427 lines
20 KiB
Python
427 lines
20 KiB
Python
"""Deterministic journal editorial contract. Fake provider proves data flow, not live prose."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-editorial-test.sqlite")
|
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
|
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
|
|
from fastapi.testclient import TestClient
|
|
from db import get_db, init_db
|
|
from engine import load_active_prompt
|
|
from identity_store import remember_mapping
|
|
from journal_editorial import (
|
|
NOTES_TO_JOURNAL,
|
|
PROSE_EDIT,
|
|
choose_editorial_mode,
|
|
editorial_instructions,
|
|
format_style_examples,
|
|
lexical_similarity,
|
|
narration_sources_text,
|
|
select_journal_style_examples,
|
|
)
|
|
from journal_generate import pack_narration_context, unattested_journal_content
|
|
from main import app
|
|
from privacy_gateway import reset_debug
|
|
from prompt_budget import JournalBudget
|
|
from providers import ChatResult
|
|
from writing_profile_store import (
|
|
NEUTRAL_JOURNAL_STYLE,
|
|
compile_task_brief,
|
|
import_text,
|
|
set_lifecycle,
|
|
update_facet,
|
|
)
|
|
|
|
|
|
SEED_REVISION = "2026-08-27-journal-placeholders-v1"
|
|
|
|
|
|
def expect(ok: bool, message: str) -> None:
|
|
if not ok:
|
|
raise SystemExit(f"FAIL: {message}")
|
|
print(f"OK {message}")
|
|
|
|
|
|
def header(token: str) -> dict:
|
|
return {"X-Auth-Token": token}
|
|
|
|
|
|
def test_mode_choice() -> None:
|
|
expect(choose_editorial_mode(["Ich ging zum Markt. Es war voll."]) == PROSE_EDIT, "narrative sentences select prose_edit")
|
|
expect(choose_editorial_mode(["markt", "kirschen", "hafen"]) == NOTES_TO_JOURNAL, "fragments select notes_to_journal")
|
|
expect(
|
|
choose_editorial_mode(["Ich war am Markt.", "Kirschen gekauft.", "hafen später"]) == PROSE_EDIT,
|
|
"mixed default is prose_edit when at least half the blocks have sentence punctuation",
|
|
)
|
|
expect(
|
|
choose_editorial_mode(["markt", "kirschen", "Später der Hafen."]) == NOTES_TO_JOURNAL,
|
|
"mixed default is notes_to_journal when the majority lacks sentence punctuation",
|
|
)
|
|
prose = editorial_instructions(PROSE_EDIT)
|
|
notes = editorial_instructions(NOTES_TO_JOURNAL)
|
|
expect("Gute Formulierungen bewahren" in prose, "prose_edit keeps good wording")
|
|
expect("zusammenhängende Journalprosa" in notes, "notes_to_journal asks for connected prose")
|
|
expect(prose != notes, "modes produce different instructions")
|
|
expect("nicht inklusive ihrer Fehler" in notes or "Fehler hintereinanderkopieren" in notes, "notes must not be concatenated with errors")
|
|
|
|
|
|
def test_prompt_contract() -> None:
|
|
init_db()
|
|
prompt = load_active_prompt("mvp.journal_generate")
|
|
text = prompt.get("template") or ""
|
|
expect("Faktentreue ist nicht Wortlauttreue" in text, "prompt separates fact fidelity from wording")
|
|
expect("CURRENT_DAY_SOURCES" in text, "prompt labels current-day facts")
|
|
expect("WRITING_PROFILE" in text, "prompt labels the writing profile")
|
|
expect("STYLE_EXAMPLES" in text, "prompt labels style examples")
|
|
expect("EDITORIAL_MODE" in text, "prompt exposes editorial mode")
|
|
expect("keine geschützten Fakten" in text, "prompt says typos are not protected facts")
|
|
expect("{{editorial_instructions}}" in text, "prompt injects mode-specific instructions")
|
|
expect("{{style_examples}}" in text, "prompt injects style examples")
|
|
expect("[[…" not in text and "[[..." not in text, "prompt must not teach ellipsis placeholders")
|
|
expect("zeichengetreu" in text, "prompt asks to copy existing placeholders unchanged")
|
|
expect("ich gieng zum laden" in text, "prompt includes a synthetic prose_edit example")
|
|
expect("Im Laden holte ich Brot" in text, "prompt includes a synthetic notes_to_journal example")
|
|
expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone")
|
|
expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone")
|
|
with get_db() as conn:
|
|
row = conn.execute(
|
|
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
|
("mvp.journal_generate",),
|
|
).fetchone()
|
|
expect(row["seed_revision"] == SEED_REVISION, "system prompt revision is stored")
|
|
expect(row["template"] == row["default_template"], "untouched install uses the seeded template")
|
|
|
|
|
|
def test_sources_and_examples_are_separated() -> None:
|
|
artifact = {
|
|
"kind": "verified_artifact",
|
|
"coverage": "all_selected_sources",
|
|
"sources": [{"source_id": "u1", "role": "user", "text": "Heute nur der Markt."}],
|
|
}
|
|
presented = narration_sources_text(artifact)
|
|
expect("[u1]" in presented and "Heute nur der Markt." in presented, "day facts are labeled source blocks")
|
|
expect("kind" not in presented, "narration does not dump artifact JSON")
|
|
formatted = format_style_examples(
|
|
[{"kind": "journal_entry", "excerpt": "Ruhige Sätze, trockener Schnitt.", "occurred_at": "2026-07-01"}]
|
|
)
|
|
expect("nicht übernommen" in formatted, "style examples forbid copying their facts")
|
|
expect("Ruhige Sätze" in formatted, "selected example text is present")
|
|
|
|
|
|
def test_unattested_covers_title_and_body() -> None:
|
|
mappings = [{"local_label": "Maren", "token": "PERSON:01"}, {"local_label": "Hanna", "token": "PERSON:99"}]
|
|
sources = ["Heute war ich mit Maren am Markt."]
|
|
expect(
|
|
unattested_journal_content("Markttag\n\nIch ging mit Maren zum Markt.", sources, mappings, ["PERSON:01"]) is None,
|
|
"attested person in title and body is allowed",
|
|
)
|
|
expect(
|
|
unattested_journal_content("Hanna am Hafen\n\nIch ging zum Markt.", sources, mappings, ["PERSON:01"])
|
|
== "unattested_identity",
|
|
"unattested person in the title is rejected",
|
|
)
|
|
expect(
|
|
unattested_journal_content("[[PERSON:99]]\n\nIch ging zum Markt.", sources, mappings, ["PERSON:01"])
|
|
== "unattested_placeholder",
|
|
"inactive placeholder in the title is unattested",
|
|
)
|
|
expect(
|
|
unattested_journal_content("[[PERSON:01]] am Markt\n\n[[PERSON:01]] kaufte Kirschen.", sources, mappings, ["PERSON:01"])
|
|
is None,
|
|
"the same attested placeholder may appear in title and body",
|
|
)
|
|
expect(
|
|
unattested_journal_content("Um 6:30 Uhr kam [[...]] ins Wohnzimmer.", sources, mappings, ["PERSON:01"])
|
|
== "unattested_placeholder",
|
|
"ellipsis placeholder leftover is unattested",
|
|
)
|
|
expect(
|
|
unattested_journal_content("Um 6:30 Uhr kam [[…]] ins Wohnzimmer.", sources, mappings, ["PERSON:01"])
|
|
== "unattested_placeholder",
|
|
"unicode ellipsis placeholder leftover is unattested",
|
|
)
|
|
|
|
|
|
def test_budget_pack_drops_examples_first() -> None:
|
|
init_db()
|
|
prompt = {
|
|
"id": "eval-pack",
|
|
"slug": "mvp.journal_generate",
|
|
"prompt_type": "base",
|
|
"template": "PROFILE\n{{writing_profile}}\nEX\n{{style_examples}}\nDAY\n{{reconstruction}}\nOLD\n{{existing_text}}\n",
|
|
}
|
|
assembled = {
|
|
"writing_profile": "Core: kurze Sätze.",
|
|
"reconstruction": "Heute Markt.",
|
|
"editorial_mode": PROSE_EDIT,
|
|
"editorial_instructions": "x",
|
|
"style_examples": "",
|
|
"existing_text": "",
|
|
}
|
|
huge_examples = "Stil " + ("Beispielwort " * 400)
|
|
existing = "Bestehende Fassung " + ("alt " * 40)
|
|
budget = JournalBudget(
|
|
model="fake",
|
|
purpose="journal_generate",
|
|
effective_context_window=32_768,
|
|
reserved_output_tokens=256,
|
|
safety_margin=0.15,
|
|
available_input_tokens=estimate_cap(assembled, prompt, extra=80),
|
|
chars_per_token=2.0,
|
|
)
|
|
packed, dropped = pack_narration_context(
|
|
prompt,
|
|
budget,
|
|
assembled,
|
|
style_examples=huge_examples,
|
|
existing_text=existing,
|
|
include_existing=True,
|
|
)
|
|
expect("style_examples" in dropped, "style examples are dropped before day sources")
|
|
expect("Heute Markt." in packed["reconstruction"], "day sources stay")
|
|
expect("kurze Sätze" in packed["writing_profile"], "writing profile stays")
|
|
|
|
|
|
def estimate_cap(assembled: dict, prompt: dict, extra: int) -> int:
|
|
from engine import preview_prompt
|
|
from prompt_budget import estimate_tokens
|
|
|
|
base = dict(assembled)
|
|
rendered = preview_prompt(prompt, base)["rendered"]
|
|
return estimate_tokens(rendered) + extra
|
|
|
|
|
|
def intern_of(payload: dict) -> str:
|
|
for stage in (payload.get("trace") or {}).get("stages") or []:
|
|
if stage.get("purpose") == "journal_generate":
|
|
return stage.get("intern") or ""
|
|
return (payload.get("trace") or {}).get("intern") or ""
|
|
|
|
|
|
def main() -> None:
|
|
test_mode_choice()
|
|
test_prompt_contract()
|
|
test_sources_and_examples_are_separated()
|
|
test_unattested_covers_title_and_body()
|
|
test_budget_pack_drops_examples_first()
|
|
expect(lexical_similarity("a b c", "a b c") > 0.9, "similarity helper is diagnostic")
|
|
expect(NEUTRAL_JOURNAL_STYLE.startswith("Neutraler Journalstil"), "neutral fallback exists")
|
|
|
|
reset_debug()
|
|
with TestClient(app) as client:
|
|
setup = client.post(
|
|
"/api/auth/setup",
|
|
json={"email": "ada@example.test", "name": "Ada", "password": "test-pass"},
|
|
)
|
|
headers = header(setup.json()["token"])
|
|
profile_id = setup.json()["profile_id"]
|
|
expect(compile_task_brief(profile_id) == NEUTRAL_JOURNAL_STYLE, "unconfirmed profile is not a style authority")
|
|
|
|
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Editorial"})
|
|
day = client.post(
|
|
f"/api/journal/spaces/{space.json()['id']}/days",
|
|
headers=headers,
|
|
json={"calendar_date": "2026-08-26"},
|
|
)
|
|
conv = client.post(
|
|
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
|
headers=headers,
|
|
json={"title": "Tag"},
|
|
)
|
|
prose_body = (
|
|
"ich bin dan zum markt gegangen und da war es zimlich voll. "
|
|
"vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. "
|
|
"die rote tasche lag im auto. ich bin dan zum markt gegangen."
|
|
)
|
|
turn = client.post(
|
|
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
|
headers=headers,
|
|
json={"body": prose_body},
|
|
)
|
|
expect(turn.status_code == 200, f"turn {turn.status_code}")
|
|
gen = client.post(
|
|
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
|
headers=headers,
|
|
json={"conversation_ids": [conv.json()["id"]]},
|
|
)
|
|
expect(gen.status_code == 200, f"generate {gen.status_code}")
|
|
intern = intern_of(gen.json())
|
|
expect("EDITORIAL_MODE: prose_edit" in intern, "narrative source selects prose_edit")
|
|
expect("Gute Formulierungen bewahren" in intern, "prose_edit instructions reach the model")
|
|
expect("CURRENT_DAY_SOURCES" in intern, "day facts are labeled")
|
|
expect("STYLE_EXAMPLES" in intern, "style examples are labeled")
|
|
expect("WRITING_PROFILE" in intern, "writing profile is labeled")
|
|
expect("Neutraler Journalstil" in intern, "without a confirmed profile the neutral fallback is used")
|
|
expect("zimlich" in intern.split("CURRENT_DAY_SOURCES")[-1], "today remains content, including typos")
|
|
expect("zimlich" not in intern.split("CURRENT_DAY_SOURCES")[0], "today is not a style authority")
|
|
expect(sum(1 for item in (gen.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "normal path is one generate call")
|
|
expect((gen.json().get("trace") or {}).get("editorial_mode") == PROSE_EDIT, "editorial mode is in the admin trace")
|
|
|
|
notes_day = client.post(
|
|
f"/api/journal/spaces/{space.json()['id']}/days",
|
|
headers=headers,
|
|
json={"calendar_date": "2026-08-25"},
|
|
)
|
|
notes_conv = client.post(
|
|
f"/api/journal/days/{notes_day.json()['day']['id']}/conversations",
|
|
headers=headers,
|
|
json={"title": "Notizen"},
|
|
)
|
|
client.post(
|
|
f"/api/journal/conversations/{notes_conv.json()['id']}/turn",
|
|
headers=headers,
|
|
json={"body": "markt\nkirschen\nspäter hafen"},
|
|
)
|
|
notes_gen = client.post(
|
|
f"/api/journal/days/{notes_day.json()['day']['id']}/generate",
|
|
headers=headers,
|
|
json={"conversation_ids": [notes_conv.json()["id"]]},
|
|
)
|
|
expect(notes_gen.status_code == 200, f"notes generate {notes_gen.text}")
|
|
notes_intern = intern_of(notes_gen.json())
|
|
expect("EDITORIAL_MODE: notes_to_journal" in notes_intern, "fragments select notes_to_journal")
|
|
expect("zusammenhängende Journalprosa" in notes_intern, "notes mode reaches the model")
|
|
expect("Gute Formulierungen bewahren" not in notes_intern, "prose_edit instructions are not used for notes")
|
|
|
|
import_text(
|
|
profile_id,
|
|
"Ich schreibe in kurzen, ruhigen Sätzen und lasse den Tag stehen. Der Hafen blieb hinter der Fähre.",
|
|
occurred_at="2026-07-01",
|
|
)
|
|
examples = select_journal_style_examples(profile_id, exclude_dates=["2026-08-26"])
|
|
expect(examples and "ruhigen Sätzen" in examples[0]["excerpt"], "imported text is a style example when no finals exist")
|
|
expect(all("zimlich" not in (item.get("excerpt") or "") for item in examples), "current day is not selected as style")
|
|
|
|
saved = client.post(
|
|
"/api/journal/entries",
|
|
headers=headers,
|
|
json={
|
|
"journal_day_id": notes_day.json()["day"]["id"],
|
|
"title": "Eigene Fassung",
|
|
"body": "Heute blieb ich beim klaren Schnitt und schrieb den Markt in eigenen Worten, ohne Pathos.",
|
|
"origin": "user_edit",
|
|
"source_conversation_ids": [notes_conv.json()["id"]],
|
|
},
|
|
)
|
|
expect(saved.status_code == 200, f"save {saved.text}")
|
|
ranked = select_journal_style_examples(profile_id, exclude_dates=["2026-08-26"])
|
|
expect(ranked and ranked[0]["kind"] == "journal_entry", "final journal text outranks imports")
|
|
expect("klaren Schnitt" in ranked[0]["excerpt"], "user-edited journal text is the preferred example")
|
|
expect(len(ranked) <= 2, "at most two style examples")
|
|
|
|
update_facet(profile_id, "core", value="Kurze Sätze, trockener Schnitt, keine Pathoswolken.")
|
|
expect("Core:" not in compile_task_brief(profile_id), "unconfirmed core is not a style authority")
|
|
set_lifecycle(profile_id, "confirmed")
|
|
with_core = compile_task_brief(profile_id)
|
|
expect("Core:" in with_core and "trockener Schnitt" in with_core, "confirmed core reaches the task brief")
|
|
expect("Neutraler Journalstil" not in with_core, "confirmed profile replaces the neutral fallback")
|
|
|
|
remember_mapping(profile_id, "Hanna", "PERSON:99")
|
|
|
|
def rewritten(_messages, _policy):
|
|
return ChatResult(
|
|
content=(
|
|
"Markttag\n\n"
|
|
"Ich ging dann zum Markt, auf dem es ziemlich voll war. "
|
|
"Vielleicht bleibe ich kürzer. "
|
|
"Brot holen wollte ich noch, habe es aber nicht gemacht. "
|
|
"Die rote Tasche lag im Auto."
|
|
),
|
|
model="fake",
|
|
usage={},
|
|
context_compression="disabled",
|
|
)
|
|
|
|
with patch("privacy_gateway.complete_model", rewritten):
|
|
second = client.post(
|
|
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
|
headers=headers,
|
|
json={"conversation_ids": [conv.json()["id"]]},
|
|
)
|
|
expect(second.status_code == 200, f"rewritten generate {second.text}")
|
|
body = second.json().get("body") or ""
|
|
expect("ziemlich voll" in body, "spelling is corrected")
|
|
expect("zimlich" not in body, "source typo is not kept")
|
|
expect("Ich ging dann zum Markt" in body, "good rewritten wording may stay")
|
|
expect("Vielleicht bleibe ich kürzer" in body, "uncertainty stays uncertainty")
|
|
expect("wollte" in body.lower() and "nicht gemacht" in body, "plan is not turned into completion")
|
|
expect("nicht gemacht" in body, "negation is kept")
|
|
expect("rote Tasche" in body, "unique attested detail is kept")
|
|
expect(body.lower().count("markt") <= 2, "repetition is allowed to be reduced")
|
|
expect("traurig" not in body.lower() and "weil" not in body.lower(), "no invented feeling or cause in the patched rewrite")
|
|
intern2 = intern_of(second.json())
|
|
expect("trockener Schnitt" in intern2.split("CURRENT_DAY_SOURCES")[0], "confirmed writing profile reaches generate")
|
|
expect("Hafen blieb hinter der Fähre" not in body, "historical style facts are not copied into today")
|
|
expect(
|
|
sum(1 for item in (second.json().get("run_log") or []) if item.get("kind") == "model_call") == 1,
|
|
"high similarity does not trigger a retry",
|
|
)
|
|
expect((second.json().get("trace") or {}).get("lexical_similarity") is not None, "similarity is a diagnostic only")
|
|
|
|
def notes_prose(_messages, _policy):
|
|
return ChatResult(
|
|
content="Notizen\n\nAm Markt holte ich Kirschen. Später war ich am Hafen.",
|
|
model="fake",
|
|
usage={},
|
|
context_compression="disabled",
|
|
)
|
|
|
|
with patch("privacy_gateway.complete_model", notes_prose):
|
|
notes_out = client.post(
|
|
f"/api/journal/days/{notes_day.json()['day']['id']}/generate",
|
|
headers=headers,
|
|
json={"conversation_ids": [notes_conv.json()["id"]]},
|
|
)
|
|
notes_body = notes_out.json().get("body") or ""
|
|
expect("Am Markt holte ich Kirschen" in notes_body, "notes become connected prose")
|
|
expect("kirschen\nspäter" not in notes_body.lower(), "notes are not concatenated as fragments")
|
|
|
|
def invent_title(_messages, _policy):
|
|
return ChatResult(
|
|
content="Hanna am Hafen\n\nIch ging zum Markt, auf dem es ziemlich voll war.",
|
|
model="fake",
|
|
usage={},
|
|
context_compression="disabled",
|
|
)
|
|
|
|
with patch("privacy_gateway.complete_model", invent_title):
|
|
blocked = client.post(
|
|
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
|
headers=headers,
|
|
json={"conversation_ids": [conv.json()["id"]]},
|
|
)
|
|
expect(any(item.get("reason") == "unattested_identity" for item in (blocked.json().get("run_log") or [])), "title identity uses local fallback")
|
|
expect("Hanna" not in (blocked.json().get("title") or "") and "Hanna" not in (blocked.json().get("body") or ""), "unattested title identity is not kept")
|
|
|
|
with get_db() as conn:
|
|
conn.execute(
|
|
"UPDATE ai_prompts SET template = 'CUSTOM JOURNAL PROMPT' WHERE slug = ?",
|
|
("mvp.journal_generate",),
|
|
)
|
|
init_db()
|
|
custom = load_active_prompt("mvp.journal_generate")
|
|
expect(custom["template"] == "CUSTOM JOURNAL PROMPT", "independently edited prompt is not overwritten")
|
|
with get_db() as conn:
|
|
row = conn.execute(
|
|
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
|
|
("mvp.journal_generate",),
|
|
).fetchone()
|
|
expect("Faktentreue ist nicht Wortlauttreue" in (row["default_template"] or ""), "default template still tracks the seed")
|
|
expect(row["seed_revision"] == SEED_REVISION, "revision updates even when template is custom")
|
|
|
|
print("journal editorial tests passed.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|