This commit is contained in:
Lars committed 2026-08-28 08:40:28 +02:00
1 parent f17b3083e8
commit 4f37991ad1
48 files changed
+4919 -540

No files matched your search

+4 -2
View File
@@ -223,8 +223,10 @@ def main() -> None:
"space_title",
"writing_profile",
"style_examples",
"editorial_mode",
"editorial_instructions",
"transformation_instructions",
"detail_instructions",
"voice_instructions",
"narrative_instructions",
"interaction_hint",
},
"system and mvp context keys",
+31 -21
View File
@@ -874,7 +874,7 @@ def test_generate_flow(client: TestClient, headers: dict) -> None:
expect("assistant:" not in (gen.json().get("body") or ""), "draft body has no assistant lines")
expect("WRITING_PROFILE" in narrate or "Neutraler Journalstil" in narrate or "Core:" in narrate, "writing profile reaches generate")
expect("STYLE_EXAMPLES" in narrate, "style examples are labeled separately from day facts")
expect("Erzählmerkmale" not in (narrate.split("CURRENT_DAY_SOURCES")[0] if "CURRENT_DAY_SOURCES" in narrate else narrate), "day dialogue is not a style brief")
expect("Erzählmerkmale" not in (narrate.split("\nCURRENT_DAY_SOURCES\n")[0] if "\nCURRENT_DAY_SOURCES\n" in narrate else narrate), "day dialogue is not a style brief")
reset_debug()
short_recorder = install_test_recorder()
@@ -995,7 +995,7 @@ def test_generate_flow(client: TestClient, headers: dict) -> None:
reset_catalog()
def test_generate_identity_leak_local_fallback(client: TestClient, headers: dict, profile_id: str) -> None:
def test_generate_attested_cleartext_is_normalized(client: TestClient, headers: dict, profile_id: str) -> None:
from identity_store import remember_mapping
from providers import ChatResult
@@ -1016,13 +1016,15 @@ def test_generate_identity_leak_local_fallback(client: TestClient, headers: dict
headers=headers,
json={"body": "Heute war ich mit Anna am Markt, danach Kirschen."},
)
expect(turn.status_code == 200, f"identity-leak setup turn {turn.status_code}")
expect(turn.status_code == 200, f"identity-cleartext setup turn {turn.status_code}")
calls = {"n": 0}
def leak(_messages, _policy):
calls["n"] += 1
return ChatResult(
content="Anna stand den ganzen Nachmittag am Markt.",
model="fake",
usage={},
usage={"prompt_tokens": 11, "completion_tokens": 9, "total_tokens": 20, "cost": 0.004},
context_compression="disabled",
)
@@ -1032,24 +1034,20 @@ def test_generate_identity_leak_local_fallback(client: TestClient, headers: dict
headers=headers,
json={"conversation_ids": [conv.json()["id"]]},
)
expect(gen.status_code == 200, f"identity leak still yields a local draft {gen.text}")
expect(gen.status_code == 200, f"attested cleartext is accepted locally {gen.text}")
payload = gen.json()
leak_log = payload.get("run_log") or (payload.get("trace") or {}).get("log") or []
expect(any(item.get("kind") == "retry" for item in leak_log), "identity leak records a retry")
expect(
any(item.get("reason") == "identity_leak_blocked" for item in leak_log),
"identity leak records local fallback",
)
expect(calls["n"] == 1, "attested cleartext does not retry")
expect(not any(item.get("kind") == "retry" for item in leak_log), "no privacy retry event")
body = payload.get("body") or ""
expect("Markt" in body and "Kirschen" in body, "local draft keeps attested user wording")
expect("Anna" in body, "local draft may keep names that the user actually wrote")
expect("Anna" in body, "source-attested name is demasked")
expect("Markt" in body, "model wording is kept after normalization")
stages = {item.get("purpose"): item for item in (payload.get("trace") or {}).get("stages") or []}
reconstruct = stages.get("local_source_artifact") or {}
narrate = stages.get("journal_generate") or {}
expect(reconstruct.get("status") == "local_ok", "stage 1 stays local and does not call reconstruct")
expect(narrate.get("guard") == "identity_leak_blocked", "stage 2 does not use the leaking reply")
expect(sum(1 for item in leak_log if item.get("kind") == "retry") == 1, "active leak retries at most once")
expect(sum(1 for item in leak_log if item.get("kind") == "model_call") <= 2, "retry is the only extra model call")
expect(narrate.get("model_text_accepted") is True, "stage 2 accepts the normalized model text")
expect((payload.get("trace") or {}).get("generate_calls") == 1 or (narrate.get("generate_calls") == 1), "exactly one generate call")
def test_generate_inactive_mapping_keeps_model_text(client: TestClient, headers: dict, profile_id: str) -> None:
@@ -1091,17 +1089,29 @@ def test_generate_inactive_mapping_keeps_model_text(client: TestClient, headers:
headers=headers,
json={"conversation_ids": [conv.json()["id"]]},
)
expect(gen.status_code == 200, f"inactive mapping generate {gen.text}")
payload = gen.json()
run_log = payload.get("run_log") or (payload.get("trace") or {}).get("log") or []
expect(gen.status_code == 409, f"inactive mapping generate {gen.text}")
detail = gen.json().get("detail") or {}
expect(detail.get("code") == "journal_generation_not_accepted", "historical-only name is not stored as a draft")
expect(detail.get("message") == "Generierung nicht übernommen.", "API names the rejection")
diag = detail.get("diagnostics") or {}
run_log = diag.get("log") or (diag.get("trace") or {}).get("log") or []
expect(calls["n"] == 1, "inactive mapping does not trigger a retry")
expect(not any(item.get("kind") == "retry" for item in run_log), "inactive mapping does not retry")
expect(
any(item.get("reason") == "unattested_identity" for item in run_log),
"invented historical name is unattested journal content",
)
expect("Clarissa" not in (payload.get("body") or ""), "unattested name is not accepted as narration")
expect("Markt" in (payload.get("body") or "") and "Kirschen" in (payload.get("body") or ""), "fallback keeps user sources")
expect(diag.get("model_text_accepted") is False, "unattested model text is not accepted")
day_after = client.get(f"/api/journal/days/{day.json()['day']['id']}", headers=headers)
expect(not (day_after.json().get("current_draft")), "rejected generate does not insert a draft")
trace = diag.get("trace") or {}
expect(trace.get("abort_reason") == "unattested_identity", "abort reason remains on the error trace")
expect(
(trace.get("generate_calls") == 1)
or ((trace.get("budget") or {}).get("generate_calls") == 1)
or diag.get("generate_calls") == 1,
"error trace keeps the generate-call count",
)
def test_traces_are_request_scoped(client: TestClient, headers: dict) -> None:
@@ -1247,7 +1257,7 @@ def main() -> None:
profile_id = setup.json()["profile_id"]
test_task_brief_and_dedupe(client, headers, profile_id)
test_generate_flow(client, headers)
test_generate_identity_leak_local_fallback(client, headers, profile_id)
test_generate_attested_cleartext_is_normalized(client, headers, profile_id)
test_generate_inactive_mapping_keeps_model_text(client, headers, profile_id)
test_traces_are_request_scoped(client, headers)
test_two_conversations_same_day(client, headers)
+147 -45
View File
@@ -20,11 +20,9 @@ from db import get_db, init_db
from engine import load_active_prompt
from identity_store import remember_mapping
from journal_editorial import (
NOTES_TO_JOURNAL,
PROSE_EDIT,
choose_editorial_mode,
editorial_instructions,
GENERATE_SEED_REVISION,
format_style_examples,
incomplete_syntax_markers,
lexical_similarity,
narration_sources_text,
select_journal_style_examples,
@@ -43,7 +41,13 @@ from writing_profile_store import (
)
SEED_REVISION = "2026-08-27-journal-placeholders-v1"
SEED_REVISION = GENERATE_SEED_REVISION
MIXED_SOURCES_INSTRUCTION = (
"Die Quellen können aus Fließtext, Stichpunkten und Satzfragmenten bestehen. "
"Behandle jede Passage entsprechend ihrer Form: Redigiere vorhandenen Fließtext, "
"verwandle Stichpunkte und Fragmente in vollständige Prosa und verbinde beides zu einem einheitlichen Eintrag. "
"Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus."
)
def expect(ok: bool, message: str) -> None:
@@ -56,43 +60,36 @@ def header(token: str) -> dict:
return {"X-Auth-Token": token}
def test_mode_choice() -> None:
expect(choose_editorial_mode(["Ich ging zum Markt. Es war voll."]) == PROSE_EDIT, "narrative sentences select prose_edit")
expect(choose_editorial_mode(["markt", "kirschen", "hafen"]) == NOTES_TO_JOURNAL, "fragments select notes_to_journal")
expect(
choose_editorial_mode(["Ich war am Markt.", "Kirschen gekauft.", "hafen später"]) == PROSE_EDIT,
"mixed default is prose_edit when at least half the blocks have sentence punctuation",
)
expect(
choose_editorial_mode(["markt", "kirschen", "Später der Hafen."]) == NOTES_TO_JOURNAL,
"mixed default is notes_to_journal when the majority lacks sentence punctuation",
)
prose = editorial_instructions(PROSE_EDIT)
notes = editorial_instructions(NOTES_TO_JOURNAL)
expect("Gute Formulierungen bewahren" in prose, "prose_edit keeps good wording")
expect("zusammenhängende Journalprosa" in notes, "notes_to_journal asks for connected prose")
expect(prose != notes, "modes produce different instructions")
expect("nicht inklusive ihrer Fehler" in notes or "Fehler hintereinanderkopieren" in notes, "notes must not be concatenated with errors")
def test_syntax_diagnostics() -> None:
expect(incomplete_syntax_markers("Danach sprach ich mit dem.") > 0, "dangling determiner is diagnostic")
expect(incomplete_syntax_markers("Danach sprach ich und kam zurück.") == 0, "complete sentence is not flagged")
def test_prompt_contract() -> None:
init_db()
prompt = load_active_prompt("mvp.journal_generate")
text = prompt.get("template") or ""
expect("Faktentreue ist nicht Wortlauttreue" in text, "prompt separates fact fidelity from wording")
expect("INHALTSTREUE" in text, "prompt keeps hard content rules")
expect("CURRENT_DAY_SOURCES" in text, "prompt labels current-day facts")
expect("WRITING_PROFILE" in text, "prompt labels the writing profile")
expect("STYLE_EXAMPLES" in text, "prompt labels style examples")
expect("EDITORIAL_MODE" in text, "prompt exposes editorial mode")
expect("keine geschützten Fakten" in text, "prompt says typos are not protected facts")
expect("{{editorial_instructions}}" in text, "prompt injects mode-specific instructions")
expect("WRITING_PROFILE" in text, "prompt labels the writing profile")
expect("{{transformation_instructions}}" in text, "prompt injects compiled transformation instructions")
expect("{{detail_instructions}}" in text, "prompt injects compiled detail instructions")
expect("{{voice_instructions}}" in text, "prompt injects compiled voice instructions")
expect("{{narrative_instructions}}" in text, "prompt injects compiled narrative instructions")
expect("{{source_mode_instructions}}" not in text, "prompt has no source-mode placeholder")
expect(MIXED_SOURCES_INSTRUCTION in text, "prompt contains the mixed-source instruction")
expect("{{writing_profile}}" in text, "prompt injects writing profile")
expect("{{style_examples}}" in text, "prompt injects style examples")
expect("{{reconstruction}}" in text, "prompt injects current-day sources")
expect("{{existing_text}}" in text, "prompt injects existing text")
expect("[[…" not in text and "[[..." not in text, "prompt must not teach ellipsis placeholders")
expect("zeichengetreu" in text, "prompt asks to copy existing placeholders unchanged")
expect("ich gieng zum laden" in text, "prompt includes a synthetic prose_edit example")
expect("Im Laden holte ich Brot" in text, "prompt includes a synthetic notes_to_journal example")
expect("ich gieng zum laden" not in text, "synthetic prose_edit example is gone")
expect("nachbarhund im garten" not in text, "synthetic notes_to_journal example is gone")
expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone")
expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone")
expect("EDITORIAL_MODE" not in text, "source mode is no longer a labeled user setting")
expect("Faktentreue ist nicht Wortlauttreue" not in text, "old redundant fidelity lecture is gone")
with get_db() as conn:
row = conn.execute(
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
@@ -163,8 +160,6 @@ def test_budget_pack_drops_examples_first() -> None:
assembled = {
"writing_profile": "Core: kurze Sätze.",
"reconstruction": "Heute Markt.",
"editorial_mode": PROSE_EDIT,
"editorial_instructions": "x",
"style_examples": "",
"existing_text": "",
}
@@ -209,7 +204,7 @@ def intern_of(payload: dict) -> str:
def main() -> None:
test_mode_choice()
test_syntax_diagnostics()
test_prompt_contract()
test_sources_and_examples_are_separated()
test_unattested_covers_title_and_body()
@@ -256,16 +251,22 @@ def main() -> None:
)
expect(gen.status_code == 200, f"generate {gen.status_code}")
intern = intern_of(gen.json())
expect("EDITORIAL_MODE: prose_edit" in intern, "narrative source selects prose_edit")
expect("Gute Formulierungen bewahren" in intern, "prose_edit instructions reach the model")
expect(MIXED_SOURCES_INSTRUCTION in intern, "unified mixed-source instruction reaches the model")
expect("CURRENT_DAY_SOURCES" in intern, "day facts are labeled")
expect("STYLE_EXAMPLES" in intern, "style examples are labeled")
expect("WRITING_PROFILE" in intern, "writing profile is labeled")
expect("Überarbeite den Text substanziell" in intern, "default transformation policy reaches the prompt")
expect("Erhalte sämtliche belegten Ereignisse" in intern, "default detail policy reaches the prompt")
expect("Neutraler Journalstil" in intern, "without a confirmed profile the neutral fallback is used")
expect("zimlich" in intern.split("CURRENT_DAY_SOURCES")[-1], "today remains content, including typos")
expect("zimlich" not in intern.split("CURRENT_DAY_SOURCES")[0], "today is not a style authority")
expect("zimlich" in intern.split("\nCURRENT_DAY_SOURCES\n")[-1], "today remains content, including typos")
expect("zimlich" not in intern.split("\nCURRENT_DAY_SOURCES\n")[0], "today is not a style authority")
expect(sum(1 for item in (gen.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "normal path is one generate call")
expect((gen.json().get("trace") or {}).get("editorial_mode") == PROSE_EDIT, "editorial mode is in the admin trace")
expect("editorial_mode" not in (gen.json().get("trace") or {}), "trace has no abandoned editorial mode")
expect((gen.json().get("trace") or {}).get("narration_source") == "model", "accepted model path is recorded")
expect((gen.json().get("trace") or {}).get("prompt_revision") == SEED_REVISION, "prompt revision is in the admin trace")
expect((gen.json().get("trace") or {}).get("writing_profile", {}).get("neutral_fallback") is True, "unconfirmed profile is visible as fallback")
expect((gen.json().get("trace") or {}).get("style_examples", {}).get("count") == 0, "no historical examples yet")
expect((gen.json().get("trace") or {}).get("dropped_optional_blocks") == [], "nothing dropped on a small day")
notes_day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
@@ -289,9 +290,8 @@ def main() -> None:
)
expect(notes_gen.status_code == 200, f"notes generate {notes_gen.text}")
notes_intern = intern_of(notes_gen.json())
expect("EDITORIAL_MODE: notes_to_journal" in notes_intern, "fragments select notes_to_journal")
expect("zusammenhängende Journalprosa" in notes_intern, "notes mode reaches the model")
expect("Gute Formulierungen bewahren" not in notes_intern, "prose_edit instructions are not used for notes")
expect(MIXED_SOURCES_INSTRUCTION in notes_intern, "notes use the same mixed-source instruction")
expect("markt" in notes_intern.lower() and "kirschen" in notes_intern.lower(), "notes sources remain complete")
import_text(
profile_id,
@@ -360,7 +360,7 @@ def main() -> None:
expect(body.lower().count("markt") <= 2, "repetition is allowed to be reduced")
expect("traurig" not in body.lower() and "weil" not in body.lower(), "no invented feeling or cause in the patched rewrite")
intern2 = intern_of(second.json())
expect("trockener Schnitt" in intern2.split("CURRENT_DAY_SOURCES")[0], "confirmed writing profile reaches generate")
expect("trockener Schnitt" in intern2.split("\nCURRENT_DAY_SOURCES\n")[0], "confirmed writing profile reaches generate")
expect("Hafen blieb hinter der Fähre" not in body, "historical style facts are not copied into today")
expect(
sum(1 for item in (second.json().get("run_log") or []) if item.get("kind") == "model_call") == 1,
@@ -386,6 +386,103 @@ def main() -> None:
expect("Am Markt holte ich Kirschen" in notes_body, "notes become connected prose")
expect("kirschen\nspäter" not in notes_body.lower(), "notes are not concatenated as fragments")
broken_day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": "2026-08-24"},
)
broken_conv = client.post(
f"/api/journal/days/{broken_day.json()['day']['id']}/conversations",
headers=headers,
json={"title": "Bruch"},
)
client.post(
f"/api/journal/conversations/{broken_conv.json()['id']}/turn",
headers=headers,
json={"body": "ich gieng zum laden. danach sprach ich mit dem und kam zurück. es war kald."},
)
def rebuilt(_messages, _policy):
return ChatResult(
content="Ladengang\n\nIch ging zum Laden. Danach sprach ich und kam zurück. Es war kalt.",
model="fake",
usage={},
context_compression="disabled",
)
with patch("privacy_gateway.complete_model", rebuilt):
broken_out = client.post(
f"/api/journal/days/{broken_day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [broken_conv.json()["id"]]},
)
broken_body = broken_out.json().get("body") or ""
expect("gieng" not in broken_body.lower(), "general spelling error is not kept")
expect("kald" not in broken_body.lower(), "second general spelling error is not kept")
expect("mit dem" not in broken_body.lower(), "dangling determiner is not kept")
expect("kam zurück" in broken_body, "attested continuation stays")
expect(incomplete_syntax_markers(broken_body) == 0, "patched rewrite has no incomplete syntax")
expect((broken_out.json().get("trace") or {}).get("incomplete_syntax") == 0, "incomplete syntax is a diagnostic")
weight_day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": "2026-08-23"},
)
weight_conv = client.post(
f"/api/journal/days/{weight_day.json()['day']['id']}/conversations",
headers=headers,
json={"title": "Notizen"},
)
client.post(
f"/api/journal/conversations/{weight_conv.json()['id']}/turn",
headers=headers,
json={"body": "morgens tee\nspäter markt\ngegen abend unerwartet der nachbarhund im garten"},
)
def weighted(_messages, _policy):
return ChatResult(
content=(
"Nachbarhund\n\n"
"Morgens trank ich Tee, später war ich am Markt. "
"Es war ein gewöhnlicher Tag – bis gegen Abend unerwartet der Nachbarhund im Garten war."
),
model="fake",
usage={},
context_compression="disabled",
)
with patch("privacy_gateway.complete_model", weighted):
weight_out = client.post(
f"/api/journal/days/{weight_day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [weight_conv.json()["id"]]},
)
weight_intern = intern_of(weight_out.json())
expect(MIXED_SOURCES_INSTRUCTION in weight_intern, "fragment notes still use the unified instruction")
weight_body = weight_out.json().get("body") or ""
expect("bis gegen Abend" in weight_body or "unerwartet" in weight_body, "attested standout may be weighted")
expect("morgens tee\nspäter markt" not in weight_body.lower(), "notes are not a concatenated list")
expect("glücklich" not in weight_body.lower() and "weil" not in weight_body.lower(), "weighting does not invent feeling or cause")
update_facet(profile_id, "core", value="Lange, ruhig fließende Sätze, behutsame Wortwahl, leise Reflexion.")
set_lifecycle(profile_id, "confirmed")
brief_b = compile_task_brief(profile_id)
expect("fließende Sätze" in brief_b, "second confirmed core compiles")
with patch("privacy_gateway.complete_model", rewritten):
third = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [conv.json()["id"]]},
)
intern3 = intern_of(third.json())
style3 = intern3.split("\nCURRENT_DAY_SOURCES\n")[0]
expect("fließende Sätze" in style3, "second writing profile reaches the rendered prompt")
expect("trockener Schnitt" not in style3, "replaced core is not still the style authority")
expect((third.json().get("trace") or {}).get("writing_profile", {}).get("has_core") is True, "core presence is in the trace")
expect((third.json().get("trace") or {}).get("writing_profile", {}).get("present") is True, "confirmed profile is marked present")
expect(sum(1 for item in (third.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "profile A/B still uses one generate call")
def invent_title(_messages, _policy):
return ChatResult(
content="Hanna am Hafen\n\nIch ging zum Markt, auf dem es ziemlich voll war.",
@@ -400,8 +497,13 @@ def main() -> None:
headers=headers,
json={"conversation_ids": [conv.json()["id"]]},
)
expect(any(item.get("reason") == "unattested_identity" for item in (blocked.json().get("run_log") or [])), "title identity uses local fallback")
expect("Hanna" not in (blocked.json().get("title") or "") and "Hanna" not in (blocked.json().get("body") or ""), "unattested title identity is not kept")
expect(blocked.status_code == 409, f"unattested title {blocked.text}")
detail = blocked.json().get("detail") or {}
expect(detail.get("code") == "journal_generation_not_accepted", "unattested title is not stored as a draft")
expect(detail.get("message") == "Generierung nicht übernommen.", "API names the rejection")
log = (detail.get("diagnostics") or {}).get("log") or []
expect(any(item.get("reason") == "unattested_identity" for item in log), "title identity is a provenance reject")
expect((detail.get("diagnostics") or {}).get("trace", {}).get("model_text_accepted") is False, "model text is not accepted")
with get_db() as conn:
conn.execute(
@@ -416,7 +518,7 @@ def main() -> None:
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
("mvp.journal_generate",),
).fetchone()
expect("Faktentreue ist nicht Wortlauttreue" in (row["default_template"] or ""), "default template still tracks the seed")
expect("INHALTSTREUE" in (row["default_template"] or ""), "default template still tracks the seed")
expect(row["seed_revision"] == SEED_REVISION, "revision updates even when template is custom")
print("journal editorial tests passed.")
+53 -5
View File
@@ -15,14 +15,19 @@ os.environ["KANSHO_FAKE_DETECT"] = "1"
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
from db import init_db
from journal_editorial import GENERATE_SEED_REVISION
from journal_eval import (
FIXTURES,
PROFILE_A,
PROFILE_B,
SYNTHETIC_PROSE,
SYNTHETIC_TYPOS,
VARIANT_BASELINE,
VARIANT_CURRENT,
VARIANT_PREVIOUS,
compare_synthetic,
fixture_context,
score_output,
synthetic_context,
variant_templates,
)
@@ -37,12 +42,33 @@ def main() -> None:
init_db()
templates = variant_templates()
expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist")
expect("Überarbeite diesen Rohtext" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
expect("Erstelle aus diesen Angaben einen ansprechenden persönlichen Tagebucheintrag" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt")
expect("Faktentreue ist nicht Wortlauttreue" in templates[VARIANT_CURRENT], "current variant has the new contract")
expect("INHALTSTREUE" in templates[VARIANT_CURRENT], "current variant has the new contract")
expect("{{transformation_instructions}}" in templates[VARIANT_CURRENT], "current variant compiles transformation policy")
expect("{{source_mode_instructions}}" not in templates[VARIANT_CURRENT], "current variant has no source-mode placeholder")
expect("Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus." in templates[VARIANT_CURRENT], "current variant has mixed-source instruction")
expect("ich gieng zum laden" not in templates[VARIANT_CURRENT], "current variant has no synthetic examples")
expect(GENERATE_SEED_REVISION == "2026-08-27-journal-mixed-sources-v1", "eval tracks the seeded revision constant")
context = synthetic_context(SYNTHETIC_PROSE)
expect(context["editorial_mode"] in {"prose_edit", "notes_to_journal"}, "eval context has an editorial mode")
required_classes = {
"already_narrative_with_errors",
"bullet_points_and_fragments",
"plan_versus_completion",
"negation_and_uncertainty",
"correction_of_earlier_claim",
"imprecise_time",
"outstanding_event_among_everyday",
"recurring_people_and_projects",
"incomplete_source_rebuildable",
}
got = {item["class"] for item in FIXTURES}
expect(required_classes <= got, f"all required fixture classes exist, missing {required_classes - got}")
expect(all("example.test" not in item["source"].lower() for item in FIXTURES), "fixtures stay synthetic")
context = fixture_context(SYNTHETIC_PROSE)
expect("source_mode_instructions" not in context, "eval context has no source-mode instruction")
expect("editorial_mode" not in context, "eval context has no editorial mode")
copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS)
expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1")
expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos")
@@ -55,6 +81,28 @@ def main() -> None:
expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical")
expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored")
expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures")
notes = next(item for item in FIXTURES if item["id"] == "notes_fragments")
expect("markt" in notes["source"].lower(), "notes fixture stays notes-shaped")
incomplete = next(item for item in FIXTURES if item["id"] == "incomplete_clause")
expect("danach sprach ich mit dem" in incomplete["source"].lower(), "incomplete fixture stays incomplete prose")
report = compare_synthetic(live=False)
expect(report["live"] is False, "default eval is offline")
expect(report["live_quality_confirmed"] is False, "offline run does not confirm live quality")
expect(report["winner_declared"] is False, "harness does not declare a winner")
expect(len(report["fixtures"]) == len(FIXTURES), "offline report covers every fixture")
first = report["fixtures"][0]
expect("human_blind" in first and "prompt_1" in first["human_blind"], "each fixture has a blind pair")
expect(first["human_blind"]["hidden_mapping"]["prompt_1"] == VARIANT_CURRENT, "mapping stays machine-side")
expect("Kanshō habe gewonnen" not in report["note"], "no victory claim")
expect(any(item["variant"] == VARIANT_CURRENT and item["fake_provider"] for item in first["variants"]), "offline current variant is fake")
ab = compare_synthetic(live=False, profile_ab=True)
expect(ab["profile_ab"]["same_facts"] is True, "profile A/B keeps facts identical")
expect(ab["profile_ab"]["prompts_differ"] is True, "rendered prompts contain different style briefs")
expect(PROFILE_A[:20] != PROFILE_B[:20], "synthetic profiles are distinct")
expect("Live-Prosa" in (ab["profile_ab"]["note"] or ""), "offline A/B does not claim live prose")
print("journal eval tests passed.")
@@ -0,0 +1,961 @@
"""Named journal generation guidelines: selection, snapshot, mixed sources."""
from __future__ import annotations
import os
import re
import sys
import tempfile
from pathlib import Path
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
FRONTEND = ROOT.parent / "frontend"
sys.path.insert(0, str(ROOT))
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-guidelines-test.sqlite")
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
os.environ["KANSHO_FAKE_DETECT"] = "1"
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
from fastapi.testclient import TestClient
from db import get_db, init_db
from engine import load_active_prompt
from identity_store import confirm_identity, remember_mapping
from journal_editorial import GENERATE_SEED_REVISION
from journal_generation_policy import (
PURPOSE_JOURNAL,
SELECTION_KEYS,
SLOT_TO_ID_KEY,
CatalogError,
GenerationPolicyError,
archive_guideline,
clone_guideline,
compile_selection,
create_guideline,
default_selection_ids,
get_guideline,
get_or_create_selection,
list_guidelines,
load_seed_document,
load_selection,
overview_payload,
publish_guideline,
save_selection,
seed_generation_instructions,
snapshot_summary,
update_guideline,
validate_selection,
)
from journal_generate import unattested_journal_content
from journal_store import current_draft
from main import app
from privacy_gateway import install_test_recorder, reset_debug
from providers import ChatResult
PREVIOUS_PROMPT_CHARS = 3468
HARD_FACTS = (
"keine neuen Tatsachen",
"Plan und Vollzug",
"Verneinungen",
"Unsicherheiten",
"STYLE_EXAMPLES dienen ausschließlich als Stilreferenz",
"[[PERSON:01]]",
)
BANNED_CODE_PHRASES = (
"Überarbeite den Text substanziell",
"Erhalte sämtliche belegten Ereignisse",
"künstlich zu literarisieren",
"Modus prose_edit",
"Modus notes_to_journal",
"Korrigiere nur Rechtschreibung, Grammatik und Zeichensetzung",
)
BANNED_RUNTIME_TOKENS = (
"transformation_strength",
"detail_retention",
"voice_strength",
"narrative_shaping",
"min_value",
"max_value",
"remember_generation_policy",
"generation_policy",
)
SEED = load_seed_document()
MIXED_SOURCES_INSTRUCTION = (
"Die Quellen können aus Fließtext, Stichpunkten und Satzfragmenten bestehen. "
"Behandle jede Passage entsprechend ihrer Form: Redigiere vorhandenen Fließtext, "
"verwandle Stichpunkte und Fragmente in vollständige Prosa und verbinde beides zu einem einheitlichen Eintrag. "
"Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus."
)
REQUIRED_KEYS = {
("transformation", "correction"),
("transformation", "copyedit"),
("transformation", "reshape"),
("transformation", "substantial"),
("detail", "compact"),
("detail", "selected"),
("detail", "broad"),
("detail", "complete"),
("voice", "neutral"),
("voice", "light"),
("voice", "noticeable"),
("voice", "clear"),
("narrative", "chronicle"),
("narrative", "structured"),
("narrative", "weighted"),
("narrative", "emphasized"),
}
def has_token(text: str, token: str) -> bool:
return re.search(rf"(?<![A-Za-z0-9_]){re.escape(token)}(?![A-Za-z0-9_])", text) is not None
def expect(ok: bool, message: str) -> None:
if not ok:
raise SystemExit(f"FAIL: {message}")
print(f"OK {message}")
def header(token: str) -> dict:
return {"X-Auth-Token": token}
def intern_of(payload: dict) -> str:
for stage in (payload.get("trace") or {}).get("stages") or []:
if stage.get("purpose") == "journal_generate":
return stage.get("intern") or ""
return (payload.get("trace") or {}).get("intern") or ""
def sources_block(intern: str) -> str:
parts = (intern or "").split("\nCURRENT_DAY_SOURCES\n")
if len(parts) < 2:
return intern or ""
rest = parts[-1]
for marker in ("\nEXISTING_TEXT\n", "\nAUSGABE\n"):
if marker in rest:
rest = rest.split(marker, 1)[0]
break
return rest
def seed_item(slot: str, key: str) -> dict:
for item in (SEED.get("slots") or {}).get(slot, {}).get("variants") or []:
if item.get("guideline_key") == key:
return item
raise SystemExit(f"FAIL: missing seed {slot}/{key}")
def seed_id(slot: str, key: str) -> str:
return seed_item(slot, key)["id"]
def seed_instruction(slot: str, key: str) -> str:
return seed_item(slot, key).get("instruction") or ""
def selection_of(**keys: str) -> dict[str, str]:
return {SLOT_TO_ID_KEY[slot]: seed_id(slot, key) for slot, key in keys.items()}
def default_ids() -> dict[str, str]:
return selection_of(
transformation="substantial",
detail="complete",
voice="clear",
narrative="weighted",
)
def low_ids() -> dict[str, str]:
return selection_of(
transformation="correction",
detail="compact",
voice="neutral",
narrative="chronicle",
)
def high_ids() -> dict[str, str]:
return selection_of(
transformation="substantial",
detail="complete",
voice="clear",
narrative="emphasized",
)
def selection_meta(payload: dict) -> dict:
return (payload.get("trace") or {}).get("generation_selection") or {}
def test_runtime_has_no_numeric_policy() -> None:
files = [
ROOT / "journal_generation_policy.py",
ROOT / "journal_generate.py",
ROOT / "routers" / "journal.py",
ROOT / "routers" / "generation_instructions.py",
FRONTEND / "src" / "pages" / "JournalDayPage.jsx",
FRONTEND / "src" / "pages" / "AdminGenerationPage.jsx",
]
for path in files:
text = path.read_text(encoding="utf-8")
for token in BANNED_RUNTIME_TOKENS:
if token == "generation_policy" and path.name == "journal_generation_policy.py":
continue
if token == "generation_policy" and path.name == "generation_instructions.py":
continue
expect(not has_token(text, token), f"{path.name} has no {token}")
day = (FRONTEND / "src" / "pages" / "JournalDayPage.jsx").read_text(encoding="utf-8")
expect('type="range"' not in day, "journal page has no sliders")
expect("policy-slider" not in day, "journal page has no slider class")
expect("SLOT_SELECTS" in day, "journal page names the four selects")
expect(day.count("slot:") >= 4, "journal page has four independent slots")
expect("<select" in day, "journal page uses select fields")
expect("Diese Auswahl als Standard merken" in day, "journal page can remember explicitly")
expect("Faktenregeln und Datenschutz bleiben unabhängig" in day, "facts/privacy note remains")
editor = (FRONTEND / "src" / "pages" / "JournalEditorPage.jsx").read_text(encoding="utf-8")
expect("generation_summary" in editor, "editor can show the generation snapshot")
admin = (FRONTEND / "src" / "pages" / "AdminGenerationPage.jsx").read_text(encoding="utf-8")
expect("<textarea" not in admin.split("generation-admin-editor")[0], "admin overview has no textareas")
expect(admin.count("<textarea") == 1, "admin opens one instruction editor at a time")
expect("Öffnen" in admin, "admin overview has open action")
expect("backend/config/" not in admin, "admin copy does not advertise seed file paths")
expect("Quellenmodus" not in admin, "admin has no source-mode management")
expect("source_mode" not in admin, "admin source has no source_mode slot")
expect("prose_edit" not in admin, "admin has no prose_edit control")
expect("notes_to_journal" not in admin, "admin has no notes_to_journal control")
expect("Quellenmodus" not in day, "journal page has no source-mode control")
expect("source_mode" not in day, "journal page has no source_mode field")
expect("editorial_mode" not in day, "journal page has no editorial_mode field")
trace_ui = (FRONTEND / "src" / "components" / "CallTrace.jsx").read_text(encoding="utf-8")
expect("Editorial Mode" not in trace_ui, "trace UI does not present source mode as a run decision")
expect("source_mode" not in trace_ui, "trace UI has no source_mode label")
def test_seed_and_compiler() -> None:
init_db()
for rel in ("journal_generation_policy.py", "journal_editorial.py"):
source = (ROOT / rel).read_text(encoding="utf-8")
for phrase in BANNED_CODE_PHRASES:
expect(phrase not in source, f"{rel} does not hardcode {phrase!r}")
items = list_guidelines(PURPOSE_JOURNAL, include_instruction=True)
got = {(item["slot"], item["guideline_key"]) for item in items}
expect(REQUIRED_KEYS <= got, "seed creates every required guideline")
expect("source_mode" not in (SEED.get("slots") or {}), "seed document has no source_mode slot")
expect(sum(1 for item in items if item["slot"] == "source_mode") == 0, "source modes are not seeded")
expect(all(item["status"] == "active" for item in items if item["is_system_seed"]), "seed rows start active")
defaults = compile_selection(default_selection_ids())
expect(defaults.ids["transformation"] == seed_id("transformation", "substantial"), "default transformation is substantial")
expect(
defaults.instructions["transformation_instructions"] == seed_instruction("transformation", "substantial"),
"default transformation instruction matches the seed",
)
expect(
defaults.instructions["detail_instructions"] == seed_instruction("detail", "complete"),
"default detail instruction matches the seed",
)
expect(
defaults.instructions["voice_instructions"] == seed_instruction("voice", "clear"),
"default voice instruction matches the seed",
)
expect(
defaults.instructions["narrative_instructions"] == seed_instruction("narrative", "weighted"),
"default narrative instruction matches the seed",
)
expect(defaults.seed_revision == SEED["seed_revision"], "compiled policy names the seed revision")
expect("source_mode_instructions" not in defaults.instructions, "compiler has no source-mode instruction")
low = compile_selection(low_ids())
high = compile_selection(high_ids())
expect(low.keys["transformation"] == "correction", "low transformation is named")
expect(high.keys["narrative"] == "emphasized", "high narrative is named")
expect("source_mode_instructions" not in low.instructions, "low compile has no source mode")
expect("source_mode_instructions" not in high.instructions, "high compile has no source mode")
expect("{{" not in "".join(low.instructions.values()), "compiled instructions contain no placeholders")
for bad in (None, [], "75", True, {"transformation_strength": 75}, {"transformation_id": seed_id("transformation", "substantial")}):
try:
validate_selection(bad)
expect(False, "incomplete or numeric selection must be rejected")
except GenerationPolicyError as exc:
expect(exc.code == "invalid_generation_selection", "rejection uses invalid_generation_selection")
draft = create_guideline(
PURPOSE_JOURNAL,
"transformation",
{
"guideline_key": "drafty",
"label": "Entwurf",
"summary": "nicht wählbar",
"instruction": "Nur ein Draft.",
},
)
try:
validate_selection({**default_ids(), "transformation_id": draft["id"]})
expect(False, "drafts must not be selectable")
except GenerationPolicyError:
expect(True, "drafts are not selectable")
cloned = clone_guideline(seed_id("transformation", "correction"))
original = get_guideline(seed_id("transformation", "correction"), include_instruction=True)
update_guideline(
cloned["id"],
{
"guideline_key": original["guideline_key"],
"label": original["label"],
"summary": original["summary"],
"instruction": "CHANGED CLONE TEXT",
},
)
expect(
get_guideline(original["id"], include_instruction=True)["instruction"] == original["instruction"],
"cloning does not change the original",
)
try:
update_guideline(original["id"], {"label": "still active", "instruction": original["instruction"], "guideline_key": original["guideline_key"]})
expect(False, "published guidelines must be immutable")
except CatalogError as exc:
expect(exc.code == "guideline_immutable", "published guidelines are not silently overwritten")
published_clone = publish_guideline(cloned["id"])
archived = archive_guideline(published_clone["id"])
expect(archived["status"] == "archived", "published clones can be archived")
try:
validate_selection({**default_ids(), "transformation_id": published_clone["id"]})
expect(False, "archived guidelines must not be selectable for new runs")
except GenerationPolicyError:
expect(True, "archived guidelines are not selectable for new runs")
with get_db() as conn:
seed_generation_instructions(conn)
expect(
get_guideline(original["id"], include_instruction=True)["instruction"] == original["instruction"],
"seed does not overwrite published guidelines",
)
with get_db() as conn:
conn.execute(
"""
INSERT INTO generation_guidelines (
id, purpose, slot, guideline_key, label, summary, instruction, sort_order,
status, revision, cloned_from, is_default, is_system_seed, seed_id, seed_revision,
created, updated
)
VALUES (
'legacy-source-mode', ?, 'source_mode', 'prose_edit', 'Fließtext', '',
'LEGACY MODE TEXT', 0, 'active', 1, NULL, 0, 0, '', '', datetime('now'), datetime('now')
)
""",
(PURPOSE_JOURNAL,),
)
seed_generation_instructions(conn)
archived_legacy = conn.execute(
"SELECT status FROM generation_guidelines WHERE id = 'legacy-source-mode'"
).fetchone()
expect(archived_legacy["status"] == "archived", "persisted source modes are archived")
expect("source_mode" not in overview_payload()["slots"], "admin overview hides archived source modes")
def test_prompt_contract() -> None:
init_db()
prompt = load_active_prompt("mvp.journal_generate")
text = prompt.get("template") or ""
expect(len(text) < PREVIOUS_PROMPT_CHARS, "new prompt is shorter than the previous default")
print(f"OK prompt length {len(text)} after, {PREVIOUS_PROMPT_CHARS} before")
for key in (
"{{transformation_instructions}}",
"{{detail_instructions}}",
"{{voice_instructions}}",
"{{narrative_instructions}}",
"{{writing_profile}}",
"{{style_examples}}",
"{{reconstruction}}",
"{{existing_text}}",
):
expect(key in text, f"prompt contains {key}")
expect("{{source_mode_instructions}}" not in text, "prompt has no source-mode placeholder")
expect("{{editorial_mode}}" not in text, "prompt has no editorial_mode placeholder")
expect("{{editorial_instructions}}" not in text, "prompt has no editorial_instructions placeholder")
expect(MIXED_SOURCES_INSTRUCTION in text, "prompt contains the mixed-source instruction")
for fact in HARD_FACTS:
expect(fact in text, f"hard fact rule remains: {fact}")
expect("ich gieng zum laden" not in text, "no synthetic prose example")
expect("nachbarhund im garten" not in text, "no synthetic notes example")
expect("EDITORIAL_MODE" not in text, "source mode is not a user-facing prompt heading")
expect(GENERATE_SEED_REVISION == "2026-08-27-journal-mixed-sources-v1", "seed revision constant tracks mixed sources")
with get_db() as conn:
row = conn.execute(
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
("mvp.journal_generate",),
).fetchone()
expect(row["seed_revision"] == "2026-08-27-journal-mixed-sources-v1", "untouched default is updated to mixed sources")
expect(row["template"] == row["default_template"], "untouched template matches the seed")
expect("source_mode" not in (SEED.get("slots") or {}), "source modes stay out of the seed")
def test_unattested_alias() -> None:
mappings = [
{
"local_label": "Clarissa",
"canonical_label": "Clarissa",
"token": "PERSON:01",
"aliases": ["Sushi"],
"demask_label": "Clarissa",
}
]
expect(
unattested_journal_content(
"Clarissa kam ins Wohnzimmer.",
["Heute war Sushi im Wohnzimmer."],
mappings,
["PERSON:01"],
)
is None,
"canonical demask of an attested alias is not unattested_identity",
)
expect(
unattested_journal_content(
"Hanna kam ins Wohnzimmer.",
["Heute war Sushi im Wohnzimmer."],
mappings + [{"local_label": "Hanna", "token": "PERSON:99"}],
["PERSON:01"],
)
== "unattested_identity",
"a different person remains unattested",
)
def main() -> None:
test_runtime_has_no_numeric_policy()
test_seed_and_compiler()
test_prompt_contract()
test_unattested_alias()
reset_debug()
with TestClient(app) as client:
setup = client.post(
"/api/auth/setup",
json={"email": "ada@example.test", "name": "Ada", "password": "test-pass"},
)
headers = header(setup.json()["token"])
profile_id = setup.json()["profile_id"]
settings = client.get("/api/journal/generation-settings", headers=headers)
expect(settings.status_code == 200, f"generation-settings {settings.text}")
body = settings.json()
expect(body["selection"] == default_ids(), "profile defaults are the named system standards")
expect(set(body["selection"]) == set(SELECTION_KEYS), "settings expose the four guideline ids")
for slot, options in body["options"].items():
expect(options, f"{slot} has active options")
expect(all("instruction" not in item for item in options), f"{slot} options hide prompt text")
expect(all(item["id"] != "draft" for item in options), f"{slot} options are real ids")
stored = get_or_create_selection(profile_id)
expect({key: stored[key] for key in SELECTION_KEYS} == default_ids(), "persisted defaults match system standards")
with get_db() as conn:
conn.execute(
"INSERT INTO profiles (id, email, name, password_hash, role) VALUES (?, ?, ?, ?, 'user')",
("other-profile", "other@example.test", "Other", "x"),
)
save_selection("other-profile", high_ids())
save_selection(profile_id, low_ids())
expect(load_selection(profile_id)["transformation_id"] == seed_id("transformation", "correction"), "profile A keeps its own selection")
expect(load_selection("other-profile")["transformation_id"] == seed_id("transformation", "substantial"), "profile B is isolated")
save_selection(profile_id, default_ids())
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Policy"})
day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": "2026-08-27"},
)
conv = client.post(
f"/api/journal/days/{day.json()['day']['id']}/conversations",
headers=headers,
json={"title": "Tag"},
)
turn = client.post(
f"/api/journal/conversations/{conv.json()['id']}/turn",
headers=headers,
json={"body": "Ich war am Markt. Vielleicht bleibe ich kürzer. Brot holen wollte ich noch, habe es aber nicht gemacht."},
)
expect(turn.status_code == 200, f"turn {turn.status_code}")
rejected = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [conv.json()["id"]], "generation_selection": {"transformation_id": "missing"}},
)
expect(rejected.status_code == 400, "unknown selection is rejected")
expect((rejected.json().get("detail") or {}).get("code") == "invalid_generation_selection", "rejection uses the selection code")
after_reject = client.get("/api/journal/generation-settings", headers=headers).json()
expect(after_reject["selection"] == default_ids(), "rejected values are not stored")
draft = client.post(
"/api/admin/generation-instructions/journal_generate",
headers=headers,
json={
"slot": "voice",
"guideline_key": "hidden_draft",
"label": "versteckt",
"summary": "Draft",
"instruction": "Nicht für neue Läufe.",
},
)
expect(draft.status_code == 200, f"admin draft {draft.text}")
blocked_draft = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={
"conversation_ids": [conv.json()["id"]],
"generation_selection": {**default_ids(), "voice_id": draft.json()["id"]},
},
)
expect(blocked_draft.status_code == 400, "drafts are rejected for new runs")
reset_debug()
recorder = install_test_recorder()
snapshot_run = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={
"conversation_ids": [conv.json()["id"]],
"generation_selection": low_ids(),
"remember_generation_selection": False,
},
)
expect(snapshot_run.status_code == 200, f"low generate {snapshot_run.text}")
intern = intern_of(snapshot_run.json())
meta = selection_meta(snapshot_run.json())
expect(meta.get("source") == "request", "trace names the request snapshot")
expect(meta.get("remembered") is False, "unremembered snapshot is marked")
expect(meta.get("ids") == {
"transformation": seed_id("transformation", "correction"),
"detail": seed_id("detail", "compact"),
"voice": seed_id("voice", "neutral"),
"narrative": seed_id("narrative", "chronicle"),
}, "trace keeps the used ids")
expect(meta.get("keys", {}).get("transformation") == "correction", "trace keeps guideline keys")
expect(meta.get("revisions", {}).get("transformation") == 1, "trace keeps revisions")
expect("editorial_mode" not in meta, "trace has no abandoned editorial mode")
expect("source_mode" not in meta, "trace has no abandoned source mode")
expect("Korrigiere nur" not in str(meta), "trace does not store compiled instruction text")
expect(
client.get("/api/journal/generation-settings", headers=headers).json()["selection"] == default_ids(),
"unremembered snapshot does not change profile defaults",
)
snapshot = snapshot_run.json().get("generation_snapshot") or {}
expect(snapshot.get("transformation", {}).get("key") == "correction", "draft stores transformation key")
expect(snapshot.get("prompt_slug") == "mvp.journal_generate", "draft stores prompt slug")
expect("instruction" not in str(snapshot), "draft snapshot omits instruction text")
expect("source_mode" not in snapshot, "draft snapshot has no source_mode")
expect("editorial_mode" not in snapshot, "draft snapshot has no editorial_mode")
summary = snapshot_run.json().get("generation_summary") or ""
expect(summary.startswith("Erzeugt mit:"), "draft summary is user-readable")
expect("Quellenmodus:" not in summary, "draft summary does not name a source mode")
expect("{{transformation_instructions}}" not in intern, "transformation placeholder is resolved")
expect("{{detail_instructions}}" not in intern, "detail placeholder is resolved")
expect("{{voice_instructions}}" not in intern, "voice placeholder is resolved")
expect("{{narrative_instructions}}" not in intern, "narrative placeholder is resolved")
expect("{{source_mode_instructions}}" not in intern, "source-mode placeholder is absent")
expect(MIXED_SOURCES_INSTRUCTION in intern, "mixed-source instruction reaches the prompt")
expect(seed_instruction("transformation", "correction") in intern, "selected transformation reaches the prompt")
expect(seed_instruction("detail", "compact") in intern, "selected detail reaches the prompt")
expect(seed_instruction("voice", "neutral") in intern, "selected voice reaches the prompt")
expect(seed_instruction("narrative", "chronicle") in intern, "selected narrative reaches the prompt")
expect([item.get("purpose") for item in recorder] == ["journal_generate"], "exactly one generate call")
expect(sum(1 for item in (snapshot_run.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "run log records one model call")
expect(
not any("Korrigiere nur Rechtschreibung" in str(item) for item in (snapshot_run.json().get("run_log") or [])),
"run log omits compiled prompt instructions",
)
for fact in HARD_FACTS:
expect(fact in intern, f"hard fact remains at low policy: {fact}")
notes_day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": "2026-08-28"},
)
notes_conv = client.post(
f"/api/journal/days/{notes_day.json()['day']['id']}/conversations",
headers=headers,
json={"title": "Notizen"},
)
client.post(
f"/api/journal/conversations/{notes_conv.json()['id']}/turn",
headers=headers,
json={"body": "- Markt\n- Brot nicht geholt\n- vielleicht kürzer bleiben"},
)
notes_run = client.post(
f"/api/journal/days/{notes_day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [notes_conv.json()["id"]], "generation_selection": default_ids()},
)
expect(notes_run.status_code == 200, f"notes generate {notes_run.text}")
notes_intern = intern_of(notes_run.json())
notes_snap = notes_run.json().get("generation_snapshot") or {}
expect("source_mode" not in notes_snap, "notes snapshot has no source mode")
expect("editorial_mode" not in (notes_run.json().get("trace") or {}), "notes trace has no editorial mode")
expect(MIXED_SOURCES_INSTRUCTION in notes_intern, "notes use the unified mixed-source instruction")
expect("Markt" in sources_block(notes_intern), "notes sources remain complete")
def generate_shapes(date: str, bodies: list[str], label: str) -> None:
shaped_day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": date},
)
shaped_conv = client.post(
f"/api/journal/days/{shaped_day.json()['day']['id']}/conversations",
headers=headers,
json={"title": label},
)
for body in bodies:
client.post(
f"/api/journal/conversations/{shaped_conv.json()['id']}/turn",
headers=headers,
json={"body": body},
)
reset_debug()
rec = install_test_recorder()
run = client.post(
f"/api/journal/days/{shaped_day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [shaped_conv.json()["id"]], "generation_selection": default_ids()},
)
expect(run.status_code == 200, f"{label} generate {run.text}")
shaped_intern = intern_of(run.json())
shaped_sources = sources_block(shaped_intern)
shaped_snap = run.json().get("generation_snapshot") or {}
expect(MIXED_SOURCES_INSTRUCTION in shaped_intern, f"{label} has mixed-source instruction")
expect("{{source_mode_instructions}}" not in shaped_intern, f"{label} has no source-mode placeholder")
expect("source_mode" not in shaped_snap, f"{label} snapshot has no source_mode")
expect("editorial_mode" not in shaped_snap, f"{label} snapshot has no editorial_mode")
expect("editorial_mode" not in (run.json().get("trace") or {}), f"{label} trace has no editorial_mode")
expect([item.get("purpose") for item in rec] == ["journal_generate"], f"{label} is exactly one generate call")
for body in bodies:
for line in body.splitlines():
token = line.lstrip("- ").strip()
if token:
expect(token in shaped_sources, f"{label} keeps source {token!r}")
generate_shapes("2026-08-11", ["Ich ging zum Markt. Es war voll. Vielleicht bleibe ich kürzer."], "prose-only")
generate_shapes("2026-08-12", ["- Kirschen\n- Brot nicht geholt\n- später Hafen"], "notes-only")
generate_shapes(
"2026-08-13",
["Ich ging zum Markt. Es war voll.", "- Kirschen\n- später Hafen"],
"prose-then-notes",
)
generate_shapes(
"2026-08-14",
["- Kirschen\n- später Hafen", "Ich ging zum Markt. Es war voll."],
"notes-then-prose",
)
generate_shapes(
"2026-08-15",
["Ich ging zum Markt. Es war voll.\n- Kirschen\n- später Hafen"],
"mixed-in-message",
)
high_run = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={
"conversation_ids": [conv.json()["id"]],
"generation_selection": high_ids(),
"remember_generation_selection": True,
},
)
expect(high_run.status_code == 200, f"high generate {high_run.text}")
high_intern = intern_of(high_run.json())
expect("Überarbeite den Text substanziell" in high_intern, "high transformation reaches the prompt")
expect("Korrigiere nur Rechtschreibung" not in high_intern, "low and high prompt blocks differ")
expect(intern != high_intern, "low and high snapshots render different prompt blocks")
expect(selection_meta(high_run.json()).get("remembered") is True, "remembered snapshot is marked")
remembered = client.get("/api/journal/generation-settings", headers=headers).json()
expect(remembered["selection"] == high_ids(), "remembered snapshot becomes the profile default")
for fact in HARD_FACTS:
expect(fact in high_intern, f"hard fact remains at high policy: {fact}")
profile_run = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [conv.json()["id"]]},
)
expect(profile_run.status_code == 200, f"profile generate {profile_run.text}")
expect(selection_meta(profile_run.json()).get("source") == "profile", "omitted snapshot uses stored values")
expect(seed_instruction("narrative", "emphasized") in intern_of(profile_run.json()), "stored high narrative is reused")
confirm_identity(profile_id, "Clarissa", aliases=["Sushi"])
alias_day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": "2026-08-21"},
)
alias_conv = client.post(
f"/api/journal/days/{alias_day.json()['day']['id']}/conversations",
headers=headers,
json={"title": "Alias"},
)
client.post(
f"/api/journal/conversations/{alias_conv.json()['id']}/turn",
headers=headers,
json={"body": "Sushi kam ins Wohnzimmer."},
)
def canonical_reply(_messages, _policy):
return ChatResult(
content="Wohnzimmer\n\nClarissa kam ins Wohnzimmer.",
model="fake",
usage={},
context_compression="disabled",
)
with patch("privacy_gateway.complete_model", canonical_reply):
alias_out = client.post(
f"/api/journal/days/{alias_day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [alias_conv.json()["id"]]},
)
expect(alias_out.status_code == 200, f"confirmed alias generate {alias_out.text}")
expect("Clarissa" in (alias_out.json().get("body") or ""), "canonical spelling is kept after demask")
expect((alias_out.json().get("trace") or {}).get("model_text_accepted") is True, "alias demask is not rejected as unattested")
remember_mapping(profile_id, "Hanna", "PERSON:99")
def invent(_messages, _policy):
return ChatResult(
content="Hanna am Hafen\n\nIch war am Markt.",
model="fake",
usage={},
context_compression="disabled",
)
with patch("privacy_gateway.complete_model", invent):
blocked = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [conv.json()["id"]]},
)
expect(blocked.status_code == 409, "unattested person is still rejected")
expect((blocked.json().get("detail") or {}).get("code") == "journal_generation_not_accepted", "409 provenance rule remains")
created_user = client.post(
"/api/users",
headers=headers,
json={"email": "policy-user@example.test", "name": "User", "password": "user-pass", "role": "user"},
)
expect(created_user.status_code == 200, f"create user {created_user.text}")
user_login = client.post(
"/api/auth/login",
json={"email": "policy-user@example.test", "password": "user-pass"},
)
user_headers = header(user_login.json()["token"])
denied = client.get("/api/admin/generation-instructions/journal_generate", headers=user_headers)
expect(denied.status_code == 403, "admin catalog is protected")
denied_put = client.put(
"/api/admin/generation-instructions/journal_generate/" + seed_id("transformation", "substantial"),
headers=user_headers,
json={"label": "nein"},
)
expect(denied_put.status_code == 403, "admin catalog write is protected")
catalog = client.get("/api/admin/generation-instructions/journal_generate", headers=headers)
expect(catalog.status_code == 200, f"admin catalog {catalog.text}")
expect("source_mode" not in catalog.json()["slots"], "admin catalog has no source-mode slot")
expect("instruction" not in str(catalog.json()["slots"]["transformation"]), "admin overview omits prompt bodies")
cloned = client.post(
f"/api/admin/generation-instructions/journal_generate/{seed_id('transformation', 'substantial')}/clone",
headers=headers,
)
expect(cloned.status_code == 200, f"clone {cloned.text}")
saved = client.put(
f"/api/admin/generation-instructions/journal_generate/{cloned.json()['id']}",
headers=headers,
json={
"guideline_key": "substantial",
"label": "substanziell",
"summary": "Klon",
"instruction": "ADMIN_CUSTOM_TRANSFORMATION_BLOCK",
},
)
expect(saved.status_code == 200, f"admin draft save {saved.text}")
published = client.post(
f"/api/admin/generation-instructions/journal_generate/{cloned.json()['id']}/publish",
headers=headers,
)
expect(published.status_code == 200, f"publish {published.text}")
immutable = client.put(
f"/api/admin/generation-instructions/journal_generate/{seed_id('transformation', 'substantial')}",
headers=headers,
json={"label": "überschreiben", "instruction": "should not stick", "guideline_key": "substantial"},
)
expect(immutable.status_code == 409, "active guidelines cannot be overwritten")
preview = client.post(
"/api/admin/generation-instructions/journal_generate/preview",
headers=headers,
json={
"generation_selection": {**high_ids(), "transformation_id": cloned.json()["id"]},
},
)
expect(preview.status_code == 200, f"admin preview {preview.text}")
expect(preview.json()["transformation_instructions"] == "ADMIN_CUSTOM_TRANSFORMATION_BLOCK", "preview compiles locally")
expect("(nicht enthalten)" in (preview.json().get("rendered") or ""), "preview has no personal sources")
expect("{{transformation_instructions}}" not in (preview.json().get("rendered") or ""), "preview resolves placeholders")
custom_run = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={
"conversation_ids": [conv.json()["id"]],
"generation_selection": {**high_ids(), "transformation_id": cloned.json()["id"]},
"remember_generation_selection": False,
},
)
expect(custom_run.status_code == 200, f"custom generate {custom_run.text}")
expect("ADMIN_CUSTOM_TRANSFORMATION_BLOCK" in intern_of(custom_run.json()), "admin change reaches the next rendered prompt")
init_db()
after_seed = overview_payload()
substantial = next(
item for item in after_seed["slots"]["transformation"] if item["id"] == seed_id("transformation", "substantial")
)
expect(get_guideline(substantial["id"], include_instruction=True)["instruction"] == seed_instruction("transformation", "substantial"), "seed does not overwrite the original")
expect(get_guideline(cloned.json()["id"], include_instruction=True)["instruction"] == "ADMIN_CUSTOM_TRANSFORMATION_BLOCK", "seed does not overwrite admin clones")
restored = client.post(
"/api/admin/generation-instructions/journal_generate/reset",
headers=headers,
)
expect(restored.status_code == 200, f"reset {restored.text}")
drafts = [item for item in restored.json()["slots"]["transformation"] if item["status"] == "draft"]
expect(drafts, "reset adds seed drafts")
expect(
get_guideline(cloned.json()["id"], include_instruction=True)["instruction"] == "ADMIN_CUSTOM_TRANSFORMATION_BLOCK",
"reset does not overwrite historical variants",
)
archived = client.post(
f"/api/admin/generation-instructions/journal_generate/{cloned.json()['id']}/archive",
headers=headers,
)
expect(archived.status_code == 200, f"archive {archived.text}")
archived_run = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={
"conversation_ids": [conv.json()["id"]],
"generation_selection": {**high_ids(), "transformation_id": cloned.json()["id"]},
},
)
expect(archived_run.status_code == 400, "archived guidelines are rejected for new runs")
fresh_day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": "2026-08-16"},
)
fresh_conv = client.post(
f"/api/journal/days/{fresh_day.json()['day']['id']}/conversations",
headers=headers,
json={"title": "Legacy"},
)
client.post(
f"/api/journal/conversations/{fresh_conv.json()['id']}/turn",
headers=headers,
json={"body": "Ich war am Markt."},
)
with get_db() as conn:
current_template = conn.execute(
"SELECT template FROM ai_prompts WHERE slug = ?",
("mvp.journal_generate",),
).fetchone()["template"]
conn.execute(
"UPDATE ai_prompts SET template = ? WHERE slug = ?",
((current_template or "") + "\n{{source_mode_instructions}}\n", "mvp.journal_generate"),
)
reset_debug()
legacy_recorder = install_test_recorder()
legacy = client.post(
f"/api/journal/days/{fresh_day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [fresh_conv.json()["id"]], "generation_selection": default_ids()},
)
expect(legacy.status_code == 409, "legacy custom prompt is refused before the provider")
legacy_detail = legacy.json().get("detail") or {}
expect(legacy_detail.get("code") == "prompt_contract_incompatible", "legacy prompt uses prompt_contract_incompatible")
expect("veralteten Platzhalter" in (legacy_detail.get("message") or ""), "admin hint names the retired placeholder")
expect(legacy_recorder == [], "legacy custom prompt does not call the provider")
expect(current_draft(profile_id, fresh_day.json()["day"]["id"]) is None, "legacy custom prompt inserts no raw draft")
with get_db() as conn:
conn.execute(
"UPDATE ai_prompts SET template = default_template WHERE slug = ?",
("mvp.journal_generate",),
)
with get_db() as conn:
conn.execute(
"""
UPDATE generation_guidelines
SET instruction = ''
WHERE id = ?
""",
(seed_id("transformation", "substantial"),),
)
reset_debug()
broken_recorder = install_test_recorder()
broken = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [conv.json()["id"]], "generation_selection": default_ids()},
)
expect(broken.status_code == 409, "invalid catalog refuses generate")
expect((broken.json().get("detail") or {}).get("code") == "generation_policy_invalid", "invalid catalog uses generation_policy_invalid")
expect(broken_recorder == [], "invalid catalog does not call the provider")
with get_db() as conn:
conn.execute(
"""
UPDATE generation_guidelines
SET instruction = ?
WHERE id = ?
""",
(seed_instruction("transformation", "substantial"), seed_id("transformation", "substantial")),
)
with get_db() as conn:
conn.execute(
"UPDATE ai_prompts SET template = 'CUSTOM POLICY PROMPT {{reconstruction}}' WHERE slug = ?",
("mvp.journal_generate",),
)
init_db()
custom = load_active_prompt("mvp.journal_generate")
expect(custom["template"] == "CUSTOM POLICY PROMPT {{reconstruction}}", "independently edited prompt is not overwritten")
with get_db() as conn:
row = conn.execute(
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
("mvp.journal_generate",),
).fetchone()
expect("{{transformation_instructions}}" in (row["default_template"] or ""), "default template still tracks mixed sources")
expect(MIXED_SOURCES_INSTRUCTION in (row["default_template"] or ""), "default template has mixed-source instruction")
expect(row["seed_revision"] == "2026-08-27-journal-mixed-sources-v1", "revision updates even when template is custom")
readable = snapshot_summary(
{
"transformation": {"label": "spürbar"},
"detail": {"label": "weitgehend"},
"voice": {"label": "deutlich"},
"narrative": {"label": "gewichtet"},
}
)
expect("Erzeugt mit:\nSpürbar · Weitgehend · Deutlich · Gewichtet" in readable, "snapshot summary is readable")
expect("Quellenmodus" not in readable, "snapshot summary does not name a source mode")
print("journal generation policy tests passed.")
if __name__ == "__main__":
main()
+39 -16
View File
@@ -49,21 +49,25 @@ def test_active_prompt_contract() -> None:
init_db()
prompt = load_active_prompt("mvp.journal_generate")
text = prompt.get("template") or ""
expect("Faktentreue ist nicht Wortlauttreue" in text, "active prompt separates fact fidelity from wording")
expect("INHALTSTREUE" in text, "active prompt keeps hard content rules")
expect("CURRENT_DAY_SOURCES" in text, "active prompt labels current-day facts")
expect("STYLE_EXAMPLES" in text, "active prompt labels style examples")
expect("Rechtschreibung" in text or "korrigieren" in text, "active prompt allows spelling and grammar fixes")
expect("Unsicherheit bleibt Unsicherheit" in text, "active prompt keeps semantic uncertainty")
expect("keine neuen Informationen" in text.lower() or "Keine neuen Informationen" in text, "active prompt still forbids new facts")
expect("Verneinungen" in text and "Unsicherheiten" in text, "active prompt keeps semantic uncertainty")
expect("keine neuen tatsachen" in text.lower(), "active prompt still forbids new facts")
expect("Plan und Vollzug" in text, "active prompt keeps plan versus completion")
expect("[[…" not in text and "[[..." not in text, "active prompt must not teach ellipsis placeholders")
expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone")
expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone")
expect("{{transformation_instructions}}" in text, "active prompt uses compiled transformation instructions")
expect("{{source_mode_instructions}}" not in text, "active prompt has no source-mode placeholder")
expect("Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus." in text, "active prompt has mixed-source instruction")
with get_db() as conn:
row = conn.execute(
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
("mvp.journal_generate",),
).fetchone()
expect(row["seed_revision"] == "2026-08-27-journal-placeholders-v1", "system prompt revision is stored")
expect(row["seed_revision"] == "2026-08-27-journal-mixed-sources-v1", "system prompt revision is stored")
expect(row["template"] == row["default_template"], "untouched install uses the seeded template")
@@ -118,6 +122,24 @@ def test_unattested_identity_is_journal_not_privacy() -> None:
is None,
"food homonym of an unused mapping is not unattested identity",
)
expect(
unattested_journal_content(
"Clarissa kam ins Wohnzimmer.",
["Heute war Sushi im Wohnzimmer."],
[
{
"local_label": "Clarissa",
"canonical_label": "Clarissa",
"token": "PERSON:01",
"aliases": ["Sushi"],
"demask_label": "Clarissa",
}
],
["PERSON:01"],
)
is None,
"confirmed alias in the source attests the canonical demasked spelling",
)
def main() -> None:
@@ -165,11 +187,11 @@ def main() -> None:
for stage in (gen.json().get("trace") or {}).get("stages") or []:
if stage.get("purpose") == "journal_generate":
intern = stage.get("intern") or ""
style = intern.split("CURRENT_DAY_SOURCES")[0]
style = intern.split("\nCURRENT_DAY_SOURCES\n")[0]
expect("Neutraler Journalstil" in style, "neutral style reaches the generate prompt")
expect("Erzählmerkmale" not in style, "current day dialogue is not a style brief")
expect("zimlich" not in style, "today's typo is not a style exemplar")
expect("Faktentreue ist nicht Wortlauttreue" in intern, "runtime intern uses the new narration contract")
expect("INHALTSTREUE" in intern, "runtime intern uses the new narration contract")
expect("CURRENT_DAY_SOURCES" in intern, "runtime intern labels current-day facts")
expect("Nur den verifizierten Nutzerwortlaut" not in intern, "old output-wording rule is not in the runtime prompt")
expect("Unsicherheiten im Wortlaut" not in intern, "old uncertainty-in-wording rule is not in the runtime prompt")
@@ -252,7 +274,7 @@ def main() -> None:
for stage in (second.json().get("trace") or {}).get("stages") or []:
if stage.get("purpose") == "journal_generate":
intern2 = stage.get("intern") or ""
expect("trockener Schnitt" in intern2.split("CURRENT_DAY_SOURCES")[0], "confirmed writing profile reaches generate")
expect("trockener Schnitt" in intern2.split("\nCURRENT_DAY_SOURCES\n")[0], "confirmed writing profile reaches generate")
def invent(_messages, _policy):
return ChatResult(
@@ -268,13 +290,14 @@ def main() -> None:
headers=headers,
json={"conversation_ids": [conv.json()["id"]]},
)
expect(blocked.status_code == 200, f"unattested generate {blocked.text}")
log = blocked.json().get("run_log") or []
expect(any(item.get("reason") == "unattested_identity" for item in log), "unattested person uses local fallback")
expect(any(item.get("status") == "local_fallback" for item in log), "fallback path is distinct from the model path")
expect("Clarissa" not in (blocked.json().get("body") or ""), "unattested person is not in the accepted draft")
expect("markt" in (blocked.json().get("body") or "").lower(), "fallback still has the user sources")
expect("rote tasche" in (blocked.json().get("body") or "").lower(), "fallback keeps the unique source detail")
expect(blocked.status_code == 409, f"unattested generate {blocked.text}")
detail = blocked.json().get("detail") or {}
expect(detail.get("code") == "journal_generation_not_accepted", "unattested person is not stored as a draft")
expect(detail.get("message") == "Generierung nicht übernommen.", "API names the rejection")
log = (detail.get("diagnostics") or {}).get("log") or []
expect(any(item.get("reason") == "unattested_identity" for item in log), "unattested person is a provenance reject")
expect(any(item.get("status") == "not_accepted" for item in log), "reject path is distinct from the model path")
expect("Clarissa" not in json.dumps((detail.get("diagnostics") or {}).get("log") or []), "clear labels stay out of compact logs")
with get_db() as conn:
conn.execute(
@@ -289,8 +312,8 @@ def main() -> None:
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
("mvp.journal_generate",),
).fetchone()
expect("Faktentreue ist nicht Wortlauttreue" in (row["default_template"] or ""), "default template still tracks the seed")
expect(row["seed_revision"] == "2026-08-27-journal-placeholders-v1", "revision updates even when template is custom")
expect("INHALTSTREUE" in (row["default_template"] or ""), "default template still tracks the seed")
expect(row["seed_revision"] == "2026-08-27-journal-mixed-sources-v1", "revision updates even when template is custom")
print("journal narration tests passed.")
+1 -1
View File
@@ -328,7 +328,7 @@ def main() -> None:
"A style brief reaches generate",
)
expect("STYLE_EXAMPLES" in narrate_intern, "A style examples block is labeled")
style_part = narrate_intern.split("CURRENT_DAY_SOURCES")[0]
style_part = narrate_intern.split("\nCURRENT_DAY_SOURCES\n")[0]
expect("Erzählmerkmale" not in style_part, "A current day dialogue is not appended as style signals")
expect("Neutraler Journalstil" in style_part or "Core:" in style_part, "A empty profile uses the neutral journal voice")
expect(
+3 -6
View File
@@ -503,12 +503,9 @@ def main() -> None:
_demask("[[ person:01 ]] war ruhig.", anna_manifest) == "Anna war ruhig.",
"demask is case- and space-insensitive",
)
blocked_plain = False
try:
_validate_response("Anna war ruhig.", anna_manifest)
except PrivacyGatewayError as exc:
blocked_plain = exc.code == "response_validation_failed"
expect(blocked_plain, "plaintext identity is blocked before demask")
tokenized = _validate_response("Anna war ruhig.", anna_manifest)
expect(tokenized.startswith("[[PERSON:01]]"), "active cleartext is normalized, not discarded")
expect(_demask(tokenized, anna_manifest) == "Anna war ruhig.", "normalized plaintext demasks")
no_user = client.get("/api/admin/identities")
expect(no_user.status_code in {401, 403}, "identity registry requires auth")
+17 -2
View File
@@ -9,7 +9,7 @@ ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
os.environ.setdefault("KANSHO_FAKE_DETECT", "1")
from entity_detect_eval import run_fake
from entity_detect_eval import CASES, run_fake, score_spans
def expect(ok: bool, message: str) -> None:
@@ -22,7 +22,22 @@ def main() -> None:
payload = run_fake()
expect(payload["mode"] == "fake", "eval default is fake")
expect(payload["live_quality"] == "unconfirmed", "live quality stays unconfirmed")
expect(len(payload["cases"]) >= 3, "synthetic cases exist")
expect(payload["detect_model_unconfirmed"] == "openai/gpt-4.1-nano", "configured detect model stays unconfirmed")
expect(len(payload["cases"]) >= 8, "synthetic span cases exist")
person = next(item for item in CASES if item["id"] == "no_sentence_span")
metrics = score_spans(
[{"start": 9, "end": 13, "text": "Anna", "entity_type": "PERSON"}],
person["expected"],
)
expect(metrics["expected_found"] == 1, "exact expected span is counted")
expect(metrics["precision"] == 1.0 and metrics["recall"] == 1.0, "exact span match is precision 1")
wrong = score_spans(
[{"start": 0, "end": 28, "text": "Ich traf Anna am Nachmittag.", "entity_type": "PERSON"}],
person["expected"],
)
expect(wrong["unexpected"] == 1, "sentence-sized span is unexpected")
food = next(item for item in payload["cases"] if item["id"] == "same_word_two_roles")
expect(food["entities"]["metrics"]["expected_found"] == 1, "fake contract finds only the identity sushi")
print("All detect eval harness tests passed.")
+100 -42
View File
@@ -22,12 +22,12 @@ from identity_store import remember_mapping
from main import app
from privacy_gateway import (
ERROR_EGRESS_VALIDATION,
IDENTITY_LEAK_RETRY,
ActiveReplacement,
GatewayRequest,
MaskingManifest,
PrivacyGatewayError,
_demask,
_process_response,
_validate_response,
complete,
last_trace,
@@ -90,14 +90,10 @@ def main() -> None:
kinship = mask_prompt("Meine Frau Sushi kam später.", sushi, "journal_generate")
expect(len(kinship.replacements) == 1, "identity mention of the same word is active")
blocked = False
try:
_validate_response("Sushi kam vorbei.", kinship)
except PrivacyGatewayError as exc:
blocked = exc.code == "response_validation_failed"
expect(exc.diagnostics.get("leak_tokens") == ["PERSON:01"], "block reports the token, not the label")
expect("Sushi" not in json.dumps(exc.diagnostics), "validation diagnostics omit the clear name")
expect(blocked, "active clear name in the raw reply is a leak")
normalized, meta = _process_response("Sushi kam vorbei.", kinship)
expect(normalized.startswith("[[PERSON:01]]"), "active cleartext is remapped to the request token")
expect(meta.get("response_normalization") == "active_cleartext_normalized", "normalization is named")
expect(_demask(normalized, kinship) == "Sushi kam vorbei.", "normalized reply demasks locally")
homonym_ok = _validate_response("Danach Sushi essen.", kinship)
expect("Sushi essen" in homonym_ok, "homonym in the reply uses the same classification rule")
@@ -107,7 +103,9 @@ def main() -> None:
leftover = _demask("[[PERSON:01]] und [[PERSON:99]]", kinship)
expect("Sushi" in leftover and "[[PERSON:99]]" in leftover, "inactive placeholder is not rematerialized")
expect("Clarissa" not in leftover, "inactive mapping label is not introduced by demask")
expect("[[" not in IDENTITY_LEAK_RETRY, "retry instruction must not teach bracket placeholders")
import privacy_gateway as gw
expect(not hasattr(gw, "IDENTITY_LEAK_RETRY"), "automatic identity-leak retry instruction is gone")
expect(
_demask("[[ person:01 ]] war ruhig.", kinship) == "Sushi war ruhig.",
"demask ignores case and inner spacing",
@@ -121,6 +119,25 @@ def main() -> None:
"generic ellipsis placeholders are not rematerialized",
)
alias_rows = [
{
"local_label": "Sushi",
"token": "PERSON:07",
"canonical_label": "Clarissa",
"demask_label": "Clarissa",
"aliases": ["Sushi"],
"labels": ["Clarissa", "Sushi"],
"entity_type": "PERSON",
}
]
aliased = mask_prompt("Sushi kam ins Wohnzimmer.", alias_rows, "journal_generate")
expect(len(aliased.replacements) == 1, "confirmed alias activates its token")
spellings = {label.casefold() for label in aliased.replacements[0].all_labels()}
expect(spellings == {"sushi", "clarissa"}, "request manifest keeps every spelling of the active token")
expect(_demask("[[PERSON:07]] kam.", aliased) == "Clarissa kam.", "confirmed alias demasks to the canonical spelling")
canonical_clear, _meta = _process_response("Clarissa kam vorbei.", aliased)
expect(canonical_clear.startswith("[[PERSON:07]]"), "canonical spelling of an active alias token is remapped")
overlap = mask_prompt(
"Anna-Lena kam vorbei.",
[
@@ -133,6 +150,54 @@ def main() -> None:
expect("[[PERSON:08]]" in overlap.masked_text, "longer label is the one replaced")
expect("Anna-Lena" not in overlap.masked_text, "replaced longer label is gone")
mixed_word = "Ich aß Sushi. Sushi kam später."
second = mixed_word.find("Sushi", mixed_word.find("Sushi") + 1)
span_only = mask_prompt(
mixed_word,
[
{
"local_label": "Sushi",
"token": "PERSON:01",
"entity_type": "PERSON",
"start": second,
"end": second + 5,
"text": "Sushi",
"source": "request_local",
}
],
"journal_generate",
)
expect(span_only.masked_text.startswith("Ich aß Sushi."), "undetected same wording stays unmasked")
expect("[[PERSON:01]] kam später." in span_only.masked_text, "only the detected span is masked")
validate_pre_egress(span_only.masked_text, span_only)
overlapping_spans = mask_prompt(
"Anna-Lena kam vorbei.",
[
{
"local_label": "Anna",
"token": "PERSON:01",
"entity_type": "PERSON",
"start": 0,
"end": 4,
"text": "Anna",
"source": "request_local",
},
{
"local_label": "Anna-Lena",
"token": "PERSON:08",
"entity_type": "PERSON",
"start": 0,
"end": 9,
"text": "Anna-Lena",
"source": "request_local",
},
],
"journal_generate",
)
expect("[[PERSON:08]]" in overlapping_spans.masked_text, "longest overlapping span wins")
expect("[[PERSON:01]]" not in overlapping_spans.masked_text, "shorter overlapping span is dropped")
subject = mask_prompt(
"Clarissa kaufte Kirschen.",
[{"local_label": "Clarissa", "token": "PERSON:99"}],
@@ -166,7 +231,7 @@ def main() -> None:
second = pool.submit(check, "Heute nur Markt.", anna + lars, "Anna kam.")
code_a, _text_a, tokens_a = first.result()
code_b, text_b, tokens_b = second.result()
expect(code_a == "response_validation_failed", "parallel request A still blocks its own active leak")
expect(code_a == "ok" and "[[PERSON:01]]" in (_text_a or ""), "parallel request A normalizes its own active cleartext")
expect(tokens_a == ["PERSON:01"], "parallel request A keeps its own active set")
expect(code_b == "ok" and "Anna kam" in (text_b or ""), "parallel request B is not blocked by A's mapping")
expect(tokens_b == [], "parallel request B does not inherit A's active set")
@@ -218,30 +283,26 @@ def main() -> None:
seen = []
def leak_then_ok(messages, _policy):
def leak_once(messages, _policy):
seen.append(messages[0].get("content") or "")
if len(seen) == 1:
return ChatResult(
content="Anna stand den ganzen Nachmittag am Markt.",
model="fake",
usage={"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18, "cost": 0.01},
context_compression="not_applicable",
)
return ChatResult(
content="[[PERSON:01]] stand den ganzen Nachmittag am Markt.",
content="Anna stand den ganzen Nachmittag am Markt.",
model="fake",
usage={"prompt_tokens": 12, "completion_tokens": 8, "total_tokens": 20, "cost": 0.02},
usage={"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18, "cost": 0.01},
context_compression="not_applicable",
)
with patch("privacy_gateway.complete_model", leak_then_ok):
with patch("privacy_gateway.complete_model", leak_once):
repaired = _run_gateway(profile_id, "Ich war mit Anna am Markt.")
expect(len(seen) == 2, "a real leak retries exactly once")
expect(IDENTITY_LEAK_RETRY in seen[1], "retry uses the generic correction instruction")
expect("Anna stand den ganzen Nachmittag" not in seen[1], "discarded raw reply is not part of the retry prompt")
expect("[[PERSON:01]]" in seen[0] and "Anna" not in seen[0].replace("[[PERSON:01]]", ""), "retry keeps the already masked prompt")
expect("Anna" in (repaired.content or ""), "successful retry is demasked locally")
expect(repaired.diagnostics.get("response_validation_retry") == 1, "retry is recorded without the raw reply")
expect(len(seen) == 1, "active cleartext does not start a second generate call")
expect("[[PERSON:01]]" in seen[0] and "Anna" not in seen[0].replace("[[PERSON:01]]", ""), "egress stays masked")
expect("Anna" in (repaired.content or ""), "normalized cleartext is demasked locally")
expect(repaired.diagnostics.get("generate_calls") == 1, "exactly one generate call")
expect(
repaired.diagnostics.get("response_normalization") == "active_cleartext_normalized",
"active cleartext is recorded as local normalization",
)
expect("Anna" not in json.dumps(repaired.diagnostics), "compact diagnostics omit the clear name")
def spaced_token(_messages, _policy):
return ChatResult(
@@ -258,10 +319,10 @@ def main() -> None:
"gateway demasks case and spacing variants",
)
expect("[[" not in (restored.content or ""), "no leftover placeholder after a successful demask")
expect(repaired.diagnostics.get("prompt_tokens") == 22, "retry aggregates prompt tokens")
expect(repaired.diagnostics.get("completion_tokens") == 16, "retry aggregates completion tokens")
expect(repaired.diagnostics.get("total_tokens") == 38, "retry aggregates total tokens")
expect(abs(float(repaired.diagnostics.get("cost") or 0) - 0.03) < 1e-9, "retry aggregates cost")
expect(repaired.diagnostics.get("prompt_tokens") == 10, "single call keeps prompt tokens")
expect(repaired.diagnostics.get("completion_tokens") == 8, "single call keeps completion tokens")
expect(repaired.diagnostics.get("total_tokens") == 18, "single call keeps total tokens")
expect(abs(float(repaired.diagnostics.get("cost") or 0) - 0.01) < 1e-9, "single call keeps cost")
inactive_calls = {"n": 0}
@@ -287,20 +348,17 @@ def main() -> None:
return ChatResult(
content="Anna stand den ganzen Nachmittag am Markt.",
model="fake",
usage={},
usage={"prompt_tokens": 4, "completion_tokens": 6, "total_tokens": 10, "cost": 0.002},
context_compression="not_applicable",
)
failed = False
with patch("privacy_gateway.complete_model", always_leak):
try:
_run_gateway(profile_id, "Ich war mit Anna am Markt.")
except PrivacyGatewayError as exc:
failed = exc.code == "response_validation_failed"
expect(exc.diagnostics.get("response_validation_retry") == 1, "failed retry is still only one extra call")
expect("Anna stand" not in json.dumps(exc.diagnostics.get("log") or []), "discarded reply is not persisted in the log")
expect(failed, "second leak stays fail-closed")
expect(len(retry_fail) == 2, "gateway stops after one retry")
accepted = _run_gateway(profile_id, "Ich war mit Anna am Markt.")
expect(len(retry_fail) == 1, "cleartext in the reply does not start a second model call")
expect("Anna" in (accepted.content or ""), "source-attested name is demasked after local normalization")
expect(accepted.diagnostics.get("generate_calls") == 1, "still exactly one generate call")
expect(accepted.trace.get("model") == "fake" or accepted.diagnostics.get("model") == "fake", "error-free trace keeps the model")
expect(accepted.diagnostics.get("completion_tokens") == 6, "token usage of the only call remains")
print("All privacy manifest tests passed.")
@@ -0,0 +1,353 @@
"""Response integrity, span masking, journal rejection, and diagnosis contract."""
from __future__ import annotations
import os
import sys
import tempfile
from pathlib import Path
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
REPO = ROOT.parent
sys.path.insert(0, str(ROOT))
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-response-test.sqlite")
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
os.environ["KANSHO_FAKE_DETECT"] = "1"
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
from fastapi.testclient import TestClient
from identity_store import list_mappings, remember_mapping
from journal_generate import JOURNAL_NOT_ACCEPTED, JOURNAL_NOT_ACCEPTED_MESSAGE
from journal_store import current_draft
from main import app
from privacy_gateway import (
ERROR_EGRESS_VALIDATION,
ActiveReplacement,
GatewayRequest,
MaskingManifest,
PrivacyGatewayError,
complete,
mask_prompt,
reset_debug,
validate_pre_egress,
)
from providers import ChatResult
def expect(ok: bool, message: str) -> None:
if not ok:
raise SystemExit(f"FAIL: {message}")
print(f"OK {message}")
def header(token: str) -> dict:
return {"X-Auth-Token": token}
def _run_gateway(profile_id: str, rendered: str, *, purpose: str = "dialogue_turn"):
return complete(
GatewayRequest(
prompt_id="response-integrity",
purpose=purpose,
data_class="B",
profile_id=profile_id,
payload={"rendered": rendered, "source_text": rendered, "prompt_slug": "mvp.journal_generate"},
)
)
def _day_with_turn(client: TestClient, headers: dict, body: str, title: str = "Tag") -> tuple[str, str]:
space = client.post("/api/journal/spaces", headers=headers, json={"title": title})
day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": "2026-08-27"},
)
conv = client.post(
f"/api/journal/days/{day.json()['day']['id']}/conversations",
headers=headers,
json={"title": "Gespräch"},
)
turn = client.post(
f"/api/journal/conversations/{conv.json()['id']}/turn",
headers=headers,
json={"body": body},
)
if turn.status_code != 200:
raise SystemExit(f"FAIL: setup turn {turn.status_code}")
print("OK setup turn 200")
return day.json()["day"]["id"], conv.json()["id"]
def _generate(client: TestClient, headers: dict, day_id: str, conv_id: str, reply: str, usage=None):
calls = {"n": 0}
def fake(_messages, _policy):
calls["n"] += 1
return ChatResult(
content=reply,
model="fake-gpt",
usage=usage or {"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 12, "cost": 0.001},
context_compression="disabled",
)
with patch("privacy_gateway.complete_model", fake):
response = client.post(
f"/api/journal/days/{day_id}/generate",
headers=headers,
json={"conversation_ids": [conv_id]},
)
return response, calls["n"]
def test_span_masking_and_pre_egress() -> None:
text = "Ich aß Sushi. Sushi kam später."
second = text.find("Sushi", text.find("Sushi") + 1)
manifest = mask_prompt(
text,
[
{
"local_label": "Sushi",
"token": "PERSON:01",
"entity_type": "PERSON",
"start": second,
"end": second + 5,
"text": "Sushi",
"source": "request_local",
}
],
"journal_generate",
)
expect(manifest.masked_text == "Ich aß Sushi. [[PERSON:01]] kam später.", "only the detected span is masked")
validate_pre_egress(manifest.masked_text, manifest)
overlapping = mask_prompt(
"Anna-Lena kam vorbei.",
[
{
"local_label": "Anna",
"token": "PERSON:01",
"entity_type": "PERSON",
"start": 0,
"end": 4,
"text": "Anna",
"source": "request_local",
},
{
"local_label": "Anna-Lena",
"token": "PERSON:08",
"entity_type": "PERSON",
"start": 0,
"end": 9,
"text": "Anna-Lena",
"source": "request_local",
},
],
"journal_generate",
)
expect(overlapping.masked_text.startswith("[[PERSON:08]]"), "overlapping spans pick the longest")
leaky = MaskingManifest(
masked_text="Anna war am Markt.",
available_mapping_count=1,
replacements=(ActiveReplacement("PERSON:01", "PERSON", 1, "Anna"),),
)
blocked = False
try:
validate_pre_egress("Anna war am Markt.", leaky)
except PrivacyGatewayError as exc:
blocked = exc.code == ERROR_EGRESS_VALIDATION
expect(blocked, "privacy stays fail-closed before the provider")
def test_runtime_has_no_incident_wordlists() -> None:
forbidden = (
"Kinder",
"Delfine",
"delphine",
"Wohnzimmer",
"Balkon",
"Tagesreflektion",
"Lebenwesen",
"Baguette",
)
runtime = []
for path in ROOT.glob("*.py"):
if path.name.startswith("entity_detect_eval") or path.name.startswith("_"):
continue
text = path.read_text(encoding="utf-8")
for word in forbidden:
if word in text:
runtime.append(f"{path.name}:{word}")
expect(not runtime, f"incident words must not be hardcoded in runtime: {runtime}")
def test_frontend_degraded_copy() -> None:
page = (REPO / "frontend" / "src" / "pages" / "JournalDayPage.jsx").read_text(encoding="utf-8")
trace = (REPO / "frontend" / "src" / "components" / "CallTrace.jsx").read_text(encoding="utf-8")
log = (REPO / "frontend" / "src" / "components" / "RunLogPopup.jsx").read_text(encoding="utf-8")
expect("Generierung nicht übernommen" in page, "journal page names a rejected generate")
expect("Generierung nicht übernommen" in trace, "trace names a rejected generate")
expect("Generierung nicht übernommen" in log, "run log names a rejected generate")
expect("nur user-Zeilen" not in trace, "masking copy is not limited to user lines")
expect("zwei Modellaufrufe nacheinander" not in log, "running copy does not claim two model calls")
def main() -> None:
reset_debug()
test_span_masking_and_pre_egress()
test_runtime_has_no_incident_wordlists()
test_frontend_degraded_copy()
with TestClient(app) as client:
setup = client.post(
"/api/auth/setup",
json={"email": "response@example.test", "name": "Response", "password": "test-pass"},
)
headers = header(setup.json()["token"])
profile_id = setup.json()["profile_id"]
calls = {"n": 0}
def false_positive_cleartext(_messages, _policy):
calls["n"] += 1
return ChatResult(
content="Danach Sushi essen am Markt.",
model="fake-gpt",
usage={"prompt_tokens": 8, "completion_tokens": 6, "total_tokens": 14, "cost": 0.002},
context_compression="not_applicable",
)
remember_mapping(profile_id, "Sushi", "PERSON:01")
with patch("privacy_gateway.complete_model", false_positive_cleartext):
result = _run_gateway(profile_id, "Ich aß Sushi. Meine Frau Sushi kam später.")
expect(calls["n"] == 1, "false-positive cleartext still uses exactly one generate call")
expect("Sushi" in (result.content or ""), "reconstructed common noun is kept after local normalization")
expect(result.diagnostics.get("generate_calls") == 1, "gateway reports one generate call")
expect(
result.diagnostics.get("response_normalization") in {"active_cleartext_normalized", "none"},
"normalization is recorded without a retry",
)
remember_mapping(profile_id, "Anna", "PERSON:02")
day_id, conv_id = _day_with_turn(client, headers, "Heute war ich mit Anna am Markt.")
gen, n = _generate(
client,
headers,
day_id,
conv_id,
"Ein Markttag\n\nAnna stand den ganzen Nachmittag am Markt.",
)
expect(gen.status_code == 200, f"attested person cleartext {gen.status_code}")
expect(n == 1, "attested person cleartext does not retry")
expect("Anna" in (gen.json().get("body") or ""), "attested person is demasked")
expect((gen.json().get("trace") or {}).get("model_text_accepted") is True, "attested model text is accepted")
remember_mapping(profile_id, "Clarissa", "PERSON:99")
hist_day, hist_conv = _day_with_turn(client, headers, "Heute nur Markt und Kirschen.", title="Historisch")
blocked, n = _generate(
client,
headers,
hist_day,
hist_conv,
"Ein Markttag\n\nClarissa kaufte Kirschen am Markt.",
)
expect(blocked.status_code == 409, f"style-only person {blocked.status_code}")
detail = blocked.json().get("detail") or {}
expect(detail.get("code") == JOURNAL_NOT_ACCEPTED, "historical-only name is rejected")
expect(detail.get("message") == JOURNAL_NOT_ACCEPTED_MESSAGE, "API message is explicit")
expect(n == 1, "provenance reject does not retry")
expect(current_draft(profile_id, hist_day) is None, "rejected generate inserts no draft")
diag = detail.get("diagnostics") or {}
trace = diag.get("trace") or {}
expect(trace.get("abort_reason") == "unattested_identity", "provenance abort reason remains")
expect(trace.get("model_text_accepted") is False, "model text is marked not accepted")
expect(
trace.get("model") == "fake-gpt" or (trace.get("budget") or {}).get("model") == "fake-gpt",
"error trace keeps the model",
)
expect(
(trace.get("generate_calls") == 1)
or ((trace.get("budget") or {}).get("generate_calls") == 1)
or diag.get("generate_calls") == 1,
"error trace keeps generate_calls",
)
expect(
(trace.get("budget") or {}).get("completion_tokens") == 5 or diag.get("completion_tokens") == 5,
"error trace keeps completion tokens",
)
unknown_day, unknown_conv = _day_with_turn(client, headers, "Heute nur der Markt.", title="Platzhalter")
unknown, n = _generate(
client,
headers,
unknown_day,
unknown_conv,
"Ein Markttag\n\n[[PERSON:99]] stand am Markt.",
)
expect(unknown.status_code == 409, f"unknown placeholder {unknown.status_code}")
expect((unknown.json().get("detail") or {}).get("code") == JOURNAL_NOT_ACCEPTED, "unknown placeholder is rejected")
expect(n == 1, "unknown placeholder does not retry")
expect(current_draft(profile_id, unknown_day) is None, "unknown placeholder inserts no draft")
generic_day, generic_conv = _day_with_turn(client, headers, "Heute nur der Hafen.", title="Generisch")
generic, n = _generate(
client,
headers,
generic_day,
generic_conv,
"Ein Hafentag\n\n[[...]] blieb am Hafen.",
)
expect(generic.status_code == 409, f"generic placeholder {generic.status_code}")
expect((generic.json().get("detail") or {}).get("code") == JOURNAL_NOT_ACCEPTED, "generic placeholder is rejected")
expect(current_draft(profile_id, generic_day) is None, "generic placeholder inserts no draft")
ok_day, ok_conv = _day_with_turn(client, headers, "Heute war ich mit Anna am Hafen.", title="Tokens")
anna_token = next(
(
item.get("token")
for item in list_mappings(profile_id)
if (item.get("canonical_label") or item.get("local_label") or "") == "Anna"
),
"PERSON:02",
)
ok_gen, n = _generate(
client,
headers,
ok_day,
ok_conv,
f"Ein Hafentag\n\n[[{anna_token}]] stand am Hafen.",
)
expect(ok_gen.status_code == 200, f"active tokens {ok_gen.status_code}")
expect("Anna" in (ok_gen.json().get("body") or ""), "active tokens demask normally")
expect("[[" not in (ok_gen.json().get("body") or ""), "no leftover placeholder after demask")
expect(n == 1, "normal demask uses one generate call")
provider_calls = {"n": 0}
def boom(_messages, _policy):
provider_calls["n"] += 1
raise AssertionError("provider must not be called after pre-egress failure")
def leaky_mask(rendered, mappings, purpose):
return MaskingManifest(
masked_text=rendered,
available_mapping_count=len(mappings or []),
replacements=(ActiveReplacement("PERSON:02", "PERSON", 1, "Anna"),),
)
with patch("privacy_gateway.mask_prompt", leaky_mask), patch("privacy_gateway.complete_model", boom):
failed = False
try:
_run_gateway(profile_id, "Anna war am Markt.")
except PrivacyGatewayError as exc:
failed = exc.code == ERROR_EGRESS_VALIDATION
expect(failed, "pre-egress still fail-closed")
expect(provider_calls["n"] == 0, "provider is not called when confirmed identity remains")
print("All privacy response integrity tests passed.")
if __name__ == "__main__":
main()