MVP 0.94
This commit is contained in:
1 parent
f17b3083e8
commit
4f37991ad1
48 files changed
+4919
-540
No files matched your search
@@ -223,8 +223,10 @@ def main() -> None:
|
||||
"space_title",
|
||||
"writing_profile",
|
||||
"style_examples",
|
||||
"editorial_mode",
|
||||
"editorial_instructions",
|
||||
"transformation_instructions",
|
||||
"detail_instructions",
|
||||
"voice_instructions",
|
||||
"narrative_instructions",
|
||||
"interaction_hint",
|
||||
},
|
||||
"system and mvp context keys",
|
||||
|
||||
@@ -874,7 +874,7 @@ def test_generate_flow(client: TestClient, headers: dict) -> None:
|
||||
expect("assistant:" not in (gen.json().get("body") or ""), "draft body has no assistant lines")
|
||||
expect("WRITING_PROFILE" in narrate or "Neutraler Journalstil" in narrate or "Core:" in narrate, "writing profile reaches generate")
|
||||
expect("STYLE_EXAMPLES" in narrate, "style examples are labeled separately from day facts")
|
||||
expect("Erzählmerkmale" not in (narrate.split("CURRENT_DAY_SOURCES")[0] if "CURRENT_DAY_SOURCES" in narrate else narrate), "day dialogue is not a style brief")
|
||||
expect("Erzählmerkmale" not in (narrate.split("\nCURRENT_DAY_SOURCES\n")[0] if "\nCURRENT_DAY_SOURCES\n" in narrate else narrate), "day dialogue is not a style brief")
|
||||
|
||||
reset_debug()
|
||||
short_recorder = install_test_recorder()
|
||||
@@ -995,7 +995,7 @@ def test_generate_flow(client: TestClient, headers: dict) -> None:
|
||||
reset_catalog()
|
||||
|
||||
|
||||
def test_generate_identity_leak_local_fallback(client: TestClient, headers: dict, profile_id: str) -> None:
|
||||
def test_generate_attested_cleartext_is_normalized(client: TestClient, headers: dict, profile_id: str) -> None:
|
||||
from identity_store import remember_mapping
|
||||
from providers import ChatResult
|
||||
|
||||
@@ -1016,13 +1016,15 @@ def test_generate_identity_leak_local_fallback(client: TestClient, headers: dict
|
||||
headers=headers,
|
||||
json={"body": "Heute war ich mit Anna am Markt, danach Kirschen."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"identity-leak setup turn {turn.status_code}")
|
||||
expect(turn.status_code == 200, f"identity-cleartext setup turn {turn.status_code}")
|
||||
calls = {"n": 0}
|
||||
|
||||
def leak(_messages, _policy):
|
||||
calls["n"] += 1
|
||||
return ChatResult(
|
||||
content="Anna stand den ganzen Nachmittag am Markt.",
|
||||
model="fake",
|
||||
usage={},
|
||||
usage={"prompt_tokens": 11, "completion_tokens": 9, "total_tokens": 20, "cost": 0.004},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
@@ -1032,24 +1034,20 @@ def test_generate_identity_leak_local_fallback(client: TestClient, headers: dict
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"identity leak still yields a local draft {gen.text}")
|
||||
expect(gen.status_code == 200, f"attested cleartext is accepted locally {gen.text}")
|
||||
payload = gen.json()
|
||||
leak_log = payload.get("run_log") or (payload.get("trace") or {}).get("log") or []
|
||||
expect(any(item.get("kind") == "retry" for item in leak_log), "identity leak records a retry")
|
||||
expect(
|
||||
any(item.get("reason") == "identity_leak_blocked" for item in leak_log),
|
||||
"identity leak records local fallback",
|
||||
)
|
||||
expect(calls["n"] == 1, "attested cleartext does not retry")
|
||||
expect(not any(item.get("kind") == "retry" for item in leak_log), "no privacy retry event")
|
||||
body = payload.get("body") or ""
|
||||
expect("Markt" in body and "Kirschen" in body, "local draft keeps attested user wording")
|
||||
expect("Anna" in body, "local draft may keep names that the user actually wrote")
|
||||
expect("Anna" in body, "source-attested name is demasked")
|
||||
expect("Markt" in body, "model wording is kept after normalization")
|
||||
stages = {item.get("purpose"): item for item in (payload.get("trace") or {}).get("stages") or []}
|
||||
reconstruct = stages.get("local_source_artifact") or {}
|
||||
narrate = stages.get("journal_generate") or {}
|
||||
expect(reconstruct.get("status") == "local_ok", "stage 1 stays local and does not call reconstruct")
|
||||
expect(narrate.get("guard") == "identity_leak_blocked", "stage 2 does not use the leaking reply")
|
||||
expect(sum(1 for item in leak_log if item.get("kind") == "retry") == 1, "active leak retries at most once")
|
||||
expect(sum(1 for item in leak_log if item.get("kind") == "model_call") <= 2, "retry is the only extra model call")
|
||||
expect(narrate.get("model_text_accepted") is True, "stage 2 accepts the normalized model text")
|
||||
expect((payload.get("trace") or {}).get("generate_calls") == 1 or (narrate.get("generate_calls") == 1), "exactly one generate call")
|
||||
|
||||
|
||||
def test_generate_inactive_mapping_keeps_model_text(client: TestClient, headers: dict, profile_id: str) -> None:
|
||||
@@ -1091,17 +1089,29 @@ def test_generate_inactive_mapping_keeps_model_text(client: TestClient, headers:
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"inactive mapping generate {gen.text}")
|
||||
payload = gen.json()
|
||||
run_log = payload.get("run_log") or (payload.get("trace") or {}).get("log") or []
|
||||
expect(gen.status_code == 409, f"inactive mapping generate {gen.text}")
|
||||
detail = gen.json().get("detail") or {}
|
||||
expect(detail.get("code") == "journal_generation_not_accepted", "historical-only name is not stored as a draft")
|
||||
expect(detail.get("message") == "Generierung nicht übernommen.", "API names the rejection")
|
||||
diag = detail.get("diagnostics") or {}
|
||||
run_log = diag.get("log") or (diag.get("trace") or {}).get("log") or []
|
||||
expect(calls["n"] == 1, "inactive mapping does not trigger a retry")
|
||||
expect(not any(item.get("kind") == "retry" for item in run_log), "inactive mapping does not retry")
|
||||
expect(
|
||||
any(item.get("reason") == "unattested_identity" for item in run_log),
|
||||
"invented historical name is unattested journal content",
|
||||
)
|
||||
expect("Clarissa" not in (payload.get("body") or ""), "unattested name is not accepted as narration")
|
||||
expect("Markt" in (payload.get("body") or "") and "Kirschen" in (payload.get("body") or ""), "fallback keeps user sources")
|
||||
expect(diag.get("model_text_accepted") is False, "unattested model text is not accepted")
|
||||
day_after = client.get(f"/api/journal/days/{day.json()['day']['id']}", headers=headers)
|
||||
expect(not (day_after.json().get("current_draft")), "rejected generate does not insert a draft")
|
||||
trace = diag.get("trace") or {}
|
||||
expect(trace.get("abort_reason") == "unattested_identity", "abort reason remains on the error trace")
|
||||
expect(
|
||||
(trace.get("generate_calls") == 1)
|
||||
or ((trace.get("budget") or {}).get("generate_calls") == 1)
|
||||
or diag.get("generate_calls") == 1,
|
||||
"error trace keeps the generate-call count",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_are_request_scoped(client: TestClient, headers: dict) -> None:
|
||||
@@ -1247,7 +1257,7 @@ def main() -> None:
|
||||
profile_id = setup.json()["profile_id"]
|
||||
test_task_brief_and_dedupe(client, headers, profile_id)
|
||||
test_generate_flow(client, headers)
|
||||
test_generate_identity_leak_local_fallback(client, headers, profile_id)
|
||||
test_generate_attested_cleartext_is_normalized(client, headers, profile_id)
|
||||
test_generate_inactive_mapping_keeps_model_text(client, headers, profile_id)
|
||||
test_traces_are_request_scoped(client, headers)
|
||||
test_two_conversations_same_day(client, headers)
|
||||
|
||||
@@ -20,11 +20,9 @@ from db import get_db, init_db
|
||||
from engine import load_active_prompt
|
||||
from identity_store import remember_mapping
|
||||
from journal_editorial import (
|
||||
NOTES_TO_JOURNAL,
|
||||
PROSE_EDIT,
|
||||
choose_editorial_mode,
|
||||
editorial_instructions,
|
||||
GENERATE_SEED_REVISION,
|
||||
format_style_examples,
|
||||
incomplete_syntax_markers,
|
||||
lexical_similarity,
|
||||
narration_sources_text,
|
||||
select_journal_style_examples,
|
||||
@@ -43,7 +41,13 @@ from writing_profile_store import (
|
||||
)
|
||||
|
||||
|
||||
SEED_REVISION = "2026-08-27-journal-placeholders-v1"
|
||||
SEED_REVISION = GENERATE_SEED_REVISION
|
||||
MIXED_SOURCES_INSTRUCTION = (
|
||||
"Die Quellen können aus Fließtext, Stichpunkten und Satzfragmenten bestehen. "
|
||||
"Behandle jede Passage entsprechend ihrer Form: Redigiere vorhandenen Fließtext, "
|
||||
"verwandle Stichpunkte und Fragmente in vollständige Prosa und verbinde beides zu einem einheitlichen Eintrag. "
|
||||
"Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus."
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
@@ -56,43 +60,36 @@ def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def test_mode_choice() -> None:
|
||||
expect(choose_editorial_mode(["Ich ging zum Markt. Es war voll."]) == PROSE_EDIT, "narrative sentences select prose_edit")
|
||||
expect(choose_editorial_mode(["markt", "kirschen", "hafen"]) == NOTES_TO_JOURNAL, "fragments select notes_to_journal")
|
||||
expect(
|
||||
choose_editorial_mode(["Ich war am Markt.", "Kirschen gekauft.", "hafen später"]) == PROSE_EDIT,
|
||||
"mixed default is prose_edit when at least half the blocks have sentence punctuation",
|
||||
)
|
||||
expect(
|
||||
choose_editorial_mode(["markt", "kirschen", "Später der Hafen."]) == NOTES_TO_JOURNAL,
|
||||
"mixed default is notes_to_journal when the majority lacks sentence punctuation",
|
||||
)
|
||||
prose = editorial_instructions(PROSE_EDIT)
|
||||
notes = editorial_instructions(NOTES_TO_JOURNAL)
|
||||
expect("Gute Formulierungen bewahren" in prose, "prose_edit keeps good wording")
|
||||
expect("zusammenhängende Journalprosa" in notes, "notes_to_journal asks for connected prose")
|
||||
expect(prose != notes, "modes produce different instructions")
|
||||
expect("nicht inklusive ihrer Fehler" in notes or "Fehler hintereinanderkopieren" in notes, "notes must not be concatenated with errors")
|
||||
def test_syntax_diagnostics() -> None:
|
||||
expect(incomplete_syntax_markers("Danach sprach ich mit dem.") > 0, "dangling determiner is diagnostic")
|
||||
expect(incomplete_syntax_markers("Danach sprach ich und kam zurück.") == 0, "complete sentence is not flagged")
|
||||
|
||||
|
||||
def test_prompt_contract() -> None:
|
||||
init_db()
|
||||
prompt = load_active_prompt("mvp.journal_generate")
|
||||
text = prompt.get("template") or ""
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in text, "prompt separates fact fidelity from wording")
|
||||
expect("INHALTSTREUE" in text, "prompt keeps hard content rules")
|
||||
expect("CURRENT_DAY_SOURCES" in text, "prompt labels current-day facts")
|
||||
expect("WRITING_PROFILE" in text, "prompt labels the writing profile")
|
||||
expect("STYLE_EXAMPLES" in text, "prompt labels style examples")
|
||||
expect("EDITORIAL_MODE" in text, "prompt exposes editorial mode")
|
||||
expect("keine geschützten Fakten" in text, "prompt says typos are not protected facts")
|
||||
expect("{{editorial_instructions}}" in text, "prompt injects mode-specific instructions")
|
||||
expect("WRITING_PROFILE" in text, "prompt labels the writing profile")
|
||||
expect("{{transformation_instructions}}" in text, "prompt injects compiled transformation instructions")
|
||||
expect("{{detail_instructions}}" in text, "prompt injects compiled detail instructions")
|
||||
expect("{{voice_instructions}}" in text, "prompt injects compiled voice instructions")
|
||||
expect("{{narrative_instructions}}" in text, "prompt injects compiled narrative instructions")
|
||||
expect("{{source_mode_instructions}}" not in text, "prompt has no source-mode placeholder")
|
||||
expect(MIXED_SOURCES_INSTRUCTION in text, "prompt contains the mixed-source instruction")
|
||||
expect("{{writing_profile}}" in text, "prompt injects writing profile")
|
||||
expect("{{style_examples}}" in text, "prompt injects style examples")
|
||||
expect("{{reconstruction}}" in text, "prompt injects current-day sources")
|
||||
expect("{{existing_text}}" in text, "prompt injects existing text")
|
||||
expect("[[…" not in text and "[[..." not in text, "prompt must not teach ellipsis placeholders")
|
||||
expect("zeichengetreu" in text, "prompt asks to copy existing placeholders unchanged")
|
||||
expect("ich gieng zum laden" in text, "prompt includes a synthetic prose_edit example")
|
||||
expect("Im Laden holte ich Brot" in text, "prompt includes a synthetic notes_to_journal example")
|
||||
expect("ich gieng zum laden" not in text, "synthetic prose_edit example is gone")
|
||||
expect("nachbarhund im garten" not in text, "synthetic notes_to_journal example is gone")
|
||||
expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone")
|
||||
expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone")
|
||||
expect("EDITORIAL_MODE" not in text, "source mode is no longer a labeled user setting")
|
||||
expect("Faktentreue ist nicht Wortlauttreue" not in text, "old redundant fidelity lecture is gone")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||
@@ -163,8 +160,6 @@ def test_budget_pack_drops_examples_first() -> None:
|
||||
assembled = {
|
||||
"writing_profile": "Core: kurze Sätze.",
|
||||
"reconstruction": "Heute Markt.",
|
||||
"editorial_mode": PROSE_EDIT,
|
||||
"editorial_instructions": "x",
|
||||
"style_examples": "",
|
||||
"existing_text": "",
|
||||
}
|
||||
@@ -209,7 +204,7 @@ def intern_of(payload: dict) -> str:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_mode_choice()
|
||||
test_syntax_diagnostics()
|
||||
test_prompt_contract()
|
||||
test_sources_and_examples_are_separated()
|
||||
test_unattested_covers_title_and_body()
|
||||
@@ -256,16 +251,22 @@ def main() -> None:
|
||||
)
|
||||
expect(gen.status_code == 200, f"generate {gen.status_code}")
|
||||
intern = intern_of(gen.json())
|
||||
expect("EDITORIAL_MODE: prose_edit" in intern, "narrative source selects prose_edit")
|
||||
expect("Gute Formulierungen bewahren" in intern, "prose_edit instructions reach the model")
|
||||
expect(MIXED_SOURCES_INSTRUCTION in intern, "unified mixed-source instruction reaches the model")
|
||||
expect("CURRENT_DAY_SOURCES" in intern, "day facts are labeled")
|
||||
expect("STYLE_EXAMPLES" in intern, "style examples are labeled")
|
||||
expect("WRITING_PROFILE" in intern, "writing profile is labeled")
|
||||
expect("Überarbeite den Text substanziell" in intern, "default transformation policy reaches the prompt")
|
||||
expect("Erhalte sämtliche belegten Ereignisse" in intern, "default detail policy reaches the prompt")
|
||||
expect("Neutraler Journalstil" in intern, "without a confirmed profile the neutral fallback is used")
|
||||
expect("zimlich" in intern.split("CURRENT_DAY_SOURCES")[-1], "today remains content, including typos")
|
||||
expect("zimlich" not in intern.split("CURRENT_DAY_SOURCES")[0], "today is not a style authority")
|
||||
expect("zimlich" in intern.split("\nCURRENT_DAY_SOURCES\n")[-1], "today remains content, including typos")
|
||||
expect("zimlich" not in intern.split("\nCURRENT_DAY_SOURCES\n")[0], "today is not a style authority")
|
||||
expect(sum(1 for item in (gen.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "normal path is one generate call")
|
||||
expect((gen.json().get("trace") or {}).get("editorial_mode") == PROSE_EDIT, "editorial mode is in the admin trace")
|
||||
expect("editorial_mode" not in (gen.json().get("trace") or {}), "trace has no abandoned editorial mode")
|
||||
expect((gen.json().get("trace") or {}).get("narration_source") == "model", "accepted model path is recorded")
|
||||
expect((gen.json().get("trace") or {}).get("prompt_revision") == SEED_REVISION, "prompt revision is in the admin trace")
|
||||
expect((gen.json().get("trace") or {}).get("writing_profile", {}).get("neutral_fallback") is True, "unconfirmed profile is visible as fallback")
|
||||
expect((gen.json().get("trace") or {}).get("style_examples", {}).get("count") == 0, "no historical examples yet")
|
||||
expect((gen.json().get("trace") or {}).get("dropped_optional_blocks") == [], "nothing dropped on a small day")
|
||||
|
||||
notes_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
@@ -289,9 +290,8 @@ def main() -> None:
|
||||
)
|
||||
expect(notes_gen.status_code == 200, f"notes generate {notes_gen.text}")
|
||||
notes_intern = intern_of(notes_gen.json())
|
||||
expect("EDITORIAL_MODE: notes_to_journal" in notes_intern, "fragments select notes_to_journal")
|
||||
expect("zusammenhängende Journalprosa" in notes_intern, "notes mode reaches the model")
|
||||
expect("Gute Formulierungen bewahren" not in notes_intern, "prose_edit instructions are not used for notes")
|
||||
expect(MIXED_SOURCES_INSTRUCTION in notes_intern, "notes use the same mixed-source instruction")
|
||||
expect("markt" in notes_intern.lower() and "kirschen" in notes_intern.lower(), "notes sources remain complete")
|
||||
|
||||
import_text(
|
||||
profile_id,
|
||||
@@ -360,7 +360,7 @@ def main() -> None:
|
||||
expect(body.lower().count("markt") <= 2, "repetition is allowed to be reduced")
|
||||
expect("traurig" not in body.lower() and "weil" not in body.lower(), "no invented feeling or cause in the patched rewrite")
|
||||
intern2 = intern_of(second.json())
|
||||
expect("trockener Schnitt" in intern2.split("CURRENT_DAY_SOURCES")[0], "confirmed writing profile reaches generate")
|
||||
expect("trockener Schnitt" in intern2.split("\nCURRENT_DAY_SOURCES\n")[0], "confirmed writing profile reaches generate")
|
||||
expect("Hafen blieb hinter der Fähre" not in body, "historical style facts are not copied into today")
|
||||
expect(
|
||||
sum(1 for item in (second.json().get("run_log") or []) if item.get("kind") == "model_call") == 1,
|
||||
@@ -386,6 +386,103 @@ def main() -> None:
|
||||
expect("Am Markt holte ich Kirschen" in notes_body, "notes become connected prose")
|
||||
expect("kirschen\nspäter" not in notes_body.lower(), "notes are not concatenated as fragments")
|
||||
|
||||
broken_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-24"},
|
||||
)
|
||||
broken_conv = client.post(
|
||||
f"/api/journal/days/{broken_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Bruch"},
|
||||
)
|
||||
client.post(
|
||||
f"/api/journal/conversations/{broken_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "ich gieng zum laden. danach sprach ich mit dem und kam zurück. es war kald."},
|
||||
)
|
||||
|
||||
def rebuilt(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Ladengang\n\nIch ging zum Laden. Danach sprach ich und kam zurück. Es war kalt.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", rebuilt):
|
||||
broken_out = client.post(
|
||||
f"/api/journal/days/{broken_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [broken_conv.json()["id"]]},
|
||||
)
|
||||
broken_body = broken_out.json().get("body") or ""
|
||||
expect("gieng" not in broken_body.lower(), "general spelling error is not kept")
|
||||
expect("kald" not in broken_body.lower(), "second general spelling error is not kept")
|
||||
expect("mit dem" not in broken_body.lower(), "dangling determiner is not kept")
|
||||
expect("kam zurück" in broken_body, "attested continuation stays")
|
||||
expect(incomplete_syntax_markers(broken_body) == 0, "patched rewrite has no incomplete syntax")
|
||||
expect((broken_out.json().get("trace") or {}).get("incomplete_syntax") == 0, "incomplete syntax is a diagnostic")
|
||||
|
||||
weight_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-23"},
|
||||
)
|
||||
weight_conv = client.post(
|
||||
f"/api/journal/days/{weight_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Notizen"},
|
||||
)
|
||||
client.post(
|
||||
f"/api/journal/conversations/{weight_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "morgens tee\nspäter markt\ngegen abend unerwartet der nachbarhund im garten"},
|
||||
)
|
||||
|
||||
def weighted(_messages, _policy):
|
||||
return ChatResult(
|
||||
content=(
|
||||
"Nachbarhund\n\n"
|
||||
"Morgens trank ich Tee, später war ich am Markt. "
|
||||
"Es war ein gewöhnlicher Tag – bis gegen Abend unerwartet der Nachbarhund im Garten war."
|
||||
),
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", weighted):
|
||||
weight_out = client.post(
|
||||
f"/api/journal/days/{weight_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [weight_conv.json()["id"]]},
|
||||
)
|
||||
weight_intern = intern_of(weight_out.json())
|
||||
expect(MIXED_SOURCES_INSTRUCTION in weight_intern, "fragment notes still use the unified instruction")
|
||||
weight_body = weight_out.json().get("body") or ""
|
||||
expect("bis gegen Abend" in weight_body or "unerwartet" in weight_body, "attested standout may be weighted")
|
||||
expect("morgens tee\nspäter markt" not in weight_body.lower(), "notes are not a concatenated list")
|
||||
expect("glücklich" not in weight_body.lower() and "weil" not in weight_body.lower(), "weighting does not invent feeling or cause")
|
||||
|
||||
update_facet(profile_id, "core", value="Lange, ruhig fließende Sätze, behutsame Wortwahl, leise Reflexion.")
|
||||
set_lifecycle(profile_id, "confirmed")
|
||||
brief_b = compile_task_brief(profile_id)
|
||||
expect("fließende Sätze" in brief_b, "second confirmed core compiles")
|
||||
with patch("privacy_gateway.complete_model", rewritten):
|
||||
third = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
intern3 = intern_of(third.json())
|
||||
style3 = intern3.split("\nCURRENT_DAY_SOURCES\n")[0]
|
||||
expect("fließende Sätze" in style3, "second writing profile reaches the rendered prompt")
|
||||
expect("trockener Schnitt" not in style3, "replaced core is not still the style authority")
|
||||
expect((third.json().get("trace") or {}).get("writing_profile", {}).get("has_core") is True, "core presence is in the trace")
|
||||
expect((third.json().get("trace") or {}).get("writing_profile", {}).get("present") is True, "confirmed profile is marked present")
|
||||
expect(sum(1 for item in (third.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "profile A/B still uses one generate call")
|
||||
|
||||
def invent_title(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Hanna am Hafen\n\nIch ging zum Markt, auf dem es ziemlich voll war.",
|
||||
@@ -400,8 +497,13 @@ def main() -> None:
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(any(item.get("reason") == "unattested_identity" for item in (blocked.json().get("run_log") or [])), "title identity uses local fallback")
|
||||
expect("Hanna" not in (blocked.json().get("title") or "") and "Hanna" not in (blocked.json().get("body") or ""), "unattested title identity is not kept")
|
||||
expect(blocked.status_code == 409, f"unattested title {blocked.text}")
|
||||
detail = blocked.json().get("detail") or {}
|
||||
expect(detail.get("code") == "journal_generation_not_accepted", "unattested title is not stored as a draft")
|
||||
expect(detail.get("message") == "Generierung nicht übernommen.", "API names the rejection")
|
||||
log = (detail.get("diagnostics") or {}).get("log") or []
|
||||
expect(any(item.get("reason") == "unattested_identity" for item in log), "title identity is a provenance reject")
|
||||
expect((detail.get("diagnostics") or {}).get("trace", {}).get("model_text_accepted") is False, "model text is not accepted")
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
@@ -416,7 +518,7 @@ def main() -> None:
|
||||
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in (row["default_template"] or ""), "default template still tracks the seed")
|
||||
expect("INHALTSTREUE" in (row["default_template"] or ""), "default template still tracks the seed")
|
||||
expect(row["seed_revision"] == SEED_REVISION, "revision updates even when template is custom")
|
||||
|
||||
print("journal editorial tests passed.")
|
||||
|
||||
@@ -15,14 +15,19 @@ os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from db import init_db
|
||||
from journal_editorial import GENERATE_SEED_REVISION
|
||||
from journal_eval import (
|
||||
FIXTURES,
|
||||
PROFILE_A,
|
||||
PROFILE_B,
|
||||
SYNTHETIC_PROSE,
|
||||
SYNTHETIC_TYPOS,
|
||||
VARIANT_BASELINE,
|
||||
VARIANT_CURRENT,
|
||||
VARIANT_PREVIOUS,
|
||||
compare_synthetic,
|
||||
fixture_context,
|
||||
score_output,
|
||||
synthetic_context,
|
||||
variant_templates,
|
||||
)
|
||||
|
||||
@@ -37,12 +42,33 @@ def main() -> None:
|
||||
init_db()
|
||||
templates = variant_templates()
|
||||
expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist")
|
||||
expect("Überarbeite diesen Rohtext" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
|
||||
expect("Erstelle aus diesen Angaben einen ansprechenden persönlichen Tagebucheintrag" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
|
||||
expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt")
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in templates[VARIANT_CURRENT], "current variant has the new contract")
|
||||
expect("INHALTSTREUE" in templates[VARIANT_CURRENT], "current variant has the new contract")
|
||||
expect("{{transformation_instructions}}" in templates[VARIANT_CURRENT], "current variant compiles transformation policy")
|
||||
expect("{{source_mode_instructions}}" not in templates[VARIANT_CURRENT], "current variant has no source-mode placeholder")
|
||||
expect("Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus." in templates[VARIANT_CURRENT], "current variant has mixed-source instruction")
|
||||
expect("ich gieng zum laden" not in templates[VARIANT_CURRENT], "current variant has no synthetic examples")
|
||||
expect(GENERATE_SEED_REVISION == "2026-08-27-journal-mixed-sources-v1", "eval tracks the seeded revision constant")
|
||||
|
||||
context = synthetic_context(SYNTHETIC_PROSE)
|
||||
expect(context["editorial_mode"] in {"prose_edit", "notes_to_journal"}, "eval context has an editorial mode")
|
||||
required_classes = {
|
||||
"already_narrative_with_errors",
|
||||
"bullet_points_and_fragments",
|
||||
"plan_versus_completion",
|
||||
"negation_and_uncertainty",
|
||||
"correction_of_earlier_claim",
|
||||
"imprecise_time",
|
||||
"outstanding_event_among_everyday",
|
||||
"recurring_people_and_projects",
|
||||
"incomplete_source_rebuildable",
|
||||
}
|
||||
got = {item["class"] for item in FIXTURES}
|
||||
expect(required_classes <= got, f"all required fixture classes exist, missing {required_classes - got}")
|
||||
expect(all("example.test" not in item["source"].lower() for item in FIXTURES), "fixtures stay synthetic")
|
||||
|
||||
context = fixture_context(SYNTHETIC_PROSE)
|
||||
expect("source_mode_instructions" not in context, "eval context has no source-mode instruction")
|
||||
expect("editorial_mode" not in context, "eval context has no editorial mode")
|
||||
copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS)
|
||||
expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1")
|
||||
expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos")
|
||||
@@ -55,6 +81,28 @@ def main() -> None:
|
||||
expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical")
|
||||
expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored")
|
||||
expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures")
|
||||
|
||||
notes = next(item for item in FIXTURES if item["id"] == "notes_fragments")
|
||||
expect("markt" in notes["source"].lower(), "notes fixture stays notes-shaped")
|
||||
incomplete = next(item for item in FIXTURES if item["id"] == "incomplete_clause")
|
||||
expect("danach sprach ich mit dem" in incomplete["source"].lower(), "incomplete fixture stays incomplete prose")
|
||||
|
||||
report = compare_synthetic(live=False)
|
||||
expect(report["live"] is False, "default eval is offline")
|
||||
expect(report["live_quality_confirmed"] is False, "offline run does not confirm live quality")
|
||||
expect(report["winner_declared"] is False, "harness does not declare a winner")
|
||||
expect(len(report["fixtures"]) == len(FIXTURES), "offline report covers every fixture")
|
||||
first = report["fixtures"][0]
|
||||
expect("human_blind" in first and "prompt_1" in first["human_blind"], "each fixture has a blind pair")
|
||||
expect(first["human_blind"]["hidden_mapping"]["prompt_1"] == VARIANT_CURRENT, "mapping stays machine-side")
|
||||
expect("Kanshō habe gewonnen" not in report["note"], "no victory claim")
|
||||
expect(any(item["variant"] == VARIANT_CURRENT and item["fake_provider"] for item in first["variants"]), "offline current variant is fake")
|
||||
|
||||
ab = compare_synthetic(live=False, profile_ab=True)
|
||||
expect(ab["profile_ab"]["same_facts"] is True, "profile A/B keeps facts identical")
|
||||
expect(ab["profile_ab"]["prompts_differ"] is True, "rendered prompts contain different style briefs")
|
||||
expect(PROFILE_A[:20] != PROFILE_B[:20], "synthetic profiles are distinct")
|
||||
expect("Live-Prosa" in (ab["profile_ab"]["note"] or ""), "offline A/B does not claim live prose")
|
||||
print("journal eval tests passed.")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,961 @@
|
||||
"""Named journal generation guidelines: selection, snapshot, mixed sources."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FRONTEND = ROOT.parent / "frontend"
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-guidelines-test.sqlite")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from db import get_db, init_db
|
||||
from engine import load_active_prompt
|
||||
from identity_store import confirm_identity, remember_mapping
|
||||
from journal_editorial import GENERATE_SEED_REVISION
|
||||
from journal_generation_policy import (
|
||||
PURPOSE_JOURNAL,
|
||||
SELECTION_KEYS,
|
||||
SLOT_TO_ID_KEY,
|
||||
CatalogError,
|
||||
GenerationPolicyError,
|
||||
archive_guideline,
|
||||
clone_guideline,
|
||||
compile_selection,
|
||||
create_guideline,
|
||||
default_selection_ids,
|
||||
get_guideline,
|
||||
get_or_create_selection,
|
||||
list_guidelines,
|
||||
load_seed_document,
|
||||
load_selection,
|
||||
overview_payload,
|
||||
publish_guideline,
|
||||
save_selection,
|
||||
seed_generation_instructions,
|
||||
snapshot_summary,
|
||||
update_guideline,
|
||||
validate_selection,
|
||||
)
|
||||
from journal_generate import unattested_journal_content
|
||||
from journal_store import current_draft
|
||||
from main import app
|
||||
from privacy_gateway import install_test_recorder, reset_debug
|
||||
from providers import ChatResult
|
||||
|
||||
|
||||
PREVIOUS_PROMPT_CHARS = 3468
|
||||
HARD_FACTS = (
|
||||
"keine neuen Tatsachen",
|
||||
"Plan und Vollzug",
|
||||
"Verneinungen",
|
||||
"Unsicherheiten",
|
||||
"STYLE_EXAMPLES dienen ausschließlich als Stilreferenz",
|
||||
"[[PERSON:01]]",
|
||||
)
|
||||
BANNED_CODE_PHRASES = (
|
||||
"Überarbeite den Text substanziell",
|
||||
"Erhalte sämtliche belegten Ereignisse",
|
||||
"künstlich zu literarisieren",
|
||||
"Modus prose_edit",
|
||||
"Modus notes_to_journal",
|
||||
"Korrigiere nur Rechtschreibung, Grammatik und Zeichensetzung",
|
||||
)
|
||||
BANNED_RUNTIME_TOKENS = (
|
||||
"transformation_strength",
|
||||
"detail_retention",
|
||||
"voice_strength",
|
||||
"narrative_shaping",
|
||||
"min_value",
|
||||
"max_value",
|
||||
"remember_generation_policy",
|
||||
"generation_policy",
|
||||
)
|
||||
SEED = load_seed_document()
|
||||
MIXED_SOURCES_INSTRUCTION = (
|
||||
"Die Quellen können aus Fließtext, Stichpunkten und Satzfragmenten bestehen. "
|
||||
"Behandle jede Passage entsprechend ihrer Form: Redigiere vorhandenen Fließtext, "
|
||||
"verwandle Stichpunkte und Fragmente in vollständige Prosa und verbinde beides zu einem einheitlichen Eintrag. "
|
||||
"Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus."
|
||||
)
|
||||
REQUIRED_KEYS = {
|
||||
("transformation", "correction"),
|
||||
("transformation", "copyedit"),
|
||||
("transformation", "reshape"),
|
||||
("transformation", "substantial"),
|
||||
("detail", "compact"),
|
||||
("detail", "selected"),
|
||||
("detail", "broad"),
|
||||
("detail", "complete"),
|
||||
("voice", "neutral"),
|
||||
("voice", "light"),
|
||||
("voice", "noticeable"),
|
||||
("voice", "clear"),
|
||||
("narrative", "chronicle"),
|
||||
("narrative", "structured"),
|
||||
("narrative", "weighted"),
|
||||
("narrative", "emphasized"),
|
||||
}
|
||||
|
||||
|
||||
def has_token(text: str, token: str) -> bool:
|
||||
return re.search(rf"(?<![A-Za-z0-9_]){re.escape(token)}(?![A-Za-z0-9_])", text) is not None
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def intern_of(payload: dict) -> str:
|
||||
for stage in (payload.get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
return stage.get("intern") or ""
|
||||
return (payload.get("trace") or {}).get("intern") or ""
|
||||
|
||||
|
||||
def sources_block(intern: str) -> str:
|
||||
parts = (intern or "").split("\nCURRENT_DAY_SOURCES\n")
|
||||
if len(parts) < 2:
|
||||
return intern or ""
|
||||
rest = parts[-1]
|
||||
for marker in ("\nEXISTING_TEXT\n", "\nAUSGABE\n"):
|
||||
if marker in rest:
|
||||
rest = rest.split(marker, 1)[0]
|
||||
break
|
||||
return rest
|
||||
|
||||
|
||||
def seed_item(slot: str, key: str) -> dict:
|
||||
for item in (SEED.get("slots") or {}).get(slot, {}).get("variants") or []:
|
||||
if item.get("guideline_key") == key:
|
||||
return item
|
||||
raise SystemExit(f"FAIL: missing seed {slot}/{key}")
|
||||
|
||||
|
||||
def seed_id(slot: str, key: str) -> str:
|
||||
return seed_item(slot, key)["id"]
|
||||
|
||||
|
||||
def seed_instruction(slot: str, key: str) -> str:
|
||||
return seed_item(slot, key).get("instruction") or ""
|
||||
|
||||
|
||||
def selection_of(**keys: str) -> dict[str, str]:
|
||||
return {SLOT_TO_ID_KEY[slot]: seed_id(slot, key) for slot, key in keys.items()}
|
||||
|
||||
|
||||
def default_ids() -> dict[str, str]:
|
||||
return selection_of(
|
||||
transformation="substantial",
|
||||
detail="complete",
|
||||
voice="clear",
|
||||
narrative="weighted",
|
||||
)
|
||||
|
||||
|
||||
def low_ids() -> dict[str, str]:
|
||||
return selection_of(
|
||||
transformation="correction",
|
||||
detail="compact",
|
||||
voice="neutral",
|
||||
narrative="chronicle",
|
||||
)
|
||||
|
||||
|
||||
def high_ids() -> dict[str, str]:
|
||||
return selection_of(
|
||||
transformation="substantial",
|
||||
detail="complete",
|
||||
voice="clear",
|
||||
narrative="emphasized",
|
||||
)
|
||||
|
||||
|
||||
def selection_meta(payload: dict) -> dict:
|
||||
return (payload.get("trace") or {}).get("generation_selection") or {}
|
||||
|
||||
|
||||
def test_runtime_has_no_numeric_policy() -> None:
|
||||
files = [
|
||||
ROOT / "journal_generation_policy.py",
|
||||
ROOT / "journal_generate.py",
|
||||
ROOT / "routers" / "journal.py",
|
||||
ROOT / "routers" / "generation_instructions.py",
|
||||
FRONTEND / "src" / "pages" / "JournalDayPage.jsx",
|
||||
FRONTEND / "src" / "pages" / "AdminGenerationPage.jsx",
|
||||
]
|
||||
for path in files:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for token in BANNED_RUNTIME_TOKENS:
|
||||
if token == "generation_policy" and path.name == "journal_generation_policy.py":
|
||||
continue
|
||||
if token == "generation_policy" and path.name == "generation_instructions.py":
|
||||
continue
|
||||
expect(not has_token(text, token), f"{path.name} has no {token}")
|
||||
day = (FRONTEND / "src" / "pages" / "JournalDayPage.jsx").read_text(encoding="utf-8")
|
||||
expect('type="range"' not in day, "journal page has no sliders")
|
||||
expect("policy-slider" not in day, "journal page has no slider class")
|
||||
expect("SLOT_SELECTS" in day, "journal page names the four selects")
|
||||
expect(day.count("slot:") >= 4, "journal page has four independent slots")
|
||||
expect("<select" in day, "journal page uses select fields")
|
||||
expect("Diese Auswahl als Standard merken" in day, "journal page can remember explicitly")
|
||||
expect("Faktenregeln und Datenschutz bleiben unabhängig" in day, "facts/privacy note remains")
|
||||
editor = (FRONTEND / "src" / "pages" / "JournalEditorPage.jsx").read_text(encoding="utf-8")
|
||||
expect("generation_summary" in editor, "editor can show the generation snapshot")
|
||||
admin = (FRONTEND / "src" / "pages" / "AdminGenerationPage.jsx").read_text(encoding="utf-8")
|
||||
expect("<textarea" not in admin.split("generation-admin-editor")[0], "admin overview has no textareas")
|
||||
expect(admin.count("<textarea") == 1, "admin opens one instruction editor at a time")
|
||||
expect("Öffnen" in admin, "admin overview has open action")
|
||||
expect("backend/config/" not in admin, "admin copy does not advertise seed file paths")
|
||||
expect("Quellenmodus" not in admin, "admin has no source-mode management")
|
||||
expect("source_mode" not in admin, "admin source has no source_mode slot")
|
||||
expect("prose_edit" not in admin, "admin has no prose_edit control")
|
||||
expect("notes_to_journal" not in admin, "admin has no notes_to_journal control")
|
||||
expect("Quellenmodus" not in day, "journal page has no source-mode control")
|
||||
expect("source_mode" not in day, "journal page has no source_mode field")
|
||||
expect("editorial_mode" not in day, "journal page has no editorial_mode field")
|
||||
trace_ui = (FRONTEND / "src" / "components" / "CallTrace.jsx").read_text(encoding="utf-8")
|
||||
expect("Editorial Mode" not in trace_ui, "trace UI does not present source mode as a run decision")
|
||||
expect("source_mode" not in trace_ui, "trace UI has no source_mode label")
|
||||
|
||||
|
||||
def test_seed_and_compiler() -> None:
|
||||
init_db()
|
||||
for rel in ("journal_generation_policy.py", "journal_editorial.py"):
|
||||
source = (ROOT / rel).read_text(encoding="utf-8")
|
||||
for phrase in BANNED_CODE_PHRASES:
|
||||
expect(phrase not in source, f"{rel} does not hardcode {phrase!r}")
|
||||
items = list_guidelines(PURPOSE_JOURNAL, include_instruction=True)
|
||||
got = {(item["slot"], item["guideline_key"]) for item in items}
|
||||
expect(REQUIRED_KEYS <= got, "seed creates every required guideline")
|
||||
expect("source_mode" not in (SEED.get("slots") or {}), "seed document has no source_mode slot")
|
||||
expect(sum(1 for item in items if item["slot"] == "source_mode") == 0, "source modes are not seeded")
|
||||
expect(all(item["status"] == "active" for item in items if item["is_system_seed"]), "seed rows start active")
|
||||
|
||||
defaults = compile_selection(default_selection_ids())
|
||||
expect(defaults.ids["transformation"] == seed_id("transformation", "substantial"), "default transformation is substantial")
|
||||
expect(
|
||||
defaults.instructions["transformation_instructions"] == seed_instruction("transformation", "substantial"),
|
||||
"default transformation instruction matches the seed",
|
||||
)
|
||||
expect(
|
||||
defaults.instructions["detail_instructions"] == seed_instruction("detail", "complete"),
|
||||
"default detail instruction matches the seed",
|
||||
)
|
||||
expect(
|
||||
defaults.instructions["voice_instructions"] == seed_instruction("voice", "clear"),
|
||||
"default voice instruction matches the seed",
|
||||
)
|
||||
expect(
|
||||
defaults.instructions["narrative_instructions"] == seed_instruction("narrative", "weighted"),
|
||||
"default narrative instruction matches the seed",
|
||||
)
|
||||
expect(defaults.seed_revision == SEED["seed_revision"], "compiled policy names the seed revision")
|
||||
expect("source_mode_instructions" not in defaults.instructions, "compiler has no source-mode instruction")
|
||||
|
||||
low = compile_selection(low_ids())
|
||||
high = compile_selection(high_ids())
|
||||
expect(low.keys["transformation"] == "correction", "low transformation is named")
|
||||
expect(high.keys["narrative"] == "emphasized", "high narrative is named")
|
||||
expect("source_mode_instructions" not in low.instructions, "low compile has no source mode")
|
||||
expect("source_mode_instructions" not in high.instructions, "high compile has no source mode")
|
||||
expect("{{" not in "".join(low.instructions.values()), "compiled instructions contain no placeholders")
|
||||
|
||||
for bad in (None, [], "75", True, {"transformation_strength": 75}, {"transformation_id": seed_id("transformation", "substantial")}):
|
||||
try:
|
||||
validate_selection(bad)
|
||||
expect(False, "incomplete or numeric selection must be rejected")
|
||||
except GenerationPolicyError as exc:
|
||||
expect(exc.code == "invalid_generation_selection", "rejection uses invalid_generation_selection")
|
||||
|
||||
draft = create_guideline(
|
||||
PURPOSE_JOURNAL,
|
||||
"transformation",
|
||||
{
|
||||
"guideline_key": "drafty",
|
||||
"label": "Entwurf",
|
||||
"summary": "nicht wählbar",
|
||||
"instruction": "Nur ein Draft.",
|
||||
},
|
||||
)
|
||||
try:
|
||||
validate_selection({**default_ids(), "transformation_id": draft["id"]})
|
||||
expect(False, "drafts must not be selectable")
|
||||
except GenerationPolicyError:
|
||||
expect(True, "drafts are not selectable")
|
||||
|
||||
cloned = clone_guideline(seed_id("transformation", "correction"))
|
||||
original = get_guideline(seed_id("transformation", "correction"), include_instruction=True)
|
||||
update_guideline(
|
||||
cloned["id"],
|
||||
{
|
||||
"guideline_key": original["guideline_key"],
|
||||
"label": original["label"],
|
||||
"summary": original["summary"],
|
||||
"instruction": "CHANGED CLONE TEXT",
|
||||
},
|
||||
)
|
||||
expect(
|
||||
get_guideline(original["id"], include_instruction=True)["instruction"] == original["instruction"],
|
||||
"cloning does not change the original",
|
||||
)
|
||||
try:
|
||||
update_guideline(original["id"], {"label": "still active", "instruction": original["instruction"], "guideline_key": original["guideline_key"]})
|
||||
expect(False, "published guidelines must be immutable")
|
||||
except CatalogError as exc:
|
||||
expect(exc.code == "guideline_immutable", "published guidelines are not silently overwritten")
|
||||
|
||||
published_clone = publish_guideline(cloned["id"])
|
||||
archived = archive_guideline(published_clone["id"])
|
||||
expect(archived["status"] == "archived", "published clones can be archived")
|
||||
try:
|
||||
validate_selection({**default_ids(), "transformation_id": published_clone["id"]})
|
||||
expect(False, "archived guidelines must not be selectable for new runs")
|
||||
except GenerationPolicyError:
|
||||
expect(True, "archived guidelines are not selectable for new runs")
|
||||
|
||||
with get_db() as conn:
|
||||
seed_generation_instructions(conn)
|
||||
expect(
|
||||
get_guideline(original["id"], include_instruction=True)["instruction"] == original["instruction"],
|
||||
"seed does not overwrite published guidelines",
|
||||
)
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO generation_guidelines (
|
||||
id, purpose, slot, guideline_key, label, summary, instruction, sort_order,
|
||||
status, revision, cloned_from, is_default, is_system_seed, seed_id, seed_revision,
|
||||
created, updated
|
||||
)
|
||||
VALUES (
|
||||
'legacy-source-mode', ?, 'source_mode', 'prose_edit', 'Fließtext', '',
|
||||
'LEGACY MODE TEXT', 0, 'active', 1, NULL, 0, 0, '', '', datetime('now'), datetime('now')
|
||||
)
|
||||
""",
|
||||
(PURPOSE_JOURNAL,),
|
||||
)
|
||||
seed_generation_instructions(conn)
|
||||
archived_legacy = conn.execute(
|
||||
"SELECT status FROM generation_guidelines WHERE id = 'legacy-source-mode'"
|
||||
).fetchone()
|
||||
expect(archived_legacy["status"] == "archived", "persisted source modes are archived")
|
||||
expect("source_mode" not in overview_payload()["slots"], "admin overview hides archived source modes")
|
||||
|
||||
|
||||
def test_prompt_contract() -> None:
|
||||
init_db()
|
||||
prompt = load_active_prompt("mvp.journal_generate")
|
||||
text = prompt.get("template") or ""
|
||||
expect(len(text) < PREVIOUS_PROMPT_CHARS, "new prompt is shorter than the previous default")
|
||||
print(f"OK prompt length {len(text)} after, {PREVIOUS_PROMPT_CHARS} before")
|
||||
for key in (
|
||||
"{{transformation_instructions}}",
|
||||
"{{detail_instructions}}",
|
||||
"{{voice_instructions}}",
|
||||
"{{narrative_instructions}}",
|
||||
"{{writing_profile}}",
|
||||
"{{style_examples}}",
|
||||
"{{reconstruction}}",
|
||||
"{{existing_text}}",
|
||||
):
|
||||
expect(key in text, f"prompt contains {key}")
|
||||
expect("{{source_mode_instructions}}" not in text, "prompt has no source-mode placeholder")
|
||||
expect("{{editorial_mode}}" not in text, "prompt has no editorial_mode placeholder")
|
||||
expect("{{editorial_instructions}}" not in text, "prompt has no editorial_instructions placeholder")
|
||||
expect(MIXED_SOURCES_INSTRUCTION in text, "prompt contains the mixed-source instruction")
|
||||
for fact in HARD_FACTS:
|
||||
expect(fact in text, f"hard fact rule remains: {fact}")
|
||||
expect("ich gieng zum laden" not in text, "no synthetic prose example")
|
||||
expect("nachbarhund im garten" not in text, "no synthetic notes example")
|
||||
expect("EDITORIAL_MODE" not in text, "source mode is not a user-facing prompt heading")
|
||||
expect(GENERATE_SEED_REVISION == "2026-08-27-journal-mixed-sources-v1", "seed revision constant tracks mixed sources")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect(row["seed_revision"] == "2026-08-27-journal-mixed-sources-v1", "untouched default is updated to mixed sources")
|
||||
expect(row["template"] == row["default_template"], "untouched template matches the seed")
|
||||
expect("source_mode" not in (SEED.get("slots") or {}), "source modes stay out of the seed")
|
||||
|
||||
|
||||
def test_unattested_alias() -> None:
|
||||
mappings = [
|
||||
{
|
||||
"local_label": "Clarissa",
|
||||
"canonical_label": "Clarissa",
|
||||
"token": "PERSON:01",
|
||||
"aliases": ["Sushi"],
|
||||
"demask_label": "Clarissa",
|
||||
}
|
||||
]
|
||||
expect(
|
||||
unattested_journal_content(
|
||||
"Clarissa kam ins Wohnzimmer.",
|
||||
["Heute war Sushi im Wohnzimmer."],
|
||||
mappings,
|
||||
["PERSON:01"],
|
||||
)
|
||||
is None,
|
||||
"canonical demask of an attested alias is not unattested_identity",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content(
|
||||
"Hanna kam ins Wohnzimmer.",
|
||||
["Heute war Sushi im Wohnzimmer."],
|
||||
mappings + [{"local_label": "Hanna", "token": "PERSON:99"}],
|
||||
["PERSON:01"],
|
||||
)
|
||||
== "unattested_identity",
|
||||
"a different person remains unattested",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_runtime_has_no_numeric_policy()
|
||||
test_seed_and_compiler()
|
||||
test_prompt_contract()
|
||||
test_unattested_alias()
|
||||
|
||||
reset_debug()
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "ada@example.test", "name": "Ada", "password": "test-pass"},
|
||||
)
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
|
||||
settings = client.get("/api/journal/generation-settings", headers=headers)
|
||||
expect(settings.status_code == 200, f"generation-settings {settings.text}")
|
||||
body = settings.json()
|
||||
expect(body["selection"] == default_ids(), "profile defaults are the named system standards")
|
||||
expect(set(body["selection"]) == set(SELECTION_KEYS), "settings expose the four guideline ids")
|
||||
for slot, options in body["options"].items():
|
||||
expect(options, f"{slot} has active options")
|
||||
expect(all("instruction" not in item for item in options), f"{slot} options hide prompt text")
|
||||
expect(all(item["id"] != "draft" for item in options), f"{slot} options are real ids")
|
||||
stored = get_or_create_selection(profile_id)
|
||||
expect({key: stored[key] for key in SELECTION_KEYS} == default_ids(), "persisted defaults match system standards")
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO profiles (id, email, name, password_hash, role) VALUES (?, ?, ?, ?, 'user')",
|
||||
("other-profile", "other@example.test", "Other", "x"),
|
||||
)
|
||||
save_selection("other-profile", high_ids())
|
||||
save_selection(profile_id, low_ids())
|
||||
expect(load_selection(profile_id)["transformation_id"] == seed_id("transformation", "correction"), "profile A keeps its own selection")
|
||||
expect(load_selection("other-profile")["transformation_id"] == seed_id("transformation", "substantial"), "profile B is isolated")
|
||||
save_selection(profile_id, default_ids())
|
||||
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Policy"})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-27"},
|
||||
)
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Tag"},
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Ich war am Markt. Vielleicht bleibe ich kürzer. Brot holen wollte ich noch, habe es aber nicht gemacht."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn {turn.status_code}")
|
||||
|
||||
rejected = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]], "generation_selection": {"transformation_id": "missing"}},
|
||||
)
|
||||
expect(rejected.status_code == 400, "unknown selection is rejected")
|
||||
expect((rejected.json().get("detail") or {}).get("code") == "invalid_generation_selection", "rejection uses the selection code")
|
||||
after_reject = client.get("/api/journal/generation-settings", headers=headers).json()
|
||||
expect(after_reject["selection"] == default_ids(), "rejected values are not stored")
|
||||
|
||||
draft = client.post(
|
||||
"/api/admin/generation-instructions/journal_generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"slot": "voice",
|
||||
"guideline_key": "hidden_draft",
|
||||
"label": "versteckt",
|
||||
"summary": "Draft",
|
||||
"instruction": "Nicht für neue Läufe.",
|
||||
},
|
||||
)
|
||||
expect(draft.status_code == 200, f"admin draft {draft.text}")
|
||||
blocked_draft = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"conversation_ids": [conv.json()["id"]],
|
||||
"generation_selection": {**default_ids(), "voice_id": draft.json()["id"]},
|
||||
},
|
||||
)
|
||||
expect(blocked_draft.status_code == 400, "drafts are rejected for new runs")
|
||||
|
||||
reset_debug()
|
||||
recorder = install_test_recorder()
|
||||
snapshot_run = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"conversation_ids": [conv.json()["id"]],
|
||||
"generation_selection": low_ids(),
|
||||
"remember_generation_selection": False,
|
||||
},
|
||||
)
|
||||
expect(snapshot_run.status_code == 200, f"low generate {snapshot_run.text}")
|
||||
intern = intern_of(snapshot_run.json())
|
||||
meta = selection_meta(snapshot_run.json())
|
||||
expect(meta.get("source") == "request", "trace names the request snapshot")
|
||||
expect(meta.get("remembered") is False, "unremembered snapshot is marked")
|
||||
expect(meta.get("ids") == {
|
||||
"transformation": seed_id("transformation", "correction"),
|
||||
"detail": seed_id("detail", "compact"),
|
||||
"voice": seed_id("voice", "neutral"),
|
||||
"narrative": seed_id("narrative", "chronicle"),
|
||||
}, "trace keeps the used ids")
|
||||
expect(meta.get("keys", {}).get("transformation") == "correction", "trace keeps guideline keys")
|
||||
expect(meta.get("revisions", {}).get("transformation") == 1, "trace keeps revisions")
|
||||
expect("editorial_mode" not in meta, "trace has no abandoned editorial mode")
|
||||
expect("source_mode" not in meta, "trace has no abandoned source mode")
|
||||
expect("Korrigiere nur" not in str(meta), "trace does not store compiled instruction text")
|
||||
expect(
|
||||
client.get("/api/journal/generation-settings", headers=headers).json()["selection"] == default_ids(),
|
||||
"unremembered snapshot does not change profile defaults",
|
||||
)
|
||||
snapshot = snapshot_run.json().get("generation_snapshot") or {}
|
||||
expect(snapshot.get("transformation", {}).get("key") == "correction", "draft stores transformation key")
|
||||
expect(snapshot.get("prompt_slug") == "mvp.journal_generate", "draft stores prompt slug")
|
||||
expect("instruction" not in str(snapshot), "draft snapshot omits instruction text")
|
||||
expect("source_mode" not in snapshot, "draft snapshot has no source_mode")
|
||||
expect("editorial_mode" not in snapshot, "draft snapshot has no editorial_mode")
|
||||
summary = snapshot_run.json().get("generation_summary") or ""
|
||||
expect(summary.startswith("Erzeugt mit:"), "draft summary is user-readable")
|
||||
expect("Quellenmodus:" not in summary, "draft summary does not name a source mode")
|
||||
expect("{{transformation_instructions}}" not in intern, "transformation placeholder is resolved")
|
||||
expect("{{detail_instructions}}" not in intern, "detail placeholder is resolved")
|
||||
expect("{{voice_instructions}}" not in intern, "voice placeholder is resolved")
|
||||
expect("{{narrative_instructions}}" not in intern, "narrative placeholder is resolved")
|
||||
expect("{{source_mode_instructions}}" not in intern, "source-mode placeholder is absent")
|
||||
expect(MIXED_SOURCES_INSTRUCTION in intern, "mixed-source instruction reaches the prompt")
|
||||
expect(seed_instruction("transformation", "correction") in intern, "selected transformation reaches the prompt")
|
||||
expect(seed_instruction("detail", "compact") in intern, "selected detail reaches the prompt")
|
||||
expect(seed_instruction("voice", "neutral") in intern, "selected voice reaches the prompt")
|
||||
expect(seed_instruction("narrative", "chronicle") in intern, "selected narrative reaches the prompt")
|
||||
expect([item.get("purpose") for item in recorder] == ["journal_generate"], "exactly one generate call")
|
||||
expect(sum(1 for item in (snapshot_run.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "run log records one model call")
|
||||
expect(
|
||||
not any("Korrigiere nur Rechtschreibung" in str(item) for item in (snapshot_run.json().get("run_log") or [])),
|
||||
"run log omits compiled prompt instructions",
|
||||
)
|
||||
for fact in HARD_FACTS:
|
||||
expect(fact in intern, f"hard fact remains at low policy: {fact}")
|
||||
|
||||
notes_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-28"},
|
||||
)
|
||||
notes_conv = client.post(
|
||||
f"/api/journal/days/{notes_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Notizen"},
|
||||
)
|
||||
client.post(
|
||||
f"/api/journal/conversations/{notes_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "- Markt\n- Brot nicht geholt\n- vielleicht kürzer bleiben"},
|
||||
)
|
||||
notes_run = client.post(
|
||||
f"/api/journal/days/{notes_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [notes_conv.json()["id"]], "generation_selection": default_ids()},
|
||||
)
|
||||
expect(notes_run.status_code == 200, f"notes generate {notes_run.text}")
|
||||
notes_intern = intern_of(notes_run.json())
|
||||
notes_snap = notes_run.json().get("generation_snapshot") or {}
|
||||
expect("source_mode" not in notes_snap, "notes snapshot has no source mode")
|
||||
expect("editorial_mode" not in (notes_run.json().get("trace") or {}), "notes trace has no editorial mode")
|
||||
expect(MIXED_SOURCES_INSTRUCTION in notes_intern, "notes use the unified mixed-source instruction")
|
||||
expect("Markt" in sources_block(notes_intern), "notes sources remain complete")
|
||||
|
||||
def generate_shapes(date: str, bodies: list[str], label: str) -> None:
|
||||
shaped_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": date},
|
||||
)
|
||||
shaped_conv = client.post(
|
||||
f"/api/journal/days/{shaped_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": label},
|
||||
)
|
||||
for body in bodies:
|
||||
client.post(
|
||||
f"/api/journal/conversations/{shaped_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": body},
|
||||
)
|
||||
reset_debug()
|
||||
rec = install_test_recorder()
|
||||
run = client.post(
|
||||
f"/api/journal/days/{shaped_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [shaped_conv.json()["id"]], "generation_selection": default_ids()},
|
||||
)
|
||||
expect(run.status_code == 200, f"{label} generate {run.text}")
|
||||
shaped_intern = intern_of(run.json())
|
||||
shaped_sources = sources_block(shaped_intern)
|
||||
shaped_snap = run.json().get("generation_snapshot") or {}
|
||||
expect(MIXED_SOURCES_INSTRUCTION in shaped_intern, f"{label} has mixed-source instruction")
|
||||
expect("{{source_mode_instructions}}" not in shaped_intern, f"{label} has no source-mode placeholder")
|
||||
expect("source_mode" not in shaped_snap, f"{label} snapshot has no source_mode")
|
||||
expect("editorial_mode" not in shaped_snap, f"{label} snapshot has no editorial_mode")
|
||||
expect("editorial_mode" not in (run.json().get("trace") or {}), f"{label} trace has no editorial_mode")
|
||||
expect([item.get("purpose") for item in rec] == ["journal_generate"], f"{label} is exactly one generate call")
|
||||
for body in bodies:
|
||||
for line in body.splitlines():
|
||||
token = line.lstrip("- ").strip()
|
||||
if token:
|
||||
expect(token in shaped_sources, f"{label} keeps source {token!r}")
|
||||
|
||||
generate_shapes("2026-08-11", ["Ich ging zum Markt. Es war voll. Vielleicht bleibe ich kürzer."], "prose-only")
|
||||
generate_shapes("2026-08-12", ["- Kirschen\n- Brot nicht geholt\n- später Hafen"], "notes-only")
|
||||
generate_shapes(
|
||||
"2026-08-13",
|
||||
["Ich ging zum Markt. Es war voll.", "- Kirschen\n- später Hafen"],
|
||||
"prose-then-notes",
|
||||
)
|
||||
generate_shapes(
|
||||
"2026-08-14",
|
||||
["- Kirschen\n- später Hafen", "Ich ging zum Markt. Es war voll."],
|
||||
"notes-then-prose",
|
||||
)
|
||||
generate_shapes(
|
||||
"2026-08-15",
|
||||
["Ich ging zum Markt. Es war voll.\n- Kirschen\n- später Hafen"],
|
||||
"mixed-in-message",
|
||||
)
|
||||
|
||||
high_run = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"conversation_ids": [conv.json()["id"]],
|
||||
"generation_selection": high_ids(),
|
||||
"remember_generation_selection": True,
|
||||
},
|
||||
)
|
||||
expect(high_run.status_code == 200, f"high generate {high_run.text}")
|
||||
high_intern = intern_of(high_run.json())
|
||||
expect("Überarbeite den Text substanziell" in high_intern, "high transformation reaches the prompt")
|
||||
expect("Korrigiere nur Rechtschreibung" not in high_intern, "low and high prompt blocks differ")
|
||||
expect(intern != high_intern, "low and high snapshots render different prompt blocks")
|
||||
expect(selection_meta(high_run.json()).get("remembered") is True, "remembered snapshot is marked")
|
||||
remembered = client.get("/api/journal/generation-settings", headers=headers).json()
|
||||
expect(remembered["selection"] == high_ids(), "remembered snapshot becomes the profile default")
|
||||
for fact in HARD_FACTS:
|
||||
expect(fact in high_intern, f"hard fact remains at high policy: {fact}")
|
||||
|
||||
profile_run = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(profile_run.status_code == 200, f"profile generate {profile_run.text}")
|
||||
expect(selection_meta(profile_run.json()).get("source") == "profile", "omitted snapshot uses stored values")
|
||||
expect(seed_instruction("narrative", "emphasized") in intern_of(profile_run.json()), "stored high narrative is reused")
|
||||
|
||||
confirm_identity(profile_id, "Clarissa", aliases=["Sushi"])
|
||||
alias_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-21"},
|
||||
)
|
||||
alias_conv = client.post(
|
||||
f"/api/journal/days/{alias_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Alias"},
|
||||
)
|
||||
client.post(
|
||||
f"/api/journal/conversations/{alias_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Sushi kam ins Wohnzimmer."},
|
||||
)
|
||||
|
||||
def canonical_reply(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Wohnzimmer\n\nClarissa kam ins Wohnzimmer.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", canonical_reply):
|
||||
alias_out = client.post(
|
||||
f"/api/journal/days/{alias_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [alias_conv.json()["id"]]},
|
||||
)
|
||||
expect(alias_out.status_code == 200, f"confirmed alias generate {alias_out.text}")
|
||||
expect("Clarissa" in (alias_out.json().get("body") or ""), "canonical spelling is kept after demask")
|
||||
expect((alias_out.json().get("trace") or {}).get("model_text_accepted") is True, "alias demask is not rejected as unattested")
|
||||
|
||||
remember_mapping(profile_id, "Hanna", "PERSON:99")
|
||||
|
||||
def invent(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Hanna am Hafen\n\nIch war am Markt.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", invent):
|
||||
blocked = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(blocked.status_code == 409, "unattested person is still rejected")
|
||||
expect((blocked.json().get("detail") or {}).get("code") == "journal_generation_not_accepted", "409 provenance rule remains")
|
||||
|
||||
created_user = client.post(
|
||||
"/api/users",
|
||||
headers=headers,
|
||||
json={"email": "policy-user@example.test", "name": "User", "password": "user-pass", "role": "user"},
|
||||
)
|
||||
expect(created_user.status_code == 200, f"create user {created_user.text}")
|
||||
user_login = client.post(
|
||||
"/api/auth/login",
|
||||
json={"email": "policy-user@example.test", "password": "user-pass"},
|
||||
)
|
||||
user_headers = header(user_login.json()["token"])
|
||||
denied = client.get("/api/admin/generation-instructions/journal_generate", headers=user_headers)
|
||||
expect(denied.status_code == 403, "admin catalog is protected")
|
||||
denied_put = client.put(
|
||||
"/api/admin/generation-instructions/journal_generate/" + seed_id("transformation", "substantial"),
|
||||
headers=user_headers,
|
||||
json={"label": "nein"},
|
||||
)
|
||||
expect(denied_put.status_code == 403, "admin catalog write is protected")
|
||||
|
||||
catalog = client.get("/api/admin/generation-instructions/journal_generate", headers=headers)
|
||||
expect(catalog.status_code == 200, f"admin catalog {catalog.text}")
|
||||
expect("source_mode" not in catalog.json()["slots"], "admin catalog has no source-mode slot")
|
||||
expect("instruction" not in str(catalog.json()["slots"]["transformation"]), "admin overview omits prompt bodies")
|
||||
cloned = client.post(
|
||||
f"/api/admin/generation-instructions/journal_generate/{seed_id('transformation', 'substantial')}/clone",
|
||||
headers=headers,
|
||||
)
|
||||
expect(cloned.status_code == 200, f"clone {cloned.text}")
|
||||
saved = client.put(
|
||||
f"/api/admin/generation-instructions/journal_generate/{cloned.json()['id']}",
|
||||
headers=headers,
|
||||
json={
|
||||
"guideline_key": "substantial",
|
||||
"label": "substanziell",
|
||||
"summary": "Klon",
|
||||
"instruction": "ADMIN_CUSTOM_TRANSFORMATION_BLOCK",
|
||||
},
|
||||
)
|
||||
expect(saved.status_code == 200, f"admin draft save {saved.text}")
|
||||
published = client.post(
|
||||
f"/api/admin/generation-instructions/journal_generate/{cloned.json()['id']}/publish",
|
||||
headers=headers,
|
||||
)
|
||||
expect(published.status_code == 200, f"publish {published.text}")
|
||||
immutable = client.put(
|
||||
f"/api/admin/generation-instructions/journal_generate/{seed_id('transformation', 'substantial')}",
|
||||
headers=headers,
|
||||
json={"label": "überschreiben", "instruction": "should not stick", "guideline_key": "substantial"},
|
||||
)
|
||||
expect(immutable.status_code == 409, "active guidelines cannot be overwritten")
|
||||
preview = client.post(
|
||||
"/api/admin/generation-instructions/journal_generate/preview",
|
||||
headers=headers,
|
||||
json={
|
||||
"generation_selection": {**high_ids(), "transformation_id": cloned.json()["id"]},
|
||||
},
|
||||
)
|
||||
expect(preview.status_code == 200, f"admin preview {preview.text}")
|
||||
expect(preview.json()["transformation_instructions"] == "ADMIN_CUSTOM_TRANSFORMATION_BLOCK", "preview compiles locally")
|
||||
expect("(nicht enthalten)" in (preview.json().get("rendered") or ""), "preview has no personal sources")
|
||||
expect("{{transformation_instructions}}" not in (preview.json().get("rendered") or ""), "preview resolves placeholders")
|
||||
|
||||
custom_run = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"conversation_ids": [conv.json()["id"]],
|
||||
"generation_selection": {**high_ids(), "transformation_id": cloned.json()["id"]},
|
||||
"remember_generation_selection": False,
|
||||
},
|
||||
)
|
||||
expect(custom_run.status_code == 200, f"custom generate {custom_run.text}")
|
||||
expect("ADMIN_CUSTOM_TRANSFORMATION_BLOCK" in intern_of(custom_run.json()), "admin change reaches the next rendered prompt")
|
||||
|
||||
init_db()
|
||||
after_seed = overview_payload()
|
||||
substantial = next(
|
||||
item for item in after_seed["slots"]["transformation"] if item["id"] == seed_id("transformation", "substantial")
|
||||
)
|
||||
expect(get_guideline(substantial["id"], include_instruction=True)["instruction"] == seed_instruction("transformation", "substantial"), "seed does not overwrite the original")
|
||||
expect(get_guideline(cloned.json()["id"], include_instruction=True)["instruction"] == "ADMIN_CUSTOM_TRANSFORMATION_BLOCK", "seed does not overwrite admin clones")
|
||||
|
||||
restored = client.post(
|
||||
"/api/admin/generation-instructions/journal_generate/reset",
|
||||
headers=headers,
|
||||
)
|
||||
expect(restored.status_code == 200, f"reset {restored.text}")
|
||||
drafts = [item for item in restored.json()["slots"]["transformation"] if item["status"] == "draft"]
|
||||
expect(drafts, "reset adds seed drafts")
|
||||
expect(
|
||||
get_guideline(cloned.json()["id"], include_instruction=True)["instruction"] == "ADMIN_CUSTOM_TRANSFORMATION_BLOCK",
|
||||
"reset does not overwrite historical variants",
|
||||
)
|
||||
|
||||
archived = client.post(
|
||||
f"/api/admin/generation-instructions/journal_generate/{cloned.json()['id']}/archive",
|
||||
headers=headers,
|
||||
)
|
||||
expect(archived.status_code == 200, f"archive {archived.text}")
|
||||
archived_run = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"conversation_ids": [conv.json()["id"]],
|
||||
"generation_selection": {**high_ids(), "transformation_id": cloned.json()["id"]},
|
||||
},
|
||||
)
|
||||
expect(archived_run.status_code == 400, "archived guidelines are rejected for new runs")
|
||||
|
||||
fresh_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-16"},
|
||||
)
|
||||
fresh_conv = client.post(
|
||||
f"/api/journal/days/{fresh_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Legacy"},
|
||||
)
|
||||
client.post(
|
||||
f"/api/journal/conversations/{fresh_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Ich war am Markt."},
|
||||
)
|
||||
with get_db() as conn:
|
||||
current_template = conn.execute(
|
||||
"SELECT template FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()["template"]
|
||||
conn.execute(
|
||||
"UPDATE ai_prompts SET template = ? WHERE slug = ?",
|
||||
((current_template or "") + "\n{{source_mode_instructions}}\n", "mvp.journal_generate"),
|
||||
)
|
||||
reset_debug()
|
||||
legacy_recorder = install_test_recorder()
|
||||
legacy = client.post(
|
||||
f"/api/journal/days/{fresh_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [fresh_conv.json()["id"]], "generation_selection": default_ids()},
|
||||
)
|
||||
expect(legacy.status_code == 409, "legacy custom prompt is refused before the provider")
|
||||
legacy_detail = legacy.json().get("detail") or {}
|
||||
expect(legacy_detail.get("code") == "prompt_contract_incompatible", "legacy prompt uses prompt_contract_incompatible")
|
||||
expect("veralteten Platzhalter" in (legacy_detail.get("message") or ""), "admin hint names the retired placeholder")
|
||||
expect(legacy_recorder == [], "legacy custom prompt does not call the provider")
|
||||
expect(current_draft(profile_id, fresh_day.json()["day"]["id"]) is None, "legacy custom prompt inserts no raw draft")
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE ai_prompts SET template = default_template WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
)
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE generation_guidelines
|
||||
SET instruction = ''
|
||||
WHERE id = ?
|
||||
""",
|
||||
(seed_id("transformation", "substantial"),),
|
||||
)
|
||||
reset_debug()
|
||||
broken_recorder = install_test_recorder()
|
||||
broken = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]], "generation_selection": default_ids()},
|
||||
)
|
||||
expect(broken.status_code == 409, "invalid catalog refuses generate")
|
||||
expect((broken.json().get("detail") or {}).get("code") == "generation_policy_invalid", "invalid catalog uses generation_policy_invalid")
|
||||
expect(broken_recorder == [], "invalid catalog does not call the provider")
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE generation_guidelines
|
||||
SET instruction = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(seed_instruction("transformation", "substantial"), seed_id("transformation", "substantial")),
|
||||
)
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE ai_prompts SET template = 'CUSTOM POLICY PROMPT {{reconstruction}}' WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
)
|
||||
init_db()
|
||||
custom = load_active_prompt("mvp.journal_generate")
|
||||
expect(custom["template"] == "CUSTOM POLICY PROMPT {{reconstruction}}", "independently edited prompt is not overwritten")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect("{{transformation_instructions}}" in (row["default_template"] or ""), "default template still tracks mixed sources")
|
||||
expect(MIXED_SOURCES_INSTRUCTION in (row["default_template"] or ""), "default template has mixed-source instruction")
|
||||
expect(row["seed_revision"] == "2026-08-27-journal-mixed-sources-v1", "revision updates even when template is custom")
|
||||
|
||||
readable = snapshot_summary(
|
||||
{
|
||||
"transformation": {"label": "spürbar"},
|
||||
"detail": {"label": "weitgehend"},
|
||||
"voice": {"label": "deutlich"},
|
||||
"narrative": {"label": "gewichtet"},
|
||||
}
|
||||
)
|
||||
expect("Erzeugt mit:\nSpürbar · Weitgehend · Deutlich · Gewichtet" in readable, "snapshot summary is readable")
|
||||
expect("Quellenmodus" not in readable, "snapshot summary does not name a source mode")
|
||||
|
||||
print("journal generation policy tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -49,21 +49,25 @@ def test_active_prompt_contract() -> None:
|
||||
init_db()
|
||||
prompt = load_active_prompt("mvp.journal_generate")
|
||||
text = prompt.get("template") or ""
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in text, "active prompt separates fact fidelity from wording")
|
||||
expect("INHALTSTREUE" in text, "active prompt keeps hard content rules")
|
||||
expect("CURRENT_DAY_SOURCES" in text, "active prompt labels current-day facts")
|
||||
expect("STYLE_EXAMPLES" in text, "active prompt labels style examples")
|
||||
expect("Rechtschreibung" in text or "korrigieren" in text, "active prompt allows spelling and grammar fixes")
|
||||
expect("Unsicherheit bleibt Unsicherheit" in text, "active prompt keeps semantic uncertainty")
|
||||
expect("keine neuen Informationen" in text.lower() or "Keine neuen Informationen" in text, "active prompt still forbids new facts")
|
||||
expect("Verneinungen" in text and "Unsicherheiten" in text, "active prompt keeps semantic uncertainty")
|
||||
expect("keine neuen tatsachen" in text.lower(), "active prompt still forbids new facts")
|
||||
expect("Plan und Vollzug" in text, "active prompt keeps plan versus completion")
|
||||
expect("[[…" not in text and "[[..." not in text, "active prompt must not teach ellipsis placeholders")
|
||||
expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone")
|
||||
expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone")
|
||||
expect("{{transformation_instructions}}" in text, "active prompt uses compiled transformation instructions")
|
||||
expect("{{source_mode_instructions}}" not in text, "active prompt has no source-mode placeholder")
|
||||
expect("Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus." in text, "active prompt has mixed-source instruction")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect(row["seed_revision"] == "2026-08-27-journal-placeholders-v1", "system prompt revision is stored")
|
||||
expect(row["seed_revision"] == "2026-08-27-journal-mixed-sources-v1", "system prompt revision is stored")
|
||||
expect(row["template"] == row["default_template"], "untouched install uses the seeded template")
|
||||
|
||||
|
||||
@@ -118,6 +122,24 @@ def test_unattested_identity_is_journal_not_privacy() -> None:
|
||||
is None,
|
||||
"food homonym of an unused mapping is not unattested identity",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content(
|
||||
"Clarissa kam ins Wohnzimmer.",
|
||||
["Heute war Sushi im Wohnzimmer."],
|
||||
[
|
||||
{
|
||||
"local_label": "Clarissa",
|
||||
"canonical_label": "Clarissa",
|
||||
"token": "PERSON:01",
|
||||
"aliases": ["Sushi"],
|
||||
"demask_label": "Clarissa",
|
||||
}
|
||||
],
|
||||
["PERSON:01"],
|
||||
)
|
||||
is None,
|
||||
"confirmed alias in the source attests the canonical demasked spelling",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -165,11 +187,11 @@ def main() -> None:
|
||||
for stage in (gen.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
intern = stage.get("intern") or ""
|
||||
style = intern.split("CURRENT_DAY_SOURCES")[0]
|
||||
style = intern.split("\nCURRENT_DAY_SOURCES\n")[0]
|
||||
expect("Neutraler Journalstil" in style, "neutral style reaches the generate prompt")
|
||||
expect("Erzählmerkmale" not in style, "current day dialogue is not a style brief")
|
||||
expect("zimlich" not in style, "today's typo is not a style exemplar")
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in intern, "runtime intern uses the new narration contract")
|
||||
expect("INHALTSTREUE" in intern, "runtime intern uses the new narration contract")
|
||||
expect("CURRENT_DAY_SOURCES" in intern, "runtime intern labels current-day facts")
|
||||
expect("Nur den verifizierten Nutzerwortlaut" not in intern, "old output-wording rule is not in the runtime prompt")
|
||||
expect("Unsicherheiten im Wortlaut" not in intern, "old uncertainty-in-wording rule is not in the runtime prompt")
|
||||
@@ -252,7 +274,7 @@ def main() -> None:
|
||||
for stage in (second.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
intern2 = stage.get("intern") or ""
|
||||
expect("trockener Schnitt" in intern2.split("CURRENT_DAY_SOURCES")[0], "confirmed writing profile reaches generate")
|
||||
expect("trockener Schnitt" in intern2.split("\nCURRENT_DAY_SOURCES\n")[0], "confirmed writing profile reaches generate")
|
||||
|
||||
def invent(_messages, _policy):
|
||||
return ChatResult(
|
||||
@@ -268,13 +290,14 @@ def main() -> None:
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(blocked.status_code == 200, f"unattested generate {blocked.text}")
|
||||
log = blocked.json().get("run_log") or []
|
||||
expect(any(item.get("reason") == "unattested_identity" for item in log), "unattested person uses local fallback")
|
||||
expect(any(item.get("status") == "local_fallback" for item in log), "fallback path is distinct from the model path")
|
||||
expect("Clarissa" not in (blocked.json().get("body") or ""), "unattested person is not in the accepted draft")
|
||||
expect("markt" in (blocked.json().get("body") or "").lower(), "fallback still has the user sources")
|
||||
expect("rote tasche" in (blocked.json().get("body") or "").lower(), "fallback keeps the unique source detail")
|
||||
expect(blocked.status_code == 409, f"unattested generate {blocked.text}")
|
||||
detail = blocked.json().get("detail") or {}
|
||||
expect(detail.get("code") == "journal_generation_not_accepted", "unattested person is not stored as a draft")
|
||||
expect(detail.get("message") == "Generierung nicht übernommen.", "API names the rejection")
|
||||
log = (detail.get("diagnostics") or {}).get("log") or []
|
||||
expect(any(item.get("reason") == "unattested_identity" for item in log), "unattested person is a provenance reject")
|
||||
expect(any(item.get("status") == "not_accepted" for item in log), "reject path is distinct from the model path")
|
||||
expect("Clarissa" not in json.dumps((detail.get("diagnostics") or {}).get("log") or []), "clear labels stay out of compact logs")
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
@@ -289,8 +312,8 @@ def main() -> None:
|
||||
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in (row["default_template"] or ""), "default template still tracks the seed")
|
||||
expect(row["seed_revision"] == "2026-08-27-journal-placeholders-v1", "revision updates even when template is custom")
|
||||
expect("INHALTSTREUE" in (row["default_template"] or ""), "default template still tracks the seed")
|
||||
expect(row["seed_revision"] == "2026-08-27-journal-mixed-sources-v1", "revision updates even when template is custom")
|
||||
|
||||
print("journal narration tests passed.")
|
||||
|
||||
|
||||
@@ -328,7 +328,7 @@ def main() -> None:
|
||||
"A style brief reaches generate",
|
||||
)
|
||||
expect("STYLE_EXAMPLES" in narrate_intern, "A style examples block is labeled")
|
||||
style_part = narrate_intern.split("CURRENT_DAY_SOURCES")[0]
|
||||
style_part = narrate_intern.split("\nCURRENT_DAY_SOURCES\n")[0]
|
||||
expect("Erzählmerkmale" not in style_part, "A current day dialogue is not appended as style signals")
|
||||
expect("Neutraler Journalstil" in style_part or "Core:" in style_part, "A empty profile uses the neutral journal voice")
|
||||
expect(
|
||||
|
||||
@@ -503,12 +503,9 @@ def main() -> None:
|
||||
_demask("[[ person:01 ]] war ruhig.", anna_manifest) == "Anna war ruhig.",
|
||||
"demask is case- and space-insensitive",
|
||||
)
|
||||
blocked_plain = False
|
||||
try:
|
||||
_validate_response("Anna war ruhig.", anna_manifest)
|
||||
except PrivacyGatewayError as exc:
|
||||
blocked_plain = exc.code == "response_validation_failed"
|
||||
expect(blocked_plain, "plaintext identity is blocked before demask")
|
||||
tokenized = _validate_response("Anna war ruhig.", anna_manifest)
|
||||
expect(tokenized.startswith("[[PERSON:01]]"), "active cleartext is normalized, not discarded")
|
||||
expect(_demask(tokenized, anna_manifest) == "Anna war ruhig.", "normalized plaintext demasks")
|
||||
|
||||
no_user = client.get("/api/admin/identities")
|
||||
expect(no_user.status_code in {401, 403}, "identity registry requires auth")
|
||||
|
||||
@@ -9,7 +9,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
os.environ.setdefault("KANSHO_FAKE_DETECT", "1")
|
||||
|
||||
from entity_detect_eval import run_fake
|
||||
from entity_detect_eval import CASES, run_fake, score_spans
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
@@ -22,7 +22,22 @@ def main() -> None:
|
||||
payload = run_fake()
|
||||
expect(payload["mode"] == "fake", "eval default is fake")
|
||||
expect(payload["live_quality"] == "unconfirmed", "live quality stays unconfirmed")
|
||||
expect(len(payload["cases"]) >= 3, "synthetic cases exist")
|
||||
expect(payload["detect_model_unconfirmed"] == "openai/gpt-4.1-nano", "configured detect model stays unconfirmed")
|
||||
expect(len(payload["cases"]) >= 8, "synthetic span cases exist")
|
||||
person = next(item for item in CASES if item["id"] == "no_sentence_span")
|
||||
metrics = score_spans(
|
||||
[{"start": 9, "end": 13, "text": "Anna", "entity_type": "PERSON"}],
|
||||
person["expected"],
|
||||
)
|
||||
expect(metrics["expected_found"] == 1, "exact expected span is counted")
|
||||
expect(metrics["precision"] == 1.0 and metrics["recall"] == 1.0, "exact span match is precision 1")
|
||||
wrong = score_spans(
|
||||
[{"start": 0, "end": 28, "text": "Ich traf Anna am Nachmittag.", "entity_type": "PERSON"}],
|
||||
person["expected"],
|
||||
)
|
||||
expect(wrong["unexpected"] == 1, "sentence-sized span is unexpected")
|
||||
food = next(item for item in payload["cases"] if item["id"] == "same_word_two_roles")
|
||||
expect(food["entities"]["metrics"]["expected_found"] == 1, "fake contract finds only the identity sushi")
|
||||
print("All detect eval harness tests passed.")
|
||||
|
||||
|
||||
|
||||
@@ -22,12 +22,12 @@ from identity_store import remember_mapping
|
||||
from main import app
|
||||
from privacy_gateway import (
|
||||
ERROR_EGRESS_VALIDATION,
|
||||
IDENTITY_LEAK_RETRY,
|
||||
ActiveReplacement,
|
||||
GatewayRequest,
|
||||
MaskingManifest,
|
||||
PrivacyGatewayError,
|
||||
_demask,
|
||||
_process_response,
|
||||
_validate_response,
|
||||
complete,
|
||||
last_trace,
|
||||
@@ -90,14 +90,10 @@ def main() -> None:
|
||||
|
||||
kinship = mask_prompt("Meine Frau Sushi kam später.", sushi, "journal_generate")
|
||||
expect(len(kinship.replacements) == 1, "identity mention of the same word is active")
|
||||
blocked = False
|
||||
try:
|
||||
_validate_response("Sushi kam vorbei.", kinship)
|
||||
except PrivacyGatewayError as exc:
|
||||
blocked = exc.code == "response_validation_failed"
|
||||
expect(exc.diagnostics.get("leak_tokens") == ["PERSON:01"], "block reports the token, not the label")
|
||||
expect("Sushi" not in json.dumps(exc.diagnostics), "validation diagnostics omit the clear name")
|
||||
expect(blocked, "active clear name in the raw reply is a leak")
|
||||
normalized, meta = _process_response("Sushi kam vorbei.", kinship)
|
||||
expect(normalized.startswith("[[PERSON:01]]"), "active cleartext is remapped to the request token")
|
||||
expect(meta.get("response_normalization") == "active_cleartext_normalized", "normalization is named")
|
||||
expect(_demask(normalized, kinship) == "Sushi kam vorbei.", "normalized reply demasks locally")
|
||||
homonym_ok = _validate_response("Danach Sushi essen.", kinship)
|
||||
expect("Sushi essen" in homonym_ok, "homonym in the reply uses the same classification rule")
|
||||
|
||||
@@ -107,7 +103,9 @@ def main() -> None:
|
||||
leftover = _demask("[[PERSON:01]] und [[PERSON:99]]", kinship)
|
||||
expect("Sushi" in leftover and "[[PERSON:99]]" in leftover, "inactive placeholder is not rematerialized")
|
||||
expect("Clarissa" not in leftover, "inactive mapping label is not introduced by demask")
|
||||
expect("[[" not in IDENTITY_LEAK_RETRY, "retry instruction must not teach bracket placeholders")
|
||||
import privacy_gateway as gw
|
||||
|
||||
expect(not hasattr(gw, "IDENTITY_LEAK_RETRY"), "automatic identity-leak retry instruction is gone")
|
||||
expect(
|
||||
_demask("[[ person:01 ]] war ruhig.", kinship) == "Sushi war ruhig.",
|
||||
"demask ignores case and inner spacing",
|
||||
@@ -121,6 +119,25 @@ def main() -> None:
|
||||
"generic ellipsis placeholders are not rematerialized",
|
||||
)
|
||||
|
||||
alias_rows = [
|
||||
{
|
||||
"local_label": "Sushi",
|
||||
"token": "PERSON:07",
|
||||
"canonical_label": "Clarissa",
|
||||
"demask_label": "Clarissa",
|
||||
"aliases": ["Sushi"],
|
||||
"labels": ["Clarissa", "Sushi"],
|
||||
"entity_type": "PERSON",
|
||||
}
|
||||
]
|
||||
aliased = mask_prompt("Sushi kam ins Wohnzimmer.", alias_rows, "journal_generate")
|
||||
expect(len(aliased.replacements) == 1, "confirmed alias activates its token")
|
||||
spellings = {label.casefold() for label in aliased.replacements[0].all_labels()}
|
||||
expect(spellings == {"sushi", "clarissa"}, "request manifest keeps every spelling of the active token")
|
||||
expect(_demask("[[PERSON:07]] kam.", aliased) == "Clarissa kam.", "confirmed alias demasks to the canonical spelling")
|
||||
canonical_clear, _meta = _process_response("Clarissa kam vorbei.", aliased)
|
||||
expect(canonical_clear.startswith("[[PERSON:07]]"), "canonical spelling of an active alias token is remapped")
|
||||
|
||||
overlap = mask_prompt(
|
||||
"Anna-Lena kam vorbei.",
|
||||
[
|
||||
@@ -133,6 +150,54 @@ def main() -> None:
|
||||
expect("[[PERSON:08]]" in overlap.masked_text, "longer label is the one replaced")
|
||||
expect("Anna-Lena" not in overlap.masked_text, "replaced longer label is gone")
|
||||
|
||||
mixed_word = "Ich aß Sushi. Sushi kam später."
|
||||
second = mixed_word.find("Sushi", mixed_word.find("Sushi") + 1)
|
||||
span_only = mask_prompt(
|
||||
mixed_word,
|
||||
[
|
||||
{
|
||||
"local_label": "Sushi",
|
||||
"token": "PERSON:01",
|
||||
"entity_type": "PERSON",
|
||||
"start": second,
|
||||
"end": second + 5,
|
||||
"text": "Sushi",
|
||||
"source": "request_local",
|
||||
}
|
||||
],
|
||||
"journal_generate",
|
||||
)
|
||||
expect(span_only.masked_text.startswith("Ich aß Sushi."), "undetected same wording stays unmasked")
|
||||
expect("[[PERSON:01]] kam später." in span_only.masked_text, "only the detected span is masked")
|
||||
validate_pre_egress(span_only.masked_text, span_only)
|
||||
|
||||
overlapping_spans = mask_prompt(
|
||||
"Anna-Lena kam vorbei.",
|
||||
[
|
||||
{
|
||||
"local_label": "Anna",
|
||||
"token": "PERSON:01",
|
||||
"entity_type": "PERSON",
|
||||
"start": 0,
|
||||
"end": 4,
|
||||
"text": "Anna",
|
||||
"source": "request_local",
|
||||
},
|
||||
{
|
||||
"local_label": "Anna-Lena",
|
||||
"token": "PERSON:08",
|
||||
"entity_type": "PERSON",
|
||||
"start": 0,
|
||||
"end": 9,
|
||||
"text": "Anna-Lena",
|
||||
"source": "request_local",
|
||||
},
|
||||
],
|
||||
"journal_generate",
|
||||
)
|
||||
expect("[[PERSON:08]]" in overlapping_spans.masked_text, "longest overlapping span wins")
|
||||
expect("[[PERSON:01]]" not in overlapping_spans.masked_text, "shorter overlapping span is dropped")
|
||||
|
||||
subject = mask_prompt(
|
||||
"Clarissa kaufte Kirschen.",
|
||||
[{"local_label": "Clarissa", "token": "PERSON:99"}],
|
||||
@@ -166,7 +231,7 @@ def main() -> None:
|
||||
second = pool.submit(check, "Heute nur Markt.", anna + lars, "Anna kam.")
|
||||
code_a, _text_a, tokens_a = first.result()
|
||||
code_b, text_b, tokens_b = second.result()
|
||||
expect(code_a == "response_validation_failed", "parallel request A still blocks its own active leak")
|
||||
expect(code_a == "ok" and "[[PERSON:01]]" in (_text_a or ""), "parallel request A normalizes its own active cleartext")
|
||||
expect(tokens_a == ["PERSON:01"], "parallel request A keeps its own active set")
|
||||
expect(code_b == "ok" and "Anna kam" in (text_b or ""), "parallel request B is not blocked by A's mapping")
|
||||
expect(tokens_b == [], "parallel request B does not inherit A's active set")
|
||||
@@ -218,30 +283,26 @@ def main() -> None:
|
||||
|
||||
seen = []
|
||||
|
||||
def leak_then_ok(messages, _policy):
|
||||
def leak_once(messages, _policy):
|
||||
seen.append(messages[0].get("content") or "")
|
||||
if len(seen) == 1:
|
||||
return ChatResult(
|
||||
content="Anna stand den ganzen Nachmittag am Markt.",
|
||||
model="fake",
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18, "cost": 0.01},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
return ChatResult(
|
||||
content="[[PERSON:01]] stand den ganzen Nachmittag am Markt.",
|
||||
content="Anna stand den ganzen Nachmittag am Markt.",
|
||||
model="fake",
|
||||
usage={"prompt_tokens": 12, "completion_tokens": 8, "total_tokens": 20, "cost": 0.02},
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18, "cost": 0.01},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", leak_then_ok):
|
||||
with patch("privacy_gateway.complete_model", leak_once):
|
||||
repaired = _run_gateway(profile_id, "Ich war mit Anna am Markt.")
|
||||
expect(len(seen) == 2, "a real leak retries exactly once")
|
||||
expect(IDENTITY_LEAK_RETRY in seen[1], "retry uses the generic correction instruction")
|
||||
expect("Anna stand den ganzen Nachmittag" not in seen[1], "discarded raw reply is not part of the retry prompt")
|
||||
expect("[[PERSON:01]]" in seen[0] and "Anna" not in seen[0].replace("[[PERSON:01]]", ""), "retry keeps the already masked prompt")
|
||||
expect("Anna" in (repaired.content or ""), "successful retry is demasked locally")
|
||||
expect(repaired.diagnostics.get("response_validation_retry") == 1, "retry is recorded without the raw reply")
|
||||
expect(len(seen) == 1, "active cleartext does not start a second generate call")
|
||||
expect("[[PERSON:01]]" in seen[0] and "Anna" not in seen[0].replace("[[PERSON:01]]", ""), "egress stays masked")
|
||||
expect("Anna" in (repaired.content or ""), "normalized cleartext is demasked locally")
|
||||
expect(repaired.diagnostics.get("generate_calls") == 1, "exactly one generate call")
|
||||
expect(
|
||||
repaired.diagnostics.get("response_normalization") == "active_cleartext_normalized",
|
||||
"active cleartext is recorded as local normalization",
|
||||
)
|
||||
expect("Anna" not in json.dumps(repaired.diagnostics), "compact diagnostics omit the clear name")
|
||||
|
||||
def spaced_token(_messages, _policy):
|
||||
return ChatResult(
|
||||
@@ -258,10 +319,10 @@ def main() -> None:
|
||||
"gateway demasks case and spacing variants",
|
||||
)
|
||||
expect("[[" not in (restored.content or ""), "no leftover placeholder after a successful demask")
|
||||
expect(repaired.diagnostics.get("prompt_tokens") == 22, "retry aggregates prompt tokens")
|
||||
expect(repaired.diagnostics.get("completion_tokens") == 16, "retry aggregates completion tokens")
|
||||
expect(repaired.diagnostics.get("total_tokens") == 38, "retry aggregates total tokens")
|
||||
expect(abs(float(repaired.diagnostics.get("cost") or 0) - 0.03) < 1e-9, "retry aggregates cost")
|
||||
expect(repaired.diagnostics.get("prompt_tokens") == 10, "single call keeps prompt tokens")
|
||||
expect(repaired.diagnostics.get("completion_tokens") == 8, "single call keeps completion tokens")
|
||||
expect(repaired.diagnostics.get("total_tokens") == 18, "single call keeps total tokens")
|
||||
expect(abs(float(repaired.diagnostics.get("cost") or 0) - 0.01) < 1e-9, "single call keeps cost")
|
||||
|
||||
inactive_calls = {"n": 0}
|
||||
|
||||
@@ -287,20 +348,17 @@ def main() -> None:
|
||||
return ChatResult(
|
||||
content="Anna stand den ganzen Nachmittag am Markt.",
|
||||
model="fake",
|
||||
usage={},
|
||||
usage={"prompt_tokens": 4, "completion_tokens": 6, "total_tokens": 10, "cost": 0.002},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
|
||||
failed = False
|
||||
with patch("privacy_gateway.complete_model", always_leak):
|
||||
try:
|
||||
_run_gateway(profile_id, "Ich war mit Anna am Markt.")
|
||||
except PrivacyGatewayError as exc:
|
||||
failed = exc.code == "response_validation_failed"
|
||||
expect(exc.diagnostics.get("response_validation_retry") == 1, "failed retry is still only one extra call")
|
||||
expect("Anna stand" not in json.dumps(exc.diagnostics.get("log") or []), "discarded reply is not persisted in the log")
|
||||
expect(failed, "second leak stays fail-closed")
|
||||
expect(len(retry_fail) == 2, "gateway stops after one retry")
|
||||
accepted = _run_gateway(profile_id, "Ich war mit Anna am Markt.")
|
||||
expect(len(retry_fail) == 1, "cleartext in the reply does not start a second model call")
|
||||
expect("Anna" in (accepted.content or ""), "source-attested name is demasked after local normalization")
|
||||
expect(accepted.diagnostics.get("generate_calls") == 1, "still exactly one generate call")
|
||||
expect(accepted.trace.get("model") == "fake" or accepted.diagnostics.get("model") == "fake", "error-free trace keeps the model")
|
||||
expect(accepted.diagnostics.get("completion_tokens") == 6, "token usage of the only call remains")
|
||||
|
||||
print("All privacy manifest tests passed.")
|
||||
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
"""Response integrity, span masking, journal rejection, and diagnosis contract."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO = ROOT.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-response-test.sqlite")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from identity_store import list_mappings, remember_mapping
|
||||
from journal_generate import JOURNAL_NOT_ACCEPTED, JOURNAL_NOT_ACCEPTED_MESSAGE
|
||||
from journal_store import current_draft
|
||||
from main import app
|
||||
from privacy_gateway import (
|
||||
ERROR_EGRESS_VALIDATION,
|
||||
ActiveReplacement,
|
||||
GatewayRequest,
|
||||
MaskingManifest,
|
||||
PrivacyGatewayError,
|
||||
complete,
|
||||
mask_prompt,
|
||||
reset_debug,
|
||||
validate_pre_egress,
|
||||
)
|
||||
from providers import ChatResult
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def _run_gateway(profile_id: str, rendered: str, *, purpose: str = "dialogue_turn"):
|
||||
return complete(
|
||||
GatewayRequest(
|
||||
prompt_id="response-integrity",
|
||||
purpose=purpose,
|
||||
data_class="B",
|
||||
profile_id=profile_id,
|
||||
payload={"rendered": rendered, "source_text": rendered, "prompt_slug": "mvp.journal_generate"},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _day_with_turn(client: TestClient, headers: dict, body: str, title: str = "Tag") -> tuple[str, str]:
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": title})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-27"},
|
||||
)
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Gespräch"},
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": body},
|
||||
)
|
||||
if turn.status_code != 200:
|
||||
raise SystemExit(f"FAIL: setup turn {turn.status_code}")
|
||||
print("OK setup turn 200")
|
||||
return day.json()["day"]["id"], conv.json()["id"]
|
||||
|
||||
|
||||
def _generate(client: TestClient, headers: dict, day_id: str, conv_id: str, reply: str, usage=None):
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake(_messages, _policy):
|
||||
calls["n"] += 1
|
||||
return ChatResult(
|
||||
content=reply,
|
||||
model="fake-gpt",
|
||||
usage=usage or {"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 12, "cost": 0.001},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", fake):
|
||||
response = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv_id]},
|
||||
)
|
||||
return response, calls["n"]
|
||||
|
||||
|
||||
def test_span_masking_and_pre_egress() -> None:
|
||||
text = "Ich aß Sushi. Sushi kam später."
|
||||
second = text.find("Sushi", text.find("Sushi") + 1)
|
||||
manifest = mask_prompt(
|
||||
text,
|
||||
[
|
||||
{
|
||||
"local_label": "Sushi",
|
||||
"token": "PERSON:01",
|
||||
"entity_type": "PERSON",
|
||||
"start": second,
|
||||
"end": second + 5,
|
||||
"text": "Sushi",
|
||||
"source": "request_local",
|
||||
}
|
||||
],
|
||||
"journal_generate",
|
||||
)
|
||||
expect(manifest.masked_text == "Ich aß Sushi. [[PERSON:01]] kam später.", "only the detected span is masked")
|
||||
validate_pre_egress(manifest.masked_text, manifest)
|
||||
|
||||
overlapping = mask_prompt(
|
||||
"Anna-Lena kam vorbei.",
|
||||
[
|
||||
{
|
||||
"local_label": "Anna",
|
||||
"token": "PERSON:01",
|
||||
"entity_type": "PERSON",
|
||||
"start": 0,
|
||||
"end": 4,
|
||||
"text": "Anna",
|
||||
"source": "request_local",
|
||||
},
|
||||
{
|
||||
"local_label": "Anna-Lena",
|
||||
"token": "PERSON:08",
|
||||
"entity_type": "PERSON",
|
||||
"start": 0,
|
||||
"end": 9,
|
||||
"text": "Anna-Lena",
|
||||
"source": "request_local",
|
||||
},
|
||||
],
|
||||
"journal_generate",
|
||||
)
|
||||
expect(overlapping.masked_text.startswith("[[PERSON:08]]"), "overlapping spans pick the longest")
|
||||
|
||||
leaky = MaskingManifest(
|
||||
masked_text="Anna war am Markt.",
|
||||
available_mapping_count=1,
|
||||
replacements=(ActiveReplacement("PERSON:01", "PERSON", 1, "Anna"),),
|
||||
)
|
||||
blocked = False
|
||||
try:
|
||||
validate_pre_egress("Anna war am Markt.", leaky)
|
||||
except PrivacyGatewayError as exc:
|
||||
blocked = exc.code == ERROR_EGRESS_VALIDATION
|
||||
expect(blocked, "privacy stays fail-closed before the provider")
|
||||
|
||||
|
||||
def test_runtime_has_no_incident_wordlists() -> None:
|
||||
forbidden = (
|
||||
"Kinder",
|
||||
"Delfine",
|
||||
"delphine",
|
||||
"Wohnzimmer",
|
||||
"Balkon",
|
||||
"Tagesreflektion",
|
||||
"Lebenwesen",
|
||||
"Baguette",
|
||||
)
|
||||
runtime = []
|
||||
for path in ROOT.glob("*.py"):
|
||||
if path.name.startswith("entity_detect_eval") or path.name.startswith("_"):
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for word in forbidden:
|
||||
if word in text:
|
||||
runtime.append(f"{path.name}:{word}")
|
||||
expect(not runtime, f"incident words must not be hardcoded in runtime: {runtime}")
|
||||
|
||||
|
||||
def test_frontend_degraded_copy() -> None:
|
||||
page = (REPO / "frontend" / "src" / "pages" / "JournalDayPage.jsx").read_text(encoding="utf-8")
|
||||
trace = (REPO / "frontend" / "src" / "components" / "CallTrace.jsx").read_text(encoding="utf-8")
|
||||
log = (REPO / "frontend" / "src" / "components" / "RunLogPopup.jsx").read_text(encoding="utf-8")
|
||||
expect("Generierung nicht übernommen" in page, "journal page names a rejected generate")
|
||||
expect("Generierung nicht übernommen" in trace, "trace names a rejected generate")
|
||||
expect("Generierung nicht übernommen" in log, "run log names a rejected generate")
|
||||
expect("nur user-Zeilen" not in trace, "masking copy is not limited to user lines")
|
||||
expect("zwei Modellaufrufe nacheinander" not in log, "running copy does not claim two model calls")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
test_span_masking_and_pre_egress()
|
||||
test_runtime_has_no_incident_wordlists()
|
||||
test_frontend_degraded_copy()
|
||||
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "response@example.test", "name": "Response", "password": "test-pass"},
|
||||
)
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
|
||||
calls = {"n": 0}
|
||||
|
||||
def false_positive_cleartext(_messages, _policy):
|
||||
calls["n"] += 1
|
||||
return ChatResult(
|
||||
content="Danach Sushi essen am Markt.",
|
||||
model="fake-gpt",
|
||||
usage={"prompt_tokens": 8, "completion_tokens": 6, "total_tokens": 14, "cost": 0.002},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
|
||||
remember_mapping(profile_id, "Sushi", "PERSON:01")
|
||||
with patch("privacy_gateway.complete_model", false_positive_cleartext):
|
||||
result = _run_gateway(profile_id, "Ich aß Sushi. Meine Frau Sushi kam später.")
|
||||
expect(calls["n"] == 1, "false-positive cleartext still uses exactly one generate call")
|
||||
expect("Sushi" in (result.content or ""), "reconstructed common noun is kept after local normalization")
|
||||
expect(result.diagnostics.get("generate_calls") == 1, "gateway reports one generate call")
|
||||
expect(
|
||||
result.diagnostics.get("response_normalization") in {"active_cleartext_normalized", "none"},
|
||||
"normalization is recorded without a retry",
|
||||
)
|
||||
|
||||
remember_mapping(profile_id, "Anna", "PERSON:02")
|
||||
day_id, conv_id = _day_with_turn(client, headers, "Heute war ich mit Anna am Markt.")
|
||||
gen, n = _generate(
|
||||
client,
|
||||
headers,
|
||||
day_id,
|
||||
conv_id,
|
||||
"Ein Markttag\n\nAnna stand den ganzen Nachmittag am Markt.",
|
||||
)
|
||||
expect(gen.status_code == 200, f"attested person cleartext {gen.status_code}")
|
||||
expect(n == 1, "attested person cleartext does not retry")
|
||||
expect("Anna" in (gen.json().get("body") or ""), "attested person is demasked")
|
||||
expect((gen.json().get("trace") or {}).get("model_text_accepted") is True, "attested model text is accepted")
|
||||
|
||||
remember_mapping(profile_id, "Clarissa", "PERSON:99")
|
||||
hist_day, hist_conv = _day_with_turn(client, headers, "Heute nur Markt und Kirschen.", title="Historisch")
|
||||
blocked, n = _generate(
|
||||
client,
|
||||
headers,
|
||||
hist_day,
|
||||
hist_conv,
|
||||
"Ein Markttag\n\nClarissa kaufte Kirschen am Markt.",
|
||||
)
|
||||
expect(blocked.status_code == 409, f"style-only person {blocked.status_code}")
|
||||
detail = blocked.json().get("detail") or {}
|
||||
expect(detail.get("code") == JOURNAL_NOT_ACCEPTED, "historical-only name is rejected")
|
||||
expect(detail.get("message") == JOURNAL_NOT_ACCEPTED_MESSAGE, "API message is explicit")
|
||||
expect(n == 1, "provenance reject does not retry")
|
||||
expect(current_draft(profile_id, hist_day) is None, "rejected generate inserts no draft")
|
||||
diag = detail.get("diagnostics") or {}
|
||||
trace = diag.get("trace") or {}
|
||||
expect(trace.get("abort_reason") == "unattested_identity", "provenance abort reason remains")
|
||||
expect(trace.get("model_text_accepted") is False, "model text is marked not accepted")
|
||||
expect(
|
||||
trace.get("model") == "fake-gpt" or (trace.get("budget") or {}).get("model") == "fake-gpt",
|
||||
"error trace keeps the model",
|
||||
)
|
||||
expect(
|
||||
(trace.get("generate_calls") == 1)
|
||||
or ((trace.get("budget") or {}).get("generate_calls") == 1)
|
||||
or diag.get("generate_calls") == 1,
|
||||
"error trace keeps generate_calls",
|
||||
)
|
||||
expect(
|
||||
(trace.get("budget") or {}).get("completion_tokens") == 5 or diag.get("completion_tokens") == 5,
|
||||
"error trace keeps completion tokens",
|
||||
)
|
||||
|
||||
unknown_day, unknown_conv = _day_with_turn(client, headers, "Heute nur der Markt.", title="Platzhalter")
|
||||
unknown, n = _generate(
|
||||
client,
|
||||
headers,
|
||||
unknown_day,
|
||||
unknown_conv,
|
||||
"Ein Markttag\n\n[[PERSON:99]] stand am Markt.",
|
||||
)
|
||||
expect(unknown.status_code == 409, f"unknown placeholder {unknown.status_code}")
|
||||
expect((unknown.json().get("detail") or {}).get("code") == JOURNAL_NOT_ACCEPTED, "unknown placeholder is rejected")
|
||||
expect(n == 1, "unknown placeholder does not retry")
|
||||
expect(current_draft(profile_id, unknown_day) is None, "unknown placeholder inserts no draft")
|
||||
|
||||
generic_day, generic_conv = _day_with_turn(client, headers, "Heute nur der Hafen.", title="Generisch")
|
||||
generic, n = _generate(
|
||||
client,
|
||||
headers,
|
||||
generic_day,
|
||||
generic_conv,
|
||||
"Ein Hafentag\n\n[[...]] blieb am Hafen.",
|
||||
)
|
||||
expect(generic.status_code == 409, f"generic placeholder {generic.status_code}")
|
||||
expect((generic.json().get("detail") or {}).get("code") == JOURNAL_NOT_ACCEPTED, "generic placeholder is rejected")
|
||||
expect(current_draft(profile_id, generic_day) is None, "generic placeholder inserts no draft")
|
||||
|
||||
ok_day, ok_conv = _day_with_turn(client, headers, "Heute war ich mit Anna am Hafen.", title="Tokens")
|
||||
anna_token = next(
|
||||
(
|
||||
item.get("token")
|
||||
for item in list_mappings(profile_id)
|
||||
if (item.get("canonical_label") or item.get("local_label") or "") == "Anna"
|
||||
),
|
||||
"PERSON:02",
|
||||
)
|
||||
ok_gen, n = _generate(
|
||||
client,
|
||||
headers,
|
||||
ok_day,
|
||||
ok_conv,
|
||||
f"Ein Hafentag\n\n[[{anna_token}]] stand am Hafen.",
|
||||
)
|
||||
expect(ok_gen.status_code == 200, f"active tokens {ok_gen.status_code}")
|
||||
expect("Anna" in (ok_gen.json().get("body") or ""), "active tokens demask normally")
|
||||
expect("[[" not in (ok_gen.json().get("body") or ""), "no leftover placeholder after demask")
|
||||
expect(n == 1, "normal demask uses one generate call")
|
||||
|
||||
provider_calls = {"n": 0}
|
||||
|
||||
def boom(_messages, _policy):
|
||||
provider_calls["n"] += 1
|
||||
raise AssertionError("provider must not be called after pre-egress failure")
|
||||
|
||||
def leaky_mask(rendered, mappings, purpose):
|
||||
return MaskingManifest(
|
||||
masked_text=rendered,
|
||||
available_mapping_count=len(mappings or []),
|
||||
replacements=(ActiveReplacement("PERSON:02", "PERSON", 1, "Anna"),),
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.mask_prompt", leaky_mask), patch("privacy_gateway.complete_model", boom):
|
||||
failed = False
|
||||
try:
|
||||
_run_gateway(profile_id, "Anna war am Markt.")
|
||||
except PrivacyGatewayError as exc:
|
||||
failed = exc.code == ERROR_EGRESS_VALIDATION
|
||||
expect(failed, "pre-egress still fail-closed")
|
||||
expect(provider_calls["n"] == 0, "provider is not called when confirmed identity remains")
|
||||
|
||||
print("All privacy response integrity tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user