MVP 0.93 - stabile, aber Writing Profile ohne erkennbaren Nutzen
This commit is contained in:
1 parent
53b1eff833
commit
f17b3083e8
56 files changed
+5366
-784
No files matched your search
@@ -222,6 +222,9 @@ def main() -> None:
|
||||
"source_text",
|
||||
"space_title",
|
||||
"writing_profile",
|
||||
"style_examples",
|
||||
"editorial_mode",
|
||||
"editorial_instructions",
|
||||
"interaction_hint",
|
||||
},
|
||||
"system and mvp context keys",
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Admin identity registry stays local. Run from backend/: python tests/test_identity_registry.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-identity-registry-test.sqlite")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from identity_store import STATUS_LEGACY, list_confirmed_identities, list_registry
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
import sqlite3
|
||||
from db import _IDENTITY_MAPPING_COLUMNS, _ensure_columns
|
||||
from identity_store import migrate_legacy_identity_rows as migrate_legacy
|
||||
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE identity_mappings (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
local_label TEXT NOT NULL DEFAULT '',
|
||||
created TEXT NOT NULL DEFAULT '2026-08-01 00:00:00'
|
||||
);
|
||||
INSERT INTO identity_mappings (id, profile_id, token, local_label)
|
||||
VALUES ('m1', 'p1', 'PERSON:01', 'Altbau');
|
||||
"""
|
||||
)
|
||||
_ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS)
|
||||
migrate_legacy(conn)
|
||||
legacy = dict(conn.execute("SELECT * FROM identity_mappings WHERE id = 'm1'").fetchone())
|
||||
expect(legacy["status"] == STATUS_LEGACY, "existing rows become legacy_review_required")
|
||||
expect(legacy["origin"] == "legacy_auto", "existing rows keep legacy origin")
|
||||
expect(legacy["canonical_label"] == "Altbau", "canonical_label is copied from local_label")
|
||||
expect(bool(legacy["updated"]), "updated is populated after ALTER")
|
||||
conn.close()
|
||||
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
headers = {"X-Auth-Token": setup.json()["token"]}
|
||||
profile_id = setup.json()["profile_id"]
|
||||
|
||||
from db import get_db
|
||||
import uuid
|
||||
|
||||
legacy_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO identity_mappings
|
||||
(id, profile_id, token, local_label, canonical_label, entity_type, status, origin)
|
||||
VALUES (?, ?, ?, ?, ?, 'PERSON', ?, 'legacy_auto')
|
||||
""",
|
||||
(legacy_id, profile_id, "PERSON:70", "Altbau", "Altbau", STATUS_LEGACY),
|
||||
)
|
||||
expect(
|
||||
not any(item["id"] == legacy_id for item in list_confirmed_identities(profile_id)),
|
||||
"legacy row is not confirmed",
|
||||
)
|
||||
expect(any(item["id"] == legacy_id for item in list_registry(profile_id)), "legacy row remains reviewable")
|
||||
|
||||
patched = client.patch(
|
||||
f"/api/admin/identities/{legacy_id}",
|
||||
headers=headers,
|
||||
json={"status": "confirmed", "entity_type": "PLACE", "canonical_label": "Altbau"},
|
||||
)
|
||||
expect(patched.status_code == 200, patched.text)
|
||||
expect(patched.json()["status"] == "confirmed", "admin can confirm a legacy row")
|
||||
expect(patched.json()["entity_type"] == "PLACE", "admin can correct the type")
|
||||
|
||||
created = client.post(
|
||||
"/api/admin/identities",
|
||||
headers=headers,
|
||||
json={"canonical_label": "Nordwerk", "entity_type": "ORG", "aliases": ["Nordwerk AG"]},
|
||||
)
|
||||
expect(created.status_code == 200, created.text)
|
||||
identity_id = created.json()["id"]
|
||||
aliases = client.patch(
|
||||
f"/api/admin/identities/{identity_id}",
|
||||
headers=headers,
|
||||
json={"aliases": ["Nordwerk AG", "Nordwerk"]},
|
||||
)
|
||||
expect(aliases.status_code == 200, aliases.text)
|
||||
expect("Nordwerk AG" in (aliases.json().get("aliases") or []), "confirmed alias can be stored")
|
||||
|
||||
calls = {"n": 0}
|
||||
|
||||
def boom(*_args, **_kwargs):
|
||||
calls["n"] += 1
|
||||
raise AssertionError("no egress")
|
||||
|
||||
with patch("entity_detect.complete_chat", boom), patch("providers.complete_chat", boom):
|
||||
gone = client.delete(f"/api/admin/identities/{identity_id}", headers=headers)
|
||||
expect(gone.status_code == 200, "admin delete stays local")
|
||||
expect(calls["n"] == 0, "registry actions do not call detect or generate")
|
||||
expect(not any(item["id"] == identity_id for item in list_registry(profile_id)), "deleted row is gone")
|
||||
|
||||
print("All identity registry tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -39,6 +39,7 @@ from writing_profile_store import (
|
||||
get_profile,
|
||||
import_text,
|
||||
replace_trait_refs,
|
||||
set_lifecycle,
|
||||
upsert_trait,
|
||||
update_facet,
|
||||
)
|
||||
@@ -769,6 +770,7 @@ def test_task_brief_and_dedupe(client: TestClient, headers: dict, profile_id: st
|
||||
}
|
||||
],
|
||||
)
|
||||
set_lifecycle(profile_id, "confirmed")
|
||||
before = get_profile(profile_id)
|
||||
brief = compile_task_brief(profile_id, "journal_generate")
|
||||
after = get_profile(profile_id)
|
||||
@@ -777,8 +779,8 @@ def test_task_brief_and_dedupe(client: TestClient, headers: dict, profile_id: st
|
||||
expect(_ends_on_word_boundary(brief), "task brief does not cut inside a word")
|
||||
for slug, statement in statements:
|
||||
expect(slug in brief or statement.split(",")[0] in brief, f"relevant trait {slug} can be included")
|
||||
expect(brief.count("Repräsentative Exemplare:") <= 1, "at most one exemplar section")
|
||||
expect(len([line for line in brief.splitlines() if line.startswith("- ") and "Beispiel" not in line]) >= 6, "six traits considered")
|
||||
expect("Repräsentative Exemplare:" not in brief, "historical exemplars are not mixed into the compact brief")
|
||||
expect(len([line for line in brief.splitlines() if line.startswith("- ")]) >= 6, "six traits considered")
|
||||
expect(before["sources"] == after["sources"], "compile_task_brief does not mutate stored sources")
|
||||
expect(
|
||||
[(item.get("slug"), item.get("statement")) for item in before["traits"]]
|
||||
@@ -836,22 +838,43 @@ def test_generate_flow(client: TestClient, headers: dict) -> None:
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"generate {gen.status_code}")
|
||||
expect([item.get("purpose") for item in recorder] == ["journal_reconstruct", "journal_generate"], "both stages ran")
|
||||
expect([item.get("purpose") for item in recorder] == ["journal_generate"], "runtime generate calls only narration")
|
||||
run_log = gen.json().get("run_log") or (gen.json().get("trace") or {}).get("log") or []
|
||||
expect(any(item.get("kind") == "model_call" for item in run_log), "run log records model calls")
|
||||
expect(
|
||||
any(item.get("kind") == "stage1_result" and item.get("stage") == "local_source_artifact" for item in run_log),
|
||||
"run log records local stage 1",
|
||||
)
|
||||
expect(sum(1 for item in run_log if item.get("kind") == "detect") <= 1, "at most one detect call")
|
||||
expect(sum(1 for item in run_log if item.get("kind") == "model_call") == 1, "exactly one generate model call")
|
||||
expect(
|
||||
not any(key in (item or {}) for item in run_log for key in ("intern", "egress", "raw", "reply", "local_label")),
|
||||
"run log omits prompt bodies and labels",
|
||||
)
|
||||
stages = {item.get("purpose"): item for item in (gen.json().get("trace") or {}).get("stages") or []}
|
||||
expect("journal_reconstruct" in stages, "stage 1 ran through the gateway")
|
||||
expect("local_source_artifact" in stages, "stage 1 is the local source artifact")
|
||||
expect("journal_generate" in stages, "stage 2 ran through the gateway")
|
||||
reconstruct = stages["journal_reconstruct"].get("intern") or ""
|
||||
expect("journal_reconstruct" not in stages, "runtime path does not call journal_reconstruct")
|
||||
reconstruct = stages["local_source_artifact"].get("intern") or ""
|
||||
narrate = stages["journal_generate"].get("intern") or ""
|
||||
expect("[u1]" in reconstruct and "user:" in reconstruct, "stage 1 sees labeled user sources")
|
||||
expect("assistant:" in reconstruct, "stage 1 may see assistant as context")
|
||||
expect(stages["local_source_artifact"].get("status") == "local_ok", "local stage 1 succeeded")
|
||||
expect(stages["local_source_artifact"].get("provider") in (None, ""), "local stage has no provider")
|
||||
expect(stages["local_source_artifact"].get("model") in (None, ""), "local stage has no model")
|
||||
expect((stages["local_source_artifact"].get("budget") or {}).get("prompt_tokens") == 0, "local stage has no tokens")
|
||||
expect("verified_artifact" in reconstruct and "u1" in reconstruct, "local artifact contains source ids")
|
||||
expect("assistant:" not in reconstruct, "assistant lines are not in the artifact")
|
||||
expect("Markt" in reconstruct and "Hafen" in reconstruct, "every selected user source is in the artifact")
|
||||
expect("assistant:" not in narrate, "stage 2 does not resend assistant lines")
|
||||
expect("user:" not in narrate, "stage 2 does not resend the raw dialogue")
|
||||
expect("verified_artifact" in narrate and "source_order" in narrate, "stage 2 receives the local verified artifact")
|
||||
expect("sources" in narrate, "stage 2 artifact contains rehydrated sources")
|
||||
expect("CURRENT_DAY_SOURCES" in narrate, "stage 2 labels current-day facts")
|
||||
expect("[u1]" in narrate, "stage 2 lists source ids")
|
||||
expect("7:30" in reconstruct or "7:30" in narrate, "times survive into generation")
|
||||
expect("Markt" in narrate and "Hafen" in narrate, "full selected user sources reach stage 2")
|
||||
expect("Vielleicht bleibe ich kürzer" in narrate, "omitted labels do not drop canonical hedges")
|
||||
expect("assistant:" not in (gen.json().get("body") or ""), "draft body has no assistant lines")
|
||||
expect("WRITING_PROFILE" in narrate or "Neutraler Journalstil" in narrate or "Core:" in narrate, "writing profile reaches generate")
|
||||
expect("STYLE_EXAMPLES" in narrate, "style examples are labeled separately from day facts")
|
||||
expect("Erzählmerkmale" not in (narrate.split("CURRENT_DAY_SOURCES")[0] if "CURRENT_DAY_SOURCES" in narrate else narrate), "day dialogue is not a style brief")
|
||||
|
||||
reset_debug()
|
||||
short_recorder = install_test_recorder()
|
||||
@@ -886,14 +909,14 @@ def test_generate_flow(client: TestClient, headers: dict) -> None:
|
||||
)
|
||||
expect(short_gen.status_code == 200, f"short-claim generate {short_gen.status_code}")
|
||||
expect(
|
||||
[item.get("purpose") for item in short_recorder] == ["journal_reconstruct", "journal_generate"],
|
||||
"short claims still run both stages because sources are rehydrated",
|
||||
[item.get("purpose") for item in short_recorder] == ["journal_generate"],
|
||||
"invalid former stage-1 JSON is irrelevant because reconstruct is not called",
|
||||
)
|
||||
short_narrate = ""
|
||||
for item in (short_gen.json().get("trace") or {}).get("stages") or []:
|
||||
if item.get("purpose") == "journal_generate":
|
||||
short_narrate = item.get("intern") or ""
|
||||
expect("Kirschen" in short_narrate and "Hafen" in short_narrate, "short claims cannot drop the rest of the sources")
|
||||
expect("Kirschen" in short_narrate and "Hafen" in short_narrate, "local artifact cannot drop selected sources")
|
||||
expect("\"feeling\"" not in short_narrate, "feeling labels do not reach stage 2 as facts")
|
||||
budget = (gen.json().get("trace") or {}).get("budget") or stages["journal_generate"].get("budget") or {}
|
||||
expect(budget.get("reserved_output_tokens") == 4096, "trace records reserved output")
|
||||
@@ -930,22 +953,23 @@ def test_generate_flow(client: TestClient, headers: dict) -> None:
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(denied_recon.status_code == 200, f"incomplete reconstruct falls back locally {denied_recon.text}")
|
||||
expect(denied_recon.status_code == 200, f"former reconstruct JSON is unused {denied_recon.text}")
|
||||
fallback_body = denied_recon.json().get("body") or ""
|
||||
expect("Markt" in fallback_body or "Hafen" in fallback_body, "local artifact still produces a draft")
|
||||
fallback_stages = {item.get("purpose"): item for item in (denied_recon.json().get("trace") or {}).get("stages") or []}
|
||||
expect(fallback_stages.get("journal_reconstruct", {}).get("stage1") == "local_fallback", "invalid stage 1 is marked local_fallback")
|
||||
expect(fallback_stages.get("local_source_artifact", {}).get("stage1") == "local_ok", "stage 1 is local and successful")
|
||||
expect(
|
||||
(fallback_stages.get("journal_reconstruct") or {}).get("reason") == "incomplete_coverage",
|
||||
"fallback records why the model JSON was rejected",
|
||||
fallback_stages.get("local_source_artifact", {}).get("coverage") == "all_selected_sources"
|
||||
or "verified_artifact" in ((fallback_stages.get("local_source_artifact") or {}).get("intern") or ""),
|
||||
"local coverage remains all selected sources",
|
||||
)
|
||||
expect("journal_generate" in fallback_stages, "stage 2 still runs on the local artifact")
|
||||
narrate_fallback = fallback_stages.get("journal_generate", {}).get("intern") or ""
|
||||
expect("Heute um 7:30 Uhr Markt" in narrate_fallback, "first user source is rehydrated locally")
|
||||
expect("Später noch der Hafen" in narrate_fallback, "omitted model source is still in the local artifact")
|
||||
expect("Später noch der Hafen" in narrate_fallback, "second user source is still in the local artifact")
|
||||
expect(
|
||||
[item.get("purpose") for item in skip_recorder] == ["journal_reconstruct", "journal_generate"],
|
||||
"stage 2 is called with local coverage after invalid reconstruction",
|
||||
[item.get("purpose") for item in skip_recorder] == ["journal_generate"],
|
||||
"no reconstruct provider call after unused model JSON",
|
||||
)
|
||||
|
||||
reset_catalog()
|
||||
@@ -1010,17 +1034,74 @@ def test_generate_identity_leak_local_fallback(client: TestClient, headers: dict
|
||||
)
|
||||
expect(gen.status_code == 200, f"identity leak still yields a local draft {gen.text}")
|
||||
payload = gen.json()
|
||||
leak_log = payload.get("run_log") or (payload.get("trace") or {}).get("log") or []
|
||||
expect(any(item.get("kind") == "retry" for item in leak_log), "identity leak records a retry")
|
||||
expect(
|
||||
any(item.get("reason") == "identity_leak_blocked" for item in leak_log),
|
||||
"identity leak records local fallback",
|
||||
)
|
||||
body = payload.get("body") or ""
|
||||
expect("Markt" in body and "Kirschen" in body, "local draft keeps attested user wording")
|
||||
expect("Anna" in body, "local draft may keep names that the user actually wrote")
|
||||
stages = {item.get("purpose"): item for item in (payload.get("trace") or {}).get("stages") or []}
|
||||
reconstruct = stages.get("journal_reconstruct") or {}
|
||||
reconstruct = stages.get("local_source_artifact") or {}
|
||||
narrate = stages.get("journal_generate") or {}
|
||||
expect(
|
||||
reconstruct.get("reason") == "identity_leak_blocked" or reconstruct.get("guard") == "identity_leak_blocked",
|
||||
"stage 1 records the blocked leak",
|
||||
)
|
||||
expect(reconstruct.get("status") == "local_ok", "stage 1 stays local and does not call reconstruct")
|
||||
expect(narrate.get("guard") == "identity_leak_blocked", "stage 2 does not use the leaking reply")
|
||||
expect(sum(1 for item in leak_log if item.get("kind") == "retry") == 1, "active leak retries at most once")
|
||||
expect(sum(1 for item in leak_log if item.get("kind") == "model_call") <= 2, "retry is the only extra model call")
|
||||
|
||||
|
||||
def test_generate_inactive_mapping_keeps_model_text(client: TestClient, headers: dict, profile_id: str) -> None:
|
||||
from identity_store import remember_mapping
|
||||
from providers import ChatResult
|
||||
|
||||
remember_mapping(profile_id, "Clarissa", "PERSON:99")
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Inaktiv"})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-26"},
|
||||
)
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Tag"},
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute Markt, danach Kirschen am Hafen."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"inactive-mapping setup {turn.status_code}")
|
||||
calls = {"n": 0}
|
||||
|
||||
def keep(_messages, _policy):
|
||||
calls["n"] += 1
|
||||
return ChatResult(
|
||||
content="Ein Markttag\n\nHeute Markt, danach Kirschen am Hafen. Clarissa blieb unerwähnt.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", keep):
|
||||
gen = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"inactive mapping generate {gen.text}")
|
||||
payload = gen.json()
|
||||
run_log = payload.get("run_log") or (payload.get("trace") or {}).get("log") or []
|
||||
expect(calls["n"] == 1, "inactive mapping does not trigger a retry")
|
||||
expect(not any(item.get("kind") == "retry" for item in run_log), "inactive mapping does not retry")
|
||||
expect(
|
||||
any(item.get("reason") == "unattested_identity" for item in run_log),
|
||||
"invented historical name is unattested journal content",
|
||||
)
|
||||
expect("Clarissa" not in (payload.get("body") or ""), "unattested name is not accepted as narration")
|
||||
expect("Markt" in (payload.get("body") or "") and "Kirschen" in (payload.get("body") or ""), "fallback keeps user sources")
|
||||
|
||||
|
||||
def test_traces_are_request_scoped(client: TestClient, headers: dict) -> None:
|
||||
@@ -1167,6 +1248,7 @@ def main() -> None:
|
||||
test_task_brief_and_dedupe(client, headers, profile_id)
|
||||
test_generate_flow(client, headers)
|
||||
test_generate_identity_leak_local_fallback(client, headers, profile_id)
|
||||
test_generate_inactive_mapping_keeps_model_text(client, headers, profile_id)
|
||||
test_traces_are_request_scoped(client, headers)
|
||||
test_two_conversations_same_day(client, headers)
|
||||
print("journal budget tests passed.")
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
"""Deterministic journal editorial contract. Fake provider proves data flow, not live prose."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-editorial-test.sqlite")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from db import get_db, init_db
|
||||
from engine import load_active_prompt
|
||||
from identity_store import remember_mapping
|
||||
from journal_editorial import (
|
||||
NOTES_TO_JOURNAL,
|
||||
PROSE_EDIT,
|
||||
choose_editorial_mode,
|
||||
editorial_instructions,
|
||||
format_style_examples,
|
||||
lexical_similarity,
|
||||
narration_sources_text,
|
||||
select_journal_style_examples,
|
||||
)
|
||||
from journal_generate import pack_narration_context, unattested_journal_content
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
from prompt_budget import JournalBudget
|
||||
from providers import ChatResult
|
||||
from writing_profile_store import (
|
||||
NEUTRAL_JOURNAL_STYLE,
|
||||
compile_task_brief,
|
||||
import_text,
|
||||
set_lifecycle,
|
||||
update_facet,
|
||||
)
|
||||
|
||||
|
||||
SEED_REVISION = "2026-08-27-journal-placeholders-v1"
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def test_mode_choice() -> None:
|
||||
expect(choose_editorial_mode(["Ich ging zum Markt. Es war voll."]) == PROSE_EDIT, "narrative sentences select prose_edit")
|
||||
expect(choose_editorial_mode(["markt", "kirschen", "hafen"]) == NOTES_TO_JOURNAL, "fragments select notes_to_journal")
|
||||
expect(
|
||||
choose_editorial_mode(["Ich war am Markt.", "Kirschen gekauft.", "hafen später"]) == PROSE_EDIT,
|
||||
"mixed default is prose_edit when at least half the blocks have sentence punctuation",
|
||||
)
|
||||
expect(
|
||||
choose_editorial_mode(["markt", "kirschen", "Später der Hafen."]) == NOTES_TO_JOURNAL,
|
||||
"mixed default is notes_to_journal when the majority lacks sentence punctuation",
|
||||
)
|
||||
prose = editorial_instructions(PROSE_EDIT)
|
||||
notes = editorial_instructions(NOTES_TO_JOURNAL)
|
||||
expect("Gute Formulierungen bewahren" in prose, "prose_edit keeps good wording")
|
||||
expect("zusammenhängende Journalprosa" in notes, "notes_to_journal asks for connected prose")
|
||||
expect(prose != notes, "modes produce different instructions")
|
||||
expect("nicht inklusive ihrer Fehler" in notes or "Fehler hintereinanderkopieren" in notes, "notes must not be concatenated with errors")
|
||||
|
||||
|
||||
def test_prompt_contract() -> None:
|
||||
init_db()
|
||||
prompt = load_active_prompt("mvp.journal_generate")
|
||||
text = prompt.get("template") or ""
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in text, "prompt separates fact fidelity from wording")
|
||||
expect("CURRENT_DAY_SOURCES" in text, "prompt labels current-day facts")
|
||||
expect("WRITING_PROFILE" in text, "prompt labels the writing profile")
|
||||
expect("STYLE_EXAMPLES" in text, "prompt labels style examples")
|
||||
expect("EDITORIAL_MODE" in text, "prompt exposes editorial mode")
|
||||
expect("keine geschützten Fakten" in text, "prompt says typos are not protected facts")
|
||||
expect("{{editorial_instructions}}" in text, "prompt injects mode-specific instructions")
|
||||
expect("{{style_examples}}" in text, "prompt injects style examples")
|
||||
expect("[[…" not in text and "[[..." not in text, "prompt must not teach ellipsis placeholders")
|
||||
expect("zeichengetreu" in text, "prompt asks to copy existing placeholders unchanged")
|
||||
expect("ich gieng zum laden" in text, "prompt includes a synthetic prose_edit example")
|
||||
expect("Im Laden holte ich Brot" in text, "prompt includes a synthetic notes_to_journal example")
|
||||
expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone")
|
||||
expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect(row["seed_revision"] == SEED_REVISION, "system prompt revision is stored")
|
||||
expect(row["template"] == row["default_template"], "untouched install uses the seeded template")
|
||||
|
||||
|
||||
def test_sources_and_examples_are_separated() -> None:
|
||||
artifact = {
|
||||
"kind": "verified_artifact",
|
||||
"coverage": "all_selected_sources",
|
||||
"sources": [{"source_id": "u1", "role": "user", "text": "Heute nur der Markt."}],
|
||||
}
|
||||
presented = narration_sources_text(artifact)
|
||||
expect("[u1]" in presented and "Heute nur der Markt." in presented, "day facts are labeled source blocks")
|
||||
expect("kind" not in presented, "narration does not dump artifact JSON")
|
||||
formatted = format_style_examples(
|
||||
[{"kind": "journal_entry", "excerpt": "Ruhige Sätze, trockener Schnitt.", "occurred_at": "2026-07-01"}]
|
||||
)
|
||||
expect("nicht übernommen" in formatted, "style examples forbid copying their facts")
|
||||
expect("Ruhige Sätze" in formatted, "selected example text is present")
|
||||
|
||||
|
||||
def test_unattested_covers_title_and_body() -> None:
|
||||
mappings = [{"local_label": "Maren", "token": "PERSON:01"}, {"local_label": "Hanna", "token": "PERSON:99"}]
|
||||
sources = ["Heute war ich mit Maren am Markt."]
|
||||
expect(
|
||||
unattested_journal_content("Markttag\n\nIch ging mit Maren zum Markt.", sources, mappings, ["PERSON:01"]) is None,
|
||||
"attested person in title and body is allowed",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Hanna am Hafen\n\nIch ging zum Markt.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_identity",
|
||||
"unattested person in the title is rejected",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("[[PERSON:99]]\n\nIch ging zum Markt.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_placeholder",
|
||||
"inactive placeholder in the title is unattested",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("[[PERSON:01]] am Markt\n\n[[PERSON:01]] kaufte Kirschen.", sources, mappings, ["PERSON:01"])
|
||||
is None,
|
||||
"the same attested placeholder may appear in title and body",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Um 6:30 Uhr kam [[...]] ins Wohnzimmer.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_placeholder",
|
||||
"ellipsis placeholder leftover is unattested",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Um 6:30 Uhr kam [[…]] ins Wohnzimmer.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_placeholder",
|
||||
"unicode ellipsis placeholder leftover is unattested",
|
||||
)
|
||||
|
||||
|
||||
def test_budget_pack_drops_examples_first() -> None:
|
||||
init_db()
|
||||
prompt = {
|
||||
"id": "eval-pack",
|
||||
"slug": "mvp.journal_generate",
|
||||
"prompt_type": "base",
|
||||
"template": "PROFILE\n{{writing_profile}}\nEX\n{{style_examples}}\nDAY\n{{reconstruction}}\nOLD\n{{existing_text}}\n",
|
||||
}
|
||||
assembled = {
|
||||
"writing_profile": "Core: kurze Sätze.",
|
||||
"reconstruction": "Heute Markt.",
|
||||
"editorial_mode": PROSE_EDIT,
|
||||
"editorial_instructions": "x",
|
||||
"style_examples": "",
|
||||
"existing_text": "",
|
||||
}
|
||||
huge_examples = "Stil " + ("Beispielwort " * 400)
|
||||
existing = "Bestehende Fassung " + ("alt " * 40)
|
||||
budget = JournalBudget(
|
||||
model="fake",
|
||||
purpose="journal_generate",
|
||||
effective_context_window=32_768,
|
||||
reserved_output_tokens=256,
|
||||
safety_margin=0.15,
|
||||
available_input_tokens=estimate_cap(assembled, prompt, extra=80),
|
||||
chars_per_token=2.0,
|
||||
)
|
||||
packed, dropped = pack_narration_context(
|
||||
prompt,
|
||||
budget,
|
||||
assembled,
|
||||
style_examples=huge_examples,
|
||||
existing_text=existing,
|
||||
include_existing=True,
|
||||
)
|
||||
expect("style_examples" in dropped, "style examples are dropped before day sources")
|
||||
expect("Heute Markt." in packed["reconstruction"], "day sources stay")
|
||||
expect("kurze Sätze" in packed["writing_profile"], "writing profile stays")
|
||||
|
||||
|
||||
def estimate_cap(assembled: dict, prompt: dict, extra: int) -> int:
|
||||
from engine import preview_prompt
|
||||
from prompt_budget import estimate_tokens
|
||||
|
||||
base = dict(assembled)
|
||||
rendered = preview_prompt(prompt, base)["rendered"]
|
||||
return estimate_tokens(rendered) + extra
|
||||
|
||||
|
||||
def intern_of(payload: dict) -> str:
|
||||
for stage in (payload.get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
return stage.get("intern") or ""
|
||||
return (payload.get("trace") or {}).get("intern") or ""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_mode_choice()
|
||||
test_prompt_contract()
|
||||
test_sources_and_examples_are_separated()
|
||||
test_unattested_covers_title_and_body()
|
||||
test_budget_pack_drops_examples_first()
|
||||
expect(lexical_similarity("a b c", "a b c") > 0.9, "similarity helper is diagnostic")
|
||||
expect(NEUTRAL_JOURNAL_STYLE.startswith("Neutraler Journalstil"), "neutral fallback exists")
|
||||
|
||||
reset_debug()
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "ada@example.test", "name": "Ada", "password": "test-pass"},
|
||||
)
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
expect(compile_task_brief(profile_id) == NEUTRAL_JOURNAL_STYLE, "unconfirmed profile is not a style authority")
|
||||
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Editorial"})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-26"},
|
||||
)
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Tag"},
|
||||
)
|
||||
prose_body = (
|
||||
"ich bin dan zum markt gegangen und da war es zimlich voll. "
|
||||
"vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. "
|
||||
"die rote tasche lag im auto. ich bin dan zum markt gegangen."
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": prose_body},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn {turn.status_code}")
|
||||
gen = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"generate {gen.status_code}")
|
||||
intern = intern_of(gen.json())
|
||||
expect("EDITORIAL_MODE: prose_edit" in intern, "narrative source selects prose_edit")
|
||||
expect("Gute Formulierungen bewahren" in intern, "prose_edit instructions reach the model")
|
||||
expect("CURRENT_DAY_SOURCES" in intern, "day facts are labeled")
|
||||
expect("STYLE_EXAMPLES" in intern, "style examples are labeled")
|
||||
expect("WRITING_PROFILE" in intern, "writing profile is labeled")
|
||||
expect("Neutraler Journalstil" in intern, "without a confirmed profile the neutral fallback is used")
|
||||
expect("zimlich" in intern.split("CURRENT_DAY_SOURCES")[-1], "today remains content, including typos")
|
||||
expect("zimlich" not in intern.split("CURRENT_DAY_SOURCES")[0], "today is not a style authority")
|
||||
expect(sum(1 for item in (gen.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "normal path is one generate call")
|
||||
expect((gen.json().get("trace") or {}).get("editorial_mode") == PROSE_EDIT, "editorial mode is in the admin trace")
|
||||
|
||||
notes_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-25"},
|
||||
)
|
||||
notes_conv = client.post(
|
||||
f"/api/journal/days/{notes_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Notizen"},
|
||||
)
|
||||
client.post(
|
||||
f"/api/journal/conversations/{notes_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "markt\nkirschen\nspäter hafen"},
|
||||
)
|
||||
notes_gen = client.post(
|
||||
f"/api/journal/days/{notes_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [notes_conv.json()["id"]]},
|
||||
)
|
||||
expect(notes_gen.status_code == 200, f"notes generate {notes_gen.text}")
|
||||
notes_intern = intern_of(notes_gen.json())
|
||||
expect("EDITORIAL_MODE: notes_to_journal" in notes_intern, "fragments select notes_to_journal")
|
||||
expect("zusammenhängende Journalprosa" in notes_intern, "notes mode reaches the model")
|
||||
expect("Gute Formulierungen bewahren" not in notes_intern, "prose_edit instructions are not used for notes")
|
||||
|
||||
import_text(
|
||||
profile_id,
|
||||
"Ich schreibe in kurzen, ruhigen Sätzen und lasse den Tag stehen. Der Hafen blieb hinter der Fähre.",
|
||||
occurred_at="2026-07-01",
|
||||
)
|
||||
examples = select_journal_style_examples(profile_id, exclude_dates=["2026-08-26"])
|
||||
expect(examples and "ruhigen Sätzen" in examples[0]["excerpt"], "imported text is a style example when no finals exist")
|
||||
expect(all("zimlich" not in (item.get("excerpt") or "") for item in examples), "current day is not selected as style")
|
||||
|
||||
saved = client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={
|
||||
"journal_day_id": notes_day.json()["day"]["id"],
|
||||
"title": "Eigene Fassung",
|
||||
"body": "Heute blieb ich beim klaren Schnitt und schrieb den Markt in eigenen Worten, ohne Pathos.",
|
||||
"origin": "user_edit",
|
||||
"source_conversation_ids": [notes_conv.json()["id"]],
|
||||
},
|
||||
)
|
||||
expect(saved.status_code == 200, f"save {saved.text}")
|
||||
ranked = select_journal_style_examples(profile_id, exclude_dates=["2026-08-26"])
|
||||
expect(ranked and ranked[0]["kind"] == "journal_entry", "final journal text outranks imports")
|
||||
expect("klaren Schnitt" in ranked[0]["excerpt"], "user-edited journal text is the preferred example")
|
||||
expect(len(ranked) <= 2, "at most two style examples")
|
||||
|
||||
update_facet(profile_id, "core", value="Kurze Sätze, trockener Schnitt, keine Pathoswolken.")
|
||||
expect("Core:" not in compile_task_brief(profile_id), "unconfirmed core is not a style authority")
|
||||
set_lifecycle(profile_id, "confirmed")
|
||||
with_core = compile_task_brief(profile_id)
|
||||
expect("Core:" in with_core and "trockener Schnitt" in with_core, "confirmed core reaches the task brief")
|
||||
expect("Neutraler Journalstil" not in with_core, "confirmed profile replaces the neutral fallback")
|
||||
|
||||
remember_mapping(profile_id, "Hanna", "PERSON:99")
|
||||
|
||||
def rewritten(_messages, _policy):
|
||||
return ChatResult(
|
||||
content=(
|
||||
"Markttag\n\n"
|
||||
"Ich ging dann zum Markt, auf dem es ziemlich voll war. "
|
||||
"Vielleicht bleibe ich kürzer. "
|
||||
"Brot holen wollte ich noch, habe es aber nicht gemacht. "
|
||||
"Die rote Tasche lag im Auto."
|
||||
),
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", rewritten):
|
||||
second = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(second.status_code == 200, f"rewritten generate {second.text}")
|
||||
body = second.json().get("body") or ""
|
||||
expect("ziemlich voll" in body, "spelling is corrected")
|
||||
expect("zimlich" not in body, "source typo is not kept")
|
||||
expect("Ich ging dann zum Markt" in body, "good rewritten wording may stay")
|
||||
expect("Vielleicht bleibe ich kürzer" in body, "uncertainty stays uncertainty")
|
||||
expect("wollte" in body.lower() and "nicht gemacht" in body, "plan is not turned into completion")
|
||||
expect("nicht gemacht" in body, "negation is kept")
|
||||
expect("rote Tasche" in body, "unique attested detail is kept")
|
||||
expect(body.lower().count("markt") <= 2, "repetition is allowed to be reduced")
|
||||
expect("traurig" not in body.lower() and "weil" not in body.lower(), "no invented feeling or cause in the patched rewrite")
|
||||
intern2 = intern_of(second.json())
|
||||
expect("trockener Schnitt" in intern2.split("CURRENT_DAY_SOURCES")[0], "confirmed writing profile reaches generate")
|
||||
expect("Hafen blieb hinter der Fähre" not in body, "historical style facts are not copied into today")
|
||||
expect(
|
||||
sum(1 for item in (second.json().get("run_log") or []) if item.get("kind") == "model_call") == 1,
|
||||
"high similarity does not trigger a retry",
|
||||
)
|
||||
expect((second.json().get("trace") or {}).get("lexical_similarity") is not None, "similarity is a diagnostic only")
|
||||
|
||||
def notes_prose(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Notizen\n\nAm Markt holte ich Kirschen. Später war ich am Hafen.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", notes_prose):
|
||||
notes_out = client.post(
|
||||
f"/api/journal/days/{notes_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [notes_conv.json()["id"]]},
|
||||
)
|
||||
notes_body = notes_out.json().get("body") or ""
|
||||
expect("Am Markt holte ich Kirschen" in notes_body, "notes become connected prose")
|
||||
expect("kirschen\nspäter" not in notes_body.lower(), "notes are not concatenated as fragments")
|
||||
|
||||
def invent_title(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Hanna am Hafen\n\nIch ging zum Markt, auf dem es ziemlich voll war.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", invent_title):
|
||||
blocked = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(any(item.get("reason") == "unattested_identity" for item in (blocked.json().get("run_log") or [])), "title identity uses local fallback")
|
||||
expect("Hanna" not in (blocked.json().get("title") or "") and "Hanna" not in (blocked.json().get("body") or ""), "unattested title identity is not kept")
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE ai_prompts SET template = 'CUSTOM JOURNAL PROMPT' WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
)
|
||||
init_db()
|
||||
custom = load_active_prompt("mvp.journal_generate")
|
||||
expect(custom["template"] == "CUSTOM JOURNAL PROMPT", "independently edited prompt is not overwritten")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in (row["default_template"] or ""), "default template still tracks the seed")
|
||||
expect(row["seed_revision"] == SEED_REVISION, "revision updates even when template is custom")
|
||||
|
||||
print("journal editorial tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Evaluation harness contract. Does not prove live model quality."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-eval-test.sqlite")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from db import init_db
|
||||
from journal_eval import (
|
||||
SYNTHETIC_PROSE,
|
||||
SYNTHETIC_TYPOS,
|
||||
VARIANT_BASELINE,
|
||||
VARIANT_CURRENT,
|
||||
VARIANT_PREVIOUS,
|
||||
score_output,
|
||||
synthetic_context,
|
||||
variant_templates,
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_db()
|
||||
templates = variant_templates()
|
||||
expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist")
|
||||
expect("Überarbeite diesen Rohtext" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
|
||||
expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt")
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in templates[VARIANT_CURRENT], "current variant has the new contract")
|
||||
|
||||
context = synthetic_context(SYNTHETIC_PROSE)
|
||||
expect(context["editorial_mode"] in {"prose_edit", "notes_to_journal"}, "eval context has an editorial mode")
|
||||
copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS)
|
||||
expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1")
|
||||
expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos")
|
||||
improved = score_output(
|
||||
SYNTHETIC_PROSE,
|
||||
"Ich ging dann zum Markt, auf dem es ziemlich voll war. Vielleicht bleibe ich kürzer.",
|
||||
typos=SYNTHETIC_TYPOS,
|
||||
)
|
||||
expect("zimlich" in improved["spelling_typos_fixed"], "metrics record typo fixes")
|
||||
expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical")
|
||||
expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored")
|
||||
expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures")
|
||||
print("journal eval tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Journal narration contract: facts stay, wording may change."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-narration-test.sqlite")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from db import get_db, init_db
|
||||
from engine import load_active_prompt
|
||||
from identity_store import remember_mapping
|
||||
from journal_generate import unattested_journal_content
|
||||
from journal_shape import shape_journal
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
from providers import ChatResult
|
||||
from writing_profile_store import (
|
||||
NEUTRAL_JOURNAL_STYLE,
|
||||
compile_task_brief,
|
||||
get_profile,
|
||||
import_text,
|
||||
set_lifecycle,
|
||||
update_facet,
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def test_active_prompt_contract() -> None:
|
||||
init_db()
|
||||
prompt = load_active_prompt("mvp.journal_generate")
|
||||
text = prompt.get("template") or ""
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in text, "active prompt separates fact fidelity from wording")
|
||||
expect("CURRENT_DAY_SOURCES" in text, "active prompt labels current-day facts")
|
||||
expect("STYLE_EXAMPLES" in text, "active prompt labels style examples")
|
||||
expect("Rechtschreibung" in text or "korrigieren" in text, "active prompt allows spelling and grammar fixes")
|
||||
expect("Unsicherheit bleibt Unsicherheit" in text, "active prompt keeps semantic uncertainty")
|
||||
expect("keine neuen Informationen" in text.lower() or "Keine neuen Informationen" in text, "active prompt still forbids new facts")
|
||||
expect("[[…" not in text and "[[..." not in text, "active prompt must not teach ellipsis placeholders")
|
||||
expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone")
|
||||
expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect(row["seed_revision"] == "2026-08-27-journal-placeholders-v1", "system prompt revision is stored")
|
||||
expect(row["template"] == row["default_template"], "untouched install uses the seeded template")
|
||||
|
||||
|
||||
def test_shape_keeps_rewritten_model_text() -> None:
|
||||
source = ["ich bin dan zum markt gegangen und da war es zimlich voll"]
|
||||
model = "Ich ging dann zum Markt, auf dem es ziemlich voll war."
|
||||
_, body = shape_journal("Markttag", model, source)
|
||||
expect(body == model, "shape_journal keeps the corrected model text")
|
||||
expect("zimlich" not in body, "source typo is not restored")
|
||||
|
||||
|
||||
def test_shape_does_not_restore_source_form() -> None:
|
||||
source = ["ich ging zum markt. danach der hafen."]
|
||||
_, punct = shape_journal("Tag", "Ich ging zum Markt. Danach der Hafen.", source)
|
||||
expect(punct == "Ich ging zum Markt. Danach der Hafen.", "corrected punctuation is not reset")
|
||||
_, casing = shape_journal("Tag", "Am Markt war es voll.", ["am markt war es voll"])
|
||||
expect(casing == "Am Markt war es voll.", "corrected capitalization is not reset")
|
||||
rewritten = "Später am Hafen packte ich die rote Tasche."
|
||||
_, structure = shape_journal("Tag", rewritten, ["danach hafen. rote tasche eingepackt."])
|
||||
expect(structure == rewritten, "changed sentence structure is not reset")
|
||||
fallback_parts = ["Zeile eins Markt.", "Zeile zwei Hafen und rote Tasche."]
|
||||
_, fallback = shape_journal("Ein Tag", "\n\n".join(fallback_parts), fallback_parts, source="fallback")
|
||||
expect("Markt" in fallback and "Hafen" in fallback and "rote Tasche" in fallback, "fallback keeps every unique user source")
|
||||
expect(fallback == "\n\n".join(fallback_parts) or "Markt" in fallback, "fallback path stays source-based")
|
||||
model_kept = "Ich war am Markt und später am Hafen."
|
||||
_, model = shape_journal("Tag", model_kept, fallback_parts, source="model")
|
||||
expect(model == model_kept, "model path is not mixed with the fallback join")
|
||||
|
||||
|
||||
def test_unattested_identity_is_journal_not_privacy() -> None:
|
||||
mappings = [
|
||||
{"local_label": "Anna", "token": "PERSON:01"},
|
||||
{"local_label": "Clarissa", "token": "PERSON:99"},
|
||||
]
|
||||
sources = ["Heute war ich mit Anna am Markt."]
|
||||
expect(
|
||||
unattested_journal_content("Ich ging mit Anna zum Markt.", sources, mappings, ["PERSON:01"]) is None,
|
||||
"attested person in rewritten prose is allowed",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Clarissa kaufte Kirschen am Markt.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_identity",
|
||||
"invented person as subject of a verb is still unattested",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("[[PERSON:99]] stand am Markt.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_placeholder",
|
||||
"inactive placeholder is unattested content",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Heute nur von Sushi essen erzählt.", sources, [{"local_label": "Sushi", "token": "PERSON:01"}], [])
|
||||
is None,
|
||||
"food homonym of an unused mapping is not unattested identity",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_active_prompt_contract()
|
||||
test_shape_keeps_rewritten_model_text()
|
||||
test_shape_does_not_restore_source_form()
|
||||
test_unattested_identity_is_journal_not_privacy()
|
||||
expect(NEUTRAL_JOURNAL_STYLE.startswith("Neutraler Journalstil"), "neutral fallback exists")
|
||||
|
||||
reset_debug()
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
empty_brief = compile_task_brief(profile_id, "journal_generate")
|
||||
expect(empty_brief == NEUTRAL_JOURNAL_STYLE, "empty profile uses the neutral journal voice")
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Narration"})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-26"},
|
||||
)
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Tag"},
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "ich bin dan zum markt gegangen und da war es zimlich voll. vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. die rote tasche lag im auto. ich bin dan zum markt gegangen."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn {turn.status_code}")
|
||||
before = get_profile(profile_id)
|
||||
gen = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"generate {gen.text}")
|
||||
intern = ""
|
||||
for stage in (gen.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
intern = stage.get("intern") or ""
|
||||
style = intern.split("CURRENT_DAY_SOURCES")[0]
|
||||
expect("Neutraler Journalstil" in style, "neutral style reaches the generate prompt")
|
||||
expect("Erzählmerkmale" not in style, "current day dialogue is not a style brief")
|
||||
expect("zimlich" not in style, "today's typo is not a style exemplar")
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in intern, "runtime intern uses the new narration contract")
|
||||
expect("CURRENT_DAY_SOURCES" in intern, "runtime intern labels current-day facts")
|
||||
expect("Nur den verifizierten Nutzerwortlaut" not in intern, "old output-wording rule is not in the runtime prompt")
|
||||
expect("Unsicherheiten im Wortlaut" not in intern, "old uncertainty-in-wording rule is not in the runtime prompt")
|
||||
after = get_profile(profile_id)
|
||||
expect(after.get("version") == before.get("version"), "generate does not create a new profile version")
|
||||
expect((after.get("suggestions") or []) == (before.get("suggestions") or []), "generate does not add suggestions")
|
||||
|
||||
import_text(profile_id, "Ich schreibe in kurzen, ruhigen Sätzen und lasse den Tag stehen.")
|
||||
from journal_editorial import select_journal_style_examples
|
||||
|
||||
imported_examples = select_journal_style_examples(profile_id)
|
||||
expect(
|
||||
imported_examples and "kurzen, ruhigen Sätzen" in imported_examples[0]["excerpt"],
|
||||
"imported own text is a style source",
|
||||
)
|
||||
imported_brief = compile_task_brief(profile_id, "journal_generate")
|
||||
expect(imported_brief == NEUTRAL_JOURNAL_STYLE, "unconfirmed profile does not become the brief")
|
||||
expect("zimlich" not in " ".join(item["excerpt"] for item in imported_examples), "today's dialogue is not mixed into the imported style")
|
||||
|
||||
saved = client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={
|
||||
"journal_day_id": day.json()["day"]["id"],
|
||||
"title": "Eigene Fassung",
|
||||
"body": "Heute blieb ich beim klaren Schnitt und schrieb den Markt in eigenen Worten.",
|
||||
"origin": "user_edit",
|
||||
"source_conversation_ids": [conv.json()["id"]],
|
||||
},
|
||||
)
|
||||
expect(saved.status_code == 200, f"save {saved.text}")
|
||||
entry_examples = select_journal_style_examples(profile_id)
|
||||
expect(
|
||||
any("klaren Schnitt" in (item.get("excerpt") or "") for item in entry_examples),
|
||||
"final user-edited journal text is a style source",
|
||||
)
|
||||
expect(all("zimlich" not in (item.get("excerpt") or "") for item in entry_examples), "current day dialogue is not a positive style reference")
|
||||
|
||||
update_facet(profile_id, "core", value="Kurze Sätze, trockener Schnitt, keine Pathoswolken.")
|
||||
set_lifecycle(profile_id, "confirmed")
|
||||
with_core = compile_task_brief(profile_id, "journal_generate")
|
||||
expect("Core:" in with_core and "trockener Schnitt" in with_core, "confirmed core reaches the task brief")
|
||||
expect("Neutraler Journalstil" not in with_core, "confirmed profile replaces the neutral fallback")
|
||||
|
||||
remember_mapping(profile_id, "Clarissa", "PERSON:99")
|
||||
|
||||
def rewritten(_messages, _policy):
|
||||
return ChatResult(
|
||||
content=(
|
||||
"Markttag\n\n"
|
||||
"Ich ging dann zum Markt, auf dem es ziemlich voll war. "
|
||||
"Vielleicht bleibe ich kürzer. "
|
||||
"Brot holen wollte ich noch, habe es aber nicht gemacht. "
|
||||
"Die rote Tasche lag im Auto."
|
||||
),
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", rewritten):
|
||||
second = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(second.status_code == 200, f"rewritten generate {second.text}")
|
||||
body = second.json().get("body") or ""
|
||||
expect("ziemlich voll" in body, "corrected spelling of the model text is kept")
|
||||
expect("zimlich" not in body, "source typo is not restored after a real model rewrite")
|
||||
expect("Vielleicht bleibe ich kürzer" in body, "semantic uncertainty is kept")
|
||||
expect("wollte" in body.lower() and "nicht gemacht" in body, "plan versus non-fulfillment stays")
|
||||
expect("rote Tasche" in body, "unique attested detail is kept after rewrite")
|
||||
expect(body.lower().count("markt") <= 2, "repeated source lines are not pasted twice")
|
||||
expect(
|
||||
any(item.get("status") == "model" for item in (second.json().get("run_log") or [])),
|
||||
"accepted model path is recorded as model",
|
||||
)
|
||||
intern2 = ""
|
||||
for stage in (second.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
intern2 = stage.get("intern") or ""
|
||||
expect("trockener Schnitt" in intern2.split("CURRENT_DAY_SOURCES")[0], "confirmed writing profile reaches generate")
|
||||
|
||||
def invent(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Clarissa blieb den ganzen Nachmittag am Hafen.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", invent):
|
||||
blocked = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(blocked.status_code == 200, f"unattested generate {blocked.text}")
|
||||
log = blocked.json().get("run_log") or []
|
||||
expect(any(item.get("reason") == "unattested_identity" for item in log), "unattested person uses local fallback")
|
||||
expect(any(item.get("status") == "local_fallback" for item in log), "fallback path is distinct from the model path")
|
||||
expect("Clarissa" not in (blocked.json().get("body") or ""), "unattested person is not in the accepted draft")
|
||||
expect("markt" in (blocked.json().get("body") or "").lower(), "fallback still has the user sources")
|
||||
expect("rote tasche" in (blocked.json().get("body") or "").lower(), "fallback keeps the unique source detail")
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE ai_prompts SET template = 'CUSTOM JOURNAL PROMPT' WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
)
|
||||
init_db()
|
||||
custom = load_active_prompt("mvp.journal_generate")
|
||||
expect(custom["template"] == "CUSTOM JOURNAL PROMPT", "independently edited prompt is not overwritten")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in (row["default_template"] or ""), "default template still tracks the seed")
|
||||
expect(row["seed_revision"] == "2026-08-27-journal-placeholders-v1", "revision updates even when template is custom")
|
||||
|
||||
print("journal narration tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -206,17 +206,17 @@ def main() -> None:
|
||||
expect(draft.status_code == 200, f"generate after opening {draft.text}")
|
||||
reconstruct = None
|
||||
for stage in (draft.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_reconstruct":
|
||||
if stage.get("purpose") == "local_source_artifact":
|
||||
reconstruct = stage
|
||||
intern = (reconstruct or {}).get("intern") or ""
|
||||
expect("Anna kaufte Brot" in intern, "user source reaches reconstruction")
|
||||
expect("[u1]" in intern, "user source id assigned")
|
||||
expect("Anna kaufte Brot" in intern, "user source reaches local artifact")
|
||||
expect("u1" in intern, "user source id assigned")
|
||||
narrate = None
|
||||
for stage in (draft.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
narrate = stage
|
||||
narrate_intern = (narrate or {}).get("intern") or ""
|
||||
expect("verified_artifact" in narrate_intern or "sources" in narrate_intern, "stage 2 consumes verified artifact")
|
||||
expect("CURRENT_DAY_SOURCES" in narrate_intern or "Anna kaufte Brot" in narrate_intern, "stage 2 consumes day sources")
|
||||
expect("Nach der Fähre kamen wir spät an" in intern, "user wording not dropped after opening")
|
||||
expect(users[0]["body"].startswith("Nach der Fähre"), "user source is the user message, not the opening")
|
||||
|
||||
|
||||
@@ -28,8 +28,24 @@ def main() -> None:
|
||||
|
||||
title, body = shape_journal("Markttag", joined, turns)
|
||||
expect(title == "Markttag", "title stays")
|
||||
expect(body.count("\n\n") == 2, "paste becomes one paragraph per turn")
|
||||
expect("Kirschen" in body and "Hafen" in body, "wording of a paste is kept")
|
||||
expect("Kirschen" in body and "Hafen" in body, "model wording is kept")
|
||||
expect(body != "\n\n".join(turns), "accepted model text is not replaced by source turns")
|
||||
|
||||
source_typo = ["ich bin dan zum markt gegangen und da war es zimlich voll"]
|
||||
rewritten = "Ich ging dann zum Markt, auf dem es ziemlich voll war."
|
||||
_, kept_rewrite = shape_journal("Markttag", rewritten, source_typo)
|
||||
expect("ziemlich" in kept_rewrite, "corrected spelling stays")
|
||||
expect("zimlich" not in kept_rewrite, "source typo is not restored")
|
||||
expect(kept_rewrite.startswith("Ich ging"), "corrected sentence structure stays")
|
||||
expect("Markt" in kept_rewrite, "capitalization stays corrected")
|
||||
|
||||
_, punct = shape_journal("Tag", "Ich ging zum Markt.", ["ich ging zum markt"])
|
||||
expect(punct == "Ich ging zum Markt.", "corrected punctuation and casing are not reset")
|
||||
|
||||
fallback_body = "\n\n".join(turns)
|
||||
_, fallback = shape_journal("Ein Tag", fallback_body, turns, source="fallback")
|
||||
expect("Kirschen" in fallback and "Hafen" in fallback, "fallback still contains every user source")
|
||||
expect("Markt" in fallback, "fallback keeps first source")
|
||||
|
||||
blob = (
|
||||
"Heute war der Markt voll. Ich bin früh gegangen. "
|
||||
|
||||
@@ -151,8 +151,9 @@ def main() -> None:
|
||||
|
||||
impulse, decision = parse_turn_payload('{"operation":"konkretisieren","impulse":"Wo war das?"}')
|
||||
expect(impulse == "Wo war das?" and decision["parsed"] is True, "operation json parsed")
|
||||
hidden = visible_for_role({"trace": {"egress": "x"}, "decision": decision, "calls": 1}, "user")
|
||||
hidden = visible_for_role({"trace": {"egress": "x"}, "run_log": [{"kind": "detect"}], "decision": decision, "calls": 1}, "user")
|
||||
expect("trace" not in hidden and "decision" not in hidden, "user does not see test spur")
|
||||
expect(hidden.get("run_log"), "compact generate log remains visible")
|
||||
expect(is_plot_continuation("Und dann mit dem Tee auf dem Balkon?"), "echo then-question rejected")
|
||||
expect(is_plot_continuation("Ihr seid dann zur Bootstour aufgebrochen."), "invented departure rejected")
|
||||
expect(is_plot_continuation("Und dann seid ihr zur Wanderung aufgebrochen."), "invented hike rejected")
|
||||
@@ -313,20 +314,25 @@ def main() -> None:
|
||||
expect(draft.status_code == 200, f"A generate {draft.text}")
|
||||
expect("Markt" in draft.json()["body"] or "Kirschen" in draft.json()["body"], "A draft keeps user words")
|
||||
stages_a = {item.get("purpose"): item for item in (draft.json().get("trace") or {}).get("stages") or []}
|
||||
reconstruct_a = stages_a.get("journal_reconstruct")
|
||||
reconstruct_a = stages_a.get("local_source_artifact")
|
||||
last_a = stages_a.get("journal_generate") or (draft.json().get("trace") or {})
|
||||
expect(reconstruct_a is not None, "A stage 1 reconstruct ran")
|
||||
expect(reconstruct_a is not None, "A stage 1 local artifact ran")
|
||||
reconstruct_intern = reconstruct_a.get("intern") or ""
|
||||
narrate_intern = last_a.get("intern") or ""
|
||||
expect("Kirschen" in reconstruct_intern or "Markt" in reconstruct_intern, "A reconstruct sees user words")
|
||||
expect("user:" in reconstruct_intern, "A reconstruct sees the day as dialogue")
|
||||
expect("Kirschen" in reconstruct_intern or "Markt" in reconstruct_intern, "A local artifact sees user words")
|
||||
expect("assistant:" not in reconstruct_intern, "A local artifact has no assistant lines")
|
||||
expect("assistant:" not in narrate_intern, "A stage 2 does not resend the day dialogue")
|
||||
expect("verified_artifact" in narrate_intern or "sources" in narrate_intern, "A stage 2 receives the reconstruction")
|
||||
expect("Stilquellen" in narrate_intern or "Dialog" in narrate_intern or "Erzählmerkmale" in narrate_intern or "Core:" in narrate_intern, "A style brief reaches generate")
|
||||
style_part = narrate_intern
|
||||
expect("Erzählmerkmale" in style_part, "A first generate still gets form hints")
|
||||
expect("CURRENT_DAY_SOURCES" in narrate_intern or "[u1]" in narrate_intern, "A stage 2 receives the day sources")
|
||||
expect(
|
||||
"Heute war der Markt voll" not in style_part.split("Verified Artifact:")[0],
|
||||
"WRITING_PROFILE" in narrate_intern or "Neutraler Journalstil" in narrate_intern or "Core:" in narrate_intern,
|
||||
"A style brief reaches generate",
|
||||
)
|
||||
expect("STYLE_EXAMPLES" in narrate_intern, "A style examples block is labeled")
|
||||
style_part = narrate_intern.split("CURRENT_DAY_SOURCES")[0]
|
||||
expect("Erzählmerkmale" not in style_part, "A current day dialogue is not appended as style signals")
|
||||
expect("Neutraler Journalstil" in style_part or "Core:" in style_part, "A empty profile uses the neutral journal voice")
|
||||
expect(
|
||||
"Heute war der Markt voll" not in style_part,
|
||||
"A source dialogue is not pasted a second time as style",
|
||||
)
|
||||
current_version = None
|
||||
@@ -364,7 +370,7 @@ def main() -> None:
|
||||
expect(day_again.json()["day"]["calendar_date"] == "2026-08-20", "B journal day stays calendar day")
|
||||
msgs = client.get(f"/api/journal/conversations/{conv['id']}", headers=headers).json()["messages"]
|
||||
expect(any("Wind" in item["body"] for item in msgs), "B later message kept")
|
||||
expect(msgs[-1]["created"] != "2026-08-20" or True, "B message time is independent")
|
||||
expect(not str(msgs[-1]["created"]).startswith("2026-08-20"), "B message time is independent of the calendar day")
|
||||
|
||||
prior_space, prior_day = open_space_day(client, headers, "Urlaub", "2026-08-01")
|
||||
prior_conv, _ = start_and_turn(
|
||||
@@ -510,7 +516,9 @@ def main() -> None:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
intern_i = stage.get("intern") or ""
|
||||
intern_i = intern_i or ((gen_i.json().get("trace") or {}).get("intern") or "")
|
||||
expect("knapp" in intern_i, "I style brief reaches generate")
|
||||
expect("STYLE_EXAMPLES" in intern_i, "I style examples reach generate")
|
||||
expect("WRITING_PROFILE" in intern_i or "Neutraler Journalstil" in intern_i, "I writing profile reaches generate")
|
||||
expect("knapp" in imported.json()["compiled_brief"], "I stored brief still lists the import")
|
||||
profile = client.get("/api/journal/writing-profile", headers=headers).json()
|
||||
expect(all(item.get("kind") != "journal_draft" for item in profile["sources"]), "I no draft kind")
|
||||
expect(not any((item.get("body") or "") == gen_i.json()["body"] for item in profile["sources"]), "I draft not a style source")
|
||||
@@ -586,7 +594,7 @@ def main() -> None:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
with_intern = stage.get("intern") or ""
|
||||
expect(marker in with_intern, "include_existing true sends the saved body only after explicit flag")
|
||||
existing_section = with_intern.split("Bestehender Text")[-1].split("Verified Artifact")[0]
|
||||
existing_section = with_intern.split("EXISTING_TEXT")[-1]
|
||||
expect(marker in existing_section, "saved Fassung is the included existing text, not a leftover draft")
|
||||
after_with = client.get(f"/api/journal/entries/{saved.json()['id']}", headers=headers).json()
|
||||
expect(after_with["current_version_id"] == marked_version, "include_existing generate still does not overwrite")
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
"""Detection + mask + demask. Run from backend/: python tests/test_privacy_detect.py"""
|
||||
"""Semantic detection contracts. Fake provider proves schema and fail-closed, not live quality.
|
||||
|
||||
Run from backend/: python tests/test_privacy_detect.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
@@ -15,9 +22,38 @@ os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from identity_store import list_mappings
|
||||
from identity_store import (
|
||||
STATUS_LEGACY,
|
||||
confirm_identity,
|
||||
list_confirmed_identities,
|
||||
list_mappings,
|
||||
list_registry,
|
||||
list_review_proposals,
|
||||
remember_mapping,
|
||||
)
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
from privacy_gateway import (
|
||||
GatewayRequest,
|
||||
PrivacyGatewayError,
|
||||
_demask,
|
||||
_validate_response,
|
||||
complete,
|
||||
mask_prompt,
|
||||
reset_debug,
|
||||
)
|
||||
from entity_detect import (
|
||||
DETECT_CHUNK_CHARS,
|
||||
DetectError,
|
||||
detect_personal_egress,
|
||||
install_test_detect_failure,
|
||||
install_test_spans,
|
||||
install_test_truncated,
|
||||
reset_detect_test_hooks,
|
||||
split_detect_chunks,
|
||||
uses_llm_detect,
|
||||
validate_detected_entity,
|
||||
)
|
||||
from providers import ProviderConfig
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
@@ -26,6 +62,18 @@ def expect(ok: bool, message: str) -> None:
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def _run(profile_id: str, rendered: str, purpose: str = "dialogue_turn"):
|
||||
return complete(
|
||||
GatewayRequest(
|
||||
prompt_id="detect-test",
|
||||
purpose=purpose,
|
||||
data_class="B",
|
||||
profile_id=profile_id,
|
||||
payload={"rendered": rendered, "source_text": rendered},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
with TestClient(app) as client:
|
||||
@@ -34,6 +82,7 @@ def main() -> None:
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
headers = {"X-Auth-Token": setup.json()["token"]}
|
||||
profile_id = setup.json()["profile_id"]
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
@@ -45,111 +94,315 @@ def main() -> None:
|
||||
headers=headers,
|
||||
json={"title": "Gespräch"},
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Ich war mit Anna in Hamburg und danach allein."},
|
||||
|
||||
balcony = _run(profile_id, "Ich ging auf den Balkon und setzte mich.")
|
||||
expect(balcony.allowed, "balcony sentence is allowed")
|
||||
expect("Balkon" in (balcony.trace.get("egress") or ""), "capitalized common noun stays unmasked")
|
||||
expect("[[PERSON:" not in (balcony.trace.get("egress") or ""), "balcony is not a person placeholder")
|
||||
|
||||
typo = _run(profile_id, "Ich ging auf den Bakon und setzte mich.")
|
||||
expect("Bakon" in (typo.trace.get("egress") or ""), "misspelled common noun is not stored as a person")
|
||||
expect(
|
||||
not any((item.get("canonical_label") or "") == "Bakon" for item in list_confirmed_identities(profile_id)),
|
||||
"misspelled noun is not a confirmed identity",
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn {turn.text}")
|
||||
trace = turn.json().get("trace") or {}
|
||||
expect(trace, "gateway returned request-scoped trace")
|
||||
expect(trace["detect_provider"] == "fake-detect", "detect provider is separate")
|
||||
expect("Anna" not in (trace.get("egress") or ""), "Anna masked before generate")
|
||||
expect("Hamburg" not in (trace.get("egress") or ""), "Hamburg masked before generate")
|
||||
expect("[[PERSON:" in (trace.get("egress") or "") or "[[PLACE:" in (trace.get("egress") or ""), "placeholders in egress")
|
||||
expect("Anna" in turn.json()["user"]["body"], "source keeps cleartext")
|
||||
mappings = list_mappings(setup.json()["profile_id"])
|
||||
labels = {item["local_label"] for item in mappings}
|
||||
expect("Anna" in labels, "Anna stored in local dictionary")
|
||||
expect("Hamburg" in labels, "Hamburg stored in local dictionary")
|
||||
expect(all(not (item.get("token") or "").startswith("[[") or True for item in mappings), "tokens stored")
|
||||
|
||||
reset_debug()
|
||||
again = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Anna war ruhig."},
|
||||
food = _run(profile_id, "Ich aß Sushi auf dem Teller.")
|
||||
expect("Sushi" in (food.trace.get("egress") or ""), "food context keeps the word")
|
||||
person_sushi = _run(profile_id, "Sushi kam ins Wohnzimmer.")
|
||||
expect("[[PERSON:" in (person_sushi.trace.get("egress") or ""), "same word can be a person in another sentence")
|
||||
expect("Sushi" not in (person_sushi.trace.get("egress") or "").replace("[[PERSON:", ""), "person Sushi is masked")
|
||||
|
||||
mixed = _run(
|
||||
profile_id,
|
||||
"Ich traf Anna. Ich arbeitete am privaten Projekt Aurora. "
|
||||
"Die Organisation Nordwerk schickte Unterlagen. Ich war in Hamburg.",
|
||||
)
|
||||
expect(again.status_code == 200, "second turn")
|
||||
last2 = again.json().get("trace") or {}
|
||||
egress = mixed.trace.get("egress") or ""
|
||||
expect("[[PERSON:" in egress, "person is typed as PERSON")
|
||||
expect("[[PROJECT:" in egress, "private project is typed as PROJECT")
|
||||
expect("[[ORG:" in egress, "organisation is typed as ORG")
|
||||
expect("[[PLACE:" in egress, "named city is typed as PLACE")
|
||||
|
||||
expect("nicht" in (last2.get("egress") or "").lower() or "Anna" not in (last2.get("egress") or ""), "dictionary remasks Anna without new invention")
|
||||
expect(last2.get("mapping_count", 0) >= 2, "dictionary reused")
|
||||
sample = "Ich traf Anna."
|
||||
start = sample.index("Anna")
|
||||
span = validate_detected_entity(
|
||||
{"start": start, "end": start + 4, "text": "Anna", "entity_type": "PERSON"},
|
||||
sample,
|
||||
0,
|
||||
)
|
||||
expect(span.text == "Anna" and span.start == start, "valid span matches the local substring")
|
||||
relocated = validate_detected_entity(
|
||||
{"start": 0, "end": 4, "text": "Anna", "entity_type": "PERSON"},
|
||||
sample,
|
||||
0,
|
||||
)
|
||||
expect(relocated is not None and relocated.start == start and relocated.text == "Anna", "wrong offsets are grounded to the local word")
|
||||
expect(
|
||||
validate_detected_entity(
|
||||
{"start": -1, "end": 4, "text": "Anna", "entity_type": "PERSON"},
|
||||
sample,
|
||||
0,
|
||||
).start
|
||||
== start,
|
||||
"negative offsets still ground when the word exists",
|
||||
)
|
||||
expect(
|
||||
validate_detected_entity(
|
||||
{"start": 0, "end": 4, "text": "Nein", "entity_type": "PERSON"},
|
||||
sample,
|
||||
0,
|
||||
)
|
||||
is None,
|
||||
"a word that is not in the chunk is skipped",
|
||||
)
|
||||
expect(
|
||||
validate_detected_entity(
|
||||
{"start": "x", "end": 4, "text": "Anna", "entity_type": "PERSON"},
|
||||
sample,
|
||||
0,
|
||||
).start
|
||||
== start,
|
||||
"non-numeric offsets still ground when the word exists",
|
||||
)
|
||||
rejected = []
|
||||
for payload, label in (
|
||||
({"start": start, "end": start + 4, "text": "Anna", "entity_type": "PET"}, "unknown type"),
|
||||
({"start": start, "end": start + 4, "text": "Anna", "entity_type": "PERSON", "token": "PERSON:01"}, "detector token"),
|
||||
({"start": start, "end": start + 4, "text": "Anna", "entity_type": "PERSON", "score": 1}, "extra field"),
|
||||
):
|
||||
try:
|
||||
validate_detected_entity(payload, sample, 0)
|
||||
except DetectError:
|
||||
rejected.append(label)
|
||||
expect(rejected == ["unknown type", "detector token", "extra field"], "invalid detector output is rejected")
|
||||
|
||||
from entity_detect import _pattern_entities
|
||||
from identity_store import is_maskable_label
|
||||
from privacy_gateway import _mask
|
||||
first = _run(profile_id, "Ich war mit Anna in Hamburg und danach allein.")
|
||||
expect(first.trace.get("detect_provider") == "fake-detect", "detect provider is separate")
|
||||
expect("Anna" not in (first.trace.get("egress") or ""), "Anna masked before generate")
|
||||
expect("Hamburg" not in (first.trace.get("egress") or ""), "Hamburg masked before generate")
|
||||
expect(first.trace.get("full_detection_coverage") is True, "successful detect records full coverage")
|
||||
expect(first.trace.get("generate_called") is True, "generate runs after complete detection")
|
||||
expect(first.diagnostics.get("detect_calls") == 1, "normal short text uses one detect call")
|
||||
labels_after = {item["canonical_label"] for item in list_confirmed_identities(profile_id)}
|
||||
expect("Anna" not in labels_after, "detect hit is not auto-confirmed")
|
||||
expect("Hamburg" not in labels_after, "place detect hit is not auto-confirmed")
|
||||
proposals = [item["observed_label"] for item in list_review_proposals(profile_id)]
|
||||
expect("Anna" in proposals, "unconfirmed proposal may be stored for review")
|
||||
|
||||
patterned = {item["text"] for item in _pattern_entities("Ich war mit Anna in Hamburg.", set())}
|
||||
expect("Anna" in patterned and "Hamburg" in patterned, "pattern finds proper names")
|
||||
expect("Himmel" not in {item["text"] for item in _pattern_entities("Der Himmel war klar.", set())}, "pattern skips weather")
|
||||
second = _run(profile_id, "Anna war ruhig.")
|
||||
expect("Anna" not in (second.trace.get("egress") or ""), "second request still runs full detection")
|
||||
expect(second.trace.get("request_local_hits", 0) >= 1, "known labels are not excluded from the new detection")
|
||||
|
||||
poisoned = remember_mapping(profile_id, "Wohnzimmer", "PERSON:77")
|
||||
expect(poisoned["status"] == "confirmed", "remember_mapping is explicit confirmation")
|
||||
update = client.patch(
|
||||
f"/api/admin/identities/{poisoned['id']}",
|
||||
headers=headers,
|
||||
json={"status": "inactive"},
|
||||
)
|
||||
expect(update.status_code == 200, "admin can deactivate a false mapping")
|
||||
expect(update.json()["status"] == "inactive", "deactivated mapping is inactive")
|
||||
later = _run(profile_id, "Ich saß im Wohnzimmer.")
|
||||
expect("Wohnzimmer" in (later.trace.get("egress") or ""), "inactive mapping does not keep masking a room")
|
||||
expect(
|
||||
not any(item.get("canonical_label") == "Wohnzimmer" and item.get("status") == "confirmed" for item in list_registry(profile_id)),
|
||||
"later request is not bound to the deactivated mapping",
|
||||
)
|
||||
|
||||
confirmed_project = confirm_identity(profile_id, "Kanshō", entity_type="PROJECT", aliases=["Kansho"])
|
||||
canonical = _run(profile_id, "Ich arbeitete an Kansho weiter.")
|
||||
expect("[[PROJECT:" in (canonical.trace.get("egress") or ""), "confirmed alias is masked")
|
||||
expect("Kansho" not in (canonical.trace.get("egress") or ""), "confirmed alias does not leak")
|
||||
project_manifest = mask_prompt(
|
||||
"Ich arbeitete an Kansho weiter.",
|
||||
[
|
||||
{
|
||||
"local_label": "Kansho",
|
||||
"token": confirmed_project["token"],
|
||||
"entity_type": "PROJECT",
|
||||
"demask_label": "Kanshō",
|
||||
}
|
||||
],
|
||||
"journal_generate",
|
||||
)
|
||||
expect(
|
||||
_demask(f"[[{confirmed_project['token']}]] weiter", project_manifest) == "Kanshō weiter",
|
||||
"confirmed alias demasks to the canonical spelling",
|
||||
)
|
||||
|
||||
observed_manifest = mask_prompt(
|
||||
"Ich traf Anna am Abend.",
|
||||
[{"local_label": "Anna", "token": "PERSON:01", "demask_label": "Anna"}],
|
||||
"journal_generate",
|
||||
)
|
||||
expect(_demask("[[PERSON:01]] am Abend.", observed_manifest) == "Anna am Abend.", "request-local hit demasks to the observed spelling")
|
||||
|
||||
leftover = _demask("[[PERSON:01]] und [[PERSON:99]]", mask_prompt("Anna war da.", [{"local_label": "Anna", "token": "PERSON:01", "demask_label": "Anna"}], "journal_generate"))
|
||||
expect("[[PERSON:99]]" in leftover, "unknown placeholder is not materialized")
|
||||
|
||||
long_text = ("Anfang. " + ("x" * 1200) + " Anna in der Mitte. " + ("y" * 1200) + " Ende.")
|
||||
with patch("entity_detect.DETECT_CHUNK_CHARS", 400), patch("entity_detect.DETECT_CHUNK_OVERLAP", 80):
|
||||
chunks = split_detect_chunks(long_text, chunk_chars=400, overlap=80)
|
||||
expect(len(chunks) >= 3, "long egress is chunked")
|
||||
covered = 0
|
||||
prev_end = 0
|
||||
for offset, chunk in chunks:
|
||||
expect(offset <= prev_end, "chunks overlap or abut")
|
||||
covered = max(covered, offset + len(chunk))
|
||||
prev_end = offset + len(chunk)
|
||||
expect(covered == len(long_text), "chunk union covers the full text")
|
||||
boundary = _run(profile_id, long_text)
|
||||
expect(boundary.trace.get("chunk_count") >= 2, "gateway chunks a long prompt")
|
||||
expect(boundary.trace.get("chunks_ok") == boundary.trace.get("chunk_count"), "every chunk is marked ok")
|
||||
expect("Anna" not in (boundary.trace.get("egress") or ""), "entity at a chunk boundary is still masked")
|
||||
expect(boundary.trace.get("full_detection_coverage") is True, "coverage is true only after every chunk")
|
||||
|
||||
generate_calls = {"n": 0}
|
||||
|
||||
def boom(_messages, _policy):
|
||||
generate_calls["n"] += 1
|
||||
raise AssertionError("generate must not run")
|
||||
|
||||
install_test_detect_failure(DetectError("detect_chunk_failed", "chunk failed"))
|
||||
blocked = False
|
||||
with patch("privacy_gateway.complete_model", boom):
|
||||
try:
|
||||
_run(profile_id, "Ich traf Anna.")
|
||||
except PrivacyGatewayError as exc:
|
||||
blocked = exc.code == "detect_chunk_failed"
|
||||
expect(exc.diagnostics.get("generate_called") is False, "failed chunk does not call generate")
|
||||
expect(exc.diagnostics.get("full_detection_coverage") is False, "failed chunk is not full coverage")
|
||||
expect("Substring" not in (exc.message or ""), "detect errors omit the internal substring wording")
|
||||
expect("Angaben" in (exc.message or ""), "detect errors use a user-facing message")
|
||||
reset_detect_test_hooks()
|
||||
expect(blocked, "failed chunk fails closed")
|
||||
expect(generate_calls["n"] == 0, "generate is not called when a chunk fails")
|
||||
|
||||
install_test_truncated(True)
|
||||
truncated = False
|
||||
generate_calls["n"] = 0
|
||||
with patch("privacy_gateway.complete_model", boom):
|
||||
try:
|
||||
_run(profile_id, "Ich traf Anna.")
|
||||
except PrivacyGatewayError as exc:
|
||||
truncated = exc.code == "detect_truncated"
|
||||
expect(exc.diagnostics.get("generate_called") is False, "truncated detect does not call generate")
|
||||
reset_detect_test_hooks()
|
||||
expect(truncated, "truncated detect output fails closed")
|
||||
|
||||
start = "Ich traf Anna.".index("Anna")
|
||||
install_test_spans([{"start": start, "end": start + 4, "text": "Anna", "entity_type": "PERSON", "token": "PERSON:09"}])
|
||||
extra = False
|
||||
with patch("privacy_gateway.complete_model", boom):
|
||||
try:
|
||||
_run(profile_id, "Ich traf Anna.")
|
||||
except PrivacyGatewayError as exc:
|
||||
extra = exc.code == "detect_invalid_output"
|
||||
reset_detect_test_hooks()
|
||||
expect(extra, "detector-assigned tokens fail closed")
|
||||
|
||||
install_test_spans([{"start": 0, "end": 4, "text": "Anna", "entity_type": "PERSON"}])
|
||||
try:
|
||||
grounded_run = _run(profile_id, "Ich traf Anna am Abend.")
|
||||
expect("Anna" not in (grounded_run.trace.get("egress") or ""), "grounded name is still masked")
|
||||
expect(grounded_run.trace.get("generate_called") is True, "wrong offsets do not block generate")
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
|
||||
results = []
|
||||
|
||||
def worker(label: str):
|
||||
reset_debug()
|
||||
out = _run(profile_id, f"Ich traf {label}.")
|
||||
results.append((label, out.trace.get("active_tokens"), threading.get_ident()))
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
one = pool.submit(worker, "Anna")
|
||||
two = pool.submit(worker, "Maren")
|
||||
one.result()
|
||||
two.result()
|
||||
labels = {item[0] for item in results}
|
||||
expect(labels == {"Anna", "Maren"}, "parallel requests keep their own manifests")
|
||||
|
||||
expect(not is_maskable_label("ich"), "pronoun ich is not a mapping")
|
||||
expect(not is_maskable_label("user"), "role user is not a mapping")
|
||||
expect(not is_maskable_label("Himmel"), "weather is not identity")
|
||||
expect(not is_maskable_label("Der Himmel"), "determiner plus weather is not identity")
|
||||
expect(not is_maskable_label("6:00 Uhr"), "clock time is not identity")
|
||||
expect(not is_maskable_label("meinen Tee"), "tea is not identity")
|
||||
expect(not is_maskable_label("Relativ"), "sentence adverb is not identity")
|
||||
expect(not is_maskable_label("Allerdings"), "conjunction is not identity")
|
||||
expect(not is_maskable_label("Tages"), "common noun is not identity")
|
||||
expect(not is_maskable_label("Brote"), "food is not identity")
|
||||
expect(not is_maskable_label("Überlegen"), "common noun is not identity")
|
||||
expect(not is_maskable_label("Ortes"), "place word is not identity")
|
||||
expect(not is_maskable_label("Zwiebeln"), "food is not identity")
|
||||
expect(not is_maskable_label("letzten"), "function word letzten is not identity")
|
||||
expect(not is_maskable_label("Getöse"), "collective noun is not identity")
|
||||
expect(not is_maskable_label("Schiff"), "vessel is not identity")
|
||||
expect(not is_maskable_label("Faden"), "prompt word is not identity")
|
||||
expect("Getöse" not in {item["text"] for item in _pattern_entities("Zweimaster mit Getöse im Hafen.", set())}, "mit plus common noun is not a name")
|
||||
expect("Anna" in {item["text"] for item in _pattern_entities("Ich war mit Anna im Hafen.", set())}, "mit plus given name stays")
|
||||
expect(is_maskable_label("Sushi"), "given name stays maskable")
|
||||
expect(is_maskable_label("meine Frau Sushi"), "kinship stripped, name remains")
|
||||
from identity_store import coalesce_mappings, normalize_token, remember_mapping
|
||||
from db import get_db
|
||||
import uuid
|
||||
|
||||
legacy_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO identity_mappings
|
||||
(id, profile_id, token, local_label, canonical_label, entity_type, status, origin)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(legacy_id, profile_id, "PERSON:55", "Wohnblock", "Wohnblock", "PERSON", STATUS_LEGACY, "legacy_auto"),
|
||||
)
|
||||
expect(
|
||||
not any(item.get("id") == legacy_id for item in list_mappings(profile_id)),
|
||||
"legacy mappings are not treated as confirmed",
|
||||
)
|
||||
legacy_run = _run(profile_id, "Der Wohnblock war ruhig.")
|
||||
expect("Wohnblock" in (legacy_run.trace.get("egress") or ""), "legacy auto-mapping does not skip detection or force masking")
|
||||
|
||||
listed = client.get("/api/admin/identities", headers=headers)
|
||||
expect(listed.status_code == 200, "admin identities list is authenticated")
|
||||
expect("registry" in listed.json() and "proposals" in listed.json(), "admin list returns registry and proposals")
|
||||
expect("local_label" not in json.dumps(first.diagnostics), "compact diagnostics omit labels")
|
||||
expect("Anna" not in json.dumps(first.diagnostics), "compact diagnostics omit clear names")
|
||||
expect("Anna" not in json.dumps(first.trace.get("log") or []), "run log omits labels")
|
||||
|
||||
provider_seen = {"n": 0}
|
||||
|
||||
def mark_provider(messages, policy):
|
||||
provider_seen["n"] += 1
|
||||
from privacy_gateway import ChatResult as _unused # noqa: F401
|
||||
from providers import ChatResult
|
||||
|
||||
return ChatResult(content="[[PERSON:01]] war ruhig.", model="fake", usage={})
|
||||
|
||||
with patch("privacy_gateway.complete_model", mark_provider):
|
||||
_run(profile_id, "Ich traf Anna.")
|
||||
expect(provider_seen["n"] == 1, "successful detection still uses exactly one generate call")
|
||||
|
||||
from identity_store import coalesce_mappings, normalize_token
|
||||
from privacy_gateway import _mask
|
||||
from pronoun_bind import bind_user_lines, bind_pronouns
|
||||
from privacy_gateway import mask_for_egress
|
||||
|
||||
expect(normalize_token("PERSON:KURZ") is None, "template token KURZ rejected")
|
||||
first = remember_mapping(setup.json()["profile_id"], "meine Frau Sushi", "PERSON:08")
|
||||
second = remember_mapping(setup.json()["profile_id"], "Sushi", "PERSON:KURZ")
|
||||
expect(first["token"] == second["token"], "alias reuses token")
|
||||
expect(second["local_label"] == "Sushi", "canonical label is the name")
|
||||
first_map = remember_mapping(profile_id, "meine Frau Sushi", "PERSON:08")
|
||||
second_map = remember_mapping(profile_id, "Sushi", "PERSON:KURZ")
|
||||
expect(first_map["token"] == second_map["token"], "alias reuses token")
|
||||
expect(second_map["canonical_label"] == "Sushi", "canonical label is the name")
|
||||
extra_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO identity_mappings (id, profile_id, token, local_label)
|
||||
VALUES (?, ?, ?, ?)
|
||||
INSERT INTO identity_mappings (id, profile_id, token, local_label, canonical_label, entity_type, status, origin)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(extra_id, setup.json()["profile_id"], "PERSON:KURZ", "meine Frau Sushi"),
|
||||
(extra_id, profile_id, "PERSON:KURZ", "meine Frau Sushi", "meine Frau Sushi", "PERSON", "confirmed", "local_authoritative"),
|
||||
)
|
||||
coalesce_mappings(setup.json()["profile_id"])
|
||||
coalesce_mappings(profile_id)
|
||||
sushi_rows = [
|
||||
item
|
||||
for item in list_mappings(setup.json()["profile_id"])
|
||||
if (item.get("local_label") or "").lower() == "sushi"
|
||||
for item in list_registry(profile_id)
|
||||
if (item.get("canonical_label") or "").lower() == "sushi"
|
||||
]
|
||||
expect(len(sushi_rows) == 1, "coalesce keeps one Sushi row")
|
||||
expect(sushi_rows[0]["token"].upper() != "PERSON:KURZ", "stable token wins over KURZ")
|
||||
joined = _mask(
|
||||
"meine Frau Sushi kam. Sushi stellte fest.",
|
||||
sushi_rows,
|
||||
)
|
||||
joined = _mask("meine Frau Sushi kam. Sushi stellte fest.", [{"local_label": "Sushi", "token": sushi_rows[0]["token"]}])
|
||||
expect(joined.count(f"[[{sushi_rows[0]['token']}]]") == 2, "both mentions share one placeholder")
|
||||
expect("Sushi" not in joined, "name fully masked")
|
||||
expect("meine Frau " in joined, "kinship stays visible")
|
||||
homonym = _mask(
|
||||
"Ich esse gerne Sushi mit meiner Frau Sushi.",
|
||||
sushi_rows,
|
||||
[{"local_label": "Sushi", "token": sushi_rows[0]["token"]}],
|
||||
)
|
||||
expect("esse gerne Sushi" in homonym, "food homonym stays")
|
||||
expect("Frau [[" in homonym or f"Frau [[{sushi_rows[0]['token']}]]" in homonym, "kinship name is masked")
|
||||
expect(homonym.count("[[") == 1, "only the person mention is masked")
|
||||
eating = _mask("Heute nur von Sushi essen erzählt.", sushi_rows)
|
||||
eating = _mask("Heute nur von Sushi essen erzählt.", [{"local_label": "Sushi", "token": sushi_rows[0]["token"]}])
|
||||
expect("Sushi essen" in eating, "noun before eat stays food")
|
||||
expect("[[" not in eating, "food-only sentence is not the person")
|
||||
from pronoun_bind import bind_user_lines, bind_pronouns
|
||||
from privacy_gateway import mask_for_egress
|
||||
|
||||
anna = [{"local_label": "Anna", "token": "PERSON:01"}]
|
||||
bound = bind_user_lines(
|
||||
@@ -159,32 +412,18 @@ def main() -> None:
|
||||
)
|
||||
)
|
||||
expect(bound.count("[[PERSON:01]]") == 3, "name and both pronouns share one token")
|
||||
expect("Sie" not in bound and "ihr" not in bound, "pronouns rebound to person")
|
||||
journal_masked = mask_for_egress(
|
||||
"user: Ich war mit Anna.\nuser: Sie setzte sich zu mir, und ich gab ihr Tee.",
|
||||
anna,
|
||||
"journal_generate",
|
||||
)
|
||||
expect(journal_masked.count("[[PERSON:01]]") == 1, "journal masks the name only")
|
||||
expect("Sie setzte" in journal_masked and "ihr Tee" in journal_masked, "journal keeps anaphora")
|
||||
dialogue_masked = mask_for_egress(
|
||||
"user: Ich war mit Anna.\nuser: Sie setzte sich zu mir, und ich gab ihr Tee.",
|
||||
anna,
|
||||
"dialogue_turn",
|
||||
)
|
||||
expect(dialogue_masked.count("[[PERSON:01]]") == 3, "dialogue still binds pronouns")
|
||||
instruction = bind_user_lines(
|
||||
"user: Anna kam.\n"
|
||||
"Der Impuls darf eine Frage sein. Er darf keine neue Station setzen.\n"
|
||||
)
|
||||
expect("Er darf keine neue Station" in instruction, "instruction pronoun stays")
|
||||
instruction_name = _mask(
|
||||
"Nenne Berg nicht in der Anweisung.\nuser: Berg kam vorbei.",
|
||||
[{"local_label": "Berg", "token": "PERSON:02"}],
|
||||
personal_lines_only=True,
|
||||
)
|
||||
expect("Nenne Berg nicht" in instruction_name, "instruction line is not remasked")
|
||||
expect("[[PERSON:02]]" in instruction_name, "user line still remasked")
|
||||
herr = [{"local_label": "Berg", "token": "PERSON:02"}]
|
||||
male, _ = bind_pronouns(_mask("Herr Berg kam. Er setzte sich.", herr))
|
||||
expect(male.count("[[PERSON:02]]") == 2, "masculine pronoun follows last man")
|
||||
@@ -192,18 +431,41 @@ def main() -> None:
|
||||
expect("ihr" in address, "address ihr is not a person")
|
||||
plural, _ = bind_pronouns(_mask("Anna kam. Sie waren zu dritt.", anna))
|
||||
expect("Sie waren" in plural or "sie waren" in plural.lower(), "plural sie stays")
|
||||
croatia = (
|
||||
"Heute bin ich ca. gegen 6:00 Uhr aufgestanden. Relativ früh gesellte sich "
|
||||
"auch meine Frau Sushi zu mir. Allerdings sollte die Bootstour erst um 11:00 Uhr starten."
|
||||
)
|
||||
patterned_live = {item["text"] for item in _pattern_entities(croatia, set())}
|
||||
expect("Sushi" in patterned_live, "pattern keeps given name")
|
||||
expect(
|
||||
patterned_live <= {"Sushi"},
|
||||
f"pattern does not treat German function words as names: {patterned_live}",
|
||||
)
|
||||
from entity_detect import uses_llm_detect
|
||||
from providers import ProviderConfig
|
||||
|
||||
from identity_store import is_maskable_label
|
||||
|
||||
expect(not is_maskable_label("ich"), "pronoun ich is not a mapping")
|
||||
expect(not is_maskable_label("user"), "role user is not a mapping")
|
||||
expect(not is_maskable_label("6:00 Uhr"), "clock time is not identity")
|
||||
expect(is_maskable_label("Sushi"), "name-shaped label stays structurally maskable")
|
||||
expect(is_maskable_label("Himmel"), "weather word is not banned by a stopword list")
|
||||
|
||||
from db import get_db as _get_db
|
||||
|
||||
with _get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.entity_detect",),
|
||||
).fetchone()
|
||||
expect(row["seed_revision"] == "2026-08-27-detect-ground-v1", "detect prompt revision is stored")
|
||||
expect("{{known_labels}}" not in (row["template"] or ""), "new detect prompt has no known_labels skip list")
|
||||
expect("Zwiebeln" not in (row["template"] or ""), "detect prompt has no food word list")
|
||||
|
||||
with _get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE ai_prompts SET template = ? WHERE slug = ?",
|
||||
("CUSTOM DETECT {{source_text}}", "mvp.entity_detect"),
|
||||
)
|
||||
from db import init_db
|
||||
|
||||
init_db()
|
||||
with _get_db() as conn:
|
||||
custom = conn.execute(
|
||||
"SELECT template, seed_revision FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.entity_detect",),
|
||||
).fetchone()
|
||||
expect(custom["template"] == "CUSTOM DETECT {{source_text}}", "user-edited detect prompt is not overwritten")
|
||||
expect(custom["seed_revision"] == "2026-08-27-detect-ground-v1", "revision still updates the default")
|
||||
|
||||
remote = ProviderConfig(
|
||||
role="detect",
|
||||
@@ -232,42 +494,60 @@ def main() -> None:
|
||||
)
|
||||
expect(uses_llm_detect(local), "production still allows local detect")
|
||||
os.environ.pop("KANSHO_ENV", None)
|
||||
poisoned = _mask(
|
||||
"Nicht erwähnt ist nicht geschehen. Ich kochte Tee.",
|
||||
[{"local_label": "ich", "token": "PERSON:05"}],
|
||||
)
|
||||
expect("N[[PERSON:05]]t" not in poisoned, "ich does not split nicht")
|
||||
expect("Nicht erwähnt" in poisoned, "instruction word bleibt")
|
||||
from privacy_gateway import PrivacyGatewayError, _demask, _validate_response
|
||||
|
||||
mapped = [{"local_label": "Anna", "token": "PERSON:01"}]
|
||||
kept = _validate_response("[[PERSON:01]] war ruhig.", mapped)
|
||||
expect(kept.startswith("[[PERSON:01]]"), "placeholders pass validation")
|
||||
expect(_demask(kept, mapped) == "Anna war ruhig.", "demask after validation")
|
||||
blocked = False
|
||||
anna_manifest = mask_prompt("Anna war am Markt.", mapped, "journal_generate")
|
||||
kept = _validate_response("[[PERSON:01]] war ruhig.", anna_manifest)
|
||||
expect(_demask(kept, anna_manifest) == "Anna war ruhig.", "demask after validation")
|
||||
expect(
|
||||
_demask("[[ person:01 ]] war ruhig.", anna_manifest) == "Anna war ruhig.",
|
||||
"demask is case- and space-insensitive",
|
||||
)
|
||||
blocked_plain = False
|
||||
try:
|
||||
_validate_response("Anna war ruhig.", mapped)
|
||||
_validate_response("Anna war ruhig.", anna_manifest)
|
||||
except PrivacyGatewayError as exc:
|
||||
blocked = exc.code == "response_validation_failed"
|
||||
expect(blocked, "plaintext identity is blocked before demask")
|
||||
sushi_map = [{"local_label": "Sushi", "token": "PERSON:01"}]
|
||||
food_reply = _validate_response("Heute nur von Sushi essen erzählt.", sushi_map)
|
||||
expect("Sushi essen" in food_reply, "food homonym in a reply is not treated as a leak")
|
||||
mixed = mask_for_egress(
|
||||
"Belegte Vorhaben (nur user-Quellen):\n- Morgen wollen wir mit Anna fahren.\nuser: Heute Markt.",
|
||||
mapped,
|
||||
"dialogue_turn",
|
||||
blocked_plain = exc.code == "response_validation_failed"
|
||||
expect(blocked_plain, "plaintext identity is blocked before demask")
|
||||
|
||||
no_user = client.get("/api/admin/identities")
|
||||
expect(no_user.status_code in {401, 403}, "identity registry requires auth")
|
||||
admin_calls = {"n": 0}
|
||||
|
||||
def no_egress(*_args, **_kwargs):
|
||||
admin_calls["n"] += 1
|
||||
raise AssertionError("admin identity actions must not call detect")
|
||||
|
||||
with patch("entity_detect.complete_chat", no_egress), patch("privacy_gateway.complete_model", no_egress):
|
||||
created = client.post(
|
||||
"/api/admin/identities",
|
||||
headers=headers,
|
||||
json={"canonical_label": "Nordlicht", "entity_type": "PROJECT", "aliases": ["Nordlicht-Projekt"]},
|
||||
)
|
||||
expect(created.status_code == 200, f"admin create stays local {created.text}")
|
||||
expect(admin_calls["n"] == 0, "admin registry actions create no external egress")
|
||||
expect(created.json()["status"] == "confirmed", "admin create confirms locally")
|
||||
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Ich war mit Anna in Hamburg und danach allein."},
|
||||
)
|
||||
expect("Anna" not in mixed, "opening context names are masked for dialogue egress")
|
||||
expect("[[PERSON:01]]" in mixed, "opening context uses the person token")
|
||||
expect("Heute Markt" in mixed, "current user wording remains")
|
||||
excerpt = mask_for_egress(
|
||||
"Frühere Einträge im Space (nur Hinweise, keine Gewissheit):\n2026-08-01: Anna war am Hafen.\nuser: Heute windstill.",
|
||||
mapped,
|
||||
"dialogue_turn",
|
||||
)
|
||||
expect("Anna" not in excerpt, "space excerpts are masked for dialogue egress")
|
||||
expect("[[PERSON:01]]" in excerpt, "space excerpt uses the person token")
|
||||
expect(turn.status_code == 200, f"turn {turn.status_code}")
|
||||
expect((turn.json().get("trace") or {}).get("generate_called") is True, "dialogue generate still runs once after detect")
|
||||
|
||||
install_test_detect_failure(DetectError("detect_invalid_output", "Detect-text entspricht nicht dem lokalen Substring."))
|
||||
try:
|
||||
held = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute nur der Markt."},
|
||||
)
|
||||
expect(held.status_code == 200, f"detect fallback turn {held.status_code} {held.text}")
|
||||
expect((held.json().get("assistant") or {}).get("body"), "detect failure still leaves a local impulse")
|
||||
expect((held.json().get("decision") or {}).get("guard") == "detect_blocked", "detect fallback is marked locally")
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
|
||||
print("All privacy detect tests passed.")
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Eval harness contract. Does not call a live provider."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
os.environ.setdefault("KANSHO_FAKE_DETECT", "1")
|
||||
|
||||
from entity_detect_eval import run_fake
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
payload = run_fake()
|
||||
expect(payload["mode"] == "fake", "eval default is fake")
|
||||
expect(payload["live_quality"] == "unconfirmed", "live quality stays unconfirmed")
|
||||
expect(len(payload["cases"]) >= 3, "synthetic cases exist")
|
||||
print("All detect eval harness tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Request-scoped masking manifest and response validation invariants."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-manifest-test.sqlite")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from identity_store import remember_mapping
|
||||
from main import app
|
||||
from privacy_gateway import (
|
||||
ERROR_EGRESS_VALIDATION,
|
||||
IDENTITY_LEAK_RETRY,
|
||||
ActiveReplacement,
|
||||
GatewayRequest,
|
||||
MaskingManifest,
|
||||
PrivacyGatewayError,
|
||||
_demask,
|
||||
_validate_response,
|
||||
complete,
|
||||
last_trace,
|
||||
mask_prompt,
|
||||
reset_debug,
|
||||
validate_pre_egress,
|
||||
)
|
||||
from providers import ChatResult
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def _no_label_keys(payload) -> bool:
|
||||
if isinstance(payload, dict):
|
||||
if "local_label" in payload:
|
||||
return False
|
||||
return all(_no_label_keys(value) for value in payload.values())
|
||||
if isinstance(payload, list):
|
||||
return all(_no_label_keys(item) for item in payload)
|
||||
return True
|
||||
|
||||
|
||||
def _run_gateway(profile_id: str, rendered: str, *, purpose: str = "dialogue_turn"):
|
||||
return complete(
|
||||
GatewayRequest(
|
||||
prompt_id="manifest-test",
|
||||
purpose=purpose,
|
||||
data_class="B",
|
||||
profile_id=profile_id,
|
||||
payload={"rendered": rendered, "source_text": rendered},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
anna = [{"local_label": "Anna", "token": "PERSON:01", "entity_type": "PERSON"}]
|
||||
lars = [{"local_label": "Lars", "token": "PERSON:02", "entity_type": "PERSON"}]
|
||||
sushi = [{"local_label": "Sushi", "token": "PERSON:01", "entity_type": "PERSON"}]
|
||||
hamburg = [{"local_label": "Hamburg", "token": "PLACE:01", "entity_type": "PLACE"}]
|
||||
|
||||
inactive = mask_prompt("Heute nur Markt und Kirschen.", anna, "journal_generate")
|
||||
expect(inactive.replacements == (), "historical mapping is inactive when absent from the prompt")
|
||||
kept = _validate_response("Anna kam später zum Markt.", inactive)
|
||||
expect("Anna kam später" in kept, "inactive mapping does not treat the same string as a leak")
|
||||
|
||||
mixed = mask_prompt("Anna war in Hamburg.", anna + lars + hamburg, "journal_generate")
|
||||
active_tokens = {item.token for item in mixed.replacements}
|
||||
expect(active_tokens == {"PERSON:01", "PLACE:01"}, "only actually masked mappings are active")
|
||||
expect("PERSON:02" not in active_tokens, "unused profile mapping stays inactive")
|
||||
|
||||
food_only = mask_prompt("Ich esse gerne Sushi.", sushi, "journal_generate")
|
||||
expect(food_only.replacements == (), "intentional homonym is not an active identity")
|
||||
food_reply = _validate_response("Heute nur von Sushi essen erzählt.", food_only)
|
||||
expect("Sushi essen" in food_reply, "unmasked homonym does not block a normal reply")
|
||||
|
||||
kinship = mask_prompt("Meine Frau Sushi kam später.", sushi, "journal_generate")
|
||||
expect(len(kinship.replacements) == 1, "identity mention of the same word is active")
|
||||
blocked = False
|
||||
try:
|
||||
_validate_response("Sushi kam vorbei.", kinship)
|
||||
except PrivacyGatewayError as exc:
|
||||
blocked = exc.code == "response_validation_failed"
|
||||
expect(exc.diagnostics.get("leak_tokens") == ["PERSON:01"], "block reports the token, not the label")
|
||||
expect("Sushi" not in json.dumps(exc.diagnostics), "validation diagnostics omit the clear name")
|
||||
expect(blocked, "active clear name in the raw reply is a leak")
|
||||
homonym_ok = _validate_response("Danach Sushi essen.", kinship)
|
||||
expect("Sushi essen" in homonym_ok, "homonym in the reply uses the same classification rule")
|
||||
|
||||
placeholder = _validate_response("[[PERSON:01]] war ruhig.", kinship)
|
||||
expect(placeholder.startswith("[[PERSON:01]]"), "active placeholder passes validation")
|
||||
expect(_demask(placeholder, kinship) == "Sushi war ruhig.", "demask happens only after validation")
|
||||
leftover = _demask("[[PERSON:01]] und [[PERSON:99]]", kinship)
|
||||
expect("Sushi" in leftover and "[[PERSON:99]]" in leftover, "inactive placeholder is not rematerialized")
|
||||
expect("Clarissa" not in leftover, "inactive mapping label is not introduced by demask")
|
||||
expect("[[" not in IDENTITY_LEAK_RETRY, "retry instruction must not teach bracket placeholders")
|
||||
expect(
|
||||
_demask("[[ person:01 ]] war ruhig.", kinship) == "Sushi war ruhig.",
|
||||
"demask ignores case and inner spacing",
|
||||
)
|
||||
expect(
|
||||
_demask("[[PERSON:1]] war ruhig.", kinship) == "Sushi war ruhig.",
|
||||
"demask accepts an unpadded numeric token",
|
||||
)
|
||||
expect(
|
||||
_demask("[[…]] und [[...]] blieben.", kinship) == "[[…]] und [[...]] blieben.",
|
||||
"generic ellipsis placeholders are not rematerialized",
|
||||
)
|
||||
|
||||
overlap = mask_prompt(
|
||||
"Anna-Lena kam vorbei.",
|
||||
[
|
||||
{"local_label": "Anna", "token": "PERSON:01"},
|
||||
{"local_label": "Anna-Lena", "token": "PERSON:08"},
|
||||
],
|
||||
"journal_generate",
|
||||
)
|
||||
expect([item.token for item in overlap.replacements] == ["PERSON:08"], "only the actually replaced overlapping label is active")
|
||||
expect("[[PERSON:08]]" in overlap.masked_text, "longer label is the one replaced")
|
||||
expect("Anna-Lena" not in overlap.masked_text, "replaced longer label is gone")
|
||||
|
||||
subject = mask_prompt(
|
||||
"Clarissa kaufte Kirschen.",
|
||||
[{"local_label": "Clarissa", "token": "PERSON:99"}],
|
||||
"journal_generate",
|
||||
)
|
||||
expect([item.token for item in subject.replacements] == ["PERSON:99"], "person as subject of kaufte remains identity")
|
||||
|
||||
leaky = MaskingManifest(
|
||||
masked_text="Anna war am Markt.",
|
||||
available_mapping_count=1,
|
||||
replacements=(ActiveReplacement("PERSON:01", "PERSON", 1, "Anna"),),
|
||||
)
|
||||
pre = False
|
||||
try:
|
||||
validate_pre_egress("Anna war am Markt.", leaky)
|
||||
except PrivacyGatewayError as exc:
|
||||
pre = exc.code == ERROR_EGRESS_VALIDATION
|
||||
expect(exc.diagnostics.get("pre_egress_validation") == "failed", "pre-egress failure is named")
|
||||
expect(pre, "remaining active identity occurrence fails closed before the provider")
|
||||
validate_pre_egress("[[PERSON:01]] war am Markt.", leaky)
|
||||
|
||||
def check(prompt: str, mappings: list[dict], reply: str):
|
||||
manifest = mask_prompt(prompt, mappings, "journal_generate")
|
||||
try:
|
||||
return "ok", _validate_response(reply, manifest), [item.token for item in manifest.replacements]
|
||||
except PrivacyGatewayError as exc:
|
||||
return exc.code, None, [item.token for item in manifest.replacements]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
first = pool.submit(check, "Anna kam vorbei.", anna, "Anna kam.")
|
||||
second = pool.submit(check, "Heute nur Markt.", anna + lars, "Anna kam.")
|
||||
code_a, _text_a, tokens_a = first.result()
|
||||
code_b, text_b, tokens_b = second.result()
|
||||
expect(code_a == "response_validation_failed", "parallel request A still blocks its own active leak")
|
||||
expect(tokens_a == ["PERSON:01"], "parallel request A keeps its own active set")
|
||||
expect(code_b == "ok" and "Anna kam" in (text_b or ""), "parallel request B is not blocked by A's mapping")
|
||||
expect(tokens_b == [], "parallel request B does not inherit A's active set")
|
||||
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
profile_id = setup.json()["profile_id"]
|
||||
remember_mapping(profile_id, "Anna", "PERSON:01")
|
||||
remember_mapping(profile_id, "Clarissa", "PERSON:99")
|
||||
|
||||
result = _run_gateway(profile_id, "Ich war mit Anna am Markt.")
|
||||
expect(result.allowed, "masked identity request is allowed")
|
||||
expect(result.diagnostics.get("available_mapping_count", 0) >= 1, "diagnostics count request mappings")
|
||||
expect(result.diagnostics.get("active_mapping_count") >= 1, "diagnostics count actually masked mappings")
|
||||
expect("PERSON:01" in (result.diagnostics.get("active_tokens") or []), "diagnostics name active tokens")
|
||||
expect(result.trace.get("pre_egress_validation") == "ok", "pre-egress is recorded")
|
||||
expect(result.trace.get("response_validation") == "ok", "response validation is recorded")
|
||||
expect(_no_label_keys(result.diagnostics), "compact diagnostics have no local_label keys")
|
||||
expect(_no_label_keys(result.trace.get("log")), "run log has no local_label keys")
|
||||
expect(_no_label_keys(last_trace()), "last compact trace has no local_label keys")
|
||||
compact_blob = json.dumps(result.diagnostics)
|
||||
expect("local_label" not in compact_blob, "compact diagnostics omit the mapping field name")
|
||||
expect("Anna" not in compact_blob, "compact diagnostics omit the clear name")
|
||||
|
||||
provider_calls = {"n": 0}
|
||||
|
||||
def boom(_messages, _policy):
|
||||
provider_calls["n"] += 1
|
||||
raise AssertionError("provider must not be called after pre-egress failure")
|
||||
|
||||
def leaky_mask(rendered, mappings, purpose):
|
||||
return MaskingManifest(
|
||||
masked_text=rendered,
|
||||
available_mapping_count=len(mappings or []),
|
||||
replacements=(ActiveReplacement("PERSON:01", "PERSON", 1, "Anna"),),
|
||||
)
|
||||
|
||||
blocked_egress = False
|
||||
with patch("privacy_gateway.mask_prompt", leaky_mask), patch("privacy_gateway.complete_model", boom):
|
||||
try:
|
||||
_run_gateway(profile_id, "Anna war am Markt.")
|
||||
except PrivacyGatewayError as exc:
|
||||
blocked_egress = exc.code == ERROR_EGRESS_VALIDATION
|
||||
expect(blocked_egress, "pre-egress failure aborts the request")
|
||||
expect(provider_calls["n"] == 0, "provider is not called when an active identity remains")
|
||||
|
||||
seen = []
|
||||
|
||||
def leak_then_ok(messages, _policy):
|
||||
seen.append(messages[0].get("content") or "")
|
||||
if len(seen) == 1:
|
||||
return ChatResult(
|
||||
content="Anna stand den ganzen Nachmittag am Markt.",
|
||||
model="fake",
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18, "cost": 0.01},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
return ChatResult(
|
||||
content="[[PERSON:01]] stand den ganzen Nachmittag am Markt.",
|
||||
model="fake",
|
||||
usage={"prompt_tokens": 12, "completion_tokens": 8, "total_tokens": 20, "cost": 0.02},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", leak_then_ok):
|
||||
repaired = _run_gateway(profile_id, "Ich war mit Anna am Markt.")
|
||||
expect(len(seen) == 2, "a real leak retries exactly once")
|
||||
expect(IDENTITY_LEAK_RETRY in seen[1], "retry uses the generic correction instruction")
|
||||
expect("Anna stand den ganzen Nachmittag" not in seen[1], "discarded raw reply is not part of the retry prompt")
|
||||
expect("[[PERSON:01]]" in seen[0] and "Anna" not in seen[0].replace("[[PERSON:01]]", ""), "retry keeps the already masked prompt")
|
||||
expect("Anna" in (repaired.content or ""), "successful retry is demasked locally")
|
||||
expect(repaired.diagnostics.get("response_validation_retry") == 1, "retry is recorded without the raw reply")
|
||||
|
||||
def spaced_token(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="[[ person:01 ]] stand den ganzen Nachmittag am Markt.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", spaced_token):
|
||||
restored = _run_gateway(profile_id, "Ich war mit Anna am Markt.")
|
||||
expect(
|
||||
"Anna stand den ganzen Nachmittag" in (restored.content or ""),
|
||||
"gateway demasks case and spacing variants",
|
||||
)
|
||||
expect("[[" not in (restored.content or ""), "no leftover placeholder after a successful demask")
|
||||
expect(repaired.diagnostics.get("prompt_tokens") == 22, "retry aggregates prompt tokens")
|
||||
expect(repaired.diagnostics.get("completion_tokens") == 16, "retry aggregates completion tokens")
|
||||
expect(repaired.diagnostics.get("total_tokens") == 38, "retry aggregates total tokens")
|
||||
expect(abs(float(repaired.diagnostics.get("cost") or 0) - 0.03) < 1e-9, "retry aggregates cost")
|
||||
|
||||
inactive_calls = {"n": 0}
|
||||
|
||||
def keep_inactive(messages, _policy):
|
||||
inactive_calls["n"] += 1
|
||||
return ChatResult(
|
||||
content="Clarissa blieb heute unerwähnt am Hafen.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", keep_inactive):
|
||||
kept_inactive = _run_gateway(profile_id, "Heute nur Markt und Kirschen am Hafen.")
|
||||
expect(inactive_calls["n"] == 1, "inactive mapping does not start a retry")
|
||||
expect(kept_inactive.trace.get("response_validation_retry") in (None, 0), "inactive mapping does not mark a retry")
|
||||
expect("Clarissa blieb heute unerwähnt" in (kept_inactive.content or ""), "inactive mapping keeps the model text")
|
||||
|
||||
retry_fail = []
|
||||
|
||||
def always_leak(messages, _policy):
|
||||
retry_fail.append(messages[0].get("content") or "")
|
||||
return ChatResult(
|
||||
content="Anna stand den ganzen Nachmittag am Markt.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
|
||||
failed = False
|
||||
with patch("privacy_gateway.complete_model", always_leak):
|
||||
try:
|
||||
_run_gateway(profile_id, "Ich war mit Anna am Markt.")
|
||||
except PrivacyGatewayError as exc:
|
||||
failed = exc.code == "response_validation_failed"
|
||||
expect(exc.diagnostics.get("response_validation_retry") == 1, "failed retry is still only one extra call")
|
||||
expect("Anna stand" not in json.dumps(exc.diagnostics.get("log") or []), "discarded reply is not persisted in the log")
|
||||
expect(failed, "second leak stays fail-closed")
|
||||
expect(len(retry_fail) == 2, "gateway stops after one retry")
|
||||
|
||||
print("All privacy manifest tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user