From f17b3083e8342895f6100fc5a33798c49e3a5ca5 Mon Sep 17 00:00:00 2001 From: Lars Date: Thu, 27 Aug 2026 10:03:05 +0200 Subject: [PATCH] MVP 0.93 - stabile, aber Writing Profile ohne erkennbaren Nutzen --- backend/config/prompts.seed.json | 10 +- backend/context_builder.py | 18 + backend/db.py | 34 +- backend/dialogue_turn.py | 5 +- backend/engine.py | 1 + backend/entity_detect.py | 802 ++++++++++++++---- backend/entity_detect_eval.py | 113 +++ backend/identity_store.py | 692 +++++++++++---- backend/journal_editorial.py | 201 +++++ backend/journal_eval.py | 273 ++++++ backend/journal_generate.py | 347 ++++++-- backend/journal_opening.py | 5 +- backend/journal_shape.py | 10 +- backend/placeholder_mvp.py | 26 +- backend/privacy_gateway.py | 674 +++++++++++++-- backend/privacy_placeholders.py | 2 +- backend/prompt_budget.py | 34 + backend/provider_settings.py | 4 +- backend/providers.py | 7 + backend/routers/admin.py | 122 +++ backend/schema.sql | 20 + backend/tests/test_frame.py | 3 + backend/tests/test_identity_registry.py | 127 +++ backend/tests/test_journal_budget.py | 130 ++- backend/tests/test_journal_editorial.py | 426 ++++++++++ backend/tests/test_journal_eval.py | 62 ++ backend/tests/test_journal_narration.py | 299 +++++++ backend/tests/test_journal_opening.py | 8 +- backend/tests/test_journal_shape.py | 20 +- backend/tests/test_mvp_journal.py | 34 +- backend/tests/test_privacy_detect.py | 550 +++++++++--- backend/tests/test_privacy_detect_eval.py | 30 + backend/tests/test_privacy_manifest.py | 309 +++++++ backend/writing_profile_store.py | 72 +- .../context_fidelity_and_regrounding.md | 2 + docs/architecture/functional/guardrails.md | 18 + .../functional/mvp_freeze_candidate.md | 9 +- .../writing_profile_and_journaling.md | 19 + .../technical/admin_diagnostics.md | 2 +- .../technical/documentation_index.md | 2 +- .../technical/frontend_pwa_shell.md | 4 +- .../technical/mvp_implementation.md | 47 +- .../architecture/technical/privacy_gateway.md | 88 +- .../technical/provenance_verification.md | 8 +- frontend/src/App.jsx | 2 + frontend/src/app.css | 60 +- frontend/src/components/CallTrace.jsx | 21 +- frontend/src/components/RunLogPopup.jsx | 136 +++ frontend/src/config/adminNav.js | 1 + frontend/src/pages/AdminHomePage.jsx | 4 +- frontend/src/pages/AdminIdentitiesPage.jsx | 190 +++++ frontend/src/pages/AdminProvidersPage.jsx | 6 +- frontend/src/pages/DialoguePage.jsx | 4 +- frontend/src/pages/JournalDayPage.jsx | 30 +- frontend/src/pages/JournalEditorPage.jsx | 23 +- frontend/src/pages/JournalSourcePage.jsx | 4 +- 56 files changed, 5366 insertions(+), 784 deletions(-) create mode 100644 backend/entity_detect_eval.py create mode 100644 backend/journal_editorial.py create mode 100644 backend/journal_eval.py create mode 100644 backend/tests/test_identity_registry.py create mode 100644 backend/tests/test_journal_editorial.py create mode 100644 backend/tests/test_journal_eval.py create mode 100644 backend/tests/test_journal_narration.py create mode 100644 backend/tests/test_privacy_detect_eval.py create mode 100644 backend/tests/test_privacy_manifest.py create mode 100644 frontend/src/components/RunLogPopup.jsx create mode 100644 frontend/src/pages/AdminIdentitiesPage.jsx diff --git a/backend/config/prompts.seed.json b/backend/config/prompts.seed.json index 171a148..f62bb0f 100644 --- a/backend/config/prompts.seed.json +++ b/backend/config/prompts.seed.json @@ -13,11 +13,12 @@ "id": "mvp-journal-generate", "slug": "mvp.journal_generate", "name": "MVP Journalentwurf", - "description": "Stufe 2: persönliche Narration aus validierter Rekonstruktion. Explizit ausgelöst. Nicht der Dialogzug.", + "description": "Stufe 2: persönliche Narration aus dem lokalen Verified Artifact. Explizit ausgelöst. Nicht der Dialogzug.", "category": "mvp", "prompt_type": "base", "required_feature": "ai_calls", - "template": "Schreibe einen Tagebucheintrag in der Ich-Form von [[SELF]].\n\nUnten liegt ausschließlich ein lokal erzeugtes Verified Artifact. Es ist die einzige Inhaltsquelle. sources[].text ist der vollständige, lokal rehydrierte Nutzerinhalt. Evidence-Ausschnitte sind nur nachgewiesene Zitate, keine Erlaubnis, den Rest wegzulassen. annotations_unverified, falls vorhanden, sind unbestätigte Modelldeutung und keine Tatsachen: daraus keine Gefühle, Widersprüche, Korrekturen oder Planänderungen machen.\n\nKein Rohdialog, keine Assistentenzeilen, keine frei erfundenen Fakten. Stimme aus dem Writing Profile, sobald dort Core, Facet-Delta oder Traits stehen. Szenen zusammenführen. Die Reihenfolge der sources-Einträge behalten. Keine Zeiten, Gefühle, Motive oder Zusammenhänge ergänzen.\n\nSchritte, alle nötig:\n1. Nur den verifizierten Nutzerwortlaut aus sources[].text.\n2. Absätze nach Phase, Ort oder Uhrzeit. Dieselbe Begebenheit bleibt ein Strang.\n3. Übergänge glätten. Wiederholungen streichen. Unsicherheiten im Wortlaut sichtbar lassen, nicht glätten und nicht aus Labels ableiten.\n4. Person: das Token nur dort, wo [[SELF]] den Namen genannt hat — in einer Passage ein- bis zweimal. Danach sie/er/ihr. Indirekte Rede nicht umbiegen.\n5. Writing Profile: Satzlänge, Wortwahl, Detailgrad, Übergänge. Keine neuen Fakten.\n\nErste Zeile: kurze Überschrift. Danach der Eintrag.\nVerboten: Assistentenimpulse als Erlebtes; erfundene Fakten, Gefühle oder Details; generischer KI-Stil; Unsicherheit wegglätten; unbestätigte Labels als Wahrheit.\n\nWriting Profile:\n{{writing_profile}}\n\nBestehender Text, nur wenn ausdrücklich gewünscht:\n{{existing_text}}\n\nVerified Artifact:\n{{reconstruction}}\n\nSchreibe jetzt den Eintrag: erste Zeile Überschrift, danach mehrere Absätze. Inhalt nur aus sources[].text, Stimme aus dem Writing Profile.\n" + "template": "Schreibe eine inhaltstreue, redaktionell verbesserte Tagebuchfassung in der Ich-Form von [[SELF]].\n\nPriorität, höher schlägt niedriger:\n1. Keine neuen Informationen erfinden.\n2. Tatsachen, Bedeutung, Unsicherheit, Verneinung sowie Plan versus Vollzug bewahren.\n3. Rechtschreibung, Grammatik und Zeichensetzung korrigieren.\n4. Lesbarkeit, Satzbau, Wiederholungen, Absätze und Übergänge verbessern.\n5. WRITING_PROFILE und STYLE_EXAMPLES anwenden.\n6. Gute Originalformulierungen erhalten; schwache Formulierungen verbessern.\n7. Nur Titel und fertigen Journaltext ausgeben.\n\nFaktentreue ist nicht Wortlauttreue. Paraphrasieren und neu strukturieren ist erlaubt. Rechtschreibfehler sind keine geschützten Fakten. Unsicherheit bleibt Unsicherheit, muss aber nicht wortgleich bleiben. Sprachlich nötige Übergänge sind erlaubt, solange sie keine neue Ursache, Bewertung oder Handlung behaupten. Direkte Rede und bewusst stilprägende Formulierungen dürfen bleiben. Der Text muss nicht künstlich vom Ausgangstext abweichen. Ein hoher Wortlautanteil ist erlaubt, wenn der Ausgangstext bereits gut ist. Ein nahezu unveränderter Text mit übernommenen Fehlern und schwachen Übergängen erfüllt den Auftrag nicht.\n\nEDITORIAL_MODE: {{editorial_mode}}\n{{editorial_instructions}}\n\nKurze synthetische Arbeitsbeispiele, keine inhaltliche Schablone für den heutigen Tag:\nprose_edit — Rohtext: «ich gieng zum laden und es war kald.» wird zu «Ich ging zum Laden, und es war kalt.»\nnotes_to_journal — Rohtext: «- laden; - brot; - später park» wird zu «Im Laden holte ich Brot. Später war ich im Park.»\n\nWRITING_PROFILE (nur Schreibweise, keine zusätzlichen Tatsachen):\n{{writing_profile}}\n\nSTYLE_EXAMPLES (nur Ton, Rhythmus, sprachliche Entscheidungen; Inhalte nicht übernehmen):\n{{style_examples}}\n\nCURRENT_DAY_SOURCES (einzige Tatsachen des heutigen Eintrags, keine Ausgabevorlage):\n{{reconstruction}}\n\nEXISTING_TEXT, nur wenn ausdrücklich gewünscht:\n{{existing_text}}\n\nNamen und Orte nur als die im Kontext bereits vorhandenen Platzhalter schreiben, zeichengetreu und unverändert. Keine Klarnamen. Keine neuen Platzhalter. Keine Auslassungspunkte in doppelten Klammern. Ausgabe: erste Zeile kurze Überschrift, danach zusammenhängende Absätze. Keine Meta-Kommentare.\n", + "seed_revision": "2026-08-27-journal-placeholders-v1" }, { "id": "mvp-journal-reconstruct", @@ -33,11 +34,12 @@ "id": "mvp-entity-detect", "slug": "mvp.entity_detect", "name": "MVP Maskierung", - "description": "Nur schützenswerte Eigennamen. Keine Dialogoperation, kein Journalentwurf. Optional; Normalweg ist Mustererkennung.", + "description": "Semantische Detection von Identitäten im persönlichen Egress. Spans und Typen, keine Tokens. Keine Dialogoperation, kein Journalentwurf.", "category": "privacy", "prompt_type": "base", "required_feature": "ai_calls", - "template": "Aufgabe: Nur identifizierende Eigennamen. Lieber zu wenig als zu viel.\n\nAufnehmen:\n- Vorname oder Rufname einer Person (Sushi, Anna)\n- benannter Ort (Hamburg, Lošinj)\n- benannte Organisation\nNur das Namenswort, nicht den ganzen Satzteil. „meine Frau Sushi“ → Sushi. Dieselbe Person immer derselbe Name, immer dasselbe Token.\n\nNicht aufnehmen:\n- Pronomen: ich, sie, wir, mich\n- Rollen und Verwandtschaft allein: Frau, Herr, Partner\n- Wetter und Natur: Himmel, Luft, Regen\n- Dinge, Speisen und Räume: Tee, Balkon, Laden, Bad, Brot, Zwiebeln, Tomaten, Paprika, Haferflocken\n- Zeit: 6:00 Uhr, 11:00, heute, gestern\n- Aktivitäten: Lektion, Bootstour, Frühstück, Tagebuch, Omlett\n- Dialogmarken: user, assistant\n\nSchon gemappt, nicht erneut und nicht unter neuem Token: {{known_labels}}\n\nBeispiel A\nText: Heute um 6:00 Uhr Tee auf dem Balkon, dann japanische Lektion. Frau Sushi kam dazu. Bootstour erst um 11:00.\nJSON: {\"entities\":[{\"text\":\"Sushi\",\"token\":\"PERSON:01\"}]}\n\nBeispiel B\nText: Ich war mit Anna in Hamburg.\nJSON: {\"entities\":[{\"text\":\"Anna\",\"token\":\"PERSON:01\"},{\"text\":\"Hamburg\",\"token\":\"PLACE:CITY\"}]}\n\nBeispiel C\nText: Der Himmel war klar, die Luft warm, ich kochte Tee.\nJSON: {\"entities\":[]}\n\nAntworte nur mit JSON: {\"entities\":[{\"text\":\"Name\",\"token\":\"PERSON:01\"}]}\nToken: PERSON:01, PERSON:02, PLACE:CITY, ORG:01. Nie PERSON:KURZ. KURZ ist kein Token. Keine sprechenden Aliase.\n\nText:\n{{source_text}}\n" + "seed_revision": "2026-08-27-detect-ground-v1", + "template": "Untersuche den gesamten Text semantisch. Entscheide kontextabhängig, nicht nach Großschreibung oder Namensähnlichkeit allein.\n\nSchützenswert ist eine konkrete Bezeichnung oder Information, durch die eine natürliche Person, ein genauer persönlicher Ort, eine Organisation, ein privates Projekt oder ein anderer im Produktrahmen definierter Identifikator erkennbar werden kann. Allgemeine Gegenstände, Tätigkeiten, Lebensmittel, Räume und Beschreibungen sind keine Identitäten, sofern sie im konkreten Satz nicht als Eigenname oder identifizierende Information verwendet werden.\n\nErlaubte entity_type-Werte ausschließlich: PERSON, PLACE, ORG, PROJECT.\n\nKontrast, keine Wortliste:\n- «Sushi kam ins Wohnzimmer» kann eine Person bezeichnen.\n- «Ich aß Sushi» bezeichnet ein Gericht.\n- «Ich arbeitete am privaten Projekt Aurora» kann ein schützenswertes Projekt bezeichnen.\n- «Ich ging auf den Balkon» bezeichnet keinen Eigennamen.\n- «Ich traf Anna» bezeichnet eine Person.\n\nNur das identifizierende Wort oder die identifizierende Wortgruppe, nicht den ganzen Satz. Offsets beziehen sich ausschließlich auf den gelieferten Text nach «Text:», 0-basiert, end ausschließlich. text muss ein exakter Substring dieses Texts sein, nicht aus den Beispielen oben. Wenn Offsets unsicher sind, kopiere trotzdem das identifizierende Wort zeichengetreu.\n\nKeine Tokens, keine Platzhalter, keine kanonischen Namen, keine Aliase, keine zusätzlichen Felder.\n\nAntworte nur mit JSON. Form: {\"entities\":[{\"start\":,\"end\":,\"text\":\"\",\"entity_type\":\"PERSON\"}]}\nNichts schützenswert: {\"entities\":[]}\n\nText:\n{{source_text}}\n" }, { "id": "mvp-profile-review", diff --git a/backend/context_builder.py b/backend/context_builder.py index cf4b412..393f3a5 100644 --- a/backend/context_builder.py +++ b/backend/context_builder.py @@ -75,6 +75,9 @@ def build_internal_context( existing_text: str = "", reconstruction: str = "", day_spec: dict | None = None, + style_examples: str = "", + editorial_mode: str = "", + editorial_instructions: str = "", ) -> dict: purpose = purpose if purpose in PURPOSES else "dialogue_turn" if conversation_id and not space_id: @@ -150,6 +153,9 @@ def build_internal_context( brief = compile_task_brief(profile_id, "journal_generate") items.append({"type": "writing_profile", "compiled_brief": brief}) + items.append({"type": "style_examples", "body": style_examples or ""}) + items.append({"type": "editorial_mode", "text": editorial_mode or ""}) + items.append({"type": "editorial_instructions", "text": editorial_instructions or ""}) if reconstruction: items.append({"type": "reconstruction", "body": reconstruction}) if include_existing and existing_text: @@ -179,6 +185,9 @@ def assemble_text(context: dict) -> dict[str, str]: interaction_hint = "" existing_text = "" reconstruction = "" + style_examples = "" + editorial_mode = "" + editorial_instructions = "" user_bodies: list[str] = [] purpose = context.get("purpose") or "" for item in context.get("items") or []: @@ -218,6 +227,12 @@ def assemble_text(context: dict) -> dict[str, str]: existing_text = item.get("body") or "" elif kind == "reconstruction": reconstruction = item.get("body") or "" + elif kind == "style_examples": + style_examples = item.get("body") or "" + elif kind == "editorial_mode": + editorial_mode = item.get("text") or "" + elif kind == "editorial_instructions": + editorial_instructions = item.get("text") or "" elif kind == "opening": pass opening_hint = "" @@ -238,6 +253,9 @@ def assemble_text(context: dict) -> dict[str, str]: return { "dialogue_context": "\n".join(dialogue_parts).strip(), "writing_profile": writing_profile, + "style_examples": style_examples, + "editorial_mode": editorial_mode, + "editorial_instructions": editorial_instructions, "interaction_hint": interaction_hint, "existing_text": existing_text, "reconstruction": reconstruction, diff --git a/backend/db.py b/backend/db.py index f92319d..b4aa256 100644 --- a/backend/db.py +++ b/backend/db.py @@ -27,6 +27,7 @@ _PROMPT_COLUMNS = { "output_schema_json": "TEXT", "required_feature": "TEXT NOT NULL DEFAULT 'ai_calls'", "default_template": "TEXT NOT NULL DEFAULT ''", + "seed_revision": "TEXT NOT NULL DEFAULT ''", "sort_order": "INTEGER NOT NULL DEFAULT 0", "updated": "TEXT NOT NULL DEFAULT (datetime('now'))", } @@ -62,6 +63,16 @@ _WRITING_VERSION_COLUMNS = { _JOURNAL_DAY_COLUMNS = { "scratch_json": "TEXT NOT NULL DEFAULT '[]'", } +_IDENTITY_MAPPING_COLUMNS = { + "canonical_label": "TEXT NOT NULL DEFAULT ''", + "entity_type": "TEXT NOT NULL DEFAULT 'PERSON'", + "status": "TEXT NOT NULL DEFAULT 'legacy_review_required'", + "origin": "TEXT NOT NULL DEFAULT 'legacy_auto'", + "aliases_json": "TEXT NOT NULL DEFAULT '[]'", + # SQLite ALTER TABLE cannot use datetime('now'); backfill in migrate_legacy_identity_rows. + "updated": "TEXT NOT NULL DEFAULT ''", + "confirmed_at": "TEXT", +} def _connect() -> sqlite3.Connection: @@ -148,10 +159,16 @@ def _seed_platform(conn: sqlite3.Connection) -> None: def _seed_prompts(conn: sqlite3.Connection) -> None: - """Prompts come from JSON/DB, never from Python string literals.""" + """Prompts come from JSON/DB, never from Python string literals. + + Untouched system prompts (template == default_template) receive the seed. + Independently edited prompts keep their template; only default_template and + seed_revision are refreshed so Reset-to-default stays possible. + """ items = json.loads(PROMPTS_SEED_PATH.read_text(encoding="utf-8")) for item in items: template = item.get("template") or "" + revision = item.get("seed_revision") or "" existing = row_to_dict( conn.execute("SELECT * FROM ai_prompts WHERE slug = ?", (item["slug"],)).fetchone() ) @@ -160,8 +177,8 @@ def _seed_prompts(conn: sqlite3.Connection) -> None: """ INSERT INTO ai_prompts (id, slug, name, description, category, prompt_type, template, - required_feature, is_system_default, default_template) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?) + required_feature, is_system_default, default_template, seed_revision) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?) """, ( item["id"], @@ -173,6 +190,7 @@ def _seed_prompts(conn: sqlite3.Connection) -> None: template, item.get("required_feature") or "ai_calls", template, + revision, ), ) continue @@ -182,6 +200,7 @@ def _seed_prompts(conn: sqlite3.Connection) -> None: UPDATE ai_prompts SET name = ?, description = ?, category = ?, prompt_type = ?, required_feature = ?, is_system_default = 1, default_template = ?, + seed_revision = ?, template = CASE WHEN ? THEN ? ELSE template END, updated = datetime('now') WHERE slug = ? @@ -193,6 +212,7 @@ def _seed_prompts(conn: sqlite3.Connection) -> None: item.get("prompt_type") or "base", item.get("required_feature") or "ai_calls", template, + revision, 1 if untouched else 0, template, item["slug"], @@ -421,12 +441,12 @@ def init_db() -> None: with get_db() as conn: conn.executescript(schema) _seed_platform(conn) + _ensure_columns(conn, "ai_prompts", _PROMPT_COLUMNS) _seed_prompts(conn) from provider_settings import seed_provider_settings seed_provider_settings(conn) _ensure_columns(conn, "profiles", _PROFILE_COLUMNS) - _ensure_columns(conn, "ai_prompts", _PROMPT_COLUMNS) _ensure_columns(conn, "conversations", _CONVERSATION_COLUMNS) _ensure_columns(conn, "journal_days", _JOURNAL_DAY_COLUMNS) _ensure_columns(conn, "writing_profiles", _WRITING_PROFILE_COLUMNS) @@ -436,6 +456,10 @@ def init_db() -> None: _ensure_columns(conn, "writing_profile_versions", _WRITING_VERSION_COLUMNS) migrate_journal_source_refs(conn) _migrate_writing_profile_shell(conn) + _ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS) + from identity_store import migrate_legacy_identity_rows + + migrate_legacy_identity_rows(conn) _mark(conn, "001_frame") _mark(conn, "002_platform") _mark(conn, "003_dialogue_memory") @@ -448,6 +472,8 @@ def init_db() -> None: _mark(conn, "010_profile_governance") _mark(conn, "011_profile_review") _mark(conn, "012_profile_shell") + _mark(conn, "013_journal_generate_narration") + _mark(conn, "014_identity_registry") from writing_profile_store import bootstrap_from_existing bootstrap_from_existing() diff --git a/backend/dialogue_turn.py b/backend/dialogue_turn.py index 51a51b5..6b96569 100644 --- a/backend/dialogue_turn.py +++ b/backend/dialogue_turn.py @@ -8,6 +8,7 @@ from context_builder import assemble_text, build_internal_context, is_closing_tu from conversation_signals import infer_signals from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals from engine import EngineError, execute_prompt, load_active_prompt +from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES from writing_profile_store import remember_dialogue_style from profile_review import consider_dialogue @@ -347,14 +348,14 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str | "guard": "local_fallback", } except EngineError as exc: - if exc.code != "response_validation_failed": + if exc.code != "response_validation_failed" and exc.code not in DETECT_DIALOGUE_FALLBACK_CODES: raise impulse = local_hold(last_user_text(assembled)) decision = { "operation": "fortfuehren", "label": OPERATIONS["fortfuehren"], "parsed": False, - "guard": "identity_leak_blocked", + "guard": "identity_leak_blocked" if exc.code == "response_validation_failed" else "detect_blocked", } assistant = append_message(profile_id, conversation_id, impulse, role="assistant") user_bodies = [ diff --git a/backend/engine.py b/backend/engine.py index f0b9ff6..4d78860 100644 --- a/backend/engine.py +++ b/backend/engine.py @@ -119,4 +119,5 @@ def execute_prompt( "provider": result.provider, "trace": result.trace, "diagnostics": result.diagnostics, + "local_identities": result.local_identities, } diff --git a/backend/entity_detect.py b/backend/entity_detect.py index ac08b5c..7761555 100644 --- a/backend/entity_detect.py +++ b/backend/entity_detect.py @@ -1,147 +1,179 @@ -"""Entity detection for the privacy gateway. May see plaintext; generate must not.""" +"""Semantic entity detection for the privacy gateway. + +Detect may see plaintext. Generate must not. Detector output is untrusted: +spans and types only, never tokens, never a durable registry write. +""" from __future__ import annotations import json import re +import time +from dataclasses import dataclass, field +from typing import Any import placeholder_mvp # noqa: F401 from db import get_db, row_to_dict from env_loader import allows_remote_plaintext_detect from identity_store import ( - coalesce_mappings, - is_given_name_candidate, + ENTITY_TYPES, + confirmed_match_labels, is_maskable_label, - list_mappings, - normalize_label, - purge_unmaskable, - remember_mapping, + list_confirmed_identities, + masking_rows_from_confirmed, + normalize_entity_type, + record_review_proposal, ) from placeholders import PlaceholderError, resolve_template -from providers import ProviderError, complete_chat, detect_provider +from providers import ChatResult, ProviderError, complete_chat, detect_provider -DETECT_MAX_CHARS = 8000 -DETECT_TIMEOUT = 20.0 -DETECT_MAX_TOKENS = 300 +DETECT_CHUNK_CHARS = 4000 +DETECT_CHUNK_OVERLAP = 250 +DETECT_TIMEOUT = 90.0 +DETECT_MAX_TOKENS = 1024 JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL) -NAME_TITLE = re.compile(r"(?:Frau|Herr)\s+([A-ZÄÖÜ][a-zäöüß]{2,})\b") -NAME_PREP = re.compile(r"(?:mit|von|bei)\s+([A-ZÄÖÜ][a-zäöüß]{2,})\b") -_LETTER = r"A-Za-zÄÖÜäöüß" -PLACE_NAME = re.compile( - r"\b([A-ZÄÖÜ][a-zäöüß]*(?:burg|stadt|dorf|haven|ingen|heim|bach|feld))\b" +ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"}) +ALLOWED_ROOT_FIELDS = frozenset({"entities"}) +TYPE_PRIORITY = {"PERSON": 0, "PROJECT": 1, "ORG": 2, "PLACE": 3} + +ERROR_DETECT_UNAVAILABLE = "detect_provider_unavailable" +ERROR_DETECT_INCOMPLETE = "detect_incomplete" +ERROR_DETECT_INVALID = "detect_invalid_output" +ERROR_DETECT_TRUNCATED = "detect_truncated" +ERROR_DETECT_CHUNK = "detect_chunk_failed" +DETECT_DIALOGUE_FALLBACK_CODES = frozenset( + { + ERROR_DETECT_INVALID, + ERROR_DETECT_TRUNCATED, + ERROR_DETECT_INCOMPLETE, + ERROR_DETECT_CHUNK, + } ) -STOPWORDS = { - "Heute", "Gestern", "Morgen", "Ich", "Wir", "Der", "Die", "Das", "Ein", "Eine", - "Und", "Oder", "Nicht", "Kein", "Keine", "Am", "Im", "Zum", "Zur", "Mit", - "Nach", "Beim", "Über", "Unter", "Aber", "Denn", "Wenn", "Dann", "Also", - "Was", "Wer", "Wie", "Wo", "Warum", "Bitte", "Danke", "Hallo", "Space", - "Journal", "Dialog", "Kontext", "Writing", "Profile", "Tagebucheintrag", - "Relativ", "Allerdings", "Zunächst", "Danach", "Deshalb", "Trotzdem", - "Außerdem", "Schließlich", "Während", "Bevor", "Seitdem", +USER_DETECT_MESSAGES = { + ERROR_DETECT_UNAVAILABLE: ( + "Persönliche Angaben können gerade nicht geschützt werden. " + "Es wurde kein Impuls erzeugt." + ), + ERROR_DETECT_INCOMPLETE: ( + "Persönliche Angaben konnten nicht vollständig geprüft werden. " + "Es wurde kein Impuls erzeugt." + ), + ERROR_DETECT_INVALID: ( + "Persönliche Angaben konnten nicht sicher zugeordnet werden. " + "Es wurde kein Impuls erzeugt." + ), + ERROR_DETECT_TRUNCATED: ( + "Die Prüfung persönlicher Angaben wurde abgebrochen. " + "Es wurde kein Impuls erzeugt." + ), + ERROR_DETECT_CHUNK: ( + "Die Prüfung persönlicher Angaben ist fehlgeschlagen. " + "Es wurde kein Impuls erzeugt." + ), } -def _detect_prompt() -> dict: - with get_db() as conn: - row = row_to_dict( - conn.execute( - "SELECT * FROM ai_prompts WHERE slug = ? AND active = 1", - ("mvp.entity_detect",), - ).fetchone() - ) - if not row or not (row.get("template") or "").strip(): - raise ProviderError( - "detect_prompt_missing", - "Prompt mvp.entity_detect fehlt in der Konfiguration.", - ) - return row +def user_detect_message(code: str, fallback: str = "") -> str: + return USER_DETECT_MESSAGES.get(code) or fallback or "Persönliche Angaben konnten nicht geprüft werden." -def _pattern_entities(text: str, known_labels: set[str]) -> list[dict]: - found: list[dict] = [] - seen: set[str] = set() - source = text or "" - candidates: list[tuple[str, str]] = [] - for match in NAME_TITLE.finditer(source): - candidates.append((match.group(1).strip(), "PERSON:01")) - for match in NAME_PREP.finditer(source): - name = match.group(1).strip() - if is_given_name_candidate(name): - candidates.append((name, "PERSON:01")) - for match in PLACE_NAME.finditer(source): - candidates.append((match.group(1).strip(), "PLACE:CITY")) - for raw, token in candidates: - label = normalize_label(raw) or raw - if label in STOPWORDS or label in known_labels or label in seen: - continue - if not is_maskable_label(label): - continue - seen.add(label) - found.append({"text": label, "token": token}) - return found +_LETTER = r"A-Za-zÄÖÜäöüß" -def _fake_entities(text: str, known_labels: set[str]) -> list[dict]: - return _pattern_entities(text, known_labels) +class DetectError(Exception): + def __init__(self, code: str, message: str, status_code: int = 503, diagnostics: dict | None = None): + super().__init__(message) + self.code = code + self.message = message + self.status_code = status_code + self.diagnostics = diagnostics or {} -def _merge_proposals(*groups: list[dict]) -> list[dict]: - merged: list[dict] = [] - seen: set[str] = set() - for group in groups: - for item in group: - label = (item.get("text") or "").strip() - if not label or label in seen: - continue - seen.add(label) - merged.append(item) - return merged +@dataclass(frozen=True) +class DetectedSpan: + start: int + end: int + text: str + entity_type: str + chunk_index: int -def _parse_entities(raw: str) -> list[dict]: - text = (raw or "").strip() - if text.startswith("```"): - text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL) - match = JSON_BLOCK.search(text) - if not match: - return [] - try: - data = json.loads(match.group(0)) - except json.JSONDecodeError: - return [] - items = data.get("entities") if isinstance(data, dict) else data - if not isinstance(items, list): - return [] - result = [] - for item in items: - if not isinstance(item, dict): - continue - label = (item.get("text") or item.get("label") or "").strip() - token = (item.get("token") or item.get("placeholder") or "").strip() - if label: - result.append({"text": label, "token": token}) - return result +@dataclass +class DetectionStats: + detect_provider: str | None = None + detect_model: str | None = None + detect_note: str | None = None + source_chars: int = 0 + chunk_count: int = 0 + chunks_ok: int = 0 + full_detection_coverage: bool = False + entity_counts: dict[str, int] = field(default_factory=dict) + confirmed_registry_hits: int = 0 + request_local_hits: int = 0 + detect_calls: int = 0 + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + cost: float = 0.0 + detect_ms: int = 0 + abort_reason: str | None = None + + def public(self) -> dict[str, Any]: + payload = { + "detect_provider": self.detect_provider, + "detect_model": self.detect_model, + "detect_note": self.detect_note, + "source_chars": self.source_chars, + "chunk_count": self.chunk_count, + "chunks_ok": self.chunks_ok, + "full_detection_coverage": self.full_detection_coverage, + "entity_counts": dict(self.entity_counts), + "confirmed_registry_hits": self.confirmed_registry_hits, + "request_local_hits": self.request_local_hits, + "detect_calls": self.detect_calls, + "detect_prompt_tokens": self.prompt_tokens, + "detect_completion_tokens": self.completion_tokens, + "detect_total_tokens": self.total_tokens, + "detect_cost": self.cost, + "detect_ms": self.detect_ms, + } + if self.abort_reason: + payload["abort_reason"] = self.abort_reason + return payload -def _label_in_text(label: str, text: str) -> bool: - if not (label or "").strip() or not (text or "").strip(): - return False - return bool( - re.search( - rf"(? bool: - label = (item.get("text") or "").strip() - if not label or not _label_in_text(label, excerpt) or not is_maskable_label(label): - return False - token = (item.get("token") or "").upper() - if token.startswith("PLACE:") or token.startswith("ORG:"): - return True - return is_given_name_candidate(label) +_injected_spans: list[dict] | None = None +_injected_fail: DetectError | None = None +_injected_truncated: bool = False + + +def install_test_spans(spans: list[dict] | None) -> None: + """Test-only span injection. Production never calls this.""" + global _injected_spans + _injected_spans = None if spans is None else [dict(item) for item in spans] + + +def install_test_detect_failure(error: DetectError | None) -> None: + global _injected_fail + _injected_fail = error + + +def install_test_truncated(flag: bool = True) -> None: + global _injected_truncated + _injected_truncated = bool(flag) + + +def reset_detect_test_hooks() -> None: + global _injected_spans, _injected_fail, _injected_truncated + _injected_spans = None + _injected_fail = None + _injected_truncated = False def uses_llm_detect(config) -> bool: @@ -153,64 +185,508 @@ def uses_llm_detect(config) -> bool: return allows_remote_plaintext_detect() -def _llm_entities(config, excerpt: str, known_labels: set[str]) -> list[dict]: - known = ", ".join(sorted(known_labels)) or "(keine)" +def _detect_prompt() -> dict: + with get_db() as conn: + row = row_to_dict( + conn.execute( + "SELECT * FROM ai_prompts WHERE slug = ? AND active = 1", + ("mvp.entity_detect",), + ).fetchone() + ) + if not row or not (row.get("template") or "").strip(): + raise DetectError( + "detect_prompt_missing", + "Prompt mvp.entity_detect fehlt in der Konfiguration.", + ) + return row + + +def split_detect_chunks(text: str, *, chunk_chars: int | None = None, overlap: int | None = None) -> list[tuple[int, str]]: + source = text or "" + if not source: + return [(0, "")] + size = max(32, int(DETECT_CHUNK_CHARS if chunk_chars is None else chunk_chars)) + overlap_n = DETECT_CHUNK_OVERLAP if overlap is None else overlap + overlap_n = max(0, min(int(overlap_n), size // 2)) + if len(source) <= size: + return [(0, source)] + chunks: list[tuple[int, str]] = [] + start = 0 + n = len(source) + while start < n: + end = min(n, start + size) + if end < n: + window = source[start:end] + cut = max(window.rfind("\n"), window.rfind(" ")) + if cut >= size // 3: + end = start + cut + 1 + chunks.append((start, source[start:end])) + if end >= n: + break + nxt = end - overlap_n + if nxt <= start: + nxt = end + start = nxt + return chunks + + +def _span_dict(text: str, label: str, entity_type: str, *, from_index: int = 0) -> dict | None: + start = text.find(label, from_index) + if start < 0: + return None + return { + "start": start, + "end": start + len(label), + "text": label, + "entity_type": entity_type, + } + + +def _contract_fake_spans(text: str) -> list[dict]: + """Deterministic contract fixture. Not a semantic quality claim.""" + source = text or "" + found: list[dict] = [] + seen: set[tuple[int, int, str]] = set() + + def add(label: str, entity_type: str, *, require: str | None = None) -> None: + if require and require not in source: + return + start = 0 + while True: + item = _span_dict(source, label, entity_type, from_index=start) + if not item: + return + key = (item["start"], item["end"], entity_type) + if key not in seen: + seen.add(key) + found.append(item) + start = item["end"] + + if re.search(r"Sushi kam", source): + add("Sushi", "PERSON") + if re.search(r"(?i)Frau\s+Sushi", source): + add("Sushi", "PERSON") + if re.search(r"(?i)Projekt\s+Aurora", source): + add("Aurora", "PROJECT") + if re.search(r"(?i)(Organisation|Firma|bei)\s+Nordwerk", source): + add("Nordwerk", "ORG") + if re.search(r"(?i)(in|nach|aus)\s+Hamburg", source): + add("Hamburg", "PLACE") + for name in ("Anna", "Clarissa", "Hanna", "Maren"): + if re.search(rf"(? dict: + text = (raw or "").strip() + if text.startswith("```"): + text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL).strip() + match = JSON_BLOCK.search(text) + if not match: + raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.") + try: + data = json.loads(match.group(0)) + except json.JSONDecodeError as exc: + raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.") from exc + if not isinstance(data, dict): + raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort muss ein Objekt mit entities sein.") + extra = set(data.keys()) - ALLOWED_ROOT_FIELDS + if extra: + raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort enthält unerwartete Felder.") + items = data.get("entities") + if not isinstance(items, list): + raise DetectError(ERROR_DETECT_INVALID, "Detect-entities muss eine Liste sein.") + return data + + +def _label_occurrences(chunk_text: str, label: str) -> list[tuple[int, int]]: + if not label: + return [] + exact = list( + re.finditer(rf"(? tuple[int, int, str] | None: + """Trust the reported word only if it exists in this chunk. Offsets are a hint.""" + source = chunk_text or "" + label = (text or "").strip() + if not label: + return None + n = len(source) + windows: list[tuple[int, int]] = [] + if 0 <= start < end <= n: + windows.append((start, end)) + if 0 <= start < end + 1 <= n and (start, end + 1) not in windows: + windows.append((start, end + 1)) + for left, right in windows: + sliced = source[left:right] + if sliced == text: + return left, right, sliced + if sliced.strip() == label: + inner = left + (len(sliced) - len(sliced.lstrip())) + if source[inner : inner + len(label)] == label: + return inner, inner + len(label), label + hits = _label_occurrences(source, label) + if not hits: + return None + best = min(hits, key=lambda item: (abs(item[0] - start), item[0])) + return best[0], best[1], source[best[0] : best[1]] + + +def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> DetectedSpan | None: + if not isinstance(item, dict): + raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity muss ein Objekt sein.") + extra = set(item.keys()) - ALLOWED_ENTITY_FIELDS + if extra: + raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity enthält unerwartete Felder.") + missing = ALLOWED_ENTITY_FIELDS - set(item.keys()) + if missing: + raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity ist unvollständig.") + try: + start = int(item["start"]) + except (TypeError, ValueError): + start = 0 + try: + end = int(item["end"]) + except (TypeError, ValueError): + end = 0 + text = item.get("text") + if not isinstance(text, str): + raise DetectError(ERROR_DETECT_INVALID, "Detect-text fehlt.") + if not text.strip(): + return None + if "token" in item or "placeholder" in item: + raise DetectError(ERROR_DETECT_INVALID, "Detect darf keine Tokens festlegen.") + kind = normalize_entity_type(item.get("entity_type"), default="") + if not kind: + raise DetectError(ERROR_DETECT_INVALID, "Detect-entity_type ist nicht erlaubt.") + grounded = _ground_span(chunk_text, start, end, text) + if not grounded: + return None + left, right, local_text = grounded + return DetectedSpan(start=left, end=right, text=local_text, entity_type=kind, chunk_index=chunk_index) + + +def resolve_overlaps(spans: list[DetectedSpan]) -> list[DetectedSpan]: + """Longest span wins, then leftmost, then type PERSON > PROJECT > ORG > PLACE.""" + ordered = sorted( + spans, + key=lambda item: ( + -(item.end - item.start), + item.start, + TYPE_PRIORITY.get(item.entity_type, 9), + item.chunk_index, + ), + ) + kept: list[DetectedSpan] = [] + occupied: list[tuple[int, int]] = [] + for span in ordered: + if any(span.start < end and span.end > start for start, end in occupied): + continue + kept.append(span) + occupied.append((span.start, span.end)) + return sorted(kept, key=lambda item: (item.start, item.end)) + + +def _add_usage(stats: DetectionStats, usage: dict | None) -> None: + data = usage or {} + stats.prompt_tokens += int(data.get("prompt_tokens") or 0) + stats.completion_tokens += int(data.get("completion_tokens") or 0) + stats.total_tokens += int(data.get("total_tokens") or 0) + try: + stats.cost += float(data.get("cost") or 0) + except (TypeError, ValueError): + pass + + +def _llm_chunk(config, excerpt: str) -> ChatResult: prompt = resolve_template( _detect_prompt()["template"], - {"source_text": excerpt, "known_labels": known}, + {"source_text": excerpt, "known_labels": ""}, ) - result = complete_chat( + return complete_chat( config, [{"role": "user", "content": prompt}], timeout=DETECT_TIMEOUT, max_tokens=DETECT_MAX_TOKENS, + disable_context_compression=True, ) - return _parse_entities(result.content) -def detect_and_remember(profile_id: str | None, source_text: str) -> tuple[list[dict], str | None, str | None]: - if profile_id: - purge_unmaskable(profile_id) - coalesce_mappings(profile_id) - mappings = list_mappings(profile_id) if profile_id else [] +def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int) -> list[DetectedSpan]: + if (result.finish_reason or "").lower() in {"length", "max_tokens"}: + raise DetectError( + ERROR_DETECT_TRUNCATED, + "Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.", + ) + data = _parse_detect_json(result.content or "") + spans: list[DetectedSpan] = [] + for item in data.get("entities") or []: + span = validate_detected_entity(item, chunk_text, chunk_index) + if span is not None: + spans.append(span) + return spans + + +def _to_global(span: DetectedSpan, offset: int) -> DetectedSpan: + return DetectedSpan( + start=span.start + offset, + end=span.end + offset, + text=span.text, + entity_type=span.entity_type, + chunk_index=span.chunk_index, + ) + + +def _dedupe_spans(spans: list[DetectedSpan]) -> list[DetectedSpan]: + unique: dict[tuple[int, int, str, str], DetectedSpan] = {} + for span in spans: + key = (span.start, span.end, span.text, span.entity_type) + unique[key] = span + return resolve_overlaps(list(unique.values())) + + +def _next_token(used: set[str], entity_type: str) -> str: + prefix = f"{entity_type}:" + n = 1 + while True: + token = f"{prefix}{n:02d}" + if token not in used: + used.add(token) + return token + n += 1 + + +def _assign_request_tokens( + spans: list[DetectedSpan], + confirmed: list[dict], +) -> tuple[list[dict], int, int]: + used_tokens = {(item.get("token") or "").upper() for item in confirmed if item.get("token")} + by_label: dict[str, dict] = {} + request_local = 0 + confirmed_hits = 0 + mappings: list[dict] = [] + + def confirmed_for(label: str) -> dict | None: + needle = (label or "").strip().casefold() + for item in confirmed: + for candidate in confirmed_match_labels(item): + if candidate.casefold() == needle: + return item + return None + + for span in spans: + label = span.text + if not is_maskable_label(label): + continue + found = confirmed_for(label) + if found: + token = found["token"] + canonical = found["canonical_label"] + source = "confirmed_registry" + confirmed_hits += 1 + entity_type = found["entity_type"] + demask = canonical + else: + key = f"{span.entity_type}:{label.casefold()}" + if key not in by_label: + by_label[key] = { + "token": _next_token(used_tokens, span.entity_type), + "entity_type": span.entity_type, + } + request_local += 1 + token = by_label[key]["token"] + entity_type = span.entity_type + source = "request_local" + demask = label + mappings.append( + { + "token": token, + "local_label": label, + "canonical_label": demask, + "demask_label": demask, + "entity_type": entity_type, + "source": source, + "start": span.start, + "end": span.end, + } + ) + return mappings, request_local, confirmed_hits + + +def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str | None) -> tuple[list[dict], int]: if not profile_id: - return mappings, None, None - known_labels = { - normalize_label(item.get("local_label") or "") or (item.get("local_label") or "").strip() - for item in mappings - if item.get("local_label") - } - excerpt = (source_text or "")[:DETECT_MAX_CHARS] - proposals = _pattern_entities(excerpt, known_labels) - detect_name = "pattern" - detect_note = "pattern" + return mappings, 0 + already = {(item.get("local_label") or "").casefold() for item in mappings} + extra: list[dict] = [] + hits = 0 + for row in masking_rows_from_confirmed(profile_id): + label = row.get("local_label") or "" + if not label or label.casefold() in already: + continue + if not re.search(rf"(? DetectionOutcome: + """Full semantic detection of the personal generate-egress. Fail closed if incomplete.""" + started = time.perf_counter() + stats = DetectionStats(source_chars=len(source_text or "")) + if _injected_fail is not None: + stats.abort_reason = _injected_fail.code + stats.detect_ms = int((time.perf_counter() - started) * 1000) + raise DetectError( + _injected_fail.code, + _injected_fail.message, + _injected_fail.status_code, + {**(_injected_fail.diagnostics or {}), **stats.public()}, + ) config = detect_provider() - if config and config.mode == "fake": - detect_name = config.name - detect_note = "fake" - elif uses_llm_detect(config): - try: - proposals = _merge_proposals( - proposals, - [item for item in _llm_entities(config, excerpt, known_labels) if _acceptable_proposal(item, excerpt)], - ) - detect_name = config.name - detect_note = "local_llm" if config.local else "remote_llm" - except PlaceholderError as exc: - raise ProviderError(exc.code, exc.message) from exc - elif config and config.mode == "http" and not config.local: - detect_note = "pattern; remote_detect_blocked_production" - for item in proposals: - label = (item.get("text") or "").strip() - core = normalize_label(label) - if not label or not core or core in known_labels: - continue - if not _acceptable_proposal(item, excerpt): - continue - try: - remember_mapping(profile_id, label, item.get("token")) - except ValueError: - continue - known_labels.add(core) - coalesce_mappings(profile_id) - return list_mappings(profile_id), detect_name, detect_note + if not config: + stats.abort_reason = ERROR_DETECT_UNAVAILABLE + stats.detect_ms = int((time.perf_counter() - started) * 1000) + raise DetectError( + ERROR_DETECT_UNAVAILABLE, + "Semantische Detection ist nicht konfiguriert. Generate wird nicht freigegeben.", + diagnostics=stats.public(), + ) + if config.mode == "http" and not config.local and not allows_remote_plaintext_detect(): + stats.abort_reason = "remote_detect_blocked_production" + stats.detect_ms = int((time.perf_counter() - started) * 1000) + raise DetectError( + ERROR_DETECT_UNAVAILABLE, + "Externes Klartext-Detect ist im Produktivmodus nicht zulässig, und ein lokales Detect-Modell fehlt.", + diagnostics=stats.public(), + ) + stats.detect_provider = config.name + stats.detect_model = config.model + chunks = split_detect_chunks(source_text or "") + stats.chunk_count = len(chunks) + collected: list[DetectedSpan] = [] + try: + for index, (offset, chunk_text) in enumerate(chunks): + if _injected_truncated: + raise DetectError( + ERROR_DETECT_TRUNCATED, + "Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.", + ) + if _injected_spans is not None: + raw_items = [ + item + for item in _injected_spans + if int(item.get("start") or 0) >= offset + and int(item.get("end") or 0) <= offset + len(chunk_text) + ] + local_items = [] + for item in raw_items: + local = dict(item) + local["start"] = int(item["start"]) - offset + local["end"] = int(item["end"]) - offset + local_items.append(local) + result = ChatResult( + content=json.dumps({"entities": local_items}), + model=config.model, + usage={}, + finish_reason="stop", + ) + stats.detect_note = "injected" + elif config.mode == "fake": + fake_items = _contract_fake_spans(chunk_text) + result = ChatResult( + content=json.dumps({"entities": fake_items}), + model=config.model, + usage={}, + finish_reason="stop", + ) + stats.detect_note = "fake" + elif uses_llm_detect(config): + try: + result = _llm_chunk(config, chunk_text) + except PlaceholderError as exc: + raise DetectError(exc.code, exc.message) from exc + except ProviderError as exc: + raise DetectError( + ERROR_DETECT_CHUNK, + "Ein Detect-Chunk ist fehlgeschlagen. Generate wird nicht freigegeben.", + exc.status_code, + ) from exc + stats.detect_note = "local_llm" if config.local else "remote_llm" + stats.detect_model = result.model or stats.detect_model + else: + raise DetectError( + ERROR_DETECT_UNAVAILABLE, + "Semantische Detection ist nicht verfügbar. Generate wird nicht freigegeben.", + ) + stats.detect_calls += 1 + _add_usage(stats, result.usage) + spans = _entities_from_result(result, chunk_text, index) + collected.extend(_to_global(span, offset) for span in spans) + stats.chunks_ok += 1 + except DetectError as exc: + stats.abort_reason = exc.code + stats.full_detection_coverage = False + stats.detect_ms = int((time.perf_counter() - started) * 1000) + raise DetectError( + exc.code, + exc.message, + exc.status_code, + {**(exc.diagnostics or {}), **stats.public()}, + ) from exc + if stats.chunks_ok != stats.chunk_count: + stats.abort_reason = ERROR_DETECT_INCOMPLETE + stats.detect_ms = int((time.perf_counter() - started) * 1000) + raise DetectError( + ERROR_DETECT_INCOMPLETE, + "Detection hat nicht alle Chunks geprüft. Generate wird nicht freigegeben.", + diagnostics=stats.public(), + ) + merged = _dedupe_spans(collected) + confirmed = list_confirmed_identities(profile_id) if profile_id else [] + mappings, request_local, confirmed_from_spans = _assign_request_tokens(merged, confirmed) + mappings, extra_confirmed = _merge_confirmed_safety_net(source_text or "", mappings, profile_id) + stats.request_local_hits = request_local + stats.confirmed_registry_hits = confirmed_from_spans + extra_confirmed + counts: dict[str, int] = {kind: 0 for kind in ENTITY_TYPES} + for item in mappings: + kind = (item.get("entity_type") or "PERSON").upper() + counts[kind] = counts.get(kind, 0) + 1 + stats.entity_counts = {key: value for key, value in counts.items() if value} + stats.full_detection_coverage = True + stats.detect_ms = int((time.perf_counter() - started) * 1000) + if profile_id: + seen_proposals: set[tuple[str, str]] = set() + for item in mappings: + if item.get("source") != "request_local": + continue + key = ((item.get("local_label") or ""), (item.get("entity_type") or "")) + if key in seen_proposals: + continue + seen_proposals.add(key) + record_review_proposal(profile_id, item.get("local_label") or "", item.get("entity_type") or "") + local_identities = [ + { + "local_label": item.get("local_label"), + "token": item.get("token"), + "entity_type": item.get("entity_type"), + "demask_label": item.get("demask_label") or item.get("local_label"), + "source": item.get("source"), + } + for item in mappings + ] + return DetectionOutcome(mappings=mappings, stats=stats, local_identities=local_identities) diff --git a/backend/entity_detect_eval.py b/backend/entity_detect_eval.py new file mode 100644 index 0000000..45b0e70 --- /dev/null +++ b/backend/entity_detect_eval.py @@ -0,0 +1,113 @@ +"""Opt-in live semantic detect comparison. Not part of production generate. + +Usage from backend/: + python entity_detect_eval.py + python entity_detect_eval.py --live + +Synthetic sentences only. No personal data. Live quality stays unconfirmed +until an explicit --live run succeeds. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +CASES = ( + { + "id": "common_noun", + "text": "Ich ging auf den Balkon und setzte mich.", + "expect_empty_types": True, + "note": "Allgemeines Substantiv, kein Eigenname.", + }, + { + "id": "food_vs_person", + "text": "Ich aß Sushi. Sushi kam ins Wohnzimmer.", + "note": "Dasselbe Wort als Gericht und als mögliche Person.", + }, + { + "id": "person", + "text": "Ich traf Anna am Nachmittag.", + "expect_types": {"PERSON"}, + "note": "Klarer Personenname.", + }, + { + "id": "project", + "text": "Ich arbeitete am privaten Projekt Aurora.", + "expect_types": {"PROJECT"}, + "note": "Privates Projekt, kein Allerweltsgegenstand.", + }, + { + "id": "place_org", + "text": "Ich war in Hamburg und sprach mit der Organisation Nordwerk.", + "expect_types": {"PLACE", "ORG"}, + "note": "Ort und Organisation.", + }, +) + + +def _summarize(entities: list[dict]) -> dict: + types = sorted({(item.get("entity_type") or "").upper() for item in entities}) + return { + "count": len(entities), + "types": types, + "has_labels": False, + } + + +def run_fake() -> dict: + from entity_detect import _contract_fake_spans + + rows = [] + for case in CASES: + entities = _contract_fake_spans(case["text"]) + rows.append({"id": case["id"], "note": case["note"], "entities": _summarize(entities), "mode": "fake"}) + return {"mode": "fake", "live_quality": "unconfirmed", "cases": rows} + + +def run_live() -> dict: + if os.environ.get("KANSHO_FAKE_DETECT"): + raise SystemExit("Live-Detect verweigert, solange KANSHO_FAKE_DETECT gesetzt ist.") + from entity_detect import detect_personal_egress + + rows = [] + for case in CASES: + started = time.perf_counter() + outcome = detect_personal_egress(None, case["text"]) + elapsed = int((time.perf_counter() - started) * 1000) + types = sorted({(item.get("entity_type") or "") for item in outcome.mappings}) + rows.append( + { + "id": case["id"], + "note": case["note"], + "types": types, + "request_local_hits": outcome.stats.request_local_hits, + "detect_calls": outcome.stats.detect_calls, + "detect_ms": elapsed, + "coverage": outcome.stats.full_detection_coverage, + "detect_tokens": outcome.stats.total_tokens, + "detect_cost": outcome.stats.cost, + } + ) + return {"mode": "live", "live_quality": "ran", "cases": rows} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Synthetic detect evaluation. No personal data.") + parser.add_argument("--live", action="store_true", help="Call the configured detect provider.") + args = parser.parse_args() + payload = run_live() if args.live else run_fake() + print(json.dumps(payload, ensure_ascii=False, indent=2)) + if payload["mode"] != "live": + print("Live-Qualität: noch nicht bestätigt. Explizit: python entity_detect_eval.py --live") + + +if __name__ == "__main__": + main() diff --git a/backend/identity_store.py b/backend/identity_store.py index 4fc4656..f6e7ab8 100644 --- a/backend/identity_store.py +++ b/backend/identity_store.py @@ -1,17 +1,20 @@ -"""Local identity mappings. Class A; never part of an external prompt.""" +"""Confirmed identity registry. Class A; never part of an external prompt. + +Detect output is not stored here. Request-local detections live only on the +request manifest. Review proposals are unconfirmed and never used for masking. +""" from __future__ import annotations import json import re import uuid -from functools import lru_cache -from pathlib import Path from db import get_db, row_to_dict -PROMPTS_SEED_PATH = Path(__file__).resolve().parent / "config" / "prompts.seed.json" - -TOKEN_RE = re.compile(r"^(SELF|PERSON:[A-Z][A-Z0-9_]{0,24}|PLACE:[A-Z][A-Z0-9_]{0,24}|ORG:[A-Z][A-Z0-9_]{0,24})$") +TOKEN_RE = re.compile( + r"^(SELF|PERSON:[A-Z][A-Z0-9_]{0,24}|PLACE:[A-Z][A-Z0-9_]{0,24}|" + r"ORG:[A-Z][A-Z0-9_]{0,24}|PROJECT:[A-Z][A-Z0-9_]{0,24})$" +) MIN_LABEL_LEN = 4 TIME_RE = re.compile(r"\d|:\d|\buhr\b|\bminuten\b|\bstunde\b", re.IGNORECASE) DETERMINERS = { @@ -22,135 +25,57 @@ DETERMINERS = { "unser", "unsere", "euer", "eure", } KINSHIP = {"frau", "herr", "mann", "kind", "tochter", "sohn", "partner", "partnerin"} -UNMASKABLE = { +STRUCTURAL_UNMASKABLE = { "ich", "du", "er", "sie", "es", "wir", "ihr", "mich", "mir", "mein", "meine", "dir", "dich", "uns", "euch", "sein", "seine", "ihre", "heute", "gestern", "morgen", "user", "assistant", "self", "kontext", - "himmel", "luft", "regen", "sonne", "wind", "meer", "tee", "kaffee", - "balkon", "laden", "brot", "brote", "bad", "gasse", "gassen", "weg", - "ort", "ortes", "stadt", "zimmer", "haus", - "frau", "herr", "mann", "kind", "lektion", "bootstour", "frühstück", - "tagebuch", "omlett", "haferflocken", "speiseplan", - "impuls", "tatsache", "motivation", "erzählung", "operation", - "uhr", "zeit", "stunde", "minute", "tag", "tages", "abend", "nacht", - "warm", "klar", "draußen", "menge", "teil", "stück", - "relativ", "allerdings", "zunächst", "danach", "deshalb", "trotzdem", - "außerdem", "schließlich", "während", "bevor", "seitdem", "deswegen", - "überlegen", "feststellen", "frage", "antwort", - "markt", "kirschen", "hafen", "wetter", "vormittag", "nachmittag", - "anreise", "details", "absatz", "essen", "licht", - "zwiebeln", "zwiebel", "tomaten", "tomate", "paprika", - "omelett", "trockenobst", "trockenfrüchte", "glutenfrei", - "schiff", "boot", "zweimaster", "fähre", } -FUNCTION_STEMS = ("letzt", "nächst") -NOUN_SUFFIXES = ("heit", "keit", "schaft", "tum") -COLLECTIVE_NOUN = re.compile(r"^ge[a-zäöüß]+öse$", re.IGNORECASE) RESERVED_TOKEN_SUFFIXES = {"KURZ", "NAME", "TEXT", "LABEL", "EXAMPLE", "FOO", "BAR"} +ENTITY_TYPES = ("PERSON", "PLACE", "ORG", "PROJECT") +STATUS_CONFIRMED = "confirmed" +STATUS_INACTIVE = "inactive" +STATUS_LEGACY = "legacy_review_required" +ORIGIN_USER_CONFIRMED = "user_confirmed" +ORIGIN_LOCAL = "local_authoritative" +ORIGIN_LEGACY = "legacy_auto" +PROPOSAL_UNCONFIRMED = "unconfirmed" +PROPOSAL_DISMISSED = "dismissed" +PROPOSAL_ORIGIN = "detect_proposal" -@lru_cache(maxsize=1) -def instruction_vocab() -> frozenset[str]: - """Words from Kanshō prompts must never become identity labels.""" - items = json.loads(PROMPTS_SEED_PATH.read_text(encoding="utf-8")) - words: set[str] = set() - for item in items: - if (item.get("slug") or "") == "mvp.entity_detect": - continue - template = item.get("template") or "" - template = re.sub(r"\{\{[^}]+\}\}|\[\[[^\]]+\]\]", " ", template) - for word in re.findall(r"[A-Za-zÄÖÜäöüß]{4,}", template): - words.add(word.casefold()) - return frozenset(words) - - -def is_function_label(label: str) -> bool: - word = (label or "").strip().casefold() - if not word: - return False - for stem in FUNCTION_STEMS: - if word == stem or word.startswith(stem): - return True - return False - - -def is_common_noun_shape(label: str) -> bool: - word = (label or "").strip() - if COLLECTIVE_NOUN.match(word): - return True - lower = word.casefold() - if lower.endswith("ung") and len(lower) > 6: - return True - return any(lower.endswith(suffix) for suffix in NOUN_SUFFIXES) - - -def is_given_name_candidate(label: str) -> bool: - core = normalize_label(label) - if not core or " " in core: - return False - if is_function_label(core) or is_common_noun_shape(core): - return False - return is_maskable_label(core) +# Backward-compatible alias. Not a detection stopword list. +UNMASKABLE = STRUCTURAL_UNMASKABLE def normalize_label(label: str) -> str: - words = re.findall(r"[A-Za-zÄÖÜäöüß0-9.:]+", label or "") + words = re.findall(r"[0-9A-Za-zÀ-žÄÖÜäöüß.:]+", label or "", re.UNICODE) keep = [ word for word in words - if word.lower() not in DETERMINERS - and word.lower() not in KINSHIP - and word.lower() not in UNMASKABLE + if word.lower() not in DETERMINERS and word.lower() not in KINSHIP ] return " ".join(keep).strip() def is_maskable_label(label: str) -> bool: + """Structural gate only: pronouns, roles, clocks. Not a food/weather word list.""" core = normalize_label(label) if len(core) < MIN_LABEL_LEN: return False - if core.lower() in UNMASKABLE: - return False - if is_function_label(core) or is_common_noun_shape(core): - return False - if core.casefold() in instruction_vocab(): + if core.lower() in STRUCTURAL_UNMASKABLE: return False if TIME_RE.search(core): return False if re.fullmatch(r"user|assistant|self", core, re.IGNORECASE): return False - if not re.search(r"[A-Za-zÄÖÜäöüß]", core): + if not re.search(r"[A-Za-zÀ-žÄÖÜäöüß]", core): return False return True -def purge_unmaskable(profile_id: str) -> int: - removed = 0 - with get_db() as conn: - rows = conn.execute( - "SELECT id, local_label FROM identity_mappings WHERE profile_id = ?", - (profile_id,), - ).fetchall() - for row in rows: - if is_maskable_label(row["local_label"]): - continue - conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],)) - removed += 1 - return removed - - -def list_mappings(profile_id: str) -> list[dict]: - with get_db() as conn: - rows = conn.execute( - "SELECT token, local_label FROM identity_mappings WHERE profile_id = ? ORDER BY created", - (profile_id,), - ).fetchall() - return [row_to_dict(row) for row in rows] - - def _token_suffix(token: str) -> str: raw = (token or "").upper() - return raw.split(":")[-1] if ":" in raw else raw + return raw.split(":")[-1] if ":" in token else raw def normalize_token(token: str | None) -> str | None: @@ -165,10 +90,121 @@ def normalize_token(token: str | None) -> str | None: return raw -def _same_core(left: str, right: str) -> bool: - a = normalize_label(left).casefold() - b = normalize_label(right).casefold() - return bool(a and b and a == b) +def normalize_entity_type(value: str | None, *, default: str = "PERSON") -> str | None: + raw = (value or "").strip().upper() + if not raw: + raw = default + if raw not in ENTITY_TYPES: + return None + return raw + + +def parse_aliases(raw) -> list[str]: + if isinstance(raw, list): + items = raw + else: + try: + items = json.loads(raw or "[]") + except (TypeError, json.JSONDecodeError): + items = [] + seen: set[str] = set() + result: list[str] = [] + for item in items: + label = (item or "").strip() + if not label or label.casefold() in seen or not is_maskable_label(label): + continue + seen.add(label.casefold()) + result.append(label) + return result + + +def _row_public(row: dict | None) -> dict | None: + if not row: + return None + canonical = (row.get("canonical_label") or row.get("local_label") or "").strip() + aliases = [ + item + for item in parse_aliases(row.get("aliases_json")) + if item.casefold() != canonical.casefold() + ] + return { + "id": row.get("id"), + "token": row.get("token"), + "local_label": canonical, + "canonical_label": canonical, + "entity_type": (row.get("entity_type") or "PERSON").upper(), + "status": row.get("status") or STATUS_LEGACY, + "origin": row.get("origin") or ORIGIN_LEGACY, + "aliases": aliases, + "created": row.get("created"), + "updated": row.get("updated"), + "confirmed_at": row.get("confirmed_at"), + } + + +def _select_all(profile_id: str) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + """ + SELECT * FROM identity_mappings + WHERE profile_id = ? + ORDER BY created + """, + (profile_id,), + ).fetchall() + return [_row_public(row_to_dict(row)) for row in rows] + + +def list_registry(profile_id: str, *, include_inactive: bool = True) -> list[dict]: + rows = [item for item in _select_all(profile_id) if item] + if include_inactive: + return rows + return [item for item in rows if item.get("status") == STATUS_CONFIRMED] + + +def list_confirmed_identities(profile_id: str) -> list[dict]: + return [item for item in _select_all(profile_id) if item and item.get("status") == STATUS_CONFIRMED] + + +def list_mappings(profile_id: str) -> list[dict]: + """Confirmed identities only. Legacy and inactive rows are not a masking dictionary.""" + return list_confirmed_identities(profile_id) + + +def confirmed_match_labels(item: dict) -> list[str]: + labels = [] + canonical = (item.get("canonical_label") or item.get("local_label") or "").strip() + if canonical: + labels.append(canonical) + for alias in item.get("aliases") or []: + if alias and alias.casefold() != canonical.casefold(): + labels.append(alias) + return labels + + +def masking_rows_from_confirmed(profile_id: str) -> list[dict]: + """One masking row per confirmed canonical label or alias. Demask uses canonical.""" + rows: list[dict] = [] + for item in list_confirmed_identities(profile_id): + token = (item.get("token") or "").strip() + canonical = (item.get("canonical_label") or "").strip() + entity_type = (item.get("entity_type") or "PERSON").upper() + if not token or not canonical: + continue + for label in confirmed_match_labels(item): + if not is_maskable_label(label): + continue + rows.append( + { + "token": token, + "local_label": label, + "canonical_label": canonical, + "demask_label": canonical, + "entity_type": entity_type, + "source": "confirmed_registry", + } + ) + return rows def find_existing_mapping(existing: list[dict], label: str) -> dict | None: @@ -176,62 +212,15 @@ def find_existing_mapping(existing: list[dict], label: str) -> dict | None: if not core: return None for item in existing: - stored = (item.get("local_label") or "").strip() - if stored.casefold() == label.casefold() or _same_core(stored, label): - return item + candidates = [item.get("canonical_label") or "", item.get("local_label") or ""] + candidates.extend(item.get("aliases") or []) + for stored in candidates: + stored_core = normalize_label(stored).casefold() + if stored.casefold() == label.casefold() or (stored_core and stored_core == core): + return item return None -def _pick_keeper(rows: list) -> dict: - def sort_key(row: dict) -> tuple: - suffix = _token_suffix(row["token"] or "") - reserved = 1 if suffix in RESERVED_TOKEN_SUFFIXES else 0 - numeric = 0 if suffix.isdigit() else 1 - return (reserved, numeric, row["created"] or "", row["id"] or "") - - return min(rows, key=sort_key) - - -def coalesce_mappings(profile_id: str) -> int: - """One token per normalized name. Template tokens like PERSON:KURZ lose against PERSON:08.""" - merged = 0 - with get_db() as conn: - rows = [ - row_to_dict(row) - for row in conn.execute( - """ - SELECT id, token, local_label, created - FROM identity_mappings - WHERE profile_id = ? - ORDER BY created - """, - (profile_id,), - ).fetchall() - ] - groups: dict[str, list[dict]] = {} - for row in rows: - core = normalize_label(row.get("local_label") or "") - if not core or not is_maskable_label(core): - conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],)) - merged += 1 - continue - groups.setdefault(core.casefold(), []).append({**row, "core": core}) - for items in groups.values(): - keeper = _pick_keeper(items) - core = items[0]["core"] - if (keeper.get("local_label") or "") != core: - conn.execute( - "UPDATE identity_mappings SET local_label = ? WHERE id = ?", - (core, keeper["id"]), - ) - for row in items: - if row["id"] == keeper["id"]: - continue - conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],)) - merged += 1 - return merged - - def _next_generic(existing: list[dict], prefix: str) -> str: used = set() for item in existing: @@ -246,37 +235,368 @@ def _next_generic(existing: list[dict], prefix: str) -> str: return f"{prefix}{n:02d}" -def remember_mapping(profile_id: str, local_label: str, suggested_token: str | None = None) -> dict: - label = normalize_label(local_label) +def _prefix_for_type(entity_type: str) -> str: + return f"{entity_type}:" + + +def confirm_identity( + profile_id: str, + canonical_label: str, + *, + entity_type: str = "PERSON", + token: str | None = None, + aliases: list[str] | None = None, + origin: str = ORIGIN_USER_CONFIRMED, + identity_id: str | None = None, +) -> dict: + label = normalize_label(canonical_label) or (canonical_label or "").strip() if not label or not is_maskable_label(label): raise ValueError("empty_label") - existing = list_mappings(profile_id) - found = find_existing_mapping(existing, label) + kind = normalize_entity_type(entity_type) + if not kind: + raise ValueError("invalid_entity_type") + alias_list = parse_aliases(aliases or []) + alias_list = [item for item in alias_list if item.casefold() != label.casefold()] + existing = _select_all(profile_id) + found = None + if identity_id: + found = next((item for item in existing if item.get("id") == identity_id), None) + if found is None: + found = find_existing_mapping(existing, label) + chosen_token = normalize_token(token) if found: - stored = (found.get("local_label") or "").strip() - if stored != label: - with get_db() as conn: - conn.execute( - """ - UPDATE identity_mappings - SET local_label = ? - WHERE profile_id = ? AND token = ? - """, - (label, profile_id, found["token"]), - ) - found = {**found, "local_label": label} - return found - token = normalize_token(suggested_token) - if token and any((item.get("token") or "").upper() == token for item in existing): - token = None - if not token: - token = _next_generic(existing, "PERSON:") + chosen_token = found["token"] + elif chosen_token and any((item.get("token") or "").upper() == chosen_token for item in existing): + chosen_token = None + if not chosen_token: + chosen_token = _next_generic(existing, _prefix_for_type(kind)) + row_id = (found or {}).get("id") or str(uuid.uuid4()) with get_db() as conn: conn.execute( """ - INSERT INTO identity_mappings (id, profile_id, token, local_label) - VALUES (?, ?, ?, ?) + INSERT INTO identity_mappings ( + id, profile_id, token, local_label, canonical_label, entity_type, + status, origin, aliases_json, confirmed_at, updated + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now')) + ON CONFLICT(id) DO UPDATE SET + token = excluded.token, + local_label = excluded.local_label, + canonical_label = excluded.canonical_label, + entity_type = excluded.entity_type, + status = excluded.status, + origin = excluded.origin, + aliases_json = excluded.aliases_json, + confirmed_at = datetime('now'), + updated = datetime('now') """, - (str(uuid.uuid4()), profile_id, token, label), + ( + row_id, + profile_id, + chosen_token, + label, + label, + kind, + STATUS_CONFIRMED, + origin if origin in {ORIGIN_USER_CONFIRMED, ORIGIN_LOCAL} else ORIGIN_USER_CONFIRMED, + json.dumps(alias_list, ensure_ascii=False), + ), ) - return {"token": token, "local_label": label} + return next(item for item in _select_all(profile_id) if item.get("id") == row_id) + + +def remember_mapping(profile_id: str, local_label: str, suggested_token: str | None = None) -> dict: + """Explicit local confirmation helper for tests and admin. Not the detect path.""" + token = normalize_token(suggested_token) + entity_type = "PERSON" + if token and ":" in token: + entity_type = token.split(":", 1)[0] + return confirm_identity( + profile_id, + local_label, + entity_type=entity_type if entity_type in ENTITY_TYPES else "PERSON", + token=suggested_token, + origin=ORIGIN_LOCAL, + ) + + +def update_identity( + profile_id: str, + identity_id: str, + *, + canonical_label: str | None = None, + entity_type: str | None = None, + aliases: list[str] | None = None, + status: str | None = None, +) -> dict: + rows = _select_all(profile_id) + found = next((item for item in rows if item.get("id") == identity_id), None) + if not found: + raise ValueError("identity_missing") + label = found["canonical_label"] + if canonical_label is not None: + label = normalize_label(canonical_label) or canonical_label.strip() + if not label or not is_maskable_label(label): + raise ValueError("empty_label") + kind = found["entity_type"] + if entity_type is not None: + kind = normalize_entity_type(entity_type) + if not kind: + raise ValueError("invalid_entity_type") + alias_list = found.get("aliases") or [] + if aliases is not None: + alias_list = [item for item in parse_aliases(aliases) if item.casefold() != label.casefold()] + next_status = found["status"] + if status is not None: + if status not in {STATUS_CONFIRMED, STATUS_INACTIVE, STATUS_LEGACY}: + raise ValueError("invalid_status") + next_status = status + with get_db() as conn: + conn.execute( + """ + UPDATE identity_mappings + SET local_label = ?, canonical_label = ?, entity_type = ?, + aliases_json = ?, status = ?, + confirmed_at = CASE WHEN ? = 'confirmed' THEN datetime('now') ELSE confirmed_at END, + origin = CASE WHEN ? = 'confirmed' AND origin = 'legacy_auto' THEN 'user_confirmed' ELSE origin END, + updated = datetime('now') + WHERE id = ? AND profile_id = ? + """, + ( + label, + label, + kind, + json.dumps(alias_list, ensure_ascii=False), + next_status, + next_status, + next_status, + identity_id, + profile_id, + ), + ) + return next(item for item in _select_all(profile_id) if item.get("id") == identity_id) + + +def deactivate_identity(profile_id: str, identity_id: str) -> dict: + return update_identity(profile_id, identity_id, status=STATUS_INACTIVE) + + +def delete_identity(profile_id: str, identity_id: str) -> None: + with get_db() as conn: + conn.execute( + "DELETE FROM identity_mappings WHERE id = ? AND profile_id = ?", + (identity_id, profile_id), + ) + + +def list_review_proposals(profile_id: str, *, include_dismissed: bool = False) -> list[dict]: + with get_db() as conn: + rows = conn.execute( + """ + SELECT * FROM identity_review_proposals + WHERE profile_id = ? + ORDER BY last_seen DESC + """, + (profile_id,), + ).fetchall() + result = [] + for row in rows: + item = row_to_dict(row) + if not include_dismissed and item.get("status") != PROPOSAL_UNCONFIRMED: + continue + result.append(item) + return result + + +def record_review_proposal(profile_id: str, observed_label: str, entity_type: str) -> None: + """Local unconfirmed note only. Never used for masking or later detection skip.""" + label = (observed_label or "").strip() + kind = normalize_entity_type(entity_type) + if not profile_id or not label or not kind or not is_maskable_label(label): + return + if find_existing_mapping(list_confirmed_identities(profile_id), label): + return + with get_db() as conn: + existing = row_to_dict( + conn.execute( + """ + SELECT id FROM identity_review_proposals + WHERE profile_id = ? AND observed_label = ? AND entity_type = ? + """, + (profile_id, label, kind), + ).fetchone() + ) + if existing: + conn.execute( + """ + UPDATE identity_review_proposals + SET last_seen = datetime('now'), status = CASE + WHEN status = 'dismissed' THEN status ELSE 'unconfirmed' END + WHERE id = ? + """, + (existing["id"],), + ) + return + conn.execute( + """ + INSERT INTO identity_review_proposals + (id, profile_id, observed_label, entity_type, status, origin) + VALUES (?, ?, ?, ?, ?, ?) + """, + (str(uuid.uuid4()), profile_id, label, kind, PROPOSAL_UNCONFIRMED, PROPOSAL_ORIGIN), + ) + + +def confirm_review_proposal(profile_id: str, proposal_id: str, **overrides) -> dict: + with get_db() as conn: + row = row_to_dict( + conn.execute( + "SELECT * FROM identity_review_proposals WHERE id = ? AND profile_id = ?", + (proposal_id, profile_id), + ).fetchone() + ) + if not row: + raise ValueError("proposal_missing") + confirmed = confirm_identity( + profile_id, + overrides.get("canonical_label") or row["observed_label"], + entity_type=overrides.get("entity_type") or row["entity_type"], + aliases=overrides.get("aliases"), + origin=ORIGIN_USER_CONFIRMED, + ) + with get_db() as conn: + conn.execute( + "DELETE FROM identity_review_proposals WHERE id = ? AND profile_id = ?", + (proposal_id, profile_id), + ) + return confirmed + + +def dismiss_review_proposal(profile_id: str, proposal_id: str) -> None: + with get_db() as conn: + conn.execute( + """ + UPDATE identity_review_proposals + SET status = ?, last_seen = datetime('now') + WHERE id = ? AND profile_id = ? + """, + (PROPOSAL_DISMISSED, proposal_id, profile_id), + ) + + +def delete_review_proposal(profile_id: str, proposal_id: str) -> None: + with get_db() as conn: + conn.execute( + "DELETE FROM identity_review_proposals WHERE id = ? AND profile_id = ?", + (proposal_id, profile_id), + ) + + +def purge_unmaskable(profile_id: str) -> int: + removed = 0 + with get_db() as conn: + rows = conn.execute( + "SELECT id, local_label, canonical_label FROM identity_mappings WHERE profile_id = ?", + (profile_id,), + ).fetchall() + for row in rows: + label = row["canonical_label"] or row["local_label"] + if is_maskable_label(label): + continue + conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],)) + removed += 1 + return removed + + +def coalesce_mappings(profile_id: str) -> int: + """One confirmed/legacy row per normalized canonical name.""" + merged = 0 + with get_db() as conn: + rows = [ + row_to_dict(row) + for row in conn.execute( + """ + SELECT id, token, local_label, canonical_label, entity_type, status, origin, created + FROM identity_mappings + WHERE profile_id = ? + ORDER BY created + """, + (profile_id,), + ).fetchall() + ] + groups: dict[str, list[dict]] = {} + for row in rows: + core = normalize_label(row.get("canonical_label") or row.get("local_label") or "") + if not core or not is_maskable_label(core): + conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],)) + merged += 1 + continue + groups.setdefault(core.casefold(), []).append({**row, "core": core}) + for items in groups.values(): + keeper = min( + items, + key=lambda row: ( + 0 if (row.get("status") == STATUS_CONFIRMED) else 1, + 0 if _token_suffix(row.get("token") or "").isdigit() else 1, + row.get("created") or "", + row.get("id") or "", + ), + ) + core = items[0]["core"] + if (keeper.get("canonical_label") or keeper.get("local_label") or "") != core: + conn.execute( + """ + UPDATE identity_mappings + SET local_label = ?, canonical_label = ?, updated = datetime('now') + WHERE id = ? + """, + (core, core, keeper["id"]), + ) + for row in items: + if row["id"] == keeper["id"]: + continue + conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],)) + merged += 1 + return merged + + +def migrate_legacy_identity_rows(conn) -> int: + """Mark pre-existing auto-mappings as review-required. Never auto-confirm them.""" + names = {row["name"] for row in conn.execute("PRAGMA table_info(identity_mappings)").fetchall()} + if "status" not in names: + return 0 + conn.execute( + """ + UPDATE identity_mappings + SET updated = COALESCE(NULLIF(updated, ''), created) + WHERE updated IS NULL OR updated = '' + """ + ) + marked = 0 + rows = conn.execute( + "SELECT id, local_label, canonical_label, status, origin FROM identity_mappings" + ).fetchall() + for row in rows: + canonical = (row["canonical_label"] or row["local_label"] or "").strip() + status = row["status"] or "" + origin = row["origin"] or "" + updates: list[str] = [] + values: list[str] = [] + if not (row["canonical_label"] or "").strip() and canonical: + updates.append("canonical_label = ?") + values.append(canonical) + if not status or (status == STATUS_CONFIRMED and origin == ORIGIN_LEGACY): + updates.append("status = ?") + values.append(STATUS_LEGACY) + if not origin: + updates.append("origin = ?") + values.append(ORIGIN_LEGACY) + if not updates: + continue + values.append(row["id"]) + conn.execute( + f"UPDATE identity_mappings SET {', '.join(updates)}, updated = datetime('now') WHERE id = ?", + values, + ) + marked += 1 + return marked diff --git a/backend/journal_editorial.py b/backend/journal_editorial.py new file mode 100644 index 0000000..a94c87b --- /dev/null +++ b/backend/journal_editorial.py @@ -0,0 +1,201 @@ +"""Journal-adapter editorial policy. Not a general provenance or privacy rule. + +Fact fidelity is not wording fidelity. Editorial mode is chosen locally, without +a second model call. Historical texts are style references, never today's facts. +""" +from __future__ import annotations + +import hashlib +import re +from difflib import SequenceMatcher + +from journal_body import plain_text +from writing_profile_schema import is_meta_style_text, recency_weight +from writing_profile_store import ( + clip_field, + get_profile, + has_confirmed_profile, + list_style_sources, +) + +PROSE_EDIT = "prose_edit" +NOTES_TO_JOURNAL = "notes_to_journal" +EDITORIAL_MODES = (PROSE_EDIT, NOTES_TO_JOURNAL) + +STYLE_EXAMPLE_MAX = 2 +STYLE_EXAMPLE_CHARS = 900 +MIN_EXAMPLE_CHARS = 40 + +INSTRUCTIONS = { + PROSE_EDIT: ( + "Modus prose_edit: Der Rohtext ist bereits erzählerisch. " + "Gute Formulierungen bewahren. Rechtschreibung, Grammatik und Zeichensetzung " + "korrigieren. Holprige Stellen glätten, Wiederholungen reduzieren, Absätze und " + "Übergänge verbessern. Die persönliche Schreibstimme anwenden. " + "Keine unnötige vollständige Neufassung erzwingen." + ), + NOTES_TO_JOURNAL: ( + "Modus notes_to_journal: Die Quellen sind Stichpunkte, Kurztexte oder Fragmente. " + "Daraus zusammenhängende Journalprosa bilden. Nur sprachlich nötige Verbindungen " + "herstellen. Keine neuen Tatsachen, Ursachen oder Bewertungen ergänzen. " + "Die Stichpunkte nicht inklusive ihrer Fehler hintereinanderkopieren." + ), +} + +EMPTY_STYLE_EXAMPLES = ( + "Keine historischen Stilbeispiele. Es gilt nur WRITING_PROFILE " + "(oder der neutrale Journalstil)." +) + + +def choose_editorial_mode(user_bodies: list[str]) -> str: + """MVP mode choice. No classifier model. + + A source block counts as already narrative when it contains `.`, `!` or `?`. + Mixed default: `prose_edit` when at least half of the non-empty blocks are + narrative; otherwise `notes_to_journal`. + """ + blocks = [(item or "").strip() for item in user_bodies if (item or "").strip()] + if not blocks: + return NOTES_TO_JOURNAL + narrative = sum(1 for block in blocks if any(mark in block for mark in ".!?")) + if narrative * 2 >= len(blocks): + return PROSE_EDIT + return NOTES_TO_JOURNAL + + +def editorial_instructions(mode: str) -> str: + return INSTRUCTIONS.get(mode) or INSTRUCTIONS[PROSE_EDIT] + + +def lexical_similarity(left: str, right: str) -> float: + """Diagnostic only. Must not reject a draft or trigger a retry.""" + a = re.sub(r"\s+", " ", (left or "").strip().lower()) + b = re.sub(r"\s+", " ", (right or "").strip().lower()) + if not a or not b: + return 0.0 + return round(SequenceMatcher(None, a, b).ratio(), 3) + + +def narration_sources_text(artifact: dict) -> str: + """Present attested day facts to the model. Not a wording template, not JSON.""" + parts: list[str] = [] + for item in artifact.get("sources") or []: + if not isinstance(item, dict): + continue + text = (item.get("text") or "").strip() + if not text: + continue + source_id = (item.get("source_id") or "").strip() + prefix = f"[{source_id}]\n" if source_id else "" + parts.append(prefix + text) + if parts: + return "\n\n".join(parts) + from journal_reconstruct import claim_texts + + fallback = [str(part).strip() for part in claim_texts(artifact) if str(part).strip()] + return "\n\n".join(fallback) + + +def format_style_examples(examples: list[dict]) -> str: + if not examples: + return EMPTY_STYLE_EXAMPLES + lines = [ + "Nur Ton, Rhythmus und sprachliche Entscheidungen. " + "Ereignisse, Personen, Orte und Bewertungen aus diesen Beispielen " + "sind keine Tatsachen des heutigen Eintrags und dürfen nicht übernommen werden." + ] + for index, item in enumerate(examples, start=1): + kind = item.get("kind") or "style" + when = (item.get("occurred_at") or "")[:10] + header = f"Beispiel {index} ({kind}" + (f", {when}" if when else "") + "):" + lines.append(header) + lines.append((item.get("excerpt") or "").strip()) + return "\n".join(part for part in lines if part).strip() + + +def _content_digest(text: str) -> str: + body = re.sub(r"\s+", " ", plain_text(text or "")).strip().lower() + return hashlib.sha256(body.encode("utf-8")).hexdigest() + + +def _eligible_excerpt(body: str, limit: int) -> str: + text = plain_text(body or "").strip() + if len(text) < MIN_EXAMPLE_CHARS or is_meta_style_text(text): + return "" + return clip_field(text, limit) + + +def select_journal_style_examples( + profile_id: str, + *, + exclude_dates: list[str] | None = None, + exclude_entry_ids: list[str] | None = None, + max_n: int = STYLE_EXAMPLE_MAX, + max_chars: int = STYLE_EXAMPLE_CHARS, +) -> list[dict]: + """Final accepted journal texts first, then imports, then trait excerpts. + + Current-day sources are excluded. Dialogue is never a style authority. + """ + excluded_dates = {(item or "")[:10] for item in (exclude_dates or []) if item} + excluded_ids = {item for item in (exclude_entry_ids or []) if item} + ranked = list_style_sources(profile_id) + buckets = [ + ("journal_entry", ranked.get("journal_entry") or []), + ("imported_text", ranked.get("imported_text") or []), + ] + picked: list[dict] = [] + seen: set[str] = set() + + def consider(kind: str, item: dict, excerpt: str) -> None: + if len(picked) >= max_n or not excerpt: + return + digest = _content_digest(excerpt) + if digest in seen: + return + seen.add(digest) + picked.append( + { + "kind": kind, + "excerpt": excerpt, + "occurred_at": item.get("occurred_at") or item.get("created"), + "entry_id": item.get("entry_id"), + "weight": item.get("weight") or recency_weight(item.get("occurred_at")), + } + ) + + for kind, rows in buckets: + ordered = sorted( + rows, + key=lambda row: ( + -float(row.get("weight") or 0), + -recency_weight(row.get("occurred_at") or row.get("created")), + ), + ) + for item in ordered: + if len(picked) >= max_n: + return picked + entry_id = (item.get("entry_id") or "").strip() + if entry_id and entry_id in excluded_ids: + continue + when = (item.get("occurred_at") or "")[:10] + if when and when in excluded_dates: + continue + consider(kind, item, _eligible_excerpt(item.get("body") or "", max_chars)) + + if picked: + return picked + + profile = get_profile(profile_id) + if not has_confirmed_profile(profile_id): + return picked + for trait in profile.get("traits") or []: + if len(picked) >= max_n: + break + for ref in trait.get("exemplars") or []: + excerpt = _eligible_excerpt(ref.get("excerpt") or "", max_chars) + consider("trait_exemplar", ref, excerpt) + if len(picked) >= max_n: + break + return picked diff --git a/backend/journal_eval.py b/backend/journal_eval.py new file mode 100644 index 0000000..69aff20 --- /dev/null +++ b/backend/journal_eval.py @@ -0,0 +1,273 @@ +"""Opt-in journal quality comparison. Not part of production generate. + +Usage from backend/: + python journal_eval.py # synthetic, fake provider, contract only + python journal_eval.py --live --profile-id + +Never writes private texts into the repository. Live quality stays unconfirmed +until an explicit --live run succeeds. +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from journal_editorial import lexical_similarity + +VARIANT_BASELINE = "baseline_simple" +VARIANT_PREVIOUS = "kansho_previous" +VARIANT_CURRENT = "kansho_current" + +BASELINE_TEMPLATE = ( + "Überarbeite diesen Rohtext zu einem ansprechenden Tagebucheintrag in meinem Stil.\n\n" + "{{reconstruction}}\n" +) + +PREVIOUS_TEMPLATE = ( + "Schreibe einen eigenständigen, gut lesbaren Tagebucheintrag in der Ich-Form von [[SELF]].\n" + "Verwende ausschließlich die verifizierten Informationen aus sources[].text. " + "Der Quellwortlaut ist keine Ausgabevorlage.\n" + "Keine Dialogabschrift. Unsicherheiten im Wortlaut erhalten.\n" + "Writing Profile:\n{{writing_profile}}\n\n" + "Verified Artifact:\n{{reconstruction}}\n" +) + +SYNTHETIC_PROSE = ( + "ich bin dan zum markt gegangen und da war es zimlich voll. " + "vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. " + "die rote tasche lag im auto." +) +SYNTHETIC_NOTES = "- markt\n- kirschen kaufen\n- später hafen" +SYNTHETIC_TYPOS = ("zimlich", "dan zum") + + +def _words(text: str) -> list[str]: + return re.findall(r"[A-Za-zÄÖÜäöüß]+", (text or "").lower()) + + +def _sentences(text: str) -> list[str]: + parts = [item.strip() for item in re.split(r"(?<=[.!?])\s+", text or "") if item.strip()] + return parts or ([text.strip()] if (text or "").strip() else []) + + +def score_output(source: str, output: str, *, profile: str = "", typos: tuple[str, ...] = ()) -> dict[str, Any]: + """Diagnostic scores. Not a production gate and not a live-quality certificate.""" + body = output or "" + src_words = set(_words(source)) + out_words = set(_words(body)) + sentences = _sentences(body) + paragraphs = [item for item in re.split(r"\n\s*\n", body) if item.strip()] + avg_len = round(sum(len(item.split()) for item in sentences) / max(1, len(sentences)), 2) + transitions = len(re.findall(r"(?i)\b(?:danach|später|dann|zuerst|schließlich)\b", body)) + profile_words = set(_words(profile)) + style_overlap = round(len(out_words & profile_words) / max(1, len(profile_words)), 3) if profile_words else 0.0 + kept = round(len(src_words & out_words) / max(1, len(src_words)), 3) + extra = sorted(out_words - src_words - profile_words) + lost = sorted(src_words - out_words) + remaining_typos = [item for item in typos if item.lower() in body.lower()] + return { + "spelling_typos_remaining": remaining_typos, + "spelling_typos_fixed": [item for item in typos if item.lower() not in body.lower()], + "readability_sentence_count": len(sentences), + "readability_avg_sentence_words": avg_len, + "readability_paragraphs": max(1, len(paragraphs)), + "transitions": transitions, + "style_token_overlap": style_overlap, + "fact_token_keep": kept, + "new_info_tokens": extra[:24], + "lost_info_tokens": lost[:24], + "lexical_similarity": lexical_similarity(source, body), + } + + +def variant_templates() -> dict[str, str]: + from engine import load_active_prompt + + current = (load_active_prompt("mvp.journal_generate") or {}).get("template") or "" + return { + VARIANT_BASELINE: BASELINE_TEMPLATE, + VARIANT_PREVIOUS: PREVIOUS_TEMPLATE, + VARIANT_CURRENT: current, + } + + +def run_variant( + name: str, + template: str, + context: dict[str, str], + profile_id: str, + *, + live: bool, +) -> dict[str, Any]: + from engine import execute_prompt + from prompt_budget import plan_journal_budget + from providers import generate_provider + from model_catalog import resolve_generate_metadata + + prompt = { + "id": f"eval-{name}", + "slug": "mvp.journal_generate" if name == VARIANT_CURRENT else f"eval.{name}", + "prompt_type": "base", + "required_feature": "ai_calls", + "template": template, + } + started = time.perf_counter() + config = generate_provider() + window = resolve_generate_metadata(config) if config else None + budget = plan_journal_budget(window, purpose="journal_generate") if window else None + result = execute_prompt( + prompt, + profile_id, + purpose="journal_generate", + data_class="B", + context=context, + max_tokens=budget.reserved_output_tokens if budget else 1024, + disable_context_compression=True, + budget=budget, + ) + elapsed_ms = int((time.perf_counter() - started) * 1000) + diag = result.get("diagnostics") or {} + return { + "variant": name, + "live": live, + "content": result.get("content") or "", + "prompt_tokens": diag.get("prompt_tokens"), + "completion_tokens": diag.get("completion_tokens"), + "total_tokens": diag.get("total_tokens"), + "cost": diag.get("cost"), + "runtime_ms": elapsed_ms, + "model": (result.get("trace") or {}).get("model") or diag.get("actual_model"), + "fake_provider": not live, + } + + +def synthetic_context(source: str) -> dict[str, str]: + artifact = { + "kind": "verified_artifact", + "coverage": "all_selected_sources", + "sources": [{"source_id": "u1", "role": "user", "text": source}], + } + from journal_editorial import ( + NOTES_TO_JOURNAL, + PROSE_EDIT, + choose_editorial_mode, + editorial_instructions, + narration_sources_text, + ) + from writing_profile_store import NEUTRAL_JOURNAL_STYLE + + mode = choose_editorial_mode([source]) + return { + "reconstruction": narration_sources_text(artifact) or source, + "writing_profile": NEUTRAL_JOURNAL_STYLE, + "style_examples": "Keine historischen Stilbeispiele.", + "editorial_mode": mode, + "editorial_instructions": editorial_instructions(mode), + "existing_text": "", + "space_title": "Eval", + "expected_mode": PROSE_EDIT if mode == PROSE_EDIT else NOTES_TO_JOURNAL, + } + + +def compare_synthetic(*, live: bool = False, profile_id: str | None = None) -> dict[str, Any]: + from placeholders import resolve_template + import placeholder_mvp # noqa: F401 + from privacy_gateway import _fake_complete + + templates = variant_templates() + source = SYNTHETIC_PROSE + context = synthetic_context(source) + rows = [] + for name, template in templates.items(): + if live: + if not profile_id: + raise SystemExit("--live requires --profile-id") + row = run_variant(name, template, context, profile_id=profile_id, live=True) + else: + rendered = resolve_template(template, context) + started = time.perf_counter() + content = _fake_complete("journal_generate", rendered) + row = { + "variant": name, + "live": False, + "content": content, + "prompt_tokens": None, + "completion_tokens": None, + "total_tokens": None, + "cost": None, + "runtime_ms": int((time.perf_counter() - started) * 1000), + "model": "fake", + "fake_provider": True, + } + row["scores"] = score_output( + source, + row["content"], + profile=context.get("writing_profile") or "", + typos=SYNTHETIC_TYPOS, + ) + row["source"] = "synthetic" + rows.append(row) + return { + "live": live, + "live_quality_confirmed": False, + "note": ( + "Fake-Provider-Lauf: beweist den Vergleichsvertrag, nicht echte Modellprosa." + if not live + else "Live-Lauf über das Privacy Gateway. Qualitative Bewertung bleibt manuell." + ), + "source_kind": "synthetic", + "variants": rows, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Journal narration quality comparison (opt-in).") + parser.add_argument("--live", action="store_true", help="Call the configured generate provider. Costs money.") + parser.add_argument("--profile-id", help="Required with --live. Uses the Privacy Gateway.") + parser.add_argument("--json", action="store_true", help="Print JSON instead of text.") + args = parser.parse_args(argv) + if args.live: + import os + + if (os.environ.get("KANSHO_FAKE_PROVIDER") or "").strip() in {"1", "true", "yes"}: + print("Refusing --live while KANSHO_FAKE_PROVIDER is set.", file=sys.stderr) + return 2 + if not args.profile_id: + print("--live requires --profile-id", file=sys.stderr) + return 2 + else: + import os + + os.environ.setdefault("KANSHO_FAKE_PROVIDER", "1") + os.environ.setdefault("KANSHO_FAKE_DETECT", "1") + from db import init_db + + init_db() + report = compare_synthetic(live=bool(args.live), profile_id=args.profile_id) + if args.json: + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 + print(report["note"]) + if not report["live"]: + print("Live-Qualität: noch nicht bestätigt.") + for item in report["variants"]: + scores = item["scores"] + print( + f"{item['variant']}: similarity={scores['lexical_similarity']} " + f"keep={scores['fact_token_keep']} typos_left={scores['spelling_typos_remaining']} " + f"tokens={item.get('total_tokens')} cost={item.get('cost')} ms={item['runtime_ms']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/journal_generate.py b/backend/journal_generate.py index e8a446e..495b7bb 100644 --- a/backend/journal_generate.py +++ b/backend/journal_generate.py @@ -1,17 +1,26 @@ """Explicit journal draft generation. Never overwrites the current user entry.""" from __future__ import annotations +import re + from context_builder import assemble_text, build_internal_context from dialogue_store import StoreError, list_conversations_for_day, list_messages -from engine import EngineError, execute_prompt, load_active_prompt +from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt from journal_policy import require_explicit_generate, source_conversation_ids -from identity_store import list_mappings +from identity_store import is_maskable_label, list_mappings from journal_body import clean_title +from journal_editorial import ( + choose_editorial_mode, + editorial_instructions, + format_style_examples, + lexical_similarity, + narration_sources_text, + select_journal_style_examples, +) from journal_reconstruct import ( assign_source_ids, claim_texts, local_verified_artifact, - reconstruction_from_model, reconstruction_text, ) from journal_shape import shape_journal @@ -19,11 +28,25 @@ from journal_store import current_draft, current_entries, get_day, insert_draft from model_catalog import resolve_generate_metadata from prompt_budget import ( JournalBudgetError, - estimate_tokens, + assert_input_fits, plan_journal_budget, ) +from privacy_gateway import ( + GENERIC_PLACEHOLDER_INNER, + canonical_token, + identity_label_pattern, + identity_occurrence_count, + is_identity_mention, +) from providers import generate_provider -from writing_profile_store import compile_style_signals, remember_dialogue_style +from retrieval import retrieve +from writing_profile_store import remember_dialogue_style + + +IDENTITY_PLACEHOLDER = re.compile( + r"\[\[\s*(?:SELF|PERSON:[^\]]+|PLACE:[^\]]+|ORG:[^\]]+|PROJECT:[^\]]+|…|\.{2,})\s*\]\]", + re.IGNORECASE, +) def _message_ids(profile_id: str, conversation_ids: list[str]) -> list[str]: @@ -53,17 +76,85 @@ def _local_narration(reconstruction: dict) -> tuple[str, str]: return "Ein Tag", body +def unattested_journal_content( + text: str, + source_user: list[str], + mappings: list[dict], + active_tokens: list[str] | None = None, +) -> str | None: + """Journal-only: historical names not in the sources are unattested facts, not a privacy leak. + + Title and body are checked together. Placeholders must stay consistent across both. + """ + body = text or "" + allowed = {canonical_token(token).upper().replace(" ", "") for token in (active_tokens or [])} + allowed.add("SELF") + for match in IDENTITY_PLACEHOLDER.finditer(body): + token = canonical_token(match.group(0)).upper().replace(" ", "") + if token == "SELF": + continue + if token in GENERIC_PLACEHOLDER_INNER or token not in allowed: + return "unattested_placeholder" + sources = "\n".join(source_user or []) + for item in mappings or []: + label = (item.get("local_label") or "").strip() + token = (item.get("token") or "").strip() + if not label or not token or not is_maskable_label(label): + continue + if identity_occurrence_count(sources, label, token) > 0: + continue + for match in identity_label_pattern(label).finditer(body): + if is_identity_mention(body, match.start(), match.end(), token): + return "unattested_identity" + return None + + def _identity_leak_result(purpose: str, exc: EngineError) -> dict: + diag = getattr(exc, "diagnostics", None) or {} return { "content": "", "trace": { "purpose": purpose, "guard": "identity_leak_blocked", + "log": list(diag.get("log") or []), + "response_validation_retry": diag.get("response_validation_retry"), }, - "diagnostics": getattr(exc, "diagnostics", None) or {}, + "diagnostics": diag, } +def _log_items(source: dict | BaseException | None) -> list[dict]: + if source is None: + return [] + if isinstance(source, BaseException): + diag = getattr(source, "diagnostics", None) or {} + return list(diag.get("log") or []) + if not isinstance(source, dict): + return [] + if source.get("log"): + return list(source["log"]) + trace = source.get("trace") or {} + if trace.get("log"): + return list(trace["log"]) + diag = source.get("diagnostics") or {} + return list(diag.get("log") or []) + + +def _stamp_log(run_log: list[dict], stage: str, items: list[dict]) -> None: + for item in items: + row = dict(item) + row.setdefault("stage", stage) + run_log.append(row) + + +def _event(run_log: list[dict], stage: str, kind: str, **fields) -> None: + row = {"stage": stage, "kind": kind} + for key, value in fields.items(): + if value is not None and value != "": + row[key] = value + run_log.append(row) + + def _raise_budget(exc: JournalBudgetError) -> None: raise EngineError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc @@ -78,11 +169,28 @@ def _stage_trace(result: dict, fallback_purpose: str) -> dict: return trace -def _day_messages_from_context(context: dict) -> list[dict]: - for item in context.get("items") or []: - if item.get("type") == "day_messages": - return list(item.get("messages") or []) - return [] +def _local_stage_trace(artifact: dict, *, source_count: int) -> dict: + return { + "purpose": "local_source_artifact", + "layer": "journalquellen", + "provider": None, + "model": None, + "coverage": artifact.get("coverage") or "all_selected_sources", + "status": "local_ok", + "stage1": "local_ok", + "source_count": source_count, + "intern": reconstruction_text(artifact), + "budget": { + "purpose": "local_source_artifact", + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "cost": 0, + "budget_ok": True, + "status": "local", + "context_compression": "not_applicable", + }, + } def _existing_text(profile_id: str, journal_day_id: str) -> str: @@ -97,6 +205,46 @@ def _existing_text(profile_id: str, journal_day_id: str) -> str: return "" +def pack_narration_context( + prompt: dict, + budget, + assembled: dict[str, str], + *, + style_examples: str, + existing_text: str, + include_existing: bool, +) -> tuple[dict[str, str], list[str]]: + """Drop optional blocks locally if they overflow. Never drop day sources or the profile. + + No extra model call. Order: try all, then drop style examples, then existing text. + """ + dropped: list[str] = [] + attempts = [ + (style_examples, existing_text if include_existing else ""), + ("", existing_text if include_existing else ""), + ("", ""), + ] + last_error: JournalBudgetError | None = None + for examples, existing in attempts: + candidate = dict(assembled) + candidate["style_examples"] = examples + candidate["existing_text"] = existing + rendered = preview_prompt(prompt, candidate)["rendered"] + try: + assert_input_fits(budget, rendered) + if not examples and style_examples: + dropped.append("style_examples") + if include_existing and existing_text and not existing: + dropped.append("existing_text") + return candidate, dropped + except JournalBudgetError as exc: + last_error = exc + continue + if last_error: + raise last_error + return assembled, dropped + + def generate_draft( profile_id: str, journal_day_id: str, @@ -124,75 +272,51 @@ def generate_draft( ) try: window = resolve_generate_metadata(config) - reconstruct_budget = plan_journal_budget(window, purpose="journal_reconstruct") narrate_budget = plan_journal_budget(window, purpose="journal_generate") except JournalBudgetError as exc: _raise_budget(exc) - reconstruct_prompt = load_active_prompt("mvp.journal_reconstruct") - static_tokens = estimate_tokens(reconstruct_prompt.get("template") or "") - available_for_day = reconstruct_budget.available_input_tokens - static_tokens - if available_for_day < 256: - raise EngineError( - "prompt_budget_exceeded", - "Der Tagesdialog ist für eine sichere Verarbeitung zu umfangreich. " - "Kanshō hat nichts stillschweigend aus der Mitte entfernt.", - 422, - diagnostics=reconstruct_budget.as_diagnostics(), - ) - try: - reconstruct_context = build_internal_context( + source_messages = retrieve( profile_id, - space_id=day["space_id"], - journal_day_id=journal_day_id, - purpose="journal_reconstruct", - conversation_ids=selected, - conversation_id=selected[0] if selected else None, - day_spec={ + { + "kind": "day_messages", + "journal_day_id": journal_day_id, + "conversation_ids": selected, "overflow": "abort", - "max_estimated_tokens": available_for_day, }, ) + reconstruction = local_verified_artifact(source_messages) except JournalBudgetError as exc: _raise_budget(exc) - reconstruct_assembled = assemble_text(reconstruct_context) - source_messages = assign_source_ids(_day_messages_from_context(reconstruct_context)) source_user = [ message.get("body") or "" - for message in source_messages - if message.get("role") == "user" + for message in assign_source_ids(source_messages) + if message.get("role") == "user" and (message.get("body") or "").strip() ] - reconstruct_result = {"trace": {"purpose": "journal_reconstruct"}, "content": "", "diagnostics": {}} - try: - reconstruct_result = execute_prompt( - reconstruct_prompt, - profile_id, - purpose="journal_reconstruct", - data_class="B", - context=reconstruct_assembled, - max_tokens=reconstruct_budget.reserved_output_tokens, - disable_context_compression=True, - budget=reconstruct_budget, - ) - reconstruction, stage1 = reconstruction_from_model( - reconstruct_result.get("content") or "", - source_messages, - ) - except EngineError as exc: - if exc.code != "response_validation_failed": - raise - reconstruction = local_verified_artifact(source_messages) - stage1 = { - "stage1": "local_fallback", - "reason": "identity_leak_blocked", - "model_rejected": exc.code, - } - reconstruct_result = _identity_leak_result("journal_reconstruct", exc) - except JournalBudgetError as exc: - _raise_budget(exc) + editorial_mode = choose_editorial_mode(source_user) + style_example_rows = select_journal_style_examples( + profile_id, + exclude_dates=[day.get("calendar_date") or ""], + ) + style_examples = format_style_examples(style_example_rows) + run_log: list[dict] = [] + _event( + run_log, + "local_source_artifact", + "stage1_result", + stage1="local_ok", + coverage=reconstruction.get("coverage") or "all_selected_sources", + status="local_ok", + source_count=len(source_user), + editorial_mode=editorial_mode, + ) + reconstruct_result = { + "trace": _local_stage_trace(reconstruction, source_count=len(source_user)), + "content": reconstruction_text(reconstruction), + } - signals = compile_style_signals(source_user) + narrate_prompt = load_active_prompt("mvp.journal_generate") narrate_context = build_internal_context( profile_id, space_id=day["space_id"], @@ -202,13 +326,28 @@ def generate_draft( conversation_ids=selected, existing_text=existing_text, conversation_id=selected[0] if selected else None, - reconstruction=reconstruction_text(reconstruction), + reconstruction=narration_sources_text(reconstruction), + style_examples=style_examples, + editorial_mode=editorial_mode, + editorial_instructions=editorial_instructions(editorial_mode), ) assembled = assemble_text(narrate_context) - profile = assembled.get("writing_profile") or "" - if signals and signals not in profile and len(profile) < 3500: - assembled["writing_profile"] = "\n\n".join(part for part in (profile, signals) if part) - narrate_prompt = load_active_prompt("mvp.journal_generate") + try: + assembled, dropped = pack_narration_context( + narrate_prompt, + narrate_budget, + assembled, + style_examples=style_examples, + existing_text=existing_text, + include_existing=include_existing, + ) + except JournalBudgetError as exc: + _raise_budget(exc) + if dropped: + _event(run_log, "journal_generate", "budget_pack", dropped=",".join(dropped)) + shape_source = "model" + mappings = list_mappings(profile_id) + narrate_result: dict = {} try: narrate_result = execute_prompt( narrate_prompt, @@ -220,20 +359,70 @@ def generate_draft( disable_context_compression=True, budget=narrate_budget, ) + _stamp_log(run_log, "journal_generate", _log_items(narrate_result)) title, body = _split_title(narrate_result.get("content") or "") + seen = {(item.get("local_label") or "").casefold() for item in mappings} + for item in narrate_result.get("local_identities") or []: + label = (item.get("local_label") or "").strip() + if not label or label.casefold() in seen: + continue + mappings.append(item) + seen.add(label.casefold()) + active_tokens = list((narrate_result.get("diagnostics") or {}).get("active_tokens") or []) + combined = f"{title}\n\n{body}".strip() + unattested = unattested_journal_content(combined, source_user, mappings, active_tokens) + if unattested: + title, body = _local_narration(reconstruction) + shape_source = "fallback" + narrate_result = { + **narrate_result, + "content": f"{title}\n\n{body}".strip(), + "trace": { + **(narrate_result.get("trace") or {}), + "guard": "unattested_content_blocked", + "reason": unattested, + }, + } + _event( + run_log, + "journal_generate", + "narration_result", + status="local_fallback", + reason=unattested, + ) + else: + _event( + run_log, + "journal_generate", + "narration_result", + status="model", + editorial_mode=editorial_mode, + lexical_similarity=lexical_similarity("\n".join(source_user), body), + ) except EngineError as exc: + _stamp_log(run_log, "journal_generate", _log_items(exc)) if exc.code != "response_validation_failed": - raise + diag = dict(exc.diagnostics or {}) + diag["log"] = run_log + raise EngineError(exc.code, exc.message, exc.status_code, diag) from exc title, body = _local_narration(reconstruction) + shape_source = "fallback" narrate_result = _identity_leak_result("journal_generate", exc) narrate_result["content"] = f"{title}\n\n{body}".strip() + _event(run_log, "journal_generate", "narration_result", status="local_fallback", reason="identity_leak_blocked") person_labels = [ (item.get("local_label") or "").strip() - for item in list_mappings(profile_id) + for item in mappings if (item.get("local_label") or "").strip() and str(item.get("token") or "").upper().lstrip("[").startswith("PERSON:") ] - title, body = shape_journal(title, body, source_user, person_labels=person_labels) + title, body = shape_journal( + title, + body, + source_user, + person_labels=person_labels, + source=shape_source, + ) before_entries = {item["id"]: item.get("current_version_id") for item in current_entries(profile_id, journal_day_id)} draft = insert_draft( profile_id, @@ -248,11 +437,19 @@ def generate_draft( previous = before_entries.get(item["id"]) if previous is not None and previous != item.get("current_version_id"): raise StoreError("policy_violation", "Generate darf die Nutzerfassung nicht verändern", 500) - reconstruct_trace = _stage_trace(reconstruct_result, "journal_reconstruct") - reconstruct_trace.update(stage1) + reconstruct_trace = _stage_trace(reconstruct_result, "local_source_artifact") + reconstruct_trace.setdefault("purpose", "local_source_artifact") + reconstruct_trace.setdefault("status", "local_ok") + reconstruct_trace.setdefault("stage1", "local_ok") narrate_trace = _stage_trace(narrate_result, "journal_generate") + narrate_trace["editorial_mode"] = editorial_mode + if shape_source == "model": + narrate_trace["lexical_similarity"] = lexical_similarity("\n".join(source_user), body) + draft["run_log"] = run_log draft["trace"] = { **narrate_trace, + "editorial_mode": editorial_mode, + "log": run_log, "stages": [reconstruct_trace, narrate_trace], } return draft diff --git a/backend/journal_opening.py b/backend/journal_opening.py index de32a25..09f4adb 100644 --- a/backend/journal_opening.py +++ b/backend/journal_opening.py @@ -12,6 +12,7 @@ from context_builder import assemble_text, build_internal_context from dialogue_store import StoreError, append_message, get_conversation, list_conversations_for_day, list_messages from dialogue_turn import OPERATIONS, needs_repair, parse_turn_payload, repair_note from engine import EngineError, execute_prompt, load_active_prompt +from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES from journal_store import get_day from retrieval import retrieve @@ -260,7 +261,7 @@ def start_journal_opening(profile_id: str, conversation_id: str) -> dict: "guard": "local_fallback", } except EngineError as exc: - if exc.code != "response_validation_failed": + if exc.code != "response_validation_failed" and exc.code not in DETECT_DIALOGUE_FALLBACK_CODES: raise return _store_opening( profile_id, @@ -275,7 +276,7 @@ def start_journal_opening(profile_id: str, conversation_id: str) -> dict: "operation": "fortfuehren", "label": OPERATIONS["fortfuehren"], "parsed": False, - "guard": "identity_leak_blocked", + "guard": "identity_leak_blocked" if exc.code == "response_validation_failed" else "detect_blocked", }, "trace": result.get("trace") if result else None, }, diff --git a/backend/journal_shape.py b/backend/journal_shape.py index 85391f1..6bf3008 100644 --- a/backend/journal_shape.py +++ b/backend/journal_shape.py @@ -46,6 +46,7 @@ def _turns(user_bodies: list[str]) -> list[str]: def is_verbatim_join(body: str, user_bodies: list[str]) -> bool: + """Diagnostic helper. Must not be used to overwrite accepted model text.""" turns = _turns(user_bodies) if not turns or not (body or "").strip(): return False @@ -172,14 +173,17 @@ def shape_journal( body: str, user_bodies: list[str], person_labels: list[str] | None = None, + *, + source: str = "model", ) -> tuple[str, str]: + """Local post-shape. Never replaces accepted model wording with source turns.""" turns = _turns(user_bodies) text = (body or "").strip() heading = (title or "").strip() text = _strip_roles(text) - if is_verbatim_join(text, turns) and len(turns) > 1: - text = "\n\n".join(turns) - else: + if source != "fallback": + text = paragraphize(text) + elif "\n\n" not in text: text = paragraphize(text) if person_labels: text = naturalize_person_mentions(text, person_labels, turns) diff --git a/backend/placeholder_mvp.py b/backend/placeholder_mvp.py index 7c72982..d86c475 100644 --- a/backend/placeholder_mvp.py +++ b/backend/placeholder_mvp.py @@ -24,6 +24,30 @@ register( resolver=_from_ctx("writing_profile"), ) ) +register( + Placeholder( + key="style_examples", + description="Historische Stilreferenzen. Keine Tatsachen des aktuellen Tages.", + data_class="B", + resolver=_from_ctx("style_examples"), + ) +) +register( + Placeholder( + key="editorial_mode", + description="Lokaler redaktioneller Journalmodus. Kein zweiter Modellaufruf.", + data_class="C", + resolver=_from_ctx("editorial_mode"), + ) +) +register( + Placeholder( + key="editorial_instructions", + description="Modusabhängige Journalinstruktion. Kein allgemeiner Provenienzvertrag.", + data_class="C", + resolver=_from_ctx("editorial_instructions"), + ) +) register( Placeholder( key="interaction_hint", @@ -59,7 +83,7 @@ register( register( Placeholder( key="known_labels", - description="Bereits gemappte lokale Labels, damit Detection sie nicht neu vorschlägt.", + description="Legacy-Platzhalter. Der aktuelle Detect-Prompt nutzt keine Ausnahmeliste bekannter Labels.", data_class="C", resolver=_from_ctx("known_labels"), ) diff --git a/backend/privacy_gateway.py b/backend/privacy_gateway.py index 287809f..e429d48 100644 --- a/backend/privacy_gateway.py +++ b/backend/privacy_gateway.py @@ -6,9 +6,10 @@ from datetime import datetime, timezone from typing import Any import contextvars import re +import time -from entity_detect import detect_and_remember -from identity_store import KINSHIP, is_maskable_label, list_mappings +from entity_detect import DetectError, detect_personal_egress, reset_detect_test_hooks, user_detect_message +from identity_store import KINSHIP, is_maskable_label from journal_reconstruct import claim_texts, fake_reconstruction, is_dialogue_role_line from prompt_budget import ( ERROR_PROVIDER_CONTEXT_LENGTH, @@ -17,6 +18,7 @@ from prompt_budget import ( USER_MESSAGES, assert_input_fits, merge_usage, + sum_usages, ) from pronoun_bind import bind_user_lines from providers import ChatResult, ProviderError, complete_chat, generate_provider @@ -47,8 +49,37 @@ COMPACT_DIAGNOSTIC_KEYS = ( "budget_ok", "abort_reason", "status", + "available_mapping_count", + "active_mapping_count", + "masked_occurrence_count", + "active_tokens", + "active_entity_types", + "leak_tokens", + "leak_entity_types", + "pre_egress_validation", + "response_validation", + "response_validation_retry", + "full_detection_coverage", + "detect_provider", + "detect_model", + "detect_note", + "source_chars", + "chunk_count", + "chunks_ok", + "entity_counts", + "confirmed_registry_hits", + "request_local_hits", + "detect_calls", + "detect_prompt_tokens", + "detect_completion_tokens", + "detect_total_tokens", + "detect_cost", + "detect_ms", + "generate_called", ) +ERROR_EGRESS_VALIDATION = "egress_validation_failed" + class PrivacyGatewayError(Exception): def __init__(self, code: str, message: str, status_code: int = 503, diagnostics: dict | None = None): @@ -76,6 +107,7 @@ class GatewayResult: content: str | None = None diagnostics: dict[str, Any] = field(default_factory=dict) trace: dict[str, Any] | None = None + local_identities: list[dict[str, Any]] = field(default_factory=list) checked_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) @@ -83,6 +115,39 @@ ALLOWED_CLASSES = {"A", "B", "C"} LOCAL_ONLY_CLASS = "A" +@dataclass(frozen=True) +class ActiveReplacement: + """One mapping that was actually replaced as identity in this request.""" + + token: str + entity_type: str + occurrence_count: int + local_label: str + demask_label: str = "" + + def restore_label(self) -> str: + return self.demask_label or self.local_label + + +@dataclass +class MaskingManifest: + """Request-scoped mask result. local_label stays on this object only.""" + + masked_text: str + available_mapping_count: int + replacements: tuple[ActiveReplacement, ...] = () + restore_by_token: dict[str, str] = field(default_factory=dict) + + def public_summary(self) -> dict[str, Any]: + return { + "available_mapping_count": self.available_mapping_count, + "active_mapping_count": len(self.replacements), + "masked_occurrence_count": sum(item.occurrence_count for item in self.replacements), + "active_tokens": [item.token for item in self.replacements], + "active_entity_types": sorted({item.entity_type for item in self.replacements}), + } + + def compact_diagnostics(data: dict[str, Any] | None) -> dict[str, Any]: payload = {} for key in COMPACT_DIAGNOSTIC_KEYS: @@ -91,6 +156,63 @@ def compact_diagnostics(data: dict[str, Any] | None) -> dict[str, Any]: return payload +_LOG_FORBIDDEN_KEYS = frozenset( + { + "local_label", + "label", + "masked_text", + "rendered", + "intern", + "egress", + "raw", + "reply", + "content", + "messages", + "prompt", + "mapping_table", + "replacements", + } +) + + +def _log_event(events: list[dict[str, Any]], started: float, kind: str, **fields: Any) -> None: + """Compact run log. No prompts, no answers, no mapping labels.""" + row: dict[str, Any] = { + "kind": kind, + "t_ms": int((time.perf_counter() - started) * 1000), + } + for key, value in fields.items(): + if value is None or value == "" or key in _LOG_FORBIDDEN_KEYS: + continue + row[key] = value + events.append(row) + + +def _usage_bits(usage: dict[str, Any] | None) -> dict[str, Any]: + data = usage or {} + bits: dict[str, Any] = {} + for key in ("prompt_tokens", "completion_tokens", "total_tokens", "cost"): + if data.get(key) is not None: + bits[key] = data[key] + return bits + + +def _raise_with_log( + exc: PrivacyGatewayError, + events: list[dict[str, Any]], + *, + purpose: str | None = None, + extra: dict[str, Any] | None = None, +) -> None: + diagnostics = dict(exc.diagnostics or {}) + diagnostics["log"] = events + if purpose: + diagnostics["purpose"] = purpose + if extra: + diagnostics.update({key: value for key, value in extra.items() if value is not None}) + raise PrivacyGatewayError(exc.code, exc.message, exc.status_code, diagnostics) from exc + + def install_test_recorder() -> list[dict[str, Any]]: """Test-local observer. Production never installs this.""" records: list[dict[str, Any]] = [] @@ -109,6 +231,7 @@ def reset_debug() -> None: last_compact = None debug_calls = 0 _test_recorder.set(None) + reset_detect_test_hooks() def inspect(request: GatewayRequest) -> GatewayResult: @@ -143,6 +266,10 @@ THING_GOVERNORS = { "bestelle", "bestellen", "bestellte", "hole", "holen", "holte", } +THING_OBJECT_AFTER = { + "essen", "kochen", "trinken", "kaufen", "bestellen", "holen", + "gegessen", "gekocht", "getrunken", "gekauft", +} def _words_before(text: str, index: int, n: int = 4) -> list[str]: @@ -164,25 +291,95 @@ def _mask_person_hit(text: str, start: int, end: int) -> bool: nxt = _words_after(text, end) if prev and prev[-1] in KINSHIP: return True - if any(word in THING_GOVERNORS for word in prev + nxt): + if any(word in THING_GOVERNORS for word in prev): + return False + if nxt and nxt[0] in THING_OBJECT_AFTER: return False return True -def _is_identity_mention(text: str, start: int, end: int, token: str) -> bool: - """Same rule as masking: food/thing homonyms are not identity.""" +def canonical_token(token: str) -> str: + """Public, intent-neutral: placeholder token without surrounding [[ ]].""" + raw = (token or "").strip() + if raw.startswith("[[") and raw.endswith("]]"): + return raw[2:-2] + return raw + + +def identity_label_pattern(label: str) -> re.Pattern[str]: + """Public, intent-neutral: word-boundary pattern for a local identity label.""" + return re.compile( + rf"(? bool: + """Public, intent-neutral: same rule as masking. Food/thing homonyms are not identity.""" if not _is_person_token(token): return True return _mask_person_hit(text, start, end) -def _label_pattern(label: str) -> re.Pattern[str]: - return re.compile( - rf"(? int: + """Public, intent-neutral count of identity mentions for one mapping.""" + if not label or not is_maskable_label(label): + return 0 + return sum( + 1 + for match in identity_label_pattern(label).finditer(text or "") + if is_identity_mention(text, match.start(), match.end(), token) ) +def _is_identity_mention(text: str, start: int, end: int, token: str) -> bool: + return is_identity_mention(text, start, end, token) + + +def _label_pattern(label: str) -> re.Pattern[str]: + return identity_label_pattern(label) + + +def _canonical_token(token: str) -> str: + return canonical_token(token) + + +def _entity_type_of(item: dict) -> str: + explicit = (item.get("entity_type") or "").strip().upper() + if explicit: + return explicit + token = _canonical_token(item.get("token") or "").upper() + if ":" in token: + return token.split(":", 1)[0] + return token or "UNKNOWN" + + +def _identity_occurrence_count(text: str, label: str, token: str) -> int: + return identity_occurrence_count(text, label, token) + + +def _active_replacements(text: str, mappings: list[dict]) -> tuple[ActiveReplacement, ...]: + found: list[ActiveReplacement] = [] + for item in mappings or []: + label = (item.get("local_label") or "").strip() + token = (item.get("token") or "").strip() + if not label or not token or not is_maskable_label(label): + continue + count = _identity_occurrence_count(text, label, token) + if count <= 0: + continue + found.append( + ActiveReplacement( + token=_canonical_token(token), + entity_type=_entity_type_of(item), + occurrence_count=count, + local_label=label, + demask_label=(item.get("demask_label") or item.get("canonical_label") or label), + ) + ) + return tuple(found) + + def _mask_body(text: str, mappings: list[dict]) -> str: masked = text for item in sorted(mappings, key=lambda row: len(row.get("local_label") or ""), reverse=True): @@ -217,57 +414,196 @@ def _mask(text: str, mappings: list[dict], *, personal_lines_only: bool = False) IDENTITY_LEAK_RETRY = ( - "Korrektur: Keine Klartext-Identität. Nur die vorhandenen Platzhalter [[…]], " - "keine Klarnamen, keine Klarorte." + "Korrektur: Keine Klartext-Identität. Kopiere die im Auftragstext bereits " + "vorhandenen Platzhalter zeichengetreu. Keine Klarnamen, keine Klarorte, " + "keine neuen Platzhalter, keine Auslassungspunkte." ) +PLACEHOLDER_RE = re.compile(r"\[\[\s*([^\[\]]+?)\s*\]\]") +GENERIC_PLACEHOLDER_INNER = frozenset({"…", "...", "..", "...."}) + + +def _restore_table(manifest: MaskingManifest | None) -> dict[str, str]: + table: dict[str, str] = {} + if not manifest: + return table + for token, label in (manifest.restore_by_token or {}).items(): + key = canonical_token(str(token)).upper().replace(" ", "") + if key and label: + table[key] = label + for item in manifest.replacements: + key = canonical_token(item.token).upper().replace(" ", "") + label = item.restore_label() + if key and label: + table.setdefault(key, label) + return table + + +def _lookup_restore(table: dict[str, str], inner: str) -> str | None: + raw = canonical_token(inner or "").upper().replace(" ", "") + if not raw or raw in GENERIC_PLACEHOLDER_INNER: + return None + if raw in table: + return table[raw] + if ":" in raw: + prefix, suffix = raw.split(":", 1) + if suffix.isdigit(): + padded = f"{prefix}:{int(suffix):02d}" + if padded in table: + return table[padded] + unpadded = f"{prefix}:{int(suffix)}" + if unpadded in table: + return table[unpadded] + return None + + +def _restore_map_from_mappings(mappings: list[dict] | None) -> dict[str, str]: + table: dict[str, str] = {} + for item in mappings or []: + token = canonical_token(item.get("token") or "").upper().replace(" ", "") + label = (item.get("demask_label") or item.get("canonical_label") or item.get("local_label") or "").strip() + if token and label: + table[token] = label + return table + + +def mask_prompt(rendered: str, mappings: list[dict], purpose: str) -> MaskingManifest: + """Request-scoped mask result. Active mappings are those actually replaced.""" + masked = rendered or "" + replacements: list[ActiveReplacement] = [] + for item in sorted(mappings or [], key=lambda row: len(row.get("local_label") or ""), reverse=True): + label = (item.get("local_label") or "").strip() + token = (item.get("token") or "").strip() + if not label or not token or not is_maskable_label(label): + continue + placeholder = token if token.startswith("[[") else f"[[{_canonical_token(token)}]]" + pattern = _label_pattern(label) + current = masked + count = 0 + + def repl(match: re.Match, *, _token=token, _ph=placeholder, _text=current) -> str: + nonlocal count + if not _is_identity_mention(_text, match.start(), match.end(), _token): + return match.group(0) + count += 1 + return _ph + + masked = pattern.sub(repl, current) + if count <= 0: + continue + replacements.append( + ActiveReplacement( + token=_canonical_token(token), + entity_type=_entity_type_of(item), + occurrence_count=count, + local_label=label, + demask_label=(item.get("demask_label") or item.get("canonical_label") or label), + ) + ) + if purpose == "dialogue_turn": + masked = bind_user_lines(masked) + return MaskingManifest( + masked_text=masked, + available_mapping_count=len(mappings or []), + replacements=tuple(replacements), + restore_by_token=_restore_map_from_mappings(mappings), + ) + def mask_for_egress(rendered: str, mappings: list[dict], purpose: str) -> str: - """Mask names in the full rendered prompt. Bind pronouns only on user lines. - - Opening hints, Space-Ausschnitte and titles are personal context, not instructions. - Journal generation keeps sie/er/ihr so demask does not turn every reference into the name. - """ - masked = _mask(rendered, mappings) - if purpose == "dialogue_turn": - return bind_user_lines(masked) - return masked + """String wrapper for callers that only need the masked egress text.""" + return mask_prompt(rendered, mappings, purpose).masked_text -def _demask(text: str, mappings: list[dict]) -> str: +def validate_pre_egress(masked_text: str, manifest: MaskingManifest) -> None: + """Fail closed before the provider if an active identity remains as plaintext.""" + leaked_tokens: list[str] = [] + leaked_types: list[str] = [] + for item in manifest.replacements: + for match in _label_pattern(item.local_label).finditer(masked_text or ""): + if _is_identity_mention(masked_text, match.start(), match.end(), item.token): + leaked_tokens.append(item.token) + leaked_types.append(item.entity_type) + break + if leaked_tokens: + raise PrivacyGatewayError( + ERROR_EGRESS_VALIDATION, + "Aktive Identität war nach der Maskierung noch als Klartext im Egress.", + diagnostics={ + "pre_egress_validation": "failed", + "leak_tokens": leaked_tokens, + "leak_entity_types": leaked_types, + **manifest.public_summary(), + }, + ) + + +def _demask(text: str, manifest: MaskingManifest | None) -> str: + """Rehydrate placeholders that belong to this request. Unknown tokens stay placeholders.""" result = text or "" - for item in mappings: - label = (item.get("local_label") or "").strip() - token = (item.get("token") or "").strip() - if not label or not token: - continue - placeholder = token if token.startswith("[[") else f"[[{token}]]" - result = result.replace(placeholder, label) - return result + table = _restore_table(manifest) + if not table: + return result + + def repl(match: re.Match) -> str: + inner = match.group(1) or "" + restored = _lookup_restore(table, inner) + return restored if restored else match.group(0) + + return PLACEHOLDER_RE.sub(repl, result) -def _validate_response(content: str, mappings: list[dict] | None = None) -> str: +def _validate_response(content: str, manifest: MaskingManifest | None = None) -> str: text = (content or "").strip() if not text: raise PrivacyGatewayError("empty_provider_response", "Der Provider lieferte keine Antwort.") - leaked: list[str] = [] - for item in mappings or []: - label = (item.get("local_label") or "").strip() - token = (item.get("token") or "").strip() - if not label or not is_maskable_label(label): - continue - for match in _label_pattern(label).finditer(text): - if _is_identity_mention(text, match.start(), match.end(), token): - leaked.append(label) + leaked_tokens: list[str] = [] + leaked_types: list[str] = [] + for item in manifest.replacements if manifest else (): + for match in _label_pattern(item.local_label).finditer(text): + if _is_identity_mention(text, match.start(), match.end(), item.token): + leaked_tokens.append(item.token) + leaked_types.append(item.entity_type) break - if leaked: + if leaked_tokens: raise PrivacyGatewayError( "response_validation_failed", "Antwort enthielt Klartext-Identität vor der Demaskierung.", + diagnostics={ + "response_validation": "failed", + "leak_tokens": leaked_tokens, + "leak_entity_types": leaked_types, + }, ) return text +def _fake_journal_from_sources(rendered: str) -> str: + """Deterministic fake body from labeled CURRENT_DAY_SOURCES. Not a quality claim.""" + text = rendered or "" + marker = "CURRENT_DAY_SOURCES" + if marker not in text: + return "" + after = text.split(marker, 1)[1] + for stop in ("WRITING_PROFILE", "STYLE_EXAMPLES", "EXISTING_TEXT", "EDITORIAL_MODE"): + if f"\n{stop}" in after: + after = after.split(f"\n{stop}", 1)[0] + parts: list[str] = [] + current: list[str] = [] + for line in after.splitlines(): + stripped = line.strip() + if stripped.startswith("[u") and stripped.endswith("]"): + if current: + parts.append(" ".join(current).strip()) + current = [] + continue + if stripped and not stripped.startswith("(") and "einzige Tatsachen" not in stripped: + current.append(stripped) + if current: + parts.append(" ".join(current).strip()) + return " ".join(part for part in parts if part).strip() + + def _fake_complete(purpose: str, rendered: str) -> str: if purpose == "profile_review": return ( @@ -283,6 +619,9 @@ def _fake_complete(purpose: str, rendered: str) -> str: if purpose == "journal_reconstruct": return fake_reconstruction(rendered) if purpose == "journal_generate": + body = _fake_journal_from_sources(rendered) + if body: + return f"Ein Tag\n\n{body}" match = re.search(r"\{.*\}", rendered, re.DOTALL) if match: try: @@ -344,41 +683,103 @@ def complete_model(messages: list[dict], policy: dict[str, Any]) -> ChatResult: def complete(request: GatewayRequest) -> GatewayResult: global last_compact, debug_calls + started = time.perf_counter() + events: list[dict[str, Any]] = [] result = inspect(request) if not result.allowed: + _log_event(events, started, "blocked", purpose=request.purpose, code=result.reason) raise PrivacyGatewayError( result.reason, "Persönlicher KI-Aufruf wurde vom Privacy Gateway blockiert. " "Es ist kein Egress-Provider konfiguriert." if result.reason == "no_egress_provider_configured" else "Persönlicher KI-Aufruf wurde vom Privacy Gateway blockiert.", + diagnostics={"log": events, "purpose": request.purpose}, ) rendered = _minimize(str((request.payload or {}).get("rendered") or ""), request.purpose) - source_text = str((request.payload or {}).get("source_text") or rendered) detect_name = None + detect_note = None + detect_stats: dict[str, Any] = {} + local_identities: list[dict[str, Any]] = [] try: - mappings, detect_name, detect_note = detect_and_remember(request.profile_id, source_text) - except ProviderError as exc: - raise PrivacyGatewayError(exc.code, exc.message, exc.status_code) from exc - if not mappings and request.profile_id: - mappings = list_mappings(request.profile_id) - masked = mask_for_egress(rendered, mappings, request.purpose) + outcome = detect_personal_egress(request.profile_id, rendered) + mappings = outcome.mappings + detect_stats = outcome.stats.public() + detect_name = outcome.stats.detect_provider + detect_note = outcome.stats.detect_note + local_identities = outcome.local_identities + except DetectError as exc: + detect_stats = dict(exc.diagnostics or {}) + _log_event( + events, + started, + "detect", + status="error", + code=exc.code, + full_detection_coverage=False, + generate_called=False, + chunk_count=detect_stats.get("chunk_count"), + chunks_ok=detect_stats.get("chunks_ok"), + detect_calls=detect_stats.get("detect_calls"), + ) + raise PrivacyGatewayError( + exc.code, + user_detect_message(exc.code, exc.message), + exc.status_code, + {"log": events, "purpose": request.purpose, "generate_called": False, **detect_stats}, + ) from exc + _log_event( + events, + started, + "detect", + status="ok", + detect_provider=detect_name, + detect_note=detect_note, + detect_model=detect_stats.get("detect_model"), + full_detection_coverage=detect_stats.get("full_detection_coverage"), + chunk_count=detect_stats.get("chunk_count"), + chunks_ok=detect_stats.get("chunks_ok"), + detect_calls=detect_stats.get("detect_calls"), + request_local_hits=detect_stats.get("request_local_hits"), + confirmed_registry_hits=detect_stats.get("confirmed_registry_hits"), + entity_counts=detect_stats.get("entity_counts"), + detect_ms=detect_stats.get("detect_ms"), + detect_prompt_tokens=detect_stats.get("detect_prompt_tokens"), + detect_completion_tokens=detect_stats.get("detect_completion_tokens"), + detect_total_tokens=detect_stats.get("detect_total_tokens"), + detect_cost=detect_stats.get("detect_cost"), + mapping_count=len(mappings), + ) + manifest = mask_prompt(rendered, mappings, request.purpose) + masked = manifest.masked_text + summary = manifest.public_summary() budget = (request.payload or {}).get("budget") diagnostics = dict((request.payload or {}).get("diagnostics") or {}) + diagnostics.update(summary) + diagnostics.update({key: value for key, value in detect_stats.items() if value is not None}) + diagnostics["generate_called"] = False if request.purpose in JOURNAL_PURPOSES: if budget is None: + _log_event(events, started, "budget", status="error", code="model_metadata_unknown") raise PrivacyGatewayError( "model_metadata_unknown", USER_MESSAGES["model_metadata_unknown"], 503, - diagnostics={"reason": "journal_budget_missing"}, + diagnostics={"reason": "journal_budget_missing", "log": events}, ) try: assert_input_fits(budget, masked) except JournalBudgetError as exc: - raise PrivacyGatewayError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc + _log_event(events, started, "budget", status="error", code=exc.code) + raise PrivacyGatewayError( + exc.code, + exc.message, + exc.status_code, + {**(exc.diagnostics or {}), "log": events}, + ) from exc diagnostics.update(budget.as_diagnostics()) diagnostics["estimated_input_tokens"] = budget.estimated_input_tokens + _log_event(events, started, "budget", status="ok") debug_calls += 1 config = generate_provider() layer = { @@ -400,13 +801,29 @@ def complete(request: GatewayRequest) -> GatewayResult: "prompt_slug": (request.payload or {}).get("prompt_slug"), "rendered": rendered, "masked": masked, - "mask_input": source_text, + "mask_input": rendered, "provider": result.provider, "model": config.model if config else None, "detect_provider": detect_name, "detect_note": detect_note, + "detect_model": detect_stats.get("detect_model"), + "full_detection_coverage": detect_stats.get("full_detection_coverage"), + "chunk_count": detect_stats.get("chunk_count"), + "chunks_ok": detect_stats.get("chunks_ok"), + "entity_counts": detect_stats.get("entity_counts"), + "confirmed_registry_hits": detect_stats.get("confirmed_registry_hits"), + "request_local_hits": detect_stats.get("request_local_hits"), + "detect_calls": detect_stats.get("detect_calls"), + "detect_ms": detect_stats.get("detect_ms"), + "generate_called": False, "mapping_count": len(mappings), + "available_mapping_count": summary["available_mapping_count"], + "active_mapping_count": summary["active_mapping_count"], + "masked_occurrence_count": summary["masked_occurrence_count"], + "active_tokens": summary["active_tokens"], + "active_entity_types": summary["active_entity_types"], "budget": compact_diagnostics(diagnostics), + "log": events, } model_policy = { "purpose": request.purpose, @@ -418,21 +835,30 @@ def complete(request: GatewayRequest) -> GatewayResult: ), } try: - chat = complete_model([{"role": "user", "content": masked}], model_policy) - raw = chat.content - try: - validated = _validate_response(raw, mappings) - except PrivacyGatewayError as exc: - if exc.code != "response_validation_failed": - raise - diagnostics["response_validation_retry"] = 1 - chat = complete_model( - [{"role": "user", "content": masked + "\n\n" + IDENTITY_LEAK_RETRY}], - model_policy, - ) - raw = chat.content - validated = _validate_response(raw, mappings) + validate_pre_egress(masked, manifest) + diagnostics["pre_egress_validation"] = "ok" + request_trace["pre_egress_validation"] = "ok" + _log_event( + events, + started, + "pre_egress_validation", + status="ok", + active_mapping_count=summary["active_mapping_count"], + masked_occurrence_count=summary["masked_occurrence_count"], + ) except PrivacyGatewayError as exc: + leak = dict(exc.diagnostics or {}) + diagnostics["pre_egress_validation"] = "failed" + request_trace["pre_egress_validation"] = "failed" + _log_event( + events, + started, + "pre_egress_validation", + status="failed", + code=exc.code, + leak_tokens=leak.get("leak_tokens"), + leak_entity_types=leak.get("leak_entity_types"), + ) failed = merge_usage( {**diagnostics, "budget_ok": False, "abort_reason": exc.code, "status": "error"}, None, @@ -440,24 +866,111 @@ def complete(request: GatewayRequest) -> GatewayResult: ) last_compact = compact_diagnostics(failed) _record_test({"purpose": request.purpose, "ok": False, "code": exc.code}) - raise + _raise_with_log(exc, events, purpose=request.purpose) + usages: list[dict] = [] + try: + _log_event( + events, + started, + "model_call", + attempt=1, + purpose=request.purpose, + model=config.model if config else None, + ) + chat = complete_model([{"role": "user", "content": masked}], model_policy) + usages.append(chat.usage or {}) + diagnostics["generate_called"] = True + request_trace["generate_called"] = True + _log_event(events, started, "model_result", attempt=1, **_usage_bits(chat.usage)) + raw = chat.content + try: + validated = _validate_response(raw, manifest) + diagnostics["response_validation"] = "ok" + _log_event(events, started, "response_validation", attempt=1, status="ok") + except PrivacyGatewayError as exc: + if exc.code != "response_validation_failed": + raise + leak = dict(exc.diagnostics or {}) + diagnostics["response_validation"] = "failed" + _log_event( + events, + started, + "response_validation", + attempt=1, + status="failed", + code=exc.code, + leak_tokens=leak.get("leak_tokens"), + leak_entity_types=leak.get("leak_entity_types"), + ) + _log_event(events, started, "retry", attempt=2, reason="response_validation_failed") + diagnostics["response_validation_retry"] = 1 + request_trace["response_validation_retry"] = 1 + _log_event( + events, + started, + "model_call", + attempt=2, + purpose=request.purpose, + model=config.model if config else None, + ) + chat = complete_model( + [{"role": "user", "content": masked + "\n\n" + IDENTITY_LEAK_RETRY}], + model_policy, + ) + usages.append(chat.usage or {}) + _log_event(events, started, "model_result", attempt=2, **_usage_bits(chat.usage)) + raw = chat.content + validated = _validate_response(raw, manifest) + diagnostics["response_validation"] = "ok" + _log_event(events, started, "response_validation", attempt=2, status="ok") + except PrivacyGatewayError as exc: + if exc.code == "response_validation_failed": + leak = dict(exc.diagnostics or {}) + diagnostics["response_validation"] = "failed" + _log_event( + events, + started, + "response_validation", + attempt=2 if diagnostics.get("response_validation_retry") else 1, + status="failed", + code=exc.code, + leak_tokens=leak.get("leak_tokens"), + leak_entity_types=leak.get("leak_entity_types"), + ) + failed = merge_usage( + {**diagnostics, "budget_ok": False, "abort_reason": exc.code, "status": "error"}, + sum_usages(usages) or None, + config.model if config else None, + ) + last_compact = compact_diagnostics(failed) + _record_test({"purpose": request.purpose, "ok": False, "code": exc.code}) + _raise_with_log( + exc, + events, + purpose=request.purpose, + extra={"response_validation_retry": diagnostics.get("response_validation_retry")}, + ) diagnostics = merge_usage( { **diagnostics, "context_compression": chat.context_compression, "budget_ok": True, "status": "ok", + "response_validation": "ok", }, - chat.usage, + sum_usages(usages) or chat.usage, chat.model or (config.model if config else None), ) - result.content = _demask(validated, mappings) + result.content = _demask(validated, manifest) result.allowed = True + result.local_identities = local_identities result.diagnostics = compact_diagnostics(diagnostics) request_trace["raw"] = validated request_trace["reply"] = result.content request_trace["model"] = chat.model or request_trace.get("model") + request_trace["response_validation"] = "ok" request_trace["budget"] = compact_diagnostics(diagnostics) + request_trace["log"] = events result.trace = public_trace(request_trace) last_compact = compact_diagnostics(diagnostics) _record_test({"purpose": request.purpose, "ok": True, "prompt_slug": request_trace.get("prompt_slug")}) @@ -485,12 +998,39 @@ def public_trace(trace: dict | None) -> dict | None: "model": trace.get("model"), "detect_provider": trace.get("detect_provider"), "detect_note": trace.get("detect_note"), + "detect_model": trace.get("detect_model"), + "full_detection_coverage": trace.get("full_detection_coverage"), + "chunk_count": trace.get("chunk_count"), + "chunks_ok": trace.get("chunks_ok"), + "entity_counts": trace.get("entity_counts"), + "confirmed_registry_hits": trace.get("confirmed_registry_hits"), + "request_local_hits": trace.get("request_local_hits"), + "detect_calls": trace.get("detect_calls"), + "detect_ms": trace.get("detect_ms"), + "generate_called": trace.get("generate_called"), "mapping_count": trace.get("mapping_count"), + "available_mapping_count": trace.get("available_mapping_count"), + "active_mapping_count": trace.get("active_mapping_count"), + "masked_occurrence_count": trace.get("masked_occurrence_count"), + "active_tokens": trace.get("active_tokens"), + "active_entity_types": trace.get("active_entity_types"), + "pre_egress_validation": trace.get("pre_egress_validation"), + "response_validation": trace.get("response_validation"), + "leak_tokens": trace.get("leak_tokens"), + "leak_entity_types": trace.get("leak_entity_types"), "intern": trace.get("rendered") or trace.get("intern"), "egress": trace.get("masked") or trace.get("egress"), "mask_input": trace.get("mask_input"), "raw": trace.get("raw"), "reply": trace.get("reply"), "budget": budget or None, + "log": trace.get("log") or None, + "response_validation_retry": trace.get("response_validation_retry"), + "stage1": trace.get("stage1"), + "reason": trace.get("reason"), + "guard": trace.get("guard"), "stages": trace.get("stages"), + "status": trace.get("status"), + "coverage": trace.get("coverage"), + "source_count": trace.get("source_count"), } diff --git a/backend/privacy_placeholders.py b/backend/privacy_placeholders.py index dc350a2..807e54b 100644 --- a/backend/privacy_placeholders.py +++ b/backend/privacy_placeholders.py @@ -7,7 +7,7 @@ from placeholders import PlaceholderError PRIVACY_PATTERN = re.compile(r"\[\[([A-Z][A-Z0-9_]*(?::[A-Z0-9_]+)?)\]\]") KNOWN_EXACT = {"SELF"} -KNOWN_PREFIXES = ("PERSON:", "PLACE:") +KNOWN_PREFIXES = ("PERSON:", "PLACE:", "ORG:", "PROJECT:") def catalog() -> list[dict]: diff --git a/backend/prompt_budget.py b/backend/prompt_budget.py index 2d9db88..fc31f10 100644 --- a/backend/prompt_budget.py +++ b/backend/prompt_budget.py @@ -275,3 +275,37 @@ def merge_usage(diagnostics: dict[str, Any], usage: dict[str, Any] | None, model cost = data.get("total_cost") payload["cost"] = cost return payload + + +def sum_usages(parts: list[dict[str, Any] | None]) -> dict[str, Any]: + prompt = 0 + completion = 0 + total = 0 + cost = 0.0 + saw_tokens = False + saw_cost = False + for item in parts: + data = item or {} + if data.get("prompt_tokens") is not None: + prompt += int(data.get("prompt_tokens") or 0) + saw_tokens = True + if data.get("completion_tokens") is not None: + completion += int(data.get("completion_tokens") or 0) + saw_tokens = True + if data.get("total_tokens") is not None: + total += int(data.get("total_tokens") or 0) + saw_tokens = True + extra = data.get("cost") + if extra is None: + extra = data.get("total_cost") + if extra is not None: + cost += float(extra) + saw_cost = True + payload: dict[str, Any] = {} + if saw_tokens: + payload["prompt_tokens"] = prompt + payload["completion_tokens"] = completion + payload["total_tokens"] = total or (prompt + completion) + if saw_cost: + payload["cost"] = cost + return payload diff --git a/backend/provider_settings.py b/backend/provider_settings.py index e04fb70..382fec9 100644 --- a/backend/provider_settings.py +++ b/backend/provider_settings.py @@ -23,7 +23,7 @@ ROLE_META = { }, "detect": { "title": "Maskierung", - "task": "Eigener Detect-Endpunkt. Nur Namen, keine Operation. Externes Klartext-Detect nur Development/Test. Produktiv: lokal oder Muster.", + "task": "Eigener Detect-Endpunkt. Vollständige semantische Detection des persönlichen Egress. Externes Klartext-Detect nur Development/Test. Produktiv: lokales Detect-Modell. Kein Pattern-Fallback.", }, } @@ -149,5 +149,5 @@ def public_status() -> dict: return { "roles": [public_role_status(role) for role in ROLES], "runtime_env": runtime_env(), - "note": "Zwei Einträge: Maskierung (Detect) und Sprachmodell. Externes Klartext-Detect nur Development/Test. Produktiv lokal oder Muster. Keys nur in backend/.env.", + "note": "Zwei Einträge: Maskierung (Detect) und Sprachmodell. Semantische Detection ist Pflicht vor Generate. Externes Klartext-Detect nur Development/Test. Produktiv lokal. Keys nur in backend/.env.", } diff --git a/backend/providers.py b/backend/providers.py index 3c7a24e..15d06f7 100644 --- a/backend/providers.py +++ b/backend/providers.py @@ -47,6 +47,7 @@ class ChatResult: model: str | None = None usage: dict = field(default_factory=dict) context_compression: str = "not_applicable" + finish_reason: str | None = None def is_openrouter(config: ProviderConfig) -> bool: @@ -215,9 +216,15 @@ def complete_chat( except (KeyError, IndexError, TypeError) as exc: raise ProviderError("provider_shape", f"Unerwartete {config.role}-Antwort.") from exc usage = data.get("usage") if isinstance(data.get("usage"), dict) else {} + finish_reason = None + try: + finish_reason = data["choices"][0].get("finish_reason") + except (KeyError, IndexError, TypeError, AttributeError): + finish_reason = None return ChatResult( content=content or "", model=data.get("model") or config.model, usage=usage, context_compression=compression, + finish_reason=finish_reason, ) diff --git a/backend/routers/admin.py b/backend/routers/admin.py index 57dcef8..bfbd4e2 100644 --- a/backend/routers/admin.py +++ b/backend/routers/admin.py @@ -4,6 +4,17 @@ from pydantic import BaseModel, Field from auth import require_admin_dep from db import get_db from dialogue_store import StoreError, get_conversation, inventory, list_conversations, list_derived_for_conversation, list_messages +from identity_store import ( + confirm_identity, + confirm_review_proposal, + deactivate_identity, + delete_identity, + delete_review_proposal, + dismiss_review_proposal, + list_registry, + list_review_proposals, + update_identity, +) from provider_settings import ROLES, SettingsError, public_status, set_role_key, upsert_setting from version import APP_VERSION, BUILD_DATE, MODULE_VERSIONS @@ -95,3 +106,114 @@ def admin_update_provider(role: str, body: ProviderUpdate, session: dict = Depen except SettingsError as exc: raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc return public_status() + + +class IdentityWrite(BaseModel): + canonical_label: str = "" + entity_type: str = "PERSON" + aliases: list[str] = Field(default_factory=list) + token: str | None = None + status: str | None = None + + +class IdentityPatch(BaseModel): + canonical_label: str | None = None + entity_type: str | None = None + aliases: list[str] | None = None + status: str | None = None + + +def _identity_error(exc: ValueError) -> HTTPException: + code = str(exc) + messages = { + "empty_label": "Bezeichnung ist leer oder nicht maskierbar.", + "invalid_entity_type": "Unzulässiger Entitätstyp.", + "invalid_status": "Unzulässiger Status.", + "identity_missing": "Registry-Eintrag fehlt.", + "proposal_missing": "Vorschlag fehlt.", + } + return HTTPException(status_code=400, detail={"code": code, "message": messages.get(code, "Ungültige Identitätsangabe.")}) + + +@router.get("/identities") +def admin_identities(session: dict = Depends(require_admin_dep)): + profile_id = session["profile_id"] + return { + "registry": list_registry(profile_id), + "proposals": list_review_proposals(profile_id, include_dismissed=True), + "note": "Nur lokale bestätigte Registry und unbestätigte Vorschläge. Kein externer Egress.", + "backup": ( + "Vor einer Bereinigung die lokale Datei backend/data/kansho.sqlite kopieren. " + "Wiederherstellung: Anwendung beenden und die Kopie zurücklegen." + ), + } + + +@router.post("/identities") +def admin_create_identity(body: IdentityWrite, session: dict = Depends(require_admin_dep)): + try: + return confirm_identity( + session["profile_id"], + body.canonical_label, + entity_type=body.entity_type, + token=body.token, + aliases=body.aliases, + ) + except ValueError as exc: + raise _identity_error(exc) from exc + + +@router.patch("/identities/{identity_id}") +def admin_update_identity(identity_id: str, body: IdentityPatch, session: dict = Depends(require_admin_dep)): + try: + return update_identity( + session["profile_id"], + identity_id, + canonical_label=body.canonical_label, + entity_type=body.entity_type, + aliases=body.aliases, + status=body.status, + ) + except ValueError as exc: + raise _identity_error(exc) from exc + + +@router.post("/identities/{identity_id}/deactivate") +def admin_deactivate_identity(identity_id: str, session: dict = Depends(require_admin_dep)): + try: + return deactivate_identity(session["profile_id"], identity_id) + except ValueError as exc: + raise _identity_error(exc) from exc + + +@router.delete("/identities/{identity_id}") +def admin_delete_identity(identity_id: str, session: dict = Depends(require_admin_dep)): + delete_identity(session["profile_id"], identity_id) + return {"ok": True} + + +@router.post("/identities/proposals/{proposal_id}/confirm") +def admin_confirm_proposal(proposal_id: str, body: IdentityPatch | None = None, session: dict = Depends(require_admin_dep)): + payload = body or IdentityPatch() + try: + return confirm_review_proposal( + session["profile_id"], + proposal_id, + canonical_label=payload.canonical_label, + entity_type=payload.entity_type, + aliases=payload.aliases, + ) + except ValueError as exc: + raise _identity_error(exc) from exc + + +@router.post("/identities/proposals/{proposal_id}/dismiss") +def admin_dismiss_proposal(proposal_id: str, session: dict = Depends(require_admin_dep)): + dismiss_review_proposal(session["profile_id"], proposal_id) + return {"ok": True} + + +@router.delete("/identities/proposals/{proposal_id}") +def admin_delete_proposal(proposal_id: str, session: dict = Depends(require_admin_dep)): + delete_review_proposal(session["profile_id"], proposal_id) + return {"ok": True} diff --git a/backend/schema.sql b/backend/schema.sql index 6307fa9..2562812 100644 --- a/backend/schema.sql +++ b/backend/schema.sql @@ -81,6 +81,7 @@ CREATE TABLE IF NOT EXISTS ai_prompts ( active INTEGER NOT NULL DEFAULT 1, is_system_default INTEGER NOT NULL DEFAULT 0, default_template TEXT NOT NULL DEFAULT '', + seed_revision TEXT NOT NULL DEFAULT '', sort_order INTEGER NOT NULL DEFAULT 0, created TEXT NOT NULL DEFAULT (datetime('now')), updated TEXT NOT NULL DEFAULT (datetime('now')) @@ -206,10 +207,29 @@ CREATE TABLE IF NOT EXISTS identity_mappings ( profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE, token TEXT NOT NULL, local_label TEXT NOT NULL DEFAULT '', + canonical_label TEXT NOT NULL DEFAULT '', + entity_type TEXT NOT NULL DEFAULT 'PERSON', + status TEXT NOT NULL DEFAULT 'legacy_review_required', + origin TEXT NOT NULL DEFAULT 'legacy_auto', + aliases_json TEXT NOT NULL DEFAULT '[]', created TEXT NOT NULL DEFAULT (datetime('now')), + updated TEXT NOT NULL DEFAULT (datetime('now')), + confirmed_at TEXT, UNIQUE (profile_id, token) ); +CREATE TABLE IF NOT EXISTS identity_review_proposals ( + id TEXT PRIMARY KEY, + profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE, + observed_label TEXT NOT NULL DEFAULT '', + entity_type TEXT NOT NULL DEFAULT 'PERSON', + status TEXT NOT NULL DEFAULT 'unconfirmed', + origin TEXT NOT NULL DEFAULT 'detect_proposal', + last_seen TEXT NOT NULL DEFAULT (datetime('now')), + created TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (profile_id, observed_label, entity_type) +); + CREATE TABLE IF NOT EXISTS journal_days ( id TEXT PRIMARY KEY, profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE, diff --git a/backend/tests/test_frame.py b/backend/tests/test_frame.py index 47c9bdf..553b6dd 100644 --- a/backend/tests/test_frame.py +++ b/backend/tests/test_frame.py @@ -222,6 +222,9 @@ def main() -> None: "source_text", "space_title", "writing_profile", + "style_examples", + "editorial_mode", + "editorial_instructions", "interaction_hint", }, "system and mvp context keys", diff --git a/backend/tests/test_identity_registry.py b/backend/tests/test_identity_registry.py new file mode 100644 index 0000000..330fda7 --- /dev/null +++ b/backend/tests/test_identity_registry.py @@ -0,0 +1,127 @@ +"""Admin identity registry stays local. Run from backend/: python tests/test_identity_registry.py""" +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-identity-registry-test.sqlite") +os.environ["KANSHO_FAKE_PROVIDER"] = "1" +os.environ["KANSHO_FAKE_DETECT"] = "1" +Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) + +from fastapi.testclient import TestClient +from identity_store import STATUS_LEGACY, list_confirmed_identities, list_registry +from main import app +from privacy_gateway import reset_debug + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def main() -> None: + reset_debug() + import sqlite3 + from db import _IDENTITY_MAPPING_COLUMNS, _ensure_columns + from identity_store import migrate_legacy_identity_rows as migrate_legacy + + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.executescript( + """ + CREATE TABLE identity_mappings ( + id TEXT PRIMARY KEY, + profile_id TEXT NOT NULL, + token TEXT NOT NULL, + local_label TEXT NOT NULL DEFAULT '', + created TEXT NOT NULL DEFAULT '2026-08-01 00:00:00' + ); + INSERT INTO identity_mappings (id, profile_id, token, local_label) + VALUES ('m1', 'p1', 'PERSON:01', 'Altbau'); + """ + ) + _ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS) + migrate_legacy(conn) + legacy = dict(conn.execute("SELECT * FROM identity_mappings WHERE id = 'm1'").fetchone()) + expect(legacy["status"] == STATUS_LEGACY, "existing rows become legacy_review_required") + expect(legacy["origin"] == "legacy_auto", "existing rows keep legacy origin") + expect(legacy["canonical_label"] == "Altbau", "canonical_label is copied from local_label") + expect(bool(legacy["updated"]), "updated is populated after ALTER") + conn.close() + + with TestClient(app) as client: + setup = client.post( + "/api/auth/setup", + json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"}, + ) + headers = {"X-Auth-Token": setup.json()["token"]} + profile_id = setup.json()["profile_id"] + + from db import get_db + import uuid + + legacy_id = str(uuid.uuid4()) + with get_db() as conn: + conn.execute( + """ + INSERT INTO identity_mappings + (id, profile_id, token, local_label, canonical_label, entity_type, status, origin) + VALUES (?, ?, ?, ?, ?, 'PERSON', ?, 'legacy_auto') + """, + (legacy_id, profile_id, "PERSON:70", "Altbau", "Altbau", STATUS_LEGACY), + ) + expect( + not any(item["id"] == legacy_id for item in list_confirmed_identities(profile_id)), + "legacy row is not confirmed", + ) + expect(any(item["id"] == legacy_id for item in list_registry(profile_id)), "legacy row remains reviewable") + + patched = client.patch( + f"/api/admin/identities/{legacy_id}", + headers=headers, + json={"status": "confirmed", "entity_type": "PLACE", "canonical_label": "Altbau"}, + ) + expect(patched.status_code == 200, patched.text) + expect(patched.json()["status"] == "confirmed", "admin can confirm a legacy row") + expect(patched.json()["entity_type"] == "PLACE", "admin can correct the type") + + created = client.post( + "/api/admin/identities", + headers=headers, + json={"canonical_label": "Nordwerk", "entity_type": "ORG", "aliases": ["Nordwerk AG"]}, + ) + expect(created.status_code == 200, created.text) + identity_id = created.json()["id"] + aliases = client.patch( + f"/api/admin/identities/{identity_id}", + headers=headers, + json={"aliases": ["Nordwerk AG", "Nordwerk"]}, + ) + expect(aliases.status_code == 200, aliases.text) + expect("Nordwerk AG" in (aliases.json().get("aliases") or []), "confirmed alias can be stored") + + calls = {"n": 0} + + def boom(*_args, **_kwargs): + calls["n"] += 1 + raise AssertionError("no egress") + + with patch("entity_detect.complete_chat", boom), patch("providers.complete_chat", boom): + gone = client.delete(f"/api/admin/identities/{identity_id}", headers=headers) + expect(gone.status_code == 200, "admin delete stays local") + expect(calls["n"] == 0, "registry actions do not call detect or generate") + expect(not any(item["id"] == identity_id for item in list_registry(profile_id)), "deleted row is gone") + + print("All identity registry tests passed.") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_journal_budget.py b/backend/tests/test_journal_budget.py index 12f1663..de9e8b7 100644 --- a/backend/tests/test_journal_budget.py +++ b/backend/tests/test_journal_budget.py @@ -39,6 +39,7 @@ from writing_profile_store import ( get_profile, import_text, replace_trait_refs, + set_lifecycle, upsert_trait, update_facet, ) @@ -769,6 +770,7 @@ def test_task_brief_and_dedupe(client: TestClient, headers: dict, profile_id: st } ], ) + set_lifecycle(profile_id, "confirmed") before = get_profile(profile_id) brief = compile_task_brief(profile_id, "journal_generate") after = get_profile(profile_id) @@ -777,8 +779,8 @@ def test_task_brief_and_dedupe(client: TestClient, headers: dict, profile_id: st expect(_ends_on_word_boundary(brief), "task brief does not cut inside a word") for slug, statement in statements: expect(slug in brief or statement.split(",")[0] in brief, f"relevant trait {slug} can be included") - expect(brief.count("Repräsentative Exemplare:") <= 1, "at most one exemplar section") - expect(len([line for line in brief.splitlines() if line.startswith("- ") and "Beispiel" not in line]) >= 6, "six traits considered") + expect("Repräsentative Exemplare:" not in brief, "historical exemplars are not mixed into the compact brief") + expect(len([line for line in brief.splitlines() if line.startswith("- ")]) >= 6, "six traits considered") expect(before["sources"] == after["sources"], "compile_task_brief does not mutate stored sources") expect( [(item.get("slug"), item.get("statement")) for item in before["traits"]] @@ -836,22 +838,43 @@ def test_generate_flow(client: TestClient, headers: dict) -> None: json={"conversation_ids": [conv.json()["id"]]}, ) expect(gen.status_code == 200, f"generate {gen.status_code}") - expect([item.get("purpose") for item in recorder] == ["journal_reconstruct", "journal_generate"], "both stages ran") + expect([item.get("purpose") for item in recorder] == ["journal_generate"], "runtime generate calls only narration") + run_log = gen.json().get("run_log") or (gen.json().get("trace") or {}).get("log") or [] + expect(any(item.get("kind") == "model_call" for item in run_log), "run log records model calls") + expect( + any(item.get("kind") == "stage1_result" and item.get("stage") == "local_source_artifact" for item in run_log), + "run log records local stage 1", + ) + expect(sum(1 for item in run_log if item.get("kind") == "detect") <= 1, "at most one detect call") + expect(sum(1 for item in run_log if item.get("kind") == "model_call") == 1, "exactly one generate model call") + expect( + not any(key in (item or {}) for item in run_log for key in ("intern", "egress", "raw", "reply", "local_label")), + "run log omits prompt bodies and labels", + ) stages = {item.get("purpose"): item for item in (gen.json().get("trace") or {}).get("stages") or []} - expect("journal_reconstruct" in stages, "stage 1 ran through the gateway") + expect("local_source_artifact" in stages, "stage 1 is the local source artifact") expect("journal_generate" in stages, "stage 2 ran through the gateway") - reconstruct = stages["journal_reconstruct"].get("intern") or "" + expect("journal_reconstruct" not in stages, "runtime path does not call journal_reconstruct") + reconstruct = stages["local_source_artifact"].get("intern") or "" narrate = stages["journal_generate"].get("intern") or "" - expect("[u1]" in reconstruct and "user:" in reconstruct, "stage 1 sees labeled user sources") - expect("assistant:" in reconstruct, "stage 1 may see assistant as context") + expect(stages["local_source_artifact"].get("status") == "local_ok", "local stage 1 succeeded") + expect(stages["local_source_artifact"].get("provider") in (None, ""), "local stage has no provider") + expect(stages["local_source_artifact"].get("model") in (None, ""), "local stage has no model") + expect((stages["local_source_artifact"].get("budget") or {}).get("prompt_tokens") == 0, "local stage has no tokens") + expect("verified_artifact" in reconstruct and "u1" in reconstruct, "local artifact contains source ids") + expect("assistant:" not in reconstruct, "assistant lines are not in the artifact") + expect("Markt" in reconstruct and "Hafen" in reconstruct, "every selected user source is in the artifact") expect("assistant:" not in narrate, "stage 2 does not resend assistant lines") expect("user:" not in narrate, "stage 2 does not resend the raw dialogue") - expect("verified_artifact" in narrate and "source_order" in narrate, "stage 2 receives the local verified artifact") - expect("sources" in narrate, "stage 2 artifact contains rehydrated sources") + expect("CURRENT_DAY_SOURCES" in narrate, "stage 2 labels current-day facts") + expect("[u1]" in narrate, "stage 2 lists source ids") expect("7:30" in reconstruct or "7:30" in narrate, "times survive into generation") expect("Markt" in narrate and "Hafen" in narrate, "full selected user sources reach stage 2") expect("Vielleicht bleibe ich kürzer" in narrate, "omitted labels do not drop canonical hedges") expect("assistant:" not in (gen.json().get("body") or ""), "draft body has no assistant lines") + expect("WRITING_PROFILE" in narrate or "Neutraler Journalstil" in narrate or "Core:" in narrate, "writing profile reaches generate") + expect("STYLE_EXAMPLES" in narrate, "style examples are labeled separately from day facts") + expect("Erzählmerkmale" not in (narrate.split("CURRENT_DAY_SOURCES")[0] if "CURRENT_DAY_SOURCES" in narrate else narrate), "day dialogue is not a style brief") reset_debug() short_recorder = install_test_recorder() @@ -886,14 +909,14 @@ def test_generate_flow(client: TestClient, headers: dict) -> None: ) expect(short_gen.status_code == 200, f"short-claim generate {short_gen.status_code}") expect( - [item.get("purpose") for item in short_recorder] == ["journal_reconstruct", "journal_generate"], - "short claims still run both stages because sources are rehydrated", + [item.get("purpose") for item in short_recorder] == ["journal_generate"], + "invalid former stage-1 JSON is irrelevant because reconstruct is not called", ) short_narrate = "" for item in (short_gen.json().get("trace") or {}).get("stages") or []: if item.get("purpose") == "journal_generate": short_narrate = item.get("intern") or "" - expect("Kirschen" in short_narrate and "Hafen" in short_narrate, "short claims cannot drop the rest of the sources") + expect("Kirschen" in short_narrate and "Hafen" in short_narrate, "local artifact cannot drop selected sources") expect("\"feeling\"" not in short_narrate, "feeling labels do not reach stage 2 as facts") budget = (gen.json().get("trace") or {}).get("budget") or stages["journal_generate"].get("budget") or {} expect(budget.get("reserved_output_tokens") == 4096, "trace records reserved output") @@ -930,22 +953,23 @@ def test_generate_flow(client: TestClient, headers: dict) -> None: headers=headers, json={"conversation_ids": [conv.json()["id"]]}, ) - expect(denied_recon.status_code == 200, f"incomplete reconstruct falls back locally {denied_recon.text}") + expect(denied_recon.status_code == 200, f"former reconstruct JSON is unused {denied_recon.text}") fallback_body = denied_recon.json().get("body") or "" expect("Markt" in fallback_body or "Hafen" in fallback_body, "local artifact still produces a draft") fallback_stages = {item.get("purpose"): item for item in (denied_recon.json().get("trace") or {}).get("stages") or []} - expect(fallback_stages.get("journal_reconstruct", {}).get("stage1") == "local_fallback", "invalid stage 1 is marked local_fallback") + expect(fallback_stages.get("local_source_artifact", {}).get("stage1") == "local_ok", "stage 1 is local and successful") expect( - (fallback_stages.get("journal_reconstruct") or {}).get("reason") == "incomplete_coverage", - "fallback records why the model JSON was rejected", + fallback_stages.get("local_source_artifact", {}).get("coverage") == "all_selected_sources" + or "verified_artifact" in ((fallback_stages.get("local_source_artifact") or {}).get("intern") or ""), + "local coverage remains all selected sources", ) expect("journal_generate" in fallback_stages, "stage 2 still runs on the local artifact") narrate_fallback = fallback_stages.get("journal_generate", {}).get("intern") or "" expect("Heute um 7:30 Uhr Markt" in narrate_fallback, "first user source is rehydrated locally") - expect("Später noch der Hafen" in narrate_fallback, "omitted model source is still in the local artifact") + expect("Später noch der Hafen" in narrate_fallback, "second user source is still in the local artifact") expect( - [item.get("purpose") for item in skip_recorder] == ["journal_reconstruct", "journal_generate"], - "stage 2 is called with local coverage after invalid reconstruction", + [item.get("purpose") for item in skip_recorder] == ["journal_generate"], + "no reconstruct provider call after unused model JSON", ) reset_catalog() @@ -1010,17 +1034,74 @@ def test_generate_identity_leak_local_fallback(client: TestClient, headers: dict ) expect(gen.status_code == 200, f"identity leak still yields a local draft {gen.text}") payload = gen.json() + leak_log = payload.get("run_log") or (payload.get("trace") or {}).get("log") or [] + expect(any(item.get("kind") == "retry" for item in leak_log), "identity leak records a retry") + expect( + any(item.get("reason") == "identity_leak_blocked" for item in leak_log), + "identity leak records local fallback", + ) body = payload.get("body") or "" expect("Markt" in body and "Kirschen" in body, "local draft keeps attested user wording") expect("Anna" in body, "local draft may keep names that the user actually wrote") stages = {item.get("purpose"): item for item in (payload.get("trace") or {}).get("stages") or []} - reconstruct = stages.get("journal_reconstruct") or {} + reconstruct = stages.get("local_source_artifact") or {} narrate = stages.get("journal_generate") or {} - expect( - reconstruct.get("reason") == "identity_leak_blocked" or reconstruct.get("guard") == "identity_leak_blocked", - "stage 1 records the blocked leak", - ) + expect(reconstruct.get("status") == "local_ok", "stage 1 stays local and does not call reconstruct") expect(narrate.get("guard") == "identity_leak_blocked", "stage 2 does not use the leaking reply") + expect(sum(1 for item in leak_log if item.get("kind") == "retry") == 1, "active leak retries at most once") + expect(sum(1 for item in leak_log if item.get("kind") == "model_call") <= 2, "retry is the only extra model call") + + +def test_generate_inactive_mapping_keeps_model_text(client: TestClient, headers: dict, profile_id: str) -> None: + from identity_store import remember_mapping + from providers import ChatResult + + remember_mapping(profile_id, "Clarissa", "PERSON:99") + space = client.post("/api/journal/spaces", headers=headers, json={"title": "Inaktiv"}) + day = client.post( + f"/api/journal/spaces/{space.json()['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-26"}, + ) + conv = client.post( + f"/api/journal/days/{day.json()['day']['id']}/conversations", + headers=headers, + json={"title": "Tag"}, + ) + turn = client.post( + f"/api/journal/conversations/{conv.json()['id']}/turn", + headers=headers, + json={"body": "Heute Markt, danach Kirschen am Hafen."}, + ) + expect(turn.status_code == 200, f"inactive-mapping setup {turn.status_code}") + calls = {"n": 0} + + def keep(_messages, _policy): + calls["n"] += 1 + return ChatResult( + content="Ein Markttag\n\nHeute Markt, danach Kirschen am Hafen. Clarissa blieb unerwähnt.", + model="fake", + usage={}, + context_compression="disabled", + ) + + with patch("privacy_gateway.complete_model", keep): + gen = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(gen.status_code == 200, f"inactive mapping generate {gen.text}") + payload = gen.json() + run_log = payload.get("run_log") or (payload.get("trace") or {}).get("log") or [] + expect(calls["n"] == 1, "inactive mapping does not trigger a retry") + expect(not any(item.get("kind") == "retry" for item in run_log), "inactive mapping does not retry") + expect( + any(item.get("reason") == "unattested_identity" for item in run_log), + "invented historical name is unattested journal content", + ) + expect("Clarissa" not in (payload.get("body") or ""), "unattested name is not accepted as narration") + expect("Markt" in (payload.get("body") or "") and "Kirschen" in (payload.get("body") or ""), "fallback keeps user sources") def test_traces_are_request_scoped(client: TestClient, headers: dict) -> None: @@ -1167,6 +1248,7 @@ def main() -> None: test_task_brief_and_dedupe(client, headers, profile_id) test_generate_flow(client, headers) test_generate_identity_leak_local_fallback(client, headers, profile_id) + test_generate_inactive_mapping_keeps_model_text(client, headers, profile_id) test_traces_are_request_scoped(client, headers) test_two_conversations_same_day(client, headers) print("journal budget tests passed.") diff --git a/backend/tests/test_journal_editorial.py b/backend/tests/test_journal_editorial.py new file mode 100644 index 0000000..910914a --- /dev/null +++ b/backend/tests/test_journal_editorial.py @@ -0,0 +1,426 @@ +"""Deterministic journal editorial contract. Fake provider proves data flow, not live prose.""" +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-editorial-test.sqlite") +os.environ["KANSHO_FAKE_PROVIDER"] = "1" +os.environ["KANSHO_FAKE_DETECT"] = "1" +Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) + +from fastapi.testclient import TestClient +from db import get_db, init_db +from engine import load_active_prompt +from identity_store import remember_mapping +from journal_editorial import ( + NOTES_TO_JOURNAL, + PROSE_EDIT, + choose_editorial_mode, + editorial_instructions, + format_style_examples, + lexical_similarity, + narration_sources_text, + select_journal_style_examples, +) +from journal_generate import pack_narration_context, unattested_journal_content +from main import app +from privacy_gateway import reset_debug +from prompt_budget import JournalBudget +from providers import ChatResult +from writing_profile_store import ( + NEUTRAL_JOURNAL_STYLE, + compile_task_brief, + import_text, + set_lifecycle, + update_facet, +) + + +SEED_REVISION = "2026-08-27-journal-placeholders-v1" + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def header(token: str) -> dict: + return {"X-Auth-Token": token} + + +def test_mode_choice() -> None: + expect(choose_editorial_mode(["Ich ging zum Markt. Es war voll."]) == PROSE_EDIT, "narrative sentences select prose_edit") + expect(choose_editorial_mode(["markt", "kirschen", "hafen"]) == NOTES_TO_JOURNAL, "fragments select notes_to_journal") + expect( + choose_editorial_mode(["Ich war am Markt.", "Kirschen gekauft.", "hafen später"]) == PROSE_EDIT, + "mixed default is prose_edit when at least half the blocks have sentence punctuation", + ) + expect( + choose_editorial_mode(["markt", "kirschen", "Später der Hafen."]) == NOTES_TO_JOURNAL, + "mixed default is notes_to_journal when the majority lacks sentence punctuation", + ) + prose = editorial_instructions(PROSE_EDIT) + notes = editorial_instructions(NOTES_TO_JOURNAL) + expect("Gute Formulierungen bewahren" in prose, "prose_edit keeps good wording") + expect("zusammenhängende Journalprosa" in notes, "notes_to_journal asks for connected prose") + expect(prose != notes, "modes produce different instructions") + expect("nicht inklusive ihrer Fehler" in notes or "Fehler hintereinanderkopieren" in notes, "notes must not be concatenated with errors") + + +def test_prompt_contract() -> None: + init_db() + prompt = load_active_prompt("mvp.journal_generate") + text = prompt.get("template") or "" + expect("Faktentreue ist nicht Wortlauttreue" in text, "prompt separates fact fidelity from wording") + expect("CURRENT_DAY_SOURCES" in text, "prompt labels current-day facts") + expect("WRITING_PROFILE" in text, "prompt labels the writing profile") + expect("STYLE_EXAMPLES" in text, "prompt labels style examples") + expect("EDITORIAL_MODE" in text, "prompt exposes editorial mode") + expect("keine geschützten Fakten" in text, "prompt says typos are not protected facts") + expect("{{editorial_instructions}}" in text, "prompt injects mode-specific instructions") + expect("{{style_examples}}" in text, "prompt injects style examples") + expect("[[…" not in text and "[[..." not in text, "prompt must not teach ellipsis placeholders") + expect("zeichengetreu" in text, "prompt asks to copy existing placeholders unchanged") + expect("ich gieng zum laden" in text, "prompt includes a synthetic prose_edit example") + expect("Im Laden holte ich Brot" in text, "prompt includes a synthetic notes_to_journal example") + expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone") + expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone") + with get_db() as conn: + row = conn.execute( + "SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?", + ("mvp.journal_generate",), + ).fetchone() + expect(row["seed_revision"] == SEED_REVISION, "system prompt revision is stored") + expect(row["template"] == row["default_template"], "untouched install uses the seeded template") + + +def test_sources_and_examples_are_separated() -> None: + artifact = { + "kind": "verified_artifact", + "coverage": "all_selected_sources", + "sources": [{"source_id": "u1", "role": "user", "text": "Heute nur der Markt."}], + } + presented = narration_sources_text(artifact) + expect("[u1]" in presented and "Heute nur der Markt." in presented, "day facts are labeled source blocks") + expect("kind" not in presented, "narration does not dump artifact JSON") + formatted = format_style_examples( + [{"kind": "journal_entry", "excerpt": "Ruhige Sätze, trockener Schnitt.", "occurred_at": "2026-07-01"}] + ) + expect("nicht übernommen" in formatted, "style examples forbid copying their facts") + expect("Ruhige Sätze" in formatted, "selected example text is present") + + +def test_unattested_covers_title_and_body() -> None: + mappings = [{"local_label": "Maren", "token": "PERSON:01"}, {"local_label": "Hanna", "token": "PERSON:99"}] + sources = ["Heute war ich mit Maren am Markt."] + expect( + unattested_journal_content("Markttag\n\nIch ging mit Maren zum Markt.", sources, mappings, ["PERSON:01"]) is None, + "attested person in title and body is allowed", + ) + expect( + unattested_journal_content("Hanna am Hafen\n\nIch ging zum Markt.", sources, mappings, ["PERSON:01"]) + == "unattested_identity", + "unattested person in the title is rejected", + ) + expect( + unattested_journal_content("[[PERSON:99]]\n\nIch ging zum Markt.", sources, mappings, ["PERSON:01"]) + == "unattested_placeholder", + "inactive placeholder in the title is unattested", + ) + expect( + unattested_journal_content("[[PERSON:01]] am Markt\n\n[[PERSON:01]] kaufte Kirschen.", sources, mappings, ["PERSON:01"]) + is None, + "the same attested placeholder may appear in title and body", + ) + expect( + unattested_journal_content("Um 6:30 Uhr kam [[...]] ins Wohnzimmer.", sources, mappings, ["PERSON:01"]) + == "unattested_placeholder", + "ellipsis placeholder leftover is unattested", + ) + expect( + unattested_journal_content("Um 6:30 Uhr kam [[…]] ins Wohnzimmer.", sources, mappings, ["PERSON:01"]) + == "unattested_placeholder", + "unicode ellipsis placeholder leftover is unattested", + ) + + +def test_budget_pack_drops_examples_first() -> None: + init_db() + prompt = { + "id": "eval-pack", + "slug": "mvp.journal_generate", + "prompt_type": "base", + "template": "PROFILE\n{{writing_profile}}\nEX\n{{style_examples}}\nDAY\n{{reconstruction}}\nOLD\n{{existing_text}}\n", + } + assembled = { + "writing_profile": "Core: kurze Sätze.", + "reconstruction": "Heute Markt.", + "editorial_mode": PROSE_EDIT, + "editorial_instructions": "x", + "style_examples": "", + "existing_text": "", + } + huge_examples = "Stil " + ("Beispielwort " * 400) + existing = "Bestehende Fassung " + ("alt " * 40) + budget = JournalBudget( + model="fake", + purpose="journal_generate", + effective_context_window=32_768, + reserved_output_tokens=256, + safety_margin=0.15, + available_input_tokens=estimate_cap(assembled, prompt, extra=80), + chars_per_token=2.0, + ) + packed, dropped = pack_narration_context( + prompt, + budget, + assembled, + style_examples=huge_examples, + existing_text=existing, + include_existing=True, + ) + expect("style_examples" in dropped, "style examples are dropped before day sources") + expect("Heute Markt." in packed["reconstruction"], "day sources stay") + expect("kurze Sätze" in packed["writing_profile"], "writing profile stays") + + +def estimate_cap(assembled: dict, prompt: dict, extra: int) -> int: + from engine import preview_prompt + from prompt_budget import estimate_tokens + + base = dict(assembled) + rendered = preview_prompt(prompt, base)["rendered"] + return estimate_tokens(rendered) + extra + + +def intern_of(payload: dict) -> str: + for stage in (payload.get("trace") or {}).get("stages") or []: + if stage.get("purpose") == "journal_generate": + return stage.get("intern") or "" + return (payload.get("trace") or {}).get("intern") or "" + + +def main() -> None: + test_mode_choice() + test_prompt_contract() + test_sources_and_examples_are_separated() + test_unattested_covers_title_and_body() + test_budget_pack_drops_examples_first() + expect(lexical_similarity("a b c", "a b c") > 0.9, "similarity helper is diagnostic") + expect(NEUTRAL_JOURNAL_STYLE.startswith("Neutraler Journalstil"), "neutral fallback exists") + + reset_debug() + with TestClient(app) as client: + setup = client.post( + "/api/auth/setup", + json={"email": "ada@example.test", "name": "Ada", "password": "test-pass"}, + ) + headers = header(setup.json()["token"]) + profile_id = setup.json()["profile_id"] + expect(compile_task_brief(profile_id) == NEUTRAL_JOURNAL_STYLE, "unconfirmed profile is not a style authority") + + space = client.post("/api/journal/spaces", headers=headers, json={"title": "Editorial"}) + day = client.post( + f"/api/journal/spaces/{space.json()['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-26"}, + ) + conv = client.post( + f"/api/journal/days/{day.json()['day']['id']}/conversations", + headers=headers, + json={"title": "Tag"}, + ) + prose_body = ( + "ich bin dan zum markt gegangen und da war es zimlich voll. " + "vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. " + "die rote tasche lag im auto. ich bin dan zum markt gegangen." + ) + turn = client.post( + f"/api/journal/conversations/{conv.json()['id']}/turn", + headers=headers, + json={"body": prose_body}, + ) + expect(turn.status_code == 200, f"turn {turn.status_code}") + gen = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(gen.status_code == 200, f"generate {gen.status_code}") + intern = intern_of(gen.json()) + expect("EDITORIAL_MODE: prose_edit" in intern, "narrative source selects prose_edit") + expect("Gute Formulierungen bewahren" in intern, "prose_edit instructions reach the model") + expect("CURRENT_DAY_SOURCES" in intern, "day facts are labeled") + expect("STYLE_EXAMPLES" in intern, "style examples are labeled") + expect("WRITING_PROFILE" in intern, "writing profile is labeled") + expect("Neutraler Journalstil" in intern, "without a confirmed profile the neutral fallback is used") + expect("zimlich" in intern.split("CURRENT_DAY_SOURCES")[-1], "today remains content, including typos") + expect("zimlich" not in intern.split("CURRENT_DAY_SOURCES")[0], "today is not a style authority") + expect(sum(1 for item in (gen.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "normal path is one generate call") + expect((gen.json().get("trace") or {}).get("editorial_mode") == PROSE_EDIT, "editorial mode is in the admin trace") + + notes_day = client.post( + f"/api/journal/spaces/{space.json()['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-25"}, + ) + notes_conv = client.post( + f"/api/journal/days/{notes_day.json()['day']['id']}/conversations", + headers=headers, + json={"title": "Notizen"}, + ) + client.post( + f"/api/journal/conversations/{notes_conv.json()['id']}/turn", + headers=headers, + json={"body": "markt\nkirschen\nspäter hafen"}, + ) + notes_gen = client.post( + f"/api/journal/days/{notes_day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [notes_conv.json()["id"]]}, + ) + expect(notes_gen.status_code == 200, f"notes generate {notes_gen.text}") + notes_intern = intern_of(notes_gen.json()) + expect("EDITORIAL_MODE: notes_to_journal" in notes_intern, "fragments select notes_to_journal") + expect("zusammenhängende Journalprosa" in notes_intern, "notes mode reaches the model") + expect("Gute Formulierungen bewahren" not in notes_intern, "prose_edit instructions are not used for notes") + + import_text( + profile_id, + "Ich schreibe in kurzen, ruhigen Sätzen und lasse den Tag stehen. Der Hafen blieb hinter der Fähre.", + occurred_at="2026-07-01", + ) + examples = select_journal_style_examples(profile_id, exclude_dates=["2026-08-26"]) + expect(examples and "ruhigen Sätzen" in examples[0]["excerpt"], "imported text is a style example when no finals exist") + expect(all("zimlich" not in (item.get("excerpt") or "") for item in examples), "current day is not selected as style") + + saved = client.post( + "/api/journal/entries", + headers=headers, + json={ + "journal_day_id": notes_day.json()["day"]["id"], + "title": "Eigene Fassung", + "body": "Heute blieb ich beim klaren Schnitt und schrieb den Markt in eigenen Worten, ohne Pathos.", + "origin": "user_edit", + "source_conversation_ids": [notes_conv.json()["id"]], + }, + ) + expect(saved.status_code == 200, f"save {saved.text}") + ranked = select_journal_style_examples(profile_id, exclude_dates=["2026-08-26"]) + expect(ranked and ranked[0]["kind"] == "journal_entry", "final journal text outranks imports") + expect("klaren Schnitt" in ranked[0]["excerpt"], "user-edited journal text is the preferred example") + expect(len(ranked) <= 2, "at most two style examples") + + update_facet(profile_id, "core", value="Kurze Sätze, trockener Schnitt, keine Pathoswolken.") + expect("Core:" not in compile_task_brief(profile_id), "unconfirmed core is not a style authority") + set_lifecycle(profile_id, "confirmed") + with_core = compile_task_brief(profile_id) + expect("Core:" in with_core and "trockener Schnitt" in with_core, "confirmed core reaches the task brief") + expect("Neutraler Journalstil" not in with_core, "confirmed profile replaces the neutral fallback") + + remember_mapping(profile_id, "Hanna", "PERSON:99") + + def rewritten(_messages, _policy): + return ChatResult( + content=( + "Markttag\n\n" + "Ich ging dann zum Markt, auf dem es ziemlich voll war. " + "Vielleicht bleibe ich kürzer. " + "Brot holen wollte ich noch, habe es aber nicht gemacht. " + "Die rote Tasche lag im Auto." + ), + model="fake", + usage={}, + context_compression="disabled", + ) + + with patch("privacy_gateway.complete_model", rewritten): + second = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(second.status_code == 200, f"rewritten generate {second.text}") + body = second.json().get("body") or "" + expect("ziemlich voll" in body, "spelling is corrected") + expect("zimlich" not in body, "source typo is not kept") + expect("Ich ging dann zum Markt" in body, "good rewritten wording may stay") + expect("Vielleicht bleibe ich kürzer" in body, "uncertainty stays uncertainty") + expect("wollte" in body.lower() and "nicht gemacht" in body, "plan is not turned into completion") + expect("nicht gemacht" in body, "negation is kept") + expect("rote Tasche" in body, "unique attested detail is kept") + expect(body.lower().count("markt") <= 2, "repetition is allowed to be reduced") + expect("traurig" not in body.lower() and "weil" not in body.lower(), "no invented feeling or cause in the patched rewrite") + intern2 = intern_of(second.json()) + expect("trockener Schnitt" in intern2.split("CURRENT_DAY_SOURCES")[0], "confirmed writing profile reaches generate") + expect("Hafen blieb hinter der Fähre" not in body, "historical style facts are not copied into today") + expect( + sum(1 for item in (second.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, + "high similarity does not trigger a retry", + ) + expect((second.json().get("trace") or {}).get("lexical_similarity") is not None, "similarity is a diagnostic only") + + def notes_prose(_messages, _policy): + return ChatResult( + content="Notizen\n\nAm Markt holte ich Kirschen. Später war ich am Hafen.", + model="fake", + usage={}, + context_compression="disabled", + ) + + with patch("privacy_gateway.complete_model", notes_prose): + notes_out = client.post( + f"/api/journal/days/{notes_day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [notes_conv.json()["id"]]}, + ) + notes_body = notes_out.json().get("body") or "" + expect("Am Markt holte ich Kirschen" in notes_body, "notes become connected prose") + expect("kirschen\nspäter" not in notes_body.lower(), "notes are not concatenated as fragments") + + def invent_title(_messages, _policy): + return ChatResult( + content="Hanna am Hafen\n\nIch ging zum Markt, auf dem es ziemlich voll war.", + model="fake", + usage={}, + context_compression="disabled", + ) + + with patch("privacy_gateway.complete_model", invent_title): + blocked = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(any(item.get("reason") == "unattested_identity" for item in (blocked.json().get("run_log") or [])), "title identity uses local fallback") + expect("Hanna" not in (blocked.json().get("title") or "") and "Hanna" not in (blocked.json().get("body") or ""), "unattested title identity is not kept") + + with get_db() as conn: + conn.execute( + "UPDATE ai_prompts SET template = 'CUSTOM JOURNAL PROMPT' WHERE slug = ?", + ("mvp.journal_generate",), + ) + init_db() + custom = load_active_prompt("mvp.journal_generate") + expect(custom["template"] == "CUSTOM JOURNAL PROMPT", "independently edited prompt is not overwritten") + with get_db() as conn: + row = conn.execute( + "SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?", + ("mvp.journal_generate",), + ).fetchone() + expect("Faktentreue ist nicht Wortlauttreue" in (row["default_template"] or ""), "default template still tracks the seed") + expect(row["seed_revision"] == SEED_REVISION, "revision updates even when template is custom") + + print("journal editorial tests passed.") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_journal_eval.py b/backend/tests/test_journal_eval.py new file mode 100644 index 0000000..e9870c5 --- /dev/null +++ b/backend/tests/test_journal_eval.py @@ -0,0 +1,62 @@ +"""Evaluation harness contract. Does not prove live model quality.""" +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-eval-test.sqlite") +os.environ["KANSHO_FAKE_PROVIDER"] = "1" +os.environ["KANSHO_FAKE_DETECT"] = "1" +Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) + +from db import init_db +from journal_eval import ( + SYNTHETIC_PROSE, + SYNTHETIC_TYPOS, + VARIANT_BASELINE, + VARIANT_CURRENT, + VARIANT_PREVIOUS, + score_output, + synthetic_context, + variant_templates, +) + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def main() -> None: + init_db() + templates = variant_templates() + expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist") + expect("Überarbeite diesen Rohtext" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt") + expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt") + expect("Faktentreue ist nicht Wortlauttreue" in templates[VARIANT_CURRENT], "current variant has the new contract") + + context = synthetic_context(SYNTHETIC_PROSE) + expect(context["editorial_mode"] in {"prose_edit", "notes_to_journal"}, "eval context has an editorial mode") + copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS) + expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1") + expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos") + improved = score_output( + SYNTHETIC_PROSE, + "Ich ging dann zum Markt, auf dem es ziemlich voll war. Vielleicht bleibe ich kürzer.", + typos=SYNTHETIC_TYPOS, + ) + expect("zimlich" in improved["spelling_typos_fixed"], "metrics record typo fixes") + expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical") + expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored") + expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures") + print("journal eval tests passed.") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_journal_narration.py b/backend/tests/test_journal_narration.py new file mode 100644 index 0000000..cdf05e1 --- /dev/null +++ b/backend/tests/test_journal_narration.py @@ -0,0 +1,299 @@ +"""Journal narration contract: facts stay, wording may change.""" +from __future__ import annotations + +import json +import os +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-narration-test.sqlite") +os.environ["KANSHO_FAKE_PROVIDER"] = "1" +os.environ["KANSHO_FAKE_DETECT"] = "1" +Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) + +from fastapi.testclient import TestClient +from db import get_db, init_db +from engine import load_active_prompt +from identity_store import remember_mapping +from journal_generate import unattested_journal_content +from journal_shape import shape_journal +from main import app +from privacy_gateway import reset_debug +from providers import ChatResult +from writing_profile_store import ( + NEUTRAL_JOURNAL_STYLE, + compile_task_brief, + get_profile, + import_text, + set_lifecycle, + update_facet, +) + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def header(token: str) -> dict: + return {"X-Auth-Token": token} + + +def test_active_prompt_contract() -> None: + init_db() + prompt = load_active_prompt("mvp.journal_generate") + text = prompt.get("template") or "" + expect("Faktentreue ist nicht Wortlauttreue" in text, "active prompt separates fact fidelity from wording") + expect("CURRENT_DAY_SOURCES" in text, "active prompt labels current-day facts") + expect("STYLE_EXAMPLES" in text, "active prompt labels style examples") + expect("Rechtschreibung" in text or "korrigieren" in text, "active prompt allows spelling and grammar fixes") + expect("Unsicherheit bleibt Unsicherheit" in text, "active prompt keeps semantic uncertainty") + expect("keine neuen Informationen" in text.lower() or "Keine neuen Informationen" in text, "active prompt still forbids new facts") + expect("[[…" not in text and "[[..." not in text, "active prompt must not teach ellipsis placeholders") + expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone") + expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone") + with get_db() as conn: + row = conn.execute( + "SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?", + ("mvp.journal_generate",), + ).fetchone() + expect(row["seed_revision"] == "2026-08-27-journal-placeholders-v1", "system prompt revision is stored") + expect(row["template"] == row["default_template"], "untouched install uses the seeded template") + + +def test_shape_keeps_rewritten_model_text() -> None: + source = ["ich bin dan zum markt gegangen und da war es zimlich voll"] + model = "Ich ging dann zum Markt, auf dem es ziemlich voll war." + _, body = shape_journal("Markttag", model, source) + expect(body == model, "shape_journal keeps the corrected model text") + expect("zimlich" not in body, "source typo is not restored") + + +def test_shape_does_not_restore_source_form() -> None: + source = ["ich ging zum markt. danach der hafen."] + _, punct = shape_journal("Tag", "Ich ging zum Markt. Danach der Hafen.", source) + expect(punct == "Ich ging zum Markt. Danach der Hafen.", "corrected punctuation is not reset") + _, casing = shape_journal("Tag", "Am Markt war es voll.", ["am markt war es voll"]) + expect(casing == "Am Markt war es voll.", "corrected capitalization is not reset") + rewritten = "Später am Hafen packte ich die rote Tasche." + _, structure = shape_journal("Tag", rewritten, ["danach hafen. rote tasche eingepackt."]) + expect(structure == rewritten, "changed sentence structure is not reset") + fallback_parts = ["Zeile eins Markt.", "Zeile zwei Hafen und rote Tasche."] + _, fallback = shape_journal("Ein Tag", "\n\n".join(fallback_parts), fallback_parts, source="fallback") + expect("Markt" in fallback and "Hafen" in fallback and "rote Tasche" in fallback, "fallback keeps every unique user source") + expect(fallback == "\n\n".join(fallback_parts) or "Markt" in fallback, "fallback path stays source-based") + model_kept = "Ich war am Markt und später am Hafen." + _, model = shape_journal("Tag", model_kept, fallback_parts, source="model") + expect(model == model_kept, "model path is not mixed with the fallback join") + + +def test_unattested_identity_is_journal_not_privacy() -> None: + mappings = [ + {"local_label": "Anna", "token": "PERSON:01"}, + {"local_label": "Clarissa", "token": "PERSON:99"}, + ] + sources = ["Heute war ich mit Anna am Markt."] + expect( + unattested_journal_content("Ich ging mit Anna zum Markt.", sources, mappings, ["PERSON:01"]) is None, + "attested person in rewritten prose is allowed", + ) + expect( + unattested_journal_content("Clarissa kaufte Kirschen am Markt.", sources, mappings, ["PERSON:01"]) + == "unattested_identity", + "invented person as subject of a verb is still unattested", + ) + expect( + unattested_journal_content("[[PERSON:99]] stand am Markt.", sources, mappings, ["PERSON:01"]) + == "unattested_placeholder", + "inactive placeholder is unattested content", + ) + expect( + unattested_journal_content("Heute nur von Sushi essen erzählt.", sources, [{"local_label": "Sushi", "token": "PERSON:01"}], []) + is None, + "food homonym of an unused mapping is not unattested identity", + ) + + +def main() -> None: + test_active_prompt_contract() + test_shape_keeps_rewritten_model_text() + test_shape_does_not_restore_source_form() + test_unattested_identity_is_journal_not_privacy() + expect(NEUTRAL_JOURNAL_STYLE.startswith("Neutraler Journalstil"), "neutral fallback exists") + + reset_debug() + with TestClient(app) as client: + setup = client.post( + "/api/auth/setup", + json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"}, + ) + headers = header(setup.json()["token"]) + profile_id = setup.json()["profile_id"] + empty_brief = compile_task_brief(profile_id, "journal_generate") + expect(empty_brief == NEUTRAL_JOURNAL_STYLE, "empty profile uses the neutral journal voice") + space = client.post("/api/journal/spaces", headers=headers, json={"title": "Narration"}) + day = client.post( + f"/api/journal/spaces/{space.json()['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-26"}, + ) + conv = client.post( + f"/api/journal/days/{day.json()['day']['id']}/conversations", + headers=headers, + json={"title": "Tag"}, + ) + turn = client.post( + f"/api/journal/conversations/{conv.json()['id']}/turn", + headers=headers, + json={"body": "ich bin dan zum markt gegangen und da war es zimlich voll. vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. die rote tasche lag im auto. ich bin dan zum markt gegangen."}, + ) + expect(turn.status_code == 200, f"turn {turn.status_code}") + before = get_profile(profile_id) + gen = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(gen.status_code == 200, f"generate {gen.text}") + intern = "" + for stage in (gen.json().get("trace") or {}).get("stages") or []: + if stage.get("purpose") == "journal_generate": + intern = stage.get("intern") or "" + style = intern.split("CURRENT_DAY_SOURCES")[0] + expect("Neutraler Journalstil" in style, "neutral style reaches the generate prompt") + expect("Erzählmerkmale" not in style, "current day dialogue is not a style brief") + expect("zimlich" not in style, "today's typo is not a style exemplar") + expect("Faktentreue ist nicht Wortlauttreue" in intern, "runtime intern uses the new narration contract") + expect("CURRENT_DAY_SOURCES" in intern, "runtime intern labels current-day facts") + expect("Nur den verifizierten Nutzerwortlaut" not in intern, "old output-wording rule is not in the runtime prompt") + expect("Unsicherheiten im Wortlaut" not in intern, "old uncertainty-in-wording rule is not in the runtime prompt") + after = get_profile(profile_id) + expect(after.get("version") == before.get("version"), "generate does not create a new profile version") + expect((after.get("suggestions") or []) == (before.get("suggestions") or []), "generate does not add suggestions") + + import_text(profile_id, "Ich schreibe in kurzen, ruhigen Sätzen und lasse den Tag stehen.") + from journal_editorial import select_journal_style_examples + + imported_examples = select_journal_style_examples(profile_id) + expect( + imported_examples and "kurzen, ruhigen Sätzen" in imported_examples[0]["excerpt"], + "imported own text is a style source", + ) + imported_brief = compile_task_brief(profile_id, "journal_generate") + expect(imported_brief == NEUTRAL_JOURNAL_STYLE, "unconfirmed profile does not become the brief") + expect("zimlich" not in " ".join(item["excerpt"] for item in imported_examples), "today's dialogue is not mixed into the imported style") + + saved = client.post( + "/api/journal/entries", + headers=headers, + json={ + "journal_day_id": day.json()["day"]["id"], + "title": "Eigene Fassung", + "body": "Heute blieb ich beim klaren Schnitt und schrieb den Markt in eigenen Worten.", + "origin": "user_edit", + "source_conversation_ids": [conv.json()["id"]], + }, + ) + expect(saved.status_code == 200, f"save {saved.text}") + entry_examples = select_journal_style_examples(profile_id) + expect( + any("klaren Schnitt" in (item.get("excerpt") or "") for item in entry_examples), + "final user-edited journal text is a style source", + ) + expect(all("zimlich" not in (item.get("excerpt") or "") for item in entry_examples), "current day dialogue is not a positive style reference") + + update_facet(profile_id, "core", value="Kurze Sätze, trockener Schnitt, keine Pathoswolken.") + set_lifecycle(profile_id, "confirmed") + with_core = compile_task_brief(profile_id, "journal_generate") + expect("Core:" in with_core and "trockener Schnitt" in with_core, "confirmed core reaches the task brief") + expect("Neutraler Journalstil" not in with_core, "confirmed profile replaces the neutral fallback") + + remember_mapping(profile_id, "Clarissa", "PERSON:99") + + def rewritten(_messages, _policy): + return ChatResult( + content=( + "Markttag\n\n" + "Ich ging dann zum Markt, auf dem es ziemlich voll war. " + "Vielleicht bleibe ich kürzer. " + "Brot holen wollte ich noch, habe es aber nicht gemacht. " + "Die rote Tasche lag im Auto." + ), + model="fake", + usage={}, + context_compression="disabled", + ) + + with patch("privacy_gateway.complete_model", rewritten): + second = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(second.status_code == 200, f"rewritten generate {second.text}") + body = second.json().get("body") or "" + expect("ziemlich voll" in body, "corrected spelling of the model text is kept") + expect("zimlich" not in body, "source typo is not restored after a real model rewrite") + expect("Vielleicht bleibe ich kürzer" in body, "semantic uncertainty is kept") + expect("wollte" in body.lower() and "nicht gemacht" in body, "plan versus non-fulfillment stays") + expect("rote Tasche" in body, "unique attested detail is kept after rewrite") + expect(body.lower().count("markt") <= 2, "repeated source lines are not pasted twice") + expect( + any(item.get("status") == "model" for item in (second.json().get("run_log") or [])), + "accepted model path is recorded as model", + ) + intern2 = "" + for stage in (second.json().get("trace") or {}).get("stages") or []: + if stage.get("purpose") == "journal_generate": + intern2 = stage.get("intern") or "" + expect("trockener Schnitt" in intern2.split("CURRENT_DAY_SOURCES")[0], "confirmed writing profile reaches generate") + + def invent(_messages, _policy): + return ChatResult( + content="Clarissa blieb den ganzen Nachmittag am Hafen.", + model="fake", + usage={}, + context_compression="disabled", + ) + + with patch("privacy_gateway.complete_model", invent): + blocked = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(blocked.status_code == 200, f"unattested generate {blocked.text}") + log = blocked.json().get("run_log") or [] + expect(any(item.get("reason") == "unattested_identity" for item in log), "unattested person uses local fallback") + expect(any(item.get("status") == "local_fallback" for item in log), "fallback path is distinct from the model path") + expect("Clarissa" not in (blocked.json().get("body") or ""), "unattested person is not in the accepted draft") + expect("markt" in (blocked.json().get("body") or "").lower(), "fallback still has the user sources") + expect("rote tasche" in (blocked.json().get("body") or "").lower(), "fallback keeps the unique source detail") + + with get_db() as conn: + conn.execute( + "UPDATE ai_prompts SET template = 'CUSTOM JOURNAL PROMPT' WHERE slug = ?", + ("mvp.journal_generate",), + ) + init_db() + custom = load_active_prompt("mvp.journal_generate") + expect(custom["template"] == "CUSTOM JOURNAL PROMPT", "independently edited prompt is not overwritten") + with get_db() as conn: + row = conn.execute( + "SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?", + ("mvp.journal_generate",), + ).fetchone() + expect("Faktentreue ist nicht Wortlauttreue" in (row["default_template"] or ""), "default template still tracks the seed") + expect(row["seed_revision"] == "2026-08-27-journal-placeholders-v1", "revision updates even when template is custom") + + print("journal narration tests passed.") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_journal_opening.py b/backend/tests/test_journal_opening.py index 6301598..0976e3d 100644 --- a/backend/tests/test_journal_opening.py +++ b/backend/tests/test_journal_opening.py @@ -206,17 +206,17 @@ def main() -> None: expect(draft.status_code == 200, f"generate after opening {draft.text}") reconstruct = None for stage in (draft.json().get("trace") or {}).get("stages") or []: - if stage.get("purpose") == "journal_reconstruct": + if stage.get("purpose") == "local_source_artifact": reconstruct = stage intern = (reconstruct or {}).get("intern") or "" - expect("Anna kaufte Brot" in intern, "user source reaches reconstruction") - expect("[u1]" in intern, "user source id assigned") + expect("Anna kaufte Brot" in intern, "user source reaches local artifact") + expect("u1" in intern, "user source id assigned") narrate = None for stage in (draft.json().get("trace") or {}).get("stages") or []: if stage.get("purpose") == "journal_generate": narrate = stage narrate_intern = (narrate or {}).get("intern") or "" - expect("verified_artifact" in narrate_intern or "sources" in narrate_intern, "stage 2 consumes verified artifact") + expect("CURRENT_DAY_SOURCES" in narrate_intern or "Anna kaufte Brot" in narrate_intern, "stage 2 consumes day sources") expect("Nach der Fähre kamen wir spät an" in intern, "user wording not dropped after opening") expect(users[0]["body"].startswith("Nach der Fähre"), "user source is the user message, not the opening") diff --git a/backend/tests/test_journal_shape.py b/backend/tests/test_journal_shape.py index 3ff7a1b..97c05f6 100644 --- a/backend/tests/test_journal_shape.py +++ b/backend/tests/test_journal_shape.py @@ -28,8 +28,24 @@ def main() -> None: title, body = shape_journal("Markttag", joined, turns) expect(title == "Markttag", "title stays") - expect(body.count("\n\n") == 2, "paste becomes one paragraph per turn") - expect("Kirschen" in body and "Hafen" in body, "wording of a paste is kept") + expect("Kirschen" in body and "Hafen" in body, "model wording is kept") + expect(body != "\n\n".join(turns), "accepted model text is not replaced by source turns") + + source_typo = ["ich bin dan zum markt gegangen und da war es zimlich voll"] + rewritten = "Ich ging dann zum Markt, auf dem es ziemlich voll war." + _, kept_rewrite = shape_journal("Markttag", rewritten, source_typo) + expect("ziemlich" in kept_rewrite, "corrected spelling stays") + expect("zimlich" not in kept_rewrite, "source typo is not restored") + expect(kept_rewrite.startswith("Ich ging"), "corrected sentence structure stays") + expect("Markt" in kept_rewrite, "capitalization stays corrected") + + _, punct = shape_journal("Tag", "Ich ging zum Markt.", ["ich ging zum markt"]) + expect(punct == "Ich ging zum Markt.", "corrected punctuation and casing are not reset") + + fallback_body = "\n\n".join(turns) + _, fallback = shape_journal("Ein Tag", fallback_body, turns, source="fallback") + expect("Kirschen" in fallback and "Hafen" in fallback, "fallback still contains every user source") + expect("Markt" in fallback, "fallback keeps first source") blob = ( "Heute war der Markt voll. Ich bin früh gegangen. " diff --git a/backend/tests/test_mvp_journal.py b/backend/tests/test_mvp_journal.py index 42283b2..b913a57 100644 --- a/backend/tests/test_mvp_journal.py +++ b/backend/tests/test_mvp_journal.py @@ -151,8 +151,9 @@ def main() -> None: impulse, decision = parse_turn_payload('{"operation":"konkretisieren","impulse":"Wo war das?"}') expect(impulse == "Wo war das?" and decision["parsed"] is True, "operation json parsed") - hidden = visible_for_role({"trace": {"egress": "x"}, "decision": decision, "calls": 1}, "user") + hidden = visible_for_role({"trace": {"egress": "x"}, "run_log": [{"kind": "detect"}], "decision": decision, "calls": 1}, "user") expect("trace" not in hidden and "decision" not in hidden, "user does not see test spur") + expect(hidden.get("run_log"), "compact generate log remains visible") expect(is_plot_continuation("Und dann mit dem Tee auf dem Balkon?"), "echo then-question rejected") expect(is_plot_continuation("Ihr seid dann zur Bootstour aufgebrochen."), "invented departure rejected") expect(is_plot_continuation("Und dann seid ihr zur Wanderung aufgebrochen."), "invented hike rejected") @@ -313,20 +314,25 @@ def main() -> None: expect(draft.status_code == 200, f"A generate {draft.text}") expect("Markt" in draft.json()["body"] or "Kirschen" in draft.json()["body"], "A draft keeps user words") stages_a = {item.get("purpose"): item for item in (draft.json().get("trace") or {}).get("stages") or []} - reconstruct_a = stages_a.get("journal_reconstruct") + reconstruct_a = stages_a.get("local_source_artifact") last_a = stages_a.get("journal_generate") or (draft.json().get("trace") or {}) - expect(reconstruct_a is not None, "A stage 1 reconstruct ran") + expect(reconstruct_a is not None, "A stage 1 local artifact ran") reconstruct_intern = reconstruct_a.get("intern") or "" narrate_intern = last_a.get("intern") or "" - expect("Kirschen" in reconstruct_intern or "Markt" in reconstruct_intern, "A reconstruct sees user words") - expect("user:" in reconstruct_intern, "A reconstruct sees the day as dialogue") + expect("Kirschen" in reconstruct_intern or "Markt" in reconstruct_intern, "A local artifact sees user words") + expect("assistant:" not in reconstruct_intern, "A local artifact has no assistant lines") expect("assistant:" not in narrate_intern, "A stage 2 does not resend the day dialogue") - expect("verified_artifact" in narrate_intern or "sources" in narrate_intern, "A stage 2 receives the reconstruction") - expect("Stilquellen" in narrate_intern or "Dialog" in narrate_intern or "Erzählmerkmale" in narrate_intern or "Core:" in narrate_intern, "A style brief reaches generate") - style_part = narrate_intern - expect("Erzählmerkmale" in style_part, "A first generate still gets form hints") + expect("CURRENT_DAY_SOURCES" in narrate_intern or "[u1]" in narrate_intern, "A stage 2 receives the day sources") expect( - "Heute war der Markt voll" not in style_part.split("Verified Artifact:")[0], + "WRITING_PROFILE" in narrate_intern or "Neutraler Journalstil" in narrate_intern or "Core:" in narrate_intern, + "A style brief reaches generate", + ) + expect("STYLE_EXAMPLES" in narrate_intern, "A style examples block is labeled") + style_part = narrate_intern.split("CURRENT_DAY_SOURCES")[0] + expect("Erzählmerkmale" not in style_part, "A current day dialogue is not appended as style signals") + expect("Neutraler Journalstil" in style_part or "Core:" in style_part, "A empty profile uses the neutral journal voice") + expect( + "Heute war der Markt voll" not in style_part, "A source dialogue is not pasted a second time as style", ) current_version = None @@ -364,7 +370,7 @@ def main() -> None: expect(day_again.json()["day"]["calendar_date"] == "2026-08-20", "B journal day stays calendar day") msgs = client.get(f"/api/journal/conversations/{conv['id']}", headers=headers).json()["messages"] expect(any("Wind" in item["body"] for item in msgs), "B later message kept") - expect(msgs[-1]["created"] != "2026-08-20" or True, "B message time is independent") + expect(not str(msgs[-1]["created"]).startswith("2026-08-20"), "B message time is independent of the calendar day") prior_space, prior_day = open_space_day(client, headers, "Urlaub", "2026-08-01") prior_conv, _ = start_and_turn( @@ -510,7 +516,9 @@ def main() -> None: if stage.get("purpose") == "journal_generate": intern_i = stage.get("intern") or "" intern_i = intern_i or ((gen_i.json().get("trace") or {}).get("intern") or "") - expect("knapp" in intern_i, "I style brief reaches generate") + expect("STYLE_EXAMPLES" in intern_i, "I style examples reach generate") + expect("WRITING_PROFILE" in intern_i or "Neutraler Journalstil" in intern_i, "I writing profile reaches generate") + expect("knapp" in imported.json()["compiled_brief"], "I stored brief still lists the import") profile = client.get("/api/journal/writing-profile", headers=headers).json() expect(all(item.get("kind") != "journal_draft" for item in profile["sources"]), "I no draft kind") expect(not any((item.get("body") or "") == gen_i.json()["body"] for item in profile["sources"]), "I draft not a style source") @@ -586,7 +594,7 @@ def main() -> None: if stage.get("purpose") == "journal_generate": with_intern = stage.get("intern") or "" expect(marker in with_intern, "include_existing true sends the saved body only after explicit flag") - existing_section = with_intern.split("Bestehender Text")[-1].split("Verified Artifact")[0] + existing_section = with_intern.split("EXISTING_TEXT")[-1] expect(marker in existing_section, "saved Fassung is the included existing text, not a leftover draft") after_with = client.get(f"/api/journal/entries/{saved.json()['id']}", headers=headers).json() expect(after_with["current_version_id"] == marked_version, "include_existing generate still does not overwrite") diff --git a/backend/tests/test_privacy_detect.py b/backend/tests/test_privacy_detect.py index b30bf28..064776b 100644 --- a/backend/tests/test_privacy_detect.py +++ b/backend/tests/test_privacy_detect.py @@ -1,10 +1,17 @@ -"""Detection + mask + demask. Run from backend/: python tests/test_privacy_detect.py""" +"""Semantic detection contracts. Fake provider proves schema and fail-closed, not live quality. + +Run from backend/: python tests/test_privacy_detect.py +""" from __future__ import annotations +import json import os import sys import tempfile +import threading +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) @@ -15,9 +22,38 @@ os.environ["KANSHO_FAKE_DETECT"] = "1" Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient -from identity_store import list_mappings +from identity_store import ( + STATUS_LEGACY, + confirm_identity, + list_confirmed_identities, + list_mappings, + list_registry, + list_review_proposals, + remember_mapping, +) from main import app -from privacy_gateway import reset_debug +from privacy_gateway import ( + GatewayRequest, + PrivacyGatewayError, + _demask, + _validate_response, + complete, + mask_prompt, + reset_debug, +) +from entity_detect import ( + DETECT_CHUNK_CHARS, + DetectError, + detect_personal_egress, + install_test_detect_failure, + install_test_spans, + install_test_truncated, + reset_detect_test_hooks, + split_detect_chunks, + uses_llm_detect, + validate_detected_entity, +) +from providers import ProviderConfig def expect(ok: bool, message: str) -> None: @@ -26,6 +62,18 @@ def expect(ok: bool, message: str) -> None: print(f"OK {message}") +def _run(profile_id: str, rendered: str, purpose: str = "dialogue_turn"): + return complete( + GatewayRequest( + prompt_id="detect-test", + purpose=purpose, + data_class="B", + profile_id=profile_id, + payload={"rendered": rendered, "source_text": rendered}, + ) + ) + + def main() -> None: reset_debug() with TestClient(app) as client: @@ -34,6 +82,7 @@ def main() -> None: json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"}, ) headers = {"X-Auth-Token": setup.json()["token"]} + profile_id = setup.json()["profile_id"] space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"}) day = client.post( f"/api/journal/spaces/{space.json()['id']}/days", @@ -45,111 +94,315 @@ def main() -> None: headers=headers, json={"title": "Gespräch"}, ) - turn = client.post( - f"/api/journal/conversations/{conv.json()['id']}/turn", - headers=headers, - json={"body": "Ich war mit Anna in Hamburg und danach allein."}, + + balcony = _run(profile_id, "Ich ging auf den Balkon und setzte mich.") + expect(balcony.allowed, "balcony sentence is allowed") + expect("Balkon" in (balcony.trace.get("egress") or ""), "capitalized common noun stays unmasked") + expect("[[PERSON:" not in (balcony.trace.get("egress") or ""), "balcony is not a person placeholder") + + typo = _run(profile_id, "Ich ging auf den Bakon und setzte mich.") + expect("Bakon" in (typo.trace.get("egress") or ""), "misspelled common noun is not stored as a person") + expect( + not any((item.get("canonical_label") or "") == "Bakon" for item in list_confirmed_identities(profile_id)), + "misspelled noun is not a confirmed identity", ) - expect(turn.status_code == 200, f"turn {turn.text}") - trace = turn.json().get("trace") or {} - expect(trace, "gateway returned request-scoped trace") - expect(trace["detect_provider"] == "fake-detect", "detect provider is separate") - expect("Anna" not in (trace.get("egress") or ""), "Anna masked before generate") - expect("Hamburg" not in (trace.get("egress") or ""), "Hamburg masked before generate") - expect("[[PERSON:" in (trace.get("egress") or "") or "[[PLACE:" in (trace.get("egress") or ""), "placeholders in egress") - expect("Anna" in turn.json()["user"]["body"], "source keeps cleartext") - mappings = list_mappings(setup.json()["profile_id"]) - labels = {item["local_label"] for item in mappings} - expect("Anna" in labels, "Anna stored in local dictionary") - expect("Hamburg" in labels, "Hamburg stored in local dictionary") - expect(all(not (item.get("token") or "").startswith("[[") or True for item in mappings), "tokens stored") - reset_debug() - again = client.post( - f"/api/journal/conversations/{conv.json()['id']}/turn", - headers=headers, - json={"body": "Anna war ruhig."}, + food = _run(profile_id, "Ich aß Sushi auf dem Teller.") + expect("Sushi" in (food.trace.get("egress") or ""), "food context keeps the word") + person_sushi = _run(profile_id, "Sushi kam ins Wohnzimmer.") + expect("[[PERSON:" in (person_sushi.trace.get("egress") or ""), "same word can be a person in another sentence") + expect("Sushi" not in (person_sushi.trace.get("egress") or "").replace("[[PERSON:", ""), "person Sushi is masked") + + mixed = _run( + profile_id, + "Ich traf Anna. Ich arbeitete am privaten Projekt Aurora. " + "Die Organisation Nordwerk schickte Unterlagen. Ich war in Hamburg.", ) - expect(again.status_code == 200, "second turn") - last2 = again.json().get("trace") or {} + egress = mixed.trace.get("egress") or "" + expect("[[PERSON:" in egress, "person is typed as PERSON") + expect("[[PROJECT:" in egress, "private project is typed as PROJECT") + expect("[[ORG:" in egress, "organisation is typed as ORG") + expect("[[PLACE:" in egress, "named city is typed as PLACE") - expect("nicht" in (last2.get("egress") or "").lower() or "Anna" not in (last2.get("egress") or ""), "dictionary remasks Anna without new invention") - expect(last2.get("mapping_count", 0) >= 2, "dictionary reused") + sample = "Ich traf Anna." + start = sample.index("Anna") + span = validate_detected_entity( + {"start": start, "end": start + 4, "text": "Anna", "entity_type": "PERSON"}, + sample, + 0, + ) + expect(span.text == "Anna" and span.start == start, "valid span matches the local substring") + relocated = validate_detected_entity( + {"start": 0, "end": 4, "text": "Anna", "entity_type": "PERSON"}, + sample, + 0, + ) + expect(relocated is not None and relocated.start == start and relocated.text == "Anna", "wrong offsets are grounded to the local word") + expect( + validate_detected_entity( + {"start": -1, "end": 4, "text": "Anna", "entity_type": "PERSON"}, + sample, + 0, + ).start + == start, + "negative offsets still ground when the word exists", + ) + expect( + validate_detected_entity( + {"start": 0, "end": 4, "text": "Nein", "entity_type": "PERSON"}, + sample, + 0, + ) + is None, + "a word that is not in the chunk is skipped", + ) + expect( + validate_detected_entity( + {"start": "x", "end": 4, "text": "Anna", "entity_type": "PERSON"}, + sample, + 0, + ).start + == start, + "non-numeric offsets still ground when the word exists", + ) + rejected = [] + for payload, label in ( + ({"start": start, "end": start + 4, "text": "Anna", "entity_type": "PET"}, "unknown type"), + ({"start": start, "end": start + 4, "text": "Anna", "entity_type": "PERSON", "token": "PERSON:01"}, "detector token"), + ({"start": start, "end": start + 4, "text": "Anna", "entity_type": "PERSON", "score": 1}, "extra field"), + ): + try: + validate_detected_entity(payload, sample, 0) + except DetectError: + rejected.append(label) + expect(rejected == ["unknown type", "detector token", "extra field"], "invalid detector output is rejected") - from entity_detect import _pattern_entities - from identity_store import is_maskable_label - from privacy_gateway import _mask + first = _run(profile_id, "Ich war mit Anna in Hamburg und danach allein.") + expect(first.trace.get("detect_provider") == "fake-detect", "detect provider is separate") + expect("Anna" not in (first.trace.get("egress") or ""), "Anna masked before generate") + expect("Hamburg" not in (first.trace.get("egress") or ""), "Hamburg masked before generate") + expect(first.trace.get("full_detection_coverage") is True, "successful detect records full coverage") + expect(first.trace.get("generate_called") is True, "generate runs after complete detection") + expect(first.diagnostics.get("detect_calls") == 1, "normal short text uses one detect call") + labels_after = {item["canonical_label"] for item in list_confirmed_identities(profile_id)} + expect("Anna" not in labels_after, "detect hit is not auto-confirmed") + expect("Hamburg" not in labels_after, "place detect hit is not auto-confirmed") + proposals = [item["observed_label"] for item in list_review_proposals(profile_id)] + expect("Anna" in proposals, "unconfirmed proposal may be stored for review") - patterned = {item["text"] for item in _pattern_entities("Ich war mit Anna in Hamburg.", set())} - expect("Anna" in patterned and "Hamburg" in patterned, "pattern finds proper names") - expect("Himmel" not in {item["text"] for item in _pattern_entities("Der Himmel war klar.", set())}, "pattern skips weather") + second = _run(profile_id, "Anna war ruhig.") + expect("Anna" not in (second.trace.get("egress") or ""), "second request still runs full detection") + expect(second.trace.get("request_local_hits", 0) >= 1, "known labels are not excluded from the new detection") + + poisoned = remember_mapping(profile_id, "Wohnzimmer", "PERSON:77") + expect(poisoned["status"] == "confirmed", "remember_mapping is explicit confirmation") + update = client.patch( + f"/api/admin/identities/{poisoned['id']}", + headers=headers, + json={"status": "inactive"}, + ) + expect(update.status_code == 200, "admin can deactivate a false mapping") + expect(update.json()["status"] == "inactive", "deactivated mapping is inactive") + later = _run(profile_id, "Ich saß im Wohnzimmer.") + expect("Wohnzimmer" in (later.trace.get("egress") or ""), "inactive mapping does not keep masking a room") + expect( + not any(item.get("canonical_label") == "Wohnzimmer" and item.get("status") == "confirmed" for item in list_registry(profile_id)), + "later request is not bound to the deactivated mapping", + ) + + confirmed_project = confirm_identity(profile_id, "Kanshō", entity_type="PROJECT", aliases=["Kansho"]) + canonical = _run(profile_id, "Ich arbeitete an Kansho weiter.") + expect("[[PROJECT:" in (canonical.trace.get("egress") or ""), "confirmed alias is masked") + expect("Kansho" not in (canonical.trace.get("egress") or ""), "confirmed alias does not leak") + project_manifest = mask_prompt( + "Ich arbeitete an Kansho weiter.", + [ + { + "local_label": "Kansho", + "token": confirmed_project["token"], + "entity_type": "PROJECT", + "demask_label": "Kanshō", + } + ], + "journal_generate", + ) + expect( + _demask(f"[[{confirmed_project['token']}]] weiter", project_manifest) == "Kanshō weiter", + "confirmed alias demasks to the canonical spelling", + ) + + observed_manifest = mask_prompt( + "Ich traf Anna am Abend.", + [{"local_label": "Anna", "token": "PERSON:01", "demask_label": "Anna"}], + "journal_generate", + ) + expect(_demask("[[PERSON:01]] am Abend.", observed_manifest) == "Anna am Abend.", "request-local hit demasks to the observed spelling") + + leftover = _demask("[[PERSON:01]] und [[PERSON:99]]", mask_prompt("Anna war da.", [{"local_label": "Anna", "token": "PERSON:01", "demask_label": "Anna"}], "journal_generate")) + expect("[[PERSON:99]]" in leftover, "unknown placeholder is not materialized") + + long_text = ("Anfang. " + ("x" * 1200) + " Anna in der Mitte. " + ("y" * 1200) + " Ende.") + with patch("entity_detect.DETECT_CHUNK_CHARS", 400), patch("entity_detect.DETECT_CHUNK_OVERLAP", 80): + chunks = split_detect_chunks(long_text, chunk_chars=400, overlap=80) + expect(len(chunks) >= 3, "long egress is chunked") + covered = 0 + prev_end = 0 + for offset, chunk in chunks: + expect(offset <= prev_end, "chunks overlap or abut") + covered = max(covered, offset + len(chunk)) + prev_end = offset + len(chunk) + expect(covered == len(long_text), "chunk union covers the full text") + boundary = _run(profile_id, long_text) + expect(boundary.trace.get("chunk_count") >= 2, "gateway chunks a long prompt") + expect(boundary.trace.get("chunks_ok") == boundary.trace.get("chunk_count"), "every chunk is marked ok") + expect("Anna" not in (boundary.trace.get("egress") or ""), "entity at a chunk boundary is still masked") + expect(boundary.trace.get("full_detection_coverage") is True, "coverage is true only after every chunk") + + generate_calls = {"n": 0} + + def boom(_messages, _policy): + generate_calls["n"] += 1 + raise AssertionError("generate must not run") + + install_test_detect_failure(DetectError("detect_chunk_failed", "chunk failed")) + blocked = False + with patch("privacy_gateway.complete_model", boom): + try: + _run(profile_id, "Ich traf Anna.") + except PrivacyGatewayError as exc: + blocked = exc.code == "detect_chunk_failed" + expect(exc.diagnostics.get("generate_called") is False, "failed chunk does not call generate") + expect(exc.diagnostics.get("full_detection_coverage") is False, "failed chunk is not full coverage") + expect("Substring" not in (exc.message or ""), "detect errors omit the internal substring wording") + expect("Angaben" in (exc.message or ""), "detect errors use a user-facing message") + reset_detect_test_hooks() + expect(blocked, "failed chunk fails closed") + expect(generate_calls["n"] == 0, "generate is not called when a chunk fails") + + install_test_truncated(True) + truncated = False + generate_calls["n"] = 0 + with patch("privacy_gateway.complete_model", boom): + try: + _run(profile_id, "Ich traf Anna.") + except PrivacyGatewayError as exc: + truncated = exc.code == "detect_truncated" + expect(exc.diagnostics.get("generate_called") is False, "truncated detect does not call generate") + reset_detect_test_hooks() + expect(truncated, "truncated detect output fails closed") + + start = "Ich traf Anna.".index("Anna") + install_test_spans([{"start": start, "end": start + 4, "text": "Anna", "entity_type": "PERSON", "token": "PERSON:09"}]) + extra = False + with patch("privacy_gateway.complete_model", boom): + try: + _run(profile_id, "Ich traf Anna.") + except PrivacyGatewayError as exc: + extra = exc.code == "detect_invalid_output" + reset_detect_test_hooks() + expect(extra, "detector-assigned tokens fail closed") + + install_test_spans([{"start": 0, "end": 4, "text": "Anna", "entity_type": "PERSON"}]) + try: + grounded_run = _run(profile_id, "Ich traf Anna am Abend.") + expect("Anna" not in (grounded_run.trace.get("egress") or ""), "grounded name is still masked") + expect(grounded_run.trace.get("generate_called") is True, "wrong offsets do not block generate") + finally: + reset_detect_test_hooks() + + results = [] + + def worker(label: str): + reset_debug() + out = _run(profile_id, f"Ich traf {label}.") + results.append((label, out.trace.get("active_tokens"), threading.get_ident())) + + with ThreadPoolExecutor(max_workers=2) as pool: + one = pool.submit(worker, "Anna") + two = pool.submit(worker, "Maren") + one.result() + two.result() + labels = {item[0] for item in results} + expect(labels == {"Anna", "Maren"}, "parallel requests keep their own manifests") - expect(not is_maskable_label("ich"), "pronoun ich is not a mapping") - expect(not is_maskable_label("user"), "role user is not a mapping") - expect(not is_maskable_label("Himmel"), "weather is not identity") - expect(not is_maskable_label("Der Himmel"), "determiner plus weather is not identity") - expect(not is_maskable_label("6:00 Uhr"), "clock time is not identity") - expect(not is_maskable_label("meinen Tee"), "tea is not identity") - expect(not is_maskable_label("Relativ"), "sentence adverb is not identity") - expect(not is_maskable_label("Allerdings"), "conjunction is not identity") - expect(not is_maskable_label("Tages"), "common noun is not identity") - expect(not is_maskable_label("Brote"), "food is not identity") - expect(not is_maskable_label("Überlegen"), "common noun is not identity") - expect(not is_maskable_label("Ortes"), "place word is not identity") - expect(not is_maskable_label("Zwiebeln"), "food is not identity") - expect(not is_maskable_label("letzten"), "function word letzten is not identity") - expect(not is_maskable_label("Getöse"), "collective noun is not identity") - expect(not is_maskable_label("Schiff"), "vessel is not identity") - expect(not is_maskable_label("Faden"), "prompt word is not identity") - expect("Getöse" not in {item["text"] for item in _pattern_entities("Zweimaster mit Getöse im Hafen.", set())}, "mit plus common noun is not a name") - expect("Anna" in {item["text"] for item in _pattern_entities("Ich war mit Anna im Hafen.", set())}, "mit plus given name stays") - expect(is_maskable_label("Sushi"), "given name stays maskable") - expect(is_maskable_label("meine Frau Sushi"), "kinship stripped, name remains") - from identity_store import coalesce_mappings, normalize_token, remember_mapping from db import get_db import uuid + legacy_id = str(uuid.uuid4()) + with get_db() as conn: + conn.execute( + """ + INSERT INTO identity_mappings + (id, profile_id, token, local_label, canonical_label, entity_type, status, origin) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + (legacy_id, profile_id, "PERSON:55", "Wohnblock", "Wohnblock", "PERSON", STATUS_LEGACY, "legacy_auto"), + ) + expect( + not any(item.get("id") == legacy_id for item in list_mappings(profile_id)), + "legacy mappings are not treated as confirmed", + ) + legacy_run = _run(profile_id, "Der Wohnblock war ruhig.") + expect("Wohnblock" in (legacy_run.trace.get("egress") or ""), "legacy auto-mapping does not skip detection or force masking") + + listed = client.get("/api/admin/identities", headers=headers) + expect(listed.status_code == 200, "admin identities list is authenticated") + expect("registry" in listed.json() and "proposals" in listed.json(), "admin list returns registry and proposals") + expect("local_label" not in json.dumps(first.diagnostics), "compact diagnostics omit labels") + expect("Anna" not in json.dumps(first.diagnostics), "compact diagnostics omit clear names") + expect("Anna" not in json.dumps(first.trace.get("log") or []), "run log omits labels") + + provider_seen = {"n": 0} + + def mark_provider(messages, policy): + provider_seen["n"] += 1 + from privacy_gateway import ChatResult as _unused # noqa: F401 + from providers import ChatResult + + return ChatResult(content="[[PERSON:01]] war ruhig.", model="fake", usage={}) + + with patch("privacy_gateway.complete_model", mark_provider): + _run(profile_id, "Ich traf Anna.") + expect(provider_seen["n"] == 1, "successful detection still uses exactly one generate call") + + from identity_store import coalesce_mappings, normalize_token + from privacy_gateway import _mask + from pronoun_bind import bind_user_lines, bind_pronouns + from privacy_gateway import mask_for_egress + expect(normalize_token("PERSON:KURZ") is None, "template token KURZ rejected") - first = remember_mapping(setup.json()["profile_id"], "meine Frau Sushi", "PERSON:08") - second = remember_mapping(setup.json()["profile_id"], "Sushi", "PERSON:KURZ") - expect(first["token"] == second["token"], "alias reuses token") - expect(second["local_label"] == "Sushi", "canonical label is the name") + first_map = remember_mapping(profile_id, "meine Frau Sushi", "PERSON:08") + second_map = remember_mapping(profile_id, "Sushi", "PERSON:KURZ") + expect(first_map["token"] == second_map["token"], "alias reuses token") + expect(second_map["canonical_label"] == "Sushi", "canonical label is the name") extra_id = str(uuid.uuid4()) with get_db() as conn: conn.execute( """ - INSERT INTO identity_mappings (id, profile_id, token, local_label) - VALUES (?, ?, ?, ?) + INSERT INTO identity_mappings (id, profile_id, token, local_label, canonical_label, entity_type, status, origin) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, - (extra_id, setup.json()["profile_id"], "PERSON:KURZ", "meine Frau Sushi"), + (extra_id, profile_id, "PERSON:KURZ", "meine Frau Sushi", "meine Frau Sushi", "PERSON", "confirmed", "local_authoritative"), ) - coalesce_mappings(setup.json()["profile_id"]) + coalesce_mappings(profile_id) sushi_rows = [ item - for item in list_mappings(setup.json()["profile_id"]) - if (item.get("local_label") or "").lower() == "sushi" + for item in list_registry(profile_id) + if (item.get("canonical_label") or "").lower() == "sushi" ] expect(len(sushi_rows) == 1, "coalesce keeps one Sushi row") expect(sushi_rows[0]["token"].upper() != "PERSON:KURZ", "stable token wins over KURZ") - joined = _mask( - "meine Frau Sushi kam. Sushi stellte fest.", - sushi_rows, - ) + joined = _mask("meine Frau Sushi kam. Sushi stellte fest.", [{"local_label": "Sushi", "token": sushi_rows[0]["token"]}]) expect(joined.count(f"[[{sushi_rows[0]['token']}]]") == 2, "both mentions share one placeholder") - expect("Sushi" not in joined, "name fully masked") expect("meine Frau " in joined, "kinship stays visible") homonym = _mask( "Ich esse gerne Sushi mit meiner Frau Sushi.", - sushi_rows, + [{"local_label": "Sushi", "token": sushi_rows[0]["token"]}], ) expect("esse gerne Sushi" in homonym, "food homonym stays") - expect("Frau [[" in homonym or f"Frau [[{sushi_rows[0]['token']}]]" in homonym, "kinship name is masked") expect(homonym.count("[[") == 1, "only the person mention is masked") - eating = _mask("Heute nur von Sushi essen erzählt.", sushi_rows) + eating = _mask("Heute nur von Sushi essen erzählt.", [{"local_label": "Sushi", "token": sushi_rows[0]["token"]}]) expect("Sushi essen" in eating, "noun before eat stays food") - expect("[[" not in eating, "food-only sentence is not the person") - from pronoun_bind import bind_user_lines, bind_pronouns - from privacy_gateway import mask_for_egress anna = [{"local_label": "Anna", "token": "PERSON:01"}] bound = bind_user_lines( @@ -159,32 +412,18 @@ def main() -> None: ) ) expect(bound.count("[[PERSON:01]]") == 3, "name and both pronouns share one token") - expect("Sie" not in bound and "ihr" not in bound, "pronouns rebound to person") journal_masked = mask_for_egress( "user: Ich war mit Anna.\nuser: Sie setzte sich zu mir, und ich gab ihr Tee.", anna, "journal_generate", ) expect(journal_masked.count("[[PERSON:01]]") == 1, "journal masks the name only") - expect("Sie setzte" in journal_masked and "ihr Tee" in journal_masked, "journal keeps anaphora") dialogue_masked = mask_for_egress( "user: Ich war mit Anna.\nuser: Sie setzte sich zu mir, und ich gab ihr Tee.", anna, "dialogue_turn", ) expect(dialogue_masked.count("[[PERSON:01]]") == 3, "dialogue still binds pronouns") - instruction = bind_user_lines( - "user: Anna kam.\n" - "Der Impuls darf eine Frage sein. Er darf keine neue Station setzen.\n" - ) - expect("Er darf keine neue Station" in instruction, "instruction pronoun stays") - instruction_name = _mask( - "Nenne Berg nicht in der Anweisung.\nuser: Berg kam vorbei.", - [{"local_label": "Berg", "token": "PERSON:02"}], - personal_lines_only=True, - ) - expect("Nenne Berg nicht" in instruction_name, "instruction line is not remasked") - expect("[[PERSON:02]]" in instruction_name, "user line still remasked") herr = [{"local_label": "Berg", "token": "PERSON:02"}] male, _ = bind_pronouns(_mask("Herr Berg kam. Er setzte sich.", herr)) expect(male.count("[[PERSON:02]]") == 2, "masculine pronoun follows last man") @@ -192,18 +431,41 @@ def main() -> None: expect("ihr" in address, "address ihr is not a person") plural, _ = bind_pronouns(_mask("Anna kam. Sie waren zu dritt.", anna)) expect("Sie waren" in plural or "sie waren" in plural.lower(), "plural sie stays") - croatia = ( - "Heute bin ich ca. gegen 6:00 Uhr aufgestanden. Relativ früh gesellte sich " - "auch meine Frau Sushi zu mir. Allerdings sollte die Bootstour erst um 11:00 Uhr starten." - ) - patterned_live = {item["text"] for item in _pattern_entities(croatia, set())} - expect("Sushi" in patterned_live, "pattern keeps given name") - expect( - patterned_live <= {"Sushi"}, - f"pattern does not treat German function words as names: {patterned_live}", - ) - from entity_detect import uses_llm_detect - from providers import ProviderConfig + + from identity_store import is_maskable_label + + expect(not is_maskable_label("ich"), "pronoun ich is not a mapping") + expect(not is_maskable_label("user"), "role user is not a mapping") + expect(not is_maskable_label("6:00 Uhr"), "clock time is not identity") + expect(is_maskable_label("Sushi"), "name-shaped label stays structurally maskable") + expect(is_maskable_label("Himmel"), "weather word is not banned by a stopword list") + + from db import get_db as _get_db + + with _get_db() as conn: + row = conn.execute( + "SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?", + ("mvp.entity_detect",), + ).fetchone() + expect(row["seed_revision"] == "2026-08-27-detect-ground-v1", "detect prompt revision is stored") + expect("{{known_labels}}" not in (row["template"] or ""), "new detect prompt has no known_labels skip list") + expect("Zwiebeln" not in (row["template"] or ""), "detect prompt has no food word list") + + with _get_db() as conn: + conn.execute( + "UPDATE ai_prompts SET template = ? WHERE slug = ?", + ("CUSTOM DETECT {{source_text}}", "mvp.entity_detect"), + ) + from db import init_db + + init_db() + with _get_db() as conn: + custom = conn.execute( + "SELECT template, seed_revision FROM ai_prompts WHERE slug = ?", + ("mvp.entity_detect",), + ).fetchone() + expect(custom["template"] == "CUSTOM DETECT {{source_text}}", "user-edited detect prompt is not overwritten") + expect(custom["seed_revision"] == "2026-08-27-detect-ground-v1", "revision still updates the default") remote = ProviderConfig( role="detect", @@ -232,42 +494,60 @@ def main() -> None: ) expect(uses_llm_detect(local), "production still allows local detect") os.environ.pop("KANSHO_ENV", None) - poisoned = _mask( - "Nicht erwähnt ist nicht geschehen. Ich kochte Tee.", - [{"local_label": "ich", "token": "PERSON:05"}], - ) - expect("N[[PERSON:05]]t" not in poisoned, "ich does not split nicht") - expect("Nicht erwähnt" in poisoned, "instruction word bleibt") - from privacy_gateway import PrivacyGatewayError, _demask, _validate_response mapped = [{"local_label": "Anna", "token": "PERSON:01"}] - kept = _validate_response("[[PERSON:01]] war ruhig.", mapped) - expect(kept.startswith("[[PERSON:01]]"), "placeholders pass validation") - expect(_demask(kept, mapped) == "Anna war ruhig.", "demask after validation") - blocked = False + anna_manifest = mask_prompt("Anna war am Markt.", mapped, "journal_generate") + kept = _validate_response("[[PERSON:01]] war ruhig.", anna_manifest) + expect(_demask(kept, anna_manifest) == "Anna war ruhig.", "demask after validation") + expect( + _demask("[[ person:01 ]] war ruhig.", anna_manifest) == "Anna war ruhig.", + "demask is case- and space-insensitive", + ) + blocked_plain = False try: - _validate_response("Anna war ruhig.", mapped) + _validate_response("Anna war ruhig.", anna_manifest) except PrivacyGatewayError as exc: - blocked = exc.code == "response_validation_failed" - expect(blocked, "plaintext identity is blocked before demask") - sushi_map = [{"local_label": "Sushi", "token": "PERSON:01"}] - food_reply = _validate_response("Heute nur von Sushi essen erzählt.", sushi_map) - expect("Sushi essen" in food_reply, "food homonym in a reply is not treated as a leak") - mixed = mask_for_egress( - "Belegte Vorhaben (nur user-Quellen):\n- Morgen wollen wir mit Anna fahren.\nuser: Heute Markt.", - mapped, - "dialogue_turn", + blocked_plain = exc.code == "response_validation_failed" + expect(blocked_plain, "plaintext identity is blocked before demask") + + no_user = client.get("/api/admin/identities") + expect(no_user.status_code in {401, 403}, "identity registry requires auth") + admin_calls = {"n": 0} + + def no_egress(*_args, **_kwargs): + admin_calls["n"] += 1 + raise AssertionError("admin identity actions must not call detect") + + with patch("entity_detect.complete_chat", no_egress), patch("privacy_gateway.complete_model", no_egress): + created = client.post( + "/api/admin/identities", + headers=headers, + json={"canonical_label": "Nordlicht", "entity_type": "PROJECT", "aliases": ["Nordlicht-Projekt"]}, + ) + expect(created.status_code == 200, f"admin create stays local {created.text}") + expect(admin_calls["n"] == 0, "admin registry actions create no external egress") + expect(created.json()["status"] == "confirmed", "admin create confirms locally") + + turn = client.post( + f"/api/journal/conversations/{conv.json()['id']}/turn", + headers=headers, + json={"body": "Ich war mit Anna in Hamburg und danach allein."}, ) - expect("Anna" not in mixed, "opening context names are masked for dialogue egress") - expect("[[PERSON:01]]" in mixed, "opening context uses the person token") - expect("Heute Markt" in mixed, "current user wording remains") - excerpt = mask_for_egress( - "Frühere Einträge im Space (nur Hinweise, keine Gewissheit):\n2026-08-01: Anna war am Hafen.\nuser: Heute windstill.", - mapped, - "dialogue_turn", - ) - expect("Anna" not in excerpt, "space excerpts are masked for dialogue egress") - expect("[[PERSON:01]]" in excerpt, "space excerpt uses the person token") + expect(turn.status_code == 200, f"turn {turn.status_code}") + expect((turn.json().get("trace") or {}).get("generate_called") is True, "dialogue generate still runs once after detect") + + install_test_detect_failure(DetectError("detect_invalid_output", "Detect-text entspricht nicht dem lokalen Substring.")) + try: + held = client.post( + f"/api/journal/conversations/{conv.json()['id']}/turn", + headers=headers, + json={"body": "Heute nur der Markt."}, + ) + expect(held.status_code == 200, f"detect fallback turn {held.status_code} {held.text}") + expect((held.json().get("assistant") or {}).get("body"), "detect failure still leaves a local impulse") + expect((held.json().get("decision") or {}).get("guard") == "detect_blocked", "detect fallback is marked locally") + finally: + reset_detect_test_hooks() print("All privacy detect tests passed.") diff --git a/backend/tests/test_privacy_detect_eval.py b/backend/tests/test_privacy_detect_eval.py new file mode 100644 index 0000000..59800e3 --- /dev/null +++ b/backend/tests/test_privacy_detect_eval.py @@ -0,0 +1,30 @@ +"""Eval harness contract. Does not call a live provider.""" +from __future__ import annotations + +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +os.environ.setdefault("KANSHO_FAKE_DETECT", "1") + +from entity_detect_eval import run_fake + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def main() -> None: + payload = run_fake() + expect(payload["mode"] == "fake", "eval default is fake") + expect(payload["live_quality"] == "unconfirmed", "live quality stays unconfirmed") + expect(len(payload["cases"]) >= 3, "synthetic cases exist") + print("All detect eval harness tests passed.") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_privacy_manifest.py b/backend/tests/test_privacy_manifest.py new file mode 100644 index 0000000..2d7d971 --- /dev/null +++ b/backend/tests/test_privacy_manifest.py @@ -0,0 +1,309 @@ +"""Request-scoped masking manifest and response validation invariants.""" +from __future__ import annotations + +import json +import os +import sys +import tempfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-manifest-test.sqlite") +os.environ["KANSHO_FAKE_PROVIDER"] = "1" +os.environ["KANSHO_FAKE_DETECT"] = "1" +Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) + +from fastapi.testclient import TestClient +from identity_store import remember_mapping +from main import app +from privacy_gateway import ( + ERROR_EGRESS_VALIDATION, + IDENTITY_LEAK_RETRY, + ActiveReplacement, + GatewayRequest, + MaskingManifest, + PrivacyGatewayError, + _demask, + _validate_response, + complete, + last_trace, + mask_prompt, + reset_debug, + validate_pre_egress, +) +from providers import ChatResult + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def _no_label_keys(payload) -> bool: + if isinstance(payload, dict): + if "local_label" in payload: + return False + return all(_no_label_keys(value) for value in payload.values()) + if isinstance(payload, list): + return all(_no_label_keys(item) for item in payload) + return True + + +def _run_gateway(profile_id: str, rendered: str, *, purpose: str = "dialogue_turn"): + return complete( + GatewayRequest( + prompt_id="manifest-test", + purpose=purpose, + data_class="B", + profile_id=profile_id, + payload={"rendered": rendered, "source_text": rendered}, + ) + ) + + +def main() -> None: + reset_debug() + anna = [{"local_label": "Anna", "token": "PERSON:01", "entity_type": "PERSON"}] + lars = [{"local_label": "Lars", "token": "PERSON:02", "entity_type": "PERSON"}] + sushi = [{"local_label": "Sushi", "token": "PERSON:01", "entity_type": "PERSON"}] + hamburg = [{"local_label": "Hamburg", "token": "PLACE:01", "entity_type": "PLACE"}] + + inactive = mask_prompt("Heute nur Markt und Kirschen.", anna, "journal_generate") + expect(inactive.replacements == (), "historical mapping is inactive when absent from the prompt") + kept = _validate_response("Anna kam später zum Markt.", inactive) + expect("Anna kam später" in kept, "inactive mapping does not treat the same string as a leak") + + mixed = mask_prompt("Anna war in Hamburg.", anna + lars + hamburg, "journal_generate") + active_tokens = {item.token for item in mixed.replacements} + expect(active_tokens == {"PERSON:01", "PLACE:01"}, "only actually masked mappings are active") + expect("PERSON:02" not in active_tokens, "unused profile mapping stays inactive") + + food_only = mask_prompt("Ich esse gerne Sushi.", sushi, "journal_generate") + expect(food_only.replacements == (), "intentional homonym is not an active identity") + food_reply = _validate_response("Heute nur von Sushi essen erzählt.", food_only) + expect("Sushi essen" in food_reply, "unmasked homonym does not block a normal reply") + + kinship = mask_prompt("Meine Frau Sushi kam später.", sushi, "journal_generate") + expect(len(kinship.replacements) == 1, "identity mention of the same word is active") + blocked = False + try: + _validate_response("Sushi kam vorbei.", kinship) + except PrivacyGatewayError as exc: + blocked = exc.code == "response_validation_failed" + expect(exc.diagnostics.get("leak_tokens") == ["PERSON:01"], "block reports the token, not the label") + expect("Sushi" not in json.dumps(exc.diagnostics), "validation diagnostics omit the clear name") + expect(blocked, "active clear name in the raw reply is a leak") + homonym_ok = _validate_response("Danach Sushi essen.", kinship) + expect("Sushi essen" in homonym_ok, "homonym in the reply uses the same classification rule") + + placeholder = _validate_response("[[PERSON:01]] war ruhig.", kinship) + expect(placeholder.startswith("[[PERSON:01]]"), "active placeholder passes validation") + expect(_demask(placeholder, kinship) == "Sushi war ruhig.", "demask happens only after validation") + leftover = _demask("[[PERSON:01]] und [[PERSON:99]]", kinship) + expect("Sushi" in leftover and "[[PERSON:99]]" in leftover, "inactive placeholder is not rematerialized") + expect("Clarissa" not in leftover, "inactive mapping label is not introduced by demask") + expect("[[" not in IDENTITY_LEAK_RETRY, "retry instruction must not teach bracket placeholders") + expect( + _demask("[[ person:01 ]] war ruhig.", kinship) == "Sushi war ruhig.", + "demask ignores case and inner spacing", + ) + expect( + _demask("[[PERSON:1]] war ruhig.", kinship) == "Sushi war ruhig.", + "demask accepts an unpadded numeric token", + ) + expect( + _demask("[[…]] und [[...]] blieben.", kinship) == "[[…]] und [[...]] blieben.", + "generic ellipsis placeholders are not rematerialized", + ) + + overlap = mask_prompt( + "Anna-Lena kam vorbei.", + [ + {"local_label": "Anna", "token": "PERSON:01"}, + {"local_label": "Anna-Lena", "token": "PERSON:08"}, + ], + "journal_generate", + ) + expect([item.token for item in overlap.replacements] == ["PERSON:08"], "only the actually replaced overlapping label is active") + expect("[[PERSON:08]]" in overlap.masked_text, "longer label is the one replaced") + expect("Anna-Lena" not in overlap.masked_text, "replaced longer label is gone") + + subject = mask_prompt( + "Clarissa kaufte Kirschen.", + [{"local_label": "Clarissa", "token": "PERSON:99"}], + "journal_generate", + ) + expect([item.token for item in subject.replacements] == ["PERSON:99"], "person as subject of kaufte remains identity") + + leaky = MaskingManifest( + masked_text="Anna war am Markt.", + available_mapping_count=1, + replacements=(ActiveReplacement("PERSON:01", "PERSON", 1, "Anna"),), + ) + pre = False + try: + validate_pre_egress("Anna war am Markt.", leaky) + except PrivacyGatewayError as exc: + pre = exc.code == ERROR_EGRESS_VALIDATION + expect(exc.diagnostics.get("pre_egress_validation") == "failed", "pre-egress failure is named") + expect(pre, "remaining active identity occurrence fails closed before the provider") + validate_pre_egress("[[PERSON:01]] war am Markt.", leaky) + + def check(prompt: str, mappings: list[dict], reply: str): + manifest = mask_prompt(prompt, mappings, "journal_generate") + try: + return "ok", _validate_response(reply, manifest), [item.token for item in manifest.replacements] + except PrivacyGatewayError as exc: + return exc.code, None, [item.token for item in manifest.replacements] + + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(check, "Anna kam vorbei.", anna, "Anna kam.") + second = pool.submit(check, "Heute nur Markt.", anna + lars, "Anna kam.") + code_a, _text_a, tokens_a = first.result() + code_b, text_b, tokens_b = second.result() + expect(code_a == "response_validation_failed", "parallel request A still blocks its own active leak") + expect(tokens_a == ["PERSON:01"], "parallel request A keeps its own active set") + expect(code_b == "ok" and "Anna kam" in (text_b or ""), "parallel request B is not blocked by A's mapping") + expect(tokens_b == [], "parallel request B does not inherit A's active set") + + with TestClient(app) as client: + setup = client.post( + "/api/auth/setup", + json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"}, + ) + profile_id = setup.json()["profile_id"] + remember_mapping(profile_id, "Anna", "PERSON:01") + remember_mapping(profile_id, "Clarissa", "PERSON:99") + + result = _run_gateway(profile_id, "Ich war mit Anna am Markt.") + expect(result.allowed, "masked identity request is allowed") + expect(result.diagnostics.get("available_mapping_count", 0) >= 1, "diagnostics count request mappings") + expect(result.diagnostics.get("active_mapping_count") >= 1, "diagnostics count actually masked mappings") + expect("PERSON:01" in (result.diagnostics.get("active_tokens") or []), "diagnostics name active tokens") + expect(result.trace.get("pre_egress_validation") == "ok", "pre-egress is recorded") + expect(result.trace.get("response_validation") == "ok", "response validation is recorded") + expect(_no_label_keys(result.diagnostics), "compact diagnostics have no local_label keys") + expect(_no_label_keys(result.trace.get("log")), "run log has no local_label keys") + expect(_no_label_keys(last_trace()), "last compact trace has no local_label keys") + compact_blob = json.dumps(result.diagnostics) + expect("local_label" not in compact_blob, "compact diagnostics omit the mapping field name") + expect("Anna" not in compact_blob, "compact diagnostics omit the clear name") + + provider_calls = {"n": 0} + + def boom(_messages, _policy): + provider_calls["n"] += 1 + raise AssertionError("provider must not be called after pre-egress failure") + + def leaky_mask(rendered, mappings, purpose): + return MaskingManifest( + masked_text=rendered, + available_mapping_count=len(mappings or []), + replacements=(ActiveReplacement("PERSON:01", "PERSON", 1, "Anna"),), + ) + + blocked_egress = False + with patch("privacy_gateway.mask_prompt", leaky_mask), patch("privacy_gateway.complete_model", boom): + try: + _run_gateway(profile_id, "Anna war am Markt.") + except PrivacyGatewayError as exc: + blocked_egress = exc.code == ERROR_EGRESS_VALIDATION + expect(blocked_egress, "pre-egress failure aborts the request") + expect(provider_calls["n"] == 0, "provider is not called when an active identity remains") + + seen = [] + + def leak_then_ok(messages, _policy): + seen.append(messages[0].get("content") or "") + if len(seen) == 1: + return ChatResult( + content="Anna stand den ganzen Nachmittag am Markt.", + model="fake", + usage={"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18, "cost": 0.01}, + context_compression="not_applicable", + ) + return ChatResult( + content="[[PERSON:01]] stand den ganzen Nachmittag am Markt.", + model="fake", + usage={"prompt_tokens": 12, "completion_tokens": 8, "total_tokens": 20, "cost": 0.02}, + context_compression="not_applicable", + ) + + with patch("privacy_gateway.complete_model", leak_then_ok): + repaired = _run_gateway(profile_id, "Ich war mit Anna am Markt.") + expect(len(seen) == 2, "a real leak retries exactly once") + expect(IDENTITY_LEAK_RETRY in seen[1], "retry uses the generic correction instruction") + expect("Anna stand den ganzen Nachmittag" not in seen[1], "discarded raw reply is not part of the retry prompt") + expect("[[PERSON:01]]" in seen[0] and "Anna" not in seen[0].replace("[[PERSON:01]]", ""), "retry keeps the already masked prompt") + expect("Anna" in (repaired.content or ""), "successful retry is demasked locally") + expect(repaired.diagnostics.get("response_validation_retry") == 1, "retry is recorded without the raw reply") + + def spaced_token(_messages, _policy): + return ChatResult( + content="[[ person:01 ]] stand den ganzen Nachmittag am Markt.", + model="fake", + usage={}, + context_compression="not_applicable", + ) + + with patch("privacy_gateway.complete_model", spaced_token): + restored = _run_gateway(profile_id, "Ich war mit Anna am Markt.") + expect( + "Anna stand den ganzen Nachmittag" in (restored.content or ""), + "gateway demasks case and spacing variants", + ) + expect("[[" not in (restored.content or ""), "no leftover placeholder after a successful demask") + expect(repaired.diagnostics.get("prompt_tokens") == 22, "retry aggregates prompt tokens") + expect(repaired.diagnostics.get("completion_tokens") == 16, "retry aggregates completion tokens") + expect(repaired.diagnostics.get("total_tokens") == 38, "retry aggregates total tokens") + expect(abs(float(repaired.diagnostics.get("cost") or 0) - 0.03) < 1e-9, "retry aggregates cost") + + inactive_calls = {"n": 0} + + def keep_inactive(messages, _policy): + inactive_calls["n"] += 1 + return ChatResult( + content="Clarissa blieb heute unerwähnt am Hafen.", + model="fake", + usage={}, + context_compression="not_applicable", + ) + + with patch("privacy_gateway.complete_model", keep_inactive): + kept_inactive = _run_gateway(profile_id, "Heute nur Markt und Kirschen am Hafen.") + expect(inactive_calls["n"] == 1, "inactive mapping does not start a retry") + expect(kept_inactive.trace.get("response_validation_retry") in (None, 0), "inactive mapping does not mark a retry") + expect("Clarissa blieb heute unerwähnt" in (kept_inactive.content or ""), "inactive mapping keeps the model text") + + retry_fail = [] + + def always_leak(messages, _policy): + retry_fail.append(messages[0].get("content") or "") + return ChatResult( + content="Anna stand den ganzen Nachmittag am Markt.", + model="fake", + usage={}, + context_compression="not_applicable", + ) + + failed = False + with patch("privacy_gateway.complete_model", always_leak): + try: + _run_gateway(profile_id, "Ich war mit Anna am Markt.") + except PrivacyGatewayError as exc: + failed = exc.code == "response_validation_failed" + expect(exc.diagnostics.get("response_validation_retry") == 1, "failed retry is still only one extra call") + expect("Anna stand" not in json.dumps(exc.diagnostics.get("log") or []), "discarded reply is not persisted in the log") + expect(failed, "second leak stays fail-closed") + expect(len(retry_fail) == 2, "gateway stops after one retry") + + print("All privacy manifest tests passed.") + + +if __name__ == "__main__": + main() diff --git a/backend/writing_profile_store.py b/backend/writing_profile_store.py index 4656e66..92c0c3d 100644 --- a/backend/writing_profile_store.py +++ b/backend/writing_profile_store.py @@ -53,6 +53,11 @@ TASK_BRIEF_TRAIT_CHARS = 280 TASK_BRIEF_EXEMPLAR_CHARS = 220 TASK_BRIEF_MAX_TRAITS = 6 TASK_BRIEF_MAX_EXEMPLARS = 2 +NEUTRAL_JOURNAL_STYLE = ( + "Neutraler Journalstil (kein individuelles Writing Profile): " + "klare Ich-Form, kurze Überschrift, ruhige Übergänge, natürliche Absätze. " + "Keine künstliche Literarisierung. Keine Stilableitung aus dem aktuellen Tagesdialog." +) DIALOGUE_KIND = "dialogue_style" MAX_DIALOGUE_MESSAGES = 8 MAX_DIALOGUE_CHARS = 500 @@ -67,6 +72,7 @@ __all__ = [ "LIFECYCLE", "SEED_FACETS", "compile_style_signals", + "NEUTRAL_JOURNAL_STYLE", "get_profile", "import_text", "import_corpus", @@ -94,6 +100,8 @@ __all__ = [ "seed_catalog", "INITIAL_BUILD_SOURCES", "compile_task_brief", + "clip_field", + "list_style_sources", ] @@ -1015,7 +1023,7 @@ def _dedupe_brief_sources(items: list[dict], *, seen_keys: set[tuple[str, str]] return result -def _clip_field(text: str, limit: int) -> str: +def clip_field(text: str, limit: int) -> str: raw = (text or "").strip() if limit <= 0 or not raw: return "" @@ -1031,6 +1039,14 @@ def _clip_field(text: str, limit: int) -> str: return cut.rstrip(" \t\n,;:-") +_clip_field = clip_field + + +def list_style_sources(profile_id: str) -> dict[str, list[dict]]: + """Intent-neutral ranked style sources. Journal policy decides which to send.""" + return _load_ranked_sources(profile_id) + + def _norm_overlap(left: str, right: str) -> float: a = re.sub(r"\s+", " ", (left or "").strip().lower()) b = re.sub(r"\s+", " ", (right or "").strip().lower()) @@ -1171,14 +1187,19 @@ def _assemble_brief(profile_id: str) -> None: def compile_task_brief(profile_id: str, task: str = "journal_generate") -> str: - """Runtime brief for one task. Not an export, import, or proposal format.""" - from writing_profile_schema import is_meta_style_text + """Compact confirmed style brief. Historical texts are selected separately. + Unconfirmed profiles do not become a style authority. Journal adapters attach + final entries as STYLE_EXAMPLES, not as extra rules in this brief. + """ profile = get_profile(profile_id) if task != "journal_generate": core = (profile.get("core") or {}).get("value") or "" return _clip_field(core, TASK_BRIEF_MAX_CHARS) + if not has_confirmed_profile(profile_id): + return NEUTRAL_JOURNAL_STYLE + parts: list[str] = [] core_text = ((profile.get("core") or {}).get("value") or "").strip() if core_text: @@ -1209,41 +1230,7 @@ def compile_task_brief(profile_id: str, task: str = "journal_generate") -> str: continue parts.append(f"- {item.get('label') or item.get('slug')}: {clipped}") used_statements.append(statement) - exemplars: list[tuple[float, str]] = [] - seen_ex = set() - for item in selected: - for ref in item.get("exemplars") or []: - excerpt = (ref.get("excerpt") or "").strip() - if not excerpt or is_meta_style_text(excerpt): - continue - digest = hashlib.sha256(re.sub(r"\s+", " ", excerpt.lower()).encode("utf-8")).hexdigest() - if digest in seen_ex: - continue - seen_ex.add(digest) - clipped = _clip_field(excerpt, TASK_BRIEF_EXEMPLAR_CHARS) - if not clipped: - continue - weight = recency_weight(ref.get("occurred_at") or _trait_recency_stamp(item)) - exemplars.append((weight, clipped)) - exemplars.sort(key=lambda row: -row[0]) - chosen = [] - for _, excerpt in exemplars: - if any(_norm_overlap(excerpt, prev) >= 0.8 for prev in chosen): - continue - chosen.append(excerpt) - if len(chosen) >= TASK_BRIEF_MAX_EXEMPLARS: - break - if chosen: - parts.append("Repräsentative Exemplare:") - for excerpt in chosen: - parts.append("- " + excerpt) brief = "\n".join(part for part in parts if part).strip() - if len(brief) > TASK_BRIEF_MAX_CHARS: - without_examples = [part for part in parts if not part.startswith("Repräsentative Exemplare") and not ( - part.startswith("- ") and chosen and any(part == "- " + excerpt for excerpt in chosen) - )] - brief = "\n".join(without_examples).strip() - parts = without_examples while len(brief) > TASK_BRIEF_MAX_CHARS and parts: last = parts.pop() remain = TASK_BRIEF_MAX_CHARS - (len("\n".join(parts)) + (1 if parts else 0)) @@ -1253,16 +1240,7 @@ def compile_task_brief(profile_id: str, task: str = "journal_generate") -> str: brief = "\n".join(parts).strip() break brief = "\n".join(parts).strip() - if brief: - return brief - ranked = _load_ranked_sources(profile_id) - snippets = (ranked.get("journal_entry") or [])[:2] + (ranked.get("imported_text") or [])[:1] - lines = [] - for item in snippets: - body = _clip_field(plain_text(item.get("body") or ""), 280) - if body and not is_meta_style_text(body): - lines.append(body) - return "\n\n".join(lines).strip() + return brief or NEUTRAL_JOURNAL_STYLE def has_facets(profile_id: str) -> bool: diff --git a/docs/architecture/functional/context_fidelity_and_regrounding.md b/docs/architecture/functional/context_fidelity_and_regrounding.md index 657cbdf..410cb29 100644 --- a/docs/architecture/functional/context_fidelity_and_regrounding.md +++ b/docs/architecture/functional/context_fidelity_and_regrounding.md @@ -226,3 +226,5 @@ Wenn sich die bisherige Einordnung ändert, sollte nachvollziehbar bleiben: Die konkrete Berechnung eines Drift- oder Fidelity-Werts wird erst in der technischen Architektur spezifiziert. +**Additiv 2026-08-26 (Journal):** Für den Journalentwurf gilt Fidelity als Faktentreue, nicht als Wortlautkopie. Ereignisse, Unsicherheit, Plan versus Vollzug und einmalige Details bleiben an die ausgewählten Nutzerquellen gebunden. Orthografie, Satzbau und Dialogform dürfen in der Narration verändert werden. Neue Personen, Orte, Zeiten, Motive oder Kausalität bleiben Drift. + diff --git a/docs/architecture/functional/guardrails.md b/docs/architecture/functional/guardrails.md index fa37fef..ce85fc7 100644 --- a/docs/architecture/functional/guardrails.md +++ b/docs/architecture/functional/guardrails.md @@ -656,9 +656,27 @@ Die Konfiguration soll weitgehend als Policy hinterlegt sein. | Response Validation | lokale Prüfung vor Demasking | entschieden | | Rehydration | ausschließlich lokal | entschieden | | Guardrails | providerunabhängig definieren | entschieden | +| Entity Detection | semantisch, request-lokal; Detect-Ausgabe nicht automatisch aktive Identität | entschieden (Verfahren 2026-08-26) | +| Entitätstypen im MVP | PERSON, PLACE, ORG, PROJECT | entschieden für den aktuellen Scope | + +Die konkrete Ausprägung von Quasi-Identifikatoren bleibt offen. | Interne Originalquellen | standardmäßig bewahren; keine automatische Löschung wegen Alter, Menge oder geringer Relevanz | entschieden | | Löschhoheit | konkrete Nutzeraufforderung oder berechtigte administrative Handlung; Ausprägung und Löschkaskade offen | fachliche Baseline entschieden | +## 22.1 Semantische Detection und bestätigte Identitäten (2026-08-26) + +Additiv. Technische Umsetzung: `../technical/privacy_gateway.md` §9.5. + +**Entschieden:** Jeder persönliche externe Generate-Egress wird vor dem Versand vollständig und semantisch untersucht. Ergebnisse einer einzelnen Detection gelten zunächst nur für diesen Request. Ein Detect-Modell darf niemals allein durch seine Ausgabe eine dauerhaft aktive Identität erzeugen. + +Schützenswert ist eine konkrete Bezeichnung, durch die eine natürliche Person, ein genauer persönlicher Ort, eine Organisation, ein privates Projekt oder ein anderer im Produktrahmen definierter Identifikator erkennbar wird. Allgemeine Gegenstände, Tätigkeiten, Lebensmittel, Räume und Beschreibungen sind keine Identitäten, sofern sie im Satz nicht als Eigenname verwendet werden. Großschreibung allein reicht nicht. + +**Entitätstypen im aktuellen Scope:** `PERSON`, `PLACE`, `ORG`, `PROJECT`. `PROJECT` ist ausdrücklich für benannte private Vorhaben und Produktbezeichnungen aufgenommen, nicht als beliebige neue Kategorie. + +Wortlisten einzelner Lebensmittel, Räume oder Wetterbegriffe sind **überholt**. Pattern-Heuristiken dürfen keine dauerhafte Identität behaupten und keinen semantischen Detect-Ausfall ersetzen. + +Unbestätigte Request-Treffer werden in der beobachteten Schreibweise demaskiert. Bestätigte Registry-Einträge und bestätigte Aliase werden kanonisch demaskiert. Legacy-Auto-Mappings sind nicht bestätigt. + --- # 23. Externe Referenzquellen diff --git a/docs/architecture/functional/mvp_freeze_candidate.md b/docs/architecture/functional/mvp_freeze_candidate.md index 4ac7958..c2d80a9 100644 --- a/docs/architecture/functional/mvp_freeze_candidate.md +++ b/docs/architecture/functional/mvp_freeze_candidate.md @@ -32,7 +32,7 @@ Nicht durch Fake-Provider bewiesen: - Erster Impuls mit einem echten Modell: belegter Plan, offener Tagesfaden, neutraler Fallback - Dialogqualität der sechs Operationen (Cases C/D/E bleiben modellabhängig) -- Journalentwurf Stufe 1/2 mit einem echten Modell: Quelltreue, keine Assistentenfakten +- Journalentwurf Stufe 1 lokal / Stufe 2 mit einem echten Modell: Quelltreue, keine Assistentenfakten, höchstens ein Narrations-Call im Normalfall - `include_existing` in der Oberfläche: Checkbox aus, Checkbox an, Nutzerfassung unangetastet - Papierkorb: Soft Delete, Wiederherstellen, endgültiges Löschen inkl. Medien - Startseite: Fortsetzen wenn ein Day existiert, sonst Space-Einstieg @@ -63,5 +63,8 @@ Kein Codefehler des Freeze Candidate, sondern Betriebsreife: - Browser-Zurück: vorhandene Unsaved-Warnung und `beforeunload` reichen; keine Data-Router-Migration. - Space-Kontext darf den Dialogzug stützen, nicht still die Generate-Quelle werden. -- Stufe 1 bleibt untrusted Auswahl/Annotation; autoritativ sind lokale Source Registry und `VerifiedArtifact`. Ungültiges Modell-JSON bricht Generate nicht ab: lokal volle Nutzerquellen, Abbruch nur ohne Nutzertext. -- Qualitative Modellfälle nicht als automatisch vollständig bewiesen bezeichnen. +- Stufe 1 ist lokal (`local_source_artifact`); autoritativ sind lokale Source Registry und `VerifiedArtifact`. Es gibt keinen externen Rekonstruktions-Call im Runtime-Pfad. Abbruch nur ohne Nutzertext. +- Semantische Detection jedes persönlichen Egress; Detect-Treffer nicht automatisch bestätigt; Legacy-Mappings lokal reviewen (`/admin/identities`) +- Response Validation prüft nur im aktuellen Request aktiv maskierte Identitäten. Historische Mappings und Homonyme blockieren keinen gültigen Modelltext. +- Journal-Generate muss eine inhaltstreue, redaktionell verbesserte Fassung liefern, keine Dialogkopie. Faktentreue bleibt; Wortlaut darf (und soll) lesbar werden. `prose_edit` und `notes_to_journal` sind lokal. Ähnlichkeit ist Diagnose, kein Retry-Grund. +- Qualitative Modellfälle nicht als automatisch vollständig bewiesen bezeichnen. Vergleich: `backend/journal_eval.py`. Live-Qualität ist unbestätigt, bis ein kontrollierter Modellvergleich läuft. diff --git a/docs/architecture/functional/writing_profile_and_journaling.md b/docs/architecture/functional/writing_profile_and_journaling.md index ce4811f..5f6ba10 100644 --- a/docs/architecture/functional/writing_profile_and_journaling.md +++ b/docs/architecture/functional/writing_profile_and_journaling.md @@ -95,6 +95,25 @@ Dabei werden verwendet: Damit werden Inhaltstreue und sprachliche Gestaltung bewusst getrennt. +**Additiv 2026-08-26 (Narration, nicht Wortlautkopie):** Provenienztreue bedeutet Faktentreue, nicht wörtliche Wiedergabe. Das lokale `VerifiedArtifact` ist autoritative Inhaltsquelle, das Writing Profile autoritative Stilquelle. Der aktuelle Tagesdialog ist Inhalt, nicht automatisch Schreibstil. Offensichtliche Rechtschreib- und Grammatikfehler sollen im Journalentwurf korrigiert werden; semantische Unsicherheit, Plan versus Vollzug und einmalige Details bleiben. Stufe 2 darf keine Dialogabschrift und keine Aneinanderreihung der Quellenzeilen liefern. + +**Additiv 2026-08-26 (redaktionelles Ziel):** Die Journalgenerierung erzeugt eine inhaltstreue, redaktionell verbesserte Tagebuchfassung in der persönlichen Schreibstimme. Hohe Wortlautähnlichkeit ist kein Fehler, wenn der Ausgangstext bereits gut ist. Ein Fehler liegt vor, wenn kein erkennbarer redaktioneller Mehrwert entsteht (übernommene Fehler, holprige Übergänge, reine Kopie). Kanshō erzwingt keine künstliche Originalität. + +Faktentreue und Wortlauttreue sind getrennt. Unsicherheit, Verneinung, Korrekturen sowie Plan versus Vollzug müssen semantisch erhalten bleiben, nicht wortidentisch. Sprachlich nötige Übergänge sind erlaubt, solange sie keine neue Ursache, Bewertung oder Handlung behaupten. Gute Originalformulierungen und direkte Rede dürfen bleiben. + +Zwei journal-spezifische Editorial Modes, lokal und ohne zweiten Modellaufruf: + +- `prose_edit`: bereits erzählerischer Rohtext wird behutsam überarbeitet. +- `notes_to_journal`: Stichpunkte und Fragmente werden zu zusammenhängender Journalprosa. Mischfall: `prose_edit`, wenn mindestens die Hälfte der Quellenblöcke Satzzeichen `.?!` enthält, sonst `notes_to_journal`. + +Das bestätigte Writing Profile ist die primäre Stilautorität (Core, Journal-Facet, aktive Traits). Historische finale, vom Nutzer akzeptierte oder bearbeitete Tagebucheinträge sind reine Stilreferenz; importierte Texte danach; sonst neutraler Journalstil. Der aktuelle Tagesdialog ist keine Stilautorität. Historische Beispiele dürfen keine aktuellen Tatsachen liefern. + +Kosten: im Normalfall genau ein externer `journal_generate`-Aufruf. Kein Rekonstruktionsmodell, keine Modusklassifikation per Modell, kein Qualitätsretry wegen Textähnlichkeit. Privacy-Retry nur bei request-spezifischem Identitätsverstoß. Textähnlichkeit ist Diagnose, keine Annahme- oder Verwerfungsregel. + +Automatisierte Tests beweisen Datenfluss und Promptvertrag, nicht die Prosaqualität eines echten Modells. Vergleichbare Live-Evaluation: `backend/journal_eval.py` (manuell, nicht im Produktionslauf). Live-Qualität bleibt unbestätigt, bis ein kontrollierter Modellvergleich ausgeführt wurde. + +Allgemeine Infrastruktur (Provenienz, Maskierung, Budget, bestätigte Profilstände, Stilreferenz-Auswahl, Traces) bleibt intent-neutral. Journalregeln (Editorial Modes, Journal-Facet, Ich-Form, Titel, Absatzform) liegen nur im Journal-Adapter. + --- ## Journal Entry als häufiger, aber nicht zwingender Abschluss einer Tagesreflexion diff --git a/docs/architecture/technical/admin_diagnostics.md b/docs/architecture/technical/admin_diagnostics.md index 5d5d9a4..73e6b85 100644 --- a/docs/architecture/technical/admin_diagnostics.md +++ b/docs/architecture/technical/admin_diagnostics.md @@ -53,7 +53,7 @@ Nicht übernehmen: Mitai-Admin für Körpertarife, Coupons, Training Types als K ## 4.2 Implementierungsstand (Dialog-Testspur) -**Status: Code vorhanden, Testphase.** Nach jedem Dialogzug und nach Generate erhält die Admin-Rolle `decision` und `trace` in der API-Antwort. Die Journal- und Dialogseite zeigen Zweck, Prompt, Provider/Modell, Detect, maskierten Egress und Antworten. Dialogzug bleibt einstufig. Nach Journal-Generate zeigt die Testspur zwei Stufen (Rekonstruktion, Journalformulierung) plus `trace.budget` (geschätzte und tatsächliche Tokens, Kosten soweit geliefert, Fenster, Output-Reserve, Marge, Compression-Status, Abbruchgrund). Klasse-A-Mapping bleibt unsichtbar. Ohne Admin-Rolle fehlen diese Felder. Nicht persistiert, nicht für den Regelbetrieb gedacht. Volle Prompts bleiben Test-Einblick des aktuellen Responses, nicht Audit-Default und nicht globale Historie. +**Additiv 2026-08-26:** Lokale Identitätsregistry unter `/admin/identities`. Detect-Vorschläge sind unbestätigt. Compact-Diagnose enthält Detect-Abdeckung, Chunks, Kosten und Laufzeit, aber keine Labels. ## 5. Entscheidungsstand diff --git a/docs/architecture/technical/documentation_index.md b/docs/architecture/technical/documentation_index.md index 4be4f1b..1917f3a 100644 --- a/docs/architecture/technical/documentation_index.md +++ b/docs/architecture/technical/documentation_index.md @@ -134,5 +134,5 @@ Vor Vereinfachungen die Invariantenliste in `technische_zielarchitektur.md` §3. ## 5. Code-Gerüst -Lokaler Produktrahmen in `frontend/` und `backend/`. Zusammenhängende technische Umsetzung: `mvp_implementation.md`. Auth, Nutzerverwaltung, Prompt-DB, Platzhalter, Feature-Check, Privacy Gateway. **Dialog-Layer 0:** Conversations/Messages/`usage_sessions`, Thread-/Space-Identität, Derived-Hülle mit Provenance. **MVP-Journal-Slice:** nutzersichtbare Spaces, Journal Days, Dialogzug (ein Call), explizite Draft-Generierung, versionierte Entries (Übernehmen = neue Version, Dirty-Schutz im Editor), Inline-Medien, lokale Stichpunkte ohne Egress, Writing Profile (Hülle mit dynamischen Traits, Initial Profile Build). Fit-Gap: `../functional/mvp_stand_und_abgleich.md` (2026-08-25). Konfigurierbare Prompts `mvp.dialogue_turn`, `mvp.journal_reconstruct`, `mvp.journal_generate`, `mvp.entity_detect`, `mvp.profile_review`. Persönliche Folgestufen konsumieren ein lokal erzeugtes `VerifiedArtifact` (`provenance_verification.md`); der Journal-Adapter setzt die Policy. Provider-URL/Modell/Policy unter Admin → Schnittstellen; Keys nur in `backend/.env`. Schichten: Maskierung, Dialogzug, Journalentwurf, explizite Profile-Review. Admin-Testspur am Dialogzug (Egress, Antwort, Operation). Privacy-Pfad durchgängig, Security Layer nicht vollständig (`privacy_gateway.md` §9.2). Kein Stripe, keine Threads-UI, kein semantisches Retrieval. SQLite nur lokal. +Lokaler Produktrahmen in `frontend/` und `backend/`. Zusammenhängende technische Umsetzung: `mvp_implementation.md`. Auth, Nutzerverwaltung, Prompt-DB, Platzhalter, Feature-Check, Privacy Gateway. **Dialog-Layer 0:** Conversations/Messages/`usage_sessions`, Thread-/Space-Identität, Derived-Hülle mit Provenance. **MVP-Journal-Slice:** nutzersichtbare Spaces, Journal Days, Dialogzug (ein Call), explizite Draft-Generierung, versionierte Entries (Übernehmen = neue Version, Dirty-Schutz im Editor), Inline-Medien, lokale Stichpunkte ohne Egress, Writing Profile (Hülle mit dynamischen Traits, Initial Profile Build). Fit-Gap: `../functional/mvp_stand_und_abgleich.md` (2026-08-25). Konfigurierbare Prompts `mvp.dialogue_turn`, `mvp.journal_reconstruct`, `mvp.journal_generate`, `mvp.entity_detect`, `mvp.profile_review`. Persönliche Folgestufen konsumieren ein lokal erzeugtes `VerifiedArtifact` (`provenance_verification.md`); der Journal-Adapter setzt die Policy. Journal-Generate: Stufe 1 lokal (`local_source_artifact`), Stufe 2 ein Narrations-Call (Faktentreue, eigenständige Journalprosa, nicht Wortlautkopie). Request-scoped Maskierungsmanifest und Pre-Egress: `privacy_gateway.md` §9.4. Semantische Request-Detection und bestätigte Registry: `privacy_gateway.md` §9.5. Provider-URL/Modell/Policy unter Admin → Schnittstellen; Keys nur in `backend/.env`. Schichten: Maskierung, Dialogzug, Journalentwurf, explizite Profile-Review. Admin-Testspur am Dialogzug (Egress, Antwort, Operation). Privacy-Pfad durchgängig, Security Layer nicht vollständig (`privacy_gateway.md` §9.2). Kein Stripe, keine Threads-UI, kein semantisches Retrieval. SQLite nur lokal. diff --git a/docs/architecture/technical/frontend_pwa_shell.md b/docs/architecture/technical/frontend_pwa_shell.md index 443a091..e5281c6 100644 --- a/docs/architecture/technical/frontend_pwa_shell.md +++ b/docs/architecture/technical/frontend_pwa_shell.md @@ -91,7 +91,9 @@ Die Shell-Nav bleibt Start / Journal / Einstellungen. Der MVP-Slice füllt `/jou Admin sieht unter dem Gespräch eine **Testspur**: intern gerenderter Prompt, maskierter Egress, Rohantwort, demaskierte Antwort, gewählte Dialogoperation. Keine Mapping-Tabelle. Nicht die produktive Nutzer-IA. -Die Dialogfläche bleibt in der Inhaltsäule der Shell. Lange Nachrichten und die Testspur umbrechen oder scrollen intern; sie dürfen die Seite nicht in der Breite aufweiten. +Die Dialogfläche bleibt in der Inhaltsäule der Shell. Lange Nachrichten und die Testspur umbrechen oder scrollen intern; sie dürfen die Seite nicht in der Breite aufweiten. Zeilenumbrüche und Leerzeilen in Nutzer- und Impulstexten bleiben in der Darstellung erhalten. + +Mobile (unter 1024px): Bottom-Nav, Kopfzeile, volle Breite, Safe-Area, Gespräche als kompakte Umschalter oben, Verlauf scrollt, Eingabe darunter in Daumennähe, Stichpunkte unter dem Dialog. Desktop (ab 1024px): Sidebar, Lesespalte für Standardseiten, Dialogfläche voll in der Inhaltsäule, Stichpunkte rechts daneben. Kein weiterer Breakpoint. Der Journal-Tag zeigt den Dialog zuerst; Draft und Einträge nur als kurze Verweise. Mehrere Gespräche werden erst beim Erzeugen des Tagebuchs zur Wahl gestellt. Mobile (unter 1024px): Bottom-Nav, Kopfzeile, volle Breite, Safe-Area, Gespräche als kompakte Umschalter oben, Verlauf scrollt, Eingabe darunter in Daumennähe, Stichpunkte unter dem Dialog. Desktop (ab 1024px): Sidebar, Lesespalte für Standardseiten, Dialogfläche voll in der Inhaltsäule, Stichpunkte rechts daneben. Kein weiterer Breakpoint. Der Journal-Tag zeigt den Dialog zuerst; Draft und Einträge nur als kurze Verweise. Mehrere Gespräche werden erst beim Erzeugen des Tagebuchs zur Wahl gestellt. diff --git a/docs/architecture/technical/mvp_implementation.md b/docs/architecture/technical/mvp_implementation.md index 2c5868e..187f3de 100644 --- a/docs/architecture/technical/mvp_implementation.md +++ b/docs/architecture/technical/mvp_implementation.md @@ -71,8 +71,8 @@ Einstieg: `backend/main.py` (Router: auth, users, dialogue, journal, prompts, pl | `routers/journal.py` | Produkt-API `/api/journal` | | `routers/dialogue.py` | Low-Level-Source `/api/dialogue`; derselbe Turn-Pfad | | `dialogue_turn.py` | Ein generativer Call, JSON `{operation, impulse}`, lokale Register-/Wächter, optional Repair | -| `journal_generate.py` | Expliziter Draft: Stufe 1 Rekonstruktion, Stufe 2 Narration | -| `journal_reconstruct.py` | Lokale Validierung der Stufe-1-JSON | +| `journal_generate.py` | Expliziter Draft: Stufe 1 lokales Quellenartefakt, Stufe 2 Narration | +| `journal_reconstruct.py` | Parser/Validator und `local_verified_artifact`; kein Runtime-Modellpfad | | `prompt_budget.py` / `model_catalog.py` | Kontextfenster, Output-Reserve, konservative Tokenschätzung | | `journal_store.py` | Days, Drafts, Entries, Versionen, Scratch | | `journal_policy.py` | Explizites Generate, Origins, Konsolidierungsangebot aus lokalen Signalen | @@ -130,7 +130,7 @@ Kernpfade: Context Builder → Gateway (`purpose=dialogue_turn`) → ein Generate-Call → Impuls speichern. Pronomenbindung nur hier in `user:`-Zeilen. 2. **Journalentwurf** `POST /days/{id}/generate` - Nur explizit. Optional `conversation_ids`. Ohne Auswahl kein stilles Mergen. Zwei Gateway-Zwecke nacheinander: `journal_reconstruct`, dann `journal_generate`. Pronomen ungebunden. Entry-`current_version_id` bleibt unangetastet. UI öffnet `?draft=1`. + Nur explizit. Optional `conversation_ids`. Ohne Auswahl kein stilles Mergen. Stufe 1 lokal (`local_source_artifact`), Stufe 2 ein Gateway-Zweck `journal_generate`. Pronomen ungebunden. Entry-`current_version_id` bleibt unangetastet. UI öffnet `?draft=1`. 3. **Speichern** `POST /entries` Mit `entry_id` + `origin=accepted_draft`: neue Version desselben Entry. Sonst neuer Entry oder `user_edit`. @@ -170,17 +170,19 @@ Editor speichert Markdown, nicht HTML. Medien-Token bleiben lokal. Dirty: In-App | Schicht | Rolle | Call | |---|---|---| -| Maskierung | `detect` | Muster immer. Externes Klartext-Detect nur Development/Test (`KANSHO_ENV`). Produktiv: Muster oder lokale Detect-URL. | +| Maskierung | `detect` | Vollständige semantische Detection des Generate-Egress. Externes Klartext-Detect nur Development/Test (`KANSHO_ENV`). Produktiv: lokales Detect-Modell. Kein Pattern-Fallback. | | Dialogzug | `generate` | Operation + Impuls | | Journalentwurf | `generate` | Explizit, getrennt | Explizite Writing-Profile-Review ist ein weiterer Generate-Zweck (`profile_review`), nicht der Dialogzug und nicht nach jedem Turn. -**Additiv 2026-08-25:** Der Journalentwurf ist fachlich zwei Stufen (Rekonstruktion, dann Narration), also zwei Generate-Calls hinter derselben expliziten Generate-Aktion. Dialogzug bleibt ein Call. Details: §16. +**Additiv 2026-08-25:** Der Journalentwurf ist fachlich zwei Stufen (Quellenartefakt, dann Narration). Der Runtime-Pfad ruft dafür kein Rekonstruktionsmodell mehr auf. Dialogzug bleibt ein Call. Details: §16. -Interner Context-Builder (Klartext, lokal): fordert Selection-Specs an (`day_messages`, `space_entries`, `space_recent_sources`, `writing_profile`). Recency-Kappen (Day 80 Messages für den Dialogzug; Journal-Generate nutzt Tokenbudget statt der 80er-Kappe, Overflow bricht ab; 5 Space-Entries × 400 Zeichen; 3 Space-Source-Conversations × 400 Zeichen User-Ausschnitt) liegen in `retrieval.py`, nicht in der Context-Builder-API. Dialogzug: aktuelle Conversation primär, plus begrenzte frühere Original-Conversations desselben Space (ohne denselben Journal Day voll zu laden). Generate-Stufe 1: Day-Messages mit Budget. Generate-Stufe 2: Writing Profile plus validierte Rekonstruktion, ohne den vollen Dialog. `WritingProfile` steuert Journaltext, nicht den Dialogue Kernel; der Turn hat einen leeren `interaction_hint`-Slot. Retrieval austauschbar, keine Embeddings. +**Additiv 2026-08-26 (Narrationsvertrag):** Stufe 2 formuliert eigenständige Journalprosa. Das `VerifiedArtifact` bestimmt, was gesagt werden darf; das Writing Profile, wie es gesagt wird. Der Quellwortlaut ist keine Ausgabevorlage. `shape_journal` überschreibt akzeptierten Modelltext nicht mit Dialogzeilen. Der aktuelle Tagesdialog ist keine Stilquelle desselben Laufs. -Gateway-Verfahren und offener Security Layer: `privacy_gateway.md` §9.1–9.2. +Interner Context-Builder (Klartext, lokal): fordert Selection-Specs an (`day_messages`, `space_entries`, `space_recent_sources`, `writing_profile`). Recency-Kappen (Day 80 Messages für den Dialogzug; Journal-Generate nutzt Tokenbudget statt der 80er-Kappe, Overflow bricht ab; 5 Space-Entries × 400 Zeichen; 3 Space-Source-Conversations × 400 Zeichen User-Ausschnitt) liegen in `retrieval.py`, nicht in der Context-Builder-API. Dialogzug: aktuelle Conversation primär, plus begrenzte frühere Original-Conversations desselben Space (ohne denselben Journal Day voll zu laden). Generate-Stufe 1: lokale Day-Messages, vollständiges `VerifiedArtifact`. Generate-Stufe 2: Writing Profile plus Artefakt, ohne den vollen Dialog. `WritingProfile` steuert Journaltext, nicht den Dialogue Kernel; der Turn hat einen leeren `interaction_hint`-Slot. Retrieval austauschbar, keine Embeddings. + +Gateway-Verfahren und offener Security Layer: `privacy_gateway.md` §9.1–9.2 und §9.5. --- @@ -192,6 +194,10 @@ Gateway-Verfahren und offener Security Layer: `privacy_gateway.md` §9.1–9.2. | Journal-Slice, Scratch-Egress, Versionsschutz | `backend/tests/test_mvp_journal.py` | | Architecture Correction (Provenance, Space-Source, Detect-Env, Kernel-Grenzen) | `backend/tests/test_architecture_correction.py` | | Detect, Pronomen Dialog vs Journal, Response Validation | `backend/tests/test_privacy_detect.py` | +| Request-scoped Maskierungsmanifest / Pre-Egress | `backend/tests/test_privacy_manifest.py` | +| Journal-Narration (Faktentreue, nicht Wortlaut) | `backend/tests/test_journal_narration.py` | +| Journal-Editorial Modes, Profil, Stilreferenzen | `backend/tests/test_journal_editorial.py` | +| Journal-Eval-Vertrag (kein Live-Qualitätsbeleg) | `backend/tests/test_journal_eval.py` | | Shape / Namen | `backend/tests/test_journal_shape.py` | | Writing Profile | `backend/tests/test_writing_profile.py` | | Profile Governance | `backend/tests/test_profile_governance.py` | @@ -215,7 +221,7 @@ Nur die **technische** Lage. Bewertung gegen Spec und Vision: Fit-Gap. | Thema | Ist | Zielrahmen / Produktiv | |---|---|---| | DB | SQLite Datei | PostgreSQL | -| Detect | Muster immer; externes Klartext-Detect nur Development/Test; Produktiv fällt auf Muster zurück. Lokales HTTP-Detect bleibt Provider-Rolle. | lokales Modell (Ollama) als Ziel | +| Detect | Semantische Request-Detection; Chunking; fail-closed ohne vollständige Abdeckung; Detect-Treffer nicht auto-persistiert. Externes Klartext-Detect nur Development/Test. | lokales Modell (Ollama) als Ziel | | Fehlerkörper | oft Objekt | `{detail: string}` | | `/dialog` | Admin-Harness, aus der Produkt-Nav entfernt | nicht MVP-Screen | | Response Validation | blockiert Klartext-Identität in der Rohantwort vor Demask; keine Quasi-Identifikatoren | `privacy_gateway.md` §9.2 voll | @@ -235,7 +241,7 @@ Additiv zum Slice, 2026-08-25. Kein Target-Model-Vorbau. - Provenance von Draft und Entry-Version ist relational (`journal_*_source_refs`); JSON-ID-Listen werden verlustfrei migriert. Source-Messages bleiben unangetastet. - Context Builder spricht Selection-Specs; Recency bleibt die aktuelle Retrieval-Heuristik, nicht die fachliche API. - Dialogzug kann begrenzte jüngere Original-Conversations desselben Space sehen, ohne denselben Journal Day voll zu laden. -- Externes Klartext-Detect ist kein Produktmodus (`KANSHO_ENV=production` fällt auf Muster zurück). Lokales Detect bleibt austauschbare Rolle. +- Externes Klartext-Detect ist kein Produktmodus (`KANSHO_ENV=production` ohne lokales Detect fail-closed). Lokales Detect bleibt austauschbare Rolle. Detect-Ausgabe persistiert keine aktive Identität. - Response Validation blockiert Klartext-Identität vor Demask. Dialog-Egress maskiert den ganzen Prompt inkl. Opening/Space-Kontext; ein Leak bricht den Zug nicht leer ab. Journal-Generate bricht bei einem Leak nach Retry nicht ab: lokales Quellenartefakt bzw. lokaler Entwurf, die leckende Modellantwort wird nicht verwendet. - Konsolidierung nutzt lokale Signale, nicht `count >= 2`. - `/dialog` ist Admin-Harness, nicht Produkt-IA. @@ -369,7 +375,9 @@ Additiv zu §7 und §15. Keine stillschweigende Kürzung autobiografischer Fakte ## 16.1 Ablauf -Explizites Generate durchläuft zwei maskierte Gateway-Calls, beide Klasse B, Fail Closed, ZDR unverändert: +**Additiv 2026-08-26:** Der historische Ablauf mit zwei maskierten Gateway-Calls (`journal_reconstruct` dann `journal_generate`) gilt nicht mehr für den Runtime-Pfad. Stufe 1 ist lokal; nur Stufe 2 geht nach außen. Der ältere Text darunter bleibt als Herkunft der Claim-Bindung und Provenienzregeln. + +Explizites Generate durchlief zuvor zwei maskierte Gateway-Calls, beide Klasse B, Fail Closed, ZDR unverändert: 1. **Stufe 1 – inhaltliche Rekonstruktion** (`mvp.journal_reconstruct`, `purpose=journal_reconstruct`). Eingabe: budgetierter Tagesdialog (alle Nutzerzeilen vollständig; vorausgehende Assistant-Zeile nur als Antwortkontext derselben Conversation). Jede nichtleere Nutzerzeile erhält lokal eine stabile Quellen-ID (`u1`, `u2`, …), Assistant-Zeilen `a1`, `a2`, …. Ausgabe: strukturiertes JSON mit `source_order` (lokale Quellenreihenfolge) und `chronology` (Ereignischronologie, nur umordenbar bei expliziten Zeit-/Reihenfolgeangaben). Lokal fail-closed validiert: vollständige Abdeckung der Nutzer-IDs, keine Umordnung oder Neunummerierung vor der Prüfung, kein Umschreiben ungültiger `source`-Werte, wörtliche `evidence` in der zitierten Nutzerzeile, keine Assistant-Tatsachen, Uhrzeiten/Unsicherheiten/Widersprüche/Korrekturen/Planänderungen bleiben. Ungültige Rekonstruktion geht nicht an Stufe 2; es gibt keine stillen Auto-Reparaturen. 2. **Stufe 2 – persönliche Narration** (`mvp.journal_generate`, `purpose=journal_generate`). Eingabe: validierte Rekonstruktion, kompakter aufgabenspezifischer Writing Brief, optional bestehender Text. Kein erneutes Mitschicken des vollen Tagesdialogs. @@ -382,6 +390,15 @@ Explizites Generate durchläuft zwei maskierte Gateway-Calls, beide Klasse B, Fa **Additiv 2026-08-26 (Identitätsleak):** Response Validation bleibt fail-closed. Klartext-Identität in der Rohantwort wird nicht demaskiert und nicht als Entwurf übernommen. Nach einem Korrekturversuch erzeugt der Journal-Adapter lokal denselben Coverage-Pfad (Stufe 1) bzw. einen Entwurf aus dem lokalen Artefakt (Stufe 2). Homonyme (Speise vs. Personenname) gelten nicht als Leak. +**Additiv 2026-08-26 (lokale Stufe 1, ein Narrations-Call):** Der Runtime-Pfad führt Stufe 1 nicht mehr als `mvp.journal_reconstruct` aus. Ablauf: + +1. Tagesquellen lokal laden, Quellen-IDs vergeben, `local_verified_artifact` aus allen ausgewählten nichtleeren Nutzerquellen erzeugen. Assistant-Zeilen gehören nicht zum Artefakt. Reihenfolge bleibt die Begegnungsreihenfolge. Keine neue modellgestützte Semantik. +2. Stufe 2 (`mvp.journal_generate`) erhält nur Verified Artifact, Writing Profile und optional bestehenden Text. Das ist der einzige externe Generierungs-Call des normalen Laufs. + +`mvp.journal_reconstruct` bleibt als Prompt und Parser/Validator vorhanden, ist aber kein aktiver Modellpfad. Ungültiges ehemaliges Stufe-1-Modell-JSON ist deshalb im Runtime-Pfad irrelevant. Budget: kein separates Rekonstruktionsfenster; Artefakt plus Writing Profile plus optionaler Bestand müssen in das Stufe-2-Fenster passen. Übergröße bleibt `prompt_budget_exceeded` ohne stille Kürzung. Mindestfenster 32K und deaktivierte Context Compression bleiben. Detect prüft den gesamten Stufe-2-Egress (ggf. gechunkt). Nach erfolgreicher Detection: ein Narrations-Call, nur bei echtem aktivem Response-Leak maximal ein weiterer Narrationsversuch. + +**Additiv 2026-08-26 (semantische Detection):** Detect ist intent-neutral. Der Journal-Adapter wählt nur, welcher Klartext in den Generate-Egress kommt. Keine Journalbegriffe in Provenienz oder Privacy. Details: `privacy_gateway.md` §9.5. + Das vollständige Writing Profile bleibt lokale Source of Truth. `compile_task_brief("journal_generate")` kompiliert nur den Aufgabenbrief (Core knapp, Facet-Delta `autobiographical_journal`, höchstens sechs relevante Traits, ein bis zwei Exemplare, Zielgröße ca. 3.000–4.000 Zeichen, feldweise an Wortgrenzen, keine Wortmitte). ## 16.2 Budgetmodell @@ -422,13 +439,19 @@ Nutzertext plus `diagnostics` ohne Prompts, Antworten, Klarnamen, Mapping. ## 16.5 Diagnose -Admin-Trace (`trace.budget`, `trace.stages`): verwendetes Modell, geschätzte Input-Tokens, tatsächliche `prompt_tokens` / `completion_tokens` / `total_tokens` / Kosten soweit geliefert, Fenster, Reserve, Marge, Budget-OK, Compression disabled/not_applicable, Abbruchgrund. Vollständige Prompts und Antworten nur im direkten Admin-Response des aktuellen Requests, nicht in einer globalen Historie oder Datenbank. Compact-Diagnose ohne Promptkörper darf prozessweit nur den letzten kompakten Status halten. +Admin-Trace (`trace.budget`, `trace.stages`): verwendetes Modell, geschätzte Input-Tokens, tatsächliche `prompt_tokens` / `completion_tokens` / `total_tokens` / Kosten soweit geliefert, Fenster, Reserve, Marge, Budget-OK, Compression disabled/not_applicable, Abbruchgrund. Stufe 1 erscheint als `purpose=local_source_artifact` ohne Provider/Modell, Tokens/Kosten 0, Coverage `all_selected_sources`, Status lokal erfolgreich. Vollständige Prompts und Antworten nur im direkten Admin-Response des aktuellen Requests, nicht in einer globalen Historie oder Datenbank. Compact-Diagnose ohne Promptkörper darf prozessweit nur den letzten kompakten Status halten. `local_label`, Mapping-Tabelle, Klartextname und verworfene Rohantworten gehören nicht in persistente Diagnose. ## 16.6 Bewusst offen Chunk-and-Merge für übergroße Tage, Tokenizer je Modellfamilie, persistente Audit-Tabelle ohne Prompt-Inhalt, Quasi-Identifikatoren. -Tests: `backend/tests/test_journal_budget.py`. Allgemeine Provenienzschicht: `provenance_verification.md`, `backend/tests/test_provenance.py`. +**Additiv 2026-08-26 (Journalprosa):** `mvp.journal_generate` formuliert aus verifizierten Informationen eigenständige Journalprosa. Der Quellwortlaut ist keine Ausgabevorlage. `shape_journal` ersetzt akzeptierten Modelltext nicht durch Dialogzeilen. Der aktuelle Tagesdialog wird nicht als Stilprofil an denselben Lauf angehängt; ohne individuelles Writing Profile gilt ein neutraler Journalstil. Systemprompts mit unverändertem Template werden über `seed_revision` `2026-08-26-journal-narration-v1` idempotent aktualisiert; unabhängig editierte Prompts bleiben. + +**Additiv 2026-08-26 (Editorial Modes und wirksames Profil):** Journal-Generate wählt lokal `prose_edit` oder `notes_to_journal` (`journal_editorial.py`), ohne zweiten Modellaufruf. Der Narrationsprompt (`seed_revision` `2026-08-26-journal-editorial-v1`) trennt `CURRENT_DAY_SOURCES`, `WRITING_PROFILE` und `STYLE_EXAMPLES`. Stufe 2 erhält den kompakten bestätigten Task Brief (Core, Journal-Facet, höchstens sechs Traits) plus höchstens zwei historische finale Einträge oder Importe als Stilreferenz; der aktuelle Kalendertag ist ausgeschlossen. Unbestätigte Profile sind keine Stilautorität (neutraler Fallback). Textähnlichkeit steht als Diagnose im Trace, löst keinen Retry aus und verwirft keinen Text. Identitätsprüfung gilt für Titel und Textkörper. Private Gateway-Hilfsfunktionen werden über öffentliche, intent-neutrale Namen genutzt (`canonical_token`, `is_identity_mention`, `identity_occurrence_count`, `identity_label_pattern`). + +Opt-in-Vergleich: `backend/journal_eval.py` (Baseline / vorheriger Vertrag / aktueller Prompt). Nicht Teil des Produktionslaufs. Live nur mit `--live --profile-id` über das Privacy Gateway. + +Tests: `backend/tests/test_journal_budget.py`, `backend/tests/test_journal_narration.py`, `backend/tests/test_journal_editorial.py`, `backend/tests/test_journal_eval.py`, `backend/tests/test_journal_shape.py`. Allgemeine Provenienzschicht: `provenance_verification.md`, `backend/tests/test_provenance.py`. --- diff --git a/docs/architecture/technical/privacy_gateway.md b/docs/architecture/technical/privacy_gateway.md index b481eed..09adb60 100644 --- a/docs/architecture/technical/privacy_gateway.md +++ b/docs/architecture/technical/privacy_gateway.md @@ -101,15 +101,13 @@ Interview H1 (Encryption at rest, vollständiges Löschen, Portabilität, DSFA b | OpenRouter | Kandidat, nicht Gateway-Ersatz | entschieden | | EU-Routing | bevorzugt | offen in Ausprägung | | Encryption at rest, Delete, DSFA | Phase H | offen | -| Implementierung Entity Detection | Wörterbuch + optionaler Detect-Provider; Qualität offen | bevorzugte Richtung im Code, nicht abgeschlossen | +| Implementierung Entity Detection | Semantische Request-Detection; Detect-Ausgabe nicht auto-persistiert; Qualität des Modells offen | entschieden im Verfahren, Modellqualität unbestätigt | ## 9.1 Implementierungsstand (MVP-Egress) **Status: Code vorhanden. Zwei getrennt konfigurierbare Provider.** Fit-Gap und Guardrail-Abstand (Detect-Testphase, Security Layer): `../functional/mvp_stand_und_abgleich.md`. -`backend/privacy_gateway.py` ist die einzige Stelle für den **generativen** Egress. Reihenfolge: Policy-Check → lokale Wörterbuch-Maskierung mit Wortgrenzen (kein Teilstring, kein `ich` in `nicht`) → Entity Detection nur auf user-Zeilen (Muster + Wörterbuch, in der Testphase zusätzlich OpenRouter-Detect) → Remaskierung: im Dialogzug nur `user:`/`assistant:`-Zeilen, beim Journalentwurf der ganze Prompt → ein Generative-Call nur mit Platzhaltern → Antwortprüfung → Demaskierung. - -Detect-Vorschläge werden nur gespeichert, wenn das Wort im user-Text steht, kein Funktions- oder Stoffwort ist und nicht zum Promptvokabular gehört. `mit` plus Allerweltsnomen (`mit Getöse`) ist kein Personenname; `mit Anna` bleibt einer. Bereits gespeicherte unzulässige Labels werden beim nächsten Detect entfernt. +`backend/privacy_gateway.py` ist die einzige Stelle für den **generativen** Egress. **Überholt (2026-08-26):** Muster-Fallback, `known_labels`, automatisches Speichern jedes Detect-Treffers und stilles Abschneiden nach `DETECT_MAX_CHARS`. Aktuell: §9.5. Intern in der Testspur ist keine dritte Anfrage, sondern die lokale Klartext-Vorlage desselben Dialogzugs. Detect und Dialogzug bleiben getrennte Calls. @@ -119,12 +117,12 @@ Zwei Provider, getrennt konfigurierbar (`providers.py`, Admin **Schnittstellen** Drei Schichten, zwei technische Endpunkte (`provider_settings`-Rollen `generate` / `detect` sind Implementierungsnamen): -- **Maskierung** (Rolle `detect`): nur schützenswerte Muster. Keine Operation, kein Impuls, kein Journaltext. Testphase: OpenRouter-Detect auf user-Zeilen. Produktivziel: Muster + lokales Modell. Wiederverwendbar für spätere Dialogtypen. +- **Maskierung** (Rolle `detect`): vollständige semantische Detection des persönlichen Generate-Egress. Keine Operation, kein Impuls, kein Journaltext. Testphase: OpenRouter-Detect darf Klartext sehen. Produktivziel: lokales Modell. Kein Pattern-Fallback. Wiederverwendbar für spätere Dialogtypen. - **Sprachmodell** (Rolle `generate`): maskierter Kontext. Dient zwei fachlichen Calls: **Dialogzug** (Operation + Impuls) und **Journalentwurf** (explizit). Das größere Modell gehört hierhin, nicht an die Maskierung. URL, Modell und Policy liegen in `provider_settings` (Seed `config/providers.seed.json`, Admin `GET/PUT /api/admin/providers`). Keys nur in `backend/.env`, nie in der DB und nie in der API-Antwort. Env überschreibt Admin-Werte, wenn gesetzt. -Lokale URLs (localhost / 127.0.0.1) brauchen keine ZDR-Flags. Die Mapping-Tabelle bleibt Klasse A und lokal; neue Treffer werden dort persistiert, damit der nächste Prompt dieselben Platzhalter per Muster nutzen kann, ohne Detection-Call. +Lokale URLs (localhost / 127.0.0.1) brauchen keine ZDR-Flags. Die Mapping-Tabelle bleibt Klasse A und lokal. Detect-Ausgaben erzeugen keine aktiven Registry-Einträge; siehe §9.5. Klasse A verlässt die Zone nicht. Mapping wird nicht mitgeschickt. Kein stiller Fallback auf einen unsicheren Generate-Provider. @@ -141,10 +139,10 @@ Admin-Testspur (`trace`) zeigt intern gerenderten Prompt und maskierten Egress. Noch offen im Code: 1. **Inhaltliche Context Minimization.** Es geht der zusammengebaute Dialog, nur durch eine Zeichenkappe begrenzt. Minimum-sufficient-context nach Aufgabe fehlt. -2. **Robuste Entity Detection.** Wörterbuch plus Muster plus Detect-Call. Testphase: externes kleines Modell über OpenRouter. Ziel: lokales Ollama. Keine belastbare Erkennung von Quasi-Identifikatoren (Beruf + Ort + Familie, seltene Konstellationen). -3. **Response Validation.** Minimale Prüfung: Klartext-Identität in der Rohantwort vor Demask wird blockiert. Dieselbe Homonym-Regel wie beim Maskieren (Speise vs. Person). Dialog-Egress maskiert den **gesamten** gerenderten Prompt (Opening-Hinweis, Space-Ausschnitte, Titel), nicht nur `user:`-Zeilen; Detect sieht denselben gerenderten Prompt. Bei einem Leak gibt es einen Korrekturversuch. Der Dialogzug fällt lokal zurück statt leer zu bleiben. Journal-Generate übernimmt ungültiges oder leckendes Stufe-1-JSON nicht; lokal vollständige Nutzerquellen, Stufe 2 analog mit lokalem Entwurf statt der leckenden Antwort. Keine Quasi-Identifikatoren, keine vollständige Platzhalterprüfung, keine inhaltliche Minimierung. +2. **Robuste Entity Detection.** Semantische Request-Detection mit Chunking ist gebaut (`§9.5`). Testphase: externes Detect-Modell über OpenRouter. Ziel: lokales Ollama. Fake-Provider-Tests beweisen nicht die semantische Modellqualität. Quasi-Identifikatoren (Beruf + Ort + Familie) bleiben unerkannt. +3. **Response Validation.** Prüfung nur gegen Identitäten, die im aktuellen Request tatsächlich als Identität maskiert wurden (`privacy_gateway.md` §9.4). Dieselbe Homonym-Regel wie beim Maskieren. Dialog-Egress maskiert den **gesamten** gerenderten Prompt. Bei einem echten Leak höchstens ein Korrekturversuch mit dem bereits maskierten Originalprompt plus generischer Anweisung; verworfene Rohantwort und Klarname gehen nicht erneut an den Provider. Der Dialogzug fällt lokal zurück statt leer zu bleiben. Journal-Generate übernimmt eine leckende Narration nicht; lokal entsteht ein Entwurf aus dem Quellenartefakt. Keine Quasi-Identifikatoren, keine vollständige Platzhalterprüfung, keine inhaltliche Minimierung. Keine pauschale Nachmaskierung der Modellantwort. 4. **Audit ohne Prompt-Inhalt.** Es gibt keine persistente Audit-Spur (Request-ID, Policy, Provider, ZDR-Status, maskierte Entitätstypen) ohne volle Prompts oder Mapping. -5. **Mapping-Härtung.** `identity_mappings` ist lokal, aber unverschlüsselt; keine Pflege-UI; keine Verschlüsselung at rest. +5. **Mapping-Härtung.** `identity_mappings` ist lokal, aber unverschlüsselt. Admin-Review unter `/admin/identities` existiert. Keine Verschlüsselung at rest. Legacy-Zeilen sind `legacy_review_required`, nicht automatisch bestätigt. 6. **Detect-Klartext in der Testphase.** Der Detect-Provider darf Klartext sehen. Bei externem Detect (OpenRouter) ist das ein bewusster Übergang, nicht der Zielpfad. Produktiv nur lokales Modell. 7. **Weitere Egress-Klassen.** Tool-, Web- und Speech-Egress bleiben getrennt und ungebaut. 8. **Löschen, Export, DSFA.** Journal-Entries haben Soft Delete, Papierkorb und explizites Purge. Kein vollständiges Account-Löschen, keine Portabilität als öffentliches Produkt, keine Datenschutz-Folgenabschätzung. @@ -155,11 +153,77 @@ Diese Punkte dürfen nicht stillschweigend als erledigt gelten, nur weil Generat Additiv zu §9.1. Fachliche Trennung Inhalt/Stil: `../functional/writing_profile_and_journaling.md` §16. Technische Umsetzung: `mvp_implementation.md` §16. -Persönlicher Journal-Egress bleibt ausschließlich über dieses Gateway. Zwei Zwecke: `journal_reconstruct` dann `journal_generate`. Kein stiller Fallback auf ein anderes Modell. OpenRouter-Context-Compression ist für Journal-Calls ausdrücklich deaktiviert (`plugins.id=context-compression`, `enabled=false`). Diese Plugin-Felder gehen nicht an lokale oder fremde OpenAI-kompatible Endpunkte. +Persönlicher Journal-Egress bleibt ausschließlich über dieses Gateway. Der normale Journal-Lauf hat genau einen generativen Zweck: `journal_generate`. Stufe 1 ist lokal (`local_source_artifact`) und erzeugt kein Providerbudget. Kein stiller Fallback auf ein anderes Modell. OpenRouter-Context-Compression ist für Journal-Calls ausdrücklich deaktiviert (`plugins.id=context-compression`, `enabled=false`). Diese Plugin-Felder gehen nicht an lokale oder fremde OpenAI-kompatible Endpunkte. -Journal-Prompts werden nicht mehr durch `MAX_EGRESS_CHARS` still in der Mitte abgeschnitten. Passt der Tagesdialog nicht ins Budget, bricht Kanshō mit `prompt_budget_exceeded` ab. Stufe 2 erhält den vollständigen lokal rehydrierten Nutzerinhalt als `VerifiedArtifact` (`provenance_verification.md`); bei Übergröße ebenfalls Abbruch, kein stilles Kürzen. +Journal-Prompts werden nicht mehr durch `MAX_EGRESS_CHARS` still in der Mitte abgeschnitten. Passt Artefakt, Writing Profile und optionaler Bestand nicht ins Stufe-2-Budget, bricht Kanshō mit `prompt_budget_exceeded` ab. Stufe 2 erhält den vollständigen lokal rehydrierten Nutzerinhalt als `VerifiedArtifact` (`provenance_verification.md`); bei Übergröße ebenfalls Abbruch, kein stilles Kürzen. -Usage-Metadaten (Tokens, Kosten, Fenster, Reserve, Marge, Compression-Status) liegen in der Admin-Testspur `trace.budget` bzw. `trace.stages`. Journal-Generate setzt beide Stufen aus den Rückgabewerten der jeweiligen Gateway-Calls zusammen; es gibt keine globale `debug_history` und keinen globalen Promptspeicher. Vollständige Prompts und Antworten höchstens im direkten Admin-Response des aktuellen Requests. Mapping bleibt ausgeschlossen. +Usage-Metadaten (Tokens, Kosten, Fenster, Reserve, Marge, Compression-Status) liegen in der Admin-Testspur `trace.budget` bzw. `trace.stages`. Journal-Generate setzt lokale Stufe 1 und den Gateway-Call der Stufe 2 zusammen; `run_log` sammelt Detect, Pre-Egress, Versuche, Antwortprüfung, Retries und Stufe-1-lokal ohne Promptkörper. Es gibt keine globale `debug_history` und keinen globalen Promptspeicher. Vollständige Prompts und Antworten höchstens im direkten Admin-Response des aktuellen Requests. Mapping bleibt ausgeschlossen. + +## 9.4 Request-scoped Maskierungsmanifest (2026-08-26) + +Intent-neutral, im Gateway. Journalspezifische Fallback-Policy bleibt im Journal-Adapter. + +Ein Mapping ist nicht allein deshalb aktiv, weil es in der Profiltabelle existiert. Aktiv ist nur, was im aktuellen gerenderten Prompt nach derselben Identitätsregel wie die Maskierung tatsächlich ersetzt wurde. Homonyme, die absichtlich unmaskiert bleiben, werden dadurch nicht aktiv. + +Das Manifest ist request-lokal: maskierter Egress-Text, aktive Tokens und Entitätstypen, optionale lokale Vorkommenszahl. `local_label` bleibt auf dem Request-Objekt und geht nicht in Logs, Compact-Diagnose oder persistente Spuren. Keine globale mutable Speicherung; parallele Requests und Profile bleiben getrennt. `mask_for_egress()` bleibt als String-Hülle. + +Vor dem Provider: Pre-Egress-Validierung. Für aktive Einträge darf kein als Identität klassifiziertes Klartextvorkommen im Egress bleiben. Homonyme derselben Regel dürfen bleiben. Fehlschlag: `egress_validation_failed`, Provider wird nicht aufgerufen. + +Response Validation prüft die Rohantwort ausschließlich gegen dieses aktive Manifest. Historische oder ungenutzte Mappings und unmaskierte Homonyme blockieren nicht. Vorhandene Platzhalter passieren und werden erst danach lokal demaskiert. Ein aktiver Klarname als Identität bleibt `response_validation_failed`. Es gibt keine pauschale Reparatur, die alle bekannten Namen in der Modellantwort durch Tokens ersetzt und die Antwort danach akzeptiert. + +Retry nur bei einem echten Treffer aus dem aktiven Manifest: maximal einer, bereits maskierter Originalprompt plus generische Korrektur, ohne verworfene Rohantwort und ohne gefundenen Klarnamen. Compact-Diagnose: verfügbare vs. aktive Mappings, maskierte Vorkommen, betroffene Tokens/Typen bei Blockade, Pre-Egress, Response Validation, Retry ja/nein, Modell- vs. lokaler Fallback, Laufzeit und Tokenverbrauch je tatsächlichem Modellaufruf. Tokens und Kosten beider Versuche werden aggregiert. Demaskierung nur mit den aktiven Tokens dieses Manifests; unbekannte Platzhalter führen keinen Klarname ein. Ein Personenname als Subjekt („Clarissa kaufte …“) bleibt Identität; nur Objekt- und Infinitivkonstruktionen wie „Sushi essen“ gelten als Homonym. + +Öffentliche, intent-neutrale Identitätsprüfung: `canonical_token`, `identity_label_pattern`, `is_identity_mention`, `identity_occurrence_count`. Journalspezifische Policy (unbelegte Personen in Titel und Textkörper) bleibt im Journal-Adapter. + +Tests: `backend/tests/test_privacy_manifest.py`, `backend/tests/test_privacy_detect.py`. + +## 9.5 Semantische Request-Detection und bestätigte Registry (2026-08-26) + +Intent-neutral, im Gateway. Journalbegriffe gehören nicht in diese Schicht. + +**Überholt:** wortlistenbasierter Detect-Prompt, `known_labels` als Ausnahme von der Prüfung, Pattern-Heuristiken (`mit`/`bei`/`von` plus Großschreibung) als Ersatz für semantische Detection, stilles Abschneiden nach `DETECT_MAX_CHARS`, Pattern-only-Freigabe des Generate-Calls, automatisches Speichern jedes Detect-Treffers als aktive Identität. + +### Verfahren + +Jeder persönliche Generate-Egress wird vollständig semantisch untersucht, bevor der Generate-Provider aufgerufen wird. Der geprüfte Klartext ist derselbe gerenderte Prompt, der anschließend maskiert versendet wird. Bekannte Labels werden nicht ausgenommen. Ergebnisse gelten zuerst nur für diesen Request. + +Überschreitet der Egress das Detect-Fenster, wird lokal in überlappende Chunks geteilt. Jeder Chunk muss syntaktisch gültig und lokal gegen den Chunktext prüfbar sein. Erst nach erfolgreicher Prüfung aller Chunks wird Generate freigegeben. Fehlschlag, Abbruch, unvollständiger Output oder `finish_reason=length` sind fail-closed: kein Generate, kein Pattern-Fallback, keine Weiterarbeit mit veralteten Mappings. + +### Detect-Schema + +Untrusted Input: + +```json +{"entities":[{"start":42,"end":47,"text":"Anna","entity_type":"PERSON"}]} +``` + +Erlaubte Typen: `PERSON`, `PLACE`, `ORG`, `PROJECT`. `PROJECT` ist additiv entschieden für benannte private Vorhaben und Produktbezeichnungen, nicht für Allerweltsgegenstände. Zusätzliche Felder und Tokens werden abgelehnt. Falsche Offsets werden lokal am exakten Wortlaut im Chunk verankert, sofern das gemeldete `text` dort vorkommt; ein Wort, das im Chunk nicht existiert, wird verworfen, ohne Generate zu sperren. Überlappende Spans: längste Spanne, dann links, dann Typreihenfolge PERSON > PROJECT > ORG > PLACE. Tokens vergibt nur die lokale Schicht, request-scoped. Schemaverletzungen, Abbruch und unvollständige Chunks bleiben fail-closed. Im Dialogzug erzeugt ein solcher Fail-closed lokal einen Halte-Impuls, ohne den Generate-Provider zu rufen. Der Nutzer sieht keine interne Substring-Diagnose. + +### RequestDetectionManifest vs. ConfirmedIdentityRegistry + +- **RequestDetectionManifest:** erkannte Spans, beobachtete Schreibweise, Typ, lokal vergebener Platzhalter. Keine globale Speicherung. Steuert Maskierung, Response Validation und Demaskierung dieses Requests. +- **ConfirmedIdentityRegistry:** nur ausdrücklich bestätigte oder lokal autoritative Einträge. Typ, kanonische Schreibweise, optionale bestätigte Aliase, Status, Herkunft, Zeitstempel. Detect-Ausgabe erzeugt niemals automatisch einen aktiven Eintrag. +- **Review-Proposals:** optional, unbestätigt, nie maskierend, nie detection-skip, nicht in externe Prompts oder Compact-Traces. + +Nach der semantischen Detection prüft die bestätigte Registry den vollen Egress als Sicherheitsnetz. Legacy-Zeilen (`legacy_review_required`) zählen nicht als bestätigt. + +### Demaskierung + +Unbestätigte Request-Treffer werden in der beobachteten Form demaskiert. Bestätigte Identitäten und bestätigte Aliase werden auf die kanonische Schreibweise demaskiert. Unbekannte Platzhalter materialisieren keine Namen. Die lokale Wiederherstellung erkennt Platzhalter unabhängig von Groß-/Kleinschreibung, Innenabstand und optionaler Nullauffüllung (`PERSON:1` / `PERSON:01`). Generische Auslassungsplatzhalter (`[[…]]`, `[[...]]`) werden nicht demaskiert; im Journalentwurf gelten sie als unbestätigter Inhalt und führen zum lokalen Fallback. Die Retry-Anweisung und der Journal-Seed dürfen keine Beispiel-Klammern `[[…]]` enthalten, damit das Modell sie nicht in den Text kopiert. + +### Persistenz und Migration + +Bestehende `identity_mappings` werden nicht gelöscht und nicht pauschal bestätigt. Additive Migration setzt `status=legacy_review_required` und `origin=legacy_auto`. Vor lokaler Bereinigung die Datei `backend/data/kansho.sqlite` kopieren; Wiederherstellung durch Zurücklegen der Kopie bei gestoppter Anwendung. Admin `/admin/identities` und `/api/admin/identities` bestätigen, korrigieren, aliasieren, deaktivieren oder entfernen lokal. Kein externer Egress, keine Klartextlabels in Compact-Logs. + +### Kosten und Diagnose + +Mehrere Detect-Calls bei langen Prompts sind zulässig. Der Nutzer akzeptiert die Laufzeit. Compact-Trace ohne Labels: Detect-Provider/Modell, Zeichen, Chunks, `full_detection_coverage`, Entitäten nach Typ, Registry- vs. Request-Treffer, Detect-Aufrufe, Tokens/Kosten/Laufzeit, `generate_called`, Abbruchgrund. + +Normalfall nach erfolgreicher Detection: genau ein Generate-Call. Privacy-Retry der Narrationsantwort bleibt unverändert (nur echter Identitätsverstoß, maximal einer). + +### Tests und Live-Qualität + +Contract-Tests: `backend/tests/test_privacy_detect.py`, `backend/tests/test_identity_registry.py`. Sie beweisen Schema, Fail-closed und Datenfluss, nicht semantische Modellleistung. Opt-in: `python entity_detect_eval.py --live` mit synthetischen Sätzen. Ohne diesen Lauf bleibt die Live-Qualität unbestätigt. ## 10. Offene Fragen diff --git a/docs/architecture/technical/provenance_verification.md b/docs/architecture/technical/provenance_verification.md index 4f288b2..7f2a76f 100644 --- a/docs/architecture/technical/provenance_verification.md +++ b/docs/architecture/technical/provenance_verification.md @@ -93,11 +93,17 @@ Nur im Adapter `journal_reconstruct.py` und in den Journal-Prompts: **Additiv 2026-08-26:** Generate reicht ungültiges Stufe-1-JSON nicht an Stufe 2 weiter und repariert es nicht. Der Adapter materialisiert stattdessen lokal alle ausgewählten Nutzerquellen (`local_verified_artifact`). Generate-Abbruch nur, wenn es keine Nutzerquelle gibt (`no_user_sources`). Eine Rohantwort mit Klartext-Identität wird nach Retry ebenfalls nicht übernommen. +**Additiv 2026-08-26 (Runtime ohne Rekonstruktionsmodell):** Der Journal-Adapter erzeugt Stufe 1 lokal als vollständiges `VerifiedArtifact` (`purpose=local_source_artifact`). Der Runtime-Pfad ruft `mvp.journal_reconstruct` nicht mehr auf. Parser und Validator bleiben für Tests und Abgrenzung erhalten, sind aber nicht der aktive Modellpfad. Stufe 2 bleibt der einzige externe Narrations-Call und konsumiert ausschließlich das lokale Artefakt plus Writing Profile plus optionalen Bestand. + +**Additiv 2026-08-26 (Faktentreue, nicht Wortlaut):** Das Artefakt autorisiert Informationen, nicht die Ausgabeformulierung. Stufe 2 darf Orthografie und Satzbau ändern, solange Ereignisse, Unsicherheit, Plan versus Vollzug und einmalige Details erhalten bleiben. Ein inaktiver historischer Name in der Narration ist kein Privacy-Leak, aber journalspezifisch unbelegter Inhalt und kein gültiger Entwurf. + +**Additiv 2026-08-26 (Editorial, Adapter):** Journalform, Editorial Modes, Journal-Facet und historische Stilreferenzen bleiben im Adapter. Die Provenienzschicht kennt sie nicht. Stufe 2 präsentiert dem Modell `CURRENT_DAY_SOURCES` als Tatsachen und `STYLE_EXAMPLES` nur als Ton. + Das Artefakt ist geprüftes Quellenmaterial, kein ungeprüfter Rohdialog mit Assistentenzeilen. ## 4. Aktuelle MVP-Nutzung -Zwei Gateway-Calls Klasse B, request-scoped Trace, Budget unverändert (`mvp_implementation.md` §16, `privacy_gateway.md` §9.3). Stufe 2 konsumiert `reconstruction` = serialisiertes `VerifiedArtifact`. Passt der vollständige Nutzerinhalt nicht ins Budget: `prompt_budget_exceeded`, nichts still kürzen, kein Chunk-and-Merge. +Ein Gateway-Call Klasse B für die Narration, plus optionales Detect auf dem versendeten Stufe-2-Kontext. Request-scoped Trace, Budget nur für Stufe 2 (`mvp_implementation.md` §16, `privacy_gateway.md` §9.3–9.4). Stufe 2 konsumiert `reconstruction` = serialisiertes `VerifiedArtifact`. Passt der vollständige Nutzerinhalt nicht ins Budget: `prompt_budget_exceeded`, nichts still kürzen, kein Chunk-and-Merge. Tests: `backend/tests/test_provenance.py`, `backend/tests/test_journal_budget.py`. diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 3126239..d7c007f 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -20,6 +20,7 @@ import AdminPlaceholdersPage from './pages/AdminPlaceholdersPage.jsx' import AdminFeaturesPage from './pages/AdminFeaturesPage.jsx' import AdminDialoguePage from './pages/AdminDialoguePage.jsx' import AdminProvidersPage from './pages/AdminProvidersPage.jsx' +import AdminIdentitiesPage from './pages/AdminIdentitiesPage.jsx' import RequireAdmin from './layouts/RequireAdmin.jsx' import AdminShell from './layouts/AdminShell.jsx' import { UnsavedProvider, useUnsavedChanges } from './context/UnsavedChanges.jsx' @@ -147,6 +148,7 @@ export default function App() { } /> } /> } /> + } /> } /> diff --git a/frontend/src/app.css b/frontend/src/app.css index 75efb72..256b651 100644 --- a/frontend/src/app.css +++ b/frontend/src/app.css @@ -265,7 +265,11 @@ ul.stack { padding: 0; } margin-bottom: 0.7rem; -webkit-overflow-scrolling: touch; } -.message-list .msg { margin: 0 0 0.75rem; overflow-wrap: anywhere; } +.message-list .msg { + margin: 0 0 0.75rem; + overflow-wrap: anywhere; + white-space: pre-wrap; +} .message-list .msg.assistant { border-left: 3px solid var(--accent); padding-left: 0.65rem; @@ -517,3 +521,57 @@ pre.code { .dialogue-composer button { width: auto; justify-self: start; } .writing-brief { max-height: none; } } + +.runlog-overlay { + position: fixed; + inset: 0; + z-index: 80; + display: grid; + place-items: center; + padding: 1rem; + background: rgba(28, 25, 23, 0.45); +} +.runlog-modal { + width: min(42rem, 100%); + max-height: min(88dvh, 52rem); + overflow: auto; + background: var(--card); + color: var(--ink); + border: 1px solid var(--line); + border-radius: 12px; + padding: 1rem 1.1rem 1.2rem; + box-shadow: 0 18px 40px rgba(28, 25, 23, 0.18); +} +.runlog-head { + display: flex; + justify-content: space-between; + gap: 0.8rem; + align-items: flex-start; +} +.runlog-head h2 { margin: 0; font-size: 1.15rem; } +.runlog-status { margin: 0.2rem 0 0; font-size: 0.9rem; color: var(--muted); } +.runlog-status.ok { color: var(--accent); } +.runlog-status.warn { color: #8a6a28; } +.runlog-status.bad { color: var(--danger); } +.runlog-list { + list-style: none; + margin: 0.8rem 0 0; + padding: 0; + display: grid; + gap: 0.55rem; +} +.runlog-list li { + display: grid; + grid-template-columns: 4.5rem minmax(0, 1fr); + gap: 0.6rem; + padding: 0.45rem 0.5rem; + border: 1px solid var(--line); + border-radius: 8px; +} +.runlog-list li p { margin: 0.2rem 0 0; color: var(--muted); font-size: 0.9rem; } +.runlog-time { color: var(--muted); font-variant-numeric: tabular-nums; font-size: 0.85rem; } +.runlog-list li.ok { border-color: #c5d4c9; } +.runlog-list li.warn { border-color: #e4d2a3; } +.runlog-list li.bad { border-color: #e3c0c0; } +.runlog-trace { margin-top: 0.8rem; } +.runlog-trace .trace-panel { border-top: 1px dashed var(--line); } diff --git a/frontend/src/components/CallTrace.jsx b/frontend/src/components/CallTrace.jsx index d020b3a..e3259a5 100644 --- a/frontend/src/components/CallTrace.jsx +++ b/frontend/src/components/CallTrace.jsx @@ -18,13 +18,18 @@ function isJournalTrace(trace) { if (Array.isArray(trace.stages) && trace.stages.length > 0) return true return ( trace.purpose === 'journal_reconstruct' || + trace.purpose === 'local_source_artifact' || trace.purpose === 'journal_generate' || trace.layer === 'journalentwurf' || + trace.layer === 'journalquellen' || trace.layer === 'journalrekonstruktion' ) } function layerLabel(trace) { + if (trace?.purpose === 'local_source_artifact' || trace?.layer === 'journalquellen') { + return 'Lokale Quellen' + } if (trace?.purpose === 'journal_reconstruct' || trace?.layer === 'journalrekonstruktion') { return 'Journalrekonstruktion' } @@ -39,6 +44,9 @@ function layerLabel(trace) { function stageTitle(stage, index) { const layer = layerLabel(stage) + if (stage?.purpose === 'local_source_artifact' || stage?.layer === 'journalquellen') { + return `Stufe ${index + 1}: lokale Quellen (kein Modell)` + } if (stage?.purpose === 'journal_reconstruct' || stage?.layer === 'journalrekonstruktion') { return `Stufe ${index + 1}: Rekonstruktion` } @@ -49,6 +57,9 @@ function stageTitle(stage, index) { } function detectLabel(trace) { + if (trace?.purpose === 'local_source_artifact' || trace?.status === 'local_ok') { + return 'kein Detect — lokal' + } if (trace?.detect_note === 'remote_llm') { return `${trace.detect_provider || 'openrouter'} (Detect-Call, Testphase)` } @@ -155,13 +166,13 @@ export default function CallTrace({ decision, trace }) { {journal ? ( <>
  • - Die Journalgenerierung sind zwei getrennte Sprachmodell-Calls: zuerst Rekonstruktion, - danach Journalformulierung. Intern ist die lokale Klartext-Vorlage (nicht gesendet). - Egress ist dieselbe Vorlage, maskiert. + Die Journalgenerierung bleibt logisch zweistufig: zuerst ein lokales Quellenartefakt + (kein Provider, keine Tokens), danach ein Narrations-Call. Intern der Stufe 1 ist das + lokale Artefakt. Egress gibt es nur in Stufe 2, maskiert.
  • - Stufe 2 läuft nur, wenn Stufe 1 lokal vollständig validiert wurde. Eine fehlende Stufe - bedeutet, dass dieser Call nicht stattgefunden hat. + Response Validation prüft nur Identitäten, die in genau diesem Request maskiert wurden. + Ein echter Leak löst höchstens einen Korrekturversuch aus; sonst entsteht ein lokaler Entwurf.
    • ) : ( diff --git a/frontend/src/components/RunLogPopup.jsx b/frontend/src/components/RunLogPopup.jsx new file mode 100644 index 0000000..61792fc --- /dev/null +++ b/frontend/src/components/RunLogPopup.jsx @@ -0,0 +1,136 @@ +import { useEffect } from 'react' +import CallTrace from './CallTrace.jsx' + +const KIND_LABEL = { + blocked: 'Gateway blockiert', + detect: 'Maskierung', + budget: 'Budgetprüfung', + pre_egress_validation: 'Pre-Egress-Prüfung', + model_call: 'Modellaufruf gestartet', + model_result: 'Modellantwort', + response_validation: 'Antwortprüfung', + retry: 'Korrekturversuch', + stage1_result: 'Stufe 1 Ergebnis', + narration_result: 'Stufe 2 Ergebnis', +} + +const STAGE_LABEL = { + local_source_artifact: 'Stufe 1 · lokale Quellen', + journal_reconstruct: 'Stufe 1 · Rekonstruktion (nicht im Runtime-Pfad)', + journal_generate: 'Stufe 2 · Narration', + dialogue_turn: 'Dialogzug', +} + +function stageLabel(item) { + return STAGE_LABEL[item?.stage] || item?.stage || '' +} + +function kindLabel(item) { + return KIND_LABEL[item?.kind] || item?.kind || 'Ereignis' +} + +function detail(item) { + const parts = [] + if (item?.attempt) parts.push(`Versuch ${item.attempt}`) + if (item?.status === 'ok') parts.push('ok') + if (item?.status === 'failed' || item?.status === 'error') parts.push('fehlgeschlagen') + if (item?.status === 'model') parts.push('Modelltext übernommen') + if (item?.status === 'local_fallback') parts.push('lokaler Entwurf') + if (item?.stage1 === 'model_accepted') parts.push('Modell-JSON gültig, übernommen') + if (item?.stage1 === 'local_ok') parts.push('lokal vollständig') + if (item?.stage1 === 'local_source_artifact') parts.push('lokal vollständig') + if (item?.stage1 === 'local_fallback') parts.push('Modell-JSON nicht übernommen, lokale Quellen') + if (item?.coverage) parts.push(item.coverage) + if (item?.active_mapping_count != null) parts.push(`${item.active_mapping_count} aktiv`) + if (item?.reason) parts.push(item.reason) + if (item?.code) parts.push(item.code) + if (item?.detect_note) parts.push(item.detect_note) + if (item?.detect_provider) parts.push(item.detect_provider) + if (item?.mapping_count != null) parts.push(`${item.mapping_count} Masken`) + if (item?.model) parts.push(item.model) + if (item?.prompt_tokens != null) parts.push(`${item.prompt_tokens} in`) + if (item?.completion_tokens != null) parts.push(`${item.completion_tokens} out`) + return parts.join(' · ') +} + +function tone(item) { + if (item?.status === 'failed' || item?.status === 'error' || item?.kind === 'blocked') return 'bad' + if (item?.kind === 'retry') return 'warn' + if (item?.stage1 === 'local_fallback' || item?.status === 'local_fallback') return 'warn' + if (item?.stage1 === 'model_accepted' || item?.stage1 === 'local_ok' || item?.status === 'ok' || item?.status === 'local_ok') return 'ok' + return '' +} + +export default function RunLogPopup({ + open, + status = 'ok', + title = 'Ablauf', + log = [], + error = '', + trace = null, + decision = null, + showTrace = false, + onClose, +}) { + useEffect(() => { + if (!open) return undefined + const onKey = (event) => { + if (event.key === 'Escape') onClose?.() + } + window.addEventListener('keydown', onKey) + return () => window.removeEventListener('keydown', onKey) + }, [open, onClose]) + + if (!open) return null + + const running = status === 'running' + const failed = status === 'error' + const events = Array.isArray(log) ? log : [] + + return ( +
    +
    event.stopPropagation()} + > +
    +
    +

    {title}

    +

    + {running ? 'Läuft … zwei Modellaufrufe nacheinander' : failed ? 'Abgebrochen' : 'Fertig'} +

    +
    + +
    + {error &&

    {error}

    } + {running && events.length === 0 && ( +

    Warte auf Maskierung, Stufe 1 und Stufe 2. Retries erscheinen hier.

    + )} +
      + {events.map((item, index) => ( +
    1. + {item.t_ms != null ? `${item.t_ms} ms` : ''} +
      + + {stageLabel(item) ? `${stageLabel(item)} · ` : ''} + {kindLabel(item)} + + {detail(item) &&

      {detail(item)}

      } +
      +
      2. + ))} +
    + {showTrace && (trace || decision) && ( +
    + +
    + )} +
    +
    + ) +} diff --git a/frontend/src/config/adminNav.js b/frontend/src/config/adminNav.js index 13460c9..a799edf 100644 --- a/frontend/src/config/adminNav.js +++ b/frontend/src/config/adminNav.js @@ -6,6 +6,7 @@ export function getAdminNavItems() { { to: '/admin/placeholders', label: 'Platzhalter' }, { to: '/admin/features', label: 'Kontingente' }, { to: '/admin/providers', label: 'Schnittstellen' }, + { to: '/admin/identities', label: 'Identitäten' }, { to: '/admin/dialogue', label: 'Dialog' } ] } diff --git a/frontend/src/pages/AdminHomePage.jsx b/frontend/src/pages/AdminHomePage.jsx index 95286a1..bdeb7d7 100644 --- a/frontend/src/pages/AdminHomePage.jsx +++ b/frontend/src/pages/AdminHomePage.jsx @@ -33,9 +33,9 @@ export default function AdminHomePage() {
    Sprachmodell
    {health.inventory?.providers?.generate?.ready ? 'bereit' : 'nicht bereit'}
    Maskierung
    -
    {health.inventory?.providers?.detect?.ready ? 'bereit (LLM)' : 'Muster'}
    +
    {health.inventory?.providers?.detect?.ready ? 'bereit (semantisch)' : 'nicht bereit, fail-closed'}
    -

    Weiter: Nutzer · Prompts · Platzhalter · Kontingente · Schnittstellen · Dialog

    +

    Weiter: Nutzer · Prompts · Platzhalter · Kontingente · Schnittstellen · Identitäten · Dialog

    )} diff --git a/frontend/src/pages/AdminIdentitiesPage.jsx b/frontend/src/pages/AdminIdentitiesPage.jsx new file mode 100644 index 0000000..5930c87 --- /dev/null +++ b/frontend/src/pages/AdminIdentitiesPage.jsx @@ -0,0 +1,190 @@ +import { useEffect, useState } from 'react' +import { api } from '../api.js' +import { useAuth } from '../context/AuthContext.jsx' + +const emptyForm = { canonical_label: '', entity_type: 'PERSON', aliases: '' } + +export default function AdminIdentitiesPage() { + const { session } = useAuth() + const [data, setData] = useState(null) + const [form, setForm] = useState(emptyForm) + const [error, setError] = useState('') + const [notice, setNotice] = useState('') + + const load = () => + api('/api/admin/identities', { token: session.token }) + .then(setData) + .catch((e) => setError(e.message)) + + useEffect(() => { load() }, [session.token]) + + const aliasesOf = (value) => + value + .split(',') + .map((item) => item.trim()) + .filter(Boolean) + + const create = async (e) => { + e.preventDefault() + setError('') + setNotice('') + try { + await api('/api/admin/identities', { + token: session.token, + method: 'POST', + body: { + canonical_label: form.canonical_label, + entity_type: form.entity_type, + aliases: aliasesOf(form.aliases) + } + }) + setForm(emptyForm) + setNotice('Eintrag bestätigt und lokal gespeichert. Kein externer Aufruf.') + await load() + } catch (err) { + setError(err.message) + } + } + + const patch = async (id, body) => { + setError('') + try { + await api(`/api/admin/identities/${id}`, { + token: session.token, + method: 'PATCH', + body + }) + await load() + } catch (err) { + setError(err.message) + } + } + + const remove = async (id) => { + setError('') + try { + await api(`/api/admin/identities/${id}`, { token: session.token, method: 'DELETE' }) + await load() + } catch (err) { + setError(err.message) + } + } + + const confirmProposal = async (id) => { + setError('') + try { + await api(`/api/admin/identities/proposals/${id}/confirm`, { + token: session.token, + method: 'POST', + body: {} + }) + await load() + } catch (err) { + setError(err.message) + } + } + + const dismissProposal = async (id) => { + setError('') + try { + await api(`/api/admin/identities/proposals/${id}/dismiss`, { + token: session.token, + method: 'POST', + body: {} + }) + await load() + } catch (err) { + setError(err.message) + } + } + + return ( +
    +

    Identitäten

    +

    + Nur lokale bestätigte Registry. Detect-Treffer werden nicht automatisch aktiv. + Unbestätigte Vorschläge gelten nicht als bekannte Identität. Kein externer Egress. +

    + {error &&

    {error}

    } + {notice &&

    {notice}

    } + {data?.backup &&

    {data.backup}

    } +
    + + + + + + + + + + + {(data?.registry || []).map((item) => ( + + + + + + + + ))} + +
    BezeichnungTypStatusAliase +
    {item.canonical_label} + + {item.status}{(item.aliases || []).join(', ')} + {item.status !== 'confirmed' && ( + + )} + {item.status !== 'inactive' && ( + + )} + +
    +
    +
    +

    Identität bestätigen

    + + + + +
    +

    Unbestätigte Detect-Vorschläge

    +

    Wirken nicht auf spätere Requests und ersetzen keine vollständige Detection.

    +
      + {(data?.proposals || []).map((item) => ( +
    • + {item.observed_label} · {item.entity_type} · {item.status} + {item.status === 'unconfirmed' && ( + <> + {' '} + + + + )} +
      • + ))} +
    +
    + ) +} diff --git a/frontend/src/pages/AdminProvidersPage.jsx b/frontend/src/pages/AdminProvidersPage.jsx index 1275b63..ba35fe2 100644 --- a/frontend/src/pages/AdminProvidersPage.jsx +++ b/frontend/src/pages/AdminProvidersPage.jsx @@ -13,7 +13,7 @@ const emptyRole = () => ({ }) function statusLabel(role) { - if (role.role === 'detect' && !role.url) return 'Muster, kein LLM' + if (role.role === 'detect' && !role.url) return 'nicht konfiguriert, fail-closed' if (role.role === 'detect' && role.url && !role.local && role.ready) { return role.remote_plaintext_allowed === false ? 'extern konfiguriert, Produktiv blockiert' @@ -93,8 +93,8 @@ export default function AdminProvidersPage() {

    Schnittstellen

    - Zwei eigene Einträge: Maskierung (Detect-Call) und Sprachmodell (Dialogzug und Journalentwurf). - Externes Klartext-Detect ist nur in Development/Test zulässig. Produktiv bleibt Detect lokal oder bei Mustern. + Zwei eigene Einträge: Maskierung (vollständige semantische Detection) und Sprachmodell (Dialogzug und Journalentwurf). + Externes Klartext-Detect ist nur in Development/Test zulässig. Produktiv ist ein lokales Detect-Modell nötig. Ohne Detection kein Generate. Der Secret-Key wird nie angezeigt und nie in der Datenbank gespeichert.

    {error &&

    {error}

    } diff --git a/frontend/src/pages/DialoguePage.jsx b/frontend/src/pages/DialoguePage.jsx index 1998d43..6fc3ddc 100644 --- a/frontend/src/pages/DialoguePage.jsx +++ b/frontend/src/pages/DialoguePage.jsx @@ -151,9 +151,9 @@ export default function DialoguePage() {
    {messages.map((item) => ( -

    +

    {item.role === 'assistant' ? 'Kanshō' : 'Du'}: {item.body} -

    +
    ))} {!activeId &&

    Noch kein Gespräch. Lege eines an und erzähle.

    }
    diff --git a/frontend/src/pages/JournalDayPage.jsx b/frontend/src/pages/JournalDayPage.jsx index fb66e61..5323997 100644 --- a/frontend/src/pages/JournalDayPage.jsx +++ b/frontend/src/pages/JournalDayPage.jsx @@ -3,6 +3,7 @@ import { Link, useNavigate, useParams } from 'react-router-dom' import { api } from '../api.js' import CallTrace from '../components/CallTrace.jsx' import DayScratch from '../components/DayScratch.jsx' +import RunLogPopup from '../components/RunLogPopup.jsx' import { useAuth } from '../context/AuthContext.jsx' import { entryTitle } from '../journal/document.js' @@ -26,6 +27,9 @@ export default function JournalDayPage() { const [decision, setDecision] = useState(null) const [chooseSources, setChooseSources] = useState(false) const [includeExisting, setIncludeExisting] = useState(false) + const [logOpen, setLogOpen] = useState(false) + const [logStatus, setLogStatus] = useState('ok') + const [runLog, setRunLog] = useState([]) const loadDay = async (preferId) => { const data = await api(`/api/journal/days/${dayId}`, { token: session.token }) @@ -112,6 +116,9 @@ export default function JournalDayPage() { setError('') setBusy(true) setChooseSources(false) + setLogOpen(true) + setLogStatus('running') + setRunLog([]) try { const body = {} if (conversationIds?.length) body.conversation_ids = conversationIds @@ -121,11 +128,18 @@ export default function JournalDayPage() { method: 'POST', body }) + const log = result.run_log || result.trace?.log || [] setDecision(null) setTrace(result.trace || null) + setRunLog(log) await loadDay(activeId) - navigate(`/journal/${spaceId}/${dayId}/entry?draft=1`, { state: { trace: result.trace || null } }) + navigate(`/journal/${spaceId}/${dayId}/entry?draft=1`, { + state: { trace: result.trace || null, run_log: log } + }) } catch (err) { + const log = err.payload?.detail?.diagnostics?.log || [] + setRunLog(log) + setLogStatus('error') setError(err.message) } finally { setBusy(false) @@ -240,9 +254,9 @@ export default function JournalDayPage() {
    {messages.map((item) => ( -

    +

    {item.role === 'assistant' ? 'Kanshō' : 'Du'}: {item.body} -

    +
    ))} {!activeId &&

    Noch kein Gespräch. Lege eines an — Kanshō setzt den ersten Impuls.

    }
    @@ -267,6 +281,16 @@ export default function JournalDayPage() { onError={setError} />
    + setLogOpen(false)} + />
    ) } diff --git a/frontend/src/pages/JournalEditorPage.jsx b/frontend/src/pages/JournalEditorPage.jsx index 42c7cfd..09404c1 100644 --- a/frontend/src/pages/JournalEditorPage.jsx +++ b/frontend/src/pages/JournalEditorPage.jsx @@ -3,6 +3,7 @@ import { Link, useLocation, useNavigate, useParams, useSearchParams } from 'reac import { api, apiBlob, apiUpload } from '../api.js' import CallTrace from '../components/CallTrace.jsx' import JournalDocumentEditor from '../components/JournalDocumentEditor.jsx' +import RunLogPopup from '../components/RunLogPopup.jsx' import { useAuth } from '../context/AuthContext.jsx' import { useUnsavedChanges } from '../context/UnsavedChanges.jsx' import { entryTitle, formatWhen, originLabel } from '../journal/document.js' @@ -33,6 +34,9 @@ export default function JournalEditorPage() { const [currentVersionId, setCurrentVersionId] = useState('') const { dirty, setDirty } = useUnsavedChanges() const savedRef = useRef({ title: '', body: '' }) + const [logOpen, setLogOpen] = useState( + Boolean((location.state?.run_log || []).length || location.state?.trace) + ) const markClean = (nextTitle, nextBody) => { const titleValue = entryTitle(nextTitle || '') @@ -260,6 +264,14 @@ export default function JournalEditorPage() {

    Zurück zum Tag {day?.day?.calendar_date ? ` · ${day.day.calendar_date}` : ''} + {(location.state?.run_log || location.state?.trace) && ( + <> + {' · '} + + + )}

    {wantDraft ? 'Neuer Entwurf' : 'Journal Editor'}{dirty ? ' · ungespeichert' : ''}

    @@ -336,7 +348,7 @@ export default function JournalEditorPage() { )} - {isAdmin && location.state?.trace && } + {isAdmin && location.state?.trace && !logOpen && } {versions.length > 0 && (

    Versionen

    @@ -370,6 +382,15 @@ export default function JournalEditorPage() {
    )} + setLogOpen(false)} + /> ) } diff --git a/frontend/src/pages/JournalSourcePage.jsx b/frontend/src/pages/JournalSourcePage.jsx index 147df63..2fa8465 100644 --- a/frontend/src/pages/JournalSourcePage.jsx +++ b/frontend/src/pages/JournalSourcePage.jsx @@ -23,7 +23,9 @@ export default function JournalSourcePage() { {error &&

    {error}

    }
    {(data?.messages || []).map((item) => ( -

    {item.role}: {item.body}

    +
    + {item.role === 'assistant' ? 'Kanshō' : 'Du'}: {item.body} +
    ))}