diff --git a/.gitignore b/.gitignore index cce80a1..7e80cda 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,6 @@ tmp/ temp/ data/local/ backend/data/ +local-backups/ +*.kansho-backup.zip *.db diff --git a/README.md b/README.md index f15a8b9..2fab3f3 100644 --- a/README.md +++ b/README.md @@ -53,14 +53,21 @@ cd frontend npm run dev ``` -Kanshō nutzt eigene lokale Ports, nicht die Vite-/FastAPI-Defaults: Frontend **5188**, Backend **8018** (`strictPort`, kein Ausweichen auf 5173/5174). Dann http://localhost:5188 – erster Start legt das Admin-Profil an. Frame-Test: +Kanshō nutzt eigene lokale Ports, nicht die Vite-/FastAPI-Defaults: Frontend **5188**, Backend **8018** (`strictPort`, kein Ausweichen auf 5173/5174). Dann http://localhost:5188 – erster Start legt das Admin-Profil an. + +Frame-Test und lokaler Abnahmelauf: + +```powershell +.\scripts\test-mvp.ps1 +``` + +Der Lauf setzt UTF-8, leere Provider-Keys und isolierte Temp-Daten. Er ändert nicht `backend/data/kansho.sqlite` und nicht die echten Medien. Fail-closed bleibt prüfbar; Suites, die einen Fake-Provider brauchen, setzen ihn selbst. Einzelne Suites bleiben möglich: ```powershell cd backend +$env:PYTHONUTF8 = "1" .\.venv\Scripts\python tests\test_frame.py -.\.venv\Scripts\python tests\test_dialogue_memory.py .\.venv\Scripts\python tests\test_mvp_journal.py -.\.venv\Scripts\python tests\test_privacy_detect.py ``` Zwei Provider. URL, Modell und ZDR/No-Train unter **Admin → Schnittstellen**. Der Secret-Key nur in `backend/.env` (Vorlage: `backend/.env.example`, wird nicht committet) oder ebenfalls dort eintragen, nie in der Datenbank. @@ -69,6 +76,17 @@ Das Sprachmodell (Dialogzug und Journalentwurf) bleibt fail-closed ohne Key plus Tests: `KANSHO_FAKE_PROVIDER=1`, optional `KANSHO_FAKE_DETECT=1`. Zusätzlich `tests/test_privacy_detect.py`. +Lokales Backup der Urlaubs-Testdaten (persönlich, unverschlüsselt, ohne `.env` oder Keys): + +```powershell +.\scripts\backup-local.ps1 create +.\scripts\backup-local.ps1 restore -Archive .\local-backups\.zip -Confirm +# bestehenden Stand ersetzen, nach automatischem Sicherheitsbackup: +.\scripts\backup-local.ps1 restore -Archive .\local-backups\.zip -Confirm -Replace +``` + +Restore bestätigt ausdrücklich, legt vorher ein Sicherheitsbackup an und überschreibt nie still. Backend währenddessen beenden. + ## Lokal weiterarbeiten ```powershell diff --git a/backend/config/prompts.seed.json b/backend/config/prompts.seed.json index 9a2c462..171a148 100644 --- a/backend/config/prompts.seed.json +++ b/backend/config/prompts.seed.json @@ -7,17 +7,27 @@ "category": "mvp", "prompt_type": "base", "required_feature": "ai_calls", - "template": "Du sitzt mit [[SELF]] im Reflection Space {{space_title}} und bist Kanshō: jemand, der zugehört hat. Warm, genau, auf Augenhöhe. Ein Mensch würde dich für einen Vertrauten halten, nicht für ein Programm.\n\n{{register_hint}}\n\n{{interaction_hint}}\n\nDie gesprochene Zeile (impulse) klingt wie mündliches Deutsch in einem Gespräch: flüssig, nicht steif, nicht literarisch, nicht wie eine App. Kurze Sätze sind in Ordnung. Kein „also“-Referat, kein „interessant“, kein „danke, dass du das teilst“, kein „lass uns“, kein „zusammengefasst“, keine Operationsnamen, kein JSON, keine Registerworte.\n\nNur user:-Zeilen sind Erlebtes. assistant:-Zeilen sind frühere Impulse, keine Tatsachen.\nErfinde nichts: keinen nächsten Ort, kein Gefühl, keine Motivation. Keine Tatsachen, Gefühle oder Motivationen erfinden. Ein Plan ist kein Vollzug. Nicht erwähnt ist nicht geschehen.\n\nLänge und Zahl der Anschlussimpulse stehen in der Interaction-Angabe oben, nicht als anonyme Produktregel. Leite einen echten Gedanken oder eine knappe Frage aus dem Gerade-Gesagten ab. Zusammenfassung des Zugs ist nicht der Default.\nNacherzählen nur, um einen Widerspruch oder Logikbruch zu klären.\n\nIntern eine Operation wählen, sie nicht aussprechen:\n- fortfuehren: am offenen Faden bleiben; bei Schluss den Schluss halten\n- konkretisieren: eine wirklich fehlende, tragende Angabe\n- plan_aufgreifen: ein genanntes Vorhaben als Vorhaben\n- abweichung: eine selbst markierte Veränderung\n- erleben_vertiefen: nur wenn [[SELF]] Wahrnehmung oder Wirkung öffnet\n- bedeutung: einen erkennbaren Punkt, ohne Tiefe zu erzwingen\n\nBei geschlossener Szene: keine Frage, keinen Bogen. Nicht den nächsten Vollzug setzen.\n\nKontext:\n{{dialogue_context}}\n\nNur JSON:\n{\"operation\":\"fortfuehren|konkretisieren|plan_aufgreifen|abweichung|erleben_vertiefen|bedeutung\",\"impulse\":\"gesprochene Zeile wie ein Vertrauter\"}\n" + "template": "Du sitzt mit [[SELF]] im Reflection Space {{space_title}} und bist Kanshō: jemand, der zugehört hat. Warm, genau, auf Augenhöhe. Ein Mensch würde dich für einen Vertrauten halten, nicht für ein Programm.\n\n{{register_hint}}\n\n{{opening_hint}}\n\n{{interaction_hint}}\n\nDie gesprochene Zeile (impulse) klingt wie mündliches Deutsch in einem Gespräch: flüssig, nicht steif, nicht literarisch, nicht wie eine App. Kurze Sätze sind in Ordnung. Kein „also“-Referat, kein „interessant“, kein „danke, dass du das teilst“, kein „lass uns“, kein „zusammengefasst“, keine Operationsnamen, kein JSON, keine Registerworte.\n\nNur user:-Zeilen sind Erlebtes. assistant:-Zeilen sind frühere Impulse, keine Tatsachen.\nErfinde nichts: keinen nächsten Ort, kein Gefühl, keine Motivation. Keine Tatsachen, Gefühle oder Motivationen erfinden. Ein Plan ist kein Vollzug. Nicht erwähnt ist nicht geschehen.\n\nLänge und Zahl der Anschlussimpulse stehen in der Interaction-Angabe oben, nicht als anonyme Produktregel. Leite einen echten Gedanken oder eine knappe Frage aus dem Gerade-Gesagten ab. Zusammenfassung des Zugs ist nicht der Default.\nNacherzählen nur, um einen Widerspruch oder Logikbruch zu klären.\n\nIntern eine Operation wählen, sie nicht aussprechen:\n- fortfuehren: am offenen Faden bleiben; bei Schluss den Schluss halten\n- konkretisieren: eine wirklich fehlende, tragende Angabe\n- plan_aufgreifen: ein genanntes Vorhaben als Vorhaben\n- abweichung: eine selbst markierte Veränderung\n- erleben_vertiefen: nur wenn [[SELF]] Wahrnehmung oder Wirkung öffnet\n- bedeutung: einen erkennbaren Punkt, ohne Tiefe zu erzwingen\n\nBei geschlossener Szene: keine Frage, keinen Bogen. Nicht den nächsten Vollzug setzen.\n\nKontext:\n{{dialogue_context}}\n\nNur JSON:\n{\"operation\":\"fortfuehren|konkretisieren|plan_aufgreifen|abweichung|erleben_vertiefen|bedeutung\",\"impulse\":\"gesprochene Zeile wie ein Vertrauter\"}\n" }, { "id": "mvp-journal-generate", "slug": "mvp.journal_generate", "name": "MVP Journalentwurf", - "description": "Tagebuchtext aus den Worten der Person. Explizit ausgelöst. Nicht der Dialogzug.", + "description": "Stufe 2: persönliche Narration aus validierter Rekonstruktion. Explizit ausgelöst. Nicht der Dialogzug.", "category": "mvp", "prompt_type": "base", "required_feature": "ai_calls", - "template": "Schreibe einen Tagebucheintrag in der Ich-Form von [[SELF]].\n\nAus den user:-Zeilen den Inhalt nehmen; die Stimme aus dem Writing Profile, sobald dort finale Fassungen oder Importe stehen. Szenen, Wartezeiten und Erzählstränge zusammenführen, nicht den Dialog hintereinanderkopieren. Uhrzeiten und Reihenfolge aus dem Gesagten behalten. Keine Zeiten erfinden, nichts umstellen, das die Chronologie verfälscht.\n\nSchritte, alle nötig:\n1. Nur Erlebtes aus user:. assistant:-Zeilen sind Kanshōs Impulse, keine Tatsachen. Sie zeigen nur, worauf die nächste user-Zeile antwortet; nicht in den Eintrag schreiben.\n2. Absätze nach Phase, Ort oder Uhrzeit. Nicht jede user-Zeile ein eigener Absatz. Dieselbe Begebenheit bleibt ein Strang.\n3. Übergänge glätten. Wiederholungen und Rückfragen streichen. Aufeinanderfolgende Züge zur selben Szene verbinden.\n4. Person: das Token nur dort, wo [[SELF]] den Namen genannt hat — in einer Passage ein- bis zweimal. Danach sie/er/ihr wie im user-Text. Indirekte Rede nicht umbiegen: „sie sagte, dass sie …“, nicht den Namen in die Selbstaussage setzen.\n5. Writing Profile: finale Fassungen sind die Stimme (Satzlänge, Wortwahl, Detailgrad, Übergänge). Kurze oder stichwortartige user-Zeilen zu ganzen Sätzen und Absätzen in genau dieser Stimme ausformulieren. Keine neuen Fakten, Gefühle oder Zeiten. Fehlt das Profile, die user-Formulierungen verbinden und glätten.\n\nErste Zeile: kurze Überschrift. Danach der Eintrag.\nBehalten: konkrete Wörter, Uhrzeiten, Details und selbst genannte Eindrücke — in der Stimme des Writing Profile, nicht als Dialogkopie.\nVerboten: den Dialog 1:1 kopieren; Kanshō-Fragen als Erlebtes; nur Überschrift plus denselben Text; ein einziger Absatz für den ganzen Tag; erfundene Fakten, Gefühle oder Details; generischer KI-Stil; jeden Bezug auf eine Person durch den Namen ersetzen.\n\nWriting Profile:\n{{writing_profile}}\n\nBestehender Text, nur wenn ausdrücklich gewünscht:\n{{existing_text}}\n\nDialogquellen:\n{{dialogue_context}}\n\nSchreibe jetzt den Eintrag: erste Zeile Überschrift, danach mehrere Absätze. Inhalt aus user:, Stimme aus dem Writing Profile. Verbinde Szenen und Zeiten, kopiere die user-Zeilen nicht.\n" + "template": "Schreibe einen Tagebucheintrag in der Ich-Form von [[SELF]].\n\nUnten liegt ausschließlich ein lokal erzeugtes Verified Artifact. Es ist die einzige Inhaltsquelle. sources[].text ist der vollständige, lokal rehydrierte Nutzerinhalt. Evidence-Ausschnitte sind nur nachgewiesene Zitate, keine Erlaubnis, den Rest wegzulassen. annotations_unverified, falls vorhanden, sind unbestätigte Modelldeutung und keine Tatsachen: daraus keine Gefühle, Widersprüche, Korrekturen oder Planänderungen machen.\n\nKein Rohdialog, keine Assistentenzeilen, keine frei erfundenen Fakten. Stimme aus dem Writing Profile, sobald dort Core, Facet-Delta oder Traits stehen. Szenen zusammenführen. Die Reihenfolge der sources-Einträge behalten. Keine Zeiten, Gefühle, Motive oder Zusammenhänge ergänzen.\n\nSchritte, alle nötig:\n1. Nur den verifizierten Nutzerwortlaut aus sources[].text.\n2. Absätze nach Phase, Ort oder Uhrzeit. Dieselbe Begebenheit bleibt ein Strang.\n3. Übergänge glätten. Wiederholungen streichen. Unsicherheiten im Wortlaut sichtbar lassen, nicht glätten und nicht aus Labels ableiten.\n4. Person: das Token nur dort, wo [[SELF]] den Namen genannt hat — in einer Passage ein- bis zweimal. Danach sie/er/ihr. Indirekte Rede nicht umbiegen.\n5. Writing Profile: Satzlänge, Wortwahl, Detailgrad, Übergänge. Keine neuen Fakten.\n\nErste Zeile: kurze Überschrift. Danach der Eintrag.\nVerboten: Assistentenimpulse als Erlebtes; erfundene Fakten, Gefühle oder Details; generischer KI-Stil; Unsicherheit wegglätten; unbestätigte Labels als Wahrheit.\n\nWriting Profile:\n{{writing_profile}}\n\nBestehender Text, nur wenn ausdrücklich gewünscht:\n{{existing_text}}\n\nVerified Artifact:\n{{reconstruction}}\n\nSchreibe jetzt den Eintrag: erste Zeile Überschrift, danach mehrere Absätze. Inhalt nur aus sources[].text, Stimme aus dem Writing Profile.\n" + }, + { + "id": "mvp-journal-reconstruct", + "slug": "mvp.journal_reconstruct", + "name": "MVP Journal Rekonstruktion", + "description": "Stufe 1: inhaltliche Rekonstruktion des Tagesdialogs. Keine Narration, kein Writing Profile.", + "category": "mvp", + "prompt_type": "base", + "required_feature": "ai_calls", + "template": "Referenziere die lokal vergebenen Quellen. Du entscheidest nicht, welche ausgewählten Nutzerpassagen verloren gehen dürfen. Du lieferst keine autoritative Semantik.\n\nJede nichtleere Nutzerzeile hat eine lokale Quellen-ID in eckigen Klammern: [u1], [u2], … in der ursprünglichen Reihenfolge. Assistant-Zeilen haben [a1], [a2], … und sind nur Antwortkontext, niemals Inhaltsquelle oder Evidence.\n\nJeder Claim ist ausschließlich ein wörtlicher Ausschnitt aus genau der genannten Nutzerzeile. Keine Paraphrasen. Keine Assistentenformulierungen. kind-Werte und contradictions/uncertainties/plan_changes/corrections sind unverbindliche Hinweise, keine festgestellten Tatsachen. Eine Evidence legitimiert keine zusätzliche Behauptung.\n\nsource_order ist die lokale Nutzerreihenfolge. chronology darf davon nur abweichen, wenn Uhrzeiten in der jeweils zugeordneten Nutzerzeile nachweisbar sind. Relative Angaben wie später oder danach nicht zum Umsortieren verwenden, wenn die Reihenfolge nicht aus bestätigten Uhrzeiten folgt. Gleiche oder mehrdeutige Uhrzeiten in Quellenreihenfolge lassen.\n\nAntworte ausschließlich mit JSON:\n{\"source_order\":[\"u1\",\"u2\"],\"chronology\":[{\"source_id\":\"u1\",\"source\":\"user\",\"time\":null,\"claims\":[{\"kind\":\"event\",\"evidence\":\"wörtlicher Ausschnitt aus genau dieser Nutzerzeile\"}]}],\"contradictions\":[],\"uncertainties\":[],\"plan_changes\":[],\"corrections\":[]}\n\nsource_order muss jede Nutzer-ID genau einmal in der lokalen Reihenfolge enthalten. chronology muss jede Nutzer-ID genau einmal abdecken. source ist Pflicht und immer genau user. time nur als reines Zeitstoken aus derselben Nutzerzeile (6:00, 06:00, 6:00 Uhr). Assistant-IDs und jedes andere source sind verboten. Keine events-Listen, keine allgemeine Evidence für mehrere Behauptungen. Du darfst den kanonischen Quellentext nicht ersetzen oder kürzen.\n\nDialog:\n{{dialogue_context}}\n" }, { "id": "mvp-entity-detect", diff --git a/backend/context_builder.py b/backend/context_builder.py index 580b44f..cf4b412 100644 --- a/backend/context_builder.py +++ b/backend/context_builder.py @@ -11,7 +11,7 @@ from dialogue_store import get_conversation from retrieval import retrieve from journal_body import plain_text -PURPOSES = {"dialogue_turn", "journal_generate"} +PURPOSES = {"dialogue_turn", "journal_generate", "journal_reconstruct"} CLOSURE = re.compile( r"\bauf den heimweg\b|" r"\bheimweg\b|" @@ -73,6 +73,8 @@ def build_internal_context( include_existing: bool = False, conversation_ids: list[str] | None = None, existing_text: str = "", + reconstruction: str = "", + day_spec: dict | None = None, ) -> dict: purpose = purpose if purpose in PURPOSES else "dialogue_turn" if conversation_id and not space_id: @@ -96,17 +98,17 @@ def build_internal_context( selected_ids.append(conversation_id) day_messages = [] - if journal_day_id: - day_messages = retrieve( - profile_id, - { - "kind": "day_messages", - "journal_day_id": journal_day_id, - "conversation_ids": selected_ids or None, - }, - ) + if journal_day_id and purpose != "journal_generate": + spec = { + "kind": "day_messages", + "journal_day_id": journal_day_id, + "conversation_ids": selected_ids or None, + } + if day_spec: + spec.update({key: value for key, value in day_spec.items() if key != "kind"}) + day_messages = retrieve(profile_id, spec) items.append({"type": "day_messages", "messages": day_messages}) - elif conversation_id: + elif conversation_id and purpose == "dialogue_turn": day_messages = read("conversation_messages", profile_id=profile_id, context={"conversation_id": conversation_id}) items.append({"type": "day_messages", "messages": day_messages}) @@ -148,6 +150,8 @@ def build_internal_context( brief = compile_task_brief(profile_id, "journal_generate") items.append({"type": "writing_profile", "compiled_brief": brief}) + if reconstruction: + items.append({"type": "reconstruction", "body": reconstruction}) if include_existing and existing_text: items.append({"type": "existing_text", "body": plain_text(existing_text)}) elif purpose == "dialogue_turn": @@ -174,15 +178,25 @@ def assemble_text(context: dict) -> dict[str, str]: writing_profile = "" interaction_hint = "" existing_text = "" + reconstruction = "" user_bodies: list[str] = [] purpose = context.get("purpose") or "" for item in context.get("items") or []: kind = item.get("type") if kind == "day_messages": - for message in item.get("messages") or []: + day_messages = item.get("messages") or [] + if purpose == "journal_reconstruct": + from journal_reconstruct import assign_source_ids + + day_messages = assign_source_ids(day_messages) + for message in day_messages: role = message.get("role") or "user" body = message.get("body") or "" - dialogue_parts.append(f"{role}: {body}") + source_id = message.get("source_id") + if purpose == "journal_reconstruct" and source_id: + dialogue_parts.append(f"[{source_id}] {role}: {body}") + else: + dialogue_parts.append(f"{role}: {body}") if role == "user": user_bodies.append(body) elif kind == "prior_entries": @@ -202,6 +216,11 @@ def assemble_text(context: dict) -> dict[str, str]: interaction_hint = item.get("text") or "" elif kind == "existing_text": existing_text = item.get("body") or "" + elif kind == "reconstruction": + reconstruction = item.get("body") or "" + elif kind == "opening": + pass + opening_hint = "" if purpose == "dialogue_turn": from conversation_signals import infer_signals from interaction_profile_store import assemble_interaction_hint @@ -221,6 +240,8 @@ def assemble_text(context: dict) -> dict[str, str]: "writing_profile": writing_profile, "interaction_hint": interaction_hint, "existing_text": existing_text, + "reconstruction": reconstruction, "space_title": context.get("space_title") or "", "register_hint": infer_register(user_bodies), + "opening_hint": opening_hint, } diff --git a/backend/dialogue_store.py b/backend/dialogue_store.py index 871daa7..b5721ec 100644 --- a/backend/dialogue_store.py +++ b/backend/dialogue_store.py @@ -339,7 +339,7 @@ def list_conversations_for_day(profile_id: str, journal_day_id: str) -> list[dic (SELECT COUNT(*) FROM messages m WHERE m.conversation_id = c.id) AS message_count FROM conversations c WHERE c.profile_id = ? AND c.journal_day_id = ? - ORDER BY c.created + ORDER BY c.created, c.rowid """, (profile_id, journal_day_id), ).fetchall() diff --git a/backend/dialogue_turn.py b/backend/dialogue_turn.py index 382c7cf..51a51b5 100644 --- a/backend/dialogue_turn.py +++ b/backend/dialogue_turn.py @@ -290,6 +290,13 @@ def visible_for_role(payload: dict, role: str | None) -> dict: cleaned = dict(payload) cleaned.pop("trace", None) cleaned.pop("decision", None) + opening = cleaned.get("opening") + if isinstance(opening, dict): + opening = dict(opening) + opening.pop("trace", None) + opening.pop("decision", None) + opening.pop("opening_context", None) + cleaned["opening"] = opening return cleaned @@ -339,8 +346,16 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str | "parsed": parsed_ok, "guard": "local_fallback", } - except EngineError: - raise + except EngineError as exc: + if exc.code != "response_validation_failed": + raise + impulse = local_hold(last_user_text(assembled)) + decision = { + "operation": "fortfuehren", + "label": OPERATIONS["fortfuehren"], + "parsed": False, + "guard": "identity_leak_blocked", + } assistant = append_message(profile_id, conversation_id, impulse, role="assistant") user_bodies = [ item.get("body") or "" diff --git a/backend/engine.py b/backend/engine.py index a68e27f..f0b9ff6 100644 --- a/backend/engine.py +++ b/backend/engine.py @@ -9,15 +9,17 @@ import privacy_placeholders from db import get_db, row_to_dict from entitlements import EntitlementError, check_feature_access, increment_feature_usage from placeholders import PlaceholderError, resolve_template -from privacy_gateway import GatewayRequest, PrivacyGatewayError, complete, last_trace, public_trace +from privacy_gateway import GatewayRequest, PrivacyGatewayError, complete +from journal_reconstruct import parse_dialogue_line class EngineError(Exception): - def __init__(self, code: str, message: str, status_code: int = 400): + def __init__(self, code: str, message: str, status_code: int = 400, diagnostics: dict | None = None): super().__init__(message) self.code = code self.message = message self.status_code = status_code + self.diagnostics = diagnostics or {} def load_active_prompt(slug: str) -> dict: @@ -54,12 +56,26 @@ def preview_prompt(prompt: dict, context: dict[str, Any] | None = None) -> dict: def user_source_text(dialogue_context: str) -> str: parts = [] for line in (dialogue_context or "").splitlines(): - if line.startswith("user:"): + parsed = parse_dialogue_line(line) + if parsed and parsed["role"] == "user": + parts.append(parsed["body"]) + elif line.startswith("user:"): parts.append(line[5:].lstrip()) return "\n".join(parts) -def execute_prompt(prompt: dict, profile_id: str, purpose: str, data_class: str, context: dict[str, Any] | None = None) -> dict: +def execute_prompt( + prompt: dict, + profile_id: str, + purpose: str, + data_class: str, + context: dict[str, Any] | None = None, + *, + max_tokens: int | None = None, + disable_context_compression: bool = False, + budget=None, + diagnostics: dict[str, Any] | None = None, +) -> dict: preview = preview_prompt(prompt, context) feature_id = prompt.get("required_feature") or "ai_calls" try: @@ -68,19 +84,13 @@ def execute_prompt(prompt: dict, profile_id: str, purpose: str, data_class: str, raise EngineError(exc.code, exc.message, exc.status_code) from exc try: ctx = context or {} - source_text = user_source_text(ctx.get("dialogue_context") or "") - extras = [] - if purpose == "journal_generate": - extras = [part for part in (ctx.get("existing_text"), ctx.get("writing_profile")) if part] - elif purpose == "profile_review": - source_text = ctx.get("source_text") or "" - extras = [part for part in (ctx.get("review_package"),) if part] - if extras: - source_text = "\n".join(part for part in (source_text, *extras) if part) - elif not source_text: + source_text = preview["rendered"] + if purpose == "profile_review" and ctx.get("source_text"): source_text = "\n".join( - part for part in (ctx.get("existing_text"), ctx.get("writing_profile")) if part + part for part in (ctx.get("source_text"), ctx.get("review_package"), source_text) if part ) + elif not (source_text or "").strip(): + source_text = user_source_text(ctx.get("dialogue_context") or "") result = complete( GatewayRequest( prompt_id=prompt["id"], @@ -92,16 +102,21 @@ def execute_prompt(prompt: dict, profile_id: str, purpose: str, data_class: str, "source_text": source_text, "privacy_tokens": preview["privacy_tokens"], "prompt_slug": prompt.get("slug"), + "max_tokens": max_tokens, + "disable_context_compression": disable_context_compression, + "budget": budget, + "diagnostics": diagnostics or {}, }, ) ) except PrivacyGatewayError as exc: - raise EngineError(exc.code, exc.message, exc.status_code) from exc + raise EngineError(exc.code, exc.message, exc.status_code, getattr(exc, "diagnostics", None)) from exc increment_feature_usage(profile_id, feature_id) return { **preview, "llm": True, "content": result.content, "provider": result.provider, - "trace": public_trace(last_trace()), + "trace": result.trace, + "diagnostics": result.diagnostics, } diff --git a/backend/entity_detect.py b/backend/entity_detect.py index bc051cb..ac08b5c 100644 --- a/backend/entity_detect.py +++ b/backend/entity_detect.py @@ -159,13 +159,13 @@ def _llm_entities(config, excerpt: str, known_labels: set[str]) -> list[dict]: _detect_prompt()["template"], {"source_text": excerpt, "known_labels": known}, ) - raw = complete_chat( + result = complete_chat( config, [{"role": "user", "content": prompt}], timeout=DETECT_TIMEOUT, max_tokens=DETECT_MAX_TOKENS, ) - return _parse_entities(raw) + return _parse_entities(result.content) def detect_and_remember(profile_id: str | None, source_text: str) -> tuple[list[dict], str | None, str | None]: diff --git a/backend/journal_generate.py b/backend/journal_generate.py index 012e216..e8a446e 100644 --- a/backend/journal_generate.py +++ b/backend/journal_generate.py @@ -3,12 +3,26 @@ from __future__ import annotations from context_builder import assemble_text, build_internal_context from dialogue_store import StoreError, list_conversations_for_day, list_messages -from engine import execute_prompt, load_active_prompt +from engine import EngineError, execute_prompt, load_active_prompt from journal_policy import require_explicit_generate, source_conversation_ids from identity_store import list_mappings from journal_body import clean_title +from journal_reconstruct import ( + assign_source_ids, + claim_texts, + local_verified_artifact, + reconstruction_from_model, + reconstruction_text, +) from journal_shape import shape_journal from journal_store import current_draft, current_entries, get_day, insert_draft +from model_catalog import resolve_generate_metadata +from prompt_budget import ( + JournalBudgetError, + estimate_tokens, + plan_journal_budget, +) +from providers import generate_provider from writing_profile_store import compile_style_signals, remember_dialogue_style @@ -32,6 +46,57 @@ def _split_title(content: str) -> tuple[str, str]: return title, body or text +def _local_narration(reconstruction: dict) -> tuple[str, str]: + """Fail-closed draft from verified local sources. No leaking model text.""" + parts = [str(part).strip() for part in claim_texts(reconstruction) if str(part).strip()] + body = "\n\n".join(parts) + return "Ein Tag", body + + +def _identity_leak_result(purpose: str, exc: EngineError) -> dict: + return { + "content": "", + "trace": { + "purpose": purpose, + "guard": "identity_leak_blocked", + }, + "diagnostics": getattr(exc, "diagnostics", None) or {}, + } + + +def _raise_budget(exc: JournalBudgetError) -> None: + raise EngineError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc + + +def _stage_trace(result: dict, fallback_purpose: str) -> dict: + trace = dict(result.get("trace") or {}) + if result.get("diagnostics"): + trace["budget"] = result.get("diagnostics") + elif not trace.get("budget"): + trace["budget"] = None + trace["purpose"] = trace.get("purpose") or fallback_purpose + return trace + + +def _day_messages_from_context(context: dict) -> list[dict]: + for item in context.get("items") or []: + if item.get("type") == "day_messages": + return list(item.get("messages") or []) + return [] + + +def _existing_text(profile_id: str, journal_day_id: str) -> str: + """Saved user entries are the Fassung. A leftover draft is only used if none exist.""" + entries = current_entries(profile_id, journal_day_id) + bodies = [(item.get("body") or "").strip() for item in entries if (item.get("body") or "").strip()] + if bodies: + return "\n\n".join(bodies) + draft = current_draft(profile_id, journal_day_id) + if draft: + return (draft.get("body") or "").strip() + return "" + + def generate_draft( profile_id: str, journal_day_id: str, @@ -47,15 +112,88 @@ def generate_draft( [item["id"] for item in day_conversations], ) remember_dialogue_style(profile_id, exclude_conversation_ids=selected) - existing_text = "" - if include_existing: - draft = current_draft(profile_id, journal_day_id) - entries = current_entries(profile_id, journal_day_id) - if draft: - existing_text = draft.get("body") or "" - elif entries: - existing_text = entries[0].get("body") or "" - context = build_internal_context( + existing_text = _existing_text(profile_id, journal_day_id) if include_existing else "" + + config = generate_provider() + if not config: + raise EngineError( + "no_egress_provider_configured", + "Persönlicher KI-Aufruf wurde vom Privacy Gateway blockiert. " + "Es ist kein Egress-Provider konfiguriert.", + 503, + ) + try: + window = resolve_generate_metadata(config) + reconstruct_budget = plan_journal_budget(window, purpose="journal_reconstruct") + narrate_budget = plan_journal_budget(window, purpose="journal_generate") + except JournalBudgetError as exc: + _raise_budget(exc) + + reconstruct_prompt = load_active_prompt("mvp.journal_reconstruct") + static_tokens = estimate_tokens(reconstruct_prompt.get("template") or "") + available_for_day = reconstruct_budget.available_input_tokens - static_tokens + if available_for_day < 256: + raise EngineError( + "prompt_budget_exceeded", + "Der Tagesdialog ist für eine sichere Verarbeitung zu umfangreich. " + "Kanshō hat nichts stillschweigend aus der Mitte entfernt.", + 422, + diagnostics=reconstruct_budget.as_diagnostics(), + ) + + try: + reconstruct_context = build_internal_context( + profile_id, + space_id=day["space_id"], + journal_day_id=journal_day_id, + purpose="journal_reconstruct", + conversation_ids=selected, + conversation_id=selected[0] if selected else None, + day_spec={ + "overflow": "abort", + "max_estimated_tokens": available_for_day, + }, + ) + except JournalBudgetError as exc: + _raise_budget(exc) + reconstruct_assembled = assemble_text(reconstruct_context) + source_messages = assign_source_ids(_day_messages_from_context(reconstruct_context)) + source_user = [ + message.get("body") or "" + for message in source_messages + if message.get("role") == "user" + ] + reconstruct_result = {"trace": {"purpose": "journal_reconstruct"}, "content": "", "diagnostics": {}} + try: + reconstruct_result = execute_prompt( + reconstruct_prompt, + profile_id, + purpose="journal_reconstruct", + data_class="B", + context=reconstruct_assembled, + max_tokens=reconstruct_budget.reserved_output_tokens, + disable_context_compression=True, + budget=reconstruct_budget, + ) + reconstruction, stage1 = reconstruction_from_model( + reconstruct_result.get("content") or "", + source_messages, + ) + except EngineError as exc: + if exc.code != "response_validation_failed": + raise + reconstruction = local_verified_artifact(source_messages) + stage1 = { + "stage1": "local_fallback", + "reason": "identity_leak_blocked", + "model_rejected": exc.code, + } + reconstruct_result = _identity_leak_result("journal_reconstruct", exc) + except JournalBudgetError as exc: + _raise_budget(exc) + + signals = compile_style_signals(source_user) + narrate_context = build_internal_context( profile_id, space_id=day["space_id"], journal_day_id=journal_day_id, @@ -64,27 +202,31 @@ def generate_draft( conversation_ids=selected, existing_text=existing_text, conversation_id=selected[0] if selected else None, + reconstruction=reconstruction_text(reconstruction), ) - assembled = assemble_text(context) - source_user = [ - message.get("body") or "" - for conversation_id in selected - for message in list_messages(profile_id, conversation_id) - if message.get("role") == "user" - ] - signals = compile_style_signals(source_user) + assembled = assemble_text(narrate_context) profile = assembled.get("writing_profile") or "" - if signals and signals not in profile: + if signals and signals not in profile and len(profile) < 3500: assembled["writing_profile"] = "\n\n".join(part for part in (profile, signals) if part) - prompt = load_active_prompt("mvp.journal_generate") - result = execute_prompt( - prompt, - profile_id, - purpose="journal_generate", - data_class="B", - context=assembled, - ) - title, body = _split_title(result.get("content") or "") + narrate_prompt = load_active_prompt("mvp.journal_generate") + try: + narrate_result = execute_prompt( + narrate_prompt, + profile_id, + purpose="journal_generate", + data_class="B", + context=assembled, + max_tokens=narrate_budget.reserved_output_tokens, + disable_context_compression=True, + budget=narrate_budget, + ) + title, body = _split_title(narrate_result.get("content") or "") + except EngineError as exc: + if exc.code != "response_validation_failed": + raise + title, body = _local_narration(reconstruction) + narrate_result = _identity_leak_result("journal_generate", exc) + narrate_result["content"] = f"{title}\n\n{body}".strip() person_labels = [ (item.get("local_label") or "").strip() for item in list_mappings(profile_id) @@ -106,5 +248,11 @@ def generate_draft( previous = before_entries.get(item["id"]) if previous is not None and previous != item.get("current_version_id"): raise StoreError("policy_violation", "Generate darf die Nutzerfassung nicht verändern", 500) - draft["trace"] = result.get("trace") + reconstruct_trace = _stage_trace(reconstruct_result, "journal_reconstruct") + reconstruct_trace.update(stage1) + narrate_trace = _stage_trace(narrate_result, "journal_generate") + draft["trace"] = { + **narrate_trace, + "stages": [reconstruct_trace, narrate_trace], + } return draft diff --git a/backend/journal_opening.py b/backend/journal_opening.py new file mode 100644 index 0000000..de32a25 --- /dev/null +++ b/backend/journal_opening.py @@ -0,0 +1,313 @@ +"""Journal-specific first impulse. Not a generic continuation or intent engine. + +Uses the existing Privacy Gateway, dialogue prompt, guards and request-scoped +trace. Attested plans and open day points come only from user-role sources. +Assistant text is conversation context, never a user fact. Recency is not a pattern. +""" +from __future__ import annotations + +import re + +from context_builder import assemble_text, build_internal_context +from dialogue_store import StoreError, append_message, get_conversation, list_conversations_for_day, list_messages +from dialogue_turn import OPERATIONS, needs_repair, parse_turn_payload, repair_note +from engine import EngineError, execute_prompt, load_active_prompt +from journal_store import get_day +from retrieval import retrieve + +NEUTRAL_OPENING = "Wenn du magst, fang einfach an – ich höre zu." +OPENING_POLICY = ( + "Dies ist der erste Impuls dieses Gesprächs. Es gibt hier noch keine Nutzerzeile. " + "Nur belegte Vorhaben aus user-Quellen oder offene user-Punkte desselben Journal Day aufgreifen. " + "Ein Recency-Treffer ist kein bewiesenes wiederkehrendes Muster. " + "Nicht erwähnt ist nicht geschehen. Ein Plan ist kein Vollzug. " + "Assistententext ist Gesprächskontext, kein Nutzerfakt. " + "Keine Behauptung, ein Vorhaben sei ausgeführt worden." +) + +FUTURE_INTENT = re.compile( + r"(?:" + r"\b(" + r"morgen|übermorgen|" + r"wollen(?: wir| sie)?|will(?:st)?|werde|werden wir|" + r"vorhaben|geplant|" + r"steht(?: heute| morgen)? an|" + r"nächste[nrs]?\s+(?:woche|monat|tag|tage)" + r")\b" + r"|(?:habe|haben|hat|habt)\s+vor\b" + r")", + re.I, +) +UNEARNED_PATTERN = re.compile( + r"\b(die letzten tage|häufig|immer|jedes mal|wiederkehr|typischerweise|muster)\b", + re.I, +) + + +def is_attested_plan(text: str) -> bool: + """User wording that marks a plan or intention. Not past completion, not overlap.""" + body = (text or "").strip() + if not body: + return False + return bool(FUTURE_INTENT.search(body)) + + +def attested_plans_from_user_texts(texts: list[str]) -> list[str]: + plans: list[str] = [] + seen: set[str] = set() + for text in texts: + body = (text or "").strip() + if not body or body in seen or not is_attested_plan(body): + continue + seen.add(body) + plans.append(body) + return plans + + +def _user_bodies(messages: list[dict]) -> list[str]: + return [ + (item.get("body") or "").strip() + for item in messages + if item.get("role") == "user" and (item.get("body") or "").strip() + ] + + +def _assistant_bodies(messages: list[dict]) -> list[str]: + return [ + (item.get("body") or "").strip() + for item in messages + if item.get("role") == "assistant" and (item.get("body") or "").strip() + ] + + +def collect_opening_context(profile_id: str, conversation_id: str) -> dict: + conversation = get_conversation(profile_id, conversation_id) + space_id = conversation.get("space_id") + journal_day_id = conversation.get("journal_day_id") + if not space_id or not journal_day_id: + raise StoreError("not_journal_conversation", "Erster Impuls nur für einen Journal-Dialog.") + get_day(profile_id, journal_day_id) + + day_conversations = list_conversations_for_day(profile_id, journal_day_id) + open_points: list[str] = [] + day_assistant: list[str] = [] + for item in day_conversations: + messages = list_messages(profile_id, item["id"]) + if item["id"] == conversation_id: + continue + open_points.extend(_user_bodies(messages)) + day_assistant.extend(_assistant_bodies(messages)) + + prior_entries = retrieve( + profile_id, + {"kind": "space_entries", "space_id": space_id, "exclude_day_id": journal_day_id}, + ) + recent_sources = retrieve( + profile_id, + { + "kind": "space_recent_sources", + "space_id": space_id, + "exclude_conversation_id": conversation_id, + "exclude_journal_day_id": journal_day_id, + }, + ) + recency_user: list[str] = [] + for entry in prior_entries: + excerpt = (entry.get("excerpt") or "").strip() + if excerpt: + recency_user.append(excerpt) + for source in recent_sources: + excerpt = (source.get("excerpt") or "").strip() + if excerpt: + recency_user.append(excerpt) + + user_pool = [*open_points, *recency_user] + plans = attested_plans_from_user_texts(user_pool) + has_relevant = bool(plans or open_points) + return { + "space_id": space_id, + "journal_day_id": journal_day_id, + "conversation_id": conversation_id, + "user_texts": user_pool, + "assistant_texts": day_assistant, + "attested_plans": plans, + "open_day_points": open_points, + "recency_excerpts": recency_user, + "has_relevant_context": has_relevant, + "recency_is_not_pattern": True, + "assistant_is_not_user_fact": True, + } + + +def format_opening_hint(facts: dict) -> str: + lines = [OPENING_POLICY] + plans = facts.get("attested_plans") or [] + points = facts.get("open_day_points") or [] + if plans: + lines.append("Belegte Vorhaben (nur user-Quellen):") + lines.extend(f"- {item}" for item in plans) + else: + lines.append("Belegte Vorhaben: keine.") + if points: + lines.append("Offene Punkte dieses Journal Day (nur user):") + lines.extend(f"- {item}" for item in points) + else: + lines.append("Offene Punkte dieses Journal Day: keine.") + lines.append( + "Recency-Ausschnitte dürfen den Impuls nicht als Muster oder als Vollzug begründen." + ) + return "\n".join(lines) + + +def _store_opening(profile_id: str, conversation_id: str, impulse: str, extra: dict) -> dict: + assistant = append_message(profile_id, conversation_id, impulse, role="assistant") + return { + "conversation": get_conversation(profile_id, conversation_id), + "assistant": assistant, + "messages": list_messages(profile_id, conversation_id), + **extra, + } + + +def start_journal_opening(profile_id: str, conversation_id: str) -> dict: + """Create the first Kanshō line. Invalid model output does not write a message.""" + existing = list_messages(profile_id, conversation_id) + if existing: + return { + "opened": False, + "reason": "already_started", + "conversation": get_conversation(profile_id, conversation_id), + "messages": existing, + } + facts = collect_opening_context(profile_id, conversation_id) + if not facts["has_relevant_context"]: + return _store_opening( + profile_id, + conversation_id, + NEUTRAL_OPENING, + { + "opened": True, + "kind": "local_neutral", + "calls": 0, + "opening_context": facts, + "decision": { + "operation": "fortfuehren", + "label": OPERATIONS["fortfuehren"], + "parsed": True, + "guard": "local_neutral_opening", + }, + "trace": None, + }, + ) + + conversation = get_conversation(profile_id, conversation_id) + day_ids = [item["id"] for item in list_conversations_for_day(profile_id, conversation["journal_day_id"])] + context = build_internal_context( + profile_id, + conversation_id=conversation_id, + space_id=conversation.get("space_id"), + journal_day_id=conversation.get("journal_day_id"), + purpose="dialogue_turn", + conversation_ids=day_ids, + ) + assembled = assemble_text(context) + assembled = dict(assembled) + assembled["opening_hint"] = format_opening_hint(facts) + assembled["register_hint"] = ( + "Erster Impuls. Noch keine Nutzerzeile in diesem Gespräch. " + "Nur belegte Vorhaben oder offene user-Punkte. Kein Muster aus Recency." + ) + prompt = load_active_prompt("mvp.dialogue_turn") + calls = 0 + result = None + impulse = "" + decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False} + try: + while calls < 2: + result = execute_prompt( + prompt, + profile_id, + purpose="dialogue_turn", + data_class="B", + context=assembled, + ) + calls += 1 + content = (result.get("content") or "").strip() + if not content: + raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.") + impulse, decision = parse_turn_payload(content) + if UNEARNED_PATTERN.search(impulse or ""): + decision = {**decision, "guard": "pattern_rejected"} + assembled = dict(assembled) + assembled["dialogue_context"] = ( + (assembled.get("dialogue_context") or "") + + "\n\nKorrektur: Kein Recency-Treffer als wiederkehrendes Muster." + ) + continue + if not needs_repair(impulse, assembled): + break + decision = {**decision, "guard": "impulse_rejected"} + assembled = dict(assembled) + assembled["dialogue_context"] = ( + (assembled.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, assembled) + ) + if needs_repair(impulse, assembled) or UNEARNED_PATTERN.search(impulse or "") or not impulse: + impulse = NEUTRAL_OPENING + decision = { + "operation": "fortfuehren", + "label": OPERATIONS["fortfuehren"], + "parsed": bool(decision.get("parsed")), + "guard": "local_fallback", + } + except EngineError as exc: + if exc.code != "response_validation_failed": + raise + return _store_opening( + profile_id, + conversation_id, + NEUTRAL_OPENING, + { + "opened": True, + "kind": "local_neutral", + "calls": calls, + "opening_context": facts, + "decision": { + "operation": "fortfuehren", + "label": OPERATIONS["fortfuehren"], + "parsed": False, + "guard": "identity_leak_blocked", + }, + "trace": result.get("trace") if result else None, + }, + ) + return _store_opening( + profile_id, + conversation_id, + impulse, + { + "opened": True, + "kind": "model", + "calls": calls, + "opening_context": facts, + "decision": decision, + "trace": result.get("trace") if result else None, + }, + ) + + +def maybe_open_journal_conversation(profile_id: str, conversation: dict) -> dict: + payload = dict(conversation) + try: + opening = start_journal_opening(profile_id, conversation["id"]) + except EngineError as exc: + payload["opening"] = { + "opened": False, + "reason": exc.code, + "message": exc.message, + "kind": "failed_closed", + } + payload["messages"] = list_messages(profile_id, conversation["id"]) + return payload + payload["opening"] = opening + payload["messages"] = opening.get("messages") or [] + return payload diff --git a/backend/journal_reconstruct.py b/backend/journal_reconstruct.py new file mode 100644 index 0000000..96f2e69 --- /dev/null +++ b/backend/journal_reconstruct.py @@ -0,0 +1,529 @@ +"""Journal adapter over the intent-neutral provenance layer. No egress. + +Journal policy: selected user sources, full canonical rehydration, assistant never +a content source, source-local clocks, reorder only with confirmed times. +Semantic labels from the model are not verified content. +""" +from __future__ import annotations + +import json +import re +from typing import Any + +from prompt_budget import ERROR_NO_USER_SOURCES, ERROR_RECONSTRUCTION_INVALID, JournalBudgetError +from provenance import ( + COVERAGE_ALL_SELECTED_SOURCES, + EvidenceRef, + ProvenanceError, + SourceRecord, + SourceRegistry, + VerificationPolicy, + texts_from_payload, + verify, +) + +JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL) +TIME_RE = re.compile(r"\b\d{1,2}:\d{2}\b|\b\d{1,2}\s*Uhr\b", re.I) +CLOCK_STAMP_RE = re.compile(r"^(?:\d{1,2}:\d{2}(?:\s*Uhr)?|\d{1,2}\s*Uhr)$", re.I) +DIALOGUE_LINE = re.compile( + r"^(?:\[(?Pu\d+|a\d+)\]\s*)?(?Puser|assistant):\s*(?P.*)$" +) +USER_SOURCE_ID = re.compile(r"^u\d+$") +ASSISTANT_SOURCE_ID = re.compile(r"^a\d+$") +REQUIRED_TOP = ("source_order", "chronology") +LEGACY_FACT_KEYS = ( + "events", + "people", + "places", + "perceptions", + "feelings_stated", + "evaluations", + "quotes", + "uncertainties", + "plan_changes", + "corrections", + "evidence", + "text", +) +SOURCED_LIST_KEYS = ("contradictions", "uncertainties", "plan_changes", "corrections") +ALLOWED_CLAIM_KEYS = frozenset({"kind", "evidence"}) +ALLOWED_ITEM_KEYS = frozenset({"source_id", "source", "time", "claims"}) +ALLOWED_SOURCED_KEYS = frozenset({"source_id", "evidence"}) +CONTENT_ROLE = "user" +JOURNAL_ALLOWED_ROLES = frozenset({CONTENT_ROLE}) + + +def _fail(reason: str, **extra: Any) -> None: + raise JournalBudgetError( + ERROR_RECONSTRUCTION_INVALID, + diagnostics={"reason": reason, **extra}, + ) + + +def parse_dialogue_line(line: str) -> dict[str, str] | None: + match = DIALOGUE_LINE.match((line or "").rstrip("\n")) + if not match: + return None + return { + "source_id": match.group("source_id") or "", + "role": match.group("role"), + "body": match.group("body") or "", + } + + +def is_dialogue_role_line(line: str) -> bool: + return parse_dialogue_line(line) is not None + + +def assign_source_ids(messages: list[dict]) -> list[dict]: + """Local, stable IDs in encounter order. User u1…; assistant a1….""" + labeled: list[dict] = [] + user_n = 0 + assistant_n = 0 + for message in messages or []: + item = dict(message) + role = item.get("role") or "user" + body = (item.get("body") or "").strip() + if role == "user" and body: + user_n += 1 + item["source_id"] = f"u{user_n}" + elif role == "assistant" and body: + assistant_n += 1 + item["source_id"] = f"a{assistant_n}" + else: + item["source_id"] = None + labeled.append(item) + return labeled + + +def expected_user_source_ids(messages: list[dict]) -> list[str]: + return [ + item["source_id"] + for item in assign_source_ids(messages) + if item.get("role") == "user" and item.get("source_id") + ] + + +def parse_reconstruction(raw: str) -> dict[str, Any]: + text = (raw or "").strip() + if text.startswith("```"): + text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL) + match = JSON_BLOCK.search(text) + if not match: + _fail("not_json") + try: + data = json.loads(match.group(0)) + except json.JSONDecodeError as exc: + raise JournalBudgetError( + ERROR_RECONSTRUCTION_INVALID, + diagnostics={"reason": "json_parse"}, + ) from exc + if not isinstance(data, dict): + _fail("not_object") + return data + + +def parse_clock(stamp: str) -> tuple[int, int] | None: + text = (stamp or "").strip() + match = re.search(r"(\d{1,2}):(\d{2})", text) + if match: + hour, minute = int(match.group(1)), int(match.group(2)) + if 0 <= hour <= 23 and 0 <= minute <= 59: + return hour, minute + return None + match = re.search(r"(\d{1,2})\s*Uhr", text, re.I) + if match: + hour = int(match.group(1)) + if 0 <= hour <= 23: + return hour, 0 + return None + + +def clocks_in_text(text: str) -> set[tuple[int, int]]: + found: set[tuple[int, int]] = set() + for match in TIME_RE.finditer(text or ""): + parsed = parse_clock(match.group(0)) + if parsed: + found.add(parsed) + return found + + +def clock_in_source(stamp: str, source_body: str) -> bool: + parsed = parse_clock(stamp) + if parsed is None: + return False + return parsed in clocks_in_text(source_body) + + +def _as_list(value: Any) -> list: + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + +def _legacy_payload(raw: dict[str, Any]) -> bool: + for key in LEGACY_FACT_KEYS: + values = [str(part).strip() for part in _as_list(raw.get(key)) if str(part).strip()] + if values: + return True + return False + + +def _normalize_claim(raw: Any, *, index: int, claim_index: int) -> dict[str, str]: + if not isinstance(raw, dict): + _fail("claim_shape", index=index, claim_index=claim_index) + extra = {key: raw.get(key) for key in raw.keys() if key not in ALLOWED_CLAIM_KEYS} + if any(value not in (None, "", [], {}) for value in extra.values()): + _fail("unverified_claim", index=index, claim_index=claim_index) + evidence = str(raw.get("evidence") or "").strip() + if not evidence: + _fail("evidence_missing", index=index, claim_index=claim_index) + kind = raw.get("kind") + return {"kind": "" if kind is None else str(kind), "evidence": evidence} + + +def _normalize_sourced_item(raw: Any, *, key: str, index: int) -> dict[str, str]: + if isinstance(raw, str) and raw.strip(): + _fail("unverified_claim", key=key, index=index) + if not isinstance(raw, dict): + _fail("sourced_item_shape", key=key, index=index) + extra = {item: raw.get(item) for item in raw.keys() if item not in ALLOWED_SOURCED_KEYS} + if any(value not in (None, "", [], {}) for value in extra.values()): + _fail("unverified_claim", key=key, index=index) + source_id = str(raw.get("source_id") or "").strip() + evidence = str(raw.get("evidence") or "").strip() + if not source_id: + _fail("missing_source_id", key=key, index=index) + if not evidence: + _fail("evidence_missing", key=key, index=index) + return {"source_id": source_id, "evidence": evidence} + + +def _confirmed_minutes(stamp: str | None, cited: str) -> int | None: + if not stamp: + return None + parsed = parse_clock(stamp) + if parsed is None or parsed not in clocks_in_text(cited): + return None + return parsed[0] * 60 + parsed[1] + + +def _reorder_allowed( + items: list[dict[str, Any]], + expected_ids: list[str], + user_by_id: dict[str, str], +) -> bool: + chrono_ids = [item["source_id"] for item in items] + if chrono_ids == expected_ids: + return True + minutes: list[int] = [] + for item in items: + value = _confirmed_minutes(item.get("time"), user_by_id.get(item["source_id"]) or "") + if value is None: + return False + minutes.append(value) + for index in range(len(items) - 1): + if minutes[index] > minutes[index + 1]: + return False + if minutes[index] == minutes[index + 1]: + left = expected_ids.index(items[index]["source_id"]) + right = expected_ids.index(items[index + 1]["source_id"]) + if left > right: + return False + return True + + +def _registry_from_messages(labeled: list[dict]) -> SourceRegistry: + registry = SourceRegistry() + for item in labeled: + source_id = item.get("source_id") + role = item.get("role") or "" + body = item.get("body") or "" + if not source_id or not body.strip(): + continue + registry.add( + SourceRecord( + source_id=source_id, + role=role, + text=body, + metadata={"conversation_id": item.get("conversation_id")}, + ) + ) + return registry + + +def _map_provenance(exc: ProvenanceError) -> None: + reason = exc.reason + details = dict(exc.details or {}) + if reason == "unknown_source": + _fail("unknown_source_id", **details) + if reason == "role_not_allowed": + role = details.get("role") + source_id = str(details.get("source_id") or "") + if role != CONTENT_ROLE or ASSISTANT_SOURCE_ID.match(source_id): + _fail("assistant_as_fact", **details) + _fail("invalid_source", **details) + if reason in {"unverified_excerpt", "canonical_overwrite"}: + _fail("unverified_claim", **details) + if reason == "wrong_source": + _fail("wrong_source", **details) + if reason == "duplicate_source_id": + _fail("duplicate_source_id", **details) + if reason == "missing_source_id": + _fail("missing_source_id", **details) + _fail(reason, **details) + + +def _journal_policy(expected_ids: list[str], chronology_ids: list[str]) -> VerificationPolicy: + return VerificationPolicy( + allowed_roles=JOURNAL_ALLOWED_ROLES, + coverage=COVERAGE_ALL_SELECTED_SOURCES, + selected_ids=tuple(chronology_ids), + source_order=tuple(expected_ids), + include_unverified_annotations=False, + ) + + +def validate_reconstruction( + data: dict[str, Any], + *, + messages: list[dict] | None = None, + user_bodies: list[str] | None = None, + assistant_bodies: list[str] | None = None, +) -> dict[str, Any]: + for key in REQUIRED_TOP: + if key not in data: + _fail("missing_key", key=key) + chronology = data.get("chronology") + source_order_raw = data.get("source_order") + if not isinstance(chronology, list): + _fail("chronology_missing") + if not isinstance(source_order_raw, list): + _fail("source_order_shape") + + if messages is not None: + labeled = assign_source_ids(messages) + else: + labeled = [] + for index, body in enumerate(user_bodies or [], start=1): + if (body or "").strip(): + labeled.append({"role": "user", "body": body, "source_id": f"u{index}"}) + for index, body in enumerate(assistant_bodies or [], start=1): + if (body or "").strip(): + labeled.append({"role": "assistant", "body": body, "source_id": f"a{index}"}) + + expected_ids = [ + item["source_id"] for item in labeled if item.get("role") == CONTENT_ROLE and item.get("source_id") + ] + user_by_id = { + item["source_id"]: item.get("body") or "" + for item in labeled + if item.get("role") == CONTENT_ROLE and item.get("source_id") + } + source_order = [str(part).strip() for part in source_order_raw] + if source_order != expected_ids: + _fail("source_order_mismatch", expected=expected_ids, got=source_order) + + items: list[dict[str, Any]] = [] + evidence_refs: list[EvidenceRef] = [] + seen: list[str] = [] + for index, raw in enumerate(chronology): + if not isinstance(raw, dict): + _fail("chronology_item_shape", index=index) + if _legacy_payload(raw): + _fail("unverified_claim", index=index) + extra_keys = [ + key for key in raw.keys() if key not in ALLOWED_ITEM_KEYS and raw.get(key) not in (None, "", [], {}) + ] + if extra_keys: + _fail("unverified_claim", index=index, keys=extra_keys) + if "source" not in raw or raw.get("source") is None: + source = None + elif not isinstance(raw.get("source"), str): + _fail("invalid_source", source=raw.get("source"), index=index) + elif raw.get("source") == "": + source = None + else: + source = raw.get("source") + claims_raw = raw.get("claims") + if claims_raw is None: + claims_raw = [] + if not isinstance(claims_raw, list): + _fail("claims_shape", index=index) + source_id = str(raw.get("source_id") or "").strip() + if not source_id: + _fail("missing_source_id", index=index) + if ASSISTANT_SOURCE_ID.match(source_id): + _fail("assistant_as_fact", source_id=source_id, index=index) + if source_id not in user_by_id: + _fail("unknown_source_id", source_id=source_id, index=index) + if source_id in seen: + _fail("duplicate_source_id", source_id=source_id, index=index) + seen.append(source_id) + if source is None: + _fail("missing_source", source_id=source_id, index=index) + if source != CONTENT_ROLE: + _fail("invalid_source", source=source, source_id=source_id, index=index) + claims = [ + _normalize_claim(claim, index=index, claim_index=claim_index) + for claim_index, claim in enumerate(claims_raw) + ] + if not claims: + _fail("empty_event", source_id=source_id, index=index) + stamp = raw.get("time") + if isinstance(stamp, str): + stamp = stamp.strip() or None + elif stamp is None: + stamp = None + else: + stamp = str(stamp).strip() or None + cited = user_by_id[source_id] + if stamp: + if ( + not CLOCK_STAMP_RE.match(stamp) + or parse_clock(stamp) is None + or not clock_in_source(stamp, cited) + ): + _fail("time_source_mismatch", source_id=source_id, time=stamp, index=index) + items.append({"source_id": source_id, "time": stamp, "claims": claims}) + for claim in claims: + evidence_refs.append(EvidenceRef(source_id, claim["evidence"])) + + missing = [sid for sid in expected_ids if sid not in seen] + if missing: + _fail("incomplete_coverage", missing=missing) + if expected_ids and not items: + _fail("empty_chronology") + + if not _reorder_allowed(items, expected_ids, user_by_id): + _fail( + "reorder_unjustified", + expected=expected_ids, + chronology=[item["source_id"] for item in items], + ) + + for key in SOURCED_LIST_KEYS: + raw_list = data.get(key) + if raw_list is None: + continue + if not isinstance(raw_list, list): + _fail("sourced_list_shape", key=key) + for index, raw in enumerate(raw_list): + row = _normalize_sourced_item(raw, key=key, index=index) + if ASSISTANT_SOURCE_ID.match(row["source_id"]): + _fail("assistant_as_fact", source_id=row["source_id"], key=key) + evidence_refs.append(EvidenceRef(row["source_id"], row["evidence"])) + + registry = _registry_from_messages(labeled) + policy = _journal_policy(expected_ids, [item["source_id"] for item in items]) + try: + artifact = verify(registry, policy=policy, evidence=evidence_refs) + except ProvenanceError as exc: + _map_provenance(exc) + return artifact.to_payload() + + +def local_verified_artifact(messages: list[dict]) -> dict[str, Any]: + """Authoritative coverage from local user sources. No model JSON required.""" + labeled = assign_source_ids(messages) + users = [ + item + for item in labeled + if item.get("role") == CONTENT_ROLE and item.get("source_id") and (item.get("body") or "").strip() + ] + if not users: + raise JournalBudgetError( + ERROR_NO_USER_SOURCES, + diagnostics={"reason": "no_user_sources"}, + ) + expected_ids = [item["source_id"] for item in users] + registry = _registry_from_messages(labeled) + evidence = [EvidenceRef(item["source_id"], (item.get("body") or "").strip()) for item in users] + policy = _journal_policy(expected_ids, expected_ids) + try: + artifact = verify(registry, policy=policy, evidence=evidence) + except ProvenanceError as exc: + _map_provenance(exc) + return artifact.to_payload() + + +def reconstruction_from_model(raw_content: str, messages: list[dict]) -> tuple[dict[str, Any], dict[str, Any]]: + """Stage 1 model is untrusted. Invalid output never drops local user sources.""" + try: + parsed = parse_reconstruction(raw_content) + artifact = validate_reconstruction(parsed, messages=messages) + return artifact, {"stage1": "model_accepted"} + except JournalBudgetError as exc: + if exc.code == ERROR_NO_USER_SOURCES: + raise + artifact = local_verified_artifact(messages) + return artifact, { + "stage1": "local_fallback", + "model_rejected": exc.code, + "reason": (exc.diagnostics or {}).get("reason") or exc.code, + } + + +def reconstruction_text(data: dict[str, Any]) -> str: + return json.dumps(data, ensure_ascii=False, indent=2) + + +def _claim_for_body(body: str) -> list[dict[str, str]]: + claims = [{"kind": "event", "evidence": body}] + if TIME_RE.search(body): + stamp = TIME_RE.search(body).group(0) + claims.append({"kind": "time", "evidence": stamp}) + return claims + + +def fake_reconstruction(rendered: str) -> str: + """Deterministic stage-1 stand-in. Claims remain untrusted; the adapter rehydrates.""" + chronology = [] + source_order: list[str] = [] + user_n = 0 + for line in (rendered or "").splitlines(): + parsed = parse_dialogue_line(line) + if parsed is None and line.startswith("user:"): + parsed = {"role": "user", "body": line[5:].strip(), "source_id": ""} + if not parsed or parsed["role"] != "user": + continue + body = (parsed["body"] or "").strip() + if not body: + continue + user_n += 1 + source_id = parsed["source_id"] if USER_SOURCE_ID.match(parsed["source_id"] or "") else f"u{user_n}" + source_order.append(source_id) + times = TIME_RE.findall(body) + chronology.append( + { + "source_id": source_id, + "source": "user", + "time": times[0] if times else None, + "claims": _claim_for_body(body), + } + ) + payload = { + "source_order": source_order, + "chronology": chronology, + "contradictions": [], + "uncertainties": [], + "plan_changes": [], + "corrections": [], + } + return json.dumps(payload, ensure_ascii=False) + + +def claim_texts(data: dict[str, Any]) -> list[str]: + parts = texts_from_payload(data) + if parts: + return parts + collected: list[str] = [] + for item in data.get("chronology") or []: + if not isinstance(item, dict): + continue + for claim in item.get("claims") or []: + if isinstance(claim, dict) and claim.get("evidence"): + collected.append(str(claim["evidence"])) + return collected diff --git a/backend/journal_store.py b/backend/journal_store.py index 0b9d7dd..e816b88 100644 --- a/backend/journal_store.py +++ b/backend/journal_store.py @@ -5,6 +5,7 @@ import json import re import uuid from datetime import datetime, timezone +from pathlib import Path from dialogue_store import ( StoreError, @@ -270,7 +271,7 @@ def current_entries(profile_id: str, journal_day_id: str) -> list[dict]: FROM journal_entries e LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id WHERE e.profile_id = ? AND e.journal_day_id = ? AND e.deleted_at IS NULL - ORDER BY e.created + ORDER BY e.created, e.rowid """, (profile_id, journal_day_id), ).fetchall() @@ -472,3 +473,304 @@ def day_payload(profile_id: str, journal_day_id: str) -> dict: "scratch": _parse_scratch(day.get("scratch_json")), "consolidation_offer": consolidation_offer(conversations), } + + +def _entry_list_row(conn, row) -> dict: + item = row_to_dict(row) + item["title"] = clean_title(item.get("title") or "") + item["version_count"] = int(item.get("version_count") or 0) + return item + + +def list_space_entries(profile_id: str, space_id: str) -> list[dict]: + """Chronological journal entries of a space. Deleted entries are excluded.""" + get_user_space(profile_id, space_id) + with get_db() as conn: + rows = conn.execute( + """ + SELECT e.id, e.profile_id, e.space_id, e.journal_day_id, e.current_version_id, + e.created, e.updated, e.deleted_at, + d.calendar_date, v.title, v.origin, + (SELECT COUNT(*) FROM journal_entry_versions ver + WHERE ver.entry_id = e.id AND ver.profile_id = e.profile_id) AS version_count, + (SELECT r.source_id FROM journal_entry_version_source_refs r + WHERE r.version_id = e.current_version_id AND r.source_kind = 'conversation' + ORDER BY r.sort_order LIMIT 1) AS source_conversation_id + FROM journal_entries e + JOIN journal_days d ON d.id = e.journal_day_id + LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id + WHERE e.profile_id = ? AND e.space_id = ? AND e.deleted_at IS NULL + ORDER BY d.calendar_date DESC, e.created ASC, e.rowid ASC + """, + (profile_id, space_id), + ).fetchall() + return [_entry_list_row(conn, row) for row in rows] + + +def list_deleted_entries(profile_id: str, space_id: str | None = None) -> list[dict]: + if space_id: + get_user_space(profile_id, space_id) + params: list = [profile_id] + space_sql = "" + if space_id: + space_sql = "AND e.space_id = ?" + params.append(space_id) + with get_db() as conn: + rows = conn.execute( + f""" + SELECT e.id, e.profile_id, e.space_id, e.journal_day_id, e.current_version_id, + e.created, e.updated, e.deleted_at, + d.calendar_date, v.title, v.origin, + (SELECT COUNT(*) FROM journal_entry_versions ver + WHERE ver.entry_id = e.id AND ver.profile_id = e.profile_id) AS version_count, + (SELECT r.source_id FROM journal_entry_version_source_refs r + WHERE r.version_id = e.current_version_id AND r.source_kind = 'conversation' + ORDER BY r.sort_order LIMIT 1) AS source_conversation_id + FROM journal_entries e + JOIN journal_days d ON d.id = e.journal_day_id + LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id + WHERE e.profile_id = ? {space_sql} AND e.deleted_at IS NOT NULL + ORDER BY e.deleted_at DESC, e.id ASC + """, + params, + ).fetchall() + return [_entry_list_row(conn, row) for row in rows] + + +def undelete_entry(profile_id: str, entry_id: str) -> dict: + """Return a soft-deleted entry to the space list. Not a version restore.""" + entry = get_entry(profile_id, entry_id, include_deleted=True) + if not entry.get("deleted_at"): + raise StoreError("not_deleted", "Eintrag ist nicht im Papierkorb.") + with get_db() as conn: + conn.execute( + """ + UPDATE journal_entries + SET deleted_at = NULL, updated = datetime('now') + WHERE id = ? AND profile_id = ? + """, + (entry_id, profile_id), + ) + return get_entry(profile_id, entry_id) + + +def continuable_journal_day(profile_id: str) -> dict | None: + """Most recently active journal day that already has a stored message.""" + with get_db() as conn: + row = row_to_dict( + conn.execute( + """ + SELECT d.id AS journal_day_id, d.space_id, d.calendar_date, + s.title AS space_title, c.id AS conversation_id, m.created AS last_message_at + FROM messages m + JOIN conversations c ON c.id = m.conversation_id AND c.profile_id = m.profile_id + JOIN journal_days d ON d.id = c.journal_day_id AND d.profile_id = m.profile_id + JOIN spaces s ON s.id = d.space_id AND s.profile_id = m.profile_id + WHERE m.profile_id = ? AND s.visibility = 'user' + ORDER BY m.created DESC, m.seq DESC, m.id DESC + LIMIT 1 + """, + (profile_id,), + ).fetchone() + ) + if not row: + return None + return { + "space_id": row["space_id"], + "space_title": row.get("space_title") or "", + "journal_day_id": row["journal_day_id"], + "calendar_date": row["calendar_date"], + "conversation_id": row["conversation_id"], + "last_message_at": row.get("last_message_at"), + "reason": "recent_dialogue", + } + + +def start_payload(profile_id: str) -> dict: + spaces = list_user_spaces(profile_id) + return { + "continuable": continuable_journal_day(profile_id), + "has_spaces": bool(spaces), + "space_count": len(spaces), + "spaces": [{"id": item["id"], "title": item.get("title") or ""} for item in spaces], + } + + +def _bodies_referencing_media(conn, profile_id: str, media_id: str, exclude_entry_id: str) -> list[str]: + from journal_body import media_ids + + rows = conn.execute( + """ + SELECT v.entry_id, v.body + FROM journal_entry_versions v + WHERE v.profile_id = ? AND v.entry_id != ? + """, + (profile_id, exclude_entry_id), + ).fetchall() + owners: list[str] = [] + seen: set[str] = set() + for row in rows: + if media_id in media_ids(row["body"] or "") and row["entry_id"] not in seen: + seen.add(row["entry_id"]) + owners.append(row["entry_id"]) + return owners + + +def inventory_entry_for_purge(profile_id: str, entry_id: str) -> dict: + entry = get_entry(profile_id, entry_id, include_deleted=True) + with get_db() as conn: + versions = [ + row_to_dict(row) + for row in conn.execute( + "SELECT id, origin, created FROM journal_entry_versions WHERE profile_id = ? AND entry_id = ? ORDER BY created", + (profile_id, entry_id), + ).fetchall() + ] + version_ids = [item["id"] for item in versions] + source_refs = 0 + if version_ids: + placeholders = ",".join("?" * len(version_ids)) + source_refs = conn.execute( + f""" + SELECT COUNT(*) AS n FROM journal_entry_version_source_refs + WHERE version_id IN ({placeholders}) AND profile_id = ? + """, + (*version_ids, profile_id), + ).fetchone()["n"] + writing_sources = [ + row_to_dict(row) + for row in conn.execute( + "SELECT id, kind FROM writing_profile_sources WHERE profile_id = ? AND entry_id = ?", + (profile_id, entry_id), + ).fetchall() + ] + evidence = [ + row_to_dict(row) + for row in conn.execute( + "SELECT id FROM writing_profile_evidence WHERE profile_id = ? AND source_id = ?", + (profile_id, entry_id), + ).fetchall() + ] + media_rows = [ + row_to_dict(row) + for row in conn.execute( + "SELECT * FROM media_assets WHERE profile_id = ? AND entry_id = ?", + (profile_id, entry_id), + ).fetchall() + ] + media = [] + from media_store import media_root + + root = media_root() + for row in media_rows: + owners = _bodies_referencing_media(conn, profile_id, row["id"], entry_id) + path = root / row["rel_path"] + media.append( + { + "id": row["id"], + "rel_path": row["rel_path"], + "path_exists": path.is_file(), + "referenced_by_entries": owners, + "keep_file": bool(owners), + "reassign_to": owners[0] if owners else None, + } + ) + return { + "entry": {"id": entry["id"], "space_id": entry["space_id"], "deleted_at": entry.get("deleted_at")}, + "versions": versions, + "source_ref_count": int(source_refs or 0), + "writing_profile_sources": writing_sources, + "writing_profile_evidence": evidence, + "media": media, + "conversations_untouched": True, + } + + +def purge_entry(profile_id: str, entry_id: str, *, confirm: bool) -> dict: + """Irreversible delete after explicit confirm. Source dialogues stay.""" + if not confirm: + raise StoreError("confirm_required", "Endgültiges Löschen braucht eine ausdrückliche Bestätigung.") + entry = get_entry(profile_id, entry_id, include_deleted=True) + if not entry.get("deleted_at"): + raise StoreError("not_in_trash", "Endgültiges Löschen nur aus dem Papierkorb.") + inventory = inventory_entry_for_purge(profile_id, entry_id) + from media_store import media_root + + root = media_root() + files_to_delete: list[Path] = [] + try: + with get_db() as conn: + for item in inventory["media"]: + if item.get("keep_file") and item.get("reassign_to"): + other = row_to_dict( + conn.execute( + "SELECT id FROM journal_entries WHERE id = ? AND profile_id = ?", + (item["reassign_to"], profile_id), + ).fetchone() + ) + if not other: + raise StoreError("purge_conflict", "Geteiltes Medium hat keinen sicheren neuen Träger.") + conn.execute( + "UPDATE media_assets SET entry_id = ? WHERE id = ? AND profile_id = ?", + (item["reassign_to"], item["id"], profile_id), + ) + elif not item.get("keep_file"): + files_to_delete.append(root / item["rel_path"]) + conn.execute( + "DELETE FROM writing_profile_sources WHERE profile_id = ? AND entry_id = ?", + (profile_id, entry_id), + ) + conn.execute( + "DELETE FROM writing_profile_evidence WHERE profile_id = ? AND source_id = ?", + (profile_id, entry_id), + ) + conn.execute( + "DELETE FROM journal_entries WHERE id = ? AND profile_id = ?", + (entry_id, profile_id), + ) + leftover = conn.execute( + "SELECT id FROM journal_entries WHERE id = ? AND profile_id = ?", + (entry_id, profile_id), + ).fetchone() + if leftover: + raise StoreError("purge_failed", "Eintrag konnte nicht vollständig gelöscht werden.", 500) + leftover_media = conn.execute( + "SELECT id FROM media_assets WHERE profile_id = ? AND entry_id = ?", + (profile_id, entry_id), + ).fetchone() + if leftover_media: + raise StoreError("purge_failed", "Medienverweise konnten nicht aufgelöst werden.", 500) + leftover_sources = conn.execute( + "SELECT id FROM writing_profile_sources WHERE profile_id = ? AND entry_id = ?", + (profile_id, entry_id), + ).fetchone() + if leftover_sources: + raise StoreError("purge_failed", "Writing-Profile-Quellen konnten nicht gelöscht werden.", 500) + except StoreError: + raise + except Exception as exc: + raise StoreError("purge_failed", "Endgültiges Löschen wurde abgebrochen, nichts wurde entfernt.", 500) from exc + + removed_files: list[str] = [] + missing_files: list[str] = [] + for path in files_to_delete: + if not path.is_file(): + missing_files.append(str(path)) + continue + try: + path.unlink() + removed_files.append(str(path)) + except OSError as exc: + raise StoreError( + "purge_failed", + "Datenbank entfernt, Mediendatei konnte nicht gelöscht werden. Vorgang fail-closed gemeldet.", + 500, + ) from exc + return { + "id": entry_id, + "purged": True, + "inventory": inventory, + "removed_files": len(removed_files), + "missing_files": missing_files, + "conversations_untouched": True, + } diff --git a/backend/local_backup.py b/backend/local_backup.py new file mode 100644 index 0000000..25ee0c0 --- /dev/null +++ b/backend/local_backup.py @@ -0,0 +1,395 @@ +"""Consistent local backup and restore for the holiday test phase. + +Uses the SQLite backup API (not a live-file copy). Includes journal media. +Never packs .env, provider keys, or logs. The archive can contain personal +unencrypted journal data. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import sqlite3 +import sys +import tempfile +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +FORMAT_KIND = "kansho.local_backup" +FORMAT_VERSION = 1 +DB_NAME = "kansho.sqlite" +MANIFEST_NAME = "manifest.json" +README_NAME = "README.txt" +MEDIA_PREFIX = "media/" +DEFAULT_DIR_NAME = "local-backups" +PERSONAL_DATA_WARNING = ( + "Dieses Archiv kann persönliche, unverschlüsselte Journalinhalte und Medien enthalten. " + "Es enthält keine .env-Dateien, Provider-Keys oder Logs, ist aber selbst nicht verschlüsselt." +) + +EXCLUDE_NAMES = {".env", ".env.example"} +EXCLUDE_SUFFIXES = {".log", ".pem", ".key"} +EXCLUDE_DIR_NAMES = {"logs", "__pycache__"} + + +class BackupError(Exception): + def __init__(self, code: str, message: str, status_code: int = 400, details: dict | None = None): + super().__init__(message) + self.code = code + self.message = message + self.status_code = status_code + self.details = details or {} + + +def _now_stamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + + +def _now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[1] + + +def default_backup_dir() -> Path: + env = (os.environ.get("KANSHO_BACKUP_DIR") or "").strip() + if env: + return Path(env) + return _repo_root() / DEFAULT_DIR_NAME + + +def default_db_path() -> Path: + env = (os.environ.get("KANSHO_DB_PATH") or "").strip() + if env: + return Path(env) + data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip() + if data_dir: + return Path(data_dir) / "kansho.sqlite" + return Path(__file__).resolve().parent / "data" / "kansho.sqlite" + + +def default_media_root() -> Path: + env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip() + if env: + return Path(env) + data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip() + if data_dir: + return Path(data_dir) / "media" + return Path(__file__).resolve().parent / "data" / "media" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def _is_excluded(path: Path) -> bool: + if path.name in EXCLUDE_NAMES: + return True + if path.suffix.lower() in EXCLUDE_SUFFIXES: + return True + return any(part in EXCLUDE_DIR_NAMES for part in path.parts) + + +def list_media_files(media_root: Path) -> list[Path]: + if not media_root.exists(): + return [] + files: list[Path] = [] + for path in sorted(media_root.rglob("*")): + if not path.is_file() or _is_excluded(path): + continue + files.append(path) + return files + + +def sqlite_snapshot(src: Path, dest: Path) -> None: + if not src.exists(): + raise BackupError("db_missing", f"Keine Datenbank unter {src}") + dest.parent.mkdir(parents=True, exist_ok=True) + source = sqlite3.connect(str(src), timeout=5) + try: + target = sqlite3.connect(str(dest)) + try: + source.backup(target) + finally: + target.close() + except sqlite3.OperationalError as exc: + raise BackupError("db_unavailable", f"SQLite-Snapshot fehlgeschlagen: {exc}") from exc + finally: + source.close() + + +def assert_db_idle(path: Path) -> None: + if not path.exists(): + return + conn = sqlite3.connect(str(path), timeout=1) + try: + conn.execute("BEGIN EXCLUSIVE") + conn.rollback() + except sqlite3.OperationalError as exc: + raise BackupError( + "db_in_use", + "Die Datenbank ist geöffnet. Backend beenden und Restore erneut ausführen.", + ) from exc + finally: + conn.close() + + +def build_manifest(created: str, db_file: Path, media_files: list[tuple[str, Path]]) -> dict: + checksums = {DB_NAME: sha256_file(db_file)} + media_entries = [] + for rel, path in media_files: + checksums[f"{MEDIA_PREFIX}{rel}"] = sha256_file(path) + media_entries.append({"path": rel, "bytes": path.stat().st_size}) + return { + "kind": FORMAT_KIND, + "format_version": FORMAT_VERSION, + "created": created, + "warning": PERSONAL_DATA_WARNING, + "contains": ["sqlite", "media"], + "excludes": [".env", "provider keys", "logs"], + "parts": { + "sqlite": {"path": DB_NAME, "bytes": db_file.stat().st_size}, + "media": media_entries, + }, + "checksums": checksums, + } + + +def create_backup( + dest: Path | None = None, + *, + db_path: Path | None = None, + media_root: Path | None = None, +) -> dict: + db_path = Path(db_path or default_db_path()) + media_root = Path(media_root or default_media_root()) + if dest is None: + dest = default_backup_dir() / f"kansho-{_now_stamp()}.zip" + dest = Path(dest) + dest.parent.mkdir(parents=True, exist_ok=True) + created = _now_iso() + with tempfile.TemporaryDirectory(prefix="kansho-backup-") as raw: + tmp = Path(raw) + snapshot = tmp / DB_NAME + sqlite_snapshot(db_path, snapshot) + media_pairs: list[tuple[str, Path]] = [] + for path in list_media_files(media_root): + rel = path.relative_to(media_root).as_posix() + media_pairs.append((rel, path)) + manifest = build_manifest(created, snapshot, media_pairs) + staging = tmp / "archive" + staging.mkdir() + shutil.copy2(snapshot, staging / DB_NAME) + (staging / README_NAME).write_text(PERSONAL_DATA_WARNING + "\n", encoding="utf-8") + (staging / MANIFEST_NAME).write_text( + json.dumps(manifest, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + media_dir = staging / "media" + for rel, path in media_pairs: + target = media_dir / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + if dest.exists(): + raise BackupError("backup_exists", f"Zieldatei existiert bereits: {dest}") + with zipfile.ZipFile(dest, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for path in staging.rglob("*"): + if path.is_file(): + zf.write(path, path.relative_to(staging).as_posix()) + return {"path": str(dest), "manifest": manifest} + + +def _read_zip_text(zf: zipfile.ZipFile, name: str) -> str: + try: + return zf.read(name).decode("utf-8") + except KeyError as exc: + raise BackupError("archive_invalid", f"Archivteil fehlt: {name}") from exc + + +def verify_archive(archive: Path) -> dict: + archive = Path(archive) + if not archive.is_file(): + raise BackupError("archive_missing", f"Archiv nicht gefunden: {archive}") + with zipfile.ZipFile(archive) as zf: + names = set(zf.namelist()) + if MANIFEST_NAME not in names: + raise BackupError("archive_invalid", "Manifest fehlt.") + try: + manifest = json.loads(_read_zip_text(zf, MANIFEST_NAME)) + except json.JSONDecodeError as exc: + raise BackupError("archive_invalid", "Manifest ist kein JSON.") from exc + if manifest.get("kind") != FORMAT_KIND: + raise BackupError("archive_invalid", "Unbekanntes Backup-Format.") + if int(manifest.get("format_version") or 0) != FORMAT_VERSION: + raise BackupError("archive_unsupported", "Backup-Formatversion wird nicht unterstützt.") + checksums = manifest.get("checksums") or {} + if not isinstance(checksums, dict) or DB_NAME not in checksums: + raise BackupError("archive_invalid", "Checksummen fehlen.") + extra = sorted( + name + for name in names + if not name.endswith("/") + and name not in {MANIFEST_NAME, README_NAME} + and name not in checksums + ) + missing = sorted(key for key in checksums if key not in names) + if missing: + raise BackupError( + "checksum_mismatch", + "Im Archiv fehlen deklarierte Dateien.", + details={"missing": missing}, + ) + with tempfile.TemporaryDirectory(prefix="kansho-verify-") as raw: + tmp = Path(raw) + for name, expected in checksums.items(): + target = tmp / name + target.parent.mkdir(parents=True, exist_ok=True) + with zf.open(name) as src, target.open("wb") as dest: + shutil.copyfileobj(src, dest) + actual = sha256_file(target) + if actual != expected: + raise BackupError( + "checksum_mismatch", + f"Checksumme stimmt nicht: {name}", + details={"path": name}, + ) + return { + "manifest": manifest, + "skipped_extra": extra, + "names": sorted(names), + } + + +def restore_backup( + archive: Path, + *, + db_path: Path | None = None, + media_root: Path | None = None, + confirm: bool = False, + allow_overwrite: bool = False, + safety_dir: Path | None = None, +) -> dict: + if not confirm: + raise BackupError( + "confirm_required", + "Restore überschreibt lokale Daten nur mit ausdrücklicher Bestätigung (--confirm).", + ) + db_path = Path(db_path or default_db_path()) + media_root = Path(media_root or default_media_root()) + verified = verify_archive(archive) + current_exists = db_path.exists() or (media_root.exists() and any(media_root.rglob("*"))) + if current_exists and not allow_overwrite: + raise BackupError( + "would_overwrite", + "Ein aktueller Datenbestand existiert. Restore nicht still. " + "Mit Bestätigung und --replace nach Sicherheitsbackup fortsetzen.", + ) + assert_db_idle(db_path) + safety_path = None + if current_exists: + safety_dir = Path(safety_dir or default_backup_dir()) + safety_path = safety_dir / f"pre-restore-{_now_stamp()}.zip" + create_backup(safety_path, db_path=db_path, media_root=media_root) + with tempfile.TemporaryDirectory(prefix="kansho-restore-") as raw: + extracted = Path(raw) / "extracted" + extracted.mkdir() + with zipfile.ZipFile(archive) as zf: + checksums = verified["manifest"]["checksums"] + for name in checksums: + target = extracted / name + target.parent.mkdir(parents=True, exist_ok=True) + with zf.open(name) as src, target.open("wb") as dest: + shutil.copyfileobj(src, dest) + if sha256_file(target) != checksums[name]: + raise BackupError("checksum_mismatch", f"Checksumme nach Entpacken ungültig: {name}") + new_db = extracted / DB_NAME + new_media = extracted / "media" + db_path.parent.mkdir(parents=True, exist_ok=True) + tmp_db = db_path.with_suffix(db_path.suffix + ".restore-tmp") + if tmp_db.exists(): + tmp_db.unlink() + shutil.copy2(new_db, tmp_db) + os.replace(tmp_db, db_path) + incoming_root = Path(raw) / "media-incoming" + incoming_root.mkdir() + if new_media.exists(): + for path in new_media.rglob("*"): + if path.is_file(): + rel = path.relative_to(new_media) + dest = incoming_root / rel + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, dest) + swap = media_root.with_name(media_root.name + ".restore-old") + if swap.exists(): + shutil.rmtree(swap) + if media_root.exists(): + media_root.rename(swap) + incoming_root.rename(media_root) + if swap.exists(): + shutil.rmtree(swap, ignore_errors=True) + return { + "restored": str(archive), + "db_path": str(db_path), + "media_root": str(media_root), + "safety_backup": str(safety_path) if safety_path else None, + "skipped_extra": verified["skipped_extra"], + "warning": PERSONAL_DATA_WARNING, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Lokales Kanshō-Backup (SQLite-Snapshot + Medien). Persönliche unverschlüsselte Daten." + ) + sub = parser.add_subparsers(dest="action", required=True) + create = sub.add_parser("create", help="Konsistentes lokales Backup erzeugen") + create.add_argument("--out", help="Zieldatei. Standard: local-backups/kansho-.zip") + restore = sub.add_parser("restore", help="Backup prüfen und einspielen") + restore.add_argument("archive", help="Backup-Archiv") + restore.add_argument("--confirm", action="store_true", help="Ausdrückliche Bestätigung, sonst Abbruch") + restore.add_argument("--replace", action="store_true", help="Nach Sicherheitsbackup aktuellen Stand ersetzen") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + if args.action == "create": + result = create_backup(Path(args.out) if args.out else None) + print(PERSONAL_DATA_WARNING) + print(f"Backup: {result['path']}") + print(f"Teile: sqlite + {len(result['manifest']['parts']['media'])} Mediendateien") + return 0 + result = restore_backup( + Path(args.archive), + confirm=bool(args.confirm), + allow_overwrite=bool(args.replace), + ) + print(PERSONAL_DATA_WARNING) + print(f"Restore: {result['db_path']}") + if result["safety_backup"]: + print(f"Sicherheitsbackup: {result['safety_backup']}") + if result["skipped_extra"]: + print("Zusätzliche Archivdateien ignoriert: " + ", ".join(result["skipped_extra"])) + return 0 + except BackupError as exc: + print(f"ERROR {exc.code}: {exc.message}", file=sys.stderr) + if exc.details: + print(json.dumps(exc.details, ensure_ascii=False), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/media_store.py b/backend/media_store.py index 7f5589e..e345254 100644 --- a/backend/media_store.py +++ b/backend/media_store.py @@ -5,6 +5,7 @@ Video containers are stored as-is; GPS in video metadata is not stripped yet. """ from __future__ import annotations +import os import uuid from pathlib import Path @@ -27,10 +28,19 @@ VIDEO_TYPES = { ALLOWED_TYPES = {**IMAGE_TYPES, **VIDEO_TYPES} IMAGE_MAX_BYTES = 8 * 1024 * 1024 VIDEO_MAX_BYTES = 64 * 1024 * 1024 -MEDIA_ROOT = DATA_DIR / "media" MAX_BYTES = IMAGE_MAX_BYTES # tests and older imports +def media_root() -> Path: + env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip() + if env: + return Path(env) + return DATA_DIR / "media" + + +MEDIA_ROOT = DATA_DIR / "media" # default; call media_root() at use time + + def _owned(conn, table: str, record_id: str, profile_id: str) -> dict | None: return row_to_dict( conn.execute(f"SELECT * FROM {table} WHERE id = ? AND profile_id = ?", (record_id, profile_id)).fetchone() @@ -150,7 +160,7 @@ def save_media(profile_id: str, entry_id: str, filename: str, content_type: str, asset_id = str(uuid.uuid4()) ext = ALLOWED_TYPES[kind] rel_path = f"{profile_id}/{asset_id}{ext}" - dest = MEDIA_ROOT / rel_path + dest = media_root() / rel_path dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(payload) prefix = "video" if is_video else "image" @@ -172,8 +182,8 @@ def save_image(profile_id: str, entry_id: str, filename: str, content_type: str, return save_media(profile_id, entry_id, filename, content_type, data) -def list_media(profile_id: str, entry_id: str) -> list[dict]: - get_entry(profile_id, entry_id) +def list_media(profile_id: str, entry_id: str, *, include_deleted: bool = False) -> list[dict]: + get_entry(profile_id, entry_id, include_deleted=include_deleted) with get_db() as conn: rows = conn.execute( """ @@ -191,7 +201,7 @@ def get_media(profile_id: str, media_id: str) -> tuple[dict, Path]: row = _owned(conn, "media_assets", media_id, profile_id) if not row: raise StoreError("not_found", "Medium nicht gefunden", 404) - path = MEDIA_ROOT / row["rel_path"] + path = media_root() / row["rel_path"] if not path.is_file(): raise StoreError("not_found", "Mediendatei fehlt", 404) return public_media(row), path diff --git a/backend/model_catalog.py b/backend/model_catalog.py new file mode 100644 index 0000000..420aa63 --- /dev/null +++ b/backend/model_catalog.py @@ -0,0 +1,251 @@ +"""Cached model window metadata for journal budget checks. + +OpenRouter exposes context_length and top_provider.max_completion_tokens via +GET /api/v1/models. The catalog is cached so journal calls do not refetch on +every generate. Missing metadata fails closed for journal generation. +""" +from __future__ import annotations + +import os +import threading +import time +from urllib.parse import urlparse + +import httpx + +from prompt_budget import ( + ERROR_MODEL_METADATA_UNKNOWN, + JOURNAL_TARGET_COMPLETION_TOKENS, + JournalBudgetError, + ModelWindow, + _int_env, +) + +DEFAULT_TTL_SECONDS = 3600 +FAKE_CONTEXT = 32_768 +_lock = threading.Lock() +# url -> (fetched_at, {model_id: ModelWindow fields as dict}) +_catalog: dict[str, tuple[float, dict[str, dict]]] = {} +_overrides: dict[str, ModelWindow] = {} + + +def catalog_ttl() -> int: + raw = (os.environ.get("KANSHO_MODEL_CATALOG_TTL_SECONDS") or "").strip() + if not raw: + return DEFAULT_TTL_SECONDS + try: + value = int(raw) + except ValueError: + return DEFAULT_TTL_SECONDS + return max(60, value) + + +def reset_catalog() -> None: + with _lock: + _catalog.clear() + _overrides.clear() + + +def set_metadata_override(model: str, window: ModelWindow) -> None: + with _lock: + _overrides[model] = window + + +def is_openrouter_url(url: str) -> bool: + host = (urlparse(url or "").hostname or "").lower() + return host.endswith("openrouter.ai") + + +def models_url(chat_url: str) -> str: + raw = (chat_url or "").rstrip("/") + if raw.endswith("/chat/completions"): + return raw[: -len("/chat/completions")] + "/models" + if is_openrouter_url(raw): + return "https://openrouter.ai/api/v1/models" + return raw + "/models" if raw else "https://openrouter.ai/api/v1/models" + + +def _from_env(model: str, *, provider: str | None, source: str) -> ModelWindow | None: + context = _int_env("KANSHO_PROVIDER_CONTEXT_LENGTH", 0) + completion = _int_env("KANSHO_PROVIDER_MAX_COMPLETION_TOKENS", 0) + if context <= 0: + return None + return ModelWindow( + model=model, + context_length=context, + max_completion_tokens=completion or JOURNAL_TARGET_COMPLETION_TOKENS, + source=source, + provider=provider, + cached=False, + ) + + +def _parse_model_row(row: dict) -> ModelWindow | None: + model_id = str(row.get("id") or "").strip() + if not model_id: + return None + top = row.get("top_provider") if isinstance(row.get("top_provider"), dict) else {} + context = row.get("context_length") or top.get("context_length") + completion = top.get("max_completion_tokens") + if completion is None: + limits = row.get("per_request_limits") if isinstance(row.get("per_request_limits"), dict) else {} + completion = limits.get("completion_tokens") or limits.get("max_tokens") + try: + context_n = int(context) + except (TypeError, ValueError): + return None + try: + completion_n = int(completion) if completion is not None else None + except (TypeError, ValueError): + completion_n = None + if context_n <= 0: + return None + return ModelWindow( + model=model_id, + context_length=context_n, + max_completion_tokens=completion_n, + source="openrouter_models", + provider="openrouter", + cached=True, + ) + + +def _fetch_catalog(url: str, key: str) -> dict[str, dict]: + headers = {"Content-Type": "application/json"} + if key: + headers["Authorization"] = f"Bearer {key}" + try: + response = httpx.get(url, headers=headers, timeout=15.0) + except httpx.HTTPError as exc: + raise JournalBudgetError( + ERROR_MODEL_METADATA_UNKNOWN, + status_code=503, + diagnostics={"models_url": url, "reason": "unreachable"}, + ) from exc + if response.status_code >= 400: + raise JournalBudgetError( + ERROR_MODEL_METADATA_UNKNOWN, + status_code=503, + diagnostics={"models_url": url, "http_status": response.status_code}, + ) + try: + payload = response.json() + except ValueError as exc: + raise JournalBudgetError( + ERROR_MODEL_METADATA_UNKNOWN, + status_code=503, + diagnostics={"models_url": url, "reason": "invalid_json"}, + ) from exc + rows = payload.get("data") if isinstance(payload, dict) else payload + if not isinstance(rows, list): + raise JournalBudgetError( + ERROR_MODEL_METADATA_UNKNOWN, + status_code=503, + diagnostics={"models_url": url, "reason": "unexpected_shape"}, + ) + parsed: dict[str, dict] = {} + for row in rows: + if not isinstance(row, dict): + continue + window = _parse_model_row(row) + if window: + parsed[window.model] = { + "model": window.model, + "context_length": window.context_length, + "max_completion_tokens": window.max_completion_tokens, + "source": window.source, + "provider": window.provider, + } + if not parsed: + raise JournalBudgetError( + ERROR_MODEL_METADATA_UNKNOWN, + status_code=503, + diagnostics={"models_url": url, "reason": "empty_catalog"}, + ) + return parsed + + +def _window_from_cache(fields: dict, *, model: str) -> ModelWindow: + return ModelWindow( + model=model, + context_length=int(fields["context_length"]), + max_completion_tokens=fields.get("max_completion_tokens"), + source=fields.get("source") or "openrouter_models", + provider=fields.get("provider"), + cached=True, + ) + + +def resolve_generate_metadata(config, *, now: float | None = None) -> ModelWindow: + """Return a window for the generate provider. Fail closed if unknown.""" + model = getattr(config, "model", None) or "" + with _lock: + override = _overrides.get(model) + if override: + return override + if getattr(config, "mode", None) == "fake" or model == "fake": + return ModelWindow( + model=model or "fake", + context_length=FAKE_CONTEXT, + max_completion_tokens=JOURNAL_TARGET_COMPLETION_TOKENS, + source="fake", + provider="fake", + cached=False, + ) + env_window = _from_env(model, provider=getattr(config, "name", None), source="env") + if getattr(config, "local", False): + if env_window: + return env_window + raise JournalBudgetError( + ERROR_MODEL_METADATA_UNKNOWN, + status_code=503, + diagnostics={ + "model": model, + "provider": getattr(config, "name", None), + "reason": "local_context_unconfigured", + }, + ) + url = getattr(config, "url", "") or "" + if not is_openrouter_url(url): + if env_window: + return env_window + raise JournalBudgetError( + ERROR_MODEL_METADATA_UNKNOWN, + status_code=503, + diagnostics={ + "model": model, + "provider": getattr(config, "name", None), + "reason": "non_openrouter_unconfigured", + }, + ) + catalog_key = models_url(url) + clock = time.time() if now is None else now + ttl = catalog_ttl() + with _lock: + cached = _catalog.get(catalog_key) + if cached and clock - cached[0] < ttl and model in cached[1]: + return _window_from_cache(cached[1][model], model=model) + stale = cached + try: + fetched = _fetch_catalog(catalog_key, getattr(config, "key", "") or "") + except JournalBudgetError: + if stale and model in stale[1]: + window = _window_from_cache(stale[1][model], model=model) + window.source = "openrouter_models_stale" + return window + if env_window: + env_window.source = "env_fallback" + return env_window + raise + with _lock: + _catalog[catalog_key] = (clock, fetched) + if model not in fetched: + if env_window: + env_window.source = "env_fallback" + return env_window + raise JournalBudgetError( + ERROR_MODEL_METADATA_UNKNOWN, + status_code=503, + diagnostics={"model": model, "reason": "model_not_in_catalog"}, + ) + return _window_from_cache(fetched[model], model=model) diff --git a/backend/placeholder_mvp.py b/backend/placeholder_mvp.py index d62011e..7c72982 100644 --- a/backend/placeholder_mvp.py +++ b/backend/placeholder_mvp.py @@ -72,6 +72,22 @@ register( resolver=_from_ctx("register_hint"), ) ) +register( + Placeholder( + key="opening_hint", + description="Journalspezifischer erster Impuls. Leer bei späteren Zügen. Keine Intent-Engine.", + data_class="B", + resolver=_from_ctx("opening_hint"), + ) +) +register( + Placeholder( + key="reconstruction", + description="Validierte inhaltliche Rekonstruktion der Stufe 1. Kein vollständiger Tagesdialog.", + data_class="B", + resolver=_from_ctx("reconstruction"), + ) +) register( Placeholder( key="review_package", diff --git a/backend/privacy_gateway.py b/backend/privacy_gateway.py index 42820f4..287809f 100644 --- a/backend/privacy_gateway.py +++ b/backend/privacy_gateway.py @@ -4,25 +4,59 @@ from __future__ import annotations from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any - +import contextvars import re from entity_detect import detect_and_remember from identity_store import KINSHIP, is_maskable_label, list_mappings +from journal_reconstruct import claim_texts, fake_reconstruction, is_dialogue_role_line +from prompt_budget import ( + ERROR_PROVIDER_CONTEXT_LENGTH, + JOURNAL_PURPOSES, + JournalBudgetError, + USER_MESSAGES, + assert_input_fits, + merge_usage, +) from pronoun_bind import bind_user_lines -from providers import ProviderError, complete_chat, generate_provider +from providers import ChatResult, ProviderError, complete_chat, generate_provider MAX_EGRESS_CHARS = 24000 -debug_last: dict[str, Any] | None = None debug_calls = 0 +last_compact: dict[str, Any] | None = None +_test_recorder: contextvars.ContextVar[list | None] = contextvars.ContextVar( + "kansho_gateway_recorder", + default=None, +) +COMPACT_DIAGNOSTIC_KEYS = ( + "model", + "actual_model", + "provider", + "purpose", + "prompt_slug", + "effective_context_window", + "estimated_input_tokens", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "reserved_output_tokens", + "max_tokens", + "safety_margin", + "context_compression", + "cost", + "budget_ok", + "abort_reason", + "status", +) class PrivacyGatewayError(Exception): - def __init__(self, code: str, message: str, status_code: int = 503): + def __init__(self, code: str, message: str, status_code: int = 503, diagnostics: dict | None = None): super().__init__(message) self.code = code self.message = message self.status_code = status_code + self.diagnostics = diagnostics or {} @dataclass @@ -40,6 +74,8 @@ class GatewayResult: reason: str provider: str | None = None content: str | None = None + diagnostics: dict[str, Any] = field(default_factory=dict) + trace: dict[str, Any] | None = None checked_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) @@ -47,10 +83,32 @@ ALLOWED_CLASSES = {"A", "B", "C"} LOCAL_ONLY_CLASS = "A" +def compact_diagnostics(data: dict[str, Any] | None) -> dict[str, Any]: + payload = {} + for key in COMPACT_DIAGNOSTIC_KEYS: + if data and key in data and data[key] is not None: + payload[key] = data[key] + return payload + + +def install_test_recorder() -> list[dict[str, Any]]: + """Test-local observer. Production never installs this.""" + records: list[dict[str, Any]] = [] + _test_recorder.set(records) + return records + + +def _record_test(event: dict[str, Any]) -> None: + records = _test_recorder.get() + if records is not None: + records.append(event) + + def reset_debug() -> None: - global debug_last, debug_calls - debug_last = None + global last_compact, debug_calls + last_compact = None debug_calls = 0 + _test_recorder.set(None) def inspect(request: GatewayRequest) -> GatewayResult: @@ -66,8 +124,10 @@ def inspect(request: GatewayRequest) -> GatewayResult: return GatewayResult(allowed=True, reason="policy_ok", provider=config.name) -def _minimize(text: str) -> str: +def _minimize(text: str, purpose: str = "") -> str: cleaned = (text or "").strip() + if purpose in JOURNAL_PURPOSES: + return cleaned if len(cleaned) <= MAX_EGRESS_CHARS: return cleaned return cleaned[:MAX_EGRESS_CHARS] @@ -109,6 +169,20 @@ def _mask_person_hit(text: str, start: int, end: int) -> bool: return True +def _is_identity_mention(text: str, start: int, end: int, token: str) -> bool: + """Same rule as masking: food/thing homonyms are not identity.""" + if not _is_person_token(token): + return True + return _mask_person_hit(text, start, end) + + +def _label_pattern(label: str) -> re.Pattern[str]: + return re.compile( + rf"(? str: masked = text for item in sorted(mappings, key=lambda row: len(row.get("local_label") or ""), reverse=True): @@ -117,14 +191,10 @@ def _mask_body(text: str, mappings: list[dict]) -> str: if not label or not token or not is_maskable_label(label): continue placeholder = token if token.startswith("[[") else f"[[{token}]]" - pattern = re.compile( - rf"(? str: - if _person and not _mask_person_hit(masked, match.start(), match.end()): + def repl(match: re.Match, *, _token=token, _ph=placeholder) -> str: + if not _is_identity_mention(masked, match.start(), match.end(), _token): return match.group(0) return _ph @@ -139,19 +209,26 @@ def _mask(text: str, mappings: list[dict], *, personal_lines_only: bool = False) for line in (text or "").splitlines(keepends=True): raw = line[:-1] if line.endswith("\n") else line newline = "\n" if line.endswith("\n") else "" - if raw.startswith("user:") or raw.startswith("assistant:"): + if is_dialogue_role_line(raw): parts.append(_mask_body(raw, mappings) + newline) else: parts.append(line) return "".join(parts) -def mask_for_egress(rendered: str, mappings: list[dict], purpose: str) -> str: - """Mask names. Bind pronouns to person tokens only in the dialogue turn. +IDENTITY_LEAK_RETRY = ( + "Korrektur: Keine Klartext-Identität. Nur die vorhandenen Platzhalter [[…]], " + "keine Klarnamen, keine Klarorte." +) + +def mask_for_egress(rendered: str, mappings: list[dict], purpose: str) -> str: + """Mask names in the full rendered prompt. Bind pronouns only on user lines. + + Opening hints, Space-Ausschnitte and titles are personal context, not instructions. Journal generation keeps sie/er/ihr so demask does not turn every reference into the name. """ - masked = _mask(rendered, mappings, personal_lines_only=purpose == "dialogue_turn") + masked = _mask(rendered, mappings) if purpose == "dialogue_turn": return bind_user_lines(masked) return masked @@ -176,14 +253,13 @@ def _validate_response(content: str, mappings: list[dict] | None = None) -> str: leaked: list[str] = [] for item in mappings or []: label = (item.get("local_label") or "").strip() + token = (item.get("token") or "").strip() if not label or not is_maskable_label(label): continue - if re.search( - rf"(? str: '"rationale":"Fake-Review: Journal-Evidenz trägt Humor in autobiographical_journal, nicht als globalen Core.",' '"evidence_ids":[],"exemplars":[{"excerpt":"haha das war irgendwie lustig","role":"exemplar","evidence_basis":["kansho_sources"]}]}]}' ) + if purpose == "journal_reconstruct": + return fake_reconstruction(rendered) if purpose == "journal_generate": + match = re.search(r"\{.*\}", rendered, re.DOTALL) + if match: + try: + import json + + data = json.loads(match.group(0)) + parts = claim_texts(data) + body = " ".join(str(part) for part in parts if part) + if body: + return f"Ein Tag\n\n{body}" + except (ValueError, TypeError): + pass lines = [] for line in rendered.splitlines(): if line.startswith("user:"): @@ -221,21 +311,39 @@ def _fake_complete(purpose: str, rendered: str) -> str: return '{"operation":"erleben_vertiefen","impulse":"' + impulse + '"}' -def complete_model(messages: list[dict], policy: dict[str, Any]) -> str: +def complete_model(messages: list[dict], policy: dict[str, Any]) -> ChatResult: config = generate_provider() if not config: raise PrivacyGatewayError("no_egress_provider_configured", "Es ist kein Egress-Provider konfiguriert.") if config.mode == "fake": rendered = "\n".join(item.get("content") or "" for item in messages) - return _fake_complete(policy.get("purpose") or "", rendered) + return ChatResult( + content=_fake_complete(policy.get("purpose") or "", rendered), + model=config.model, + usage={}, + context_compression="disabled" if policy.get("disable_context_compression") else "not_applicable", + ) try: - return complete_chat(config, messages, timeout=60) + return complete_chat( + config, + messages, + timeout=90 if (policy.get("purpose") in JOURNAL_PURPOSES) else 60, + max_tokens=policy.get("max_tokens"), + disable_context_compression=bool(policy.get("disable_context_compression")), + ) except ProviderError as exc: - raise PrivacyGatewayError(exc.code, exc.message, exc.status_code) from exc + if exc.code == "provider_context_length_rejected": + raise PrivacyGatewayError( + ERROR_PROVIDER_CONTEXT_LENGTH, + USER_MESSAGES[ERROR_PROVIDER_CONTEXT_LENGTH], + exc.status_code, + diagnostics=exc.diagnostics, + ) from exc + raise PrivacyGatewayError(exc.code, exc.message, exc.status_code, getattr(exc, "diagnostics", None)) from exc def complete(request: GatewayRequest) -> GatewayResult: - global debug_last, debug_calls + global last_compact, debug_calls result = inspect(request) if not result.allowed: raise PrivacyGatewayError( @@ -245,7 +353,7 @@ def complete(request: GatewayRequest) -> GatewayResult: if result.reason == "no_egress_provider_configured" else "Persönlicher KI-Aufruf wurde vom Privacy Gateway blockiert.", ) - rendered = _minimize(str((request.payload or {}).get("rendered") or "")) + rendered = _minimize(str((request.payload or {}).get("rendered") or ""), request.purpose) source_text = str((request.payload or {}).get("source_text") or rendered) detect_name = None try: @@ -255,14 +363,39 @@ def complete(request: GatewayRequest) -> GatewayResult: if not mappings and request.profile_id: mappings = list_mappings(request.profile_id) masked = mask_for_egress(rendered, mappings, request.purpose) + budget = (request.payload or {}).get("budget") + diagnostics = dict((request.payload or {}).get("diagnostics") or {}) + if request.purpose in JOURNAL_PURPOSES: + if budget is None: + raise PrivacyGatewayError( + "model_metadata_unknown", + USER_MESSAGES["model_metadata_unknown"], + 503, + diagnostics={"reason": "journal_budget_missing"}, + ) + try: + assert_input_fits(budget, masked) + except JournalBudgetError as exc: + raise PrivacyGatewayError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc + diagnostics.update(budget.as_diagnostics()) + diagnostics["estimated_input_tokens"] = budget.estimated_input_tokens debug_calls += 1 config = generate_provider() - debug_last = { + layer = { + "journal_generate": "journalentwurf", + "journal_reconstruct": "journalrekonstruktion", + "profile_review": "profilreview", + }.get(request.purpose, "dialogzug") + max_tokens = (request.payload or {}).get("max_tokens") + if max_tokens is not None: + diagnostics["max_tokens"] = max_tokens + diagnostics["provider"] = result.provider + diagnostics["purpose"] = request.purpose + diagnostics["prompt_slug"] = (request.payload or {}).get("prompt_slug") + diagnostics["model"] = config.model if config else None + request_trace = { "purpose": request.purpose, - "layer": { - "journal_generate": "journalentwurf", - "profile_review": "profilreview", - }.get(request.purpose, "dialogzug"), + "layer": layer, "data_class": request.data_class, "prompt_slug": (request.payload or {}).get("prompt_slug"), "rendered": rendered, @@ -273,27 +406,76 @@ def complete(request: GatewayRequest) -> GatewayResult: "detect_provider": detect_name, "detect_note": detect_note, "mapping_count": len(mappings), + "budget": compact_diagnostics(diagnostics), } - raw = complete_model( - [{"role": "user", "content": masked}], - {"purpose": request.purpose, "zdr": True, "no_train": True}, + model_policy = { + "purpose": request.purpose, + "zdr": True, + "no_train": True, + "max_tokens": max_tokens, + "disable_context_compression": bool( + (request.payload or {}).get("disable_context_compression") + ), + } + try: + chat = complete_model([{"role": "user", "content": masked}], model_policy) + raw = chat.content + try: + validated = _validate_response(raw, mappings) + except PrivacyGatewayError as exc: + if exc.code != "response_validation_failed": + raise + diagnostics["response_validation_retry"] = 1 + chat = complete_model( + [{"role": "user", "content": masked + "\n\n" + IDENTITY_LEAK_RETRY}], + model_policy, + ) + raw = chat.content + validated = _validate_response(raw, mappings) + except PrivacyGatewayError as exc: + failed = merge_usage( + {**diagnostics, "budget_ok": False, "abort_reason": exc.code, "status": "error"}, + None, + config.model if config else None, + ) + last_compact = compact_diagnostics(failed) + _record_test({"purpose": request.purpose, "ok": False, "code": exc.code}) + raise + diagnostics = merge_usage( + { + **diagnostics, + "context_compression": chat.context_compression, + "budget_ok": True, + "status": "ok", + }, + chat.usage, + chat.model or (config.model if config else None), ) - validated = _validate_response(raw, mappings) result.content = _demask(validated, mappings) result.allowed = True - debug_last["raw"] = validated - debug_last["reply"] = result.content + result.diagnostics = compact_diagnostics(diagnostics) + request_trace["raw"] = validated + request_trace["reply"] = result.content + request_trace["model"] = chat.model or request_trace.get("model") + request_trace["budget"] = compact_diagnostics(diagnostics) + result.trace = public_trace(request_trace) + last_compact = compact_diagnostics(diagnostics) + _record_test({"purpose": request.purpose, "ok": True, "prompt_slug": request_trace.get("prompt_slug")}) return result def last_trace() -> dict | None: - return debug_last + """Compact diagnostics only. Full prompts live on the current GatewayResult.""" + return last_compact def public_trace(trace: dict | None) -> dict | None: - """Admin test view. No mapping table, no secrets.""" + """Admin test view for the current response. No mapping table, no secrets.""" if not trace: return None + budget = dict(trace.get("budget") or {}) + for key in ("rendered", "masked", "intern", "raw", "reply", "prompt", "messages"): + budget.pop(key, None) return { "purpose": trace.get("purpose"), "layer": trace.get("layer"), @@ -304,9 +486,11 @@ def public_trace(trace: dict | None) -> dict | None: "detect_provider": trace.get("detect_provider"), "detect_note": trace.get("detect_note"), "mapping_count": trace.get("mapping_count"), - "intern": trace.get("rendered"), - "egress": trace.get("masked"), + "intern": trace.get("rendered") or trace.get("intern"), + "egress": trace.get("masked") or trace.get("egress"), "mask_input": trace.get("mask_input"), "raw": trace.get("raw"), "reply": trace.get("reply"), + "budget": budget or None, + "stages": trace.get("stages"), } diff --git a/backend/prompt_budget.py b/backend/prompt_budget.py new file mode 100644 index 0000000..2d9db88 --- /dev/null +++ b/backend/prompt_budget.py @@ -0,0 +1,277 @@ +"""Conservative prompt budget for personal journal generation. + +Token counts are estimates, not tokenizer-accurate. Kanshō overestimates input +so a request is refused before it can overflow a context window. There is no +silent middle-out deletion of the day. +""" +from __future__ import annotations + +import math +import os +from dataclasses import dataclass, field +from typing import Any + +# Journal generation requires a wide window so a full day can be reconstructed +# without dropping middle events. 32_768 is the product default, not a claim +# that every model tokenizer uses this unit identically. +JOURNAL_MIN_CONTEXT_TOKENS = 32_768 +JOURNAL_TARGET_COMPLETION_TOKENS = 4_096 +# Overestimate: German mixed with prompt English is typically closer to 3–4 +# characters per token. Two characters per token is deliberately conservative. +DEFAULT_CHARS_PER_TOKEN = 2.0 +DEFAULT_SAFETY_MARGIN = 0.15 +CHAT_FORMAT_OVERHEAD_TOKENS = 64 +JOURNAL_PURPOSES = {"journal_generate", "journal_reconstruct"} + +ERROR_MODEL_CONTEXT_TOO_SMALL = "model_context_too_small" +ERROR_PROMPT_BUDGET_EXCEEDED = "prompt_budget_exceeded" +ERROR_MODEL_METADATA_UNKNOWN = "model_metadata_unknown" +ERROR_RECONSTRUCTION_INVALID = "reconstruction_invalid" +ERROR_NO_USER_SOURCES = "no_user_sources" +ERROR_OUTPUT_LIMIT_UNSUPPORTED = "output_limit_unsupported" +ERROR_PROVIDER_CONTEXT_LENGTH = "provider_context_length_rejected" + +USER_MESSAGES = { + ERROR_MODEL_CONTEXT_TOO_SMALL: ( + "Das gewählte Modell hat ein zu kleines Kontextfenster für die " + "Journalgenerierung. Es sind mindestens 32.000 Tokens erforderlich. " + "Es wurde kein anderes Modell gewählt." + ), + ERROR_PROMPT_BUDGET_EXCEEDED: ( + "Der Tagesdialog ist für eine sichere Verarbeitung zu umfangreich. " + "Kanshō hat nichts stillschweigend aus der Mitte entfernt. " + "Bitte wähle weniger Gespräche oder ein Modell mit größerem Kontext." + ), + ERROR_MODEL_METADATA_UNKNOWN: ( + "Die Kontextgrenzen des Modells konnten nicht sicher bestimmt werden. " + "Der Aufruf wurde nicht gesendet." + ), + ERROR_RECONSTRUCTION_INVALID: ( + "Die inhaltliche Rekonstruktion war unvollständig oder ungültig. " + "Es wurde kein Journalentwurf erzeugt." + ), + ERROR_NO_USER_SOURCES: ( + "In den ausgewählten Gesprächen gibt es noch keinen Nutzertext. " + "Es wurde kein Journalentwurf erzeugt." + ), + ERROR_OUTPUT_LIMIT_UNSUPPORTED: ( + "Das Modell unterstützt die für die Journalgenerierung reservierte " + "Ausgabelänge nicht. Der Aufruf wurde nicht gesendet." + ), + ERROR_PROVIDER_CONTEXT_LENGTH: ( + "Der Anbieter hat die Anfrage wegen der Kontextlänge abgelehnt. " + "Kanshō hat den Tagesdialog nicht automatisch gekürzt." + ), +} + + +class JournalBudgetError(Exception): + def __init__( + self, + code: str, + message: str | None = None, + status_code: int = 422, + diagnostics: dict[str, Any] | None = None, + ): + super().__init__(message or USER_MESSAGES.get(code) or code) + self.code = code + self.message = message or USER_MESSAGES.get(code) or code + self.status_code = status_code + self.diagnostics = diagnostics or {} + + +@dataclass +class ModelWindow: + model: str + context_length: int + max_completion_tokens: int | None + source: str + provider: str | None = None + cached: bool = False + + +@dataclass +class JournalBudget: + model: str + purpose: str + effective_context_window: int + reserved_output_tokens: int + safety_margin: float + estimated_input_tokens: int = 0 + available_input_tokens: int = 0 + chars_per_token: float = DEFAULT_CHARS_PER_TOKEN + budget_ok: bool = True + abort_reason: str | None = None + context_compression: str = "disabled" + extras: dict[str, Any] = field(default_factory=dict) + + def as_diagnostics(self) -> dict[str, Any]: + payload = { + "model": self.model, + "purpose": self.purpose, + "estimated_input_tokens": self.estimated_input_tokens, + "effective_context_window": self.effective_context_window, + "reserved_output_tokens": self.reserved_output_tokens, + "safety_margin": self.safety_margin, + "available_input_tokens": self.available_input_tokens, + "chars_per_token": self.chars_per_token, + "budget_ok": self.budget_ok, + "context_compression": self.context_compression, + "abort_reason": self.abort_reason, + "estimation": "conservative_char_ratio", + } + payload.update(self.extras) + return payload + + +def _float_env(name: str, default: float) -> float: + raw = (os.environ.get(name) or "").strip() + if not raw: + return default + try: + value = float(raw) + except ValueError: + return default + return value if value > 0 else default + + +def _int_env(name: str, default: int) -> int: + raw = (os.environ.get(name) or "").strip() + if not raw: + return default + try: + value = int(raw) + except ValueError: + return default + return value if value > 0 else default + + +def chars_per_token() -> float: + return _float_env("KANSHO_TOKEN_CHARS_PER_TOKEN", DEFAULT_CHARS_PER_TOKEN) + + +def safety_margin() -> float: + return min(0.4, max(0.05, _float_env("KANSHO_JOURNAL_SAFETY_MARGIN", DEFAULT_SAFETY_MARGIN))) + + +def min_context_tokens() -> int: + return _int_env("KANSHO_JOURNAL_MIN_CONTEXT_TOKENS", JOURNAL_MIN_CONTEXT_TOKENS) + + +def target_completion_tokens() -> int: + return _int_env("KANSHO_JOURNAL_MAX_COMPLETION_TOKENS", JOURNAL_TARGET_COMPLETION_TOKENS) + + +def day_message_safety_cap() -> int: + return _int_env("KANSHO_JOURNAL_DAY_MAX_MESSAGES", 500) + + +def estimate_tokens(text: str, *, extra: int = 0) -> int: + """Conservative character ratio. Not a model tokenizer.""" + n = len(text or "") + if n <= 0: + return extra + return max(1, math.ceil(n / chars_per_token())) + extra + + +def usable_context_tokens(window: ModelWindow) -> int: + margin = safety_margin() + usable = math.floor(window.context_length * (1.0 - margin)) + return max(0, usable) + + +def reserved_output_tokens(window: ModelWindow) -> int: + wanted = target_completion_tokens() + supported = window.max_completion_tokens + if supported is None or supported <= 0: + raise JournalBudgetError( + ERROR_OUTPUT_LIMIT_UNSUPPORTED, + diagnostics={"model": window.model, "max_completion_tokens": supported}, + ) + reserved = min(wanted, int(supported)) + if reserved < 256: + raise JournalBudgetError( + ERROR_OUTPUT_LIMIT_UNSUPPORTED, + diagnostics={ + "model": window.model, + "max_completion_tokens": supported, + "reserved_output_tokens": reserved, + }, + ) + return reserved + + +def plan_journal_budget(window: ModelWindow, *, purpose: str) -> JournalBudget: + required = min_context_tokens() + if window.context_length < required: + raise JournalBudgetError( + ERROR_MODEL_CONTEXT_TOO_SMALL, + diagnostics={ + "model": window.model, + "effective_context_window": window.context_length, + "required_context_window": required, + "source": window.source, + }, + ) + reserved = reserved_output_tokens(window) + usable = usable_context_tokens(window) + available = usable - reserved - CHAT_FORMAT_OVERHEAD_TOKENS + if available < 512: + raise JournalBudgetError( + ERROR_PROMPT_BUDGET_EXCEEDED, + diagnostics={ + "model": window.model, + "effective_context_window": window.context_length, + "reserved_output_tokens": reserved, + "available_input_tokens": available, + }, + ) + compression = "disabled" if purpose in JOURNAL_PURPOSES else "provider_default" + return JournalBudget( + model=window.model, + purpose=purpose, + effective_context_window=window.context_length, + reserved_output_tokens=reserved, + safety_margin=safety_margin(), + available_input_tokens=available, + chars_per_token=chars_per_token(), + context_compression=compression, + extras={"metadata_source": window.source, "cached_metadata": window.cached}, + ) + + +def assert_input_fits(budget: JournalBudget, text: str) -> JournalBudget: + estimated = estimate_tokens(text, extra=CHAT_FORMAT_OVERHEAD_TOKENS) + budget.estimated_input_tokens = estimated + total = estimated + budget.reserved_output_tokens + ceiling = math.floor(budget.effective_context_window * (1.0 - budget.safety_margin)) + if estimated > budget.available_input_tokens or total > ceiling: + budget.budget_ok = False + budget.abort_reason = ERROR_PROMPT_BUDGET_EXCEEDED + raise JournalBudgetError( + ERROR_PROMPT_BUDGET_EXCEEDED, + diagnostics={ + **budget.as_diagnostics(), + "estimated_input_tokens": estimated, + "required_tokens": total, + "usable_tokens": ceiling, + }, + ) + budget.budget_ok = True + budget.abort_reason = None + return budget + + +def merge_usage(diagnostics: dict[str, Any], usage: dict[str, Any] | None, model: str | None = None) -> dict[str, Any]: + payload = dict(diagnostics) + data = usage or {} + if model: + payload["actual_model"] = model + payload["prompt_tokens"] = data.get("prompt_tokens") + payload["completion_tokens"] = data.get("completion_tokens") + payload["total_tokens"] = data.get("total_tokens") + cost = data.get("cost") + if cost is None: + cost = data.get("total_cost") + payload["cost"] = cost + return payload diff --git a/backend/provenance.py b/backend/provenance.py new file mode 100644 index 0000000..fc67d6d --- /dev/null +++ b/backend/provenance.py @@ -0,0 +1,242 @@ +"""Intent-neutral provenance verification. Model output is untrusted input. + +Canonical source texts come only from a locally built SourceRegistry. +Downstream stages consume a locally materialized VerifiedArtifact. +This module has no journal, dialogue, or product-intent vocabulary. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + +COVERAGE_ALL_SELECTED_SOURCES = "all_selected_sources" +COVERAGE_SELECTED_EVIDENCE = "selected_evidence" +COVERAGE_MODES = frozenset({COVERAGE_ALL_SELECTED_SOURCES, COVERAGE_SELECTED_EVIDENCE}) +ARTIFACT_KIND = "verified_artifact" + + +class ProvenanceError(Exception): + """Fail-closed verification error. Domain mapping happens at the caller boundary.""" + + def __init__(self, reason: str, **details: Any): + super().__init__(reason) + self.reason = reason + self.details = details + + +@dataclass(frozen=True) +class SourceRecord: + source_id: str + role: str + text: str + metadata: Mapping[str, Any] = field(default_factory=dict) + + +class SourceRegistry: + """Local authority for original content. Model text never replaces these records.""" + + def __init__(self, sources: list[SourceRecord] | None = None): + self._sources: dict[str, SourceRecord] = {} + self._order: list[str] = [] + for source in sources or []: + self.add(source) + + def add(self, source: SourceRecord) -> None: + source_id = (source.source_id or "").strip() + if not source_id: + raise ProvenanceError("missing_source_id") + if source_id in self._sources: + raise ProvenanceError("duplicate_source_id", source_id=source_id) + self._sources[source_id] = SourceRecord( + source_id=source_id, + role=source.role, + text=source.text, + metadata=dict(source.metadata or {}), + ) + self._order.append(source_id) + + def get(self, source_id: str) -> SourceRecord | None: + return self._sources.get(source_id) + + def order(self) -> list[str]: + return list(self._order) + + def items(self) -> list[SourceRecord]: + return [self._sources[source_id] for source_id in self._order] + + +@dataclass(frozen=True) +class EvidenceRef: + source_id: str + excerpt: str + + +@dataclass(frozen=True) +class VerifiedEvidence: + source_id: str + excerpt: str + start: int + end: int + + +@dataclass(frozen=True) +class VerificationPolicy: + allowed_roles: frozenset[str] + coverage: str = COVERAGE_ALL_SELECTED_SOURCES + selected_ids: tuple[str, ...] = () + source_order: tuple[str, ...] = () + include_unverified_annotations: bool = False + + +@dataclass(frozen=True) +class VerifiedArtifact: + coverage: str + source_order: tuple[str, ...] + sources: tuple[dict[str, str], ...] + evidence: tuple[VerifiedEvidence, ...] + annotations_unverified: tuple[dict[str, Any], ...] = () + + def texts(self) -> list[str]: + return [item.get("text") or "" for item in self.sources if item.get("text")] + + def to_payload(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "kind": ARTIFACT_KIND, + "coverage": self.coverage, + "source_order": list(self.source_order), + "order": [item.get("source_id") for item in self.sources], + "sources": [dict(item) for item in self.sources], + "evidence": [ + { + "source_id": item.source_id, + "excerpt": item.excerpt, + "start": item.start, + "end": item.end, + } + for item in self.evidence + ], + } + if self.annotations_unverified: + payload["annotations_unverified"] = [dict(item) for item in self.annotations_unverified] + return payload + + +def locate_excerpt(text: str, excerpt: str) -> tuple[int, int] | None: + """Local span from a confirmed literal excerpt. Model offsets are ignored.""" + if not excerpt: + return None + start = (text or "").find(excerpt) + if start < 0: + return None + return start, start + len(excerpt) + + +def texts_from_payload(data: Mapping[str, Any] | None) -> list[str]: + if not isinstance(data, Mapping): + return [] + parts: list[str] = [] + for item in data.get("sources") or []: + if isinstance(item, Mapping) and item.get("text"): + parts.append(str(item["text"])) + return parts + + +def verify_evidence( + registry: SourceRegistry, + ref: EvidenceRef, + policy: VerificationPolicy, +) -> VerifiedEvidence: + source_id = (ref.source_id or "").strip() + excerpt = ref.excerpt or "" + if not source_id: + raise ProvenanceError("missing_source_id") + if not excerpt.strip(): + raise ProvenanceError("unverified_excerpt", source_id=source_id) + if policy.selected_ids and source_id not in policy.selected_ids: + raise ProvenanceError("unknown_source", source_id=source_id) + source = registry.get(source_id) + if source is None: + raise ProvenanceError("unknown_source", source_id=source_id) + if source.role not in policy.allowed_roles: + raise ProvenanceError("role_not_allowed", source_id=source_id, role=source.role) + located = locate_excerpt(source.text, excerpt) + if located is not None: + return VerifiedEvidence(source_id, excerpt, located[0], located[1]) + for other in registry.items(): + if other.source_id == source_id or excerpt not in (other.text or ""): + continue + if other.role not in policy.allowed_roles: + raise ProvenanceError("role_not_allowed", source_id=other.source_id, role=other.role) + raise ProvenanceError("wrong_source", source_id=source_id, excerpt=excerpt[:120]) + raise ProvenanceError("unverified_excerpt", source_id=source_id, excerpt=excerpt[:120]) + + +def _selected_records(registry: SourceRegistry, policy: VerificationPolicy) -> list[SourceRecord]: + if policy.coverage not in COVERAGE_MODES: + raise ProvenanceError("unknown_coverage", coverage=policy.coverage) + selected = tuple(item for item in policy.selected_ids if item) + if not selected: + raise ProvenanceError("selection_empty") + if len(set(selected)) != len(selected): + raise ProvenanceError("duplicate_source_id", ids=list(selected)) + records: list[SourceRecord] = [] + for source_id in selected: + source = registry.get(source_id) + if source is None: + raise ProvenanceError("unknown_source", source_id=source_id) + if source.role not in policy.allowed_roles: + raise ProvenanceError("role_not_allowed", source_id=source_id, role=source.role) + records.append(source) + return records + + +def verify( + registry: SourceRegistry, + *, + policy: VerificationPolicy, + evidence: list[EvidenceRef] | None = None, + annotations: list[dict[str, Any]] | None = None, + proposed_texts: Mapping[str, str] | None = None, +) -> VerifiedArtifact: + """Materialize a VerifiedArtifact from local sources plus untrusted references.""" + selected = _selected_records(registry, policy) + for source in selected: + proposed = (proposed_texts or {}).get(source.source_id) + if proposed is not None and proposed != source.text: + raise ProvenanceError("canonical_overwrite", source_id=source.source_id) + + verified: list[VerifiedEvidence] = [] + for ref in evidence or []: + verified.append(verify_evidence(registry, ref, policy)) + + if policy.coverage == COVERAGE_ALL_SELECTED_SOURCES: + sources = tuple( + {"source_id": source.source_id, "role": source.role, "text": source.text} for source in selected + ) + else: + grouped: dict[str, list[str]] = {source.source_id: [] for source in selected} + for item in verified: + grouped[item.source_id].append(item.excerpt) + sources = tuple( + { + "source_id": source.source_id, + "role": source.role, + "text": "\n".join(grouped[source.source_id]), + } + for source in selected + ) + + unverified: tuple[dict[str, Any], ...] = () + if policy.include_unverified_annotations and annotations: + unverified = tuple(dict(item) for item in annotations) + + encounter = tuple(item for item in policy.source_order if item) or tuple( + source.source_id for source in selected + ) + return VerifiedArtifact( + coverage=policy.coverage, + source_order=encounter, + sources=sources, + evidence=tuple(verified), + annotations_unverified=unverified, + ) diff --git a/backend/providers.py b/backend/providers.py index 0699ab1..3c7a24e 100644 --- a/backend/providers.py +++ b/backend/providers.py @@ -2,7 +2,8 @@ from __future__ import annotations import os -from dataclasses import dataclass +import re +from dataclasses import dataclass, field from urllib.parse import urlparse import httpx @@ -11,13 +12,20 @@ from env_loader import load_env_file load_env_file() +CONTEXT_LENGTH_REJECT = re.compile( + r"context.?length|maximum context|prompt is too long|too many tokens|" + r"context window|max context|exceeds? (?:the )?(?:maximum|context)", + re.I, +) + class ProviderError(Exception): - def __init__(self, code: str, message: str, status_code: int = 503): + def __init__(self, code: str, message: str, status_code: int = 503, diagnostics: dict | None = None): super().__init__(message) self.code = code self.message = message self.status_code = status_code + self.diagnostics = diagnostics or {} @dataclass(frozen=True) @@ -33,6 +41,19 @@ class ProviderConfig: no_train: bool +@dataclass +class ChatResult: + content: str + model: str | None = None + usage: dict = field(default_factory=dict) + context_compression: str = "not_applicable" + + +def is_openrouter(config: ProviderConfig) -> bool: + host = (urlparse(config.url or "").hostname or "").lower() + return host.endswith("openrouter.ai") or (config.name or "").lower() == "openrouter" + + def _truthy(name: str) -> bool: return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} @@ -146,12 +167,23 @@ def detect_provider() -> ProviderConfig | None: ) -def complete_chat(config: ProviderConfig, messages: list[dict], *, timeout: float, max_tokens: int | None = None) -> str: +def complete_chat( + config: ProviderConfig, + messages: list[dict], + *, + timeout: float, + max_tokens: int | None = None, + disable_context_compression: bool = False, +) -> ChatResult: payload: dict = {"model": config.model, "messages": messages} if max_tokens is not None: payload["max_tokens"] = max_tokens + compression = "not_applicable" if not config.local: payload["provider"] = {"data_collection": "deny"} + if disable_context_compression and is_openrouter(config) and not config.local: + payload["plugins"] = [{"id": "context-compression", "enabled": False}] + compression = "disabled" headers = {"Content-Type": "application/json"} if config.key: headers["Authorization"] = f"Bearer {config.key}" @@ -163,9 +195,29 @@ def complete_chat(config: ProviderConfig, messages: list[dict], *, timeout: floa except httpx.HTTPError as exc: raise ProviderError("provider_unreachable", f"{config.role}-Provider ist nicht erreichbar.") from exc if response.status_code >= 400: + body = "" + try: + body = response.text or "" + except Exception: + body = "" + if CONTEXT_LENGTH_REJECT.search(body): + raise ProviderError( + "provider_context_length_rejected", + "Der Anbieter hat die Anfrage wegen der Kontextlänge abgelehnt. " + "Kanshō hat den Tagesdialog nicht automatisch gekürzt.", + 502, + diagnostics={"http_status": response.status_code, "context_compression": compression}, + ) raise ProviderError("provider_rejected", f"{config.role}-Provider hat die Anfrage abgelehnt.") data = response.json() try: - return data["choices"][0]["message"]["content"] + content = data["choices"][0]["message"]["content"] except (KeyError, IndexError, TypeError) as exc: raise ProviderError("provider_shape", f"Unerwartete {config.role}-Antwort.") from exc + usage = data.get("usage") if isinstance(data.get("usage"), dict) else {} + return ChatResult( + content=content or "", + model=data.get("model") or config.model, + usage=usage, + context_compression=compression, + ) diff --git a/backend/retrieval.py b/backend/retrieval.py index 988dcc7..271d35a 100644 --- a/backend/retrieval.py +++ b/backend/retrieval.py @@ -5,6 +5,12 @@ from typing import Any from db import get_db, row_to_dict from journal_body import plain_text +from prompt_budget import ( + ERROR_PROMPT_BUDGET_EXCEEDED, + JournalBudgetError, + day_message_safety_cap, + estimate_tokens, +) # Implementation caps. Not part of the selection-spec contract. _DAY_MESSAGE_CAP = 80 @@ -28,11 +34,58 @@ def retrieve(profile_id: str, spec: dict[str, Any]) -> list[dict]: return [] +def format_day_messages(messages: list[dict], *, with_source_ids: bool = False) -> str: + parts = [] + for message in messages: + role = message.get("role") or "user" + body = message.get("body") or "" + source_id = message.get("source_id") + if with_source_ids and source_id: + parts.append(f"[{source_id}] {role}: {body}") + else: + parts.append(f"{role}: {body}") + return "\n".join(parts).strip() + + +def pair_user_priority_messages(messages: list[dict]) -> list[dict]: + """Keep every user message. Assistant context stays inside the same conversation.""" + selected: list[dict] = [] + pending_assistant = None + pending_conversation = None + for message in messages: + conversation_id = message.get("conversation_id") + role = message.get("role") or "user" + if pending_assistant is not None and conversation_id != pending_conversation: + pending_assistant = None + pending_conversation = None + if role == "assistant": + pending_assistant = message + pending_conversation = conversation_id + continue + if role == "user": + if pending_assistant is not None and pending_conversation == conversation_id: + selected.append(pending_assistant) + selected.append(message) + pending_assistant = None + pending_conversation = None + continue + pending_assistant = None + pending_conversation = None + return selected + + def _day_messages(profile_id: str, spec: dict[str, Any]) -> list[dict]: journal_day_id = spec.get("journal_day_id") conversation_ids = spec.get("conversation_ids") or [] if not journal_day_id: return [] + overflow = spec.get("overflow") or "limit" + if overflow == "abort": + cap = int(spec.get("message_cap") or day_message_safety_cap()) + fetch_limit = cap + 1 + else: + cap = int(spec.get("message_cap") or _DAY_MESSAGE_CAP) + fetch_limit = cap with get_db() as conn: if conversation_ids: placeholders = ",".join("?" * len(conversation_ids)) @@ -42,10 +95,10 @@ def _day_messages(profile_id: str, spec: dict[str, Any]) -> list[dict]: FROM messages m JOIN conversations c ON c.id = m.conversation_id WHERE m.profile_id = ? AND c.journal_day_id = ? AND m.conversation_id IN ({placeholders}) - ORDER BY c.created, m.seq + ORDER BY c.created, c.rowid, m.seq, m.id LIMIT ? """, - (profile_id, journal_day_id, *conversation_ids, _DAY_MESSAGE_CAP), + (profile_id, journal_day_id, *conversation_ids, fetch_limit), ).fetchall() else: rows = conn.execute( @@ -54,12 +107,36 @@ def _day_messages(profile_id: str, spec: dict[str, Any]) -> list[dict]: FROM messages m JOIN conversations c ON c.id = m.conversation_id WHERE m.profile_id = ? AND c.journal_day_id = ? - ORDER BY c.created, m.seq + ORDER BY c.created, c.rowid, m.seq, m.id LIMIT ? """, - (profile_id, journal_day_id, _DAY_MESSAGE_CAP), + (profile_id, journal_day_id, fetch_limit), ).fetchall() - return [row_to_dict(row) for row in rows] + messages = [row_to_dict(row) for row in rows] + if overflow == "abort": + if len(messages) > cap: + raise JournalBudgetError( + ERROR_PROMPT_BUDGET_EXCEEDED, + diagnostics={"reason": "day_message_cap", "message_cap": cap, "fetched": len(messages)}, + ) + from journal_reconstruct import assign_source_ids + + paired = assign_source_ids(pair_user_priority_messages(messages)) + max_tokens = spec.get("max_estimated_tokens") + if max_tokens is not None: + used = estimate_tokens(format_day_messages(paired, with_source_ids=True)) + if used > int(max_tokens): + raise JournalBudgetError( + ERROR_PROMPT_BUDGET_EXCEEDED, + diagnostics={ + "reason": "day_dialogue_tokens", + "estimated_tokens": used, + "available_tokens": int(max_tokens), + "message_count": len(paired), + }, + ) + return paired + return messages def _space_entries(profile_id: str, spec: dict[str, Any]) -> list[dict]: diff --git a/backend/routers/dialogue.py b/backend/routers/dialogue.py index a497312..0664112 100644 --- a/backend/routers/dialogue.py +++ b/backend/routers/dialogue.py @@ -31,7 +31,11 @@ router = APIRouter(prefix="/api/dialogue", tags=["dialogue"]) def _http(exc: StoreError | EngineError): - raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc + detail = {"code": exc.code, "message": exc.message} + extra = getattr(exc, "diagnostics", None) + if extra: + detail["diagnostics"] = extra + raise HTTPException(status_code=exc.status_code, detail=detail) from exc class SessionWrite(BaseModel): diff --git a/backend/routers/journal.py b/backend/routers/journal.py index ab3327b..bf07014 100644 --- a/backend/routers/journal.py +++ b/backend/routers/journal.py @@ -10,6 +10,7 @@ from dialogue_turn import run_turn, visible_for_role from privacy_gateway import GatewayRequest, inspect from engine import EngineError from journal_generate import generate_draft +from journal_opening import maybe_open_journal_conversation from journal_policy import PolicyError from journal_store import ( create_day_conversation, @@ -20,13 +21,18 @@ from journal_store import ( get_or_create_day, get_user_space, list_days, + list_deleted_entries, + list_space_entries, list_user_spaces, list_versions, + purge_entry, rename_user_space, restore_entry, save_day_scratch, save_entry, soft_delete_entry, + start_payload, + undelete_entry, ) from media_store import delete_media, get_media, list_media, save_media from writing_profile_store import ( @@ -69,7 +75,11 @@ router = APIRouter(prefix="/api/journal", tags=["journal"]) def _http(exc: StoreError | PolicyError | EngineError): - raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc + detail = {"code": exc.code, "message": exc.message} + extra = getattr(exc, "diagnostics", None) + if extra: + detail["diagnostics"] = extra + raise HTTPException(status_code=exc.status_code, detail=detail) from exc class SpaceWrite(BaseModel): @@ -108,6 +118,10 @@ class RestoreWrite(BaseModel): version_id: str +class PurgeWrite(BaseModel): + confirm: bool = False + + class ScratchItem(BaseModel): id: str | None = None text: str = "" @@ -163,6 +177,11 @@ class ReviewAcceptWrite(BaseModel): baseline: bool = False +@router.get("/continuity") +def continuity(session: dict = Depends(require_auth)): + return start_payload(session["profile_id"]) + + @router.get("/spaces") def spaces(session: dict = Depends(require_auth)): return list_user_spaces(session["profile_id"]) @@ -180,7 +199,27 @@ def create_space(body: SpaceWrite, session: dict = Depends(require_auth)): def space_detail(space_id: str, session: dict = Depends(require_auth)): try: space = get_user_space(session["profile_id"], space_id) - return {"space": space, "days": list_days(session["profile_id"], space_id)} + return { + "space": space, + "days": list_days(session["profile_id"], space_id), + "entries": list_space_entries(session["profile_id"], space_id), + } + except StoreError as exc: + _http(exc) + + +@router.get("/spaces/{space_id}/entries") +def space_entries(space_id: str, session: dict = Depends(require_auth)): + try: + return list_space_entries(session["profile_id"], space_id) + except StoreError as exc: + _http(exc) + + +@router.get("/spaces/{space_id}/trash") +def space_trash(space_id: str, session: dict = Depends(require_auth)): + try: + return list_deleted_entries(session["profile_id"], space_id) except StoreError as exc: _http(exc) @@ -227,7 +266,9 @@ def write_scratch(journal_day_id: str, body: ScratchWrite, session: dict = Depen @router.post("/days/{journal_day_id}/conversations") def start_conversation(journal_day_id: str, body: ConversationWrite, session: dict = Depends(require_auth)): try: - return create_day_conversation(session["profile_id"], journal_day_id, body.title) + conversation = create_day_conversation(session["profile_id"], journal_day_id, body.title) + opened = maybe_open_journal_conversation(session["profile_id"], conversation) + return visible_for_role(opened, session.get("role")) except StoreError as exc: _http(exc) @@ -356,6 +397,22 @@ def entry_restore(entry_id: str, body: RestoreWrite, session: dict = Depends(req _http(exc) +@router.post("/entries/{entry_id}/undelete") +def entry_undelete(entry_id: str, session: dict = Depends(require_auth)): + try: + return undelete_entry(session["profile_id"], entry_id) + except StoreError as exc: + _http(exc) + + +@router.post("/entries/{entry_id}/purge") +def entry_purge(entry_id: str, body: PurgeWrite, session: dict = Depends(require_auth)): + try: + return purge_entry(session["profile_id"], entry_id, confirm=body.confirm) + except StoreError as exc: + _http(exc) + + @router.delete("/entries/{entry_id}") def entry_delete(entry_id: str, session: dict = Depends(require_auth)): try: diff --git a/backend/tests/test_architecture_correction.py b/backend/tests/test_architecture_correction.py index a795e0a..5143cb9 100644 --- a/backend/tests/test_architecture_correction.py +++ b/backend/tests/test_architecture_correction.py @@ -186,9 +186,7 @@ def main() -> None: json={"body": "Heute früh aufgestanden."}, ) expect(later_turn.status_code == 200, "later day turn") - from privacy_gateway import debug_last - - rendered = debug_last["rendered"] + rendered = (later_turn.json().get("trace") or {}).get("intern") or "" expect("Hafen" in rendered, "earlier space source can supply a known plan") expect("Heute früh aufgestanden" in rendered, "current conversation remains primary") @@ -204,9 +202,7 @@ def main() -> None: json={"body": "Nur ein kurzer Einkauf."}, ) expect(sibling_turn.status_code == 200, "sibling turn") - from privacy_gateway import debug_last as last_sibling - - sibling_rendered = last_sibling["rendered"] + sibling_rendered = (sibling_turn.json().get("trace") or {}).get("intern") or "" expect("Heute früh aufgestanden" not in sibling_rendered, "same-day sibling is not fully loaded") expect("Nur ein kurzer Einkauf" in sibling_rendered, "current sibling conversation is primary") diff --git a/backend/tests/test_frame.py b/backend/tests/test_frame.py index 6be55e4..47c9bdf 100644 --- a/backend/tests/test_frame.py +++ b/backend/tests/test_frame.py @@ -32,7 +32,7 @@ def auth_header(token: str) -> dict: def main() -> None: seed_prompts = json.loads((ROOT / "config" / "prompts.seed.json").read_text(encoding="utf-8")) slugs = {item["slug"] for item in seed_prompts} - expect(slugs == {"mvp.dialogue_turn", "mvp.journal_generate", "mvp.entity_detect", "mvp.profile_review"}, "mvp prompts seeded") + expect(slugs == {"mvp.dialogue_turn", "mvp.journal_generate", "mvp.journal_reconstruct", "mvp.entity_detect", "mvp.profile_review"}, "mvp prompts seeded") with TestClient(app) as client: health = client.get("/api/health") @@ -78,7 +78,7 @@ def main() -> None: admin = client.get("/api/admin/health", headers=headers) expect(admin.status_code == 200, "admin health") - expect(admin.json()["inventory"]["prompts"] == 3, "three mvp prompts") + expect(admin.json()["inventory"]["prompts"] == 5, "five seeded prompts") providers = client.get("/api/admin/providers", headers=headers) expect(providers.status_code == 200, "admin providers") @@ -216,6 +216,8 @@ def main() -> None: "existing_text", "known_labels", "register_hint", + "opening_hint", + "reconstruction", "review_package", "source_text", "space_title", diff --git a/backend/tests/test_journal_budget.py b/backend/tests/test_journal_budget.py new file mode 100644 index 0000000..12f1663 --- /dev/null +++ b/backend/tests/test_journal_budget.py @@ -0,0 +1,1176 @@ +"""Journal budget, two-stage generation, provider payload. Run from backend/: python tests/test_journal_budget.py""" +from __future__ import annotations + +import json +import os +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-budget-test.sqlite") +os.environ["KANSHO_FAKE_PROVIDER"] = "1" +os.environ["KANSHO_FAKE_DETECT"] = "1" +Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) + +from fastapi.testclient import TestClient +from main import app +from journal_reconstruct import parse_reconstruction, reconstruction_from_model, validate_reconstruction +from model_catalog import ModelWindow, reset_catalog, set_metadata_override +from prompt_budget import ( + ERROR_MODEL_CONTEXT_TOO_SMALL, + ERROR_NO_USER_SOURCES, + ERROR_OUTPUT_LIMIT_UNSUPPORTED, + ERROR_PROMPT_BUDGET_EXCEEDED, + ERROR_RECONSTRUCTION_INVALID, + JournalBudgetError, + assert_input_fits, + estimate_tokens, + plan_journal_budget, +) +from providers import ChatResult, ProviderConfig, complete_chat, is_openrouter +from retrieval import pair_user_priority_messages +from writing_profile_store import ( + TASK_BRIEF_MAX_CHARS, + compile_task_brief, + get_profile, + import_text, + replace_trait_refs, + upsert_trait, + update_facet, +) + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def header(token: str) -> dict: + return {"X-Auth-Token": token} + + +def _window(context: int, completion: int = 4096, model: str = "test/model") -> ModelWindow: + return ModelWindow( + model=model, + context_length=context, + max_completion_tokens=completion, + source="test", + provider="test", + ) + + +def _openrouter_config(**kwargs) -> ProviderConfig: + data = dict( + role="generate", + name="openrouter", + mode="http", + url="https://openrouter.ai/api/v1/chat/completions", + model="openai/gpt-4o", + key="sk-test", + local=False, + zdr=True, + no_train=True, + ) + data.update(kwargs) + return ProviderConfig(**data) + + +def test_budget_math() -> None: + window = _window(32_768) + budget = plan_journal_budget(window, purpose="journal_generate") + expect(budget.reserved_output_tokens == 4096, "journal reserves 4096 completion tokens") + expect(budget.safety_margin == 0.15, "documented safety margin") + expect(budget.available_input_tokens > 0, "input budget remains after output reserve") + expect( + budget.estimated_input_tokens + budget.reserved_output_tokens + <= int(budget.effective_context_window * (1 - budget.safety_margin)) + or budget.estimated_input_tokens == 0, + "empty estimate still has a planned ceiling", + ) + typical = "user: " + ("Heute Markt. " * 40) + fitted = assert_input_fits(budget, typical) + expect(fitted.budget_ok is True, "typical current journal request fits 32K") + expect(fitted.estimated_input_tokens >= estimate_tokens(typical), "estimate includes overhead") + huge = "Wort " * 80_000 + try: + assert_input_fits(plan_journal_budget(window, purpose="journal_generate"), huge) + raise SystemExit("FAIL: oversized input should miss the budget") + except JournalBudgetError as exc: + expect(exc.code == ERROR_PROMPT_BUDGET_EXCEEDED, "oversized input is prompt_budget_exceeded") + + try: + plan_journal_budget(_window(8192), purpose="journal_generate") + raise SystemExit("FAIL: 8K model should be rejected") + except JournalBudgetError as exc: + expect(exc.code == ERROR_MODEL_CONTEXT_TOO_SMALL, "8K model is model_context_too_small") + + try: + plan_journal_budget(_window(32_768, completion=100), purpose="journal_generate") + raise SystemExit("FAIL: tiny completion window should be rejected") + except JournalBudgetError as exc: + expect(exc.code == ERROR_OUTPUT_LIMIT_UNSUPPORTED, "tiny output window is output_limit_unsupported") + + capped = plan_journal_budget(_window(32_768, completion=2048), purpose="journal_generate") + expect(capped.reserved_output_tokens == 2048, "output reserve is capped by the model") + four_char_guess = 100 # 400 chars / 4 + expect(estimate_tokens("abcd" * 100) >= four_char_guess, "conservative estimate is not looser than 4 chars/token") + + +def _claim(kind: str, evidence: str) -> dict: + return {"kind": kind, "evidence": evidence} + + +def _chrono(source_id: str, *, time: str | None = None, claims: list | None = None, source: str | None = "user") -> dict: + item = { + "source_id": source_id, + "time": time, + "claims": claims or [], + } + if source is not None: + item["source"] = source + return item + + +def _sourced(source_id: str, evidence: str) -> dict: + return {"source_id": source_id, "evidence": evidence} + + +def _expect_invalid(payload, *, messages=None, user_bodies=None, assistant_bodies=None, reason=None) -> None: + try: + validate_reconstruction( + payload, + messages=messages, + user_bodies=user_bodies, + assistant_bodies=assistant_bodies, + ) + raise SystemExit(f"FAIL: reconstruction should be invalid ({reason or 'expected invalid'})") + except JournalBudgetError as exc: + expect(exc.code == ERROR_RECONSTRUCTION_INVALID, f"{reason or 'invalid'} is reconstruction_invalid") + if reason: + actual = (exc.diagnostics or {}).get("reason") + expect(actual == reason, f"reason is {reason}, got {actual}") + + +def test_reconstruction_rules() -> None: + user = [ + "Heute um 6:00 Uhr Tee. Vielleicht fahren wir doch nicht.", + "Um 11:00 Uhr war die Bootstour, oder doch erst später.", + ] + assistant = ["Was davon möchtest du festhalten? Bitte erzähl den nächsten Schritt."] + messages = [ + {"role": "user", "body": user[0], "conversation_id": "c1"}, + {"role": "assistant", "body": assistant[0], "conversation_id": "c1"}, + {"role": "user", "body": user[1], "conversation_id": "c1"}, + ] + good = { + "source_order": ["u1", "u2"], + "chronology": [ + _chrono( + "u1", + time="6:00 Uhr", + claims=[ + _claim("event", "Heute um 6:00 Uhr Tee"), + _claim("uncertainty", "Vielleicht fahren wir doch nicht."), + ], + ), + _chrono( + "u2", + time="11:00 Uhr", + claims=[ + _claim("event", "Um 11:00 Uhr war die Bootstour"), + _claim("uncertainty", "oder doch erst später"), + ], + ), + ], + "contradictions": [_sourced("u2", "oder doch erst später")], + "uncertainties": [_sourced("u1", "Vielleicht fahren wir doch nicht.")], + } + validated = validate_reconstruction(good, messages=messages) + blob = "\n".join(item.get("text") or "" for item in validated.get("sources") or []) + expect(validated.get("kind") == "verified_artifact", "stage 2 payload is a verified artifact") + expect("6:00" in blob, "canonical first source keeps its time") + expect(validated["source_order"] == ["u1", "u2"], "stage 1 keeps local source order") + expect("Vielleicht" in blob and "oder doch" in blob, "hedges remain as canonical wording") + expect(all(item.get("role") == "user" for item in validated["sources"]), "artifact sources are user role") + expect("chronology" not in validated, "untrusted chronology is not the stage 2 payload") + expect("contradictions" not in validated, "unverified contradiction labels are not verified content") + expect("feeling" not in json.dumps(validated.get("sources")), "source records carry no semantic kind") + expect(assistant[0] not in blob, "assistant wording is not in the artifact") + + dropped_middle = json.loads(json.dumps(good)) + dropped_middle["chronology"] = [good["chronology"][0]] + _expect_invalid(dropped_middle, messages=messages, reason="incomplete_coverage") + + three_users = [ + {"role": "user", "body": "Anfang des Tages am Markt.", "conversation_id": "c1"}, + {"role": "user", "body": "Mitte: Kirschen gekauft.", "conversation_id": "c1"}, + {"role": "user", "body": "Ende am Hafen um 18:00 Uhr.", "conversation_id": "c1"}, + ] + skip_middle = { + "source_order": ["u1", "u2", "u3"], + "chronology": [ + _chrono("u1", claims=[_claim("event", "Anfang des Tages am Markt.")]), + _chrono( + "u3", + time="18:00 Uhr", + claims=[_claim("event", "Ende am Hafen um 18:00 Uhr")], + ), + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(skip_middle, messages=three_users, reason="incomplete_coverage") + + swapped = json.loads(json.dumps(good)) + swapped["source_order"] = ["u2", "u1"] + _expect_invalid(swapped, messages=messages, reason="source_order_mismatch") + + swapped_chrono = json.loads(json.dumps(good)) + swapped_chrono["chronology"] = [good["chronology"][1], good["chronology"][0]] + _expect_invalid(swapped_chrono, messages=messages, reason="reorder_unjustified") + + duplicate = json.loads(json.dumps(good)) + duplicate["chronology"] = [good["chronology"][0], {**good["chronology"][0]}] + _expect_invalid(duplicate, messages=messages, reason="duplicate_source_id") + + unknown = json.loads(json.dumps(good)) + unknown["chronology"][1]["source_id"] = "u9" + _expect_invalid(unknown, messages=messages, reason="unknown_source_id") + + as_assistant = json.loads(json.dumps(good)) + as_assistant["chronology"][0]["source"] = "assistant" + _expect_invalid(as_assistant, messages=messages, reason="invalid_source") + + bogus_source = json.loads(json.dumps(good)) + bogus_source["chronology"][0]["source"] = "unknown" + _expect_invalid(bogus_source, messages=messages, reason="invalid_source") + + missing_source = json.loads(json.dumps(good)) + del missing_source["chronology"][0]["source"] + _expect_invalid(missing_source, messages=messages, reason="missing_source") + + empty_source = json.loads(json.dumps(good)) + empty_source["chronology"][0]["source"] = "" + _expect_invalid(empty_source, messages=messages, reason="missing_source") + + assistant_id = json.loads(json.dumps(good)) + assistant_id["chronology"][0]["source_id"] = "a1" + assistant_id["chronology"][0]["claims"] = [_claim("event", assistant[0])] + _expect_invalid(assistant_id, messages=messages, reason="assistant_as_fact") + + stolen = json.loads(json.dumps(good)) + stolen["chronology"][0]["claims"] = [_claim("event", assistant[0])] + _expect_invalid(stolen, messages=messages, reason="assistant_as_fact") + + missing_time = json.loads(json.dumps(good)) + missing_time["chronology"][0]["time"] = None + missing_time["chronology"][0]["claims"] = [_claim("event", "Tee")] + missing_time["chronology"][1]["time"] = None + missing_time["chronology"][1]["claims"] = [_claim("event", "Bootstour")] + kept_times = validate_reconstruction(missing_time, messages=messages) + kept_blob = "\n".join(item.get("text") or "" for item in kept_times["sources"]) + expect("6:00" in kept_blob and "11:00" in kept_blob, "omitted clock claims do not drop canonical times") + + dropped_uncertainty = json.loads(json.dumps(good)) + dropped_uncertainty["uncertainties"] = [] + dropped_uncertainty["chronology"][0]["claims"] = [_claim("event", "Heute um 6:00 Uhr Tee")] + dropped_uncertainty["chronology"][1]["claims"] = [_claim("event", "Um 11:00 Uhr war die Bootstour")] + kept_hedge = validate_reconstruction(dropped_uncertainty, messages=messages) + hedge_blob = "\n".join(item.get("text") or "" for item in kept_hedge["sources"]) + expect("Vielleicht" in hedge_blob, "canonical uncertainty wording is rehydrated without labels") + + dropped_contradiction = json.loads(json.dumps(good)) + dropped_contradiction["contradictions"] = [] + kept_contra = validate_reconstruction(dropped_contradiction, messages=messages) + expect("contradictions" not in kept_contra, "omitted contradiction labels are not required as facts") + expect("oder doch" in "\n".join(item.get("text") or "" for item in kept_contra["sources"]), "canonical hedge remains") + + capitalized_source = json.loads(json.dumps(good)) + capitalized_source["chronology"][0]["source"] = "User" + _expect_invalid(capitalized_source, messages=messages, reason="invalid_source") + + +def test_reconstruction_claim_and_time_binding() -> None: + tea_messages = [ + {"role": "user", "body": "Heute trank ich Tee.", "conversation_id": "c1"}, + {"role": "assistant", "body": "Bist du anschließend mit Anna nach Berlin gefahren?", "conversation_id": "c1"}, + ] + invented = { + "source_order": ["u1"], + "chronology": [ + { + "source_id": "u1", + "source": "user", + "time": None, + "events": ["Ich fuhr anschließend mit Anna nach Berlin."], + "evidence": "Heute trank ich Tee.", + } + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(invented, messages=tea_messages, reason="unverified_claim") + + paraphrased = { + "source_order": ["u1"], + "chronology": [ + _chrono( + "u1", + claims=[ + _claim("event", "Heute trank ich Tee."), + _claim("event", "Ich fuhr anschließend mit Anna nach Berlin."), + ], + ) + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(paraphrased, messages=tea_messages, reason="unverified_claim") + + identical_assistant = { + "source_order": ["u1"], + "chronology": [ + _chrono("u1", claims=[_claim("event", "Bist du anschließend mit Anna nach Berlin gefahren?")]) + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(identical_assistant, messages=tea_messages, reason="assistant_as_fact") + + two_users = [ + {"role": "user", "body": "Heute trank ich Tee.", "conversation_id": "c1"}, + {"role": "user", "body": "Später ging ich zum Markt.", "conversation_id": "c1"}, + ] + other_source = { + "source_order": ["u1", "u2"], + "chronology": [ + _chrono("u1", claims=[_claim("event", "Später ging ich zum Markt.")]), + _chrono("u2", claims=[_claim("event", "Heute trank ich Tee.")]), + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(other_source, messages=two_users, reason="wrong_source") + + each_claim_own_source = { + "source_order": ["u1"], + "chronology": [ + _chrono( + "u1", + claims=[ + _claim("event", "Heute trank ich Tee."), + _claim("event", "Ich fuhr anschließend mit Anna nach Berlin."), + ], + ) + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(each_claim_own_source, messages=tea_messages, reason="unverified_claim") + + clock_messages = [ + {"role": "user", "body": "Um 6:00 Uhr trank ich Tee.", "conversation_id": "c1"}, + {"role": "user", "body": "Um 11:00 Uhr begann die Bootstour.", "conversation_id": "c1"}, + ] + swapped_times = { + "source_order": ["u1", "u2"], + "chronology": [ + _chrono( + "u2", + time="6:00 Uhr", + claims=[_claim("event", "Um 11:00 Uhr begann die Bootstour.")], + ), + _chrono( + "u1", + time="11:00 Uhr", + claims=[_claim("event", "Um 6:00 Uhr trank ich Tee.")], + ), + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(swapped_times, messages=clock_messages, reason="time_source_mismatch") + + polluted_time = { + "source_order": ["u1", "u2"], + "chronology": [ + _chrono( + "u1", + time="6:00 Uhr mit Anna nach Berlin", + claims=[_claim("event", "Um 6:00 Uhr trank ich Tee.")], + ), + _chrono( + "u2", + time="11:00 Uhr", + claims=[_claim("event", "Um 11:00 Uhr begann die Bootstour.")], + ), + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(polluted_time, messages=clock_messages, reason="time_source_mismatch") + + justified = { + "source_order": ["u1", "u2"], + "chronology": [ + _chrono( + "u2", + time="06:00", + claims=[_claim("event", "Um 6:00 Uhr trank ich Tee.")], + ), + _chrono( + "u1", + time="11:00 Uhr", + claims=[_claim("event", "Um 11:00 Uhr begann die Bootstour.")], + ), + ], + "contradictions": [], + "uncertainties": [], + } + # messages are 6:00 then 11:00 in source order; justified uses later-mentioned morning first + morning_first = [ + {"role": "user", "body": "Um 11:00 Uhr begann die Bootstour.", "conversation_id": "c1"}, + {"role": "user", "body": "Um 6:00 Uhr trank ich Tee.", "conversation_id": "c1"}, + ] + accepted = validate_reconstruction(justified, messages=morning_first) + expect([item["source_id"] for item in accepted["sources"]] == ["u2", "u1"], "confirmed clocks may reorder events") + expect(accepted["source_order"] == ["u1", "u2"], "source_order stays the local encounter order") + expect(accepted.get("order") == ["u2", "u1"], "artifact order follows justified chronology") + + relative_messages = [ + {"role": "user", "body": "Zuerst der Markt.", "conversation_id": "c1"}, + {"role": "user", "body": "Danach der Hafen.", "conversation_id": "c1"}, + ] + unjustified = { + "source_order": ["u1", "u2"], + "chronology": [ + _chrono("u2", claims=[_claim("event", "Danach der Hafen.")]), + _chrono("u1", claims=[_claim("event", "Zuerst der Markt.")]), + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(unjustified, messages=relative_messages, reason="reorder_unjustified") + + equal_clocks = [ + {"role": "user", "body": "Um 6:00 Uhr Tee.", "conversation_id": "c1"}, + {"role": "user", "body": "Um 6:00 Uhr noch Brot.", "conversation_id": "c1"}, + ] + equal_reorder = { + "source_order": ["u1", "u2"], + "chronology": [ + _chrono("u2", time="6:00 Uhr", claims=[_claim("event", "Um 6:00 Uhr noch Brot.")]), + _chrono("u1", time="6:00 Uhr", claims=[_claim("event", "Um 6:00 Uhr Tee.")]), + ], + "contradictions": [], + "uncertainties": [], + } + _expect_invalid(equal_reorder, messages=equal_clocks, reason="reorder_unjustified") + + +def test_reconstruction_completeness_and_labels() -> None: + meal = "Morgens kaufte ich Brot. Mittags traf ich Anna am Markt. Abends kochte ich Suppe." + meal_messages = [ + {"role": "user", "body": meal, "conversation_id": "c1"}, + {"role": "assistant", "body": "War die Suppe für Gäste?", "conversation_id": "c1"}, + ] + morning_only = { + "source_order": ["u1"], + "chronology": [_chrono("u1", claims=[_claim("event", "Morgens")])], + "contradictions": [], + "uncertainties": [], + } + full = validate_reconstruction(morning_only, messages=meal_messages) + blob = "\n".join(item.get("text") or "" for item in full["sources"]) + expect(blob == meal, "bread/anna/soup source is rehydrated in full") + expect("Mittags traf ich Anna am Markt" in blob, "anna sentence survives a short claim") + expect("Abends kochte ich Suppe" in blob, "soup sentence survives a short claim") + expect(full.get("kind") == "verified_artifact", "stage 2 payload is the local artifact") + expect("Was die Suppe für Gäste" not in blob and "Gäste" not in blob, "assistant line is not rehydrated") + + rain = "Es regnete am Bahnsteig. Der Zug hatte Verspätung. Im Café bestellte ich Kaffee." + rain_messages = [{"role": "user", "body": rain, "conversation_id": "c1"}] + rain_only = { + "source_order": ["u1"], + "chronology": [_chrono("u1", claims=[_claim("event", "Es regnete")])], + "contradictions": [], + "uncertainties": [], + } + rain_full = validate_reconstruction(rain_only, messages=rain_messages) + rain_blob = "\n".join(item.get("text") or "" for item in rain_full["sources"]) + expect("Verspätung" in rain_blob and "Kaffee" in rain_blob, "a second long source is also fully rehydrated") + + feeling = { + "source_order": ["u1"], + "chronology": [_chrono("u1", claims=[_claim("feeling", "Morgens kaufte ich Brot.")])], + "contradictions": [], + "uncertainties": [], + } + labeled = validate_reconstruction(feeling, messages=meal_messages) + dumped = json.dumps(labeled) + expect("\"feeling\"" not in dumped, "kind feeling is not verified content") + expect(labeled["sources"][0]["text"] == meal, "feeling label does not replace canonical wording") + expect("annotations_unverified" not in labeled, "journal stage 2 does not receive unverified labels") + + as_contradiction = { + "source_order": ["u1"], + "chronology": [_chrono("u1", claims=[_claim("event", "Morgens kaufte ich Brot.")])], + "contradictions": [_sourced("u1", "Morgens kaufte ich Brot.")], + "uncertainties": [], + } + not_contra = validate_reconstruction(as_contradiction, messages=meal_messages) + expect("contradictions" not in not_contra, "a normal event is not a verified contradiction") + expect(not_contra["sources"][0]["text"] == meal, "canonical meal text stays intact") + + two_users = [ + {"role": "user", "body": meal, "conversation_id": "c1"}, + {"role": "user", "body": rain, "conversation_id": "c1"}, + ] + both = { + "source_order": ["u1", "u2"], + "chronology": [ + _chrono("u1", claims=[_claim("event", "Morgens")]), + _chrono("u2", claims=[_claim("event", "Es regnete")]), + ], + "contradictions": [], + "uncertainties": [], + } + combined = validate_reconstruction(both, messages=two_users) + combined_blob = "\n".join(item.get("text") or "" for item in combined["sources"]) + expect(meal in combined_blob and rain in combined_blob, "every selected user source is fully present") + + +def test_local_fallback_covers_users() -> None: + messages = [ + {"role": "assistant", "body": "Wenn du magst, fang einfach an – ich höre zu."}, + {"role": "user", "body": "Heute Markt, Kirschen."}, + {"role": "assistant", "body": "Was davon möchtest du festhalten?"}, + {"role": "user", "body": "Später der Hafen."}, + ] + artifact, info = reconstruction_from_model("das ist kein JSON", messages) + expect(info.get("stage1") == "local_fallback", "broken JSON uses local coverage") + texts = " ".join(item.get("text") or "" for item in artifact.get("sources") or []) + expect("Kirschen" in texts and "Hafen" in texts, "both user sources survive broken stage 1") + expect(all(item.get("role") == "user" for item in artifact.get("sources") or []), "assistant is not a source") + accepted, ok = reconstruction_from_model( + json.dumps( + { + "source_order": ["u1", "u2"], + "chronology": [ + { + "source_id": "u1", + "source": "user", + "time": None, + "claims": [{"kind": "event", "evidence": "Heute Markt, Kirschen."}], + }, + { + "source_id": "u2", + "source": "user", + "time": None, + "claims": [{"kind": "event", "evidence": "Später der Hafen."}], + }, + ], + "contradictions": [], + "uncertainties": [], + "plan_changes": [], + "corrections": [], + } + ), + messages, + ) + expect(ok.get("stage1") == "model_accepted", "valid stage 1 still accepted") + expect("Kirschen" in " ".join(item.get("text") or "" for item in accepted.get("sources") or []), "accepted path keeps user text") + try: + reconstruction_from_model("{}", [{"role": "assistant", "body": "Hallo"}]) + raise SystemExit("FAIL: expected no_user_sources") + except JournalBudgetError as exc: + expect(exc.code == ERROR_NO_USER_SOURCES, "assistant-only day has no journal source") + + +def test_day_messages_keep_middle() -> None: + messages = [ + {"role": "assistant", "body": "Frage A"}, + {"role": "user", "body": "Anfang"}, + {"role": "assistant", "body": "Frage B"}, + {"role": "user", "body": "Mitte des Tages, Markt"}, + {"role": "assistant", "body": "Frage C"}, + {"role": "user", "body": "Ende"}, + ] + paired = pair_user_priority_messages(messages) + bodies = [item["body"] for item in paired if item["role"] == "user"] + expect(bodies == ["Anfang", "Mitte des Tages, Markt", "Ende"], "middle user content stays") + expect(all(item["role"] in {"user", "assistant"} for item in paired), "only dialogue roles") + expect(paired[0]["role"] == "assistant" and paired[1]["body"] == "Anfang", "assistant kept only as reply context") + + +def test_conversation_boundary_pairing() -> None: + messages = [ + {"role": "user", "body": "Tag in Gespräch A", "conversation_id": "A", "seq": 1}, + {"role": "assistant", "body": "Frage aus A", "conversation_id": "A", "seq": 2}, + {"role": "user", "body": "Tag in Gespräch B", "conversation_id": "B", "seq": 1}, + ] + paired = pair_user_priority_messages(messages) + bodies = [item["body"] for item in paired] + expect("Tag in Gespräch A" in bodies and "Tag in Gespräch B" in bodies, "all user messages remain") + expect("Frage aus A" not in bodies, "trailing assistant from A is dropped at the conversation boundary") + expect(bodies.index("Tag in Gespräch A") < bodies.index("Tag in Gespräch B"), "conversation order stays stable") + + opening = [ + {"role": "assistant", "body": "Impuls A", "conversation_id": "A", "seq": 1}, + {"role": "user", "body": "Tag in Gespräch A", "conversation_id": "A", "seq": 2}, + {"role": "assistant", "body": "Frage aus A", "conversation_id": "A", "seq": 3}, + {"role": "assistant", "body": "Impuls B", "conversation_id": "B", "seq": 1}, + {"role": "user", "body": "Tag in Gespräch B", "conversation_id": "B", "seq": 2}, + ] + opened = pair_user_priority_messages(opening) + opened_bodies = [item["body"] for item in opened] + expect(opened_bodies == ["Impuls A", "Tag in Gespräch A", "Impuls B", "Tag in Gespräch B"], "opening assistant stays with its own user") + expect("Frage aus A" not in opened_bodies, "trailing A reply is still dropped after an opening") + for index, item in enumerate(opened): + if item["conversation_id"] == "B" and item["role"] == "user" and index: + prev = opened[index - 1] + expect( + not (prev["role"] == "assistant" and prev["conversation_id"] == "A"), + "A assistant is never glued onto B's user", + ) + + + +def test_provider_payload() -> None: + captured = {} + + class Response: + status_code = 200 + + def json(self): + return { + "model": "openai/gpt-4o", + "choices": [{"message": {"content": "ok"}}], + "usage": { + "prompt_tokens": 21, + "completion_tokens": 7, + "total_tokens": 28, + "cost": 0.0012, + }, + } + + def fake_post(url, json=None, headers=None, timeout=None): + captured["url"] = url + captured["json"] = json + return Response() + + remote = _openrouter_config() + with patch("providers.httpx.post", fake_post): + result = complete_chat( + remote, + [{"role": "user", "content": "ping"}], + timeout=5, + max_tokens=4096, + disable_context_compression=True, + ) + expect(isinstance(result, ChatResult), "complete_chat returns usage-aware result") + expect(captured["json"]["max_tokens"] == 4096, "journal sets max_tokens") + expect( + captured["json"]["plugins"] == [{"id": "context-compression", "enabled": False}], + "OpenRouter journal disables context compression", + ) + expect(result.usage.get("prompt_tokens") == 21, "usage prompt_tokens kept") + expect(result.usage.get("cost") == 0.0012, "usage cost kept") + expect(result.context_compression == "disabled", "compression flag recorded") + + local = ProviderConfig( + role="generate", + name="ollama", + mode="http", + url="http://127.0.0.1:11434/v1/chat/completions", + model="llama", + key="", + local=True, + zdr=True, + no_train=True, + ) + captured.clear() + with patch("providers.httpx.post", fake_post): + complete_chat( + local, + [{"role": "user", "content": "ping"}], + timeout=5, + max_tokens=4096, + disable_context_compression=True, + ) + expect("plugins" not in captured["json"], "local providers do not get OpenRouter plugins") + expect("provider" not in captured["json"], "local providers do not get OpenRouter provider block") + expect(is_openrouter(remote) is True, "openrouter url is detected") + expect(is_openrouter(local) is False, "localhost is not openrouter") + + class Reject: + status_code = 400 + text = "This endpoint's maximum context length is 8192 tokens" + + def json(self): + return {"error": {"message": self.text}} + + with patch("providers.httpx.post", lambda *args, **kwargs: Reject()): + try: + complete_chat(remote, [{"role": "user", "content": "x"}], timeout=5, max_tokens=4096) + raise SystemExit("FAIL: context-length reject should raise") + except Exception as exc: + expect(getattr(exc, "code", "") == "provider_context_length_rejected", "provider context reject is distinct") + + +def _ends_on_word_boundary(text: str) -> bool: + if not text: + return True + return text[-1].isalnum() is False or text.split()[-1].isalnum() + + +def test_task_brief_and_dedupe(client: TestClient, headers: dict, profile_id: str) -> None: + long_core = ( + "Lange Sätze mit konkreten Uhren und Orten, selten Pathos, oft ein trockener Schnitt. " + * 20 + ) + update_facet(profile_id, "core", value=long_core) + update_facet( + profile_id, + "autobiographical_journal", + value="Urlaubstagebücher bleiben chronologisch und nennen Zeiten, ohne den Core zu wiederholen.", + ) + statements = [ + ("rhythm", "Wechselt zwischen kurzen Schnitten und längeren Sätzen."), + ("detail", "Behält Uhren, Orte und kleine Gegenstände."), + ("chronology", "Erzählt in der Reihenfolge des Tages, ohne Rückblenden zu erfinden."), + ("lexicon", "Alltagswörter, wenig Schmuck, eigene Wiederholungen erlaubt."), + ("humor", "Gelegentlich trocken, nie aufgesetzt."), + ("transitions", "Kommt mit danach, später, irgendwann von Szene zu Szene."), + ] + for slug, statement in statements: + upsert_trait( + profile_id, + slug=slug, + facet_key="autobiographical_journal", + label=slug, + statement=statement, + origin="manual", + force=True, + ) + replace_trait_refs( + profile_id, + slug, + [ + { + "role": "exemplar", + "excerpt": f"Langes Beispiel für {slug} " + ("Wortfolge " * 40), + "occurred_at": "2026-08-24", + } + ], + ) + before = get_profile(profile_id) + brief = compile_task_brief(profile_id, "journal_generate") + after = get_profile(profile_id) + expect(len(brief) <= TASK_BRIEF_MAX_CHARS, f"task brief stays in budget ({len(brief)})") + expect(brief == brief.strip(), "task brief is not mid-trim whitespace") + expect(_ends_on_word_boundary(brief), "task brief does not cut inside a word") + for slug, statement in statements: + expect(slug in brief or statement.split(",")[0] in brief, f"relevant trait {slug} can be included") + expect(brief.count("Repräsentative Exemplare:") <= 1, "at most one exemplar section") + expect(len([line for line in brief.splitlines() if line.startswith("- ") and "Beispiel" not in line]) >= 6, "six traits considered") + expect(before["sources"] == after["sources"], "compile_task_brief does not mutate stored sources") + expect( + [(item.get("slug"), item.get("statement")) for item in before["traits"]] + == [(item.get("slug"), item.get("statement")) for item in after["traits"]], + "stored traits stay the source of truth", + ) + + body = "Am 24.08. war der Hafen ruhig, später der Markt." + import_text(profile_id, body, occurred_at="2026-08-24") + import_text(profile_id, body, occurred_at="2026-08-24") + imported = get_profile(profile_id) + same_day = [ + item + for item in imported["sources"] + if item.get("kind") == "imported_text" and (item.get("occurred_at") or "").startswith("2026-08-24") + ] + expect(len(same_day) >= 2, "duplicate originals remain stored") + display = imported.get("compiled_brief") or "" + expect(display.count("Hafen ruhig") == 1, "duplicate brief sources are not shown twice") + + +def test_generate_flow(client: TestClient, headers: dict) -> None: + import privacy_gateway + from privacy_gateway import install_test_recorder, reset_debug + + expect(not hasattr(privacy_gateway, "debug_history"), "production has no global prompt history") + space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"}) + day = client.post( + f"/api/journal/spaces/{space.json()['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-25"}, + ) + conv = client.post( + f"/api/journal/days/{day.json()['day']['id']}/conversations", + headers=headers, + json={"title": "Tag"}, + ) + turn = client.post( + f"/api/journal/conversations/{conv.json()['id']}/turn", + headers=headers, + json={"body": "Heute um 7:30 Uhr Markt, danach Kirschen. Vielleicht bleibe ich kürzer."}, + ) + expect(turn.status_code == 200, f"turn {turn.status_code}") + turn2 = client.post( + f"/api/journal/conversations/{conv.json()['id']}/turn", + headers=headers, + json={"body": "Später noch der Hafen, ungefähr 18:00 Uhr."}, + ) + expect(turn2.status_code == 200, f"second turn {turn2.status_code}") + reset_debug() + recorder = install_test_recorder() + gen = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(gen.status_code == 200, f"generate {gen.status_code}") + expect([item.get("purpose") for item in recorder] == ["journal_reconstruct", "journal_generate"], "both stages ran") + stages = {item.get("purpose"): item for item in (gen.json().get("trace") or {}).get("stages") or []} + expect("journal_reconstruct" in stages, "stage 1 ran through the gateway") + expect("journal_generate" in stages, "stage 2 ran through the gateway") + reconstruct = stages["journal_reconstruct"].get("intern") or "" + narrate = stages["journal_generate"].get("intern") or "" + expect("[u1]" in reconstruct and "user:" in reconstruct, "stage 1 sees labeled user sources") + expect("assistant:" in reconstruct, "stage 1 may see assistant as context") + expect("assistant:" not in narrate, "stage 2 does not resend assistant lines") + expect("user:" not in narrate, "stage 2 does not resend the raw dialogue") + expect("verified_artifact" in narrate and "source_order" in narrate, "stage 2 receives the local verified artifact") + expect("sources" in narrate, "stage 2 artifact contains rehydrated sources") + expect("7:30" in reconstruct or "7:30" in narrate, "times survive into generation") + expect("Markt" in narrate and "Hafen" in narrate, "full selected user sources reach stage 2") + expect("Vielleicht bleibe ich kürzer" in narrate, "omitted labels do not drop canonical hedges") + expect("assistant:" not in (gen.json().get("body") or ""), "draft body has no assistant lines") + + reset_debug() + short_recorder = install_test_recorder() + with patch( + "privacy_gateway.fake_reconstruction", + lambda rendered: json.dumps( + { + "source_order": ["u1", "u2"], + "chronology": [ + { + "source_id": "u1", + "source": "user", + "time": None, + "claims": [{"kind": "feeling", "evidence": "Heute"}], + }, + { + "source_id": "u2", + "source": "user", + "time": None, + "claims": [{"kind": "event", "evidence": "Später"}], + }, + ], + "contradictions": [], + "uncertainties": [], + } + ), + ): + short_gen = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(short_gen.status_code == 200, f"short-claim generate {short_gen.status_code}") + expect( + [item.get("purpose") for item in short_recorder] == ["journal_reconstruct", "journal_generate"], + "short claims still run both stages because sources are rehydrated", + ) + short_narrate = "" + for item in (short_gen.json().get("trace") or {}).get("stages") or []: + if item.get("purpose") == "journal_generate": + short_narrate = item.get("intern") or "" + expect("Kirschen" in short_narrate and "Hafen" in short_narrate, "short claims cannot drop the rest of the sources") + expect("\"feeling\"" not in short_narrate, "feeling labels do not reach stage 2 as facts") + budget = (gen.json().get("trace") or {}).get("budget") or stages["journal_generate"].get("budget") or {} + expect(budget.get("reserved_output_tokens") == 4096, "trace records reserved output") + expect(budget.get("context_compression") == "disabled", "trace records compression off") + expect(budget.get("budget_ok") is True, "trace records budget ok") + expect("rendered" not in budget and "masked" not in budget, "budget diagnostics omit prompt bodies") + expect((gen.json().get("trace") or {}).get("stages"), "admin trace has both stages") + + reset_debug() + skip_recorder = install_test_recorder() + with patch( + "privacy_gateway.fake_reconstruction", + lambda rendered: json.dumps( + { + "source_order": ["u1", "u2"], + "chronology": [ + { + "source_id": "u1", + "source": "user", + "time": "7:30 Uhr", + "claims": [ + {"kind": "event", "evidence": "Heute um 7:30 Uhr Markt"}, + {"kind": "uncertainty", "evidence": "Vielleicht bleibe ich kürzer."}, + ], + } + ], + "contradictions": [], + "uncertainties": [{"source_id": "u1", "evidence": "Vielleicht bleibe ich kürzer."}], + } + ), + ): + denied_recon = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(denied_recon.status_code == 200, f"incomplete reconstruct falls back locally {denied_recon.text}") + fallback_body = denied_recon.json().get("body") or "" + expect("Markt" in fallback_body or "Hafen" in fallback_body, "local artifact still produces a draft") + fallback_stages = {item.get("purpose"): item for item in (denied_recon.json().get("trace") or {}).get("stages") or []} + expect(fallback_stages.get("journal_reconstruct", {}).get("stage1") == "local_fallback", "invalid stage 1 is marked local_fallback") + expect( + (fallback_stages.get("journal_reconstruct") or {}).get("reason") == "incomplete_coverage", + "fallback records why the model JSON was rejected", + ) + expect("journal_generate" in fallback_stages, "stage 2 still runs on the local artifact") + narrate_fallback = fallback_stages.get("journal_generate", {}).get("intern") or "" + expect("Heute um 7:30 Uhr Markt" in narrate_fallback, "first user source is rehydrated locally") + expect("Später noch der Hafen" in narrate_fallback, "omitted model source is still in the local artifact") + expect( + [item.get("purpose") for item in skip_recorder] == ["journal_reconstruct", "journal_generate"], + "stage 2 is called with local coverage after invalid reconstruction", + ) + + reset_catalog() + set_metadata_override( + "fake", + ModelWindow( + model="fake", + context_length=8192, + max_completion_tokens=4096, + source="test", + provider="fake", + ), + ) + denied = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(denied.status_code == 422, f"8K generate is refused {denied.text}") + detail = denied.json().get("detail") or {} + expect(detail.get("code") == ERROR_MODEL_CONTEXT_TOO_SMALL, "8K refusal uses model_context_too_small") + expect(detail.get("diagnostics"), "refusal carries diagnostics") + reset_catalog() + + +def test_generate_identity_leak_local_fallback(client: TestClient, headers: dict, profile_id: str) -> None: + from identity_store import remember_mapping + from providers import ChatResult + + remember_mapping(profile_id, "Anna", "PERSON:01") + space = client.post("/api/journal/spaces", headers=headers, json={"title": "Identität"}) + day = client.post( + f"/api/journal/spaces/{space.json()['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-26"}, + ) + conv = client.post( + f"/api/journal/days/{day.json()['day']['id']}/conversations", + headers=headers, + json={"title": "Tag"}, + ) + turn = client.post( + f"/api/journal/conversations/{conv.json()['id']}/turn", + headers=headers, + json={"body": "Heute war ich mit Anna am Markt, danach Kirschen."}, + ) + expect(turn.status_code == 200, f"identity-leak setup turn {turn.status_code}") + + def leak(_messages, _policy): + return ChatResult( + content="Anna stand den ganzen Nachmittag am Markt.", + model="fake", + usage={}, + context_compression="disabled", + ) + + with patch("privacy_gateway.complete_model", leak): + gen = client.post( + f"/api/journal/days/{day.json()['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [conv.json()["id"]]}, + ) + expect(gen.status_code == 200, f"identity leak still yields a local draft {gen.text}") + payload = gen.json() + body = payload.get("body") or "" + expect("Markt" in body and "Kirschen" in body, "local draft keeps attested user wording") + expect("Anna" in body, "local draft may keep names that the user actually wrote") + stages = {item.get("purpose"): item for item in (payload.get("trace") or {}).get("stages") or []} + reconstruct = stages.get("journal_reconstruct") or {} + narrate = stages.get("journal_generate") or {} + expect( + reconstruct.get("reason") == "identity_leak_blocked" or reconstruct.get("guard") == "identity_leak_blocked", + "stage 1 records the blocked leak", + ) + expect(narrate.get("guard") == "identity_leak_blocked", "stage 2 does not use the leaking reply") + + +def test_traces_are_request_scoped(client: TestClient, headers: dict) -> None: + from journal_generate import generate_draft + + other = client.post( + "/api/users", + headers=headers, + json={"email": "second@example.test", "name": "Second", "password": "test-pass", "role": "user"}, + ) + expect(other.status_code == 200, f"second profile {other.text}") + login = client.post("/api/auth/login", json={"email": "second@example.test", "password": "test-pass"}) + other_headers = header(login.json()["token"]) + profile_b = login.json()["profile_id"] + + space_a = client.post("/api/journal/spaces", headers=headers, json={"title": "Profil A"}) + day_a = client.post( + f"/api/journal/spaces/{space_a.json()['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-26"}, + ) + conv_a = client.post( + f"/api/journal/days/{day_a.json()['day']['id']}/conversations", + headers=headers, + json={"title": "A"}, + ) + token_a = "UNIQUA-ALPHA-KIRSCHEN-9921" + client.post( + f"/api/journal/conversations/{conv_a.json()['id']}/turn", + headers=headers, + json={"body": f"Heute {token_a} am Markt."}, + ) + + space_b = client.post("/api/journal/spaces", headers=other_headers, json={"title": "Profil B"}) + day_b = client.post( + f"/api/journal/spaces/{space_b.json()['id']}/days", + headers=other_headers, + json={"calendar_date": "2026-08-26"}, + ) + conv_b = client.post( + f"/api/journal/days/{day_b.json()['day']['id']}/conversations", + headers=other_headers, + json={"title": "B"}, + ) + token_b = "UNIQUB-BETA-HAFEN-7744" + client.post( + f"/api/journal/conversations/{conv_b.json()['id']}/turn", + headers=other_headers, + json={"body": f"Heute {token_b} am Hafen."}, + ) + + draft_a = generate_draft( + client.get("/api/auth/me", headers=headers).json()["id"], + day_a.json()["day"]["id"], + [conv_a.json()["id"]], + ) + draft_b = generate_draft(profile_b, day_b.json()["day"]["id"], [conv_b.json()["id"]]) + intern_a = " ".join(stage.get("intern") or "" for stage in (draft_a.get("trace") or {}).get("stages") or []) + intern_b = " ".join(stage.get("intern") or "" for stage in (draft_b.get("trace") or {}).get("stages") or []) + expect(token_a in intern_a, "profile A reconstruct contains its own text") + expect(token_b not in intern_a, "profile A trace is not mixed with profile B") + expect(token_b in intern_b, "profile B reconstruct contains its own text") + expect(token_a not in intern_b, "profile B trace is not mixed with profile A") + + +def test_two_conversations_same_day(client: TestClient, headers: dict) -> None: + from retrieval import format_day_messages, retrieve + + space = client.post("/api/journal/spaces", headers=headers, json={"title": "Zwei Gespräche"}) + day = client.post( + f"/api/journal/spaces/{space.json()['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-27"}, + ) + conv_a = client.post( + f"/api/journal/days/{day.json()['day']['id']}/conversations", + headers=headers, + json={"title": "A"}, + ) + client.post( + f"/api/journal/conversations/{conv_a.json()['id']}/turn", + headers=headers, + json={"body": "Nur in Gespräch A: der Vormittag."}, + ) + conv_b = client.post( + f"/api/journal/days/{day.json()['day']['id']}/conversations", + headers=headers, + json={"title": "B"}, + ) + client.post( + f"/api/journal/conversations/{conv_b.json()['id']}/turn", + headers=headers, + json={"body": "Nur in Gespräch B: der Nachmittag."}, + ) + profile_id = client.get("/api/auth/me", headers=headers).json()["id"] + messages = retrieve( + profile_id, + { + "kind": "day_messages", + "journal_day_id": day.json()["day"]["id"], + "conversation_ids": [conv_a.json()["id"], conv_b.json()["id"]], + "overflow": "abort", + "max_estimated_tokens": 20_000, + }, + ) + formatted = format_day_messages(messages, with_source_ids=True) + expect("Nur in Gespräch A" in formatted and "Nur in Gespräch B" in formatted, "both conversations remain") + id_a = conv_a.json()["id"] + id_b = conv_b.json()["id"] + a_idx = [index for index, item in enumerate(messages) if item.get("conversation_id") == id_a] + b_idx = [index for index, item in enumerate(messages) if item.get("conversation_id") == id_b] + user_b = [index for index, item in enumerate(messages) if item.get("conversation_id") == id_b and item.get("role") == "user"] + expect(user_b, "conversation B user message is present") + expect(a_idx and b_idx and max(a_idx) < min(b_idx), "conversation A stays entirely before B") + a_kept = [item for item in messages if item.get("conversation_id") == id_a] + expect(a_kept and a_kept[-1]["role"] == "user", "trailing assistant from A is dropped; last kept A line is the user") + for index in user_b: + if index == 0: + continue + prev = messages[index - 1] + expect( + not (prev.get("role") == "assistant" and prev.get("conversation_id") == id_a), + "assistant from A is not glued to user text from B", + ) + + +def main() -> None: + test_budget_math() + test_reconstruction_rules() + test_reconstruction_claim_and_time_binding() + test_reconstruction_completeness_and_labels() + test_local_fallback_covers_users() + test_day_messages_keep_middle() + test_conversation_boundary_pairing() + test_provider_payload() + reset_catalog() + with TestClient(app) as client: + setup = client.post( + "/api/auth/setup", + json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"}, + ) + headers = header(setup.json()["token"]) + profile_id = setup.json()["profile_id"] + test_task_brief_and_dedupe(client, headers, profile_id) + test_generate_flow(client, headers) + test_generate_identity_leak_local_fallback(client, headers, profile_id) + test_traces_are_request_scoped(client, headers) + test_two_conversations_same_day(client, headers) + print("journal budget tests passed.") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_journal_opening.py b/backend/tests/test_journal_opening.py new file mode 100644 index 0000000..6301598 --- /dev/null +++ b/backend/tests/test_journal_opening.py @@ -0,0 +1,257 @@ +"""First journal impulse invariants. Run from backend/: python tests/test_journal_opening.py""" +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-opening-test.sqlite") +os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-opening-media") +os.environ["KANSHO_FAKE_PROVIDER"] = "1" +os.environ["KANSHO_FAKE_DETECT"] = "1" +Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) +Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True) + +from fastapi.testclient import TestClient +from main import app +from journal_opening import ( + NEUTRAL_OPENING, + attested_plans_from_user_texts, + collect_opening_context, + is_attested_plan, +) +from journal_reconstruct import assign_source_ids +from retrieval import pair_user_priority_messages + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def header(token: str) -> dict: + return {"X-Auth-Token": token} + + +def open_space_day(client, headers, title, date): + space = client.post("/api/journal/spaces", headers=headers, json={"title": title}) + day = client.post( + f"/api/journal/spaces/{space.json()['id']}/days", + headers=headers, + json={"calendar_date": date}, + ) + return space.json(), day.json() + + +def start_conv(client, headers, day_id): + return client.post(f"/api/journal/days/{day_id}/conversations", headers=headers, json={"title": "Gespräch"}) + + +def main() -> None: + expect(is_attested_plan("Morgen wollen wir nach Cres fahren."), "future intent is a plan") + expect(is_attested_plan("Wir haben vor, später an den Hafen zu gehen."), "vorhaben is a plan") + expect(not is_attested_plan("Heute war der Hafen ruhig. Kirschen am Stand."), "past report is not a plan") + expect(not is_attested_plan("Wir standen vor dem Hafen und aßen Kirschen."), "vor without haben is not a plan") + expect(not is_attested_plan("Hafen Kirschen Wind"), "word overlap is not a plan") + plans = attested_plans_from_user_texts( + ["Heute war der Hafen ruhig.", "Morgen wollen wir nach Cres fahren.", "Hafen Kirschen"] + ) + expect(len(plans) == 1 and "Cres" in plans[0], "only attested plan sentences are kept") + expect( + not any(is_attested_plan(text) for text in ["Was davon möchtest du festhalten?", "Die Fahrt nach Cres war schön."]), + "assistant-like completion is not treated as a user plan by the helper", + ) + + with TestClient(app) as client: + setup = client.post("/api/auth/setup", json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"}) + headers = header(setup.json()["token"]) + other = client.post( + "/api/users", + headers=headers, + json={"email": "ute@example.test", "name": "Ute", "password": "user-pass", "role": "user"}, + ) + expect(other.status_code == 200, "second profile") + other_headers = header( + client.post("/api/auth/login", json={"email": "ute@example.test", "password": "user-pass"}).json()["token"] + ) + + empty_space, empty_day = open_space_day(client, headers, "Leer", "2026-08-21") + empty = start_conv(client, headers, empty_day["day"]["id"]) + expect(empty.status_code == 200, f"empty opening {empty.text}") + expect(empty.json().get("opening", {}).get("kind") == "local_neutral", "no context uses local neutral") + expect(empty.json()["opening"]["assistant"]["body"] == NEUTRAL_OPENING, "neutral impulse stored") + expect(empty.json()["opening"]["calls"] == 0, "neutral opening does not call a model") + empty_msgs = empty.json().get("messages") or [] + expect(len(empty_msgs) == 1 and empty_msgs[0]["role"] == "assistant", "neutral opening is assistant-first") + + plan_space, plan_day = open_space_day(client, headers, "Urlaub", "2026-08-01") + plan_conv = start_conv(client, headers, plan_day["day"]["id"]) + client.post( + f"/api/journal/conversations/{plan_conv.json()['id']}/turn", + headers=headers, + json={"body": "Morgen wollen wir nach Cres fahren. Die Fähre ist schon bedacht."}, + ) + later = client.post( + f"/api/journal/spaces/{plan_space['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-02"}, + ) + planned = start_conv(client, headers, later.json()["day"]["id"]) + facts = collect_opening_context(setup.json()["profile_id"], planned.json()["id"]) + expect(facts["has_relevant_context"], "attested plan counts as relevant context") + expect(any("Cres" in item and is_attested_plan(item) for item in facts["attested_plans"]), "plan comes from user wording") + expect(all(item in facts["user_texts"] for item in facts["attested_plans"]), "plans are a subset of user texts") + expect(planned.json().get("opening", {}).get("kind") == "model", "attested plan uses the dialogue path") + intern = (planned.json().get("opening") or {}).get("trace") or {} + intern_text = intern.get("intern") or "" + expect("Belegte Vorhaben" in intern_text, "opening policy reaches the prompt") + expect("kein bewiesenes wiederkehrendes Muster" in intern_text or "kein bewiesenes" in intern_text, "pattern rule is in the opening hint") + + thread_space, thread_day = open_space_day(client, headers, "Tagfaden", "2026-08-10") + first = start_conv(client, headers, thread_day["day"]["id"]) + client.post( + f"/api/journal/conversations/{first.json()['id']}/turn", + headers=headers, + json={"body": "Morgens am Markt, danach noch offen, wie der Nachmittag wird."}, + ) + second = start_conv(client, headers, thread_day["day"]["id"]) + thread_facts = collect_opening_context(setup.json()["profile_id"], second.json()["id"]) + expect(thread_facts["open_day_points"], "same-day user text is an open point") + expect( + any("Markt" in item or "Nachmittag" in item for item in thread_facts["open_day_points"]), + "open point is the day's user wording", + ) + expect(second.json().get("opening", {}).get("kind") == "model", "open day thread uses the dialogue path") + + overlap_space, overlap_day = open_space_day(client, headers, "Overlap", "2026-08-03") + overlap_conv = start_conv(client, headers, overlap_day["day"]["id"]) + client.post( + f"/api/journal/conversations/{overlap_conv.json()['id']}/turn", + headers=headers, + json={"body": "Heute war der Hafen ruhig. Kirschen am Stand, Wind von See."}, + ) + client.post( + "/api/journal/entries", + headers=headers, + json={ + "journal_day_id": overlap_day["day"]["id"], + "title": "Hafen", + "body": "Heute war der Hafen ruhig. Kirschen am Stand.", + "origin": "user_edit", + }, + ) + later_overlap = client.post( + f"/api/journal/spaces/{overlap_space['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-04"}, + ) + overlap_open = start_conv(client, headers, later_overlap.json()["day"]["id"]) + overlap_facts = collect_opening_context(setup.json()["profile_id"], overlap_open.json()["id"]) + expect(not overlap_facts["attested_plans"], "random overlap is not an attested plan") + expect(not overlap_facts["has_relevant_context"], "recency without plan or open day is not relevant opening context") + expect(overlap_open.json().get("opening", {}).get("kind") == "local_neutral", "overlap falls back to neutral") + expect("Muster" not in (overlap_open.json().get("opening") or {}).get("assistant", {}).get("body", ""), "neutral impulse does not claim a pattern") + + asst_space, asst_day = open_space_day(client, headers, "Assistent", "2026-08-05") + asst_conv = start_conv(client, headers, asst_day["day"]["id"]) + from dialogue_store import append_message + + append_message( + setup.json()["profile_id"], + asst_conv.json()["id"], + "Morgen wollen wir nach Cres fahren.", + role="assistant", + ) + next_asst = client.post( + f"/api/journal/spaces/{asst_space['id']}/days", + headers=headers, + json={"calendar_date": "2026-08-06"}, + ) + asst_open = start_conv(client, headers, next_asst.json()["day"]["id"]) + asst_facts = collect_opening_context(setup.json()["profile_id"], asst_open.json()["id"]) + expect(not asst_facts["attested_plans"], "assistant wording is not a user plan") + expect("Cres" not in " ".join(asst_facts["attested_plans"]), "assistant plan sentence stays out of attested plans") + expect(asst_facts["assistant_is_not_user_fact"], "adapter marks assistant as non-fact") + + pair_space, pair_day = open_space_day(client, headers, "Paarung", "2026-08-07") + pair_conv = start_conv(client, headers, pair_day["day"]["id"]) + expect((pair_conv.json().get("messages") or [])[0]["role"] == "assistant", "assistant-first stored") + user_turn = client.post( + f"/api/journal/conversations/{pair_conv.json()['id']}/turn", + headers=headers, + json={"body": "Nach der Fähre kamen wir spät an. Anna kaufte Brot."}, + ) + expect(user_turn.status_code == 200, "user turn after opening") + msgs = user_turn.json()["messages"] + users = [item for item in msgs if item["role"] == "user"] + expect(len(users) == 1, "exactly one user source after opening plus turn") + paired = pair_user_priority_messages(msgs) + paired_users = [item for item in paired if item["role"] == "user"] + expect([item["body"] for item in paired_users] == [item["body"] for item in users], "pairing keeps every user source") + labeled = assign_source_ids(paired) + expect( + [item["source_id"] for item in labeled if item["role"] == "user"] == ["u1"], + "first user source is u1 even after an opening assistant", + ) + draft = client.post( + f"/api/journal/days/{pair_day['day']['id']}/generate", + headers=headers, + json={"conversation_ids": [pair_conv.json()["id"]]}, + ) + expect(draft.status_code == 200, f"generate after opening {draft.text}") + reconstruct = None + for stage in (draft.json().get("trace") or {}).get("stages") or []: + if stage.get("purpose") == "journal_reconstruct": + reconstruct = stage + intern = (reconstruct or {}).get("intern") or "" + expect("Anna kaufte Brot" in intern, "user source reaches reconstruction") + expect("[u1]" in intern, "user source id assigned") + narrate = None + for stage in (draft.json().get("trace") or {}).get("stages") or []: + if stage.get("purpose") == "journal_generate": + narrate = stage + narrate_intern = (narrate or {}).get("intern") or "" + expect("verified_artifact" in narrate_intern or "sources" in narrate_intern, "stage 2 consumes verified artifact") + expect("Nach der Fähre kamen wir spät an" in intern, "user wording not dropped after opening") + expect(users[0]["body"].startswith("Nach der Fähre"), "user source is the user message, not the opening") + + stolen = client.post( + f"/api/journal/days/{empty_day['day']['id']}/conversations", + headers=other_headers, + json={"title": "fremd"}, + ) + expect(stolen.status_code == 404, "foreign profile cannot open a conversation") + stolen_turn = client.post( + f"/api/journal/conversations/{empty.json()['id']}/turn", + headers=other_headers, + json={"body": "Hallo"}, + ) + expect(stolen_turn.status_code == 404, "foreign profile cannot turn into the opening conversation") + user_login_space = client.post("/api/journal/spaces", headers=other_headers, json={"title": "Ute"}) + user_day = client.post( + f"/api/journal/spaces/{user_login_space.json()['id']}/days", + headers=other_headers, + json={"calendar_date": "2026-08-21"}, + ) + user_open = start_conv(client, other_headers, user_day.json()["day"]["id"]) + expect(user_open.status_code == 200, "non-admin can start their own day") + expect("trace" not in (user_open.json().get("opening") or {}), "non-admin opening has no trace") + expect("opening_context" not in (user_open.json().get("opening") or {}), "non-admin opening has no opening_context") + expect("trace" not in user_open.json(), "non-admin response has no top-level trace") + + start = client.get("/api/journal/continuity", headers=headers) + expect(start.status_code == 200 and start.json().get("continuable"), "continuable day for the author") + expect(start.json()["continuable"]["space_id"] == start.json()["continuable"]["space_id"], "continuity names a space") + other_start = client.get("/api/journal/continuity", headers=other_headers) + expect(other_start.json()["continuable"]["space_id"] == user_login_space.json()["id"], "continuity is profile-scoped") + stolen_cont = client.get(f"/api/journal/spaces/{plan_space['id']}", headers=other_headers) + expect(stolen_cont.status_code == 404, "continuity space stays isolated") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_journal_trash.py b/backend/tests/test_journal_trash.py new file mode 100644 index 0000000..41401d4 --- /dev/null +++ b/backend/tests/test_journal_trash.py @@ -0,0 +1,155 @@ +"""Journal trash, undelete, and hard delete. Run from backend/: python tests/test_journal_trash.py""" +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-trash-test.sqlite") +os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-trash-media") +os.environ["KANSHO_FAKE_PROVIDER"] = "1" +os.environ["KANSHO_FAKE_DETECT"] = "1" +Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) +Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True) + +from fastapi.testclient import TestClient +from main import app +from media_store import media_root + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def header(token: str) -> dict: + return {"X-Auth-Token": token} + + +PNG = bytes.fromhex( + "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489" + "0000000a49444154789c6360000002000100ffff03000006000557bf0000000049454e44ae426082" +) + + +def main() -> None: + with TestClient(app) as client: + setup = client.post("/api/auth/setup", json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"}) + headers = header(setup.json()["token"]) + other = client.post( + "/api/users", + headers=headers, + json={"email": "ute@example.test", "name": "Ute", "password": "user-pass", "role": "user"}, + ) + expect(other.status_code == 200, "second profile") + other_headers = header(client.post("/api/auth/login", json={"email": "ute@example.test", "password": "user-pass"}).json()["token"]) + + space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"}) + space_id = space.json()["id"] + day = client.post(f"/api/journal/spaces/{space_id}/days", headers=headers, json={"calendar_date": "2026-08-20"}) + day_id = day.json()["day"]["id"] + conv = client.post(f"/api/journal/days/{day_id}/conversations", headers=headers, json={"title": "Gespräch"}) + client.post(f"/api/journal/conversations/{conv.json()['id']}/turn", headers=headers, json={"body": "Heute war der Markt voll."}) + saved = client.post( + "/api/journal/entries", + headers=headers, + json={"journal_day_id": day_id, "title": "Markt", "body": "Heute war der Markt voll.", "origin": "user_edit"}, + ) + entry_id = saved.json()["id"] + second = client.post( + "/api/journal/entries", + headers=headers, + json={"journal_day_id": day_id, "title": "Abend", "body": "Später am Hafen.", "origin": "user_edit"}, + ) + second_id = second.json()["id"] + media = client.post( + f"/api/journal/entries/{entry_id}/media", + headers=headers, + files={"file": ("shot.png", PNG, "image/png")}, + ) + expect(media.status_code == 200, "media attached") + media_id = media.json()["id"] + rel = media.json()["rel_path"] + expect((media_root() / rel).is_file(), "media file exists") + + listed = client.get(f"/api/journal/spaces/{space_id}/entries", headers=headers).json() + expect(len(listed) == 2, "two active entries") + expect(listed[0]["calendar_date"] == "2026-08-20", "same day") + expect(listed[0]["id"] != listed[1]["id"], "same-day entries are distinct") + expect(all(item["id"] in {entry_id, second_id} for item in listed), "list is the two entries") + + hidden_list = client.get(f"/api/journal/spaces/{space_id}/entries", headers=other_headers) + expect(hidden_list.status_code == 404, "entry list isolated") + hidden_trash = client.get(f"/api/journal/spaces/{space_id}/trash", headers=other_headers) + expect(hidden_trash.status_code == 404, "trash list isolated") + + deleted = client.delete(f"/api/journal/entries/{entry_id}", headers=headers) + expect(deleted.status_code == 200, "soft delete") + after_soft = client.get(f"/api/journal/spaces/{space_id}/entries", headers=headers).json() + expect(all(item["id"] != entry_id for item in after_soft), "soft-deleted entry leaves chronology") + expect(any(item["id"] == second_id for item in after_soft), "other entry remains") + trash = client.get(f"/api/journal/spaces/{space_id}/trash", headers=headers).json() + expect(any(item["id"] == entry_id for item in trash), "soft-deleted appears in trash") + missing = client.get(f"/api/journal/entries/{entry_id}", headers=headers) + expect(missing.status_code == 404, "deleted entry is not a normal GET") + + stolen_delete = client.delete(f"/api/journal/entries/{second_id}", headers=other_headers) + expect(stolen_delete.status_code == 404, "foreign profile cannot soft-delete") + stolen_undelete = client.post(f"/api/journal/entries/{entry_id}/undelete", headers=other_headers, json={}) + expect(stolen_undelete.status_code == 404, "foreign profile cannot undelete") + stolen_purge = client.post(f"/api/journal/entries/{entry_id}/purge", headers=other_headers, json={"confirm": True}) + expect(stolen_purge.status_code == 404, "foreign profile cannot purge") + + version_restore = client.post( + f"/api/journal/entries/{entry_id}/restore", + headers=headers, + json={"version_id": saved.json()["current_version_id"]}, + ) + expect(version_restore.status_code == 404, "version restore is not trash undelete") + + back = client.post(f"/api/journal/entries/{entry_id}/undelete", headers=headers, json={}) + expect(back.status_code == 200, f"undelete {back.text}") + expect(back.json().get("deleted_at") in (None, ""), "undelete clears deleted_at") + restored_list = client.get(f"/api/journal/spaces/{space_id}/entries", headers=headers).json() + expect(any(item["id"] == entry_id for item in restored_list), "undeleted entry returns to chronology") + expect(client.get(f"/api/journal/spaces/{space_id}/trash", headers=headers).json() == [], "trash empty after undelete") + versions = client.get(f"/api/journal/entries/{entry_id}/versions", headers=headers).json() + expect(len(versions) == 1, "undelete does not append a version") + + client.delete(f"/api/journal/entries/{entry_id}", headers=headers) + unconfirmed = client.post(f"/api/journal/entries/{entry_id}/purge", headers=headers, json={"confirm": False}) + expect(unconfirmed.status_code == 400, "purge without confirm fails closed") + still = client.get(f"/api/journal/spaces/{space_id}/trash", headers=headers).json() + expect(any(item["id"] == entry_id for item in still), "unconfirmed purge leaves the entry") + + active_purge = client.post(f"/api/journal/entries/{second_id}/purge", headers=headers, json={"confirm": True}) + expect(active_purge.status_code == 400, "purge of active entry is rejected") + + conv_before = client.get(f"/api/journal/conversations/{conv.json()['id']}", headers=headers) + expect(conv_before.status_code == 200, "source dialogue still there before purge") + purged = client.post(f"/api/journal/entries/{entry_id}/purge", headers=headers, json={"confirm": True}) + expect(purged.status_code == 200 and purged.json().get("purged"), f"purge {purged.text}") + expect(purged.json().get("conversations_untouched") is True, "source dialogues are not deleted") + expect(client.get(f"/api/journal/conversations/{conv.json()['id']}", headers=headers).status_code == 200, "dialogue remains") + expect(client.get(f"/api/journal/entries/{entry_id}", headers=headers).status_code == 404, "purged entry gone") + expect(client.get(f"/api/journal/spaces/{space_id}/trash", headers=headers).json() == [], "purged entry leaves trash") + expect(not (media_root() / rel).exists(), "unreferenced media file removed") + expect(client.get(f"/api/journal/media/{media_id}", headers=headers).status_code == 404, "media row gone") + + again = client.post(f"/api/journal/entries/{entry_id}/purge", headers=headers, json={"confirm": True}) + expect(again.status_code == 404, "second purge of same id is not found") + again_undelete = client.post(f"/api/journal/entries/{entry_id}/undelete", headers=headers, json={}) + expect(again_undelete.status_code == 404, "cannot undelete a purged entry") + + leftover = client.get(f"/api/journal/spaces/{space_id}/entries", headers=headers).json() + expect(all(item["id"] != entry_id for item in leftover), "purged id absent from chronology") + expect(any(item["id"] == second_id for item in leftover), "untouched entry remains") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_local_backup.py b/backend/tests/test_local_backup.py new file mode 100644 index 0000000..5ac2c94 --- /dev/null +++ b/backend/tests/test_local_backup.py @@ -0,0 +1,161 @@ +"""Local backup/restore. Run from backend/: python tests/test_local_backup.py""" +from __future__ import annotations + +import json +import os +import sqlite3 +import sys +import tempfile +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from local_backup import BackupError, create_backup, restore_backup, verify_archive + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def write_db(path: Path, value: str) -> None: + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE IF NOT EXISTS note (id INTEGER PRIMARY KEY, body TEXT)") + conn.execute("DELETE FROM note") + conn.execute("INSERT INTO note (body) VALUES (?)", (value,)) + conn.commit() + conn.close() + + +def read_db(path: Path) -> str: + conn = sqlite3.connect(path) + row = conn.execute("SELECT body FROM note").fetchone() + conn.close() + return row[0] if row else "" + + +def main() -> None: + with tempfile.TemporaryDirectory(prefix="kansho-backup-test-") as raw: + base = Path(raw) + db = base / "live" / "kansho.sqlite" + media = base / "live" / "media" + out = base / "archives" + db.parent.mkdir(parents=True) + media.mkdir(parents=True) + write_db(db, "original-body") + sample = media / "profile" / "shot.png" + sample.parent.mkdir(parents=True) + sample.write_bytes(b"png-bytes") + secret = base / "live" / ".env" + secret.write_text("KANSHO_PROVIDER_KEY=sk-secret\n", encoding="utf-8") + + archive = out / "first.zip" + created = create_backup(archive, db_path=db, media_root=media) + expect(archive.is_file(), "backup file created") + expect(created["manifest"]["kind"] == "kansho.local_backup", "manifest kind") + expect("persönliche, unverschlüsselte" in created["manifest"]["warning"].lower() or "unverschl" in created["manifest"]["warning"], "personal-data warning") + verified = verify_archive(archive) + expect(verified["manifest"]["checksums"]["kansho.sqlite"], "sqlite checksum present") + with zipfile.ZipFile(archive) as zf: + names = zf.namelist() + expect("kansho.sqlite" in names, "sqlite in archive") + expect("media/profile/shot.png" in names, "media in archive") + expect(".env" not in names, ".env excluded") + expect(all("sk-secret" not in zf.read(name).decode("latin1", errors="ignore") for name in names if name.endswith(".json") or name.endswith(".txt")), "no secret in manifest/readme") + + try: + restore_backup(archive, db_path=db, media_root=media, confirm=False) + expect(False, "restore without confirm must fail") + except BackupError as exc: + expect(exc.code == "confirm_required", "restore requires explicit confirm") + + try: + restore_backup(archive, db_path=db, media_root=media, confirm=True, allow_overwrite=False) + expect(False, "restore must not silently overwrite") + except BackupError as exc: + expect(exc.code == "would_overwrite", "restore refuses existing data without replace") + + target_db = base / "restore" / "kansho.sqlite" + target_media = base / "restore" / "media" + restored = restore_backup( + archive, + db_path=target_db, + media_root=target_media, + confirm=True, + allow_overwrite=False, + safety_dir=out, + ) + expect(read_db(target_db) == "original-body", "restored sqlite content") + expect((target_media / "profile" / "shot.png").read_bytes() == b"png-bytes", "restored media") + expect(restored["safety_backup"] is None, "empty target needs no safety backup") + + write_db(target_db, "changed-live") + (target_media / "profile" / "extra.bin").write_bytes(b"extra") + replaced = restore_backup( + archive, + db_path=target_db, + media_root=target_media, + confirm=True, + allow_overwrite=True, + safety_dir=out, + ) + expect(read_db(target_db) == "original-body", "replace restores snapshot") + expect(not (target_media / "profile" / "extra.bin").exists(), "extra live media not kept") + expect(Path(replaced["safety_backup"]).is_file(), "safety backup written before replace") + safety = verify_archive(Path(replaced["safety_backup"])) + expect(safety["manifest"]["kind"] == "kansho.local_backup", "safety backup is a real archive") + + broken = out / "broken.zip" + with zipfile.ZipFile(archive) as src, zipfile.ZipFile(broken, "w") as dest: + for info in src.infolist(): + data = src.read(info.filename) + if info.filename == "kansho.sqlite": + data = data + b"tamper" + dest.writestr(info, data) + before = read_db(target_db) + try: + restore_backup(broken, db_path=target_db, media_root=target_media, confirm=True, allow_overwrite=True) + expect(False, "tampered archive must fail") + except BackupError as exc: + expect(exc.code == "checksum_mismatch", "tamper detected before restore") + expect(read_db(target_db) == before, "failed restore leaves current db") + + extra_zip = out / "extra.zip" + with zipfile.ZipFile(archive) as src, zipfile.ZipFile(extra_zip, "w") as dest: + for info in src.infolist(): + dest.writestr(info, src.read(info.filename)) + dest.writestr("notes.txt", "not in manifest") + extra_verified = verify_archive(extra_zip) + expect("notes.txt" in extra_verified["skipped_extra"], "extra archive files are skipped deterministically") + + locked_db = base / "locked" / "kansho.sqlite" + locked_media = base / "locked" / "media" + locked_db.parent.mkdir() + locked_media.mkdir() + write_db(locked_db, "locked") + holder = sqlite3.connect(str(locked_db), timeout=1) + holder.execute("BEGIN EXCLUSIVE") + try: + restore_backup( + archive, + db_path=locked_db, + media_root=locked_media, + confirm=True, + allow_overwrite=True, + safety_dir=out, + ) + expect(False, "in-use db must abort") + except BackupError as exc: + expect(exc.code in {"db_in_use", "db_unavailable", "backup_exists", "would_overwrite"} or "in_use" in exc.code or "unavailable" in exc.code, f"busy db fail-closed ({exc.code})") + finally: + holder.rollback() + holder.close() + + expect(True, "backup tests used only temporary directories") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_model_catalog.py b/backend/tests/test_model_catalog.py new file mode 100644 index 0000000..d8d9de1 --- /dev/null +++ b/backend/tests/test_model_catalog.py @@ -0,0 +1,193 @@ +"""Network-free OpenRouter catalog tests. Run from backend/: python tests/test_model_catalog.py""" +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-model-catalog-test.sqlite") +Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) + +from model_catalog import ( + ModelWindow, + catalog_ttl, + reset_catalog, + resolve_generate_metadata, +) +from prompt_budget import ERROR_MODEL_METADATA_UNKNOWN, JournalBudgetError +from providers import ProviderConfig + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +class FakeResponse: + def __init__(self, payload, status_code: int = 200, text: str = ""): + self._payload = payload + self.status_code = status_code + self.text = text + + def json(self): + if isinstance(self._payload, Exception): + raise self._payload + return self._payload + + +def _openrouter_config(model: str = "openai/gpt-4o") -> ProviderConfig: + return ProviderConfig( + role="generate", + name="openrouter", + mode="http", + url="https://openrouter.ai/api/v1/chat/completions", + model=model, + key="sk-test", + local=False, + zdr=True, + no_train=True, + ) + + +def _clear_env(*names: str) -> dict[str, str | None]: + previous = {} + for name in names: + previous[name] = os.environ.pop(name, None) + return previous + + +def _restore_env(previous: dict[str, str | None]) -> None: + for name, value in previous.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def test_reads_context_and_completion() -> None: + reset_catalog() + previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS") + payload = { + "data": [ + { + "id": "openai/gpt-4o", + "context_length": 128000, + "top_provider": {"max_completion_tokens": 16384}, + } + ] + } + with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)): + window = resolve_generate_metadata(_openrouter_config()) + expect(window.context_length == 128000, "catalog reads context_length") + expect(window.max_completion_tokens == 16384, "catalog reads max completion") + expect(window.source == "openrouter_models", "window is from the catalog") + _restore_env(previous) + + +def test_model_not_in_catalog() -> None: + reset_catalog() + previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS") + payload = {"data": [{"id": "other/model", "context_length": 32000, "top_provider": {"max_completion_tokens": 4096}}]} + with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)): + try: + resolve_generate_metadata(_openrouter_config("missing/model")) + raise SystemExit("FAIL: missing catalog model should fail closed") + except JournalBudgetError as exc: + expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "model not in catalog is fail-closed") + expect((exc.diagnostics or {}).get("reason") == "model_not_in_catalog", "reason is model_not_in_catalog") + _restore_env(previous) + + +def test_incomplete_metadata() -> None: + reset_catalog() + previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS") + payload = {"data": [{"id": "openai/gpt-4o", "top_provider": {"max_completion_tokens": 4096}}]} + with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)): + try: + resolve_generate_metadata(_openrouter_config()) + raise SystemExit("FAIL: incomplete metadata should fail closed") + except JournalBudgetError as exc: + expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "incomplete metadata is fail-closed") + _restore_env(previous) + + +def test_cache_hit_and_ttl_expiry() -> None: + reset_catalog() + previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS") + calls = {"n": 0} + + def fake_get(*args, **kwargs): + calls["n"] += 1 + return FakeResponse( + { + "data": [ + { + "id": "openai/gpt-4o", + "context_length": 64000 + calls["n"], + "top_provider": {"max_completion_tokens": 4096}, + } + ] + } + ) + + with patch("model_catalog.httpx.get", fake_get): + first = resolve_generate_metadata(_openrouter_config(), now=1000.0) + second = resolve_generate_metadata(_openrouter_config(), now=1000.0 + catalog_ttl() - 1) + expect(calls["n"] == 1, "catalog is cached within TTL") + expect(first.context_length == second.context_length, "cache hit returns the same window") + expect(second.cached is True, "cached window is marked cached") + third = resolve_generate_metadata(_openrouter_config(), now=1000.0 + catalog_ttl() + 1) + expect(calls["n"] == 2, "catalog is refetched after TTL") + expect(third.context_length != first.context_length, "refetch reads a new catalog row") + _restore_env(previous) + + +def test_env_fallback_and_fail_closed() -> None: + reset_catalog() + previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS") + payload = {"data": [{"id": "other/model", "context_length": 32000, "top_provider": {"max_completion_tokens": 2048}}]} + os.environ["KANSHO_PROVIDER_CONTEXT_LENGTH"] = "65536" + os.environ["KANSHO_PROVIDER_MAX_COMPLETION_TOKENS"] = "8192" + with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)): + window = resolve_generate_metadata(_openrouter_config("missing/model")) + expect(window.context_length == 65536, "documented env fallback supplies the window") + expect(window.max_completion_tokens == 8192, "env fallback supplies completion limit") + expect(window.source == "env_fallback", "fallback is labeled as env_fallback") + + reset_catalog() + os.environ.pop("KANSHO_PROVIDER_CONTEXT_LENGTH", None) + os.environ.pop("KANSHO_PROVIDER_MAX_COMPLETION_TOKENS", None) + with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)): + try: + resolve_generate_metadata(_openrouter_config("missing/model")) + raise SystemExit("FAIL: no window should fail closed") + except JournalBudgetError as exc: + expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "no window is fail-closed") + + reset_catalog() + with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse({"nope": True})): + try: + resolve_generate_metadata(_openrouter_config()) + raise SystemExit("FAIL: bad catalog shape should fail closed") + except JournalBudgetError as exc: + expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "bad metadata shape is fail-closed") + _restore_env(previous) + + +def main() -> None: + test_reads_context_and_completion() + test_model_not_in_catalog() + test_incomplete_metadata() + test_cache_hit_and_ttl_expiry() + test_env_fallback_and_fail_closed() + print("model catalog tests passed.") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_mvp_journal.py b/backend/tests/test_mvp_journal.py index 5cf41d1..42283b2 100644 --- a/backend/tests/test_mvp_journal.py +++ b/backend/tests/test_mvp_journal.py @@ -10,9 +10,11 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-mvp-journal-test.sqlite") +os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-mvp-journal-media") os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) +Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True) from fastapi.testclient import TestClient from main import app @@ -302,22 +304,29 @@ def main() -> None: expect("Schluss" in infer_register(["Wir machten uns müde auf den Heimweg."]), "register sees the close first") originals = client.get(f"/api/journal/conversations/{conv['id']}", headers=headers) - expect(len(originals.json()["messages"]) == 2, "A two source messages") + source_msgs = originals.json()["messages"] + expect(sum(1 for item in source_msgs if item["role"] == "user") == 1, "A one user source") + expect(any(item["role"] == "assistant" for item in source_msgs), "A assistant source present") reset_debug() draft = client.post(f"/api/journal/days/{day_id}/generate", headers=headers, json={"conversation_ids": [conv["id"]]}) expect(draft.status_code == 200, f"A generate {draft.text}") expect("Markt" in draft.json()["body"] or "Kirschen" in draft.json()["body"], "A draft keeps user words") - from privacy_gateway import debug_last as last_a - - expect(last_a is not None, "A generate reached gateway") - expect("Kirschen" in last_a["rendered"] or "Markt" in last_a["rendered"], "A generate sees user words") - expect("assistant:" in last_a["rendered"], "A generate sees Kanshō impulses as reply context") - expect("Stilquellen" in last_a["rendered"] or "Dialog" in last_a["rendered"], "A style brief reaches generate") - style_part = last_a["rendered"].split("Dialogquellen:")[0] + stages_a = {item.get("purpose"): item for item in (draft.json().get("trace") or {}).get("stages") or []} + reconstruct_a = stages_a.get("journal_reconstruct") + last_a = stages_a.get("journal_generate") or (draft.json().get("trace") or {}) + expect(reconstruct_a is not None, "A stage 1 reconstruct ran") + reconstruct_intern = reconstruct_a.get("intern") or "" + narrate_intern = last_a.get("intern") or "" + expect("Kirschen" in reconstruct_intern or "Markt" in reconstruct_intern, "A reconstruct sees user words") + expect("user:" in reconstruct_intern, "A reconstruct sees the day as dialogue") + expect("assistant:" not in narrate_intern, "A stage 2 does not resend the day dialogue") + expect("verified_artifact" in narrate_intern or "sources" in narrate_intern, "A stage 2 receives the reconstruction") + expect("Stilquellen" in narrate_intern or "Dialog" in narrate_intern or "Erzählmerkmale" in narrate_intern or "Core:" in narrate_intern, "A style brief reaches generate") + style_part = narrate_intern expect("Erzählmerkmale" in style_part, "A first generate still gets form hints") expect( - "Heute war der Markt voll" not in style_part, + "Heute war der Markt voll" not in style_part.split("Verified Artifact:")[0], "A source dialogue is not pasted a second time as style", ) current_version = None @@ -387,11 +396,10 @@ def main() -> None: next_day.json()["day"]["id"], "Der Vormittag war ruhig.", ) - from privacy_gateway import debug_last as last_c - - expect(last_c is not None, "C gateway saw a request") - expect("Hafen" in last_c["rendered"], "C prior plan in context") - expect("nicht geschehen" in last_c["rendered"], "C not-mentioned rule in prompt") + expect(turn_c is not None, "C gateway saw a request") + intern_c = (turn_c.get("trace") or {}).get("intern") or "" + expect("Hafen" in intern_c, "C prior plan in context") + expect("nicht geschehen" in intern_c, "C not-mentioned rule in prompt") reset_debug() dense = "Erster Absatz mit vielen Details über die Anreise.\n\nZweiter Absatz über das Licht am Abend.\n\nDritter Absatz über das Essen." @@ -410,9 +418,8 @@ def main() -> None: "Ich bin traurig über den Abschied, das ist klar.", title="Emotion", ) - from privacy_gateway import debug_last as last_e - - expect("Motivationen erfinden" in last_e["rendered"], "E no invented emotion rule") + intern_e = (turn_e.get("trace") or {}).get("intern") or "" + expect("Motivationen erfinden" in intern_e, "E no invented emotion rule") expect(turn_e["calls"] == 1, "E one follow-up") day_f = client.post( @@ -498,9 +505,12 @@ def main() -> None: json={"conversation_ids": [conv["id"]]}, ) expect(gen_i.status_code == 200, "I generate after import") - from privacy_gateway import debug_last as last_i - - expect("knapp" in last_i["rendered"], "I style brief reaches generate") + intern_i = "" + for stage in (gen_i.json().get("trace") or {}).get("stages") or []: + if stage.get("purpose") == "journal_generate": + intern_i = stage.get("intern") or "" + intern_i = intern_i or ((gen_i.json().get("trace") or {}).get("intern") or "") + expect("knapp" in intern_i, "I style brief reaches generate") profile = client.get("/api/journal/writing-profile", headers=headers).json() expect(all(item.get("kind") != "journal_draft" for item in profile["sources"]), "I no draft kind") expect(not any((item.get("body") or "") == gen_i.json()["body"] for item in profile["sources"]), "I draft not a style source") @@ -538,6 +548,50 @@ def main() -> None: (day_with_draft["current_draft"] or {}).get("body") != after_gen["body"], "J new draft is not the saved user text", ) + marker = "EXISTING_FASSUNG_XYZ_UNIQUE" + marked = client.post( + "/api/journal/entries", + headers=headers, + json={ + "journal_day_id": day_id, + "entry_id": saved.json()["id"], + "title": "Markiert", + "body": marker, + "origin": "user_edit", + }, + ) + expect(marked.status_code == 200, "include_existing setup save") + marked_version = marked.json()["current_version_id"] + without_flag = client.post( + f"/api/journal/days/{day_id}/generate", + headers=headers, + json={"conversation_ids": [conv["id"]]}, + ) + expect(without_flag.status_code == 200, f"generate default {without_flag.text}") + without_intern = "" + for stage in (without_flag.json().get("trace") or {}).get("stages") or []: + if stage.get("purpose") == "journal_generate": + without_intern = stage.get("intern") or "" + expect(marker not in without_intern, "include_existing default does not send the saved body") + after_without = client.get(f"/api/journal/entries/{saved.json()['id']}", headers=headers).json() + expect(after_without["current_version_id"] == marked_version, "default generate leaves entry version") + with_flag = client.post( + f"/api/journal/days/{day_id}/generate", + headers=headers, + json={"conversation_ids": [conv["id"]], "include_existing": True}, + ) + expect(with_flag.status_code == 200, f"generate include_existing {with_flag.text}") + with_intern = "" + for stage in (with_flag.json().get("trace") or {}).get("stages") or []: + if stage.get("purpose") == "journal_generate": + with_intern = stage.get("intern") or "" + expect(marker in with_intern, "include_existing true sends the saved body only after explicit flag") + existing_section = with_intern.split("Bestehender Text")[-1].split("Verified Artifact")[0] + expect(marker in existing_section, "saved Fassung is the included existing text, not a leftover draft") + after_with = client.get(f"/api/journal/entries/{saved.json()['id']}", headers=headers).json() + expect(after_with["current_version_id"] == marked_version, "include_existing generate still does not overwrite") + expect(after_with["body"] == marker, "saved user body remains") + day_with_draft = client.get(f"/api/journal/days/{day_id}", headers=headers).json() n_before = len(client.get(f"/api/journal/entries/{saved.json()['id']}/versions", headers=headers).json()) adopted = client.post( "/api/journal/entries", @@ -662,13 +716,10 @@ def main() -> None: json={"body": "Noch ein Satz zum Abend."}, ) expect(turn_scratch.status_code == 200, "turn after scratch") - from privacy_gateway import debug_last as last_scratch_turn - blob = " ".join( - str(last_scratch_turn.get(key) or "") - for key in ("rendered", "masked", "mask_input", "raw", "reply") - ) - intern = (turn_scratch.json().get("trace") or {}).get("intern") or "" - expect(scratch_mark not in blob and scratch_mark not in intern, "scratch not in dialogue egress") + blob = (turn_scratch.json().get("trace") or {}).get("intern") or "" + intern = blob + egress = (turn_scratch.json().get("trace") or {}).get("egress") or "" + expect(scratch_mark not in blob and scratch_mark not in intern and scratch_mark not in egress, "scratch not in dialogue egress") reset_debug() gen_scratch = client.post( f"/api/journal/days/{day_id}/generate", @@ -676,12 +727,13 @@ def main() -> None: json={"conversation_ids": [conv["id"]]}, ) expect(gen_scratch.status_code == 200, "generate after scratch") - from privacy_gateway import debug_last as last_scratch_gen + gen_trace = gen_scratch.json().get("trace") or {} gen_blob = " ".join( - str(last_scratch_gen.get(key) or "") - for key in ("rendered", "masked", "mask_input", "raw", "reply") + str(stage.get(key) or "") + for stage in (gen_trace.get("stages") or [gen_trace]) + for key in ("intern", "egress", "mask_input", "raw", "reply") ) - gen_intern = (gen_scratch.json().get("trace") or {}).get("intern") or "" + gen_intern = gen_trace.get("intern") or "" expect(scratch_mark not in gen_blob and scratch_mark not in gen_intern, "scratch not in generate egress") from media_store import strip_jpeg_exif diff --git a/backend/tests/test_privacy_detect.py b/backend/tests/test_privacy_detect.py index 81b38fc..b30bf28 100644 --- a/backend/tests/test_privacy_detect.py +++ b/backend/tests/test_privacy_detect.py @@ -51,13 +51,12 @@ def main() -> None: json={"body": "Ich war mit Anna in Hamburg und danach allein."}, ) expect(turn.status_code == 200, f"turn {turn.text}") - from privacy_gateway import debug_last - - expect(debug_last is not None, "gateway recorded request") - expect(debug_last["detect_provider"] == "fake-detect", "detect provider is separate") - expect("Anna" not in debug_last["masked"], "Anna masked before generate") - expect("Hamburg" not in debug_last["masked"], "Hamburg masked before generate") - expect("[[PERSON:" in debug_last["masked"] or "[[PLACE:" in debug_last["masked"], "placeholders in egress") + trace = turn.json().get("trace") or {} + expect(trace, "gateway returned request-scoped trace") + expect(trace["detect_provider"] == "fake-detect", "detect provider is separate") + expect("Anna" not in (trace.get("egress") or ""), "Anna masked before generate") + expect("Hamburg" not in (trace.get("egress") or ""), "Hamburg masked before generate") + expect("[[PERSON:" in (trace.get("egress") or "") or "[[PLACE:" in (trace.get("egress") or ""), "placeholders in egress") expect("Anna" in turn.json()["user"]["body"], "source keeps cleartext") mappings = list_mappings(setup.json()["profile_id"]) labels = {item["local_label"] for item in mappings} @@ -72,10 +71,10 @@ def main() -> None: json={"body": "Anna war ruhig."}, ) expect(again.status_code == 200, "second turn") - from privacy_gateway import debug_last as last2 + last2 = again.json().get("trace") or {} - expect("nicht" in last2["masked"].lower() or "Anna" not in last2["masked"], "dictionary remasks Anna without new invention") - expect(last2["mapping_count"] >= 2, "dictionary reused") + expect("nicht" in (last2.get("egress") or "").lower() or "Anna" not in (last2.get("egress") or ""), "dictionary remasks Anna without new invention") + expect(last2.get("mapping_count", 0) >= 2, "dictionary reused") from entity_detect import _pattern_entities from identity_store import is_maskable_label @@ -251,6 +250,25 @@ def main() -> None: except PrivacyGatewayError as exc: blocked = exc.code == "response_validation_failed" expect(blocked, "plaintext identity is blocked before demask") + sushi_map = [{"local_label": "Sushi", "token": "PERSON:01"}] + food_reply = _validate_response("Heute nur von Sushi essen erzählt.", sushi_map) + expect("Sushi essen" in food_reply, "food homonym in a reply is not treated as a leak") + mixed = mask_for_egress( + "Belegte Vorhaben (nur user-Quellen):\n- Morgen wollen wir mit Anna fahren.\nuser: Heute Markt.", + mapped, + "dialogue_turn", + ) + expect("Anna" not in mixed, "opening context names are masked for dialogue egress") + expect("[[PERSON:01]]" in mixed, "opening context uses the person token") + expect("Heute Markt" in mixed, "current user wording remains") + excerpt = mask_for_egress( + "Frühere Einträge im Space (nur Hinweise, keine Gewissheit):\n2026-08-01: Anna war am Hafen.\nuser: Heute windstill.", + mapped, + "dialogue_turn", + ) + expect("Anna" not in excerpt, "space excerpts are masked for dialogue egress") + expect("[[PERSON:01]]" in excerpt, "space excerpt uses the person token") + print("All privacy detect tests passed.") diff --git a/backend/tests/test_profile_governance.py b/backend/tests/test_profile_governance.py index f36c6be..38fc034 100644 --- a/backend/tests/test_profile_governance.py +++ b/backend/tests/test_profile_governance.py @@ -183,11 +183,11 @@ def main() -> None: expect(interaction["suggestions"] == [], "silence is not an interaction suggestion") expect(any(item["pref_key"] == "reply_length" for item in interaction["defaults"]), "labeled product defaults") - from privacy_gateway import debug_last + intern = (turn.json().get("trace") or {}).get("intern") or "" - expect("Produktdefault" in (debug_last["rendered"] if debug_last else ""), "hint reaches dialogue prompt") - expect("Ein bis drei Sätze" in (debug_last["rendered"] if debug_last else ""), "length default is in hint not anonymous") - expect("anonyme Produktregel" in (debug_last["rendered"] if debug_last else ""), "prompt labels the moved preference") + expect("Produktdefault" in intern, "hint reaches dialogue prompt") + expect("Ein bis drei Sätze" in intern, "length default is in hint not anonymous") + expect("anonyme Produktregel" in intern, "prompt labels the moved preference") proposed = propose_observation( profile_id, @@ -223,10 +223,10 @@ def main() -> None: json={"body": "Heute Vormittag zum Markt, danach Kirschen, dann noch der Hafen bei 18:00 Uhr."}, ) expect(later.status_code == 200, "second turn") - from privacy_gateway import debug_last as last_hint + intern = (later.json().get("trace") or {}).get("intern") or "" - expect("Meist nur ein Satz" in last_hint["rendered"], "personal preference reaches hint") - expect("persönlich" in last_hint["rendered"], "personal override is labeled") + expect("Meist nur ein Satz" in intern, "personal preference reaches hint") + expect("persönlich" in intern, "personal override is labeled") expect(later.json()["conversation"].get("narrative_mode"), "dialogue state remains automatic") expect("long_story" in later.json()["conversation"], "long_story is conversation state") diff --git a/backend/tests/test_profile_review.py b/backend/tests/test_profile_review.py index d4dba94..9eaff77 100644 --- a/backend/tests/test_profile_review.py +++ b/backend/tests/test_profile_review.py @@ -260,9 +260,9 @@ def main() -> None: json={"body": long_body}, ) expect(long_turn.status_code == 200, "long turn") - from privacy_gateway import debug_last as last_long + last_long = long_turn.json().get("trace") or {} - expect(last_long["purpose"] == "dialogue_turn", "long turn still has no review call") + expect(last_long.get("purpose") == "dialogue_turn", "long turn still has no review call") status = client.get("/api/journal/writing-profile/review", headers=headers).json() expect("dialogue_sample" not in pending_triggers(status), "dialogue is not sampled before confirm") @@ -514,10 +514,10 @@ def main() -> None: reset_debug() api_review = client.post("/api/journal/writing-profile/review/api", headers=headers) expect(api_review.status_code == 200, f"api review {api_review.text}") - from privacy_gateway import debug_last as api_trace + api_trace = api_review.json().get("trace") or {} - expect(api_trace["purpose"] == "profile_review", "api review uses dedicated purpose") - expect(api_trace["layer"] == "profilreview", "trace is not a dialogue turn") + expect(api_trace.get("purpose") == "profile_review", "api review uses dedicated purpose") + expect(api_trace.get("layer") == "profilreview", "trace is not a dialogue turn") expect(api_review.json()["applied"] == 0, "api result is a proposal, not an auto-apply") expect(api_review.json()["suggested"] >= 1, "advising queues review as suggestion") expect( diff --git a/backend/tests/test_provenance.py b/backend/tests/test_provenance.py new file mode 100644 index 0000000..6c817b7 --- /dev/null +++ b/backend/tests/test_provenance.py @@ -0,0 +1,255 @@ +"""Intent-neutral provenance verification. Run from backend/: python tests/test_provenance.py""" +from __future__ import annotations + +import os +import sys +import tempfile +import threading +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +os.environ.setdefault( + "KANSHO_DB_PATH", + str(Path(tempfile.gettempdir()) / "kansho-provenance-test.sqlite"), +) + +from provenance import ( + COVERAGE_ALL_SELECTED_SOURCES, + COVERAGE_SELECTED_EVIDENCE, + EvidenceRef, + ProvenanceError, + SourceRecord, + SourceRegistry, + VerificationPolicy, + verify, +) + + +def expect(ok: bool, message: str) -> None: + if not ok: + raise SystemExit(f"FAIL: {message}") + print(f"OK {message}") + + +def _expect_error(reason: str, **kwargs) -> None: + try: + verify(**kwargs) + raise SystemExit(f"FAIL: expected {reason}") + except ProvenanceError as exc: + expect(exc.reason == reason, f"reason is {reason}, got {exc.reason}") + + +CHANGELOG = ( + "Build 14.2 ersetzt den Hash-Algorithmus. " + "Die Checksummen-Datei liegt unter /var/lib/app/SHA256SUMS. " + "Der Paketname bleibt libcore." +) +NOTE = "Der Review-Kommentar betrifft nur die Signaturprüfung." + + +def _registry() -> SourceRegistry: + return SourceRegistry( + [ + SourceRecord("s1", "primary", CHANGELOG), + SourceRecord("s2", "note", NOTE), + ] + ) + + +def test_unknown_source() -> None: + registry = _registry() + policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s9",)) + _expect_error("unknown_source", registry=registry, policy=policy) + + +def test_disallowed_role() -> None: + registry = _registry() + policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s2",)) + _expect_error("role_not_allowed", registry=registry, policy=policy) + + +def test_wrong_source_excerpt() -> None: + registry = _registry() + policy = VerificationPolicy(allowed_roles=frozenset({"primary", "note"}), selected_ids=("s1", "s2")) + _expect_error( + "wrong_source", + registry=registry, + policy=policy, + evidence=[EvidenceRef("s1", "Signaturprüfung")], + ) + + +def test_invented_excerpt() -> None: + registry = _registry() + policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s1",)) + _expect_error( + "unverified_excerpt", + registry=registry, + policy=policy, + evidence=[EvidenceRef("s1", "Die Lizenz ändert sich auf GPL.")], + ) + + +def test_all_selected_rehydrates_full_text() -> None: + registry = _registry() + policy = VerificationPolicy( + allowed_roles=frozenset({"primary"}), + coverage=COVERAGE_ALL_SELECTED_SOURCES, + selected_ids=("s1",), + ) + artifact = verify( + registry, + policy=policy, + evidence=[EvidenceRef("s1", "Build 14.2")], + ) + expect(artifact.sources[0]["text"] == CHANGELOG, "full canonical changelog is rehydrated") + expect("SHA256SUMS" in artifact.sources[0]["text"], "later changelog sentences stay") + expect("libcore" in artifact.sources[0]["text"], "final changelog sentence stays") + + +def test_excerpt_cannot_drop_remainder() -> None: + registry = _registry() + policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s1",)) + artifact = verify(registry, policy=policy, evidence=[EvidenceRef("s1", "Build 14.2")]) + expect(artifact.sources[0]["text"] != "Build 14.2", "a short excerpt is not the source") + expect(artifact.sources[0]["text"].startswith("Build 14.2 ersetzt"), "canonical prefix remains") + expect("Paketname bleibt libcore" in artifact.sources[0]["text"], "unread remainder is not dropped") + + +def test_model_cannot_overwrite_canonical() -> None: + registry = _registry() + policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s1",)) + _expect_error( + "canonical_overwrite", + registry=registry, + policy=policy, + proposed_texts={"s1": "Build 14.2 ist ein Sicherheitsrelease ohne Dateipfade."}, + ) + artifact = verify( + registry, + policy=policy, + proposed_texts={"s1": CHANGELOG}, + ) + expect(artifact.sources[0]["text"] == CHANGELOG, "identical proposed text is ignored as authority") + + +def test_annotations_are_not_verified_content() -> None: + registry = _registry() + excluded = VerificationPolicy( + allowed_roles=frozenset({"primary"}), + selected_ids=("s1",), + include_unverified_annotations=False, + ) + artifact = verify( + registry, + policy=excluded, + evidence=[EvidenceRef("s1", CHANGELOG)], + annotations=[{"kind": "risk", "source_id": "s1", "excerpt": CHANGELOG}], + ) + payload = artifact.to_payload() + expect("annotations_unverified" not in payload, "excluded annotations do not appear") + expect(all("kind" not in item for item in payload["sources"]), "source records have no semantic kind") + dumped = str(payload) + expect("risk" not in dumped, "unverified label is not in verified payload") + + separated = VerificationPolicy( + allowed_roles=frozenset({"primary"}), + selected_ids=("s1",), + include_unverified_annotations=True, + ) + held = verify( + registry, + policy=separated, + annotations=[{"kind": "risk", "source_id": "s1", "excerpt": CHANGELOG}], + ) + expect(held.annotations_unverified[0]["kind"] == "risk", "unverified labels can be held separately") + expect(held.sources[0]["text"] == CHANGELOG, "canonical text is independent of the label") + + +def test_policies_differ_without_domain_logic() -> None: + registry = _registry() + primary_only = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s2",)) + _expect_error("role_not_allowed", registry=registry, policy=primary_only) + + with_notes = VerificationPolicy( + allowed_roles=frozenset({"primary", "note"}), + coverage=COVERAGE_ALL_SELECTED_SOURCES, + selected_ids=("s2",), + ) + accepted = verify(registry, policy=with_notes) + expect(accepted.sources[0]["text"] == NOTE, "a second policy can allow another role") + + selective = VerificationPolicy( + allowed_roles=frozenset({"primary"}), + coverage=COVERAGE_SELECTED_EVIDENCE, + selected_ids=("s1",), + ) + excerpt_only = verify( + registry, + policy=selective, + evidence=[EvidenceRef("s1", "Build 14.2")], + ) + expect(excerpt_only.sources[0]["text"] == "Build 14.2", "selected-evidence mode keeps only verified excerpts") + expect("SHA256SUMS" not in excerpt_only.sources[0]["text"], "selected-evidence mode does not rehydrate the rest") + + full = verify( + registry, + policy=VerificationPolicy( + allowed_roles=frozenset({"primary"}), + coverage=COVERAGE_ALL_SELECTED_SOURCES, + selected_ids=("s1",), + ), + evidence=[EvidenceRef("s1", "Build 14.2")], + ) + expect("SHA256SUMS" in full.sources[0]["text"], "all-selected mode still rehydrates the same registry") + + +def test_parallel_requests_are_isolated() -> None: + registry_ok = SourceRegistry([SourceRecord("a1", "primary", "Alpha-Release 3 ist signiert.")]) + registry_bad = SourceRegistry([SourceRecord("b1", "primary", "Beta-Kanal bleibt intern.")]) + ok_policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("a1",)) + bad_policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("missing",)) + results: dict[str, object] = {} + errors: dict[str, str] = {} + + def succeed() -> None: + results["ok"] = verify(registry_ok, policy=ok_policy) + + def fail() -> None: + try: + verify(registry_bad, policy=bad_policy) + except ProvenanceError as exc: + errors["bad"] = exc.reason + + first = threading.Thread(target=fail) + second = threading.Thread(target=succeed) + first.start() + second.start() + first.join() + second.join() + expect(errors.get("bad") == "unknown_source", "failing request keeps its own reason") + artifact = results.get("ok") + expect(artifact is not None, "successful request still returns an artifact") + expect(artifact.sources[0]["text"] == "Alpha-Release 3 ist signiert.", "parallel failure does not rewrite the other registry") + later = verify(registry_ok, policy=ok_policy) + expect(later.sources[0]["text"] == "Alpha-Release 3 ist signiert.", "later request is unaffected") + + +def main() -> None: + test_unknown_source() + test_disallowed_role() + test_wrong_source_excerpt() + test_invented_excerpt() + test_all_selected_rehydrates_full_text() + test_excerpt_cannot_drop_remainder() + test_model_cannot_overwrite_canonical() + test_annotations_are_not_verified_content() + test_policies_differ_without_domain_logic() + test_parallel_requests_are_isolated() + print("provenance tests passed.") + + +if __name__ == "__main__": + main() diff --git a/backend/writing_profile_store.py b/backend/writing_profile_store.py index c8721f2..4656e66 100644 --- a/backend/writing_profile_store.py +++ b/backend/writing_profile_store.py @@ -1,7 +1,9 @@ """Writing profile store with facets, governance, and evidence. AI drafts are never a positive style source.""" from __future__ import annotations +import hashlib import json +import re import uuid from db import get_db, row_to_dict @@ -44,6 +46,13 @@ BRIEF_IMPORT_EXCERPTS = 6 INITIAL_BUILD_SOURCES = 40 EXCERPT_CHARS = 1200 TRAIT_EXCERPT_CHARS = 280 +TASK_BRIEF_MAX_CHARS = 4000 +TASK_BRIEF_CORE_CHARS = 800 +TASK_BRIEF_FACET_CHARS = 700 +TASK_BRIEF_TRAIT_CHARS = 280 +TASK_BRIEF_EXEMPLAR_CHARS = 220 +TASK_BRIEF_MAX_TRAITS = 6 +TASK_BRIEF_MAX_EXEMPLARS = 2 DIALOGUE_KIND = "dialogue_style" MAX_DIALOGUE_MESSAGES = 8 MAX_DIALOGUE_CHARS = 500 @@ -980,6 +989,104 @@ def _load_ranked_sources(profile_id: str) -> dict[str, list[dict]]: return {"journal_entry": entries, "imported_text": imports, "dialogue_style": dialogue} +def _source_content_digest(item: dict) -> str: + body = re.sub(r"\s+", " ", plain_text(item.get("body") or "")).strip().lower() + return hashlib.sha256(body.encode("utf-8")).hexdigest() + + +def _dedupe_brief_sources(items: list[dict], *, seen_keys: set[tuple[str, str]] | None = None) -> list[dict]: + """Display-only. Originals remain in writing_profile_sources.""" + seen_entry: set[str] = set() + seen = seen_keys if seen_keys is not None else set() + result = [] + for item in items: + entry_id = (item.get("entry_id") or "").strip() + if entry_id: + if entry_id in seen_entry: + continue + seen_entry.add(entry_id) + when = (item.get("occurred_at") or "")[:10] + digest = _source_content_digest(item) + key = (when, digest) + if key in seen: + continue + seen.add(key) + result.append(item) + return result + + +def _clip_field(text: str, limit: int) -> str: + raw = (text or "").strip() + if limit <= 0 or not raw: + return "" + if len(raw) <= limit: + return raw + cut = raw[:limit] + if not raw[limit].isspace(): + sp = max(cut.rfind(" "), cut.rfind("\n"), cut.rfind("\t")) + if sp >= max(1, limit // 3): + cut = cut[:sp] + else: + return "" + return cut.rstrip(" \t\n,;:-") + + +def _norm_overlap(left: str, right: str) -> float: + a = re.sub(r"\s+", " ", (left or "").strip().lower()) + b = re.sub(r"\s+", " ", (right or "").strip().lower()) + if not a or not b: + return 0.0 + if a == b or a in b or b in a: + return min(len(a), len(b)) / max(len(a), len(b)) + return 0.0 + + +def _trait_recency_stamp(item: dict) -> str | None: + stamps = [item.get("updated"), item.get("observed_to"), item.get("observed_from")] + for ref in item.get("exemplars") or []: + stamps.append(ref.get("occurred_at")) + stamps = [stamp for stamp in stamps if stamp] + return max(stamps) if stamps else None + + +def _select_task_traits(traits: list[dict]) -> list[dict]: + relevant = [ + item + for item in traits + if item.get("status") == "active" + and (item.get("statement") or "") + and item.get("facet_key") in {"autobiographical_journal", "core", ""} + ] + if not relevant: + relevant = [item for item in traits if item.get("status") == "active" and (item.get("statement") or "")] + current = [] + long_term = [] + for item in relevant: + stamp = _trait_recency_stamp(item) + role = recency_role(stamp) + weight = recency_weight(stamp) + scored = (item, weight, role) + if role == "current_expression": + current.append(scored) + else: + long_term.append(scored) + current.sort(key=lambda row: (-row[1], row[0].get("slug") or "")) + long_term.sort(key=lambda row: (-row[1], row[0].get("slug") or "")) + picked: list[dict] = [] + for item, _, _ in current[:4]: + picked.append(item) + for item, _, _ in long_term: + if len(picked) >= TASK_BRIEF_MAX_TRAITS: + break + picked.append(item) + if len(picked) < TASK_BRIEF_MAX_TRAITS: + for item, _, _ in current[4:]: + if len(picked) >= TASK_BRIEF_MAX_TRAITS: + break + picked.append(item) + return picked[:TASK_BRIEF_MAX_TRAITS] + + def _source_line(item: dict) -> str: when = (item.get("occurred_at") or "")[:10] role = recency_role(item.get("occurred_at") or item.get("created")) @@ -989,8 +1096,9 @@ def _source_line(item: dict) -> str: def _assemble_brief(profile_id: str) -> None: ranked = _load_ranked_sources(profile_id) - entries = ranked["journal_entry"][:BRIEF_ENTRY_EXCERPTS] - imports = ranked["imported_text"][:BRIEF_IMPORT_EXCERPTS] + display_keys: set[tuple[str, str]] = set() + entries = _dedupe_brief_sources(ranked["journal_entry"], seen_keys=display_keys)[:BRIEF_ENTRY_EXCERPTS] + imports = _dedupe_brief_sources(ranked["imported_text"], seen_keys=display_keys)[:BRIEF_IMPORT_EXCERPTS] dialogue = ranked["dialogue_style"] profile_row = ensure_profile(profile_id) with get_db() as conn: @@ -1067,46 +1175,91 @@ def compile_task_brief(profile_id: str, task: str = "journal_generate") -> str: from writing_profile_schema import is_meta_style_text profile = get_profile(profile_id) - parts = [] - core = profile.get("core") or {} - if core.get("value"): - parts.append("Core: " + (core.get("value") or "")) - traits = [item for item in profile.get("traits") or [] if item.get("status") == "active"] - if task == "journal_generate": - facet = next( - ( - item - for item in profile.get("facets") or [] - if item.get("facet_key") == "autobiographical_journal" and (item.get("value") or "") - ), - None, - ) - if facet: - parts.append("Autobiografisches Journaling (Facet-Delta): " + (facet.get("value") or "")) - relevant = [ + if task != "journal_generate": + core = (profile.get("core") or {}).get("value") or "" + return _clip_field(core, TASK_BRIEF_MAX_CHARS) + + parts: list[str] = [] + core_text = ((profile.get("core") or {}).get("value") or "").strip() + if core_text: + clipped = _clip_field(core_text, TASK_BRIEF_CORE_CHARS) + if clipped: + parts.append("Core: " + clipped) + facet = next( + ( item - for item in traits - if item.get("facet_key") in {"autobiographical_journal", "core", ""} and (item.get("statement") or "") - ] - if not relevant: - relevant = [item for item in traits if item.get("statement")] - for item in relevant[:6]: - line = f"- {item.get('label') or item.get('slug')}: {item.get('statement')}" - exemplars = [ - ref.get("excerpt") - for ref in item.get("exemplars") or [] - if ref.get("excerpt") and not is_meta_style_text(ref.get("excerpt")) - ] - if exemplars: - line += " Beispiel: " + exemplars[0][:TRAIT_EXCERPT_CHARS] - parts.append(line) - if parts: - return "\n".join(parts).strip() + for item in profile.get("facets") or [] + if item.get("facet_key") == "autobiographical_journal" and (item.get("value") or "") + ), + None, + ) + facet_text = ((facet or {}).get("value") or "").strip() + if facet_text and _norm_overlap(facet_text, core_text) < 0.8: + clipped = _clip_field(facet_text, TASK_BRIEF_FACET_CHARS) + if clipped: + parts.append("Autobiografisches Journaling (Facet-Delta): " + clipped) + selected = _select_task_traits(profile.get("traits") or []) + used_statements = [core_text, facet_text] + for item in selected: + statement = (item.get("statement") or "").strip() + if any(_norm_overlap(statement, previous) >= 0.85 for previous in used_statements if previous): + continue + clipped = _clip_field(statement, TASK_BRIEF_TRAIT_CHARS) + if not clipped: + continue + parts.append(f"- {item.get('label') or item.get('slug')}: {clipped}") + used_statements.append(statement) + exemplars: list[tuple[float, str]] = [] + seen_ex = set() + for item in selected: + for ref in item.get("exemplars") or []: + excerpt = (ref.get("excerpt") or "").strip() + if not excerpt or is_meta_style_text(excerpt): + continue + digest = hashlib.sha256(re.sub(r"\s+", " ", excerpt.lower()).encode("utf-8")).hexdigest() + if digest in seen_ex: + continue + seen_ex.add(digest) + clipped = _clip_field(excerpt, TASK_BRIEF_EXEMPLAR_CHARS) + if not clipped: + continue + weight = recency_weight(ref.get("occurred_at") or _trait_recency_stamp(item)) + exemplars.append((weight, clipped)) + exemplars.sort(key=lambda row: -row[0]) + chosen = [] + for _, excerpt in exemplars: + if any(_norm_overlap(excerpt, prev) >= 0.8 for prev in chosen): + continue + chosen.append(excerpt) + if len(chosen) >= TASK_BRIEF_MAX_EXEMPLARS: + break + if chosen: + parts.append("Repräsentative Exemplare:") + for excerpt in chosen: + parts.append("- " + excerpt) + brief = "\n".join(part for part in parts if part).strip() + if len(brief) > TASK_BRIEF_MAX_CHARS: + without_examples = [part for part in parts if not part.startswith("Repräsentative Exemplare") and not ( + part.startswith("- ") and chosen and any(part == "- " + excerpt for excerpt in chosen) + )] + brief = "\n".join(without_examples).strip() + parts = without_examples + while len(brief) > TASK_BRIEF_MAX_CHARS and parts: + last = parts.pop() + remain = TASK_BRIEF_MAX_CHARS - (len("\n".join(parts)) + (1 if parts else 0)) + clipped = _clip_field(last, remain) + if clipped: + parts.append(clipped) + brief = "\n".join(parts).strip() + break + brief = "\n".join(parts).strip() + if brief: + return brief ranked = _load_ranked_sources(profile_id) snippets = (ranked.get("journal_entry") or [])[:2] + (ranked.get("imported_text") or [])[:1] lines = [] for item in snippets: - body = plain_text(item.get("body") or "")[:280] + body = _clip_field(plain_text(item.get("body") or ""), 280) if body and not is_meta_style_text(body): lines.append(body) return "\n\n".join(lines).strip() diff --git a/docs/architecture/functional/documentation_index.md b/docs/architecture/functional/documentation_index.md index 747d07d..bf9987f 100644 --- a/docs/architecture/functional/documentation_index.md +++ b/docs/architecture/functional/documentation_index.md @@ -37,6 +37,7 @@ Dieses Dokument dient dazu, für weitere Konzeptarbeit nur die tatsächlich ben | `mvp.md` | Erster vertikaler Slice (dialoggeführtes Journal) | | `implementation_foundation.md` | Verbindliche Leitplanke zwischen Zielmodell und Slice | | `mvp_stand_und_abgleich.md` | **Kanonische Fit-Gap-Analyse** (2026-08-25): Code gegen Foundation, MVP-Slice und Gesamtziel; Gaps, Ungenauigkeiten, Prüfbrief für Gegenlesung; Profile-Analyse-Export 2.5 | +| `mvp_freeze_candidate.md` | Abnahme-Checkliste für den lokalen MVP Freeze Candidate (Urlaubs-Testphase) | ## 3. Empfohlene Context Bundles @@ -80,10 +81,11 @@ Kanonisches Home der Fit-Gap-Analyse ist `mvp_stand_und_abgleich.md`. Es ersetzt 1. `mvp.md` 2. `implementation_foundation.md` 3. `mvp_stand_und_abgleich.md` -4. `writing_profile_and_journaling.md` -5. `guardrails.md` -6. bei Gegenlesung gegen das Gesamtziel zusätzlich `produktvision_und_produktidentitaet.md` -7. technisch: `../technical/mvp_implementation.md` und `../technical/documentation_index.md` Bundle „MVP-Journal-Slice / technische Umsetzung“ +4. `mvp_freeze_candidate.md` +5. `writing_profile_and_journaling.md` +6. `guardrails.md` +7. bei Gegenlesung gegen das Gesamtziel zusätzlich `produktvision_und_produktidentitaet.md` +8. technisch: `../technical/mvp_implementation.md` und `../technical/documentation_index.md` Bundle „MVP-Journal-Slice / technische Umsetzung“ ### Datenschutz / externe KI diff --git a/docs/architecture/functional/mvp.md b/docs/architecture/functional/mvp.md index 08c292d..3b76b05 100644 --- a/docs/architecture/functional/mvp.md +++ b/docs/architecture/functional/mvp.md @@ -384,6 +384,12 @@ Nur wenn kein sinnvoller Kontext verfügbar ist, wird ein neutraler Einstieg ver Der erste Gesprächsimpuls unterliegt derselben Dialog-Policy wie spätere Folgeimpulse. +### 6.1 Implementierungsstand (Freeze Candidate 2026-08-25) + +**Status: Code vorhanden, journalspezifisch, keine universelle Continuation-Engine.** + +Ein neu gestarteter Journal-Dialog erhält einen ersten Kanshō-Impuls. Belegte Vorhaben und offene user-Punkte desselben Journal Day dürfen den Einstieg stützen. Recency allein ist kein Muster. Assistententext ist Kontext, kein Nutzerfakt. Ohne relevanten Kontext: kurzer neutraler lokaler Einstieg. Fehlgeschlagener Modelloutput speichert keine beschädigte Nachricht. + --- # 7. Unterbrechen und Fortsetzen @@ -427,6 +433,12 @@ Für die Generierung dürfen verwendet werden: Im MVP ist der Space-Kontext im Dialogzug die letzten gespeicherten Journal Entries desselben Spaces als kurze Recency-Ausschnitte, keine inhaltliche Space-Memory (keine Threads, keine wiederkehrenden Aktivitäten, keine Relevanzauswahl). Für den laufenden Tag ist der Dialog des Journal Day die führende Quelle. Ein Space Content Memory bleibt später. +### 8.2.1 Freeze Candidate: Space-Kontext bei Generate (2026-08-25) + +**Status: bewusst begrenzt, entschieden für diesen Slice.** + +Fachliche Inhaltsquelle der Journalgenerierung sind die ausdrücklich ausgewählten Conversations des Journal Day. Stufe 2 erhält ausschließlich das lokal erzeugte Verified Artifact. Frühere Space Entries sind keine zusätzliche Faktenquelle in Generate. Das Writing Profile darf nur die Form beeinflussen. Space-Kontext kann die Dialogführung stützen, wird aber nicht still zur Journalquelle. + --- ## 8.3 Generierungsprinzip @@ -574,6 +586,12 @@ Nicht Bestandteil des MVP: Einzelne, vom Nutzer eingebettete Videos im Journaltext sind damit Teil des Slices; eine Videoverwaltung ist es nicht. +### 10.2 Dialogmedien (Freeze Candidate 2026-08-25) + +**Status: bewusst nicht im Slice.** Zielarchitektur für Voice/visuellen Egress bleibt unberührt. + +Keine Medien als direkte Dialogquelle in diesem MVP. Bilder und Videos können im Journal Editor ergänzt werden. Sicherer visueller Modell-Egress, Maskierung und Provenance sind nicht Teil dieses Slices. Dialogmedien werden in einem eigenen späteren Slice umgesetzt. + --- ## 10.1 Tagesstichpunkte (Arbeitsstand) @@ -601,7 +619,7 @@ Beispiel: - v3 – Textkorrektur und Bilder - v4 – spätere Ergänzung -Frühere gespeicherte Versionen bleiben wiederherstellbar. Die aktuell geladene Fassung darf in der Liste stehen, wird dort aber nicht wiederhergestellt. Offene, noch nicht gespeicherte Editoränderungen können lokal verworfen werden („Alle Änderungen rückgängig machen“). Verlassen des Editors mit solchen Änderungen wird bestätigt, nicht still verworfen. +Frühere gespeicherte Versionen bleiben wiederherstellbar. Die aktuell geladene Fassung darf in der Liste stehen, wird dort aber nicht wiederhergestellt. Offene, noch nicht gespeicherte Editoränderungen können lokal verworfen werden („Alle Änderungen rückgängig machen“). Verlassen des Editors mit solchen Änderungen wird bestätigt, nicht still verworfen. Browser-Zurück bleibt für diesen Freeze über die vorhandene Warnung und `beforeunload` ausreichend; keine Router-Großmigration nur für einen perfekten Zurück-Blocker. --- @@ -628,6 +646,10 @@ Für den MVP: - endgültiges Löschen ist eine separate explizite Aktion, - Version History schützt zusätzlich vor unbeabsichtigten Textänderungen. +### 11.3.1 Implementierungsstand (Freeze Candidate 2026-08-25) + +**Status: Code vorhanden.** Soft Delete legt in den Papierkorb. Wiederherstellen aus dem Papierkorb ist nicht die Versions-Restore-Semantik. Endgültiges Löschen ist ausdrücklich bestätigt, transaktional, löscht Quelldialoge nicht und entfernt nur unreferenzierte Medien. + --- # 12. Journal als Nachschlagewerk diff --git a/docs/architecture/functional/mvp_freeze_candidate.md b/docs/architecture/functional/mvp_freeze_candidate.md new file mode 100644 index 0000000..4ac7958 --- /dev/null +++ b/docs/architecture/functional/mvp_freeze_candidate.md @@ -0,0 +1,67 @@ +--- +title: "Kanshō – MVP Freeze Candidate (lokale Urlaubs-Testphase)" +status: "Abnahme / Freeze Candidate" +date: "2026-08-25" +product_family: "Jinkendo" +document_role: "Kurze Abnahme-Checkliste für den lokalen Journal-MVP Freeze Candidate" +parent_document: "mvp_stand_und_abgleich.md" +--- + +# Kanshō – MVP Freeze Candidate + +Dieses Blatt ersetzt weder `mvp.md` noch `mvp_stand_und_abgleich.md`. Es sagt, was für die **lokale Urlaubs-Testphase** automatisch, manuell und später in Produktion zu prüfen ist, und was bewusst nicht zu diesem Slice gehört. + +**Stand:** 2026-08-25. Kein Docker, kein PostgreSQL, keine echten Provider-Aufrufe in der automatischen Suite. + +## 1. Automatisierte lokale Prüfungen + +Einstieg: `.\scripts\test-mvp.ps1` + +Die Suite setzt UTF-8, leere Provider-Keys und isolierte Temp-Daten. Suites, die einen Fake-Provider brauchen, setzen ihn selbst; Fail-closed bleibt prüfbar. Sie darf `backend/data/kansho.sqlite` und echte Medien nicht verändern. + +- alle `backend/tests/test_*.py` (Abbruch beim ersten Fehler) +- `frontend/src/journal/document.test.js` +- Frontend-Produktions-Build +- `git diff --check` (separat, vor einem späteren Commit) + +Neu in diesem Freeze: Backup/Restore, Papierkorb/Purge, Entry-Liste, Opening-Invarianten, `include_existing` beider Varianten. + +## 2. Spätere manuelle Modell-/Browserprüfungen + +Nicht durch Fake-Provider bewiesen: + +- Erster Impuls mit einem echten Modell: belegter Plan, offener Tagesfaden, neutraler Fallback +- Dialogqualität der sechs Operationen (Cases C/D/E bleiben modellabhängig) +- Journalentwurf Stufe 1/2 mit einem echten Modell: Quelltreue, keine Assistentenfakten +- `include_existing` in der Oberfläche: Checkbox aus, Checkbox an, Nutzerfassung unangetastet +- Papierkorb: Soft Delete, Wiederherstellen, endgültiges Löschen inkl. Medien +- Startseite: Fortsetzen wenn ein Day existiert, sonst Space-Einstieg +- Backup erzeugen und an einem zweiten Temp-Pfad wiederherstellen, inkl. Bestätigung +- Browser: ungespeicherte Editoränderungen, `beforeunload`; Zurück bleibt bewusst ohne Router-Migration + +## 3. Spätere Produktions-Gates + +Kein Codefehler des Freeze Candidate, sondern Betriebsreife: + +- PostgreSQL, Docker/Compose, TLS, Reverse Proxy +- Gitea-Pipelines / CI +- Ollama bzw. lokales Detect-Modell +- Verschlüsselung at rest, DSFA, persistentes Audit ohne Prompts +- Quasi-Identifikatoren, vollständige Platzhalterprüfung, inhaltliche Minimierung +- Vollständiger Datenexport für ein öffentliches Produkt + +## 4. Bewusst ausgeschlossene Funktionen des nächsten Slices + +- Neue Reflection-Intents, universelle Continuation-Engine +- Self Model, semantisches Retrieval, Space Content Memory +- Chunk-and-Merge +- Voice und visuelle LLM-Verarbeitung +- Dialogmedien als direkte Dialogquelle (Bilder/Videos gehören im MVP in den Journal Editor) +- Frühere Space-Entries als Faktenquelle der Journalgenerierung + +## 5. Bewusste MVP-Grenzen dieses Freeze + +- Browser-Zurück: vorhandene Unsaved-Warnung und `beforeunload` reichen; keine Data-Router-Migration. +- Space-Kontext darf den Dialogzug stützen, nicht still die Generate-Quelle werden. +- Stufe 1 bleibt untrusted Auswahl/Annotation; autoritativ sind lokale Source Registry und `VerifiedArtifact`. Ungültiges Modell-JSON bricht Generate nicht ab: lokal volle Nutzerquellen, Abbruch nur ohne Nutzertext. +- Qualitative Modellfälle nicht als automatisch vollständig bewiesen bezeichnen. diff --git a/docs/architecture/functional/mvp_stand_und_abgleich.md b/docs/architecture/functional/mvp_stand_und_abgleich.md index 624035d..be7ee31 100644 --- a/docs/architecture/functional/mvp_stand_und_abgleich.md +++ b/docs/architecture/functional/mvp_stand_und_abgleich.md @@ -91,7 +91,7 @@ Kanshō ist lokal ein dialoggeführtes Journal auf FastAPI + SQLite und einer Re **Technische Keimzelle:** Auth-Sessions, Prompt-DB, Platzhalter, Feature-Check, Privacy Gateway, zwei Provider-Rollen (Maskierung / Sprachmodell), Context Builder, Retrieval als SQL-Schnittstelle, Writing Profile ohne Extra-LLM, Pronomenbindung nur im Dialogzug. -**Nicht gebaut und nicht behauptet als gebaut:** Space Content Memory, Threads-UI, semantisches Retrieval, Self Model, Insights, Related Spaces in der Nutzung, Papierkorb-UI, PostgreSQL, lokales Detect-Modell, Voice, Offline-Sync, Obsidian/mindnet/Kairo-Handoff, Contextual Continuation als Start, zweistufige Journalgenerierung. +**Nicht gebaut und nicht behauptet als gebaut:** Space Content Memory, Threads-UI, semantisches Retrieval, Self Model, Insights, Related Spaces in der Nutzung, PostgreSQL, lokales Detect-Modell, Voice, Offline-Sync, Obsidian/mindnet/Kairo-Handoff, Contextual Continuation als Ziel-IA. Papierkorb-UI, Entry-Liste, `include_existing` in der UI, Startkontinuität und erster Journal-Impuls: siehe §2.7 (seit 2026-08-25 im Code). Zweistufige Journalgenerierung: siehe §2.6. **Produktidentität im Slice:** der Kern ist *Erzählen → reflektierender Impuls → Journalentwurf*. Das ist eine legitime Keimzelle von „Personal Reflection Companion“, noch kein Begleiter über Monate, keine Wertearbeit, keine offenen Fäden. @@ -201,6 +201,47 @@ Tests: `backend/tests/test_profile_review.py`. --- +# 2.6 Journal-Budget und zweistufige Generierung (2026-08-25) + +Additiv zu 2.5. Schließt die frühere Slice-Lücke „zweistufige Journalgenerierung ist nicht gebaut“, ohne das Fachkapitel `writing_profile_and_journaling.md` zu ersetzen. Technische How-to: `../technical/mvp_implementation.md` §16. + +| Thema | Ist-Stand | Bewusst offen | +|---|---|---| +| Stufen | Stufe 1 Rekonstruktion (`mvp.journal_reconstruct`) lokal validiert; Stufe 2 Narration (`mvp.journal_generate`) ohne vollen Dialog. Beides Privacy Gateway, Klasse B. | Chunk-and-Merge für Tage über dem Fenster | +| Budget | 32K-Mindestfenster, Output-Reserve bis 4096, 15 % Marge, konservative 2-Zeichen/Token-Schätzung. OpenRouter-Metadaten gecacht. | Modellfamilien-Tokenizer | +| Compression | OpenRouter Context Compression für Journal explizit aus. Kein stilles Middle-Out. | — | +| Writing Brief | Task-Compiler feldweise ~3–4k Zeichen; volle Profile-Quelle unverändert. Anzeige-Dedup ohne Löschen. | Weitere Task-Compiler | +| Fehler | Kontrollierte Codes, kein Modellwechsel, Fail Closed/ZDR unverändert. | Persistente Audit-Tabelle ohne Prompts | + +Widerspruch, nicht stillschweigend eingeebnet: `mvp.md` §5.3.2 beschreibt den Journalentwurf als einen maskierten Call im Slice-Schnitt (Dialogzug vs. Entwurf). `writing_profile_and_journaling.md` §16 bevorzugt fachlich zwei Stufen. Die Umsetzung folgt der fachlichen Trennung Inhalt/Stil; Dialogzug bleibt ein Call. Beide Journal-Stufen gehen nur über das Privacy Gateway. + +Tests: `backend/tests/test_journal_budget.py`. + +**Additiv 2026-08-26:** Ungültiges Stufe-1-JSON bricht Generate nicht mehr ab. Stufe 2 erhält ein lokal erzeugtes `VerifiedArtifact` aus allen ausgewählten Nutzerquellen. Abbruch nur ohne Nutzertext (`no_user_sources`). Das Modell-JSON wird nicht still repariert. Eine Rohantwort mit Klartext-Identität wird nach Retry ebenfalls nicht übernommen; lokal entsteht ein Entwurf aus den Nutzerquellen. + +--- + +# 2.7 Freeze Candidate (2026-08-25) + +Additiv zu 2.6. Schließt lokal realisierbare MVP-Lücken für die Urlaubs-Testphase. Kein Target-Model-Vorbau. Abnahmeblatt: `mvp_freeze_candidate.md`. + +| Thema | Ist-Stand | Bewusst offen | +|---|---|---| +| Lokaler Testeinstieg | `scripts/test-mvp.ps1`: UTF-8, alle `test_*.py`, Dokumenttest, Frontend-Build, isolierte Temp-Daten, keine Live-Keys | Echte Modell-/Browserläufe | +| Backup/Restore | SQLite-Backup-API + Medien + Manifest/Checksummen; Restore prüft, bestätigt, legt Sicherheitsbackup an, überschreibt nicht still | Verschlüsselung, Produktions-Backup | +| Papierkorb | Liste, UI, Undelete, explizites Purge, Isolation | Hard-Delete von Dialogen, Account-Löschen | +| Chronologische Entries | Space-Entry-Liste neben der Tagesliste; gelöschte nur im Papierkorb | Semantische Suche | +| `include_existing` | UI-Auswahl, Default aus, Generate überschreibt nicht | — | +| Start `/` | Kontinuität: Fortsetzen-Aktion zum letzten aktiven Journal Day, sonst Space-Einstieg | Contextual Continuation als Ziel-IA | +| Erster Impuls | Journalspezifischer Adapter, Gateway, gleiche Wächter; ohne Kontext lokal neutral | Intent-Engine, Pattern-Infrastruktur | +| Dialogmedien | Nicht als Dialogquelle | Eigener späterer Slice | +| Generate-Quellen | Day-Conversations + Verified Artifact; Space-Entries nicht als Generate-Fakten | Space Content Memory | +| Browser-Zurück | Unsaved-Warnung + `beforeunload` | Data-Router-Blocker | + +Tests: `test_local_backup.py`, `test_journal_trash.py`, `test_journal_opening.py`, Erweiterungen in `test_mvp_journal.py`. + +--- + # 3. Abgleich gegen das Gesamtziel Quelle: `produktvision_und_produktidentitaet.md`, ergänzt um Dialog-, Memory- und Output-Kapitel. Das ist **keine** Slice-Checkliste. Ein „Gap“ hier heißt: das Zielmodell ist nicht da — erwartet, solange Foundation-Regel 23/24 gilt. Die spannenden Zeilen sind **Spannung** und **Drift-Risiko**. @@ -216,13 +257,13 @@ Quelle: `produktvision_und_produktidentitaet.md`, ergänzt um Dialog-, Memory- u | Erinnerung mit Herkunft | **Teilweise** | Messages bleiben. Draft/Entry-Version tragen relationale Source-Refs (API weiterhin ID-Listen). Nutzer sieht „zum Quelldialog“, keine Provenance-Kette, kein Re-Grounding. | | Hypothese statt Gewissheit | **Teilweise** | Impulse sind Nachrichten, nicht Fakten. Keine Hypothesen-Objekte, keine Confidence, keine Bestätigung vor Self-Aussagen (kein Self Model). | | Mensch entscheidet über Identität | **Erfüllt** als Abwesenheit | Kein Self-Model-Schreibpfad. Writing Profile ist als Formhinweis gekennzeichnet. | -| Persönliche Sprache | **Teilweise** | Lokaler Brief + Exzerpte. Keine Wortwahl-/Übergangs-Signale. Eine generative Stufe, nicht Inhalt/Stil getrennt. | +| Persönliche Sprache | **Teilweise** | Lokaler Brief + Exzerpte plus kompakter Task-Brief. Journalentwurf trennt Rekonstruktion und Narration (2026-08-25). Langfristige stilistische Entwicklung und Exemplar-Korpus bleiben begrenzt. | | Reflection before Action | **Bewusst offen** | Keine Action Candidates, keine Kairo-Übergabe. Scratch ist lokale Merkhilfe, kein Aufgabenmanager — Grenze gehalten. | | Keine Jinkendo-Duplikation | **Erfüllt** im Slice | Kein Tracker, keine Diagnose-UI, kein Kairo-Todo. `/dialog` ist Admin-Harness, kein zweiter produktiver Gesprächsraum. | | Menschenlesbares Langzeitarchiv (Obsidian) | **Gap** zum Ziel | Entries liegen in SQLite. Kein Export. | | Langfristige Kontinuität | **Teilweise** | Transcript und versionierte Entries überdauern Sitzungen. Kein Thread Memory, keine offenen Fragen, kein Point-in-Time Self. | | Einfache UX, differenzierter Kern | **Spannung** | UX ist einfach. Der Kern ist noch nicht differenziert (ein Call, Recency, Regex-Register). Das ist Slice-angemessen; es darf nicht als „der Kern *ist* Recency“ kanonisch werden. | -| Start = Kontinuität des Dialogs, kein Dashboard | **Gap** zur Identität | `/` ist ein Stub mit Link zum Journal. `/dialog` ist nicht mehr Teil der Nutzer-IA. Primärstart ist nicht der offene Day. | +| Start = Kontinuität des Dialogs, kein Dashboard | **Erfüllt** als MVP-Keim | `/` führt zum fortsetzbaren Journal Day oder zum Space-Einstieg. Kein Dashboard. Contextual Continuation als Ziel-IA bleibt später. | ## 3.2 Dialog-IA (Ziel vs Slice) @@ -354,7 +395,7 @@ Foundation-Verletzungen, die ein Refactor erzwingen würden, sind **nicht** gefu | Anforderung | Stand | Befund | |---|---|---| | Dialoggeführtes Journal, Entwurf aus dem Gespräch | **Erfüllt** | Explizites Generate, Öffnen als Entwurf | -| Redigieren, Medien, chronologisch wiederfinden | **Teilweise** | Editor mit Absätzen, Listen, Inline-Bild/Video und Unterschrift. Navigation über Tage, nicht eine Entry-Liste | +| Redigieren, Medien, chronologisch wiederfinden | **Erfüllt** im Slice-Schnitt | Editor mit Absätzen, Listen, Inline-Bild/Video. Navigation über Entry-Liste und Tage. | | Dialog berücksichtigt Tagesverlauf, Space, frühere Aussagen, Erzählweise | **Teilweise** | Tagesverlauf: voller Day-Dialog. Erzählweise: Formhinweise. Space/Früheres: Recency-Ausschnitte gespeicherter Entries, 5×400 Zeichen. **Ungenauigkeit** von §1 gegenüber der Umsetzung in §5.1 letzter Absatz und §8.2 | ## 5.2 Sichtbare Einheiten @@ -362,7 +403,7 @@ Foundation-Verletzungen, die ein Refactor erzwingen würden, sind **nicht** gefu | Einheit | Stand | Befund | |---|---|---| | Space anlegen, umbenennen, auswählen | **Erfüllt** | | -| Chronologische Entries des Spaces | **Teilweise** | Space-Seite zeigt **Tage**. Entries liegen unter dem Tag. | +| Chronologische Entries des Spaces | **Erfüllt** | Space-Seite zeigt Einträge (Datum, Titel, Versionshinweis) und weiterhin Tage. Gelöschte nur im Papierkorb. | | Journal Day ≠ Dialogzeit | **Erfüllt** | | | Originaldialog erhalten | **Erfüllt** | | | Gespräch aus dem Tag entfernen | **Erfüllt** | Hartes Löschen der Conversation. **Ungenauigkeit:** härter als Entry-Soft-Delete; kein Papierkorb für Dialoge | @@ -388,7 +429,7 @@ Foundation-Verletzungen, die ein Refactor erzwingen würden, sind **nicht** gefu | Bekannter Tagesplan (Cres-Beispiel) | **Teilweise / Ungenauigkeit** | Plan kann aus begrenzten Recency-Ausschnitten **gespeicherter Entries** oder **jüngerer Original-Conversations** desselben Space kommen. Kein Plan-Objekt, keine zeitliche Vor/Nach-Logik, kein semantisches Retrieval. | | Wiederkehrende Aktivität (Japanisch-Beispiel) | **Gap** | Keine Zählung über Morgenbeschreibungen. Recency kann den Treffer zufällig enthalten oder nicht. | | Keine triviale Gefühlsfrage nach selbst benannter Emotion | **Teilweise** | Prompt + `is_unearned_stance`. Kein Dialogzustand „bereits beantwortet“. | -| Kontextsensitiver Einstieg statt „Erzähl von deinem Tag“ | **Gap** | Nutzer schreibt die erste Zeile. Kanshō eröffnet nicht aus Plan/Space. Neutraler Start ist der Default, nicht der Fallback. | +| Kontextsensitiver Einstieg statt „Erzähl von deinem Tag“ | **Erfüllt** als begrenzte Keimzelle | Erster Impuls aus belegtem Plan oder offenem Tagesfaden; sonst neutral. Keine Pattern-Engine. | | Unterbrechen und Fortsetzen am selben Day | **Erfüllt** | Transcript bleibt. **Teilweise:** keine lokale Karte „welche Tagesabschnitte schon da sind“. | | Folgetag-Fortsetzung dem ursprünglichen Day zuordnen | **Erfüllt** | Nutzer öffnet den Day; Message-Zeit ist unabhängig. | @@ -400,8 +441,8 @@ Foundation-Verletzungen, die ein Refactor erzwingen würden, sind **nicht** gefu | Quellen: Day-Dialog inkl. Impulse als Kontext | **Erfüllt** | `user:` Erlebtes, `assistant:` Anschluss | | Space-Kontext soweit nötig | **Teilweise** | Generate bekommt **keinen** Prior-Entry-Block; nur Writing Profile + Day-Dialog. Dialogzug sieht Prior-Entries. | | Writing Profile | **Erfüllt** | Lokal, ohne Extra-LLM | -| Bestehenden Text einbeziehen | **Teilweise** | API `include_existing`. **Gap:** UI setzt das Flag nicht. | -| Verbinden, nicht durchreichen | **Teilweise** | Prompt: Szenen verbinden und Stimme aus dem Writing Profile; kurze Passagen ausformulieren ohne neue Fakten. `journal_shape` formt 1:1-Kopien und dünnt Namenswiederholungen. Qualität modellabhängig. Zweistufige Generierung (Inhalt, dann Stil) aus dem Fachkapitel: **nicht** gebaut, ein Call. | +| Bestehenden Text einbeziehen | **Erfüllt** | API und UI. Default nicht einbeziehen; Flag nur nach ausdrücklicher Auswahl. Nutzerfassung unangetastet. | +| Verbinden, nicht durchreichen | **Teilweise** | Prompt: Szenen verbinden und Stimme aus dem Writing Profile; kurze Passagen ausformulieren ohne neue Fakten. `journal_shape` formt 1:1-Kopien und dünnt Namenswiederholungen. Qualität modellabhängig. Zweistufige Generierung (Rekonstruktion, dann Narration) ist im Code (2026-08-25). Stufe 1 ist untrusted Auswahl/Annotation, nicht autoritative Semantik. | | Nicht erfinden | **Teilweise** | Prompt; keine lokale Faktenprüfung gegen den Dialog | ## 5.6 Writing Profile (`mvp.md` §9) @@ -426,15 +467,15 @@ Foundation-Verletzungen, die ein Refactor erzwingen würden, sind **nicht** gefu | Bilder ergänzen, anordnen | **Erfüllt** | Fließtext, Einfügen an der Schreibmarke, Ziehen im Text, Unterschrift. Alte Galerie-Dateien ohne Referenz: „Noch nicht im Text“. | | Videos im Eintrag darstellen | **Erfüllt** | MP4/WebM/MOV, lokal, direkt abspielen. Keine Videobibliothek. Metadaten in Video-Containern werden **nicht** gestrippt (**Gap** / bewusst dünn). | | Zeichnungen als Bilddatei | **Erfüllt** | als Upload | -| Soft Delete / Papierkorb | **Teilweise** | `deleted_at` + DELETE-API. Editor-Text: „In den Papierkorb“. **Gap:** keine Papierkorb-UI, kein Wiederherstellen Gelöschter, kein zweiter Schritt „endgültig löschen“. | +| Soft Delete / Papierkorb | **Erfüllt** | Soft Delete, Papierkorb-UI, Wiederherstellen, explizites endgültiges Löschen. Isolation geprüft. | | Version History sichtbar, Restore | **Erfüllt** | Aktuelle Fassung ohne Restore auf sich selbst; offene Edits lokal verwerfen. Navigation mit Dirty-Flag bestätigt (Links, Abmelden, `beforeunload`). Browser-Zurück nicht hart blockiert. | | Generate überschreibt Nutzerfassung nicht | **Erfüllt** | | -| Chronologische Liste, Space, Datum, Entry, Quelldialog | **Teilweise** | Über Space → Tage → Tag/Editor/Source. Keine Entry-Indexseite. | +| Chronologische Liste, Space, Datum, Entry, Quelldialog | **Erfüllt** | Space → Einträge und Tage → Editor/Source. | | Screen Journal/Spaces | **Erfüllt** | | -| Screen Journal Day / Dialog | **Teilweise** | Chat + Generate + lokale Stichpunkte (Return fokussiert neue Zeile). **Gap:** keine Medienoption am Dialog (MVP §13.2 Dialogquellen). Draft/Entry nur Links. | +| Screen Journal Day / Dialog | **Teilweise** | Chat + Generate + lokale Stichpunkte + `include_existing`. **Bewusst begrenzt:** keine Medienoption am Dialog (siehe `mvp.md` §10.2). | | Screen Editor | **Erfüllt** | Fließtexteditor, Markdown, Inline-Medien, Restore, Dirty-Schutz | | Screen Source Dialog | **Erfüllt** | | -| Start = Kontinuität des Dialogs | **Gap** zur Produktidentität | `/` ist ein Stub. Primärstart ist nicht der offene Day. `/dialog` ist Admin-Harness, nicht Nutzer-IA. | +| Start = Kontinuität des Dialogs | **Erfüllt** als MVP-Keim | `/` Fortsetzen oder Journal/Space. `/dialog` bleibt Admin-Harness. | | Tagesstichpunkte §10.1 | **Erfüllt** als lokale Liste | Klasse A, kein Egress (Test). Kein Aufgabenmanager. Offen bleibt eine spätere bewusste, maskierte Nutzung als Kontext. | ## 5.8 Acceptance Cases @@ -451,7 +492,7 @@ Foundation-Verletzungen, die ein Refactor erzwingen würden, sind **nicht** gefu | **H** Zwei Spaces, gleicher Tag | **Erfüllt** | | | **I** Persönlicher Stil | **Teilweise** | Import und Entries im Brief; Formhinweise. „Nicht in KI-Stil driftet“ ist Promptziel, nicht messbar erfüllt. | | **J** Versionsschutz | **Erfüllt** | Restore hängt Version an; Generate ändert Entry nicht; Entwurf öffnet separat; Übernehmen versioniert. | -| **K** Nachschlagen | **Erfüllt** | Space → Datum → Entry → Source. | +| **K** Nachschlagen | **Erfüllt** | Space → Entry-Liste und Datum → Entry → Source. | --- @@ -460,18 +501,18 @@ Foundation-Verletzungen, die ein Refactor erzwingen würden, sind **nicht** gefu 1. **`mvp.md` §1 und §5.1** versprechen „relevante frühere Aussagen“ und „wiederkehrende Aktivitäten“. Der Code liefert Recency-Ausschnitte finaler Entries plus begrenzte Original-Ausschnitte jüngerer Space-Conversations. Der klarstellende Absatz in §5.1 und §8.2 ist der gültige Implementierungsstand; die Beispiele §5.4–5.5 und Case C bleiben **schärfer als der Code** (kein Plan-Objekt, keine Wiederholungszählung). 2. **`mvp.md` §17** sprach vom „nächsten Schritt: technischen Plan ableiten“. Der Slice ist gebaut. Dieses Dokument ist der Abgleich; die Formulierung in §17 bleibt historisch. 3. **`mvp.md` §4.1 Konsolidierung** beschreibt narrative Kriterien. Der Slice nutzt lokale Signale derselben Dimensionen, keine LLM-Typklassifikation. -4. **`mvp.md` §6 Einstieg** verlangt einen kontextsensitiven ersten Impuls. Implementiert ist: der Mensch beginnt. -5. **`mvp.md` §8.2 Punkt 2** Space-Kontext bei der Generierung: Generate sieht Prior-Entries nicht. +4. **`mvp.md` §6 Einstieg** verlangt einen kontextsensitiven ersten Impuls. Seit 2026-08-25: journalspezifischer Opening-Adapter; ohne belegten Plan oder offenen Tagesfaden lokal neutral. +5. **`mvp.md` §8.2 Punkt 2** Space-Kontext bei der Generierung: Generate sieht Prior-Entries nicht. Für diesen Freeze bewusst so begrenzt (`mvp.md` §8.2.1). 6. **`mvp.md` §9.3** listet Lernsignale, von denen nur ein Teil heuristisch existiert. -7. **`mvp.md` §11.3 / §13.1–13.2** Papierkorb-UI und Dialog-Medien fehlen weiter. Inline-Medien, Unterschriften, Versions-UI und Dirty-Schutz im Editor sind vorhanden. +7. **`mvp.md` §11.3 / §13.1–13.2** Papierkorb-UI und Entry-Liste sind vorhanden. Dialog-Medien bleiben bewusst aus (`mvp.md` §10.2). 8. **Acceptance-Tests C/D/E** sind oft Prompt- oder Call-Zähl-Proxies, keine fachlichen Verhaltensbeweise. 9. **`backend_and_api.md`** beschreibt PostgreSQL als Rahmen; der Slice läuft auf **SQLite**. **Abweichung** vom technischen Ziel, lokal zulässig. 10. **Fehlerformat:** Rahmen will `{detail: string}`. Viele Router liefern `{detail: {code, message}}`. -11. **Nav `/dialog`** ist aus der Produkt-IA entfernt (Admin-Harness). Stub-Home bleibt. +11. **Nav `/dialog`** ist aus der Produkt-IA entfernt (Admin-Harness). Home ist Kontinuität, kein Dashboard. 12. **Detect über OpenRouter** darf Klartext nur in Development/Test sehen. Produktivmodus fällt auf Muster oder lokale Detect-URL zurück. Guardrail-Abstand zum lokalen Detect-Modell bleibt. -13. **Home `/`** widerspricht der Produktidentität „Kontinuität des Dialogs, kein Funktions-Dashboard“ nur schwach (Stub), erfüllt sie aber nicht. +13. **Home `/`** führt zum fortsetzbaren Journal Day oder zum Space-Einstieg. Contextual Continuation als Ziel-IA bleibt später. 14. **`handover.md`** ist Konzept-Bootstrap (Interviewstand 2026-08-19) und **kein** Code-Stand. Code-Stand ist dieses Dokument. -15. **Zweistufige Journalgenerierung** im Fachkapitel Writing Profile ist Zielmodell. Der Slice hat einen Generate-Call. `mvp.md` verlangt das Zweistufenmodell nicht zwingend; das Fachkapitel schon. Nicht als Slice-Bug lesen, nicht als erfüllt behaupten. +15. **Zweistufige Journalgenerierung** ist im Slice umgesetzt. Stufe 1 ist untrusted Auswahl/Annotation; lokale Source Registry und Verified Artifact sind autoritativ. Stufe 2 erhält vollständige lokal rehydrierte Nutzquellen. Ungültiges Modell-JSON der Stufe 1 führt seit 2026-08-26 nicht mehr zum Generate-Abbruch, sondern zum lokalen Artefakt. Qualitative Modellfälle bleiben nicht automatisch bewiesen. 16. **Tagesreflexion-Journal-Default** (Hintergrund-Entwurf am Abschluss, dezent anbieten) ist Fachstand der Nutzungssituation, nicht MVP-Text. Der Slice verlangt den Button. Kein Widerspruch, solange später kein stilles Überschreiben entsteht. --- @@ -488,10 +529,9 @@ Zielmodell und Non-Goals, nicht „vergessen im MVP“: - Tagesstichpunkte als Dialogkontext (bewusst lokal, siehe `mvp.md` §10.1) - Encryption at rest, DSFA, Multi-Gerät-Sync, PWA-Offline-Ausprägung - Lokales Detect-Modell (Ollama) als Ablösung der Testphase -- `include_existing` in der UI (API existiert) -- Zweistufige Journalgenerierung (Inhalt, dann Stil) -- Contextual Continuation als Start-IA +- Contextual Continuation als Ziel-IA (der Freeze hat nur eine journalspezifische Startkeimzelle) - Browser-Zurück als harter Unsaved-Blocker (Data-Router) +- Dialogmedien als direkte Dialogquelle --- @@ -501,12 +541,11 @@ Keine neue Horizontal-Konzeption. Getrennt halten, sonst baut ein Agent das Targ ## 8.1 Distanz zum **freigegebenen MVP-Text** (`mvp.md`) -1. **Kontextsensitiver Einstieg und Case C** – Plan/Wiederholung nur, wenn sie wirklich im Space-Kontext liegen; sonst ehrlich kein Cres-Beispiel, oder §5.4–5.5 an Recency anpassen. -2. **Generate und Space-Ausschnitte** – klären, ob Generate Prior-Entries braucht oder §8.2 korrigiert bleibt (Day-Dialog reicht). -3. **Editor-Lücken** – Papierkorb oder zweiter Löschschritt; Dialog-Medien oder Streichung in §13.2. -4. **Startlogik** – offenen Journal Day statt Stub-Home. -5. **Lokales Detect-Modell** – Klartext an externe Detect-Provider ist in Production blockiert; Ollama als Ablösung der Dev/Test-Phase bleibt offen. -6. **`include_existing`** – UI oder Flag aus der Spezifikation nehmen. +1. **Case C Wiederholung** – wiederkehrende Aktivität bleibt Recency, keine Zählung. Der erste Impuls greift nur belegte Pläne oder offene Tagesfäden auf. +2. **Generate und Space-Ausschnitte** – für diesen Freeze entschieden: Day-Conversations plus Verified Artifact, keine Prior-Entries als Generate-Fakten (`mvp.md` §8.2.1). +3. **Dialog-Medien** – bewusst gestrichen für den Slice (`mvp.md` §10.2); Editor-Medien bleiben. +4. **Lokales Detect-Modell** – Klartext an externe Detect-Provider ist in Production blockiert; Ollama als Ablösung der Dev/Test-Phase bleibt offen. +5. **Qualitative Modellfälle** – C/D/E/I bleiben modellabhängig, nicht durch Tests vollständig bewiesen. ## 8.2 Distanz zum **Gesamtziel** (nicht jetzt bauen) diff --git a/docs/architecture/technical/admin_diagnostics.md b/docs/architecture/technical/admin_diagnostics.md index 332e7ac..5d5d9a4 100644 --- a/docs/architecture/technical/admin_diagnostics.md +++ b/docs/architecture/technical/admin_diagnostics.md @@ -53,7 +53,7 @@ Nicht übernehmen: Mitai-Admin für Körpertarife, Coupons, Training Types als K ## 4.2 Implementierungsstand (Dialog-Testspur) -**Status: Code vorhanden, Testphase.** Nach jedem Dialogzug und nach Generate erhält die Admin-Rolle `decision` und `trace` in der API-Antwort. Die Journal- und Dialogseite zeigen Zweck, Prompt, Provider/Modell, Detect, maskierten Egress und Antworten. Klasse-A-Mapping bleibt unsichtbar. Ohne Admin-Rolle fehlen diese Felder. Nicht persistiert, nicht für den Regelbetrieb gedacht. +**Status: Code vorhanden, Testphase.** Nach jedem Dialogzug und nach Generate erhält die Admin-Rolle `decision` und `trace` in der API-Antwort. Die Journal- und Dialogseite zeigen Zweck, Prompt, Provider/Modell, Detect, maskierten Egress und Antworten. Dialogzug bleibt einstufig. Nach Journal-Generate zeigt die Testspur zwei Stufen (Rekonstruktion, Journalformulierung) plus `trace.budget` (geschätzte und tatsächliche Tokens, Kosten soweit geliefert, Fenster, Output-Reserve, Marge, Compression-Status, Abbruchgrund). Klasse-A-Mapping bleibt unsichtbar. Ohne Admin-Rolle fehlen diese Felder. Nicht persistiert, nicht für den Regelbetrieb gedacht. Volle Prompts bleiben Test-Einblick des aktuellen Responses, nicht Audit-Default und nicht globale Historie. ## 5. Entscheidungsstand diff --git a/docs/architecture/technical/ai_architecture.md b/docs/architecture/technical/ai_architecture.md index fe06fbb..44e30d4 100644 --- a/docs/architecture/technical/ai_architecture.md +++ b/docs/architecture/technical/ai_architecture.md @@ -51,11 +51,14 @@ Technische Mindestanforderung unabhängig von der späteren Zerlegung: - Context Builder (wann immer er existiert) erzeugt Internal Context. - Gateway erzeugt External Model Context. - Antworten werden validiert und demaskiert, bevor sie lokal weiterverwendet werden. +- Untrusted Modellausgabe, die als Inhaltsquelle für eine Folgestufe dienen soll, durchläuft die lokale Provenienzprüfung (`provenance_verification.md`). Das gilt unabhängig vom späteren Intent; die konkrete Coverage-Policy setzt der jeweilige Adapter. - Der aktuelle Fachstand zur Strukturierungsautonomie ändert weder Egress-Klasse noch Nutzerhoheit über Identität. ## 5. Context-Fenster -Der heutige Fachstand geht davon aus, dass ein LLM-Kontextfenster nicht ausreicht (`memory_and_context.md`). Orchestrierung von Ausschnitten und Re-Grounding bleibt an diesen Stand gekoppelt, ist aber **kein** vorgezogenes Speicherschema. Algorithmen und Token-Budgets: offen. +Der heutige Fachstand geht davon aus, dass ein LLM-Kontextfenster nicht ausreicht (`memory_and_context.md`). Orchestrierung von Ausschnitten und Re-Grounding bleibt an diesen Stand gekoppelt, ist aber **kein** vorgezogenes Speicherschema. + +**Journalgenerierung (2026-08-25):** Für den expliziten Journalentwurf gibt es ein lokales Budgetmodell (`mvp_implementation.md` §16, `privacy_gateway.md` §9.3): Mindestfenster 32K, Output-Reserve, Sicherheitsmarge, konservative Zeichenschätzung, kein stilles Middle-Out. Das ist keine globale Token-Orchestrierung für Dialog, Memory oder Resurfacing. Resurfacing und Saturation (aktueller Fachstand) steuern Relevanz, nicht die Egress-Menge. Guardrails bleiben vorrangig. @@ -82,4 +85,4 @@ Siehe Interview F3. Zusätzlich: Wie wird das Writing Profile in die Journal-Pip ## 9. Querverweise - Fachlich: `../functional/guardrails.md`, `../functional/reflection_intelligence.md`, `../functional/writing_profile_and_journaling.md` -- Technisch: `platform_extensibility.md`, `privacy_gateway.md`, `admin_diagnostics.md` +- Technisch: `platform_extensibility.md`, `privacy_gateway.md`, `admin_diagnostics.md`, `provenance_verification.md` diff --git a/docs/architecture/technical/backend_and_api.md b/docs/architecture/technical/backend_and_api.md index e967c9f..8f323c7 100644 --- a/docs/architecture/technical/backend_and_api.md +++ b/docs/architecture/technical/backend_and_api.md @@ -89,19 +89,20 @@ Konkreter Pfad: offen, Muster übernommen. Produkt-Endpunkte hinter Session-Auth, Isolation über `profile_id`: -- Spaces: `GET/POST /spaces`, `GET/PATCH /spaces/{id}`, `POST /spaces/{id}/days` +- Spaces: `GET/POST /spaces`, `GET/PATCH /spaces/{id}`, `POST /spaces/{id}/days`, `GET /spaces/{id}/entries`, `GET /spaces/{id}/trash` - Days: `GET /days/{id}` inkl. Conversations, Current Draft, Entries, `consolidation_offer` -- Dialog: `POST /days/{id}/conversations`, `POST /conversations/{id}/turn` (ein Modell-Call, bei Erzähler-Fortsetzung einmal Korrektur), `GET /conversations/{id}`, `DELETE /conversations/{id}` (Profil-isoliert; Nachrichten und Derived dieser Conversation gehen mit) +- Start: `GET /continuity` +- Dialog: `POST /days/{id}/conversations` (erster Impuls), `POST /conversations/{id}/turn` (ein Modell-Call, bei Erzähler-Fortsetzung einmal Korrektur), `GET /conversations/{id}`, `DELETE /conversations/{id}` (Profil-isoliert; Nachrichten und Derived dieser Conversation gehen mit) - Status: `GET /egress-status` (Provider bereit / fail-closed) - Derselbe Zug auch unter `POST /api/dialogue/conversations/{id}/turn` - Admin-Antwort zusätzlich `decision` und `trace` (Testphase) -- Generate: `POST /days/{id}/generate` nur explizit; optionale `conversation_ids`; ohne Auswahl kein stilles Mergen -- Entries: `POST /entries`, `GET /entries/{id}`, Versionen, Restore, Soft-Delete. Speichern aus einem Entwurf mit `entry_id` + `origin=accepted_draft` legt eine neue Version desselben Entry an. +- Generate: `POST /days/{id}/generate` nur explizit; optionale `conversation_ids`; `include_existing` nur nach ausdrücklicher UI-Auswahl; ohne Auswahl kein stilles Mergen +- Entries: `POST /entries`, `GET /entries/{id}`, Versionen, Versions-Restore, Soft-Delete, `POST /entries/{id}/undelete`, `POST /entries/{id}/purge`. Speichern aus einem Entwurf mit `entry_id` + `origin=accepted_draft` legt eine neue Version desselben Entry an. - Media: Upload/GET/DELETE; Bilder und einzelne Videos; Position/Unterschrift im Entry-Body als Markdown `![…](kansho-media:)` - Tagesstichpunkte: `PATCH /days/{id}/scratch` — lokal, nicht Context-Builder - Writing Profile: `GET /writing-profile`, Import (optional `occurred_at`, `context_hint`), Korpus `GET/POST /writing-profile/corpus`, Rebuild, `PATCH` Governance, Facet-Edit/Lock, Trait-Edit `PATCH /writing-profile/traits/{slug}`, Vorschläge annehmen/verwerfen. Brief ist eine abgeleitete Sicht aus Traits und zeitgestempelten Quellen. Media-Token werden vor dem Brief entfernt. JSON-Export/Restore: `GET /writing-profile/export`, `POST /writing-profile/restore`. Review: `GET /writing-profile/review`, `POST /writing-profile/review/paste`, `POST /writing-profile/review/api`, `POST /writing-profile/review/import`, `POST /writing-profile/review/accept`, `POST /writing-profile/review/reject`. Initial Build: `POST /writing-profile/initial-build/paste`, `POST /writing-profile/initial-build/api`. API und Copy/Paste nutzen denselben Vertrag `kansho.profile_analysis_package` / `kansho.profile_analysis_result` (`mode` `initial_build`\|`review`, `corpus`, `existing_before_new`, `expected_result`). Legacy-`kind` `kansho.profile_review_*` bleibt lesbar. Import erzeugt ein Proposal, kein Current Profile; Übernahme erst nach Accept. Der Current Brief ist kein Import-/Exportformat. - Interaction Profile: `GET /interaction-profile`, `PATCH` Governance und Präferenzen, Vorschläge annehmen/verwerfen. Keine Ableitung aus Nicht-Widersprechen. Der Slot liegt unter `/api/journal` nur als Settings-Nachbar, nicht als Journal-Artefakt. JSON-Export/Restore: `GET /interaction-profile/export`, `POST /interaction-profile/restore`. -- Beide Profile: `GET /profiles/export`, `POST /profiles/restore` (`kind: kansho.profiles`). Kein Journal-Backup. +- Beide Profile: `GET /profiles/export`, `POST /profiles/restore` (`kind: kansho.profiles`). Journal-Backup ist `local_backup.py`, nicht dieser Profil-Export. `/api/dialogue/*` bleibt Low-Level-Source und Admin-Diagnose, nicht die Produkt-IA. Context Builder und Retrieval liegen nicht im Frontend. diff --git a/docs/architecture/technical/documentation_index.md b/docs/architecture/technical/documentation_index.md index 5f0ec20..4be4f1b 100644 --- a/docs/architecture/technical/documentation_index.md +++ b/docs/architecture/technical/documentation_index.md @@ -38,12 +38,13 @@ Fachlich zusätzlich immer bei Querschnittsentscheidungen: | `data_architecture.md` | Persistenzgrenzen, Entitäten | | `memory_storage_and_offline.md` | Dialogspeicher, Sync, Offline | | `privacy_gateway.md` | Trust Zones, Egress, Demasking | +| `provenance_verification.md` | Intent-neutrale Provenienz- und Verifikationsschicht, `VerifiedArtifact` | | `ai_architecture.md` | Inferenzpfad, Agenten | | `integrations_technical.md` | Technische Handoffs | | `admin_diagnostics.md` | Diagnoseansicht | | `voice_and_media.md` | Sprache und Transkription | | `security.md` | Security-Baseline | -| `mvp_implementation.md` | **Kanonisches Home der technischen MVP-Umsetzung** (Laufzeit, Module, Daten, API, Pfade; Profile-Analyse Export/Import §15). Fit-Gap bleibt fachlich. | +| `mvp_implementation.md` | **Kanonisches Home der technischen MVP-Umsetzung** (Laufzeit, Module, Daten, API, Pfade; Profile-Analyse Export/Import §15). Fit-Gap bleibt fachlich. Freeze Candidate: `../functional/mvp_freeze_candidate.md`. | ## 3. Empfohlene Context Bundles @@ -73,9 +74,10 @@ Fachlich zusätzlich immer bei Querschnittsentscheidungen: 2. `privacy_gateway.md` 3. `platform_extensibility.md` 4. `ai_architecture.md` -5. `security.md` -6. fachlich: `../functional/guardrails.md` -7. bei Bedarf: `../functional/self_model_and_lived_experience.md`, `../functional/memory_and_context.md` +5. `provenance_verification.md` +6. `security.md` +7. fachlich: `../functional/guardrails.md` +8. bei Bedarf: `../functional/self_model_and_lived_experience.md`, `../functional/memory_and_context.md` ### Dialogspeicher / Memory / Offline @@ -105,15 +107,16 @@ Fachlich zusätzlich immer bei Querschnittsentscheidungen: Kanonisches Home **wie** der Slice gebaut ist: `mvp_implementation.md`. Fachliches Fit-Gap-Home bleibt `../functional/mvp_stand_und_abgleich.md`. 1. `mvp_implementation.md` -2. `memory_storage_and_offline.md` §6.2 -3. `backend_and_api.md` §5.1 -4. `privacy_gateway.md` §9.1–9.2 -5. `frontend_pwa_shell.md` §6.1 -6. fachlich: `../functional/mvp.md` -7. `../functional/implementation_foundation.md` -8. `../functional/mvp_stand_und_abgleich.md` -9. bei Gegenlesung gegen das Gesamtziel: `../functional/produktvision_und_produktidentitaet.md` -10. bei Bedarf: `../functional/writing_profile_and_journaling.md`, `../functional/guardrails.md` +2. `provenance_verification.md` +3. `memory_storage_and_offline.md` §6.2 +4. `backend_and_api.md` §5.1 +5. `privacy_gateway.md` §9.1–9.2 +6. `frontend_pwa_shell.md` §6.1 +7. fachlich: `../functional/mvp.md` +8. `../functional/implementation_foundation.md` +9. `../functional/mvp_stand_und_abgleich.md` +10. bei Gegenlesung gegen das Gesamtziel: `../functional/produktvision_und_produktidentitaet.md` +11. bei Bedarf: `../functional/writing_profile_and_journaling.md`, `../functional/guardrails.md` ### Voice / PWA-Offline @@ -131,5 +134,5 @@ Vor Vereinfachungen die Invariantenliste in `technische_zielarchitektur.md` §3. ## 5. Code-Gerüst -Lokaler Produktrahmen in `frontend/` und `backend/`. Zusammenhängende technische Umsetzung: `mvp_implementation.md`. Auth, Nutzerverwaltung, Prompt-DB, Platzhalter, Feature-Check, Privacy Gateway. **Dialog-Layer 0:** Conversations/Messages/`usage_sessions`, Thread-/Space-Identität, Derived-Hülle mit Provenance. **MVP-Journal-Slice:** nutzersichtbare Spaces, Journal Days, Dialogzug (ein Call), explizite Draft-Generierung, versionierte Entries (Übernehmen = neue Version, Dirty-Schutz im Editor), Inline-Medien, lokale Stichpunkte ohne Egress, Writing Profile (Hülle mit dynamischen Traits, Initial Profile Build). Fit-Gap: `../functional/mvp_stand_und_abgleich.md` (2026-08-25). Konfigurierbare Prompts `mvp.dialogue_turn`, `mvp.journal_generate`, `mvp.entity_detect`, `mvp.profile_review`. Provider-URL/Modell/Policy unter Admin → Schnittstellen; Keys nur in `backend/.env`. Schichten: Maskierung, Dialogzug, Journalentwurf, explizite Profile-Review. Admin-Testspur am Dialogzug (Egress, Antwort, Operation). Privacy-Pfad durchgängig, Security Layer nicht vollständig (`privacy_gateway.md` §9.2). Kein Stripe, keine Threads-UI, kein semantisches Retrieval. SQLite nur lokal. +Lokaler Produktrahmen in `frontend/` und `backend/`. Zusammenhängende technische Umsetzung: `mvp_implementation.md`. Auth, Nutzerverwaltung, Prompt-DB, Platzhalter, Feature-Check, Privacy Gateway. **Dialog-Layer 0:** Conversations/Messages/`usage_sessions`, Thread-/Space-Identität, Derived-Hülle mit Provenance. **MVP-Journal-Slice:** nutzersichtbare Spaces, Journal Days, Dialogzug (ein Call), explizite Draft-Generierung, versionierte Entries (Übernehmen = neue Version, Dirty-Schutz im Editor), Inline-Medien, lokale Stichpunkte ohne Egress, Writing Profile (Hülle mit dynamischen Traits, Initial Profile Build). Fit-Gap: `../functional/mvp_stand_und_abgleich.md` (2026-08-25). Konfigurierbare Prompts `mvp.dialogue_turn`, `mvp.journal_reconstruct`, `mvp.journal_generate`, `mvp.entity_detect`, `mvp.profile_review`. Persönliche Folgestufen konsumieren ein lokal erzeugtes `VerifiedArtifact` (`provenance_verification.md`); der Journal-Adapter setzt die Policy. Provider-URL/Modell/Policy unter Admin → Schnittstellen; Keys nur in `backend/.env`. Schichten: Maskierung, Dialogzug, Journalentwurf, explizite Profile-Review. Admin-Testspur am Dialogzug (Egress, Antwort, Operation). Privacy-Pfad durchgängig, Security Layer nicht vollständig (`privacy_gateway.md` §9.2). Kein Stripe, keine Threads-UI, kein semantisches Retrieval. SQLite nur lokal. diff --git a/docs/architecture/technical/frontend_pwa_shell.md b/docs/architecture/technical/frontend_pwa_shell.md index 8fdc0b0..443a091 100644 --- a/docs/architecture/technical/frontend_pwa_shell.md +++ b/docs/architecture/technical/frontend_pwa_shell.md @@ -78,15 +78,16 @@ Business-Logik bleibt im Backend (`backend_and_api.md`). **Status: Code vorhanden, keine finale Start-IA.** Technische Umsetzung der Routen: `mvp_implementation.md` §6. Fit-Gap zu Produktstart und `/dialog`: `../functional/mvp_stand_und_abgleich.md`. -Die Shell-Nav bleibt Start / Dialog / Journal / Einstellungen. Der MVP-Slice füllt `/journal`: +Die Shell-Nav bleibt Start / Journal / Einstellungen. Der MVP-Slice füllt `/journal`: - `/journal` Space-Liste -- `/journal/:spaceId` Chronologie -- `/journal/:spaceId/:dayId` Dialog, Generate und lokale Tagesstichpunkte +- `/journal/:spaceId` Entry-Chronologie und Tage +- `/journal/:spaceId/trash` Papierkorb +- `/journal/:spaceId/:dayId` Dialog, Generate, `include_existing`, lokale Tagesstichpunkte - `/journal/:spaceId/:dayId/entry` Fließtexteditor (Markdown, Inline-Bild/Video, Versionen, Dirty-Flag) - `/journal/:spaceId/:dayId/source/:conversationId` Quelldialog -`/` bleibt dünn und verweist auf das Journal. `/dialog` ruft in der Testphase denselben Dialogzug auf wie der Journal Day (`POST .../turn`), bleibt aber ohne Space-Chronologie und ist nicht die zweite Produkt-IA. Home wird kein Dashboard. Der Journal-Editor führt ein Dirty-Flag; In-App-Navigation mit ungespeicherten Änderungen wird bestätigt. Stichpunkte: Return erzeugt eine neue Zeile und fokussiert sie. +`/` ist Kontinuität: Fortsetzen des letzten aktiven Journal Day oder Space-Einstieg, kein Dashboard. `/dialog` bleibt Admin-Harness. Der Journal-Editor führt ein Dirty-Flag; In-App-Navigation mit ungespeicherten Änderungen wird bestätigt. Browser-Zurück bleibt über `beforeunload` bewusst dünn. Stichpunkte: Return erzeugt eine neue Zeile und fokussiert sie. Admin sieht unter dem Gespräch eine **Testspur**: intern gerenderter Prompt, maskierter Egress, Rohantwort, demaskierte Antwort, gewählte Dialogoperation. Keine Mapping-Tabelle. Nicht die produktive Nutzer-IA. diff --git a/docs/architecture/technical/mvp_implementation.md b/docs/architecture/technical/mvp_implementation.md index ba097f1..2c5868e 100644 --- a/docs/architecture/technical/mvp_implementation.md +++ b/docs/architecture/technical/mvp_implementation.md @@ -71,7 +71,9 @@ Einstieg: `backend/main.py` (Router: auth, users, dialogue, journal, prompts, pl | `routers/journal.py` | Produkt-API `/api/journal` | | `routers/dialogue.py` | Low-Level-Source `/api/dialogue`; derselbe Turn-Pfad | | `dialogue_turn.py` | Ein generativer Call, JSON `{operation, impulse}`, lokale Register-/Wächter, optional Repair | -| `journal_generate.py` | Expliziter Draft-Call | +| `journal_generate.py` | Expliziter Draft: Stufe 1 Rekonstruktion, Stufe 2 Narration | +| `journal_reconstruct.py` | Lokale Validierung der Stufe-1-JSON | +| `prompt_budget.py` / `model_catalog.py` | Kontextfenster, Output-Reserve, konservative Tokenschätzung | | `journal_store.py` | Days, Drafts, Entries, Versionen, Scratch | | `journal_policy.py` | Explizites Generate, Origins, Konsolidierungsangebot aus lokalen Signalen | | `journal_body.py` / `journal_shape.py` | Titel, Markdown-Body, lokale Nachformung | @@ -88,7 +90,7 @@ Einstieg: `backend/main.py` (Router: auth, users, dialogue, journal, prompts, pl | `providers.py` / `provider_settings.py` | Provider-Rollen | | `db.py` + `schema.sql` | SQLite, Migrationen | -Prompts in der DB, nicht im Anwendungscode: `mvp.dialogue_turn`, `mvp.journal_generate`, `mvp.entity_detect`, `mvp.profile_review`. Seed: `backend/config/prompts.seed.json`. Unangetastete Templates aktualisiert `init_db`. +Prompts in der DB, nicht im Anwendungscode: `mvp.dialogue_turn`, `mvp.journal_reconstruct`, `mvp.journal_generate`, `mvp.entity_detect`, `mvp.profile_review`. Seed: `backend/config/prompts.seed.json`. Unangetastete Templates aktualisiert `init_db`. --- @@ -128,7 +130,7 @@ Kernpfade: Context Builder → Gateway (`purpose=dialogue_turn`) → ein Generate-Call → Impuls speichern. Pronomenbindung nur hier in `user:`-Zeilen. 2. **Journalentwurf** `POST /days/{id}/generate` - Nur explizit. Optional `conversation_ids`. Ohne Auswahl kein stilles Mergen. Gateway `purpose=journal_generate`. Pronomen ungebunden. Entry-`current_version_id` bleibt unangetastet. UI öffnet `?draft=1`. + Nur explizit. Optional `conversation_ids`. Ohne Auswahl kein stilles Mergen. Zwei Gateway-Zwecke nacheinander: `journal_reconstruct`, dann `journal_generate`. Pronomen ungebunden. Entry-`current_version_id` bleibt unangetastet. UI öffnet `?draft=1`. 3. **Speichern** `POST /entries` Mit `entry_id` + `origin=accepted_draft`: neue Version desselben Entry. Sonst neuer Entry oder `user_edit`. @@ -141,7 +143,7 @@ Kernpfade: 7. **Profile Review** `GET/POST /writing-profile/review…` → lokale Evidenz, explizite API- oder Paste-Review, kein Call am Dialogzug. -`include_existing` existiert in der API, die UI setzt es nicht. +`include_existing` existiert in der API und in der UI; Default ist aus. Die Nutzerfassung bleibt unangetastet. Bei gesetztem Flag gilt die gespeicherte Entry-Fassung; ein liegengebliebener Generate-Draft wird nur verwendet, wenn es noch keinen Entry gibt. Fehlerkörper oft `{detail: {code, message}}`, Rahmen will `{detail: string}` — Abweichung, Fit-Gap. @@ -153,7 +155,7 @@ Shell: `frontend/src/App.jsx`, Nav `config/appNav.js` (Start / Journal / Einstel | Route | Seite | Technik | |---|---|---| -| `/` | `HomePage.jsx` | Stub, Link zum Journal | +| `/` | `HomePage.jsx` | Kontinuität: Fortsetzen des letzten aktiven Journal Day oder Space-Einstieg | | `/dialog` | `DialoguePage.jsx` | **Admin-only Dev/Test-Harness**, nicht in der Produkt-Nav. Unzugeordnete Conversations. | | `/journal` … `/journal/:spaceId/:dayId` | Spaces, Space, `JournalDayPage.jsx` | Chat, Generate, `DayScratch.jsx` | | `…/entry` | `JournalEditorPage.jsx` | TipTap, Markdown in `journal/document.js`, Dirty-Flag `context/UnsavedChanges.jsx` | @@ -174,7 +176,9 @@ Editor speichert Markdown, nicht HTML. Medien-Token bleiben lokal. Dirty: In-App Explizite Writing-Profile-Review ist ein weiterer Generate-Zweck (`profile_review`), nicht der Dialogzug und nicht nach jedem Turn. -Interner Context-Builder (Klartext, lokal): fordert Selection-Specs an (`day_messages`, `space_entries`, `space_recent_sources`, `writing_profile`). Recency-Kappen (Day 80 Messages; 5 Space-Entries × 400 Zeichen; 3 Space-Source-Conversations × 400 Zeichen User-Ausschnitt) liegen in `retrieval.py`, nicht in der Context-Builder-API. Dialogzug: aktuelle Conversation primär, plus begrenzte frühere Original-Conversations desselben Space (ohne denselben Journal Day voll zu laden). Generate ohne Prior-Entries, mit Writing Profile. `WritingProfile` steuert Journaltext, nicht den Dialogue Kernel; der Turn hat einen leeren `interaction_hint`-Slot. Retrieval austauschbar, keine Embeddings. +**Additiv 2026-08-25:** Der Journalentwurf ist fachlich zwei Stufen (Rekonstruktion, dann Narration), also zwei Generate-Calls hinter derselben expliziten Generate-Aktion. Dialogzug bleibt ein Call. Details: §16. + +Interner Context-Builder (Klartext, lokal): fordert Selection-Specs an (`day_messages`, `space_entries`, `space_recent_sources`, `writing_profile`). Recency-Kappen (Day 80 Messages für den Dialogzug; Journal-Generate nutzt Tokenbudget statt der 80er-Kappe, Overflow bricht ab; 5 Space-Entries × 400 Zeichen; 3 Space-Source-Conversations × 400 Zeichen User-Ausschnitt) liegen in `retrieval.py`, nicht in der Context-Builder-API. Dialogzug: aktuelle Conversation primär, plus begrenzte frühere Original-Conversations desselben Space (ohne denselben Journal Day voll zu laden). Generate-Stufe 1: Day-Messages mit Budget. Generate-Stufe 2: Writing Profile plus validierte Rekonstruktion, ohne den vollen Dialog. `WritingProfile` steuert Journaltext, nicht den Dialogue Kernel; der Turn hat einen leeren `interaction_hint`-Slot. Retrieval austauschbar, keine Embeddings. Gateway-Verfahren und offener Security Layer: `privacy_gateway.md` §9.1–9.2. @@ -192,8 +196,13 @@ Gateway-Verfahren und offener Security Layer: `privacy_gateway.md` §9.1–9.2. | Writing Profile | `backend/tests/test_writing_profile.py` | | Profile Governance | `backend/tests/test_profile_governance.py` | | Profile Review Pipeline | `backend/tests/test_profile_review.py` | +| Journal-Budget / zweistufige Generierung | `backend/tests/test_journal_budget.py` | +| Lokales Backup/Restore | `backend/tests/test_local_backup.py` | +| Papierkorb / Purge | `backend/tests/test_journal_trash.py` | +| Erster Journal-Impuls | `backend/tests/test_journal_opening.py` | | Writing-Profile-Hülle / Initial Build | `backend/tests/test_profile_review.py`, `backend/tests/test_profile_governance.py` | | Markdown-Runde | `frontend/src/journal/document.test.js` | +| Lokaler Abnahmelauf | `scripts/test-mvp.ps1` | Tests beweisen API-Verträge und Invarianten, nicht Dialogqualität. @@ -213,7 +222,7 @@ Nur die **technische** Lage. Bewertung gegen Spec und Vision: Fit-Gap. | Security Layer | Pfad da; Minimierung/Audit/Encryption fehlen weiter | `privacy_gateway.md` §9.2 | | Offline-Sync, Voice-Egress, Compose-Prod | ungebaut | jeweiliges Kapitel | -Nicht im Code: Threads-UI, semantisches Retrieval, Session-Summaries, Related-Space-UI, Papierkorb-UI, lokales Detect-Modell, Obsidian/mindnet/Kairo-Handoff. +Nicht im Code: Threads-UI, semantisches Retrieval, Session-Summaries, Related-Space-UI, lokales Detect-Modell, Obsidian/mindnet/Kairo-Handoff. Papierkorb-UI, Entry-Liste, lokales Backup/Restore und erster Journal-Impuls: 2026-08-25. --- @@ -227,7 +236,7 @@ Additiv zum Slice, 2026-08-25. Kein Target-Model-Vorbau. - Context Builder spricht Selection-Specs; Recency bleibt die aktuelle Retrieval-Heuristik, nicht die fachliche API. - Dialogzug kann begrenzte jüngere Original-Conversations desselben Space sehen, ohne denselben Journal Day voll zu laden. - Externes Klartext-Detect ist kein Produktmodus (`KANSHO_ENV=production` fällt auf Muster zurück). Lokales Detect bleibt austauschbare Rolle. -- Response Validation blockiert Klartext-Identität vor Demask. +- Response Validation blockiert Klartext-Identität vor Demask. Dialog-Egress maskiert den ganzen Prompt inkl. Opening/Space-Kontext; ein Leak bricht den Zug nicht leer ab. Journal-Generate bricht bei einem Leak nach Retry nicht ab: lokales Quellenartefakt bzw. lokaler Entwurf, die leckende Modellantwort wird nicht verwendet. - Konsolidierung nutzt lokale Signale, nicht `count >= 2`. - `/dialog` ist Admin-Harness, nicht Produkt-IA. - `WritingProfile` hängt nicht am Dialogue Kernel; `interaction_hint` ist ein leerer Slot. @@ -239,7 +248,21 @@ Additiv zum Slice, 2026-08-25. Kein Target-Model-Vorbau. - Interaction-Profile-Engine, UI und Persistenz - Reflection-Space-Typen, Journal-Default am Gesprächsabschluss - Lokales Detect-Modell, Audit ohne Prompt, Encryption, inhaltliche Minimierung -- Stub-Home / Contextual Continuation als Start-IA +- Contextual Continuation als Ziel-IA (Home ist eine journalspezifische Fortsetzen-Keimzelle) + +--- + +# 17. Freeze Candidate, Backup, Papierkorb, Opening (2026-08-25) + +Additiv zum Slice. Fachliche Abnahme: `../functional/mvp_freeze_candidate.md`. + +- Lokaler Testeinstieg `scripts/test-mvp.ps1` (UTF-8, isolierte Temp-Daten, keine Live-Keys; Fake nur in den Suites, die ihn setzen). +- Backup: `backend/local_backup.py`, Einstieg `scripts/backup-local.ps1`. SQLite-Backup-API, Medien, Manifest, keine Secrets. Restore bestätigt, prüft Checksummen, legt Sicherheitsbackup an, bricht bei geöffneter DB ab. +- Papierkorb: `GET /spaces/{id}/entries`, `GET /spaces/{id}/trash`, `POST /entries/{id}/undelete`, `POST /entries/{id}/purge`. +- Erster Impuls: `journal_opening.py`, derselbe Gateway-Pfad, `{{opening_hint}}`. +- Home: `GET /continuity`. + +Bewusst nicht: Docker, Postgres, Ollama, Verschlüsselung, Dialogmedien als Quelle, Space-Entries als Generate-Fakten. --- @@ -340,12 +363,82 @@ Tests: `backend/tests/test_profile_review.py`. --- +# 16. Journal-Budget und zweistufige Generierung (2026-08-25) + +Additiv zu §7 und §15. Keine stillschweigende Kürzung autobiografischer Fakten. Chunk-and-Merge bleibt eine spätere Option, ist in diesem Schritt nicht gebaut. + +## 16.1 Ablauf + +Explizites Generate durchläuft zwei maskierte Gateway-Calls, beide Klasse B, Fail Closed, ZDR unverändert: + +1. **Stufe 1 – inhaltliche Rekonstruktion** (`mvp.journal_reconstruct`, `purpose=journal_reconstruct`). Eingabe: budgetierter Tagesdialog (alle Nutzerzeilen vollständig; vorausgehende Assistant-Zeile nur als Antwortkontext derselben Conversation). Jede nichtleere Nutzerzeile erhält lokal eine stabile Quellen-ID (`u1`, `u2`, …), Assistant-Zeilen `a1`, `a2`, …. Ausgabe: strukturiertes JSON mit `source_order` (lokale Quellenreihenfolge) und `chronology` (Ereignischronologie, nur umordenbar bei expliziten Zeit-/Reihenfolgeangaben). Lokal fail-closed validiert: vollständige Abdeckung der Nutzer-IDs, keine Umordnung oder Neunummerierung vor der Prüfung, kein Umschreiben ungültiger `source`-Werte, wörtliche `evidence` in der zitierten Nutzerzeile, keine Assistant-Tatsachen, Uhrzeiten/Unsicherheiten/Widersprüche/Korrekturen/Planänderungen bleiben. Ungültige Rekonstruktion geht nicht an Stufe 2; es gibt keine stillen Auto-Reparaturen. +2. **Stufe 2 – persönliche Narration** (`mvp.journal_generate`, `purpose=journal_generate`). Eingabe: validierte Rekonstruktion, kompakter aufgabenspezifischer Writing Brief, optional bestehender Text. Kein erneutes Mitschicken des vollen Tagesdialogs. + +**Additiv 2026-08-25 (Claim-Bindung):** Chronologieeinträge tragen keine freien Faktenlisten mehr. Jedes inhaltliche Element ist ein Claim `{kind, evidence}` mit eigener, lokal geprüfter Evidence: ein wörtlicher Ausschnitt genau der genannten Nutzerzeile. `source` ist Pflicht und nur der exakte Wert `user`. Legacy-Felder (`events`, allgemeine `evidence`, …) werden abgelehnt. Eine gültige Evidence legitimiert keine zweite, unbelegte Behauptung. Assistententext ist niemals Evidence. Uhrzeiten in `time` müssen ein reines Zeitstoken sein (`6:00`, `06:00`, `6:00 Uhr`) und in derselben `source_id` vorkommen; vertauschte Uhrzeiten sind `time_source_mismatch`. Chronologische Umordnung nur, wenn jede dafür verwendete Uhrzeit source-lokal bestätigt ist und die Minutenfolge nicht abfällt; gleiche oder relative Angaben bleiben in Quellenreihenfolge (`reorder_unjustified`). Top-Level-`contradictions` / `uncertainties` / `plan_changes` / `corrections` sind sourced `{source_id, evidence}`. Stufe 2 konsumiert ausschließlich diese geprüften Ausschnitte plus bestätigte Zeittoken. Damit gilt: kein Fakten-, Zeit-, Gefühls-, Wahrnehmungs-, Bewertungs-, Zitat-, Korrektur- oder Planänderungselement erreicht Stufe 2, ohne dass seine Herkunft aus der angegebenen Nutzernachricht lokal überprüft wurde. Stufe-2-Narration kann stilistisch umformulieren; sie erhält keine unbestätigten Modellparaphrasen als Inhaltsquelle. + +**Additiv 2026-08-25 (Provenienz / Vollständigkeit):** Die Claim-Bindung allein sichert Herkunft, nicht Vollständigkeit und nicht die Semantik von `kind`. Der Journal-Adapter nutzt die allgemeine Schicht `provenance_verification.md`: Coverage `all_selected_sources` rehydriert den kanonischen Volltext jeder ausgewählten Nutzerquelle lokal. Ein Claim `Morgens` entfernt weder Brot noch Anna noch Suppe. Modell-Labels sind unverifiziert und fehlen im Stufe-2-Artefakt. Stufe 2 erhält nur das lokal erzeugte `VerifiedArtifact` (`kind=verified_artifact`, `sources[].text`). Allgemeine Invarianten und Tests: `backend/provenance.py`, `backend/tests/test_provenance.py`. Journal-Adapter: `journal_reconstruct.py`. + +**Additiv 2026-08-26 (lokale Coverage-Autorität):** Stufe-1-JSON bleibt untrusted. Ist es unvollständig oder ungültig, wird es **nicht** still repariert und **nicht** an Stufe 2 weitergereicht. Stattdessen erzeugt der Journal-Adapter lokal ein `VerifiedArtifact` aus allen ausgewählten Nutzerquellen (volle Rehydration). Stufe 2 läuft damit weiter. Abbruch nur, wenn es gar keinen Nutzertext gibt (`no_user_sources`). Das ist keine Auto-Reparatur des Modell-JSON. + +**Additiv 2026-08-26 (Identitätsleak):** Response Validation bleibt fail-closed. Klartext-Identität in der Rohantwort wird nicht demaskiert und nicht als Entwurf übernommen. Nach einem Korrekturversuch erzeugt der Journal-Adapter lokal denselben Coverage-Pfad (Stufe 1) bzw. einen Entwurf aus dem lokalen Artefakt (Stufe 2). Homonyme (Speise vs. Personenname) gelten nicht als Leak. + +Das vollständige Writing Profile bleibt lokale Source of Truth. `compile_task_brief("journal_generate")` kompiliert nur den Aufgabenbrief (Core knapp, Facet-Delta `autobiographical_journal`, höchstens sechs relevante Traits, ein bis zwei Exemplare, Zielgröße ca. 3.000–4.000 Zeichen, feldweise an Wortgrenzen, keine Wortmitte). + +## 16.2 Budgetmodell + +Vor jedem Journal-Egress: + +| Größe | Herkunft | +|---|---| +| Effektives Kontextfenster | OpenRouter Models-API `context_length` / `top_provider.context_length`, sonst Env `KANSHO_PROVIDER_CONTEXT_LENGTH` | +| Maximale Ausgabelänge | `top_provider.max_completion_tokens`, sonst Env `KANSHO_PROVIDER_MAX_COMPLETION_TOKENS` | +| Reservierte Ausgabe | `min(4096, unterstützte max_completion_tokens)` | +| Sicherheitsmarge | 15 % des Fensters (`KANSHO_JOURNAL_SAFETY_MARGIN`) | +| Input-Schätzung | konservativ 2 Zeichen/Token (`KANSHO_TOKEN_CHARS_PER_TOKEN`); kein Modell-Tokenizer, keine Scheingenauigkeit | + +Mindestfenster für Journalgenerierung: **32.768 Tokens**. Darunter kontrollierte Ablehnung, kein Modellwechsel. + +Katalog: `backend/model_catalog.py`, Cache TTL 3600 s (`KANSHO_MODEL_CATALOG_TTL_SECONDS`). Bei Fetch-Fehler gilt ein noch gültiger Stale-Cache; sonst Env-Fallback; sonst `model_metadata_unknown`. Fake-Provider: 32K/4096. + +Request nur, wenn geschätzter Input + reservierte Ausgabe in das geminderte Fenster passen. Tagesdialog: konfigurierbare Nachrichten-Sicherheitskappe `KANSHO_JOURNAL_DAY_MAX_MESSAGES` (Default 500) plus Tokenbudget. Overflow bricht ab; keine stillen Mittelkürzungen, keine Wortmitte. + +## 16.3 OpenRouter + +Journal-Calls setzen `max_tokens` auf die reservierte Ausgabe und `plugins: [{"id":"context-compression","enabled":false}]`. Nur OpenRouter. Lokale und andere Provider erhalten diese Felder nicht. `provider.data_collection=deny` bleibt der bestehende ZDR-Pfad. + +## 16.4 Fehlercodes + +| Code | Bedeutung | +|---|---| +| `model_context_too_small` | Effektives Fenster unter 32K | +| `prompt_budget_exceeded` | Input plus Reserve plus Marge passt nicht; nichts wurde gekürzt | +| `model_metadata_unknown` | Fenster nicht sicher bestimmbar | +| `reconstruction_invalid` | Modell-JSON der Stufe 1 unvollständig oder Assistant als Tatsache (Validator; Generate bricht daran nicht ab) | +| `no_user_sources` | Ausgewählte Gespräche enthalten keinen Nutzertext; Generate bricht ab | +| `output_limit_unsupported` | Reservierte Ausgabelänge nicht tragfähig | +| `provider_context_length_rejected` | Anbieter lehnte wegen Kontextlänge ab | + +Nutzertext plus `diagnostics` ohne Prompts, Antworten, Klarnamen, Mapping. + +## 16.5 Diagnose + +Admin-Trace (`trace.budget`, `trace.stages`): verwendetes Modell, geschätzte Input-Tokens, tatsächliche `prompt_tokens` / `completion_tokens` / `total_tokens` / Kosten soweit geliefert, Fenster, Reserve, Marge, Budget-OK, Compression disabled/not_applicable, Abbruchgrund. Vollständige Prompts und Antworten nur im direkten Admin-Response des aktuellen Requests, nicht in einer globalen Historie oder Datenbank. Compact-Diagnose ohne Promptkörper darf prozessweit nur den letzten kompakten Status halten. + +## 16.6 Bewusst offen + +Chunk-and-Merge für übergroße Tage, Tokenizer je Modellfamilie, persistente Audit-Tabelle ohne Prompt-Inhalt, Quasi-Identifikatoren. + +Tests: `backend/tests/test_journal_budget.py`. Allgemeine Provenienzschicht: `provenance_verification.md`, `backend/tests/test_provenance.py`. + +--- + # 10. Querverweise - Fit-Gap: `../functional/mvp_stand_und_abgleich.md` - API-Liste: `backend_and_api.md` §5.1 - Tabellen: `memory_storage_and_offline.md` §6 - Egress: `privacy_gateway.md` §9 +- Journal-Budget und zweistufige Generierung: §16 - Routen: `frontend_pwa_shell.md` §6.1 - Stack-Ziel: `product_frame_and_stack.md` - Ports: `runtime_and_deploy.md` diff --git a/docs/architecture/technical/privacy_gateway.md b/docs/architecture/technical/privacy_gateway.md index f22025e..b481eed 100644 --- a/docs/architecture/technical/privacy_gateway.md +++ b/docs/architecture/technical/privacy_gateway.md @@ -128,7 +128,7 @@ Lokale URLs (localhost / 127.0.0.1) brauchen keine ZDR-Flags. Die Mapping-Tabell Klasse A verlässt die Zone nicht. Mapping wird nicht mitgeschickt. Kein stiller Fallback auf einen unsicheren Generate-Provider. -Prompts für Dialog, Journal, Detection und Profile-Review liegen in der DB (`mvp.dialogue_turn`, `mvp.journal_generate`, `mvp.entity_detect`, `mvp.profile_review`), nicht im Anwendungscode. +Prompts für Dialog, Journal-Rekonstruktion, Journal-Narration, Detection und Profile-Review liegen in der DB (`mvp.dialogue_turn`, `mvp.journal_reconstruct`, `mvp.journal_generate`, `mvp.entity_detect`, `mvp.profile_review`), nicht im Anwendungscode. `mvp.dialogue_turn` bleibt ein generativer Call (Foundation: ein Hauptcall pro Turn). Ton, Tiefe und Erzählmodus kommen lokal als `register_hint` plus Writing-Profile-Hinweis in denselben Prompt, nicht als zweites Modell. JSON `{operation, impulse}`. Persistiert wird nur `impulse`. @@ -142,15 +142,25 @@ Noch offen im Code: 1. **Inhaltliche Context Minimization.** Es geht der zusammengebaute Dialog, nur durch eine Zeichenkappe begrenzt. Minimum-sufficient-context nach Aufgabe fehlt. 2. **Robuste Entity Detection.** Wörterbuch plus Muster plus Detect-Call. Testphase: externes kleines Modell über OpenRouter. Ziel: lokales Ollama. Keine belastbare Erkennung von Quasi-Identifikatoren (Beruf + Ort + Familie, seltene Konstellationen). -3. **Response Validation.** Es wird nur auf leere Antworten geprüft. Keine Prüfung, ob Platzhalter erhalten blieben oder das Modell Klarnamen erraten hat. +3. **Response Validation.** Minimale Prüfung: Klartext-Identität in der Rohantwort vor Demask wird blockiert. Dieselbe Homonym-Regel wie beim Maskieren (Speise vs. Person). Dialog-Egress maskiert den **gesamten** gerenderten Prompt (Opening-Hinweis, Space-Ausschnitte, Titel), nicht nur `user:`-Zeilen; Detect sieht denselben gerenderten Prompt. Bei einem Leak gibt es einen Korrekturversuch. Der Dialogzug fällt lokal zurück statt leer zu bleiben. Journal-Generate übernimmt ungültiges oder leckendes Stufe-1-JSON nicht; lokal vollständige Nutzerquellen, Stufe 2 analog mit lokalem Entwurf statt der leckenden Antwort. Keine Quasi-Identifikatoren, keine vollständige Platzhalterprüfung, keine inhaltliche Minimierung. 4. **Audit ohne Prompt-Inhalt.** Es gibt keine persistente Audit-Spur (Request-ID, Policy, Provider, ZDR-Status, maskierte Entitätstypen) ohne volle Prompts oder Mapping. 5. **Mapping-Härtung.** `identity_mappings` ist lokal, aber unverschlüsselt; keine Pflege-UI; keine Verschlüsselung at rest. 6. **Detect-Klartext in der Testphase.** Der Detect-Provider darf Klartext sehen. Bei externem Detect (OpenRouter) ist das ein bewusster Übergang, nicht der Zielpfad. Produktiv nur lokales Modell. 7. **Weitere Egress-Klassen.** Tool-, Web- und Speech-Egress bleiben getrennt und ungebaut. -8. **Löschen, Export, DSFA.** Kein Hard-Delete, keine Portabilität, keine Datenschutz-Folgenabschätzung. +8. **Löschen, Export, DSFA.** Journal-Entries haben Soft Delete, Papierkorb und explizites Purge. Kein vollständiges Account-Löschen, keine Portabilität als öffentliches Produkt, keine Datenschutz-Folgenabschätzung. Diese Punkte dürfen nicht stillschweigend als erledigt gelten, nur weil Generate bereits maskiert. +## 9.3 Journal-Budget und Context Compression (2026-08-25) + +Additiv zu §9.1. Fachliche Trennung Inhalt/Stil: `../functional/writing_profile_and_journaling.md` §16. Technische Umsetzung: `mvp_implementation.md` §16. + +Persönlicher Journal-Egress bleibt ausschließlich über dieses Gateway. Zwei Zwecke: `journal_reconstruct` dann `journal_generate`. Kein stiller Fallback auf ein anderes Modell. OpenRouter-Context-Compression ist für Journal-Calls ausdrücklich deaktiviert (`plugins.id=context-compression`, `enabled=false`). Diese Plugin-Felder gehen nicht an lokale oder fremde OpenAI-kompatible Endpunkte. + +Journal-Prompts werden nicht mehr durch `MAX_EGRESS_CHARS` still in der Mitte abgeschnitten. Passt der Tagesdialog nicht ins Budget, bricht Kanshō mit `prompt_budget_exceeded` ab. Stufe 2 erhält den vollständigen lokal rehydrierten Nutzerinhalt als `VerifiedArtifact` (`provenance_verification.md`); bei Übergröße ebenfalls Abbruch, kein stilles Kürzen. + +Usage-Metadaten (Tokens, Kosten, Fenster, Reserve, Marge, Compression-Status) liegen in der Admin-Testspur `trace.budget` bzw. `trace.stages`. Journal-Generate setzt beide Stufen aus den Rückgabewerten der jeweiligen Gateway-Calls zusammen; es gibt keine globale `debug_history` und keinen globalen Promptspeicher. Vollständige Prompts und Antworten höchstens im direkten Admin-Response des aktuellen Requests. Mapping bleibt ausgeschlossen. + ## 10. Offene Fragen Übernommen aus `guardrails.md` §21, hier nicht vorentschieden: Entitätstypen, Quasi-Identifikatoren, Pseudonym-Stabilität, Mehrnutzer-Trennung der Mappings, Verschlüsselung der Mapping-Tabelle, rechtliche DSFA bei Mehrbenutzer-Produktbetrieb. diff --git a/docs/architecture/technical/provenance_verification.md b/docs/architecture/technical/provenance_verification.md new file mode 100644 index 0000000..4f288b2 --- /dev/null +++ b/docs/architecture/technical/provenance_verification.md @@ -0,0 +1,111 @@ +--- +title: "Kanshō – Provenienz und lokale Verifikation untrusted Modellausgabe" +version: "0.1" +status: "Arbeitsstand" +date: "2026-08-25" +product_family: "Jinkendo" +document_role: "Technical Chapter / Provenance / Verification Contract" +parent_document: "technische_zielarchitektur.md" +--- +# Kanshō – Provenienz und lokale Verifikation + +Kanonisches Home für den **allgemeinen** technischen Vertrag: externe Modellantworten sind untrusted input; kanonischer Inhalt kommt nur aus einer lokalen Quellenregistry; Downstream-Stufen konsumieren ein lokal erzeugtes `VerifiedArtifact`. + +Journalregeln, Writing Profile, Chronologie und Tagebuchstil gehören **nicht** in diese Schicht. Journal-Adapter und MVP-Nutzung: `mvp_implementation.md` §16.7. Privacy-Egress bleibt `privacy_gateway.md`. Fachliche Quellenbindung und Re-Grounding bleiben `../functional/context_fidelity_and_regrounding.md`. + +## 1. Decision Record (2026-08-25) + +**Entscheidung:** Persönliche KI-Folgestufen dürfen freie Modellantworten nicht als Inhaltsautorität behandeln. Es gibt eine kleine, intent-neutrale Provenienz- und Verifikationsschicht (`backend/provenance.py`). Fachgrenzen mappen Domainfehler erst dort auf Nutzertexte und Fehlercodes. + +| Thema | Stand | Status | +|---|---|---| +| Modellantworten sind untrusted input | ja | **Entschieden** (technisch, 2026-08-25) | +| Lokale Quellenregistry ist Autorität für Originaltext | ja | **Entschieden** (technisch, 2026-08-25) | +| Evidence lokal gegen Registry prüfen | ja | **Entschieden** (technisch, 2026-08-25) | +| Downstream nur `VerifiedArtifact` | ja | **Entschieden** (technisch, 2026-08-25) | +| Semantische Modell-Labels standardmäßig unverifiziert | ja | **Entschieden** (technisch, 2026-08-25) | +| Vollständigkeit über explizite Policy (`coverage`) | ja | **Entschieden** (technisch, 2026-08-25) | +| Fail Closed, keine stillen Kürzungen in dieser Schicht | ja | **Entschieden** (technisch, 2026-08-25) | +| Konkrete spätere Intent-Policies (Reflexion, Synthese, …) | nicht gebaut | **Offen** / später prüfen | +| Selektiver Evidence-Modus als Produktfunktion | nur Schnittstelle `selected_evidence` | **Nicht** als Produkt entschieden | + +Keine Vorwegnahme von Agentengraph, Workflow-Framework oder Pluginarchitektur. `ai_architecture.md`: Multi-Agent-Zerlegung bleibt **offen**. + +## 2. Allgemeiner Vertrag + +Die Schicht kennt Quellen, Rollen, Evidence-Referenzen, Coverage-Modi und ein geprüftes Artefakt. Sie kennt keine Tagebuch-, Space- oder Writing-Profile-Begriffe. + +### 2.1 Quellenregistry + +Eine Quelle hat mindestens `source_id`, `role`, kanonischen Originaltext und optionale lokale Metadaten. IDs und Texte entstehen lokal. Das Modell darf sie referenzieren, nicht ersetzen. Ein abweichender `proposed_texts`-Eintrag ist `canonical_overwrite`. + +### 2.2 Evidence + +Untrusted Input: `{source_id, excerpt}`. Lokal: + +- Source existiert. +- Rolle ist laut Policy erlaubt. +- Excerpt kommt **exakt** in genau dieser Source vor. +- Excerpt nur in einer anderen Source: `wrong_source`. +- Excerpt in einer unzulässigen Rolle: `role_not_allowed`. +- Erfundener Text: `unverified_excerpt`. + +Spans (`start`/`end`) werden lokal aus dem bestätigten Zitat materialisiert. Modell-Offsets auf maskiertem Text werden nicht verwendet. Mehrdeutige gleiche Ausschnitte: erstes Vorkommen in der kanonischen Source. + +Maskierung/Demaskierung: Verifikation läuft auf dem lokalen Klartext der Registry, nicht auf Egress-Platzhaltern. Schlägt die Demaskierung fehl, findet der Excerpt nicht, die Anfrage bleibt fail-closed. + +### 2.3 Verification Policy + +Ausdrücklich zu setzen: + +| Feld | Bedeutung | +|---|---| +| `allowed_roles` | welche Registry-Rollen Inhaltsquelle sein dürfen | +| `coverage` | Vollständigkeitsmodus | +| `selected_ids` | welche Quellen in welcher Materialisierungsreihenfolge | +| `source_order` | optionale Begegnungsreihenfolge, unabhängig von `selected_ids` | +| `include_unverified_annotations` | ob Modell-Labels separat gehalten werden | + +Aktuell implementierte Coverage-Modi: + +- `all_selected_sources`: der **vollständige** kanonische Text jeder ausgewählten zulässigen Quelle wird lokal rehydriert. Ein kurzer Evidence-Teilstring entfernt den Rest nicht. +- `selected_evidence`: Schnittstelle für spätere selektive Intents; materialisiert nur geprüfte Excerpts. **Keine** Produktfunktion, kein Journal-Default. + +### 2.4 Verified Artifact + +Mindestens: `kind=verified_artifact`, `coverage`, `source_order`, `order`, `sources[]` (`source_id`, `role`, `text`), geprüfte `evidence[]`. Unverified Annotations nur wenn die Policy das ausdrücklich verlangt, und dann unter `annotations_unverified`, niemals als `sources[].text`. + +Invariante: + +> Verifizierter Wortlaut und modellgelieferte Interpretation sind unterschiedliche Dinge. + +## 3. Journalspezifische Policy + +Nur im Adapter `journal_reconstruct.py` und in den Journal-Prompts: + +- zulässige Inhaltsrolle: `user` +- Assistent darf Stufe 1 als Dialogkontext sehen, nicht ins Artefakt +- Coverage: `all_selected_sources` für alle ausgewählten nichtleeren Nutzerquellen +- `source_order` = Begegnungsreihenfolge; Umordnung der Materialisierung nur mit source-lokal bestätigten Uhrzeiten +- Modell-`kind` und Top-Level-Kategorien (`contradictions`, …) nicht autoritativ; Stufe 2 erhält sie nicht +- Domainfehler des Modell-JSON → `reconstruction_invalid` plus `diagnostics.reason` +- Stufe 2 formuliert Ich-Form, Titel, Absätze, Writing Profile + +**Additiv 2026-08-26:** Generate reicht ungültiges Stufe-1-JSON nicht an Stufe 2 weiter und repariert es nicht. Der Adapter materialisiert stattdessen lokal alle ausgewählten Nutzerquellen (`local_verified_artifact`). Generate-Abbruch nur, wenn es keine Nutzerquelle gibt (`no_user_sources`). Eine Rohantwort mit Klartext-Identität wird nach Retry ebenfalls nicht übernommen. + +Das Artefakt ist geprüftes Quellenmaterial, kein ungeprüfter Rohdialog mit Assistentenzeilen. + +## 4. Aktuelle MVP-Nutzung + +Zwei Gateway-Calls Klasse B, request-scoped Trace, Budget unverändert (`mvp_implementation.md` §16, `privacy_gateway.md` §9.3). Stufe 2 konsumiert `reconstruction` = serialisiertes `VerifiedArtifact`. Passt der vollständige Nutzerinhalt nicht ins Budget: `prompt_budget_exceeded`, nichts still kürzen, kein Chunk-and-Merge. + +Tests: `backend/tests/test_provenance.py`, `backend/tests/test_journal_budget.py`. + +## 5. Bewusst nicht implementiert + +Zukünftige Intent-Policies, selektive Evidence als Produktweg, Chunk-and-Merge, persistente Audit-Tabelle, semantische Klassifikation über wachsende RegEx-Listen. + +## 6. Querverweise + +- Technisch: `mvp_implementation.md` §16, `privacy_gateway.md`, `ai_architecture.md` +- Fachlich: `../functional/guardrails.md`, `../functional/context_fidelity_and_regrounding.md`, `../functional/writing_profile_and_journaling.md` diff --git a/docs/architecture/technical/runtime_and_deploy.md b/docs/architecture/technical/runtime_and_deploy.md index 0ade6ee..cc06932 100644 --- a/docs/architecture/technical/runtime_and_deploy.md +++ b/docs/architecture/technical/runtime_and_deploy.md @@ -96,6 +96,15 @@ Kanshō-Repo liegt bereits auf Gitea (`Lars/Kansho`). HTTPS-Push ist eingerichte 2. Gemeinsames oder separates Postgres? 3. Backup-Rhythmus und Restore-Übung vor erstem persönlichen Dialogdatenbestand. +### 7.1 Lokales Backup für die Urlaubs-Testphase (2026-08-25) + +Kein Ersatz für Produktions-Backup, Docker oder Verschlüsselung. + +- Erzeugen: `.\scripts\backup-local.ps1 create` oder `python backend\local_backup.py create --out ` +- Standardziel: `local-backups/` (gitignoriert) +- Wiederherstellen: `.\scripts\backup-local.ps1 restore -Archive -Confirm`; bestehender Stand nur mit `--replace` nach automatischem Sicherheitsbackup +- SQLite-Backup-API, Journalmedien, Manifest mit Checksummen. Keine `.env` oder Keys. Das Archiv kann persönliche unverschlüsselte Daten enthalten. + ## 8. Querverweise - Technisch: `product_frame_and_stack.md`, `security.md` diff --git a/docs/architecture/technical/security.md b/docs/architecture/technical/security.md index dd71c4f..29b6531 100644 --- a/docs/architecture/technical/security.md +++ b/docs/architecture/technical/security.md @@ -35,7 +35,7 @@ Kanonisches Home für Zugriffssicherheit des Mitai-Rahmens (Auth, TLS, Secrets, | TLS nach außen | ja, Betriebsstandard der Familie | entschieden als Anforderung | | Verschlüsselung at rest (DB, Backups, Gerät) | – | offen | | Client-Token in localStorage | XSS-Risiko bekannt | bevorzugte Richtung, später bewerten | -| Vollständiges Löschen / selektives Vergessen | fachlich offen | offen | +| Vollständiges Löschen / selektives Vergessen | Journal-Entry: Soft Delete + explizites Purge; Account/DSFA offen | Slice teilweise, H1 offen | | Export/Portabilität | fachlich gewünscht (menschenlesbar) | offen in Technik | ## 4. Dependency- und Deploy-Hygiene diff --git a/docs/architecture/technical/technische_zielarchitektur.md b/docs/architecture/technical/technische_zielarchitektur.md index 1ad1484..20b79a4 100644 --- a/docs/architecture/technical/technische_zielarchitektur.md +++ b/docs/architecture/technical/technische_zielarchitektur.md @@ -204,6 +204,7 @@ mvp_implementation.md | `data_architecture.md` | Persistenzgrenzen, Entitäten ohne Schema | Arbeitsstand, nicht final | | `memory_storage_and_offline.md` | Dialogspeicher, Sync, Offline | Arbeitsstand, Sync offen | | `privacy_gateway.md` | Trust Zones, Egress, Demasking | Invariante entschieden; Verfahren offen | +| `provenance_verification.md` | Untrusted Modellausgabe, lokale Registry, `VerifiedArtifact` | Technisch entschieden 2026-08-25; zukünftige Intent-Policies offen | | `ai_architecture.md` | Inferenzpfad, Agenten (später) | Gateway bindend; Agenten offen | | `integrations_technical.md` | Handoffs zur Produktfamilie | Arbeitsstand, Verträge offen | | `admin_diagnostics.md` | Diagnoseansicht | Mitai-Admin-Realm; Kanshō-Inhalte später | diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 8e3cdf1..3126239 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -8,6 +8,7 @@ import HomePage from './pages/HomePage.jsx' import DialoguePage from './pages/DialoguePage.jsx' import JournalSpacesPage from './pages/JournalSpacesPage.jsx' import JournalSpacePage from './pages/JournalSpacePage.jsx' +import JournalTrashPage from './pages/JournalTrashPage.jsx' import JournalDayPage from './pages/JournalDayPage.jsx' import JournalEditorPage from './pages/JournalEditorPage.jsx' import JournalSourcePage from './pages/JournalSourcePage.jsx' @@ -132,6 +133,7 @@ export default function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/app.css b/frontend/src/app.css index cda861a..75efb72 100644 --- a/frontend/src/app.css +++ b/frontend/src/app.css @@ -82,6 +82,17 @@ button.ghost { padding: 0.4rem 0.6rem; } button:disabled { opacity: 0.6; } +a.button-link { + display: inline-flex; + align-items: center; + justify-content: center; + background: var(--accent); + color: white; + border-radius: 10px; + padding: 0.75rem 1rem; + min-height: 2.75rem; + text-decoration: none; +} .app-shell { min-height: 100dvh; width: 100%; max-width: 100%; } .app-main { @@ -320,6 +331,15 @@ label.button-file { .trace-panel ol { padding-left: 1.2rem; margin: 0.4rem 0 0.8rem; } .trace-panel details { margin: 0.5rem 0; } .trace-panel summary { cursor: pointer; color: var(--muted); } +.trace-stage { + margin-top: 1rem; + padding-top: 0.8rem; + border-top: 1px solid var(--line); +} +.trace-stage h3 { + margin: 0 0 0.5rem; + font-size: 1rem; +} button.ghost.active { color: var(--accent); } .media-row { display: flex; flex-wrap: wrap; gap: 0.8rem; } .media-row figure { margin: 0; width: 7.5rem; } diff --git a/frontend/src/components/CallTrace.jsx b/frontend/src/components/CallTrace.jsx index 373dce2..d020b3a 100644 --- a/frontend/src/components/CallTrace.jsx +++ b/frontend/src/components/CallTrace.jsx @@ -8,12 +8,46 @@ function Block({ title, value }) { ) } +function unavailable(value) { + if (value == null || value === '') return 'nicht verfügbar' + return value +} + +function isJournalTrace(trace) { + if (!trace) return false + if (Array.isArray(trace.stages) && trace.stages.length > 0) return true + return ( + trace.purpose === 'journal_reconstruct' || + trace.purpose === 'journal_generate' || + trace.layer === 'journalentwurf' || + trace.layer === 'journalrekonstruktion' + ) +} + function layerLabel(trace) { - if (trace?.layer === 'journalentwurf') return 'Journalentwurf' - if (trace?.purpose === 'journal_generate') return 'Journalentwurf' + if (trace?.purpose === 'journal_reconstruct' || trace?.layer === 'journalrekonstruktion') { + return 'Journalrekonstruktion' + } + if (trace?.layer === 'journalentwurf' || trace?.purpose === 'journal_generate') { + return 'Journalformulierung' + } + if (trace?.layer === 'profilreview' || trace?.purpose === 'profile_review') { + return 'Profilreview' + } return 'Dialogzug' } +function stageTitle(stage, index) { + const layer = layerLabel(stage) + if (stage?.purpose === 'journal_reconstruct' || stage?.layer === 'journalrekonstruktion') { + return `Stufe ${index + 1}: Rekonstruktion` + } + if (stage?.purpose === 'journal_generate' || stage?.layer === 'journalentwurf') { + return `Stufe ${index + 1}: Journalformulierung` + } + return `Stufe ${index + 1}: ${layer}` +} + function detectLabel(trace) { if (trace?.detect_note === 'remote_llm') { return `${trace.detect_provider || 'openrouter'} (Detect-Call, Testphase)` @@ -27,9 +61,89 @@ function detectLabel(trace) { return trace.detect_provider } +function BudgetMeta({ budget, trace }) { + const data = budget || {} + const model = data.actual_model || data.model || trace?.model + return ( +
+
Zweck
+
{unavailable(trace?.purpose)}
+
Prompt
+
{unavailable(trace?.prompt_slug)}
+
Provider
+
{unavailable(trace?.provider || data.provider)}
+
Modell
+
{unavailable(model)}
+
Kontextfenster
+
{unavailable(data.effective_context_window)}
+
Geschätzte Eingabetokens
+
{unavailable(data.estimated_input_tokens)}
+
Prompt-Tokens
+
{unavailable(data.prompt_tokens)}
+
Completion-Tokens
+
{unavailable(data.completion_tokens)}
+
Gesamttokens
+
{unavailable(data.total_tokens)}
+
Reservierte Ausgabe / max_tokens
+
+ {unavailable(data.reserved_output_tokens)} + {' / '} + {unavailable(data.max_tokens)} +
+
Sicherheitsmarge
+
{unavailable(data.safety_margin)}
+
Kontextkompression
+
{unavailable(data.context_compression)}
+
Budgetstatus
+
+ {data.budget_ok == null ? 'nicht verfügbar' : data.budget_ok ? 'ok' : 'abgebrochen'} + {data.abort_reason ? ` · ${data.abort_reason}` : ''} +
+
Kosten
+
{unavailable(data.cost)}
+
+ ) +} + +function StageView({ stage, index, journal }) { + const title = journal ? stageTitle(stage, index) : layerLabel(stage) + return ( +
+

{title}

+ +
+
Schicht
+
{layerLabel(stage)}
+
Maskierung
+
{detectLabel(stage)}
+
Klasse
+
{unavailable(stage?.data_class)}
+
Masken
+
{stage?.mapping_count ?? 'nicht verfügbar'}
+
+ + + + + +
+ ) +} + export default function CallTrace({ decision, trace }) { if (!decision && !trace) return null - const layer = layerLabel(trace) + const journal = isJournalTrace(trace) + const stages = Array.isArray(trace?.stages) && trace.stages.length > 0 + ? trace.stages + : trace + ? [trace] + : [] return ( ) } diff --git a/frontend/src/pages/HomePage.jsx b/frontend/src/pages/HomePage.jsx index f0e8b21..033956d 100644 --- a/frontend/src/pages/HomePage.jsx +++ b/frontend/src/pages/HomePage.jsx @@ -1,14 +1,50 @@ +import { useEffect, useState } from 'react' import { Link } from 'react-router-dom' +import { api } from '../api.js' +import { useAuth } from '../context/AuthContext.jsx' export default function HomePage() { + const { session } = useAuth() + const [payload, setPayload] = useState(null) + const [error, setError] = useState('') + + useEffect(() => { + api('/api/journal/continuity', { token: session.token }) + .then(setPayload) + .catch((err) => setError(err.message)) + }, [session.token]) + + if (!payload && !error) { + return

Lädt …

+ } + + const continuable = payload?.continuable + const hasSpaces = payload?.has_spaces + return (
-

Start

-

- Der Einstieg bleibt bewusst dünn: kein Funktions-Dashboard. Der MVP-Kern liegt im Journal. +

Weitergehen

+

+ Kein Dashboard. Wenn ein Journal Day offen ist, geht es dort weiter.

-

Zum Journal

-

Keine Tracking-Kacheln, keine Aufgabenliste, keine Diagnose.

+ {error &&

{error}

} + {continuable && ( +

+ + {continuable.calendar_date} in {continuable.space_title || 'Space'} fortsetzen + +

+ )} + {!continuable && hasSpaces && ( +

Zum Journal

+ )} + {!continuable && !hasSpaces && ( +

+ Noch kein Reflection Space. Lege einen an und starte den ersten Journal Day. + {' '} + Space anlegen +

+ )}
) } diff --git a/frontend/src/pages/JournalDayPage.jsx b/frontend/src/pages/JournalDayPage.jsx index 6cf55d8..fb66e61 100644 --- a/frontend/src/pages/JournalDayPage.jsx +++ b/frontend/src/pages/JournalDayPage.jsx @@ -25,6 +25,7 @@ export default function JournalDayPage() { const [trace, setTrace] = useState(null) const [decision, setDecision] = useState(null) const [chooseSources, setChooseSources] = useState(false) + const [includeExisting, setIncludeExisting] = useState(false) const loadDay = async (preferId) => { const data = await api(`/api/journal/days/${dayId}`, { token: session.token }) @@ -114,6 +115,7 @@ export default function JournalDayPage() { try { const body = {} if (conversationIds?.length) body.conversation_ids = conversationIds + if (includeExisting) body.include_existing = true const result = await api(`/api/journal/days/${dayId}/generate`, { token: session.token, method: 'POST', @@ -122,7 +124,7 @@ export default function JournalDayPage() { setDecision(null) setTrace(result.trace || null) await loadDay(activeId) - navigate(`/journal/${spaceId}/${dayId}/entry?draft=1`) + navigate(`/journal/${spaceId}/${dayId}/entry?draft=1`, { state: { trace: result.trace || null } }) } catch (err) { setError(err.message) } finally { @@ -202,6 +204,25 @@ export default function JournalDayPage() { )} + {(payload.current_draft || payload.entries?.length > 0) && ( +
+

+ Es gibt bereits einen Entwurf oder Eintrag. Die gespeicherte Fassung bleibt unangetastet. + Ein neuer Entwurf entsteht getrennt und wird erst durch Übernehmen zur Version. +

+ +

+ Standard: nicht einbeziehen. Das Flag wird nur nach dieser Auswahl gesetzt. +

+
+ )} {(payload.current_draft || payload.entries?.length > 0) && (

{payload.current_draft && ( @@ -223,7 +244,7 @@ export default function JournalDayPage() { {item.role === 'assistant' ? 'Kanshō' : 'Du'}: {item.body}

))} - {!activeId &&

Noch kein Gespräch. Lege eines an und erzähle.

} + {!activeId &&

Noch kein Gespräch. Lege eines an — Kanshō setzt den ersten Impuls.

} {activeId && (
diff --git a/frontend/src/pages/JournalEditorPage.jsx b/frontend/src/pages/JournalEditorPage.jsx index 9166919..42c7cfd 100644 --- a/frontend/src/pages/JournalEditorPage.jsx +++ b/frontend/src/pages/JournalEditorPage.jsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from 'react' -import { Link, useNavigate, useParams, useSearchParams } from 'react-router-dom' +import { Link, useLocation, useNavigate, useParams, useSearchParams } from 'react-router-dom' import { api, apiBlob, apiUpload } from '../api.js' +import CallTrace from '../components/CallTrace.jsx' import JournalDocumentEditor from '../components/JournalDocumentEditor.jsx' import { useAuth } from '../context/AuthContext.jsx' import { useUnsavedChanges } from '../context/UnsavedChanges.jsx' @@ -9,9 +10,10 @@ import { entryTitle, formatWhen, originLabel } from '../journal/document.js' export default function JournalEditorPage() { const { spaceId, dayId } = useParams() const [params] = useSearchParams() + const location = useLocation() const entryQuery = params.get('entry') || '' const wantDraft = params.get('draft') === '1' - const { session } = useAuth() + const { session, isAdmin } = useAuth() const navigate = useNavigate() const editorRef = useRef(null) const entryIdRef = useRef(entryQuery) @@ -224,13 +226,13 @@ export default function JournalEditorPage() { const remove = async () => { if (!entryId) return - if (!window.confirm('Eintrag in den Papierkorb legen? Das ist noch nicht endgültig.')) return + if (!window.confirm('Eintrag in den Papierkorb legen? Das ist noch nicht endgültig. Endgültiges Löschen geschieht später im Papierkorb.')) return setError('') setNotice('') try { await api(`/api/journal/entries/${entryId}`, { token: session.token, method: 'DELETE' }) showLoaded(null) - setNotice('Im Papierkorb.') + navigate(`/journal/${spaceId}/trash`) } catch (err) { setError(err.message) } @@ -334,6 +336,7 @@ export default function JournalEditorPage() { )}
+ {isAdmin && location.state?.trace && } {versions.length > 0 && (

Versionen

diff --git a/frontend/src/pages/JournalSpacePage.jsx b/frontend/src/pages/JournalSpacePage.jsx index 4abfb85..9e47fb4 100644 --- a/frontend/src/pages/JournalSpacePage.jsx +++ b/frontend/src/pages/JournalSpacePage.jsx @@ -2,6 +2,7 @@ import { useEffect, useState } from 'react' import { Link, useNavigate, useParams } from 'react-router-dom' import { api } from '../api.js' import { useAuth } from '../context/AuthContext.jsx' +import { entryTitle, originLabel } from '../journal/document.js' function todayLocal() { const now = new Date() @@ -16,16 +17,20 @@ export default function JournalSpacePage() { const { session } = useAuth() const [space, setSpace] = useState(null) const [days, setDays] = useState([]) + const [entries, setEntries] = useState([]) const [date, setDate] = useState(todayLocal()) const [error, setError] = useState('') - useEffect(() => { + const load = () => api(`/api/journal/spaces/${spaceId}`, { token: session.token }) .then((data) => { setSpace(data.space) setDays(data.days || []) + setEntries(data.entries || []) }) - .catch((err) => setError(err.message)) + + useEffect(() => { + load().catch((err) => setError(err.message)) }, [session.token, spaceId]) const openDay = async (calendarDate) => { @@ -44,9 +49,16 @@ export default function JournalSpacePage() { return (
-

Alle Spaces

+

+ Alle Spaces + {' · '} + Papierkorb +

{space?.title || 'Space'}

-

Tage chronologisch. Der Kalendertag ist nicht die Dialogzeit.

+

+ Einträge chronologisch. Mehrere Einträge desselben Tages bleiben einzeln erreichbar. + Gelöschte Einträge liegen nur im Papierkorb. +

{error &&

{error}

}
+

Einträge

+ {entries.length === 0 &&

Noch keine Einträge in diesem Space.

} +
    + {entries.map((item) => ( +
  • + + {item.calendar_date} + {' · '} + {entryTitle(item.title) || 'Ohne Titel'} + {item.version_count > 1 ? ` · ${item.version_count} Versionen` : ''} + {item.origin ? ` · ${originLabel(item.origin)}` : ''} + + + Tag + {item.source_conversation_id && ( + + Quelldialog + + )} + +
    • + ))} +
+

Tage

    {days.map((item) => (
  • diff --git a/frontend/src/pages/JournalTrashPage.jsx b/frontend/src/pages/JournalTrashPage.jsx new file mode 100644 index 0000000..ee3b07f --- /dev/null +++ b/frontend/src/pages/JournalTrashPage.jsx @@ -0,0 +1,101 @@ +import { useEffect, useState } from 'react' +import { Link, useParams } from 'react-router-dom' +import { api } from '../api.js' +import { useAuth } from '../context/AuthContext.jsx' +import { entryTitle } from '../journal/document.js' + +export default function JournalTrashPage() { + const { spaceId } = useParams() + const { session } = useAuth() + const [space, setSpace] = useState(null) + const [items, setItems] = useState([]) + const [error, setError] = useState('') + const [notice, setNotice] = useState('') + const [busy, setBusy] = useState(false) + + const load = async () => { + const detail = await api(`/api/journal/spaces/${spaceId}`, { token: session.token }) + setSpace(detail.space) + const trash = await api(`/api/journal/spaces/${spaceId}/trash`, { token: session.token }) + setItems(Array.isArray(trash) ? trash : []) + } + + useEffect(() => { + load().catch((err) => setError(err.message)) + }, [session.token, spaceId]) + + const undelete = async (entryId) => { + setError('') + setNotice('') + setBusy(true) + try { + await api(`/api/journal/entries/${entryId}/undelete`, { token: session.token, method: 'POST', body: {} }) + setNotice('Eintrag aus dem Papierkorb zurückgeholt. Das ist keine Versionswiederherstellung.') + await load() + } catch (err) { + setError(err.message) + } finally { + setBusy(false) + } + } + + const purge = async (entryId) => { + if (!window.confirm( + 'Endgültig löschen? Text, Versionen und nur diesem Eintrag gehörende Medien werden unwiderruflich entfernt. Quelldialoge bleiben. Geteilte Medien bleiben, wenn sie noch in einem anderen Eintrag stehen.' + )) return + const typed = window.prompt('Zum Bestätigen ENDGÜLTIG eingeben.') + if (typed !== 'ENDGÜLTIG') { + setNotice('Endgültiges Löschen abgebrochen.') + return + } + setError('') + setNotice('') + setBusy(true) + try { + await api(`/api/journal/entries/${entryId}/purge`, { + token: session.token, + method: 'POST', + body: { confirm: true } + }) + setNotice('Eintrag endgültig gelöscht.') + await load() + } catch (err) { + setError(err.message) + } finally { + setBusy(false) + } + } + + return ( +
    +

    {space?.title || 'Space'}

    +

    Papierkorb

    +

    + Im Papierkorb liegen nur soft gelöschte Einträge. Wiederherstellen holt den Eintrag zurück. + Endgültiges Löschen ist eine eigene, irreversible Aktion. +

    + {error &&

    {error}

    } + {notice &&

    {notice}

    } + {items.length === 0 &&

    Der Papierkorb ist leer.

    } +
      + {items.map((item) => ( +
    • + + {item.calendar_date} + {' · '} + {entryTitle(item.title) || 'Ohne Titel'} + + + + + +
      • + ))} +
    +
    + ) +} diff --git a/frontend/src/pages/SettingsPage.jsx b/frontend/src/pages/SettingsPage.jsx index ef33f63..3907fee 100644 --- a/frontend/src/pages/SettingsPage.jsx +++ b/frontend/src/pages/SettingsPage.jsx @@ -678,9 +678,7 @@ export default function SettingsPage() {
)} {!profile?.compiled_brief && ( -

- Noch kein Stilbrief. Sobald Einträge oder Importe mit Zeitbezug vorliegen, entstehen Exzerpte. Traits folgen erst aus dem Initial Build. -

+

Noch kein Stilbrief. Lege eigene Texte in den Korpus oder übernimm einen Journal-Eintrag.

)} {profile?.sources?.length > 0 && (
    @@ -697,7 +695,10 @@ export default function SettingsPage() { {profile?.compiled_brief && (

    Aktueller Brief

    -

    Abgeleiteter Runtime-Brief, nicht Source of Truth und kein Import-/Exportformat. Nach einer übernommenen Profiländerung wird er neu kompiliert.

    +

    + Abgeleiteter Runtime-Brief, nicht Source of Truth und nicht das Import-/Exportformat. + Facets, Traits und Quellen bleiben maßgeblich. +

    {profile.compiled_brief}

    )} diff --git a/scripts/backup-local.ps1 b/scripts/backup-local.ps1 new file mode 100644 index 0000000..59c42de --- /dev/null +++ b/scripts/backup-local.ps1 @@ -0,0 +1,44 @@ +# Local SQLite+media backup/restore for the holiday test phase. +# The archive can contain personal unencrypted journal data. No .env or keys. +[CmdletBinding()] +param( + [Parameter(Position = 0)] + [ValidateSet("create", "restore")] + [string]$Action = "create", + [string]$Out, + [string]$Archive, + [switch]$Confirm, + [switch]$Replace +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +$root = Split-Path -Parent $PSScriptRoot +$py = Join-Path $root "backend\.venv\Scripts\python.exe" +if (-not (Test-Path $py)) { + $py = "python" +} + +$env:PYTHONUTF8 = "1" +$env:PYTHONIOENCODING = "utf-8" + +$argsList = @((Join-Path $root "backend\local_backup.py"), $Action) +if ($Action -eq "create" -and $Out) { + $argsList += @("--out", $Out) +} +if ($Action -eq "restore") { + if (-not $Archive) { + throw "Restore braucht -Archive " + } + $argsList += @($Archive) + if ($Confirm) { + $argsList += "--confirm" + } + if ($Replace) { + $argsList += "--replace" + } +} + +& $py @argsList +exit $LASTEXITCODE diff --git a/scripts/test-mvp.ps1 b/scripts/test-mvp.ps1 new file mode 100644 index 0000000..c3e9237 --- /dev/null +++ b/scripts/test-mvp.ps1 @@ -0,0 +1,131 @@ +# Local MVP acceptance runner. No Docker, no real providers, no production DB/media. +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +$root = Split-Path -Parent $PSScriptRoot +$failed = $false +$ran = 0 +$passed = 0 +$results = @() + +function Use-Utf8 { + $env:PYTHONUTF8 = "1" + $env:PYTHONIOENCODING = "utf-8" + try { + [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 + [Console]::InputEncoding = [System.Text.Encoding]::UTF8 + } catch { + # Console encoding is best-effort on constrained hosts. + } + chcp 65001 | Out-Null +} + +function Resolve-Python { + $venv = Join-Path $root "backend\.venv\Scripts\python.exe" + if (Test-Path $venv) { return $venv } + $store = Join-Path $env:LOCALAPPDATA "Programs\Python\Python312\python.exe" + if (Test-Path $store) { return $store } + return "python" +} + +function New-IsolatedDirs { + $stamp = Get-Date -Format "yyyyMMdd-HHmmss" + $base = Join-Path $env:TEMP "kansho-mvp-test-$stamp-$PID" + New-Item -ItemType Directory -Force -Path $base | Out-Null + $media = Join-Path $base "media" + New-Item -ItemType Directory -Force -Path $media | Out-Null + return @{ Base = $base; Media = $media; Db = (Join-Path $base "fallback.sqlite") } +} + +function Invoke-Step { + param( + [string]$Name, + [scriptblock]$Action + ) + $script:ran++ + Write-Host "" + Write-Host "=== $Name ===" + try { + & $Action + if ($LASTEXITCODE -ne 0 -and $null -ne $LASTEXITCODE) { + throw "exit $LASTEXITCODE" + } + $script:passed++ + $script:results += [pscustomobject]@{ Name = $Name; Status = "OK" } + Write-Host "OK $Name" + } catch { + $script:failed = $true + $script:results += [pscustomobject]@{ Name = $Name; Status = "FAIL"; Detail = $_.ToString() } + Write-Host "FAIL $Name" + Write-Host $_ + throw + } +} + +Use-Utf8 +$py = Resolve-Python +$iso = New-IsolatedDirs + +Remove-Item Env:KANSHO_FAKE_PROVIDER -ErrorAction SilentlyContinue +Remove-Item Env:KANSHO_FAKE_DETECT -ErrorAction SilentlyContinue +$env:KANSHO_PROVIDER_KEY = "" +$env:KANSHO_DETECT_PROVIDER_KEY = "" +$env:KANSHO_MEDIA_ROOT = $iso.Media +$env:KANSHO_DB_PATH = $iso.Db +$env:KANSHO_DATA_DIR = $iso.Base + +Write-Host "Kansho local MVP tests" +Write-Host "Python: $py" +Write-Host "Isolation: $($iso.Base)" +Write-Host "No live provider/detect calls. Production DB/media untouched." +Write-Host "Fail-closed stays testable; suites that need a fake provider set it themselves." + +try { + $backendTests = Get-ChildItem -Path (Join-Path $root "backend\tests") -Filter "test_*.py" | Sort-Object Name + if (-not $backendTests) { + throw "no backend/tests/test_*.py files found" + } + + foreach ($test in $backendTests) { + Invoke-Step -Name "backend $($test.Name)" -Action { + & $py $test.FullName + } + } + + Invoke-Step -Name "frontend document.test.js" -Action { + Push-Location (Join-Path $root "frontend") + try { + $node = Get-Command node -ErrorAction Stop + & $node.Source --test "src/journal/document.test.js" + } finally { + Pop-Location + } + } + + Invoke-Step -Name "frontend production build" -Action { + Push-Location (Join-Path $root "frontend") + try { + npm run build + } finally { + Pop-Location + } + } +} catch { + $failed = $true +} finally { + if (Test-Path $iso.Base) { + Remove-Item -Recurse -Force $iso.Base -ErrorAction SilentlyContinue + } +} + +Write-Host "" +Write-Host "=== Zusammenfassung ===" +foreach ($row in $results) { + Write-Host ("{0,-42} {1}" -f $row.Name, $row.Status) +} +Write-Host ("{0} von {1} Schritten erfolgreich." -f $passed, $ran) +if ($failed) { + exit 1 +} +Write-Host "MVP local acceptance: OK" +exit 0