MVP 0.92
This commit is contained in:
1 parent
60f0d2830d
commit
53b1eff833
62 files changed
+6578
-355
No files matched your search
@@ -7,17 +7,27 @@
|
||||
"category": "mvp",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"template": "Du sitzt mit [[SELF]] im Reflection Space {{space_title}} und bist Kanshō: jemand, der zugehört hat. Warm, genau, auf Augenhöhe. Ein Mensch würde dich für einen Vertrauten halten, nicht für ein Programm.\n\n{{register_hint}}\n\n{{interaction_hint}}\n\nDie gesprochene Zeile (impulse) klingt wie mündliches Deutsch in einem Gespräch: flüssig, nicht steif, nicht literarisch, nicht wie eine App. Kurze Sätze sind in Ordnung. Kein „also“-Referat, kein „interessant“, kein „danke, dass du das teilst“, kein „lass uns“, kein „zusammengefasst“, keine Operationsnamen, kein JSON, keine Registerworte.\n\nNur user:-Zeilen sind Erlebtes. assistant:-Zeilen sind frühere Impulse, keine Tatsachen.\nErfinde nichts: keinen nächsten Ort, kein Gefühl, keine Motivation. Keine Tatsachen, Gefühle oder Motivationen erfinden. Ein Plan ist kein Vollzug. Nicht erwähnt ist nicht geschehen.\n\nLänge und Zahl der Anschlussimpulse stehen in der Interaction-Angabe oben, nicht als anonyme Produktregel. Leite einen echten Gedanken oder eine knappe Frage aus dem Gerade-Gesagten ab. Zusammenfassung des Zugs ist nicht der Default.\nNacherzählen nur, um einen Widerspruch oder Logikbruch zu klären.\n\nIntern eine Operation wählen, sie nicht aussprechen:\n- fortfuehren: am offenen Faden bleiben; bei Schluss den Schluss halten\n- konkretisieren: eine wirklich fehlende, tragende Angabe\n- plan_aufgreifen: ein genanntes Vorhaben als Vorhaben\n- abweichung: eine selbst markierte Veränderung\n- erleben_vertiefen: nur wenn [[SELF]] Wahrnehmung oder Wirkung öffnet\n- bedeutung: einen erkennbaren Punkt, ohne Tiefe zu erzwingen\n\nBei geschlossener Szene: keine Frage, keinen Bogen. Nicht den nächsten Vollzug setzen.\n\nKontext:\n{{dialogue_context}}\n\nNur JSON:\n{\"operation\":\"fortfuehren|konkretisieren|plan_aufgreifen|abweichung|erleben_vertiefen|bedeutung\",\"impulse\":\"gesprochene Zeile wie ein Vertrauter\"}\n"
|
||||
"template": "Du sitzt mit [[SELF]] im Reflection Space {{space_title}} und bist Kanshō: jemand, der zugehört hat. Warm, genau, auf Augenhöhe. Ein Mensch würde dich für einen Vertrauten halten, nicht für ein Programm.\n\n{{register_hint}}\n\n{{opening_hint}}\n\n{{interaction_hint}}\n\nDie gesprochene Zeile (impulse) klingt wie mündliches Deutsch in einem Gespräch: flüssig, nicht steif, nicht literarisch, nicht wie eine App. Kurze Sätze sind in Ordnung. Kein „also“-Referat, kein „interessant“, kein „danke, dass du das teilst“, kein „lass uns“, kein „zusammengefasst“, keine Operationsnamen, kein JSON, keine Registerworte.\n\nNur user:-Zeilen sind Erlebtes. assistant:-Zeilen sind frühere Impulse, keine Tatsachen.\nErfinde nichts: keinen nächsten Ort, kein Gefühl, keine Motivation. Keine Tatsachen, Gefühle oder Motivationen erfinden. Ein Plan ist kein Vollzug. Nicht erwähnt ist nicht geschehen.\n\nLänge und Zahl der Anschlussimpulse stehen in der Interaction-Angabe oben, nicht als anonyme Produktregel. Leite einen echten Gedanken oder eine knappe Frage aus dem Gerade-Gesagten ab. Zusammenfassung des Zugs ist nicht der Default.\nNacherzählen nur, um einen Widerspruch oder Logikbruch zu klären.\n\nIntern eine Operation wählen, sie nicht aussprechen:\n- fortfuehren: am offenen Faden bleiben; bei Schluss den Schluss halten\n- konkretisieren: eine wirklich fehlende, tragende Angabe\n- plan_aufgreifen: ein genanntes Vorhaben als Vorhaben\n- abweichung: eine selbst markierte Veränderung\n- erleben_vertiefen: nur wenn [[SELF]] Wahrnehmung oder Wirkung öffnet\n- bedeutung: einen erkennbaren Punkt, ohne Tiefe zu erzwingen\n\nBei geschlossener Szene: keine Frage, keinen Bogen. Nicht den nächsten Vollzug setzen.\n\nKontext:\n{{dialogue_context}}\n\nNur JSON:\n{\"operation\":\"fortfuehren|konkretisieren|plan_aufgreifen|abweichung|erleben_vertiefen|bedeutung\",\"impulse\":\"gesprochene Zeile wie ein Vertrauter\"}\n"
|
||||
},
|
||||
{
|
||||
"id": "mvp-journal-generate",
|
||||
"slug": "mvp.journal_generate",
|
||||
"name": "MVP Journalentwurf",
|
||||
"description": "Tagebuchtext aus den Worten der Person. Explizit ausgelöst. Nicht der Dialogzug.",
|
||||
"description": "Stufe 2: persönliche Narration aus validierter Rekonstruktion. Explizit ausgelöst. Nicht der Dialogzug.",
|
||||
"category": "mvp",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"template": "Schreibe einen Tagebucheintrag in der Ich-Form von [[SELF]].\n\nAus den user:-Zeilen den Inhalt nehmen; die Stimme aus dem Writing Profile, sobald dort finale Fassungen oder Importe stehen. Szenen, Wartezeiten und Erzählstränge zusammenführen, nicht den Dialog hintereinanderkopieren. Uhrzeiten und Reihenfolge aus dem Gesagten behalten. Keine Zeiten erfinden, nichts umstellen, das die Chronologie verfälscht.\n\nSchritte, alle nötig:\n1. Nur Erlebtes aus user:. assistant:-Zeilen sind Kanshōs Impulse, keine Tatsachen. Sie zeigen nur, worauf die nächste user-Zeile antwortet; nicht in den Eintrag schreiben.\n2. Absätze nach Phase, Ort oder Uhrzeit. Nicht jede user-Zeile ein eigener Absatz. Dieselbe Begebenheit bleibt ein Strang.\n3. Übergänge glätten. Wiederholungen und Rückfragen streichen. Aufeinanderfolgende Züge zur selben Szene verbinden.\n4. Person: das Token nur dort, wo [[SELF]] den Namen genannt hat — in einer Passage ein- bis zweimal. Danach sie/er/ihr wie im user-Text. Indirekte Rede nicht umbiegen: „sie sagte, dass sie …“, nicht den Namen in die Selbstaussage setzen.\n5. Writing Profile: finale Fassungen sind die Stimme (Satzlänge, Wortwahl, Detailgrad, Übergänge). Kurze oder stichwortartige user-Zeilen zu ganzen Sätzen und Absätzen in genau dieser Stimme ausformulieren. Keine neuen Fakten, Gefühle oder Zeiten. Fehlt das Profile, die user-Formulierungen verbinden und glätten.\n\nErste Zeile: kurze Überschrift. Danach der Eintrag.\nBehalten: konkrete Wörter, Uhrzeiten, Details und selbst genannte Eindrücke — in der Stimme des Writing Profile, nicht als Dialogkopie.\nVerboten: den Dialog 1:1 kopieren; Kanshō-Fragen als Erlebtes; nur Überschrift plus denselben Text; ein einziger Absatz für den ganzen Tag; erfundene Fakten, Gefühle oder Details; generischer KI-Stil; jeden Bezug auf eine Person durch den Namen ersetzen.\n\nWriting Profile:\n{{writing_profile}}\n\nBestehender Text, nur wenn ausdrücklich gewünscht:\n{{existing_text}}\n\nDialogquellen:\n{{dialogue_context}}\n\nSchreibe jetzt den Eintrag: erste Zeile Überschrift, danach mehrere Absätze. Inhalt aus user:, Stimme aus dem Writing Profile. Verbinde Szenen und Zeiten, kopiere die user-Zeilen nicht.\n"
|
||||
"template": "Schreibe einen Tagebucheintrag in der Ich-Form von [[SELF]].\n\nUnten liegt ausschließlich ein lokal erzeugtes Verified Artifact. Es ist die einzige Inhaltsquelle. sources[].text ist der vollständige, lokal rehydrierte Nutzerinhalt. Evidence-Ausschnitte sind nur nachgewiesene Zitate, keine Erlaubnis, den Rest wegzulassen. annotations_unverified, falls vorhanden, sind unbestätigte Modelldeutung und keine Tatsachen: daraus keine Gefühle, Widersprüche, Korrekturen oder Planänderungen machen.\n\nKein Rohdialog, keine Assistentenzeilen, keine frei erfundenen Fakten. Stimme aus dem Writing Profile, sobald dort Core, Facet-Delta oder Traits stehen. Szenen zusammenführen. Die Reihenfolge der sources-Einträge behalten. Keine Zeiten, Gefühle, Motive oder Zusammenhänge ergänzen.\n\nSchritte, alle nötig:\n1. Nur den verifizierten Nutzerwortlaut aus sources[].text.\n2. Absätze nach Phase, Ort oder Uhrzeit. Dieselbe Begebenheit bleibt ein Strang.\n3. Übergänge glätten. Wiederholungen streichen. Unsicherheiten im Wortlaut sichtbar lassen, nicht glätten und nicht aus Labels ableiten.\n4. Person: das Token nur dort, wo [[SELF]] den Namen genannt hat — in einer Passage ein- bis zweimal. Danach sie/er/ihr. Indirekte Rede nicht umbiegen.\n5. Writing Profile: Satzlänge, Wortwahl, Detailgrad, Übergänge. Keine neuen Fakten.\n\nErste Zeile: kurze Überschrift. Danach der Eintrag.\nVerboten: Assistentenimpulse als Erlebtes; erfundene Fakten, Gefühle oder Details; generischer KI-Stil; Unsicherheit wegglätten; unbestätigte Labels als Wahrheit.\n\nWriting Profile:\n{{writing_profile}}\n\nBestehender Text, nur wenn ausdrücklich gewünscht:\n{{existing_text}}\n\nVerified Artifact:\n{{reconstruction}}\n\nSchreibe jetzt den Eintrag: erste Zeile Überschrift, danach mehrere Absätze. Inhalt nur aus sources[].text, Stimme aus dem Writing Profile.\n"
|
||||
},
|
||||
{
|
||||
"id": "mvp-journal-reconstruct",
|
||||
"slug": "mvp.journal_reconstruct",
|
||||
"name": "MVP Journal Rekonstruktion",
|
||||
"description": "Stufe 1: inhaltliche Rekonstruktion des Tagesdialogs. Keine Narration, kein Writing Profile.",
|
||||
"category": "mvp",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"template": "Referenziere die lokal vergebenen Quellen. Du entscheidest nicht, welche ausgewählten Nutzerpassagen verloren gehen dürfen. Du lieferst keine autoritative Semantik.\n\nJede nichtleere Nutzerzeile hat eine lokale Quellen-ID in eckigen Klammern: [u1], [u2], … in der ursprünglichen Reihenfolge. Assistant-Zeilen haben [a1], [a2], … und sind nur Antwortkontext, niemals Inhaltsquelle oder Evidence.\n\nJeder Claim ist ausschließlich ein wörtlicher Ausschnitt aus genau der genannten Nutzerzeile. Keine Paraphrasen. Keine Assistentenformulierungen. kind-Werte und contradictions/uncertainties/plan_changes/corrections sind unverbindliche Hinweise, keine festgestellten Tatsachen. Eine Evidence legitimiert keine zusätzliche Behauptung.\n\nsource_order ist die lokale Nutzerreihenfolge. chronology darf davon nur abweichen, wenn Uhrzeiten in der jeweils zugeordneten Nutzerzeile nachweisbar sind. Relative Angaben wie später oder danach nicht zum Umsortieren verwenden, wenn die Reihenfolge nicht aus bestätigten Uhrzeiten folgt. Gleiche oder mehrdeutige Uhrzeiten in Quellenreihenfolge lassen.\n\nAntworte ausschließlich mit JSON:\n{\"source_order\":[\"u1\",\"u2\"],\"chronology\":[{\"source_id\":\"u1\",\"source\":\"user\",\"time\":null,\"claims\":[{\"kind\":\"event\",\"evidence\":\"wörtlicher Ausschnitt aus genau dieser Nutzerzeile\"}]}],\"contradictions\":[],\"uncertainties\":[],\"plan_changes\":[],\"corrections\":[]}\n\nsource_order muss jede Nutzer-ID genau einmal in der lokalen Reihenfolge enthalten. chronology muss jede Nutzer-ID genau einmal abdecken. source ist Pflicht und immer genau user. time nur als reines Zeitstoken aus derselben Nutzerzeile (6:00, 06:00, 6:00 Uhr). Assistant-IDs und jedes andere source sind verboten. Keine events-Listen, keine allgemeine Evidence für mehrere Behauptungen. Du darfst den kanonischen Quellentext nicht ersetzen oder kürzen.\n\nDialog:\n{{dialogue_context}}\n"
|
||||
},
|
||||
{
|
||||
"id": "mvp-entity-detect",
|
||||
|
||||
+34
-13
@@ -11,7 +11,7 @@ from dialogue_store import get_conversation
|
||||
from retrieval import retrieve
|
||||
from journal_body import plain_text
|
||||
|
||||
PURPOSES = {"dialogue_turn", "journal_generate"}
|
||||
PURPOSES = {"dialogue_turn", "journal_generate", "journal_reconstruct"}
|
||||
CLOSURE = re.compile(
|
||||
r"\bauf den heimweg\b|"
|
||||
r"\bheimweg\b|"
|
||||
@@ -73,6 +73,8 @@ def build_internal_context(
|
||||
include_existing: bool = False,
|
||||
conversation_ids: list[str] | None = None,
|
||||
existing_text: str = "",
|
||||
reconstruction: str = "",
|
||||
day_spec: dict | None = None,
|
||||
) -> dict:
|
||||
purpose = purpose if purpose in PURPOSES else "dialogue_turn"
|
||||
if conversation_id and not space_id:
|
||||
@@ -96,17 +98,17 @@ def build_internal_context(
|
||||
selected_ids.append(conversation_id)
|
||||
|
||||
day_messages = []
|
||||
if journal_day_id:
|
||||
day_messages = retrieve(
|
||||
profile_id,
|
||||
{
|
||||
"kind": "day_messages",
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_ids": selected_ids or None,
|
||||
},
|
||||
)
|
||||
if journal_day_id and purpose != "journal_generate":
|
||||
spec = {
|
||||
"kind": "day_messages",
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_ids": selected_ids or None,
|
||||
}
|
||||
if day_spec:
|
||||
spec.update({key: value for key, value in day_spec.items() if key != "kind"})
|
||||
day_messages = retrieve(profile_id, spec)
|
||||
items.append({"type": "day_messages", "messages": day_messages})
|
||||
elif conversation_id:
|
||||
elif conversation_id and purpose == "dialogue_turn":
|
||||
day_messages = read("conversation_messages", profile_id=profile_id, context={"conversation_id": conversation_id})
|
||||
items.append({"type": "day_messages", "messages": day_messages})
|
||||
|
||||
@@ -148,6 +150,8 @@ def build_internal_context(
|
||||
|
||||
brief = compile_task_brief(profile_id, "journal_generate")
|
||||
items.append({"type": "writing_profile", "compiled_brief": brief})
|
||||
if reconstruction:
|
||||
items.append({"type": "reconstruction", "body": reconstruction})
|
||||
if include_existing and existing_text:
|
||||
items.append({"type": "existing_text", "body": plain_text(existing_text)})
|
||||
elif purpose == "dialogue_turn":
|
||||
@@ -174,15 +178,25 @@ def assemble_text(context: dict) -> dict[str, str]:
|
||||
writing_profile = ""
|
||||
interaction_hint = ""
|
||||
existing_text = ""
|
||||
reconstruction = ""
|
||||
user_bodies: list[str] = []
|
||||
purpose = context.get("purpose") or ""
|
||||
for item in context.get("items") or []:
|
||||
kind = item.get("type")
|
||||
if kind == "day_messages":
|
||||
for message in item.get("messages") or []:
|
||||
day_messages = item.get("messages") or []
|
||||
if purpose == "journal_reconstruct":
|
||||
from journal_reconstruct import assign_source_ids
|
||||
|
||||
day_messages = assign_source_ids(day_messages)
|
||||
for message in day_messages:
|
||||
role = message.get("role") or "user"
|
||||
body = message.get("body") or ""
|
||||
dialogue_parts.append(f"{role}: {body}")
|
||||
source_id = message.get("source_id")
|
||||
if purpose == "journal_reconstruct" and source_id:
|
||||
dialogue_parts.append(f"[{source_id}] {role}: {body}")
|
||||
else:
|
||||
dialogue_parts.append(f"{role}: {body}")
|
||||
if role == "user":
|
||||
user_bodies.append(body)
|
||||
elif kind == "prior_entries":
|
||||
@@ -202,6 +216,11 @@ def assemble_text(context: dict) -> dict[str, str]:
|
||||
interaction_hint = item.get("text") or ""
|
||||
elif kind == "existing_text":
|
||||
existing_text = item.get("body") or ""
|
||||
elif kind == "reconstruction":
|
||||
reconstruction = item.get("body") or ""
|
||||
elif kind == "opening":
|
||||
pass
|
||||
opening_hint = ""
|
||||
if purpose == "dialogue_turn":
|
||||
from conversation_signals import infer_signals
|
||||
from interaction_profile_store import assemble_interaction_hint
|
||||
@@ -221,6 +240,8 @@ def assemble_text(context: dict) -> dict[str, str]:
|
||||
"writing_profile": writing_profile,
|
||||
"interaction_hint": interaction_hint,
|
||||
"existing_text": existing_text,
|
||||
"reconstruction": reconstruction,
|
||||
"space_title": context.get("space_title") or "",
|
||||
"register_hint": infer_register(user_bodies),
|
||||
"opening_hint": opening_hint,
|
||||
}
|
||||
@@ -339,7 +339,7 @@ def list_conversations_for_day(profile_id: str, journal_day_id: str) -> list[dic
|
||||
(SELECT COUNT(*) FROM messages m WHERE m.conversation_id = c.id) AS message_count
|
||||
FROM conversations c
|
||||
WHERE c.profile_id = ? AND c.journal_day_id = ?
|
||||
ORDER BY c.created
|
||||
ORDER BY c.created, c.rowid
|
||||
""",
|
||||
(profile_id, journal_day_id),
|
||||
).fetchall()
|
||||
|
||||
@@ -290,6 +290,13 @@ def visible_for_role(payload: dict, role: str | None) -> dict:
|
||||
cleaned = dict(payload)
|
||||
cleaned.pop("trace", None)
|
||||
cleaned.pop("decision", None)
|
||||
opening = cleaned.get("opening")
|
||||
if isinstance(opening, dict):
|
||||
opening = dict(opening)
|
||||
opening.pop("trace", None)
|
||||
opening.pop("decision", None)
|
||||
opening.pop("opening_context", None)
|
||||
cleaned["opening"] = opening
|
||||
return cleaned
|
||||
|
||||
|
||||
@@ -339,8 +346,16 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||
"parsed": parsed_ok,
|
||||
"guard": "local_fallback",
|
||||
}
|
||||
except EngineError:
|
||||
raise
|
||||
except EngineError as exc:
|
||||
if exc.code != "response_validation_failed":
|
||||
raise
|
||||
impulse = local_hold(last_user_text(assembled))
|
||||
decision = {
|
||||
"operation": "fortfuehren",
|
||||
"label": OPERATIONS["fortfuehren"],
|
||||
"parsed": False,
|
||||
"guard": "identity_leak_blocked",
|
||||
}
|
||||
assistant = append_message(profile_id, conversation_id, impulse, role="assistant")
|
||||
user_bodies = [
|
||||
item.get("body") or ""
|
||||
|
||||
+32
-17
@@ -9,15 +9,17 @@ import privacy_placeholders
|
||||
from db import get_db, row_to_dict
|
||||
from entitlements import EntitlementError, check_feature_access, increment_feature_usage
|
||||
from placeholders import PlaceholderError, resolve_template
|
||||
from privacy_gateway import GatewayRequest, PrivacyGatewayError, complete, last_trace, public_trace
|
||||
from privacy_gateway import GatewayRequest, PrivacyGatewayError, complete
|
||||
from journal_reconstruct import parse_dialogue_line
|
||||
|
||||
|
||||
class EngineError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 400):
|
||||
def __init__(self, code: str, message: str, status_code: int = 400, diagnostics: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
self.diagnostics = diagnostics or {}
|
||||
|
||||
|
||||
def load_active_prompt(slug: str) -> dict:
|
||||
@@ -54,12 +56,26 @@ def preview_prompt(prompt: dict, context: dict[str, Any] | None = None) -> dict:
|
||||
def user_source_text(dialogue_context: str) -> str:
|
||||
parts = []
|
||||
for line in (dialogue_context or "").splitlines():
|
||||
if line.startswith("user:"):
|
||||
parsed = parse_dialogue_line(line)
|
||||
if parsed and parsed["role"] == "user":
|
||||
parts.append(parsed["body"])
|
||||
elif line.startswith("user:"):
|
||||
parts.append(line[5:].lstrip())
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def execute_prompt(prompt: dict, profile_id: str, purpose: str, data_class: str, context: dict[str, Any] | None = None) -> dict:
|
||||
def execute_prompt(
|
||||
prompt: dict,
|
||||
profile_id: str,
|
||||
purpose: str,
|
||||
data_class: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
*,
|
||||
max_tokens: int | None = None,
|
||||
disable_context_compression: bool = False,
|
||||
budget=None,
|
||||
diagnostics: dict[str, Any] | None = None,
|
||||
) -> dict:
|
||||
preview = preview_prompt(prompt, context)
|
||||
feature_id = prompt.get("required_feature") or "ai_calls"
|
||||
try:
|
||||
@@ -68,19 +84,13 @@ def execute_prompt(prompt: dict, profile_id: str, purpose: str, data_class: str,
|
||||
raise EngineError(exc.code, exc.message, exc.status_code) from exc
|
||||
try:
|
||||
ctx = context or {}
|
||||
source_text = user_source_text(ctx.get("dialogue_context") or "")
|
||||
extras = []
|
||||
if purpose == "journal_generate":
|
||||
extras = [part for part in (ctx.get("existing_text"), ctx.get("writing_profile")) if part]
|
||||
elif purpose == "profile_review":
|
||||
source_text = ctx.get("source_text") or ""
|
||||
extras = [part for part in (ctx.get("review_package"),) if part]
|
||||
if extras:
|
||||
source_text = "\n".join(part for part in (source_text, *extras) if part)
|
||||
elif not source_text:
|
||||
source_text = preview["rendered"]
|
||||
if purpose == "profile_review" and ctx.get("source_text"):
|
||||
source_text = "\n".join(
|
||||
part for part in (ctx.get("existing_text"), ctx.get("writing_profile")) if part
|
||||
part for part in (ctx.get("source_text"), ctx.get("review_package"), source_text) if part
|
||||
)
|
||||
elif not (source_text or "").strip():
|
||||
source_text = user_source_text(ctx.get("dialogue_context") or "")
|
||||
result = complete(
|
||||
GatewayRequest(
|
||||
prompt_id=prompt["id"],
|
||||
@@ -92,16 +102,21 @@ def execute_prompt(prompt: dict, profile_id: str, purpose: str, data_class: str,
|
||||
"source_text": source_text,
|
||||
"privacy_tokens": preview["privacy_tokens"],
|
||||
"prompt_slug": prompt.get("slug"),
|
||||
"max_tokens": max_tokens,
|
||||
"disable_context_compression": disable_context_compression,
|
||||
"budget": budget,
|
||||
"diagnostics": diagnostics or {},
|
||||
},
|
||||
)
|
||||
)
|
||||
except PrivacyGatewayError as exc:
|
||||
raise EngineError(exc.code, exc.message, exc.status_code) from exc
|
||||
raise EngineError(exc.code, exc.message, exc.status_code, getattr(exc, "diagnostics", None)) from exc
|
||||
increment_feature_usage(profile_id, feature_id)
|
||||
return {
|
||||
**preview,
|
||||
"llm": True,
|
||||
"content": result.content,
|
||||
"provider": result.provider,
|
||||
"trace": public_trace(last_trace()),
|
||||
"trace": result.trace,
|
||||
"diagnostics": result.diagnostics,
|
||||
}
|
||||
@@ -159,13 +159,13 @@ def _llm_entities(config, excerpt: str, known_labels: set[str]) -> list[dict]:
|
||||
_detect_prompt()["template"],
|
||||
{"source_text": excerpt, "known_labels": known},
|
||||
)
|
||||
raw = complete_chat(
|
||||
result = complete_chat(
|
||||
config,
|
||||
[{"role": "user", "content": prompt}],
|
||||
timeout=DETECT_TIMEOUT,
|
||||
max_tokens=DETECT_MAX_TOKENS,
|
||||
)
|
||||
return _parse_entities(raw)
|
||||
return _parse_entities(result.content)
|
||||
|
||||
|
||||
def detect_and_remember(profile_id: str | None, source_text: str) -> tuple[list[dict], str | None, str | None]:
|
||||
|
||||
+177
-29
@@ -3,12 +3,26 @@ from __future__ import annotations
|
||||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
from dialogue_store import StoreError, list_conversations_for_day, list_messages
|
||||
from engine import execute_prompt, load_active_prompt
|
||||
from engine import EngineError, execute_prompt, load_active_prompt
|
||||
from journal_policy import require_explicit_generate, source_conversation_ids
|
||||
from identity_store import list_mappings
|
||||
from journal_body import clean_title
|
||||
from journal_reconstruct import (
|
||||
assign_source_ids,
|
||||
claim_texts,
|
||||
local_verified_artifact,
|
||||
reconstruction_from_model,
|
||||
reconstruction_text,
|
||||
)
|
||||
from journal_shape import shape_journal
|
||||
from journal_store import current_draft, current_entries, get_day, insert_draft
|
||||
from model_catalog import resolve_generate_metadata
|
||||
from prompt_budget import (
|
||||
JournalBudgetError,
|
||||
estimate_tokens,
|
||||
plan_journal_budget,
|
||||
)
|
||||
from providers import generate_provider
|
||||
from writing_profile_store import compile_style_signals, remember_dialogue_style
|
||||
|
||||
|
||||
@@ -32,6 +46,57 @@ def _split_title(content: str) -> tuple[str, str]:
|
||||
return title, body or text
|
||||
|
||||
|
||||
def _local_narration(reconstruction: dict) -> tuple[str, str]:
|
||||
"""Fail-closed draft from verified local sources. No leaking model text."""
|
||||
parts = [str(part).strip() for part in claim_texts(reconstruction) if str(part).strip()]
|
||||
body = "\n\n".join(parts)
|
||||
return "Ein Tag", body
|
||||
|
||||
|
||||
def _identity_leak_result(purpose: str, exc: EngineError) -> dict:
|
||||
return {
|
||||
"content": "",
|
||||
"trace": {
|
||||
"purpose": purpose,
|
||||
"guard": "identity_leak_blocked",
|
||||
},
|
||||
"diagnostics": getattr(exc, "diagnostics", None) or {},
|
||||
}
|
||||
|
||||
|
||||
def _raise_budget(exc: JournalBudgetError) -> None:
|
||||
raise EngineError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc
|
||||
|
||||
|
||||
def _stage_trace(result: dict, fallback_purpose: str) -> dict:
|
||||
trace = dict(result.get("trace") or {})
|
||||
if result.get("diagnostics"):
|
||||
trace["budget"] = result.get("diagnostics")
|
||||
elif not trace.get("budget"):
|
||||
trace["budget"] = None
|
||||
trace["purpose"] = trace.get("purpose") or fallback_purpose
|
||||
return trace
|
||||
|
||||
|
||||
def _day_messages_from_context(context: dict) -> list[dict]:
|
||||
for item in context.get("items") or []:
|
||||
if item.get("type") == "day_messages":
|
||||
return list(item.get("messages") or [])
|
||||
return []
|
||||
|
||||
|
||||
def _existing_text(profile_id: str, journal_day_id: str) -> str:
|
||||
"""Saved user entries are the Fassung. A leftover draft is only used if none exist."""
|
||||
entries = current_entries(profile_id, journal_day_id)
|
||||
bodies = [(item.get("body") or "").strip() for item in entries if (item.get("body") or "").strip()]
|
||||
if bodies:
|
||||
return "\n\n".join(bodies)
|
||||
draft = current_draft(profile_id, journal_day_id)
|
||||
if draft:
|
||||
return (draft.get("body") or "").strip()
|
||||
return ""
|
||||
|
||||
|
||||
def generate_draft(
|
||||
profile_id: str,
|
||||
journal_day_id: str,
|
||||
@@ -47,15 +112,88 @@ def generate_draft(
|
||||
[item["id"] for item in day_conversations],
|
||||
)
|
||||
remember_dialogue_style(profile_id, exclude_conversation_ids=selected)
|
||||
existing_text = ""
|
||||
if include_existing:
|
||||
draft = current_draft(profile_id, journal_day_id)
|
||||
entries = current_entries(profile_id, journal_day_id)
|
||||
if draft:
|
||||
existing_text = draft.get("body") or ""
|
||||
elif entries:
|
||||
existing_text = entries[0].get("body") or ""
|
||||
context = build_internal_context(
|
||||
existing_text = _existing_text(profile_id, journal_day_id) if include_existing else ""
|
||||
|
||||
config = generate_provider()
|
||||
if not config:
|
||||
raise EngineError(
|
||||
"no_egress_provider_configured",
|
||||
"Persönlicher KI-Aufruf wurde vom Privacy Gateway blockiert. "
|
||||
"Es ist kein Egress-Provider konfiguriert.",
|
||||
503,
|
||||
)
|
||||
try:
|
||||
window = resolve_generate_metadata(config)
|
||||
reconstruct_budget = plan_journal_budget(window, purpose="journal_reconstruct")
|
||||
narrate_budget = plan_journal_budget(window, purpose="journal_generate")
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
|
||||
reconstruct_prompt = load_active_prompt("mvp.journal_reconstruct")
|
||||
static_tokens = estimate_tokens(reconstruct_prompt.get("template") or "")
|
||||
available_for_day = reconstruct_budget.available_input_tokens - static_tokens
|
||||
if available_for_day < 256:
|
||||
raise EngineError(
|
||||
"prompt_budget_exceeded",
|
||||
"Der Tagesdialog ist für eine sichere Verarbeitung zu umfangreich. "
|
||||
"Kanshō hat nichts stillschweigend aus der Mitte entfernt.",
|
||||
422,
|
||||
diagnostics=reconstruct_budget.as_diagnostics(),
|
||||
)
|
||||
|
||||
try:
|
||||
reconstruct_context = build_internal_context(
|
||||
profile_id,
|
||||
space_id=day["space_id"],
|
||||
journal_day_id=journal_day_id,
|
||||
purpose="journal_reconstruct",
|
||||
conversation_ids=selected,
|
||||
conversation_id=selected[0] if selected else None,
|
||||
day_spec={
|
||||
"overflow": "abort",
|
||||
"max_estimated_tokens": available_for_day,
|
||||
},
|
||||
)
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
reconstruct_assembled = assemble_text(reconstruct_context)
|
||||
source_messages = assign_source_ids(_day_messages_from_context(reconstruct_context))
|
||||
source_user = [
|
||||
message.get("body") or ""
|
||||
for message in source_messages
|
||||
if message.get("role") == "user"
|
||||
]
|
||||
reconstruct_result = {"trace": {"purpose": "journal_reconstruct"}, "content": "", "diagnostics": {}}
|
||||
try:
|
||||
reconstruct_result = execute_prompt(
|
||||
reconstruct_prompt,
|
||||
profile_id,
|
||||
purpose="journal_reconstruct",
|
||||
data_class="B",
|
||||
context=reconstruct_assembled,
|
||||
max_tokens=reconstruct_budget.reserved_output_tokens,
|
||||
disable_context_compression=True,
|
||||
budget=reconstruct_budget,
|
||||
)
|
||||
reconstruction, stage1 = reconstruction_from_model(
|
||||
reconstruct_result.get("content") or "",
|
||||
source_messages,
|
||||
)
|
||||
except EngineError as exc:
|
||||
if exc.code != "response_validation_failed":
|
||||
raise
|
||||
reconstruction = local_verified_artifact(source_messages)
|
||||
stage1 = {
|
||||
"stage1": "local_fallback",
|
||||
"reason": "identity_leak_blocked",
|
||||
"model_rejected": exc.code,
|
||||
}
|
||||
reconstruct_result = _identity_leak_result("journal_reconstruct", exc)
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
|
||||
signals = compile_style_signals(source_user)
|
||||
narrate_context = build_internal_context(
|
||||
profile_id,
|
||||
space_id=day["space_id"],
|
||||
journal_day_id=journal_day_id,
|
||||
@@ -64,27 +202,31 @@ def generate_draft(
|
||||
conversation_ids=selected,
|
||||
existing_text=existing_text,
|
||||
conversation_id=selected[0] if selected else None,
|
||||
reconstruction=reconstruction_text(reconstruction),
|
||||
)
|
||||
assembled = assemble_text(context)
|
||||
source_user = [
|
||||
message.get("body") or ""
|
||||
for conversation_id in selected
|
||||
for message in list_messages(profile_id, conversation_id)
|
||||
if message.get("role") == "user"
|
||||
]
|
||||
signals = compile_style_signals(source_user)
|
||||
assembled = assemble_text(narrate_context)
|
||||
profile = assembled.get("writing_profile") or ""
|
||||
if signals and signals not in profile:
|
||||
if signals and signals not in profile and len(profile) < 3500:
|
||||
assembled["writing_profile"] = "\n\n".join(part for part in (profile, signals) if part)
|
||||
prompt = load_active_prompt("mvp.journal_generate")
|
||||
result = execute_prompt(
|
||||
prompt,
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
data_class="B",
|
||||
context=assembled,
|
||||
)
|
||||
title, body = _split_title(result.get("content") or "")
|
||||
narrate_prompt = load_active_prompt("mvp.journal_generate")
|
||||
try:
|
||||
narrate_result = execute_prompt(
|
||||
narrate_prompt,
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
data_class="B",
|
||||
context=assembled,
|
||||
max_tokens=narrate_budget.reserved_output_tokens,
|
||||
disable_context_compression=True,
|
||||
budget=narrate_budget,
|
||||
)
|
||||
title, body = _split_title(narrate_result.get("content") or "")
|
||||
except EngineError as exc:
|
||||
if exc.code != "response_validation_failed":
|
||||
raise
|
||||
title, body = _local_narration(reconstruction)
|
||||
narrate_result = _identity_leak_result("journal_generate", exc)
|
||||
narrate_result["content"] = f"{title}\n\n{body}".strip()
|
||||
person_labels = [
|
||||
(item.get("local_label") or "").strip()
|
||||
for item in list_mappings(profile_id)
|
||||
@@ -106,5 +248,11 @@ def generate_draft(
|
||||
previous = before_entries.get(item["id"])
|
||||
if previous is not None and previous != item.get("current_version_id"):
|
||||
raise StoreError("policy_violation", "Generate darf die Nutzerfassung nicht verändern", 500)
|
||||
draft["trace"] = result.get("trace")
|
||||
reconstruct_trace = _stage_trace(reconstruct_result, "journal_reconstruct")
|
||||
reconstruct_trace.update(stage1)
|
||||
narrate_trace = _stage_trace(narrate_result, "journal_generate")
|
||||
draft["trace"] = {
|
||||
**narrate_trace,
|
||||
"stages": [reconstruct_trace, narrate_trace],
|
||||
}
|
||||
return draft
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Journal-specific first impulse. Not a generic continuation or intent engine.
|
||||
|
||||
Uses the existing Privacy Gateway, dialogue prompt, guards and request-scoped
|
||||
trace. Attested plans and open day points come only from user-role sources.
|
||||
Assistant text is conversation context, never a user fact. Recency is not a pattern.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
from dialogue_store import StoreError, append_message, get_conversation, list_conversations_for_day, list_messages
|
||||
from dialogue_turn import OPERATIONS, needs_repair, parse_turn_payload, repair_note
|
||||
from engine import EngineError, execute_prompt, load_active_prompt
|
||||
from journal_store import get_day
|
||||
from retrieval import retrieve
|
||||
|
||||
NEUTRAL_OPENING = "Wenn du magst, fang einfach an – ich höre zu."
|
||||
OPENING_POLICY = (
|
||||
"Dies ist der erste Impuls dieses Gesprächs. Es gibt hier noch keine Nutzerzeile. "
|
||||
"Nur belegte Vorhaben aus user-Quellen oder offene user-Punkte desselben Journal Day aufgreifen. "
|
||||
"Ein Recency-Treffer ist kein bewiesenes wiederkehrendes Muster. "
|
||||
"Nicht erwähnt ist nicht geschehen. Ein Plan ist kein Vollzug. "
|
||||
"Assistententext ist Gesprächskontext, kein Nutzerfakt. "
|
||||
"Keine Behauptung, ein Vorhaben sei ausgeführt worden."
|
||||
)
|
||||
|
||||
FUTURE_INTENT = re.compile(
|
||||
r"(?:"
|
||||
r"\b("
|
||||
r"morgen|übermorgen|"
|
||||
r"wollen(?: wir| sie)?|will(?:st)?|werde|werden wir|"
|
||||
r"vorhaben|geplant|"
|
||||
r"steht(?: heute| morgen)? an|"
|
||||
r"nächste[nrs]?\s+(?:woche|monat|tag|tage)"
|
||||
r")\b"
|
||||
r"|(?:habe|haben|hat|habt)\s+vor\b"
|
||||
r")",
|
||||
re.I,
|
||||
)
|
||||
UNEARNED_PATTERN = re.compile(
|
||||
r"\b(die letzten tage|häufig|immer|jedes mal|wiederkehr|typischerweise|muster)\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def is_attested_plan(text: str) -> bool:
|
||||
"""User wording that marks a plan or intention. Not past completion, not overlap."""
|
||||
body = (text or "").strip()
|
||||
if not body:
|
||||
return False
|
||||
return bool(FUTURE_INTENT.search(body))
|
||||
|
||||
|
||||
def attested_plans_from_user_texts(texts: list[str]) -> list[str]:
|
||||
plans: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for text in texts:
|
||||
body = (text or "").strip()
|
||||
if not body or body in seen or not is_attested_plan(body):
|
||||
continue
|
||||
seen.add(body)
|
||||
plans.append(body)
|
||||
return plans
|
||||
|
||||
|
||||
def _user_bodies(messages: list[dict]) -> list[str]:
|
||||
return [
|
||||
(item.get("body") or "").strip()
|
||||
for item in messages
|
||||
if item.get("role") == "user" and (item.get("body") or "").strip()
|
||||
]
|
||||
|
||||
|
||||
def _assistant_bodies(messages: list[dict]) -> list[str]:
|
||||
return [
|
||||
(item.get("body") or "").strip()
|
||||
for item in messages
|
||||
if item.get("role") == "assistant" and (item.get("body") or "").strip()
|
||||
]
|
||||
|
||||
|
||||
def collect_opening_context(profile_id: str, conversation_id: str) -> dict:
|
||||
conversation = get_conversation(profile_id, conversation_id)
|
||||
space_id = conversation.get("space_id")
|
||||
journal_day_id = conversation.get("journal_day_id")
|
||||
if not space_id or not journal_day_id:
|
||||
raise StoreError("not_journal_conversation", "Erster Impuls nur für einen Journal-Dialog.")
|
||||
get_day(profile_id, journal_day_id)
|
||||
|
||||
day_conversations = list_conversations_for_day(profile_id, journal_day_id)
|
||||
open_points: list[str] = []
|
||||
day_assistant: list[str] = []
|
||||
for item in day_conversations:
|
||||
messages = list_messages(profile_id, item["id"])
|
||||
if item["id"] == conversation_id:
|
||||
continue
|
||||
open_points.extend(_user_bodies(messages))
|
||||
day_assistant.extend(_assistant_bodies(messages))
|
||||
|
||||
prior_entries = retrieve(
|
||||
profile_id,
|
||||
{"kind": "space_entries", "space_id": space_id, "exclude_day_id": journal_day_id},
|
||||
)
|
||||
recent_sources = retrieve(
|
||||
profile_id,
|
||||
{
|
||||
"kind": "space_recent_sources",
|
||||
"space_id": space_id,
|
||||
"exclude_conversation_id": conversation_id,
|
||||
"exclude_journal_day_id": journal_day_id,
|
||||
},
|
||||
)
|
||||
recency_user: list[str] = []
|
||||
for entry in prior_entries:
|
||||
excerpt = (entry.get("excerpt") or "").strip()
|
||||
if excerpt:
|
||||
recency_user.append(excerpt)
|
||||
for source in recent_sources:
|
||||
excerpt = (source.get("excerpt") or "").strip()
|
||||
if excerpt:
|
||||
recency_user.append(excerpt)
|
||||
|
||||
user_pool = [*open_points, *recency_user]
|
||||
plans = attested_plans_from_user_texts(user_pool)
|
||||
has_relevant = bool(plans or open_points)
|
||||
return {
|
||||
"space_id": space_id,
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_id": conversation_id,
|
||||
"user_texts": user_pool,
|
||||
"assistant_texts": day_assistant,
|
||||
"attested_plans": plans,
|
||||
"open_day_points": open_points,
|
||||
"recency_excerpts": recency_user,
|
||||
"has_relevant_context": has_relevant,
|
||||
"recency_is_not_pattern": True,
|
||||
"assistant_is_not_user_fact": True,
|
||||
}
|
||||
|
||||
|
||||
def format_opening_hint(facts: dict) -> str:
|
||||
lines = [OPENING_POLICY]
|
||||
plans = facts.get("attested_plans") or []
|
||||
points = facts.get("open_day_points") or []
|
||||
if plans:
|
||||
lines.append("Belegte Vorhaben (nur user-Quellen):")
|
||||
lines.extend(f"- {item}" for item in plans)
|
||||
else:
|
||||
lines.append("Belegte Vorhaben: keine.")
|
||||
if points:
|
||||
lines.append("Offene Punkte dieses Journal Day (nur user):")
|
||||
lines.extend(f"- {item}" for item in points)
|
||||
else:
|
||||
lines.append("Offene Punkte dieses Journal Day: keine.")
|
||||
lines.append(
|
||||
"Recency-Ausschnitte dürfen den Impuls nicht als Muster oder als Vollzug begründen."
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _store_opening(profile_id: str, conversation_id: str, impulse: str, extra: dict) -> dict:
|
||||
assistant = append_message(profile_id, conversation_id, impulse, role="assistant")
|
||||
return {
|
||||
"conversation": get_conversation(profile_id, conversation_id),
|
||||
"assistant": assistant,
|
||||
"messages": list_messages(profile_id, conversation_id),
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def start_journal_opening(profile_id: str, conversation_id: str) -> dict:
|
||||
"""Create the first Kanshō line. Invalid model output does not write a message."""
|
||||
existing = list_messages(profile_id, conversation_id)
|
||||
if existing:
|
||||
return {
|
||||
"opened": False,
|
||||
"reason": "already_started",
|
||||
"conversation": get_conversation(profile_id, conversation_id),
|
||||
"messages": existing,
|
||||
}
|
||||
facts = collect_opening_context(profile_id, conversation_id)
|
||||
if not facts["has_relevant_context"]:
|
||||
return _store_opening(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
NEUTRAL_OPENING,
|
||||
{
|
||||
"opened": True,
|
||||
"kind": "local_neutral",
|
||||
"calls": 0,
|
||||
"opening_context": facts,
|
||||
"decision": {
|
||||
"operation": "fortfuehren",
|
||||
"label": OPERATIONS["fortfuehren"],
|
||||
"parsed": True,
|
||||
"guard": "local_neutral_opening",
|
||||
},
|
||||
"trace": None,
|
||||
},
|
||||
)
|
||||
|
||||
conversation = get_conversation(profile_id, conversation_id)
|
||||
day_ids = [item["id"] for item in list_conversations_for_day(profile_id, conversation["journal_day_id"])]
|
||||
context = build_internal_context(
|
||||
profile_id,
|
||||
conversation_id=conversation_id,
|
||||
space_id=conversation.get("space_id"),
|
||||
journal_day_id=conversation.get("journal_day_id"),
|
||||
purpose="dialogue_turn",
|
||||
conversation_ids=day_ids,
|
||||
)
|
||||
assembled = assemble_text(context)
|
||||
assembled = dict(assembled)
|
||||
assembled["opening_hint"] = format_opening_hint(facts)
|
||||
assembled["register_hint"] = (
|
||||
"Erster Impuls. Noch keine Nutzerzeile in diesem Gespräch. "
|
||||
"Nur belegte Vorhaben oder offene user-Punkte. Kein Muster aus Recency."
|
||||
)
|
||||
prompt = load_active_prompt("mvp.dialogue_turn")
|
||||
calls = 0
|
||||
result = None
|
||||
impulse = ""
|
||||
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
|
||||
try:
|
||||
while calls < 2:
|
||||
result = execute_prompt(
|
||||
prompt,
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
data_class="B",
|
||||
context=assembled,
|
||||
)
|
||||
calls += 1
|
||||
content = (result.get("content") or "").strip()
|
||||
if not content:
|
||||
raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.")
|
||||
impulse, decision = parse_turn_payload(content)
|
||||
if UNEARNED_PATTERN.search(impulse or ""):
|
||||
decision = {**decision, "guard": "pattern_rejected"}
|
||||
assembled = dict(assembled)
|
||||
assembled["dialogue_context"] = (
|
||||
(assembled.get("dialogue_context") or "")
|
||||
+ "\n\nKorrektur: Kein Recency-Treffer als wiederkehrendes Muster."
|
||||
)
|
||||
continue
|
||||
if not needs_repair(impulse, assembled):
|
||||
break
|
||||
decision = {**decision, "guard": "impulse_rejected"}
|
||||
assembled = dict(assembled)
|
||||
assembled["dialogue_context"] = (
|
||||
(assembled.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, assembled)
|
||||
)
|
||||
if needs_repair(impulse, assembled) or UNEARNED_PATTERN.search(impulse or "") or not impulse:
|
||||
impulse = NEUTRAL_OPENING
|
||||
decision = {
|
||||
"operation": "fortfuehren",
|
||||
"label": OPERATIONS["fortfuehren"],
|
||||
"parsed": bool(decision.get("parsed")),
|
||||
"guard": "local_fallback",
|
||||
}
|
||||
except EngineError as exc:
|
||||
if exc.code != "response_validation_failed":
|
||||
raise
|
||||
return _store_opening(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
NEUTRAL_OPENING,
|
||||
{
|
||||
"opened": True,
|
||||
"kind": "local_neutral",
|
||||
"calls": calls,
|
||||
"opening_context": facts,
|
||||
"decision": {
|
||||
"operation": "fortfuehren",
|
||||
"label": OPERATIONS["fortfuehren"],
|
||||
"parsed": False,
|
||||
"guard": "identity_leak_blocked",
|
||||
},
|
||||
"trace": result.get("trace") if result else None,
|
||||
},
|
||||
)
|
||||
return _store_opening(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
impulse,
|
||||
{
|
||||
"opened": True,
|
||||
"kind": "model",
|
||||
"calls": calls,
|
||||
"opening_context": facts,
|
||||
"decision": decision,
|
||||
"trace": result.get("trace") if result else None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def maybe_open_journal_conversation(profile_id: str, conversation: dict) -> dict:
|
||||
payload = dict(conversation)
|
||||
try:
|
||||
opening = start_journal_opening(profile_id, conversation["id"])
|
||||
except EngineError as exc:
|
||||
payload["opening"] = {
|
||||
"opened": False,
|
||||
"reason": exc.code,
|
||||
"message": exc.message,
|
||||
"kind": "failed_closed",
|
||||
}
|
||||
payload["messages"] = list_messages(profile_id, conversation["id"])
|
||||
return payload
|
||||
payload["opening"] = opening
|
||||
payload["messages"] = opening.get("messages") or []
|
||||
return payload
|
||||
@@ -0,0 +1,529 @@
|
||||
"""Journal adapter over the intent-neutral provenance layer. No egress.
|
||||
|
||||
Journal policy: selected user sources, full canonical rehydration, assistant never
|
||||
a content source, source-local clocks, reorder only with confirmed times.
|
||||
Semantic labels from the model are not verified content.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from prompt_budget import ERROR_NO_USER_SOURCES, ERROR_RECONSTRUCTION_INVALID, JournalBudgetError
|
||||
from provenance import (
|
||||
COVERAGE_ALL_SELECTED_SOURCES,
|
||||
EvidenceRef,
|
||||
ProvenanceError,
|
||||
SourceRecord,
|
||||
SourceRegistry,
|
||||
VerificationPolicy,
|
||||
texts_from_payload,
|
||||
verify,
|
||||
)
|
||||
|
||||
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
||||
TIME_RE = re.compile(r"\b\d{1,2}:\d{2}\b|\b\d{1,2}\s*Uhr\b", re.I)
|
||||
CLOCK_STAMP_RE = re.compile(r"^(?:\d{1,2}:\d{2}(?:\s*Uhr)?|\d{1,2}\s*Uhr)$", re.I)
|
||||
DIALOGUE_LINE = re.compile(
|
||||
r"^(?:\[(?P<source_id>u\d+|a\d+)\]\s*)?(?P<role>user|assistant):\s*(?P<body>.*)$"
|
||||
)
|
||||
USER_SOURCE_ID = re.compile(r"^u\d+$")
|
||||
ASSISTANT_SOURCE_ID = re.compile(r"^a\d+$")
|
||||
REQUIRED_TOP = ("source_order", "chronology")
|
||||
LEGACY_FACT_KEYS = (
|
||||
"events",
|
||||
"people",
|
||||
"places",
|
||||
"perceptions",
|
||||
"feelings_stated",
|
||||
"evaluations",
|
||||
"quotes",
|
||||
"uncertainties",
|
||||
"plan_changes",
|
||||
"corrections",
|
||||
"evidence",
|
||||
"text",
|
||||
)
|
||||
SOURCED_LIST_KEYS = ("contradictions", "uncertainties", "plan_changes", "corrections")
|
||||
ALLOWED_CLAIM_KEYS = frozenset({"kind", "evidence"})
|
||||
ALLOWED_ITEM_KEYS = frozenset({"source_id", "source", "time", "claims"})
|
||||
ALLOWED_SOURCED_KEYS = frozenset({"source_id", "evidence"})
|
||||
CONTENT_ROLE = "user"
|
||||
JOURNAL_ALLOWED_ROLES = frozenset({CONTENT_ROLE})
|
||||
|
||||
|
||||
def _fail(reason: str, **extra: Any) -> None:
|
||||
raise JournalBudgetError(
|
||||
ERROR_RECONSTRUCTION_INVALID,
|
||||
diagnostics={"reason": reason, **extra},
|
||||
)
|
||||
|
||||
|
||||
def parse_dialogue_line(line: str) -> dict[str, str] | None:
|
||||
match = DIALOGUE_LINE.match((line or "").rstrip("\n"))
|
||||
if not match:
|
||||
return None
|
||||
return {
|
||||
"source_id": match.group("source_id") or "",
|
||||
"role": match.group("role"),
|
||||
"body": match.group("body") or "",
|
||||
}
|
||||
|
||||
|
||||
def is_dialogue_role_line(line: str) -> bool:
|
||||
return parse_dialogue_line(line) is not None
|
||||
|
||||
|
||||
def assign_source_ids(messages: list[dict]) -> list[dict]:
|
||||
"""Local, stable IDs in encounter order. User u1…; assistant a1…."""
|
||||
labeled: list[dict] = []
|
||||
user_n = 0
|
||||
assistant_n = 0
|
||||
for message in messages or []:
|
||||
item = dict(message)
|
||||
role = item.get("role") or "user"
|
||||
body = (item.get("body") or "").strip()
|
||||
if role == "user" and body:
|
||||
user_n += 1
|
||||
item["source_id"] = f"u{user_n}"
|
||||
elif role == "assistant" and body:
|
||||
assistant_n += 1
|
||||
item["source_id"] = f"a{assistant_n}"
|
||||
else:
|
||||
item["source_id"] = None
|
||||
labeled.append(item)
|
||||
return labeled
|
||||
|
||||
|
||||
def expected_user_source_ids(messages: list[dict]) -> list[str]:
|
||||
return [
|
||||
item["source_id"]
|
||||
for item in assign_source_ids(messages)
|
||||
if item.get("role") == "user" and item.get("source_id")
|
||||
]
|
||||
|
||||
|
||||
def parse_reconstruction(raw: str) -> dict[str, Any]:
|
||||
text = (raw or "").strip()
|
||||
if text.startswith("```"):
|
||||
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL)
|
||||
match = JSON_BLOCK.search(text)
|
||||
if not match:
|
||||
_fail("not_json")
|
||||
try:
|
||||
data = json.loads(match.group(0))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise JournalBudgetError(
|
||||
ERROR_RECONSTRUCTION_INVALID,
|
||||
diagnostics={"reason": "json_parse"},
|
||||
) from exc
|
||||
if not isinstance(data, dict):
|
||||
_fail("not_object")
|
||||
return data
|
||||
|
||||
|
||||
def parse_clock(stamp: str) -> tuple[int, int] | None:
|
||||
text = (stamp or "").strip()
|
||||
match = re.search(r"(\d{1,2}):(\d{2})", text)
|
||||
if match:
|
||||
hour, minute = int(match.group(1)), int(match.group(2))
|
||||
if 0 <= hour <= 23 and 0 <= minute <= 59:
|
||||
return hour, minute
|
||||
return None
|
||||
match = re.search(r"(\d{1,2})\s*Uhr", text, re.I)
|
||||
if match:
|
||||
hour = int(match.group(1))
|
||||
if 0 <= hour <= 23:
|
||||
return hour, 0
|
||||
return None
|
||||
|
||||
|
||||
def clocks_in_text(text: str) -> set[tuple[int, int]]:
|
||||
found: set[tuple[int, int]] = set()
|
||||
for match in TIME_RE.finditer(text or ""):
|
||||
parsed = parse_clock(match.group(0))
|
||||
if parsed:
|
||||
found.add(parsed)
|
||||
return found
|
||||
|
||||
|
||||
def clock_in_source(stamp: str, source_body: str) -> bool:
|
||||
parsed = parse_clock(stamp)
|
||||
if parsed is None:
|
||||
return False
|
||||
return parsed in clocks_in_text(source_body)
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return [value]
|
||||
|
||||
|
||||
def _legacy_payload(raw: dict[str, Any]) -> bool:
|
||||
for key in LEGACY_FACT_KEYS:
|
||||
values = [str(part).strip() for part in _as_list(raw.get(key)) if str(part).strip()]
|
||||
if values:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _normalize_claim(raw: Any, *, index: int, claim_index: int) -> dict[str, str]:
|
||||
if not isinstance(raw, dict):
|
||||
_fail("claim_shape", index=index, claim_index=claim_index)
|
||||
extra = {key: raw.get(key) for key in raw.keys() if key not in ALLOWED_CLAIM_KEYS}
|
||||
if any(value not in (None, "", [], {}) for value in extra.values()):
|
||||
_fail("unverified_claim", index=index, claim_index=claim_index)
|
||||
evidence = str(raw.get("evidence") or "").strip()
|
||||
if not evidence:
|
||||
_fail("evidence_missing", index=index, claim_index=claim_index)
|
||||
kind = raw.get("kind")
|
||||
return {"kind": "" if kind is None else str(kind), "evidence": evidence}
|
||||
|
||||
|
||||
def _normalize_sourced_item(raw: Any, *, key: str, index: int) -> dict[str, str]:
|
||||
if isinstance(raw, str) and raw.strip():
|
||||
_fail("unverified_claim", key=key, index=index)
|
||||
if not isinstance(raw, dict):
|
||||
_fail("sourced_item_shape", key=key, index=index)
|
||||
extra = {item: raw.get(item) for item in raw.keys() if item not in ALLOWED_SOURCED_KEYS}
|
||||
if any(value not in (None, "", [], {}) for value in extra.values()):
|
||||
_fail("unverified_claim", key=key, index=index)
|
||||
source_id = str(raw.get("source_id") or "").strip()
|
||||
evidence = str(raw.get("evidence") or "").strip()
|
||||
if not source_id:
|
||||
_fail("missing_source_id", key=key, index=index)
|
||||
if not evidence:
|
||||
_fail("evidence_missing", key=key, index=index)
|
||||
return {"source_id": source_id, "evidence": evidence}
|
||||
|
||||
|
||||
def _confirmed_minutes(stamp: str | None, cited: str) -> int | None:
|
||||
if not stamp:
|
||||
return None
|
||||
parsed = parse_clock(stamp)
|
||||
if parsed is None or parsed not in clocks_in_text(cited):
|
||||
return None
|
||||
return parsed[0] * 60 + parsed[1]
|
||||
|
||||
|
||||
def _reorder_allowed(
|
||||
items: list[dict[str, Any]],
|
||||
expected_ids: list[str],
|
||||
user_by_id: dict[str, str],
|
||||
) -> bool:
|
||||
chrono_ids = [item["source_id"] for item in items]
|
||||
if chrono_ids == expected_ids:
|
||||
return True
|
||||
minutes: list[int] = []
|
||||
for item in items:
|
||||
value = _confirmed_minutes(item.get("time"), user_by_id.get(item["source_id"]) or "")
|
||||
if value is None:
|
||||
return False
|
||||
minutes.append(value)
|
||||
for index in range(len(items) - 1):
|
||||
if minutes[index] > minutes[index + 1]:
|
||||
return False
|
||||
if minutes[index] == minutes[index + 1]:
|
||||
left = expected_ids.index(items[index]["source_id"])
|
||||
right = expected_ids.index(items[index + 1]["source_id"])
|
||||
if left > right:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _registry_from_messages(labeled: list[dict]) -> SourceRegistry:
|
||||
registry = SourceRegistry()
|
||||
for item in labeled:
|
||||
source_id = item.get("source_id")
|
||||
role = item.get("role") or ""
|
||||
body = item.get("body") or ""
|
||||
if not source_id or not body.strip():
|
||||
continue
|
||||
registry.add(
|
||||
SourceRecord(
|
||||
source_id=source_id,
|
||||
role=role,
|
||||
text=body,
|
||||
metadata={"conversation_id": item.get("conversation_id")},
|
||||
)
|
||||
)
|
||||
return registry
|
||||
|
||||
|
||||
def _map_provenance(exc: ProvenanceError) -> None:
|
||||
reason = exc.reason
|
||||
details = dict(exc.details or {})
|
||||
if reason == "unknown_source":
|
||||
_fail("unknown_source_id", **details)
|
||||
if reason == "role_not_allowed":
|
||||
role = details.get("role")
|
||||
source_id = str(details.get("source_id") or "")
|
||||
if role != CONTENT_ROLE or ASSISTANT_SOURCE_ID.match(source_id):
|
||||
_fail("assistant_as_fact", **details)
|
||||
_fail("invalid_source", **details)
|
||||
if reason in {"unverified_excerpt", "canonical_overwrite"}:
|
||||
_fail("unverified_claim", **details)
|
||||
if reason == "wrong_source":
|
||||
_fail("wrong_source", **details)
|
||||
if reason == "duplicate_source_id":
|
||||
_fail("duplicate_source_id", **details)
|
||||
if reason == "missing_source_id":
|
||||
_fail("missing_source_id", **details)
|
||||
_fail(reason, **details)
|
||||
|
||||
|
||||
def _journal_policy(expected_ids: list[str], chronology_ids: list[str]) -> VerificationPolicy:
|
||||
return VerificationPolicy(
|
||||
allowed_roles=JOURNAL_ALLOWED_ROLES,
|
||||
coverage=COVERAGE_ALL_SELECTED_SOURCES,
|
||||
selected_ids=tuple(chronology_ids),
|
||||
source_order=tuple(expected_ids),
|
||||
include_unverified_annotations=False,
|
||||
)
|
||||
|
||||
|
||||
def validate_reconstruction(
|
||||
data: dict[str, Any],
|
||||
*,
|
||||
messages: list[dict] | None = None,
|
||||
user_bodies: list[str] | None = None,
|
||||
assistant_bodies: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
for key in REQUIRED_TOP:
|
||||
if key not in data:
|
||||
_fail("missing_key", key=key)
|
||||
chronology = data.get("chronology")
|
||||
source_order_raw = data.get("source_order")
|
||||
if not isinstance(chronology, list):
|
||||
_fail("chronology_missing")
|
||||
if not isinstance(source_order_raw, list):
|
||||
_fail("source_order_shape")
|
||||
|
||||
if messages is not None:
|
||||
labeled = assign_source_ids(messages)
|
||||
else:
|
||||
labeled = []
|
||||
for index, body in enumerate(user_bodies or [], start=1):
|
||||
if (body or "").strip():
|
||||
labeled.append({"role": "user", "body": body, "source_id": f"u{index}"})
|
||||
for index, body in enumerate(assistant_bodies or [], start=1):
|
||||
if (body or "").strip():
|
||||
labeled.append({"role": "assistant", "body": body, "source_id": f"a{index}"})
|
||||
|
||||
expected_ids = [
|
||||
item["source_id"] for item in labeled if item.get("role") == CONTENT_ROLE and item.get("source_id")
|
||||
]
|
||||
user_by_id = {
|
||||
item["source_id"]: item.get("body") or ""
|
||||
for item in labeled
|
||||
if item.get("role") == CONTENT_ROLE and item.get("source_id")
|
||||
}
|
||||
source_order = [str(part).strip() for part in source_order_raw]
|
||||
if source_order != expected_ids:
|
||||
_fail("source_order_mismatch", expected=expected_ids, got=source_order)
|
||||
|
||||
items: list[dict[str, Any]] = []
|
||||
evidence_refs: list[EvidenceRef] = []
|
||||
seen: list[str] = []
|
||||
for index, raw in enumerate(chronology):
|
||||
if not isinstance(raw, dict):
|
||||
_fail("chronology_item_shape", index=index)
|
||||
if _legacy_payload(raw):
|
||||
_fail("unverified_claim", index=index)
|
||||
extra_keys = [
|
||||
key for key in raw.keys() if key not in ALLOWED_ITEM_KEYS and raw.get(key) not in (None, "", [], {})
|
||||
]
|
||||
if extra_keys:
|
||||
_fail("unverified_claim", index=index, keys=extra_keys)
|
||||
if "source" not in raw or raw.get("source") is None:
|
||||
source = None
|
||||
elif not isinstance(raw.get("source"), str):
|
||||
_fail("invalid_source", source=raw.get("source"), index=index)
|
||||
elif raw.get("source") == "":
|
||||
source = None
|
||||
else:
|
||||
source = raw.get("source")
|
||||
claims_raw = raw.get("claims")
|
||||
if claims_raw is None:
|
||||
claims_raw = []
|
||||
if not isinstance(claims_raw, list):
|
||||
_fail("claims_shape", index=index)
|
||||
source_id = str(raw.get("source_id") or "").strip()
|
||||
if not source_id:
|
||||
_fail("missing_source_id", index=index)
|
||||
if ASSISTANT_SOURCE_ID.match(source_id):
|
||||
_fail("assistant_as_fact", source_id=source_id, index=index)
|
||||
if source_id not in user_by_id:
|
||||
_fail("unknown_source_id", source_id=source_id, index=index)
|
||||
if source_id in seen:
|
||||
_fail("duplicate_source_id", source_id=source_id, index=index)
|
||||
seen.append(source_id)
|
||||
if source is None:
|
||||
_fail("missing_source", source_id=source_id, index=index)
|
||||
if source != CONTENT_ROLE:
|
||||
_fail("invalid_source", source=source, source_id=source_id, index=index)
|
||||
claims = [
|
||||
_normalize_claim(claim, index=index, claim_index=claim_index)
|
||||
for claim_index, claim in enumerate(claims_raw)
|
||||
]
|
||||
if not claims:
|
||||
_fail("empty_event", source_id=source_id, index=index)
|
||||
stamp = raw.get("time")
|
||||
if isinstance(stamp, str):
|
||||
stamp = stamp.strip() or None
|
||||
elif stamp is None:
|
||||
stamp = None
|
||||
else:
|
||||
stamp = str(stamp).strip() or None
|
||||
cited = user_by_id[source_id]
|
||||
if stamp:
|
||||
if (
|
||||
not CLOCK_STAMP_RE.match(stamp)
|
||||
or parse_clock(stamp) is None
|
||||
or not clock_in_source(stamp, cited)
|
||||
):
|
||||
_fail("time_source_mismatch", source_id=source_id, time=stamp, index=index)
|
||||
items.append({"source_id": source_id, "time": stamp, "claims": claims})
|
||||
for claim in claims:
|
||||
evidence_refs.append(EvidenceRef(source_id, claim["evidence"]))
|
||||
|
||||
missing = [sid for sid in expected_ids if sid not in seen]
|
||||
if missing:
|
||||
_fail("incomplete_coverage", missing=missing)
|
||||
if expected_ids and not items:
|
||||
_fail("empty_chronology")
|
||||
|
||||
if not _reorder_allowed(items, expected_ids, user_by_id):
|
||||
_fail(
|
||||
"reorder_unjustified",
|
||||
expected=expected_ids,
|
||||
chronology=[item["source_id"] for item in items],
|
||||
)
|
||||
|
||||
for key in SOURCED_LIST_KEYS:
|
||||
raw_list = data.get(key)
|
||||
if raw_list is None:
|
||||
continue
|
||||
if not isinstance(raw_list, list):
|
||||
_fail("sourced_list_shape", key=key)
|
||||
for index, raw in enumerate(raw_list):
|
||||
row = _normalize_sourced_item(raw, key=key, index=index)
|
||||
if ASSISTANT_SOURCE_ID.match(row["source_id"]):
|
||||
_fail("assistant_as_fact", source_id=row["source_id"], key=key)
|
||||
evidence_refs.append(EvidenceRef(row["source_id"], row["evidence"]))
|
||||
|
||||
registry = _registry_from_messages(labeled)
|
||||
policy = _journal_policy(expected_ids, [item["source_id"] for item in items])
|
||||
try:
|
||||
artifact = verify(registry, policy=policy, evidence=evidence_refs)
|
||||
except ProvenanceError as exc:
|
||||
_map_provenance(exc)
|
||||
return artifact.to_payload()
|
||||
|
||||
|
||||
def local_verified_artifact(messages: list[dict]) -> dict[str, Any]:
|
||||
"""Authoritative coverage from local user sources. No model JSON required."""
|
||||
labeled = assign_source_ids(messages)
|
||||
users = [
|
||||
item
|
||||
for item in labeled
|
||||
if item.get("role") == CONTENT_ROLE and item.get("source_id") and (item.get("body") or "").strip()
|
||||
]
|
||||
if not users:
|
||||
raise JournalBudgetError(
|
||||
ERROR_NO_USER_SOURCES,
|
||||
diagnostics={"reason": "no_user_sources"},
|
||||
)
|
||||
expected_ids = [item["source_id"] for item in users]
|
||||
registry = _registry_from_messages(labeled)
|
||||
evidence = [EvidenceRef(item["source_id"], (item.get("body") or "").strip()) for item in users]
|
||||
policy = _journal_policy(expected_ids, expected_ids)
|
||||
try:
|
||||
artifact = verify(registry, policy=policy, evidence=evidence)
|
||||
except ProvenanceError as exc:
|
||||
_map_provenance(exc)
|
||||
return artifact.to_payload()
|
||||
|
||||
|
||||
def reconstruction_from_model(raw_content: str, messages: list[dict]) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Stage 1 model is untrusted. Invalid output never drops local user sources."""
|
||||
try:
|
||||
parsed = parse_reconstruction(raw_content)
|
||||
artifact = validate_reconstruction(parsed, messages=messages)
|
||||
return artifact, {"stage1": "model_accepted"}
|
||||
except JournalBudgetError as exc:
|
||||
if exc.code == ERROR_NO_USER_SOURCES:
|
||||
raise
|
||||
artifact = local_verified_artifact(messages)
|
||||
return artifact, {
|
||||
"stage1": "local_fallback",
|
||||
"model_rejected": exc.code,
|
||||
"reason": (exc.diagnostics or {}).get("reason") or exc.code,
|
||||
}
|
||||
|
||||
|
||||
def reconstruction_text(data: dict[str, Any]) -> str:
|
||||
return json.dumps(data, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def _claim_for_body(body: str) -> list[dict[str, str]]:
|
||||
claims = [{"kind": "event", "evidence": body}]
|
||||
if TIME_RE.search(body):
|
||||
stamp = TIME_RE.search(body).group(0)
|
||||
claims.append({"kind": "time", "evidence": stamp})
|
||||
return claims
|
||||
|
||||
|
||||
def fake_reconstruction(rendered: str) -> str:
|
||||
"""Deterministic stage-1 stand-in. Claims remain untrusted; the adapter rehydrates."""
|
||||
chronology = []
|
||||
source_order: list[str] = []
|
||||
user_n = 0
|
||||
for line in (rendered or "").splitlines():
|
||||
parsed = parse_dialogue_line(line)
|
||||
if parsed is None and line.startswith("user:"):
|
||||
parsed = {"role": "user", "body": line[5:].strip(), "source_id": ""}
|
||||
if not parsed or parsed["role"] != "user":
|
||||
continue
|
||||
body = (parsed["body"] or "").strip()
|
||||
if not body:
|
||||
continue
|
||||
user_n += 1
|
||||
source_id = parsed["source_id"] if USER_SOURCE_ID.match(parsed["source_id"] or "") else f"u{user_n}"
|
||||
source_order.append(source_id)
|
||||
times = TIME_RE.findall(body)
|
||||
chronology.append(
|
||||
{
|
||||
"source_id": source_id,
|
||||
"source": "user",
|
||||
"time": times[0] if times else None,
|
||||
"claims": _claim_for_body(body),
|
||||
}
|
||||
)
|
||||
payload = {
|
||||
"source_order": source_order,
|
||||
"chronology": chronology,
|
||||
"contradictions": [],
|
||||
"uncertainties": [],
|
||||
"plan_changes": [],
|
||||
"corrections": [],
|
||||
}
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def claim_texts(data: dict[str, Any]) -> list[str]:
|
||||
parts = texts_from_payload(data)
|
||||
if parts:
|
||||
return parts
|
||||
collected: list[str] = []
|
||||
for item in data.get("chronology") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
for claim in item.get("claims") or []:
|
||||
if isinstance(claim, dict) and claim.get("evidence"):
|
||||
collected.append(str(claim["evidence"]))
|
||||
return collected
|
||||
+303
-1
@@ -5,6 +5,7 @@ import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from dialogue_store import (
|
||||
StoreError,
|
||||
@@ -270,7 +271,7 @@ def current_entries(profile_id: str, journal_day_id: str) -> list[dict]:
|
||||
FROM journal_entries e
|
||||
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
||||
WHERE e.profile_id = ? AND e.journal_day_id = ? AND e.deleted_at IS NULL
|
||||
ORDER BY e.created
|
||||
ORDER BY e.created, e.rowid
|
||||
""",
|
||||
(profile_id, journal_day_id),
|
||||
).fetchall()
|
||||
@@ -472,3 +473,304 @@ def day_payload(profile_id: str, journal_day_id: str) -> dict:
|
||||
"scratch": _parse_scratch(day.get("scratch_json")),
|
||||
"consolidation_offer": consolidation_offer(conversations),
|
||||
}
|
||||
|
||||
|
||||
def _entry_list_row(conn, row) -> dict:
|
||||
item = row_to_dict(row)
|
||||
item["title"] = clean_title(item.get("title") or "")
|
||||
item["version_count"] = int(item.get("version_count") or 0)
|
||||
return item
|
||||
|
||||
|
||||
def list_space_entries(profile_id: str, space_id: str) -> list[dict]:
|
||||
"""Chronological journal entries of a space. Deleted entries are excluded."""
|
||||
get_user_space(profile_id, space_id)
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT e.id, e.profile_id, e.space_id, e.journal_day_id, e.current_version_id,
|
||||
e.created, e.updated, e.deleted_at,
|
||||
d.calendar_date, v.title, v.origin,
|
||||
(SELECT COUNT(*) FROM journal_entry_versions ver
|
||||
WHERE ver.entry_id = e.id AND ver.profile_id = e.profile_id) AS version_count,
|
||||
(SELECT r.source_id FROM journal_entry_version_source_refs r
|
||||
WHERE r.version_id = e.current_version_id AND r.source_kind = 'conversation'
|
||||
ORDER BY r.sort_order LIMIT 1) AS source_conversation_id
|
||||
FROM journal_entries e
|
||||
JOIN journal_days d ON d.id = e.journal_day_id
|
||||
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
||||
WHERE e.profile_id = ? AND e.space_id = ? AND e.deleted_at IS NULL
|
||||
ORDER BY d.calendar_date DESC, e.created ASC, e.rowid ASC
|
||||
""",
|
||||
(profile_id, space_id),
|
||||
).fetchall()
|
||||
return [_entry_list_row(conn, row) for row in rows]
|
||||
|
||||
|
||||
def list_deleted_entries(profile_id: str, space_id: str | None = None) -> list[dict]:
|
||||
if space_id:
|
||||
get_user_space(profile_id, space_id)
|
||||
params: list = [profile_id]
|
||||
space_sql = ""
|
||||
if space_id:
|
||||
space_sql = "AND e.space_id = ?"
|
||||
params.append(space_id)
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT e.id, e.profile_id, e.space_id, e.journal_day_id, e.current_version_id,
|
||||
e.created, e.updated, e.deleted_at,
|
||||
d.calendar_date, v.title, v.origin,
|
||||
(SELECT COUNT(*) FROM journal_entry_versions ver
|
||||
WHERE ver.entry_id = e.id AND ver.profile_id = e.profile_id) AS version_count,
|
||||
(SELECT r.source_id FROM journal_entry_version_source_refs r
|
||||
WHERE r.version_id = e.current_version_id AND r.source_kind = 'conversation'
|
||||
ORDER BY r.sort_order LIMIT 1) AS source_conversation_id
|
||||
FROM journal_entries e
|
||||
JOIN journal_days d ON d.id = e.journal_day_id
|
||||
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
||||
WHERE e.profile_id = ? {space_sql} AND e.deleted_at IS NOT NULL
|
||||
ORDER BY e.deleted_at DESC, e.id ASC
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
return [_entry_list_row(conn, row) for row in rows]
|
||||
|
||||
|
||||
def undelete_entry(profile_id: str, entry_id: str) -> dict:
|
||||
"""Return a soft-deleted entry to the space list. Not a version restore."""
|
||||
entry = get_entry(profile_id, entry_id, include_deleted=True)
|
||||
if not entry.get("deleted_at"):
|
||||
raise StoreError("not_deleted", "Eintrag ist nicht im Papierkorb.")
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE journal_entries
|
||||
SET deleted_at = NULL, updated = datetime('now')
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(entry_id, profile_id),
|
||||
)
|
||||
return get_entry(profile_id, entry_id)
|
||||
|
||||
|
||||
def continuable_journal_day(profile_id: str) -> dict | None:
|
||||
"""Most recently active journal day that already has a stored message."""
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT d.id AS journal_day_id, d.space_id, d.calendar_date,
|
||||
s.title AS space_title, c.id AS conversation_id, m.created AS last_message_at
|
||||
FROM messages m
|
||||
JOIN conversations c ON c.id = m.conversation_id AND c.profile_id = m.profile_id
|
||||
JOIN journal_days d ON d.id = c.journal_day_id AND d.profile_id = m.profile_id
|
||||
JOIN spaces s ON s.id = d.space_id AND s.profile_id = m.profile_id
|
||||
WHERE m.profile_id = ? AND s.visibility = 'user'
|
||||
ORDER BY m.created DESC, m.seq DESC, m.id DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"space_id": row["space_id"],
|
||||
"space_title": row.get("space_title") or "",
|
||||
"journal_day_id": row["journal_day_id"],
|
||||
"calendar_date": row["calendar_date"],
|
||||
"conversation_id": row["conversation_id"],
|
||||
"last_message_at": row.get("last_message_at"),
|
||||
"reason": "recent_dialogue",
|
||||
}
|
||||
|
||||
|
||||
def start_payload(profile_id: str) -> dict:
|
||||
spaces = list_user_spaces(profile_id)
|
||||
return {
|
||||
"continuable": continuable_journal_day(profile_id),
|
||||
"has_spaces": bool(spaces),
|
||||
"space_count": len(spaces),
|
||||
"spaces": [{"id": item["id"], "title": item.get("title") or ""} for item in spaces],
|
||||
}
|
||||
|
||||
|
||||
def _bodies_referencing_media(conn, profile_id: str, media_id: str, exclude_entry_id: str) -> list[str]:
|
||||
from journal_body import media_ids
|
||||
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT v.entry_id, v.body
|
||||
FROM journal_entry_versions v
|
||||
WHERE v.profile_id = ? AND v.entry_id != ?
|
||||
""",
|
||||
(profile_id, exclude_entry_id),
|
||||
).fetchall()
|
||||
owners: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for row in rows:
|
||||
if media_id in media_ids(row["body"] or "") and row["entry_id"] not in seen:
|
||||
seen.add(row["entry_id"])
|
||||
owners.append(row["entry_id"])
|
||||
return owners
|
||||
|
||||
|
||||
def inventory_entry_for_purge(profile_id: str, entry_id: str) -> dict:
|
||||
entry = get_entry(profile_id, entry_id, include_deleted=True)
|
||||
with get_db() as conn:
|
||||
versions = [
|
||||
row_to_dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT id, origin, created FROM journal_entry_versions WHERE profile_id = ? AND entry_id = ? ORDER BY created",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
]
|
||||
version_ids = [item["id"] for item in versions]
|
||||
source_refs = 0
|
||||
if version_ids:
|
||||
placeholders = ",".join("?" * len(version_ids))
|
||||
source_refs = conn.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS n FROM journal_entry_version_source_refs
|
||||
WHERE version_id IN ({placeholders}) AND profile_id = ?
|
||||
""",
|
||||
(*version_ids, profile_id),
|
||||
).fetchone()["n"]
|
||||
writing_sources = [
|
||||
row_to_dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT id, kind FROM writing_profile_sources WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
]
|
||||
evidence = [
|
||||
row_to_dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM writing_profile_evidence WHERE profile_id = ? AND source_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
]
|
||||
media_rows = [
|
||||
row_to_dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT * FROM media_assets WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
]
|
||||
media = []
|
||||
from media_store import media_root
|
||||
|
||||
root = media_root()
|
||||
for row in media_rows:
|
||||
owners = _bodies_referencing_media(conn, profile_id, row["id"], entry_id)
|
||||
path = root / row["rel_path"]
|
||||
media.append(
|
||||
{
|
||||
"id": row["id"],
|
||||
"rel_path": row["rel_path"],
|
||||
"path_exists": path.is_file(),
|
||||
"referenced_by_entries": owners,
|
||||
"keep_file": bool(owners),
|
||||
"reassign_to": owners[0] if owners else None,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"entry": {"id": entry["id"], "space_id": entry["space_id"], "deleted_at": entry.get("deleted_at")},
|
||||
"versions": versions,
|
||||
"source_ref_count": int(source_refs or 0),
|
||||
"writing_profile_sources": writing_sources,
|
||||
"writing_profile_evidence": evidence,
|
||||
"media": media,
|
||||
"conversations_untouched": True,
|
||||
}
|
||||
|
||||
|
||||
def purge_entry(profile_id: str, entry_id: str, *, confirm: bool) -> dict:
|
||||
"""Irreversible delete after explicit confirm. Source dialogues stay."""
|
||||
if not confirm:
|
||||
raise StoreError("confirm_required", "Endgültiges Löschen braucht eine ausdrückliche Bestätigung.")
|
||||
entry = get_entry(profile_id, entry_id, include_deleted=True)
|
||||
if not entry.get("deleted_at"):
|
||||
raise StoreError("not_in_trash", "Endgültiges Löschen nur aus dem Papierkorb.")
|
||||
inventory = inventory_entry_for_purge(profile_id, entry_id)
|
||||
from media_store import media_root
|
||||
|
||||
root = media_root()
|
||||
files_to_delete: list[Path] = []
|
||||
try:
|
||||
with get_db() as conn:
|
||||
for item in inventory["media"]:
|
||||
if item.get("keep_file") and item.get("reassign_to"):
|
||||
other = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT id FROM journal_entries WHERE id = ? AND profile_id = ?",
|
||||
(item["reassign_to"], profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not other:
|
||||
raise StoreError("purge_conflict", "Geteiltes Medium hat keinen sicheren neuen Träger.")
|
||||
conn.execute(
|
||||
"UPDATE media_assets SET entry_id = ? WHERE id = ? AND profile_id = ?",
|
||||
(item["reassign_to"], item["id"], profile_id),
|
||||
)
|
||||
elif not item.get("keep_file"):
|
||||
files_to_delete.append(root / item["rel_path"])
|
||||
conn.execute(
|
||||
"DELETE FROM writing_profile_sources WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM writing_profile_evidence WHERE profile_id = ? AND source_id = ?",
|
||||
(profile_id, entry_id),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM journal_entries WHERE id = ? AND profile_id = ?",
|
||||
(entry_id, profile_id),
|
||||
)
|
||||
leftover = conn.execute(
|
||||
"SELECT id FROM journal_entries WHERE id = ? AND profile_id = ?",
|
||||
(entry_id, profile_id),
|
||||
).fetchone()
|
||||
if leftover:
|
||||
raise StoreError("purge_failed", "Eintrag konnte nicht vollständig gelöscht werden.", 500)
|
||||
leftover_media = conn.execute(
|
||||
"SELECT id FROM media_assets WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchone()
|
||||
if leftover_media:
|
||||
raise StoreError("purge_failed", "Medienverweise konnten nicht aufgelöst werden.", 500)
|
||||
leftover_sources = conn.execute(
|
||||
"SELECT id FROM writing_profile_sources WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchone()
|
||||
if leftover_sources:
|
||||
raise StoreError("purge_failed", "Writing-Profile-Quellen konnten nicht gelöscht werden.", 500)
|
||||
except StoreError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise StoreError("purge_failed", "Endgültiges Löschen wurde abgebrochen, nichts wurde entfernt.", 500) from exc
|
||||
|
||||
removed_files: list[str] = []
|
||||
missing_files: list[str] = []
|
||||
for path in files_to_delete:
|
||||
if not path.is_file():
|
||||
missing_files.append(str(path))
|
||||
continue
|
||||
try:
|
||||
path.unlink()
|
||||
removed_files.append(str(path))
|
||||
except OSError as exc:
|
||||
raise StoreError(
|
||||
"purge_failed",
|
||||
"Datenbank entfernt, Mediendatei konnte nicht gelöscht werden. Vorgang fail-closed gemeldet.",
|
||||
500,
|
||||
) from exc
|
||||
return {
|
||||
"id": entry_id,
|
||||
"purged": True,
|
||||
"inventory": inventory,
|
||||
"removed_files": len(removed_files),
|
||||
"missing_files": missing_files,
|
||||
"conversations_untouched": True,
|
||||
}
|
||||
@@ -0,0 +1,395 @@
|
||||
"""Consistent local backup and restore for the holiday test phase.
|
||||
|
||||
Uses the SQLite backup API (not a live-file copy). Includes journal media.
|
||||
Never packs .env, provider keys, or logs. The archive can contain personal
|
||||
unencrypted journal data.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
FORMAT_KIND = "kansho.local_backup"
|
||||
FORMAT_VERSION = 1
|
||||
DB_NAME = "kansho.sqlite"
|
||||
MANIFEST_NAME = "manifest.json"
|
||||
README_NAME = "README.txt"
|
||||
MEDIA_PREFIX = "media/"
|
||||
DEFAULT_DIR_NAME = "local-backups"
|
||||
PERSONAL_DATA_WARNING = (
|
||||
"Dieses Archiv kann persönliche, unverschlüsselte Journalinhalte und Medien enthalten. "
|
||||
"Es enthält keine .env-Dateien, Provider-Keys oder Logs, ist aber selbst nicht verschlüsselt."
|
||||
)
|
||||
|
||||
EXCLUDE_NAMES = {".env", ".env.example"}
|
||||
EXCLUDE_SUFFIXES = {".log", ".pem", ".key"}
|
||||
EXCLUDE_DIR_NAMES = {"logs", "__pycache__"}
|
||||
|
||||
|
||||
class BackupError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 400, details: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
self.details = details or {}
|
||||
|
||||
|
||||
def _now_stamp() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def default_backup_dir() -> Path:
|
||||
env = (os.environ.get("KANSHO_BACKUP_DIR") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
return _repo_root() / DEFAULT_DIR_NAME
|
||||
|
||||
|
||||
def default_db_path() -> Path:
|
||||
env = (os.environ.get("KANSHO_DB_PATH") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
if data_dir:
|
||||
return Path(data_dir) / "kansho.sqlite"
|
||||
return Path(__file__).resolve().parent / "data" / "kansho.sqlite"
|
||||
|
||||
|
||||
def default_media_root() -> Path:
|
||||
env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
if data_dir:
|
||||
return Path(data_dir) / "media"
|
||||
return Path(__file__).resolve().parent / "data" / "media"
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _is_excluded(path: Path) -> bool:
|
||||
if path.name in EXCLUDE_NAMES:
|
||||
return True
|
||||
if path.suffix.lower() in EXCLUDE_SUFFIXES:
|
||||
return True
|
||||
return any(part in EXCLUDE_DIR_NAMES for part in path.parts)
|
||||
|
||||
|
||||
def list_media_files(media_root: Path) -> list[Path]:
|
||||
if not media_root.exists():
|
||||
return []
|
||||
files: list[Path] = []
|
||||
for path in sorted(media_root.rglob("*")):
|
||||
if not path.is_file() or _is_excluded(path):
|
||||
continue
|
||||
files.append(path)
|
||||
return files
|
||||
|
||||
|
||||
def sqlite_snapshot(src: Path, dest: Path) -> None:
|
||||
if not src.exists():
|
||||
raise BackupError("db_missing", f"Keine Datenbank unter {src}")
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
source = sqlite3.connect(str(src), timeout=5)
|
||||
try:
|
||||
target = sqlite3.connect(str(dest))
|
||||
try:
|
||||
source.backup(target)
|
||||
finally:
|
||||
target.close()
|
||||
except sqlite3.OperationalError as exc:
|
||||
raise BackupError("db_unavailable", f"SQLite-Snapshot fehlgeschlagen: {exc}") from exc
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
|
||||
def assert_db_idle(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
conn = sqlite3.connect(str(path), timeout=1)
|
||||
try:
|
||||
conn.execute("BEGIN EXCLUSIVE")
|
||||
conn.rollback()
|
||||
except sqlite3.OperationalError as exc:
|
||||
raise BackupError(
|
||||
"db_in_use",
|
||||
"Die Datenbank ist geöffnet. Backend beenden und Restore erneut ausführen.",
|
||||
) from exc
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def build_manifest(created: str, db_file: Path, media_files: list[tuple[str, Path]]) -> dict:
|
||||
checksums = {DB_NAME: sha256_file(db_file)}
|
||||
media_entries = []
|
||||
for rel, path in media_files:
|
||||
checksums[f"{MEDIA_PREFIX}{rel}"] = sha256_file(path)
|
||||
media_entries.append({"path": rel, "bytes": path.stat().st_size})
|
||||
return {
|
||||
"kind": FORMAT_KIND,
|
||||
"format_version": FORMAT_VERSION,
|
||||
"created": created,
|
||||
"warning": PERSONAL_DATA_WARNING,
|
||||
"contains": ["sqlite", "media"],
|
||||
"excludes": [".env", "provider keys", "logs"],
|
||||
"parts": {
|
||||
"sqlite": {"path": DB_NAME, "bytes": db_file.stat().st_size},
|
||||
"media": media_entries,
|
||||
},
|
||||
"checksums": checksums,
|
||||
}
|
||||
|
||||
|
||||
def create_backup(
|
||||
dest: Path | None = None,
|
||||
*,
|
||||
db_path: Path | None = None,
|
||||
media_root: Path | None = None,
|
||||
) -> dict:
|
||||
db_path = Path(db_path or default_db_path())
|
||||
media_root = Path(media_root or default_media_root())
|
||||
if dest is None:
|
||||
dest = default_backup_dir() / f"kansho-{_now_stamp()}.zip"
|
||||
dest = Path(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
created = _now_iso()
|
||||
with tempfile.TemporaryDirectory(prefix="kansho-backup-") as raw:
|
||||
tmp = Path(raw)
|
||||
snapshot = tmp / DB_NAME
|
||||
sqlite_snapshot(db_path, snapshot)
|
||||
media_pairs: list[tuple[str, Path]] = []
|
||||
for path in list_media_files(media_root):
|
||||
rel = path.relative_to(media_root).as_posix()
|
||||
media_pairs.append((rel, path))
|
||||
manifest = build_manifest(created, snapshot, media_pairs)
|
||||
staging = tmp / "archive"
|
||||
staging.mkdir()
|
||||
shutil.copy2(snapshot, staging / DB_NAME)
|
||||
(staging / README_NAME).write_text(PERSONAL_DATA_WARNING + "\n", encoding="utf-8")
|
||||
(staging / MANIFEST_NAME).write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
media_dir = staging / "media"
|
||||
for rel, path in media_pairs:
|
||||
target = media_dir / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, target)
|
||||
if dest.exists():
|
||||
raise BackupError("backup_exists", f"Zieldatei existiert bereits: {dest}")
|
||||
with zipfile.ZipFile(dest, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for path in staging.rglob("*"):
|
||||
if path.is_file():
|
||||
zf.write(path, path.relative_to(staging).as_posix())
|
||||
return {"path": str(dest), "manifest": manifest}
|
||||
|
||||
|
||||
def _read_zip_text(zf: zipfile.ZipFile, name: str) -> str:
|
||||
try:
|
||||
return zf.read(name).decode("utf-8")
|
||||
except KeyError as exc:
|
||||
raise BackupError("archive_invalid", f"Archivteil fehlt: {name}") from exc
|
||||
|
||||
|
||||
def verify_archive(archive: Path) -> dict:
|
||||
archive = Path(archive)
|
||||
if not archive.is_file():
|
||||
raise BackupError("archive_missing", f"Archiv nicht gefunden: {archive}")
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
names = set(zf.namelist())
|
||||
if MANIFEST_NAME not in names:
|
||||
raise BackupError("archive_invalid", "Manifest fehlt.")
|
||||
try:
|
||||
manifest = json.loads(_read_zip_text(zf, MANIFEST_NAME))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise BackupError("archive_invalid", "Manifest ist kein JSON.") from exc
|
||||
if manifest.get("kind") != FORMAT_KIND:
|
||||
raise BackupError("archive_invalid", "Unbekanntes Backup-Format.")
|
||||
if int(manifest.get("format_version") or 0) != FORMAT_VERSION:
|
||||
raise BackupError("archive_unsupported", "Backup-Formatversion wird nicht unterstützt.")
|
||||
checksums = manifest.get("checksums") or {}
|
||||
if not isinstance(checksums, dict) or DB_NAME not in checksums:
|
||||
raise BackupError("archive_invalid", "Checksummen fehlen.")
|
||||
extra = sorted(
|
||||
name
|
||||
for name in names
|
||||
if not name.endswith("/")
|
||||
and name not in {MANIFEST_NAME, README_NAME}
|
||||
and name not in checksums
|
||||
)
|
||||
missing = sorted(key for key in checksums if key not in names)
|
||||
if missing:
|
||||
raise BackupError(
|
||||
"checksum_mismatch",
|
||||
"Im Archiv fehlen deklarierte Dateien.",
|
||||
details={"missing": missing},
|
||||
)
|
||||
with tempfile.TemporaryDirectory(prefix="kansho-verify-") as raw:
|
||||
tmp = Path(raw)
|
||||
for name, expected in checksums.items():
|
||||
target = tmp / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(name) as src, target.open("wb") as dest:
|
||||
shutil.copyfileobj(src, dest)
|
||||
actual = sha256_file(target)
|
||||
if actual != expected:
|
||||
raise BackupError(
|
||||
"checksum_mismatch",
|
||||
f"Checksumme stimmt nicht: {name}",
|
||||
details={"path": name},
|
||||
)
|
||||
return {
|
||||
"manifest": manifest,
|
||||
"skipped_extra": extra,
|
||||
"names": sorted(names),
|
||||
}
|
||||
|
||||
|
||||
def restore_backup(
|
||||
archive: Path,
|
||||
*,
|
||||
db_path: Path | None = None,
|
||||
media_root: Path | None = None,
|
||||
confirm: bool = False,
|
||||
allow_overwrite: bool = False,
|
||||
safety_dir: Path | None = None,
|
||||
) -> dict:
|
||||
if not confirm:
|
||||
raise BackupError(
|
||||
"confirm_required",
|
||||
"Restore überschreibt lokale Daten nur mit ausdrücklicher Bestätigung (--confirm).",
|
||||
)
|
||||
db_path = Path(db_path or default_db_path())
|
||||
media_root = Path(media_root or default_media_root())
|
||||
verified = verify_archive(archive)
|
||||
current_exists = db_path.exists() or (media_root.exists() and any(media_root.rglob("*")))
|
||||
if current_exists and not allow_overwrite:
|
||||
raise BackupError(
|
||||
"would_overwrite",
|
||||
"Ein aktueller Datenbestand existiert. Restore nicht still. "
|
||||
"Mit Bestätigung und --replace nach Sicherheitsbackup fortsetzen.",
|
||||
)
|
||||
assert_db_idle(db_path)
|
||||
safety_path = None
|
||||
if current_exists:
|
||||
safety_dir = Path(safety_dir or default_backup_dir())
|
||||
safety_path = safety_dir / f"pre-restore-{_now_stamp()}.zip"
|
||||
create_backup(safety_path, db_path=db_path, media_root=media_root)
|
||||
with tempfile.TemporaryDirectory(prefix="kansho-restore-") as raw:
|
||||
extracted = Path(raw) / "extracted"
|
||||
extracted.mkdir()
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
checksums = verified["manifest"]["checksums"]
|
||||
for name in checksums:
|
||||
target = extracted / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(name) as src, target.open("wb") as dest:
|
||||
shutil.copyfileobj(src, dest)
|
||||
if sha256_file(target) != checksums[name]:
|
||||
raise BackupError("checksum_mismatch", f"Checksumme nach Entpacken ungültig: {name}")
|
||||
new_db = extracted / DB_NAME
|
||||
new_media = extracted / "media"
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_db = db_path.with_suffix(db_path.suffix + ".restore-tmp")
|
||||
if tmp_db.exists():
|
||||
tmp_db.unlink()
|
||||
shutil.copy2(new_db, tmp_db)
|
||||
os.replace(tmp_db, db_path)
|
||||
incoming_root = Path(raw) / "media-incoming"
|
||||
incoming_root.mkdir()
|
||||
if new_media.exists():
|
||||
for path in new_media.rglob("*"):
|
||||
if path.is_file():
|
||||
rel = path.relative_to(new_media)
|
||||
dest = incoming_root / rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, dest)
|
||||
swap = media_root.with_name(media_root.name + ".restore-old")
|
||||
if swap.exists():
|
||||
shutil.rmtree(swap)
|
||||
if media_root.exists():
|
||||
media_root.rename(swap)
|
||||
incoming_root.rename(media_root)
|
||||
if swap.exists():
|
||||
shutil.rmtree(swap, ignore_errors=True)
|
||||
return {
|
||||
"restored": str(archive),
|
||||
"db_path": str(db_path),
|
||||
"media_root": str(media_root),
|
||||
"safety_backup": str(safety_path) if safety_path else None,
|
||||
"skipped_extra": verified["skipped_extra"],
|
||||
"warning": PERSONAL_DATA_WARNING,
|
||||
}
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Lokales Kanshō-Backup (SQLite-Snapshot + Medien). Persönliche unverschlüsselte Daten."
|
||||
)
|
||||
sub = parser.add_subparsers(dest="action", required=True)
|
||||
create = sub.add_parser("create", help="Konsistentes lokales Backup erzeugen")
|
||||
create.add_argument("--out", help="Zieldatei. Standard: local-backups/kansho-<zeit>.zip")
|
||||
restore = sub.add_parser("restore", help="Backup prüfen und einspielen")
|
||||
restore.add_argument("archive", help="Backup-Archiv")
|
||||
restore.add_argument("--confirm", action="store_true", help="Ausdrückliche Bestätigung, sonst Abbruch")
|
||||
restore.add_argument("--replace", action="store_true", help="Nach Sicherheitsbackup aktuellen Stand ersetzen")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
try:
|
||||
if args.action == "create":
|
||||
result = create_backup(Path(args.out) if args.out else None)
|
||||
print(PERSONAL_DATA_WARNING)
|
||||
print(f"Backup: {result['path']}")
|
||||
print(f"Teile: sqlite + {len(result['manifest']['parts']['media'])} Mediendateien")
|
||||
return 0
|
||||
result = restore_backup(
|
||||
Path(args.archive),
|
||||
confirm=bool(args.confirm),
|
||||
allow_overwrite=bool(args.replace),
|
||||
)
|
||||
print(PERSONAL_DATA_WARNING)
|
||||
print(f"Restore: {result['db_path']}")
|
||||
if result["safety_backup"]:
|
||||
print(f"Sicherheitsbackup: {result['safety_backup']}")
|
||||
if result["skipped_extra"]:
|
||||
print("Zusätzliche Archivdateien ignoriert: " + ", ".join(result["skipped_extra"]))
|
||||
return 0
|
||||
except BackupError as exc:
|
||||
print(f"ERROR {exc.code}: {exc.message}", file=sys.stderr)
|
||||
if exc.details:
|
||||
print(json.dumps(exc.details, ensure_ascii=False), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+15
-5
@@ -5,6 +5,7 @@ Video containers are stored as-is; GPS in video metadata is not stripped yet.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
@@ -27,10 +28,19 @@ VIDEO_TYPES = {
|
||||
ALLOWED_TYPES = {**IMAGE_TYPES, **VIDEO_TYPES}
|
||||
IMAGE_MAX_BYTES = 8 * 1024 * 1024
|
||||
VIDEO_MAX_BYTES = 64 * 1024 * 1024
|
||||
MEDIA_ROOT = DATA_DIR / "media"
|
||||
MAX_BYTES = IMAGE_MAX_BYTES # tests and older imports
|
||||
|
||||
|
||||
def media_root() -> Path:
|
||||
env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
return DATA_DIR / "media"
|
||||
|
||||
|
||||
MEDIA_ROOT = DATA_DIR / "media" # default; call media_root() at use time
|
||||
|
||||
|
||||
def _owned(conn, table: str, record_id: str, profile_id: str) -> dict | None:
|
||||
return row_to_dict(
|
||||
conn.execute(f"SELECT * FROM {table} WHERE id = ? AND profile_id = ?", (record_id, profile_id)).fetchone()
|
||||
@@ -150,7 +160,7 @@ def save_media(profile_id: str, entry_id: str, filename: str, content_type: str,
|
||||
asset_id = str(uuid.uuid4())
|
||||
ext = ALLOWED_TYPES[kind]
|
||||
rel_path = f"{profile_id}/{asset_id}{ext}"
|
||||
dest = MEDIA_ROOT / rel_path
|
||||
dest = media_root() / rel_path
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(payload)
|
||||
prefix = "video" if is_video else "image"
|
||||
@@ -172,8 +182,8 @@ def save_image(profile_id: str, entry_id: str, filename: str, content_type: str,
|
||||
return save_media(profile_id, entry_id, filename, content_type, data)
|
||||
|
||||
|
||||
def list_media(profile_id: str, entry_id: str) -> list[dict]:
|
||||
get_entry(profile_id, entry_id)
|
||||
def list_media(profile_id: str, entry_id: str, *, include_deleted: bool = False) -> list[dict]:
|
||||
get_entry(profile_id, entry_id, include_deleted=include_deleted)
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
@@ -191,7 +201,7 @@ def get_media(profile_id: str, media_id: str) -> tuple[dict, Path]:
|
||||
row = _owned(conn, "media_assets", media_id, profile_id)
|
||||
if not row:
|
||||
raise StoreError("not_found", "Medium nicht gefunden", 404)
|
||||
path = MEDIA_ROOT / row["rel_path"]
|
||||
path = media_root() / row["rel_path"]
|
||||
if not path.is_file():
|
||||
raise StoreError("not_found", "Mediendatei fehlt", 404)
|
||||
return public_media(row), path
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Cached model window metadata for journal budget checks.
|
||||
|
||||
OpenRouter exposes context_length and top_provider.max_completion_tokens via
|
||||
GET /api/v1/models. The catalog is cached so journal calls do not refetch on
|
||||
every generate. Missing metadata fails closed for journal generation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from prompt_budget import (
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
JOURNAL_TARGET_COMPLETION_TOKENS,
|
||||
JournalBudgetError,
|
||||
ModelWindow,
|
||||
_int_env,
|
||||
)
|
||||
|
||||
DEFAULT_TTL_SECONDS = 3600
|
||||
FAKE_CONTEXT = 32_768
|
||||
_lock = threading.Lock()
|
||||
# url -> (fetched_at, {model_id: ModelWindow fields as dict})
|
||||
_catalog: dict[str, tuple[float, dict[str, dict]]] = {}
|
||||
_overrides: dict[str, ModelWindow] = {}
|
||||
|
||||
|
||||
def catalog_ttl() -> int:
|
||||
raw = (os.environ.get("KANSHO_MODEL_CATALOG_TTL_SECONDS") or "").strip()
|
||||
if not raw:
|
||||
return DEFAULT_TTL_SECONDS
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return DEFAULT_TTL_SECONDS
|
||||
return max(60, value)
|
||||
|
||||
|
||||
def reset_catalog() -> None:
|
||||
with _lock:
|
||||
_catalog.clear()
|
||||
_overrides.clear()
|
||||
|
||||
|
||||
def set_metadata_override(model: str, window: ModelWindow) -> None:
|
||||
with _lock:
|
||||
_overrides[model] = window
|
||||
|
||||
|
||||
def is_openrouter_url(url: str) -> bool:
|
||||
host = (urlparse(url or "").hostname or "").lower()
|
||||
return host.endswith("openrouter.ai")
|
||||
|
||||
|
||||
def models_url(chat_url: str) -> str:
|
||||
raw = (chat_url or "").rstrip("/")
|
||||
if raw.endswith("/chat/completions"):
|
||||
return raw[: -len("/chat/completions")] + "/models"
|
||||
if is_openrouter_url(raw):
|
||||
return "https://openrouter.ai/api/v1/models"
|
||||
return raw + "/models" if raw else "https://openrouter.ai/api/v1/models"
|
||||
|
||||
|
||||
def _from_env(model: str, *, provider: str | None, source: str) -> ModelWindow | None:
|
||||
context = _int_env("KANSHO_PROVIDER_CONTEXT_LENGTH", 0)
|
||||
completion = _int_env("KANSHO_PROVIDER_MAX_COMPLETION_TOKENS", 0)
|
||||
if context <= 0:
|
||||
return None
|
||||
return ModelWindow(
|
||||
model=model,
|
||||
context_length=context,
|
||||
max_completion_tokens=completion or JOURNAL_TARGET_COMPLETION_TOKENS,
|
||||
source=source,
|
||||
provider=provider,
|
||||
cached=False,
|
||||
)
|
||||
|
||||
|
||||
def _parse_model_row(row: dict) -> ModelWindow | None:
|
||||
model_id = str(row.get("id") or "").strip()
|
||||
if not model_id:
|
||||
return None
|
||||
top = row.get("top_provider") if isinstance(row.get("top_provider"), dict) else {}
|
||||
context = row.get("context_length") or top.get("context_length")
|
||||
completion = top.get("max_completion_tokens")
|
||||
if completion is None:
|
||||
limits = row.get("per_request_limits") if isinstance(row.get("per_request_limits"), dict) else {}
|
||||
completion = limits.get("completion_tokens") or limits.get("max_tokens")
|
||||
try:
|
||||
context_n = int(context)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
try:
|
||||
completion_n = int(completion) if completion is not None else None
|
||||
except (TypeError, ValueError):
|
||||
completion_n = None
|
||||
if context_n <= 0:
|
||||
return None
|
||||
return ModelWindow(
|
||||
model=model_id,
|
||||
context_length=context_n,
|
||||
max_completion_tokens=completion_n,
|
||||
source="openrouter_models",
|
||||
provider="openrouter",
|
||||
cached=True,
|
||||
)
|
||||
|
||||
|
||||
def _fetch_catalog(url: str, key: str) -> dict[str, dict]:
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if key:
|
||||
headers["Authorization"] = f"Bearer {key}"
|
||||
try:
|
||||
response = httpx.get(url, headers=headers, timeout=15.0)
|
||||
except httpx.HTTPError as exc:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "reason": "unreachable"},
|
||||
) from exc
|
||||
if response.status_code >= 400:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "http_status": response.status_code},
|
||||
)
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "reason": "invalid_json"},
|
||||
) from exc
|
||||
rows = payload.get("data") if isinstance(payload, dict) else payload
|
||||
if not isinstance(rows, list):
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "reason": "unexpected_shape"},
|
||||
)
|
||||
parsed: dict[str, dict] = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
window = _parse_model_row(row)
|
||||
if window:
|
||||
parsed[window.model] = {
|
||||
"model": window.model,
|
||||
"context_length": window.context_length,
|
||||
"max_completion_tokens": window.max_completion_tokens,
|
||||
"source": window.source,
|
||||
"provider": window.provider,
|
||||
}
|
||||
if not parsed:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "reason": "empty_catalog"},
|
||||
)
|
||||
return parsed
|
||||
|
||||
|
||||
def _window_from_cache(fields: dict, *, model: str) -> ModelWindow:
|
||||
return ModelWindow(
|
||||
model=model,
|
||||
context_length=int(fields["context_length"]),
|
||||
max_completion_tokens=fields.get("max_completion_tokens"),
|
||||
source=fields.get("source") or "openrouter_models",
|
||||
provider=fields.get("provider"),
|
||||
cached=True,
|
||||
)
|
||||
|
||||
|
||||
def resolve_generate_metadata(config, *, now: float | None = None) -> ModelWindow:
|
||||
"""Return a window for the generate provider. Fail closed if unknown."""
|
||||
model = getattr(config, "model", None) or ""
|
||||
with _lock:
|
||||
override = _overrides.get(model)
|
||||
if override:
|
||||
return override
|
||||
if getattr(config, "mode", None) == "fake" or model == "fake":
|
||||
return ModelWindow(
|
||||
model=model or "fake",
|
||||
context_length=FAKE_CONTEXT,
|
||||
max_completion_tokens=JOURNAL_TARGET_COMPLETION_TOKENS,
|
||||
source="fake",
|
||||
provider="fake",
|
||||
cached=False,
|
||||
)
|
||||
env_window = _from_env(model, provider=getattr(config, "name", None), source="env")
|
||||
if getattr(config, "local", False):
|
||||
if env_window:
|
||||
return env_window
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={
|
||||
"model": model,
|
||||
"provider": getattr(config, "name", None),
|
||||
"reason": "local_context_unconfigured",
|
||||
},
|
||||
)
|
||||
url = getattr(config, "url", "") or ""
|
||||
if not is_openrouter_url(url):
|
||||
if env_window:
|
||||
return env_window
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={
|
||||
"model": model,
|
||||
"provider": getattr(config, "name", None),
|
||||
"reason": "non_openrouter_unconfigured",
|
||||
},
|
||||
)
|
||||
catalog_key = models_url(url)
|
||||
clock = time.time() if now is None else now
|
||||
ttl = catalog_ttl()
|
||||
with _lock:
|
||||
cached = _catalog.get(catalog_key)
|
||||
if cached and clock - cached[0] < ttl and model in cached[1]:
|
||||
return _window_from_cache(cached[1][model], model=model)
|
||||
stale = cached
|
||||
try:
|
||||
fetched = _fetch_catalog(catalog_key, getattr(config, "key", "") or "")
|
||||
except JournalBudgetError:
|
||||
if stale and model in stale[1]:
|
||||
window = _window_from_cache(stale[1][model], model=model)
|
||||
window.source = "openrouter_models_stale"
|
||||
return window
|
||||
if env_window:
|
||||
env_window.source = "env_fallback"
|
||||
return env_window
|
||||
raise
|
||||
with _lock:
|
||||
_catalog[catalog_key] = (clock, fetched)
|
||||
if model not in fetched:
|
||||
if env_window:
|
||||
env_window.source = "env_fallback"
|
||||
return env_window
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"model": model, "reason": "model_not_in_catalog"},
|
||||
)
|
||||
return _window_from_cache(fetched[model], model=model)
|
||||
@@ -72,6 +72,22 @@ register(
|
||||
resolver=_from_ctx("register_hint"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="opening_hint",
|
||||
description="Journalspezifischer erster Impuls. Leer bei späteren Zügen. Keine Intent-Engine.",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("opening_hint"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="reconstruction",
|
||||
description="Validierte inhaltliche Rekonstruktion der Stufe 1. Kein vollständiger Tagesdialog.",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("reconstruction"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="review_package",
|
||||
|
||||
+229
-45
@@ -4,25 +4,59 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import contextvars
|
||||
import re
|
||||
|
||||
from entity_detect import detect_and_remember
|
||||
from identity_store import KINSHIP, is_maskable_label, list_mappings
|
||||
from journal_reconstruct import claim_texts, fake_reconstruction, is_dialogue_role_line
|
||||
from prompt_budget import (
|
||||
ERROR_PROVIDER_CONTEXT_LENGTH,
|
||||
JOURNAL_PURPOSES,
|
||||
JournalBudgetError,
|
||||
USER_MESSAGES,
|
||||
assert_input_fits,
|
||||
merge_usage,
|
||||
)
|
||||
from pronoun_bind import bind_user_lines
|
||||
from providers import ProviderError, complete_chat, generate_provider
|
||||
from providers import ChatResult, ProviderError, complete_chat, generate_provider
|
||||
|
||||
MAX_EGRESS_CHARS = 24000
|
||||
debug_last: dict[str, Any] | None = None
|
||||
debug_calls = 0
|
||||
last_compact: dict[str, Any] | None = None
|
||||
_test_recorder: contextvars.ContextVar[list | None] = contextvars.ContextVar(
|
||||
"kansho_gateway_recorder",
|
||||
default=None,
|
||||
)
|
||||
COMPACT_DIAGNOSTIC_KEYS = (
|
||||
"model",
|
||||
"actual_model",
|
||||
"provider",
|
||||
"purpose",
|
||||
"prompt_slug",
|
||||
"effective_context_window",
|
||||
"estimated_input_tokens",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"reserved_output_tokens",
|
||||
"max_tokens",
|
||||
"safety_margin",
|
||||
"context_compression",
|
||||
"cost",
|
||||
"budget_ok",
|
||||
"abort_reason",
|
||||
"status",
|
||||
)
|
||||
|
||||
|
||||
class PrivacyGatewayError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 503):
|
||||
def __init__(self, code: str, message: str, status_code: int = 503, diagnostics: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
self.diagnostics = diagnostics or {}
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -40,6 +74,8 @@ class GatewayResult:
|
||||
reason: str
|
||||
provider: str | None = None
|
||||
content: str | None = None
|
||||
diagnostics: dict[str, Any] = field(default_factory=dict)
|
||||
trace: dict[str, Any] | None = None
|
||||
checked_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
|
||||
|
||||
|
||||
@@ -47,10 +83,32 @@ ALLOWED_CLASSES = {"A", "B", "C"}
|
||||
LOCAL_ONLY_CLASS = "A"
|
||||
|
||||
|
||||
def compact_diagnostics(data: dict[str, Any] | None) -> dict[str, Any]:
|
||||
payload = {}
|
||||
for key in COMPACT_DIAGNOSTIC_KEYS:
|
||||
if data and key in data and data[key] is not None:
|
||||
payload[key] = data[key]
|
||||
return payload
|
||||
|
||||
|
||||
def install_test_recorder() -> list[dict[str, Any]]:
|
||||
"""Test-local observer. Production never installs this."""
|
||||
records: list[dict[str, Any]] = []
|
||||
_test_recorder.set(records)
|
||||
return records
|
||||
|
||||
|
||||
def _record_test(event: dict[str, Any]) -> None:
|
||||
records = _test_recorder.get()
|
||||
if records is not None:
|
||||
records.append(event)
|
||||
|
||||
|
||||
def reset_debug() -> None:
|
||||
global debug_last, debug_calls
|
||||
debug_last = None
|
||||
global last_compact, debug_calls
|
||||
last_compact = None
|
||||
debug_calls = 0
|
||||
_test_recorder.set(None)
|
||||
|
||||
|
||||
def inspect(request: GatewayRequest) -> GatewayResult:
|
||||
@@ -66,8 +124,10 @@ def inspect(request: GatewayRequest) -> GatewayResult:
|
||||
return GatewayResult(allowed=True, reason="policy_ok", provider=config.name)
|
||||
|
||||
|
||||
def _minimize(text: str) -> str:
|
||||
def _minimize(text: str, purpose: str = "") -> str:
|
||||
cleaned = (text or "").strip()
|
||||
if purpose in JOURNAL_PURPOSES:
|
||||
return cleaned
|
||||
if len(cleaned) <= MAX_EGRESS_CHARS:
|
||||
return cleaned
|
||||
return cleaned[:MAX_EGRESS_CHARS]
|
||||
@@ -109,6 +169,20 @@ def _mask_person_hit(text: str, start: int, end: int) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _is_identity_mention(text: str, start: int, end: int, token: str) -> bool:
|
||||
"""Same rule as masking: food/thing homonyms are not identity."""
|
||||
if not _is_person_token(token):
|
||||
return True
|
||||
return _mask_person_hit(text, start, end)
|
||||
|
||||
|
||||
def _label_pattern(label: str) -> re.Pattern[str]:
|
||||
return re.compile(
|
||||
rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _mask_body(text: str, mappings: list[dict]) -> str:
|
||||
masked = text
|
||||
for item in sorted(mappings, key=lambda row: len(row.get("local_label") or ""), reverse=True):
|
||||
@@ -117,14 +191,10 @@ def _mask_body(text: str, mappings: list[dict]) -> str:
|
||||
if not label or not token or not is_maskable_label(label):
|
||||
continue
|
||||
placeholder = token if token.startswith("[[") else f"[[{token}]]"
|
||||
pattern = re.compile(
|
||||
rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
person = _is_person_token(token)
|
||||
pattern = _label_pattern(label)
|
||||
|
||||
def repl(match: re.Match, *, _person=person, _ph=placeholder) -> str:
|
||||
if _person and not _mask_person_hit(masked, match.start(), match.end()):
|
||||
def repl(match: re.Match, *, _token=token, _ph=placeholder) -> str:
|
||||
if not _is_identity_mention(masked, match.start(), match.end(), _token):
|
||||
return match.group(0)
|
||||
return _ph
|
||||
|
||||
@@ -139,19 +209,26 @@ def _mask(text: str, mappings: list[dict], *, personal_lines_only: bool = False)
|
||||
for line in (text or "").splitlines(keepends=True):
|
||||
raw = line[:-1] if line.endswith("\n") else line
|
||||
newline = "\n" if line.endswith("\n") else ""
|
||||
if raw.startswith("user:") or raw.startswith("assistant:"):
|
||||
if is_dialogue_role_line(raw):
|
||||
parts.append(_mask_body(raw, mappings) + newline)
|
||||
else:
|
||||
parts.append(line)
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def mask_for_egress(rendered: str, mappings: list[dict], purpose: str) -> str:
|
||||
"""Mask names. Bind pronouns to person tokens only in the dialogue turn.
|
||||
IDENTITY_LEAK_RETRY = (
|
||||
"Korrektur: Keine Klartext-Identität. Nur die vorhandenen Platzhalter [[…]], "
|
||||
"keine Klarnamen, keine Klarorte."
|
||||
)
|
||||
|
||||
|
||||
def mask_for_egress(rendered: str, mappings: list[dict], purpose: str) -> str:
|
||||
"""Mask names in the full rendered prompt. Bind pronouns only on user lines.
|
||||
|
||||
Opening hints, Space-Ausschnitte and titles are personal context, not instructions.
|
||||
Journal generation keeps sie/er/ihr so demask does not turn every reference into the name.
|
||||
"""
|
||||
masked = _mask(rendered, mappings, personal_lines_only=purpose == "dialogue_turn")
|
||||
masked = _mask(rendered, mappings)
|
||||
if purpose == "dialogue_turn":
|
||||
return bind_user_lines(masked)
|
||||
return masked
|
||||
@@ -176,14 +253,13 @@ def _validate_response(content: str, mappings: list[dict] | None = None) -> str:
|
||||
leaked: list[str] = []
|
||||
for item in mappings or []:
|
||||
label = (item.get("local_label") or "").strip()
|
||||
token = (item.get("token") or "").strip()
|
||||
if not label or not is_maskable_label(label):
|
||||
continue
|
||||
if re.search(
|
||||
rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
|
||||
text,
|
||||
re.IGNORECASE,
|
||||
):
|
||||
leaked.append(label)
|
||||
for match in _label_pattern(label).finditer(text):
|
||||
if _is_identity_mention(text, match.start(), match.end(), token):
|
||||
leaked.append(label)
|
||||
break
|
||||
if leaked:
|
||||
raise PrivacyGatewayError(
|
||||
"response_validation_failed",
|
||||
@@ -204,7 +280,21 @@ def _fake_complete(purpose: str, rendered: str) -> str:
|
||||
'"rationale":"Fake-Review: Journal-Evidenz trägt Humor in autobiographical_journal, nicht als globalen Core.",'
|
||||
'"evidence_ids":[],"exemplars":[{"excerpt":"haha das war irgendwie lustig","role":"exemplar","evidence_basis":["kansho_sources"]}]}]}'
|
||||
)
|
||||
if purpose == "journal_reconstruct":
|
||||
return fake_reconstruction(rendered)
|
||||
if purpose == "journal_generate":
|
||||
match = re.search(r"\{.*\}", rendered, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
import json
|
||||
|
||||
data = json.loads(match.group(0))
|
||||
parts = claim_texts(data)
|
||||
body = " ".join(str(part) for part in parts if part)
|
||||
if body:
|
||||
return f"Ein Tag\n\n{body}"
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
lines = []
|
||||
for line in rendered.splitlines():
|
||||
if line.startswith("user:"):
|
||||
@@ -221,21 +311,39 @@ def _fake_complete(purpose: str, rendered: str) -> str:
|
||||
return '{"operation":"erleben_vertiefen","impulse":"' + impulse + '"}'
|
||||
|
||||
|
||||
def complete_model(messages: list[dict], policy: dict[str, Any]) -> str:
|
||||
def complete_model(messages: list[dict], policy: dict[str, Any]) -> ChatResult:
|
||||
config = generate_provider()
|
||||
if not config:
|
||||
raise PrivacyGatewayError("no_egress_provider_configured", "Es ist kein Egress-Provider konfiguriert.")
|
||||
if config.mode == "fake":
|
||||
rendered = "\n".join(item.get("content") or "" for item in messages)
|
||||
return _fake_complete(policy.get("purpose") or "", rendered)
|
||||
return ChatResult(
|
||||
content=_fake_complete(policy.get("purpose") or "", rendered),
|
||||
model=config.model,
|
||||
usage={},
|
||||
context_compression="disabled" if policy.get("disable_context_compression") else "not_applicable",
|
||||
)
|
||||
try:
|
||||
return complete_chat(config, messages, timeout=60)
|
||||
return complete_chat(
|
||||
config,
|
||||
messages,
|
||||
timeout=90 if (policy.get("purpose") in JOURNAL_PURPOSES) else 60,
|
||||
max_tokens=policy.get("max_tokens"),
|
||||
disable_context_compression=bool(policy.get("disable_context_compression")),
|
||||
)
|
||||
except ProviderError as exc:
|
||||
raise PrivacyGatewayError(exc.code, exc.message, exc.status_code) from exc
|
||||
if exc.code == "provider_context_length_rejected":
|
||||
raise PrivacyGatewayError(
|
||||
ERROR_PROVIDER_CONTEXT_LENGTH,
|
||||
USER_MESSAGES[ERROR_PROVIDER_CONTEXT_LENGTH],
|
||||
exc.status_code,
|
||||
diagnostics=exc.diagnostics,
|
||||
) from exc
|
||||
raise PrivacyGatewayError(exc.code, exc.message, exc.status_code, getattr(exc, "diagnostics", None)) from exc
|
||||
|
||||
|
||||
def complete(request: GatewayRequest) -> GatewayResult:
|
||||
global debug_last, debug_calls
|
||||
global last_compact, debug_calls
|
||||
result = inspect(request)
|
||||
if not result.allowed:
|
||||
raise PrivacyGatewayError(
|
||||
@@ -245,7 +353,7 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
if result.reason == "no_egress_provider_configured"
|
||||
else "Persönlicher KI-Aufruf wurde vom Privacy Gateway blockiert.",
|
||||
)
|
||||
rendered = _minimize(str((request.payload or {}).get("rendered") or ""))
|
||||
rendered = _minimize(str((request.payload or {}).get("rendered") or ""), request.purpose)
|
||||
source_text = str((request.payload or {}).get("source_text") or rendered)
|
||||
detect_name = None
|
||||
try:
|
||||
@@ -255,14 +363,39 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
if not mappings and request.profile_id:
|
||||
mappings = list_mappings(request.profile_id)
|
||||
masked = mask_for_egress(rendered, mappings, request.purpose)
|
||||
budget = (request.payload or {}).get("budget")
|
||||
diagnostics = dict((request.payload or {}).get("diagnostics") or {})
|
||||
if request.purpose in JOURNAL_PURPOSES:
|
||||
if budget is None:
|
||||
raise PrivacyGatewayError(
|
||||
"model_metadata_unknown",
|
||||
USER_MESSAGES["model_metadata_unknown"],
|
||||
503,
|
||||
diagnostics={"reason": "journal_budget_missing"},
|
||||
)
|
||||
try:
|
||||
assert_input_fits(budget, masked)
|
||||
except JournalBudgetError as exc:
|
||||
raise PrivacyGatewayError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc
|
||||
diagnostics.update(budget.as_diagnostics())
|
||||
diagnostics["estimated_input_tokens"] = budget.estimated_input_tokens
|
||||
debug_calls += 1
|
||||
config = generate_provider()
|
||||
debug_last = {
|
||||
layer = {
|
||||
"journal_generate": "journalentwurf",
|
||||
"journal_reconstruct": "journalrekonstruktion",
|
||||
"profile_review": "profilreview",
|
||||
}.get(request.purpose, "dialogzug")
|
||||
max_tokens = (request.payload or {}).get("max_tokens")
|
||||
if max_tokens is not None:
|
||||
diagnostics["max_tokens"] = max_tokens
|
||||
diagnostics["provider"] = result.provider
|
||||
diagnostics["purpose"] = request.purpose
|
||||
diagnostics["prompt_slug"] = (request.payload or {}).get("prompt_slug")
|
||||
diagnostics["model"] = config.model if config else None
|
||||
request_trace = {
|
||||
"purpose": request.purpose,
|
||||
"layer": {
|
||||
"journal_generate": "journalentwurf",
|
||||
"profile_review": "profilreview",
|
||||
}.get(request.purpose, "dialogzug"),
|
||||
"layer": layer,
|
||||
"data_class": request.data_class,
|
||||
"prompt_slug": (request.payload or {}).get("prompt_slug"),
|
||||
"rendered": rendered,
|
||||
@@ -273,27 +406,76 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
"detect_provider": detect_name,
|
||||
"detect_note": detect_note,
|
||||
"mapping_count": len(mappings),
|
||||
"budget": compact_diagnostics(diagnostics),
|
||||
}
|
||||
raw = complete_model(
|
||||
[{"role": "user", "content": masked}],
|
||||
{"purpose": request.purpose, "zdr": True, "no_train": True},
|
||||
model_policy = {
|
||||
"purpose": request.purpose,
|
||||
"zdr": True,
|
||||
"no_train": True,
|
||||
"max_tokens": max_tokens,
|
||||
"disable_context_compression": bool(
|
||||
(request.payload or {}).get("disable_context_compression")
|
||||
),
|
||||
}
|
||||
try:
|
||||
chat = complete_model([{"role": "user", "content": masked}], model_policy)
|
||||
raw = chat.content
|
||||
try:
|
||||
validated = _validate_response(raw, mappings)
|
||||
except PrivacyGatewayError as exc:
|
||||
if exc.code != "response_validation_failed":
|
||||
raise
|
||||
diagnostics["response_validation_retry"] = 1
|
||||
chat = complete_model(
|
||||
[{"role": "user", "content": masked + "\n\n" + IDENTITY_LEAK_RETRY}],
|
||||
model_policy,
|
||||
)
|
||||
raw = chat.content
|
||||
validated = _validate_response(raw, mappings)
|
||||
except PrivacyGatewayError as exc:
|
||||
failed = merge_usage(
|
||||
{**diagnostics, "budget_ok": False, "abort_reason": exc.code, "status": "error"},
|
||||
None,
|
||||
config.model if config else None,
|
||||
)
|
||||
last_compact = compact_diagnostics(failed)
|
||||
_record_test({"purpose": request.purpose, "ok": False, "code": exc.code})
|
||||
raise
|
||||
diagnostics = merge_usage(
|
||||
{
|
||||
**diagnostics,
|
||||
"context_compression": chat.context_compression,
|
||||
"budget_ok": True,
|
||||
"status": "ok",
|
||||
},
|
||||
chat.usage,
|
||||
chat.model or (config.model if config else None),
|
||||
)
|
||||
validated = _validate_response(raw, mappings)
|
||||
result.content = _demask(validated, mappings)
|
||||
result.allowed = True
|
||||
debug_last["raw"] = validated
|
||||
debug_last["reply"] = result.content
|
||||
result.diagnostics = compact_diagnostics(diagnostics)
|
||||
request_trace["raw"] = validated
|
||||
request_trace["reply"] = result.content
|
||||
request_trace["model"] = chat.model or request_trace.get("model")
|
||||
request_trace["budget"] = compact_diagnostics(diagnostics)
|
||||
result.trace = public_trace(request_trace)
|
||||
last_compact = compact_diagnostics(diagnostics)
|
||||
_record_test({"purpose": request.purpose, "ok": True, "prompt_slug": request_trace.get("prompt_slug")})
|
||||
return result
|
||||
|
||||
|
||||
def last_trace() -> dict | None:
|
||||
return debug_last
|
||||
"""Compact diagnostics only. Full prompts live on the current GatewayResult."""
|
||||
return last_compact
|
||||
|
||||
|
||||
def public_trace(trace: dict | None) -> dict | None:
|
||||
"""Admin test view. No mapping table, no secrets."""
|
||||
"""Admin test view for the current response. No mapping table, no secrets."""
|
||||
if not trace:
|
||||
return None
|
||||
budget = dict(trace.get("budget") or {})
|
||||
for key in ("rendered", "masked", "intern", "raw", "reply", "prompt", "messages"):
|
||||
budget.pop(key, None)
|
||||
return {
|
||||
"purpose": trace.get("purpose"),
|
||||
"layer": trace.get("layer"),
|
||||
@@ -304,9 +486,11 @@ def public_trace(trace: dict | None) -> dict | None:
|
||||
"detect_provider": trace.get("detect_provider"),
|
||||
"detect_note": trace.get("detect_note"),
|
||||
"mapping_count": trace.get("mapping_count"),
|
||||
"intern": trace.get("rendered"),
|
||||
"egress": trace.get("masked"),
|
||||
"intern": trace.get("rendered") or trace.get("intern"),
|
||||
"egress": trace.get("masked") or trace.get("egress"),
|
||||
"mask_input": trace.get("mask_input"),
|
||||
"raw": trace.get("raw"),
|
||||
"reply": trace.get("reply"),
|
||||
"budget": budget or None,
|
||||
"stages": trace.get("stages"),
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Conservative prompt budget for personal journal generation.
|
||||
|
||||
Token counts are estimates, not tokenizer-accurate. Kanshō overestimates input
|
||||
so a request is refused before it can overflow a context window. There is no
|
||||
silent middle-out deletion of the day.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
# Journal generation requires a wide window so a full day can be reconstructed
|
||||
# without dropping middle events. 32_768 is the product default, not a claim
|
||||
# that every model tokenizer uses this unit identically.
|
||||
JOURNAL_MIN_CONTEXT_TOKENS = 32_768
|
||||
JOURNAL_TARGET_COMPLETION_TOKENS = 4_096
|
||||
# Overestimate: German mixed with prompt English is typically closer to 3–4
|
||||
# characters per token. Two characters per token is deliberately conservative.
|
||||
DEFAULT_CHARS_PER_TOKEN = 2.0
|
||||
DEFAULT_SAFETY_MARGIN = 0.15
|
||||
CHAT_FORMAT_OVERHEAD_TOKENS = 64
|
||||
JOURNAL_PURPOSES = {"journal_generate", "journal_reconstruct"}
|
||||
|
||||
ERROR_MODEL_CONTEXT_TOO_SMALL = "model_context_too_small"
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED = "prompt_budget_exceeded"
|
||||
ERROR_MODEL_METADATA_UNKNOWN = "model_metadata_unknown"
|
||||
ERROR_RECONSTRUCTION_INVALID = "reconstruction_invalid"
|
||||
ERROR_NO_USER_SOURCES = "no_user_sources"
|
||||
ERROR_OUTPUT_LIMIT_UNSUPPORTED = "output_limit_unsupported"
|
||||
ERROR_PROVIDER_CONTEXT_LENGTH = "provider_context_length_rejected"
|
||||
|
||||
USER_MESSAGES = {
|
||||
ERROR_MODEL_CONTEXT_TOO_SMALL: (
|
||||
"Das gewählte Modell hat ein zu kleines Kontextfenster für die "
|
||||
"Journalgenerierung. Es sind mindestens 32.000 Tokens erforderlich. "
|
||||
"Es wurde kein anderes Modell gewählt."
|
||||
),
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED: (
|
||||
"Der Tagesdialog ist für eine sichere Verarbeitung zu umfangreich. "
|
||||
"Kanshō hat nichts stillschweigend aus der Mitte entfernt. "
|
||||
"Bitte wähle weniger Gespräche oder ein Modell mit größerem Kontext."
|
||||
),
|
||||
ERROR_MODEL_METADATA_UNKNOWN: (
|
||||
"Die Kontextgrenzen des Modells konnten nicht sicher bestimmt werden. "
|
||||
"Der Aufruf wurde nicht gesendet."
|
||||
),
|
||||
ERROR_RECONSTRUCTION_INVALID: (
|
||||
"Die inhaltliche Rekonstruktion war unvollständig oder ungültig. "
|
||||
"Es wurde kein Journalentwurf erzeugt."
|
||||
),
|
||||
ERROR_NO_USER_SOURCES: (
|
||||
"In den ausgewählten Gesprächen gibt es noch keinen Nutzertext. "
|
||||
"Es wurde kein Journalentwurf erzeugt."
|
||||
),
|
||||
ERROR_OUTPUT_LIMIT_UNSUPPORTED: (
|
||||
"Das Modell unterstützt die für die Journalgenerierung reservierte "
|
||||
"Ausgabelänge nicht. Der Aufruf wurde nicht gesendet."
|
||||
),
|
||||
ERROR_PROVIDER_CONTEXT_LENGTH: (
|
||||
"Der Anbieter hat die Anfrage wegen der Kontextlänge abgelehnt. "
|
||||
"Kanshō hat den Tagesdialog nicht automatisch gekürzt."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class JournalBudgetError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
code: str,
|
||||
message: str | None = None,
|
||||
status_code: int = 422,
|
||||
diagnostics: dict[str, Any] | None = None,
|
||||
):
|
||||
super().__init__(message or USER_MESSAGES.get(code) or code)
|
||||
self.code = code
|
||||
self.message = message or USER_MESSAGES.get(code) or code
|
||||
self.status_code = status_code
|
||||
self.diagnostics = diagnostics or {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelWindow:
|
||||
model: str
|
||||
context_length: int
|
||||
max_completion_tokens: int | None
|
||||
source: str
|
||||
provider: str | None = None
|
||||
cached: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class JournalBudget:
|
||||
model: str
|
||||
purpose: str
|
||||
effective_context_window: int
|
||||
reserved_output_tokens: int
|
||||
safety_margin: float
|
||||
estimated_input_tokens: int = 0
|
||||
available_input_tokens: int = 0
|
||||
chars_per_token: float = DEFAULT_CHARS_PER_TOKEN
|
||||
budget_ok: bool = True
|
||||
abort_reason: str | None = None
|
||||
context_compression: str = "disabled"
|
||||
extras: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_diagnostics(self) -> dict[str, Any]:
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"purpose": self.purpose,
|
||||
"estimated_input_tokens": self.estimated_input_tokens,
|
||||
"effective_context_window": self.effective_context_window,
|
||||
"reserved_output_tokens": self.reserved_output_tokens,
|
||||
"safety_margin": self.safety_margin,
|
||||
"available_input_tokens": self.available_input_tokens,
|
||||
"chars_per_token": self.chars_per_token,
|
||||
"budget_ok": self.budget_ok,
|
||||
"context_compression": self.context_compression,
|
||||
"abort_reason": self.abort_reason,
|
||||
"estimation": "conservative_char_ratio",
|
||||
}
|
||||
payload.update(self.extras)
|
||||
return payload
|
||||
|
||||
|
||||
def _float_env(name: str, default: float) -> float:
|
||||
raw = (os.environ.get(name) or "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
def _int_env(name: str, default: int) -> int:
|
||||
raw = (os.environ.get(name) or "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
def chars_per_token() -> float:
|
||||
return _float_env("KANSHO_TOKEN_CHARS_PER_TOKEN", DEFAULT_CHARS_PER_TOKEN)
|
||||
|
||||
|
||||
def safety_margin() -> float:
|
||||
return min(0.4, max(0.05, _float_env("KANSHO_JOURNAL_SAFETY_MARGIN", DEFAULT_SAFETY_MARGIN)))
|
||||
|
||||
|
||||
def min_context_tokens() -> int:
|
||||
return _int_env("KANSHO_JOURNAL_MIN_CONTEXT_TOKENS", JOURNAL_MIN_CONTEXT_TOKENS)
|
||||
|
||||
|
||||
def target_completion_tokens() -> int:
|
||||
return _int_env("KANSHO_JOURNAL_MAX_COMPLETION_TOKENS", JOURNAL_TARGET_COMPLETION_TOKENS)
|
||||
|
||||
|
||||
def day_message_safety_cap() -> int:
|
||||
return _int_env("KANSHO_JOURNAL_DAY_MAX_MESSAGES", 500)
|
||||
|
||||
|
||||
def estimate_tokens(text: str, *, extra: int = 0) -> int:
|
||||
"""Conservative character ratio. Not a model tokenizer."""
|
||||
n = len(text or "")
|
||||
if n <= 0:
|
||||
return extra
|
||||
return max(1, math.ceil(n / chars_per_token())) + extra
|
||||
|
||||
|
||||
def usable_context_tokens(window: ModelWindow) -> int:
|
||||
margin = safety_margin()
|
||||
usable = math.floor(window.context_length * (1.0 - margin))
|
||||
return max(0, usable)
|
||||
|
||||
|
||||
def reserved_output_tokens(window: ModelWindow) -> int:
|
||||
wanted = target_completion_tokens()
|
||||
supported = window.max_completion_tokens
|
||||
if supported is None or supported <= 0:
|
||||
raise JournalBudgetError(
|
||||
ERROR_OUTPUT_LIMIT_UNSUPPORTED,
|
||||
diagnostics={"model": window.model, "max_completion_tokens": supported},
|
||||
)
|
||||
reserved = min(wanted, int(supported))
|
||||
if reserved < 256:
|
||||
raise JournalBudgetError(
|
||||
ERROR_OUTPUT_LIMIT_UNSUPPORTED,
|
||||
diagnostics={
|
||||
"model": window.model,
|
||||
"max_completion_tokens": supported,
|
||||
"reserved_output_tokens": reserved,
|
||||
},
|
||||
)
|
||||
return reserved
|
||||
|
||||
|
||||
def plan_journal_budget(window: ModelWindow, *, purpose: str) -> JournalBudget:
|
||||
required = min_context_tokens()
|
||||
if window.context_length < required:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_CONTEXT_TOO_SMALL,
|
||||
diagnostics={
|
||||
"model": window.model,
|
||||
"effective_context_window": window.context_length,
|
||||
"required_context_window": required,
|
||||
"source": window.source,
|
||||
},
|
||||
)
|
||||
reserved = reserved_output_tokens(window)
|
||||
usable = usable_context_tokens(window)
|
||||
available = usable - reserved - CHAT_FORMAT_OVERHEAD_TOKENS
|
||||
if available < 512:
|
||||
raise JournalBudgetError(
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
diagnostics={
|
||||
"model": window.model,
|
||||
"effective_context_window": window.context_length,
|
||||
"reserved_output_tokens": reserved,
|
||||
"available_input_tokens": available,
|
||||
},
|
||||
)
|
||||
compression = "disabled" if purpose in JOURNAL_PURPOSES else "provider_default"
|
||||
return JournalBudget(
|
||||
model=window.model,
|
||||
purpose=purpose,
|
||||
effective_context_window=window.context_length,
|
||||
reserved_output_tokens=reserved,
|
||||
safety_margin=safety_margin(),
|
||||
available_input_tokens=available,
|
||||
chars_per_token=chars_per_token(),
|
||||
context_compression=compression,
|
||||
extras={"metadata_source": window.source, "cached_metadata": window.cached},
|
||||
)
|
||||
|
||||
|
||||
def assert_input_fits(budget: JournalBudget, text: str) -> JournalBudget:
|
||||
estimated = estimate_tokens(text, extra=CHAT_FORMAT_OVERHEAD_TOKENS)
|
||||
budget.estimated_input_tokens = estimated
|
||||
total = estimated + budget.reserved_output_tokens
|
||||
ceiling = math.floor(budget.effective_context_window * (1.0 - budget.safety_margin))
|
||||
if estimated > budget.available_input_tokens or total > ceiling:
|
||||
budget.budget_ok = False
|
||||
budget.abort_reason = ERROR_PROMPT_BUDGET_EXCEEDED
|
||||
raise JournalBudgetError(
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
diagnostics={
|
||||
**budget.as_diagnostics(),
|
||||
"estimated_input_tokens": estimated,
|
||||
"required_tokens": total,
|
||||
"usable_tokens": ceiling,
|
||||
},
|
||||
)
|
||||
budget.budget_ok = True
|
||||
budget.abort_reason = None
|
||||
return budget
|
||||
|
||||
|
||||
def merge_usage(diagnostics: dict[str, Any], usage: dict[str, Any] | None, model: str | None = None) -> dict[str, Any]:
|
||||
payload = dict(diagnostics)
|
||||
data = usage or {}
|
||||
if model:
|
||||
payload["actual_model"] = model
|
||||
payload["prompt_tokens"] = data.get("prompt_tokens")
|
||||
payload["completion_tokens"] = data.get("completion_tokens")
|
||||
payload["total_tokens"] = data.get("total_tokens")
|
||||
cost = data.get("cost")
|
||||
if cost is None:
|
||||
cost = data.get("total_cost")
|
||||
payload["cost"] = cost
|
||||
return payload
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Intent-neutral provenance verification. Model output is untrusted input.
|
||||
|
||||
Canonical source texts come only from a locally built SourceRegistry.
|
||||
Downstream stages consume a locally materialized VerifiedArtifact.
|
||||
This module has no journal, dialogue, or product-intent vocabulary.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping
|
||||
|
||||
COVERAGE_ALL_SELECTED_SOURCES = "all_selected_sources"
|
||||
COVERAGE_SELECTED_EVIDENCE = "selected_evidence"
|
||||
COVERAGE_MODES = frozenset({COVERAGE_ALL_SELECTED_SOURCES, COVERAGE_SELECTED_EVIDENCE})
|
||||
ARTIFACT_KIND = "verified_artifact"
|
||||
|
||||
|
||||
class ProvenanceError(Exception):
|
||||
"""Fail-closed verification error. Domain mapping happens at the caller boundary."""
|
||||
|
||||
def __init__(self, reason: str, **details: Any):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
self.details = details
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceRecord:
|
||||
source_id: str
|
||||
role: str
|
||||
text: str
|
||||
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class SourceRegistry:
|
||||
"""Local authority for original content. Model text never replaces these records."""
|
||||
|
||||
def __init__(self, sources: list[SourceRecord] | None = None):
|
||||
self._sources: dict[str, SourceRecord] = {}
|
||||
self._order: list[str] = []
|
||||
for source in sources or []:
|
||||
self.add(source)
|
||||
|
||||
def add(self, source: SourceRecord) -> None:
|
||||
source_id = (source.source_id or "").strip()
|
||||
if not source_id:
|
||||
raise ProvenanceError("missing_source_id")
|
||||
if source_id in self._sources:
|
||||
raise ProvenanceError("duplicate_source_id", source_id=source_id)
|
||||
self._sources[source_id] = SourceRecord(
|
||||
source_id=source_id,
|
||||
role=source.role,
|
||||
text=source.text,
|
||||
metadata=dict(source.metadata or {}),
|
||||
)
|
||||
self._order.append(source_id)
|
||||
|
||||
def get(self, source_id: str) -> SourceRecord | None:
|
||||
return self._sources.get(source_id)
|
||||
|
||||
def order(self) -> list[str]:
|
||||
return list(self._order)
|
||||
|
||||
def items(self) -> list[SourceRecord]:
|
||||
return [self._sources[source_id] for source_id in self._order]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvidenceRef:
|
||||
source_id: str
|
||||
excerpt: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerifiedEvidence:
|
||||
source_id: str
|
||||
excerpt: str
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerificationPolicy:
|
||||
allowed_roles: frozenset[str]
|
||||
coverage: str = COVERAGE_ALL_SELECTED_SOURCES
|
||||
selected_ids: tuple[str, ...] = ()
|
||||
source_order: tuple[str, ...] = ()
|
||||
include_unverified_annotations: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerifiedArtifact:
|
||||
coverage: str
|
||||
source_order: tuple[str, ...]
|
||||
sources: tuple[dict[str, str], ...]
|
||||
evidence: tuple[VerifiedEvidence, ...]
|
||||
annotations_unverified: tuple[dict[str, Any], ...] = ()
|
||||
|
||||
def texts(self) -> list[str]:
|
||||
return [item.get("text") or "" for item in self.sources if item.get("text")]
|
||||
|
||||
def to_payload(self) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"kind": ARTIFACT_KIND,
|
||||
"coverage": self.coverage,
|
||||
"source_order": list(self.source_order),
|
||||
"order": [item.get("source_id") for item in self.sources],
|
||||
"sources": [dict(item) for item in self.sources],
|
||||
"evidence": [
|
||||
{
|
||||
"source_id": item.source_id,
|
||||
"excerpt": item.excerpt,
|
||||
"start": item.start,
|
||||
"end": item.end,
|
||||
}
|
||||
for item in self.evidence
|
||||
],
|
||||
}
|
||||
if self.annotations_unverified:
|
||||
payload["annotations_unverified"] = [dict(item) for item in self.annotations_unverified]
|
||||
return payload
|
||||
|
||||
|
||||
def locate_excerpt(text: str, excerpt: str) -> tuple[int, int] | None:
|
||||
"""Local span from a confirmed literal excerpt. Model offsets are ignored."""
|
||||
if not excerpt:
|
||||
return None
|
||||
start = (text or "").find(excerpt)
|
||||
if start < 0:
|
||||
return None
|
||||
return start, start + len(excerpt)
|
||||
|
||||
|
||||
def texts_from_payload(data: Mapping[str, Any] | None) -> list[str]:
|
||||
if not isinstance(data, Mapping):
|
||||
return []
|
||||
parts: list[str] = []
|
||||
for item in data.get("sources") or []:
|
||||
if isinstance(item, Mapping) and item.get("text"):
|
||||
parts.append(str(item["text"]))
|
||||
return parts
|
||||
|
||||
|
||||
def verify_evidence(
|
||||
registry: SourceRegistry,
|
||||
ref: EvidenceRef,
|
||||
policy: VerificationPolicy,
|
||||
) -> VerifiedEvidence:
|
||||
source_id = (ref.source_id or "").strip()
|
||||
excerpt = ref.excerpt or ""
|
||||
if not source_id:
|
||||
raise ProvenanceError("missing_source_id")
|
||||
if not excerpt.strip():
|
||||
raise ProvenanceError("unverified_excerpt", source_id=source_id)
|
||||
if policy.selected_ids and source_id not in policy.selected_ids:
|
||||
raise ProvenanceError("unknown_source", source_id=source_id)
|
||||
source = registry.get(source_id)
|
||||
if source is None:
|
||||
raise ProvenanceError("unknown_source", source_id=source_id)
|
||||
if source.role not in policy.allowed_roles:
|
||||
raise ProvenanceError("role_not_allowed", source_id=source_id, role=source.role)
|
||||
located = locate_excerpt(source.text, excerpt)
|
||||
if located is not None:
|
||||
return VerifiedEvidence(source_id, excerpt, located[0], located[1])
|
||||
for other in registry.items():
|
||||
if other.source_id == source_id or excerpt not in (other.text or ""):
|
||||
continue
|
||||
if other.role not in policy.allowed_roles:
|
||||
raise ProvenanceError("role_not_allowed", source_id=other.source_id, role=other.role)
|
||||
raise ProvenanceError("wrong_source", source_id=source_id, excerpt=excerpt[:120])
|
||||
raise ProvenanceError("unverified_excerpt", source_id=source_id, excerpt=excerpt[:120])
|
||||
|
||||
|
||||
def _selected_records(registry: SourceRegistry, policy: VerificationPolicy) -> list[SourceRecord]:
|
||||
if policy.coverage not in COVERAGE_MODES:
|
||||
raise ProvenanceError("unknown_coverage", coverage=policy.coverage)
|
||||
selected = tuple(item for item in policy.selected_ids if item)
|
||||
if not selected:
|
||||
raise ProvenanceError("selection_empty")
|
||||
if len(set(selected)) != len(selected):
|
||||
raise ProvenanceError("duplicate_source_id", ids=list(selected))
|
||||
records: list[SourceRecord] = []
|
||||
for source_id in selected:
|
||||
source = registry.get(source_id)
|
||||
if source is None:
|
||||
raise ProvenanceError("unknown_source", source_id=source_id)
|
||||
if source.role not in policy.allowed_roles:
|
||||
raise ProvenanceError("role_not_allowed", source_id=source_id, role=source.role)
|
||||
records.append(source)
|
||||
return records
|
||||
|
||||
|
||||
def verify(
|
||||
registry: SourceRegistry,
|
||||
*,
|
||||
policy: VerificationPolicy,
|
||||
evidence: list[EvidenceRef] | None = None,
|
||||
annotations: list[dict[str, Any]] | None = None,
|
||||
proposed_texts: Mapping[str, str] | None = None,
|
||||
) -> VerifiedArtifact:
|
||||
"""Materialize a VerifiedArtifact from local sources plus untrusted references."""
|
||||
selected = _selected_records(registry, policy)
|
||||
for source in selected:
|
||||
proposed = (proposed_texts or {}).get(source.source_id)
|
||||
if proposed is not None and proposed != source.text:
|
||||
raise ProvenanceError("canonical_overwrite", source_id=source.source_id)
|
||||
|
||||
verified: list[VerifiedEvidence] = []
|
||||
for ref in evidence or []:
|
||||
verified.append(verify_evidence(registry, ref, policy))
|
||||
|
||||
if policy.coverage == COVERAGE_ALL_SELECTED_SOURCES:
|
||||
sources = tuple(
|
||||
{"source_id": source.source_id, "role": source.role, "text": source.text} for source in selected
|
||||
)
|
||||
else:
|
||||
grouped: dict[str, list[str]] = {source.source_id: [] for source in selected}
|
||||
for item in verified:
|
||||
grouped[item.source_id].append(item.excerpt)
|
||||
sources = tuple(
|
||||
{
|
||||
"source_id": source.source_id,
|
||||
"role": source.role,
|
||||
"text": "\n".join(grouped[source.source_id]),
|
||||
}
|
||||
for source in selected
|
||||
)
|
||||
|
||||
unverified: tuple[dict[str, Any], ...] = ()
|
||||
if policy.include_unverified_annotations and annotations:
|
||||
unverified = tuple(dict(item) for item in annotations)
|
||||
|
||||
encounter = tuple(item for item in policy.source_order if item) or tuple(
|
||||
source.source_id for source in selected
|
||||
)
|
||||
return VerifiedArtifact(
|
||||
coverage=policy.coverage,
|
||||
source_order=encounter,
|
||||
sources=sources,
|
||||
evidence=tuple(verified),
|
||||
annotations_unverified=unverified,
|
||||
)
|
||||
+56
-4
@@ -2,7 +2,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
@@ -11,13 +12,20 @@ from env_loader import load_env_file
|
||||
|
||||
load_env_file()
|
||||
|
||||
CONTEXT_LENGTH_REJECT = re.compile(
|
||||
r"context.?length|maximum context|prompt is too long|too many tokens|"
|
||||
r"context window|max context|exceeds? (?:the )?(?:maximum|context)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
class ProviderError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 503):
|
||||
def __init__(self, code: str, message: str, status_code: int = 503, diagnostics: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
self.diagnostics = diagnostics or {}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -33,6 +41,19 @@ class ProviderConfig:
|
||||
no_train: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatResult:
|
||||
content: str
|
||||
model: str | None = None
|
||||
usage: dict = field(default_factory=dict)
|
||||
context_compression: str = "not_applicable"
|
||||
|
||||
|
||||
def is_openrouter(config: ProviderConfig) -> bool:
|
||||
host = (urlparse(config.url or "").hostname or "").lower()
|
||||
return host.endswith("openrouter.ai") or (config.name or "").lower() == "openrouter"
|
||||
|
||||
|
||||
def _truthy(name: str) -> bool:
|
||||
return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
@@ -146,12 +167,23 @@ def detect_provider() -> ProviderConfig | None:
|
||||
)
|
||||
|
||||
|
||||
def complete_chat(config: ProviderConfig, messages: list[dict], *, timeout: float, max_tokens: int | None = None) -> str:
|
||||
def complete_chat(
|
||||
config: ProviderConfig,
|
||||
messages: list[dict],
|
||||
*,
|
||||
timeout: float,
|
||||
max_tokens: int | None = None,
|
||||
disable_context_compression: bool = False,
|
||||
) -> ChatResult:
|
||||
payload: dict = {"model": config.model, "messages": messages}
|
||||
if max_tokens is not None:
|
||||
payload["max_tokens"] = max_tokens
|
||||
compression = "not_applicable"
|
||||
if not config.local:
|
||||
payload["provider"] = {"data_collection": "deny"}
|
||||
if disable_context_compression and is_openrouter(config) and not config.local:
|
||||
payload["plugins"] = [{"id": "context-compression", "enabled": False}]
|
||||
compression = "disabled"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if config.key:
|
||||
headers["Authorization"] = f"Bearer {config.key}"
|
||||
@@ -163,9 +195,29 @@ def complete_chat(config: ProviderConfig, messages: list[dict], *, timeout: floa
|
||||
except httpx.HTTPError as exc:
|
||||
raise ProviderError("provider_unreachable", f"{config.role}-Provider ist nicht erreichbar.") from exc
|
||||
if response.status_code >= 400:
|
||||
body = ""
|
||||
try:
|
||||
body = response.text or ""
|
||||
except Exception:
|
||||
body = ""
|
||||
if CONTEXT_LENGTH_REJECT.search(body):
|
||||
raise ProviderError(
|
||||
"provider_context_length_rejected",
|
||||
"Der Anbieter hat die Anfrage wegen der Kontextlänge abgelehnt. "
|
||||
"Kanshō hat den Tagesdialog nicht automatisch gekürzt.",
|
||||
502,
|
||||
diagnostics={"http_status": response.status_code, "context_compression": compression},
|
||||
)
|
||||
raise ProviderError("provider_rejected", f"{config.role}-Provider hat die Anfrage abgelehnt.")
|
||||
data = response.json()
|
||||
try:
|
||||
return data["choices"][0]["message"]["content"]
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ProviderError("provider_shape", f"Unerwartete {config.role}-Antwort.") from exc
|
||||
usage = data.get("usage") if isinstance(data.get("usage"), dict) else {}
|
||||
return ChatResult(
|
||||
content=content or "",
|
||||
model=data.get("model") or config.model,
|
||||
usage=usage,
|
||||
context_compression=compression,
|
||||
)
|
||||
+82
-5
@@ -5,6 +5,12 @@ from typing import Any
|
||||
|
||||
from db import get_db, row_to_dict
|
||||
from journal_body import plain_text
|
||||
from prompt_budget import (
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
JournalBudgetError,
|
||||
day_message_safety_cap,
|
||||
estimate_tokens,
|
||||
)
|
||||
|
||||
# Implementation caps. Not part of the selection-spec contract.
|
||||
_DAY_MESSAGE_CAP = 80
|
||||
@@ -28,11 +34,58 @@ def retrieve(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||
return []
|
||||
|
||||
|
||||
def format_day_messages(messages: list[dict], *, with_source_ids: bool = False) -> str:
|
||||
parts = []
|
||||
for message in messages:
|
||||
role = message.get("role") or "user"
|
||||
body = message.get("body") or ""
|
||||
source_id = message.get("source_id")
|
||||
if with_source_ids and source_id:
|
||||
parts.append(f"[{source_id}] {role}: {body}")
|
||||
else:
|
||||
parts.append(f"{role}: {body}")
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
def pair_user_priority_messages(messages: list[dict]) -> list[dict]:
|
||||
"""Keep every user message. Assistant context stays inside the same conversation."""
|
||||
selected: list[dict] = []
|
||||
pending_assistant = None
|
||||
pending_conversation = None
|
||||
for message in messages:
|
||||
conversation_id = message.get("conversation_id")
|
||||
role = message.get("role") or "user"
|
||||
if pending_assistant is not None and conversation_id != pending_conversation:
|
||||
pending_assistant = None
|
||||
pending_conversation = None
|
||||
if role == "assistant":
|
||||
pending_assistant = message
|
||||
pending_conversation = conversation_id
|
||||
continue
|
||||
if role == "user":
|
||||
if pending_assistant is not None and pending_conversation == conversation_id:
|
||||
selected.append(pending_assistant)
|
||||
selected.append(message)
|
||||
pending_assistant = None
|
||||
pending_conversation = None
|
||||
continue
|
||||
pending_assistant = None
|
||||
pending_conversation = None
|
||||
return selected
|
||||
|
||||
|
||||
def _day_messages(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||
journal_day_id = spec.get("journal_day_id")
|
||||
conversation_ids = spec.get("conversation_ids") or []
|
||||
if not journal_day_id:
|
||||
return []
|
||||
overflow = spec.get("overflow") or "limit"
|
||||
if overflow == "abort":
|
||||
cap = int(spec.get("message_cap") or day_message_safety_cap())
|
||||
fetch_limit = cap + 1
|
||||
else:
|
||||
cap = int(spec.get("message_cap") or _DAY_MESSAGE_CAP)
|
||||
fetch_limit = cap
|
||||
with get_db() as conn:
|
||||
if conversation_ids:
|
||||
placeholders = ",".join("?" * len(conversation_ids))
|
||||
@@ -42,10 +95,10 @@ def _day_messages(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||
FROM messages m
|
||||
JOIN conversations c ON c.id = m.conversation_id
|
||||
WHERE m.profile_id = ? AND c.journal_day_id = ? AND m.conversation_id IN ({placeholders})
|
||||
ORDER BY c.created, m.seq
|
||||
ORDER BY c.created, c.rowid, m.seq, m.id
|
||||
LIMIT ?
|
||||
""",
|
||||
(profile_id, journal_day_id, *conversation_ids, _DAY_MESSAGE_CAP),
|
||||
(profile_id, journal_day_id, *conversation_ids, fetch_limit),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
@@ -54,12 +107,36 @@ def _day_messages(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||
FROM messages m
|
||||
JOIN conversations c ON c.id = m.conversation_id
|
||||
WHERE m.profile_id = ? AND c.journal_day_id = ?
|
||||
ORDER BY c.created, m.seq
|
||||
ORDER BY c.created, c.rowid, m.seq, m.id
|
||||
LIMIT ?
|
||||
""",
|
||||
(profile_id, journal_day_id, _DAY_MESSAGE_CAP),
|
||||
(profile_id, journal_day_id, fetch_limit),
|
||||
).fetchall()
|
||||
return [row_to_dict(row) for row in rows]
|
||||
messages = [row_to_dict(row) for row in rows]
|
||||
if overflow == "abort":
|
||||
if len(messages) > cap:
|
||||
raise JournalBudgetError(
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
diagnostics={"reason": "day_message_cap", "message_cap": cap, "fetched": len(messages)},
|
||||
)
|
||||
from journal_reconstruct import assign_source_ids
|
||||
|
||||
paired = assign_source_ids(pair_user_priority_messages(messages))
|
||||
max_tokens = spec.get("max_estimated_tokens")
|
||||
if max_tokens is not None:
|
||||
used = estimate_tokens(format_day_messages(paired, with_source_ids=True))
|
||||
if used > int(max_tokens):
|
||||
raise JournalBudgetError(
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
diagnostics={
|
||||
"reason": "day_dialogue_tokens",
|
||||
"estimated_tokens": used,
|
||||
"available_tokens": int(max_tokens),
|
||||
"message_count": len(paired),
|
||||
},
|
||||
)
|
||||
return paired
|
||||
return messages
|
||||
|
||||
|
||||
def _space_entries(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||
|
||||
@@ -31,7 +31,11 @@ router = APIRouter(prefix="/api/dialogue", tags=["dialogue"])
|
||||
|
||||
|
||||
def _http(exc: StoreError | EngineError):
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
detail = {"code": exc.code, "message": exc.message}
|
||||
extra = getattr(exc, "diagnostics", None)
|
||||
if extra:
|
||||
detail["diagnostics"] = extra
|
||||
raise HTTPException(status_code=exc.status_code, detail=detail) from exc
|
||||
|
||||
|
||||
class SessionWrite(BaseModel):
|
||||
|
||||
@@ -10,6 +10,7 @@ from dialogue_turn import run_turn, visible_for_role
|
||||
from privacy_gateway import GatewayRequest, inspect
|
||||
from engine import EngineError
|
||||
from journal_generate import generate_draft
|
||||
from journal_opening import maybe_open_journal_conversation
|
||||
from journal_policy import PolicyError
|
||||
from journal_store import (
|
||||
create_day_conversation,
|
||||
@@ -20,13 +21,18 @@ from journal_store import (
|
||||
get_or_create_day,
|
||||
get_user_space,
|
||||
list_days,
|
||||
list_deleted_entries,
|
||||
list_space_entries,
|
||||
list_user_spaces,
|
||||
list_versions,
|
||||
purge_entry,
|
||||
rename_user_space,
|
||||
restore_entry,
|
||||
save_day_scratch,
|
||||
save_entry,
|
||||
soft_delete_entry,
|
||||
start_payload,
|
||||
undelete_entry,
|
||||
)
|
||||
from media_store import delete_media, get_media, list_media, save_media
|
||||
from writing_profile_store import (
|
||||
@@ -69,7 +75,11 @@ router = APIRouter(prefix="/api/journal", tags=["journal"])
|
||||
|
||||
|
||||
def _http(exc: StoreError | PolicyError | EngineError):
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
detail = {"code": exc.code, "message": exc.message}
|
||||
extra = getattr(exc, "diagnostics", None)
|
||||
if extra:
|
||||
detail["diagnostics"] = extra
|
||||
raise HTTPException(status_code=exc.status_code, detail=detail) from exc
|
||||
|
||||
|
||||
class SpaceWrite(BaseModel):
|
||||
@@ -108,6 +118,10 @@ class RestoreWrite(BaseModel):
|
||||
version_id: str
|
||||
|
||||
|
||||
class PurgeWrite(BaseModel):
|
||||
confirm: bool = False
|
||||
|
||||
|
||||
class ScratchItem(BaseModel):
|
||||
id: str | None = None
|
||||
text: str = ""
|
||||
@@ -163,6 +177,11 @@ class ReviewAcceptWrite(BaseModel):
|
||||
baseline: bool = False
|
||||
|
||||
|
||||
@router.get("/continuity")
|
||||
def continuity(session: dict = Depends(require_auth)):
|
||||
return start_payload(session["profile_id"])
|
||||
|
||||
|
||||
@router.get("/spaces")
|
||||
def spaces(session: dict = Depends(require_auth)):
|
||||
return list_user_spaces(session["profile_id"])
|
||||
@@ -180,7 +199,27 @@ def create_space(body: SpaceWrite, session: dict = Depends(require_auth)):
|
||||
def space_detail(space_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
space = get_user_space(session["profile_id"], space_id)
|
||||
return {"space": space, "days": list_days(session["profile_id"], space_id)}
|
||||
return {
|
||||
"space": space,
|
||||
"days": list_days(session["profile_id"], space_id),
|
||||
"entries": list_space_entries(session["profile_id"], space_id),
|
||||
}
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/spaces/{space_id}/entries")
|
||||
def space_entries(space_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return list_space_entries(session["profile_id"], space_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/spaces/{space_id}/trash")
|
||||
def space_trash(space_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return list_deleted_entries(session["profile_id"], space_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
@@ -227,7 +266,9 @@ def write_scratch(journal_day_id: str, body: ScratchWrite, session: dict = Depen
|
||||
@router.post("/days/{journal_day_id}/conversations")
|
||||
def start_conversation(journal_day_id: str, body: ConversationWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return create_day_conversation(session["profile_id"], journal_day_id, body.title)
|
||||
conversation = create_day_conversation(session["profile_id"], journal_day_id, body.title)
|
||||
opened = maybe_open_journal_conversation(session["profile_id"], conversation)
|
||||
return visible_for_role(opened, session.get("role"))
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
@@ -356,6 +397,22 @@ def entry_restore(entry_id: str, body: RestoreWrite, session: dict = Depends(req
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/entries/{entry_id}/undelete")
|
||||
def entry_undelete(entry_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return undelete_entry(session["profile_id"], entry_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/entries/{entry_id}/purge")
|
||||
def entry_purge(entry_id: str, body: PurgeWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return purge_entry(session["profile_id"], entry_id, confirm=body.confirm)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.delete("/entries/{entry_id}")
|
||||
def entry_delete(entry_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
|
||||
@@ -186,9 +186,7 @@ def main() -> None:
|
||||
json={"body": "Heute früh aufgestanden."},
|
||||
)
|
||||
expect(later_turn.status_code == 200, "later day turn")
|
||||
from privacy_gateway import debug_last
|
||||
|
||||
rendered = debug_last["rendered"]
|
||||
rendered = (later_turn.json().get("trace") or {}).get("intern") or ""
|
||||
expect("Hafen" in rendered, "earlier space source can supply a known plan")
|
||||
expect("Heute früh aufgestanden" in rendered, "current conversation remains primary")
|
||||
|
||||
@@ -204,9 +202,7 @@ def main() -> None:
|
||||
json={"body": "Nur ein kurzer Einkauf."},
|
||||
)
|
||||
expect(sibling_turn.status_code == 200, "sibling turn")
|
||||
from privacy_gateway import debug_last as last_sibling
|
||||
|
||||
sibling_rendered = last_sibling["rendered"]
|
||||
sibling_rendered = (sibling_turn.json().get("trace") or {}).get("intern") or ""
|
||||
expect("Heute früh aufgestanden" not in sibling_rendered, "same-day sibling is not fully loaded")
|
||||
expect("Nur ein kurzer Einkauf" in sibling_rendered, "current sibling conversation is primary")
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ def auth_header(token: str) -> dict:
|
||||
def main() -> None:
|
||||
seed_prompts = json.loads((ROOT / "config" / "prompts.seed.json").read_text(encoding="utf-8"))
|
||||
slugs = {item["slug"] for item in seed_prompts}
|
||||
expect(slugs == {"mvp.dialogue_turn", "mvp.journal_generate", "mvp.entity_detect", "mvp.profile_review"}, "mvp prompts seeded")
|
||||
expect(slugs == {"mvp.dialogue_turn", "mvp.journal_generate", "mvp.journal_reconstruct", "mvp.entity_detect", "mvp.profile_review"}, "mvp prompts seeded")
|
||||
|
||||
with TestClient(app) as client:
|
||||
health = client.get("/api/health")
|
||||
@@ -78,7 +78,7 @@ def main() -> None:
|
||||
|
||||
admin = client.get("/api/admin/health", headers=headers)
|
||||
expect(admin.status_code == 200, "admin health")
|
||||
expect(admin.json()["inventory"]["prompts"] == 3, "three mvp prompts")
|
||||
expect(admin.json()["inventory"]["prompts"] == 5, "five seeded prompts")
|
||||
|
||||
providers = client.get("/api/admin/providers", headers=headers)
|
||||
expect(providers.status_code == 200, "admin providers")
|
||||
@@ -216,6 +216,8 @@ def main() -> None:
|
||||
"existing_text",
|
||||
"known_labels",
|
||||
"register_hint",
|
||||
"opening_hint",
|
||||
"reconstruction",
|
||||
"review_package",
|
||||
"source_text",
|
||||
"space_title",
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,257 @@
|
||||
"""First journal impulse invariants. Run from backend/: python tests/test_journal_opening.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-opening-test.sqlite")
|
||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-opening-media")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from main import app
|
||||
from journal_opening import (
|
||||
NEUTRAL_OPENING,
|
||||
attested_plans_from_user_texts,
|
||||
collect_opening_context,
|
||||
is_attested_plan,
|
||||
)
|
||||
from journal_reconstruct import assign_source_ids
|
||||
from retrieval import pair_user_priority_messages
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def open_space_day(client, headers, title, date):
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": title})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": date},
|
||||
)
|
||||
return space.json(), day.json()
|
||||
|
||||
|
||||
def start_conv(client, headers, day_id):
|
||||
return client.post(f"/api/journal/days/{day_id}/conversations", headers=headers, json={"title": "Gespräch"})
|
||||
|
||||
|
||||
def main() -> None:
|
||||
expect(is_attested_plan("Morgen wollen wir nach Cres fahren."), "future intent is a plan")
|
||||
expect(is_attested_plan("Wir haben vor, später an den Hafen zu gehen."), "vorhaben is a plan")
|
||||
expect(not is_attested_plan("Heute war der Hafen ruhig. Kirschen am Stand."), "past report is not a plan")
|
||||
expect(not is_attested_plan("Wir standen vor dem Hafen und aßen Kirschen."), "vor without haben is not a plan")
|
||||
expect(not is_attested_plan("Hafen Kirschen Wind"), "word overlap is not a plan")
|
||||
plans = attested_plans_from_user_texts(
|
||||
["Heute war der Hafen ruhig.", "Morgen wollen wir nach Cres fahren.", "Hafen Kirschen"]
|
||||
)
|
||||
expect(len(plans) == 1 and "Cres" in plans[0], "only attested plan sentences are kept")
|
||||
expect(
|
||||
not any(is_attested_plan(text) for text in ["Was davon möchtest du festhalten?", "Die Fahrt nach Cres war schön."]),
|
||||
"assistant-like completion is not treated as a user plan by the helper",
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
setup = client.post("/api/auth/setup", json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"})
|
||||
headers = header(setup.json()["token"])
|
||||
other = client.post(
|
||||
"/api/users",
|
||||
headers=headers,
|
||||
json={"email": "ute@example.test", "name": "Ute", "password": "user-pass", "role": "user"},
|
||||
)
|
||||
expect(other.status_code == 200, "second profile")
|
||||
other_headers = header(
|
||||
client.post("/api/auth/login", json={"email": "ute@example.test", "password": "user-pass"}).json()["token"]
|
||||
)
|
||||
|
||||
empty_space, empty_day = open_space_day(client, headers, "Leer", "2026-08-21")
|
||||
empty = start_conv(client, headers, empty_day["day"]["id"])
|
||||
expect(empty.status_code == 200, f"empty opening {empty.text}")
|
||||
expect(empty.json().get("opening", {}).get("kind") == "local_neutral", "no context uses local neutral")
|
||||
expect(empty.json()["opening"]["assistant"]["body"] == NEUTRAL_OPENING, "neutral impulse stored")
|
||||
expect(empty.json()["opening"]["calls"] == 0, "neutral opening does not call a model")
|
||||
empty_msgs = empty.json().get("messages") or []
|
||||
expect(len(empty_msgs) == 1 and empty_msgs[0]["role"] == "assistant", "neutral opening is assistant-first")
|
||||
|
||||
plan_space, plan_day = open_space_day(client, headers, "Urlaub", "2026-08-01")
|
||||
plan_conv = start_conv(client, headers, plan_day["day"]["id"])
|
||||
client.post(
|
||||
f"/api/journal/conversations/{plan_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Morgen wollen wir nach Cres fahren. Die Fähre ist schon bedacht."},
|
||||
)
|
||||
later = client.post(
|
||||
f"/api/journal/spaces/{plan_space['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-02"},
|
||||
)
|
||||
planned = start_conv(client, headers, later.json()["day"]["id"])
|
||||
facts = collect_opening_context(setup.json()["profile_id"], planned.json()["id"])
|
||||
expect(facts["has_relevant_context"], "attested plan counts as relevant context")
|
||||
expect(any("Cres" in item and is_attested_plan(item) for item in facts["attested_plans"]), "plan comes from user wording")
|
||||
expect(all(item in facts["user_texts"] for item in facts["attested_plans"]), "plans are a subset of user texts")
|
||||
expect(planned.json().get("opening", {}).get("kind") == "model", "attested plan uses the dialogue path")
|
||||
intern = (planned.json().get("opening") or {}).get("trace") or {}
|
||||
intern_text = intern.get("intern") or ""
|
||||
expect("Belegte Vorhaben" in intern_text, "opening policy reaches the prompt")
|
||||
expect("kein bewiesenes wiederkehrendes Muster" in intern_text or "kein bewiesenes" in intern_text, "pattern rule is in the opening hint")
|
||||
|
||||
thread_space, thread_day = open_space_day(client, headers, "Tagfaden", "2026-08-10")
|
||||
first = start_conv(client, headers, thread_day["day"]["id"])
|
||||
client.post(
|
||||
f"/api/journal/conversations/{first.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Morgens am Markt, danach noch offen, wie der Nachmittag wird."},
|
||||
)
|
||||
second = start_conv(client, headers, thread_day["day"]["id"])
|
||||
thread_facts = collect_opening_context(setup.json()["profile_id"], second.json()["id"])
|
||||
expect(thread_facts["open_day_points"], "same-day user text is an open point")
|
||||
expect(
|
||||
any("Markt" in item or "Nachmittag" in item for item in thread_facts["open_day_points"]),
|
||||
"open point is the day's user wording",
|
||||
)
|
||||
expect(second.json().get("opening", {}).get("kind") == "model", "open day thread uses the dialogue path")
|
||||
|
||||
overlap_space, overlap_day = open_space_day(client, headers, "Overlap", "2026-08-03")
|
||||
overlap_conv = start_conv(client, headers, overlap_day["day"]["id"])
|
||||
client.post(
|
||||
f"/api/journal/conversations/{overlap_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute war der Hafen ruhig. Kirschen am Stand, Wind von See."},
|
||||
)
|
||||
client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={
|
||||
"journal_day_id": overlap_day["day"]["id"],
|
||||
"title": "Hafen",
|
||||
"body": "Heute war der Hafen ruhig. Kirschen am Stand.",
|
||||
"origin": "user_edit",
|
||||
},
|
||||
)
|
||||
later_overlap = client.post(
|
||||
f"/api/journal/spaces/{overlap_space['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-04"},
|
||||
)
|
||||
overlap_open = start_conv(client, headers, later_overlap.json()["day"]["id"])
|
||||
overlap_facts = collect_opening_context(setup.json()["profile_id"], overlap_open.json()["id"])
|
||||
expect(not overlap_facts["attested_plans"], "random overlap is not an attested plan")
|
||||
expect(not overlap_facts["has_relevant_context"], "recency without plan or open day is not relevant opening context")
|
||||
expect(overlap_open.json().get("opening", {}).get("kind") == "local_neutral", "overlap falls back to neutral")
|
||||
expect("Muster" not in (overlap_open.json().get("opening") or {}).get("assistant", {}).get("body", ""), "neutral impulse does not claim a pattern")
|
||||
|
||||
asst_space, asst_day = open_space_day(client, headers, "Assistent", "2026-08-05")
|
||||
asst_conv = start_conv(client, headers, asst_day["day"]["id"])
|
||||
from dialogue_store import append_message
|
||||
|
||||
append_message(
|
||||
setup.json()["profile_id"],
|
||||
asst_conv.json()["id"],
|
||||
"Morgen wollen wir nach Cres fahren.",
|
||||
role="assistant",
|
||||
)
|
||||
next_asst = client.post(
|
||||
f"/api/journal/spaces/{asst_space['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-06"},
|
||||
)
|
||||
asst_open = start_conv(client, headers, next_asst.json()["day"]["id"])
|
||||
asst_facts = collect_opening_context(setup.json()["profile_id"], asst_open.json()["id"])
|
||||
expect(not asst_facts["attested_plans"], "assistant wording is not a user plan")
|
||||
expect("Cres" not in " ".join(asst_facts["attested_plans"]), "assistant plan sentence stays out of attested plans")
|
||||
expect(asst_facts["assistant_is_not_user_fact"], "adapter marks assistant as non-fact")
|
||||
|
||||
pair_space, pair_day = open_space_day(client, headers, "Paarung", "2026-08-07")
|
||||
pair_conv = start_conv(client, headers, pair_day["day"]["id"])
|
||||
expect((pair_conv.json().get("messages") or [])[0]["role"] == "assistant", "assistant-first stored")
|
||||
user_turn = client.post(
|
||||
f"/api/journal/conversations/{pair_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Nach der Fähre kamen wir spät an. Anna kaufte Brot."},
|
||||
)
|
||||
expect(user_turn.status_code == 200, "user turn after opening")
|
||||
msgs = user_turn.json()["messages"]
|
||||
users = [item for item in msgs if item["role"] == "user"]
|
||||
expect(len(users) == 1, "exactly one user source after opening plus turn")
|
||||
paired = pair_user_priority_messages(msgs)
|
||||
paired_users = [item for item in paired if item["role"] == "user"]
|
||||
expect([item["body"] for item in paired_users] == [item["body"] for item in users], "pairing keeps every user source")
|
||||
labeled = assign_source_ids(paired)
|
||||
expect(
|
||||
[item["source_id"] for item in labeled if item["role"] == "user"] == ["u1"],
|
||||
"first user source is u1 even after an opening assistant",
|
||||
)
|
||||
draft = client.post(
|
||||
f"/api/journal/days/{pair_day['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [pair_conv.json()["id"]]},
|
||||
)
|
||||
expect(draft.status_code == 200, f"generate after opening {draft.text}")
|
||||
reconstruct = None
|
||||
for stage in (draft.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_reconstruct":
|
||||
reconstruct = stage
|
||||
intern = (reconstruct or {}).get("intern") or ""
|
||||
expect("Anna kaufte Brot" in intern, "user source reaches reconstruction")
|
||||
expect("[u1]" in intern, "user source id assigned")
|
||||
narrate = None
|
||||
for stage in (draft.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
narrate = stage
|
||||
narrate_intern = (narrate or {}).get("intern") or ""
|
||||
expect("verified_artifact" in narrate_intern or "sources" in narrate_intern, "stage 2 consumes verified artifact")
|
||||
expect("Nach der Fähre kamen wir spät an" in intern, "user wording not dropped after opening")
|
||||
expect(users[0]["body"].startswith("Nach der Fähre"), "user source is the user message, not the opening")
|
||||
|
||||
stolen = client.post(
|
||||
f"/api/journal/days/{empty_day['day']['id']}/conversations",
|
||||
headers=other_headers,
|
||||
json={"title": "fremd"},
|
||||
)
|
||||
expect(stolen.status_code == 404, "foreign profile cannot open a conversation")
|
||||
stolen_turn = client.post(
|
||||
f"/api/journal/conversations/{empty.json()['id']}/turn",
|
||||
headers=other_headers,
|
||||
json={"body": "Hallo"},
|
||||
)
|
||||
expect(stolen_turn.status_code == 404, "foreign profile cannot turn into the opening conversation")
|
||||
user_login_space = client.post("/api/journal/spaces", headers=other_headers, json={"title": "Ute"})
|
||||
user_day = client.post(
|
||||
f"/api/journal/spaces/{user_login_space.json()['id']}/days",
|
||||
headers=other_headers,
|
||||
json={"calendar_date": "2026-08-21"},
|
||||
)
|
||||
user_open = start_conv(client, other_headers, user_day.json()["day"]["id"])
|
||||
expect(user_open.status_code == 200, "non-admin can start their own day")
|
||||
expect("trace" not in (user_open.json().get("opening") or {}), "non-admin opening has no trace")
|
||||
expect("opening_context" not in (user_open.json().get("opening") or {}), "non-admin opening has no opening_context")
|
||||
expect("trace" not in user_open.json(), "non-admin response has no top-level trace")
|
||||
|
||||
start = client.get("/api/journal/continuity", headers=headers)
|
||||
expect(start.status_code == 200 and start.json().get("continuable"), "continuable day for the author")
|
||||
expect(start.json()["continuable"]["space_id"] == start.json()["continuable"]["space_id"], "continuity names a space")
|
||||
other_start = client.get("/api/journal/continuity", headers=other_headers)
|
||||
expect(other_start.json()["continuable"]["space_id"] == user_login_space.json()["id"], "continuity is profile-scoped")
|
||||
stolen_cont = client.get(f"/api/journal/spaces/{plan_space['id']}", headers=other_headers)
|
||||
expect(stolen_cont.status_code == 404, "continuity space stays isolated")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Journal trash, undelete, and hard delete. Run from backend/: python tests/test_journal_trash.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-trash-test.sqlite")
|
||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-trash-media")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from main import app
|
||||
from media_store import media_root
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
PNG = bytes.fromhex(
|
||||
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
|
||||
"0000000a49444154789c6360000002000100ffff03000006000557bf0000000049454e44ae426082"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
with TestClient(app) as client:
|
||||
setup = client.post("/api/auth/setup", json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"})
|
||||
headers = header(setup.json()["token"])
|
||||
other = client.post(
|
||||
"/api/users",
|
||||
headers=headers,
|
||||
json={"email": "ute@example.test", "name": "Ute", "password": "user-pass", "role": "user"},
|
||||
)
|
||||
expect(other.status_code == 200, "second profile")
|
||||
other_headers = header(client.post("/api/auth/login", json={"email": "ute@example.test", "password": "user-pass"}).json()["token"])
|
||||
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"})
|
||||
space_id = space.json()["id"]
|
||||
day = client.post(f"/api/journal/spaces/{space_id}/days", headers=headers, json={"calendar_date": "2026-08-20"})
|
||||
day_id = day.json()["day"]["id"]
|
||||
conv = client.post(f"/api/journal/days/{day_id}/conversations", headers=headers, json={"title": "Gespräch"})
|
||||
client.post(f"/api/journal/conversations/{conv.json()['id']}/turn", headers=headers, json={"body": "Heute war der Markt voll."})
|
||||
saved = client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={"journal_day_id": day_id, "title": "Markt", "body": "Heute war der Markt voll.", "origin": "user_edit"},
|
||||
)
|
||||
entry_id = saved.json()["id"]
|
||||
second = client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={"journal_day_id": day_id, "title": "Abend", "body": "Später am Hafen.", "origin": "user_edit"},
|
||||
)
|
||||
second_id = second.json()["id"]
|
||||
media = client.post(
|
||||
f"/api/journal/entries/{entry_id}/media",
|
||||
headers=headers,
|
||||
files={"file": ("shot.png", PNG, "image/png")},
|
||||
)
|
||||
expect(media.status_code == 200, "media attached")
|
||||
media_id = media.json()["id"]
|
||||
rel = media.json()["rel_path"]
|
||||
expect((media_root() / rel).is_file(), "media file exists")
|
||||
|
||||
listed = client.get(f"/api/journal/spaces/{space_id}/entries", headers=headers).json()
|
||||
expect(len(listed) == 2, "two active entries")
|
||||
expect(listed[0]["calendar_date"] == "2026-08-20", "same day")
|
||||
expect(listed[0]["id"] != listed[1]["id"], "same-day entries are distinct")
|
||||
expect(all(item["id"] in {entry_id, second_id} for item in listed), "list is the two entries")
|
||||
|
||||
hidden_list = client.get(f"/api/journal/spaces/{space_id}/entries", headers=other_headers)
|
||||
expect(hidden_list.status_code == 404, "entry list isolated")
|
||||
hidden_trash = client.get(f"/api/journal/spaces/{space_id}/trash", headers=other_headers)
|
||||
expect(hidden_trash.status_code == 404, "trash list isolated")
|
||||
|
||||
deleted = client.delete(f"/api/journal/entries/{entry_id}", headers=headers)
|
||||
expect(deleted.status_code == 200, "soft delete")
|
||||
after_soft = client.get(f"/api/journal/spaces/{space_id}/entries", headers=headers).json()
|
||||
expect(all(item["id"] != entry_id for item in after_soft), "soft-deleted entry leaves chronology")
|
||||
expect(any(item["id"] == second_id for item in after_soft), "other entry remains")
|
||||
trash = client.get(f"/api/journal/spaces/{space_id}/trash", headers=headers).json()
|
||||
expect(any(item["id"] == entry_id for item in trash), "soft-deleted appears in trash")
|
||||
missing = client.get(f"/api/journal/entries/{entry_id}", headers=headers)
|
||||
expect(missing.status_code == 404, "deleted entry is not a normal GET")
|
||||
|
||||
stolen_delete = client.delete(f"/api/journal/entries/{second_id}", headers=other_headers)
|
||||
expect(stolen_delete.status_code == 404, "foreign profile cannot soft-delete")
|
||||
stolen_undelete = client.post(f"/api/journal/entries/{entry_id}/undelete", headers=other_headers, json={})
|
||||
expect(stolen_undelete.status_code == 404, "foreign profile cannot undelete")
|
||||
stolen_purge = client.post(f"/api/journal/entries/{entry_id}/purge", headers=other_headers, json={"confirm": True})
|
||||
expect(stolen_purge.status_code == 404, "foreign profile cannot purge")
|
||||
|
||||
version_restore = client.post(
|
||||
f"/api/journal/entries/{entry_id}/restore",
|
||||
headers=headers,
|
||||
json={"version_id": saved.json()["current_version_id"]},
|
||||
)
|
||||
expect(version_restore.status_code == 404, "version restore is not trash undelete")
|
||||
|
||||
back = client.post(f"/api/journal/entries/{entry_id}/undelete", headers=headers, json={})
|
||||
expect(back.status_code == 200, f"undelete {back.text}")
|
||||
expect(back.json().get("deleted_at") in (None, ""), "undelete clears deleted_at")
|
||||
restored_list = client.get(f"/api/journal/spaces/{space_id}/entries", headers=headers).json()
|
||||
expect(any(item["id"] == entry_id for item in restored_list), "undeleted entry returns to chronology")
|
||||
expect(client.get(f"/api/journal/spaces/{space_id}/trash", headers=headers).json() == [], "trash empty after undelete")
|
||||
versions = client.get(f"/api/journal/entries/{entry_id}/versions", headers=headers).json()
|
||||
expect(len(versions) == 1, "undelete does not append a version")
|
||||
|
||||
client.delete(f"/api/journal/entries/{entry_id}", headers=headers)
|
||||
unconfirmed = client.post(f"/api/journal/entries/{entry_id}/purge", headers=headers, json={"confirm": False})
|
||||
expect(unconfirmed.status_code == 400, "purge without confirm fails closed")
|
||||
still = client.get(f"/api/journal/spaces/{space_id}/trash", headers=headers).json()
|
||||
expect(any(item["id"] == entry_id for item in still), "unconfirmed purge leaves the entry")
|
||||
|
||||
active_purge = client.post(f"/api/journal/entries/{second_id}/purge", headers=headers, json={"confirm": True})
|
||||
expect(active_purge.status_code == 400, "purge of active entry is rejected")
|
||||
|
||||
conv_before = client.get(f"/api/journal/conversations/{conv.json()['id']}", headers=headers)
|
||||
expect(conv_before.status_code == 200, "source dialogue still there before purge")
|
||||
purged = client.post(f"/api/journal/entries/{entry_id}/purge", headers=headers, json={"confirm": True})
|
||||
expect(purged.status_code == 200 and purged.json().get("purged"), f"purge {purged.text}")
|
||||
expect(purged.json().get("conversations_untouched") is True, "source dialogues are not deleted")
|
||||
expect(client.get(f"/api/journal/conversations/{conv.json()['id']}", headers=headers).status_code == 200, "dialogue remains")
|
||||
expect(client.get(f"/api/journal/entries/{entry_id}", headers=headers).status_code == 404, "purged entry gone")
|
||||
expect(client.get(f"/api/journal/spaces/{space_id}/trash", headers=headers).json() == [], "purged entry leaves trash")
|
||||
expect(not (media_root() / rel).exists(), "unreferenced media file removed")
|
||||
expect(client.get(f"/api/journal/media/{media_id}", headers=headers).status_code == 404, "media row gone")
|
||||
|
||||
again = client.post(f"/api/journal/entries/{entry_id}/purge", headers=headers, json={"confirm": True})
|
||||
expect(again.status_code == 404, "second purge of same id is not found")
|
||||
again_undelete = client.post(f"/api/journal/entries/{entry_id}/undelete", headers=headers, json={})
|
||||
expect(again_undelete.status_code == 404, "cannot undelete a purged entry")
|
||||
|
||||
leftover = client.get(f"/api/journal/spaces/{space_id}/entries", headers=headers).json()
|
||||
expect(all(item["id"] != entry_id for item in leftover), "purged id absent from chronology")
|
||||
expect(any(item["id"] == second_id for item in leftover), "untouched entry remains")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Local backup/restore. Run from backend/: python tests/test_local_backup.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from local_backup import BackupError, create_backup, restore_backup, verify_archive
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def write_db(path: Path, value: str) -> None:
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS note (id INTEGER PRIMARY KEY, body TEXT)")
|
||||
conn.execute("DELETE FROM note")
|
||||
conn.execute("INSERT INTO note (body) VALUES (?)", (value,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def read_db(path: Path) -> str:
|
||||
conn = sqlite3.connect(path)
|
||||
row = conn.execute("SELECT body FROM note").fetchone()
|
||||
conn.close()
|
||||
return row[0] if row else ""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="kansho-backup-test-") as raw:
|
||||
base = Path(raw)
|
||||
db = base / "live" / "kansho.sqlite"
|
||||
media = base / "live" / "media"
|
||||
out = base / "archives"
|
||||
db.parent.mkdir(parents=True)
|
||||
media.mkdir(parents=True)
|
||||
write_db(db, "original-body")
|
||||
sample = media / "profile" / "shot.png"
|
||||
sample.parent.mkdir(parents=True)
|
||||
sample.write_bytes(b"png-bytes")
|
||||
secret = base / "live" / ".env"
|
||||
secret.write_text("KANSHO_PROVIDER_KEY=sk-secret\n", encoding="utf-8")
|
||||
|
||||
archive = out / "first.zip"
|
||||
created = create_backup(archive, db_path=db, media_root=media)
|
||||
expect(archive.is_file(), "backup file created")
|
||||
expect(created["manifest"]["kind"] == "kansho.local_backup", "manifest kind")
|
||||
expect("persönliche, unverschlüsselte" in created["manifest"]["warning"].lower() or "unverschl" in created["manifest"]["warning"], "personal-data warning")
|
||||
verified = verify_archive(archive)
|
||||
expect(verified["manifest"]["checksums"]["kansho.sqlite"], "sqlite checksum present")
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
names = zf.namelist()
|
||||
expect("kansho.sqlite" in names, "sqlite in archive")
|
||||
expect("media/profile/shot.png" in names, "media in archive")
|
||||
expect(".env" not in names, ".env excluded")
|
||||
expect(all("sk-secret" not in zf.read(name).decode("latin1", errors="ignore") for name in names if name.endswith(".json") or name.endswith(".txt")), "no secret in manifest/readme")
|
||||
|
||||
try:
|
||||
restore_backup(archive, db_path=db, media_root=media, confirm=False)
|
||||
expect(False, "restore without confirm must fail")
|
||||
except BackupError as exc:
|
||||
expect(exc.code == "confirm_required", "restore requires explicit confirm")
|
||||
|
||||
try:
|
||||
restore_backup(archive, db_path=db, media_root=media, confirm=True, allow_overwrite=False)
|
||||
expect(False, "restore must not silently overwrite")
|
||||
except BackupError as exc:
|
||||
expect(exc.code == "would_overwrite", "restore refuses existing data without replace")
|
||||
|
||||
target_db = base / "restore" / "kansho.sqlite"
|
||||
target_media = base / "restore" / "media"
|
||||
restored = restore_backup(
|
||||
archive,
|
||||
db_path=target_db,
|
||||
media_root=target_media,
|
||||
confirm=True,
|
||||
allow_overwrite=False,
|
||||
safety_dir=out,
|
||||
)
|
||||
expect(read_db(target_db) == "original-body", "restored sqlite content")
|
||||
expect((target_media / "profile" / "shot.png").read_bytes() == b"png-bytes", "restored media")
|
||||
expect(restored["safety_backup"] is None, "empty target needs no safety backup")
|
||||
|
||||
write_db(target_db, "changed-live")
|
||||
(target_media / "profile" / "extra.bin").write_bytes(b"extra")
|
||||
replaced = restore_backup(
|
||||
archive,
|
||||
db_path=target_db,
|
||||
media_root=target_media,
|
||||
confirm=True,
|
||||
allow_overwrite=True,
|
||||
safety_dir=out,
|
||||
)
|
||||
expect(read_db(target_db) == "original-body", "replace restores snapshot")
|
||||
expect(not (target_media / "profile" / "extra.bin").exists(), "extra live media not kept")
|
||||
expect(Path(replaced["safety_backup"]).is_file(), "safety backup written before replace")
|
||||
safety = verify_archive(Path(replaced["safety_backup"]))
|
||||
expect(safety["manifest"]["kind"] == "kansho.local_backup", "safety backup is a real archive")
|
||||
|
||||
broken = out / "broken.zip"
|
||||
with zipfile.ZipFile(archive) as src, zipfile.ZipFile(broken, "w") as dest:
|
||||
for info in src.infolist():
|
||||
data = src.read(info.filename)
|
||||
if info.filename == "kansho.sqlite":
|
||||
data = data + b"tamper"
|
||||
dest.writestr(info, data)
|
||||
before = read_db(target_db)
|
||||
try:
|
||||
restore_backup(broken, db_path=target_db, media_root=target_media, confirm=True, allow_overwrite=True)
|
||||
expect(False, "tampered archive must fail")
|
||||
except BackupError as exc:
|
||||
expect(exc.code == "checksum_mismatch", "tamper detected before restore")
|
||||
expect(read_db(target_db) == before, "failed restore leaves current db")
|
||||
|
||||
extra_zip = out / "extra.zip"
|
||||
with zipfile.ZipFile(archive) as src, zipfile.ZipFile(extra_zip, "w") as dest:
|
||||
for info in src.infolist():
|
||||
dest.writestr(info, src.read(info.filename))
|
||||
dest.writestr("notes.txt", "not in manifest")
|
||||
extra_verified = verify_archive(extra_zip)
|
||||
expect("notes.txt" in extra_verified["skipped_extra"], "extra archive files are skipped deterministically")
|
||||
|
||||
locked_db = base / "locked" / "kansho.sqlite"
|
||||
locked_media = base / "locked" / "media"
|
||||
locked_db.parent.mkdir()
|
||||
locked_media.mkdir()
|
||||
write_db(locked_db, "locked")
|
||||
holder = sqlite3.connect(str(locked_db), timeout=1)
|
||||
holder.execute("BEGIN EXCLUSIVE")
|
||||
try:
|
||||
restore_backup(
|
||||
archive,
|
||||
db_path=locked_db,
|
||||
media_root=locked_media,
|
||||
confirm=True,
|
||||
allow_overwrite=True,
|
||||
safety_dir=out,
|
||||
)
|
||||
expect(False, "in-use db must abort")
|
||||
except BackupError as exc:
|
||||
expect(exc.code in {"db_in_use", "db_unavailable", "backup_exists", "would_overwrite"} or "in_use" in exc.code or "unavailable" in exc.code, f"busy db fail-closed ({exc.code})")
|
||||
finally:
|
||||
holder.rollback()
|
||||
holder.close()
|
||||
|
||||
expect(True, "backup tests used only temporary directories")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Network-free OpenRouter catalog tests. Run from backend/: python tests/test_model_catalog.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-model-catalog-test.sqlite")
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from model_catalog import (
|
||||
ModelWindow,
|
||||
catalog_ttl,
|
||||
reset_catalog,
|
||||
resolve_generate_metadata,
|
||||
)
|
||||
from prompt_budget import ERROR_MODEL_METADATA_UNKNOWN, JournalBudgetError
|
||||
from providers import ProviderConfig
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload, status_code: int = 200, text: str = ""):
|
||||
self._payload = payload
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
|
||||
def json(self):
|
||||
if isinstance(self._payload, Exception):
|
||||
raise self._payload
|
||||
return self._payload
|
||||
|
||||
|
||||
def _openrouter_config(model: str = "openai/gpt-4o") -> ProviderConfig:
|
||||
return ProviderConfig(
|
||||
role="generate",
|
||||
name="openrouter",
|
||||
mode="http",
|
||||
url="https://openrouter.ai/api/v1/chat/completions",
|
||||
model=model,
|
||||
key="sk-test",
|
||||
local=False,
|
||||
zdr=True,
|
||||
no_train=True,
|
||||
)
|
||||
|
||||
|
||||
def _clear_env(*names: str) -> dict[str, str | None]:
|
||||
previous = {}
|
||||
for name in names:
|
||||
previous[name] = os.environ.pop(name, None)
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_env(previous: dict[str, str | None]) -> None:
|
||||
for name, value in previous.items():
|
||||
if value is None:
|
||||
os.environ.pop(name, None)
|
||||
else:
|
||||
os.environ[name] = value
|
||||
|
||||
|
||||
def test_reads_context_and_completion() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
payload = {
|
||||
"data": [
|
||||
{
|
||||
"id": "openai/gpt-4o",
|
||||
"context_length": 128000,
|
||||
"top_provider": {"max_completion_tokens": 16384},
|
||||
}
|
||||
]
|
||||
}
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
window = resolve_generate_metadata(_openrouter_config())
|
||||
expect(window.context_length == 128000, "catalog reads context_length")
|
||||
expect(window.max_completion_tokens == 16384, "catalog reads max completion")
|
||||
expect(window.source == "openrouter_models", "window is from the catalog")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def test_model_not_in_catalog() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
payload = {"data": [{"id": "other/model", "context_length": 32000, "top_provider": {"max_completion_tokens": 4096}}]}
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
try:
|
||||
resolve_generate_metadata(_openrouter_config("missing/model"))
|
||||
raise SystemExit("FAIL: missing catalog model should fail closed")
|
||||
except JournalBudgetError as exc:
|
||||
expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "model not in catalog is fail-closed")
|
||||
expect((exc.diagnostics or {}).get("reason") == "model_not_in_catalog", "reason is model_not_in_catalog")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def test_incomplete_metadata() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
payload = {"data": [{"id": "openai/gpt-4o", "top_provider": {"max_completion_tokens": 4096}}]}
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
try:
|
||||
resolve_generate_metadata(_openrouter_config())
|
||||
raise SystemExit("FAIL: incomplete metadata should fail closed")
|
||||
except JournalBudgetError as exc:
|
||||
expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "incomplete metadata is fail-closed")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def test_cache_hit_and_ttl_expiry() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_get(*args, **kwargs):
|
||||
calls["n"] += 1
|
||||
return FakeResponse(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "openai/gpt-4o",
|
||||
"context_length": 64000 + calls["n"],
|
||||
"top_provider": {"max_completion_tokens": 4096},
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
with patch("model_catalog.httpx.get", fake_get):
|
||||
first = resolve_generate_metadata(_openrouter_config(), now=1000.0)
|
||||
second = resolve_generate_metadata(_openrouter_config(), now=1000.0 + catalog_ttl() - 1)
|
||||
expect(calls["n"] == 1, "catalog is cached within TTL")
|
||||
expect(first.context_length == second.context_length, "cache hit returns the same window")
|
||||
expect(second.cached is True, "cached window is marked cached")
|
||||
third = resolve_generate_metadata(_openrouter_config(), now=1000.0 + catalog_ttl() + 1)
|
||||
expect(calls["n"] == 2, "catalog is refetched after TTL")
|
||||
expect(third.context_length != first.context_length, "refetch reads a new catalog row")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def test_env_fallback_and_fail_closed() -> None:
|
||||
reset_catalog()
|
||||
previous = _clear_env("KANSHO_PROVIDER_CONTEXT_LENGTH", "KANSHO_PROVIDER_MAX_COMPLETION_TOKENS")
|
||||
payload = {"data": [{"id": "other/model", "context_length": 32000, "top_provider": {"max_completion_tokens": 2048}}]}
|
||||
os.environ["KANSHO_PROVIDER_CONTEXT_LENGTH"] = "65536"
|
||||
os.environ["KANSHO_PROVIDER_MAX_COMPLETION_TOKENS"] = "8192"
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
window = resolve_generate_metadata(_openrouter_config("missing/model"))
|
||||
expect(window.context_length == 65536, "documented env fallback supplies the window")
|
||||
expect(window.max_completion_tokens == 8192, "env fallback supplies completion limit")
|
||||
expect(window.source == "env_fallback", "fallback is labeled as env_fallback")
|
||||
|
||||
reset_catalog()
|
||||
os.environ.pop("KANSHO_PROVIDER_CONTEXT_LENGTH", None)
|
||||
os.environ.pop("KANSHO_PROVIDER_MAX_COMPLETION_TOKENS", None)
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse(payload)):
|
||||
try:
|
||||
resolve_generate_metadata(_openrouter_config("missing/model"))
|
||||
raise SystemExit("FAIL: no window should fail closed")
|
||||
except JournalBudgetError as exc:
|
||||
expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "no window is fail-closed")
|
||||
|
||||
reset_catalog()
|
||||
with patch("model_catalog.httpx.get", lambda *args, **kwargs: FakeResponse({"nope": True})):
|
||||
try:
|
||||
resolve_generate_metadata(_openrouter_config())
|
||||
raise SystemExit("FAIL: bad catalog shape should fail closed")
|
||||
except JournalBudgetError as exc:
|
||||
expect(exc.code == ERROR_MODEL_METADATA_UNKNOWN, "bad metadata shape is fail-closed")
|
||||
_restore_env(previous)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_reads_context_and_completion()
|
||||
test_model_not_in_catalog()
|
||||
test_incomplete_metadata()
|
||||
test_cache_hit_and_ttl_expiry()
|
||||
test_env_fallback_and_fail_closed()
|
||||
print("model catalog tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -10,9 +10,11 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-mvp-journal-test.sqlite")
|
||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-mvp-journal-media")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from main import app
|
||||
@@ -302,22 +304,29 @@ def main() -> None:
|
||||
|
||||
expect("Schluss" in infer_register(["Wir machten uns müde auf den Heimweg."]), "register sees the close first")
|
||||
originals = client.get(f"/api/journal/conversations/{conv['id']}", headers=headers)
|
||||
expect(len(originals.json()["messages"]) == 2, "A two source messages")
|
||||
source_msgs = originals.json()["messages"]
|
||||
expect(sum(1 for item in source_msgs if item["role"] == "user") == 1, "A one user source")
|
||||
expect(any(item["role"] == "assistant" for item in source_msgs), "A assistant source present")
|
||||
|
||||
reset_debug()
|
||||
draft = client.post(f"/api/journal/days/{day_id}/generate", headers=headers, json={"conversation_ids": [conv["id"]]})
|
||||
expect(draft.status_code == 200, f"A generate {draft.text}")
|
||||
expect("Markt" in draft.json()["body"] or "Kirschen" in draft.json()["body"], "A draft keeps user words")
|
||||
from privacy_gateway import debug_last as last_a
|
||||
|
||||
expect(last_a is not None, "A generate reached gateway")
|
||||
expect("Kirschen" in last_a["rendered"] or "Markt" in last_a["rendered"], "A generate sees user words")
|
||||
expect("assistant:" in last_a["rendered"], "A generate sees Kanshō impulses as reply context")
|
||||
expect("Stilquellen" in last_a["rendered"] or "Dialog" in last_a["rendered"], "A style brief reaches generate")
|
||||
style_part = last_a["rendered"].split("Dialogquellen:")[0]
|
||||
stages_a = {item.get("purpose"): item for item in (draft.json().get("trace") or {}).get("stages") or []}
|
||||
reconstruct_a = stages_a.get("journal_reconstruct")
|
||||
last_a = stages_a.get("journal_generate") or (draft.json().get("trace") or {})
|
||||
expect(reconstruct_a is not None, "A stage 1 reconstruct ran")
|
||||
reconstruct_intern = reconstruct_a.get("intern") or ""
|
||||
narrate_intern = last_a.get("intern") or ""
|
||||
expect("Kirschen" in reconstruct_intern or "Markt" in reconstruct_intern, "A reconstruct sees user words")
|
||||
expect("user:" in reconstruct_intern, "A reconstruct sees the day as dialogue")
|
||||
expect("assistant:" not in narrate_intern, "A stage 2 does not resend the day dialogue")
|
||||
expect("verified_artifact" in narrate_intern or "sources" in narrate_intern, "A stage 2 receives the reconstruction")
|
||||
expect("Stilquellen" in narrate_intern or "Dialog" in narrate_intern or "Erzählmerkmale" in narrate_intern or "Core:" in narrate_intern, "A style brief reaches generate")
|
||||
style_part = narrate_intern
|
||||
expect("Erzählmerkmale" in style_part, "A first generate still gets form hints")
|
||||
expect(
|
||||
"Heute war der Markt voll" not in style_part,
|
||||
"Heute war der Markt voll" not in style_part.split("Verified Artifact:")[0],
|
||||
"A source dialogue is not pasted a second time as style",
|
||||
)
|
||||
current_version = None
|
||||
@@ -387,11 +396,10 @@ def main() -> None:
|
||||
next_day.json()["day"]["id"],
|
||||
"Der Vormittag war ruhig.",
|
||||
)
|
||||
from privacy_gateway import debug_last as last_c
|
||||
|
||||
expect(last_c is not None, "C gateway saw a request")
|
||||
expect("Hafen" in last_c["rendered"], "C prior plan in context")
|
||||
expect("nicht geschehen" in last_c["rendered"], "C not-mentioned rule in prompt")
|
||||
expect(turn_c is not None, "C gateway saw a request")
|
||||
intern_c = (turn_c.get("trace") or {}).get("intern") or ""
|
||||
expect("Hafen" in intern_c, "C prior plan in context")
|
||||
expect("nicht geschehen" in intern_c, "C not-mentioned rule in prompt")
|
||||
|
||||
reset_debug()
|
||||
dense = "Erster Absatz mit vielen Details über die Anreise.\n\nZweiter Absatz über das Licht am Abend.\n\nDritter Absatz über das Essen."
|
||||
@@ -410,9 +418,8 @@ def main() -> None:
|
||||
"Ich bin traurig über den Abschied, das ist klar.",
|
||||
title="Emotion",
|
||||
)
|
||||
from privacy_gateway import debug_last as last_e
|
||||
|
||||
expect("Motivationen erfinden" in last_e["rendered"], "E no invented emotion rule")
|
||||
intern_e = (turn_e.get("trace") or {}).get("intern") or ""
|
||||
expect("Motivationen erfinden" in intern_e, "E no invented emotion rule")
|
||||
expect(turn_e["calls"] == 1, "E one follow-up")
|
||||
|
||||
day_f = client.post(
|
||||
@@ -498,9 +505,12 @@ def main() -> None:
|
||||
json={"conversation_ids": [conv["id"]]},
|
||||
)
|
||||
expect(gen_i.status_code == 200, "I generate after import")
|
||||
from privacy_gateway import debug_last as last_i
|
||||
|
||||
expect("knapp" in last_i["rendered"], "I style brief reaches generate")
|
||||
intern_i = ""
|
||||
for stage in (gen_i.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
intern_i = stage.get("intern") or ""
|
||||
intern_i = intern_i or ((gen_i.json().get("trace") or {}).get("intern") or "")
|
||||
expect("knapp" in intern_i, "I style brief reaches generate")
|
||||
profile = client.get("/api/journal/writing-profile", headers=headers).json()
|
||||
expect(all(item.get("kind") != "journal_draft" for item in profile["sources"]), "I no draft kind")
|
||||
expect(not any((item.get("body") or "") == gen_i.json()["body"] for item in profile["sources"]), "I draft not a style source")
|
||||
@@ -538,6 +548,50 @@ def main() -> None:
|
||||
(day_with_draft["current_draft"] or {}).get("body") != after_gen["body"],
|
||||
"J new draft is not the saved user text",
|
||||
)
|
||||
marker = "EXISTING_FASSUNG_XYZ_UNIQUE"
|
||||
marked = client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={
|
||||
"journal_day_id": day_id,
|
||||
"entry_id": saved.json()["id"],
|
||||
"title": "Markiert",
|
||||
"body": marker,
|
||||
"origin": "user_edit",
|
||||
},
|
||||
)
|
||||
expect(marked.status_code == 200, "include_existing setup save")
|
||||
marked_version = marked.json()["current_version_id"]
|
||||
without_flag = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv["id"]]},
|
||||
)
|
||||
expect(without_flag.status_code == 200, f"generate default {without_flag.text}")
|
||||
without_intern = ""
|
||||
for stage in (without_flag.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
without_intern = stage.get("intern") or ""
|
||||
expect(marker not in without_intern, "include_existing default does not send the saved body")
|
||||
after_without = client.get(f"/api/journal/entries/{saved.json()['id']}", headers=headers).json()
|
||||
expect(after_without["current_version_id"] == marked_version, "default generate leaves entry version")
|
||||
with_flag = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv["id"]], "include_existing": True},
|
||||
)
|
||||
expect(with_flag.status_code == 200, f"generate include_existing {with_flag.text}")
|
||||
with_intern = ""
|
||||
for stage in (with_flag.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
with_intern = stage.get("intern") or ""
|
||||
expect(marker in with_intern, "include_existing true sends the saved body only after explicit flag")
|
||||
existing_section = with_intern.split("Bestehender Text")[-1].split("Verified Artifact")[0]
|
||||
expect(marker in existing_section, "saved Fassung is the included existing text, not a leftover draft")
|
||||
after_with = client.get(f"/api/journal/entries/{saved.json()['id']}", headers=headers).json()
|
||||
expect(after_with["current_version_id"] == marked_version, "include_existing generate still does not overwrite")
|
||||
expect(after_with["body"] == marker, "saved user body remains")
|
||||
day_with_draft = client.get(f"/api/journal/days/{day_id}", headers=headers).json()
|
||||
n_before = len(client.get(f"/api/journal/entries/{saved.json()['id']}/versions", headers=headers).json())
|
||||
adopted = client.post(
|
||||
"/api/journal/entries",
|
||||
@@ -662,13 +716,10 @@ def main() -> None:
|
||||
json={"body": "Noch ein Satz zum Abend."},
|
||||
)
|
||||
expect(turn_scratch.status_code == 200, "turn after scratch")
|
||||
from privacy_gateway import debug_last as last_scratch_turn
|
||||
blob = " ".join(
|
||||
str(last_scratch_turn.get(key) or "")
|
||||
for key in ("rendered", "masked", "mask_input", "raw", "reply")
|
||||
)
|
||||
intern = (turn_scratch.json().get("trace") or {}).get("intern") or ""
|
||||
expect(scratch_mark not in blob and scratch_mark not in intern, "scratch not in dialogue egress")
|
||||
blob = (turn_scratch.json().get("trace") or {}).get("intern") or ""
|
||||
intern = blob
|
||||
egress = (turn_scratch.json().get("trace") or {}).get("egress") or ""
|
||||
expect(scratch_mark not in blob and scratch_mark not in intern and scratch_mark not in egress, "scratch not in dialogue egress")
|
||||
reset_debug()
|
||||
gen_scratch = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
@@ -676,12 +727,13 @@ def main() -> None:
|
||||
json={"conversation_ids": [conv["id"]]},
|
||||
)
|
||||
expect(gen_scratch.status_code == 200, "generate after scratch")
|
||||
from privacy_gateway import debug_last as last_scratch_gen
|
||||
gen_trace = gen_scratch.json().get("trace") or {}
|
||||
gen_blob = " ".join(
|
||||
str(last_scratch_gen.get(key) or "")
|
||||
for key in ("rendered", "masked", "mask_input", "raw", "reply")
|
||||
str(stage.get(key) or "")
|
||||
for stage in (gen_trace.get("stages") or [gen_trace])
|
||||
for key in ("intern", "egress", "mask_input", "raw", "reply")
|
||||
)
|
||||
gen_intern = (gen_scratch.json().get("trace") or {}).get("intern") or ""
|
||||
gen_intern = gen_trace.get("intern") or ""
|
||||
expect(scratch_mark not in gen_blob and scratch_mark not in gen_intern, "scratch not in generate egress")
|
||||
|
||||
from media_store import strip_jpeg_exif
|
||||
|
||||
@@ -51,13 +51,12 @@ def main() -> None:
|
||||
json={"body": "Ich war mit Anna in Hamburg und danach allein."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn {turn.text}")
|
||||
from privacy_gateway import debug_last
|
||||
|
||||
expect(debug_last is not None, "gateway recorded request")
|
||||
expect(debug_last["detect_provider"] == "fake-detect", "detect provider is separate")
|
||||
expect("Anna" not in debug_last["masked"], "Anna masked before generate")
|
||||
expect("Hamburg" not in debug_last["masked"], "Hamburg masked before generate")
|
||||
expect("[[PERSON:" in debug_last["masked"] or "[[PLACE:" in debug_last["masked"], "placeholders in egress")
|
||||
trace = turn.json().get("trace") or {}
|
||||
expect(trace, "gateway returned request-scoped trace")
|
||||
expect(trace["detect_provider"] == "fake-detect", "detect provider is separate")
|
||||
expect("Anna" not in (trace.get("egress") or ""), "Anna masked before generate")
|
||||
expect("Hamburg" not in (trace.get("egress") or ""), "Hamburg masked before generate")
|
||||
expect("[[PERSON:" in (trace.get("egress") or "") or "[[PLACE:" in (trace.get("egress") or ""), "placeholders in egress")
|
||||
expect("Anna" in turn.json()["user"]["body"], "source keeps cleartext")
|
||||
mappings = list_mappings(setup.json()["profile_id"])
|
||||
labels = {item["local_label"] for item in mappings}
|
||||
@@ -72,10 +71,10 @@ def main() -> None:
|
||||
json={"body": "Anna war ruhig."},
|
||||
)
|
||||
expect(again.status_code == 200, "second turn")
|
||||
from privacy_gateway import debug_last as last2
|
||||
last2 = again.json().get("trace") or {}
|
||||
|
||||
expect("nicht" in last2["masked"].lower() or "Anna" not in last2["masked"], "dictionary remasks Anna without new invention")
|
||||
expect(last2["mapping_count"] >= 2, "dictionary reused")
|
||||
expect("nicht" in (last2.get("egress") or "").lower() or "Anna" not in (last2.get("egress") or ""), "dictionary remasks Anna without new invention")
|
||||
expect(last2.get("mapping_count", 0) >= 2, "dictionary reused")
|
||||
|
||||
from entity_detect import _pattern_entities
|
||||
from identity_store import is_maskable_label
|
||||
@@ -251,6 +250,25 @@ def main() -> None:
|
||||
except PrivacyGatewayError as exc:
|
||||
blocked = exc.code == "response_validation_failed"
|
||||
expect(blocked, "plaintext identity is blocked before demask")
|
||||
sushi_map = [{"local_label": "Sushi", "token": "PERSON:01"}]
|
||||
food_reply = _validate_response("Heute nur von Sushi essen erzählt.", sushi_map)
|
||||
expect("Sushi essen" in food_reply, "food homonym in a reply is not treated as a leak")
|
||||
mixed = mask_for_egress(
|
||||
"Belegte Vorhaben (nur user-Quellen):\n- Morgen wollen wir mit Anna fahren.\nuser: Heute Markt.",
|
||||
mapped,
|
||||
"dialogue_turn",
|
||||
)
|
||||
expect("Anna" not in mixed, "opening context names are masked for dialogue egress")
|
||||
expect("[[PERSON:01]]" in mixed, "opening context uses the person token")
|
||||
expect("Heute Markt" in mixed, "current user wording remains")
|
||||
excerpt = mask_for_egress(
|
||||
"Frühere Einträge im Space (nur Hinweise, keine Gewissheit):\n2026-08-01: Anna war am Hafen.\nuser: Heute windstill.",
|
||||
mapped,
|
||||
"dialogue_turn",
|
||||
)
|
||||
expect("Anna" not in excerpt, "space excerpts are masked for dialogue egress")
|
||||
expect("[[PERSON:01]]" in excerpt, "space excerpt uses the person token")
|
||||
|
||||
print("All privacy detect tests passed.")
|
||||
|
||||
|
||||
|
||||
@@ -183,11 +183,11 @@ def main() -> None:
|
||||
expect(interaction["suggestions"] == [], "silence is not an interaction suggestion")
|
||||
expect(any(item["pref_key"] == "reply_length" for item in interaction["defaults"]), "labeled product defaults")
|
||||
|
||||
from privacy_gateway import debug_last
|
||||
intern = (turn.json().get("trace") or {}).get("intern") or ""
|
||||
|
||||
expect("Produktdefault" in (debug_last["rendered"] if debug_last else ""), "hint reaches dialogue prompt")
|
||||
expect("Ein bis drei Sätze" in (debug_last["rendered"] if debug_last else ""), "length default is in hint not anonymous")
|
||||
expect("anonyme Produktregel" in (debug_last["rendered"] if debug_last else ""), "prompt labels the moved preference")
|
||||
expect("Produktdefault" in intern, "hint reaches dialogue prompt")
|
||||
expect("Ein bis drei Sätze" in intern, "length default is in hint not anonymous")
|
||||
expect("anonyme Produktregel" in intern, "prompt labels the moved preference")
|
||||
|
||||
proposed = propose_observation(
|
||||
profile_id,
|
||||
@@ -223,10 +223,10 @@ def main() -> None:
|
||||
json={"body": "Heute Vormittag zum Markt, danach Kirschen, dann noch der Hafen bei 18:00 Uhr."},
|
||||
)
|
||||
expect(later.status_code == 200, "second turn")
|
||||
from privacy_gateway import debug_last as last_hint
|
||||
intern = (later.json().get("trace") or {}).get("intern") or ""
|
||||
|
||||
expect("Meist nur ein Satz" in last_hint["rendered"], "personal preference reaches hint")
|
||||
expect("persönlich" in last_hint["rendered"], "personal override is labeled")
|
||||
expect("Meist nur ein Satz" in intern, "personal preference reaches hint")
|
||||
expect("persönlich" in intern, "personal override is labeled")
|
||||
expect(later.json()["conversation"].get("narrative_mode"), "dialogue state remains automatic")
|
||||
expect("long_story" in later.json()["conversation"], "long_story is conversation state")
|
||||
|
||||
|
||||
@@ -260,9 +260,9 @@ def main() -> None:
|
||||
json={"body": long_body},
|
||||
)
|
||||
expect(long_turn.status_code == 200, "long turn")
|
||||
from privacy_gateway import debug_last as last_long
|
||||
last_long = long_turn.json().get("trace") or {}
|
||||
|
||||
expect(last_long["purpose"] == "dialogue_turn", "long turn still has no review call")
|
||||
expect(last_long.get("purpose") == "dialogue_turn", "long turn still has no review call")
|
||||
status = client.get("/api/journal/writing-profile/review", headers=headers).json()
|
||||
expect("dialogue_sample" not in pending_triggers(status), "dialogue is not sampled before confirm")
|
||||
|
||||
@@ -514,10 +514,10 @@ def main() -> None:
|
||||
reset_debug()
|
||||
api_review = client.post("/api/journal/writing-profile/review/api", headers=headers)
|
||||
expect(api_review.status_code == 200, f"api review {api_review.text}")
|
||||
from privacy_gateway import debug_last as api_trace
|
||||
api_trace = api_review.json().get("trace") or {}
|
||||
|
||||
expect(api_trace["purpose"] == "profile_review", "api review uses dedicated purpose")
|
||||
expect(api_trace["layer"] == "profilreview", "trace is not a dialogue turn")
|
||||
expect(api_trace.get("purpose") == "profile_review", "api review uses dedicated purpose")
|
||||
expect(api_trace.get("layer") == "profilreview", "trace is not a dialogue turn")
|
||||
expect(api_review.json()["applied"] == 0, "api result is a proposal, not an auto-apply")
|
||||
expect(api_review.json()["suggested"] >= 1, "advising queues review as suggestion")
|
||||
expect(
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Intent-neutral provenance verification. Run from backend/: python tests/test_provenance.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ.setdefault(
|
||||
"KANSHO_DB_PATH",
|
||||
str(Path(tempfile.gettempdir()) / "kansho-provenance-test.sqlite"),
|
||||
)
|
||||
|
||||
from provenance import (
|
||||
COVERAGE_ALL_SELECTED_SOURCES,
|
||||
COVERAGE_SELECTED_EVIDENCE,
|
||||
EvidenceRef,
|
||||
ProvenanceError,
|
||||
SourceRecord,
|
||||
SourceRegistry,
|
||||
VerificationPolicy,
|
||||
verify,
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def _expect_error(reason: str, **kwargs) -> None:
|
||||
try:
|
||||
verify(**kwargs)
|
||||
raise SystemExit(f"FAIL: expected {reason}")
|
||||
except ProvenanceError as exc:
|
||||
expect(exc.reason == reason, f"reason is {reason}, got {exc.reason}")
|
||||
|
||||
|
||||
CHANGELOG = (
|
||||
"Build 14.2 ersetzt den Hash-Algorithmus. "
|
||||
"Die Checksummen-Datei liegt unter /var/lib/app/SHA256SUMS. "
|
||||
"Der Paketname bleibt libcore."
|
||||
)
|
||||
NOTE = "Der Review-Kommentar betrifft nur die Signaturprüfung."
|
||||
|
||||
|
||||
def _registry() -> SourceRegistry:
|
||||
return SourceRegistry(
|
||||
[
|
||||
SourceRecord("s1", "primary", CHANGELOG),
|
||||
SourceRecord("s2", "note", NOTE),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_source() -> None:
|
||||
registry = _registry()
|
||||
policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s9",))
|
||||
_expect_error("unknown_source", registry=registry, policy=policy)
|
||||
|
||||
|
||||
def test_disallowed_role() -> None:
|
||||
registry = _registry()
|
||||
policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s2",))
|
||||
_expect_error("role_not_allowed", registry=registry, policy=policy)
|
||||
|
||||
|
||||
def test_wrong_source_excerpt() -> None:
|
||||
registry = _registry()
|
||||
policy = VerificationPolicy(allowed_roles=frozenset({"primary", "note"}), selected_ids=("s1", "s2"))
|
||||
_expect_error(
|
||||
"wrong_source",
|
||||
registry=registry,
|
||||
policy=policy,
|
||||
evidence=[EvidenceRef("s1", "Signaturprüfung")],
|
||||
)
|
||||
|
||||
|
||||
def test_invented_excerpt() -> None:
|
||||
registry = _registry()
|
||||
policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s1",))
|
||||
_expect_error(
|
||||
"unverified_excerpt",
|
||||
registry=registry,
|
||||
policy=policy,
|
||||
evidence=[EvidenceRef("s1", "Die Lizenz ändert sich auf GPL.")],
|
||||
)
|
||||
|
||||
|
||||
def test_all_selected_rehydrates_full_text() -> None:
|
||||
registry = _registry()
|
||||
policy = VerificationPolicy(
|
||||
allowed_roles=frozenset({"primary"}),
|
||||
coverage=COVERAGE_ALL_SELECTED_SOURCES,
|
||||
selected_ids=("s1",),
|
||||
)
|
||||
artifact = verify(
|
||||
registry,
|
||||
policy=policy,
|
||||
evidence=[EvidenceRef("s1", "Build 14.2")],
|
||||
)
|
||||
expect(artifact.sources[0]["text"] == CHANGELOG, "full canonical changelog is rehydrated")
|
||||
expect("SHA256SUMS" in artifact.sources[0]["text"], "later changelog sentences stay")
|
||||
expect("libcore" in artifact.sources[0]["text"], "final changelog sentence stays")
|
||||
|
||||
|
||||
def test_excerpt_cannot_drop_remainder() -> None:
|
||||
registry = _registry()
|
||||
policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s1",))
|
||||
artifact = verify(registry, policy=policy, evidence=[EvidenceRef("s1", "Build 14.2")])
|
||||
expect(artifact.sources[0]["text"] != "Build 14.2", "a short excerpt is not the source")
|
||||
expect(artifact.sources[0]["text"].startswith("Build 14.2 ersetzt"), "canonical prefix remains")
|
||||
expect("Paketname bleibt libcore" in artifact.sources[0]["text"], "unread remainder is not dropped")
|
||||
|
||||
|
||||
def test_model_cannot_overwrite_canonical() -> None:
|
||||
registry = _registry()
|
||||
policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s1",))
|
||||
_expect_error(
|
||||
"canonical_overwrite",
|
||||
registry=registry,
|
||||
policy=policy,
|
||||
proposed_texts={"s1": "Build 14.2 ist ein Sicherheitsrelease ohne Dateipfade."},
|
||||
)
|
||||
artifact = verify(
|
||||
registry,
|
||||
policy=policy,
|
||||
proposed_texts={"s1": CHANGELOG},
|
||||
)
|
||||
expect(artifact.sources[0]["text"] == CHANGELOG, "identical proposed text is ignored as authority")
|
||||
|
||||
|
||||
def test_annotations_are_not_verified_content() -> None:
|
||||
registry = _registry()
|
||||
excluded = VerificationPolicy(
|
||||
allowed_roles=frozenset({"primary"}),
|
||||
selected_ids=("s1",),
|
||||
include_unverified_annotations=False,
|
||||
)
|
||||
artifact = verify(
|
||||
registry,
|
||||
policy=excluded,
|
||||
evidence=[EvidenceRef("s1", CHANGELOG)],
|
||||
annotations=[{"kind": "risk", "source_id": "s1", "excerpt": CHANGELOG}],
|
||||
)
|
||||
payload = artifact.to_payload()
|
||||
expect("annotations_unverified" not in payload, "excluded annotations do not appear")
|
||||
expect(all("kind" not in item for item in payload["sources"]), "source records have no semantic kind")
|
||||
dumped = str(payload)
|
||||
expect("risk" not in dumped, "unverified label is not in verified payload")
|
||||
|
||||
separated = VerificationPolicy(
|
||||
allowed_roles=frozenset({"primary"}),
|
||||
selected_ids=("s1",),
|
||||
include_unverified_annotations=True,
|
||||
)
|
||||
held = verify(
|
||||
registry,
|
||||
policy=separated,
|
||||
annotations=[{"kind": "risk", "source_id": "s1", "excerpt": CHANGELOG}],
|
||||
)
|
||||
expect(held.annotations_unverified[0]["kind"] == "risk", "unverified labels can be held separately")
|
||||
expect(held.sources[0]["text"] == CHANGELOG, "canonical text is independent of the label")
|
||||
|
||||
|
||||
def test_policies_differ_without_domain_logic() -> None:
|
||||
registry = _registry()
|
||||
primary_only = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("s2",))
|
||||
_expect_error("role_not_allowed", registry=registry, policy=primary_only)
|
||||
|
||||
with_notes = VerificationPolicy(
|
||||
allowed_roles=frozenset({"primary", "note"}),
|
||||
coverage=COVERAGE_ALL_SELECTED_SOURCES,
|
||||
selected_ids=("s2",),
|
||||
)
|
||||
accepted = verify(registry, policy=with_notes)
|
||||
expect(accepted.sources[0]["text"] == NOTE, "a second policy can allow another role")
|
||||
|
||||
selective = VerificationPolicy(
|
||||
allowed_roles=frozenset({"primary"}),
|
||||
coverage=COVERAGE_SELECTED_EVIDENCE,
|
||||
selected_ids=("s1",),
|
||||
)
|
||||
excerpt_only = verify(
|
||||
registry,
|
||||
policy=selective,
|
||||
evidence=[EvidenceRef("s1", "Build 14.2")],
|
||||
)
|
||||
expect(excerpt_only.sources[0]["text"] == "Build 14.2", "selected-evidence mode keeps only verified excerpts")
|
||||
expect("SHA256SUMS" not in excerpt_only.sources[0]["text"], "selected-evidence mode does not rehydrate the rest")
|
||||
|
||||
full = verify(
|
||||
registry,
|
||||
policy=VerificationPolicy(
|
||||
allowed_roles=frozenset({"primary"}),
|
||||
coverage=COVERAGE_ALL_SELECTED_SOURCES,
|
||||
selected_ids=("s1",),
|
||||
),
|
||||
evidence=[EvidenceRef("s1", "Build 14.2")],
|
||||
)
|
||||
expect("SHA256SUMS" in full.sources[0]["text"], "all-selected mode still rehydrates the same registry")
|
||||
|
||||
|
||||
def test_parallel_requests_are_isolated() -> None:
|
||||
registry_ok = SourceRegistry([SourceRecord("a1", "primary", "Alpha-Release 3 ist signiert.")])
|
||||
registry_bad = SourceRegistry([SourceRecord("b1", "primary", "Beta-Kanal bleibt intern.")])
|
||||
ok_policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("a1",))
|
||||
bad_policy = VerificationPolicy(allowed_roles=frozenset({"primary"}), selected_ids=("missing",))
|
||||
results: dict[str, object] = {}
|
||||
errors: dict[str, str] = {}
|
||||
|
||||
def succeed() -> None:
|
||||
results["ok"] = verify(registry_ok, policy=ok_policy)
|
||||
|
||||
def fail() -> None:
|
||||
try:
|
||||
verify(registry_bad, policy=bad_policy)
|
||||
except ProvenanceError as exc:
|
||||
errors["bad"] = exc.reason
|
||||
|
||||
first = threading.Thread(target=fail)
|
||||
second = threading.Thread(target=succeed)
|
||||
first.start()
|
||||
second.start()
|
||||
first.join()
|
||||
second.join()
|
||||
expect(errors.get("bad") == "unknown_source", "failing request keeps its own reason")
|
||||
artifact = results.get("ok")
|
||||
expect(artifact is not None, "successful request still returns an artifact")
|
||||
expect(artifact.sources[0]["text"] == "Alpha-Release 3 ist signiert.", "parallel failure does not rewrite the other registry")
|
||||
later = verify(registry_ok, policy=ok_policy)
|
||||
expect(later.sources[0]["text"] == "Alpha-Release 3 ist signiert.", "later request is unaffected")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_unknown_source()
|
||||
test_disallowed_role()
|
||||
test_wrong_source_excerpt()
|
||||
test_invented_excerpt()
|
||||
test_all_selected_rehydrates_full_text()
|
||||
test_excerpt_cannot_drop_remainder()
|
||||
test_model_cannot_overwrite_canonical()
|
||||
test_annotations_are_not_verified_content()
|
||||
test_policies_differ_without_domain_logic()
|
||||
test_parallel_requests_are_isolated()
|
||||
print("provenance tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Writing profile store with facets, governance, and evidence. AI drafts are never a positive style source."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from db import get_db, row_to_dict
|
||||
@@ -44,6 +46,13 @@ BRIEF_IMPORT_EXCERPTS = 6
|
||||
INITIAL_BUILD_SOURCES = 40
|
||||
EXCERPT_CHARS = 1200
|
||||
TRAIT_EXCERPT_CHARS = 280
|
||||
TASK_BRIEF_MAX_CHARS = 4000
|
||||
TASK_BRIEF_CORE_CHARS = 800
|
||||
TASK_BRIEF_FACET_CHARS = 700
|
||||
TASK_BRIEF_TRAIT_CHARS = 280
|
||||
TASK_BRIEF_EXEMPLAR_CHARS = 220
|
||||
TASK_BRIEF_MAX_TRAITS = 6
|
||||
TASK_BRIEF_MAX_EXEMPLARS = 2
|
||||
DIALOGUE_KIND = "dialogue_style"
|
||||
MAX_DIALOGUE_MESSAGES = 8
|
||||
MAX_DIALOGUE_CHARS = 500
|
||||
@@ -980,6 +989,104 @@ def _load_ranked_sources(profile_id: str) -> dict[str, list[dict]]:
|
||||
return {"journal_entry": entries, "imported_text": imports, "dialogue_style": dialogue}
|
||||
|
||||
|
||||
def _source_content_digest(item: dict) -> str:
|
||||
body = re.sub(r"\s+", " ", plain_text(item.get("body") or "")).strip().lower()
|
||||
return hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _dedupe_brief_sources(items: list[dict], *, seen_keys: set[tuple[str, str]] | None = None) -> list[dict]:
|
||||
"""Display-only. Originals remain in writing_profile_sources."""
|
||||
seen_entry: set[str] = set()
|
||||
seen = seen_keys if seen_keys is not None else set()
|
||||
result = []
|
||||
for item in items:
|
||||
entry_id = (item.get("entry_id") or "").strip()
|
||||
if entry_id:
|
||||
if entry_id in seen_entry:
|
||||
continue
|
||||
seen_entry.add(entry_id)
|
||||
when = (item.get("occurred_at") or "")[:10]
|
||||
digest = _source_content_digest(item)
|
||||
key = (when, digest)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def _clip_field(text: str, limit: int) -> str:
|
||||
raw = (text or "").strip()
|
||||
if limit <= 0 or not raw:
|
||||
return ""
|
||||
if len(raw) <= limit:
|
||||
return raw
|
||||
cut = raw[:limit]
|
||||
if not raw[limit].isspace():
|
||||
sp = max(cut.rfind(" "), cut.rfind("\n"), cut.rfind("\t"))
|
||||
if sp >= max(1, limit // 3):
|
||||
cut = cut[:sp]
|
||||
else:
|
||||
return ""
|
||||
return cut.rstrip(" \t\n,;:-")
|
||||
|
||||
|
||||
def _norm_overlap(left: str, right: str) -> float:
|
||||
a = re.sub(r"\s+", " ", (left or "").strip().lower())
|
||||
b = re.sub(r"\s+", " ", (right or "").strip().lower())
|
||||
if not a or not b:
|
||||
return 0.0
|
||||
if a == b or a in b or b in a:
|
||||
return min(len(a), len(b)) / max(len(a), len(b))
|
||||
return 0.0
|
||||
|
||||
|
||||
def _trait_recency_stamp(item: dict) -> str | None:
|
||||
stamps = [item.get("updated"), item.get("observed_to"), item.get("observed_from")]
|
||||
for ref in item.get("exemplars") or []:
|
||||
stamps.append(ref.get("occurred_at"))
|
||||
stamps = [stamp for stamp in stamps if stamp]
|
||||
return max(stamps) if stamps else None
|
||||
|
||||
|
||||
def _select_task_traits(traits: list[dict]) -> list[dict]:
|
||||
relevant = [
|
||||
item
|
||||
for item in traits
|
||||
if item.get("status") == "active"
|
||||
and (item.get("statement") or "")
|
||||
and item.get("facet_key") in {"autobiographical_journal", "core", ""}
|
||||
]
|
||||
if not relevant:
|
||||
relevant = [item for item in traits if item.get("status") == "active" and (item.get("statement") or "")]
|
||||
current = []
|
||||
long_term = []
|
||||
for item in relevant:
|
||||
stamp = _trait_recency_stamp(item)
|
||||
role = recency_role(stamp)
|
||||
weight = recency_weight(stamp)
|
||||
scored = (item, weight, role)
|
||||
if role == "current_expression":
|
||||
current.append(scored)
|
||||
else:
|
||||
long_term.append(scored)
|
||||
current.sort(key=lambda row: (-row[1], row[0].get("slug") or ""))
|
||||
long_term.sort(key=lambda row: (-row[1], row[0].get("slug") or ""))
|
||||
picked: list[dict] = []
|
||||
for item, _, _ in current[:4]:
|
||||
picked.append(item)
|
||||
for item, _, _ in long_term:
|
||||
if len(picked) >= TASK_BRIEF_MAX_TRAITS:
|
||||
break
|
||||
picked.append(item)
|
||||
if len(picked) < TASK_BRIEF_MAX_TRAITS:
|
||||
for item, _, _ in current[4:]:
|
||||
if len(picked) >= TASK_BRIEF_MAX_TRAITS:
|
||||
break
|
||||
picked.append(item)
|
||||
return picked[:TASK_BRIEF_MAX_TRAITS]
|
||||
|
||||
|
||||
def _source_line(item: dict) -> str:
|
||||
when = (item.get("occurred_at") or "")[:10]
|
||||
role = recency_role(item.get("occurred_at") or item.get("created"))
|
||||
@@ -989,8 +1096,9 @@ def _source_line(item: dict) -> str:
|
||||
|
||||
def _assemble_brief(profile_id: str) -> None:
|
||||
ranked = _load_ranked_sources(profile_id)
|
||||
entries = ranked["journal_entry"][:BRIEF_ENTRY_EXCERPTS]
|
||||
imports = ranked["imported_text"][:BRIEF_IMPORT_EXCERPTS]
|
||||
display_keys: set[tuple[str, str]] = set()
|
||||
entries = _dedupe_brief_sources(ranked["journal_entry"], seen_keys=display_keys)[:BRIEF_ENTRY_EXCERPTS]
|
||||
imports = _dedupe_brief_sources(ranked["imported_text"], seen_keys=display_keys)[:BRIEF_IMPORT_EXCERPTS]
|
||||
dialogue = ranked["dialogue_style"]
|
||||
profile_row = ensure_profile(profile_id)
|
||||
with get_db() as conn:
|
||||
@@ -1067,46 +1175,91 @@ def compile_task_brief(profile_id: str, task: str = "journal_generate") -> str:
|
||||
from writing_profile_schema import is_meta_style_text
|
||||
|
||||
profile = get_profile(profile_id)
|
||||
parts = []
|
||||
core = profile.get("core") or {}
|
||||
if core.get("value"):
|
||||
parts.append("Core: " + (core.get("value") or ""))
|
||||
traits = [item for item in profile.get("traits") or [] if item.get("status") == "active"]
|
||||
if task == "journal_generate":
|
||||
facet = next(
|
||||
(
|
||||
item
|
||||
for item in profile.get("facets") or []
|
||||
if item.get("facet_key") == "autobiographical_journal" and (item.get("value") or "")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if facet:
|
||||
parts.append("Autobiografisches Journaling (Facet-Delta): " + (facet.get("value") or ""))
|
||||
relevant = [
|
||||
if task != "journal_generate":
|
||||
core = (profile.get("core") or {}).get("value") or ""
|
||||
return _clip_field(core, TASK_BRIEF_MAX_CHARS)
|
||||
|
||||
parts: list[str] = []
|
||||
core_text = ((profile.get("core") or {}).get("value") or "").strip()
|
||||
if core_text:
|
||||
clipped = _clip_field(core_text, TASK_BRIEF_CORE_CHARS)
|
||||
if clipped:
|
||||
parts.append("Core: " + clipped)
|
||||
facet = next(
|
||||
(
|
||||
item
|
||||
for item in traits
|
||||
if item.get("facet_key") in {"autobiographical_journal", "core", ""} and (item.get("statement") or "")
|
||||
]
|
||||
if not relevant:
|
||||
relevant = [item for item in traits if item.get("statement")]
|
||||
for item in relevant[:6]:
|
||||
line = f"- {item.get('label') or item.get('slug')}: {item.get('statement')}"
|
||||
exemplars = [
|
||||
ref.get("excerpt")
|
||||
for ref in item.get("exemplars") or []
|
||||
if ref.get("excerpt") and not is_meta_style_text(ref.get("excerpt"))
|
||||
]
|
||||
if exemplars:
|
||||
line += " Beispiel: " + exemplars[0][:TRAIT_EXCERPT_CHARS]
|
||||
parts.append(line)
|
||||
if parts:
|
||||
return "\n".join(parts).strip()
|
||||
for item in profile.get("facets") or []
|
||||
if item.get("facet_key") == "autobiographical_journal" and (item.get("value") or "")
|
||||
),
|
||||
None,
|
||||
)
|
||||
facet_text = ((facet or {}).get("value") or "").strip()
|
||||
if facet_text and _norm_overlap(facet_text, core_text) < 0.8:
|
||||
clipped = _clip_field(facet_text, TASK_BRIEF_FACET_CHARS)
|
||||
if clipped:
|
||||
parts.append("Autobiografisches Journaling (Facet-Delta): " + clipped)
|
||||
selected = _select_task_traits(profile.get("traits") or [])
|
||||
used_statements = [core_text, facet_text]
|
||||
for item in selected:
|
||||
statement = (item.get("statement") or "").strip()
|
||||
if any(_norm_overlap(statement, previous) >= 0.85 for previous in used_statements if previous):
|
||||
continue
|
||||
clipped = _clip_field(statement, TASK_BRIEF_TRAIT_CHARS)
|
||||
if not clipped:
|
||||
continue
|
||||
parts.append(f"- {item.get('label') or item.get('slug')}: {clipped}")
|
||||
used_statements.append(statement)
|
||||
exemplars: list[tuple[float, str]] = []
|
||||
seen_ex = set()
|
||||
for item in selected:
|
||||
for ref in item.get("exemplars") or []:
|
||||
excerpt = (ref.get("excerpt") or "").strip()
|
||||
if not excerpt or is_meta_style_text(excerpt):
|
||||
continue
|
||||
digest = hashlib.sha256(re.sub(r"\s+", " ", excerpt.lower()).encode("utf-8")).hexdigest()
|
||||
if digest in seen_ex:
|
||||
continue
|
||||
seen_ex.add(digest)
|
||||
clipped = _clip_field(excerpt, TASK_BRIEF_EXEMPLAR_CHARS)
|
||||
if not clipped:
|
||||
continue
|
||||
weight = recency_weight(ref.get("occurred_at") or _trait_recency_stamp(item))
|
||||
exemplars.append((weight, clipped))
|
||||
exemplars.sort(key=lambda row: -row[0])
|
||||
chosen = []
|
||||
for _, excerpt in exemplars:
|
||||
if any(_norm_overlap(excerpt, prev) >= 0.8 for prev in chosen):
|
||||
continue
|
||||
chosen.append(excerpt)
|
||||
if len(chosen) >= TASK_BRIEF_MAX_EXEMPLARS:
|
||||
break
|
||||
if chosen:
|
||||
parts.append("Repräsentative Exemplare:")
|
||||
for excerpt in chosen:
|
||||
parts.append("- " + excerpt)
|
||||
brief = "\n".join(part for part in parts if part).strip()
|
||||
if len(brief) > TASK_BRIEF_MAX_CHARS:
|
||||
without_examples = [part for part in parts if not part.startswith("Repräsentative Exemplare") and not (
|
||||
part.startswith("- ") and chosen and any(part == "- " + excerpt for excerpt in chosen)
|
||||
)]
|
||||
brief = "\n".join(without_examples).strip()
|
||||
parts = without_examples
|
||||
while len(brief) > TASK_BRIEF_MAX_CHARS and parts:
|
||||
last = parts.pop()
|
||||
remain = TASK_BRIEF_MAX_CHARS - (len("\n".join(parts)) + (1 if parts else 0))
|
||||
clipped = _clip_field(last, remain)
|
||||
if clipped:
|
||||
parts.append(clipped)
|
||||
brief = "\n".join(parts).strip()
|
||||
break
|
||||
brief = "\n".join(parts).strip()
|
||||
if brief:
|
||||
return brief
|
||||
ranked = _load_ranked_sources(profile_id)
|
||||
snippets = (ranked.get("journal_entry") or [])[:2] + (ranked.get("imported_text") or [])[:1]
|
||||
lines = []
|
||||
for item in snippets:
|
||||
body = plain_text(item.get("body") or "")[:280]
|
||||
body = _clip_field(plain_text(item.get("body") or ""), 280)
|
||||
if body and not is_meta_style_text(body):
|
||||
lines.append(body)
|
||||
return "\n\n".join(lines).strip()
|
||||
|
||||
Reference in new issue
Block a user