This commit is contained in:
Lars committed 2026-08-29 11:04:34 +02:00
1 parent 4f37991ad1
commit be11f82f84
39 files changed
+2490 -239

No files matched your search

+10
View File
@@ -66,6 +66,13 @@ _JOURNAL_DAY_COLUMNS = {
_JOURNAL_DRAFT_COLUMNS = {
"generation_snapshot": "TEXT NOT NULL DEFAULT '{}'",
}
_DEBUG_RUN_COLUMNS = {
"space_id": "TEXT",
"journal_day_id": "TEXT",
"space_title": "TEXT NOT NULL DEFAULT ''",
"calendar_date": "TEXT NOT NULL DEFAULT ''",
"conversation_title": "TEXT NOT NULL DEFAULT ''",
}
_IDENTITY_MAPPING_COLUMNS = {
"canonical_label": "TEXT NOT NULL DEFAULT ''",
"entity_type": "TEXT NOT NULL DEFAULT 'PERSON'",
@@ -485,6 +492,9 @@ def init_db() -> None:
_mark(conn, "015_journal_generation_settings")
_mark(conn, "016_generation_instruction_fragments")
_mark(conn, "017_generation_guidelines")
_mark(conn, "018_debug_runs")
_ensure_columns(conn, "debug_runs", _DEBUG_RUN_COLUMNS)
_mark(conn, "019_debug_run_placement")
from writing_profile_store import bootstrap_from_existing
bootstrap_from_existing()
+837
View File
@@ -0,0 +1,837 @@
"""Opt-in local development debug persist. Mapping labels and secrets stay out."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any
import uuid
from db import get_db, row_to_dict
from version import APP_VERSION, BUILD_DATE
SETTING_KEY = "debug.persist_traces"
EXPORT_KIND = "kansho.debug_export"
EXPORT_VERSION = 2
UNASSIGNED_SPACE_TITLE = "Ohne Space"
UNASSIGNED_DAY_LABEL = "Ohne Tag"
FORBIDDEN_KEYS = frozenset(
{
"local_label",
"demask_label",
"mapping_table",
"replacements",
"restore_by_token",
"password",
"password_hash",
"key",
"api_key",
"secret",
"token",
}
)
LAYER_FOR_PURPOSE = {
"dialogue_turn": "dialogzug",
"journal_generate": "journalentwurf",
"local_source_artifact": "journalquellen",
"journal_reconstruct": "journalrekonstruktion",
"profile_review": "profilreview",
}
def _now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def persist_enabled() -> bool:
with get_db() as conn:
row = row_to_dict(
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
)
return (row or {}).get("value") == "1"
def set_persist_enabled(enabled: bool) -> dict:
value = "1" if enabled else "0"
with get_db() as conn:
conn.execute(
"""
INSERT INTO app_settings (key, value, updated)
VALUES (?, ?, datetime('now'))
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
""",
(SETTING_KEY, value),
)
return settings_payload()
def _is_admin(profile_id: str) -> bool:
with get_db() as conn:
row = row_to_dict(
conn.execute("SELECT role FROM profiles WHERE id = ?", (profile_id,)).fetchone()
)
return (row or {}).get("role") == "admin"
def should_persist(profile_id: str | None) -> bool:
if not profile_id:
return False
return persist_enabled() and _is_admin(profile_id)
def sanitize(value: Any) -> Any:
if isinstance(value, dict):
cleaned = {}
for key, item in value.items():
if key in FORBIDDEN_KEYS:
continue
cleaned[key] = sanitize(item)
return cleaned
if isinstance(value, list):
return [sanitize(item) for item in value]
if isinstance(value, tuple):
return [sanitize(item) for item in value]
return value
def settings_payload(profile_id: str | None = None) -> dict:
enabled = persist_enabled()
count = 0
if profile_id:
with get_db() as conn:
count = conn.execute(
"SELECT COUNT(*) AS n FROM debug_runs WHERE profile_id = ?",
(profile_id,),
).fetchone()["n"]
return {
"persist_enabled": enabled,
"run_count": count,
"note": (
"Entwicklungsoption. Speichert die Admin-Testspur lokal je Dialogschritt. "
"Zuordnung Space / Tag / Gespräch. Mapping-Tabelle, Secrets und Klarnamen-Labels "
"bleiben ausgeschlossen. Nur Aktionen des Admin-Profils. Default aus."
),
}
def resolve_placement(
profile_id: str,
*,
conversation_id: str | None = None,
journal_day_id: str | None = None,
space_id: str | None = None,
) -> dict:
place = {
"space_id": space_id or None,
"journal_day_id": journal_day_id or None,
"conversation_id": conversation_id or None,
"space_title": "",
"calendar_date": "",
"conversation_title": "",
}
with get_db() as conn:
if conversation_id:
conv = row_to_dict(
conn.execute(
"""
SELECT id, title, space_id, journal_day_id
FROM conversations
WHERE id = ? AND profile_id = ?
""",
(conversation_id, profile_id),
).fetchone()
)
if conv:
place["conversation_id"] = conv["id"]
place["conversation_title"] = conv.get("title") or ""
place["space_id"] = place["space_id"] or conv.get("space_id")
place["journal_day_id"] = place["journal_day_id"] or conv.get("journal_day_id")
if place["journal_day_id"]:
day = row_to_dict(
conn.execute(
"""
SELECT id, calendar_date, space_id
FROM journal_days
WHERE id = ? AND profile_id = ?
""",
(place["journal_day_id"], profile_id),
).fetchone()
)
if day:
place["calendar_date"] = day.get("calendar_date") or ""
place["space_id"] = place["space_id"] or day.get("space_id")
if place["space_id"]:
space = row_to_dict(
conn.execute(
"""
SELECT id, title FROM spaces
WHERE id = ? AND profile_id = ?
""",
(place["space_id"], profile_id),
).fetchone()
)
if space:
place["space_title"] = space.get("title") or ""
return place
def persist_step(
profile_id: str,
*,
purpose: str,
status: str = "ok",
layer: str = "",
subject_type: str = "",
subject_id: str = "",
conversation_id: str | None = None,
message_id: str | None = None,
journal_day_id: str | None = None,
space_id: str | None = None,
decision: dict | None = None,
trace: dict | None = None,
extra: dict | None = None,
) -> str | None:
if not should_persist(profile_id):
return None
if not journal_day_id and subject_type == "journal_day" and subject_id:
journal_day_id = subject_id
place = resolve_placement(
profile_id,
conversation_id=conversation_id,
journal_day_id=journal_day_id,
space_id=space_id,
)
payload = sanitize(
{
"decision": decision,
"trace": trace,
**(extra or {}),
}
)
run_id = str(uuid.uuid4())
with get_db() as conn:
conn.execute(
"""
INSERT INTO debug_runs (
id, profile_id, purpose, layer, status,
subject_type, subject_id, conversation_id, message_id,
space_id, journal_day_id, space_title, calendar_date, conversation_title,
payload_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
run_id,
profile_id,
purpose,
layer or LAYER_FOR_PURPOSE.get(purpose, ""),
status,
subject_type or "",
subject_id or "",
place["conversation_id"],
message_id,
place["space_id"],
place["journal_day_id"],
place["space_title"],
place["calendar_date"],
place["conversation_title"],
json.dumps(payload, ensure_ascii=False, default=str),
),
)
return run_id
def persist_engine_error(
profile_id: str,
*,
purpose: str,
exc,
layer: str = "",
subject_type: str = "",
subject_id: str = "",
conversation_id: str | None = None,
journal_day_id: str | None = None,
extra: dict | None = None,
) -> str | None:
diag = dict(getattr(exc, "diagnostics", None) or {})
merged = dict(extra or {})
merged.setdefault("code", getattr(exc, "code", None))
merged.setdefault("message", getattr(exc, "message", None) or str(exc))
if diag.get("log") is not None:
merged.setdefault("run_log", diag.get("log"))
return persist_step(
profile_id,
purpose=purpose,
status="error",
layer=layer,
subject_type=subject_type,
subject_id=subject_id,
conversation_id=conversation_id,
journal_day_id=journal_day_id,
trace=diag.get("trace"),
extra=merged,
)
def _parse_payload(raw: str | None) -> dict:
if not raw:
return {}
try:
data = json.loads(raw)
except json.JSONDecodeError:
return {}
return data if isinstance(data, dict) else {}
def _row_public(row: dict, *, include_payload: bool = True) -> dict:
payload = _parse_payload(row.get("payload_json"))
item = {
"id": row["id"],
"created": row["created"],
"purpose": row["purpose"],
"layer": row["layer"],
"status": row["status"],
"subject_type": row.get("subject_type") or "",
"subject_id": row.get("subject_id") or "",
"conversation_id": row.get("conversation_id"),
"message_id": row.get("message_id"),
"space_id": row.get("space_id"),
"journal_day_id": row.get("journal_day_id"),
"space_title": row.get("space_title") or "",
"calendar_date": row.get("calendar_date") or "",
"conversation_title": row.get("conversation_title") or "",
"decision": payload.get("decision"),
"summary": _summary(row, payload),
}
if include_payload:
item["payload"] = payload
item["trace"] = payload.get("trace")
item["run_log"] = payload.get("run_log") or ((payload.get("trace") or {}).get("log"))
return item
def _summary(row: dict, payload: dict) -> str:
decision = payload.get("decision") or {}
label = decision.get("label") or decision.get("operation") or ""
purpose = row.get("purpose") or ""
status = row.get("status") or ""
parts = [purpose]
if label:
parts.append(str(label))
if status and status != "ok":
parts.append(status)
code = payload.get("code")
if code:
parts.append(str(code))
return " · ".join(parts)
def list_runs(profile_id: str, *, limit: int = 200) -> list[dict]:
capped = max(1, min(int(limit or 200), 500))
with get_db() as conn:
rows = [
dict(item)
for item in conn.execute(
"""
SELECT * FROM debug_runs
WHERE profile_id = ?
ORDER BY created DESC, id DESC
LIMIT ?
""",
(profile_id, capped),
).fetchall()
]
return [_row_public(row, include_payload=False) for row in rows]
def latest_run(
profile_id: str,
*,
journal_day_id: str | None = None,
purpose: str | None = None,
) -> dict | None:
clauses = ["profile_id = ?"]
args: list[Any] = [profile_id]
if journal_day_id:
clauses.append("journal_day_id = ?")
args.append(journal_day_id)
if purpose:
clauses.append("purpose = ?")
args.append(purpose)
with get_db() as conn:
row = row_to_dict(
conn.execute(
f"""
SELECT * FROM debug_runs
WHERE {' AND '.join(clauses)}
ORDER BY created DESC, id DESC
LIMIT 1
""",
args,
).fetchone()
)
if not row:
return None
items = _fill_placement(profile_id, [_row_public(row, include_payload=True)])
return items[0] if items else None
def get_run(profile_id: str, run_id: str) -> dict | None:
with get_db() as conn:
row = row_to_dict(
conn.execute(
"SELECT * FROM debug_runs WHERE id = ? AND profile_id = ?",
(run_id, profile_id),
).fetchone()
)
if not row:
return None
items = _fill_placement(profile_id, [_row_public(row, include_payload=True)])
return items[0]
def delete_run(profile_id: str, run_id: str) -> bool:
with get_db() as conn:
cursor = conn.execute(
"DELETE FROM debug_runs WHERE id = ? AND profile_id = ?",
(run_id, profile_id),
)
return cursor.rowcount > 0
def clear_runs(profile_id: str) -> int:
with get_db() as conn:
cursor = conn.execute("DELETE FROM debug_runs WHERE profile_id = ?", (profile_id,))
return cursor.rowcount
def list_runs_full(profile_id: str) -> list[dict]:
with get_db() as conn:
rows = [
dict(item)
for item in conn.execute(
"""
SELECT * FROM debug_runs
WHERE profile_id = ?
ORDER BY created ASC, id ASC
""",
(profile_id,),
).fetchall()
]
items = [_row_public(row, include_payload=True) for row in rows]
return _fill_placement(profile_id, items)
def _fill_placement(profile_id: str, items: list[dict]) -> list[dict]:
missing_conv = [
item.get("conversation_id")
for item in items
if item.get("conversation_id") and not item.get("space_id") and not item.get("journal_day_id")
]
missing_days = [
item.get("journal_day_id")
for item in items
if item.get("journal_day_id") and not item.get("space_id")
]
conv_ids = list(dict.fromkeys(missing_conv))
day_ids = list(dict.fromkeys(missing_days))
convs: dict[str, dict] = {}
days: dict[str, dict] = {}
spaces: dict[str, dict] = {}
with get_db() as conn:
if conv_ids:
marks = ",".join("?" * len(conv_ids))
for row in conn.execute(
f"SELECT id, title, space_id, journal_day_id FROM conversations WHERE profile_id = ? AND id IN ({marks})",
[profile_id, *conv_ids],
):
convs[row["id"]] = dict(row)
if row["journal_day_id"]:
day_ids.append(row["journal_day_id"])
day_ids = list(dict.fromkeys(day_ids))
if day_ids:
marks = ",".join("?" * len(day_ids))
for row in conn.execute(
f"SELECT id, calendar_date, space_id FROM journal_days WHERE profile_id = ? AND id IN ({marks})",
[profile_id, *day_ids],
):
days[row["id"]] = dict(row)
space_ids = [
item
for item in (
[row.get("space_id") for row in convs.values()]
+ [row.get("space_id") for row in days.values()]
+ [item.get("space_id") for item in items]
)
if item
]
space_ids = list(dict.fromkeys(space_ids))
if space_ids:
marks = ",".join("?" * len(space_ids))
for row in conn.execute(
f"SELECT id, title FROM spaces WHERE profile_id = ? AND id IN ({marks})",
[profile_id, *space_ids],
):
spaces[row["id"]] = dict(row)
for item in items:
conv = convs.get(item.get("conversation_id") or "")
if conv:
item["conversation_title"] = item.get("conversation_title") or conv.get("title") or ""
item["space_id"] = item.get("space_id") or conv.get("space_id")
item["journal_day_id"] = item.get("journal_day_id") or conv.get("journal_day_id")
day = days.get(item.get("journal_day_id") or "")
if day:
item["calendar_date"] = item.get("calendar_date") or day.get("calendar_date") or ""
item["space_id"] = item.get("space_id") or day.get("space_id")
space = spaces.get(item.get("space_id") or "")
if space:
item["space_title"] = item.get("space_title") or space.get("title") or ""
return items
def _export_run(item: dict) -> dict:
return {
"id": item["id"],
"created": item["created"],
"purpose": item["purpose"],
"layer": item["layer"],
"status": item["status"],
"subject_type": item["subject_type"],
"subject_id": item["subject_id"],
"conversation_id": item.get("conversation_id"),
"message_id": item.get("message_id"),
"space_id": item.get("space_id"),
"journal_day_id": item.get("journal_day_id"),
"space_title": item.get("space_title") or "",
"calendar_date": item.get("calendar_date") or "",
"conversation_title": item.get("conversation_title") or "",
"summary": item["summary"],
**(item.get("payload") or {}),
}
def _step_summary(item: dict) -> dict:
return {
"id": item["id"],
"created": item["created"],
"purpose": item["purpose"],
"layer": item["layer"],
"status": item["status"],
"summary": item["summary"],
"message_id": item.get("message_id"),
}
def _filter_runs(
runs: list[dict],
*,
conversation_id: str | None = None,
journal_day_id: str | None = None,
space_id: str | None = None,
purpose: str | None = None,
) -> list[dict]:
selected = runs
if conversation_id:
selected = [
item
for item in selected
if item.get("conversation_id") == conversation_id and item.get("purpose") != "journal_generate"
]
elif journal_day_id:
selected = [item for item in selected if item.get("journal_day_id") == journal_day_id]
if space_id:
selected = [item for item in selected if item.get("space_id") == space_id]
if purpose:
selected = [item for item in selected if item.get("purpose") == purpose]
return selected
def group_tree(runs: list[dict], *, include_payload: bool = False) -> list[dict]:
spaces_map: dict[str, dict] = {}
space_order: list[str] = []
for run in runs:
exported = _export_run(run) if include_payload else _step_summary(run)
sid = run.get("space_id") or ""
space_key = sid or "_unassigned"
if space_key not in spaces_map:
spaces_map[space_key] = {
"id": sid or None,
"title": run.get("space_title") or (UNASSIGNED_SPACE_TITLE if not sid else ""),
"days_map": {},
"day_order": [],
}
space_order.append(space_key)
space = spaces_map[space_key]
if run.get("space_title") and not space["title"]:
space["title"] = run.get("space_title")
did = run.get("journal_day_id") or ""
day_key = did or "_unassigned"
if day_key not in space["days_map"]:
space["days_map"][day_key] = {
"id": did or None,
"calendar_date": run.get("calendar_date") or (UNASSIGNED_DAY_LABEL if not did else ""),
"conversations_map": {},
"conversation_order": [],
"generates": [],
}
space["day_order"].append(day_key)
day = space["days_map"][day_key]
if run.get("calendar_date") and (not day["calendar_date"] or day["calendar_date"] == UNASSIGNED_DAY_LABEL):
day["calendar_date"] = run.get("calendar_date")
if run.get("purpose") == "journal_generate":
day["generates"].append(exported)
continue
cid = run.get("conversation_id") or ""
conv_key = cid or "_none"
if conv_key not in day["conversations_map"]:
day["conversations_map"][conv_key] = {
"id": cid or None,
"title": run.get("conversation_title") or "Gespräch",
"steps": [],
}
day["conversation_order"].append(conv_key)
conv = day["conversations_map"][conv_key]
if run.get("conversation_title"):
conv["title"] = run.get("conversation_title")
conv["steps"].append(exported)
spaces = []
for space_key in space_order:
space = spaces_map[space_key]
days = []
for day_key in space["day_order"]:
day = space["days_map"][day_key]
conversations = []
for conv_key in day["conversation_order"]:
conv = day["conversations_map"][conv_key]
conversations.append(
{
"id": conv["id"],
"title": conv["title"] or "Gespräch",
"step_count": len(conv["steps"]),
"steps": conv["steps"],
}
)
days.append(
{
"id": day["id"],
"calendar_date": day["calendar_date"] or UNASSIGNED_DAY_LABEL,
"conversations": conversations,
"generates": day["generates"],
}
)
spaces.append({"id": space["id"], "title": space["title"] or UNASSIGNED_SPACE_TITLE, "days": days})
return spaces
def debug_tree(profile_id: str) -> dict:
runs = list_runs_full(profile_id)
return {"spaces": group_tree(runs, include_payload=False), "run_count": len(runs)}
def export_document(
profile_id: str,
*,
conversation_id: str | None = None,
journal_day_id: str | None = None,
purpose: str | None = None,
) -> dict:
runs = list_runs_full(profile_id)
selected = _filter_runs(
runs,
conversation_id=conversation_id,
journal_day_id=journal_day_id,
purpose=purpose,
)
scope = "all"
if purpose == "journal_generate":
scope = "journal_generate"
elif conversation_id:
scope = "conversation"
elif journal_day_id:
scope = "day"
spaces = group_tree(selected, include_payload=True)
first_space = spaces[0] if spaces else None
first_day = ((first_space or {}).get("days") or [None])[0] if first_space else None
first_conv = ((first_day or {}).get("conversations") or [None])[0] if first_day else None
return {
"kind": EXPORT_KIND,
"version": EXPORT_VERSION,
"exported_at": _now(),
"app_version": APP_VERSION,
"build_date": BUILD_DATE,
"scope": scope,
"note": (
"Lokale Entwicklungsdiagnose, gegliedert nach Space / Tag / Gespräch / Schritt. "
"Mapping-Tabelle und Secrets sind ausgeschlossen. "
"Enthält Klartext-Vorlagen (intern) und maskierten Egress des eigenen Admin-Profils. "
"Nicht an Provider senden, außer bewusst zur lokalen Auswertung."
),
"persist_enabled": persist_enabled(),
"run_count": len(selected),
"space": {"id": (first_space or {}).get("id"), "title": (first_space or {}).get("title")} if scope != "all" else None,
"day": (
{"id": (first_day or {}).get("id"), "calendar_date": (first_day or {}).get("calendar_date")}
if scope in {"conversation", "day", "journal_generate"}
else None
),
"conversation": (
{"id": (first_conv or {}).get("id"), "title": (first_conv or {}).get("title")}
if scope == "conversation"
else None
),
"spaces": spaces,
"runs": [_export_run(item) for item in selected],
}
def _fence(text: str | None) -> str:
body = (text or "").replace("```", "'''")
return f"```\n{body}\n```" if body.strip() else "_leer_"
def _kv(title: str, value: Any) -> str:
if value is None or value == "":
return f"- {title}: nicht verfügbar"
if isinstance(value, (dict, list)):
return f"- {title}: `{json.dumps(value, ensure_ascii=False)}`"
return f"- {title}: {value}"
def _render_trace_markdown(trace: dict | None, heading: str) -> list[str]:
if not trace:
return [f"### {heading}", "", "_keine Testspur_", ""]
lines = [f"### {heading}", ""]
lines.append(_kv("Zweck", trace.get("purpose")))
lines.append(_kv("Schicht", trace.get("layer")))
lines.append(_kv("Prompt", trace.get("prompt_slug")))
lines.append(_kv("Prompt-Revision", trace.get("prompt_revision")))
lines.append(_kv("Provider", trace.get("provider")))
lines.append(_kv("Modell", trace.get("model")))
lines.append(_kv("Detect", trace.get("detect_note") or trace.get("detect_provider")))
lines.append(_kv("Detect-Modell", trace.get("detect_model")))
lines.append(_kv("Abbruchgrund", trace.get("abort_reason")))
lines.append(_kv("Modelltext übernommen", trace.get("model_text_accepted")))
budget = trace.get("budget") or {}
if budget:
lines.append(_kv("Prompt-Tokens", budget.get("prompt_tokens")))
lines.append(_kv("Completion-Tokens", budget.get("completion_tokens")))
lines.append(_kv("Kosten", budget.get("cost")))
lines.append(_kv("Generate-ms", budget.get("generate_ms") or trace.get("generate_ms")))
lines.extend(["", "#### Intern (Klartext-Vorlage, nicht gesendet)", "", _fence(trace.get("intern")), ""])
lines.extend(["#### Egress (maskiert, gesendet)", "", _fence(trace.get("egress")), ""])
lines.extend(["#### Maskierung, Eingabe", "", _fence(trace.get("mask_input")), ""])
lines.extend(["#### Modellantwort roh", "", _fence(trace.get("raw")), ""])
lines.extend(["#### Lokal demaskiert", "", _fence(trace.get("reply")), ""])
stored = "\n\n".join(part for part in (trace.get("stored_title"), trace.get("stored_body")) if part)
if stored:
lines.extend(["#### Gespeicherter Entwurf", "", _fence(stored), ""])
if trace.get("source_preview"):
lines.extend(["#### Lokale Quellenansicht", "", _fence(trace.get("source_preview")), ""])
return lines
def render_markdown(document: dict) -> str:
lines = [
"# Kanshō Debug-Export",
"",
_kv("kind", document.get("kind")),
_kv("version", document.get("version")),
_kv("scope", document.get("scope")),
_kv("exported_at", document.get("exported_at")),
_kv("app_version", document.get("app_version")),
_kv("build_date", document.get("build_date")),
_kv("run_count", document.get("run_count")),
_kv("persist_enabled", document.get("persist_enabled")),
"",
document.get("note") or "",
"",
"Gliederung: Space / Tag / Gespräch / Schritt. Die Schritte eines Gesprächs sind chronologisch.",
"",
]
if (document.get("space") or {}).get("title"):
lines.append(_kv("Space", (document.get("space") or {}).get("title")))
if (document.get("day") or {}).get("calendar_date"):
lines.append(_kv("Tag", (document.get("day") or {}).get("calendar_date")))
if (document.get("conversation") or {}).get("title"):
lines.append(_kv("Gespräch", (document.get("conversation") or {}).get("title")))
lines.append("")
spaces = document.get("spaces") or []
if spaces:
for space in spaces:
lines.extend(["---", "", f"# Space: {space.get('title') or UNASSIGNED_SPACE_TITLE}", ""])
for day in space.get("days") or []:
lines.extend([f"## Tag: {day.get('calendar_date') or UNASSIGNED_DAY_LABEL}", ""])
for conv in day.get("conversations") or []:
lines.extend([f"### Gespräch: {conv.get('title') or 'Gespräch'}", ""])
for index, run in enumerate(conv.get("steps") or [], start=1):
lines.extend(_render_run_markdown(run, index))
for index, run in enumerate(day.get("generates") or [], start=1):
lines.extend([f"### Journalentwurf {index}", ""])
lines.extend(_render_run_markdown(run, index))
else:
for index, run in enumerate(document.get("runs") or [], start=1):
lines.extend(_render_run_markdown(run, index))
lines.append("")
return "\n".join(lines)
def _render_run_markdown(run: dict, index: int) -> list[str]:
created = run.get("created") or ""
purpose = run.get("purpose") or ""
status = run.get("status") or ""
lines = [
"---",
"",
f"#### {index}. {purpose} · {created} · {status}",
"",
_kv("id", run.get("id")),
_kv("layer", run.get("layer")),
_kv("conversation_id", run.get("conversation_id")),
_kv("code", run.get("code")),
]
decision = run.get("decision")
if decision:
lines.append(_kv("Operation", decision.get("label") or decision.get("operation")))
if decision.get("guard"):
lines.append(_kv("Guard", decision.get("guard")))
user = run.get("user") or {}
assistant = run.get("assistant") or {}
if user.get("body"):
lines.extend(["", "##### Nutzer", "", _fence(user.get("body")), ""])
if assistant.get("body"):
lines.extend(["##### Antwort", "", _fence(assistant.get("body")), ""])
traces = []
if run.get("call_traces"):
traces = list(run.get("call_traces") or [])
elif run.get("trace"):
nested = (run.get("trace") or {}).get("stages")
traces = list(nested) if nested else [run.get("trace")]
if not traces and run.get("trace"):
traces = [run.get("trace")]
for offset, stage in enumerate(traces):
title = f"Schritt {offset + 1}"
if isinstance(stage, dict) and stage.get("purpose"):
title = f"Schritt {offset + 1}: {stage.get('purpose')}"
lines.extend(_render_trace_markdown(stage if isinstance(stage, dict) else None, title))
log = run.get("run_log") or ((run.get("trace") or {}).get("log") if isinstance(run.get("trace"), dict) else None)
if log:
lines.extend(
[
"##### Kompaktes Laufprotokoll (ohne Promptkörper)",
"",
_fence(json.dumps(log, ensure_ascii=False, indent=2)),
"",
]
)
return lines
def export_filename(fmt: str, document: dict | None = None) -> str:
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
suffix = "md" if fmt == "markdown" else "json"
scope = (document or {}).get("scope") or "all"
if scope == "conversation":
return f"kansho-debug-conversation-{stamp}.{suffix}"
if scope == "journal_generate":
return f"kansho-debug-journal-{stamp}.{suffix}"
if scope == "day":
return f"kansho-debug-day-{stamp}.{suffix}"
return f"kansho-debug-{stamp}.{suffix}"
+38 -1
View File
@@ -6,6 +6,7 @@ import re
from context_builder import assemble_text, build_internal_context, is_closing_turn
from conversation_signals import infer_signals
from debug_store import persist_engine_error, persist_step
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
from engine import EngineError, execute_prompt, load_active_prompt
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES
@@ -317,6 +318,7 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
result = None
impulse = ""
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
call_traces: list[dict] = []
try:
while calls < 2:
result = execute_prompt(
@@ -327,6 +329,8 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
context=assembled,
)
calls += 1
if result.get("trace"):
call_traces.append(result.get("trace"))
content = (result.get("content") or "").strip()
if not content:
raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.")
@@ -348,7 +352,22 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
"guard": "local_fallback",
}
except EngineError as exc:
if result and result.get("trace") and result.get("trace") not in call_traces:
call_traces.append(result.get("trace"))
if exc.code != "response_validation_failed" and exc.code not in DETECT_DIALOGUE_FALLBACK_CODES:
persist_engine_error(
profile_id,
purpose="dialogue_turn",
exc=exc,
subject_type="conversation",
subject_id=conversation_id,
conversation_id=conversation_id,
extra={
"calls": calls,
"call_traces": call_traces,
"user": {"id": user.get("id"), "body": user.get("body")},
},
)
raise
impulse = local_hold(last_user_text(assembled))
decision = {
@@ -370,6 +389,24 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
)
remember_dialogue_style(profile_id)
consider_dialogue(profile_id, body)
trace = result.get("trace") if result else None
persist_step(
profile_id,
purpose="dialogue_turn",
status="ok",
subject_type="conversation",
subject_id=conversation_id,
conversation_id=conversation_id,
message_id=assistant.get("id"),
decision=decision,
trace=trace,
extra={
"calls": calls,
"call_traces": call_traces,
"user": {"id": user.get("id"), "body": user.get("body")},
"assistant": {"id": assistant.get("id"), "body": assistant.get("body")},
},
)
return {
"conversation": get_conversation(profile_id, conversation_id),
"user": user,
@@ -377,5 +414,5 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
"calls": calls,
"messages": list_messages(profile_id, conversation_id),
"decision": decision,
"trace": result.get("trace") if result else None,
"trace": trace,
}
+13 -2
View File
@@ -18,6 +18,7 @@ from identity_store import (
ENTITY_TYPES,
confirmed_match_labels,
is_maskable_label,
is_registry_maskable_label,
list_confirmed_identities,
masking_rows_from_confirmed,
normalize_entity_type,
@@ -109,6 +110,8 @@ class DetectionStats:
entity_counts: dict[str, int] = field(default_factory=dict)
confirmed_registry_hits: int = 0
request_local_hits: int = 0
confirmed_registry_applied: bool = False
semantic_identity_guaranteed: bool = False
detect_calls: int = 0
prompt_tokens: int = 0
completion_tokens: int = 0
@@ -128,7 +131,9 @@ class DetectionStats:
"full_detection_coverage": self.full_detection_coverage,
"entity_counts": dict(self.entity_counts),
"confirmed_registry_hits": self.confirmed_registry_hits,
"confirmed_registry_applied": self.confirmed_registry_applied,
"request_local_hits": self.request_local_hits,
"semantic_identity_guaranteed": False,
"detect_calls": self.detect_calls,
"detect_prompt_tokens": self.prompt_tokens,
"detect_completion_tokens": self.completion_tokens,
@@ -547,12 +552,16 @@ def _assign_request_tokens(
def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str | None) -> tuple[list[dict], int]:
if not profile_id:
return mappings, 0
already_labels = {(item.get("local_label") or "").casefold() for item in mappings if item.get("start") is None}
already_labels = {
(item.get("local_label") or "").casefold()
for item in mappings
if item.get("start") is None and item.get("source") == "confirmed_registry"
}
extra: list[dict] = []
hits = 0
for row in masking_rows_from_confirmed(profile_id):
label = row.get("local_label") or ""
if not label or label.casefold() in already_labels:
if not label or not is_registry_maskable_label(label) or label.casefold() in already_labels:
continue
if not re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text or "", re.IGNORECASE):
continue
@@ -680,6 +689,8 @@ def detect_personal_egress(profile_id: str | None, source_text: str) -> Detectio
mappings, extra_confirmed = _merge_confirmed_safety_net(source_text or "", mappings, profile_id)
stats.request_local_hits = request_local
stats.confirmed_registry_hits = confirmed_from_spans + extra_confirmed
stats.confirmed_registry_applied = bool(profile_id)
stats.semantic_identity_guaranteed = False
counts: dict[str, int] = {kind: 0 for kind in ENTITY_TYPES}
for item in mappings:
kind = (item.get("entity_type") or "PERSON").upper()
+23 -7
View File
@@ -73,6 +73,22 @@ def is_maskable_label(label: str) -> bool:
return True
def is_registry_maskable_label(label: str) -> bool:
"""Confirmed names may be shorter than the detect gate. Not a semantic word list."""
core = (label or "").strip()
if len(core) < 2:
return False
if core.lower() in STRUCTURAL_UNMASKABLE:
return False
if TIME_RE.search(core):
return False
if re.fullmatch(r"user|assistant|self", core, re.IGNORECASE):
return False
if not re.search(r"[A-Za-zÀ-žÄÖÜäöüß]", core):
return False
return True
def _token_suffix(token: str) -> str:
raw = (token or "").upper()
return raw.split(":")[-1] if ":" in token else raw
@@ -111,7 +127,7 @@ def parse_aliases(raw) -> list[str]:
result: list[str] = []
for item in items:
label = (item or "").strip()
if not label or label.casefold() in seen or not is_maskable_label(label):
if not label or label.casefold() in seen or not is_registry_maskable_label(label):
continue
seen.add(label.casefold())
result.append(label)
@@ -189,7 +205,7 @@ def mapping_spellings(item: dict) -> list[str]:
def add(raw) -> None:
label = (raw or "").strip()
if not label or label.casefold() in seen or not is_maskable_label(label):
if not label or label.casefold() in seen or not is_registry_maskable_label(label):
return
seen.add(label.casefold())
result.append(label)
@@ -214,7 +230,7 @@ def masking_rows_from_confirmed(profile_id: str) -> list[dict]:
if not token or not canonical:
continue
for label in confirmed_match_labels(item):
if not is_maskable_label(label):
if not is_registry_maskable_label(label):
continue
rows.append(
{
@@ -274,7 +290,7 @@ def confirm_identity(
identity_id: str | None = None,
) -> dict:
label = normalize_label(canonical_label) or (canonical_label or "").strip()
if not label or not is_maskable_label(label):
if not label or not is_registry_maskable_label(label):
raise ValueError("empty_label")
kind = normalize_entity_type(entity_type)
if not kind:
@@ -360,7 +376,7 @@ def update_identity(
label = found["canonical_label"]
if canonical_label is not None:
label = normalize_label(canonical_label) or canonical_label.strip()
if not label or not is_maskable_label(label):
if not label or not is_registry_maskable_label(label):
raise ValueError("empty_label")
kind = found["entity_type"]
if entity_type is not None:
@@ -525,7 +541,7 @@ def purge_unmaskable(profile_id: str) -> int:
).fetchall()
for row in rows:
label = row["canonical_label"] or row["local_label"]
if is_maskable_label(label):
if is_registry_maskable_label(label):
continue
conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],))
removed += 1
@@ -551,7 +567,7 @@ def coalesce_mappings(profile_id: str) -> int:
groups: dict[str, list[dict]] = {}
for row in rows:
core = normalize_label(row.get("canonical_label") or row.get("local_label") or "")
if not core or not is_maskable_label(core):
if not core or not is_registry_maskable_label(core):
conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],))
merged += 1
continue
+56
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import re
from context_builder import assemble_text, build_internal_context
from debug_store import persist_engine_error, persist_step
from dialogue_store import StoreError, list_conversations_for_day, list_messages
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
from journal_policy import require_explicit_generate, source_conversation_ids
@@ -356,6 +357,57 @@ def generate_draft(
explicit: bool = True,
generation_selection: dict | None = None,
remember_generation_selection: bool = False,
) -> dict:
try:
draft = _generate_draft(
profile_id,
journal_day_id,
conversation_ids=conversation_ids,
include_existing=include_existing,
explicit=explicit,
generation_selection=generation_selection,
remember_generation_selection=remember_generation_selection,
)
except EngineError as exc:
persist_engine_error(
profile_id,
purpose="journal_generate",
exc=exc,
subject_type="journal_day",
subject_id=journal_day_id,
journal_day_id=journal_day_id,
extra={"conversation_ids": conversation_ids or []},
)
raise
persist_step(
profile_id,
purpose="journal_generate",
status="ok",
subject_type="journal_day",
subject_id=journal_day_id,
journal_day_id=journal_day_id,
conversation_id=(conversation_ids[0] if conversation_ids else None),
decision=None,
trace=draft.get("trace"),
extra={
"draft_id": draft.get("id"),
"run_log": draft.get("run_log"),
"conversation_ids": conversation_ids or [],
"stored_title": draft.get("title"),
"stored_body": draft.get("body"),
},
)
return draft
def _generate_draft(
profile_id: str,
journal_day_id: str,
conversation_ids: list[str] | None = None,
include_existing: bool = False,
explicit: bool = True,
generation_selection: dict | None = None,
remember_generation_selection: bool = False,
) -> dict:
require_explicit_generate(explicit)
day = get_day(profile_id, journal_day_id)
@@ -602,6 +654,7 @@ def generate_draft(
person_labels=person_labels,
source="model",
)
stored = f"{title}\n\n{body}".strip() if title else (body or "").strip()
before_entries = {item["id"]: item.get("current_version_id") for item in current_entries(profile_id, journal_day_id)}
model = (
((narrate_result.get("diagnostics") or {}).get("actual_model"))
@@ -647,6 +700,9 @@ def generate_draft(
"lexical_similarity": lexical_similarity("\n".join(source_user), body),
"incomplete_syntax": incomplete_syntax_markers(body),
"generation_selection": policy_meta,
"stored_title": title,
"stored_body": body,
"reply": stored,
},
)
draft["run_log"] = run_log
+30 -1
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import re
from context_builder import assemble_text, build_internal_context
from debug_store import persist_engine_error, persist_step
from dialogue_store import StoreError, append_message, get_conversation, list_conversations_for_day, list_messages
from dialogue_turn import OPERATIONS, needs_repair, parse_turn_payload, repair_note
from engine import EngineError, execute_prompt, load_active_prompt
@@ -162,12 +163,31 @@ def format_opening_hint(facts: dict) -> str:
def _store_opening(profile_id: str, conversation_id: str, impulse: str, extra: dict) -> dict:
assistant = append_message(profile_id, conversation_id, impulse, role="assistant")
return {
payload = {
"conversation": get_conversation(profile_id, conversation_id),
"assistant": assistant,
"messages": list_messages(profile_id, conversation_id),
**extra,
}
persist_step(
profile_id,
purpose="dialogue_turn",
status="ok",
layer="dialogzug",
subject_type="conversation",
subject_id=conversation_id,
conversation_id=conversation_id,
message_id=assistant.get("id"),
decision=extra.get("decision"),
trace=extra.get("trace"),
extra={
"kind": extra.get("kind"),
"calls": extra.get("calls"),
"opened": extra.get("opened"),
"assistant": {"id": assistant.get("id"), "body": assistant.get("body")},
},
)
return payload
def start_journal_opening(profile_id: str, conversation_id: str) -> dict:
@@ -262,6 +282,15 @@ def start_journal_opening(profile_id: str, conversation_id: str) -> dict:
}
except EngineError as exc:
if exc.code != "response_validation_failed" and exc.code not in DETECT_DIALOGUE_FALLBACK_CODES:
persist_engine_error(
profile_id,
purpose="dialogue_turn",
exc=exc,
subject_type="conversation",
subject_id=conversation_id,
conversation_id=conversation_id,
extra={"kind": "opening_failed", "calls": calls},
)
raise
return _store_opening(
profile_id,
+6 -123
View File
@@ -1,4 +1,4 @@
"""Local journal shape after the model. Does not invent content."""
"""Local journal shape after the model. Does not invent or rewrite content."""
from __future__ import annotations
import re
@@ -10,26 +10,6 @@ NEW_PHASE = re.compile(
r"Als\s+(?:wir|ich)\b)",
re.I,
)
_LETTER = r"A-Za-zÄÖÜäöüß"
_WORD = re.compile(rf"[{_LETTER}]+")
DATIVE_PREPS = {
"mit", "bei", "von", "zu", "nach", "aus", "außer", "gegenüber", "gemäß",
}
NOM_CUES = {
"dass", "daß", "ob", "weil", "wenn", "als", "während", "obwohl", "damit",
"nachdem", "bevor", "sodass", "sofern",
}
THING_GOVERNORS = {
"esse", "essen", "isst", "aß", "aßest", "gegessen",
"koche", "kochen", "kochte", "gekocht",
"trinke", "trinken", "trank", "getrunken",
"kaufe", "kaufen", "kaufte", "gekauft",
"bestelle", "bestellen", "bestellte",
"hole", "holen", "holte",
}
FEM_HINTS = {"sie", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres"}
MASC_HINTS = {"er", "ihn", "ihm", "seine", "seinem", "seinen", "seiner", "seines"}
KEEP_NAMES_PER_PARAGRAPH = 2
def _strip_roles(text: str) -> str:
@@ -55,6 +35,7 @@ def is_verbatim_join(body: str, user_bodies: list[str]) -> bool:
def paragraphize(body: str) -> str:
"""Optional local preview helper. Must not rewrite an accepted model draft."""
text = (body or "").strip()
if "\n\n" in text:
return text
@@ -75,99 +56,6 @@ def paragraphize(body: str) -> str:
return "\n\n".join(chunks)
def _words_before(text: str, index: int, n: int = 4) -> list[str]:
return [word.lower() for word in _WORD.findall(text[:index])[-n:]]
def _words_after(text: str, index: int, n: int = 3) -> list[str]:
return [word.lower() for word in _WORD.findall(text[index:])[:n]]
def _is_thing_mention(text: str, start: int, end: int) -> bool:
prev = _words_before(text, start)
nxt = _words_after(text, end)
return any(word in THING_GOVERNORS for word in prev + nxt)
def _sentence_start(text: str, index: int) -> bool:
prefix = text[:index].rstrip()
return not prefix or prefix[-1] in ".!?\n"
def infer_person_gender(label: str, source_texts: list[str]) -> str | None:
blob = " ".join(source_texts or [])
if not label or not blob:
return None
pattern = re.compile(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", re.I)
first = pattern.search(blob)
if not first:
return None
after = [word.lower() for word in _WORD.findall(blob[first.start():])]
fem = sum(1 for word in after if word in FEM_HINTS)
masc = sum(1 for word in after if word in MASC_HINTS)
if fem > masc:
return "f"
if masc > fem:
return "m"
return None
def _pronoun(gender: str, prev: str, sentence_start: bool) -> str:
dative = prev in DATIVE_PREPS
if gender == "f":
form = "ihr" if dative else "sie"
elif sentence_start or prev in NOM_CUES:
form = "er"
elif dative:
form = "ihm"
else:
form = "ihn"
if sentence_start:
return form[:1].upper() + form[1:]
return form
def naturalize_person_mentions(body: str, labels: list[str], source_texts: list[str]) -> str:
"""After demask: keep the name once or twice per passage, then pronouns. No new facts."""
text = body or ""
people = []
for label in sorted({item.strip() for item in (labels or []) if (item or "").strip()}, key=len, reverse=True):
gender = infer_person_gender(label, source_texts)
if gender:
people.append((label, gender))
if not people:
return text
paragraphs = re.split(r"(\n\n+)", text)
out: list[str] = []
for block in paragraphs:
if not block or block.startswith("\n"):
out.append(block)
continue
piece = block
for label, gender in people:
pattern = re.compile(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", re.I)
matches = [
match
for match in pattern.finditer(piece)
if not _is_thing_mention(piece, match.start(), match.end())
]
if len(matches) <= KEEP_NAMES_PER_PARAGRAPH:
continue
replacements: list[tuple[int, int, str]] = []
kept = 0
for match in matches:
kept += 1
if kept <= KEEP_NAMES_PER_PARAGRAPH:
continue
prev = (_words_before(piece, match.start(), 1) or [""])[0]
form = _pronoun(gender, prev, _sentence_start(piece, match.start()))
replacements.append((match.start(), match.end(), form))
for start, end, form in reversed(replacements):
piece = piece[:start] + form + piece[end:]
out.append(piece)
return "".join(out)
def shape_journal(
title: str,
body: str,
@@ -176,15 +64,10 @@ def shape_journal(
*,
source: str = "model",
) -> tuple[str, str]:
"""Local post-shape. Never replaces accepted model wording with source turns."""
turns = _turns(user_bodies)
text = (body or "").strip()
"""Safe technical trim only. Never replaces names, pronouns, or accepted wording."""
del user_bodies, person_labels
heading = (title or "").strip()
text = _strip_roles(text)
if source != "fallback":
text = _strip_roles(body or "").strip()
if source == "fallback" and "\n\n" not in text:
text = paragraphize(text)
elif "\n\n" not in text:
text = paragraphize(text)
if person_labels:
text = naturalize_person_mentions(text, person_labels, turns)
return heading, text
+35 -11
View File
@@ -9,7 +9,7 @@ import re
import time
from entity_detect import DetectError, detect_personal_egress, reset_detect_test_hooks, user_detect_message
from identity_store import KINSHIP, is_maskable_label, mapping_spellings
from identity_store import KINSHIP, is_maskable_label, is_registry_maskable_label, mapping_spellings
from journal_reconstruct import claim_texts, fake_reconstruction, is_dialogue_role_line
from prompt_budget import (
ERROR_PROVIDER_CONTEXT_LENGTH,
@@ -75,7 +75,9 @@ COMPACT_DIAGNOSTIC_KEYS = (
"chunks_ok",
"entity_counts",
"confirmed_registry_hits",
"confirmed_registry_applied",
"request_local_hits",
"semantic_identity_guaranteed",
"detect_calls",
"detect_prompt_tokens",
"detect_completion_tokens",
@@ -338,7 +340,7 @@ def is_identity_mention(text: str, start: int, end: int, token: str) -> bool:
def identity_occurrence_count(text: str, label: str, token: str) -> int:
"""Public, intent-neutral count of identity mentions for one mapping."""
if not label or not is_maskable_label(label):
if not label or not is_registry_maskable_label(label):
return 0
return sum(
1
@@ -373,12 +375,22 @@ def _identity_occurrence_count(text: str, label: str, token: str) -> int:
return identity_occurrence_count(text, label, token)
def _mapping_label_allowed(item: dict) -> bool:
label = (item.get("local_label") or "").strip()
token = (item.get("token") or "").strip()
if not label or not token:
return False
if (item.get("source") or "") == "confirmed_registry":
return is_registry_maskable_label(label)
return is_maskable_label(label)
def _active_replacements(text: str, mappings: list[dict]) -> tuple[ActiveReplacement, ...]:
found: list[ActiveReplacement] = []
for item in mappings or []:
label = (item.get("local_label") or "").strip()
token = (item.get("token") or "").strip()
if not label or not token or not is_maskable_label(label):
if not _mapping_label_allowed(item):
continue
count = _identity_occurrence_count(text, label, token)
if count <= 0:
@@ -447,7 +459,7 @@ def _mask_body(text: str, mappings: list[dict]) -> str:
for item in sorted(mappings, key=lambda row: len(row.get("local_label") or ""), reverse=True):
label = (item.get("local_label") or "").strip()
token = (item.get("token") or "").strip()
if not label or not token or not is_maskable_label(label):
if not _mapping_label_allowed(item):
continue
placeholder = _placeholder_of(token)
pattern = _label_pattern(label)
@@ -572,7 +584,7 @@ def mask_prompt(rendered: str, mappings: list[dict], purpose: str) -> MaskingMan
for item in mappings or []:
label = (item.get("local_label") or "").strip()
token = (item.get("token") or "").strip()
if not label or not token or not is_maskable_label(label):
if not _mapping_label_allowed(item):
continue
has_span_fields = item.get("start") is not None and item.get("end") is not None
grounded = _ground_mapping_span(text, item) if has_span_fields else None
@@ -683,8 +695,13 @@ def _demask(text: str, manifest: MaskingManifest | None) -> str:
return PLACEHOLDER_RE.sub(repl, result)
def _visible_response(content: str, manifest: MaskingManifest | None) -> str:
"""Stored output: original model text with only emitted placeholders rehydrated."""
return _demask((content or "").strip(), manifest)
def _normalize_active_cleartext(text: str, manifest: MaskingManifest | None) -> tuple[str, int]:
"""Map active identity cleartext back to the request token. Local only, no retry."""
"""Map active identity cleartext back to the request token. Check copy only; never stored."""
if not manifest or not manifest.replacements:
return text or "", 0
hits: list[tuple[int, int, str]] = []
@@ -711,7 +728,7 @@ def _normalize_active_cleartext(text: str, manifest: MaskingManifest | None) ->
def _process_response(content: str, manifest: MaskingManifest | None = None) -> tuple[str, dict[str, Any]]:
"""Integrity check after the model reply. Does not undo egress and does not retry."""
"""Integrity check after the model reply. Tokenized copy is not the visible output."""
text = (content or "").strip()
if not text:
raise PrivacyGatewayError("empty_provider_response", "Der Provider lieferte keine Antwort.")
@@ -725,7 +742,7 @@ def _process_response(content: str, manifest: MaskingManifest | None = None) ->
def _validate_response(content: str, manifest: MaskingManifest | None = None) -> str:
"""Compatibility wrapper: normalize active cleartext, then return tokenized text."""
"""Compatibility wrapper: tokenized check copy. Not the stored user-facing text."""
text, _meta = _process_response(content, manifest)
return text
@@ -910,6 +927,8 @@ def complete(request: GatewayRequest) -> GatewayResult:
detect_calls=detect_stats.get("detect_calls"),
request_local_hits=detect_stats.get("request_local_hits"),
confirmed_registry_hits=detect_stats.get("confirmed_registry_hits"),
confirmed_registry_applied=detect_stats.get("confirmed_registry_applied"),
semantic_identity_guaranteed=False,
entity_counts=detect_stats.get("entity_counts"),
detect_ms=detect_stats.get("detect_ms"),
detect_prompt_tokens=detect_stats.get("detect_prompt_tokens"),
@@ -982,7 +1001,9 @@ def complete(request: GatewayRequest) -> GatewayResult:
"chunks_ok": detect_stats.get("chunks_ok"),
"entity_counts": detect_stats.get("entity_counts"),
"confirmed_registry_hits": detect_stats.get("confirmed_registry_hits"),
"confirmed_registry_applied": detect_stats.get("confirmed_registry_applied"),
"request_local_hits": detect_stats.get("request_local_hits"),
"semantic_identity_guaranteed": False,
"detect_calls": detect_stats.get("detect_calls"),
"detect_ms": detect_stats.get("detect_ms"),
"generate_called": False,
@@ -1048,7 +1069,6 @@ def complete(request: GatewayRequest) -> GatewayResult:
)
usages: list[dict] = []
raw = ""
validated = ""
chat = None
try:
_log_event(
@@ -1067,7 +1087,7 @@ def complete(request: GatewayRequest) -> GatewayResult:
request_trace["generate_calls"] = 1
_log_event(events, started, "model_result", attempt=1, **_usage_bits(chat.usage))
raw = chat.content
validated, process_meta = _process_response(raw, manifest)
_tokenized, process_meta = _process_response(raw, manifest)
diagnostics["response_validation"] = process_meta["response_validation"]
diagnostics["response_normalization"] = process_meta["response_normalization"]
diagnostics["normalized_cleartext_count"] = process_meta["normalized_cleartext_count"]
@@ -1124,7 +1144,7 @@ def complete(request: GatewayRequest) -> GatewayResult:
sum_usages(usages) or chat.usage,
chat.model or (config.model if config else None),
)
result.content = _demask(validated, manifest)
result.content = _visible_response(raw, manifest)
result.allowed = True
result.local_identities = local_identities
result.diagnostics = compact_diagnostics(diagnostics)
@@ -1169,8 +1189,12 @@ def public_trace(trace: dict | None) -> dict | None:
"chunks_ok": trace.get("chunks_ok"),
"entity_counts": trace.get("entity_counts"),
"confirmed_registry_hits": trace.get("confirmed_registry_hits"),
"confirmed_registry_applied": trace.get("confirmed_registry_applied"),
"request_local_hits": trace.get("request_local_hits"),
"semantic_identity_guaranteed": False,
"detect_calls": trace.get("detect_calls"),
"stored_title": trace.get("stored_title"),
"stored_body": trace.get("stored_body"),
"detect_ms": trace.get("detect_ms"),
"generate_ms": (trace.get("budget") or {}).get("generate_ms") if isinstance(trace.get("budget"), dict) else trace.get("generate_ms"),
"generate_called": trace.get("generate_called"),
+34 -14
View File
@@ -13,6 +13,7 @@ import uuid
from datetime import datetime, timedelta, timezone
from db import get_db, row_to_dict
from debug_store import persist_engine_error, persist_step
from dialogue_store import StoreError
from journal_body import plain_text
from journal_store import list_versions
@@ -867,7 +868,7 @@ def open_api_review(profile_id: str, *, mode: str | None = None) -> dict:
def run_api_review(profile_id: str, review_id: str | None = None, *, mode: str | None = None) -> dict:
from engine import execute_prompt, load_active_prompt
from engine import EngineError, execute_prompt, load_active_prompt
if review_id:
with get_db() as conn:
@@ -887,25 +888,44 @@ def run_api_review(profile_id: str, review_id: str | None = None, *, mode: str |
review_id = opened["id"]
package = opened["package"]
prompt = load_active_prompt("mvp.profile_review")
result = execute_prompt(
prompt,
profile_id,
purpose="profile_review",
data_class="B",
context={
"review_package": json.dumps(package, ensure_ascii=False, indent=2),
"source_text": "\n".join(
[item.get("excerpt") or "" for item in package.get("evidences") or []]
+ [item.get("excerpt") or "" for item in package.get("corpus") or []]
),
},
)
try:
result = execute_prompt(
prompt,
profile_id,
purpose="profile_review",
data_class="B",
context={
"review_package": json.dumps(package, ensure_ascii=False, indent=2),
"source_text": "\n".join(
[item.get("excerpt") or "" for item in package.get("evidences") or []]
+ [item.get("excerpt") or "" for item in package.get("corpus") or []]
),
},
)
except EngineError as exc:
persist_engine_error(
profile_id,
purpose="profile_review",
exc=exc,
subject_type="writing_profile_review",
subject_id=review_id or "",
)
raise
parsed = parse_result(result.get("content") or "")
if not parsed.get("mode"):
parsed["mode"] = package.get("mode") or ""
staged = _stage_proposal(profile_id, parsed, review_id=review_id, package=package)
staged["trace"] = result.get("trace")
staged["review_id"] = review_id
persist_step(
profile_id,
purpose="profile_review",
status="ok",
subject_type="writing_profile_review",
subject_id=review_id or "",
trace=result.get("trace"),
extra={"review_id": review_id},
)
return staged
+117 -1
View File
@@ -1,8 +1,23 @@
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import Response
from pydantic import BaseModel, Field
import json
from auth import require_admin_dep
from db import get_db
from debug_store import (
clear_runs,
debug_tree,
delete_run,
export_document,
export_filename,
get_run,
latest_run,
list_runs,
render_markdown,
set_persist_enabled,
settings_payload,
)
from dialogue_store import StoreError, get_conversation, inventory, list_conversations, list_derived_for_conversation, list_messages
from identity_store import (
confirm_identity,
@@ -29,6 +44,7 @@ def admin_health(session: dict = Depends(require_admin_dep)):
features = conn.execute("SELECT COUNT(*) AS n FROM features").fetchone()["n"]
dialogue = inventory()
providers = public_status()
debug = settings_payload(session["profile_id"])
return {
"ok": True,
"service": "kansho",
@@ -40,6 +56,8 @@ def admin_health(session: dict = Depends(require_admin_dep)):
"prompts": prompts,
"features": features,
"dialogue": dialogue,
"debug_runs": debug["run_count"],
"debug_persist_enabled": debug["persist_enabled"],
"providers": {
role["role"]: {"ready": role["ready"], "key_present": role["key_present"]}
for role in providers["roles"]
@@ -74,6 +92,104 @@ def admin_conversation(conversation_id: str, session: dict = Depends(require_adm
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
class DebugToggle(BaseModel):
persist_enabled: bool
@router.get("/debug")
def admin_debug(session: dict = Depends(require_admin_dep)):
return settings_payload(session["profile_id"])
@router.put("/debug")
def admin_update_debug(body: DebugToggle, session: dict = Depends(require_admin_dep)):
payload = set_persist_enabled(body.persist_enabled)
payload["run_count"] = settings_payload(session["profile_id"])["run_count"]
return payload
@router.get("/debug/tree")
def admin_debug_tree(session: dict = Depends(require_admin_dep)):
return debug_tree(session["profile_id"])
@router.get("/debug/runs")
def admin_debug_runs(limit: int = Query(default=200, ge=1, le=500), session: dict = Depends(require_admin_dep)):
return {"runs": list_runs(session["profile_id"], limit=limit)}
@router.get("/debug/latest")
def admin_debug_latest(
journal_day_id: str | None = None,
purpose: str | None = None,
session: dict = Depends(require_admin_dep),
):
return {
"run": latest_run(
session["profile_id"],
journal_day_id=journal_day_id,
purpose=purpose,
)
}
@router.get("/debug/export")
def admin_debug_export(
format: str = Query(default="json"),
conversation_id: str | None = None,
journal_day_id: str | None = None,
purpose: str | None = None,
session: dict = Depends(require_admin_dep),
):
fmt = (format or "json").strip().lower()
if fmt not in {"json", "markdown"}:
raise HTTPException(
status_code=400,
detail={"code": "invalid_debug_export_format", "message": "Format ist json oder markdown."},
)
document = export_document(
session["profile_id"],
conversation_id=conversation_id,
journal_day_id=journal_day_id,
purpose=purpose,
)
filename = export_filename(fmt, document)
if fmt == "markdown":
body = render_markdown(document)
return Response(
content=body.encode("utf-8"),
media_type="text/markdown; charset=utf-8",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
body = json.dumps(document, ensure_ascii=False, indent=2)
return Response(
content=body.encode("utf-8"),
media_type="application/json; charset=utf-8",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.post("/debug/runs/clear")
def admin_clear_debug_runs(session: dict = Depends(require_admin_dep)):
deleted = clear_runs(session["profile_id"])
return {"ok": True, "deleted": deleted}
@router.get("/debug/runs/{run_id}")
def admin_debug_run(run_id: str, session: dict = Depends(require_admin_dep)):
item = get_run(session["profile_id"], run_id)
if not item:
raise HTTPException(status_code=404, detail={"code": "debug_run_missing", "message": "Debug-Lauf fehlt."})
return item
@router.delete("/debug/runs/{run_id}")
def admin_delete_debug_run(run_id: str, session: dict = Depends(require_admin_dep)):
if not delete_run(session["profile_id"], run_id):
raise HTTPException(status_code=404, detail={"code": "debug_run_missing", "message": "Debug-Lauf fehlt."})
return {"ok": True}
class ProviderUpdate(BaseModel):
name: str = ""
url: str = ""
+31
View File
@@ -494,3 +494,34 @@ CREATE TABLE IF NOT EXISTS generation_guidelines (
created TEXT NOT NULL DEFAULT (datetime('now')),
updated TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS app_settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL DEFAULT '',
updated TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS debug_runs (
id TEXT PRIMARY KEY,
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
purpose TEXT NOT NULL,
layer TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'ok',
subject_type TEXT NOT NULL DEFAULT '',
subject_id TEXT NOT NULL DEFAULT '',
conversation_id TEXT,
message_id TEXT,
space_id TEXT,
journal_day_id TEXT,
space_title TEXT NOT NULL DEFAULT '',
calendar_date TEXT NOT NULL DEFAULT '',
conversation_title TEXT NOT NULL DEFAULT '',
payload_json TEXT NOT NULL DEFAULT '{}',
created TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_debug_runs_profile_created
ON debug_runs (profile_id, created DESC);
CREATE INDEX IF NOT EXISTS idx_debug_runs_conversation
ON debug_runs (profile_id, conversation_id, created);
+309
View File
@@ -0,0 +1,309 @@
"""Opt-in debug persist for the admin test spur. Run from backend/: python tests/test_debug_persist.py"""
from __future__ import annotations
import json
import os
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
REPO = ROOT.parent
sys.path.insert(0, str(ROOT))
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-test.sqlite")
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-media")
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
os.environ["KANSHO_FAKE_DETECT"] = "1"
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
from fastapi.testclient import TestClient
from identity_store import remember_mapping
from main import app
from privacy_gateway import reset_debug
def expect(ok: bool, message: str) -> None:
if not ok:
raise SystemExit(f"FAIL: {message}")
print(f"OK {message}")
def header(token: str) -> dict:
return {"X-Auth-Token": token}
def open_space_day(client, headers, title="Alltag", date="2026-08-28"):
space = client.post("/api/journal/spaces", headers=headers, json={"title": title})
expect(space.status_code == 200, f"create space {space.status_code}")
day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": date},
)
expect(day.status_code == 200, f"open day {day.status_code}")
return space.json(), day.json()
def blob(data) -> str:
return json.dumps(data, ensure_ascii=False)
def main() -> None:
reset_debug()
ui = (REPO / "frontend" / "src" / "pages" / "AdminDebugPage.jsx").read_text(encoding="utf-8")
expect("Debug persistieren" in ui, "admin page has the persist toggle")
expect("Space / Tag / Gespräche" in ui, "admin page uses space/day/conversation structure")
expect("Gespräch-JSON" in ui, "admin page can download one conversation")
expect("JSON herunterladen" not in ui, "admin page no longer uses a flat download label as primary")
nav = (REPO / "frontend" / "src" / "config" / "adminNav.js").read_text(encoding="utf-8")
expect("/admin/debug" in nav, "admin nav lists debug")
dialogue_ui = (REPO / "frontend" / "src" / "pages" / "DialoguePage.jsx").read_text(encoding="utf-8")
expect("CallTrace" not in dialogue_ui, "dialogue page no longer shows the live test spur")
expect("Debug herunterladen" in dialogue_ui, "dialogue page can download the conversation debug log")
day_ui = (REPO / "frontend" / "src" / "pages" / "JournalDayPage.jsx").read_text(encoding="utf-8")
expect("import CallTrace" not in day_ui, "journal day does not render CallTrace under the thread")
expect("Debug herunterladen" in day_ui, "journal day can download the conversation debug log")
editor_ui = (REPO / "frontend" / "src" / "pages" / "JournalEditorPage.jsx").read_text(encoding="utf-8")
expect("import CallTrace" in editor_ui, "journal editor still shows the generate trace")
expect("<h2>Verlauf</h2>" in editor_ui, "journal editor names the generate trace")
expect("Debug herunterladen" in editor_ui, "journal editor can download the generate debug log")
expect("purpose=journal_generate" in editor_ui, "journal editor downloads the generate scope")
with TestClient(app) as client:
setup = client.post(
"/api/auth/setup",
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
)
expect(setup.status_code == 200, f"setup {setup.status_code}")
headers = header(setup.json()["token"])
profile_id = setup.json()["profile_id"]
other = client.post(
"/api/users",
headers=headers,
json={"email": "ute@example.test", "name": "Ute", "password": "user-pass", "role": "user"},
)
expect(other.status_code == 200, "second profile")
other_headers = header(
client.post("/api/auth/login", json={"email": "ute@example.test", "password": "user-pass"}).json()["token"]
)
settings = client.get("/api/admin/debug", headers=headers)
expect(settings.status_code == 200, "admin can read debug settings")
expect(settings.json()["persist_enabled"] is False, "persist is off by default")
expect(settings.json()["run_count"] == 0, "no runs at start")
denied = client.get("/api/admin/debug", headers=other_headers)
expect(denied.status_code == 403, "non-admin cannot read debug settings")
space, day = open_space_day(client, headers)
day_id = day["day"]["id"]
conv = client.post(
f"/api/journal/days/{day_id}/conversations",
headers=headers,
json={"title": "Gespräch"},
)
expect(conv.status_code == 200, f"start conversation {conv.status_code}")
conv_id = conv.json()["id"]
reset_debug()
turn = client.post(
f"/api/journal/conversations/{conv_id}/turn",
headers=headers,
json={"body": "Heute war der Markt voll. Ich habe Kirschen gekauft."},
)
expect(turn.status_code == 200, f"turn while persist off {turn.status_code}")
listed = client.get("/api/admin/debug/runs", headers=headers)
expect(listed.json()["runs"] == [], "off means no persisted steps")
enabled = client.put("/api/admin/debug", headers=headers, json={"persist_enabled": True})
expect(enabled.status_code == 200, "enable persist")
expect(enabled.json()["persist_enabled"] is True, "persist flag stored")
remember_mapping(profile_id, "Anna")
reset_debug()
first = client.post(
f"/api/journal/conversations/{conv_id}/turn",
headers=headers,
json={"body": "Anna war am Stand. Die Kirschen waren reif."},
)
expect(first.status_code == 200, f"first persisted turn {first.status_code}")
reset_debug()
second = client.post(
f"/api/journal/conversations/{conv_id}/turn",
headers=headers,
json={"body": "Später noch der Hafen, ungefähr 18:00 Uhr."},
)
expect(second.status_code == 200, f"second persisted turn {second.status_code}")
listed = client.get("/api/admin/debug/runs", headers=headers)
expect(listed.status_code == 200, "list runs")
runs = listed.json()["runs"]
expect(len(runs) >= 2, "each turn is stored")
expect(all(item.get("purpose") == "dialogue_turn" for item in runs[:2]), "listed steps are dialogue turns")
newest = client.get(f"/api/admin/debug/runs/{runs[0]['id']}", headers=headers)
expect(newest.status_code == 200, "read run detail")
payload = newest.json().get("payload") or {}
stored = blob(payload)
expect("local_label" not in stored, "mapping labels are not persisted")
expect("mapping_table" not in stored, "mapping table is not persisted")
expect("replacements" not in stored, "replacement objects are not persisted")
details = [
client.get(f"/api/admin/debug/runs/{item['id']}", headers=headers).json()
for item in runs[:2]
]
expect(
any("Hafen" in blob(item.get("payload") or {}) for item in details),
"intern keeps the step text",
)
hafen = next(item.get("payload") or {} for item in details if "Hafen" in blob(item.get("payload") or {}))
expect((hafen.get("trace") or {}).get("egress") or (hafen.get("call_traces")), "egress of the step is stored")
expect((hafen.get("user") or {}).get("body"), "user line of the step is stored")
expect((hafen.get("assistant") or {}).get("body"), "assistant line of the step is stored")
expect(hafen.get("call_traces") or (hafen.get("trace") or {}).get("egress"), "gateway calls of the step are stored")
user_space = client.post("/api/journal/spaces", headers=other_headers, json={"title": "Utes Tag"})
user_day = client.post(
f"/api/journal/spaces/{user_space.json()['id']}/days",
headers=other_headers,
json={"calendar_date": "2026-08-28"},
)
user_conv = client.post(
f"/api/journal/days/{user_day.json()['day']['id']}/conversations",
headers=other_headers,
json={"title": "Gespräch"},
)
user_turn = client.post(
f"/api/journal/conversations/{user_conv.json()['id']}/turn",
headers=other_headers,
json={"body": "Utes privater Satz bleibt ohne Debug-Speicher."},
)
expect(user_turn.status_code == 200, "non-admin can still talk")
after_user = client.get("/api/admin/debug/runs", headers=headers).json()["runs"]
expect(len(after_user) == len(runs), "non-admin turns are not persisted")
expect(
not any("Utes privater Satz" in blob(item) for item in after_user),
"admin list does not contain the other profile",
)
reset_debug()
gen = client.post(
f"/api/journal/days/{day_id}/generate",
headers=headers,
json={"conversation_ids": [conv_id]},
)
expect(gen.status_code == 200, f"generate {gen.status_code}")
after_gen = client.get("/api/admin/debug/runs", headers=headers).json()["runs"]
expect(any(item.get("purpose") == "journal_generate" for item in after_gen), "journal generate is a stored step")
exported = client.get("/api/admin/debug/export?format=json", headers=headers)
expect(exported.status_code == 200, "json export")
expect("attachment" in (exported.headers.get("content-disposition") or ""), "json is a download")
document = exported.json()
expect(document.get("kind") == "kansho.debug_export", "export names the contract")
expect(document.get("version") == 2, "export version is nested tree")
expect(document.get("scope") == "all", "full export is unscoped")
expect(document.get("run_count") == len(document.get("runs") or []), "run_count matches runs")
expect(len(document.get("runs") or []) >= 3, "export contains the dialogue and generate steps")
expect((document.get("spaces") or []), "full export has space tree")
space = (document.get("spaces") or [{}])[0]
expect(space.get("title") == "Alltag", "tree names the space")
day_node = (space.get("days") or [{}])[0]
expect(day_node.get("calendar_date") == "2026-08-28", "tree names the day")
conv_node = (day_node.get("conversations") or [{}])[0]
expect(len(conv_node.get("steps") or []) >= 2, "tree keeps dialogue steps on the conversation")
expect(day_node.get("generates"), "journal generate sits on the day")
expect("local_label" not in blob(document), "export has no mapping labels")
created = [item.get("created") for item in document["runs"]]
expect(created == sorted(created), "export is chronological")
tree = client.get("/api/admin/debug/tree", headers=headers)
expect(tree.status_code == 200, "tree endpoint")
tree_space = (tree.json().get("spaces") or [{}])[0]
expect(tree_space.get("title") == "Alltag", "live tree names the space")
tree_conv = ((tree_space.get("days") or [{}])[0].get("conversations") or [{}])[0]
expect(tree_conv.get("id") == conv_id, "live tree points at the conversation")
expect(tree_conv.get("step_count") >= 2, "live tree counts steps")
conv_export = client.get(
f"/api/admin/debug/export?format=json&conversation_id={conv_id}",
headers=headers,
)
expect(conv_export.status_code == 200, "conversation export")
conv_doc = conv_export.json()
expect(conv_doc.get("scope") == "conversation", "conversation export is scoped")
expect(conv_doc.get("conversation", {}).get("id") == conv_id, "conversation export names the dialogue")
expect(
all(item.get("purpose") == "dialogue_turn" for item in conv_doc.get("runs") or []),
"conversation export is the dialogue steps, not the day generate",
)
expect(len(conv_doc.get("runs") or []) >= 2, "conversation export has both turns")
expect("conversation" in (conv_export.headers.get("content-disposition") or ""), "conversation filename is distinct")
latest = client.get(
f"/api/admin/debug/latest?journal_day_id={day_id}&purpose=journal_generate",
headers=headers,
)
expect(latest.status_code == 200, "latest generate endpoint")
latest_run = latest.json().get("run") or {}
expect(latest_run.get("purpose") == "journal_generate", "latest run is the journal generate")
expect(latest_run.get("journal_day_id") == day_id, "latest generate belongs to the day")
expect((latest_run.get("trace") or {}).get("purpose") == "journal_generate" or (latest_run.get("trace") or {}).get("stages"), "latest generate keeps the trace")
gen_export = client.get(
f"/api/admin/debug/export?format=json&journal_day_id={day_id}&purpose=journal_generate",
headers=headers,
)
expect(gen_export.status_code == 200, "generate export")
gen_doc = gen_export.json()
expect(gen_doc.get("scope") == "journal_generate", "generate export is scoped")
expect(
all(item.get("purpose") == "journal_generate" for item in gen_doc.get("runs") or []),
"generate export is only the journal draft steps",
)
expect(len(gen_doc.get("runs") or []) >= 1, "generate export has the draft run")
expect("journal" in (gen_export.headers.get("content-disposition") or ""), "generate filename is distinct")
markdown = client.get("/api/admin/debug/export?format=markdown", headers=headers)
expect(markdown.status_code == 200, "markdown export")
text = markdown.text
expect("Kanshō Debug-Export" in text, "markdown has a title")
expect("Intern (Klartext-Vorlage, nicht gesendet)" in text, "markdown has intern section")
expect("Egress (maskiert, gesendet)" in text, "markdown has egress section")
expect("dialogue_turn" in text, "markdown lists dialogue steps")
expect("journal_generate" in text, "markdown lists generate steps")
bad = client.get("/api/admin/debug/export?format=csv", headers=headers)
expect(bad.status_code == 400, "unknown export format is rejected")
disabled = client.put("/api/admin/debug", headers=headers, json={"persist_enabled": False})
expect(disabled.json()["persist_enabled"] is False, "persist can be turned off")
count_before = len(client.get("/api/admin/debug/runs", headers=headers).json()["runs"])
reset_debug()
later = client.post(
f"/api/journal/conversations/{conv_id}/turn",
headers=headers,
json={"body": "Noch ein Satz ohne Persistenz."},
)
expect(later.status_code == 200, "turn after disable")
count_after = len(client.get("/api/admin/debug/runs", headers=headers).json()["runs"])
expect(count_after == count_before, "disable stops new writes and keeps history")
run_id = client.get("/api/admin/debug/runs", headers=headers).json()["runs"][0]["id"]
removed = client.delete(f"/api/admin/debug/runs/{run_id}", headers=headers)
expect(removed.status_code == 200, "delete one run")
cleared = client.post("/api/admin/debug/runs/clear", headers=headers)
expect(cleared.status_code == 200, "clear remaining runs")
expect(client.get("/api/admin/debug/runs", headers=headers).json()["runs"] == [], "history empty after clear")
health = client.get("/api/admin/health", headers=headers)
expect(health.json()["inventory"]["debug_persist_enabled"] is False, "health shows persist off")
expect(health.json()["inventory"]["debug_runs"] == 0, "health shows empty debug count")
print("All debug persist tests passed.")
if __name__ == "__main__":
main()
+5
View File
@@ -79,6 +79,11 @@ def main() -> None:
admin = client.get("/api/admin/health", headers=headers)
expect(admin.status_code == 200, "admin health")
expect(admin.json()["inventory"]["prompts"] == 5, "five seeded prompts")
expect(admin.json()["inventory"]["debug_persist_enabled"] is False, "debug persist off by default")
expect(admin.json()["inventory"]["debug_runs"] == 0, "no debug runs at start")
debug = client.get("/api/admin/debug", headers=headers)
expect(debug.status_code == 200 and debug.json()["persist_enabled"] is False, "debug settings default off")
providers = client.get("/api/admin/providers", headers=headers)
expect(providers.status_code == 200, "admin providers")
+58
View File
@@ -483,6 +483,64 @@ def main() -> None:
expect((third.json().get("trace") or {}).get("writing_profile", {}).get("present") is True, "confirmed profile is marked present")
expect(sum(1 for item in (third.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "profile A/B still uses one generate call")
kept_body = (
"Die Kinder verstanden kein Wort. Später fragte ich, "
"ob die Kinder überhaupt Deutsch verstehen. Die Kinder lachten."
)
def keep_groups(_messages, _policy):
return ChatResult(
content=f"Markttag\n\n{kept_body}",
model="fake",
usage={},
context_compression="disabled",
)
with patch("privacy_gateway.complete_model", keep_groups):
kept = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [conv.json()["id"]]},
)
expect(kept.status_code == 200, f"accepted model text {kept.text}")
expect(kept.json().get("title") == "Markttag", "title split stays technical")
expect(kept.json().get("body") == kept_body, "stored body equals demasked model body")
expect("Er" not in (kept.json().get("body") or "").split(), "generic group is not rewritten to a pronoun")
stored = f"{kept.json().get('title')}\n\n{kept.json().get('body')}".strip()
expect((kept.json().get("trace") or {}).get("reply") == stored, "trace reply is the stored draft")
expect((kept.json().get("trace") or {}).get("stored_body") == kept_body, "trace stores the same body")
expect((kept.json().get("trace") or {}).get("generate_calls") == 1, "accepted text still uses one generate call")
def mixed_surface(_messages, _policy):
body = (
"Die Gruppe verstand kein Wort. "
"Später die Gruppe wieder. "
"die Gruppen standen abseits."
)
return ChatResult(
content=f"Markttag\n\n{body}",
model="fake",
usage={},
context_compression="disabled",
)
with patch("privacy_gateway.complete_model", mixed_surface):
mixed = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={"conversation_ids": [conv.json()["id"]]},
)
expect(mixed.status_code == 200, f"mixed surface generate {mixed.text}")
mixed_body = mixed.json().get("body") or ""
expect(mixed_body.startswith("Die Gruppe verstand"), "journal stores sentence-initial model casing")
expect("Später die Gruppe wieder" in mixed_body, "journal keeps mid-sentence model casing")
expect("die Gruppen standen" in mixed_body, "journal keeps unmatched plural surface form")
mixed_stored = f"{mixed.json().get('title')}\n\n{mixed_body}".strip()
expect((mixed.json().get("trace") or {}).get("reply") == mixed_stored, "journal trace reply is the stored draft")
expect((mixed.json().get("trace") or {}).get("stored_body") == mixed_body, "journal stored_body matches the draft")
expect("Die Gruppe verstand" in ((mixed.json().get("trace") or {}).get("raw") or ""), "journal trace raw keeps provider casing")
expect((mixed.json().get("trace") or {}).get("generate_calls") == 1, "mixed surface still uses one generate call")
def invent_title(_messages, _policy):
return ChatResult(
content="Hanna am Hafen\n\nIch ging zum Markt, auf dem es ziemlich voll war.",
@@ -42,6 +42,7 @@ from journal_generation_policy import (
publish_guideline,
save_selection,
seed_generation_instructions,
set_default_guideline,
snapshot_summary,
update_guideline,
validate_selection,
@@ -214,6 +215,9 @@ def test_runtime_has_no_numeric_policy() -> None:
expect(day.count("slot:") >= 4, "journal page has four independent slots")
expect("<select" in day, "journal page uses select fields")
expect("Diese Auswahl als Standard merken" in day, "journal page can remember explicitly")
expect("if (filled) return current" in day, "settings load does not overwrite a filled selection")
expect("Auswahl nicht mehr aktiv" in day, "stale guideline ids stay visible as invalid")
expect("Rev. ${option.revision}" in day, "options show the guideline revision")
expect("Faktenregeln und Datenschutz bleiben unabhängig" in day, "facts/privacy note remains")
editor = (FRONTEND / "src" / "pages" / "JournalEditorPage.jsx").read_text(encoding="utf-8")
expect("generation_summary" in editor, "editor can show the generation snapshot")
@@ -232,6 +236,11 @@ def test_runtime_has_no_numeric_policy() -> None:
trace_ui = (FRONTEND / "src" / "components" / "CallTrace.jsx").read_text(encoding="utf-8")
expect("Editorial Mode" not in trace_ui, "trace UI does not present source mode as a run decision")
expect("source_mode" not in trace_ui, "trace UI has no source_mode label")
expect("Chunks technisch verarbeitet" in trace_ui, "trace separates technical chunk coverage")
expect("Semantische Identitätserkennung" in trace_ui, "trace names semantic uncertainty")
expect("nicht garantiert" in trace_ui, "trace does not claim semantic completeness")
expect("Gespeicherter Entwurf" in trace_ui, "trace shows the stored draft")
expect("Technisch verarbeitete Chunks bedeuten keine vollständige" in trace_ui, "trace explains coverage is not identity certainty")
def test_seed_and_compiler() -> None:
@@ -493,6 +502,27 @@ def main() -> None:
expect((rejected.json().get("detail") or {}).get("code") == "invalid_generation_selection", "rejection uses the selection code")
after_reject = client.get("/api/journal/generation-settings", headers=headers).json()
expect(after_reject["selection"] == default_ids(), "rejected values are not stored")
save_selection(profile_id, low_ids())
set_default_guideline(seed_id("transformation", "substantial"))
expect(load_selection(profile_id)["transformation_id"] == seed_id("transformation", "correction"), "setting a catalog default does not rewrite a stored selection")
save_selection(profile_id, default_ids())
blocked_calls = {"n": 0}
def no_provider(*_a, **_k):
blocked_calls["n"] += 1
raise AssertionError("invalid selection must not call a provider")
with patch("privacy_gateway.complete_model", no_provider), patch("entity_detect.complete_chat", no_provider):
before_provider = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={
"conversation_ids": [conv.json()["id"]],
"generation_selection": {**default_ids(), "transformation_id": "missing"},
},
)
expect(before_provider.status_code == 400, "invalid selection still fails closed")
expect(blocked_calls["n"] == 0, "invalid selection aborts before detect and generate")
draft = client.post(
"/api/admin/generation-instructions/journal_generate",
@@ -575,6 +605,53 @@ def main() -> None:
for fact in HARD_FACTS:
expect(fact in intern, f"hard fact remains at low policy: {fact}")
copyedit_ids = selection_of(
transformation="copyedit",
detail="complete",
voice="clear",
narrative="weighted",
)
reset_debug()
copyedit_recorder = install_test_recorder()
copyedit_run = client.post(
f"/api/journal/days/{day.json()['day']['id']}/generate",
headers=headers,
json={
"conversation_ids": [conv.json()["id"]],
"generation_selection": copyedit_ids,
"remember_generation_selection": False,
},
)
expect(copyedit_run.status_code == 200, f"copyedit generate {copyedit_run.text}")
copyedit_meta = selection_meta(copyedit_run.json())
copyedit_intern = intern_of(copyedit_run.json())
copyedit_snap = copyedit_run.json().get("generation_snapshot") or {}
expect(copyedit_meta.get("ids") == {
"transformation": seed_id("transformation", "copyedit"),
"detail": seed_id("detail", "complete"),
"voice": seed_id("voice", "clear"),
"narrative": seed_id("narrative", "weighted"),
}, "copyedit request ids reach the trace")
expect(copyedit_meta.get("keys", {}).get("transformation") == "copyedit", "copyedit key reaches the trace")
expect(copyedit_meta.get("revisions", {}).get("transformation") == 1, "copyedit revision reaches the trace")
expect(copyedit_meta.get("revisions", {}).get("narrative") == 1, "weighted revision of the selected id reaches the trace")
expect(copyedit_snap.get("transformation", {}).get("id") == seed_id("transformation", "copyedit"), "snapshot stores the selected copyedit id")
expect(copyedit_snap.get("transformation", {}).get("revision") == 1, "snapshot stores the selected copyedit revision")
expect(copyedit_snap.get("narrative", {}).get("id") == seed_id("narrative", "weighted"), "snapshot stores the selected weighted id")
expect(seed_instruction("transformation", "copyedit") in copyedit_intern, "selected copyedit instruction reaches the prompt")
expect(seed_instruction("narrative", "weighted") in copyedit_intern, "selected weighted instruction reaches the prompt")
expect(seed_instruction("transformation", "substantial") not in copyedit_intern, "unselected substantial instruction is absent")
expect([item.get("purpose") for item in copyedit_recorder] == ["journal_generate"], "copyedit still uses one generate call")
preview = client.post(
"/api/admin/generation-instructions/journal_generate/preview",
headers=headers,
json={"generation_selection": copyedit_ids},
)
expect(preview.status_code == 200, f"copyedit preview {preview.text}")
expect(preview.json()["transformation_instructions"] == seed_instruction("transformation", "copyedit"), "preview compiles the same copyedit instruction")
expect(preview.json().get("selection", {}).get("ids", {}).get("transformation") == seed_id("transformation", "copyedit"), "preview reports the same copyedit id")
expect(preview.json().get("selection", {}).get("revisions", {}).get("narrative") == 1, "preview reports the same weighted revision")
notes_day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
@@ -789,6 +866,7 @@ def main() -> None:
headers=headers,
)
expect(published.status_code == 200, f"publish {published.text}")
expect(load_selection(profile_id)["transformation_id"] == seed_id("transformation", "substantial"), "publishing a clone does not rewrite the stored selection")
immutable = client.put(
f"/api/admin/generation-instructions/journal_generate/{seed_id('transformation', 'substantial')}",
headers=headers,
+20 -25
View File
@@ -60,33 +60,28 @@ def main() -> None:
kept = "Am Morgen der Markt.\n\nSpäter das Boot, noch gehetzt."
expect(shape_journal("Tag", kept, turns)[1] == kept, "already shaped text is left alone")
source = [
"Gegen 8:30 Uhr war Sushi immer noch nicht auf dem Balkon. "
"Ich fand sie unter der Dusche. Später saß sie auf dem Bett und telefonierte. "
"Auf die Frage, ob sie mit mir frühstücken wolle, antwortete sie, dass sie erst Brot holen müsse."
]
stuffed = (
"Gegen 8:30 Uhr war Sushi immer noch nicht bei mir auf dem Balkon. "
"Ich machte mich auf die Suche und fand Sushi schließlich unter der Dusche im Badezimmer. "
"Als Sushi dann immer noch nicht auftauchte, sah ich nochmals nach und musste feststellen, "
"dass Sushi auf dem Bett saß und telefonierte. Als ich gerade mit den Vorbereitungen fertig war, "
"kam Sushi aus dem Schlafzimmer. Auf meine Frage, ob Sushi mit mir frühstücken wolle, "
"antwortete sie, dass Sushi erst Brot vom Laden holen müsse."
repeated = (
"Die Kinder verstanden kein Wort. Später fragte ich, "
"ob die Kinder überhaupt Deutsch verstehen. Die Kinder lachten."
)
from journal_shape import infer_person_gender, naturalize_person_mentions
_, unchanged = shape_journal(
"Tag",
repeated,
["Die Kinder verstanden kein Wort."],
person_labels=["Die Kinder", "Kinder"],
)
expect(unchanged == repeated, "repeated generic person terms stay unchanged")
expect("Er" not in unchanged.split(), "generic group is not turned into a pronoun")
expect(unchanged.count("Kinder") == 3, "every generic mention remains")
named = (
"Maren kam zum Markt. Später rief Maren an. "
"Ob Maren noch käme, wusste ich nicht. Maren blieb weg."
)
_, named_out = shape_journal("Tag", named, named.split(". "), person_labels=["Maren"])
expect(named_out == named, "accepted person names are not replaced by pronouns")
expect("Er " not in named_out and "Sie " not in named_out, "no guessed pronoun rewrite")
expect(infer_person_gender("Sushi", source) == "f", "source pronouns set feminine")
cleaned = naturalize_person_mentions(stuffed, ["Sushi"], source)
expect(cleaned.count("Sushi") == 2, "name kept twice then pronouns")
expect("fand Sushi" in cleaned, "second mention may stay a name")
expect("dass sie erst Brot" in cleaned, "reported speech keeps sie not the name")
expect("ob sie mit mir" in cleaned, "ob-clause is sie")
expect("Als sie dann" in cleaned, "later subject becomes pronoun")
expect("esse gerne Sushi" in naturalize_person_mentions(
"Ich esse gerne Sushi mit meiner Frau Sushi. Danach aß ich Sushi.",
["Sushi"],
source,
), "food homonym is not turned into a pronoun")
print("journal shape ok")
+84 -5
View File
@@ -8,7 +8,6 @@ import json
import os
import sys
import tempfile
import threading
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from unittest.mock import patch
@@ -25,6 +24,8 @@ from fastapi.testclient import TestClient
from identity_store import (
STATUS_LEGACY,
confirm_identity,
is_maskable_label,
is_registry_maskable_label,
list_confirmed_identities,
list_mappings,
list_registry,
@@ -37,6 +38,7 @@ from privacy_gateway import (
PrivacyGatewayError,
_demask,
_validate_response,
_visible_response,
complete,
mask_prompt,
reset_debug,
@@ -181,7 +183,10 @@ def main() -> None:
expect(first.trace.get("detect_provider") == "fake-detect", "detect provider is separate")
expect("Anna" not in (first.trace.get("egress") or ""), "Anna masked before generate")
expect("Hamburg" not in (first.trace.get("egress") or ""), "Hamburg masked before generate")
expect(first.trace.get("full_detection_coverage") is True, "successful detect records full coverage")
expect(first.trace.get("full_detection_coverage") is True, "successful detect records technical chunk coverage")
expect(first.trace.get("semantic_identity_guaranteed") is False, "chunk coverage is not semantic certainty")
expect(first.diagnostics.get("semantic_identity_guaranteed") is False, "compact trace does not claim semantic completeness")
expect(first.trace.get("confirmed_registry_applied") is True, "confirmed registry is applied as a safety net")
expect(first.trace.get("generate_called") is True, "generate runs after complete detection")
expect(first.diagnostics.get("detect_calls") == 1, "normal short text uses one detect call")
labels_after = {item["canonical_label"] for item in list_confirmed_identities(profile_id)}
@@ -315,7 +320,14 @@ def main() -> None:
def worker(label: str):
reset_debug()
out = _run(profile_id, f"Ich traf {label}.")
results.append((label, out.trace.get("active_tokens"), threading.get_ident()))
results.append(
(
label,
tuple(out.trace.get("active_tokens") or []),
out.trace.get("egress") or "",
out.trace.get("intern") or "",
)
)
with ThreadPoolExecutor(max_workers=2) as pool:
one = pool.submit(worker, "Anna")
@@ -324,6 +336,10 @@ def main() -> None:
two.result()
labels = {item[0] for item in results}
expect(labels == {"Anna", "Maren"}, "parallel requests keep their own manifests")
by_label = {item[0]: item for item in results}
expect("Maren" not in by_label["Anna"][2] and "Maren" not in by_label["Anna"][3], "Anna request does not carry Maren")
expect("Anna" not in by_label["Maren"][2] and "Anna" not in by_label["Maren"][3], "Maren request does not carry Anna")
expect(by_label["Anna"][1] and by_label["Maren"][1], "each parallel request has its own tokens")
from db import get_db
import uuid
@@ -432,13 +448,14 @@ def main() -> None:
plural, _ = bind_pronouns(_mask("Anna kam. Sie waren zu dritt.", anna))
expect("Sie waren" in plural or "sie waren" in plural.lower(), "plural sie stays")
from identity_store import is_maskable_label
expect(not is_maskable_label("ich"), "pronoun ich is not a mapping")
expect(not is_maskable_label("user"), "role user is not a mapping")
expect(not is_maskable_label("6:00 Uhr"), "clock time is not identity")
expect(is_maskable_label("Sushi"), "name-shaped label stays structurally maskable")
expect(is_maskable_label("Himmel"), "weather word is not banned by a stopword list")
expect(not is_maskable_label("Lia"), "detect gate still skips short labels")
expect(is_registry_maskable_label("Lia"), "confirmed registry may mask short names")
expect(not is_registry_maskable_label("ich"), "registry gate still skips pronouns")
from db import get_db as _get_db
@@ -506,6 +523,10 @@ def main() -> None:
tokenized = _validate_response("Anna war ruhig.", anna_manifest)
expect(tokenized.startswith("[[PERSON:01]]"), "active cleartext is normalized, not discarded")
expect(_demask(tokenized, anna_manifest) == "Anna war ruhig.", "normalized plaintext demasks")
expect(
_visible_response("Anna war ruhig.", anna_manifest) == "Anna war ruhig.",
"visible output keeps model plaintext instead of the check roundtrip",
)
no_user = client.get("/api/admin/identities")
expect(no_user.status_code in {401, 403}, "identity registry requires auth")
@@ -525,6 +546,64 @@ def main() -> None:
expect(admin_calls["n"] == 0, "admin registry actions create no external egress")
expect(created.json()["status"] == "confirmed", "admin create confirms locally")
before_confirm = {item.get("canonical_label") for item in list_confirmed_identities(profile_id)}
confirm_identity(profile_id, "Lia", aliases=["Mara"])
expect(is_registry_maskable_label("Lia"), "short confirmed name remains maskable")
egress_body = (
"WRITING_PROFILE\nLia schreibt knapp.\n"
"STYLE_EXAMPLES\nMara lachte am Markt.\n"
"CURRENT_DAY_SOURCES\nLia und Mara gingen später.\n"
"EXISTING_TEXT\nLia blieb."
)
covered = _run(profile_id, egress_body)
masked_all = covered.trace.get("egress") or ""
intern = covered.trace.get("intern") or ""
expect("Lia" in intern and "Mara" in intern, "clear names remain in the local template")
expect("Lia" not in masked_all, "confirmed short name is masked in every egress section")
expect("Mara" not in masked_all, "confirmed alias is masked in every egress section")
expect(
masked_all.count("[[") >= intern.count("Lia") + intern.count("Mara"),
"canonical and alias mentions are replaced",
)
expect("WRITING_PROFILE" in masked_all and "STYLE_EXAMPLES" in masked_all, "writing profile and style examples stay in egress")
expect("CURRENT_DAY_SOURCES" in masked_all and "EXISTING_TEXT" in masked_all, "sources and existing text stay in egress")
expect(covered.trace.get("confirmed_registry_applied") is True, "registry safety net is marked applied")
expect(covered.trace.get("semantic_identity_guaranteed") is False, "registry coverage is not semantic certainty")
expect(covered.trace.get("pre_egress_validation") == "ok", "confirmed plaintext is not left in egress")
group_text = "Die Kinder verstanden kein Wort. Herkunft: Nordland."
kinder_start = group_text.index("Kinder")
nord_start = group_text.index("Nordland")
install_test_spans(
[
{
"start": kinder_start,
"end": kinder_start + len("Kinder"),
"text": "Kinder",
"entity_type": "PERSON",
},
{
"start": nord_start,
"end": nord_start + len("Nordland"),
"text": "Nordland",
"entity_type": "PLACE",
},
]
)
try:
overmasked = _run(profile_id, group_text)
proposals = list_review_proposals(profile_id)
labels = {(item.get("observed_label") or item.get("canonical_label") or item.get("local_label") or "").casefold() for item in proposals}
expect("kinder" in labels or "nordland" in labels, "semantic overmask stays request-local as a proposal")
confirmed_now = {item.get("canonical_label") for item in list_confirmed_identities(profile_id)}
expect("Kinder" not in confirmed_now, "generic group is not persisted as an identity")
expect("Nordland" not in confirmed_now, "origin designation is not persisted as an identity")
expect(confirmed_now == before_confirm | {"Lia"}, "only the confirmed registry row is durable")
expect(overmasked.trace.get("full_detection_coverage") is True, "injected spans still mark technical coverage")
expect(overmasked.trace.get("semantic_identity_guaranteed") is False, "overmasking is not claimed as certainty")
finally:
reset_detect_test_hooks()
turn = client.post(
f"/api/journal/conversations/{conv.json()['id']}/turn",
headers=headers,
+94 -1
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import json
import os
import re
import sys
import tempfile
from concurrent.futures import ThreadPoolExecutor
@@ -18,6 +19,7 @@ os.environ["KANSHO_FAKE_DETECT"] = "1"
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
from fastapi.testclient import TestClient
from entity_detect import install_test_spans, reset_detect_test_hooks
from identity_store import remember_mapping
from main import app
from privacy_gateway import (
@@ -29,6 +31,7 @@ from privacy_gateway import (
_demask,
_process_response,
_validate_response,
_visible_response,
complete,
last_trace,
mask_prompt,
@@ -137,6 +140,55 @@ def main() -> None:
expect(_demask("[[PERSON:07]] kam.", aliased) == "Clarissa kam.", "confirmed alias demasks to the canonical spelling")
canonical_clear, _meta = _process_response("Clarissa kam vorbei.", aliased)
expect(canonical_clear.startswith("[[PERSON:07]]"), "canonical spelling of an active alias token is remapped")
expect(
_visible_response("Sushi kam vorbei.", aliased) == "Sushi kam vorbei.",
"alias plaintext keeps the model surface form",
)
expect(
_visible_response("Clarissa kam vorbei.", aliased) == "Clarissa kam vorbei.",
"canonical plaintext keeps the model surface form",
)
expect(
_visible_response("[[PERSON:07]] kam vorbei.", aliased) == "Clarissa kam vorbei.",
"emitted placeholder still rehydrates to the canonical spelling",
)
group_prompt = "Heute die Gruppe am Markt."
group_start = group_prompt.index("die Gruppe")
group_rows = [
{
"local_label": "die Gruppe",
"token": "PERSON:04",
"canonical_label": "die Gruppe",
"demask_label": "die Gruppe",
"entity_type": "PERSON",
"start": group_start,
"end": group_start + len("die Gruppe"),
"source": "request_local",
}
]
group = mask_prompt(group_prompt, group_rows, "journal_generate")
expect(len(group.replacements) == 1, "observed multi-word label is active")
mixed_raw = (
"Die Gruppe verstand kein Wort. "
"Später fragte ich, ob die Gruppe noch da sei. "
"[[PERSON:04]] blieb. "
"die Gruppen standen abseits."
)
tokenized, meta = _process_response(mixed_raw, group)
expect(meta.get("response_normalization") == "active_cleartext_normalized", "check copy still remaps active cleartext")
expect(tokenized.startswith("[[PERSON:04]]"), "check copy tokenizes sentence-initial cleartext")
expect("die Gruppe verstand" in _demask(tokenized, group), "check roundtrip uses the stored label")
visible = _visible_response(mixed_raw, group)
expect(visible.startswith("Die Gruppe verstand"), "stored text keeps sentence-initial capitalization")
expect("ob die Gruppe noch da sei" in visible, "stored text keeps mid-sentence form")
expect("die Gruppe blieb" in visible, "emitted placeholder is rehydrated locally")
expect("die Gruppen standen" in visible, "unmatched plural surface form is left unchanged")
expect(visible != _demask(tokenized, group), "visible output is not the rematerialized check copy")
leftover_unknown = _visible_response("[[PERSON:04]] und [[PERSON:99]] und [[…]]", group)
expect(leftover_unknown.startswith("die Gruppe"), "known placeholder is rehydrated")
expect("[[PERSON:99]]" in leftover_unknown, "unknown placeholder is not materialized")
expect("[[…]]" in leftover_unknown, "generic placeholder is not materialized")
overlap = mask_prompt(
"Anna-Lena kam vorbei.",
@@ -296,7 +348,7 @@ def main() -> None:
repaired = _run_gateway(profile_id, "Ich war mit Anna am Markt.")
expect(len(seen) == 1, "active cleartext does not start a second generate call")
expect("[[PERSON:01]]" in seen[0] and "Anna" not in seen[0].replace("[[PERSON:01]]", ""), "egress stays masked")
expect("Anna" in (repaired.content or ""), "normalized cleartext is demasked locally")
expect("Anna" in (repaired.content or ""), "model plaintext identity spelling is kept")
expect(repaired.diagnostics.get("generate_calls") == 1, "exactly one generate call")
expect(
repaired.diagnostics.get("response_normalization") == "active_cleartext_normalized",
@@ -324,6 +376,47 @@ def main() -> None:
expect(repaired.diagnostics.get("total_tokens") == 18, "single call keeps total tokens")
expect(abs(float(repaired.diagnostics.get("cost") or 0) - 0.01) < 1e-9, "single call keeps cost")
prompt_group = "Ich traf die Gruppe."
group_at = prompt_group.index("die Gruppe")
install_test_spans(
[
{
"start": group_at,
"end": group_at + len("die Gruppe"),
"text": "die Gruppe",
"entity_type": "PERSON",
}
]
)
def mixed_surface(messages, _policy):
egress = messages[0].get("content") or ""
match = re.search(r"\[\[PERSON:\d+\]\]", egress)
token = match.group(0) if match else "[[PERSON:01]]"
return ChatResult(
content=f"Die Gruppe verstand kein Wort. {token} blieb. Später die Gruppe wieder.",
model="fake",
usage={},
context_compression="not_applicable",
)
try:
with patch("privacy_gateway.complete_model", mixed_surface):
surfaced = _run_gateway(profile_id, prompt_group)
finally:
reset_detect_test_hooks()
expect(surfaced.diagnostics.get("generate_calls") == 1, "mixed surface still uses one generate call")
expect("Die Gruppe verstand" in ((surfaced.trace or {}).get("raw") or ""), "trace raw keeps model casing")
expect("[[" in ((surfaced.trace or {}).get("raw") or ""), "trace raw keeps the emitted placeholder")
expect(surfaced.content == (surfaced.trace or {}).get("reply"), "visible content matches trace reply")
expect((surfaced.content or "").startswith("Die Gruppe verstand"), "gateway keeps sentence-initial model casing")
expect("die Gruppe blieb" in (surfaced.content or ""), "gateway rehydrates the emitted placeholder")
expect("Später die Gruppe wieder" in (surfaced.content or ""), "gateway keeps later plaintext unchanged")
expect(
surfaced.diagnostics.get("response_normalization") == "active_cleartext_normalized",
"check copy still records cleartext normalization",
)
inactive_calls = {"n": 0}
def keep_inactive(messages, _policy):