MVP 0.95
This commit is contained in:
1 parent
4f37991ad1
commit
be11f82f84
39 files changed
+2490
-239
No files matched your search
@@ -66,6 +66,13 @@ _JOURNAL_DAY_COLUMNS = {
|
||||
_JOURNAL_DRAFT_COLUMNS = {
|
||||
"generation_snapshot": "TEXT NOT NULL DEFAULT '{}'",
|
||||
}
|
||||
_DEBUG_RUN_COLUMNS = {
|
||||
"space_id": "TEXT",
|
||||
"journal_day_id": "TEXT",
|
||||
"space_title": "TEXT NOT NULL DEFAULT ''",
|
||||
"calendar_date": "TEXT NOT NULL DEFAULT ''",
|
||||
"conversation_title": "TEXT NOT NULL DEFAULT ''",
|
||||
}
|
||||
_IDENTITY_MAPPING_COLUMNS = {
|
||||
"canonical_label": "TEXT NOT NULL DEFAULT ''",
|
||||
"entity_type": "TEXT NOT NULL DEFAULT 'PERSON'",
|
||||
@@ -485,6 +492,9 @@ def init_db() -> None:
|
||||
_mark(conn, "015_journal_generation_settings")
|
||||
_mark(conn, "016_generation_instruction_fragments")
|
||||
_mark(conn, "017_generation_guidelines")
|
||||
_mark(conn, "018_debug_runs")
|
||||
_ensure_columns(conn, "debug_runs", _DEBUG_RUN_COLUMNS)
|
||||
_mark(conn, "019_debug_run_placement")
|
||||
from writing_profile_store import bootstrap_from_existing
|
||||
|
||||
bootstrap_from_existing()
|
||||
@@ -0,0 +1,837 @@
|
||||
"""Opt-in local development debug persist. Mapping labels and secrets stay out."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
import uuid
|
||||
|
||||
from db import get_db, row_to_dict
|
||||
from version import APP_VERSION, BUILD_DATE
|
||||
|
||||
SETTING_KEY = "debug.persist_traces"
|
||||
EXPORT_KIND = "kansho.debug_export"
|
||||
EXPORT_VERSION = 2
|
||||
UNASSIGNED_SPACE_TITLE = "Ohne Space"
|
||||
UNASSIGNED_DAY_LABEL = "Ohne Tag"
|
||||
FORBIDDEN_KEYS = frozenset(
|
||||
{
|
||||
"local_label",
|
||||
"demask_label",
|
||||
"mapping_table",
|
||||
"replacements",
|
||||
"restore_by_token",
|
||||
"password",
|
||||
"password_hash",
|
||||
"key",
|
||||
"api_key",
|
||||
"secret",
|
||||
"token",
|
||||
}
|
||||
)
|
||||
LAYER_FOR_PURPOSE = {
|
||||
"dialogue_turn": "dialogzug",
|
||||
"journal_generate": "journalentwurf",
|
||||
"local_source_artifact": "journalquellen",
|
||||
"journal_reconstruct": "journalrekonstruktion",
|
||||
"profile_review": "profilreview",
|
||||
}
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def persist_enabled() -> bool:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
|
||||
)
|
||||
return (row or {}).get("value") == "1"
|
||||
|
||||
|
||||
def set_persist_enabled(enabled: bool) -> dict:
|
||||
value = "1" if enabled else "0"
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO app_settings (key, value, updated)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
|
||||
""",
|
||||
(SETTING_KEY, value),
|
||||
)
|
||||
return settings_payload()
|
||||
|
||||
|
||||
def _is_admin(profile_id: str) -> bool:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT role FROM profiles WHERE id = ?", (profile_id,)).fetchone()
|
||||
)
|
||||
return (row or {}).get("role") == "admin"
|
||||
|
||||
|
||||
def should_persist(profile_id: str | None) -> bool:
|
||||
if not profile_id:
|
||||
return False
|
||||
return persist_enabled() and _is_admin(profile_id)
|
||||
|
||||
|
||||
def sanitize(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
cleaned = {}
|
||||
for key, item in value.items():
|
||||
if key in FORBIDDEN_KEYS:
|
||||
continue
|
||||
cleaned[key] = sanitize(item)
|
||||
return cleaned
|
||||
if isinstance(value, list):
|
||||
return [sanitize(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return [sanitize(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def settings_payload(profile_id: str | None = None) -> dict:
|
||||
enabled = persist_enabled()
|
||||
count = 0
|
||||
if profile_id:
|
||||
with get_db() as conn:
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM debug_runs WHERE profile_id = ?",
|
||||
(profile_id,),
|
||||
).fetchone()["n"]
|
||||
return {
|
||||
"persist_enabled": enabled,
|
||||
"run_count": count,
|
||||
"note": (
|
||||
"Entwicklungsoption. Speichert die Admin-Testspur lokal je Dialogschritt. "
|
||||
"Zuordnung Space / Tag / Gespräch. Mapping-Tabelle, Secrets und Klarnamen-Labels "
|
||||
"bleiben ausgeschlossen. Nur Aktionen des Admin-Profils. Default aus."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def resolve_placement(
|
||||
profile_id: str,
|
||||
*,
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
space_id: str | None = None,
|
||||
) -> dict:
|
||||
place = {
|
||||
"space_id": space_id or None,
|
||||
"journal_day_id": journal_day_id or None,
|
||||
"conversation_id": conversation_id or None,
|
||||
"space_title": "",
|
||||
"calendar_date": "",
|
||||
"conversation_title": "",
|
||||
}
|
||||
with get_db() as conn:
|
||||
if conversation_id:
|
||||
conv = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id, title, space_id, journal_day_id
|
||||
FROM conversations
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(conversation_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if conv:
|
||||
place["conversation_id"] = conv["id"]
|
||||
place["conversation_title"] = conv.get("title") or ""
|
||||
place["space_id"] = place["space_id"] or conv.get("space_id")
|
||||
place["journal_day_id"] = place["journal_day_id"] or conv.get("journal_day_id")
|
||||
if place["journal_day_id"]:
|
||||
day = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id, calendar_date, space_id
|
||||
FROM journal_days
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(place["journal_day_id"], profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if day:
|
||||
place["calendar_date"] = day.get("calendar_date") or ""
|
||||
place["space_id"] = place["space_id"] or day.get("space_id")
|
||||
if place["space_id"]:
|
||||
space = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id, title FROM spaces
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(place["space_id"], profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if space:
|
||||
place["space_title"] = space.get("title") or ""
|
||||
return place
|
||||
|
||||
|
||||
def persist_step(
|
||||
profile_id: str,
|
||||
*,
|
||||
purpose: str,
|
||||
status: str = "ok",
|
||||
layer: str = "",
|
||||
subject_type: str = "",
|
||||
subject_id: str = "",
|
||||
conversation_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
space_id: str | None = None,
|
||||
decision: dict | None = None,
|
||||
trace: dict | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> str | None:
|
||||
if not should_persist(profile_id):
|
||||
return None
|
||||
if not journal_day_id and subject_type == "journal_day" and subject_id:
|
||||
journal_day_id = subject_id
|
||||
place = resolve_placement(
|
||||
profile_id,
|
||||
conversation_id=conversation_id,
|
||||
journal_day_id=journal_day_id,
|
||||
space_id=space_id,
|
||||
)
|
||||
payload = sanitize(
|
||||
{
|
||||
"decision": decision,
|
||||
"trace": trace,
|
||||
**(extra or {}),
|
||||
}
|
||||
)
|
||||
run_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO debug_runs (
|
||||
id, profile_id, purpose, layer, status,
|
||||
subject_type, subject_id, conversation_id, message_id,
|
||||
space_id, journal_day_id, space_title, calendar_date, conversation_title,
|
||||
payload_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
run_id,
|
||||
profile_id,
|
||||
purpose,
|
||||
layer or LAYER_FOR_PURPOSE.get(purpose, ""),
|
||||
status,
|
||||
subject_type or "",
|
||||
subject_id or "",
|
||||
place["conversation_id"],
|
||||
message_id,
|
||||
place["space_id"],
|
||||
place["journal_day_id"],
|
||||
place["space_title"],
|
||||
place["calendar_date"],
|
||||
place["conversation_title"],
|
||||
json.dumps(payload, ensure_ascii=False, default=str),
|
||||
),
|
||||
)
|
||||
return run_id
|
||||
|
||||
|
||||
def persist_engine_error(
|
||||
profile_id: str,
|
||||
*,
|
||||
purpose: str,
|
||||
exc,
|
||||
layer: str = "",
|
||||
subject_type: str = "",
|
||||
subject_id: str = "",
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> str | None:
|
||||
diag = dict(getattr(exc, "diagnostics", None) or {})
|
||||
merged = dict(extra or {})
|
||||
merged.setdefault("code", getattr(exc, "code", None))
|
||||
merged.setdefault("message", getattr(exc, "message", None) or str(exc))
|
||||
if diag.get("log") is not None:
|
||||
merged.setdefault("run_log", diag.get("log"))
|
||||
return persist_step(
|
||||
profile_id,
|
||||
purpose=purpose,
|
||||
status="error",
|
||||
layer=layer,
|
||||
subject_type=subject_type,
|
||||
subject_id=subject_id,
|
||||
conversation_id=conversation_id,
|
||||
journal_day_id=journal_day_id,
|
||||
trace=diag.get("trace"),
|
||||
extra=merged,
|
||||
)
|
||||
|
||||
|
||||
def _parse_payload(raw: str | None) -> dict:
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _row_public(row: dict, *, include_payload: bool = True) -> dict:
|
||||
payload = _parse_payload(row.get("payload_json"))
|
||||
item = {
|
||||
"id": row["id"],
|
||||
"created": row["created"],
|
||||
"purpose": row["purpose"],
|
||||
"layer": row["layer"],
|
||||
"status": row["status"],
|
||||
"subject_type": row.get("subject_type") or "",
|
||||
"subject_id": row.get("subject_id") or "",
|
||||
"conversation_id": row.get("conversation_id"),
|
||||
"message_id": row.get("message_id"),
|
||||
"space_id": row.get("space_id"),
|
||||
"journal_day_id": row.get("journal_day_id"),
|
||||
"space_title": row.get("space_title") or "",
|
||||
"calendar_date": row.get("calendar_date") or "",
|
||||
"conversation_title": row.get("conversation_title") or "",
|
||||
"decision": payload.get("decision"),
|
||||
"summary": _summary(row, payload),
|
||||
}
|
||||
if include_payload:
|
||||
item["payload"] = payload
|
||||
item["trace"] = payload.get("trace")
|
||||
item["run_log"] = payload.get("run_log") or ((payload.get("trace") or {}).get("log"))
|
||||
return item
|
||||
|
||||
|
||||
def _summary(row: dict, payload: dict) -> str:
|
||||
decision = payload.get("decision") or {}
|
||||
label = decision.get("label") or decision.get("operation") or ""
|
||||
purpose = row.get("purpose") or ""
|
||||
status = row.get("status") or ""
|
||||
parts = [purpose]
|
||||
if label:
|
||||
parts.append(str(label))
|
||||
if status and status != "ok":
|
||||
parts.append(status)
|
||||
code = payload.get("code")
|
||||
if code:
|
||||
parts.append(str(code))
|
||||
return " · ".join(parts)
|
||||
|
||||
|
||||
def list_runs(profile_id: str, *, limit: int = 200) -> list[dict]:
|
||||
capped = max(1, min(int(limit or 200), 500))
|
||||
with get_db() as conn:
|
||||
rows = [
|
||||
dict(item)
|
||||
for item in conn.execute(
|
||||
"""
|
||||
SELECT * FROM debug_runs
|
||||
WHERE profile_id = ?
|
||||
ORDER BY created DESC, id DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(profile_id, capped),
|
||||
).fetchall()
|
||||
]
|
||||
return [_row_public(row, include_payload=False) for row in rows]
|
||||
|
||||
|
||||
def latest_run(
|
||||
profile_id: str,
|
||||
*,
|
||||
journal_day_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
) -> dict | None:
|
||||
clauses = ["profile_id = ?"]
|
||||
args: list[Any] = [profile_id]
|
||||
if journal_day_id:
|
||||
clauses.append("journal_day_id = ?")
|
||||
args.append(journal_day_id)
|
||||
if purpose:
|
||||
clauses.append("purpose = ?")
|
||||
args.append(purpose)
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
f"""
|
||||
SELECT * FROM debug_runs
|
||||
WHERE {' AND '.join(clauses)}
|
||||
ORDER BY created DESC, id DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
args,
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
items = _fill_placement(profile_id, [_row_public(row, include_payload=True)])
|
||||
return items[0] if items else None
|
||||
|
||||
|
||||
def get_run(profile_id: str, run_id: str) -> dict | None:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT * FROM debug_runs WHERE id = ? AND profile_id = ?",
|
||||
(run_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
items = _fill_placement(profile_id, [_row_public(row, include_payload=True)])
|
||||
return items[0]
|
||||
|
||||
|
||||
def delete_run(profile_id: str, run_id: str) -> bool:
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
"DELETE FROM debug_runs WHERE id = ? AND profile_id = ?",
|
||||
(run_id, profile_id),
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
def clear_runs(profile_id: str) -> int:
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute("DELETE FROM debug_runs WHERE profile_id = ?", (profile_id,))
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def list_runs_full(profile_id: str) -> list[dict]:
|
||||
with get_db() as conn:
|
||||
rows = [
|
||||
dict(item)
|
||||
for item in conn.execute(
|
||||
"""
|
||||
SELECT * FROM debug_runs
|
||||
WHERE profile_id = ?
|
||||
ORDER BY created ASC, id ASC
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
]
|
||||
items = [_row_public(row, include_payload=True) for row in rows]
|
||||
return _fill_placement(profile_id, items)
|
||||
|
||||
|
||||
def _fill_placement(profile_id: str, items: list[dict]) -> list[dict]:
|
||||
missing_conv = [
|
||||
item.get("conversation_id")
|
||||
for item in items
|
||||
if item.get("conversation_id") and not item.get("space_id") and not item.get("journal_day_id")
|
||||
]
|
||||
missing_days = [
|
||||
item.get("journal_day_id")
|
||||
for item in items
|
||||
if item.get("journal_day_id") and not item.get("space_id")
|
||||
]
|
||||
conv_ids = list(dict.fromkeys(missing_conv))
|
||||
day_ids = list(dict.fromkeys(missing_days))
|
||||
convs: dict[str, dict] = {}
|
||||
days: dict[str, dict] = {}
|
||||
spaces: dict[str, dict] = {}
|
||||
with get_db() as conn:
|
||||
if conv_ids:
|
||||
marks = ",".join("?" * len(conv_ids))
|
||||
for row in conn.execute(
|
||||
f"SELECT id, title, space_id, journal_day_id FROM conversations WHERE profile_id = ? AND id IN ({marks})",
|
||||
[profile_id, *conv_ids],
|
||||
):
|
||||
convs[row["id"]] = dict(row)
|
||||
if row["journal_day_id"]:
|
||||
day_ids.append(row["journal_day_id"])
|
||||
day_ids = list(dict.fromkeys(day_ids))
|
||||
if day_ids:
|
||||
marks = ",".join("?" * len(day_ids))
|
||||
for row in conn.execute(
|
||||
f"SELECT id, calendar_date, space_id FROM journal_days WHERE profile_id = ? AND id IN ({marks})",
|
||||
[profile_id, *day_ids],
|
||||
):
|
||||
days[row["id"]] = dict(row)
|
||||
space_ids = [
|
||||
item
|
||||
for item in (
|
||||
[row.get("space_id") for row in convs.values()]
|
||||
+ [row.get("space_id") for row in days.values()]
|
||||
+ [item.get("space_id") for item in items]
|
||||
)
|
||||
if item
|
||||
]
|
||||
space_ids = list(dict.fromkeys(space_ids))
|
||||
if space_ids:
|
||||
marks = ",".join("?" * len(space_ids))
|
||||
for row in conn.execute(
|
||||
f"SELECT id, title FROM spaces WHERE profile_id = ? AND id IN ({marks})",
|
||||
[profile_id, *space_ids],
|
||||
):
|
||||
spaces[row["id"]] = dict(row)
|
||||
for item in items:
|
||||
conv = convs.get(item.get("conversation_id") or "")
|
||||
if conv:
|
||||
item["conversation_title"] = item.get("conversation_title") or conv.get("title") or ""
|
||||
item["space_id"] = item.get("space_id") or conv.get("space_id")
|
||||
item["journal_day_id"] = item.get("journal_day_id") or conv.get("journal_day_id")
|
||||
day = days.get(item.get("journal_day_id") or "")
|
||||
if day:
|
||||
item["calendar_date"] = item.get("calendar_date") or day.get("calendar_date") or ""
|
||||
item["space_id"] = item.get("space_id") or day.get("space_id")
|
||||
space = spaces.get(item.get("space_id") or "")
|
||||
if space:
|
||||
item["space_title"] = item.get("space_title") or space.get("title") or ""
|
||||
return items
|
||||
|
||||
|
||||
def _export_run(item: dict) -> dict:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"created": item["created"],
|
||||
"purpose": item["purpose"],
|
||||
"layer": item["layer"],
|
||||
"status": item["status"],
|
||||
"subject_type": item["subject_type"],
|
||||
"subject_id": item["subject_id"],
|
||||
"conversation_id": item.get("conversation_id"),
|
||||
"message_id": item.get("message_id"),
|
||||
"space_id": item.get("space_id"),
|
||||
"journal_day_id": item.get("journal_day_id"),
|
||||
"space_title": item.get("space_title") or "",
|
||||
"calendar_date": item.get("calendar_date") or "",
|
||||
"conversation_title": item.get("conversation_title") or "",
|
||||
"summary": item["summary"],
|
||||
**(item.get("payload") or {}),
|
||||
}
|
||||
|
||||
|
||||
def _step_summary(item: dict) -> dict:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"created": item["created"],
|
||||
"purpose": item["purpose"],
|
||||
"layer": item["layer"],
|
||||
"status": item["status"],
|
||||
"summary": item["summary"],
|
||||
"message_id": item.get("message_id"),
|
||||
}
|
||||
|
||||
|
||||
def _filter_runs(
|
||||
runs: list[dict],
|
||||
*,
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
space_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
) -> list[dict]:
|
||||
selected = runs
|
||||
if conversation_id:
|
||||
selected = [
|
||||
item
|
||||
for item in selected
|
||||
if item.get("conversation_id") == conversation_id and item.get("purpose") != "journal_generate"
|
||||
]
|
||||
elif journal_day_id:
|
||||
selected = [item for item in selected if item.get("journal_day_id") == journal_day_id]
|
||||
if space_id:
|
||||
selected = [item for item in selected if item.get("space_id") == space_id]
|
||||
if purpose:
|
||||
selected = [item for item in selected if item.get("purpose") == purpose]
|
||||
return selected
|
||||
|
||||
|
||||
def group_tree(runs: list[dict], *, include_payload: bool = False) -> list[dict]:
|
||||
spaces_map: dict[str, dict] = {}
|
||||
space_order: list[str] = []
|
||||
for run in runs:
|
||||
exported = _export_run(run) if include_payload else _step_summary(run)
|
||||
sid = run.get("space_id") or ""
|
||||
space_key = sid or "_unassigned"
|
||||
if space_key not in spaces_map:
|
||||
spaces_map[space_key] = {
|
||||
"id": sid or None,
|
||||
"title": run.get("space_title") or (UNASSIGNED_SPACE_TITLE if not sid else ""),
|
||||
"days_map": {},
|
||||
"day_order": [],
|
||||
}
|
||||
space_order.append(space_key)
|
||||
space = spaces_map[space_key]
|
||||
if run.get("space_title") and not space["title"]:
|
||||
space["title"] = run.get("space_title")
|
||||
did = run.get("journal_day_id") or ""
|
||||
day_key = did or "_unassigned"
|
||||
if day_key not in space["days_map"]:
|
||||
space["days_map"][day_key] = {
|
||||
"id": did or None,
|
||||
"calendar_date": run.get("calendar_date") or (UNASSIGNED_DAY_LABEL if not did else ""),
|
||||
"conversations_map": {},
|
||||
"conversation_order": [],
|
||||
"generates": [],
|
||||
}
|
||||
space["day_order"].append(day_key)
|
||||
day = space["days_map"][day_key]
|
||||
if run.get("calendar_date") and (not day["calendar_date"] or day["calendar_date"] == UNASSIGNED_DAY_LABEL):
|
||||
day["calendar_date"] = run.get("calendar_date")
|
||||
if run.get("purpose") == "journal_generate":
|
||||
day["generates"].append(exported)
|
||||
continue
|
||||
cid = run.get("conversation_id") or ""
|
||||
conv_key = cid or "_none"
|
||||
if conv_key not in day["conversations_map"]:
|
||||
day["conversations_map"][conv_key] = {
|
||||
"id": cid or None,
|
||||
"title": run.get("conversation_title") or "Gespräch",
|
||||
"steps": [],
|
||||
}
|
||||
day["conversation_order"].append(conv_key)
|
||||
conv = day["conversations_map"][conv_key]
|
||||
if run.get("conversation_title"):
|
||||
conv["title"] = run.get("conversation_title")
|
||||
conv["steps"].append(exported)
|
||||
spaces = []
|
||||
for space_key in space_order:
|
||||
space = spaces_map[space_key]
|
||||
days = []
|
||||
for day_key in space["day_order"]:
|
||||
day = space["days_map"][day_key]
|
||||
conversations = []
|
||||
for conv_key in day["conversation_order"]:
|
||||
conv = day["conversations_map"][conv_key]
|
||||
conversations.append(
|
||||
{
|
||||
"id": conv["id"],
|
||||
"title": conv["title"] or "Gespräch",
|
||||
"step_count": len(conv["steps"]),
|
||||
"steps": conv["steps"],
|
||||
}
|
||||
)
|
||||
days.append(
|
||||
{
|
||||
"id": day["id"],
|
||||
"calendar_date": day["calendar_date"] or UNASSIGNED_DAY_LABEL,
|
||||
"conversations": conversations,
|
||||
"generates": day["generates"],
|
||||
}
|
||||
)
|
||||
spaces.append({"id": space["id"], "title": space["title"] or UNASSIGNED_SPACE_TITLE, "days": days})
|
||||
return spaces
|
||||
|
||||
|
||||
def debug_tree(profile_id: str) -> dict:
|
||||
runs = list_runs_full(profile_id)
|
||||
return {"spaces": group_tree(runs, include_payload=False), "run_count": len(runs)}
|
||||
|
||||
|
||||
def export_document(
|
||||
profile_id: str,
|
||||
*,
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
) -> dict:
|
||||
runs = list_runs_full(profile_id)
|
||||
selected = _filter_runs(
|
||||
runs,
|
||||
conversation_id=conversation_id,
|
||||
journal_day_id=journal_day_id,
|
||||
purpose=purpose,
|
||||
)
|
||||
scope = "all"
|
||||
if purpose == "journal_generate":
|
||||
scope = "journal_generate"
|
||||
elif conversation_id:
|
||||
scope = "conversation"
|
||||
elif journal_day_id:
|
||||
scope = "day"
|
||||
spaces = group_tree(selected, include_payload=True)
|
||||
first_space = spaces[0] if spaces else None
|
||||
first_day = ((first_space or {}).get("days") or [None])[0] if first_space else None
|
||||
first_conv = ((first_day or {}).get("conversations") or [None])[0] if first_day else None
|
||||
return {
|
||||
"kind": EXPORT_KIND,
|
||||
"version": EXPORT_VERSION,
|
||||
"exported_at": _now(),
|
||||
"app_version": APP_VERSION,
|
||||
"build_date": BUILD_DATE,
|
||||
"scope": scope,
|
||||
"note": (
|
||||
"Lokale Entwicklungsdiagnose, gegliedert nach Space / Tag / Gespräch / Schritt. "
|
||||
"Mapping-Tabelle und Secrets sind ausgeschlossen. "
|
||||
"Enthält Klartext-Vorlagen (intern) und maskierten Egress des eigenen Admin-Profils. "
|
||||
"Nicht an Provider senden, außer bewusst zur lokalen Auswertung."
|
||||
),
|
||||
"persist_enabled": persist_enabled(),
|
||||
"run_count": len(selected),
|
||||
"space": {"id": (first_space or {}).get("id"), "title": (first_space or {}).get("title")} if scope != "all" else None,
|
||||
"day": (
|
||||
{"id": (first_day or {}).get("id"), "calendar_date": (first_day or {}).get("calendar_date")}
|
||||
if scope in {"conversation", "day", "journal_generate"}
|
||||
else None
|
||||
),
|
||||
"conversation": (
|
||||
{"id": (first_conv or {}).get("id"), "title": (first_conv or {}).get("title")}
|
||||
if scope == "conversation"
|
||||
else None
|
||||
),
|
||||
"spaces": spaces,
|
||||
"runs": [_export_run(item) for item in selected],
|
||||
}
|
||||
|
||||
|
||||
def _fence(text: str | None) -> str:
|
||||
body = (text or "").replace("```", "'''")
|
||||
return f"```\n{body}\n```" if body.strip() else "_leer_"
|
||||
|
||||
|
||||
def _kv(title: str, value: Any) -> str:
|
||||
if value is None or value == "":
|
||||
return f"- {title}: nicht verfügbar"
|
||||
if isinstance(value, (dict, list)):
|
||||
return f"- {title}: `{json.dumps(value, ensure_ascii=False)}`"
|
||||
return f"- {title}: {value}"
|
||||
|
||||
|
||||
def _render_trace_markdown(trace: dict | None, heading: str) -> list[str]:
|
||||
if not trace:
|
||||
return [f"### {heading}", "", "_keine Testspur_", ""]
|
||||
lines = [f"### {heading}", ""]
|
||||
lines.append(_kv("Zweck", trace.get("purpose")))
|
||||
lines.append(_kv("Schicht", trace.get("layer")))
|
||||
lines.append(_kv("Prompt", trace.get("prompt_slug")))
|
||||
lines.append(_kv("Prompt-Revision", trace.get("prompt_revision")))
|
||||
lines.append(_kv("Provider", trace.get("provider")))
|
||||
lines.append(_kv("Modell", trace.get("model")))
|
||||
lines.append(_kv("Detect", trace.get("detect_note") or trace.get("detect_provider")))
|
||||
lines.append(_kv("Detect-Modell", trace.get("detect_model")))
|
||||
lines.append(_kv("Abbruchgrund", trace.get("abort_reason")))
|
||||
lines.append(_kv("Modelltext übernommen", trace.get("model_text_accepted")))
|
||||
budget = trace.get("budget") or {}
|
||||
if budget:
|
||||
lines.append(_kv("Prompt-Tokens", budget.get("prompt_tokens")))
|
||||
lines.append(_kv("Completion-Tokens", budget.get("completion_tokens")))
|
||||
lines.append(_kv("Kosten", budget.get("cost")))
|
||||
lines.append(_kv("Generate-ms", budget.get("generate_ms") or trace.get("generate_ms")))
|
||||
lines.extend(["", "#### Intern (Klartext-Vorlage, nicht gesendet)", "", _fence(trace.get("intern")), ""])
|
||||
lines.extend(["#### Egress (maskiert, gesendet)", "", _fence(trace.get("egress")), ""])
|
||||
lines.extend(["#### Maskierung, Eingabe", "", _fence(trace.get("mask_input")), ""])
|
||||
lines.extend(["#### Modellantwort roh", "", _fence(trace.get("raw")), ""])
|
||||
lines.extend(["#### Lokal demaskiert", "", _fence(trace.get("reply")), ""])
|
||||
stored = "\n\n".join(part for part in (trace.get("stored_title"), trace.get("stored_body")) if part)
|
||||
if stored:
|
||||
lines.extend(["#### Gespeicherter Entwurf", "", _fence(stored), ""])
|
||||
if trace.get("source_preview"):
|
||||
lines.extend(["#### Lokale Quellenansicht", "", _fence(trace.get("source_preview")), ""])
|
||||
return lines
|
||||
|
||||
|
||||
def render_markdown(document: dict) -> str:
|
||||
lines = [
|
||||
"# Kanshō Debug-Export",
|
||||
"",
|
||||
_kv("kind", document.get("kind")),
|
||||
_kv("version", document.get("version")),
|
||||
_kv("scope", document.get("scope")),
|
||||
_kv("exported_at", document.get("exported_at")),
|
||||
_kv("app_version", document.get("app_version")),
|
||||
_kv("build_date", document.get("build_date")),
|
||||
_kv("run_count", document.get("run_count")),
|
||||
_kv("persist_enabled", document.get("persist_enabled")),
|
||||
"",
|
||||
document.get("note") or "",
|
||||
"",
|
||||
"Gliederung: Space / Tag / Gespräch / Schritt. Die Schritte eines Gesprächs sind chronologisch.",
|
||||
"",
|
||||
]
|
||||
if (document.get("space") or {}).get("title"):
|
||||
lines.append(_kv("Space", (document.get("space") or {}).get("title")))
|
||||
if (document.get("day") or {}).get("calendar_date"):
|
||||
lines.append(_kv("Tag", (document.get("day") or {}).get("calendar_date")))
|
||||
if (document.get("conversation") or {}).get("title"):
|
||||
lines.append(_kv("Gespräch", (document.get("conversation") or {}).get("title")))
|
||||
lines.append("")
|
||||
spaces = document.get("spaces") or []
|
||||
if spaces:
|
||||
for space in spaces:
|
||||
lines.extend(["---", "", f"# Space: {space.get('title') or UNASSIGNED_SPACE_TITLE}", ""])
|
||||
for day in space.get("days") or []:
|
||||
lines.extend([f"## Tag: {day.get('calendar_date') or UNASSIGNED_DAY_LABEL}", ""])
|
||||
for conv in day.get("conversations") or []:
|
||||
lines.extend([f"### Gespräch: {conv.get('title') or 'Gespräch'}", ""])
|
||||
for index, run in enumerate(conv.get("steps") or [], start=1):
|
||||
lines.extend(_render_run_markdown(run, index))
|
||||
for index, run in enumerate(day.get("generates") or [], start=1):
|
||||
lines.extend([f"### Journalentwurf {index}", ""])
|
||||
lines.extend(_render_run_markdown(run, index))
|
||||
else:
|
||||
for index, run in enumerate(document.get("runs") or [], start=1):
|
||||
lines.extend(_render_run_markdown(run, index))
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _render_run_markdown(run: dict, index: int) -> list[str]:
|
||||
created = run.get("created") or ""
|
||||
purpose = run.get("purpose") or ""
|
||||
status = run.get("status") or ""
|
||||
lines = [
|
||||
"---",
|
||||
"",
|
||||
f"#### {index}. {purpose} · {created} · {status}",
|
||||
"",
|
||||
_kv("id", run.get("id")),
|
||||
_kv("layer", run.get("layer")),
|
||||
_kv("conversation_id", run.get("conversation_id")),
|
||||
_kv("code", run.get("code")),
|
||||
]
|
||||
decision = run.get("decision")
|
||||
if decision:
|
||||
lines.append(_kv("Operation", decision.get("label") or decision.get("operation")))
|
||||
if decision.get("guard"):
|
||||
lines.append(_kv("Guard", decision.get("guard")))
|
||||
user = run.get("user") or {}
|
||||
assistant = run.get("assistant") or {}
|
||||
if user.get("body"):
|
||||
lines.extend(["", "##### Nutzer", "", _fence(user.get("body")), ""])
|
||||
if assistant.get("body"):
|
||||
lines.extend(["##### Antwort", "", _fence(assistant.get("body")), ""])
|
||||
traces = []
|
||||
if run.get("call_traces"):
|
||||
traces = list(run.get("call_traces") or [])
|
||||
elif run.get("trace"):
|
||||
nested = (run.get("trace") or {}).get("stages")
|
||||
traces = list(nested) if nested else [run.get("trace")]
|
||||
if not traces and run.get("trace"):
|
||||
traces = [run.get("trace")]
|
||||
for offset, stage in enumerate(traces):
|
||||
title = f"Schritt {offset + 1}"
|
||||
if isinstance(stage, dict) and stage.get("purpose"):
|
||||
title = f"Schritt {offset + 1}: {stage.get('purpose')}"
|
||||
lines.extend(_render_trace_markdown(stage if isinstance(stage, dict) else None, title))
|
||||
log = run.get("run_log") or ((run.get("trace") or {}).get("log") if isinstance(run.get("trace"), dict) else None)
|
||||
if log:
|
||||
lines.extend(
|
||||
[
|
||||
"##### Kompaktes Laufprotokoll (ohne Promptkörper)",
|
||||
"",
|
||||
_fence(json.dumps(log, ensure_ascii=False, indent=2)),
|
||||
"",
|
||||
]
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def export_filename(fmt: str, document: dict | None = None) -> str:
|
||||
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
||||
suffix = "md" if fmt == "markdown" else "json"
|
||||
scope = (document or {}).get("scope") or "all"
|
||||
if scope == "conversation":
|
||||
return f"kansho-debug-conversation-{stamp}.{suffix}"
|
||||
if scope == "journal_generate":
|
||||
return f"kansho-debug-journal-{stamp}.{suffix}"
|
||||
if scope == "day":
|
||||
return f"kansho-debug-day-{stamp}.{suffix}"
|
||||
return f"kansho-debug-{stamp}.{suffix}"
|
||||
@@ -6,6 +6,7 @@ import re
|
||||
|
||||
from context_builder import assemble_text, build_internal_context, is_closing_turn
|
||||
from conversation_signals import infer_signals
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
|
||||
from engine import EngineError, execute_prompt, load_active_prompt
|
||||
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES
|
||||
@@ -317,6 +318,7 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||
result = None
|
||||
impulse = ""
|
||||
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
|
||||
call_traces: list[dict] = []
|
||||
try:
|
||||
while calls < 2:
|
||||
result = execute_prompt(
|
||||
@@ -327,6 +329,8 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||
context=assembled,
|
||||
)
|
||||
calls += 1
|
||||
if result.get("trace"):
|
||||
call_traces.append(result.get("trace"))
|
||||
content = (result.get("content") or "").strip()
|
||||
if not content:
|
||||
raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.")
|
||||
@@ -348,7 +352,22 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||
"guard": "local_fallback",
|
||||
}
|
||||
except EngineError as exc:
|
||||
if result and result.get("trace") and result.get("trace") not in call_traces:
|
||||
call_traces.append(result.get("trace"))
|
||||
if exc.code != "response_validation_failed" and exc.code not in DETECT_DIALOGUE_FALLBACK_CODES:
|
||||
persist_engine_error(
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
exc=exc,
|
||||
subject_type="conversation",
|
||||
subject_id=conversation_id,
|
||||
conversation_id=conversation_id,
|
||||
extra={
|
||||
"calls": calls,
|
||||
"call_traces": call_traces,
|
||||
"user": {"id": user.get("id"), "body": user.get("body")},
|
||||
},
|
||||
)
|
||||
raise
|
||||
impulse = local_hold(last_user_text(assembled))
|
||||
decision = {
|
||||
@@ -370,6 +389,24 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||
)
|
||||
remember_dialogue_style(profile_id)
|
||||
consider_dialogue(profile_id, body)
|
||||
trace = result.get("trace") if result else None
|
||||
persist_step(
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
status="ok",
|
||||
subject_type="conversation",
|
||||
subject_id=conversation_id,
|
||||
conversation_id=conversation_id,
|
||||
message_id=assistant.get("id"),
|
||||
decision=decision,
|
||||
trace=trace,
|
||||
extra={
|
||||
"calls": calls,
|
||||
"call_traces": call_traces,
|
||||
"user": {"id": user.get("id"), "body": user.get("body")},
|
||||
"assistant": {"id": assistant.get("id"), "body": assistant.get("body")},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"conversation": get_conversation(profile_id, conversation_id),
|
||||
"user": user,
|
||||
@@ -377,5 +414,5 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||
"calls": calls,
|
||||
"messages": list_messages(profile_id, conversation_id),
|
||||
"decision": decision,
|
||||
"trace": result.get("trace") if result else None,
|
||||
"trace": trace,
|
||||
}
|
||||
@@ -18,6 +18,7 @@ from identity_store import (
|
||||
ENTITY_TYPES,
|
||||
confirmed_match_labels,
|
||||
is_maskable_label,
|
||||
is_registry_maskable_label,
|
||||
list_confirmed_identities,
|
||||
masking_rows_from_confirmed,
|
||||
normalize_entity_type,
|
||||
@@ -109,6 +110,8 @@ class DetectionStats:
|
||||
entity_counts: dict[str, int] = field(default_factory=dict)
|
||||
confirmed_registry_hits: int = 0
|
||||
request_local_hits: int = 0
|
||||
confirmed_registry_applied: bool = False
|
||||
semantic_identity_guaranteed: bool = False
|
||||
detect_calls: int = 0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
@@ -128,7 +131,9 @@ class DetectionStats:
|
||||
"full_detection_coverage": self.full_detection_coverage,
|
||||
"entity_counts": dict(self.entity_counts),
|
||||
"confirmed_registry_hits": self.confirmed_registry_hits,
|
||||
"confirmed_registry_applied": self.confirmed_registry_applied,
|
||||
"request_local_hits": self.request_local_hits,
|
||||
"semantic_identity_guaranteed": False,
|
||||
"detect_calls": self.detect_calls,
|
||||
"detect_prompt_tokens": self.prompt_tokens,
|
||||
"detect_completion_tokens": self.completion_tokens,
|
||||
@@ -547,12 +552,16 @@ def _assign_request_tokens(
|
||||
def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str | None) -> tuple[list[dict], int]:
|
||||
if not profile_id:
|
||||
return mappings, 0
|
||||
already_labels = {(item.get("local_label") or "").casefold() for item in mappings if item.get("start") is None}
|
||||
already_labels = {
|
||||
(item.get("local_label") or "").casefold()
|
||||
for item in mappings
|
||||
if item.get("start") is None and item.get("source") == "confirmed_registry"
|
||||
}
|
||||
extra: list[dict] = []
|
||||
hits = 0
|
||||
for row in masking_rows_from_confirmed(profile_id):
|
||||
label = row.get("local_label") or ""
|
||||
if not label or label.casefold() in already_labels:
|
||||
if not label or not is_registry_maskable_label(label) or label.casefold() in already_labels:
|
||||
continue
|
||||
if not re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text or "", re.IGNORECASE):
|
||||
continue
|
||||
@@ -680,6 +689,8 @@ def detect_personal_egress(profile_id: str | None, source_text: str) -> Detectio
|
||||
mappings, extra_confirmed = _merge_confirmed_safety_net(source_text or "", mappings, profile_id)
|
||||
stats.request_local_hits = request_local
|
||||
stats.confirmed_registry_hits = confirmed_from_spans + extra_confirmed
|
||||
stats.confirmed_registry_applied = bool(profile_id)
|
||||
stats.semantic_identity_guaranteed = False
|
||||
counts: dict[str, int] = {kind: 0 for kind in ENTITY_TYPES}
|
||||
for item in mappings:
|
||||
kind = (item.get("entity_type") or "PERSON").upper()
|
||||
|
||||
@@ -73,6 +73,22 @@ def is_maskable_label(label: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def is_registry_maskable_label(label: str) -> bool:
|
||||
"""Confirmed names may be shorter than the detect gate. Not a semantic word list."""
|
||||
core = (label or "").strip()
|
||||
if len(core) < 2:
|
||||
return False
|
||||
if core.lower() in STRUCTURAL_UNMASKABLE:
|
||||
return False
|
||||
if TIME_RE.search(core):
|
||||
return False
|
||||
if re.fullmatch(r"user|assistant|self", core, re.IGNORECASE):
|
||||
return False
|
||||
if not re.search(r"[A-Za-zÀ-žÄÖÜäöüß]", core):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _token_suffix(token: str) -> str:
|
||||
raw = (token or "").upper()
|
||||
return raw.split(":")[-1] if ":" in token else raw
|
||||
@@ -111,7 +127,7 @@ def parse_aliases(raw) -> list[str]:
|
||||
result: list[str] = []
|
||||
for item in items:
|
||||
label = (item or "").strip()
|
||||
if not label or label.casefold() in seen or not is_maskable_label(label):
|
||||
if not label or label.casefold() in seen or not is_registry_maskable_label(label):
|
||||
continue
|
||||
seen.add(label.casefold())
|
||||
result.append(label)
|
||||
@@ -189,7 +205,7 @@ def mapping_spellings(item: dict) -> list[str]:
|
||||
|
||||
def add(raw) -> None:
|
||||
label = (raw or "").strip()
|
||||
if not label or label.casefold() in seen or not is_maskable_label(label):
|
||||
if not label or label.casefold() in seen or not is_registry_maskable_label(label):
|
||||
return
|
||||
seen.add(label.casefold())
|
||||
result.append(label)
|
||||
@@ -214,7 +230,7 @@ def masking_rows_from_confirmed(profile_id: str) -> list[dict]:
|
||||
if not token or not canonical:
|
||||
continue
|
||||
for label in confirmed_match_labels(item):
|
||||
if not is_maskable_label(label):
|
||||
if not is_registry_maskable_label(label):
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
@@ -274,7 +290,7 @@ def confirm_identity(
|
||||
identity_id: str | None = None,
|
||||
) -> dict:
|
||||
label = normalize_label(canonical_label) or (canonical_label or "").strip()
|
||||
if not label or not is_maskable_label(label):
|
||||
if not label or not is_registry_maskable_label(label):
|
||||
raise ValueError("empty_label")
|
||||
kind = normalize_entity_type(entity_type)
|
||||
if not kind:
|
||||
@@ -360,7 +376,7 @@ def update_identity(
|
||||
label = found["canonical_label"]
|
||||
if canonical_label is not None:
|
||||
label = normalize_label(canonical_label) or canonical_label.strip()
|
||||
if not label or not is_maskable_label(label):
|
||||
if not label or not is_registry_maskable_label(label):
|
||||
raise ValueError("empty_label")
|
||||
kind = found["entity_type"]
|
||||
if entity_type is not None:
|
||||
@@ -525,7 +541,7 @@ def purge_unmaskable(profile_id: str) -> int:
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
label = row["canonical_label"] or row["local_label"]
|
||||
if is_maskable_label(label):
|
||||
if is_registry_maskable_label(label):
|
||||
continue
|
||||
conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],))
|
||||
removed += 1
|
||||
@@ -551,7 +567,7 @@ def coalesce_mappings(profile_id: str) -> int:
|
||||
groups: dict[str, list[dict]] = {}
|
||||
for row in rows:
|
||||
core = normalize_label(row.get("canonical_label") or row.get("local_label") or "")
|
||||
if not core or not is_maskable_label(core):
|
||||
if not core or not is_registry_maskable_label(core):
|
||||
conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],))
|
||||
merged += 1
|
||||
continue
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
import re
|
||||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from dialogue_store import StoreError, list_conversations_for_day, list_messages
|
||||
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
||||
from journal_policy import require_explicit_generate, source_conversation_ids
|
||||
@@ -356,6 +357,57 @@ def generate_draft(
|
||||
explicit: bool = True,
|
||||
generation_selection: dict | None = None,
|
||||
remember_generation_selection: bool = False,
|
||||
) -> dict:
|
||||
try:
|
||||
draft = _generate_draft(
|
||||
profile_id,
|
||||
journal_day_id,
|
||||
conversation_ids=conversation_ids,
|
||||
include_existing=include_existing,
|
||||
explicit=explicit,
|
||||
generation_selection=generation_selection,
|
||||
remember_generation_selection=remember_generation_selection,
|
||||
)
|
||||
except EngineError as exc:
|
||||
persist_engine_error(
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
exc=exc,
|
||||
subject_type="journal_day",
|
||||
subject_id=journal_day_id,
|
||||
journal_day_id=journal_day_id,
|
||||
extra={"conversation_ids": conversation_ids or []},
|
||||
)
|
||||
raise
|
||||
persist_step(
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
status="ok",
|
||||
subject_type="journal_day",
|
||||
subject_id=journal_day_id,
|
||||
journal_day_id=journal_day_id,
|
||||
conversation_id=(conversation_ids[0] if conversation_ids else None),
|
||||
decision=None,
|
||||
trace=draft.get("trace"),
|
||||
extra={
|
||||
"draft_id": draft.get("id"),
|
||||
"run_log": draft.get("run_log"),
|
||||
"conversation_ids": conversation_ids or [],
|
||||
"stored_title": draft.get("title"),
|
||||
"stored_body": draft.get("body"),
|
||||
},
|
||||
)
|
||||
return draft
|
||||
|
||||
|
||||
def _generate_draft(
|
||||
profile_id: str,
|
||||
journal_day_id: str,
|
||||
conversation_ids: list[str] | None = None,
|
||||
include_existing: bool = False,
|
||||
explicit: bool = True,
|
||||
generation_selection: dict | None = None,
|
||||
remember_generation_selection: bool = False,
|
||||
) -> dict:
|
||||
require_explicit_generate(explicit)
|
||||
day = get_day(profile_id, journal_day_id)
|
||||
@@ -602,6 +654,7 @@ def generate_draft(
|
||||
person_labels=person_labels,
|
||||
source="model",
|
||||
)
|
||||
stored = f"{title}\n\n{body}".strip() if title else (body or "").strip()
|
||||
before_entries = {item["id"]: item.get("current_version_id") for item in current_entries(profile_id, journal_day_id)}
|
||||
model = (
|
||||
((narrate_result.get("diagnostics") or {}).get("actual_model"))
|
||||
@@ -647,6 +700,9 @@ def generate_draft(
|
||||
"lexical_similarity": lexical_similarity("\n".join(source_user), body),
|
||||
"incomplete_syntax": incomplete_syntax_markers(body),
|
||||
"generation_selection": policy_meta,
|
||||
"stored_title": title,
|
||||
"stored_body": body,
|
||||
"reply": stored,
|
||||
},
|
||||
)
|
||||
draft["run_log"] = run_log
|
||||
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
import re
|
||||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from dialogue_store import StoreError, append_message, get_conversation, list_conversations_for_day, list_messages
|
||||
from dialogue_turn import OPERATIONS, needs_repair, parse_turn_payload, repair_note
|
||||
from engine import EngineError, execute_prompt, load_active_prompt
|
||||
@@ -162,12 +163,31 @@ def format_opening_hint(facts: dict) -> str:
|
||||
|
||||
def _store_opening(profile_id: str, conversation_id: str, impulse: str, extra: dict) -> dict:
|
||||
assistant = append_message(profile_id, conversation_id, impulse, role="assistant")
|
||||
return {
|
||||
payload = {
|
||||
"conversation": get_conversation(profile_id, conversation_id),
|
||||
"assistant": assistant,
|
||||
"messages": list_messages(profile_id, conversation_id),
|
||||
**extra,
|
||||
}
|
||||
persist_step(
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
status="ok",
|
||||
layer="dialogzug",
|
||||
subject_type="conversation",
|
||||
subject_id=conversation_id,
|
||||
conversation_id=conversation_id,
|
||||
message_id=assistant.get("id"),
|
||||
decision=extra.get("decision"),
|
||||
trace=extra.get("trace"),
|
||||
extra={
|
||||
"kind": extra.get("kind"),
|
||||
"calls": extra.get("calls"),
|
||||
"opened": extra.get("opened"),
|
||||
"assistant": {"id": assistant.get("id"), "body": assistant.get("body")},
|
||||
},
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def start_journal_opening(profile_id: str, conversation_id: str) -> dict:
|
||||
@@ -262,6 +282,15 @@ def start_journal_opening(profile_id: str, conversation_id: str) -> dict:
|
||||
}
|
||||
except EngineError as exc:
|
||||
if exc.code != "response_validation_failed" and exc.code not in DETECT_DIALOGUE_FALLBACK_CODES:
|
||||
persist_engine_error(
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
exc=exc,
|
||||
subject_type="conversation",
|
||||
subject_id=conversation_id,
|
||||
conversation_id=conversation_id,
|
||||
extra={"kind": "opening_failed", "calls": calls},
|
||||
)
|
||||
raise
|
||||
return _store_opening(
|
||||
profile_id,
|
||||
|
||||
+6
-123
@@ -1,4 +1,4 @@
|
||||
"""Local journal shape after the model. Does not invent content."""
|
||||
"""Local journal shape after the model. Does not invent or rewrite content."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
@@ -10,26 +10,6 @@ NEW_PHASE = re.compile(
|
||||
r"Als\s+(?:wir|ich)\b)",
|
||||
re.I,
|
||||
)
|
||||
_LETTER = r"A-Za-zÄÖÜäöüß"
|
||||
_WORD = re.compile(rf"[{_LETTER}]+")
|
||||
DATIVE_PREPS = {
|
||||
"mit", "bei", "von", "zu", "nach", "aus", "außer", "gegenüber", "gemäß",
|
||||
}
|
||||
NOM_CUES = {
|
||||
"dass", "daß", "ob", "weil", "wenn", "als", "während", "obwohl", "damit",
|
||||
"nachdem", "bevor", "sodass", "sofern",
|
||||
}
|
||||
THING_GOVERNORS = {
|
||||
"esse", "essen", "isst", "aß", "aßest", "gegessen",
|
||||
"koche", "kochen", "kochte", "gekocht",
|
||||
"trinke", "trinken", "trank", "getrunken",
|
||||
"kaufe", "kaufen", "kaufte", "gekauft",
|
||||
"bestelle", "bestellen", "bestellte",
|
||||
"hole", "holen", "holte",
|
||||
}
|
||||
FEM_HINTS = {"sie", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres"}
|
||||
MASC_HINTS = {"er", "ihn", "ihm", "seine", "seinem", "seinen", "seiner", "seines"}
|
||||
KEEP_NAMES_PER_PARAGRAPH = 2
|
||||
|
||||
|
||||
def _strip_roles(text: str) -> str:
|
||||
@@ -55,6 +35,7 @@ def is_verbatim_join(body: str, user_bodies: list[str]) -> bool:
|
||||
|
||||
|
||||
def paragraphize(body: str) -> str:
|
||||
"""Optional local preview helper. Must not rewrite an accepted model draft."""
|
||||
text = (body or "").strip()
|
||||
if "\n\n" in text:
|
||||
return text
|
||||
@@ -75,99 +56,6 @@ def paragraphize(body: str) -> str:
|
||||
return "\n\n".join(chunks)
|
||||
|
||||
|
||||
def _words_before(text: str, index: int, n: int = 4) -> list[str]:
|
||||
return [word.lower() for word in _WORD.findall(text[:index])[-n:]]
|
||||
|
||||
|
||||
def _words_after(text: str, index: int, n: int = 3) -> list[str]:
|
||||
return [word.lower() for word in _WORD.findall(text[index:])[:n]]
|
||||
|
||||
|
||||
def _is_thing_mention(text: str, start: int, end: int) -> bool:
|
||||
prev = _words_before(text, start)
|
||||
nxt = _words_after(text, end)
|
||||
return any(word in THING_GOVERNORS for word in prev + nxt)
|
||||
|
||||
|
||||
def _sentence_start(text: str, index: int) -> bool:
|
||||
prefix = text[:index].rstrip()
|
||||
return not prefix or prefix[-1] in ".!?\n"
|
||||
|
||||
|
||||
def infer_person_gender(label: str, source_texts: list[str]) -> str | None:
|
||||
blob = " ".join(source_texts or [])
|
||||
if not label or not blob:
|
||||
return None
|
||||
pattern = re.compile(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", re.I)
|
||||
first = pattern.search(blob)
|
||||
if not first:
|
||||
return None
|
||||
after = [word.lower() for word in _WORD.findall(blob[first.start():])]
|
||||
fem = sum(1 for word in after if word in FEM_HINTS)
|
||||
masc = sum(1 for word in after if word in MASC_HINTS)
|
||||
if fem > masc:
|
||||
return "f"
|
||||
if masc > fem:
|
||||
return "m"
|
||||
return None
|
||||
|
||||
|
||||
def _pronoun(gender: str, prev: str, sentence_start: bool) -> str:
|
||||
dative = prev in DATIVE_PREPS
|
||||
if gender == "f":
|
||||
form = "ihr" if dative else "sie"
|
||||
elif sentence_start or prev in NOM_CUES:
|
||||
form = "er"
|
||||
elif dative:
|
||||
form = "ihm"
|
||||
else:
|
||||
form = "ihn"
|
||||
if sentence_start:
|
||||
return form[:1].upper() + form[1:]
|
||||
return form
|
||||
|
||||
|
||||
def naturalize_person_mentions(body: str, labels: list[str], source_texts: list[str]) -> str:
|
||||
"""After demask: keep the name once or twice per passage, then pronouns. No new facts."""
|
||||
text = body or ""
|
||||
people = []
|
||||
for label in sorted({item.strip() for item in (labels or []) if (item or "").strip()}, key=len, reverse=True):
|
||||
gender = infer_person_gender(label, source_texts)
|
||||
if gender:
|
||||
people.append((label, gender))
|
||||
if not people:
|
||||
return text
|
||||
paragraphs = re.split(r"(\n\n+)", text)
|
||||
out: list[str] = []
|
||||
for block in paragraphs:
|
||||
if not block or block.startswith("\n"):
|
||||
out.append(block)
|
||||
continue
|
||||
piece = block
|
||||
for label, gender in people:
|
||||
pattern = re.compile(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", re.I)
|
||||
matches = [
|
||||
match
|
||||
for match in pattern.finditer(piece)
|
||||
if not _is_thing_mention(piece, match.start(), match.end())
|
||||
]
|
||||
if len(matches) <= KEEP_NAMES_PER_PARAGRAPH:
|
||||
continue
|
||||
replacements: list[tuple[int, int, str]] = []
|
||||
kept = 0
|
||||
for match in matches:
|
||||
kept += 1
|
||||
if kept <= KEEP_NAMES_PER_PARAGRAPH:
|
||||
continue
|
||||
prev = (_words_before(piece, match.start(), 1) or [""])[0]
|
||||
form = _pronoun(gender, prev, _sentence_start(piece, match.start()))
|
||||
replacements.append((match.start(), match.end(), form))
|
||||
for start, end, form in reversed(replacements):
|
||||
piece = piece[:start] + form + piece[end:]
|
||||
out.append(piece)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def shape_journal(
|
||||
title: str,
|
||||
body: str,
|
||||
@@ -176,15 +64,10 @@ def shape_journal(
|
||||
*,
|
||||
source: str = "model",
|
||||
) -> tuple[str, str]:
|
||||
"""Local post-shape. Never replaces accepted model wording with source turns."""
|
||||
turns = _turns(user_bodies)
|
||||
text = (body or "").strip()
|
||||
"""Safe technical trim only. Never replaces names, pronouns, or accepted wording."""
|
||||
del user_bodies, person_labels
|
||||
heading = (title or "").strip()
|
||||
text = _strip_roles(text)
|
||||
if source != "fallback":
|
||||
text = _strip_roles(body or "").strip()
|
||||
if source == "fallback" and "\n\n" not in text:
|
||||
text = paragraphize(text)
|
||||
elif "\n\n" not in text:
|
||||
text = paragraphize(text)
|
||||
if person_labels:
|
||||
text = naturalize_person_mentions(text, person_labels, turns)
|
||||
return heading, text
|
||||
+35
-11
@@ -9,7 +9,7 @@ import re
|
||||
import time
|
||||
|
||||
from entity_detect import DetectError, detect_personal_egress, reset_detect_test_hooks, user_detect_message
|
||||
from identity_store import KINSHIP, is_maskable_label, mapping_spellings
|
||||
from identity_store import KINSHIP, is_maskable_label, is_registry_maskable_label, mapping_spellings
|
||||
from journal_reconstruct import claim_texts, fake_reconstruction, is_dialogue_role_line
|
||||
from prompt_budget import (
|
||||
ERROR_PROVIDER_CONTEXT_LENGTH,
|
||||
@@ -75,7 +75,9 @@ COMPACT_DIAGNOSTIC_KEYS = (
|
||||
"chunks_ok",
|
||||
"entity_counts",
|
||||
"confirmed_registry_hits",
|
||||
"confirmed_registry_applied",
|
||||
"request_local_hits",
|
||||
"semantic_identity_guaranteed",
|
||||
"detect_calls",
|
||||
"detect_prompt_tokens",
|
||||
"detect_completion_tokens",
|
||||
@@ -338,7 +340,7 @@ def is_identity_mention(text: str, start: int, end: int, token: str) -> bool:
|
||||
|
||||
def identity_occurrence_count(text: str, label: str, token: str) -> int:
|
||||
"""Public, intent-neutral count of identity mentions for one mapping."""
|
||||
if not label or not is_maskable_label(label):
|
||||
if not label or not is_registry_maskable_label(label):
|
||||
return 0
|
||||
return sum(
|
||||
1
|
||||
@@ -373,12 +375,22 @@ def _identity_occurrence_count(text: str, label: str, token: str) -> int:
|
||||
return identity_occurrence_count(text, label, token)
|
||||
|
||||
|
||||
def _mapping_label_allowed(item: dict) -> bool:
|
||||
label = (item.get("local_label") or "").strip()
|
||||
token = (item.get("token") or "").strip()
|
||||
if not label or not token:
|
||||
return False
|
||||
if (item.get("source") or "") == "confirmed_registry":
|
||||
return is_registry_maskable_label(label)
|
||||
return is_maskable_label(label)
|
||||
|
||||
|
||||
def _active_replacements(text: str, mappings: list[dict]) -> tuple[ActiveReplacement, ...]:
|
||||
found: list[ActiveReplacement] = []
|
||||
for item in mappings or []:
|
||||
label = (item.get("local_label") or "").strip()
|
||||
token = (item.get("token") or "").strip()
|
||||
if not label or not token or not is_maskable_label(label):
|
||||
if not _mapping_label_allowed(item):
|
||||
continue
|
||||
count = _identity_occurrence_count(text, label, token)
|
||||
if count <= 0:
|
||||
@@ -447,7 +459,7 @@ def _mask_body(text: str, mappings: list[dict]) -> str:
|
||||
for item in sorted(mappings, key=lambda row: len(row.get("local_label") or ""), reverse=True):
|
||||
label = (item.get("local_label") or "").strip()
|
||||
token = (item.get("token") or "").strip()
|
||||
if not label or not token or not is_maskable_label(label):
|
||||
if not _mapping_label_allowed(item):
|
||||
continue
|
||||
placeholder = _placeholder_of(token)
|
||||
pattern = _label_pattern(label)
|
||||
@@ -572,7 +584,7 @@ def mask_prompt(rendered: str, mappings: list[dict], purpose: str) -> MaskingMan
|
||||
for item in mappings or []:
|
||||
label = (item.get("local_label") or "").strip()
|
||||
token = (item.get("token") or "").strip()
|
||||
if not label or not token or not is_maskable_label(label):
|
||||
if not _mapping_label_allowed(item):
|
||||
continue
|
||||
has_span_fields = item.get("start") is not None and item.get("end") is not None
|
||||
grounded = _ground_mapping_span(text, item) if has_span_fields else None
|
||||
@@ -683,8 +695,13 @@ def _demask(text: str, manifest: MaskingManifest | None) -> str:
|
||||
return PLACEHOLDER_RE.sub(repl, result)
|
||||
|
||||
|
||||
def _visible_response(content: str, manifest: MaskingManifest | None) -> str:
|
||||
"""Stored output: original model text with only emitted placeholders rehydrated."""
|
||||
return _demask((content or "").strip(), manifest)
|
||||
|
||||
|
||||
def _normalize_active_cleartext(text: str, manifest: MaskingManifest | None) -> tuple[str, int]:
|
||||
"""Map active identity cleartext back to the request token. Local only, no retry."""
|
||||
"""Map active identity cleartext back to the request token. Check copy only; never stored."""
|
||||
if not manifest or not manifest.replacements:
|
||||
return text or "", 0
|
||||
hits: list[tuple[int, int, str]] = []
|
||||
@@ -711,7 +728,7 @@ def _normalize_active_cleartext(text: str, manifest: MaskingManifest | None) ->
|
||||
|
||||
|
||||
def _process_response(content: str, manifest: MaskingManifest | None = None) -> tuple[str, dict[str, Any]]:
|
||||
"""Integrity check after the model reply. Does not undo egress and does not retry."""
|
||||
"""Integrity check after the model reply. Tokenized copy is not the visible output."""
|
||||
text = (content or "").strip()
|
||||
if not text:
|
||||
raise PrivacyGatewayError("empty_provider_response", "Der Provider lieferte keine Antwort.")
|
||||
@@ -725,7 +742,7 @@ def _process_response(content: str, manifest: MaskingManifest | None = None) ->
|
||||
|
||||
|
||||
def _validate_response(content: str, manifest: MaskingManifest | None = None) -> str:
|
||||
"""Compatibility wrapper: normalize active cleartext, then return tokenized text."""
|
||||
"""Compatibility wrapper: tokenized check copy. Not the stored user-facing text."""
|
||||
text, _meta = _process_response(content, manifest)
|
||||
return text
|
||||
|
||||
@@ -910,6 +927,8 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
detect_calls=detect_stats.get("detect_calls"),
|
||||
request_local_hits=detect_stats.get("request_local_hits"),
|
||||
confirmed_registry_hits=detect_stats.get("confirmed_registry_hits"),
|
||||
confirmed_registry_applied=detect_stats.get("confirmed_registry_applied"),
|
||||
semantic_identity_guaranteed=False,
|
||||
entity_counts=detect_stats.get("entity_counts"),
|
||||
detect_ms=detect_stats.get("detect_ms"),
|
||||
detect_prompt_tokens=detect_stats.get("detect_prompt_tokens"),
|
||||
@@ -982,7 +1001,9 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
"chunks_ok": detect_stats.get("chunks_ok"),
|
||||
"entity_counts": detect_stats.get("entity_counts"),
|
||||
"confirmed_registry_hits": detect_stats.get("confirmed_registry_hits"),
|
||||
"confirmed_registry_applied": detect_stats.get("confirmed_registry_applied"),
|
||||
"request_local_hits": detect_stats.get("request_local_hits"),
|
||||
"semantic_identity_guaranteed": False,
|
||||
"detect_calls": detect_stats.get("detect_calls"),
|
||||
"detect_ms": detect_stats.get("detect_ms"),
|
||||
"generate_called": False,
|
||||
@@ -1048,7 +1069,6 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
)
|
||||
usages: list[dict] = []
|
||||
raw = ""
|
||||
validated = ""
|
||||
chat = None
|
||||
try:
|
||||
_log_event(
|
||||
@@ -1067,7 +1087,7 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
request_trace["generate_calls"] = 1
|
||||
_log_event(events, started, "model_result", attempt=1, **_usage_bits(chat.usage))
|
||||
raw = chat.content
|
||||
validated, process_meta = _process_response(raw, manifest)
|
||||
_tokenized, process_meta = _process_response(raw, manifest)
|
||||
diagnostics["response_validation"] = process_meta["response_validation"]
|
||||
diagnostics["response_normalization"] = process_meta["response_normalization"]
|
||||
diagnostics["normalized_cleartext_count"] = process_meta["normalized_cleartext_count"]
|
||||
@@ -1124,7 +1144,7 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
sum_usages(usages) or chat.usage,
|
||||
chat.model or (config.model if config else None),
|
||||
)
|
||||
result.content = _demask(validated, manifest)
|
||||
result.content = _visible_response(raw, manifest)
|
||||
result.allowed = True
|
||||
result.local_identities = local_identities
|
||||
result.diagnostics = compact_diagnostics(diagnostics)
|
||||
@@ -1169,8 +1189,12 @@ def public_trace(trace: dict | None) -> dict | None:
|
||||
"chunks_ok": trace.get("chunks_ok"),
|
||||
"entity_counts": trace.get("entity_counts"),
|
||||
"confirmed_registry_hits": trace.get("confirmed_registry_hits"),
|
||||
"confirmed_registry_applied": trace.get("confirmed_registry_applied"),
|
||||
"request_local_hits": trace.get("request_local_hits"),
|
||||
"semantic_identity_guaranteed": False,
|
||||
"detect_calls": trace.get("detect_calls"),
|
||||
"stored_title": trace.get("stored_title"),
|
||||
"stored_body": trace.get("stored_body"),
|
||||
"detect_ms": trace.get("detect_ms"),
|
||||
"generate_ms": (trace.get("budget") or {}).get("generate_ms") if isinstance(trace.get("budget"), dict) else trace.get("generate_ms"),
|
||||
"generate_called": trace.get("generate_called"),
|
||||
|
||||
+34
-14
@@ -13,6 +13,7 @@ import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from db import get_db, row_to_dict
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from dialogue_store import StoreError
|
||||
from journal_body import plain_text
|
||||
from journal_store import list_versions
|
||||
@@ -867,7 +868,7 @@ def open_api_review(profile_id: str, *, mode: str | None = None) -> dict:
|
||||
|
||||
|
||||
def run_api_review(profile_id: str, review_id: str | None = None, *, mode: str | None = None) -> dict:
|
||||
from engine import execute_prompt, load_active_prompt
|
||||
from engine import EngineError, execute_prompt, load_active_prompt
|
||||
|
||||
if review_id:
|
||||
with get_db() as conn:
|
||||
@@ -887,25 +888,44 @@ def run_api_review(profile_id: str, review_id: str | None = None, *, mode: str |
|
||||
review_id = opened["id"]
|
||||
package = opened["package"]
|
||||
prompt = load_active_prompt("mvp.profile_review")
|
||||
result = execute_prompt(
|
||||
prompt,
|
||||
profile_id,
|
||||
purpose="profile_review",
|
||||
data_class="B",
|
||||
context={
|
||||
"review_package": json.dumps(package, ensure_ascii=False, indent=2),
|
||||
"source_text": "\n".join(
|
||||
[item.get("excerpt") or "" for item in package.get("evidences") or []]
|
||||
+ [item.get("excerpt") or "" for item in package.get("corpus") or []]
|
||||
),
|
||||
},
|
||||
)
|
||||
try:
|
||||
result = execute_prompt(
|
||||
prompt,
|
||||
profile_id,
|
||||
purpose="profile_review",
|
||||
data_class="B",
|
||||
context={
|
||||
"review_package": json.dumps(package, ensure_ascii=False, indent=2),
|
||||
"source_text": "\n".join(
|
||||
[item.get("excerpt") or "" for item in package.get("evidences") or []]
|
||||
+ [item.get("excerpt") or "" for item in package.get("corpus") or []]
|
||||
),
|
||||
},
|
||||
)
|
||||
except EngineError as exc:
|
||||
persist_engine_error(
|
||||
profile_id,
|
||||
purpose="profile_review",
|
||||
exc=exc,
|
||||
subject_type="writing_profile_review",
|
||||
subject_id=review_id or "",
|
||||
)
|
||||
raise
|
||||
parsed = parse_result(result.get("content") or "")
|
||||
if not parsed.get("mode"):
|
||||
parsed["mode"] = package.get("mode") or ""
|
||||
staged = _stage_proposal(profile_id, parsed, review_id=review_id, package=package)
|
||||
staged["trace"] = result.get("trace")
|
||||
staged["review_id"] = review_id
|
||||
persist_step(
|
||||
profile_id,
|
||||
purpose="profile_review",
|
||||
status="ok",
|
||||
subject_type="writing_profile_review",
|
||||
subject_id=review_id or "",
|
||||
trace=result.get("trace"),
|
||||
extra={"review_id": review_id},
|
||||
)
|
||||
return staged
|
||||
|
||||
|
||||
|
||||
+117
-1
@@ -1,8 +1,23 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field
|
||||
import json
|
||||
|
||||
from auth import require_admin_dep
|
||||
from db import get_db
|
||||
from debug_store import (
|
||||
clear_runs,
|
||||
debug_tree,
|
||||
delete_run,
|
||||
export_document,
|
||||
export_filename,
|
||||
get_run,
|
||||
latest_run,
|
||||
list_runs,
|
||||
render_markdown,
|
||||
set_persist_enabled,
|
||||
settings_payload,
|
||||
)
|
||||
from dialogue_store import StoreError, get_conversation, inventory, list_conversations, list_derived_for_conversation, list_messages
|
||||
from identity_store import (
|
||||
confirm_identity,
|
||||
@@ -29,6 +44,7 @@ def admin_health(session: dict = Depends(require_admin_dep)):
|
||||
features = conn.execute("SELECT COUNT(*) AS n FROM features").fetchone()["n"]
|
||||
dialogue = inventory()
|
||||
providers = public_status()
|
||||
debug = settings_payload(session["profile_id"])
|
||||
return {
|
||||
"ok": True,
|
||||
"service": "kansho",
|
||||
@@ -40,6 +56,8 @@ def admin_health(session: dict = Depends(require_admin_dep)):
|
||||
"prompts": prompts,
|
||||
"features": features,
|
||||
"dialogue": dialogue,
|
||||
"debug_runs": debug["run_count"],
|
||||
"debug_persist_enabled": debug["persist_enabled"],
|
||||
"providers": {
|
||||
role["role"]: {"ready": role["ready"], "key_present": role["key_present"]}
|
||||
for role in providers["roles"]
|
||||
@@ -74,6 +92,104 @@ def admin_conversation(conversation_id: str, session: dict = Depends(require_adm
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
|
||||
|
||||
class DebugToggle(BaseModel):
|
||||
persist_enabled: bool
|
||||
|
||||
|
||||
@router.get("/debug")
|
||||
def admin_debug(session: dict = Depends(require_admin_dep)):
|
||||
return settings_payload(session["profile_id"])
|
||||
|
||||
|
||||
@router.put("/debug")
|
||||
def admin_update_debug(body: DebugToggle, session: dict = Depends(require_admin_dep)):
|
||||
payload = set_persist_enabled(body.persist_enabled)
|
||||
payload["run_count"] = settings_payload(session["profile_id"])["run_count"]
|
||||
return payload
|
||||
|
||||
|
||||
@router.get("/debug/tree")
|
||||
def admin_debug_tree(session: dict = Depends(require_admin_dep)):
|
||||
return debug_tree(session["profile_id"])
|
||||
|
||||
|
||||
@router.get("/debug/runs")
|
||||
def admin_debug_runs(limit: int = Query(default=200, ge=1, le=500), session: dict = Depends(require_admin_dep)):
|
||||
return {"runs": list_runs(session["profile_id"], limit=limit)}
|
||||
|
||||
|
||||
@router.get("/debug/latest")
|
||||
def admin_debug_latest(
|
||||
journal_day_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
session: dict = Depends(require_admin_dep),
|
||||
):
|
||||
return {
|
||||
"run": latest_run(
|
||||
session["profile_id"],
|
||||
journal_day_id=journal_day_id,
|
||||
purpose=purpose,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@router.get("/debug/export")
|
||||
def admin_debug_export(
|
||||
format: str = Query(default="json"),
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
session: dict = Depends(require_admin_dep),
|
||||
):
|
||||
fmt = (format or "json").strip().lower()
|
||||
if fmt not in {"json", "markdown"}:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "invalid_debug_export_format", "message": "Format ist json oder markdown."},
|
||||
)
|
||||
document = export_document(
|
||||
session["profile_id"],
|
||||
conversation_id=conversation_id,
|
||||
journal_day_id=journal_day_id,
|
||||
purpose=purpose,
|
||||
)
|
||||
filename = export_filename(fmt, document)
|
||||
if fmt == "markdown":
|
||||
body = render_markdown(document)
|
||||
return Response(
|
||||
content=body.encode("utf-8"),
|
||||
media_type="text/markdown; charset=utf-8",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
body = json.dumps(document, ensure_ascii=False, indent=2)
|
||||
return Response(
|
||||
content=body.encode("utf-8"),
|
||||
media_type="application/json; charset=utf-8",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/debug/runs/clear")
|
||||
def admin_clear_debug_runs(session: dict = Depends(require_admin_dep)):
|
||||
deleted = clear_runs(session["profile_id"])
|
||||
return {"ok": True, "deleted": deleted}
|
||||
|
||||
|
||||
@router.get("/debug/runs/{run_id}")
|
||||
def admin_debug_run(run_id: str, session: dict = Depends(require_admin_dep)):
|
||||
item = get_run(session["profile_id"], run_id)
|
||||
if not item:
|
||||
raise HTTPException(status_code=404, detail={"code": "debug_run_missing", "message": "Debug-Lauf fehlt."})
|
||||
return item
|
||||
|
||||
|
||||
@router.delete("/debug/runs/{run_id}")
|
||||
def admin_delete_debug_run(run_id: str, session: dict = Depends(require_admin_dep)):
|
||||
if not delete_run(session["profile_id"], run_id):
|
||||
raise HTTPException(status_code=404, detail={"code": "debug_run_missing", "message": "Debug-Lauf fehlt."})
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
class ProviderUpdate(BaseModel):
|
||||
name: str = ""
|
||||
url: str = ""
|
||||
|
||||
@@ -494,3 +494,34 @@ CREATE TABLE IF NOT EXISTS generation_guidelines (
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS app_settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL DEFAULT '',
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS debug_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
purpose TEXT NOT NULL,
|
||||
layer TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'ok',
|
||||
subject_type TEXT NOT NULL DEFAULT '',
|
||||
subject_id TEXT NOT NULL DEFAULT '',
|
||||
conversation_id TEXT,
|
||||
message_id TEXT,
|
||||
space_id TEXT,
|
||||
journal_day_id TEXT,
|
||||
space_title TEXT NOT NULL DEFAULT '',
|
||||
calendar_date TEXT NOT NULL DEFAULT '',
|
||||
conversation_title TEXT NOT NULL DEFAULT '',
|
||||
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_debug_runs_profile_created
|
||||
ON debug_runs (profile_id, created DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_debug_runs_conversation
|
||||
ON debug_runs (profile_id, conversation_id, created);
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Opt-in debug persist for the admin test spur. Run from backend/: python tests/test_debug_persist.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO = ROOT.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-test.sqlite")
|
||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-media")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from identity_store import remember_mapping
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def open_space_day(client, headers, title="Alltag", date="2026-08-28"):
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": title})
|
||||
expect(space.status_code == 200, f"create space {space.status_code}")
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": date},
|
||||
)
|
||||
expect(day.status_code == 200, f"open day {day.status_code}")
|
||||
return space.json(), day.json()
|
||||
|
||||
|
||||
def blob(data) -> str:
|
||||
return json.dumps(data, ensure_ascii=False)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
ui = (REPO / "frontend" / "src" / "pages" / "AdminDebugPage.jsx").read_text(encoding="utf-8")
|
||||
expect("Debug persistieren" in ui, "admin page has the persist toggle")
|
||||
expect("Space / Tag / Gespräche" in ui, "admin page uses space/day/conversation structure")
|
||||
expect("Gespräch-JSON" in ui, "admin page can download one conversation")
|
||||
expect("JSON herunterladen" not in ui, "admin page no longer uses a flat download label as primary")
|
||||
nav = (REPO / "frontend" / "src" / "config" / "adminNav.js").read_text(encoding="utf-8")
|
||||
expect("/admin/debug" in nav, "admin nav lists debug")
|
||||
dialogue_ui = (REPO / "frontend" / "src" / "pages" / "DialoguePage.jsx").read_text(encoding="utf-8")
|
||||
expect("CallTrace" not in dialogue_ui, "dialogue page no longer shows the live test spur")
|
||||
expect("Debug herunterladen" in dialogue_ui, "dialogue page can download the conversation debug log")
|
||||
day_ui = (REPO / "frontend" / "src" / "pages" / "JournalDayPage.jsx").read_text(encoding="utf-8")
|
||||
expect("import CallTrace" not in day_ui, "journal day does not render CallTrace under the thread")
|
||||
expect("Debug herunterladen" in day_ui, "journal day can download the conversation debug log")
|
||||
editor_ui = (REPO / "frontend" / "src" / "pages" / "JournalEditorPage.jsx").read_text(encoding="utf-8")
|
||||
expect("import CallTrace" in editor_ui, "journal editor still shows the generate trace")
|
||||
expect("<h2>Verlauf</h2>" in editor_ui, "journal editor names the generate trace")
|
||||
expect("Debug herunterladen" in editor_ui, "journal editor can download the generate debug log")
|
||||
expect("purpose=journal_generate" in editor_ui, "journal editor downloads the generate scope")
|
||||
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
expect(setup.status_code == 200, f"setup {setup.status_code}")
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
|
||||
other = client.post(
|
||||
"/api/users",
|
||||
headers=headers,
|
||||
json={"email": "ute@example.test", "name": "Ute", "password": "user-pass", "role": "user"},
|
||||
)
|
||||
expect(other.status_code == 200, "second profile")
|
||||
other_headers = header(
|
||||
client.post("/api/auth/login", json={"email": "ute@example.test", "password": "user-pass"}).json()["token"]
|
||||
)
|
||||
|
||||
settings = client.get("/api/admin/debug", headers=headers)
|
||||
expect(settings.status_code == 200, "admin can read debug settings")
|
||||
expect(settings.json()["persist_enabled"] is False, "persist is off by default")
|
||||
expect(settings.json()["run_count"] == 0, "no runs at start")
|
||||
|
||||
denied = client.get("/api/admin/debug", headers=other_headers)
|
||||
expect(denied.status_code == 403, "non-admin cannot read debug settings")
|
||||
|
||||
space, day = open_space_day(client, headers)
|
||||
day_id = day["day"]["id"]
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day_id}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Gespräch"},
|
||||
)
|
||||
expect(conv.status_code == 200, f"start conversation {conv.status_code}")
|
||||
conv_id = conv.json()["id"]
|
||||
reset_debug()
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv_id}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute war der Markt voll. Ich habe Kirschen gekauft."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn while persist off {turn.status_code}")
|
||||
listed = client.get("/api/admin/debug/runs", headers=headers)
|
||||
expect(listed.json()["runs"] == [], "off means no persisted steps")
|
||||
|
||||
enabled = client.put("/api/admin/debug", headers=headers, json={"persist_enabled": True})
|
||||
expect(enabled.status_code == 200, "enable persist")
|
||||
expect(enabled.json()["persist_enabled"] is True, "persist flag stored")
|
||||
|
||||
remember_mapping(profile_id, "Anna")
|
||||
reset_debug()
|
||||
first = client.post(
|
||||
f"/api/journal/conversations/{conv_id}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Anna war am Stand. Die Kirschen waren reif."},
|
||||
)
|
||||
expect(first.status_code == 200, f"first persisted turn {first.status_code}")
|
||||
reset_debug()
|
||||
second = client.post(
|
||||
f"/api/journal/conversations/{conv_id}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Später noch der Hafen, ungefähr 18:00 Uhr."},
|
||||
)
|
||||
expect(second.status_code == 200, f"second persisted turn {second.status_code}")
|
||||
|
||||
listed = client.get("/api/admin/debug/runs", headers=headers)
|
||||
expect(listed.status_code == 200, "list runs")
|
||||
runs = listed.json()["runs"]
|
||||
expect(len(runs) >= 2, "each turn is stored")
|
||||
expect(all(item.get("purpose") == "dialogue_turn" for item in runs[:2]), "listed steps are dialogue turns")
|
||||
|
||||
newest = client.get(f"/api/admin/debug/runs/{runs[0]['id']}", headers=headers)
|
||||
expect(newest.status_code == 200, "read run detail")
|
||||
payload = newest.json().get("payload") or {}
|
||||
stored = blob(payload)
|
||||
expect("local_label" not in stored, "mapping labels are not persisted")
|
||||
expect("mapping_table" not in stored, "mapping table is not persisted")
|
||||
expect("replacements" not in stored, "replacement objects are not persisted")
|
||||
details = [
|
||||
client.get(f"/api/admin/debug/runs/{item['id']}", headers=headers).json()
|
||||
for item in runs[:2]
|
||||
]
|
||||
expect(
|
||||
any("Hafen" in blob(item.get("payload") or {}) for item in details),
|
||||
"intern keeps the step text",
|
||||
)
|
||||
hafen = next(item.get("payload") or {} for item in details if "Hafen" in blob(item.get("payload") or {}))
|
||||
expect((hafen.get("trace") or {}).get("egress") or (hafen.get("call_traces")), "egress of the step is stored")
|
||||
expect((hafen.get("user") or {}).get("body"), "user line of the step is stored")
|
||||
expect((hafen.get("assistant") or {}).get("body"), "assistant line of the step is stored")
|
||||
expect(hafen.get("call_traces") or (hafen.get("trace") or {}).get("egress"), "gateway calls of the step are stored")
|
||||
|
||||
user_space = client.post("/api/journal/spaces", headers=other_headers, json={"title": "Utes Tag"})
|
||||
user_day = client.post(
|
||||
f"/api/journal/spaces/{user_space.json()['id']}/days",
|
||||
headers=other_headers,
|
||||
json={"calendar_date": "2026-08-28"},
|
||||
)
|
||||
user_conv = client.post(
|
||||
f"/api/journal/days/{user_day.json()['day']['id']}/conversations",
|
||||
headers=other_headers,
|
||||
json={"title": "Gespräch"},
|
||||
)
|
||||
user_turn = client.post(
|
||||
f"/api/journal/conversations/{user_conv.json()['id']}/turn",
|
||||
headers=other_headers,
|
||||
json={"body": "Utes privater Satz bleibt ohne Debug-Speicher."},
|
||||
)
|
||||
expect(user_turn.status_code == 200, "non-admin can still talk")
|
||||
after_user = client.get("/api/admin/debug/runs", headers=headers).json()["runs"]
|
||||
expect(len(after_user) == len(runs), "non-admin turns are not persisted")
|
||||
expect(
|
||||
not any("Utes privater Satz" in blob(item) for item in after_user),
|
||||
"admin list does not contain the other profile",
|
||||
)
|
||||
|
||||
reset_debug()
|
||||
gen = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv_id]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"generate {gen.status_code}")
|
||||
after_gen = client.get("/api/admin/debug/runs", headers=headers).json()["runs"]
|
||||
expect(any(item.get("purpose") == "journal_generate" for item in after_gen), "journal generate is a stored step")
|
||||
|
||||
exported = client.get("/api/admin/debug/export?format=json", headers=headers)
|
||||
expect(exported.status_code == 200, "json export")
|
||||
expect("attachment" in (exported.headers.get("content-disposition") or ""), "json is a download")
|
||||
document = exported.json()
|
||||
expect(document.get("kind") == "kansho.debug_export", "export names the contract")
|
||||
expect(document.get("version") == 2, "export version is nested tree")
|
||||
expect(document.get("scope") == "all", "full export is unscoped")
|
||||
expect(document.get("run_count") == len(document.get("runs") or []), "run_count matches runs")
|
||||
expect(len(document.get("runs") or []) >= 3, "export contains the dialogue and generate steps")
|
||||
expect((document.get("spaces") or []), "full export has space tree")
|
||||
space = (document.get("spaces") or [{}])[0]
|
||||
expect(space.get("title") == "Alltag", "tree names the space")
|
||||
day_node = (space.get("days") or [{}])[0]
|
||||
expect(day_node.get("calendar_date") == "2026-08-28", "tree names the day")
|
||||
conv_node = (day_node.get("conversations") or [{}])[0]
|
||||
expect(len(conv_node.get("steps") or []) >= 2, "tree keeps dialogue steps on the conversation")
|
||||
expect(day_node.get("generates"), "journal generate sits on the day")
|
||||
expect("local_label" not in blob(document), "export has no mapping labels")
|
||||
created = [item.get("created") for item in document["runs"]]
|
||||
expect(created == sorted(created), "export is chronological")
|
||||
|
||||
tree = client.get("/api/admin/debug/tree", headers=headers)
|
||||
expect(tree.status_code == 200, "tree endpoint")
|
||||
tree_space = (tree.json().get("spaces") or [{}])[0]
|
||||
expect(tree_space.get("title") == "Alltag", "live tree names the space")
|
||||
tree_conv = ((tree_space.get("days") or [{}])[0].get("conversations") or [{}])[0]
|
||||
expect(tree_conv.get("id") == conv_id, "live tree points at the conversation")
|
||||
expect(tree_conv.get("step_count") >= 2, "live tree counts steps")
|
||||
|
||||
conv_export = client.get(
|
||||
f"/api/admin/debug/export?format=json&conversation_id={conv_id}",
|
||||
headers=headers,
|
||||
)
|
||||
expect(conv_export.status_code == 200, "conversation export")
|
||||
conv_doc = conv_export.json()
|
||||
expect(conv_doc.get("scope") == "conversation", "conversation export is scoped")
|
||||
expect(conv_doc.get("conversation", {}).get("id") == conv_id, "conversation export names the dialogue")
|
||||
expect(
|
||||
all(item.get("purpose") == "dialogue_turn" for item in conv_doc.get("runs") or []),
|
||||
"conversation export is the dialogue steps, not the day generate",
|
||||
)
|
||||
expect(len(conv_doc.get("runs") or []) >= 2, "conversation export has both turns")
|
||||
expect("conversation" in (conv_export.headers.get("content-disposition") or ""), "conversation filename is distinct")
|
||||
|
||||
latest = client.get(
|
||||
f"/api/admin/debug/latest?journal_day_id={day_id}&purpose=journal_generate",
|
||||
headers=headers,
|
||||
)
|
||||
expect(latest.status_code == 200, "latest generate endpoint")
|
||||
latest_run = latest.json().get("run") or {}
|
||||
expect(latest_run.get("purpose") == "journal_generate", "latest run is the journal generate")
|
||||
expect(latest_run.get("journal_day_id") == day_id, "latest generate belongs to the day")
|
||||
expect((latest_run.get("trace") or {}).get("purpose") == "journal_generate" or (latest_run.get("trace") or {}).get("stages"), "latest generate keeps the trace")
|
||||
|
||||
gen_export = client.get(
|
||||
f"/api/admin/debug/export?format=json&journal_day_id={day_id}&purpose=journal_generate",
|
||||
headers=headers,
|
||||
)
|
||||
expect(gen_export.status_code == 200, "generate export")
|
||||
gen_doc = gen_export.json()
|
||||
expect(gen_doc.get("scope") == "journal_generate", "generate export is scoped")
|
||||
expect(
|
||||
all(item.get("purpose") == "journal_generate" for item in gen_doc.get("runs") or []),
|
||||
"generate export is only the journal draft steps",
|
||||
)
|
||||
expect(len(gen_doc.get("runs") or []) >= 1, "generate export has the draft run")
|
||||
expect("journal" in (gen_export.headers.get("content-disposition") or ""), "generate filename is distinct")
|
||||
|
||||
markdown = client.get("/api/admin/debug/export?format=markdown", headers=headers)
|
||||
expect(markdown.status_code == 200, "markdown export")
|
||||
text = markdown.text
|
||||
expect("Kanshō Debug-Export" in text, "markdown has a title")
|
||||
expect("Intern (Klartext-Vorlage, nicht gesendet)" in text, "markdown has intern section")
|
||||
expect("Egress (maskiert, gesendet)" in text, "markdown has egress section")
|
||||
expect("dialogue_turn" in text, "markdown lists dialogue steps")
|
||||
expect("journal_generate" in text, "markdown lists generate steps")
|
||||
|
||||
bad = client.get("/api/admin/debug/export?format=csv", headers=headers)
|
||||
expect(bad.status_code == 400, "unknown export format is rejected")
|
||||
|
||||
disabled = client.put("/api/admin/debug", headers=headers, json={"persist_enabled": False})
|
||||
expect(disabled.json()["persist_enabled"] is False, "persist can be turned off")
|
||||
count_before = len(client.get("/api/admin/debug/runs", headers=headers).json()["runs"])
|
||||
reset_debug()
|
||||
later = client.post(
|
||||
f"/api/journal/conversations/{conv_id}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Noch ein Satz ohne Persistenz."},
|
||||
)
|
||||
expect(later.status_code == 200, "turn after disable")
|
||||
count_after = len(client.get("/api/admin/debug/runs", headers=headers).json()["runs"])
|
||||
expect(count_after == count_before, "disable stops new writes and keeps history")
|
||||
|
||||
run_id = client.get("/api/admin/debug/runs", headers=headers).json()["runs"][0]["id"]
|
||||
removed = client.delete(f"/api/admin/debug/runs/{run_id}", headers=headers)
|
||||
expect(removed.status_code == 200, "delete one run")
|
||||
cleared = client.post("/api/admin/debug/runs/clear", headers=headers)
|
||||
expect(cleared.status_code == 200, "clear remaining runs")
|
||||
expect(client.get("/api/admin/debug/runs", headers=headers).json()["runs"] == [], "history empty after clear")
|
||||
|
||||
health = client.get("/api/admin/health", headers=headers)
|
||||
expect(health.json()["inventory"]["debug_persist_enabled"] is False, "health shows persist off")
|
||||
expect(health.json()["inventory"]["debug_runs"] == 0, "health shows empty debug count")
|
||||
|
||||
print("All debug persist tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -79,6 +79,11 @@ def main() -> None:
|
||||
admin = client.get("/api/admin/health", headers=headers)
|
||||
expect(admin.status_code == 200, "admin health")
|
||||
expect(admin.json()["inventory"]["prompts"] == 5, "five seeded prompts")
|
||||
expect(admin.json()["inventory"]["debug_persist_enabled"] is False, "debug persist off by default")
|
||||
expect(admin.json()["inventory"]["debug_runs"] == 0, "no debug runs at start")
|
||||
|
||||
debug = client.get("/api/admin/debug", headers=headers)
|
||||
expect(debug.status_code == 200 and debug.json()["persist_enabled"] is False, "debug settings default off")
|
||||
|
||||
providers = client.get("/api/admin/providers", headers=headers)
|
||||
expect(providers.status_code == 200, "admin providers")
|
||||
|
||||
@@ -483,6 +483,64 @@ def main() -> None:
|
||||
expect((third.json().get("trace") or {}).get("writing_profile", {}).get("present") is True, "confirmed profile is marked present")
|
||||
expect(sum(1 for item in (third.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "profile A/B still uses one generate call")
|
||||
|
||||
kept_body = (
|
||||
"Die Kinder verstanden kein Wort. Später fragte ich, "
|
||||
"ob die Kinder überhaupt Deutsch verstehen. Die Kinder lachten."
|
||||
)
|
||||
|
||||
def keep_groups(_messages, _policy):
|
||||
return ChatResult(
|
||||
content=f"Markttag\n\n{kept_body}",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", keep_groups):
|
||||
kept = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(kept.status_code == 200, f"accepted model text {kept.text}")
|
||||
expect(kept.json().get("title") == "Markttag", "title split stays technical")
|
||||
expect(kept.json().get("body") == kept_body, "stored body equals demasked model body")
|
||||
expect("Er" not in (kept.json().get("body") or "").split(), "generic group is not rewritten to a pronoun")
|
||||
stored = f"{kept.json().get('title')}\n\n{kept.json().get('body')}".strip()
|
||||
expect((kept.json().get("trace") or {}).get("reply") == stored, "trace reply is the stored draft")
|
||||
expect((kept.json().get("trace") or {}).get("stored_body") == kept_body, "trace stores the same body")
|
||||
expect((kept.json().get("trace") or {}).get("generate_calls") == 1, "accepted text still uses one generate call")
|
||||
|
||||
def mixed_surface(_messages, _policy):
|
||||
body = (
|
||||
"Die Gruppe verstand kein Wort. "
|
||||
"Später die Gruppe wieder. "
|
||||
"die Gruppen standen abseits."
|
||||
)
|
||||
return ChatResult(
|
||||
content=f"Markttag\n\n{body}",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", mixed_surface):
|
||||
mixed = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(mixed.status_code == 200, f"mixed surface generate {mixed.text}")
|
||||
mixed_body = mixed.json().get("body") or ""
|
||||
expect(mixed_body.startswith("Die Gruppe verstand"), "journal stores sentence-initial model casing")
|
||||
expect("Später die Gruppe wieder" in mixed_body, "journal keeps mid-sentence model casing")
|
||||
expect("die Gruppen standen" in mixed_body, "journal keeps unmatched plural surface form")
|
||||
mixed_stored = f"{mixed.json().get('title')}\n\n{mixed_body}".strip()
|
||||
expect((mixed.json().get("trace") or {}).get("reply") == mixed_stored, "journal trace reply is the stored draft")
|
||||
expect((mixed.json().get("trace") or {}).get("stored_body") == mixed_body, "journal stored_body matches the draft")
|
||||
expect("Die Gruppe verstand" in ((mixed.json().get("trace") or {}).get("raw") or ""), "journal trace raw keeps provider casing")
|
||||
expect((mixed.json().get("trace") or {}).get("generate_calls") == 1, "mixed surface still uses one generate call")
|
||||
|
||||
def invent_title(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Hanna am Hafen\n\nIch ging zum Markt, auf dem es ziemlich voll war.",
|
||||
|
||||
@@ -42,6 +42,7 @@ from journal_generation_policy import (
|
||||
publish_guideline,
|
||||
save_selection,
|
||||
seed_generation_instructions,
|
||||
set_default_guideline,
|
||||
snapshot_summary,
|
||||
update_guideline,
|
||||
validate_selection,
|
||||
@@ -214,6 +215,9 @@ def test_runtime_has_no_numeric_policy() -> None:
|
||||
expect(day.count("slot:") >= 4, "journal page has four independent slots")
|
||||
expect("<select" in day, "journal page uses select fields")
|
||||
expect("Diese Auswahl als Standard merken" in day, "journal page can remember explicitly")
|
||||
expect("if (filled) return current" in day, "settings load does not overwrite a filled selection")
|
||||
expect("Auswahl nicht mehr aktiv" in day, "stale guideline ids stay visible as invalid")
|
||||
expect("Rev. ${option.revision}" in day, "options show the guideline revision")
|
||||
expect("Faktenregeln und Datenschutz bleiben unabhängig" in day, "facts/privacy note remains")
|
||||
editor = (FRONTEND / "src" / "pages" / "JournalEditorPage.jsx").read_text(encoding="utf-8")
|
||||
expect("generation_summary" in editor, "editor can show the generation snapshot")
|
||||
@@ -232,6 +236,11 @@ def test_runtime_has_no_numeric_policy() -> None:
|
||||
trace_ui = (FRONTEND / "src" / "components" / "CallTrace.jsx").read_text(encoding="utf-8")
|
||||
expect("Editorial Mode" not in trace_ui, "trace UI does not present source mode as a run decision")
|
||||
expect("source_mode" not in trace_ui, "trace UI has no source_mode label")
|
||||
expect("Chunks technisch verarbeitet" in trace_ui, "trace separates technical chunk coverage")
|
||||
expect("Semantische Identitätserkennung" in trace_ui, "trace names semantic uncertainty")
|
||||
expect("nicht garantiert" in trace_ui, "trace does not claim semantic completeness")
|
||||
expect("Gespeicherter Entwurf" in trace_ui, "trace shows the stored draft")
|
||||
expect("Technisch verarbeitete Chunks bedeuten keine vollständige" in trace_ui, "trace explains coverage is not identity certainty")
|
||||
|
||||
|
||||
def test_seed_and_compiler() -> None:
|
||||
@@ -493,6 +502,27 @@ def main() -> None:
|
||||
expect((rejected.json().get("detail") or {}).get("code") == "invalid_generation_selection", "rejection uses the selection code")
|
||||
after_reject = client.get("/api/journal/generation-settings", headers=headers).json()
|
||||
expect(after_reject["selection"] == default_ids(), "rejected values are not stored")
|
||||
save_selection(profile_id, low_ids())
|
||||
set_default_guideline(seed_id("transformation", "substantial"))
|
||||
expect(load_selection(profile_id)["transformation_id"] == seed_id("transformation", "correction"), "setting a catalog default does not rewrite a stored selection")
|
||||
save_selection(profile_id, default_ids())
|
||||
blocked_calls = {"n": 0}
|
||||
|
||||
def no_provider(*_a, **_k):
|
||||
blocked_calls["n"] += 1
|
||||
raise AssertionError("invalid selection must not call a provider")
|
||||
|
||||
with patch("privacy_gateway.complete_model", no_provider), patch("entity_detect.complete_chat", no_provider):
|
||||
before_provider = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"conversation_ids": [conv.json()["id"]],
|
||||
"generation_selection": {**default_ids(), "transformation_id": "missing"},
|
||||
},
|
||||
)
|
||||
expect(before_provider.status_code == 400, "invalid selection still fails closed")
|
||||
expect(blocked_calls["n"] == 0, "invalid selection aborts before detect and generate")
|
||||
|
||||
draft = client.post(
|
||||
"/api/admin/generation-instructions/journal_generate",
|
||||
@@ -575,6 +605,53 @@ def main() -> None:
|
||||
for fact in HARD_FACTS:
|
||||
expect(fact in intern, f"hard fact remains at low policy: {fact}")
|
||||
|
||||
copyedit_ids = selection_of(
|
||||
transformation="copyedit",
|
||||
detail="complete",
|
||||
voice="clear",
|
||||
narrative="weighted",
|
||||
)
|
||||
reset_debug()
|
||||
copyedit_recorder = install_test_recorder()
|
||||
copyedit_run = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"conversation_ids": [conv.json()["id"]],
|
||||
"generation_selection": copyedit_ids,
|
||||
"remember_generation_selection": False,
|
||||
},
|
||||
)
|
||||
expect(copyedit_run.status_code == 200, f"copyedit generate {copyedit_run.text}")
|
||||
copyedit_meta = selection_meta(copyedit_run.json())
|
||||
copyedit_intern = intern_of(copyedit_run.json())
|
||||
copyedit_snap = copyedit_run.json().get("generation_snapshot") or {}
|
||||
expect(copyedit_meta.get("ids") == {
|
||||
"transformation": seed_id("transformation", "copyedit"),
|
||||
"detail": seed_id("detail", "complete"),
|
||||
"voice": seed_id("voice", "clear"),
|
||||
"narrative": seed_id("narrative", "weighted"),
|
||||
}, "copyedit request ids reach the trace")
|
||||
expect(copyedit_meta.get("keys", {}).get("transformation") == "copyedit", "copyedit key reaches the trace")
|
||||
expect(copyedit_meta.get("revisions", {}).get("transformation") == 1, "copyedit revision reaches the trace")
|
||||
expect(copyedit_meta.get("revisions", {}).get("narrative") == 1, "weighted revision of the selected id reaches the trace")
|
||||
expect(copyedit_snap.get("transformation", {}).get("id") == seed_id("transformation", "copyedit"), "snapshot stores the selected copyedit id")
|
||||
expect(copyedit_snap.get("transformation", {}).get("revision") == 1, "snapshot stores the selected copyedit revision")
|
||||
expect(copyedit_snap.get("narrative", {}).get("id") == seed_id("narrative", "weighted"), "snapshot stores the selected weighted id")
|
||||
expect(seed_instruction("transformation", "copyedit") in copyedit_intern, "selected copyedit instruction reaches the prompt")
|
||||
expect(seed_instruction("narrative", "weighted") in copyedit_intern, "selected weighted instruction reaches the prompt")
|
||||
expect(seed_instruction("transformation", "substantial") not in copyedit_intern, "unselected substantial instruction is absent")
|
||||
expect([item.get("purpose") for item in copyedit_recorder] == ["journal_generate"], "copyedit still uses one generate call")
|
||||
preview = client.post(
|
||||
"/api/admin/generation-instructions/journal_generate/preview",
|
||||
headers=headers,
|
||||
json={"generation_selection": copyedit_ids},
|
||||
)
|
||||
expect(preview.status_code == 200, f"copyedit preview {preview.text}")
|
||||
expect(preview.json()["transformation_instructions"] == seed_instruction("transformation", "copyedit"), "preview compiles the same copyedit instruction")
|
||||
expect(preview.json().get("selection", {}).get("ids", {}).get("transformation") == seed_id("transformation", "copyedit"), "preview reports the same copyedit id")
|
||||
expect(preview.json().get("selection", {}).get("revisions", {}).get("narrative") == 1, "preview reports the same weighted revision")
|
||||
|
||||
notes_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
@@ -789,6 +866,7 @@ def main() -> None:
|
||||
headers=headers,
|
||||
)
|
||||
expect(published.status_code == 200, f"publish {published.text}")
|
||||
expect(load_selection(profile_id)["transformation_id"] == seed_id("transformation", "substantial"), "publishing a clone does not rewrite the stored selection")
|
||||
immutable = client.put(
|
||||
f"/api/admin/generation-instructions/journal_generate/{seed_id('transformation', 'substantial')}",
|
||||
headers=headers,
|
||||
|
||||
@@ -60,33 +60,28 @@ def main() -> None:
|
||||
kept = "Am Morgen der Markt.\n\nSpäter das Boot, noch gehetzt."
|
||||
expect(shape_journal("Tag", kept, turns)[1] == kept, "already shaped text is left alone")
|
||||
|
||||
source = [
|
||||
"Gegen 8:30 Uhr war Sushi immer noch nicht auf dem Balkon. "
|
||||
"Ich fand sie unter der Dusche. Später saß sie auf dem Bett und telefonierte. "
|
||||
"Auf die Frage, ob sie mit mir frühstücken wolle, antwortete sie, dass sie erst Brot holen müsse."
|
||||
]
|
||||
stuffed = (
|
||||
"Gegen 8:30 Uhr war Sushi immer noch nicht bei mir auf dem Balkon. "
|
||||
"Ich machte mich auf die Suche und fand Sushi schließlich unter der Dusche im Badezimmer. "
|
||||
"Als Sushi dann immer noch nicht auftauchte, sah ich nochmals nach und musste feststellen, "
|
||||
"dass Sushi auf dem Bett saß und telefonierte. Als ich gerade mit den Vorbereitungen fertig war, "
|
||||
"kam Sushi aus dem Schlafzimmer. Auf meine Frage, ob Sushi mit mir frühstücken wolle, "
|
||||
"antwortete sie, dass Sushi erst Brot vom Laden holen müsse."
|
||||
repeated = (
|
||||
"Die Kinder verstanden kein Wort. Später fragte ich, "
|
||||
"ob die Kinder überhaupt Deutsch verstehen. Die Kinder lachten."
|
||||
)
|
||||
from journal_shape import infer_person_gender, naturalize_person_mentions
|
||||
_, unchanged = shape_journal(
|
||||
"Tag",
|
||||
repeated,
|
||||
["Die Kinder verstanden kein Wort."],
|
||||
person_labels=["Die Kinder", "Kinder"],
|
||||
)
|
||||
expect(unchanged == repeated, "repeated generic person terms stay unchanged")
|
||||
expect("Er" not in unchanged.split(), "generic group is not turned into a pronoun")
|
||||
expect(unchanged.count("Kinder") == 3, "every generic mention remains")
|
||||
|
||||
named = (
|
||||
"Maren kam zum Markt. Später rief Maren an. "
|
||||
"Ob Maren noch käme, wusste ich nicht. Maren blieb weg."
|
||||
)
|
||||
_, named_out = shape_journal("Tag", named, named.split(". "), person_labels=["Maren"])
|
||||
expect(named_out == named, "accepted person names are not replaced by pronouns")
|
||||
expect("Er " not in named_out and "Sie " not in named_out, "no guessed pronoun rewrite")
|
||||
|
||||
expect(infer_person_gender("Sushi", source) == "f", "source pronouns set feminine")
|
||||
cleaned = naturalize_person_mentions(stuffed, ["Sushi"], source)
|
||||
expect(cleaned.count("Sushi") == 2, "name kept twice then pronouns")
|
||||
expect("fand Sushi" in cleaned, "second mention may stay a name")
|
||||
expect("dass sie erst Brot" in cleaned, "reported speech keeps sie not the name")
|
||||
expect("ob sie mit mir" in cleaned, "ob-clause is sie")
|
||||
expect("Als sie dann" in cleaned, "later subject becomes pronoun")
|
||||
expect("esse gerne Sushi" in naturalize_person_mentions(
|
||||
"Ich esse gerne Sushi mit meiner Frau Sushi. Danach aß ich Sushi.",
|
||||
["Sushi"],
|
||||
source,
|
||||
), "food homonym is not turned into a pronoun")
|
||||
print("journal shape ok")
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
@@ -25,6 +24,8 @@ from fastapi.testclient import TestClient
|
||||
from identity_store import (
|
||||
STATUS_LEGACY,
|
||||
confirm_identity,
|
||||
is_maskable_label,
|
||||
is_registry_maskable_label,
|
||||
list_confirmed_identities,
|
||||
list_mappings,
|
||||
list_registry,
|
||||
@@ -37,6 +38,7 @@ from privacy_gateway import (
|
||||
PrivacyGatewayError,
|
||||
_demask,
|
||||
_validate_response,
|
||||
_visible_response,
|
||||
complete,
|
||||
mask_prompt,
|
||||
reset_debug,
|
||||
@@ -181,7 +183,10 @@ def main() -> None:
|
||||
expect(first.trace.get("detect_provider") == "fake-detect", "detect provider is separate")
|
||||
expect("Anna" not in (first.trace.get("egress") or ""), "Anna masked before generate")
|
||||
expect("Hamburg" not in (first.trace.get("egress") or ""), "Hamburg masked before generate")
|
||||
expect(first.trace.get("full_detection_coverage") is True, "successful detect records full coverage")
|
||||
expect(first.trace.get("full_detection_coverage") is True, "successful detect records technical chunk coverage")
|
||||
expect(first.trace.get("semantic_identity_guaranteed") is False, "chunk coverage is not semantic certainty")
|
||||
expect(first.diagnostics.get("semantic_identity_guaranteed") is False, "compact trace does not claim semantic completeness")
|
||||
expect(first.trace.get("confirmed_registry_applied") is True, "confirmed registry is applied as a safety net")
|
||||
expect(first.trace.get("generate_called") is True, "generate runs after complete detection")
|
||||
expect(first.diagnostics.get("detect_calls") == 1, "normal short text uses one detect call")
|
||||
labels_after = {item["canonical_label"] for item in list_confirmed_identities(profile_id)}
|
||||
@@ -315,7 +320,14 @@ def main() -> None:
|
||||
def worker(label: str):
|
||||
reset_debug()
|
||||
out = _run(profile_id, f"Ich traf {label}.")
|
||||
results.append((label, out.trace.get("active_tokens"), threading.get_ident()))
|
||||
results.append(
|
||||
(
|
||||
label,
|
||||
tuple(out.trace.get("active_tokens") or []),
|
||||
out.trace.get("egress") or "",
|
||||
out.trace.get("intern") or "",
|
||||
)
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
one = pool.submit(worker, "Anna")
|
||||
@@ -324,6 +336,10 @@ def main() -> None:
|
||||
two.result()
|
||||
labels = {item[0] for item in results}
|
||||
expect(labels == {"Anna", "Maren"}, "parallel requests keep their own manifests")
|
||||
by_label = {item[0]: item for item in results}
|
||||
expect("Maren" not in by_label["Anna"][2] and "Maren" not in by_label["Anna"][3], "Anna request does not carry Maren")
|
||||
expect("Anna" not in by_label["Maren"][2] and "Anna" not in by_label["Maren"][3], "Maren request does not carry Anna")
|
||||
expect(by_label["Anna"][1] and by_label["Maren"][1], "each parallel request has its own tokens")
|
||||
|
||||
from db import get_db
|
||||
import uuid
|
||||
@@ -432,13 +448,14 @@ def main() -> None:
|
||||
plural, _ = bind_pronouns(_mask("Anna kam. Sie waren zu dritt.", anna))
|
||||
expect("Sie waren" in plural or "sie waren" in plural.lower(), "plural sie stays")
|
||||
|
||||
from identity_store import is_maskable_label
|
||||
|
||||
expect(not is_maskable_label("ich"), "pronoun ich is not a mapping")
|
||||
expect(not is_maskable_label("user"), "role user is not a mapping")
|
||||
expect(not is_maskable_label("6:00 Uhr"), "clock time is not identity")
|
||||
expect(is_maskable_label("Sushi"), "name-shaped label stays structurally maskable")
|
||||
expect(is_maskable_label("Himmel"), "weather word is not banned by a stopword list")
|
||||
expect(not is_maskable_label("Lia"), "detect gate still skips short labels")
|
||||
expect(is_registry_maskable_label("Lia"), "confirmed registry may mask short names")
|
||||
expect(not is_registry_maskable_label("ich"), "registry gate still skips pronouns")
|
||||
|
||||
from db import get_db as _get_db
|
||||
|
||||
@@ -506,6 +523,10 @@ def main() -> None:
|
||||
tokenized = _validate_response("Anna war ruhig.", anna_manifest)
|
||||
expect(tokenized.startswith("[[PERSON:01]]"), "active cleartext is normalized, not discarded")
|
||||
expect(_demask(tokenized, anna_manifest) == "Anna war ruhig.", "normalized plaintext demasks")
|
||||
expect(
|
||||
_visible_response("Anna war ruhig.", anna_manifest) == "Anna war ruhig.",
|
||||
"visible output keeps model plaintext instead of the check roundtrip",
|
||||
)
|
||||
|
||||
no_user = client.get("/api/admin/identities")
|
||||
expect(no_user.status_code in {401, 403}, "identity registry requires auth")
|
||||
@@ -525,6 +546,64 @@ def main() -> None:
|
||||
expect(admin_calls["n"] == 0, "admin registry actions create no external egress")
|
||||
expect(created.json()["status"] == "confirmed", "admin create confirms locally")
|
||||
|
||||
before_confirm = {item.get("canonical_label") for item in list_confirmed_identities(profile_id)}
|
||||
confirm_identity(profile_id, "Lia", aliases=["Mara"])
|
||||
expect(is_registry_maskable_label("Lia"), "short confirmed name remains maskable")
|
||||
egress_body = (
|
||||
"WRITING_PROFILE\nLia schreibt knapp.\n"
|
||||
"STYLE_EXAMPLES\nMara lachte am Markt.\n"
|
||||
"CURRENT_DAY_SOURCES\nLia und Mara gingen später.\n"
|
||||
"EXISTING_TEXT\nLia blieb."
|
||||
)
|
||||
covered = _run(profile_id, egress_body)
|
||||
masked_all = covered.trace.get("egress") or ""
|
||||
intern = covered.trace.get("intern") or ""
|
||||
expect("Lia" in intern and "Mara" in intern, "clear names remain in the local template")
|
||||
expect("Lia" not in masked_all, "confirmed short name is masked in every egress section")
|
||||
expect("Mara" not in masked_all, "confirmed alias is masked in every egress section")
|
||||
expect(
|
||||
masked_all.count("[[") >= intern.count("Lia") + intern.count("Mara"),
|
||||
"canonical and alias mentions are replaced",
|
||||
)
|
||||
expect("WRITING_PROFILE" in masked_all and "STYLE_EXAMPLES" in masked_all, "writing profile and style examples stay in egress")
|
||||
expect("CURRENT_DAY_SOURCES" in masked_all and "EXISTING_TEXT" in masked_all, "sources and existing text stay in egress")
|
||||
expect(covered.trace.get("confirmed_registry_applied") is True, "registry safety net is marked applied")
|
||||
expect(covered.trace.get("semantic_identity_guaranteed") is False, "registry coverage is not semantic certainty")
|
||||
expect(covered.trace.get("pre_egress_validation") == "ok", "confirmed plaintext is not left in egress")
|
||||
|
||||
group_text = "Die Kinder verstanden kein Wort. Herkunft: Nordland."
|
||||
kinder_start = group_text.index("Kinder")
|
||||
nord_start = group_text.index("Nordland")
|
||||
install_test_spans(
|
||||
[
|
||||
{
|
||||
"start": kinder_start,
|
||||
"end": kinder_start + len("Kinder"),
|
||||
"text": "Kinder",
|
||||
"entity_type": "PERSON",
|
||||
},
|
||||
{
|
||||
"start": nord_start,
|
||||
"end": nord_start + len("Nordland"),
|
||||
"text": "Nordland",
|
||||
"entity_type": "PLACE",
|
||||
},
|
||||
]
|
||||
)
|
||||
try:
|
||||
overmasked = _run(profile_id, group_text)
|
||||
proposals = list_review_proposals(profile_id)
|
||||
labels = {(item.get("observed_label") or item.get("canonical_label") or item.get("local_label") or "").casefold() for item in proposals}
|
||||
expect("kinder" in labels or "nordland" in labels, "semantic overmask stays request-local as a proposal")
|
||||
confirmed_now = {item.get("canonical_label") for item in list_confirmed_identities(profile_id)}
|
||||
expect("Kinder" not in confirmed_now, "generic group is not persisted as an identity")
|
||||
expect("Nordland" not in confirmed_now, "origin designation is not persisted as an identity")
|
||||
expect(confirmed_now == before_confirm | {"Lia"}, "only the confirmed registry row is durable")
|
||||
expect(overmasked.trace.get("full_detection_coverage") is True, "injected spans still mark technical coverage")
|
||||
expect(overmasked.trace.get("semantic_identity_guaranteed") is False, "overmasking is not claimed as certainty")
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
@@ -18,6 +19,7 @@ os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from entity_detect import install_test_spans, reset_detect_test_hooks
|
||||
from identity_store import remember_mapping
|
||||
from main import app
|
||||
from privacy_gateway import (
|
||||
@@ -29,6 +31,7 @@ from privacy_gateway import (
|
||||
_demask,
|
||||
_process_response,
|
||||
_validate_response,
|
||||
_visible_response,
|
||||
complete,
|
||||
last_trace,
|
||||
mask_prompt,
|
||||
@@ -137,6 +140,55 @@ def main() -> None:
|
||||
expect(_demask("[[PERSON:07]] kam.", aliased) == "Clarissa kam.", "confirmed alias demasks to the canonical spelling")
|
||||
canonical_clear, _meta = _process_response("Clarissa kam vorbei.", aliased)
|
||||
expect(canonical_clear.startswith("[[PERSON:07]]"), "canonical spelling of an active alias token is remapped")
|
||||
expect(
|
||||
_visible_response("Sushi kam vorbei.", aliased) == "Sushi kam vorbei.",
|
||||
"alias plaintext keeps the model surface form",
|
||||
)
|
||||
expect(
|
||||
_visible_response("Clarissa kam vorbei.", aliased) == "Clarissa kam vorbei.",
|
||||
"canonical plaintext keeps the model surface form",
|
||||
)
|
||||
expect(
|
||||
_visible_response("[[PERSON:07]] kam vorbei.", aliased) == "Clarissa kam vorbei.",
|
||||
"emitted placeholder still rehydrates to the canonical spelling",
|
||||
)
|
||||
|
||||
group_prompt = "Heute die Gruppe am Markt."
|
||||
group_start = group_prompt.index("die Gruppe")
|
||||
group_rows = [
|
||||
{
|
||||
"local_label": "die Gruppe",
|
||||
"token": "PERSON:04",
|
||||
"canonical_label": "die Gruppe",
|
||||
"demask_label": "die Gruppe",
|
||||
"entity_type": "PERSON",
|
||||
"start": group_start,
|
||||
"end": group_start + len("die Gruppe"),
|
||||
"source": "request_local",
|
||||
}
|
||||
]
|
||||
group = mask_prompt(group_prompt, group_rows, "journal_generate")
|
||||
expect(len(group.replacements) == 1, "observed multi-word label is active")
|
||||
mixed_raw = (
|
||||
"Die Gruppe verstand kein Wort. "
|
||||
"Später fragte ich, ob die Gruppe noch da sei. "
|
||||
"[[PERSON:04]] blieb. "
|
||||
"die Gruppen standen abseits."
|
||||
)
|
||||
tokenized, meta = _process_response(mixed_raw, group)
|
||||
expect(meta.get("response_normalization") == "active_cleartext_normalized", "check copy still remaps active cleartext")
|
||||
expect(tokenized.startswith("[[PERSON:04]]"), "check copy tokenizes sentence-initial cleartext")
|
||||
expect("die Gruppe verstand" in _demask(tokenized, group), "check roundtrip uses the stored label")
|
||||
visible = _visible_response(mixed_raw, group)
|
||||
expect(visible.startswith("Die Gruppe verstand"), "stored text keeps sentence-initial capitalization")
|
||||
expect("ob die Gruppe noch da sei" in visible, "stored text keeps mid-sentence form")
|
||||
expect("die Gruppe blieb" in visible, "emitted placeholder is rehydrated locally")
|
||||
expect("die Gruppen standen" in visible, "unmatched plural surface form is left unchanged")
|
||||
expect(visible != _demask(tokenized, group), "visible output is not the rematerialized check copy")
|
||||
leftover_unknown = _visible_response("[[PERSON:04]] und [[PERSON:99]] und [[…]]", group)
|
||||
expect(leftover_unknown.startswith("die Gruppe"), "known placeholder is rehydrated")
|
||||
expect("[[PERSON:99]]" in leftover_unknown, "unknown placeholder is not materialized")
|
||||
expect("[[…]]" in leftover_unknown, "generic placeholder is not materialized")
|
||||
|
||||
overlap = mask_prompt(
|
||||
"Anna-Lena kam vorbei.",
|
||||
@@ -296,7 +348,7 @@ def main() -> None:
|
||||
repaired = _run_gateway(profile_id, "Ich war mit Anna am Markt.")
|
||||
expect(len(seen) == 1, "active cleartext does not start a second generate call")
|
||||
expect("[[PERSON:01]]" in seen[0] and "Anna" not in seen[0].replace("[[PERSON:01]]", ""), "egress stays masked")
|
||||
expect("Anna" in (repaired.content or ""), "normalized cleartext is demasked locally")
|
||||
expect("Anna" in (repaired.content or ""), "model plaintext identity spelling is kept")
|
||||
expect(repaired.diagnostics.get("generate_calls") == 1, "exactly one generate call")
|
||||
expect(
|
||||
repaired.diagnostics.get("response_normalization") == "active_cleartext_normalized",
|
||||
@@ -324,6 +376,47 @@ def main() -> None:
|
||||
expect(repaired.diagnostics.get("total_tokens") == 18, "single call keeps total tokens")
|
||||
expect(abs(float(repaired.diagnostics.get("cost") or 0) - 0.01) < 1e-9, "single call keeps cost")
|
||||
|
||||
prompt_group = "Ich traf die Gruppe."
|
||||
group_at = prompt_group.index("die Gruppe")
|
||||
install_test_spans(
|
||||
[
|
||||
{
|
||||
"start": group_at,
|
||||
"end": group_at + len("die Gruppe"),
|
||||
"text": "die Gruppe",
|
||||
"entity_type": "PERSON",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
def mixed_surface(messages, _policy):
|
||||
egress = messages[0].get("content") or ""
|
||||
match = re.search(r"\[\[PERSON:\d+\]\]", egress)
|
||||
token = match.group(0) if match else "[[PERSON:01]]"
|
||||
return ChatResult(
|
||||
content=f"Die Gruppe verstand kein Wort. {token} blieb. Später die Gruppe wieder.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="not_applicable",
|
||||
)
|
||||
|
||||
try:
|
||||
with patch("privacy_gateway.complete_model", mixed_surface):
|
||||
surfaced = _run_gateway(profile_id, prompt_group)
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(surfaced.diagnostics.get("generate_calls") == 1, "mixed surface still uses one generate call")
|
||||
expect("Die Gruppe verstand" in ((surfaced.trace or {}).get("raw") or ""), "trace raw keeps model casing")
|
||||
expect("[[" in ((surfaced.trace or {}).get("raw") or ""), "trace raw keeps the emitted placeholder")
|
||||
expect(surfaced.content == (surfaced.trace or {}).get("reply"), "visible content matches trace reply")
|
||||
expect((surfaced.content or "").startswith("Die Gruppe verstand"), "gateway keeps sentence-initial model casing")
|
||||
expect("die Gruppe blieb" in (surfaced.content or ""), "gateway rehydrates the emitted placeholder")
|
||||
expect("Später die Gruppe wieder" in (surfaced.content or ""), "gateway keeps later plaintext unchanged")
|
||||
expect(
|
||||
surfaced.diagnostics.get("response_normalization") == "active_cleartext_normalized",
|
||||
"check copy still records cleartext normalization",
|
||||
)
|
||||
|
||||
inactive_calls = {"n": 0}
|
||||
|
||||
def keep_inactive(messages, _policy):
|
||||
|
||||
Reference in new issue
Block a user