456 lines
16 KiB
Python
456 lines
16 KiB
Python
"""Explicit journal draft generation. Never overwrites the current user entry."""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
from context_builder import assemble_text, build_internal_context
|
|
from dialogue_store import StoreError, list_conversations_for_day, list_messages
|
|
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
|
from journal_policy import require_explicit_generate, source_conversation_ids
|
|
from identity_store import is_maskable_label, list_mappings
|
|
from journal_body import clean_title
|
|
from journal_editorial import (
|
|
choose_editorial_mode,
|
|
editorial_instructions,
|
|
format_style_examples,
|
|
lexical_similarity,
|
|
narration_sources_text,
|
|
select_journal_style_examples,
|
|
)
|
|
from journal_reconstruct import (
|
|
assign_source_ids,
|
|
claim_texts,
|
|
local_verified_artifact,
|
|
reconstruction_text,
|
|
)
|
|
from journal_shape import shape_journal
|
|
from journal_store import current_draft, current_entries, get_day, insert_draft
|
|
from model_catalog import resolve_generate_metadata
|
|
from prompt_budget import (
|
|
JournalBudgetError,
|
|
assert_input_fits,
|
|
plan_journal_budget,
|
|
)
|
|
from privacy_gateway import (
|
|
GENERIC_PLACEHOLDER_INNER,
|
|
canonical_token,
|
|
identity_label_pattern,
|
|
identity_occurrence_count,
|
|
is_identity_mention,
|
|
)
|
|
from providers import generate_provider
|
|
from retrieval import retrieve
|
|
from writing_profile_store import remember_dialogue_style
|
|
|
|
|
|
IDENTITY_PLACEHOLDER = re.compile(
|
|
r"\[\[\s*(?:SELF|PERSON:[^\]]+|PLACE:[^\]]+|ORG:[^\]]+|PROJECT:[^\]]+|…|\.{2,})\s*\]\]",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _message_ids(profile_id: str, conversation_ids: list[str]) -> list[str]:
|
|
ids: list[str] = []
|
|
for conversation_id in conversation_ids:
|
|
for message in list_messages(profile_id, conversation_id):
|
|
ids.append(message["id"])
|
|
return ids
|
|
|
|
|
|
def _split_title(content: str) -> tuple[str, str]:
|
|
text = (content or "").strip()
|
|
if not text:
|
|
return "", ""
|
|
lines = text.splitlines()
|
|
title = clean_title(lines[0])
|
|
body = "\n".join(lines[1:]).strip() if len(lines) > 1 else text
|
|
if not title or len(title) > 80:
|
|
return "", text
|
|
return title, body or text
|
|
|
|
|
|
def _local_narration(reconstruction: dict) -> tuple[str, str]:
|
|
"""Fail-closed draft from verified local sources. No leaking model text."""
|
|
parts = [str(part).strip() for part in claim_texts(reconstruction) if str(part).strip()]
|
|
body = "\n\n".join(parts)
|
|
return "Ein Tag", body
|
|
|
|
|
|
def unattested_journal_content(
|
|
text: str,
|
|
source_user: list[str],
|
|
mappings: list[dict],
|
|
active_tokens: list[str] | None = None,
|
|
) -> str | None:
|
|
"""Journal-only: historical names not in the sources are unattested facts, not a privacy leak.
|
|
|
|
Title and body are checked together. Placeholders must stay consistent across both.
|
|
"""
|
|
body = text or ""
|
|
allowed = {canonical_token(token).upper().replace(" ", "") for token in (active_tokens or [])}
|
|
allowed.add("SELF")
|
|
for match in IDENTITY_PLACEHOLDER.finditer(body):
|
|
token = canonical_token(match.group(0)).upper().replace(" ", "")
|
|
if token == "SELF":
|
|
continue
|
|
if token in GENERIC_PLACEHOLDER_INNER or token not in allowed:
|
|
return "unattested_placeholder"
|
|
sources = "\n".join(source_user or [])
|
|
for item in mappings or []:
|
|
label = (item.get("local_label") or "").strip()
|
|
token = (item.get("token") or "").strip()
|
|
if not label or not token or not is_maskable_label(label):
|
|
continue
|
|
if identity_occurrence_count(sources, label, token) > 0:
|
|
continue
|
|
for match in identity_label_pattern(label).finditer(body):
|
|
if is_identity_mention(body, match.start(), match.end(), token):
|
|
return "unattested_identity"
|
|
return None
|
|
|
|
|
|
def _identity_leak_result(purpose: str, exc: EngineError) -> dict:
|
|
diag = getattr(exc, "diagnostics", None) or {}
|
|
return {
|
|
"content": "",
|
|
"trace": {
|
|
"purpose": purpose,
|
|
"guard": "identity_leak_blocked",
|
|
"log": list(diag.get("log") or []),
|
|
"response_validation_retry": diag.get("response_validation_retry"),
|
|
},
|
|
"diagnostics": diag,
|
|
}
|
|
|
|
|
|
def _log_items(source: dict | BaseException | None) -> list[dict]:
|
|
if source is None:
|
|
return []
|
|
if isinstance(source, BaseException):
|
|
diag = getattr(source, "diagnostics", None) or {}
|
|
return list(diag.get("log") or [])
|
|
if not isinstance(source, dict):
|
|
return []
|
|
if source.get("log"):
|
|
return list(source["log"])
|
|
trace = source.get("trace") or {}
|
|
if trace.get("log"):
|
|
return list(trace["log"])
|
|
diag = source.get("diagnostics") or {}
|
|
return list(diag.get("log") or [])
|
|
|
|
|
|
def _stamp_log(run_log: list[dict], stage: str, items: list[dict]) -> None:
|
|
for item in items:
|
|
row = dict(item)
|
|
row.setdefault("stage", stage)
|
|
run_log.append(row)
|
|
|
|
|
|
def _event(run_log: list[dict], stage: str, kind: str, **fields) -> None:
|
|
row = {"stage": stage, "kind": kind}
|
|
for key, value in fields.items():
|
|
if value is not None and value != "":
|
|
row[key] = value
|
|
run_log.append(row)
|
|
|
|
|
|
def _raise_budget(exc: JournalBudgetError) -> None:
|
|
raise EngineError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc
|
|
|
|
|
|
def _stage_trace(result: dict, fallback_purpose: str) -> dict:
|
|
trace = dict(result.get("trace") or {})
|
|
if result.get("diagnostics"):
|
|
trace["budget"] = result.get("diagnostics")
|
|
elif not trace.get("budget"):
|
|
trace["budget"] = None
|
|
trace["purpose"] = trace.get("purpose") or fallback_purpose
|
|
return trace
|
|
|
|
|
|
def _local_stage_trace(artifact: dict, *, source_count: int) -> dict:
|
|
return {
|
|
"purpose": "local_source_artifact",
|
|
"layer": "journalquellen",
|
|
"provider": None,
|
|
"model": None,
|
|
"coverage": artifact.get("coverage") or "all_selected_sources",
|
|
"status": "local_ok",
|
|
"stage1": "local_ok",
|
|
"source_count": source_count,
|
|
"intern": reconstruction_text(artifact),
|
|
"budget": {
|
|
"purpose": "local_source_artifact",
|
|
"prompt_tokens": 0,
|
|
"completion_tokens": 0,
|
|
"total_tokens": 0,
|
|
"cost": 0,
|
|
"budget_ok": True,
|
|
"status": "local",
|
|
"context_compression": "not_applicable",
|
|
},
|
|
}
|
|
|
|
|
|
def _existing_text(profile_id: str, journal_day_id: str) -> str:
|
|
"""Saved user entries are the Fassung. A leftover draft is only used if none exist."""
|
|
entries = current_entries(profile_id, journal_day_id)
|
|
bodies = [(item.get("body") or "").strip() for item in entries if (item.get("body") or "").strip()]
|
|
if bodies:
|
|
return "\n\n".join(bodies)
|
|
draft = current_draft(profile_id, journal_day_id)
|
|
if draft:
|
|
return (draft.get("body") or "").strip()
|
|
return ""
|
|
|
|
|
|
def pack_narration_context(
|
|
prompt: dict,
|
|
budget,
|
|
assembled: dict[str, str],
|
|
*,
|
|
style_examples: str,
|
|
existing_text: str,
|
|
include_existing: bool,
|
|
) -> tuple[dict[str, str], list[str]]:
|
|
"""Drop optional blocks locally if they overflow. Never drop day sources or the profile.
|
|
|
|
No extra model call. Order: try all, then drop style examples, then existing text.
|
|
"""
|
|
dropped: list[str] = []
|
|
attempts = [
|
|
(style_examples, existing_text if include_existing else ""),
|
|
("", existing_text if include_existing else ""),
|
|
("", ""),
|
|
]
|
|
last_error: JournalBudgetError | None = None
|
|
for examples, existing in attempts:
|
|
candidate = dict(assembled)
|
|
candidate["style_examples"] = examples
|
|
candidate["existing_text"] = existing
|
|
rendered = preview_prompt(prompt, candidate)["rendered"]
|
|
try:
|
|
assert_input_fits(budget, rendered)
|
|
if not examples and style_examples:
|
|
dropped.append("style_examples")
|
|
if include_existing and existing_text and not existing:
|
|
dropped.append("existing_text")
|
|
return candidate, dropped
|
|
except JournalBudgetError as exc:
|
|
last_error = exc
|
|
continue
|
|
if last_error:
|
|
raise last_error
|
|
return assembled, dropped
|
|
|
|
|
|
def generate_draft(
|
|
profile_id: str,
|
|
journal_day_id: str,
|
|
conversation_ids: list[str] | None = None,
|
|
include_existing: bool = False,
|
|
explicit: bool = True,
|
|
) -> dict:
|
|
require_explicit_generate(explicit)
|
|
day = get_day(profile_id, journal_day_id)
|
|
day_conversations = list_conversations_for_day(profile_id, journal_day_id)
|
|
selected = source_conversation_ids(
|
|
conversation_ids,
|
|
[item["id"] for item in day_conversations],
|
|
)
|
|
remember_dialogue_style(profile_id, exclude_conversation_ids=selected)
|
|
existing_text = _existing_text(profile_id, journal_day_id) if include_existing else ""
|
|
|
|
config = generate_provider()
|
|
if not config:
|
|
raise EngineError(
|
|
"no_egress_provider_configured",
|
|
"Persönlicher KI-Aufruf wurde vom Privacy Gateway blockiert. "
|
|
"Es ist kein Egress-Provider konfiguriert.",
|
|
503,
|
|
)
|
|
try:
|
|
window = resolve_generate_metadata(config)
|
|
narrate_budget = plan_journal_budget(window, purpose="journal_generate")
|
|
except JournalBudgetError as exc:
|
|
_raise_budget(exc)
|
|
|
|
try:
|
|
source_messages = retrieve(
|
|
profile_id,
|
|
{
|
|
"kind": "day_messages",
|
|
"journal_day_id": journal_day_id,
|
|
"conversation_ids": selected,
|
|
"overflow": "abort",
|
|
},
|
|
)
|
|
reconstruction = local_verified_artifact(source_messages)
|
|
except JournalBudgetError as exc:
|
|
_raise_budget(exc)
|
|
source_user = [
|
|
message.get("body") or ""
|
|
for message in assign_source_ids(source_messages)
|
|
if message.get("role") == "user" and (message.get("body") or "").strip()
|
|
]
|
|
editorial_mode = choose_editorial_mode(source_user)
|
|
style_example_rows = select_journal_style_examples(
|
|
profile_id,
|
|
exclude_dates=[day.get("calendar_date") or ""],
|
|
)
|
|
style_examples = format_style_examples(style_example_rows)
|
|
run_log: list[dict] = []
|
|
_event(
|
|
run_log,
|
|
"local_source_artifact",
|
|
"stage1_result",
|
|
stage1="local_ok",
|
|
coverage=reconstruction.get("coverage") or "all_selected_sources",
|
|
status="local_ok",
|
|
source_count=len(source_user),
|
|
editorial_mode=editorial_mode,
|
|
)
|
|
reconstruct_result = {
|
|
"trace": _local_stage_trace(reconstruction, source_count=len(source_user)),
|
|
"content": reconstruction_text(reconstruction),
|
|
}
|
|
|
|
narrate_prompt = load_active_prompt("mvp.journal_generate")
|
|
narrate_context = build_internal_context(
|
|
profile_id,
|
|
space_id=day["space_id"],
|
|
journal_day_id=journal_day_id,
|
|
purpose="journal_generate",
|
|
include_existing=include_existing,
|
|
conversation_ids=selected,
|
|
existing_text=existing_text,
|
|
conversation_id=selected[0] if selected else None,
|
|
reconstruction=narration_sources_text(reconstruction),
|
|
style_examples=style_examples,
|
|
editorial_mode=editorial_mode,
|
|
editorial_instructions=editorial_instructions(editorial_mode),
|
|
)
|
|
assembled = assemble_text(narrate_context)
|
|
try:
|
|
assembled, dropped = pack_narration_context(
|
|
narrate_prompt,
|
|
narrate_budget,
|
|
assembled,
|
|
style_examples=style_examples,
|
|
existing_text=existing_text,
|
|
include_existing=include_existing,
|
|
)
|
|
except JournalBudgetError as exc:
|
|
_raise_budget(exc)
|
|
if dropped:
|
|
_event(run_log, "journal_generate", "budget_pack", dropped=",".join(dropped))
|
|
shape_source = "model"
|
|
mappings = list_mappings(profile_id)
|
|
narrate_result: dict = {}
|
|
try:
|
|
narrate_result = execute_prompt(
|
|
narrate_prompt,
|
|
profile_id,
|
|
purpose="journal_generate",
|
|
data_class="B",
|
|
context=assembled,
|
|
max_tokens=narrate_budget.reserved_output_tokens,
|
|
disable_context_compression=True,
|
|
budget=narrate_budget,
|
|
)
|
|
_stamp_log(run_log, "journal_generate", _log_items(narrate_result))
|
|
title, body = _split_title(narrate_result.get("content") or "")
|
|
seen = {(item.get("local_label") or "").casefold() for item in mappings}
|
|
for item in narrate_result.get("local_identities") or []:
|
|
label = (item.get("local_label") or "").strip()
|
|
if not label or label.casefold() in seen:
|
|
continue
|
|
mappings.append(item)
|
|
seen.add(label.casefold())
|
|
active_tokens = list((narrate_result.get("diagnostics") or {}).get("active_tokens") or [])
|
|
combined = f"{title}\n\n{body}".strip()
|
|
unattested = unattested_journal_content(combined, source_user, mappings, active_tokens)
|
|
if unattested:
|
|
title, body = _local_narration(reconstruction)
|
|
shape_source = "fallback"
|
|
narrate_result = {
|
|
**narrate_result,
|
|
"content": f"{title}\n\n{body}".strip(),
|
|
"trace": {
|
|
**(narrate_result.get("trace") or {}),
|
|
"guard": "unattested_content_blocked",
|
|
"reason": unattested,
|
|
},
|
|
}
|
|
_event(
|
|
run_log,
|
|
"journal_generate",
|
|
"narration_result",
|
|
status="local_fallback",
|
|
reason=unattested,
|
|
)
|
|
else:
|
|
_event(
|
|
run_log,
|
|
"journal_generate",
|
|
"narration_result",
|
|
status="model",
|
|
editorial_mode=editorial_mode,
|
|
lexical_similarity=lexical_similarity("\n".join(source_user), body),
|
|
)
|
|
except EngineError as exc:
|
|
_stamp_log(run_log, "journal_generate", _log_items(exc))
|
|
if exc.code != "response_validation_failed":
|
|
diag = dict(exc.diagnostics or {})
|
|
diag["log"] = run_log
|
|
raise EngineError(exc.code, exc.message, exc.status_code, diag) from exc
|
|
title, body = _local_narration(reconstruction)
|
|
shape_source = "fallback"
|
|
narrate_result = _identity_leak_result("journal_generate", exc)
|
|
narrate_result["content"] = f"{title}\n\n{body}".strip()
|
|
_event(run_log, "journal_generate", "narration_result", status="local_fallback", reason="identity_leak_blocked")
|
|
person_labels = [
|
|
(item.get("local_label") or "").strip()
|
|
for item in mappings
|
|
if (item.get("local_label") or "").strip()
|
|
and str(item.get("token") or "").upper().lstrip("[").startswith("PERSON:")
|
|
]
|
|
title, body = shape_journal(
|
|
title,
|
|
body,
|
|
source_user,
|
|
person_labels=person_labels,
|
|
source=shape_source,
|
|
)
|
|
before_entries = {item["id"]: item.get("current_version_id") for item in current_entries(profile_id, journal_day_id)}
|
|
draft = insert_draft(
|
|
profile_id,
|
|
journal_day_id,
|
|
title=title,
|
|
body=body,
|
|
source_conversation_ids=selected,
|
|
source_message_ids=_message_ids(profile_id, selected),
|
|
)
|
|
after_entries = current_entries(profile_id, journal_day_id)
|
|
for item in after_entries:
|
|
previous = before_entries.get(item["id"])
|
|
if previous is not None and previous != item.get("current_version_id"):
|
|
raise StoreError("policy_violation", "Generate darf die Nutzerfassung nicht verändern", 500)
|
|
reconstruct_trace = _stage_trace(reconstruct_result, "local_source_artifact")
|
|
reconstruct_trace.setdefault("purpose", "local_source_artifact")
|
|
reconstruct_trace.setdefault("status", "local_ok")
|
|
reconstruct_trace.setdefault("stage1", "local_ok")
|
|
narrate_trace = _stage_trace(narrate_result, "journal_generate")
|
|
narrate_trace["editorial_mode"] = editorial_mode
|
|
if shape_source == "model":
|
|
narrate_trace["lexical_similarity"] = lexical_similarity("\n".join(source_user), body)
|
|
draft["run_log"] = run_log
|
|
draft["trace"] = {
|
|
**narrate_trace,
|
|
"editorial_mode": editorial_mode,
|
|
"log": run_log,
|
|
"stages": [reconstruct_trace, narrate_trace],
|
|
}
|
|
return draft
|