MVP 0.93 - stabile, aber Writing Profile ohne erkennbaren Nutzen

This commit is contained in:
Lars committed 2026-08-27 10:03:05 +02:00
1 parent 53b1eff833
commit f17b3083e8
56 files changed
+5366 -784

No files matched your search

+272 -75
View File
@@ -1,17 +1,26 @@
"""Explicit journal draft generation. Never overwrites the current user entry."""
from __future__ import annotations
import re
from context_builder import assemble_text, build_internal_context
from dialogue_store import StoreError, list_conversations_for_day, list_messages
from engine import EngineError, execute_prompt, load_active_prompt
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
from journal_policy import require_explicit_generate, source_conversation_ids
from identity_store import list_mappings
from identity_store import is_maskable_label, list_mappings
from journal_body import clean_title
from journal_editorial import (
choose_editorial_mode,
editorial_instructions,
format_style_examples,
lexical_similarity,
narration_sources_text,
select_journal_style_examples,
)
from journal_reconstruct import (
assign_source_ids,
claim_texts,
local_verified_artifact,
reconstruction_from_model,
reconstruction_text,
)
from journal_shape import shape_journal
@@ -19,11 +28,25 @@ from journal_store import current_draft, current_entries, get_day, insert_draft
from model_catalog import resolve_generate_metadata
from prompt_budget import (
JournalBudgetError,
estimate_tokens,
assert_input_fits,
plan_journal_budget,
)
from privacy_gateway import (
GENERIC_PLACEHOLDER_INNER,
canonical_token,
identity_label_pattern,
identity_occurrence_count,
is_identity_mention,
)
from providers import generate_provider
from writing_profile_store import compile_style_signals, remember_dialogue_style
from retrieval import retrieve
from writing_profile_store import remember_dialogue_style
IDENTITY_PLACEHOLDER = re.compile(
r"\[\[\s*(?:SELF|PERSON:[^\]]+|PLACE:[^\]]+|ORG:[^\]]+|PROJECT:[^\]]+|…|\.{2,})\s*\]\]",
re.IGNORECASE,
)
def _message_ids(profile_id: str, conversation_ids: list[str]) -> list[str]:
@@ -53,17 +76,85 @@ def _local_narration(reconstruction: dict) -> tuple[str, str]:
return "Ein Tag", body
def unattested_journal_content(
text: str,
source_user: list[str],
mappings: list[dict],
active_tokens: list[str] | None = None,
) -> str | None:
"""Journal-only: historical names not in the sources are unattested facts, not a privacy leak.
Title and body are checked together. Placeholders must stay consistent across both.
"""
body = text or ""
allowed = {canonical_token(token).upper().replace(" ", "") for token in (active_tokens or [])}
allowed.add("SELF")
for match in IDENTITY_PLACEHOLDER.finditer(body):
token = canonical_token(match.group(0)).upper().replace(" ", "")
if token == "SELF":
continue
if token in GENERIC_PLACEHOLDER_INNER or token not in allowed:
return "unattested_placeholder"
sources = "\n".join(source_user or [])
for item in mappings or []:
label = (item.get("local_label") or "").strip()
token = (item.get("token") or "").strip()
if not label or not token or not is_maskable_label(label):
continue
if identity_occurrence_count(sources, label, token) > 0:
continue
for match in identity_label_pattern(label).finditer(body):
if is_identity_mention(body, match.start(), match.end(), token):
return "unattested_identity"
return None
def _identity_leak_result(purpose: str, exc: EngineError) -> dict:
diag = getattr(exc, "diagnostics", None) or {}
return {
"content": "",
"trace": {
"purpose": purpose,
"guard": "identity_leak_blocked",
"log": list(diag.get("log") or []),
"response_validation_retry": diag.get("response_validation_retry"),
},
"diagnostics": getattr(exc, "diagnostics", None) or {},
"diagnostics": diag,
}
def _log_items(source: dict | BaseException | None) -> list[dict]:
if source is None:
return []
if isinstance(source, BaseException):
diag = getattr(source, "diagnostics", None) or {}
return list(diag.get("log") or [])
if not isinstance(source, dict):
return []
if source.get("log"):
return list(source["log"])
trace = source.get("trace") or {}
if trace.get("log"):
return list(trace["log"])
diag = source.get("diagnostics") or {}
return list(diag.get("log") or [])
def _stamp_log(run_log: list[dict], stage: str, items: list[dict]) -> None:
for item in items:
row = dict(item)
row.setdefault("stage", stage)
run_log.append(row)
def _event(run_log: list[dict], stage: str, kind: str, **fields) -> None:
row = {"stage": stage, "kind": kind}
for key, value in fields.items():
if value is not None and value != "":
row[key] = value
run_log.append(row)
def _raise_budget(exc: JournalBudgetError) -> None:
raise EngineError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc
@@ -78,11 +169,28 @@ def _stage_trace(result: dict, fallback_purpose: str) -> dict:
return trace
def _day_messages_from_context(context: dict) -> list[dict]:
for item in context.get("items") or []:
if item.get("type") == "day_messages":
return list(item.get("messages") or [])
return []
def _local_stage_trace(artifact: dict, *, source_count: int) -> dict:
return {
"purpose": "local_source_artifact",
"layer": "journalquellen",
"provider": None,
"model": None,
"coverage": artifact.get("coverage") or "all_selected_sources",
"status": "local_ok",
"stage1": "local_ok",
"source_count": source_count,
"intern": reconstruction_text(artifact),
"budget": {
"purpose": "local_source_artifact",
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0,
"cost": 0,
"budget_ok": True,
"status": "local",
"context_compression": "not_applicable",
},
}
def _existing_text(profile_id: str, journal_day_id: str) -> str:
@@ -97,6 +205,46 @@ def _existing_text(profile_id: str, journal_day_id: str) -> str:
return ""
def pack_narration_context(
prompt: dict,
budget,
assembled: dict[str, str],
*,
style_examples: str,
existing_text: str,
include_existing: bool,
) -> tuple[dict[str, str], list[str]]:
"""Drop optional blocks locally if they overflow. Never drop day sources or the profile.
No extra model call. Order: try all, then drop style examples, then existing text.
"""
dropped: list[str] = []
attempts = [
(style_examples, existing_text if include_existing else ""),
("", existing_text if include_existing else ""),
("", ""),
]
last_error: JournalBudgetError | None = None
for examples, existing in attempts:
candidate = dict(assembled)
candidate["style_examples"] = examples
candidate["existing_text"] = existing
rendered = preview_prompt(prompt, candidate)["rendered"]
try:
assert_input_fits(budget, rendered)
if not examples and style_examples:
dropped.append("style_examples")
if include_existing and existing_text and not existing:
dropped.append("existing_text")
return candidate, dropped
except JournalBudgetError as exc:
last_error = exc
continue
if last_error:
raise last_error
return assembled, dropped
def generate_draft(
profile_id: str,
journal_day_id: str,
@@ -124,75 +272,51 @@ def generate_draft(
)
try:
window = resolve_generate_metadata(config)
reconstruct_budget = plan_journal_budget(window, purpose="journal_reconstruct")
narrate_budget = plan_journal_budget(window, purpose="journal_generate")
except JournalBudgetError as exc:
_raise_budget(exc)
reconstruct_prompt = load_active_prompt("mvp.journal_reconstruct")
static_tokens = estimate_tokens(reconstruct_prompt.get("template") or "")
available_for_day = reconstruct_budget.available_input_tokens - static_tokens
if available_for_day < 256:
raise EngineError(
"prompt_budget_exceeded",
"Der Tagesdialog ist für eine sichere Verarbeitung zu umfangreich. "
"Kanshō hat nichts stillschweigend aus der Mitte entfernt.",
422,
diagnostics=reconstruct_budget.as_diagnostics(),
)
try:
reconstruct_context = build_internal_context(
source_messages = retrieve(
profile_id,
space_id=day["space_id"],
journal_day_id=journal_day_id,
purpose="journal_reconstruct",
conversation_ids=selected,
conversation_id=selected[0] if selected else None,
day_spec={
{
"kind": "day_messages",
"journal_day_id": journal_day_id,
"conversation_ids": selected,
"overflow": "abort",
"max_estimated_tokens": available_for_day,
},
)
reconstruction = local_verified_artifact(source_messages)
except JournalBudgetError as exc:
_raise_budget(exc)
reconstruct_assembled = assemble_text(reconstruct_context)
source_messages = assign_source_ids(_day_messages_from_context(reconstruct_context))
source_user = [
message.get("body") or ""
for message in source_messages
if message.get("role") == "user"
for message in assign_source_ids(source_messages)
if message.get("role") == "user" and (message.get("body") or "").strip()
]
reconstruct_result = {"trace": {"purpose": "journal_reconstruct"}, "content": "", "diagnostics": {}}
try:
reconstruct_result = execute_prompt(
reconstruct_prompt,
profile_id,
purpose="journal_reconstruct",
data_class="B",
context=reconstruct_assembled,
max_tokens=reconstruct_budget.reserved_output_tokens,
disable_context_compression=True,
budget=reconstruct_budget,
)
reconstruction, stage1 = reconstruction_from_model(
reconstruct_result.get("content") or "",
source_messages,
)
except EngineError as exc:
if exc.code != "response_validation_failed":
raise
reconstruction = local_verified_artifact(source_messages)
stage1 = {
"stage1": "local_fallback",
"reason": "identity_leak_blocked",
"model_rejected": exc.code,
}
reconstruct_result = _identity_leak_result("journal_reconstruct", exc)
except JournalBudgetError as exc:
_raise_budget(exc)
editorial_mode = choose_editorial_mode(source_user)
style_example_rows = select_journal_style_examples(
profile_id,
exclude_dates=[day.get("calendar_date") or ""],
)
style_examples = format_style_examples(style_example_rows)
run_log: list[dict] = []
_event(
run_log,
"local_source_artifact",
"stage1_result",
stage1="local_ok",
coverage=reconstruction.get("coverage") or "all_selected_sources",
status="local_ok",
source_count=len(source_user),
editorial_mode=editorial_mode,
)
reconstruct_result = {
"trace": _local_stage_trace(reconstruction, source_count=len(source_user)),
"content": reconstruction_text(reconstruction),
}
signals = compile_style_signals(source_user)
narrate_prompt = load_active_prompt("mvp.journal_generate")
narrate_context = build_internal_context(
profile_id,
space_id=day["space_id"],
@@ -202,13 +326,28 @@ def generate_draft(
conversation_ids=selected,
existing_text=existing_text,
conversation_id=selected[0] if selected else None,
reconstruction=reconstruction_text(reconstruction),
reconstruction=narration_sources_text(reconstruction),
style_examples=style_examples,
editorial_mode=editorial_mode,
editorial_instructions=editorial_instructions(editorial_mode),
)
assembled = assemble_text(narrate_context)
profile = assembled.get("writing_profile") or ""
if signals and signals not in profile and len(profile) < 3500:
assembled["writing_profile"] = "\n\n".join(part for part in (profile, signals) if part)
narrate_prompt = load_active_prompt("mvp.journal_generate")
try:
assembled, dropped = pack_narration_context(
narrate_prompt,
narrate_budget,
assembled,
style_examples=style_examples,
existing_text=existing_text,
include_existing=include_existing,
)
except JournalBudgetError as exc:
_raise_budget(exc)
if dropped:
_event(run_log, "journal_generate", "budget_pack", dropped=",".join(dropped))
shape_source = "model"
mappings = list_mappings(profile_id)
narrate_result: dict = {}
try:
narrate_result = execute_prompt(
narrate_prompt,
@@ -220,20 +359,70 @@ def generate_draft(
disable_context_compression=True,
budget=narrate_budget,
)
_stamp_log(run_log, "journal_generate", _log_items(narrate_result))
title, body = _split_title(narrate_result.get("content") or "")
seen = {(item.get("local_label") or "").casefold() for item in mappings}
for item in narrate_result.get("local_identities") or []:
label = (item.get("local_label") or "").strip()
if not label or label.casefold() in seen:
continue
mappings.append(item)
seen.add(label.casefold())
active_tokens = list((narrate_result.get("diagnostics") or {}).get("active_tokens") or [])
combined = f"{title}\n\n{body}".strip()
unattested = unattested_journal_content(combined, source_user, mappings, active_tokens)
if unattested:
title, body = _local_narration(reconstruction)
shape_source = "fallback"
narrate_result = {
**narrate_result,
"content": f"{title}\n\n{body}".strip(),
"trace": {
**(narrate_result.get("trace") or {}),
"guard": "unattested_content_blocked",
"reason": unattested,
},
}
_event(
run_log,
"journal_generate",
"narration_result",
status="local_fallback",
reason=unattested,
)
else:
_event(
run_log,
"journal_generate",
"narration_result",
status="model",
editorial_mode=editorial_mode,
lexical_similarity=lexical_similarity("\n".join(source_user), body),
)
except EngineError as exc:
_stamp_log(run_log, "journal_generate", _log_items(exc))
if exc.code != "response_validation_failed":
raise
diag = dict(exc.diagnostics or {})
diag["log"] = run_log
raise EngineError(exc.code, exc.message, exc.status_code, diag) from exc
title, body = _local_narration(reconstruction)
shape_source = "fallback"
narrate_result = _identity_leak_result("journal_generate", exc)
narrate_result["content"] = f"{title}\n\n{body}".strip()
_event(run_log, "journal_generate", "narration_result", status="local_fallback", reason="identity_leak_blocked")
person_labels = [
(item.get("local_label") or "").strip()
for item in list_mappings(profile_id)
for item in mappings
if (item.get("local_label") or "").strip()
and str(item.get("token") or "").upper().lstrip("[").startswith("PERSON:")
]
title, body = shape_journal(title, body, source_user, person_labels=person_labels)
title, body = shape_journal(
title,
body,
source_user,
person_labels=person_labels,
source=shape_source,
)
before_entries = {item["id"]: item.get("current_version_id") for item in current_entries(profile_id, journal_day_id)}
draft = insert_draft(
profile_id,
@@ -248,11 +437,19 @@ def generate_draft(
previous = before_entries.get(item["id"])
if previous is not None and previous != item.get("current_version_id"):
raise StoreError("policy_violation", "Generate darf die Nutzerfassung nicht verändern", 500)
reconstruct_trace = _stage_trace(reconstruct_result, "journal_reconstruct")
reconstruct_trace.update(stage1)
reconstruct_trace = _stage_trace(reconstruct_result, "local_source_artifact")
reconstruct_trace.setdefault("purpose", "local_source_artifact")
reconstruct_trace.setdefault("status", "local_ok")
reconstruct_trace.setdefault("stage1", "local_ok")
narrate_trace = _stage_trace(narrate_result, "journal_generate")
narrate_trace["editorial_mode"] = editorial_mode
if shape_source == "model":
narrate_trace["lexical_similarity"] = lexical_similarity("\n".join(source_user), body)
draft["run_log"] = run_log
draft["trace"] = {
**narrate_trace,
"editorial_mode": editorial_mode,
"log": run_log,
"stages": [reconstruct_trace, narrate_trace],
}
return draft