MVP 0.93 - stabile, aber Writing Profile ohne erkennbaren Nutzen
This commit is contained in:
1 parent
53b1eff833
commit
f17b3083e8
56 files changed
+5366
-784
No files matched your search
+272
-75
@@ -1,17 +1,26 @@
|
||||
"""Explicit journal draft generation. Never overwrites the current user entry."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
from dialogue_store import StoreError, list_conversations_for_day, list_messages
|
||||
from engine import EngineError, execute_prompt, load_active_prompt
|
||||
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
||||
from journal_policy import require_explicit_generate, source_conversation_ids
|
||||
from identity_store import list_mappings
|
||||
from identity_store import is_maskable_label, list_mappings
|
||||
from journal_body import clean_title
|
||||
from journal_editorial import (
|
||||
choose_editorial_mode,
|
||||
editorial_instructions,
|
||||
format_style_examples,
|
||||
lexical_similarity,
|
||||
narration_sources_text,
|
||||
select_journal_style_examples,
|
||||
)
|
||||
from journal_reconstruct import (
|
||||
assign_source_ids,
|
||||
claim_texts,
|
||||
local_verified_artifact,
|
||||
reconstruction_from_model,
|
||||
reconstruction_text,
|
||||
)
|
||||
from journal_shape import shape_journal
|
||||
@@ -19,11 +28,25 @@ from journal_store import current_draft, current_entries, get_day, insert_draft
|
||||
from model_catalog import resolve_generate_metadata
|
||||
from prompt_budget import (
|
||||
JournalBudgetError,
|
||||
estimate_tokens,
|
||||
assert_input_fits,
|
||||
plan_journal_budget,
|
||||
)
|
||||
from privacy_gateway import (
|
||||
GENERIC_PLACEHOLDER_INNER,
|
||||
canonical_token,
|
||||
identity_label_pattern,
|
||||
identity_occurrence_count,
|
||||
is_identity_mention,
|
||||
)
|
||||
from providers import generate_provider
|
||||
from writing_profile_store import compile_style_signals, remember_dialogue_style
|
||||
from retrieval import retrieve
|
||||
from writing_profile_store import remember_dialogue_style
|
||||
|
||||
|
||||
IDENTITY_PLACEHOLDER = re.compile(
|
||||
r"\[\[\s*(?:SELF|PERSON:[^\]]+|PLACE:[^\]]+|ORG:[^\]]+|PROJECT:[^\]]+|…|\.{2,})\s*\]\]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _message_ids(profile_id: str, conversation_ids: list[str]) -> list[str]:
|
||||
@@ -53,17 +76,85 @@ def _local_narration(reconstruction: dict) -> tuple[str, str]:
|
||||
return "Ein Tag", body
|
||||
|
||||
|
||||
def unattested_journal_content(
|
||||
text: str,
|
||||
source_user: list[str],
|
||||
mappings: list[dict],
|
||||
active_tokens: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Journal-only: historical names not in the sources are unattested facts, not a privacy leak.
|
||||
|
||||
Title and body are checked together. Placeholders must stay consistent across both.
|
||||
"""
|
||||
body = text or ""
|
||||
allowed = {canonical_token(token).upper().replace(" ", "") for token in (active_tokens or [])}
|
||||
allowed.add("SELF")
|
||||
for match in IDENTITY_PLACEHOLDER.finditer(body):
|
||||
token = canonical_token(match.group(0)).upper().replace(" ", "")
|
||||
if token == "SELF":
|
||||
continue
|
||||
if token in GENERIC_PLACEHOLDER_INNER or token not in allowed:
|
||||
return "unattested_placeholder"
|
||||
sources = "\n".join(source_user or [])
|
||||
for item in mappings or []:
|
||||
label = (item.get("local_label") or "").strip()
|
||||
token = (item.get("token") or "").strip()
|
||||
if not label or not token or not is_maskable_label(label):
|
||||
continue
|
||||
if identity_occurrence_count(sources, label, token) > 0:
|
||||
continue
|
||||
for match in identity_label_pattern(label).finditer(body):
|
||||
if is_identity_mention(body, match.start(), match.end(), token):
|
||||
return "unattested_identity"
|
||||
return None
|
||||
|
||||
|
||||
def _identity_leak_result(purpose: str, exc: EngineError) -> dict:
|
||||
diag = getattr(exc, "diagnostics", None) or {}
|
||||
return {
|
||||
"content": "",
|
||||
"trace": {
|
||||
"purpose": purpose,
|
||||
"guard": "identity_leak_blocked",
|
||||
"log": list(diag.get("log") or []),
|
||||
"response_validation_retry": diag.get("response_validation_retry"),
|
||||
},
|
||||
"diagnostics": getattr(exc, "diagnostics", None) or {},
|
||||
"diagnostics": diag,
|
||||
}
|
||||
|
||||
|
||||
def _log_items(source: dict | BaseException | None) -> list[dict]:
|
||||
if source is None:
|
||||
return []
|
||||
if isinstance(source, BaseException):
|
||||
diag = getattr(source, "diagnostics", None) or {}
|
||||
return list(diag.get("log") or [])
|
||||
if not isinstance(source, dict):
|
||||
return []
|
||||
if source.get("log"):
|
||||
return list(source["log"])
|
||||
trace = source.get("trace") or {}
|
||||
if trace.get("log"):
|
||||
return list(trace["log"])
|
||||
diag = source.get("diagnostics") or {}
|
||||
return list(diag.get("log") or [])
|
||||
|
||||
|
||||
def _stamp_log(run_log: list[dict], stage: str, items: list[dict]) -> None:
|
||||
for item in items:
|
||||
row = dict(item)
|
||||
row.setdefault("stage", stage)
|
||||
run_log.append(row)
|
||||
|
||||
|
||||
def _event(run_log: list[dict], stage: str, kind: str, **fields) -> None:
|
||||
row = {"stage": stage, "kind": kind}
|
||||
for key, value in fields.items():
|
||||
if value is not None and value != "":
|
||||
row[key] = value
|
||||
run_log.append(row)
|
||||
|
||||
|
||||
def _raise_budget(exc: JournalBudgetError) -> None:
|
||||
raise EngineError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc
|
||||
|
||||
@@ -78,11 +169,28 @@ def _stage_trace(result: dict, fallback_purpose: str) -> dict:
|
||||
return trace
|
||||
|
||||
|
||||
def _day_messages_from_context(context: dict) -> list[dict]:
|
||||
for item in context.get("items") or []:
|
||||
if item.get("type") == "day_messages":
|
||||
return list(item.get("messages") or [])
|
||||
return []
|
||||
def _local_stage_trace(artifact: dict, *, source_count: int) -> dict:
|
||||
return {
|
||||
"purpose": "local_source_artifact",
|
||||
"layer": "journalquellen",
|
||||
"provider": None,
|
||||
"model": None,
|
||||
"coverage": artifact.get("coverage") or "all_selected_sources",
|
||||
"status": "local_ok",
|
||||
"stage1": "local_ok",
|
||||
"source_count": source_count,
|
||||
"intern": reconstruction_text(artifact),
|
||||
"budget": {
|
||||
"purpose": "local_source_artifact",
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
"cost": 0,
|
||||
"budget_ok": True,
|
||||
"status": "local",
|
||||
"context_compression": "not_applicable",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _existing_text(profile_id: str, journal_day_id: str) -> str:
|
||||
@@ -97,6 +205,46 @@ def _existing_text(profile_id: str, journal_day_id: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def pack_narration_context(
|
||||
prompt: dict,
|
||||
budget,
|
||||
assembled: dict[str, str],
|
||||
*,
|
||||
style_examples: str,
|
||||
existing_text: str,
|
||||
include_existing: bool,
|
||||
) -> tuple[dict[str, str], list[str]]:
|
||||
"""Drop optional blocks locally if they overflow. Never drop day sources or the profile.
|
||||
|
||||
No extra model call. Order: try all, then drop style examples, then existing text.
|
||||
"""
|
||||
dropped: list[str] = []
|
||||
attempts = [
|
||||
(style_examples, existing_text if include_existing else ""),
|
||||
("", existing_text if include_existing else ""),
|
||||
("", ""),
|
||||
]
|
||||
last_error: JournalBudgetError | None = None
|
||||
for examples, existing in attempts:
|
||||
candidate = dict(assembled)
|
||||
candidate["style_examples"] = examples
|
||||
candidate["existing_text"] = existing
|
||||
rendered = preview_prompt(prompt, candidate)["rendered"]
|
||||
try:
|
||||
assert_input_fits(budget, rendered)
|
||||
if not examples and style_examples:
|
||||
dropped.append("style_examples")
|
||||
if include_existing and existing_text and not existing:
|
||||
dropped.append("existing_text")
|
||||
return candidate, dropped
|
||||
except JournalBudgetError as exc:
|
||||
last_error = exc
|
||||
continue
|
||||
if last_error:
|
||||
raise last_error
|
||||
return assembled, dropped
|
||||
|
||||
|
||||
def generate_draft(
|
||||
profile_id: str,
|
||||
journal_day_id: str,
|
||||
@@ -124,75 +272,51 @@ def generate_draft(
|
||||
)
|
||||
try:
|
||||
window = resolve_generate_metadata(config)
|
||||
reconstruct_budget = plan_journal_budget(window, purpose="journal_reconstruct")
|
||||
narrate_budget = plan_journal_budget(window, purpose="journal_generate")
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
|
||||
reconstruct_prompt = load_active_prompt("mvp.journal_reconstruct")
|
||||
static_tokens = estimate_tokens(reconstruct_prompt.get("template") or "")
|
||||
available_for_day = reconstruct_budget.available_input_tokens - static_tokens
|
||||
if available_for_day < 256:
|
||||
raise EngineError(
|
||||
"prompt_budget_exceeded",
|
||||
"Der Tagesdialog ist für eine sichere Verarbeitung zu umfangreich. "
|
||||
"Kanshō hat nichts stillschweigend aus der Mitte entfernt.",
|
||||
422,
|
||||
diagnostics=reconstruct_budget.as_diagnostics(),
|
||||
)
|
||||
|
||||
try:
|
||||
reconstruct_context = build_internal_context(
|
||||
source_messages = retrieve(
|
||||
profile_id,
|
||||
space_id=day["space_id"],
|
||||
journal_day_id=journal_day_id,
|
||||
purpose="journal_reconstruct",
|
||||
conversation_ids=selected,
|
||||
conversation_id=selected[0] if selected else None,
|
||||
day_spec={
|
||||
{
|
||||
"kind": "day_messages",
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_ids": selected,
|
||||
"overflow": "abort",
|
||||
"max_estimated_tokens": available_for_day,
|
||||
},
|
||||
)
|
||||
reconstruction = local_verified_artifact(source_messages)
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
reconstruct_assembled = assemble_text(reconstruct_context)
|
||||
source_messages = assign_source_ids(_day_messages_from_context(reconstruct_context))
|
||||
source_user = [
|
||||
message.get("body") or ""
|
||||
for message in source_messages
|
||||
if message.get("role") == "user"
|
||||
for message in assign_source_ids(source_messages)
|
||||
if message.get("role") == "user" and (message.get("body") or "").strip()
|
||||
]
|
||||
reconstruct_result = {"trace": {"purpose": "journal_reconstruct"}, "content": "", "diagnostics": {}}
|
||||
try:
|
||||
reconstruct_result = execute_prompt(
|
||||
reconstruct_prompt,
|
||||
profile_id,
|
||||
purpose="journal_reconstruct",
|
||||
data_class="B",
|
||||
context=reconstruct_assembled,
|
||||
max_tokens=reconstruct_budget.reserved_output_tokens,
|
||||
disable_context_compression=True,
|
||||
budget=reconstruct_budget,
|
||||
)
|
||||
reconstruction, stage1 = reconstruction_from_model(
|
||||
reconstruct_result.get("content") or "",
|
||||
source_messages,
|
||||
)
|
||||
except EngineError as exc:
|
||||
if exc.code != "response_validation_failed":
|
||||
raise
|
||||
reconstruction = local_verified_artifact(source_messages)
|
||||
stage1 = {
|
||||
"stage1": "local_fallback",
|
||||
"reason": "identity_leak_blocked",
|
||||
"model_rejected": exc.code,
|
||||
}
|
||||
reconstruct_result = _identity_leak_result("journal_reconstruct", exc)
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
editorial_mode = choose_editorial_mode(source_user)
|
||||
style_example_rows = select_journal_style_examples(
|
||||
profile_id,
|
||||
exclude_dates=[day.get("calendar_date") or ""],
|
||||
)
|
||||
style_examples = format_style_examples(style_example_rows)
|
||||
run_log: list[dict] = []
|
||||
_event(
|
||||
run_log,
|
||||
"local_source_artifact",
|
||||
"stage1_result",
|
||||
stage1="local_ok",
|
||||
coverage=reconstruction.get("coverage") or "all_selected_sources",
|
||||
status="local_ok",
|
||||
source_count=len(source_user),
|
||||
editorial_mode=editorial_mode,
|
||||
)
|
||||
reconstruct_result = {
|
||||
"trace": _local_stage_trace(reconstruction, source_count=len(source_user)),
|
||||
"content": reconstruction_text(reconstruction),
|
||||
}
|
||||
|
||||
signals = compile_style_signals(source_user)
|
||||
narrate_prompt = load_active_prompt("mvp.journal_generate")
|
||||
narrate_context = build_internal_context(
|
||||
profile_id,
|
||||
space_id=day["space_id"],
|
||||
@@ -202,13 +326,28 @@ def generate_draft(
|
||||
conversation_ids=selected,
|
||||
existing_text=existing_text,
|
||||
conversation_id=selected[0] if selected else None,
|
||||
reconstruction=reconstruction_text(reconstruction),
|
||||
reconstruction=narration_sources_text(reconstruction),
|
||||
style_examples=style_examples,
|
||||
editorial_mode=editorial_mode,
|
||||
editorial_instructions=editorial_instructions(editorial_mode),
|
||||
)
|
||||
assembled = assemble_text(narrate_context)
|
||||
profile = assembled.get("writing_profile") or ""
|
||||
if signals and signals not in profile and len(profile) < 3500:
|
||||
assembled["writing_profile"] = "\n\n".join(part for part in (profile, signals) if part)
|
||||
narrate_prompt = load_active_prompt("mvp.journal_generate")
|
||||
try:
|
||||
assembled, dropped = pack_narration_context(
|
||||
narrate_prompt,
|
||||
narrate_budget,
|
||||
assembled,
|
||||
style_examples=style_examples,
|
||||
existing_text=existing_text,
|
||||
include_existing=include_existing,
|
||||
)
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
if dropped:
|
||||
_event(run_log, "journal_generate", "budget_pack", dropped=",".join(dropped))
|
||||
shape_source = "model"
|
||||
mappings = list_mappings(profile_id)
|
||||
narrate_result: dict = {}
|
||||
try:
|
||||
narrate_result = execute_prompt(
|
||||
narrate_prompt,
|
||||
@@ -220,20 +359,70 @@ def generate_draft(
|
||||
disable_context_compression=True,
|
||||
budget=narrate_budget,
|
||||
)
|
||||
_stamp_log(run_log, "journal_generate", _log_items(narrate_result))
|
||||
title, body = _split_title(narrate_result.get("content") or "")
|
||||
seen = {(item.get("local_label") or "").casefold() for item in mappings}
|
||||
for item in narrate_result.get("local_identities") or []:
|
||||
label = (item.get("local_label") or "").strip()
|
||||
if not label or label.casefold() in seen:
|
||||
continue
|
||||
mappings.append(item)
|
||||
seen.add(label.casefold())
|
||||
active_tokens = list((narrate_result.get("diagnostics") or {}).get("active_tokens") or [])
|
||||
combined = f"{title}\n\n{body}".strip()
|
||||
unattested = unattested_journal_content(combined, source_user, mappings, active_tokens)
|
||||
if unattested:
|
||||
title, body = _local_narration(reconstruction)
|
||||
shape_source = "fallback"
|
||||
narrate_result = {
|
||||
**narrate_result,
|
||||
"content": f"{title}\n\n{body}".strip(),
|
||||
"trace": {
|
||||
**(narrate_result.get("trace") or {}),
|
||||
"guard": "unattested_content_blocked",
|
||||
"reason": unattested,
|
||||
},
|
||||
}
|
||||
_event(
|
||||
run_log,
|
||||
"journal_generate",
|
||||
"narration_result",
|
||||
status="local_fallback",
|
||||
reason=unattested,
|
||||
)
|
||||
else:
|
||||
_event(
|
||||
run_log,
|
||||
"journal_generate",
|
||||
"narration_result",
|
||||
status="model",
|
||||
editorial_mode=editorial_mode,
|
||||
lexical_similarity=lexical_similarity("\n".join(source_user), body),
|
||||
)
|
||||
except EngineError as exc:
|
||||
_stamp_log(run_log, "journal_generate", _log_items(exc))
|
||||
if exc.code != "response_validation_failed":
|
||||
raise
|
||||
diag = dict(exc.diagnostics or {})
|
||||
diag["log"] = run_log
|
||||
raise EngineError(exc.code, exc.message, exc.status_code, diag) from exc
|
||||
title, body = _local_narration(reconstruction)
|
||||
shape_source = "fallback"
|
||||
narrate_result = _identity_leak_result("journal_generate", exc)
|
||||
narrate_result["content"] = f"{title}\n\n{body}".strip()
|
||||
_event(run_log, "journal_generate", "narration_result", status="local_fallback", reason="identity_leak_blocked")
|
||||
person_labels = [
|
||||
(item.get("local_label") or "").strip()
|
||||
for item in list_mappings(profile_id)
|
||||
for item in mappings
|
||||
if (item.get("local_label") or "").strip()
|
||||
and str(item.get("token") or "").upper().lstrip("[").startswith("PERSON:")
|
||||
]
|
||||
title, body = shape_journal(title, body, source_user, person_labels=person_labels)
|
||||
title, body = shape_journal(
|
||||
title,
|
||||
body,
|
||||
source_user,
|
||||
person_labels=person_labels,
|
||||
source=shape_source,
|
||||
)
|
||||
before_entries = {item["id"]: item.get("current_version_id") for item in current_entries(profile_id, journal_day_id)}
|
||||
draft = insert_draft(
|
||||
profile_id,
|
||||
@@ -248,11 +437,19 @@ def generate_draft(
|
||||
previous = before_entries.get(item["id"])
|
||||
if previous is not None and previous != item.get("current_version_id"):
|
||||
raise StoreError("policy_violation", "Generate darf die Nutzerfassung nicht verändern", 500)
|
||||
reconstruct_trace = _stage_trace(reconstruct_result, "journal_reconstruct")
|
||||
reconstruct_trace.update(stage1)
|
||||
reconstruct_trace = _stage_trace(reconstruct_result, "local_source_artifact")
|
||||
reconstruct_trace.setdefault("purpose", "local_source_artifact")
|
||||
reconstruct_trace.setdefault("status", "local_ok")
|
||||
reconstruct_trace.setdefault("stage1", "local_ok")
|
||||
narrate_trace = _stage_trace(narrate_result, "journal_generate")
|
||||
narrate_trace["editorial_mode"] = editorial_mode
|
||||
if shape_source == "model":
|
||||
narrate_trace["lexical_similarity"] = lexical_similarity("\n".join(source_user), body)
|
||||
draft["run_log"] = run_log
|
||||
draft["trace"] = {
|
||||
**narrate_trace,
|
||||
"editorial_mode": editorial_mode,
|
||||
"log": run_log,
|
||||
"stages": [reconstruct_trace, narrate_trace],
|
||||
}
|
||||
return draft
|
||||
Reference in new issue
Block a user