274 lines
9.9 KiB
Python
274 lines
9.9 KiB
Python
"""Opt-in journal quality comparison. Not part of production generate.
|
|
|
|
Usage from backend/:
|
|
python journal_eval.py # synthetic, fake provider, contract only
|
|
python journal_eval.py --live --profile-id <id>
|
|
|
|
Never writes private texts into the repository. Live quality stays unconfirmed
|
|
until an explicit --live run succeeds.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from journal_editorial import lexical_similarity
|
|
|
|
VARIANT_BASELINE = "baseline_simple"
|
|
VARIANT_PREVIOUS = "kansho_previous"
|
|
VARIANT_CURRENT = "kansho_current"
|
|
|
|
BASELINE_TEMPLATE = (
|
|
"Überarbeite diesen Rohtext zu einem ansprechenden Tagebucheintrag in meinem Stil.\n\n"
|
|
"{{reconstruction}}\n"
|
|
)
|
|
|
|
PREVIOUS_TEMPLATE = (
|
|
"Schreibe einen eigenständigen, gut lesbaren Tagebucheintrag in der Ich-Form von [[SELF]].\n"
|
|
"Verwende ausschließlich die verifizierten Informationen aus sources[].text. "
|
|
"Der Quellwortlaut ist keine Ausgabevorlage.\n"
|
|
"Keine Dialogabschrift. Unsicherheiten im Wortlaut erhalten.\n"
|
|
"Writing Profile:\n{{writing_profile}}\n\n"
|
|
"Verified Artifact:\n{{reconstruction}}\n"
|
|
)
|
|
|
|
SYNTHETIC_PROSE = (
|
|
"ich bin dan zum markt gegangen und da war es zimlich voll. "
|
|
"vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. "
|
|
"die rote tasche lag im auto."
|
|
)
|
|
SYNTHETIC_NOTES = "- markt\n- kirschen kaufen\n- später hafen"
|
|
SYNTHETIC_TYPOS = ("zimlich", "dan zum")
|
|
|
|
|
|
def _words(text: str) -> list[str]:
|
|
return re.findall(r"[A-Za-zÄÖÜäöüß]+", (text or "").lower())
|
|
|
|
|
|
def _sentences(text: str) -> list[str]:
|
|
parts = [item.strip() for item in re.split(r"(?<=[.!?])\s+", text or "") if item.strip()]
|
|
return parts or ([text.strip()] if (text or "").strip() else [])
|
|
|
|
|
|
def score_output(source: str, output: str, *, profile: str = "", typos: tuple[str, ...] = ()) -> dict[str, Any]:
|
|
"""Diagnostic scores. Not a production gate and not a live-quality certificate."""
|
|
body = output or ""
|
|
src_words = set(_words(source))
|
|
out_words = set(_words(body))
|
|
sentences = _sentences(body)
|
|
paragraphs = [item for item in re.split(r"\n\s*\n", body) if item.strip()]
|
|
avg_len = round(sum(len(item.split()) for item in sentences) / max(1, len(sentences)), 2)
|
|
transitions = len(re.findall(r"(?i)\b(?:danach|später|dann|zuerst|schließlich)\b", body))
|
|
profile_words = set(_words(profile))
|
|
style_overlap = round(len(out_words & profile_words) / max(1, len(profile_words)), 3) if profile_words else 0.0
|
|
kept = round(len(src_words & out_words) / max(1, len(src_words)), 3)
|
|
extra = sorted(out_words - src_words - profile_words)
|
|
lost = sorted(src_words - out_words)
|
|
remaining_typos = [item for item in typos if item.lower() in body.lower()]
|
|
return {
|
|
"spelling_typos_remaining": remaining_typos,
|
|
"spelling_typos_fixed": [item for item in typos if item.lower() not in body.lower()],
|
|
"readability_sentence_count": len(sentences),
|
|
"readability_avg_sentence_words": avg_len,
|
|
"readability_paragraphs": max(1, len(paragraphs)),
|
|
"transitions": transitions,
|
|
"style_token_overlap": style_overlap,
|
|
"fact_token_keep": kept,
|
|
"new_info_tokens": extra[:24],
|
|
"lost_info_tokens": lost[:24],
|
|
"lexical_similarity": lexical_similarity(source, body),
|
|
}
|
|
|
|
|
|
def variant_templates() -> dict[str, str]:
|
|
from engine import load_active_prompt
|
|
|
|
current = (load_active_prompt("mvp.journal_generate") or {}).get("template") or ""
|
|
return {
|
|
VARIANT_BASELINE: BASELINE_TEMPLATE,
|
|
VARIANT_PREVIOUS: PREVIOUS_TEMPLATE,
|
|
VARIANT_CURRENT: current,
|
|
}
|
|
|
|
|
|
def run_variant(
|
|
name: str,
|
|
template: str,
|
|
context: dict[str, str],
|
|
profile_id: str,
|
|
*,
|
|
live: bool,
|
|
) -> dict[str, Any]:
|
|
from engine import execute_prompt
|
|
from prompt_budget import plan_journal_budget
|
|
from providers import generate_provider
|
|
from model_catalog import resolve_generate_metadata
|
|
|
|
prompt = {
|
|
"id": f"eval-{name}",
|
|
"slug": "mvp.journal_generate" if name == VARIANT_CURRENT else f"eval.{name}",
|
|
"prompt_type": "base",
|
|
"required_feature": "ai_calls",
|
|
"template": template,
|
|
}
|
|
started = time.perf_counter()
|
|
config = generate_provider()
|
|
window = resolve_generate_metadata(config) if config else None
|
|
budget = plan_journal_budget(window, purpose="journal_generate") if window else None
|
|
result = execute_prompt(
|
|
prompt,
|
|
profile_id,
|
|
purpose="journal_generate",
|
|
data_class="B",
|
|
context=context,
|
|
max_tokens=budget.reserved_output_tokens if budget else 1024,
|
|
disable_context_compression=True,
|
|
budget=budget,
|
|
)
|
|
elapsed_ms = int((time.perf_counter() - started) * 1000)
|
|
diag = result.get("diagnostics") or {}
|
|
return {
|
|
"variant": name,
|
|
"live": live,
|
|
"content": result.get("content") or "",
|
|
"prompt_tokens": diag.get("prompt_tokens"),
|
|
"completion_tokens": diag.get("completion_tokens"),
|
|
"total_tokens": diag.get("total_tokens"),
|
|
"cost": diag.get("cost"),
|
|
"runtime_ms": elapsed_ms,
|
|
"model": (result.get("trace") or {}).get("model") or diag.get("actual_model"),
|
|
"fake_provider": not live,
|
|
}
|
|
|
|
|
|
def synthetic_context(source: str) -> dict[str, str]:
|
|
artifact = {
|
|
"kind": "verified_artifact",
|
|
"coverage": "all_selected_sources",
|
|
"sources": [{"source_id": "u1", "role": "user", "text": source}],
|
|
}
|
|
from journal_editorial import (
|
|
NOTES_TO_JOURNAL,
|
|
PROSE_EDIT,
|
|
choose_editorial_mode,
|
|
editorial_instructions,
|
|
narration_sources_text,
|
|
)
|
|
from writing_profile_store import NEUTRAL_JOURNAL_STYLE
|
|
|
|
mode = choose_editorial_mode([source])
|
|
return {
|
|
"reconstruction": narration_sources_text(artifact) or source,
|
|
"writing_profile": NEUTRAL_JOURNAL_STYLE,
|
|
"style_examples": "Keine historischen Stilbeispiele.",
|
|
"editorial_mode": mode,
|
|
"editorial_instructions": editorial_instructions(mode),
|
|
"existing_text": "",
|
|
"space_title": "Eval",
|
|
"expected_mode": PROSE_EDIT if mode == PROSE_EDIT else NOTES_TO_JOURNAL,
|
|
}
|
|
|
|
|
|
def compare_synthetic(*, live: bool = False, profile_id: str | None = None) -> dict[str, Any]:
|
|
from placeholders import resolve_template
|
|
import placeholder_mvp # noqa: F401
|
|
from privacy_gateway import _fake_complete
|
|
|
|
templates = variant_templates()
|
|
source = SYNTHETIC_PROSE
|
|
context = synthetic_context(source)
|
|
rows = []
|
|
for name, template in templates.items():
|
|
if live:
|
|
if not profile_id:
|
|
raise SystemExit("--live requires --profile-id")
|
|
row = run_variant(name, template, context, profile_id=profile_id, live=True)
|
|
else:
|
|
rendered = resolve_template(template, context)
|
|
started = time.perf_counter()
|
|
content = _fake_complete("journal_generate", rendered)
|
|
row = {
|
|
"variant": name,
|
|
"live": False,
|
|
"content": content,
|
|
"prompt_tokens": None,
|
|
"completion_tokens": None,
|
|
"total_tokens": None,
|
|
"cost": None,
|
|
"runtime_ms": int((time.perf_counter() - started) * 1000),
|
|
"model": "fake",
|
|
"fake_provider": True,
|
|
}
|
|
row["scores"] = score_output(
|
|
source,
|
|
row["content"],
|
|
profile=context.get("writing_profile") or "",
|
|
typos=SYNTHETIC_TYPOS,
|
|
)
|
|
row["source"] = "synthetic"
|
|
rows.append(row)
|
|
return {
|
|
"live": live,
|
|
"live_quality_confirmed": False,
|
|
"note": (
|
|
"Fake-Provider-Lauf: beweist den Vergleichsvertrag, nicht echte Modellprosa."
|
|
if not live
|
|
else "Live-Lauf über das Privacy Gateway. Qualitative Bewertung bleibt manuell."
|
|
),
|
|
"source_kind": "synthetic",
|
|
"variants": rows,
|
|
}
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Journal narration quality comparison (opt-in).")
|
|
parser.add_argument("--live", action="store_true", help="Call the configured generate provider. Costs money.")
|
|
parser.add_argument("--profile-id", help="Required with --live. Uses the Privacy Gateway.")
|
|
parser.add_argument("--json", action="store_true", help="Print JSON instead of text.")
|
|
args = parser.parse_args(argv)
|
|
if args.live:
|
|
import os
|
|
|
|
if (os.environ.get("KANSHO_FAKE_PROVIDER") or "").strip() in {"1", "true", "yes"}:
|
|
print("Refusing --live while KANSHO_FAKE_PROVIDER is set.", file=sys.stderr)
|
|
return 2
|
|
if not args.profile_id:
|
|
print("--live requires --profile-id", file=sys.stderr)
|
|
return 2
|
|
else:
|
|
import os
|
|
|
|
os.environ.setdefault("KANSHO_FAKE_PROVIDER", "1")
|
|
os.environ.setdefault("KANSHO_FAKE_DETECT", "1")
|
|
from db import init_db
|
|
|
|
init_db()
|
|
report = compare_synthetic(live=bool(args.live), profile_id=args.profile_id)
|
|
if args.json:
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
return 0
|
|
print(report["note"])
|
|
if not report["live"]:
|
|
print("Live-Qualität: noch nicht bestätigt.")
|
|
for item in report["variants"]:
|
|
scores = item["scores"]
|
|
print(
|
|
f"{item['variant']}: similarity={scores['lexical_similarity']} "
|
|
f"keep={scores['fact_token_keep']} typos_left={scores['spelling_typos_remaining']} "
|
|
f"tokens={item.get('total_tokens')} cost={item.get('cost')} ms={item['runtime_ms']}"
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|