MVP 0.93 - stabile, aber Writing Profile ohne erkennbaren Nutzen
This commit is contained in:
1 parent
53b1eff833
commit
f17b3083e8
56 files changed
+5366
-784
No files matched your search
@@ -0,0 +1,273 @@
|
||||
"""Opt-in journal quality comparison. Not part of production generate.
|
||||
|
||||
Usage from backend/:
|
||||
python journal_eval.py # synthetic, fake provider, contract only
|
||||
python journal_eval.py --live --profile-id <id>
|
||||
|
||||
Never writes private texts into the repository. Live quality stays unconfirmed
|
||||
until an explicit --live run succeeds.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from journal_editorial import lexical_similarity
|
||||
|
||||
VARIANT_BASELINE = "baseline_simple"
|
||||
VARIANT_PREVIOUS = "kansho_previous"
|
||||
VARIANT_CURRENT = "kansho_current"
|
||||
|
||||
BASELINE_TEMPLATE = (
|
||||
"Überarbeite diesen Rohtext zu einem ansprechenden Tagebucheintrag in meinem Stil.\n\n"
|
||||
"{{reconstruction}}\n"
|
||||
)
|
||||
|
||||
PREVIOUS_TEMPLATE = (
|
||||
"Schreibe einen eigenständigen, gut lesbaren Tagebucheintrag in der Ich-Form von [[SELF]].\n"
|
||||
"Verwende ausschließlich die verifizierten Informationen aus sources[].text. "
|
||||
"Der Quellwortlaut ist keine Ausgabevorlage.\n"
|
||||
"Keine Dialogabschrift. Unsicherheiten im Wortlaut erhalten.\n"
|
||||
"Writing Profile:\n{{writing_profile}}\n\n"
|
||||
"Verified Artifact:\n{{reconstruction}}\n"
|
||||
)
|
||||
|
||||
SYNTHETIC_PROSE = (
|
||||
"ich bin dan zum markt gegangen und da war es zimlich voll. "
|
||||
"vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. "
|
||||
"die rote tasche lag im auto."
|
||||
)
|
||||
SYNTHETIC_NOTES = "- markt\n- kirschen kaufen\n- später hafen"
|
||||
SYNTHETIC_TYPOS = ("zimlich", "dan zum")
|
||||
|
||||
|
||||
def _words(text: str) -> list[str]:
|
||||
return re.findall(r"[A-Za-zÄÖÜäöüß]+", (text or "").lower())
|
||||
|
||||
|
||||
def _sentences(text: str) -> list[str]:
|
||||
parts = [item.strip() for item in re.split(r"(?<=[.!?])\s+", text or "") if item.strip()]
|
||||
return parts or ([text.strip()] if (text or "").strip() else [])
|
||||
|
||||
|
||||
def score_output(source: str, output: str, *, profile: str = "", typos: tuple[str, ...] = ()) -> dict[str, Any]:
|
||||
"""Diagnostic scores. Not a production gate and not a live-quality certificate."""
|
||||
body = output or ""
|
||||
src_words = set(_words(source))
|
||||
out_words = set(_words(body))
|
||||
sentences = _sentences(body)
|
||||
paragraphs = [item for item in re.split(r"\n\s*\n", body) if item.strip()]
|
||||
avg_len = round(sum(len(item.split()) for item in sentences) / max(1, len(sentences)), 2)
|
||||
transitions = len(re.findall(r"(?i)\b(?:danach|später|dann|zuerst|schließlich)\b", body))
|
||||
profile_words = set(_words(profile))
|
||||
style_overlap = round(len(out_words & profile_words) / max(1, len(profile_words)), 3) if profile_words else 0.0
|
||||
kept = round(len(src_words & out_words) / max(1, len(src_words)), 3)
|
||||
extra = sorted(out_words - src_words - profile_words)
|
||||
lost = sorted(src_words - out_words)
|
||||
remaining_typos = [item for item in typos if item.lower() in body.lower()]
|
||||
return {
|
||||
"spelling_typos_remaining": remaining_typos,
|
||||
"spelling_typos_fixed": [item for item in typos if item.lower() not in body.lower()],
|
||||
"readability_sentence_count": len(sentences),
|
||||
"readability_avg_sentence_words": avg_len,
|
||||
"readability_paragraphs": max(1, len(paragraphs)),
|
||||
"transitions": transitions,
|
||||
"style_token_overlap": style_overlap,
|
||||
"fact_token_keep": kept,
|
||||
"new_info_tokens": extra[:24],
|
||||
"lost_info_tokens": lost[:24],
|
||||
"lexical_similarity": lexical_similarity(source, body),
|
||||
}
|
||||
|
||||
|
||||
def variant_templates() -> dict[str, str]:
|
||||
from engine import load_active_prompt
|
||||
|
||||
current = (load_active_prompt("mvp.journal_generate") or {}).get("template") or ""
|
||||
return {
|
||||
VARIANT_BASELINE: BASELINE_TEMPLATE,
|
||||
VARIANT_PREVIOUS: PREVIOUS_TEMPLATE,
|
||||
VARIANT_CURRENT: current,
|
||||
}
|
||||
|
||||
|
||||
def run_variant(
|
||||
name: str,
|
||||
template: str,
|
||||
context: dict[str, str],
|
||||
profile_id: str,
|
||||
*,
|
||||
live: bool,
|
||||
) -> dict[str, Any]:
|
||||
from engine import execute_prompt
|
||||
from prompt_budget import plan_journal_budget
|
||||
from providers import generate_provider
|
||||
from model_catalog import resolve_generate_metadata
|
||||
|
||||
prompt = {
|
||||
"id": f"eval-{name}",
|
||||
"slug": "mvp.journal_generate" if name == VARIANT_CURRENT else f"eval.{name}",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"template": template,
|
||||
}
|
||||
started = time.perf_counter()
|
||||
config = generate_provider()
|
||||
window = resolve_generate_metadata(config) if config else None
|
||||
budget = plan_journal_budget(window, purpose="journal_generate") if window else None
|
||||
result = execute_prompt(
|
||||
prompt,
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
data_class="B",
|
||||
context=context,
|
||||
max_tokens=budget.reserved_output_tokens if budget else 1024,
|
||||
disable_context_compression=True,
|
||||
budget=budget,
|
||||
)
|
||||
elapsed_ms = int((time.perf_counter() - started) * 1000)
|
||||
diag = result.get("diagnostics") or {}
|
||||
return {
|
||||
"variant": name,
|
||||
"live": live,
|
||||
"content": result.get("content") or "",
|
||||
"prompt_tokens": diag.get("prompt_tokens"),
|
||||
"completion_tokens": diag.get("completion_tokens"),
|
||||
"total_tokens": diag.get("total_tokens"),
|
||||
"cost": diag.get("cost"),
|
||||
"runtime_ms": elapsed_ms,
|
||||
"model": (result.get("trace") or {}).get("model") or diag.get("actual_model"),
|
||||
"fake_provider": not live,
|
||||
}
|
||||
|
||||
|
||||
def synthetic_context(source: str) -> dict[str, str]:
|
||||
artifact = {
|
||||
"kind": "verified_artifact",
|
||||
"coverage": "all_selected_sources",
|
||||
"sources": [{"source_id": "u1", "role": "user", "text": source}],
|
||||
}
|
||||
from journal_editorial import (
|
||||
NOTES_TO_JOURNAL,
|
||||
PROSE_EDIT,
|
||||
choose_editorial_mode,
|
||||
editorial_instructions,
|
||||
narration_sources_text,
|
||||
)
|
||||
from writing_profile_store import NEUTRAL_JOURNAL_STYLE
|
||||
|
||||
mode = choose_editorial_mode([source])
|
||||
return {
|
||||
"reconstruction": narration_sources_text(artifact) or source,
|
||||
"writing_profile": NEUTRAL_JOURNAL_STYLE,
|
||||
"style_examples": "Keine historischen Stilbeispiele.",
|
||||
"editorial_mode": mode,
|
||||
"editorial_instructions": editorial_instructions(mode),
|
||||
"existing_text": "",
|
||||
"space_title": "Eval",
|
||||
"expected_mode": PROSE_EDIT if mode == PROSE_EDIT else NOTES_TO_JOURNAL,
|
||||
}
|
||||
|
||||
|
||||
def compare_synthetic(*, live: bool = False, profile_id: str | None = None) -> dict[str, Any]:
|
||||
from placeholders import resolve_template
|
||||
import placeholder_mvp # noqa: F401
|
||||
from privacy_gateway import _fake_complete
|
||||
|
||||
templates = variant_templates()
|
||||
source = SYNTHETIC_PROSE
|
||||
context = synthetic_context(source)
|
||||
rows = []
|
||||
for name, template in templates.items():
|
||||
if live:
|
||||
if not profile_id:
|
||||
raise SystemExit("--live requires --profile-id")
|
||||
row = run_variant(name, template, context, profile_id=profile_id, live=True)
|
||||
else:
|
||||
rendered = resolve_template(template, context)
|
||||
started = time.perf_counter()
|
||||
content = _fake_complete("journal_generate", rendered)
|
||||
row = {
|
||||
"variant": name,
|
||||
"live": False,
|
||||
"content": content,
|
||||
"prompt_tokens": None,
|
||||
"completion_tokens": None,
|
||||
"total_tokens": None,
|
||||
"cost": None,
|
||||
"runtime_ms": int((time.perf_counter() - started) * 1000),
|
||||
"model": "fake",
|
||||
"fake_provider": True,
|
||||
}
|
||||
row["scores"] = score_output(
|
||||
source,
|
||||
row["content"],
|
||||
profile=context.get("writing_profile") or "",
|
||||
typos=SYNTHETIC_TYPOS,
|
||||
)
|
||||
row["source"] = "synthetic"
|
||||
rows.append(row)
|
||||
return {
|
||||
"live": live,
|
||||
"live_quality_confirmed": False,
|
||||
"note": (
|
||||
"Fake-Provider-Lauf: beweist den Vergleichsvertrag, nicht echte Modellprosa."
|
||||
if not live
|
||||
else "Live-Lauf über das Privacy Gateway. Qualitative Bewertung bleibt manuell."
|
||||
),
|
||||
"source_kind": "synthetic",
|
||||
"variants": rows,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Journal narration quality comparison (opt-in).")
|
||||
parser.add_argument("--live", action="store_true", help="Call the configured generate provider. Costs money.")
|
||||
parser.add_argument("--profile-id", help="Required with --live. Uses the Privacy Gateway.")
|
||||
parser.add_argument("--json", action="store_true", help="Print JSON instead of text.")
|
||||
args = parser.parse_args(argv)
|
||||
if args.live:
|
||||
import os
|
||||
|
||||
if (os.environ.get("KANSHO_FAKE_PROVIDER") or "").strip() in {"1", "true", "yes"}:
|
||||
print("Refusing --live while KANSHO_FAKE_PROVIDER is set.", file=sys.stderr)
|
||||
return 2
|
||||
if not args.profile_id:
|
||||
print("--live requires --profile-id", file=sys.stderr)
|
||||
return 2
|
||||
else:
|
||||
import os
|
||||
|
||||
os.environ.setdefault("KANSHO_FAKE_PROVIDER", "1")
|
||||
os.environ.setdefault("KANSHO_FAKE_DETECT", "1")
|
||||
from db import init_db
|
||||
|
||||
init_db()
|
||||
report = compare_synthetic(live=bool(args.live), profile_id=args.profile_id)
|
||||
if args.json:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
print(report["note"])
|
||||
if not report["live"]:
|
||||
print("Live-Qualität: noch nicht bestätigt.")
|
||||
for item in report["variants"]:
|
||||
scores = item["scores"]
|
||||
print(
|
||||
f"{item['variant']}: similarity={scores['lexical_similarity']} "
|
||||
f"keep={scores['fact_token_keep']} typos_left={scores['spelling_typos_remaining']} "
|
||||
f"tokens={item.get('total_tokens')} cost={item.get('cost')} ms={item['runtime_ms']}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user