MVP 0.93 - stabile, aber Writing Profile ohne erkennbaren Nutzen
This commit is contained in:
1 parent
53b1eff833
commit
f17b3083e8
56 files changed
+5366
-784
No files matched your search
@@ -0,0 +1,62 @@
|
||||
"""Evaluation harness contract. Does not prove live model quality."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-eval-test.sqlite")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from db import init_db
|
||||
from journal_eval import (
|
||||
SYNTHETIC_PROSE,
|
||||
SYNTHETIC_TYPOS,
|
||||
VARIANT_BASELINE,
|
||||
VARIANT_CURRENT,
|
||||
VARIANT_PREVIOUS,
|
||||
score_output,
|
||||
synthetic_context,
|
||||
variant_templates,
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_db()
|
||||
templates = variant_templates()
|
||||
expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist")
|
||||
expect("Überarbeite diesen Rohtext" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
|
||||
expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt")
|
||||
expect("Faktentreue ist nicht Wortlauttreue" in templates[VARIANT_CURRENT], "current variant has the new contract")
|
||||
|
||||
context = synthetic_context(SYNTHETIC_PROSE)
|
||||
expect(context["editorial_mode"] in {"prose_edit", "notes_to_journal"}, "eval context has an editorial mode")
|
||||
copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS)
|
||||
expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1")
|
||||
expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos")
|
||||
improved = score_output(
|
||||
SYNTHETIC_PROSE,
|
||||
"Ich ging dann zum Markt, auf dem es ziemlich voll war. Vielleicht bleibe ich kürzer.",
|
||||
typos=SYNTHETIC_TYPOS,
|
||||
)
|
||||
expect("zimlich" in improved["spelling_typos_fixed"], "metrics record typo fixes")
|
||||
expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical")
|
||||
expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored")
|
||||
expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures")
|
||||
print("journal eval tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user