63 lines
2.5 KiB
Python
63 lines
2.5 KiB
Python
"""Evaluation harness contract. Does not prove live model quality."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-eval-test.sqlite")
|
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
|
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
|
|
from db import init_db
|
|
from journal_eval import (
|
|
SYNTHETIC_PROSE,
|
|
SYNTHETIC_TYPOS,
|
|
VARIANT_BASELINE,
|
|
VARIANT_CURRENT,
|
|
VARIANT_PREVIOUS,
|
|
score_output,
|
|
synthetic_context,
|
|
variant_templates,
|
|
)
|
|
|
|
|
|
def expect(ok: bool, message: str) -> None:
|
|
if not ok:
|
|
raise SystemExit(f"FAIL: {message}")
|
|
print(f"OK {message}")
|
|
|
|
|
|
def main() -> None:
|
|
init_db()
|
|
templates = variant_templates()
|
|
expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist")
|
|
expect("Überarbeite diesen Rohtext" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
|
|
expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt")
|
|
expect("Faktentreue ist nicht Wortlauttreue" in templates[VARIANT_CURRENT], "current variant has the new contract")
|
|
|
|
context = synthetic_context(SYNTHETIC_PROSE)
|
|
expect(context["editorial_mode"] in {"prose_edit", "notes_to_journal"}, "eval context has an editorial mode")
|
|
copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS)
|
|
expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1")
|
|
expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos")
|
|
improved = score_output(
|
|
SYNTHETIC_PROSE,
|
|
"Ich ging dann zum Markt, auf dem es ziemlich voll war. Vielleicht bleibe ich kürzer.",
|
|
typos=SYNTHETIC_TYPOS,
|
|
)
|
|
expect("zimlich" in improved["spelling_typos_fixed"], "metrics record typo fixes")
|
|
expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical")
|
|
expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored")
|
|
expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures")
|
|
print("journal eval tests passed.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|