111 lines
5.5 KiB
Python
111 lines
5.5 KiB
Python
"""Evaluation harness contract. Does not prove live model quality."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-eval-test.sqlite")
|
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
|
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
|
|
from db import init_db
|
|
from journal_editorial import GENERATE_SEED_REVISION
|
|
from journal_eval import (
|
|
FIXTURES,
|
|
PROFILE_A,
|
|
PROFILE_B,
|
|
SYNTHETIC_PROSE,
|
|
SYNTHETIC_TYPOS,
|
|
VARIANT_BASELINE,
|
|
VARIANT_CURRENT,
|
|
VARIANT_PREVIOUS,
|
|
compare_synthetic,
|
|
fixture_context,
|
|
score_output,
|
|
variant_templates,
|
|
)
|
|
|
|
|
|
def expect(ok: bool, message: str) -> None:
|
|
if not ok:
|
|
raise SystemExit(f"FAIL: {message}")
|
|
print(f"OK {message}")
|
|
|
|
|
|
def main() -> None:
|
|
init_db()
|
|
templates = variant_templates()
|
|
expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist")
|
|
expect("Erstelle aus diesen Angaben einen ansprechenden persönlichen Tagebucheintrag" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
|
|
expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt")
|
|
expect("INHALTSTREUE" in templates[VARIANT_CURRENT], "current variant has the new contract")
|
|
expect("{{transformation_instructions}}" in templates[VARIANT_CURRENT], "current variant compiles transformation policy")
|
|
expect("{{source_mode_instructions}}" not in templates[VARIANT_CURRENT], "current variant has no source-mode placeholder")
|
|
expect("Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus." in templates[VARIANT_CURRENT], "current variant has mixed-source instruction")
|
|
expect("ich gieng zum laden" not in templates[VARIANT_CURRENT], "current variant has no synthetic examples")
|
|
expect(GENERATE_SEED_REVISION == "2026-08-27-journal-mixed-sources-v1", "eval tracks the seeded revision constant")
|
|
|
|
required_classes = {
|
|
"already_narrative_with_errors",
|
|
"bullet_points_and_fragments",
|
|
"plan_versus_completion",
|
|
"negation_and_uncertainty",
|
|
"correction_of_earlier_claim",
|
|
"imprecise_time",
|
|
"outstanding_event_among_everyday",
|
|
"recurring_people_and_projects",
|
|
"incomplete_source_rebuildable",
|
|
}
|
|
got = {item["class"] for item in FIXTURES}
|
|
expect(required_classes <= got, f"all required fixture classes exist, missing {required_classes - got}")
|
|
expect(all("example.test" not in item["source"].lower() for item in FIXTURES), "fixtures stay synthetic")
|
|
|
|
context = fixture_context(SYNTHETIC_PROSE)
|
|
expect("source_mode_instructions" not in context, "eval context has no source-mode instruction")
|
|
expect("editorial_mode" not in context, "eval context has no editorial mode")
|
|
copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS)
|
|
expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1")
|
|
expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos")
|
|
improved = score_output(
|
|
SYNTHETIC_PROSE,
|
|
"Ich ging dann zum Markt, auf dem es ziemlich voll war. Vielleicht bleibe ich kürzer.",
|
|
typos=SYNTHETIC_TYPOS,
|
|
)
|
|
expect("zimlich" in improved["spelling_typos_fixed"], "metrics record typo fixes")
|
|
expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical")
|
|
expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored")
|
|
expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures")
|
|
|
|
notes = next(item for item in FIXTURES if item["id"] == "notes_fragments")
|
|
expect("markt" in notes["source"].lower(), "notes fixture stays notes-shaped")
|
|
incomplete = next(item for item in FIXTURES if item["id"] == "incomplete_clause")
|
|
expect("danach sprach ich mit dem" in incomplete["source"].lower(), "incomplete fixture stays incomplete prose")
|
|
|
|
report = compare_synthetic(live=False)
|
|
expect(report["live"] is False, "default eval is offline")
|
|
expect(report["live_quality_confirmed"] is False, "offline run does not confirm live quality")
|
|
expect(report["winner_declared"] is False, "harness does not declare a winner")
|
|
expect(len(report["fixtures"]) == len(FIXTURES), "offline report covers every fixture")
|
|
first = report["fixtures"][0]
|
|
expect("human_blind" in first and "prompt_1" in first["human_blind"], "each fixture has a blind pair")
|
|
expect(first["human_blind"]["hidden_mapping"]["prompt_1"] == VARIANT_CURRENT, "mapping stays machine-side")
|
|
expect("Kanshō habe gewonnen" not in report["note"], "no victory claim")
|
|
expect(any(item["variant"] == VARIANT_CURRENT and item["fake_provider"] for item in first["variants"]), "offline current variant is fake")
|
|
|
|
ab = compare_synthetic(live=False, profile_ab=True)
|
|
expect(ab["profile_ab"]["same_facts"] is True, "profile A/B keeps facts identical")
|
|
expect(ab["profile_ab"]["prompts_differ"] is True, "rendered prompts contain different style briefs")
|
|
expect(PROFILE_A[:20] != PROFILE_B[:20], "synthetic profiles are distinct")
|
|
expect("Live-Prosa" in (ab["profile_ab"]["note"] or ""), "offline A/B does not claim live prose")
|
|
print("journal eval tests passed.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|