This commit is contained in:
Lars committed 2026-08-28 08:40:28 +02:00
1 parent f17b3083e8
commit 4f37991ad1
48 files changed
+4919 -540

No files matched your search

+53 -5
View File
@@ -15,14 +15,19 @@ os.environ["KANSHO_FAKE_DETECT"] = "1"
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
from db import init_db
from journal_editorial import GENERATE_SEED_REVISION
from journal_eval import (
FIXTURES,
PROFILE_A,
PROFILE_B,
SYNTHETIC_PROSE,
SYNTHETIC_TYPOS,
VARIANT_BASELINE,
VARIANT_CURRENT,
VARIANT_PREVIOUS,
compare_synthetic,
fixture_context,
score_output,
synthetic_context,
variant_templates,
)
@@ -37,12 +42,33 @@ def main() -> None:
init_db()
templates = variant_templates()
expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist")
expect("Überarbeite diesen Rohtext" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
expect("Erstelle aus diesen Angaben einen ansprechenden persönlichen Tagebucheintrag" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt")
expect("Faktentreue ist nicht Wortlauttreue" in templates[VARIANT_CURRENT], "current variant has the new contract")
expect("INHALTSTREUE" in templates[VARIANT_CURRENT], "current variant has the new contract")
expect("{{transformation_instructions}}" in templates[VARIANT_CURRENT], "current variant compiles transformation policy")
expect("{{source_mode_instructions}}" not in templates[VARIANT_CURRENT], "current variant has no source-mode placeholder")
expect("Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus." in templates[VARIANT_CURRENT], "current variant has mixed-source instruction")
expect("ich gieng zum laden" not in templates[VARIANT_CURRENT], "current variant has no synthetic examples")
expect(GENERATE_SEED_REVISION == "2026-08-27-journal-mixed-sources-v1", "eval tracks the seeded revision constant")
context = synthetic_context(SYNTHETIC_PROSE)
expect(context["editorial_mode"] in {"prose_edit", "notes_to_journal"}, "eval context has an editorial mode")
required_classes = {
"already_narrative_with_errors",
"bullet_points_and_fragments",
"plan_versus_completion",
"negation_and_uncertainty",
"correction_of_earlier_claim",
"imprecise_time",
"outstanding_event_among_everyday",
"recurring_people_and_projects",
"incomplete_source_rebuildable",
}
got = {item["class"] for item in FIXTURES}
expect(required_classes <= got, f"all required fixture classes exist, missing {required_classes - got}")
expect(all("example.test" not in item["source"].lower() for item in FIXTURES), "fixtures stay synthetic")
context = fixture_context(SYNTHETIC_PROSE)
expect("source_mode_instructions" not in context, "eval context has no source-mode instruction")
expect("editorial_mode" not in context, "eval context has no editorial mode")
copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS)
expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1")
expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos")
@@ -55,6 +81,28 @@ def main() -> None:
expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical")
expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored")
expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures")
notes = next(item for item in FIXTURES if item["id"] == "notes_fragments")
expect("markt" in notes["source"].lower(), "notes fixture stays notes-shaped")
incomplete = next(item for item in FIXTURES if item["id"] == "incomplete_clause")
expect("danach sprach ich mit dem" in incomplete["source"].lower(), "incomplete fixture stays incomplete prose")
report = compare_synthetic(live=False)
expect(report["live"] is False, "default eval is offline")
expect(report["live_quality_confirmed"] is False, "offline run does not confirm live quality")
expect(report["winner_declared"] is False, "harness does not declare a winner")
expect(len(report["fixtures"]) == len(FIXTURES), "offline report covers every fixture")
first = report["fixtures"][0]
expect("human_blind" in first and "prompt_1" in first["human_blind"], "each fixture has a blind pair")
expect(first["human_blind"]["hidden_mapping"]["prompt_1"] == VARIANT_CURRENT, "mapping stays machine-side")
expect("Kanshō habe gewonnen" not in report["note"], "no victory claim")
expect(any(item["variant"] == VARIANT_CURRENT and item["fake_provider"] for item in first["variants"]), "offline current variant is fake")
ab = compare_synthetic(live=False, profile_ab=True)
expect(ab["profile_ab"]["same_facts"] is True, "profile A/B keeps facts identical")
expect(ab["profile_ab"]["prompts_differ"] is True, "rendered prompts contain different style briefs")
expect(PROFILE_A[:20] != PROFILE_B[:20], "synthetic profiles are distinct")
expect("Live-Prosa" in (ab["profile_ab"]["note"] or ""), "offline A/B does not claim live prose")
print("journal eval tests passed.")