MVP 0.94
This commit is contained in:
1 parent
f17b3083e8
commit
4f37991ad1
48 files changed
+4919
-540
No files matched your search
@@ -9,7 +9,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
os.environ.setdefault("KANSHO_FAKE_DETECT", "1")
|
||||
|
||||
from entity_detect_eval import run_fake
|
||||
from entity_detect_eval import CASES, run_fake, score_spans
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
@@ -22,7 +22,22 @@ def main() -> None:
|
||||
payload = run_fake()
|
||||
expect(payload["mode"] == "fake", "eval default is fake")
|
||||
expect(payload["live_quality"] == "unconfirmed", "live quality stays unconfirmed")
|
||||
expect(len(payload["cases"]) >= 3, "synthetic cases exist")
|
||||
expect(payload["detect_model_unconfirmed"] == "openai/gpt-4.1-nano", "configured detect model stays unconfirmed")
|
||||
expect(len(payload["cases"]) >= 8, "synthetic span cases exist")
|
||||
person = next(item for item in CASES if item["id"] == "no_sentence_span")
|
||||
metrics = score_spans(
|
||||
[{"start": 9, "end": 13, "text": "Anna", "entity_type": "PERSON"}],
|
||||
person["expected"],
|
||||
)
|
||||
expect(metrics["expected_found"] == 1, "exact expected span is counted")
|
||||
expect(metrics["precision"] == 1.0 and metrics["recall"] == 1.0, "exact span match is precision 1")
|
||||
wrong = score_spans(
|
||||
[{"start": 0, "end": 28, "text": "Ich traf Anna am Nachmittag.", "entity_type": "PERSON"}],
|
||||
person["expected"],
|
||||
)
|
||||
expect(wrong["unexpected"] == 1, "sentence-sized span is unexpected")
|
||||
food = next(item for item in payload["cases"] if item["id"] == "same_word_two_roles")
|
||||
expect(food["entities"]["metrics"]["expected_found"] == 1, "fake contract finds only the identity sushi")
|
||||
print("All detect eval harness tests passed.")
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user