Add support for handling non-identity entity types in detection logic
- Introduced a new set of non-identity entity types (FOOD, DISH, MEAL, CUISINE, FOODSTUFF, INGREDIENT) that are omitted from detection results without causing contract violations. - Updated the `DetectionStats` class to track omitted non-identity types. - Modified validation logic to ensure that food-related mentions are not treated as identity types, preserving the context of mentions as either PERSON or omitted. - Enhanced tests to verify the correct handling of food homonyms and ensure that invalid entity types are recorded appropriately without causing errors. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
79e4fd0d4b
commit
dd7858347c
7 files changed
+170
-22
No files matched your search
@@ -101,7 +101,7 @@ def _chat(entities: list[dict], usage: dict) -> ChatResult:
|
||||
|
||||
|
||||
def _invalid_type():
|
||||
return [{"start": 0, "end": 1, "text": "x", "entity_type": "FOOD"}]
|
||||
return [{"start": 0, "end": 1, "text": "x", "entity_type": "HUMAN"}]
|
||||
|
||||
|
||||
def _extra_field():
|
||||
@@ -135,14 +135,14 @@ def main() -> None:
|
||||
sample = "Ich traf Anna."
|
||||
try:
|
||||
validate_detected_entity(
|
||||
{"start": 0, "end": 4, "text": "Anna", "entity_type": "FOOD"},
|
||||
{"start": 0, "end": 4, "text": "Anna", "entity_type": "HUMAN"},
|
||||
sample,
|
||||
0,
|
||||
)
|
||||
typed = False
|
||||
except DetectError as exc:
|
||||
typed = exc.code == "detect_invalid_output"
|
||||
expect((exc.diagnostics or {}).get("invalid_entity_type") == "FOOD", "unknown type is recorded without the span text")
|
||||
expect((exc.diagnostics or {}).get("invalid_entity_type") == "HUMAN", "unknown type is recorded without the span text")
|
||||
expect((exc.diagnostics or {}).get("contract_violation") == "unknown_entity_type", "unknown type has a violation category")
|
||||
expect("Anna" not in json.dumps(exc.diagnostics or {}), "span plaintext is not stored on the type violation")
|
||||
expect(typed, "unknown entity_type is a contract error")
|
||||
@@ -252,10 +252,10 @@ def main() -> None:
|
||||
expect(blocked.diagnostics.get("generate_called") is False, "final abort keeps generate_called false")
|
||||
expect(blocked.diagnostics.get("detect_passes") == 2, "both invalid passes are counted")
|
||||
expect(blocked.diagnostics.get("contract_violation") == "unknown_entity_type", "final abort keeps the violation category")
|
||||
expect(blocked.diagnostics.get("invalid_entity_type") == "FOOD", "delivered type is stored without span text")
|
||||
expect(blocked.diagnostics.get("invalid_entity_type") == "HUMAN", "delivered type is stored without span text")
|
||||
expect(blocked.diagnostics.get("detect_partial_discarded") is True, "failed retry discarded the first pass")
|
||||
expect(_ids(blocked.diagnostics), "failed diagnostics are acyclic")
|
||||
expect("FOOD" in json.dumps(blocked.diagnostics), "invalid type remains visible")
|
||||
expect("HUMAN" in json.dumps(blocked.diagnostics), "invalid type remains visible")
|
||||
expect("secret" not in json.dumps(blocked.diagnostics).lower(), "no extra identity payload")
|
||||
expect(blocked.status_code != 500, "detect abort is not a generic 500")
|
||||
|
||||
@@ -334,7 +334,7 @@ def main() -> None:
|
||||
def script(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
||||
seen.append((kind, attempt))
|
||||
if attempt == 1:
|
||||
entity_type = "FOOD" if kind == "alpha" else "ANIMAL"
|
||||
entity_type = "HUMAN" if kind == "alpha" else "ANIMAL"
|
||||
return _chat([{"start": 0, "end": 1, "text": "x", "entity_type": entity_type}], USAGE_A)
|
||||
return _chat([], USAGE_B)
|
||||
|
||||
@@ -357,7 +357,7 @@ def main() -> None:
|
||||
all(kind == row[0] for row in by_kind.values() for kind, _attempt in row[1]),
|
||||
"parallel scripts stay on their request",
|
||||
)
|
||||
expect(by_kind["alpha"][2] in (None, "FOOD"), "alpha does not keep beta's type")
|
||||
expect(by_kind["alpha"][2] in (None, "HUMAN"), "alpha does not keep beta's type")
|
||||
expect(by_kind["beta"][2] in (None, "ANIMAL"), "beta does not keep alpha's type")
|
||||
expect(by_kind["alpha"][4] != by_kind["beta"][4], "parallel stats objects are distinct")
|
||||
|
||||
@@ -442,7 +442,7 @@ def main() -> None:
|
||||
expect(payload.get("detect_passes") == 2, "stored run records both detect passes")
|
||||
expect(payload.get("detect_partial_discarded") is True, "stored run records discarded partials")
|
||||
expect(payload.get("contract_violation") == "unknown_entity_type", "stored run records the violation")
|
||||
expect(payload.get("invalid_entity_type") == "FOOD", "stored run records the delivered type")
|
||||
expect(payload.get("invalid_entity_type") == "HUMAN", "stored run records the delivered type")
|
||||
expect(payload.get("generation_selection") or (payload.get("trace") or {}).get("generation_selection"), "stored run keeps generation selection")
|
||||
expect("local_label" not in json.dumps(payload), "stored run has no mapping labels")
|
||||
exported = client.get(
|
||||
|
||||
@@ -113,6 +113,73 @@ def main() -> None:
|
||||
expect("[[PERSON:" in (person_sushi.trace.get("egress") or ""), "same word can be a person in another sentence")
|
||||
expect("Sushi" not in (person_sushi.trace.get("egress") or "").replace("[[PERSON:", ""), "person Sushi is masked")
|
||||
|
||||
mixed_homonym = "Ich aß Sushi. Meine Frau Sushi kam später."
|
||||
food_at = mixed_homonym.find("Sushi")
|
||||
person_at = mixed_homonym.find("Sushi", food_at + 1)
|
||||
dish = validate_detected_entity(
|
||||
{"start": food_at, "end": food_at + 5, "text": "Sushi", "entity_type": "FOOD"},
|
||||
mixed_homonym,
|
||||
0,
|
||||
)
|
||||
expect(dish is None, "FOOD on a dish mention is omitted, not a contract error")
|
||||
person_from_food = validate_detected_entity(
|
||||
{"start": person_at, "end": person_at + 5, "text": "Sushi", "entity_type": "FOOD"},
|
||||
mixed_homonym,
|
||||
0,
|
||||
)
|
||||
expect(
|
||||
person_from_food is not None and person_from_food.entity_type == "PERSON",
|
||||
"FOOD on a kinship mention stays PERSON",
|
||||
)
|
||||
arrived = "Sushi kam später."
|
||||
arrived_span = validate_detected_entity(
|
||||
{"start": 0, "end": 5, "text": "Sushi", "entity_type": "DISH"},
|
||||
arrived,
|
||||
0,
|
||||
)
|
||||
expect(
|
||||
arrived_span is not None and arrived_span.entity_type == "PERSON",
|
||||
"DISH on a person subject stays PERSON",
|
||||
)
|
||||
noted = validate_detected_entity(
|
||||
{
|
||||
"start": food_at,
|
||||
"end": food_at + 5,
|
||||
"text": "Sushi",
|
||||
"entity_type": "FOOD",
|
||||
"note": "Gericht",
|
||||
},
|
||||
mixed_homonym,
|
||||
0,
|
||||
)
|
||||
expect(noted is None, "extra fields on a FOOD dish do not fail the pass")
|
||||
install_test_spans(
|
||||
[
|
||||
{"start": food_at, "end": food_at + 5, "text": "Sushi", "entity_type": "FOOD"},
|
||||
{"start": person_at, "end": person_at + 5, "text": "Sushi", "entity_type": "FOOD"},
|
||||
]
|
||||
)
|
||||
try:
|
||||
mixed_run = _run(profile_id, mixed_homonym)
|
||||
mixed_egress = mixed_run.trace.get("egress") or ""
|
||||
expect(mixed_run.allowed, "FOOD homonym does not block generate")
|
||||
expect("aß Sushi" in mixed_egress, "reconstructed dish stays unmasked")
|
||||
expect("[[PERSON:" in mixed_egress, "person-shaped FOOD span is masked")
|
||||
expect(
|
||||
not mixed_run.diagnostics.get("contract_violation"),
|
||||
"food type is not a contract violation",
|
||||
)
|
||||
expect(
|
||||
not mixed_run.diagnostics.get("invalid_entity_type"),
|
||||
"food type is not recorded as invalid",
|
||||
)
|
||||
expect(
|
||||
"FOOD" in (mixed_run.diagnostics.get("omitted_non_identity_types") or []),
|
||||
"omitted food type is recorded without span text",
|
||||
)
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
|
||||
mixed = _run(
|
||||
profile_id,
|
||||
"Ich traf Anna. Ich arbeitete am privaten Projekt Aurora. "
|
||||
@@ -470,7 +537,7 @@ def main() -> None:
|
||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.entity_detect",),
|
||||
).fetchone()
|
||||
expect(row["seed_revision"] == "2026-08-27-detect-ground-v1", "detect prompt revision is stored")
|
||||
expect(row["seed_revision"] == "2026-09-08-detect-homonym-v1", "detect prompt revision is stored")
|
||||
expect("{{known_labels}}" not in (row["template"] or ""), "new detect prompt has no known_labels skip list")
|
||||
expect("Zwiebeln" not in (row["template"] or ""), "detect prompt has no food word list")
|
||||
|
||||
@@ -488,7 +555,7 @@ def main() -> None:
|
||||
("mvp.entity_detect",),
|
||||
).fetchone()
|
||||
expect(custom["template"] == "CUSTOM DETECT {{source_text}}", "user-edited detect prompt is not overwritten")
|
||||
expect(custom["seed_revision"] == "2026-08-27-detect-ground-v1", "revision still updates the default")
|
||||
expect(custom["seed_revision"] == "2026-09-08-detect-homonym-v1", "revision still updates the default")
|
||||
|
||||
remote = ProviderConfig(
|
||||
role="detect",
|
||||
|
||||
Reference in new issue
Block a user