Add support for handling non-identity entity types in detection logic
Deploy Development / deploy (push) Successful in 54s
Test Suite / pytest-backend (push) Successful in 2m37s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 15s
Deploy Production / deploy (push) Successful in 43s

- Introduced a new set of non-identity entity types (FOOD, DISH, MEAL, CUISINE, FOODSTUFF, INGREDIENT) that are omitted from detection results without causing contract violations.
- Updated the `DetectionStats` class to track omitted non-identity types.
- Modified validation logic to ensure that food-related mentions are not treated as identity types, preserving the context of mentions as either PERSON or omitted.
- Enhanced tests to verify the correct handling of food homonyms and ensure that invalid entity types are recorded appropriately without causing errors.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-09-08 07:35:44 +02:00
1 parent 79e4fd0d4b
commit dd7858347c
7 files changed
+170 -22

No files matched your search

+87 -10
View File
@@ -36,6 +36,16 @@ JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
TYPE_PRIORITY = {"PERSON": 0, "PROJECT": 1, "ORG": 2, "PLACE": 3}
NON_IDENTITY_ENTITY_TYPES = frozenset(
{
"FOOD",
"DISH",
"MEAL",
"CUISINE",
"FOODSTUFF",
"INGREDIENT",
}
)
ERROR_DETECT_UNAVAILABLE = "detect_provider_unavailable"
ERROR_DETECT_INCOMPLETE = "detect_incomplete"
@@ -55,6 +65,7 @@ SCHEMA_RETRY_HINT = (
"Antworte ausschließlich mit dem geforderten JSON-Objekt. "
"Nur die Felder start, end, text und entity_type sind zulässig. "
"Nur entity_type-Werte PERSON, PLACE, ORG, PROJECT. "
"Lebensmittel und Gerichte weglassen, nicht als FOOD oder anderen Typ melden. "
"Offsets sind 0-basiert und end ausschließlich, bezogen auf den Text nach «Text:». "
"text muss ein exakter Substring dieses Texts sein. "
"Keine zusätzlichen Felder, keine Tokens, keine Platzhalter, keine Erklärungen."
@@ -143,6 +154,7 @@ class DetectionStats:
detect_attempts: list[dict[str, Any]] = field(default_factory=list)
contract_violation: str | None = None
invalid_entity_type: str | None = None
omitted_non_identity_types: list[str] = field(default_factory=list)
def public(self) -> dict[str, Any]:
payload = {
@@ -179,6 +191,8 @@ class DetectionStats:
payload["contract_violation"] = self.contract_violation
if self.invalid_entity_type:
payload["invalid_entity_type"] = self.invalid_entity_type
if self.omitted_non_identity_types:
payload["omitted_non_identity_types"] = list(self.omitted_non_identity_types)
return payload
@@ -433,9 +447,56 @@ def _ground_span(chunk_text: str, start: int, end: int, text: str) -> tuple[int,
return best[0], best[1], source[best[0] : best[1]]
def _delivered_entity_type(raw: Any) -> str:
if not isinstance(raw, str):
return ""
return re.sub(r"[^A-Z0-9_:-]", "", raw.strip().upper())[:40]
def _identity_mention(chunk_text: str, start: int, end: int) -> bool:
from privacy_gateway import is_identity_mention
return is_identity_mention(chunk_text, start, end, "PERSON:00")
def _span_from_non_identity(item: dict, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
"""Food/dish types are not identities. Keep only a person-shaped mention of the same word."""
text = item.get("text")
if not isinstance(text, str) or not text.strip():
return None
try:
start = int(item.get("start") or 0)
except (TypeError, ValueError):
start = 0
try:
end = int(item.get("end") or 0)
except (TypeError, ValueError):
end = 0
grounded = _ground_span(chunk_text, start, end, text)
if not grounded:
return None
left, right, local_text = grounded
if not _identity_mention(chunk_text, left, right):
return None
return DetectedSpan(
start=left,
end=right,
text=local_text,
entity_type="PERSON",
chunk_index=chunk_index,
)
def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
if not isinstance(item, dict):
raise _contract_error("Detect-Entity muss ein Objekt sein.", violation="invalid_entity")
if "token" in item or "placeholder" in item:
raise _contract_error("Detect darf keine Tokens festlegen.", violation="extra_fields")
raw_type = item.get("entity_type")
delivered = _delivered_entity_type(raw_type)
kind = normalize_entity_type(raw_type if isinstance(raw_type, str) else "", default="")
if not kind and delivered in NON_IDENTITY_ENTITY_TYPES:
return _span_from_non_identity(item, chunk_text, chunk_index)
extra = set(item.keys()) - ALLOWED_ENTITY_FIELDS
if extra:
raise _contract_error("Detect-Entity enthält unerwartete Felder.", violation="extra_fields")
@@ -455,16 +516,12 @@ def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> De
raise _contract_error("Detect-text fehlt.", violation="missing_fields")
if not text.strip():
raise _contract_error("Detect-text ist leer.", violation="invalid_text")
if "token" in item or "placeholder" in item:
raise _contract_error("Detect darf keine Tokens festlegen.", violation="extra_fields")
raw_type = item.get("entity_type")
kind = normalize_entity_type(raw_type if isinstance(raw_type, str) else "", default="")
if not kind:
delivered = raw_type if isinstance(raw_type, str) else ""
delivered_raw = raw_type if isinstance(raw_type, str) else ""
raise _contract_error(
"Detect-entity_type ist nicht erlaubt.",
violation="unknown_entity_type",
invalid_entity_type=delivered,
invalid_entity_type=delivered_raw,
)
grounded = _ground_span(chunk_text, start, end, text)
if not grounded:
@@ -532,7 +589,19 @@ def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResul
)
def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int) -> list[DetectedSpan]:
def _note_omitted_type(stats: DetectionStats | None, delivered: str) -> None:
if not stats or not delivered or delivered not in NON_IDENTITY_ENTITY_TYPES:
return
if delivered not in stats.omitted_non_identity_types:
stats.omitted_non_identity_types.append(delivered)
def _entities_from_result(
result: ChatResult,
chunk_text: str,
chunk_index: int,
stats: DetectionStats | None = None,
) -> list[DetectedSpan]:
if (result.finish_reason or "").lower() in {"length", "max_tokens"}:
raise DetectError(
ERROR_DETECT_TRUNCATED,
@@ -543,8 +612,11 @@ def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int)
spans: list[DetectedSpan] = []
for item in data.get("entities") or []:
span = validate_detected_entity(item, chunk_text, chunk_index)
if span is not None:
spans.append(span)
if span is None:
delivered = _delivered_entity_type(item.get("entity_type") if isinstance(item, dict) else "")
_note_omitted_type(stats, delivered)
continue
spans.append(span)
return spans
@@ -674,6 +746,9 @@ def _merge_pass_stats(overall: DetectionStats, pass_stats: DetectionStats) -> No
if pass_stats.cost_known:
overall.cost_known = True
overall.cost += pass_stats.cost
for kind in pass_stats.omitted_non_identity_types:
if kind not in overall.omitted_non_identity_types:
overall.omitted_non_identity_types.append(kind)
def _pass_snapshot(
@@ -706,6 +781,8 @@ def _pass_snapshot(
row["contract_violation"] = pass_stats.contract_violation
if pass_stats.invalid_entity_type:
row["invalid_entity_type"] = pass_stats.invalid_entity_type
if pass_stats.omitted_non_identity_types:
row["omitted_non_identity_types"] = list(pass_stats.omitted_non_identity_types)
return row
@@ -822,7 +899,7 @@ def _run_detection_pass(
)
stats.detect_calls += 1
_add_usage(stats, result.usage)
spans = _entities_from_result(result, chunk_text, index)
spans = _entities_from_result(result, chunk_text, index, stats)
collected.extend(_to_global(span, offset) for span in spans)
stats.chunks_ok += 1
if stats.chunks_ok != stats.chunk_count: