Add support for handling non-identity entity types in detection logic
- Introduced a new set of non-identity entity types (FOOD, DISH, MEAL, CUISINE, FOODSTUFF, INGREDIENT) that are omitted from detection results without causing contract violations. - Updated the `DetectionStats` class to track omitted non-identity types. - Modified validation logic to ensure that food-related mentions are not treated as identity types, preserving the context of mentions as either PERSON or omitted. - Enhanced tests to verify the correct handling of food homonyms and ensure that invalid entity types are recorded appropriately without causing errors. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
79e4fd0d4b
commit
dd7858347c
7 files changed
+170
-22
No files matched your search
+87
-10
@@ -36,6 +36,16 @@ JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
||||
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
|
||||
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
|
||||
TYPE_PRIORITY = {"PERSON": 0, "PROJECT": 1, "ORG": 2, "PLACE": 3}
|
||||
NON_IDENTITY_ENTITY_TYPES = frozenset(
|
||||
{
|
||||
"FOOD",
|
||||
"DISH",
|
||||
"MEAL",
|
||||
"CUISINE",
|
||||
"FOODSTUFF",
|
||||
"INGREDIENT",
|
||||
}
|
||||
)
|
||||
|
||||
ERROR_DETECT_UNAVAILABLE = "detect_provider_unavailable"
|
||||
ERROR_DETECT_INCOMPLETE = "detect_incomplete"
|
||||
@@ -55,6 +65,7 @@ SCHEMA_RETRY_HINT = (
|
||||
"Antworte ausschließlich mit dem geforderten JSON-Objekt. "
|
||||
"Nur die Felder start, end, text und entity_type sind zulässig. "
|
||||
"Nur entity_type-Werte PERSON, PLACE, ORG, PROJECT. "
|
||||
"Lebensmittel und Gerichte weglassen, nicht als FOOD oder anderen Typ melden. "
|
||||
"Offsets sind 0-basiert und end ausschließlich, bezogen auf den Text nach «Text:». "
|
||||
"text muss ein exakter Substring dieses Texts sein. "
|
||||
"Keine zusätzlichen Felder, keine Tokens, keine Platzhalter, keine Erklärungen."
|
||||
@@ -143,6 +154,7 @@ class DetectionStats:
|
||||
detect_attempts: list[dict[str, Any]] = field(default_factory=list)
|
||||
contract_violation: str | None = None
|
||||
invalid_entity_type: str | None = None
|
||||
omitted_non_identity_types: list[str] = field(default_factory=list)
|
||||
|
||||
def public(self) -> dict[str, Any]:
|
||||
payload = {
|
||||
@@ -179,6 +191,8 @@ class DetectionStats:
|
||||
payload["contract_violation"] = self.contract_violation
|
||||
if self.invalid_entity_type:
|
||||
payload["invalid_entity_type"] = self.invalid_entity_type
|
||||
if self.omitted_non_identity_types:
|
||||
payload["omitted_non_identity_types"] = list(self.omitted_non_identity_types)
|
||||
return payload
|
||||
|
||||
|
||||
@@ -433,9 +447,56 @@ def _ground_span(chunk_text: str, start: int, end: int, text: str) -> tuple[int,
|
||||
return best[0], best[1], source[best[0] : best[1]]
|
||||
|
||||
|
||||
def _delivered_entity_type(raw: Any) -> str:
|
||||
if not isinstance(raw, str):
|
||||
return ""
|
||||
return re.sub(r"[^A-Z0-9_:-]", "", raw.strip().upper())[:40]
|
||||
|
||||
|
||||
def _identity_mention(chunk_text: str, start: int, end: int) -> bool:
|
||||
from privacy_gateway import is_identity_mention
|
||||
|
||||
return is_identity_mention(chunk_text, start, end, "PERSON:00")
|
||||
|
||||
|
||||
def _span_from_non_identity(item: dict, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
|
||||
"""Food/dish types are not identities. Keep only a person-shaped mention of the same word."""
|
||||
text = item.get("text")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
return None
|
||||
try:
|
||||
start = int(item.get("start") or 0)
|
||||
except (TypeError, ValueError):
|
||||
start = 0
|
||||
try:
|
||||
end = int(item.get("end") or 0)
|
||||
except (TypeError, ValueError):
|
||||
end = 0
|
||||
grounded = _ground_span(chunk_text, start, end, text)
|
||||
if not grounded:
|
||||
return None
|
||||
left, right, local_text = grounded
|
||||
if not _identity_mention(chunk_text, left, right):
|
||||
return None
|
||||
return DetectedSpan(
|
||||
start=left,
|
||||
end=right,
|
||||
text=local_text,
|
||||
entity_type="PERSON",
|
||||
chunk_index=chunk_index,
|
||||
)
|
||||
|
||||
|
||||
def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
|
||||
if not isinstance(item, dict):
|
||||
raise _contract_error("Detect-Entity muss ein Objekt sein.", violation="invalid_entity")
|
||||
if "token" in item or "placeholder" in item:
|
||||
raise _contract_error("Detect darf keine Tokens festlegen.", violation="extra_fields")
|
||||
raw_type = item.get("entity_type")
|
||||
delivered = _delivered_entity_type(raw_type)
|
||||
kind = normalize_entity_type(raw_type if isinstance(raw_type, str) else "", default="")
|
||||
if not kind and delivered in NON_IDENTITY_ENTITY_TYPES:
|
||||
return _span_from_non_identity(item, chunk_text, chunk_index)
|
||||
extra = set(item.keys()) - ALLOWED_ENTITY_FIELDS
|
||||
if extra:
|
||||
raise _contract_error("Detect-Entity enthält unerwartete Felder.", violation="extra_fields")
|
||||
@@ -455,16 +516,12 @@ def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> De
|
||||
raise _contract_error("Detect-text fehlt.", violation="missing_fields")
|
||||
if not text.strip():
|
||||
raise _contract_error("Detect-text ist leer.", violation="invalid_text")
|
||||
if "token" in item or "placeholder" in item:
|
||||
raise _contract_error("Detect darf keine Tokens festlegen.", violation="extra_fields")
|
||||
raw_type = item.get("entity_type")
|
||||
kind = normalize_entity_type(raw_type if isinstance(raw_type, str) else "", default="")
|
||||
if not kind:
|
||||
delivered = raw_type if isinstance(raw_type, str) else ""
|
||||
delivered_raw = raw_type if isinstance(raw_type, str) else ""
|
||||
raise _contract_error(
|
||||
"Detect-entity_type ist nicht erlaubt.",
|
||||
violation="unknown_entity_type",
|
||||
invalid_entity_type=delivered,
|
||||
invalid_entity_type=delivered_raw,
|
||||
)
|
||||
grounded = _ground_span(chunk_text, start, end, text)
|
||||
if not grounded:
|
||||
@@ -532,7 +589,19 @@ def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResul
|
||||
)
|
||||
|
||||
|
||||
def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int) -> list[DetectedSpan]:
|
||||
def _note_omitted_type(stats: DetectionStats | None, delivered: str) -> None:
|
||||
if not stats or not delivered or delivered not in NON_IDENTITY_ENTITY_TYPES:
|
||||
return
|
||||
if delivered not in stats.omitted_non_identity_types:
|
||||
stats.omitted_non_identity_types.append(delivered)
|
||||
|
||||
|
||||
def _entities_from_result(
|
||||
result: ChatResult,
|
||||
chunk_text: str,
|
||||
chunk_index: int,
|
||||
stats: DetectionStats | None = None,
|
||||
) -> list[DetectedSpan]:
|
||||
if (result.finish_reason or "").lower() in {"length", "max_tokens"}:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_TRUNCATED,
|
||||
@@ -543,8 +612,11 @@ def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int)
|
||||
spans: list[DetectedSpan] = []
|
||||
for item in data.get("entities") or []:
|
||||
span = validate_detected_entity(item, chunk_text, chunk_index)
|
||||
if span is not None:
|
||||
spans.append(span)
|
||||
if span is None:
|
||||
delivered = _delivered_entity_type(item.get("entity_type") if isinstance(item, dict) else "")
|
||||
_note_omitted_type(stats, delivered)
|
||||
continue
|
||||
spans.append(span)
|
||||
return spans
|
||||
|
||||
|
||||
@@ -674,6 +746,9 @@ def _merge_pass_stats(overall: DetectionStats, pass_stats: DetectionStats) -> No
|
||||
if pass_stats.cost_known:
|
||||
overall.cost_known = True
|
||||
overall.cost += pass_stats.cost
|
||||
for kind in pass_stats.omitted_non_identity_types:
|
||||
if kind not in overall.omitted_non_identity_types:
|
||||
overall.omitted_non_identity_types.append(kind)
|
||||
|
||||
|
||||
def _pass_snapshot(
|
||||
@@ -706,6 +781,8 @@ def _pass_snapshot(
|
||||
row["contract_violation"] = pass_stats.contract_violation
|
||||
if pass_stats.invalid_entity_type:
|
||||
row["invalid_entity_type"] = pass_stats.invalid_entity_type
|
||||
if pass_stats.omitted_non_identity_types:
|
||||
row["omitted_non_identity_types"] = list(pass_stats.omitted_non_identity_types)
|
||||
return row
|
||||
|
||||
|
||||
@@ -822,7 +899,7 @@ def _run_detection_pass(
|
||||
)
|
||||
stats.detect_calls += 1
|
||||
_add_usage(stats, result.usage)
|
||||
spans = _entities_from_result(result, chunk_text, index)
|
||||
spans = _entities_from_result(result, chunk_text, index, stats)
|
||||
collected.extend(_to_global(span, offset) for span in spans)
|
||||
stats.chunks_ok += 1
|
||||
if stats.chunks_ok != stats.chunk_count:
|
||||
|
||||
Reference in new issue
Block a user