MVP 0.95
This commit is contained in:
1 parent
4f37991ad1
commit
be11f82f84
39 files changed
+2490
-239
No files matched your search
@@ -18,6 +18,7 @@ from identity_store import (
|
||||
ENTITY_TYPES,
|
||||
confirmed_match_labels,
|
||||
is_maskable_label,
|
||||
is_registry_maskable_label,
|
||||
list_confirmed_identities,
|
||||
masking_rows_from_confirmed,
|
||||
normalize_entity_type,
|
||||
@@ -109,6 +110,8 @@ class DetectionStats:
|
||||
entity_counts: dict[str, int] = field(default_factory=dict)
|
||||
confirmed_registry_hits: int = 0
|
||||
request_local_hits: int = 0
|
||||
confirmed_registry_applied: bool = False
|
||||
semantic_identity_guaranteed: bool = False
|
||||
detect_calls: int = 0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
@@ -128,7 +131,9 @@ class DetectionStats:
|
||||
"full_detection_coverage": self.full_detection_coverage,
|
||||
"entity_counts": dict(self.entity_counts),
|
||||
"confirmed_registry_hits": self.confirmed_registry_hits,
|
||||
"confirmed_registry_applied": self.confirmed_registry_applied,
|
||||
"request_local_hits": self.request_local_hits,
|
||||
"semantic_identity_guaranteed": False,
|
||||
"detect_calls": self.detect_calls,
|
||||
"detect_prompt_tokens": self.prompt_tokens,
|
||||
"detect_completion_tokens": self.completion_tokens,
|
||||
@@ -547,12 +552,16 @@ def _assign_request_tokens(
|
||||
def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str | None) -> tuple[list[dict], int]:
|
||||
if not profile_id:
|
||||
return mappings, 0
|
||||
already_labels = {(item.get("local_label") or "").casefold() for item in mappings if item.get("start") is None}
|
||||
already_labels = {
|
||||
(item.get("local_label") or "").casefold()
|
||||
for item in mappings
|
||||
if item.get("start") is None and item.get("source") == "confirmed_registry"
|
||||
}
|
||||
extra: list[dict] = []
|
||||
hits = 0
|
||||
for row in masking_rows_from_confirmed(profile_id):
|
||||
label = row.get("local_label") or ""
|
||||
if not label or label.casefold() in already_labels:
|
||||
if not label or not is_registry_maskable_label(label) or label.casefold() in already_labels:
|
||||
continue
|
||||
if not re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text or "", re.IGNORECASE):
|
||||
continue
|
||||
@@ -680,6 +689,8 @@ def detect_personal_egress(profile_id: str | None, source_text: str) -> Detectio
|
||||
mappings, extra_confirmed = _merge_confirmed_safety_net(source_text or "", mappings, profile_id)
|
||||
stats.request_local_hits = request_local
|
||||
stats.confirmed_registry_hits = confirmed_from_spans + extra_confirmed
|
||||
stats.confirmed_registry_applied = bool(profile_id)
|
||||
stats.semantic_identity_guaranteed = False
|
||||
counts: dict[str, int] = {kind: 0 for kind in ENTITY_TYPES}
|
||||
for item in mappings:
|
||||
kind = (item.get("entity_type") or "PERSON").upper()
|
||||
|
||||
Reference in new issue
Block a user