This commit is contained in:
Lars committed 2026-08-28 08:40:28 +02:00
1 parent f17b3083e8
commit 4f37991ad1
48 files changed
+4919 -540

No files matched your search

+26 -5
View File
@@ -262,10 +262,22 @@ def _contract_fake_spans(text: str) -> list[dict]:
found.append(item)
start = item["end"]
if re.search(r"Sushi kam", source):
add("Sushi", "PERSON")
if re.search(r"(?i)Frau\s+Sushi", source):
add("Sushi", "PERSON")
else:
for match in re.finditer(r"Sushi", source):
if source[match.end() : match.end() + 4] == " kam":
key = (match.start(), match.end(), "PERSON")
if key not in seen:
seen.add(key)
found.append(
{
"start": match.start(),
"end": match.end(),
"text": "Sushi",
"entity_type": "PERSON",
}
)
if re.search(r"(?i)Projekt\s+Aurora", source):
add("Aurora", "PROJECT")
if re.search(r"(?i)(Organisation|Firma|bei)\s+Nordwerk", source):
@@ -499,6 +511,8 @@ def _assign_request_tokens(
confirmed_hits += 1
entity_type = found["entity_type"]
demask = canonical
aliases = list(found.get("aliases") or [])
labels = confirmed_match_labels(found)
else:
key = f"{span.entity_type}:{label.casefold()}"
if key not in by_label:
@@ -511,6 +525,8 @@ def _assign_request_tokens(
entity_type = span.entity_type
source = "request_local"
demask = label
aliases = []
labels = [label]
mappings.append(
{
"token": token,
@@ -521,6 +537,8 @@ def _assign_request_tokens(
"source": source,
"start": span.start,
"end": span.end,
"aliases": aliases,
"labels": labels,
}
)
return mappings, request_local, confirmed_hits
@@ -529,17 +547,17 @@ def _assign_request_tokens(
def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str | None) -> tuple[list[dict], int]:
if not profile_id:
return mappings, 0
already = {(item.get("local_label") or "").casefold() for item in mappings}
already_labels = {(item.get("local_label") or "").casefold() for item in mappings if item.get("start") is None}
extra: list[dict] = []
hits = 0
for row in masking_rows_from_confirmed(profile_id):
label = row.get("local_label") or ""
if not label or label.casefold() in already:
if not label or label.casefold() in already_labels:
continue
if not re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text or "", re.IGNORECASE):
continue
extra.append(row)
already.add(label.casefold())
already_labels.add(label.casefold())
hits += 1
return mappings + extra, hits
@@ -685,6 +703,9 @@ def detect_personal_egress(profile_id: str | None, source_text: str) -> Detectio
"token": item.get("token"),
"entity_type": item.get("entity_type"),
"demask_label": item.get("demask_label") or item.get("local_label"),
"canonical_label": item.get("canonical_label") or item.get("demask_label") or item.get("local_label"),
"aliases": list(item.get("aliases") or []),
"labels": list(item.get("labels") or []),
"source": item.get("source"),
}
for item in mappings