MVP 0.94
This commit is contained in:
1 parent
f17b3083e8
commit
4f37991ad1
48 files changed
+4919
-540
No files matched your search
@@ -262,10 +262,22 @@ def _contract_fake_spans(text: str) -> list[dict]:
|
||||
found.append(item)
|
||||
start = item["end"]
|
||||
|
||||
if re.search(r"Sushi kam", source):
|
||||
add("Sushi", "PERSON")
|
||||
if re.search(r"(?i)Frau\s+Sushi", source):
|
||||
add("Sushi", "PERSON")
|
||||
else:
|
||||
for match in re.finditer(r"Sushi", source):
|
||||
if source[match.end() : match.end() + 4] == " kam":
|
||||
key = (match.start(), match.end(), "PERSON")
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
found.append(
|
||||
{
|
||||
"start": match.start(),
|
||||
"end": match.end(),
|
||||
"text": "Sushi",
|
||||
"entity_type": "PERSON",
|
||||
}
|
||||
)
|
||||
if re.search(r"(?i)Projekt\s+Aurora", source):
|
||||
add("Aurora", "PROJECT")
|
||||
if re.search(r"(?i)(Organisation|Firma|bei)\s+Nordwerk", source):
|
||||
@@ -499,6 +511,8 @@ def _assign_request_tokens(
|
||||
confirmed_hits += 1
|
||||
entity_type = found["entity_type"]
|
||||
demask = canonical
|
||||
aliases = list(found.get("aliases") or [])
|
||||
labels = confirmed_match_labels(found)
|
||||
else:
|
||||
key = f"{span.entity_type}:{label.casefold()}"
|
||||
if key not in by_label:
|
||||
@@ -511,6 +525,8 @@ def _assign_request_tokens(
|
||||
entity_type = span.entity_type
|
||||
source = "request_local"
|
||||
demask = label
|
||||
aliases = []
|
||||
labels = [label]
|
||||
mappings.append(
|
||||
{
|
||||
"token": token,
|
||||
@@ -521,6 +537,8 @@ def _assign_request_tokens(
|
||||
"source": source,
|
||||
"start": span.start,
|
||||
"end": span.end,
|
||||
"aliases": aliases,
|
||||
"labels": labels,
|
||||
}
|
||||
)
|
||||
return mappings, request_local, confirmed_hits
|
||||
@@ -529,17 +547,17 @@ def _assign_request_tokens(
|
||||
def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str | None) -> tuple[list[dict], int]:
|
||||
if not profile_id:
|
||||
return mappings, 0
|
||||
already = {(item.get("local_label") or "").casefold() for item in mappings}
|
||||
already_labels = {(item.get("local_label") or "").casefold() for item in mappings if item.get("start") is None}
|
||||
extra: list[dict] = []
|
||||
hits = 0
|
||||
for row in masking_rows_from_confirmed(profile_id):
|
||||
label = row.get("local_label") or ""
|
||||
if not label or label.casefold() in already:
|
||||
if not label or label.casefold() in already_labels:
|
||||
continue
|
||||
if not re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text or "", re.IGNORECASE):
|
||||
continue
|
||||
extra.append(row)
|
||||
already.add(label.casefold())
|
||||
already_labels.add(label.casefold())
|
||||
hits += 1
|
||||
return mappings + extra, hits
|
||||
|
||||
@@ -685,6 +703,9 @@ def detect_personal_egress(profile_id: str | None, source_text: str) -> Detectio
|
||||
"token": item.get("token"),
|
||||
"entity_type": item.get("entity_type"),
|
||||
"demask_label": item.get("demask_label") or item.get("local_label"),
|
||||
"canonical_label": item.get("canonical_label") or item.get("demask_label") or item.get("local_label"),
|
||||
"aliases": list(item.get("aliases") or []),
|
||||
"labels": list(item.get("labels") or []),
|
||||
"source": item.get("source"),
|
||||
}
|
||||
for item in mappings
|
||||
|
||||
Reference in new issue
Block a user