217 lines
7.5 KiB
Python
217 lines
7.5 KiB
Python
"""Entity detection for the privacy gateway. May see plaintext; generate must not."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
|
|
import placeholder_mvp # noqa: F401
|
|
from db import get_db, row_to_dict
|
|
from env_loader import allows_remote_plaintext_detect
|
|
from identity_store import (
|
|
coalesce_mappings,
|
|
is_given_name_candidate,
|
|
is_maskable_label,
|
|
list_mappings,
|
|
normalize_label,
|
|
purge_unmaskable,
|
|
remember_mapping,
|
|
)
|
|
from placeholders import PlaceholderError, resolve_template
|
|
from providers import ProviderError, complete_chat, detect_provider
|
|
|
|
DETECT_MAX_CHARS = 8000
|
|
DETECT_TIMEOUT = 20.0
|
|
DETECT_MAX_TOKENS = 300
|
|
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
|
NAME_TITLE = re.compile(r"(?:Frau|Herr)\s+([A-ZÄÖÜ][a-zäöüß]{2,})\b")
|
|
NAME_PREP = re.compile(r"(?:mit|von|bei)\s+([A-ZÄÖÜ][a-zäöüß]{2,})\b")
|
|
_LETTER = r"A-Za-zÄÖÜäöüß"
|
|
PLACE_NAME = re.compile(
|
|
r"\b([A-ZÄÖÜ][a-zäöüß]*(?:burg|stadt|dorf|haven|ingen|heim|bach|feld))\b"
|
|
)
|
|
STOPWORDS = {
|
|
"Heute", "Gestern", "Morgen", "Ich", "Wir", "Der", "Die", "Das", "Ein", "Eine",
|
|
"Und", "Oder", "Nicht", "Kein", "Keine", "Am", "Im", "Zum", "Zur", "Mit",
|
|
"Nach", "Beim", "Über", "Unter", "Aber", "Denn", "Wenn", "Dann", "Also",
|
|
"Was", "Wer", "Wie", "Wo", "Warum", "Bitte", "Danke", "Hallo", "Space",
|
|
"Journal", "Dialog", "Kontext", "Writing", "Profile", "Tagebucheintrag",
|
|
"Relativ", "Allerdings", "Zunächst", "Danach", "Deshalb", "Trotzdem",
|
|
"Außerdem", "Schließlich", "Während", "Bevor", "Seitdem",
|
|
}
|
|
|
|
|
|
def _detect_prompt() -> dict:
|
|
with get_db() as conn:
|
|
row = row_to_dict(
|
|
conn.execute(
|
|
"SELECT * FROM ai_prompts WHERE slug = ? AND active = 1",
|
|
("mvp.entity_detect",),
|
|
).fetchone()
|
|
)
|
|
if not row or not (row.get("template") or "").strip():
|
|
raise ProviderError(
|
|
"detect_prompt_missing",
|
|
"Prompt mvp.entity_detect fehlt in der Konfiguration.",
|
|
)
|
|
return row
|
|
|
|
|
|
def _pattern_entities(text: str, known_labels: set[str]) -> list[dict]:
|
|
found: list[dict] = []
|
|
seen: set[str] = set()
|
|
source = text or ""
|
|
candidates: list[tuple[str, str]] = []
|
|
for match in NAME_TITLE.finditer(source):
|
|
candidates.append((match.group(1).strip(), "PERSON:01"))
|
|
for match in NAME_PREP.finditer(source):
|
|
name = match.group(1).strip()
|
|
if is_given_name_candidate(name):
|
|
candidates.append((name, "PERSON:01"))
|
|
for match in PLACE_NAME.finditer(source):
|
|
candidates.append((match.group(1).strip(), "PLACE:CITY"))
|
|
for raw, token in candidates:
|
|
label = normalize_label(raw) or raw
|
|
if label in STOPWORDS or label in known_labels or label in seen:
|
|
continue
|
|
if not is_maskable_label(label):
|
|
continue
|
|
seen.add(label)
|
|
found.append({"text": label, "token": token})
|
|
return found
|
|
|
|
|
|
def _fake_entities(text: str, known_labels: set[str]) -> list[dict]:
|
|
return _pattern_entities(text, known_labels)
|
|
|
|
|
|
def _merge_proposals(*groups: list[dict]) -> list[dict]:
|
|
merged: list[dict] = []
|
|
seen: set[str] = set()
|
|
for group in groups:
|
|
for item in group:
|
|
label = (item.get("text") or "").strip()
|
|
if not label or label in seen:
|
|
continue
|
|
seen.add(label)
|
|
merged.append(item)
|
|
return merged
|
|
|
|
|
|
def _parse_entities(raw: str) -> list[dict]:
|
|
text = (raw or "").strip()
|
|
if text.startswith("```"):
|
|
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL)
|
|
match = JSON_BLOCK.search(text)
|
|
if not match:
|
|
return []
|
|
try:
|
|
data = json.loads(match.group(0))
|
|
except json.JSONDecodeError:
|
|
return []
|
|
items = data.get("entities") if isinstance(data, dict) else data
|
|
if not isinstance(items, list):
|
|
return []
|
|
result = []
|
|
for item in items:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
label = (item.get("text") or item.get("label") or "").strip()
|
|
token = (item.get("token") or item.get("placeholder") or "").strip()
|
|
if label:
|
|
result.append({"text": label, "token": token})
|
|
return result
|
|
|
|
|
|
def _label_in_text(label: str, text: str) -> bool:
|
|
if not (label or "").strip() or not (text or "").strip():
|
|
return False
|
|
return bool(
|
|
re.search(
|
|
rf"(?<![{_LETTER}]){re.escape(label.strip())}(?![{_LETTER}])",
|
|
text,
|
|
re.IGNORECASE,
|
|
)
|
|
)
|
|
|
|
|
|
def _acceptable_proposal(item: dict, excerpt: str) -> bool:
|
|
label = (item.get("text") or "").strip()
|
|
if not label or not _label_in_text(label, excerpt) or not is_maskable_label(label):
|
|
return False
|
|
token = (item.get("token") or "").upper()
|
|
if token.startswith("PLACE:") or token.startswith("ORG:"):
|
|
return True
|
|
return is_given_name_candidate(label)
|
|
|
|
|
|
def uses_llm_detect(config) -> bool:
|
|
"""Local detect stays a provider role. Remote plaintext detect is development/test only."""
|
|
if not config or config.mode != "http":
|
|
return False
|
|
if config.local:
|
|
return True
|
|
return allows_remote_plaintext_detect()
|
|
|
|
|
|
def _llm_entities(config, excerpt: str, known_labels: set[str]) -> list[dict]:
|
|
known = ", ".join(sorted(known_labels)) or "(keine)"
|
|
prompt = resolve_template(
|
|
_detect_prompt()["template"],
|
|
{"source_text": excerpt, "known_labels": known},
|
|
)
|
|
result = complete_chat(
|
|
config,
|
|
[{"role": "user", "content": prompt}],
|
|
timeout=DETECT_TIMEOUT,
|
|
max_tokens=DETECT_MAX_TOKENS,
|
|
)
|
|
return _parse_entities(result.content)
|
|
|
|
|
|
def detect_and_remember(profile_id: str | None, source_text: str) -> tuple[list[dict], str | None, str | None]:
|
|
if profile_id:
|
|
purge_unmaskable(profile_id)
|
|
coalesce_mappings(profile_id)
|
|
mappings = list_mappings(profile_id) if profile_id else []
|
|
if not profile_id:
|
|
return mappings, None, None
|
|
known_labels = {
|
|
normalize_label(item.get("local_label") or "") or (item.get("local_label") or "").strip()
|
|
for item in mappings
|
|
if item.get("local_label")
|
|
}
|
|
excerpt = (source_text or "")[:DETECT_MAX_CHARS]
|
|
proposals = _pattern_entities(excerpt, known_labels)
|
|
detect_name = "pattern"
|
|
detect_note = "pattern"
|
|
config = detect_provider()
|
|
if config and config.mode == "fake":
|
|
detect_name = config.name
|
|
detect_note = "fake"
|
|
elif uses_llm_detect(config):
|
|
try:
|
|
proposals = _merge_proposals(
|
|
proposals,
|
|
[item for item in _llm_entities(config, excerpt, known_labels) if _acceptable_proposal(item, excerpt)],
|
|
)
|
|
detect_name = config.name
|
|
detect_note = "local_llm" if config.local else "remote_llm"
|
|
except PlaceholderError as exc:
|
|
raise ProviderError(exc.code, exc.message) from exc
|
|
elif config and config.mode == "http" and not config.local:
|
|
detect_note = "pattern; remote_detect_blocked_production"
|
|
for item in proposals:
|
|
label = (item.get("text") or "").strip()
|
|
core = normalize_label(label)
|
|
if not label or not core or core in known_labels:
|
|
continue
|
|
if not _acceptable_proposal(item, excerpt):
|
|
continue
|
|
try:
|
|
remember_mapping(profile_id, label, item.get("token"))
|
|
except ValueError:
|
|
continue
|
|
known_labels.add(core)
|
|
coalesce_mappings(profile_id)
|
|
return list_mappings(profile_id), detect_name, detect_note
|