"""Entity detection for the privacy gateway. May see plaintext; generate must not.""" from __future__ import annotations import json import re import placeholder_mvp # noqa: F401 from db import get_db, row_to_dict from env_loader import allows_remote_plaintext_detect from identity_store import ( coalesce_mappings, is_given_name_candidate, is_maskable_label, list_mappings, normalize_label, purge_unmaskable, remember_mapping, ) from placeholders import PlaceholderError, resolve_template from providers import ProviderError, complete_chat, detect_provider DETECT_MAX_CHARS = 8000 DETECT_TIMEOUT = 20.0 DETECT_MAX_TOKENS = 300 JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL) NAME_TITLE = re.compile(r"(?:Frau|Herr)\s+([A-ZÄÖÜ][a-zäöüß]{2,})\b") NAME_PREP = re.compile(r"(?:mit|von|bei)\s+([A-ZÄÖÜ][a-zäöüß]{2,})\b") _LETTER = r"A-Za-zÄÖÜäöüß" PLACE_NAME = re.compile( r"\b([A-ZÄÖÜ][a-zäöüß]*(?:burg|stadt|dorf|haven|ingen|heim|bach|feld))\b" ) STOPWORDS = { "Heute", "Gestern", "Morgen", "Ich", "Wir", "Der", "Die", "Das", "Ein", "Eine", "Und", "Oder", "Nicht", "Kein", "Keine", "Am", "Im", "Zum", "Zur", "Mit", "Nach", "Beim", "Über", "Unter", "Aber", "Denn", "Wenn", "Dann", "Also", "Was", "Wer", "Wie", "Wo", "Warum", "Bitte", "Danke", "Hallo", "Space", "Journal", "Dialog", "Kontext", "Writing", "Profile", "Tagebucheintrag", "Relativ", "Allerdings", "Zunächst", "Danach", "Deshalb", "Trotzdem", "Außerdem", "Schließlich", "Während", "Bevor", "Seitdem", } def _detect_prompt() -> dict: with get_db() as conn: row = row_to_dict( conn.execute( "SELECT * FROM ai_prompts WHERE slug = ? AND active = 1", ("mvp.entity_detect",), ).fetchone() ) if not row or not (row.get("template") or "").strip(): raise ProviderError( "detect_prompt_missing", "Prompt mvp.entity_detect fehlt in der Konfiguration.", ) return row def _pattern_entities(text: str, known_labels: set[str]) -> list[dict]: found: list[dict] = [] seen: set[str] = set() source = text or "" candidates: list[tuple[str, str]] = [] for match in NAME_TITLE.finditer(source): candidates.append((match.group(1).strip(), "PERSON:01")) for match in NAME_PREP.finditer(source): name = match.group(1).strip() if is_given_name_candidate(name): candidates.append((name, "PERSON:01")) for match in PLACE_NAME.finditer(source): candidates.append((match.group(1).strip(), "PLACE:CITY")) for raw, token in candidates: label = normalize_label(raw) or raw if label in STOPWORDS or label in known_labels or label in seen: continue if not is_maskable_label(label): continue seen.add(label) found.append({"text": label, "token": token}) return found def _fake_entities(text: str, known_labels: set[str]) -> list[dict]: return _pattern_entities(text, known_labels) def _merge_proposals(*groups: list[dict]) -> list[dict]: merged: list[dict] = [] seen: set[str] = set() for group in groups: for item in group: label = (item.get("text") or "").strip() if not label or label in seen: continue seen.add(label) merged.append(item) return merged def _parse_entities(raw: str) -> list[dict]: text = (raw or "").strip() if text.startswith("```"): text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL) match = JSON_BLOCK.search(text) if not match: return [] try: data = json.loads(match.group(0)) except json.JSONDecodeError: return [] items = data.get("entities") if isinstance(data, dict) else data if not isinstance(items, list): return [] result = [] for item in items: if not isinstance(item, dict): continue label = (item.get("text") or item.get("label") or "").strip() token = (item.get("token") or item.get("placeholder") or "").strip() if label: result.append({"text": label, "token": token}) return result def _label_in_text(label: str, text: str) -> bool: if not (label or "").strip() or not (text or "").strip(): return False return bool( re.search( rf"(? bool: label = (item.get("text") or "").strip() if not label or not _label_in_text(label, excerpt) or not is_maskable_label(label): return False token = (item.get("token") or "").upper() if token.startswith("PLACE:") or token.startswith("ORG:"): return True return is_given_name_candidate(label) def uses_llm_detect(config) -> bool: """Local detect stays a provider role. Remote plaintext detect is development/test only.""" if not config or config.mode != "http": return False if config.local: return True return allows_remote_plaintext_detect() def _llm_entities(config, excerpt: str, known_labels: set[str]) -> list[dict]: known = ", ".join(sorted(known_labels)) or "(keine)" prompt = resolve_template( _detect_prompt()["template"], {"source_text": excerpt, "known_labels": known}, ) raw = complete_chat( config, [{"role": "user", "content": prompt}], timeout=DETECT_TIMEOUT, max_tokens=DETECT_MAX_TOKENS, ) return _parse_entities(raw) def detect_and_remember(profile_id: str | None, source_text: str) -> tuple[list[dict], str | None, str | None]: if profile_id: purge_unmaskable(profile_id) coalesce_mappings(profile_id) mappings = list_mappings(profile_id) if profile_id else [] if not profile_id: return mappings, None, None known_labels = { normalize_label(item.get("local_label") or "") or (item.get("local_label") or "").strip() for item in mappings if item.get("local_label") } excerpt = (source_text or "")[:DETECT_MAX_CHARS] proposals = _pattern_entities(excerpt, known_labels) detect_name = "pattern" detect_note = "pattern" config = detect_provider() if config and config.mode == "fake": detect_name = config.name detect_note = "fake" elif uses_llm_detect(config): try: proposals = _merge_proposals( proposals, [item for item in _llm_entities(config, excerpt, known_labels) if _acceptable_proposal(item, excerpt)], ) detect_name = config.name detect_note = "local_llm" if config.local else "remote_llm" except PlaceholderError as exc: raise ProviderError(exc.code, exc.message) from exc elif config and config.mode == "http" and not config.local: detect_note = "pattern; remote_detect_blocked_production" for item in proposals: label = (item.get("text") or "").strip() core = normalize_label(label) if not label or not core or core in known_labels: continue if not _acceptable_proposal(item, excerpt): continue try: remember_mapping(profile_id, label, item.get("token")) except ValueError: continue known_labels.add(core) coalesce_mappings(profile_id) return list_mappings(profile_id), detect_name, detect_note