MVP 0.93 - stabile, aber Writing Profile ohne erkennbaren Nutzen
This commit is contained in:
1 parent
53b1eff833
commit
f17b3083e8
56 files changed
+5366
-784
No files matched your search
+639
-163
@@ -1,147 +1,179 @@
|
||||
"""Entity detection for the privacy gateway. May see plaintext; generate must not."""
|
||||
"""Semantic entity detection for the privacy gateway.
|
||||
|
||||
Detect may see plaintext. Generate must not. Detector output is untrusted:
|
||||
spans and types only, never tokens, never a durable registry write.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import placeholder_mvp # noqa: F401
|
||||
from db import get_db, row_to_dict
|
||||
from env_loader import allows_remote_plaintext_detect
|
||||
from identity_store import (
|
||||
coalesce_mappings,
|
||||
is_given_name_candidate,
|
||||
ENTITY_TYPES,
|
||||
confirmed_match_labels,
|
||||
is_maskable_label,
|
||||
list_mappings,
|
||||
normalize_label,
|
||||
purge_unmaskable,
|
||||
remember_mapping,
|
||||
list_confirmed_identities,
|
||||
masking_rows_from_confirmed,
|
||||
normalize_entity_type,
|
||||
record_review_proposal,
|
||||
)
|
||||
from placeholders import PlaceholderError, resolve_template
|
||||
from providers import ProviderError, complete_chat, detect_provider
|
||||
from providers import ChatResult, ProviderError, complete_chat, detect_provider
|
||||
|
||||
DETECT_MAX_CHARS = 8000
|
||||
DETECT_TIMEOUT = 20.0
|
||||
DETECT_MAX_TOKENS = 300
|
||||
DETECT_CHUNK_CHARS = 4000
|
||||
DETECT_CHUNK_OVERLAP = 250
|
||||
DETECT_TIMEOUT = 90.0
|
||||
DETECT_MAX_TOKENS = 1024
|
||||
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
||||
NAME_TITLE = re.compile(r"(?:Frau|Herr)\s+([A-ZÄÖÜ][a-zäöüß]{2,})\b")
|
||||
NAME_PREP = re.compile(r"(?:mit|von|bei)\s+([A-ZÄÖÜ][a-zäöüß]{2,})\b")
|
||||
_LETTER = r"A-Za-zÄÖÜäöüß"
|
||||
PLACE_NAME = re.compile(
|
||||
r"\b([A-ZÄÖÜ][a-zäöüß]*(?:burg|stadt|dorf|haven|ingen|heim|bach|feld))\b"
|
||||
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
|
||||
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
|
||||
TYPE_PRIORITY = {"PERSON": 0, "PROJECT": 1, "ORG": 2, "PLACE": 3}
|
||||
|
||||
ERROR_DETECT_UNAVAILABLE = "detect_provider_unavailable"
|
||||
ERROR_DETECT_INCOMPLETE = "detect_incomplete"
|
||||
ERROR_DETECT_INVALID = "detect_invalid_output"
|
||||
ERROR_DETECT_TRUNCATED = "detect_truncated"
|
||||
ERROR_DETECT_CHUNK = "detect_chunk_failed"
|
||||
DETECT_DIALOGUE_FALLBACK_CODES = frozenset(
|
||||
{
|
||||
ERROR_DETECT_INVALID,
|
||||
ERROR_DETECT_TRUNCATED,
|
||||
ERROR_DETECT_INCOMPLETE,
|
||||
ERROR_DETECT_CHUNK,
|
||||
}
|
||||
)
|
||||
STOPWORDS = {
|
||||
"Heute", "Gestern", "Morgen", "Ich", "Wir", "Der", "Die", "Das", "Ein", "Eine",
|
||||
"Und", "Oder", "Nicht", "Kein", "Keine", "Am", "Im", "Zum", "Zur", "Mit",
|
||||
"Nach", "Beim", "Über", "Unter", "Aber", "Denn", "Wenn", "Dann", "Also",
|
||||
"Was", "Wer", "Wie", "Wo", "Warum", "Bitte", "Danke", "Hallo", "Space",
|
||||
"Journal", "Dialog", "Kontext", "Writing", "Profile", "Tagebucheintrag",
|
||||
"Relativ", "Allerdings", "Zunächst", "Danach", "Deshalb", "Trotzdem",
|
||||
"Außerdem", "Schließlich", "Während", "Bevor", "Seitdem",
|
||||
USER_DETECT_MESSAGES = {
|
||||
ERROR_DETECT_UNAVAILABLE: (
|
||||
"Persönliche Angaben können gerade nicht geschützt werden. "
|
||||
"Es wurde kein Impuls erzeugt."
|
||||
),
|
||||
ERROR_DETECT_INCOMPLETE: (
|
||||
"Persönliche Angaben konnten nicht vollständig geprüft werden. "
|
||||
"Es wurde kein Impuls erzeugt."
|
||||
),
|
||||
ERROR_DETECT_INVALID: (
|
||||
"Persönliche Angaben konnten nicht sicher zugeordnet werden. "
|
||||
"Es wurde kein Impuls erzeugt."
|
||||
),
|
||||
ERROR_DETECT_TRUNCATED: (
|
||||
"Die Prüfung persönlicher Angaben wurde abgebrochen. "
|
||||
"Es wurde kein Impuls erzeugt."
|
||||
),
|
||||
ERROR_DETECT_CHUNK: (
|
||||
"Die Prüfung persönlicher Angaben ist fehlgeschlagen. "
|
||||
"Es wurde kein Impuls erzeugt."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _detect_prompt() -> dict:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT * FROM ai_prompts WHERE slug = ? AND active = 1",
|
||||
("mvp.entity_detect",),
|
||||
).fetchone()
|
||||
)
|
||||
if not row or not (row.get("template") or "").strip():
|
||||
raise ProviderError(
|
||||
"detect_prompt_missing",
|
||||
"Prompt mvp.entity_detect fehlt in der Konfiguration.",
|
||||
)
|
||||
return row
|
||||
def user_detect_message(code: str, fallback: str = "") -> str:
|
||||
return USER_DETECT_MESSAGES.get(code) or fallback or "Persönliche Angaben konnten nicht geprüft werden."
|
||||
|
||||
|
||||
def _pattern_entities(text: str, known_labels: set[str]) -> list[dict]:
|
||||
found: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
source = text or ""
|
||||
candidates: list[tuple[str, str]] = []
|
||||
for match in NAME_TITLE.finditer(source):
|
||||
candidates.append((match.group(1).strip(), "PERSON:01"))
|
||||
for match in NAME_PREP.finditer(source):
|
||||
name = match.group(1).strip()
|
||||
if is_given_name_candidate(name):
|
||||
candidates.append((name, "PERSON:01"))
|
||||
for match in PLACE_NAME.finditer(source):
|
||||
candidates.append((match.group(1).strip(), "PLACE:CITY"))
|
||||
for raw, token in candidates:
|
||||
label = normalize_label(raw) or raw
|
||||
if label in STOPWORDS or label in known_labels or label in seen:
|
||||
continue
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
seen.add(label)
|
||||
found.append({"text": label, "token": token})
|
||||
return found
|
||||
_LETTER = r"A-Za-zÄÖÜäöüß"
|
||||
|
||||
|
||||
def _fake_entities(text: str, known_labels: set[str]) -> list[dict]:
|
||||
return _pattern_entities(text, known_labels)
|
||||
class DetectError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 503, diagnostics: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
self.diagnostics = diagnostics or {}
|
||||
|
||||
|
||||
def _merge_proposals(*groups: list[dict]) -> list[dict]:
|
||||
merged: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
for group in groups:
|
||||
for item in group:
|
||||
label = (item.get("text") or "").strip()
|
||||
if not label or label in seen:
|
||||
continue
|
||||
seen.add(label)
|
||||
merged.append(item)
|
||||
return merged
|
||||
@dataclass(frozen=True)
|
||||
class DetectedSpan:
|
||||
start: int
|
||||
end: int
|
||||
text: str
|
||||
entity_type: str
|
||||
chunk_index: int
|
||||
|
||||
|
||||
def _parse_entities(raw: str) -> list[dict]:
|
||||
text = (raw or "").strip()
|
||||
if text.startswith("```"):
|
||||
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL)
|
||||
match = JSON_BLOCK.search(text)
|
||||
if not match:
|
||||
return []
|
||||
try:
|
||||
data = json.loads(match.group(0))
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
items = data.get("entities") if isinstance(data, dict) else data
|
||||
if not isinstance(items, list):
|
||||
return []
|
||||
result = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
label = (item.get("text") or item.get("label") or "").strip()
|
||||
token = (item.get("token") or item.get("placeholder") or "").strip()
|
||||
if label:
|
||||
result.append({"text": label, "token": token})
|
||||
return result
|
||||
@dataclass
|
||||
class DetectionStats:
|
||||
detect_provider: str | None = None
|
||||
detect_model: str | None = None
|
||||
detect_note: str | None = None
|
||||
source_chars: int = 0
|
||||
chunk_count: int = 0
|
||||
chunks_ok: int = 0
|
||||
full_detection_coverage: bool = False
|
||||
entity_counts: dict[str, int] = field(default_factory=dict)
|
||||
confirmed_registry_hits: int = 0
|
||||
request_local_hits: int = 0
|
||||
detect_calls: int = 0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
cost: float = 0.0
|
||||
detect_ms: int = 0
|
||||
abort_reason: str | None = None
|
||||
|
||||
def public(self) -> dict[str, Any]:
|
||||
payload = {
|
||||
"detect_provider": self.detect_provider,
|
||||
"detect_model": self.detect_model,
|
||||
"detect_note": self.detect_note,
|
||||
"source_chars": self.source_chars,
|
||||
"chunk_count": self.chunk_count,
|
||||
"chunks_ok": self.chunks_ok,
|
||||
"full_detection_coverage": self.full_detection_coverage,
|
||||
"entity_counts": dict(self.entity_counts),
|
||||
"confirmed_registry_hits": self.confirmed_registry_hits,
|
||||
"request_local_hits": self.request_local_hits,
|
||||
"detect_calls": self.detect_calls,
|
||||
"detect_prompt_tokens": self.prompt_tokens,
|
||||
"detect_completion_tokens": self.completion_tokens,
|
||||
"detect_total_tokens": self.total_tokens,
|
||||
"detect_cost": self.cost,
|
||||
"detect_ms": self.detect_ms,
|
||||
}
|
||||
if self.abort_reason:
|
||||
payload["abort_reason"] = self.abort_reason
|
||||
return payload
|
||||
|
||||
|
||||
def _label_in_text(label: str, text: str) -> bool:
|
||||
if not (label or "").strip() or not (text or "").strip():
|
||||
return False
|
||||
return bool(
|
||||
re.search(
|
||||
rf"(?<![{_LETTER}]){re.escape(label.strip())}(?![{_LETTER}])",
|
||||
text,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
)
|
||||
@dataclass
|
||||
class DetectionOutcome:
|
||||
mappings: list[dict]
|
||||
stats: DetectionStats
|
||||
local_identities: list[dict]
|
||||
|
||||
|
||||
def _acceptable_proposal(item: dict, excerpt: str) -> bool:
|
||||
label = (item.get("text") or "").strip()
|
||||
if not label or not _label_in_text(label, excerpt) or not is_maskable_label(label):
|
||||
return False
|
||||
token = (item.get("token") or "").upper()
|
||||
if token.startswith("PLACE:") or token.startswith("ORG:"):
|
||||
return True
|
||||
return is_given_name_candidate(label)
|
||||
_injected_spans: list[dict] | None = None
|
||||
_injected_fail: DetectError | None = None
|
||||
_injected_truncated: bool = False
|
||||
|
||||
|
||||
def install_test_spans(spans: list[dict] | None) -> None:
|
||||
"""Test-only span injection. Production never calls this."""
|
||||
global _injected_spans
|
||||
_injected_spans = None if spans is None else [dict(item) for item in spans]
|
||||
|
||||
|
||||
def install_test_detect_failure(error: DetectError | None) -> None:
|
||||
global _injected_fail
|
||||
_injected_fail = error
|
||||
|
||||
|
||||
def install_test_truncated(flag: bool = True) -> None:
|
||||
global _injected_truncated
|
||||
_injected_truncated = bool(flag)
|
||||
|
||||
|
||||
def reset_detect_test_hooks() -> None:
|
||||
global _injected_spans, _injected_fail, _injected_truncated
|
||||
_injected_spans = None
|
||||
_injected_fail = None
|
||||
_injected_truncated = False
|
||||
|
||||
|
||||
def uses_llm_detect(config) -> bool:
|
||||
@@ -153,64 +185,508 @@ def uses_llm_detect(config) -> bool:
|
||||
return allows_remote_plaintext_detect()
|
||||
|
||||
|
||||
def _llm_entities(config, excerpt: str, known_labels: set[str]) -> list[dict]:
|
||||
known = ", ".join(sorted(known_labels)) or "(keine)"
|
||||
def _detect_prompt() -> dict:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT * FROM ai_prompts WHERE slug = ? AND active = 1",
|
||||
("mvp.entity_detect",),
|
||||
).fetchone()
|
||||
)
|
||||
if not row or not (row.get("template") or "").strip():
|
||||
raise DetectError(
|
||||
"detect_prompt_missing",
|
||||
"Prompt mvp.entity_detect fehlt in der Konfiguration.",
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def split_detect_chunks(text: str, *, chunk_chars: int | None = None, overlap: int | None = None) -> list[tuple[int, str]]:
|
||||
source = text or ""
|
||||
if not source:
|
||||
return [(0, "")]
|
||||
size = max(32, int(DETECT_CHUNK_CHARS if chunk_chars is None else chunk_chars))
|
||||
overlap_n = DETECT_CHUNK_OVERLAP if overlap is None else overlap
|
||||
overlap_n = max(0, min(int(overlap_n), size // 2))
|
||||
if len(source) <= size:
|
||||
return [(0, source)]
|
||||
chunks: list[tuple[int, str]] = []
|
||||
start = 0
|
||||
n = len(source)
|
||||
while start < n:
|
||||
end = min(n, start + size)
|
||||
if end < n:
|
||||
window = source[start:end]
|
||||
cut = max(window.rfind("\n"), window.rfind(" "))
|
||||
if cut >= size // 3:
|
||||
end = start + cut + 1
|
||||
chunks.append((start, source[start:end]))
|
||||
if end >= n:
|
||||
break
|
||||
nxt = end - overlap_n
|
||||
if nxt <= start:
|
||||
nxt = end
|
||||
start = nxt
|
||||
return chunks
|
||||
|
||||
|
||||
def _span_dict(text: str, label: str, entity_type: str, *, from_index: int = 0) -> dict | None:
|
||||
start = text.find(label, from_index)
|
||||
if start < 0:
|
||||
return None
|
||||
return {
|
||||
"start": start,
|
||||
"end": start + len(label),
|
||||
"text": label,
|
||||
"entity_type": entity_type,
|
||||
}
|
||||
|
||||
|
||||
def _contract_fake_spans(text: str) -> list[dict]:
|
||||
"""Deterministic contract fixture. Not a semantic quality claim."""
|
||||
source = text or ""
|
||||
found: list[dict] = []
|
||||
seen: set[tuple[int, int, str]] = set()
|
||||
|
||||
def add(label: str, entity_type: str, *, require: str | None = None) -> None:
|
||||
if require and require not in source:
|
||||
return
|
||||
start = 0
|
||||
while True:
|
||||
item = _span_dict(source, label, entity_type, from_index=start)
|
||||
if not item:
|
||||
return
|
||||
key = (item["start"], item["end"], entity_type)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
found.append(item)
|
||||
start = item["end"]
|
||||
|
||||
if re.search(r"Sushi kam", source):
|
||||
add("Sushi", "PERSON")
|
||||
if re.search(r"(?i)Frau\s+Sushi", source):
|
||||
add("Sushi", "PERSON")
|
||||
if re.search(r"(?i)Projekt\s+Aurora", source):
|
||||
add("Aurora", "PROJECT")
|
||||
if re.search(r"(?i)(Organisation|Firma|bei)\s+Nordwerk", source):
|
||||
add("Nordwerk", "ORG")
|
||||
if re.search(r"(?i)(in|nach|aus)\s+Hamburg", source):
|
||||
add("Hamburg", "PLACE")
|
||||
for name in ("Anna", "Clarissa", "Hanna", "Maren"):
|
||||
if re.search(rf"(?<![{_LETTER}]){name}(?![{_LETTER}])", source):
|
||||
add(name, "PERSON")
|
||||
return found
|
||||
|
||||
|
||||
def _parse_detect_json(raw: str) -> dict:
|
||||
text = (raw or "").strip()
|
||||
if text.startswith("```"):
|
||||
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL).strip()
|
||||
match = JSON_BLOCK.search(text)
|
||||
if not match:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.")
|
||||
try:
|
||||
data = json.loads(match.group(0))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort muss ein Objekt mit entities sein.")
|
||||
extra = set(data.keys()) - ALLOWED_ROOT_FIELDS
|
||||
if extra:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort enthält unerwartete Felder.")
|
||||
items = data.get("entities")
|
||||
if not isinstance(items, list):
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-entities muss eine Liste sein.")
|
||||
return data
|
||||
|
||||
|
||||
def _label_occurrences(chunk_text: str, label: str) -> list[tuple[int, int]]:
|
||||
if not label:
|
||||
return []
|
||||
exact = list(
|
||||
re.finditer(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", chunk_text or "")
|
||||
)
|
||||
if exact:
|
||||
return [(match.start(), match.end()) for match in exact]
|
||||
found = list(
|
||||
re.finditer(
|
||||
rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
|
||||
chunk_text or "",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
)
|
||||
return [(match.start(), match.end()) for match in found]
|
||||
|
||||
|
||||
def _ground_span(chunk_text: str, start: int, end: int, text: str) -> tuple[int, int, str] | None:
|
||||
"""Trust the reported word only if it exists in this chunk. Offsets are a hint."""
|
||||
source = chunk_text or ""
|
||||
label = (text or "").strip()
|
||||
if not label:
|
||||
return None
|
||||
n = len(source)
|
||||
windows: list[tuple[int, int]] = []
|
||||
if 0 <= start < end <= n:
|
||||
windows.append((start, end))
|
||||
if 0 <= start < end + 1 <= n and (start, end + 1) not in windows:
|
||||
windows.append((start, end + 1))
|
||||
for left, right in windows:
|
||||
sliced = source[left:right]
|
||||
if sliced == text:
|
||||
return left, right, sliced
|
||||
if sliced.strip() == label:
|
||||
inner = left + (len(sliced) - len(sliced.lstrip()))
|
||||
if source[inner : inner + len(label)] == label:
|
||||
return inner, inner + len(label), label
|
||||
hits = _label_occurrences(source, label)
|
||||
if not hits:
|
||||
return None
|
||||
best = min(hits, key=lambda item: (abs(item[0] - start), item[0]))
|
||||
return best[0], best[1], source[best[0] : best[1]]
|
||||
|
||||
|
||||
def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
|
||||
if not isinstance(item, dict):
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity muss ein Objekt sein.")
|
||||
extra = set(item.keys()) - ALLOWED_ENTITY_FIELDS
|
||||
if extra:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity enthält unerwartete Felder.")
|
||||
missing = ALLOWED_ENTITY_FIELDS - set(item.keys())
|
||||
if missing:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity ist unvollständig.")
|
||||
try:
|
||||
start = int(item["start"])
|
||||
except (TypeError, ValueError):
|
||||
start = 0
|
||||
try:
|
||||
end = int(item["end"])
|
||||
except (TypeError, ValueError):
|
||||
end = 0
|
||||
text = item.get("text")
|
||||
if not isinstance(text, str):
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-text fehlt.")
|
||||
if not text.strip():
|
||||
return None
|
||||
if "token" in item or "placeholder" in item:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect darf keine Tokens festlegen.")
|
||||
kind = normalize_entity_type(item.get("entity_type"), default="")
|
||||
if not kind:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-entity_type ist nicht erlaubt.")
|
||||
grounded = _ground_span(chunk_text, start, end, text)
|
||||
if not grounded:
|
||||
return None
|
||||
left, right, local_text = grounded
|
||||
return DetectedSpan(start=left, end=right, text=local_text, entity_type=kind, chunk_index=chunk_index)
|
||||
|
||||
|
||||
def resolve_overlaps(spans: list[DetectedSpan]) -> list[DetectedSpan]:
|
||||
"""Longest span wins, then leftmost, then type PERSON > PROJECT > ORG > PLACE."""
|
||||
ordered = sorted(
|
||||
spans,
|
||||
key=lambda item: (
|
||||
-(item.end - item.start),
|
||||
item.start,
|
||||
TYPE_PRIORITY.get(item.entity_type, 9),
|
||||
item.chunk_index,
|
||||
),
|
||||
)
|
||||
kept: list[DetectedSpan] = []
|
||||
occupied: list[tuple[int, int]] = []
|
||||
for span in ordered:
|
||||
if any(span.start < end and span.end > start for start, end in occupied):
|
||||
continue
|
||||
kept.append(span)
|
||||
occupied.append((span.start, span.end))
|
||||
return sorted(kept, key=lambda item: (item.start, item.end))
|
||||
|
||||
|
||||
def _add_usage(stats: DetectionStats, usage: dict | None) -> None:
|
||||
data = usage or {}
|
||||
stats.prompt_tokens += int(data.get("prompt_tokens") or 0)
|
||||
stats.completion_tokens += int(data.get("completion_tokens") or 0)
|
||||
stats.total_tokens += int(data.get("total_tokens") or 0)
|
||||
try:
|
||||
stats.cost += float(data.get("cost") or 0)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _llm_chunk(config, excerpt: str) -> ChatResult:
|
||||
prompt = resolve_template(
|
||||
_detect_prompt()["template"],
|
||||
{"source_text": excerpt, "known_labels": known},
|
||||
{"source_text": excerpt, "known_labels": ""},
|
||||
)
|
||||
result = complete_chat(
|
||||
return complete_chat(
|
||||
config,
|
||||
[{"role": "user", "content": prompt}],
|
||||
timeout=DETECT_TIMEOUT,
|
||||
max_tokens=DETECT_MAX_TOKENS,
|
||||
disable_context_compression=True,
|
||||
)
|
||||
return _parse_entities(result.content)
|
||||
|
||||
|
||||
def detect_and_remember(profile_id: str | None, source_text: str) -> tuple[list[dict], str | None, str | None]:
|
||||
if profile_id:
|
||||
purge_unmaskable(profile_id)
|
||||
coalesce_mappings(profile_id)
|
||||
mappings = list_mappings(profile_id) if profile_id else []
|
||||
def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int) -> list[DetectedSpan]:
|
||||
if (result.finish_reason or "").lower() in {"length", "max_tokens"}:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_TRUNCATED,
|
||||
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
|
||||
)
|
||||
data = _parse_detect_json(result.content or "")
|
||||
spans: list[DetectedSpan] = []
|
||||
for item in data.get("entities") or []:
|
||||
span = validate_detected_entity(item, chunk_text, chunk_index)
|
||||
if span is not None:
|
||||
spans.append(span)
|
||||
return spans
|
||||
|
||||
|
||||
def _to_global(span: DetectedSpan, offset: int) -> DetectedSpan:
|
||||
return DetectedSpan(
|
||||
start=span.start + offset,
|
||||
end=span.end + offset,
|
||||
text=span.text,
|
||||
entity_type=span.entity_type,
|
||||
chunk_index=span.chunk_index,
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_spans(spans: list[DetectedSpan]) -> list[DetectedSpan]:
|
||||
unique: dict[tuple[int, int, str, str], DetectedSpan] = {}
|
||||
for span in spans:
|
||||
key = (span.start, span.end, span.text, span.entity_type)
|
||||
unique[key] = span
|
||||
return resolve_overlaps(list(unique.values()))
|
||||
|
||||
|
||||
def _next_token(used: set[str], entity_type: str) -> str:
|
||||
prefix = f"{entity_type}:"
|
||||
n = 1
|
||||
while True:
|
||||
token = f"{prefix}{n:02d}"
|
||||
if token not in used:
|
||||
used.add(token)
|
||||
return token
|
||||
n += 1
|
||||
|
||||
|
||||
def _assign_request_tokens(
|
||||
spans: list[DetectedSpan],
|
||||
confirmed: list[dict],
|
||||
) -> tuple[list[dict], int, int]:
|
||||
used_tokens = {(item.get("token") or "").upper() for item in confirmed if item.get("token")}
|
||||
by_label: dict[str, dict] = {}
|
||||
request_local = 0
|
||||
confirmed_hits = 0
|
||||
mappings: list[dict] = []
|
||||
|
||||
def confirmed_for(label: str) -> dict | None:
|
||||
needle = (label or "").strip().casefold()
|
||||
for item in confirmed:
|
||||
for candidate in confirmed_match_labels(item):
|
||||
if candidate.casefold() == needle:
|
||||
return item
|
||||
return None
|
||||
|
||||
for span in spans:
|
||||
label = span.text
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
found = confirmed_for(label)
|
||||
if found:
|
||||
token = found["token"]
|
||||
canonical = found["canonical_label"]
|
||||
source = "confirmed_registry"
|
||||
confirmed_hits += 1
|
||||
entity_type = found["entity_type"]
|
||||
demask = canonical
|
||||
else:
|
||||
key = f"{span.entity_type}:{label.casefold()}"
|
||||
if key not in by_label:
|
||||
by_label[key] = {
|
||||
"token": _next_token(used_tokens, span.entity_type),
|
||||
"entity_type": span.entity_type,
|
||||
}
|
||||
request_local += 1
|
||||
token = by_label[key]["token"]
|
||||
entity_type = span.entity_type
|
||||
source = "request_local"
|
||||
demask = label
|
||||
mappings.append(
|
||||
{
|
||||
"token": token,
|
||||
"local_label": label,
|
||||
"canonical_label": demask,
|
||||
"demask_label": demask,
|
||||
"entity_type": entity_type,
|
||||
"source": source,
|
||||
"start": span.start,
|
||||
"end": span.end,
|
||||
}
|
||||
)
|
||||
return mappings, request_local, confirmed_hits
|
||||
|
||||
|
||||
def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str | None) -> tuple[list[dict], int]:
|
||||
if not profile_id:
|
||||
return mappings, None, None
|
||||
known_labels = {
|
||||
normalize_label(item.get("local_label") or "") or (item.get("local_label") or "").strip()
|
||||
for item in mappings
|
||||
if item.get("local_label")
|
||||
}
|
||||
excerpt = (source_text or "")[:DETECT_MAX_CHARS]
|
||||
proposals = _pattern_entities(excerpt, known_labels)
|
||||
detect_name = "pattern"
|
||||
detect_note = "pattern"
|
||||
return mappings, 0
|
||||
already = {(item.get("local_label") or "").casefold() for item in mappings}
|
||||
extra: list[dict] = []
|
||||
hits = 0
|
||||
for row in masking_rows_from_confirmed(profile_id):
|
||||
label = row.get("local_label") or ""
|
||||
if not label or label.casefold() in already:
|
||||
continue
|
||||
if not re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text or "", re.IGNORECASE):
|
||||
continue
|
||||
extra.append(row)
|
||||
already.add(label.casefold())
|
||||
hits += 1
|
||||
return mappings + extra, hits
|
||||
|
||||
|
||||
def detect_personal_egress(profile_id: str | None, source_text: str) -> DetectionOutcome:
|
||||
"""Full semantic detection of the personal generate-egress. Fail closed if incomplete."""
|
||||
started = time.perf_counter()
|
||||
stats = DetectionStats(source_chars=len(source_text or ""))
|
||||
if _injected_fail is not None:
|
||||
stats.abort_reason = _injected_fail.code
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
raise DetectError(
|
||||
_injected_fail.code,
|
||||
_injected_fail.message,
|
||||
_injected_fail.status_code,
|
||||
{**(_injected_fail.diagnostics or {}), **stats.public()},
|
||||
)
|
||||
config = detect_provider()
|
||||
if config and config.mode == "fake":
|
||||
detect_name = config.name
|
||||
detect_note = "fake"
|
||||
elif uses_llm_detect(config):
|
||||
try:
|
||||
proposals = _merge_proposals(
|
||||
proposals,
|
||||
[item for item in _llm_entities(config, excerpt, known_labels) if _acceptable_proposal(item, excerpt)],
|
||||
)
|
||||
detect_name = config.name
|
||||
detect_note = "local_llm" if config.local else "remote_llm"
|
||||
except PlaceholderError as exc:
|
||||
raise ProviderError(exc.code, exc.message) from exc
|
||||
elif config and config.mode == "http" and not config.local:
|
||||
detect_note = "pattern; remote_detect_blocked_production"
|
||||
for item in proposals:
|
||||
label = (item.get("text") or "").strip()
|
||||
core = normalize_label(label)
|
||||
if not label or not core or core in known_labels:
|
||||
continue
|
||||
if not _acceptable_proposal(item, excerpt):
|
||||
continue
|
||||
try:
|
||||
remember_mapping(profile_id, label, item.get("token"))
|
||||
except ValueError:
|
||||
continue
|
||||
known_labels.add(core)
|
||||
coalesce_mappings(profile_id)
|
||||
return list_mappings(profile_id), detect_name, detect_note
|
||||
if not config:
|
||||
stats.abort_reason = ERROR_DETECT_UNAVAILABLE
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
raise DetectError(
|
||||
ERROR_DETECT_UNAVAILABLE,
|
||||
"Semantische Detection ist nicht konfiguriert. Generate wird nicht freigegeben.",
|
||||
diagnostics=stats.public(),
|
||||
)
|
||||
if config.mode == "http" and not config.local and not allows_remote_plaintext_detect():
|
||||
stats.abort_reason = "remote_detect_blocked_production"
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
raise DetectError(
|
||||
ERROR_DETECT_UNAVAILABLE,
|
||||
"Externes Klartext-Detect ist im Produktivmodus nicht zulässig, und ein lokales Detect-Modell fehlt.",
|
||||
diagnostics=stats.public(),
|
||||
)
|
||||
stats.detect_provider = config.name
|
||||
stats.detect_model = config.model
|
||||
chunks = split_detect_chunks(source_text or "")
|
||||
stats.chunk_count = len(chunks)
|
||||
collected: list[DetectedSpan] = []
|
||||
try:
|
||||
for index, (offset, chunk_text) in enumerate(chunks):
|
||||
if _injected_truncated:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_TRUNCATED,
|
||||
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
|
||||
)
|
||||
if _injected_spans is not None:
|
||||
raw_items = [
|
||||
item
|
||||
for item in _injected_spans
|
||||
if int(item.get("start") or 0) >= offset
|
||||
and int(item.get("end") or 0) <= offset + len(chunk_text)
|
||||
]
|
||||
local_items = []
|
||||
for item in raw_items:
|
||||
local = dict(item)
|
||||
local["start"] = int(item["start"]) - offset
|
||||
local["end"] = int(item["end"]) - offset
|
||||
local_items.append(local)
|
||||
result = ChatResult(
|
||||
content=json.dumps({"entities": local_items}),
|
||||
model=config.model,
|
||||
usage={},
|
||||
finish_reason="stop",
|
||||
)
|
||||
stats.detect_note = "injected"
|
||||
elif config.mode == "fake":
|
||||
fake_items = _contract_fake_spans(chunk_text)
|
||||
result = ChatResult(
|
||||
content=json.dumps({"entities": fake_items}),
|
||||
model=config.model,
|
||||
usage={},
|
||||
finish_reason="stop",
|
||||
)
|
||||
stats.detect_note = "fake"
|
||||
elif uses_llm_detect(config):
|
||||
try:
|
||||
result = _llm_chunk(config, chunk_text)
|
||||
except PlaceholderError as exc:
|
||||
raise DetectError(exc.code, exc.message) from exc
|
||||
except ProviderError as exc:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_CHUNK,
|
||||
"Ein Detect-Chunk ist fehlgeschlagen. Generate wird nicht freigegeben.",
|
||||
exc.status_code,
|
||||
) from exc
|
||||
stats.detect_note = "local_llm" if config.local else "remote_llm"
|
||||
stats.detect_model = result.model or stats.detect_model
|
||||
else:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_UNAVAILABLE,
|
||||
"Semantische Detection ist nicht verfügbar. Generate wird nicht freigegeben.",
|
||||
)
|
||||
stats.detect_calls += 1
|
||||
_add_usage(stats, result.usage)
|
||||
spans = _entities_from_result(result, chunk_text, index)
|
||||
collected.extend(_to_global(span, offset) for span in spans)
|
||||
stats.chunks_ok += 1
|
||||
except DetectError as exc:
|
||||
stats.abort_reason = exc.code
|
||||
stats.full_detection_coverage = False
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
raise DetectError(
|
||||
exc.code,
|
||||
exc.message,
|
||||
exc.status_code,
|
||||
{**(exc.diagnostics or {}), **stats.public()},
|
||||
) from exc
|
||||
if stats.chunks_ok != stats.chunk_count:
|
||||
stats.abort_reason = ERROR_DETECT_INCOMPLETE
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
raise DetectError(
|
||||
ERROR_DETECT_INCOMPLETE,
|
||||
"Detection hat nicht alle Chunks geprüft. Generate wird nicht freigegeben.",
|
||||
diagnostics=stats.public(),
|
||||
)
|
||||
merged = _dedupe_spans(collected)
|
||||
confirmed = list_confirmed_identities(profile_id) if profile_id else []
|
||||
mappings, request_local, confirmed_from_spans = _assign_request_tokens(merged, confirmed)
|
||||
mappings, extra_confirmed = _merge_confirmed_safety_net(source_text or "", mappings, profile_id)
|
||||
stats.request_local_hits = request_local
|
||||
stats.confirmed_registry_hits = confirmed_from_spans + extra_confirmed
|
||||
counts: dict[str, int] = {kind: 0 for kind in ENTITY_TYPES}
|
||||
for item in mappings:
|
||||
kind = (item.get("entity_type") or "PERSON").upper()
|
||||
counts[kind] = counts.get(kind, 0) + 1
|
||||
stats.entity_counts = {key: value for key, value in counts.items() if value}
|
||||
stats.full_detection_coverage = True
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
if profile_id:
|
||||
seen_proposals: set[tuple[str, str]] = set()
|
||||
for item in mappings:
|
||||
if item.get("source") != "request_local":
|
||||
continue
|
||||
key = ((item.get("local_label") or ""), (item.get("entity_type") or ""))
|
||||
if key in seen_proposals:
|
||||
continue
|
||||
seen_proposals.add(key)
|
||||
record_review_proposal(profile_id, item.get("local_label") or "", item.get("entity_type") or "")
|
||||
local_identities = [
|
||||
{
|
||||
"local_label": item.get("local_label"),
|
||||
"token": item.get("token"),
|
||||
"entity_type": item.get("entity_type"),
|
||||
"demask_label": item.get("demask_label") or item.get("local_label"),
|
||||
"source": item.get("source"),
|
||||
}
|
||||
for item in mappings
|
||||
]
|
||||
return DetectionOutcome(mappings=mappings, stats=stats, local_identities=local_identities)
|
||||
Reference in new issue
Block a user