693 lines
25 KiB
Python
693 lines
25 KiB
Python
"""Semantic entity detection for the privacy gateway.
|
|
|
|
Detect may see plaintext. Generate must not. Detector output is untrusted:
|
|
spans and types only, never tokens, never a durable registry write.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
import placeholder_mvp # noqa: F401
|
|
from db import get_db, row_to_dict
|
|
from env_loader import allows_remote_plaintext_detect
|
|
from identity_store import (
|
|
ENTITY_TYPES,
|
|
confirmed_match_labels,
|
|
is_maskable_label,
|
|
list_confirmed_identities,
|
|
masking_rows_from_confirmed,
|
|
normalize_entity_type,
|
|
record_review_proposal,
|
|
)
|
|
from placeholders import PlaceholderError, resolve_template
|
|
from providers import ChatResult, ProviderError, complete_chat, detect_provider
|
|
|
|
DETECT_CHUNK_CHARS = 4000
|
|
DETECT_CHUNK_OVERLAP = 250
|
|
DETECT_TIMEOUT = 90.0
|
|
DETECT_MAX_TOKENS = 1024
|
|
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
|
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
|
|
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
|
|
TYPE_PRIORITY = {"PERSON": 0, "PROJECT": 1, "ORG": 2, "PLACE": 3}
|
|
|
|
ERROR_DETECT_UNAVAILABLE = "detect_provider_unavailable"
|
|
ERROR_DETECT_INCOMPLETE = "detect_incomplete"
|
|
ERROR_DETECT_INVALID = "detect_invalid_output"
|
|
ERROR_DETECT_TRUNCATED = "detect_truncated"
|
|
ERROR_DETECT_CHUNK = "detect_chunk_failed"
|
|
DETECT_DIALOGUE_FALLBACK_CODES = frozenset(
|
|
{
|
|
ERROR_DETECT_INVALID,
|
|
ERROR_DETECT_TRUNCATED,
|
|
ERROR_DETECT_INCOMPLETE,
|
|
ERROR_DETECT_CHUNK,
|
|
}
|
|
)
|
|
USER_DETECT_MESSAGES = {
|
|
ERROR_DETECT_UNAVAILABLE: (
|
|
"Persönliche Angaben können gerade nicht geschützt werden. "
|
|
"Es wurde kein Impuls erzeugt."
|
|
),
|
|
ERROR_DETECT_INCOMPLETE: (
|
|
"Persönliche Angaben konnten nicht vollständig geprüft werden. "
|
|
"Es wurde kein Impuls erzeugt."
|
|
),
|
|
ERROR_DETECT_INVALID: (
|
|
"Persönliche Angaben konnten nicht sicher zugeordnet werden. "
|
|
"Es wurde kein Impuls erzeugt."
|
|
),
|
|
ERROR_DETECT_TRUNCATED: (
|
|
"Die Prüfung persönlicher Angaben wurde abgebrochen. "
|
|
"Es wurde kein Impuls erzeugt."
|
|
),
|
|
ERROR_DETECT_CHUNK: (
|
|
"Die Prüfung persönlicher Angaben ist fehlgeschlagen. "
|
|
"Es wurde kein Impuls erzeugt."
|
|
),
|
|
}
|
|
|
|
|
|
def user_detect_message(code: str, fallback: str = "") -> str:
|
|
return USER_DETECT_MESSAGES.get(code) or fallback or "Persönliche Angaben konnten nicht geprüft werden."
|
|
|
|
|
|
_LETTER = r"A-Za-zÄÖÜäöüß"
|
|
|
|
|
|
class DetectError(Exception):
|
|
def __init__(self, code: str, message: str, status_code: int = 503, diagnostics: dict | None = None):
|
|
super().__init__(message)
|
|
self.code = code
|
|
self.message = message
|
|
self.status_code = status_code
|
|
self.diagnostics = diagnostics or {}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DetectedSpan:
|
|
start: int
|
|
end: int
|
|
text: str
|
|
entity_type: str
|
|
chunk_index: int
|
|
|
|
|
|
@dataclass
|
|
class DetectionStats:
|
|
detect_provider: str | None = None
|
|
detect_model: str | None = None
|
|
detect_note: str | None = None
|
|
source_chars: int = 0
|
|
chunk_count: int = 0
|
|
chunks_ok: int = 0
|
|
full_detection_coverage: bool = False
|
|
entity_counts: dict[str, int] = field(default_factory=dict)
|
|
confirmed_registry_hits: int = 0
|
|
request_local_hits: int = 0
|
|
detect_calls: int = 0
|
|
prompt_tokens: int = 0
|
|
completion_tokens: int = 0
|
|
total_tokens: int = 0
|
|
cost: float = 0.0
|
|
detect_ms: int = 0
|
|
abort_reason: str | None = None
|
|
|
|
def public(self) -> dict[str, Any]:
|
|
payload = {
|
|
"detect_provider": self.detect_provider,
|
|
"detect_model": self.detect_model,
|
|
"detect_note": self.detect_note,
|
|
"source_chars": self.source_chars,
|
|
"chunk_count": self.chunk_count,
|
|
"chunks_ok": self.chunks_ok,
|
|
"full_detection_coverage": self.full_detection_coverage,
|
|
"entity_counts": dict(self.entity_counts),
|
|
"confirmed_registry_hits": self.confirmed_registry_hits,
|
|
"request_local_hits": self.request_local_hits,
|
|
"detect_calls": self.detect_calls,
|
|
"detect_prompt_tokens": self.prompt_tokens,
|
|
"detect_completion_tokens": self.completion_tokens,
|
|
"detect_total_tokens": self.total_tokens,
|
|
"detect_cost": self.cost,
|
|
"detect_ms": self.detect_ms,
|
|
}
|
|
if self.abort_reason:
|
|
payload["abort_reason"] = self.abort_reason
|
|
return payload
|
|
|
|
|
|
@dataclass
|
|
class DetectionOutcome:
|
|
mappings: list[dict]
|
|
stats: DetectionStats
|
|
local_identities: list[dict]
|
|
|
|
|
|
_injected_spans: list[dict] | None = None
|
|
_injected_fail: DetectError | None = None
|
|
_injected_truncated: bool = False
|
|
|
|
|
|
def install_test_spans(spans: list[dict] | None) -> None:
|
|
"""Test-only span injection. Production never calls this."""
|
|
global _injected_spans
|
|
_injected_spans = None if spans is None else [dict(item) for item in spans]
|
|
|
|
|
|
def install_test_detect_failure(error: DetectError | None) -> None:
|
|
global _injected_fail
|
|
_injected_fail = error
|
|
|
|
|
|
def install_test_truncated(flag: bool = True) -> None:
|
|
global _injected_truncated
|
|
_injected_truncated = bool(flag)
|
|
|
|
|
|
def reset_detect_test_hooks() -> None:
|
|
global _injected_spans, _injected_fail, _injected_truncated
|
|
_injected_spans = None
|
|
_injected_fail = None
|
|
_injected_truncated = False
|
|
|
|
|
|
def uses_llm_detect(config) -> bool:
|
|
"""Local detect stays a provider role. Remote plaintext detect is development/test only."""
|
|
if not config or config.mode != "http":
|
|
return False
|
|
if config.local:
|
|
return True
|
|
return allows_remote_plaintext_detect()
|
|
|
|
|
|
def _detect_prompt() -> dict:
|
|
with get_db() as conn:
|
|
row = row_to_dict(
|
|
conn.execute(
|
|
"SELECT * FROM ai_prompts WHERE slug = ? AND active = 1",
|
|
("mvp.entity_detect",),
|
|
).fetchone()
|
|
)
|
|
if not row or not (row.get("template") or "").strip():
|
|
raise DetectError(
|
|
"detect_prompt_missing",
|
|
"Prompt mvp.entity_detect fehlt in der Konfiguration.",
|
|
)
|
|
return row
|
|
|
|
|
|
def split_detect_chunks(text: str, *, chunk_chars: int | None = None, overlap: int | None = None) -> list[tuple[int, str]]:
|
|
source = text or ""
|
|
if not source:
|
|
return [(0, "")]
|
|
size = max(32, int(DETECT_CHUNK_CHARS if chunk_chars is None else chunk_chars))
|
|
overlap_n = DETECT_CHUNK_OVERLAP if overlap is None else overlap
|
|
overlap_n = max(0, min(int(overlap_n), size // 2))
|
|
if len(source) <= size:
|
|
return [(0, source)]
|
|
chunks: list[tuple[int, str]] = []
|
|
start = 0
|
|
n = len(source)
|
|
while start < n:
|
|
end = min(n, start + size)
|
|
if end < n:
|
|
window = source[start:end]
|
|
cut = max(window.rfind("\n"), window.rfind(" "))
|
|
if cut >= size // 3:
|
|
end = start + cut + 1
|
|
chunks.append((start, source[start:end]))
|
|
if end >= n:
|
|
break
|
|
nxt = end - overlap_n
|
|
if nxt <= start:
|
|
nxt = end
|
|
start = nxt
|
|
return chunks
|
|
|
|
|
|
def _span_dict(text: str, label: str, entity_type: str, *, from_index: int = 0) -> dict | None:
|
|
start = text.find(label, from_index)
|
|
if start < 0:
|
|
return None
|
|
return {
|
|
"start": start,
|
|
"end": start + len(label),
|
|
"text": label,
|
|
"entity_type": entity_type,
|
|
}
|
|
|
|
|
|
def _contract_fake_spans(text: str) -> list[dict]:
|
|
"""Deterministic contract fixture. Not a semantic quality claim."""
|
|
source = text or ""
|
|
found: list[dict] = []
|
|
seen: set[tuple[int, int, str]] = set()
|
|
|
|
def add(label: str, entity_type: str, *, require: str | None = None) -> None:
|
|
if require and require not in source:
|
|
return
|
|
start = 0
|
|
while True:
|
|
item = _span_dict(source, label, entity_type, from_index=start)
|
|
if not item:
|
|
return
|
|
key = (item["start"], item["end"], entity_type)
|
|
if key not in seen:
|
|
seen.add(key)
|
|
found.append(item)
|
|
start = item["end"]
|
|
|
|
if re.search(r"Sushi kam", source):
|
|
add("Sushi", "PERSON")
|
|
if re.search(r"(?i)Frau\s+Sushi", source):
|
|
add("Sushi", "PERSON")
|
|
if re.search(r"(?i)Projekt\s+Aurora", source):
|
|
add("Aurora", "PROJECT")
|
|
if re.search(r"(?i)(Organisation|Firma|bei)\s+Nordwerk", source):
|
|
add("Nordwerk", "ORG")
|
|
if re.search(r"(?i)(in|nach|aus)\s+Hamburg", source):
|
|
add("Hamburg", "PLACE")
|
|
for name in ("Anna", "Clarissa", "Hanna", "Maren"):
|
|
if re.search(rf"(?<![{_LETTER}]){name}(?![{_LETTER}])", source):
|
|
add(name, "PERSON")
|
|
return found
|
|
|
|
|
|
def _parse_detect_json(raw: str) -> dict:
|
|
text = (raw or "").strip()
|
|
if text.startswith("```"):
|
|
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL).strip()
|
|
match = JSON_BLOCK.search(text)
|
|
if not match:
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.")
|
|
try:
|
|
data = json.loads(match.group(0))
|
|
except json.JSONDecodeError as exc:
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.") from exc
|
|
if not isinstance(data, dict):
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort muss ein Objekt mit entities sein.")
|
|
extra = set(data.keys()) - ALLOWED_ROOT_FIELDS
|
|
if extra:
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort enthält unerwartete Felder.")
|
|
items = data.get("entities")
|
|
if not isinstance(items, list):
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-entities muss eine Liste sein.")
|
|
return data
|
|
|
|
|
|
def _label_occurrences(chunk_text: str, label: str) -> list[tuple[int, int]]:
|
|
if not label:
|
|
return []
|
|
exact = list(
|
|
re.finditer(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", chunk_text or "")
|
|
)
|
|
if exact:
|
|
return [(match.start(), match.end()) for match in exact]
|
|
found = list(
|
|
re.finditer(
|
|
rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
|
|
chunk_text or "",
|
|
re.IGNORECASE,
|
|
)
|
|
)
|
|
return [(match.start(), match.end()) for match in found]
|
|
|
|
|
|
def _ground_span(chunk_text: str, start: int, end: int, text: str) -> tuple[int, int, str] | None:
|
|
"""Trust the reported word only if it exists in this chunk. Offsets are a hint."""
|
|
source = chunk_text or ""
|
|
label = (text or "").strip()
|
|
if not label:
|
|
return None
|
|
n = len(source)
|
|
windows: list[tuple[int, int]] = []
|
|
if 0 <= start < end <= n:
|
|
windows.append((start, end))
|
|
if 0 <= start < end + 1 <= n and (start, end + 1) not in windows:
|
|
windows.append((start, end + 1))
|
|
for left, right in windows:
|
|
sliced = source[left:right]
|
|
if sliced == text:
|
|
return left, right, sliced
|
|
if sliced.strip() == label:
|
|
inner = left + (len(sliced) - len(sliced.lstrip()))
|
|
if source[inner : inner + len(label)] == label:
|
|
return inner, inner + len(label), label
|
|
hits = _label_occurrences(source, label)
|
|
if not hits:
|
|
return None
|
|
best = min(hits, key=lambda item: (abs(item[0] - start), item[0]))
|
|
return best[0], best[1], source[best[0] : best[1]]
|
|
|
|
|
|
def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
|
|
if not isinstance(item, dict):
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity muss ein Objekt sein.")
|
|
extra = set(item.keys()) - ALLOWED_ENTITY_FIELDS
|
|
if extra:
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity enthält unerwartete Felder.")
|
|
missing = ALLOWED_ENTITY_FIELDS - set(item.keys())
|
|
if missing:
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity ist unvollständig.")
|
|
try:
|
|
start = int(item["start"])
|
|
except (TypeError, ValueError):
|
|
start = 0
|
|
try:
|
|
end = int(item["end"])
|
|
except (TypeError, ValueError):
|
|
end = 0
|
|
text = item.get("text")
|
|
if not isinstance(text, str):
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-text fehlt.")
|
|
if not text.strip():
|
|
return None
|
|
if "token" in item or "placeholder" in item:
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect darf keine Tokens festlegen.")
|
|
kind = normalize_entity_type(item.get("entity_type"), default="")
|
|
if not kind:
|
|
raise DetectError(ERROR_DETECT_INVALID, "Detect-entity_type ist nicht erlaubt.")
|
|
grounded = _ground_span(chunk_text, start, end, text)
|
|
if not grounded:
|
|
return None
|
|
left, right, local_text = grounded
|
|
return DetectedSpan(start=left, end=right, text=local_text, entity_type=kind, chunk_index=chunk_index)
|
|
|
|
|
|
def resolve_overlaps(spans: list[DetectedSpan]) -> list[DetectedSpan]:
|
|
"""Longest span wins, then leftmost, then type PERSON > PROJECT > ORG > PLACE."""
|
|
ordered = sorted(
|
|
spans,
|
|
key=lambda item: (
|
|
-(item.end - item.start),
|
|
item.start,
|
|
TYPE_PRIORITY.get(item.entity_type, 9),
|
|
item.chunk_index,
|
|
),
|
|
)
|
|
kept: list[DetectedSpan] = []
|
|
occupied: list[tuple[int, int]] = []
|
|
for span in ordered:
|
|
if any(span.start < end and span.end > start for start, end in occupied):
|
|
continue
|
|
kept.append(span)
|
|
occupied.append((span.start, span.end))
|
|
return sorted(kept, key=lambda item: (item.start, item.end))
|
|
|
|
|
|
def _add_usage(stats: DetectionStats, usage: dict | None) -> None:
|
|
data = usage or {}
|
|
stats.prompt_tokens += int(data.get("prompt_tokens") or 0)
|
|
stats.completion_tokens += int(data.get("completion_tokens") or 0)
|
|
stats.total_tokens += int(data.get("total_tokens") or 0)
|
|
try:
|
|
stats.cost += float(data.get("cost") or 0)
|
|
except (TypeError, ValueError):
|
|
pass
|
|
|
|
|
|
def _llm_chunk(config, excerpt: str) -> ChatResult:
|
|
prompt = resolve_template(
|
|
_detect_prompt()["template"],
|
|
{"source_text": excerpt, "known_labels": ""},
|
|
)
|
|
return complete_chat(
|
|
config,
|
|
[{"role": "user", "content": prompt}],
|
|
timeout=DETECT_TIMEOUT,
|
|
max_tokens=DETECT_MAX_TOKENS,
|
|
disable_context_compression=True,
|
|
)
|
|
|
|
|
|
def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int) -> list[DetectedSpan]:
|
|
if (result.finish_reason or "").lower() in {"length", "max_tokens"}:
|
|
raise DetectError(
|
|
ERROR_DETECT_TRUNCATED,
|
|
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
|
|
)
|
|
data = _parse_detect_json(result.content or "")
|
|
spans: list[DetectedSpan] = []
|
|
for item in data.get("entities") or []:
|
|
span = validate_detected_entity(item, chunk_text, chunk_index)
|
|
if span is not None:
|
|
spans.append(span)
|
|
return spans
|
|
|
|
|
|
def _to_global(span: DetectedSpan, offset: int) -> DetectedSpan:
|
|
return DetectedSpan(
|
|
start=span.start + offset,
|
|
end=span.end + offset,
|
|
text=span.text,
|
|
entity_type=span.entity_type,
|
|
chunk_index=span.chunk_index,
|
|
)
|
|
|
|
|
|
def _dedupe_spans(spans: list[DetectedSpan]) -> list[DetectedSpan]:
|
|
unique: dict[tuple[int, int, str, str], DetectedSpan] = {}
|
|
for span in spans:
|
|
key = (span.start, span.end, span.text, span.entity_type)
|
|
unique[key] = span
|
|
return resolve_overlaps(list(unique.values()))
|
|
|
|
|
|
def _next_token(used: set[str], entity_type: str) -> str:
|
|
prefix = f"{entity_type}:"
|
|
n = 1
|
|
while True:
|
|
token = f"{prefix}{n:02d}"
|
|
if token not in used:
|
|
used.add(token)
|
|
return token
|
|
n += 1
|
|
|
|
|
|
def _assign_request_tokens(
|
|
spans: list[DetectedSpan],
|
|
confirmed: list[dict],
|
|
) -> tuple[list[dict], int, int]:
|
|
used_tokens = {(item.get("token") or "").upper() for item in confirmed if item.get("token")}
|
|
by_label: dict[str, dict] = {}
|
|
request_local = 0
|
|
confirmed_hits = 0
|
|
mappings: list[dict] = []
|
|
|
|
def confirmed_for(label: str) -> dict | None:
|
|
needle = (label or "").strip().casefold()
|
|
for item in confirmed:
|
|
for candidate in confirmed_match_labels(item):
|
|
if candidate.casefold() == needle:
|
|
return item
|
|
return None
|
|
|
|
for span in spans:
|
|
label = span.text
|
|
if not is_maskable_label(label):
|
|
continue
|
|
found = confirmed_for(label)
|
|
if found:
|
|
token = found["token"]
|
|
canonical = found["canonical_label"]
|
|
source = "confirmed_registry"
|
|
confirmed_hits += 1
|
|
entity_type = found["entity_type"]
|
|
demask = canonical
|
|
else:
|
|
key = f"{span.entity_type}:{label.casefold()}"
|
|
if key not in by_label:
|
|
by_label[key] = {
|
|
"token": _next_token(used_tokens, span.entity_type),
|
|
"entity_type": span.entity_type,
|
|
}
|
|
request_local += 1
|
|
token = by_label[key]["token"]
|
|
entity_type = span.entity_type
|
|
source = "request_local"
|
|
demask = label
|
|
mappings.append(
|
|
{
|
|
"token": token,
|
|
"local_label": label,
|
|
"canonical_label": demask,
|
|
"demask_label": demask,
|
|
"entity_type": entity_type,
|
|
"source": source,
|
|
"start": span.start,
|
|
"end": span.end,
|
|
}
|
|
)
|
|
return mappings, request_local, confirmed_hits
|
|
|
|
|
|
def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str | None) -> tuple[list[dict], int]:
|
|
if not profile_id:
|
|
return mappings, 0
|
|
already = {(item.get("local_label") or "").casefold() for item in mappings}
|
|
extra: list[dict] = []
|
|
hits = 0
|
|
for row in masking_rows_from_confirmed(profile_id):
|
|
label = row.get("local_label") or ""
|
|
if not label or label.casefold() in already:
|
|
continue
|
|
if not re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text or "", re.IGNORECASE):
|
|
continue
|
|
extra.append(row)
|
|
already.add(label.casefold())
|
|
hits += 1
|
|
return mappings + extra, hits
|
|
|
|
|
|
def detect_personal_egress(profile_id: str | None, source_text: str) -> DetectionOutcome:
|
|
"""Full semantic detection of the personal generate-egress. Fail closed if incomplete."""
|
|
started = time.perf_counter()
|
|
stats = DetectionStats(source_chars=len(source_text or ""))
|
|
if _injected_fail is not None:
|
|
stats.abort_reason = _injected_fail.code
|
|
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
|
raise DetectError(
|
|
_injected_fail.code,
|
|
_injected_fail.message,
|
|
_injected_fail.status_code,
|
|
{**(_injected_fail.diagnostics or {}), **stats.public()},
|
|
)
|
|
config = detect_provider()
|
|
if not config:
|
|
stats.abort_reason = ERROR_DETECT_UNAVAILABLE
|
|
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
|
raise DetectError(
|
|
ERROR_DETECT_UNAVAILABLE,
|
|
"Semantische Detection ist nicht konfiguriert. Generate wird nicht freigegeben.",
|
|
diagnostics=stats.public(),
|
|
)
|
|
if config.mode == "http" and not config.local and not allows_remote_plaintext_detect():
|
|
stats.abort_reason = "remote_detect_blocked_production"
|
|
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
|
raise DetectError(
|
|
ERROR_DETECT_UNAVAILABLE,
|
|
"Externes Klartext-Detect ist im Produktivmodus nicht zulässig, und ein lokales Detect-Modell fehlt.",
|
|
diagnostics=stats.public(),
|
|
)
|
|
stats.detect_provider = config.name
|
|
stats.detect_model = config.model
|
|
chunks = split_detect_chunks(source_text or "")
|
|
stats.chunk_count = len(chunks)
|
|
collected: list[DetectedSpan] = []
|
|
try:
|
|
for index, (offset, chunk_text) in enumerate(chunks):
|
|
if _injected_truncated:
|
|
raise DetectError(
|
|
ERROR_DETECT_TRUNCATED,
|
|
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
|
|
)
|
|
if _injected_spans is not None:
|
|
raw_items = [
|
|
item
|
|
for item in _injected_spans
|
|
if int(item.get("start") or 0) >= offset
|
|
and int(item.get("end") or 0) <= offset + len(chunk_text)
|
|
]
|
|
local_items = []
|
|
for item in raw_items:
|
|
local = dict(item)
|
|
local["start"] = int(item["start"]) - offset
|
|
local["end"] = int(item["end"]) - offset
|
|
local_items.append(local)
|
|
result = ChatResult(
|
|
content=json.dumps({"entities": local_items}),
|
|
model=config.model,
|
|
usage={},
|
|
finish_reason="stop",
|
|
)
|
|
stats.detect_note = "injected"
|
|
elif config.mode == "fake":
|
|
fake_items = _contract_fake_spans(chunk_text)
|
|
result = ChatResult(
|
|
content=json.dumps({"entities": fake_items}),
|
|
model=config.model,
|
|
usage={},
|
|
finish_reason="stop",
|
|
)
|
|
stats.detect_note = "fake"
|
|
elif uses_llm_detect(config):
|
|
try:
|
|
result = _llm_chunk(config, chunk_text)
|
|
except PlaceholderError as exc:
|
|
raise DetectError(exc.code, exc.message) from exc
|
|
except ProviderError as exc:
|
|
raise DetectError(
|
|
ERROR_DETECT_CHUNK,
|
|
"Ein Detect-Chunk ist fehlgeschlagen. Generate wird nicht freigegeben.",
|
|
exc.status_code,
|
|
) from exc
|
|
stats.detect_note = "local_llm" if config.local else "remote_llm"
|
|
stats.detect_model = result.model or stats.detect_model
|
|
else:
|
|
raise DetectError(
|
|
ERROR_DETECT_UNAVAILABLE,
|
|
"Semantische Detection ist nicht verfügbar. Generate wird nicht freigegeben.",
|
|
)
|
|
stats.detect_calls += 1
|
|
_add_usage(stats, result.usage)
|
|
spans = _entities_from_result(result, chunk_text, index)
|
|
collected.extend(_to_global(span, offset) for span in spans)
|
|
stats.chunks_ok += 1
|
|
except DetectError as exc:
|
|
stats.abort_reason = exc.code
|
|
stats.full_detection_coverage = False
|
|
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
|
raise DetectError(
|
|
exc.code,
|
|
exc.message,
|
|
exc.status_code,
|
|
{**(exc.diagnostics or {}), **stats.public()},
|
|
) from exc
|
|
if stats.chunks_ok != stats.chunk_count:
|
|
stats.abort_reason = ERROR_DETECT_INCOMPLETE
|
|
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
|
raise DetectError(
|
|
ERROR_DETECT_INCOMPLETE,
|
|
"Detection hat nicht alle Chunks geprüft. Generate wird nicht freigegeben.",
|
|
diagnostics=stats.public(),
|
|
)
|
|
merged = _dedupe_spans(collected)
|
|
confirmed = list_confirmed_identities(profile_id) if profile_id else []
|
|
mappings, request_local, confirmed_from_spans = _assign_request_tokens(merged, confirmed)
|
|
mappings, extra_confirmed = _merge_confirmed_safety_net(source_text or "", mappings, profile_id)
|
|
stats.request_local_hits = request_local
|
|
stats.confirmed_registry_hits = confirmed_from_spans + extra_confirmed
|
|
counts: dict[str, int] = {kind: 0 for kind in ENTITY_TYPES}
|
|
for item in mappings:
|
|
kind = (item.get("entity_type") or "PERSON").upper()
|
|
counts[kind] = counts.get(kind, 0) + 1
|
|
stats.entity_counts = {key: value for key, value in counts.items() if value}
|
|
stats.full_detection_coverage = True
|
|
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
|
if profile_id:
|
|
seen_proposals: set[tuple[str, str]] = set()
|
|
for item in mappings:
|
|
if item.get("source") != "request_local":
|
|
continue
|
|
key = ((item.get("local_label") or ""), (item.get("entity_type") or ""))
|
|
if key in seen_proposals:
|
|
continue
|
|
seen_proposals.add(key)
|
|
record_review_proposal(profile_id, item.get("local_label") or "", item.get("entity_type") or "")
|
|
local_identities = [
|
|
{
|
|
"local_label": item.get("local_label"),
|
|
"token": item.get("token"),
|
|
"entity_type": item.get("entity_type"),
|
|
"demask_label": item.get("demask_label") or item.get("local_label"),
|
|
"source": item.get("source"),
|
|
}
|
|
for item in mappings
|
|
]
|
|
return DetectionOutcome(mappings=mappings, stats=stats, local_identities=local_identities)
|