This commit is contained in:
Lars committed 2026-08-29 20:28:59 +02:00
1 parent be11f82f84
commit e42f751fdc
47 files changed
+3359 -301

No files matched your search

+370 -118
View File
@@ -5,6 +5,7 @@ spans and types only, never tokens, never a durable registry write.
"""
from __future__ import annotations
import contextvars
import json
import re
import time
@@ -41,6 +42,23 @@ ERROR_DETECT_INCOMPLETE = "detect_incomplete"
ERROR_DETECT_INVALID = "detect_invalid_output"
ERROR_DETECT_TRUNCATED = "detect_truncated"
ERROR_DETECT_CHUNK = "detect_chunk_failed"
DETECT_MAX_PASSES = 2
DETECT_CONTRACT_RETRY_CODES = frozenset(
{
ERROR_DETECT_INVALID,
ERROR_DETECT_TRUNCATED,
ERROR_DETECT_INCOMPLETE,
}
)
SCHEMA_RETRY_HINT = (
"Der vorherige Durchgang hat das verbindliche Schema nicht erfüllt. "
"Antworte ausschließlich mit dem geforderten JSON-Objekt. "
"Nur die Felder start, end, text und entity_type sind zulässig. "
"Nur entity_type-Werte PERSON, PLACE, ORG, PROJECT. "
"Offsets sind 0-basiert und end ausschließlich, bezogen auf den Text nach «Text:». "
"text muss ein exakter Substring dieses Texts sein. "
"Keine zusätzlichen Felder, keine Tokens, keine Platzhalter, keine Erklärungen."
)
DETECT_DIALOGUE_FALLBACK_CODES = frozenset(
{
ERROR_DETECT_INVALID,
@@ -117,8 +135,14 @@ class DetectionStats:
completion_tokens: int = 0
total_tokens: int = 0
cost: float = 0.0
cost_known: bool = False
detect_ms: int = 0
abort_reason: str | None = None
detect_passes: int = 0
detect_partial_discarded: bool = False
detect_attempts: list[dict[str, Any]] = field(default_factory=list)
contract_violation: str | None = None
invalid_entity_type: str | None = None
def public(self) -> dict[str, Any]:
payload = {
@@ -138,11 +162,23 @@ class DetectionStats:
"detect_prompt_tokens": self.prompt_tokens,
"detect_completion_tokens": self.completion_tokens,
"detect_total_tokens": self.total_tokens,
"detect_cost": self.cost,
"detect_ms": self.detect_ms,
"detect_passes": self.detect_passes,
"detect_partial_discarded": self.detect_partial_discarded,
"detect_attempts": [dict(item) for item in self.detect_attempts],
"generate_called": False,
}
if self.cost_known:
payload["detect_cost"] = self.cost
payload["detect_cost_unknown"] = False
else:
payload["detect_cost_unknown"] = True
if self.abort_reason:
payload["abort_reason"] = self.abort_reason
if self.contract_violation:
payload["contract_violation"] = self.contract_violation
if self.invalid_entity_type:
payload["invalid_entity_type"] = self.invalid_entity_type
return payload
@@ -153,32 +189,51 @@ class DetectionOutcome:
local_identities: list[dict]
_injected_spans: list[dict] | None = None
_injected_fail: DetectError | None = None
_injected_truncated: bool = False
_injected_spans: contextvars.ContextVar[list[dict] | None] = contextvars.ContextVar(
"kansho_detect_spans",
default=None,
)
_injected_fail: contextvars.ContextVar[DetectError | None] = contextvars.ContextVar(
"kansho_detect_fail",
default=None,
)
_injected_truncated: contextvars.ContextVar[bool] = contextvars.ContextVar(
"kansho_detect_truncated",
default=False,
)
_injected_script: contextvars.ContextVar[Any] = contextvars.ContextVar(
"kansho_detect_script",
default=None,
)
def install_test_spans(spans: list[dict] | None) -> None:
"""Test-only span injection. Production never calls this."""
global _injected_spans
_injected_spans = None if spans is None else [dict(item) for item in spans]
_injected_spans.set(None if spans is None else [dict(item) for item in spans])
def install_test_detect_failure(error: DetectError | None) -> None:
global _injected_fail
_injected_fail = error
_injected_fail.set(error)
def install_test_truncated(flag: bool = True) -> None:
global _injected_truncated
_injected_truncated = bool(flag)
_injected_truncated.set(bool(flag))
def install_test_detect_script(handler=None) -> None:
"""Test-only per-attempt chunk script. Production never calls this.
handler(attempt, chunk_index, chunk_text, offset, schema_retry=...) may return
entity lists, raw JSON, ChatResult, or DetectError. Attempt is 1-based.
"""
_injected_script.set(handler)
def reset_detect_test_hooks() -> None:
global _injected_spans, _injected_fail, _injected_truncated
_injected_spans = None
_injected_fail = None
_injected_truncated = False
_injected_spans.set(None)
_injected_fail.set(None)
_injected_truncated.set(False)
_injected_script.set(None)
def uses_llm_detect(config) -> bool:
@@ -295,25 +350,41 @@ def _contract_fake_spans(text: str) -> list[dict]:
return found
def _contract_error(
message: str,
*,
violation: str,
code: str = ERROR_DETECT_INVALID,
invalid_entity_type: str | None = None,
status_code: int = 503,
) -> DetectError:
diagnostics: dict[str, Any] = {"contract_violation": violation}
if invalid_entity_type:
delivered = re.sub(r"[^A-Z0-9_:-]", "", str(invalid_entity_type).strip().upper())[:40]
if delivered:
diagnostics["invalid_entity_type"] = delivered
return DetectError(code, message, status_code, diagnostics)
def _parse_detect_json(raw: str) -> dict:
text = (raw or "").strip()
if text.startswith("```"):
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL).strip()
match = JSON_BLOCK.search(text)
if not match:
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.")
raise _contract_error("Detect-Antwort war kein gültiges JSON.", violation="invalid_json")
try:
data = json.loads(match.group(0))
except json.JSONDecodeError as exc:
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.") from exc
raise _contract_error("Detect-Antwort war kein gültiges JSON.", violation="invalid_json") from exc
if not isinstance(data, dict):
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort muss ein Objekt mit entities sein.")
raise _contract_error("Detect-Antwort muss ein Objekt mit entities sein.", violation="invalid_json")
extra = set(data.keys()) - ALLOWED_ROOT_FIELDS
if extra:
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort enthält unerwartete Felder.")
raise _contract_error("Detect-Antwort enthält unerwartete Felder.", violation="extra_fields")
items = data.get("entities")
if not isinstance(items, list):
raise DetectError(ERROR_DETECT_INVALID, "Detect-entities muss eine Liste sein.")
raise _contract_error("Detect-entities muss eine Liste sein.", violation="invalid_json")
return data
@@ -364,13 +435,13 @@ def _ground_span(chunk_text: str, start: int, end: int, text: str) -> tuple[int,
def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
if not isinstance(item, dict):
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity muss ein Objekt sein.")
raise _contract_error("Detect-Entity muss ein Objekt sein.", violation="invalid_entity")
extra = set(item.keys()) - ALLOWED_ENTITY_FIELDS
if extra:
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity enthält unerwartete Felder.")
raise _contract_error("Detect-Entity enthält unerwartete Felder.", violation="extra_fields")
missing = ALLOWED_ENTITY_FIELDS - set(item.keys())
if missing:
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity ist unvollständig.")
raise _contract_error("Detect-Entity ist unvollständig.", violation="missing_fields")
try:
start = int(item["start"])
except (TypeError, ValueError):
@@ -381,17 +452,26 @@ def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> De
end = 0
text = item.get("text")
if not isinstance(text, str):
raise DetectError(ERROR_DETECT_INVALID, "Detect-text fehlt.")
raise _contract_error("Detect-text fehlt.", violation="missing_fields")
if not text.strip():
return None
raise _contract_error("Detect-text ist leer.", violation="invalid_text")
if "token" in item or "placeholder" in item:
raise DetectError(ERROR_DETECT_INVALID, "Detect darf keine Tokens festlegen.")
kind = normalize_entity_type(item.get("entity_type"), default="")
raise _contract_error("Detect darf keine Tokens festlegen.", violation="extra_fields")
raw_type = item.get("entity_type")
kind = normalize_entity_type(raw_type if isinstance(raw_type, str) else "", default="")
if not kind:
raise DetectError(ERROR_DETECT_INVALID, "Detect-entity_type ist nicht erlaubt.")
delivered = raw_type if isinstance(raw_type, str) else ""
raise _contract_error(
"Detect-entity_type ist nicht erlaubt.",
violation="unknown_entity_type",
invalid_entity_type=delivered,
)
grounded = _ground_span(chunk_text, start, end, text)
if not grounded:
return None
raise _contract_error(
"Detect-Offsets sind nicht im Chunk verankert.",
violation="unusable_offsets",
)
left, right, local_text = grounded
return DetectedSpan(start=left, end=right, text=local_text, entity_type=kind, chunk_index=chunk_index)
@@ -419,20 +499,30 @@ def resolve_overlaps(spans: list[DetectedSpan]) -> list[DetectedSpan]:
def _add_usage(stats: DetectionStats, usage: dict | None) -> None:
data = usage or {}
stats.prompt_tokens += int(data.get("prompt_tokens") or 0)
stats.completion_tokens += int(data.get("completion_tokens") or 0)
stats.total_tokens += int(data.get("total_tokens") or 0)
try:
stats.cost += float(data.get("cost") or 0)
except (TypeError, ValueError):
pass
if data.get("prompt_tokens") is not None:
stats.prompt_tokens += int(data.get("prompt_tokens") or 0)
if data.get("completion_tokens") is not None:
stats.completion_tokens += int(data.get("completion_tokens") or 0)
if data.get("total_tokens") is not None:
stats.total_tokens += int(data.get("total_tokens") or 0)
extra = data.get("cost")
if extra is None:
extra = data.get("total_cost")
if extra is not None:
try:
stats.cost += float(extra)
stats.cost_known = True
except (TypeError, ValueError):
pass
def _llm_chunk(config, excerpt: str) -> ChatResult:
def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResult:
prompt = resolve_template(
_detect_prompt()["template"],
{"source_text": excerpt, "known_labels": ""},
)
if schema_retry:
prompt = f"{prompt.rstrip()}\n\n{SCHEMA_RETRY_HINT}"
return complete_chat(
config,
[{"role": "user", "content": prompt}],
@@ -447,6 +537,7 @@ def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int)
raise DetectError(
ERROR_DETECT_TRUNCATED,
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
diagnostics={"contract_violation": "truncated"},
)
data = _parse_detect_json(result.content or "")
spans: list[DetectedSpan] = []
@@ -571,18 +662,192 @@ def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str
return mappings + extra, hits
def _merge_pass_stats(overall: DetectionStats, pass_stats: DetectionStats) -> None:
overall.detect_calls += pass_stats.detect_calls
overall.prompt_tokens += pass_stats.prompt_tokens
overall.completion_tokens += pass_stats.completion_tokens
overall.total_tokens += pass_stats.total_tokens
overall.chunk_count = pass_stats.chunk_count
overall.chunks_ok = pass_stats.chunks_ok
overall.detect_note = pass_stats.detect_note or overall.detect_note
overall.detect_model = pass_stats.detect_model or overall.detect_model
if pass_stats.cost_known:
overall.cost_known = True
overall.cost += pass_stats.cost
def _pass_snapshot(
pass_stats: DetectionStats,
*,
attempt: int,
coverage: bool,
abort: str | None = None,
) -> dict[str, Any]:
row: dict[str, Any] = {
"attempt": attempt,
"schema_retry": attempt > 1,
"chunk_count": pass_stats.chunk_count,
"chunks_ok": pass_stats.chunks_ok,
"detect_calls": pass_stats.detect_calls,
"full_detection_coverage": coverage,
"detect_prompt_tokens": pass_stats.prompt_tokens,
"detect_completion_tokens": pass_stats.completion_tokens,
"detect_total_tokens": pass_stats.total_tokens,
"detect_ms": pass_stats.detect_ms,
}
if pass_stats.cost_known:
row["detect_cost"] = pass_stats.cost
row["detect_cost_unknown"] = False
else:
row["detect_cost_unknown"] = True
if abort:
row["abort_reason"] = abort
if pass_stats.contract_violation:
row["contract_violation"] = pass_stats.contract_violation
if pass_stats.invalid_entity_type:
row["invalid_entity_type"] = pass_stats.invalid_entity_type
return row
def _script_result(raw: Any, *, model: str | None) -> ChatResult:
if isinstance(raw, DetectError):
raise raw
if isinstance(raw, ChatResult):
return raw
if isinstance(raw, str):
return ChatResult(content=raw, model=model, usage={}, finish_reason="stop")
if isinstance(raw, dict) and "entities" in raw:
return ChatResult(content=json.dumps(raw), model=model, usage={}, finish_reason="stop")
if isinstance(raw, list):
return ChatResult(
content=json.dumps({"entities": raw}),
model=model,
usage={},
finish_reason="stop",
)
raise _contract_error("Detect-Testskript lieferte keine verwertbare Antwort.", violation="invalid_json")
def _chunk_chat_result(
config,
chunk_text: str,
*,
offset: int,
index: int,
attempt: int,
schema_retry: bool,
stats: DetectionStats,
) -> ChatResult:
if _injected_truncated.get():
raise DetectError(
ERROR_DETECT_TRUNCATED,
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
diagnostics={"contract_violation": "truncated"},
)
script = _injected_script.get()
if script is not None:
stats.detect_note = "injected"
try:
raw = script(attempt, index, chunk_text, offset, schema_retry=schema_retry)
except TypeError:
raw = script(attempt, index, chunk_text, offset)
return _script_result(raw, model=config.model)
spans = _injected_spans.get()
if spans is not None:
raw_items = [
item
for item in spans
if int(item.get("start") or 0) >= offset
and int(item.get("end") or 0) <= offset + len(chunk_text)
]
local_items = []
for item in raw_items:
local = dict(item)
local["start"] = int(item["start"]) - offset
local["end"] = int(item["end"]) - offset
local_items.append(local)
stats.detect_note = "injected"
return ChatResult(
content=json.dumps({"entities": local_items}),
model=config.model,
usage={},
finish_reason="stop",
)
if config.mode == "fake":
stats.detect_note = "fake"
return ChatResult(
content=json.dumps({"entities": _contract_fake_spans(chunk_text)}),
model=config.model,
usage={},
finish_reason="stop",
)
if uses_llm_detect(config):
try:
result = _llm_chunk(config, chunk_text, schema_retry=schema_retry)
except PlaceholderError as exc:
raise DetectError(exc.code, exc.message) from exc
except ProviderError as exc:
raise DetectError(
ERROR_DETECT_CHUNK,
"Ein Detect-Chunk ist fehlgeschlagen. Generate wird nicht freigegeben.",
exc.status_code,
) from exc
stats.detect_note = "local_llm" if config.local else "remote_llm"
stats.detect_model = result.model or stats.detect_model
return result
raise DetectError(
ERROR_DETECT_UNAVAILABLE,
"Semantische Detection ist nicht verfügbar. Generate wird nicht freigegeben.",
)
def _run_detection_pass(
config,
chunks: list[tuple[int, str]],
*,
attempt: int,
schema_retry: bool,
stats: DetectionStats,
) -> list[DetectedSpan]:
collected: list[DetectedSpan] = []
for index, (offset, chunk_text) in enumerate(chunks):
result = _chunk_chat_result(
config,
chunk_text,
offset=offset,
index=index,
attempt=attempt,
schema_retry=schema_retry,
stats=stats,
)
stats.detect_calls += 1
_add_usage(stats, result.usage)
spans = _entities_from_result(result, chunk_text, index)
collected.extend(_to_global(span, offset) for span in spans)
stats.chunks_ok += 1
if stats.chunks_ok != stats.chunk_count:
raise DetectError(
ERROR_DETECT_INCOMPLETE,
"Detection hat nicht alle Chunks geprüft. Generate wird nicht freigegeben.",
diagnostics={"contract_violation": "incomplete"},
)
return collected
def detect_personal_egress(profile_id: str | None, source_text: str) -> DetectionOutcome:
"""Full semantic detection of the personal generate-egress. Fail closed if incomplete."""
started = time.perf_counter()
stats = DetectionStats(source_chars=len(source_text or ""))
if _injected_fail is not None:
stats.abort_reason = _injected_fail.code
injected_fail = _injected_fail.get()
if injected_fail is not None:
stats.abort_reason = injected_fail.code
stats.detect_ms = int((time.perf_counter() - started) * 1000)
stats.detect_passes = 0
raise DetectError(
_injected_fail.code,
_injected_fail.message,
_injected_fail.status_code,
{**(_injected_fail.diagnostics or {}), **stats.public()},
injected_fail.code,
injected_fail.message,
injected_fail.status_code,
{**(injected_fail.diagnostics or {}), **stats.public()},
)
config = detect_provider()
if not config:
@@ -606,83 +871,70 @@ def detect_personal_egress(profile_id: str | None, source_text: str) -> Detectio
chunks = split_detect_chunks(source_text or "")
stats.chunk_count = len(chunks)
collected: list[DetectedSpan] = []
try:
for index, (offset, chunk_text) in enumerate(chunks):
if _injected_truncated:
raise DetectError(
ERROR_DETECT_TRUNCATED,
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
)
if _injected_spans is not None:
raw_items = [
item
for item in _injected_spans
if int(item.get("start") or 0) >= offset
and int(item.get("end") or 0) <= offset + len(chunk_text)
]
local_items = []
for item in raw_items:
local = dict(item)
local["start"] = int(item["start"]) - offset
local["end"] = int(item["end"]) - offset
local_items.append(local)
result = ChatResult(
content=json.dumps({"entities": local_items}),
model=config.model,
usage={},
finish_reason="stop",
)
stats.detect_note = "injected"
elif config.mode == "fake":
fake_items = _contract_fake_spans(chunk_text)
result = ChatResult(
content=json.dumps({"entities": fake_items}),
model=config.model,
usage={},
finish_reason="stop",
)
stats.detect_note = "fake"
elif uses_llm_detect(config):
try:
result = _llm_chunk(config, chunk_text)
except PlaceholderError as exc:
raise DetectError(exc.code, exc.message) from exc
except ProviderError as exc:
raise DetectError(
ERROR_DETECT_CHUNK,
"Ein Detect-Chunk ist fehlgeschlagen. Generate wird nicht freigegeben.",
exc.status_code,
) from exc
stats.detect_note = "local_llm" if config.local else "remote_llm"
stats.detect_model = result.model or stats.detect_model
else:
raise DetectError(
ERROR_DETECT_UNAVAILABLE,
"Semantische Detection ist nicht verfügbar. Generate wird nicht freigegeben.",
)
stats.detect_calls += 1
_add_usage(stats, result.usage)
spans = _entities_from_result(result, chunk_text, index)
collected.extend(_to_global(span, offset) for span in spans)
stats.chunks_ok += 1
except DetectError as exc:
stats.abort_reason = exc.code
stats.full_detection_coverage = False
stats.detect_ms = int((time.perf_counter() - started) * 1000)
raise DetectError(
exc.code,
exc.message,
exc.status_code,
{**(exc.diagnostics or {}), **stats.public()},
) from exc
if stats.chunks_ok != stats.chunk_count:
stats.abort_reason = ERROR_DETECT_INCOMPLETE
stats.detect_ms = int((time.perf_counter() - started) * 1000)
raise DetectError(
ERROR_DETECT_INCOMPLETE,
"Detection hat nicht alle Chunks geprüft. Generate wird nicht freigegeben.",
diagnostics=stats.public(),
last_error: DetectError | None = None
for attempt in range(1, DETECT_MAX_PASSES + 1):
pass_stats = DetectionStats(
source_chars=stats.source_chars,
detect_provider=stats.detect_provider,
detect_model=stats.detect_model,
chunk_count=len(chunks),
)
pass_started = time.perf_counter()
try:
collected = _run_detection_pass(
config,
chunks,
attempt=attempt,
schema_retry=attempt > 1,
stats=pass_stats,
)
pass_stats.full_detection_coverage = True
pass_stats.detect_ms = int((time.perf_counter() - pass_started) * 1000)
_merge_pass_stats(stats, pass_stats)
stats.detect_attempts.append(
_pass_snapshot(pass_stats, attempt=attempt, coverage=True)
)
last_error = None
break
except DetectError as exc:
inner = dict(exc.diagnostics or {})
pass_stats.abort_reason = exc.code
pass_stats.contract_violation = inner.get("contract_violation")
pass_stats.invalid_entity_type = inner.get("invalid_entity_type")
pass_stats.full_detection_coverage = False
pass_stats.detect_ms = int((time.perf_counter() - pass_started) * 1000)
_merge_pass_stats(stats, pass_stats)
stats.detect_attempts.append(
_pass_snapshot(pass_stats, attempt=attempt, coverage=False, abort=exc.code)
)
collected = []
last_error = DetectError(exc.code, exc.message, exc.status_code, inner)
retryable = exc.code in DETECT_CONTRACT_RETRY_CODES
if attempt < DETECT_MAX_PASSES and retryable:
stats.detect_partial_discarded = True
continue
stats.abort_reason = exc.code
stats.contract_violation = pass_stats.contract_violation
stats.invalid_entity_type = pass_stats.invalid_entity_type
stats.full_detection_coverage = False
stats.detect_passes = attempt
stats.detect_ms = int((time.perf_counter() - started) * 1000)
raise DetectError(
exc.code,
exc.message,
exc.status_code,
{**inner, **stats.public()},
) from exc
if last_error is not None:
stats.detect_passes = len(stats.detect_attempts)
stats.detect_ms = int((time.perf_counter() - started) * 1000)
raise DetectError(
last_error.code,
last_error.message,
last_error.status_code,
{**(last_error.diagnostics or {}), **stats.public()},
)
stats.detect_passes = len(stats.detect_attempts)
merged = _dedupe_spans(collected)
confirmed = list_confirmed_identities(profile_id) if profile_id else []
mappings, request_local, confirmed_from_spans = _assign_request_tokens(merged, confirmed)