MVP 1.0
This commit is contained in:
1 parent
be11f82f84
commit
e42f751fdc
47 files changed
+3359
-301
No files matched your search
+370
-118
@@ -5,6 +5,7 @@ spans and types only, never tokens, never a durable registry write.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
@@ -41,6 +42,23 @@ ERROR_DETECT_INCOMPLETE = "detect_incomplete"
|
||||
ERROR_DETECT_INVALID = "detect_invalid_output"
|
||||
ERROR_DETECT_TRUNCATED = "detect_truncated"
|
||||
ERROR_DETECT_CHUNK = "detect_chunk_failed"
|
||||
DETECT_MAX_PASSES = 2
|
||||
DETECT_CONTRACT_RETRY_CODES = frozenset(
|
||||
{
|
||||
ERROR_DETECT_INVALID,
|
||||
ERROR_DETECT_TRUNCATED,
|
||||
ERROR_DETECT_INCOMPLETE,
|
||||
}
|
||||
)
|
||||
SCHEMA_RETRY_HINT = (
|
||||
"Der vorherige Durchgang hat das verbindliche Schema nicht erfüllt. "
|
||||
"Antworte ausschließlich mit dem geforderten JSON-Objekt. "
|
||||
"Nur die Felder start, end, text und entity_type sind zulässig. "
|
||||
"Nur entity_type-Werte PERSON, PLACE, ORG, PROJECT. "
|
||||
"Offsets sind 0-basiert und end ausschließlich, bezogen auf den Text nach «Text:». "
|
||||
"text muss ein exakter Substring dieses Texts sein. "
|
||||
"Keine zusätzlichen Felder, keine Tokens, keine Platzhalter, keine Erklärungen."
|
||||
)
|
||||
DETECT_DIALOGUE_FALLBACK_CODES = frozenset(
|
||||
{
|
||||
ERROR_DETECT_INVALID,
|
||||
@@ -117,8 +135,14 @@ class DetectionStats:
|
||||
completion_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
cost: float = 0.0
|
||||
cost_known: bool = False
|
||||
detect_ms: int = 0
|
||||
abort_reason: str | None = None
|
||||
detect_passes: int = 0
|
||||
detect_partial_discarded: bool = False
|
||||
detect_attempts: list[dict[str, Any]] = field(default_factory=list)
|
||||
contract_violation: str | None = None
|
||||
invalid_entity_type: str | None = None
|
||||
|
||||
def public(self) -> dict[str, Any]:
|
||||
payload = {
|
||||
@@ -138,11 +162,23 @@ class DetectionStats:
|
||||
"detect_prompt_tokens": self.prompt_tokens,
|
||||
"detect_completion_tokens": self.completion_tokens,
|
||||
"detect_total_tokens": self.total_tokens,
|
||||
"detect_cost": self.cost,
|
||||
"detect_ms": self.detect_ms,
|
||||
"detect_passes": self.detect_passes,
|
||||
"detect_partial_discarded": self.detect_partial_discarded,
|
||||
"detect_attempts": [dict(item) for item in self.detect_attempts],
|
||||
"generate_called": False,
|
||||
}
|
||||
if self.cost_known:
|
||||
payload["detect_cost"] = self.cost
|
||||
payload["detect_cost_unknown"] = False
|
||||
else:
|
||||
payload["detect_cost_unknown"] = True
|
||||
if self.abort_reason:
|
||||
payload["abort_reason"] = self.abort_reason
|
||||
if self.contract_violation:
|
||||
payload["contract_violation"] = self.contract_violation
|
||||
if self.invalid_entity_type:
|
||||
payload["invalid_entity_type"] = self.invalid_entity_type
|
||||
return payload
|
||||
|
||||
|
||||
@@ -153,32 +189,51 @@ class DetectionOutcome:
|
||||
local_identities: list[dict]
|
||||
|
||||
|
||||
_injected_spans: list[dict] | None = None
|
||||
_injected_fail: DetectError | None = None
|
||||
_injected_truncated: bool = False
|
||||
_injected_spans: contextvars.ContextVar[list[dict] | None] = contextvars.ContextVar(
|
||||
"kansho_detect_spans",
|
||||
default=None,
|
||||
)
|
||||
_injected_fail: contextvars.ContextVar[DetectError | None] = contextvars.ContextVar(
|
||||
"kansho_detect_fail",
|
||||
default=None,
|
||||
)
|
||||
_injected_truncated: contextvars.ContextVar[bool] = contextvars.ContextVar(
|
||||
"kansho_detect_truncated",
|
||||
default=False,
|
||||
)
|
||||
_injected_script: contextvars.ContextVar[Any] = contextvars.ContextVar(
|
||||
"kansho_detect_script",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def install_test_spans(spans: list[dict] | None) -> None:
|
||||
"""Test-only span injection. Production never calls this."""
|
||||
global _injected_spans
|
||||
_injected_spans = None if spans is None else [dict(item) for item in spans]
|
||||
_injected_spans.set(None if spans is None else [dict(item) for item in spans])
|
||||
|
||||
|
||||
def install_test_detect_failure(error: DetectError | None) -> None:
|
||||
global _injected_fail
|
||||
_injected_fail = error
|
||||
_injected_fail.set(error)
|
||||
|
||||
|
||||
def install_test_truncated(flag: bool = True) -> None:
|
||||
global _injected_truncated
|
||||
_injected_truncated = bool(flag)
|
||||
_injected_truncated.set(bool(flag))
|
||||
|
||||
|
||||
def install_test_detect_script(handler=None) -> None:
|
||||
"""Test-only per-attempt chunk script. Production never calls this.
|
||||
|
||||
handler(attempt, chunk_index, chunk_text, offset, schema_retry=...) may return
|
||||
entity lists, raw JSON, ChatResult, or DetectError. Attempt is 1-based.
|
||||
"""
|
||||
_injected_script.set(handler)
|
||||
|
||||
|
||||
def reset_detect_test_hooks() -> None:
|
||||
global _injected_spans, _injected_fail, _injected_truncated
|
||||
_injected_spans = None
|
||||
_injected_fail = None
|
||||
_injected_truncated = False
|
||||
_injected_spans.set(None)
|
||||
_injected_fail.set(None)
|
||||
_injected_truncated.set(False)
|
||||
_injected_script.set(None)
|
||||
|
||||
|
||||
def uses_llm_detect(config) -> bool:
|
||||
@@ -295,25 +350,41 @@ def _contract_fake_spans(text: str) -> list[dict]:
|
||||
return found
|
||||
|
||||
|
||||
def _contract_error(
|
||||
message: str,
|
||||
*,
|
||||
violation: str,
|
||||
code: str = ERROR_DETECT_INVALID,
|
||||
invalid_entity_type: str | None = None,
|
||||
status_code: int = 503,
|
||||
) -> DetectError:
|
||||
diagnostics: dict[str, Any] = {"contract_violation": violation}
|
||||
if invalid_entity_type:
|
||||
delivered = re.sub(r"[^A-Z0-9_:-]", "", str(invalid_entity_type).strip().upper())[:40]
|
||||
if delivered:
|
||||
diagnostics["invalid_entity_type"] = delivered
|
||||
return DetectError(code, message, status_code, diagnostics)
|
||||
|
||||
|
||||
def _parse_detect_json(raw: str) -> dict:
|
||||
text = (raw or "").strip()
|
||||
if text.startswith("```"):
|
||||
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL).strip()
|
||||
match = JSON_BLOCK.search(text)
|
||||
if not match:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.")
|
||||
raise _contract_error("Detect-Antwort war kein gültiges JSON.", violation="invalid_json")
|
||||
try:
|
||||
data = json.loads(match.group(0))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort war kein gültiges JSON.") from exc
|
||||
raise _contract_error("Detect-Antwort war kein gültiges JSON.", violation="invalid_json") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort muss ein Objekt mit entities sein.")
|
||||
raise _contract_error("Detect-Antwort muss ein Objekt mit entities sein.", violation="invalid_json")
|
||||
extra = set(data.keys()) - ALLOWED_ROOT_FIELDS
|
||||
if extra:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Antwort enthält unerwartete Felder.")
|
||||
raise _contract_error("Detect-Antwort enthält unerwartete Felder.", violation="extra_fields")
|
||||
items = data.get("entities")
|
||||
if not isinstance(items, list):
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-entities muss eine Liste sein.")
|
||||
raise _contract_error("Detect-entities muss eine Liste sein.", violation="invalid_json")
|
||||
return data
|
||||
|
||||
|
||||
@@ -364,13 +435,13 @@ def _ground_span(chunk_text: str, start: int, end: int, text: str) -> tuple[int,
|
||||
|
||||
def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
|
||||
if not isinstance(item, dict):
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity muss ein Objekt sein.")
|
||||
raise _contract_error("Detect-Entity muss ein Objekt sein.", violation="invalid_entity")
|
||||
extra = set(item.keys()) - ALLOWED_ENTITY_FIELDS
|
||||
if extra:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity enthält unerwartete Felder.")
|
||||
raise _contract_error("Detect-Entity enthält unerwartete Felder.", violation="extra_fields")
|
||||
missing = ALLOWED_ENTITY_FIELDS - set(item.keys())
|
||||
if missing:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-Entity ist unvollständig.")
|
||||
raise _contract_error("Detect-Entity ist unvollständig.", violation="missing_fields")
|
||||
try:
|
||||
start = int(item["start"])
|
||||
except (TypeError, ValueError):
|
||||
@@ -381,17 +452,26 @@ def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> De
|
||||
end = 0
|
||||
text = item.get("text")
|
||||
if not isinstance(text, str):
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-text fehlt.")
|
||||
raise _contract_error("Detect-text fehlt.", violation="missing_fields")
|
||||
if not text.strip():
|
||||
return None
|
||||
raise _contract_error("Detect-text ist leer.", violation="invalid_text")
|
||||
if "token" in item or "placeholder" in item:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect darf keine Tokens festlegen.")
|
||||
kind = normalize_entity_type(item.get("entity_type"), default="")
|
||||
raise _contract_error("Detect darf keine Tokens festlegen.", violation="extra_fields")
|
||||
raw_type = item.get("entity_type")
|
||||
kind = normalize_entity_type(raw_type if isinstance(raw_type, str) else "", default="")
|
||||
if not kind:
|
||||
raise DetectError(ERROR_DETECT_INVALID, "Detect-entity_type ist nicht erlaubt.")
|
||||
delivered = raw_type if isinstance(raw_type, str) else ""
|
||||
raise _contract_error(
|
||||
"Detect-entity_type ist nicht erlaubt.",
|
||||
violation="unknown_entity_type",
|
||||
invalid_entity_type=delivered,
|
||||
)
|
||||
grounded = _ground_span(chunk_text, start, end, text)
|
||||
if not grounded:
|
||||
return None
|
||||
raise _contract_error(
|
||||
"Detect-Offsets sind nicht im Chunk verankert.",
|
||||
violation="unusable_offsets",
|
||||
)
|
||||
left, right, local_text = grounded
|
||||
return DetectedSpan(start=left, end=right, text=local_text, entity_type=kind, chunk_index=chunk_index)
|
||||
|
||||
@@ -419,20 +499,30 @@ def resolve_overlaps(spans: list[DetectedSpan]) -> list[DetectedSpan]:
|
||||
|
||||
def _add_usage(stats: DetectionStats, usage: dict | None) -> None:
|
||||
data = usage or {}
|
||||
stats.prompt_tokens += int(data.get("prompt_tokens") or 0)
|
||||
stats.completion_tokens += int(data.get("completion_tokens") or 0)
|
||||
stats.total_tokens += int(data.get("total_tokens") or 0)
|
||||
try:
|
||||
stats.cost += float(data.get("cost") or 0)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if data.get("prompt_tokens") is not None:
|
||||
stats.prompt_tokens += int(data.get("prompt_tokens") or 0)
|
||||
if data.get("completion_tokens") is not None:
|
||||
stats.completion_tokens += int(data.get("completion_tokens") or 0)
|
||||
if data.get("total_tokens") is not None:
|
||||
stats.total_tokens += int(data.get("total_tokens") or 0)
|
||||
extra = data.get("cost")
|
||||
if extra is None:
|
||||
extra = data.get("total_cost")
|
||||
if extra is not None:
|
||||
try:
|
||||
stats.cost += float(extra)
|
||||
stats.cost_known = True
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _llm_chunk(config, excerpt: str) -> ChatResult:
|
||||
def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResult:
|
||||
prompt = resolve_template(
|
||||
_detect_prompt()["template"],
|
||||
{"source_text": excerpt, "known_labels": ""},
|
||||
)
|
||||
if schema_retry:
|
||||
prompt = f"{prompt.rstrip()}\n\n{SCHEMA_RETRY_HINT}"
|
||||
return complete_chat(
|
||||
config,
|
||||
[{"role": "user", "content": prompt}],
|
||||
@@ -447,6 +537,7 @@ def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int)
|
||||
raise DetectError(
|
||||
ERROR_DETECT_TRUNCATED,
|
||||
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
|
||||
diagnostics={"contract_violation": "truncated"},
|
||||
)
|
||||
data = _parse_detect_json(result.content or "")
|
||||
spans: list[DetectedSpan] = []
|
||||
@@ -571,18 +662,192 @@ def _merge_confirmed_safety_net(text: str, mappings: list[dict], profile_id: str
|
||||
return mappings + extra, hits
|
||||
|
||||
|
||||
def _merge_pass_stats(overall: DetectionStats, pass_stats: DetectionStats) -> None:
|
||||
overall.detect_calls += pass_stats.detect_calls
|
||||
overall.prompt_tokens += pass_stats.prompt_tokens
|
||||
overall.completion_tokens += pass_stats.completion_tokens
|
||||
overall.total_tokens += pass_stats.total_tokens
|
||||
overall.chunk_count = pass_stats.chunk_count
|
||||
overall.chunks_ok = pass_stats.chunks_ok
|
||||
overall.detect_note = pass_stats.detect_note or overall.detect_note
|
||||
overall.detect_model = pass_stats.detect_model or overall.detect_model
|
||||
if pass_stats.cost_known:
|
||||
overall.cost_known = True
|
||||
overall.cost += pass_stats.cost
|
||||
|
||||
|
||||
def _pass_snapshot(
|
||||
pass_stats: DetectionStats,
|
||||
*,
|
||||
attempt: int,
|
||||
coverage: bool,
|
||||
abort: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
row: dict[str, Any] = {
|
||||
"attempt": attempt,
|
||||
"schema_retry": attempt > 1,
|
||||
"chunk_count": pass_stats.chunk_count,
|
||||
"chunks_ok": pass_stats.chunks_ok,
|
||||
"detect_calls": pass_stats.detect_calls,
|
||||
"full_detection_coverage": coverage,
|
||||
"detect_prompt_tokens": pass_stats.prompt_tokens,
|
||||
"detect_completion_tokens": pass_stats.completion_tokens,
|
||||
"detect_total_tokens": pass_stats.total_tokens,
|
||||
"detect_ms": pass_stats.detect_ms,
|
||||
}
|
||||
if pass_stats.cost_known:
|
||||
row["detect_cost"] = pass_stats.cost
|
||||
row["detect_cost_unknown"] = False
|
||||
else:
|
||||
row["detect_cost_unknown"] = True
|
||||
if abort:
|
||||
row["abort_reason"] = abort
|
||||
if pass_stats.contract_violation:
|
||||
row["contract_violation"] = pass_stats.contract_violation
|
||||
if pass_stats.invalid_entity_type:
|
||||
row["invalid_entity_type"] = pass_stats.invalid_entity_type
|
||||
return row
|
||||
|
||||
|
||||
def _script_result(raw: Any, *, model: str | None) -> ChatResult:
|
||||
if isinstance(raw, DetectError):
|
||||
raise raw
|
||||
if isinstance(raw, ChatResult):
|
||||
return raw
|
||||
if isinstance(raw, str):
|
||||
return ChatResult(content=raw, model=model, usage={}, finish_reason="stop")
|
||||
if isinstance(raw, dict) and "entities" in raw:
|
||||
return ChatResult(content=json.dumps(raw), model=model, usage={}, finish_reason="stop")
|
||||
if isinstance(raw, list):
|
||||
return ChatResult(
|
||||
content=json.dumps({"entities": raw}),
|
||||
model=model,
|
||||
usage={},
|
||||
finish_reason="stop",
|
||||
)
|
||||
raise _contract_error("Detect-Testskript lieferte keine verwertbare Antwort.", violation="invalid_json")
|
||||
|
||||
|
||||
def _chunk_chat_result(
|
||||
config,
|
||||
chunk_text: str,
|
||||
*,
|
||||
offset: int,
|
||||
index: int,
|
||||
attempt: int,
|
||||
schema_retry: bool,
|
||||
stats: DetectionStats,
|
||||
) -> ChatResult:
|
||||
if _injected_truncated.get():
|
||||
raise DetectError(
|
||||
ERROR_DETECT_TRUNCATED,
|
||||
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
|
||||
diagnostics={"contract_violation": "truncated"},
|
||||
)
|
||||
script = _injected_script.get()
|
||||
if script is not None:
|
||||
stats.detect_note = "injected"
|
||||
try:
|
||||
raw = script(attempt, index, chunk_text, offset, schema_retry=schema_retry)
|
||||
except TypeError:
|
||||
raw = script(attempt, index, chunk_text, offset)
|
||||
return _script_result(raw, model=config.model)
|
||||
spans = _injected_spans.get()
|
||||
if spans is not None:
|
||||
raw_items = [
|
||||
item
|
||||
for item in spans
|
||||
if int(item.get("start") or 0) >= offset
|
||||
and int(item.get("end") or 0) <= offset + len(chunk_text)
|
||||
]
|
||||
local_items = []
|
||||
for item in raw_items:
|
||||
local = dict(item)
|
||||
local["start"] = int(item["start"]) - offset
|
||||
local["end"] = int(item["end"]) - offset
|
||||
local_items.append(local)
|
||||
stats.detect_note = "injected"
|
||||
return ChatResult(
|
||||
content=json.dumps({"entities": local_items}),
|
||||
model=config.model,
|
||||
usage={},
|
||||
finish_reason="stop",
|
||||
)
|
||||
if config.mode == "fake":
|
||||
stats.detect_note = "fake"
|
||||
return ChatResult(
|
||||
content=json.dumps({"entities": _contract_fake_spans(chunk_text)}),
|
||||
model=config.model,
|
||||
usage={},
|
||||
finish_reason="stop",
|
||||
)
|
||||
if uses_llm_detect(config):
|
||||
try:
|
||||
result = _llm_chunk(config, chunk_text, schema_retry=schema_retry)
|
||||
except PlaceholderError as exc:
|
||||
raise DetectError(exc.code, exc.message) from exc
|
||||
except ProviderError as exc:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_CHUNK,
|
||||
"Ein Detect-Chunk ist fehlgeschlagen. Generate wird nicht freigegeben.",
|
||||
exc.status_code,
|
||||
) from exc
|
||||
stats.detect_note = "local_llm" if config.local else "remote_llm"
|
||||
stats.detect_model = result.model or stats.detect_model
|
||||
return result
|
||||
raise DetectError(
|
||||
ERROR_DETECT_UNAVAILABLE,
|
||||
"Semantische Detection ist nicht verfügbar. Generate wird nicht freigegeben.",
|
||||
)
|
||||
|
||||
|
||||
def _run_detection_pass(
|
||||
config,
|
||||
chunks: list[tuple[int, str]],
|
||||
*,
|
||||
attempt: int,
|
||||
schema_retry: bool,
|
||||
stats: DetectionStats,
|
||||
) -> list[DetectedSpan]:
|
||||
collected: list[DetectedSpan] = []
|
||||
for index, (offset, chunk_text) in enumerate(chunks):
|
||||
result = _chunk_chat_result(
|
||||
config,
|
||||
chunk_text,
|
||||
offset=offset,
|
||||
index=index,
|
||||
attempt=attempt,
|
||||
schema_retry=schema_retry,
|
||||
stats=stats,
|
||||
)
|
||||
stats.detect_calls += 1
|
||||
_add_usage(stats, result.usage)
|
||||
spans = _entities_from_result(result, chunk_text, index)
|
||||
collected.extend(_to_global(span, offset) for span in spans)
|
||||
stats.chunks_ok += 1
|
||||
if stats.chunks_ok != stats.chunk_count:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_INCOMPLETE,
|
||||
"Detection hat nicht alle Chunks geprüft. Generate wird nicht freigegeben.",
|
||||
diagnostics={"contract_violation": "incomplete"},
|
||||
)
|
||||
return collected
|
||||
|
||||
|
||||
def detect_personal_egress(profile_id: str | None, source_text: str) -> DetectionOutcome:
|
||||
"""Full semantic detection of the personal generate-egress. Fail closed if incomplete."""
|
||||
started = time.perf_counter()
|
||||
stats = DetectionStats(source_chars=len(source_text or ""))
|
||||
if _injected_fail is not None:
|
||||
stats.abort_reason = _injected_fail.code
|
||||
injected_fail = _injected_fail.get()
|
||||
if injected_fail is not None:
|
||||
stats.abort_reason = injected_fail.code
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
stats.detect_passes = 0
|
||||
raise DetectError(
|
||||
_injected_fail.code,
|
||||
_injected_fail.message,
|
||||
_injected_fail.status_code,
|
||||
{**(_injected_fail.diagnostics or {}), **stats.public()},
|
||||
injected_fail.code,
|
||||
injected_fail.message,
|
||||
injected_fail.status_code,
|
||||
{**(injected_fail.diagnostics or {}), **stats.public()},
|
||||
)
|
||||
config = detect_provider()
|
||||
if not config:
|
||||
@@ -606,83 +871,70 @@ def detect_personal_egress(profile_id: str | None, source_text: str) -> Detectio
|
||||
chunks = split_detect_chunks(source_text or "")
|
||||
stats.chunk_count = len(chunks)
|
||||
collected: list[DetectedSpan] = []
|
||||
try:
|
||||
for index, (offset, chunk_text) in enumerate(chunks):
|
||||
if _injected_truncated:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_TRUNCATED,
|
||||
"Detect-Ausgabe wurde abgeschnitten. Generate wird nicht freigegeben.",
|
||||
)
|
||||
if _injected_spans is not None:
|
||||
raw_items = [
|
||||
item
|
||||
for item in _injected_spans
|
||||
if int(item.get("start") or 0) >= offset
|
||||
and int(item.get("end") or 0) <= offset + len(chunk_text)
|
||||
]
|
||||
local_items = []
|
||||
for item in raw_items:
|
||||
local = dict(item)
|
||||
local["start"] = int(item["start"]) - offset
|
||||
local["end"] = int(item["end"]) - offset
|
||||
local_items.append(local)
|
||||
result = ChatResult(
|
||||
content=json.dumps({"entities": local_items}),
|
||||
model=config.model,
|
||||
usage={},
|
||||
finish_reason="stop",
|
||||
)
|
||||
stats.detect_note = "injected"
|
||||
elif config.mode == "fake":
|
||||
fake_items = _contract_fake_spans(chunk_text)
|
||||
result = ChatResult(
|
||||
content=json.dumps({"entities": fake_items}),
|
||||
model=config.model,
|
||||
usage={},
|
||||
finish_reason="stop",
|
||||
)
|
||||
stats.detect_note = "fake"
|
||||
elif uses_llm_detect(config):
|
||||
try:
|
||||
result = _llm_chunk(config, chunk_text)
|
||||
except PlaceholderError as exc:
|
||||
raise DetectError(exc.code, exc.message) from exc
|
||||
except ProviderError as exc:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_CHUNK,
|
||||
"Ein Detect-Chunk ist fehlgeschlagen. Generate wird nicht freigegeben.",
|
||||
exc.status_code,
|
||||
) from exc
|
||||
stats.detect_note = "local_llm" if config.local else "remote_llm"
|
||||
stats.detect_model = result.model or stats.detect_model
|
||||
else:
|
||||
raise DetectError(
|
||||
ERROR_DETECT_UNAVAILABLE,
|
||||
"Semantische Detection ist nicht verfügbar. Generate wird nicht freigegeben.",
|
||||
)
|
||||
stats.detect_calls += 1
|
||||
_add_usage(stats, result.usage)
|
||||
spans = _entities_from_result(result, chunk_text, index)
|
||||
collected.extend(_to_global(span, offset) for span in spans)
|
||||
stats.chunks_ok += 1
|
||||
except DetectError as exc:
|
||||
stats.abort_reason = exc.code
|
||||
stats.full_detection_coverage = False
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
raise DetectError(
|
||||
exc.code,
|
||||
exc.message,
|
||||
exc.status_code,
|
||||
{**(exc.diagnostics or {}), **stats.public()},
|
||||
) from exc
|
||||
if stats.chunks_ok != stats.chunk_count:
|
||||
stats.abort_reason = ERROR_DETECT_INCOMPLETE
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
raise DetectError(
|
||||
ERROR_DETECT_INCOMPLETE,
|
||||
"Detection hat nicht alle Chunks geprüft. Generate wird nicht freigegeben.",
|
||||
diagnostics=stats.public(),
|
||||
last_error: DetectError | None = None
|
||||
for attempt in range(1, DETECT_MAX_PASSES + 1):
|
||||
pass_stats = DetectionStats(
|
||||
source_chars=stats.source_chars,
|
||||
detect_provider=stats.detect_provider,
|
||||
detect_model=stats.detect_model,
|
||||
chunk_count=len(chunks),
|
||||
)
|
||||
pass_started = time.perf_counter()
|
||||
try:
|
||||
collected = _run_detection_pass(
|
||||
config,
|
||||
chunks,
|
||||
attempt=attempt,
|
||||
schema_retry=attempt > 1,
|
||||
stats=pass_stats,
|
||||
)
|
||||
pass_stats.full_detection_coverage = True
|
||||
pass_stats.detect_ms = int((time.perf_counter() - pass_started) * 1000)
|
||||
_merge_pass_stats(stats, pass_stats)
|
||||
stats.detect_attempts.append(
|
||||
_pass_snapshot(pass_stats, attempt=attempt, coverage=True)
|
||||
)
|
||||
last_error = None
|
||||
break
|
||||
except DetectError as exc:
|
||||
inner = dict(exc.diagnostics or {})
|
||||
pass_stats.abort_reason = exc.code
|
||||
pass_stats.contract_violation = inner.get("contract_violation")
|
||||
pass_stats.invalid_entity_type = inner.get("invalid_entity_type")
|
||||
pass_stats.full_detection_coverage = False
|
||||
pass_stats.detect_ms = int((time.perf_counter() - pass_started) * 1000)
|
||||
_merge_pass_stats(stats, pass_stats)
|
||||
stats.detect_attempts.append(
|
||||
_pass_snapshot(pass_stats, attempt=attempt, coverage=False, abort=exc.code)
|
||||
)
|
||||
collected = []
|
||||
last_error = DetectError(exc.code, exc.message, exc.status_code, inner)
|
||||
retryable = exc.code in DETECT_CONTRACT_RETRY_CODES
|
||||
if attempt < DETECT_MAX_PASSES and retryable:
|
||||
stats.detect_partial_discarded = True
|
||||
continue
|
||||
stats.abort_reason = exc.code
|
||||
stats.contract_violation = pass_stats.contract_violation
|
||||
stats.invalid_entity_type = pass_stats.invalid_entity_type
|
||||
stats.full_detection_coverage = False
|
||||
stats.detect_passes = attempt
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
raise DetectError(
|
||||
exc.code,
|
||||
exc.message,
|
||||
exc.status_code,
|
||||
{**inner, **stats.public()},
|
||||
) from exc
|
||||
if last_error is not None:
|
||||
stats.detect_passes = len(stats.detect_attempts)
|
||||
stats.detect_ms = int((time.perf_counter() - started) * 1000)
|
||||
raise DetectError(
|
||||
last_error.code,
|
||||
last_error.message,
|
||||
last_error.status_code,
|
||||
{**(last_error.diagnostics or {}), **stats.public()},
|
||||
)
|
||||
stats.detect_passes = len(stats.detect_attempts)
|
||||
merged = _dedupe_spans(collected)
|
||||
confirmed = list_confirmed_identities(profile_id) if profile_id else []
|
||||
mappings, request_local, confirmed_from_spans = _assign_request_tokens(merged, confirmed)
|
||||
|
||||
Reference in new issue
Block a user