Put student-first detect learning ahead of the teacher call.
Identity-only senses no longer lock a new cue. Learning stages, journal-generate review, and additive docs follow the closed section 6. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
a65ea98b0c
commit
011cc9dd81
26 files changed
+1893
-89
No files matched your search
+322
-20
@@ -1,12 +1,13 @@
|
||||
"""Transitional learning detect: senses grow from dialogue confirmations.
|
||||
|
||||
Not a global word list. Detect still does not auto-activate identities.
|
||||
Ambiguous spellings get a local passage call when a local detect provider exists.
|
||||
Stored passages are the context a later local GLiNER/detect can use for pre-decision.
|
||||
Student path: cues, kinship, homonym passage, then GLiNER. Teacher is optional.
|
||||
Uncertain mentions go as a mini-passage to a chosen LLM profile, not to Detect.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
@@ -21,10 +22,31 @@ from identity_store import (
|
||||
normalize_label,
|
||||
)
|
||||
|
||||
MODE_STUDENT = "student"
|
||||
MODE_LEARNING_FULL = "learning_full"
|
||||
MODE_LEARNING_REDUCED = "learning_reduced"
|
||||
MODE_LEARNING_MANUAL = "learning_manual"
|
||||
MODE_BYPASS = "bypass"
|
||||
MODE_SEMANTIC = "semantic"
|
||||
MODE_LEARNING = "learning"
|
||||
MODES = (MODE_SEMANTIC, MODE_LEARNING)
|
||||
CANONICAL_MODES = (
|
||||
MODE_STUDENT,
|
||||
MODE_LEARNING_FULL,
|
||||
MODE_LEARNING_REDUCED,
|
||||
MODE_LEARNING_MANUAL,
|
||||
MODE_BYPASS,
|
||||
)
|
||||
MODE_ALIASES = {
|
||||
MODE_SEMANTIC: MODE_BYPASS,
|
||||
MODE_LEARNING: MODE_LEARNING_FULL,
|
||||
}
|
||||
MODES = CANONICAL_MODES + (MODE_SEMANTIC, MODE_LEARNING)
|
||||
LEARNING_MODES = {MODE_LEARNING_FULL, MODE_LEARNING_REDUCED, MODE_LEARNING_MANUAL}
|
||||
SETTING_KEY = "detect_operating_mode"
|
||||
GLINER_ENABLED_KEY = "gliner_enabled"
|
||||
GLINER_URL_KEY = "gliner_url"
|
||||
PASSAGE_PROFILE_KEY = "passage_profile_id"
|
||||
GLINER_DEFAULT_URL = "http://192.168.2.144:8090"
|
||||
_LETTER = r"A-Za-zÄÖÜäöüß"
|
||||
WINDOW = 80
|
||||
|
||||
@@ -34,18 +56,32 @@ _DECISION_ASK = "ask"
|
||||
_CUE_BARE = "_bare"
|
||||
|
||||
|
||||
def canonicalize_detect_mode(mode: str | None) -> str | None:
|
||||
chosen = (mode or "").strip().lower()
|
||||
if not chosen:
|
||||
return None
|
||||
mapped = MODE_ALIASES.get(chosen, chosen)
|
||||
return mapped if mapped in CANONICAL_MODES else None
|
||||
|
||||
|
||||
def _default_detect_mode() -> str:
|
||||
if os.environ.get("KANSHO_FAKE_PROVIDER") or os.environ.get("KANSHO_FAKE_DETECT"):
|
||||
return MODE_BYPASS
|
||||
return MODE_LEARNING_FULL
|
||||
|
||||
|
||||
def get_detect_operating_mode() -> str:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
|
||||
)
|
||||
value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower()
|
||||
return value if value in MODES else MODE_SEMANTIC
|
||||
stored = canonicalize_detect_mode((row or {}).get("value") or "")
|
||||
return stored or _default_detect_mode()
|
||||
|
||||
|
||||
def set_detect_operating_mode(mode: str) -> str:
|
||||
chosen = (mode or "").strip().lower()
|
||||
if chosen not in MODES:
|
||||
chosen = canonicalize_detect_mode(mode)
|
||||
if not chosen:
|
||||
raise ValueError("invalid_detect_operating_mode")
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
@@ -59,6 +95,88 @@ def set_detect_operating_mode(mode: str) -> str:
|
||||
return chosen
|
||||
|
||||
|
||||
def is_bypass_mode(mode: str | None = None) -> bool:
|
||||
return (mode or get_detect_operating_mode()) == MODE_BYPASS
|
||||
|
||||
|
||||
def is_learning_mode(mode: str | None = None) -> bool:
|
||||
return (mode or get_detect_operating_mode()) in LEARNING_MODES
|
||||
|
||||
|
||||
def learning_stage(mode: str | None = None) -> int:
|
||||
current = mode or get_detect_operating_mode()
|
||||
if current == MODE_LEARNING_REDUCED:
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
def _app_setting(key: str, default: str = "") -> str:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT value FROM app_settings WHERE key = ?", (key,)).fetchone()
|
||||
)
|
||||
if not row or row.get("value") is None:
|
||||
return default
|
||||
return str(row.get("value") or "").strip() or default
|
||||
|
||||
|
||||
def _put_app_setting(key: str, value: str) -> None:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO app_settings (key, value, updated)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
|
||||
""",
|
||||
(key, value),
|
||||
)
|
||||
|
||||
|
||||
def get_local_ner_settings() -> dict:
|
||||
from gliner_client import health, normalize_base_url
|
||||
from providers import is_local_url
|
||||
|
||||
enabled_raw = _app_setting(GLINER_ENABLED_KEY, "0").lower()
|
||||
enabled = enabled_raw in {"1", "true", "yes", "on"}
|
||||
url = normalize_base_url(_app_setting(GLINER_URL_KEY, GLINER_DEFAULT_URL) or GLINER_DEFAULT_URL)
|
||||
passage_profile_id = _app_setting(PASSAGE_PROFILE_KEY, "")
|
||||
ping = health(url) if enabled and is_local_url(url) else {"ok": False, "status": "off"}
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"url": url,
|
||||
"passage_profile_id": passage_profile_id,
|
||||
"ready": bool(enabled and ping.get("ok")),
|
||||
"health": ping,
|
||||
}
|
||||
|
||||
|
||||
def set_local_ner_settings(
|
||||
*,
|
||||
enabled: bool | None = None,
|
||||
url: str | None = None,
|
||||
passage_profile_id: str | None = None,
|
||||
) -> dict:
|
||||
from gliner_client import normalize_base_url
|
||||
from providers import is_local_url
|
||||
|
||||
if enabled is not None:
|
||||
_put_app_setting(GLINER_ENABLED_KEY, "1" if enabled else "0")
|
||||
if url is not None:
|
||||
chosen = normalize_base_url(url) if (url or "").strip() else GLINER_DEFAULT_URL
|
||||
if not is_local_url(chosen):
|
||||
raise ValueError("gliner_url_not_local")
|
||||
_put_app_setting(GLINER_URL_KEY, chosen)
|
||||
if passage_profile_id is not None:
|
||||
token = (passage_profile_id or "").strip()
|
||||
if token:
|
||||
from provider_settings import get_profile
|
||||
|
||||
if not get_profile(token):
|
||||
raise ValueError("passage_profile_missing")
|
||||
_put_app_setting(PASSAGE_PROFILE_KEY, token)
|
||||
return get_local_ner_settings()
|
||||
|
||||
|
||||
def get_sense(profile_id: str, label: str) -> dict:
|
||||
key = normalize_label(label) or (label or "").strip()
|
||||
if not key:
|
||||
@@ -376,21 +494,16 @@ def decision_for_mention(
|
||||
end: int | None,
|
||||
rendered: str = "",
|
||||
) -> str:
|
||||
"""identity / not_identity from prior review, or ask once for a new cue."""
|
||||
"""identity / not_identity from a stored cue, otherwise ask.
|
||||
|
||||
A sense with only identity hits must not lock a new cue (Sushi day 21).
|
||||
"""
|
||||
if not label:
|
||||
return _DECISION_ASK
|
||||
cue = cue_for_mention(user_body, start, end, label, rendered)
|
||||
learned = get_cue_decision(profile_id, label, cue)
|
||||
if learned:
|
||||
return learned
|
||||
sense = get_sense(profile_id, label)
|
||||
identity_shaped = cue in KINSHIP
|
||||
if sense["ambiguous"]:
|
||||
return _DECISION_ASK
|
||||
if sense["identity_hits"] > 0 and not sense["non_identity_hits"]:
|
||||
return _DECISION_IDENTITY
|
||||
if sense["non_identity_hits"] > 0 and not sense["identity_hits"]:
|
||||
return _DECISION_ASK if identity_shaped else _DECISION_NOT
|
||||
return _DECISION_ASK
|
||||
|
||||
|
||||
@@ -533,6 +646,193 @@ def supplement_kinship_candidates(
|
||||
return extra_candidates, extra_mappings
|
||||
|
||||
|
||||
def merge_gliner_candidates(
|
||||
profile_id: str,
|
||||
candidates: list[dict],
|
||||
mappings: list[dict],
|
||||
user_body: str,
|
||||
rendered: str,
|
||||
spans: list[dict] | None,
|
||||
) -> tuple[list[dict], list[dict]]:
|
||||
"""Add GLiNER spans from the user line. Learned cues still win afterwards."""
|
||||
from gliner_client import gliner_identity_type, is_gliner_non_identity
|
||||
|
||||
extra_candidates = list(candidates)
|
||||
extra_mappings = list(mappings)
|
||||
if not spans:
|
||||
return extra_candidates, extra_mappings
|
||||
occupied: set[tuple[int, int, str]] = set()
|
||||
for item in extra_candidates:
|
||||
span = _resolve_user_span(
|
||||
user_body,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if span:
|
||||
occupied.add((span[0], span[1], (item.get("text") or "").casefold()))
|
||||
food_keys: set[tuple[int, int, str]] = set()
|
||||
for raw in spans:
|
||||
label = raw.get("text") or ""
|
||||
kind = raw.get("label") or ""
|
||||
try:
|
||||
start, end = int(raw["start"]), int(raw["end"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
if start < 0 or end > len(user_body) or end <= start:
|
||||
continue
|
||||
if user_body[start:end].casefold() != label.casefold():
|
||||
resolved = _resolve_user_span(user_body, start, end, label, rendered)
|
||||
if not resolved:
|
||||
continue
|
||||
start, end = resolved
|
||||
label = user_body[start:end]
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
key = (start, end, label.casefold())
|
||||
if is_gliner_non_identity(kind):
|
||||
food_keys.add(key)
|
||||
continue
|
||||
entity_type = gliner_identity_type(kind)
|
||||
if not entity_type or key in occupied:
|
||||
continue
|
||||
if decision_for_mention(profile_id, label, user_body, start, end, rendered) != _DECISION_ASK:
|
||||
continue
|
||||
occupied.add(key)
|
||||
aligned = _align_user_span(rendered, user_body, start, end)
|
||||
extra_candidates.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": entity_type,
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(user_body, start, end, label, rendered),
|
||||
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||
"suggested": _DECISION_IDENTITY,
|
||||
}
|
||||
)
|
||||
extra_mappings.append(
|
||||
{
|
||||
"token": _token_for_label(extra_mappings, label, entity_type),
|
||||
"local_label": label,
|
||||
"canonical_label": label,
|
||||
"demask_label": label,
|
||||
"entity_type": entity_type,
|
||||
"source": "request_local",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
"aliases": [],
|
||||
"labels": [label],
|
||||
}
|
||||
)
|
||||
for item in extra_candidates:
|
||||
span = _resolve_user_span(
|
||||
user_body,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if not span:
|
||||
continue
|
||||
key = (span[0], span[1], (item.get("text") or "").casefold())
|
||||
if key in food_keys and not attachment_kinship(user_body, span[0], span[1]):
|
||||
item["suggested"] = _DECISION_NOT
|
||||
for start, end, folded in food_keys:
|
||||
if (start, end, folded) in occupied:
|
||||
continue
|
||||
label = user_body[start:end]
|
||||
if decision_for_mention(profile_id, label, user_body, start, end, rendered) != _DECISION_ASK:
|
||||
continue
|
||||
occupied.add((start, end, folded))
|
||||
aligned = _align_user_span(rendered, user_body, start, end)
|
||||
extra_candidates.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": "PERSON",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(user_body, start, end, label, rendered),
|
||||
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||
"suggested": _DECISION_NOT,
|
||||
}
|
||||
)
|
||||
extra_mappings.append(
|
||||
{
|
||||
"token": _token_for_label(extra_mappings, label),
|
||||
"local_label": label,
|
||||
"canonical_label": label,
|
||||
"demask_label": label,
|
||||
"entity_type": "PERSON",
|
||||
"source": "request_local",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
"aliases": [],
|
||||
"labels": [label],
|
||||
}
|
||||
)
|
||||
return extra_candidates, extra_mappings
|
||||
|
||||
|
||||
def apply_gliner_if_enabled(
|
||||
profile_id: str,
|
||||
candidates: list[dict],
|
||||
mappings: list[dict],
|
||||
user_body: str,
|
||||
rendered: str = "",
|
||||
) -> tuple[list[dict], list[dict]]:
|
||||
settings = get_local_ner_settings()
|
||||
if not settings.get("enabled"):
|
||||
return candidates, mappings
|
||||
from gliner_client import extract_entities
|
||||
|
||||
spans = extract_entities(user_body, url=settings.get("url"))
|
||||
if spans is None:
|
||||
return candidates, mappings
|
||||
return merge_gliner_candidates(profile_id, candidates, mappings, user_body, rendered, spans)
|
||||
|
||||
|
||||
def passage_provider():
|
||||
"""LLM profile for mini-passage decisions. Independent of the detect role."""
|
||||
import os
|
||||
|
||||
from providers import ProviderConfig, is_local_url
|
||||
from provider_settings import get_profile
|
||||
|
||||
profile_id = _app_setting(PASSAGE_PROFILE_KEY, "")
|
||||
if not profile_id:
|
||||
return None
|
||||
profile = get_profile(profile_id)
|
||||
if not profile:
|
||||
return None
|
||||
url = profile.get("url") or ""
|
||||
model = profile.get("model") or ""
|
||||
if not url or not model:
|
||||
return None
|
||||
local = is_local_url(url)
|
||||
key = ""
|
||||
if not local:
|
||||
key = (os.environ.get("KANSHO_PROVIDER_KEY") or os.environ.get("KANSHO_DETECT_PROVIDER_KEY") or "").strip()
|
||||
if not key:
|
||||
return None
|
||||
if not profile.get("zdr") or not profile.get("no_train"):
|
||||
return None
|
||||
return ProviderConfig(
|
||||
role="passage",
|
||||
name=profile.get("name") or ("ollama" if local else "openrouter"),
|
||||
mode="http",
|
||||
url=url,
|
||||
model=model,
|
||||
key=key,
|
||||
local=local,
|
||||
zdr=True if local else bool(profile.get("zdr")),
|
||||
no_train=True if local else bool(profile.get("no_train")),
|
||||
)
|
||||
|
||||
|
||||
def build_candidates(profile_id: str, mappings: list[dict], user_body: str, rendered: str = "") -> list[dict]:
|
||||
seen: set[tuple[int | None, int | None, str]] = set()
|
||||
items: list[dict] = []
|
||||
@@ -721,6 +1021,8 @@ def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | No
|
||||
return {
|
||||
"id": row["id"],
|
||||
"candidates": payload.get("candidates") or [],
|
||||
"kind": payload.get("kind") or "diff",
|
||||
"purpose": payload.get("purpose") or "dialogue_turn",
|
||||
}
|
||||
|
||||
|
||||
@@ -756,11 +1058,11 @@ def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: li
|
||||
|
||||
|
||||
def try_local_passage_decision(excerpt: str, label: str, examples: list[dict] | None = None) -> str | None:
|
||||
"""Return identity, not_identity, or None if no local model or unusable answer."""
|
||||
from providers import ProviderError, complete_chat, detect_provider
|
||||
"""Return identity, not_identity, or None. Uses the admin passage profile, not Detect."""
|
||||
from providers import ProviderError, complete_chat
|
||||
|
||||
config = detect_provider()
|
||||
if not config or not config.local or config.mode != "http":
|
||||
config = passage_provider()
|
||||
if not config or config.mode != "http":
|
||||
return None
|
||||
few_shot = ""
|
||||
lines = []
|
||||
|
||||
@@ -0,0 +1,618 @@
|
||||
"""Student-first detect pipeline with optional teacher comparison.
|
||||
|
||||
Cues stay local. OpenRouter/GLiNER never create identities. Compact counters
|
||||
carry no plaintext labels.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from detect_learning import (
|
||||
MODE_LEARNING_MANUAL,
|
||||
MODE_STUDENT,
|
||||
_CUE_BARE,
|
||||
_DECISION_ASK,
|
||||
_DECISION_IDENTITY,
|
||||
_DECISION_NOT,
|
||||
_align_user_span,
|
||||
_mention_fields,
|
||||
_resolve_user_span,
|
||||
_token_for_label,
|
||||
_word_matches,
|
||||
apply_learned_decisions,
|
||||
auto_resolve_ambiguous,
|
||||
cue_for_mention,
|
||||
decision_for_mention,
|
||||
get_cue_decision,
|
||||
get_detect_operating_mode,
|
||||
get_local_ner_settings,
|
||||
get_sense,
|
||||
is_bypass_mode,
|
||||
is_learning_mode,
|
||||
learning_stage,
|
||||
merge_gliner_candidates,
|
||||
record_cue,
|
||||
supplement_kinship_candidates,
|
||||
)
|
||||
from entity_detect import DetectError, detect_personal_egress
|
||||
from identity_store import ENTITY_TYPES, is_maskable_label
|
||||
|
||||
KIND_DIFF = "diff"
|
||||
KIND_MANUAL = "manual"
|
||||
KIND_CONFIRM_REMOTE = "confirm_remote"
|
||||
|
||||
PROTECTING_TYPES = set(ENTITY_TYPES)
|
||||
FOOD_TYPES = frozenset(
|
||||
{"FOOD", "DISH", "MEAL", "CUISINE", "FOODSTUFF", "INGREDIENT", "food", "dish"}
|
||||
)
|
||||
TYPE_MAP = {
|
||||
"person": "PERSON",
|
||||
"place": "PLACE",
|
||||
"org": "ORG",
|
||||
"organization": "ORG",
|
||||
"project": "PROJECT",
|
||||
"food": "FOOD",
|
||||
"dish": "DISH",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MentionSpan:
|
||||
text: str
|
||||
start: int
|
||||
end: int
|
||||
entity_type: str
|
||||
protection: str
|
||||
source: str
|
||||
confirmed_cue: bool = False
|
||||
cue: str = _CUE_BARE
|
||||
|
||||
def key(self) -> tuple[int, int, str]:
|
||||
return (self.start, self.end, self.text.casefold())
|
||||
|
||||
|
||||
@dataclass
|
||||
class StudentResult:
|
||||
mappings: list[dict] = field(default_factory=list)
|
||||
candidates: list[dict] = field(default_factory=list)
|
||||
spans: list[MentionSpan] = field(default_factory=list)
|
||||
gliner_missing: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class TeacherResult:
|
||||
spans: list[MentionSpan] = field(default_factory=list)
|
||||
mappings: list[dict] = field(default_factory=list)
|
||||
ok: bool = True
|
||||
error: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineResult:
|
||||
mappings: list[dict]
|
||||
candidates: list[dict]
|
||||
kind: str = KIND_DIFF
|
||||
need_review: bool = False
|
||||
need_remote_confirm: bool = False
|
||||
counters: dict[str, int] = field(default_factory=dict)
|
||||
mode: str = ""
|
||||
|
||||
|
||||
def empty_counters() -> dict[str, int]:
|
||||
return {
|
||||
"diff_agree": 0,
|
||||
"diff_only_student": 0,
|
||||
"diff_only_teacher": 0,
|
||||
"diff_type_conflict": 0,
|
||||
"diff_new_identity": 0,
|
||||
"diff_cue_reinforced": 0,
|
||||
}
|
||||
|
||||
|
||||
def map_entity_type(raw: str) -> str:
|
||||
token = (raw or "").strip()
|
||||
if token.upper() in PROTECTING_TYPES or token.upper() in {item.upper() for item in FOOD_TYPES}:
|
||||
mapped = TYPE_MAP.get(token.lower())
|
||||
return mapped or token.upper()
|
||||
return TYPE_MAP.get(token.lower()) or token.upper()
|
||||
|
||||
|
||||
def protects(entity_type: str) -> bool:
|
||||
kind = map_entity_type(entity_type)
|
||||
if kind in FOOD_TYPES or kind.upper() in {item.upper() for item in FOOD_TYPES}:
|
||||
return False
|
||||
return kind in PROTECTING_TYPES
|
||||
|
||||
|
||||
def types_equivalent(left: str, right: str) -> bool:
|
||||
a, b = map_entity_type(left), map_entity_type(right)
|
||||
if a in FOOD_TYPES or b in FOOD_TYPES or a in {"FOOD", "DISH"} or b in {"FOOD", "DISH"}:
|
||||
return a == b
|
||||
if protects(a) and protects(b):
|
||||
return True
|
||||
return a == b
|
||||
|
||||
|
||||
def _contains(outer: MentionSpan, inner: MentionSpan) -> bool:
|
||||
return outer.start <= inner.start and inner.end <= outer.end and outer.end > outer.start
|
||||
|
||||
|
||||
def same_mention(left: MentionSpan, right: MentionSpan) -> bool:
|
||||
if left.text.casefold() == right.text.casefold() and left.start == right.start and left.end == right.end:
|
||||
return True
|
||||
if _contains(left, right) or _contains(right, left):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _collapse_same_protection(spans: list[MentionSpan]) -> list[MentionSpan]:
|
||||
if not spans:
|
||||
return []
|
||||
remaining = list(spans)
|
||||
chosen: list[MentionSpan] = []
|
||||
while remaining:
|
||||
current = remaining.pop(0)
|
||||
cluster = [current]
|
||||
rest = []
|
||||
for item in remaining:
|
||||
if same_mention(current, item):
|
||||
cluster.append(item)
|
||||
else:
|
||||
rest.append(item)
|
||||
remaining = rest
|
||||
protections = {item.protection for item in cluster}
|
||||
if len(protections) > 1:
|
||||
chosen.extend(cluster)
|
||||
continue
|
||||
chosen.append(max(cluster, key=lambda item: (item.end - item.start, -item.start)))
|
||||
return chosen
|
||||
|
||||
|
||||
def _user_span_for_mapping(mapping: dict, user_body: str, rendered: str = "") -> tuple[int, int] | None:
|
||||
return _resolve_user_span(
|
||||
user_body,
|
||||
mapping.get("start"),
|
||||
mapping.get("end"),
|
||||
mapping.get("local_label") or "",
|
||||
rendered,
|
||||
)
|
||||
|
||||
|
||||
def _mapping_from_user_span(
|
||||
mappings: list[dict],
|
||||
label: str,
|
||||
user_body: str,
|
||||
start: int,
|
||||
end: int,
|
||||
entity_type: str,
|
||||
rendered: str,
|
||||
) -> dict:
|
||||
aligned = _align_user_span(rendered, user_body, start, end)
|
||||
kind = entity_type if entity_type in ENTITY_TYPES else "PERSON"
|
||||
return {
|
||||
"token": _token_for_label(mappings, label, kind),
|
||||
"local_label": label,
|
||||
"canonical_label": label,
|
||||
"demask_label": label,
|
||||
"entity_type": kind,
|
||||
"source": "request_local",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
"aliases": [],
|
||||
"labels": [label],
|
||||
}
|
||||
|
||||
|
||||
def run_student_path(profile_id: str, user_body: str, rendered: str = "") -> StudentResult:
|
||||
mappings: list[dict] = []
|
||||
candidates: list[dict] = []
|
||||
spans: list[MentionSpan] = []
|
||||
text = user_body or ""
|
||||
for match in _word_matches(text):
|
||||
label = match.group(0)
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
start, end = match.start(), match.end()
|
||||
verdict = decision_for_mention(profile_id, label, text, start, end, rendered)
|
||||
cue = cue_for_mention(text, start, end, label, rendered)
|
||||
if verdict == _DECISION_IDENTITY:
|
||||
mappings.append(_mapping_from_user_span(mappings, label, text, start, end, "PERSON", rendered))
|
||||
spans.append(
|
||||
MentionSpan(label, start, end, "PERSON", _DECISION_IDENTITY, "cue", True, cue)
|
||||
)
|
||||
elif verdict == _DECISION_NOT:
|
||||
spans.append(
|
||||
MentionSpan(label, start, end, "FOOD", _DECISION_NOT, "cue", True, cue)
|
||||
)
|
||||
elif get_sense(profile_id, label)["ambiguous"]:
|
||||
aligned = _align_user_span(rendered, text, start, end)
|
||||
candidates.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": "PERSON",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(text, start, end, label, rendered),
|
||||
"ambiguous": True,
|
||||
}
|
||||
)
|
||||
candidates, mappings = supplement_kinship_candidates(
|
||||
profile_id, candidates, mappings, text, rendered
|
||||
)
|
||||
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
|
||||
bound = {(span.start, span.end, span.text.casefold()) for span in spans}
|
||||
for item in candidates:
|
||||
resolved = _resolve_user_span(
|
||||
text,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if resolved:
|
||||
bound.add((resolved[0], resolved[1], (item.get("text") or "").casefold()))
|
||||
settings = get_local_ner_settings()
|
||||
gliner_missing = False
|
||||
if settings.get("enabled"):
|
||||
from gliner_client import extract_entities
|
||||
|
||||
raw = extract_entities(text, url=settings.get("url"))
|
||||
if raw is None:
|
||||
gliner_missing = True
|
||||
else:
|
||||
usable = []
|
||||
for item in raw:
|
||||
try:
|
||||
start, end = int(item["start"]), int(item["end"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
label = (item.get("text") or "").casefold()
|
||||
if any(
|
||||
not (end <= left or start >= right) and folded == label
|
||||
for left, right, folded in bound
|
||||
) or any(left <= start and end <= right for left, right, _folded in bound):
|
||||
continue
|
||||
usable.append(item)
|
||||
candidates, mappings = merge_gliner_candidates(
|
||||
profile_id, candidates, mappings, text, rendered, usable
|
||||
)
|
||||
candidates, mappings = apply_learned_decisions(profile_id, candidates, mappings, text, rendered)
|
||||
for item in candidates:
|
||||
resolved = _resolve_user_span(
|
||||
text,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if not resolved:
|
||||
continue
|
||||
suggested = item.get("suggested") or _DECISION_IDENTITY
|
||||
kind = item.get("entity_type") or "PERSON"
|
||||
protection = _DECISION_NOT if suggested == _DECISION_NOT else _DECISION_IDENTITY
|
||||
entity = "FOOD" if protection == _DECISION_NOT else kind
|
||||
spans.append(
|
||||
MentionSpan(
|
||||
item.get("text") or "",
|
||||
resolved[0],
|
||||
resolved[1],
|
||||
entity,
|
||||
protection,
|
||||
"candidate",
|
||||
False,
|
||||
item.get("cue") or _CUE_BARE,
|
||||
)
|
||||
)
|
||||
return StudentResult(
|
||||
mappings=mappings,
|
||||
candidates=candidates,
|
||||
spans=_collapse_same_protection(spans),
|
||||
gliner_missing=gliner_missing,
|
||||
)
|
||||
|
||||
|
||||
def blank_cue_safe_forms(profile_id: str, user_body: str, rendered: str = "") -> str:
|
||||
chars = list(user_body or "")
|
||||
for match in _word_matches(user_body or ""):
|
||||
label = match.group(0)
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
verdict = decision_for_mention(
|
||||
profile_id, label, user_body, match.start(), match.end(), rendered
|
||||
)
|
||||
if verdict not in {_DECISION_IDENTITY, _DECISION_NOT}:
|
||||
continue
|
||||
for index in range(match.start(), match.end()):
|
||||
chars[index] = " "
|
||||
return "".join(chars)
|
||||
|
||||
|
||||
def _teacher_span(label: str, start: int, end: int, entity_type: str) -> MentionSpan:
|
||||
kind = map_entity_type(entity_type)
|
||||
protection = _DECISION_IDENTITY if protects(kind) else _DECISION_NOT
|
||||
return MentionSpan(label, start, end, kind, protection, "teacher", False)
|
||||
|
||||
|
||||
def run_teacher(
|
||||
profile_id: str,
|
||||
user_body: str,
|
||||
*,
|
||||
stage: int = 1,
|
||||
rendered: str = "",
|
||||
) -> TeacherResult:
|
||||
source = user_body or ""
|
||||
if stage >= 2:
|
||||
source = blank_cue_safe_forms(profile_id, user_body, rendered)
|
||||
try:
|
||||
outcome = detect_personal_egress(profile_id, source)
|
||||
except DetectError as exc:
|
||||
return TeacherResult(ok=False, error=exc.code)
|
||||
spans: list[MentionSpan] = []
|
||||
for mapping in outcome.mappings or []:
|
||||
label = mapping.get("local_label") or ""
|
||||
start, end = mapping.get("start"), mapping.get("end")
|
||||
resolved = None
|
||||
if start is not None and end is not None:
|
||||
left, right = int(start), int(end)
|
||||
if 0 <= left < right <= len(user_body) and user_body[left:right].casefold() == label.casefold():
|
||||
resolved = (left, right)
|
||||
if not resolved:
|
||||
resolved = _resolve_user_span(user_body, start, end, label, user_body)
|
||||
if not resolved:
|
||||
continue
|
||||
spans.append(
|
||||
_teacher_span(label, resolved[0], resolved[1], mapping.get("entity_type") or "PERSON")
|
||||
)
|
||||
omitted = list((outcome.stats.omitted_non_identity_types or []) if outcome.stats else [])
|
||||
for kind in omitted:
|
||||
if map_entity_type(kind) in FOOD_TYPES or kind.upper() in {"FOOD", "DISH"}:
|
||||
continue
|
||||
return TeacherResult(spans=_collapse_same_protection(spans), mappings=list(outcome.mappings or []), ok=True)
|
||||
|
||||
|
||||
def _candidate_from_span(
|
||||
span: MentionSpan,
|
||||
user_body: str,
|
||||
rendered: str,
|
||||
*,
|
||||
suggested: str,
|
||||
reason: str,
|
||||
) -> dict:
|
||||
aligned = _align_user_span(rendered, user_body, span.start, span.end)
|
||||
kind = span.entity_type if span.entity_type in ENTITY_TYPES else "PERSON"
|
||||
return {
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": span.text,
|
||||
"entity_type": kind,
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(user_body, span.start, span.end, span.text, rendered),
|
||||
"ambiguous": False,
|
||||
"suggested": suggested,
|
||||
"reason": reason,
|
||||
}
|
||||
|
||||
|
||||
def compare_mentions(
|
||||
profile_id: str,
|
||||
user_body: str,
|
||||
student_spans: list[MentionSpan],
|
||||
teacher_spans: list[MentionSpan],
|
||||
rendered: str = "",
|
||||
) -> tuple[list[dict], dict[str, int]]:
|
||||
counters = empty_counters()
|
||||
review: list[dict] = []
|
||||
used_teacher: set[int] = set()
|
||||
student_list = _collapse_same_protection(student_spans)
|
||||
teacher_list = _collapse_same_protection(teacher_spans)
|
||||
|
||||
def take_teacher(index: int) -> MentionSpan:
|
||||
used_teacher.add(index)
|
||||
return teacher_list[index]
|
||||
|
||||
for student in student_list:
|
||||
match_index = next(
|
||||
(
|
||||
index
|
||||
for index, teacher in enumerate(teacher_list)
|
||||
if index not in used_teacher and same_mention(student, teacher)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if match_index is None:
|
||||
counters["diff_only_student"] += 1
|
||||
if student.confirmed_cue:
|
||||
continue
|
||||
if student.protection == _DECISION_IDENTITY:
|
||||
counters["diff_new_identity"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_student"
|
||||
)
|
||||
)
|
||||
continue
|
||||
teacher = take_teacher(match_index)
|
||||
if student.protection != teacher.protection or (
|
||||
protects(student.entity_type) != protects(teacher.entity_type)
|
||||
):
|
||||
counters["diff_type_conflict"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
student,
|
||||
user_body,
|
||||
rendered,
|
||||
suggested=student.protection,
|
||||
reason="type_conflict",
|
||||
)
|
||||
)
|
||||
continue
|
||||
if types_equivalent(student.entity_type, teacher.entity_type) or (
|
||||
student.protection == teacher.protection
|
||||
):
|
||||
counters["diff_agree"] += 1
|
||||
if student.confirmed_cue:
|
||||
record_cue(profile_id, student.text, student.cue, student.protection)
|
||||
counters["diff_cue_reinforced"] += 1
|
||||
elif student.protection == _DECISION_IDENTITY:
|
||||
learned = get_cue_decision(profile_id, student.text, student.cue)
|
||||
if learned == _DECISION_IDENTITY:
|
||||
record_cue(profile_id, student.text, student.cue, _DECISION_IDENTITY)
|
||||
counters["diff_cue_reinforced"] += 1
|
||||
else:
|
||||
counters["diff_new_identity"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="new_identity"
|
||||
)
|
||||
)
|
||||
for index, teacher in enumerate(teacher_list):
|
||||
if index in used_teacher:
|
||||
continue
|
||||
counters["diff_only_teacher"] += 1
|
||||
cue = cue_for_mention(user_body, teacher.start, teacher.end, teacher.text, rendered)
|
||||
learned = get_cue_decision(profile_id, teacher.text, cue)
|
||||
if learned:
|
||||
continue
|
||||
if teacher.protection == _DECISION_IDENTITY:
|
||||
counters["diff_new_identity"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
teacher, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_teacher"
|
||||
)
|
||||
)
|
||||
return review, counters
|
||||
|
||||
|
||||
def _remote_confirm_candidate(user_body: str) -> dict:
|
||||
snippet = (user_body or "")[:80]
|
||||
return {
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": "Klartext-Detect",
|
||||
"entity_type": "PERSON",
|
||||
"start": 0,
|
||||
"end": 0,
|
||||
"excerpt": snippet,
|
||||
"highlight_start": None,
|
||||
"highlight_end": None,
|
||||
"user_start": None,
|
||||
"user_end": None,
|
||||
"cue": _CUE_BARE,
|
||||
"ambiguous": False,
|
||||
"suggested": _DECISION_NOT,
|
||||
"reason": KIND_CONFIRM_REMOTE,
|
||||
}
|
||||
|
||||
|
||||
def prepare_detect_pipeline(profile_id: str, user_body: str, rendered: str = "") -> PipelineResult:
|
||||
mode = get_detect_operating_mode()
|
||||
counters = empty_counters()
|
||||
if is_bypass_mode(mode):
|
||||
return PipelineResult(mappings=[], candidates=[], kind=KIND_DIFF, mode=mode, counters=counters)
|
||||
student = run_student_path(profile_id, user_body, rendered)
|
||||
if mode == MODE_STUDENT:
|
||||
if student.gliner_missing:
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=[_remote_confirm_candidate(user_body)],
|
||||
kind=KIND_CONFIRM_REMOTE,
|
||||
need_review=True,
|
||||
need_remote_confirm=True,
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=[],
|
||||
kind=KIND_DIFF,
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
if mode == MODE_LEARNING_MANUAL or student.gliner_missing:
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=student.candidates,
|
||||
kind=KIND_MANUAL,
|
||||
need_review=bool(student.candidates),
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
teacher = run_teacher(profile_id, user_body, stage=learning_stage(mode), rendered=rendered)
|
||||
if not teacher.ok:
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=student.candidates,
|
||||
kind=KIND_MANUAL,
|
||||
need_review=bool(student.candidates),
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
review, counters = compare_mentions(
|
||||
profile_id, user_body, student.spans, teacher.spans, rendered
|
||||
)
|
||||
merged = list(student.mappings)
|
||||
seen = {
|
||||
(
|
||||
item.get("start"),
|
||||
item.get("end"),
|
||||
(item.get("local_label") or "").casefold(),
|
||||
)
|
||||
for item in merged
|
||||
}
|
||||
for mapping in teacher.mappings:
|
||||
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||
if key in seen:
|
||||
continue
|
||||
label = mapping.get("local_label") or ""
|
||||
resolved = _resolve_user_span(
|
||||
user_body, mapping.get("start"), mapping.get("end"), label, user_body
|
||||
)
|
||||
if resolved:
|
||||
cue = cue_for_mention(user_body, resolved[0], resolved[1], label, rendered)
|
||||
if get_cue_decision(profile_id, label, cue) == _DECISION_NOT:
|
||||
continue
|
||||
merged.append(mapping)
|
||||
seen.add(key)
|
||||
return PipelineResult(
|
||||
mappings=merged,
|
||||
candidates=review,
|
||||
kind=KIND_DIFF,
|
||||
need_review=bool(review),
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
|
||||
|
||||
def compact_diff_counters(counters: dict[str, int] | None) -> dict[str, int]:
|
||||
payload = empty_counters()
|
||||
for key in payload:
|
||||
payload[key] = int((counters or {}).get(key) or 0)
|
||||
return payload
|
||||
|
||||
|
||||
def apply_remote_confirm(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
|
||||
"""After the user allows plaintext detect, run teacher on the user line."""
|
||||
approved = any((item.get("decision") or "").strip() == _DECISION_IDENTITY for item in decisions)
|
||||
if not approved:
|
||||
raise DetectError(
|
||||
"remote_detect_declined",
|
||||
"Klartext-Detect wurde nicht bestätigt.",
|
||||
403,
|
||||
)
|
||||
user_body = pending.get("user_body") or ""
|
||||
teacher = run_teacher(profile_id, user_body, stage=1)
|
||||
if not teacher.ok:
|
||||
raise DetectError(teacher.error or "detect_provider_unavailable", "Lehrer-Detect ist nicht erreichbar.")
|
||||
merged = list(pending.get("mappings") or [])
|
||||
seen = {
|
||||
(item.get("start"), item.get("end"), (item.get("local_label") or "").casefold())
|
||||
for item in merged
|
||||
}
|
||||
for mapping in teacher.mappings:
|
||||
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||
if key in seen:
|
||||
continue
|
||||
merged.append(mapping)
|
||||
seen.add(key)
|
||||
return merged
|
||||
+37
-31
@@ -8,22 +8,18 @@ from context_builder import assemble_text, build_internal_context, is_closing_tu
|
||||
from conversation_signals import infer_signals
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from detect_learning import (
|
||||
MODE_LEARNING,
|
||||
apply_learned_decisions,
|
||||
apply_review_decisions,
|
||||
auto_resolve_ambiguous,
|
||||
build_candidates,
|
||||
confirm_known_identity_spans,
|
||||
drop_pending,
|
||||
get_detect_operating_mode,
|
||||
is_bypass_mode,
|
||||
load_pending,
|
||||
save_pending,
|
||||
supplement_kinship_candidates,
|
||||
suppress_known_non_identity,
|
||||
)
|
||||
from detect_pipeline import KIND_CONFIRM_REMOTE, prepare_detect_pipeline
|
||||
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
|
||||
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
||||
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError, detect_personal_egress
|
||||
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError
|
||||
from writing_profile_store import remember_dialogue_style
|
||||
from profile_review import consider_dialogue
|
||||
|
||||
@@ -331,6 +327,7 @@ def _finish_turn(
|
||||
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
|
||||
call_traces: list[dict] = []
|
||||
working = dict(assembled)
|
||||
working["user_body"] = user.get("body") or working.get("user_body") or ""
|
||||
try:
|
||||
while calls < 2:
|
||||
result = execute_prompt(
|
||||
@@ -431,24 +428,22 @@ def _finish_turn(
|
||||
}
|
||||
|
||||
|
||||
def _review_pause_payload(review_id: str, candidates: list[dict], kind: str, purpose: str = "dialogue_turn") -> dict:
|
||||
return {
|
||||
"id": review_id,
|
||||
"candidates": candidates,
|
||||
"kind": kind,
|
||||
"purpose": purpose,
|
||||
}
|
||||
|
||||
|
||||
def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled: dict, prompt: dict) -> dict | None:
|
||||
preview = preview_prompt(prompt, assembled)
|
||||
rendered = preview.get("rendered") or ""
|
||||
try:
|
||||
outcome = detect_personal_egress(profile_id, rendered)
|
||||
except DetectError:
|
||||
return None
|
||||
user_body = user.get("body") or ""
|
||||
mappings = list(outcome.mappings or [])
|
||||
candidates = build_candidates(profile_id, mappings, user_body, rendered)
|
||||
candidates, mappings = supplement_kinship_candidates(
|
||||
profile_id, candidates, mappings, user_body, rendered
|
||||
)
|
||||
candidates, mappings = apply_learned_decisions(
|
||||
profile_id, candidates, mappings, user_body, rendered
|
||||
)
|
||||
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
|
||||
if not candidates:
|
||||
result = prepare_detect_pipeline(profile_id, user_body, rendered)
|
||||
mappings = list(result.mappings or [])
|
||||
if not result.need_review and not result.candidates:
|
||||
confirm_known_identity_spans(profile_id, mappings, user_body)
|
||||
mappings = suppress_known_non_identity(profile_id, mappings, user_body)
|
||||
return _finish_turn(
|
||||
@@ -465,8 +460,11 @@ def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled
|
||||
user.get("id") or "",
|
||||
{
|
||||
"mappings": mappings,
|
||||
"candidates": candidates,
|
||||
"candidates": result.candidates,
|
||||
"user_body": user_body,
|
||||
"kind": result.kind,
|
||||
"counters": result.counters,
|
||||
"purpose": "dialogue_turn",
|
||||
},
|
||||
)
|
||||
return {
|
||||
@@ -475,10 +473,7 @@ def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled
|
||||
"assistant": None,
|
||||
"calls": 0,
|
||||
"messages": list_messages(profile_id, conversation_id),
|
||||
"pending_mask_review": {
|
||||
"id": review_id,
|
||||
"candidates": candidates,
|
||||
},
|
||||
"pending_mask_review": _review_pause_payload(review_id, result.candidates, result.kind),
|
||||
}
|
||||
|
||||
|
||||
@@ -494,10 +489,11 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||
)
|
||||
assembled = assemble_text(context)
|
||||
prompt = load_active_prompt("mvp.dialogue_turn")
|
||||
if get_detect_operating_mode() == MODE_LEARNING:
|
||||
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
|
||||
if paused is not None:
|
||||
return paused
|
||||
if is_bypass_mode():
|
||||
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
|
||||
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
|
||||
if paused is not None:
|
||||
return paused
|
||||
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
|
||||
|
||||
|
||||
@@ -505,7 +501,17 @@ def continue_turn(profile_id: str, conversation_id: str, review_id: str, decisio
|
||||
pending = load_pending(profile_id, review_id)
|
||||
if not pending or pending.get("conversation_id") != conversation_id:
|
||||
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||
mappings = apply_review_decisions(profile_id, pending, decisions)
|
||||
if pending.get("purpose") == "journal_generate":
|
||||
raise EngineError("mask_review_missing", "Diese Prüfung gehört zum Journalentwurf.", 409)
|
||||
if pending.get("kind") == KIND_CONFIRM_REMOTE:
|
||||
try:
|
||||
from detect_pipeline import apply_remote_confirm
|
||||
|
||||
mappings = apply_remote_confirm(profile_id, pending, decisions)
|
||||
except DetectError as exc:
|
||||
raise EngineError(exc.code, exc.message, getattr(exc, "status_code", 403)) from exc
|
||||
else:
|
||||
mappings = apply_review_decisions(profile_id, pending, decisions)
|
||||
drop_pending(profile_id, review_id)
|
||||
conversation = get_conversation(profile_id, conversation_id)
|
||||
context = build_internal_context(
|
||||
|
||||
@@ -76,6 +76,7 @@ def execute_prompt(
|
||||
budget=None,
|
||||
diagnostics: dict[str, Any] | None = None,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
detect_counters: dict | None = None,
|
||||
) -> dict:
|
||||
preview = preview_prompt(prompt, context)
|
||||
feature_id = prompt.get("required_feature") or "ai_calls"
|
||||
@@ -101,6 +102,7 @@ def execute_prompt(
|
||||
payload={
|
||||
"rendered": preview["rendered"],
|
||||
"source_text": source_text,
|
||||
"user_body": (ctx.get("user_body") or user_source_text(ctx.get("dialogue_context") or "")).strip(),
|
||||
"privacy_tokens": preview["privacy_tokens"],
|
||||
"prompt_slug": prompt.get("slug"),
|
||||
"prompt_revision": prompt.get("seed_revision") or "",
|
||||
@@ -109,6 +111,7 @@ def execute_prompt(
|
||||
"budget": budget,
|
||||
"diagnostics": diagnostics or {},
|
||||
"precomputed_mappings": precomputed_mappings,
|
||||
"detect_counters": detect_counters or {},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Stateless client for the LAN GLiNER spike. Fail-soft; never logs request text."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
import httpx
|
||||
|
||||
from providers import is_local_url
|
||||
|
||||
log = logging.getLogger("gliner_client")
|
||||
|
||||
DEFAULT_URL = "http://192.168.2.144:8090"
|
||||
DEFAULT_LABELS = ("person", "place", "organization", "project", "dish", "food")
|
||||
IDENTITY_LABELS = {
|
||||
"person": "PERSON",
|
||||
"place": "PLACE",
|
||||
"organization": "ORG",
|
||||
"project": "PROJECT",
|
||||
}
|
||||
NON_IDENTITY_LABELS = frozenset({"dish", "food"})
|
||||
EXTRACT_TIMEOUT = 8.0
|
||||
HEALTH_TIMEOUT = 2.0
|
||||
|
||||
|
||||
def normalize_base_url(url: str | None) -> str:
|
||||
raw = (url or DEFAULT_URL).strip() or DEFAULT_URL
|
||||
parsed = urlparse(raw)
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
return DEFAULT_URL
|
||||
path = (parsed.path or "").rstrip("/")
|
||||
return urlunparse((parsed.scheme, parsed.netloc, path, "", "", ""))
|
||||
|
||||
|
||||
def gliner_identity_type(label: str) -> str | None:
|
||||
return IDENTITY_LABELS.get((label or "").strip().lower())
|
||||
|
||||
|
||||
def is_gliner_non_identity(label: str) -> bool:
|
||||
return (label or "").strip().lower() in NON_IDENTITY_LABELS
|
||||
|
||||
|
||||
def health(url: str | None = None) -> dict[str, Any]:
|
||||
"""Ping GLiNER. Never include request text. Fail-soft."""
|
||||
base = normalize_base_url(url)
|
||||
if not is_local_url(base):
|
||||
return {"ok": False, "status": "rejected", "reason": "not_local"}
|
||||
try:
|
||||
response = httpx.get(f"{base}/health", timeout=HEALTH_TIMEOUT)
|
||||
except httpx.HTTPError:
|
||||
return {"ok": False, "status": "unreachable", "url": base}
|
||||
if response.status_code >= 400:
|
||||
return {"ok": False, "status": "http_error", "http_status": response.status_code}
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return {"ok": False, "status": "invalid"}
|
||||
if not isinstance(payload, dict):
|
||||
return {"ok": False, "status": "invalid"}
|
||||
status = (payload.get("status") or "").strip() or "ok"
|
||||
return {
|
||||
"ok": status == "ok",
|
||||
"status": status,
|
||||
"model": payload.get("model") or "",
|
||||
"offline": bool(payload.get("offline")),
|
||||
}
|
||||
|
||||
|
||||
def extract_entities(text: str, url: str | None = None) -> list[dict] | None:
|
||||
"""Return spans or None if the service cannot be used. Empty list is a valid miss."""
|
||||
body = text or ""
|
||||
if not body.strip():
|
||||
return []
|
||||
base = normalize_base_url(url)
|
||||
if not is_local_url(base):
|
||||
log.info("gliner skipped: url is not local")
|
||||
return None
|
||||
try:
|
||||
response = httpx.post(
|
||||
f"{base}/v1/extract",
|
||||
json={"text": body, "labels": list(DEFAULT_LABELS)},
|
||||
timeout=EXTRACT_TIMEOUT,
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
log.info("gliner extract unreachable chars=%s", len(body))
|
||||
return None
|
||||
if response.status_code >= 400:
|
||||
log.info("gliner extract http=%s chars=%s", response.status_code, len(body))
|
||||
return None
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return None
|
||||
raw = payload.get("spans") if isinstance(payload, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
return None
|
||||
spans = []
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
try:
|
||||
start = int(item["start"])
|
||||
end = int(item["end"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
label = str(item.get("text") or "")
|
||||
kind = str(item.get("label") or "")
|
||||
if start < 0 or end <= start or not label or not kind:
|
||||
continue
|
||||
spans.append(
|
||||
{
|
||||
"text": label,
|
||||
"start": start,
|
||||
"end": end,
|
||||
"label": kind,
|
||||
"score": item.get("score"),
|
||||
}
|
||||
)
|
||||
log.info("gliner extract chars=%s spans=%s", len(body), len(spans))
|
||||
return spans
|
||||
@@ -5,8 +5,17 @@ import re
|
||||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from detect_learning import (
|
||||
apply_review_decisions,
|
||||
drop_pending,
|
||||
is_bypass_mode,
|
||||
load_pending,
|
||||
save_pending,
|
||||
)
|
||||
from detect_pipeline import KIND_CONFIRM_REMOTE, apply_remote_confirm, prepare_detect_pipeline
|
||||
from dialogue_store import StoreError, list_conversations_for_day, list_messages
|
||||
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
||||
from entity_detect import DetectError
|
||||
from journal_policy import require_explicit_generate, source_conversation_ids
|
||||
from identity_store import is_maskable_label, list_mappings, mapping_spellings
|
||||
from journal_body import clean_title
|
||||
@@ -372,6 +381,7 @@ def generate_draft(
|
||||
explicit: bool = True,
|
||||
generation_selection: dict | None = None,
|
||||
remember_generation_selection: bool = False,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
) -> dict:
|
||||
try:
|
||||
draft = _generate_draft(
|
||||
@@ -382,6 +392,7 @@ def generate_draft(
|
||||
explicit=explicit,
|
||||
generation_selection=generation_selection,
|
||||
remember_generation_selection=remember_generation_selection,
|
||||
precomputed_mappings=precomputed_mappings,
|
||||
)
|
||||
except EngineError as exc:
|
||||
persist_engine_error(
|
||||
@@ -394,6 +405,8 @@ def generate_draft(
|
||||
extra={"conversation_ids": conversation_ids or []},
|
||||
)
|
||||
raise
|
||||
if draft.get("pending_mask_review"):
|
||||
return draft
|
||||
persist_step(
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
@@ -423,6 +436,7 @@ def _generate_draft(
|
||||
explicit: bool = True,
|
||||
generation_selection: dict | None = None,
|
||||
remember_generation_selection: bool = False,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
) -> dict:
|
||||
require_explicit_generate(explicit)
|
||||
day = get_day(profile_id, journal_day_id)
|
||||
@@ -564,6 +578,44 @@ def _generate_draft(
|
||||
)
|
||||
rendered_preview = preview_prompt(narrate_prompt, assembled)
|
||||
assert_template_resolved(rendered_preview.get("rendered") or "")
|
||||
detect_mappings = precomputed_mappings
|
||||
if detect_mappings is None and not is_bypass_mode():
|
||||
user_text = "\n".join(source_user)
|
||||
pipe = prepare_detect_pipeline(profile_id, user_text, "")
|
||||
if (pipe.need_review or pipe.candidates) and selected:
|
||||
review_id = save_pending(
|
||||
profile_id,
|
||||
selected[0],
|
||||
"",
|
||||
{
|
||||
"mappings": pipe.mappings,
|
||||
"candidates": pipe.candidates,
|
||||
"user_body": user_text,
|
||||
"kind": pipe.kind,
|
||||
"counters": pipe.counters,
|
||||
"purpose": "journal_generate",
|
||||
"generate": {
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_ids": selected,
|
||||
"include_existing": include_existing,
|
||||
"generation_selection": generation_selection,
|
||||
"remember_generation_selection": remember_generation_selection,
|
||||
},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"pending_mask_review": {
|
||||
"id": review_id,
|
||||
"candidates": pipe.candidates,
|
||||
"kind": pipe.kind,
|
||||
"purpose": "journal_generate",
|
||||
},
|
||||
"run_log": run_log,
|
||||
"trace": None,
|
||||
}
|
||||
detect_mappings = pipe.mappings
|
||||
assembled = dict(assembled)
|
||||
assembled["user_body"] = "\n".join(source_user)
|
||||
narrate_result = execute_prompt(
|
||||
narrate_prompt,
|
||||
profile_id,
|
||||
@@ -573,6 +625,7 @@ def _generate_draft(
|
||||
max_tokens=narrate_budget.reserved_output_tokens,
|
||||
disable_context_compression=True,
|
||||
budget=narrate_budget,
|
||||
precomputed_mappings=detect_mappings,
|
||||
)
|
||||
_stamp_log(run_log, "journal_generate", _log_items(narrate_result))
|
||||
title, body = _split_title(narrate_result.get("content") or "")
|
||||
@@ -745,3 +798,30 @@ def _generate_draft(
|
||||
draft["run_log"] = run_log
|
||||
draft["trace"] = bundle
|
||||
return draft
|
||||
|
||||
|
||||
def continue_journal_generate(profile_id: str, journal_day_id: str, review_id: str, decisions: list[dict]) -> dict:
|
||||
pending = load_pending(profile_id, review_id)
|
||||
if not pending or pending.get("purpose") != "journal_generate":
|
||||
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||
generate = pending.get("generate") or {}
|
||||
if generate.get("journal_day_id") != journal_day_id:
|
||||
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||
if pending.get("kind") == KIND_CONFIRM_REMOTE:
|
||||
try:
|
||||
mappings = apply_remote_confirm(profile_id, pending, decisions)
|
||||
except DetectError as exc:
|
||||
raise EngineError(exc.code, exc.message, getattr(exc, "status_code", 403)) from exc
|
||||
else:
|
||||
mappings = apply_review_decisions(profile_id, pending, decisions)
|
||||
drop_pending(profile_id, review_id)
|
||||
return generate_draft(
|
||||
profile_id,
|
||||
journal_day_id,
|
||||
conversation_ids=generate.get("conversation_ids"),
|
||||
include_existing=bool(generate.get("include_existing")),
|
||||
explicit=True,
|
||||
generation_selection=generate.get("generation_selection"),
|
||||
remember_generation_selection=bool(generate.get("remember_generation_selection")),
|
||||
precomputed_mappings=mappings,
|
||||
)
|
||||
@@ -98,6 +98,12 @@ COMPACT_DIAGNOSTIC_KEYS = (
|
||||
"invalid_entity_type",
|
||||
"omitted_non_identity_types",
|
||||
"detect_cost_unknown",
|
||||
"diff_agree",
|
||||
"diff_only_student",
|
||||
"diff_only_teacher",
|
||||
"diff_type_conflict",
|
||||
"diff_new_identity",
|
||||
"diff_cue_reinforced",
|
||||
"generation_selection",
|
||||
"style_application",
|
||||
)
|
||||
@@ -1059,10 +1065,12 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
try:
|
||||
if precomputed is not None:
|
||||
mappings = list(precomputed)
|
||||
extra = dict((request.payload or {}).get("detect_counters") or {})
|
||||
detect_stats = {
|
||||
"detect_note": "precomputed_learning_review",
|
||||
"full_detection_coverage": True,
|
||||
"semantic_identity_guaranteed": False,
|
||||
**{key: extra[key] for key in extra if key.startswith("diff_")},
|
||||
}
|
||||
local_identities = [
|
||||
{
|
||||
@@ -1075,12 +1083,37 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||
]
|
||||
detect_name = "learning_review"
|
||||
else:
|
||||
outcome = detect_personal_egress(request.profile_id, rendered)
|
||||
mappings = outcome.mappings
|
||||
detect_stats = outcome.stats.public()
|
||||
detect_name = outcome.stats.detect_provider
|
||||
detect_note = outcome.stats.detect_note
|
||||
local_identities = outcome.local_identities
|
||||
from detect_learning import is_bypass_mode
|
||||
from detect_pipeline import compact_diff_counters, prepare_detect_pipeline
|
||||
|
||||
user_body = str((request.payload or {}).get("user_body") or "").strip()
|
||||
if request.profile_id and not is_bypass_mode() and user_body:
|
||||
pipe = prepare_detect_pipeline(request.profile_id, user_body, rendered)
|
||||
mappings = list(pipe.mappings or [])
|
||||
detect_stats = {
|
||||
"detect_note": "student_path" if not pipe.need_review else "student_path_unreviewed",
|
||||
"full_detection_coverage": True,
|
||||
"semantic_identity_guaranteed": False,
|
||||
**compact_diff_counters(pipe.counters),
|
||||
}
|
||||
detect_name = "student_path"
|
||||
detect_note = detect_stats["detect_note"]
|
||||
local_identities = [
|
||||
{
|
||||
"local_label": item.get("local_label"),
|
||||
"token": item.get("token"),
|
||||
"entity_type": item.get("entity_type"),
|
||||
"source": item.get("source"),
|
||||
}
|
||||
for item in mappings
|
||||
]
|
||||
else:
|
||||
outcome = detect_personal_egress(request.profile_id, rendered)
|
||||
mappings = outcome.mappings
|
||||
detect_stats = outcome.stats.public()
|
||||
detect_name = outcome.stats.detect_provider
|
||||
detect_note = outcome.stats.detect_note
|
||||
local_identities = outcome.local_identities
|
||||
except DetectError as exc:
|
||||
detect_stats = dict(exc.diagnostics or {})
|
||||
_log_event(
|
||||
|
||||
@@ -285,6 +285,10 @@ def delete_profile(profile_id: str) -> None:
|
||||
"profile_in_use",
|
||||
f"Profil ist noch der Stufe {roles} zugeordnet. Zuerst ein anderes Profil wählen.",
|
||||
)
|
||||
from detect_learning import get_local_ner_settings, set_local_ner_settings
|
||||
|
||||
if get_local_ner_settings().get("passage_profile_id") == profile_id:
|
||||
set_local_ner_settings(passage_profile_id="")
|
||||
conn.execute("DELETE FROM llm_profiles WHERE id = ?", (profile_id,))
|
||||
|
||||
|
||||
@@ -462,4 +466,11 @@ def public_status() -> dict:
|
||||
"remote_plaintext_allowed": allows_remote_plaintext_detect(),
|
||||
"ollama_url": OLLAMA_LAN_URL,
|
||||
"detect_operating_mode": get_detect_operating_mode(),
|
||||
"local_ner": _local_ner_status(),
|
||||
}
|
||||
|
||||
|
||||
def _local_ner_status() -> dict:
|
||||
from detect_learning import get_local_ner_settings
|
||||
|
||||
return get_local_ner_settings()
|
||||
@@ -231,6 +231,12 @@ class DetectModeWrite(BaseModel):
|
||||
mode: str
|
||||
|
||||
|
||||
class LocalNerWrite(BaseModel):
|
||||
enabled: bool | None = None
|
||||
url: str | None = None
|
||||
passage_profile_id: str | None = None
|
||||
|
||||
|
||||
@router.put("/providers/detect-mode")
|
||||
def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require_admin_dep)):
|
||||
from detect_learning import set_detect_operating_mode
|
||||
@@ -243,12 +249,45 @@ def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "invalid_detect_operating_mode",
|
||||
"message": "Detect-Modus muss semantic oder learning sein.",
|
||||
"message": "Detect-Modus muss student, learning_full, learning_reduced, learning_manual oder bypass sein.",
|
||||
},
|
||||
)
|
||||
return provider_public_status()
|
||||
|
||||
|
||||
@router.put("/providers/local-ner")
|
||||
def admin_set_local_ner(body: LocalNerWrite, session: dict = Depends(require_admin_dep)):
|
||||
from detect_learning import set_local_ner_settings
|
||||
from provider_settings import public_status as provider_public_status
|
||||
|
||||
try:
|
||||
set_local_ner_settings(
|
||||
enabled=body.enabled,
|
||||
url=body.url,
|
||||
passage_profile_id=body.passage_profile_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
code = str(exc)
|
||||
if code == "gliner_url_not_local":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": code,
|
||||
"message": "GLiNER-URL muss in der lokalen Trusted Zone liegen.",
|
||||
},
|
||||
) from exc
|
||||
if code == "passage_profile_missing":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": code, "message": "Das Klärungs-Profil fehlt."},
|
||||
) from exc
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "invalid_local_ner", "message": "Lokale NER-Einstellung ungültig."},
|
||||
) from exc
|
||||
return provider_public_status()
|
||||
|
||||
|
||||
@router.get("/providers")
|
||||
def admin_providers(session: dict = Depends(require_admin_dep)):
|
||||
return public_status()
|
||||
|
||||
@@ -10,7 +10,7 @@ from dialogue_store import StoreError, delete_conversation, get_conversation, li
|
||||
from dialogue_turn import continue_turn, run_turn, visible_for_role
|
||||
from privacy_gateway import GatewayRequest, inspect, public_error_detail
|
||||
from engine import EngineError
|
||||
from journal_generate import generate_draft
|
||||
from journal_generate import continue_journal_generate, generate_draft
|
||||
from journal_generation_policy import settings_payload
|
||||
from journal_opening import maybe_open_journal_conversation
|
||||
from journal_policy import PolicyError
|
||||
@@ -353,6 +353,22 @@ def generate(journal_day_id: str, body: GenerateWrite, session: dict = Depends(r
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/days/{journal_day_id}/generate/review")
|
||||
def generate_review(journal_day_id: str, body: MaskReviewWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return visible_for_role(
|
||||
continue_journal_generate(
|
||||
session["profile_id"],
|
||||
journal_day_id,
|
||||
body.review_id,
|
||||
body.decisions,
|
||||
),
|
||||
session.get("role"),
|
||||
)
|
||||
except (StoreError, PolicyError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/entries")
|
||||
def write_entry(body: EntryWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
|
||||
@@ -36,6 +36,8 @@ UNIT_MODULES = frozenset(
|
||||
"test_journal_shape",
|
||||
"test_journal_body",
|
||||
"test_writing_profile",
|
||||
"test_gliner_client",
|
||||
"test_detect_pipeline",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -18,12 +18,27 @@ os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
from fastapi.testclient import TestClient
|
||||
from detect_learning import (
|
||||
cue_for_mention,
|
||||
decision_for_mention,
|
||||
excerpt_view,
|
||||
get_cue_decision,
|
||||
get_detect_operating_mode,
|
||||
get_sense,
|
||||
merge_gliner_candidates,
|
||||
passage_provider,
|
||||
record_cue,
|
||||
record_sense,
|
||||
set_detect_operating_mode,
|
||||
try_local_passage_decision,
|
||||
)
|
||||
from detect_pipeline import (
|
||||
MentionSpan,
|
||||
compact_diff_counters,
|
||||
compare_mentions,
|
||||
map_entity_type,
|
||||
prepare_detect_pipeline,
|
||||
same_mention,
|
||||
)
|
||||
from providers import ChatResult, ProviderConfig
|
||||
from dialogue_turn import visible_for_role
|
||||
from identity_store import list_confirmed_identities
|
||||
from main import app
|
||||
@@ -75,8 +90,28 @@ def main() -> None:
|
||||
|
||||
providers = client.get("/api/admin/providers", headers=headers)
|
||||
expect(providers.status_code == 200, "providers status")
|
||||
expect(providers.json()["detect_operating_mode"] == "semantic", "default detect mode is semantic")
|
||||
expect(get_detect_operating_mode() == "semantic", "store default is semantic")
|
||||
expect(providers.json()["detect_operating_mode"] == "bypass", "fake-env default is bypass")
|
||||
expect(get_detect_operating_mode() == "bypass", "store default in tests is bypass")
|
||||
ner = providers.json().get("local_ner") or {}
|
||||
expect(ner.get("enabled") is False, "GLiNER is off by default")
|
||||
detect_before = next(item for item in providers.json()["roles"] if item["role"] == "detect")
|
||||
remote_gliner = client.put(
|
||||
"/api/admin/providers/local-ner",
|
||||
headers=headers,
|
||||
json={"url": "https://openrouter.ai/api/v1"},
|
||||
)
|
||||
expect(remote_gliner.status_code == 400, "GLiNER URL must stay local")
|
||||
enabled = client.put(
|
||||
"/api/admin/providers/local-ner",
|
||||
headers=headers,
|
||||
json={"enabled": True, "url": "http://192.168.2.144:8090"},
|
||||
)
|
||||
status_ok(enabled, "enable local GLiNER")
|
||||
expect(enabled.json()["local_ner"]["enabled"] is True, "GLiNER enabled flag persisted")
|
||||
expect(enabled.json()["local_ner"]["url"] == "http://192.168.2.144:8090", "GLiNER LAN URL persisted")
|
||||
detect_after = next(item for item in enabled.json()["roles"] if item["role"] == "detect")
|
||||
expect(detect_after.get("url") == detect_before.get("url"), "OpenRouter detect URL is unchanged")
|
||||
client.put("/api/admin/providers/local-ner", headers=headers, json={"enabled": False})
|
||||
|
||||
bad = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "wordlist"})
|
||||
expect(bad.status_code == 400, "invalid detect mode is rejected")
|
||||
@@ -87,13 +122,13 @@ def main() -> None:
|
||||
headers=headers,
|
||||
json={"body": "Anna kam vorbei."},
|
||||
)
|
||||
status_ok(semantic_turn, "semantic turn")
|
||||
expect(not semantic_turn.json().get("pending_mask_review"), "semantic mode does not pause for review")
|
||||
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "semantic turn gets an assistant")
|
||||
status_ok(semantic_turn, "bypass turn")
|
||||
expect(not semantic_turn.json().get("pending_mask_review"), "bypass mode does not pause for review")
|
||||
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "bypass turn gets an assistant")
|
||||
|
||||
switched = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning"})
|
||||
status_ok(switched, "switch to learning")
|
||||
expect(switched.json()["detect_operating_mode"] == "learning", "learning mode persisted")
|
||||
status_ok(switched, "switch to learning alias")
|
||||
expect(switched.json()["detect_operating_mode"] == "learning_full", "learning alias stores learning_full")
|
||||
|
||||
conv = open_conv(client, headers)
|
||||
paused = client.post(
|
||||
@@ -174,7 +209,11 @@ def main() -> None:
|
||||
json={"body": "Hanna aß mit uns."},
|
||||
)
|
||||
status_ok(hanna_again, "second Hanna turn")
|
||||
expect(not hanna_again.json().get("pending_mask_review"), "known non-identity skips the popup")
|
||||
hanna_conflict = hanna_again.json().get("pending_mask_review") or {}
|
||||
expect(
|
||||
bool(hanna_conflict.get("id")),
|
||||
"teacher PERSON versus stored not_identity cue stays a visible type conflict",
|
||||
)
|
||||
|
||||
record_sense(profile_id, "Clarissa", identity=True)
|
||||
record_sense(profile_id, "Clarissa", identity=False)
|
||||
@@ -311,8 +350,89 @@ def main() -> None:
|
||||
status_ok(sushi_again, "repeat homonym sentence")
|
||||
expect(not sushi_again.json().get("pending_mask_review"), "same reviewed sentence does not re-ask")
|
||||
|
||||
dish = "Ich aß Sushi."
|
||||
dish_at = dish.index("Sushi")
|
||||
food_spans = [{"text": "Sushi", "start": dish_at, "end": dish_at + 5, "label": "food", "score": 0.8}]
|
||||
food_cands, _maps = merge_gliner_candidates(profile_id, [], [], dish, dish, food_spans)
|
||||
expect(
|
||||
any(item.get("text") == "Sushi" and item.get("suggested") == "not_identity" for item in food_cands),
|
||||
"GLiNER food span suggests not_identity",
|
||||
)
|
||||
person_line = "Ich war mit meiner Frau Sushi unterwegs."
|
||||
person_at = person_line.index("Sushi")
|
||||
gliner_miss = [{"text": "Sushi", "start": person_at, "end": person_at + 5, "label": "food", "score": 0.9}]
|
||||
merged, _maps = merge_gliner_candidates(profile_id, [], [], person_line, person_line, gliner_miss)
|
||||
expect(
|
||||
not any(item.get("suggested") == "not_identity" for item in merged),
|
||||
"GLiNER food does not override Frau attachment",
|
||||
)
|
||||
expect(
|
||||
decision_for_mention(profile_id, "Sushi", person_line, person_at, person_at + 5, person_line) == "identity",
|
||||
"learned Frau cue wins over GLiNER food",
|
||||
)
|
||||
dish_line = "Ich aß Sushi."
|
||||
dish_pos = dish_line.index("Sushi")
|
||||
expect(
|
||||
decision_for_mention(profile_id, "Sushi", dish_line, dish_pos, dish_pos + 5, dish_line) == "ask",
|
||||
"Tag-21: identity hits plus new cue stay visible",
|
||||
)
|
||||
person = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "cue", True, "frau")
|
||||
food = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "teacher", False, "aß")
|
||||
expect(same_mention(person, food), "same offsets are the same mention")
|
||||
wrapped = MentionSpan("Frau Sushi", 15, 25, "PERSON", "identity", "teacher", False, "frau")
|
||||
expect(same_mention(person, wrapped), "contained Sushi matches Frau Sushi")
|
||||
expect(map_entity_type("person") == "PERSON", "GLiNER person maps to PERSON")
|
||||
review, counters = compare_mentions(profile_id, dish_line, [person], [food], dish_line)
|
||||
expect(counters["diff_type_conflict"] >= 1, "FOOD versus PERSON is a type conflict")
|
||||
expect(any(item.get("reason") == "type_conflict" for item in review), "type conflict is offered")
|
||||
compact = compact_diff_counters(counters)
|
||||
dumped = str(compact)
|
||||
expect("Sushi" not in dumped and "Frau" not in dumped, "compact diff counters carry no plaintext")
|
||||
expect(set(compact) == {
|
||||
"diff_agree",
|
||||
"diff_only_student",
|
||||
"diff_only_teacher",
|
||||
"diff_type_conflict",
|
||||
"diff_new_identity",
|
||||
"diff_cue_reinforced",
|
||||
}, "compact counters are the agreed keys")
|
||||
set_detect_operating_mode("learning_reduced")
|
||||
expect(get_detect_operating_mode() == "learning_reduced", "stage 2 mode persists")
|
||||
set_detect_operating_mode("learning_manual")
|
||||
expect(get_detect_operating_mode() == "learning_manual", "manual mode persists")
|
||||
set_detect_operating_mode("student")
|
||||
pipe = prepare_detect_pipeline(profile_id, "Anna kam vorbei.", "Anna kam vorbei.")
|
||||
expect(pipe.mode == "student", "student path is selectable")
|
||||
expect(not pipe.need_review, "zielpfad does not open a teacher popup")
|
||||
record_sense(profile_id, "Sushi", identity=True)
|
||||
record_cue(profile_id, "Sushi", "frau", "identity", "meine Frau Sushi")
|
||||
expect(
|
||||
decision_for_mention(profile_id, "Sushi", "Ich aß Sushi.", dish_pos, dish_pos + 5) == "ask",
|
||||
"Tag-21 remains ask after more identity hits",
|
||||
)
|
||||
|
||||
fake_passage = ProviderConfig(
|
||||
role="passage",
|
||||
name="ollama",
|
||||
mode="http",
|
||||
url="http://192.168.2.144:11434/v1/chat/completions",
|
||||
model="phi3:mini",
|
||||
key="",
|
||||
local=True,
|
||||
zdr=True,
|
||||
no_train=True,
|
||||
)
|
||||
with patch("detect_learning.passage_provider", return_value=fake_passage):
|
||||
with patch("providers.complete_chat", return_value=ChatResult(content='{"decision":"not_identity"}')) as chat:
|
||||
decided = try_local_passage_decision("Rohan Sushi essen", "Sushi")
|
||||
expect(decided == "not_identity", "passage profile can decide without local Detect")
|
||||
expect(chat.called, "passage uses complete_chat")
|
||||
expect(passage_provider() is None, "no passage profile means no silent detect fallback")
|
||||
|
||||
back = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "semantic"})
|
||||
expect(back.json()["detect_operating_mode"] == "semantic", "mode can return to semantic")
|
||||
expect(back.json()["detect_operating_mode"] == "bypass", "semantic alias stores bypass")
|
||||
full = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning_full"})
|
||||
expect(full.json()["detect_operating_mode"] == "learning_full", "learning_full is accepted")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Unit tests for student/teacher comparison. No database."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
|
||||
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from detect_learning import canonicalize_detect_mode, decision_for_mention
|
||||
from detect_pipeline import (
|
||||
MentionSpan,
|
||||
compact_diff_counters,
|
||||
map_entity_type,
|
||||
protects,
|
||||
same_mention,
|
||||
types_equivalent,
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
expect(canonicalize_detect_mode("learning") == "learning_full", "learning aliases to learning_full")
|
||||
expect(canonicalize_detect_mode("semantic") == "bypass", "semantic aliases to bypass")
|
||||
expect(canonicalize_detect_mode("wordlist") is None, "unknown mode is rejected")
|
||||
expect(map_entity_type("person") == "PERSON", "person maps to PERSON")
|
||||
expect(map_entity_type("place") == "PLACE", "place maps to PLACE")
|
||||
expect(protects("PERSON") and not protects("FOOD"), "PERSON protects, FOOD does not")
|
||||
expect(types_equivalent("PERSON", "PLACE"), "similar protectable types are equivalent")
|
||||
expect(not types_equivalent("FOOD", "PERSON"), "FOOD versus PERSON is never equivalent")
|
||||
inner = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "cue", True, "frau")
|
||||
outer = MentionSpan("Frau Sushi", 15, 25, "PERSON", "identity", "teacher", False, "frau")
|
||||
expect(same_mention(inner, outer), "contained span is the same mention")
|
||||
food = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "teacher", False, "aß")
|
||||
expect(same_mention(inner, food), "same offsets stay one mention even when types differ")
|
||||
compact = compact_diff_counters({"diff_agree": 2, "diff_type_conflict": 1, "label": "Sushi"})
|
||||
expect(compact["diff_agree"] == 2 and compact["diff_type_conflict"] == 1, "known counters are kept")
|
||||
expect("label" not in compact and "Sushi" not in str(compact), "compact counters drop plaintext")
|
||||
with patch("detect_learning.get_cue_decision", return_value=None), patch(
|
||||
"detect_learning.get_sense",
|
||||
return_value={"identity_hits": 3, "non_identity_hits": 0, "ambiguous": False},
|
||||
), patch("detect_learning.cue_for_mention", return_value="aß"):
|
||||
expect(
|
||||
decision_for_mention("p", "Sushi", "Ich aß Sushi.", 7, 12) == "ask",
|
||||
"Tag-21: identity-only sense does not lock a new cue",
|
||||
)
|
||||
with patch("detect_learning.get_cue_decision", return_value="identity"), patch(
|
||||
"detect_learning.cue_for_mention", return_value="frau"
|
||||
):
|
||||
expect(
|
||||
decision_for_mention("p", "Sushi", "meine Frau Sushi", 11, 16) == "identity",
|
||||
"stored cue remains binding",
|
||||
)
|
||||
print("detect pipeline unit ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -89,7 +89,7 @@ def main() -> None:
|
||||
expect(set(roles) == {"generate", "detect"}, "two provider roles")
|
||||
expect(providers.json()["remote_plaintext_allowed"] is True, "test runtime allows remote detect")
|
||||
expect(providers.json()["remote_plaintext_reason"] == "non_production", "tests are not production")
|
||||
expect(providers.json().get("detect_operating_mode") == "semantic", "detect mode defaults to semantic")
|
||||
expect(providers.json().get("detect_operating_mode") == "learning_full", "detect mode defaults to learning stage 1")
|
||||
expect(roles["generate"]["ready"] is False, "generate fail-closed without key")
|
||||
expect("sk-" not in providers.text, "provider status has no secret")
|
||||
expect("KANSHO_PROVIDER_KEY=" not in providers.text, "env assignment not leaked")
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""GLiNER client: local URL only, fail-soft, no request text in logs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
|
||||
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
|
||||
|
||||
from gliner_client import DEFAULT_URL, extract_entities, health, normalize_base_url
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status_code: int, payload):
|
||||
self.status_code = status_code
|
||||
self._payload = payload
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
def main() -> None:
|
||||
expect(normalize_base_url("http://192.168.2.144:8090/") == DEFAULT_URL, "default GLiNER URL is normalized")
|
||||
expect(extract_entities("Anna", url="https://openrouter.ai/api/v1") is None, "OpenRouter URL is rejected")
|
||||
expect(health("https://openrouter.ai").get("reason") == "not_local", "health rejects non-local URL")
|
||||
|
||||
with patch("gliner_client.httpx.post", return_value=_Resp(200, {"spans": [{"text": "Anna", "start": 0, "end": 4, "label": "person", "score": 0.9}]})):
|
||||
spans = extract_entities("Anna kam.", url="http://192.168.2.144:8090")
|
||||
expect(spans == [{"text": "Anna", "start": 0, "end": 4, "label": "person", "score": 0.9}], "extract returns spans")
|
||||
|
||||
with patch("gliner_client.httpx.post", side_effect=httpx.ConnectError("down")):
|
||||
expect(extract_entities("Anna kam.", url="http://192.168.2.144:8090") is None, "extract is fail-soft on errors")
|
||||
|
||||
with patch("gliner_client.httpx.get", return_value=_Resp(200, {"status": "ok", "model": "urchade/gliner_multi-v2.1", "offline": True})):
|
||||
ping = health("http://192.168.2.144:8090")
|
||||
expect(ping.get("ok") is True and ping.get("offline") is True, "health reports ready offline model")
|
||||
print("gliner client: OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -31,7 +31,7 @@ def expect(ok: bool, message: str) -> None:
|
||||
def main() -> None:
|
||||
expect(is_local_url("http://127.0.0.1:11434/v1/chat/completions"), "loopback is local")
|
||||
expect(is_local_url("http://localhost:11434/v1/chat/completions"), "localhost is local")
|
||||
expect(is_local_url("http://192.168.2.144:11434/v1/chat/completions"), "RFC1918 Ollama host is local")
|
||||
expect(is_local_url("http://192.168.2.144:8090"), "RFC1918 GLiNER host is local")
|
||||
expect(is_local_url("http://10.0.0.8:11434/v1/chat/completions"), "10/8 is local")
|
||||
expect(is_local_url("http://ollama:11434/v1/chat/completions"), "compose hostname ollama is local")
|
||||
expect(not is_local_url("https://openrouter.ai/api/v1/chat/completions"), "OpenRouter is not local")
|
||||
|
||||
Reference in new issue
Block a user