Put student-first detect learning ahead of the teacher call.
Deploy Development / deploy (push) Successful in 1m0s
Test Suite / pytest-backend (push) Failing after 2m58s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 23s

Identity-only senses no longer lock a new cue. Learning stages, journal-generate review, and additive docs follow the closed section 6.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-10-12 08:59:18 +02:00
1 parent a65ea98b0c
commit 011cc9dd81
26 files changed
+1893 -89

No files matched your search

+322 -20
View File
@@ -1,12 +1,13 @@
"""Transitional learning detect: senses grow from dialogue confirmations.
Not a global word list. Detect still does not auto-activate identities.
Ambiguous spellings get a local passage call when a local detect provider exists.
Stored passages are the context a later local GLiNER/detect can use for pre-decision.
Student path: cues, kinship, homonym passage, then GLiNER. Teacher is optional.
Uncertain mentions go as a mini-passage to a chosen LLM profile, not to Detect.
"""
from __future__ import annotations
import json
import os
import re
import uuid
from typing import Any
@@ -21,10 +22,31 @@ from identity_store import (
normalize_label,
)
MODE_STUDENT = "student"
MODE_LEARNING_FULL = "learning_full"
MODE_LEARNING_REDUCED = "learning_reduced"
MODE_LEARNING_MANUAL = "learning_manual"
MODE_BYPASS = "bypass"
MODE_SEMANTIC = "semantic"
MODE_LEARNING = "learning"
MODES = (MODE_SEMANTIC, MODE_LEARNING)
CANONICAL_MODES = (
MODE_STUDENT,
MODE_LEARNING_FULL,
MODE_LEARNING_REDUCED,
MODE_LEARNING_MANUAL,
MODE_BYPASS,
)
MODE_ALIASES = {
MODE_SEMANTIC: MODE_BYPASS,
MODE_LEARNING: MODE_LEARNING_FULL,
}
MODES = CANONICAL_MODES + (MODE_SEMANTIC, MODE_LEARNING)
LEARNING_MODES = {MODE_LEARNING_FULL, MODE_LEARNING_REDUCED, MODE_LEARNING_MANUAL}
SETTING_KEY = "detect_operating_mode"
GLINER_ENABLED_KEY = "gliner_enabled"
GLINER_URL_KEY = "gliner_url"
PASSAGE_PROFILE_KEY = "passage_profile_id"
GLINER_DEFAULT_URL = "http://192.168.2.144:8090"
_LETTER = r"A-Za-zÄÖÜäöüß"
WINDOW = 80
@@ -34,18 +56,32 @@ _DECISION_ASK = "ask"
_CUE_BARE = "_bare"
def canonicalize_detect_mode(mode: str | None) -> str | None:
chosen = (mode or "").strip().lower()
if not chosen:
return None
mapped = MODE_ALIASES.get(chosen, chosen)
return mapped if mapped in CANONICAL_MODES else None
def _default_detect_mode() -> str:
if os.environ.get("KANSHO_FAKE_PROVIDER") or os.environ.get("KANSHO_FAKE_DETECT"):
return MODE_BYPASS
return MODE_LEARNING_FULL
def get_detect_operating_mode() -> str:
with get_db() as conn:
row = row_to_dict(
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
)
value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower()
return value if value in MODES else MODE_SEMANTIC
stored = canonicalize_detect_mode((row or {}).get("value") or "")
return stored or _default_detect_mode()
def set_detect_operating_mode(mode: str) -> str:
chosen = (mode or "").strip().lower()
if chosen not in MODES:
chosen = canonicalize_detect_mode(mode)
if not chosen:
raise ValueError("invalid_detect_operating_mode")
with get_db() as conn:
conn.execute(
@@ -59,6 +95,88 @@ def set_detect_operating_mode(mode: str) -> str:
return chosen
def is_bypass_mode(mode: str | None = None) -> bool:
return (mode or get_detect_operating_mode()) == MODE_BYPASS
def is_learning_mode(mode: str | None = None) -> bool:
return (mode or get_detect_operating_mode()) in LEARNING_MODES
def learning_stage(mode: str | None = None) -> int:
current = mode or get_detect_operating_mode()
if current == MODE_LEARNING_REDUCED:
return 2
return 1
def _app_setting(key: str, default: str = "") -> str:
with get_db() as conn:
row = row_to_dict(
conn.execute("SELECT value FROM app_settings WHERE key = ?", (key,)).fetchone()
)
if not row or row.get("value") is None:
return default
return str(row.get("value") or "").strip() or default
def _put_app_setting(key: str, value: str) -> None:
with get_db() as conn:
conn.execute(
"""
INSERT INTO app_settings (key, value, updated)
VALUES (?, ?, datetime('now'))
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
""",
(key, value),
)
def get_local_ner_settings() -> dict:
from gliner_client import health, normalize_base_url
from providers import is_local_url
enabled_raw = _app_setting(GLINER_ENABLED_KEY, "0").lower()
enabled = enabled_raw in {"1", "true", "yes", "on"}
url = normalize_base_url(_app_setting(GLINER_URL_KEY, GLINER_DEFAULT_URL) or GLINER_DEFAULT_URL)
passage_profile_id = _app_setting(PASSAGE_PROFILE_KEY, "")
ping = health(url) if enabled and is_local_url(url) else {"ok": False, "status": "off"}
return {
"enabled": enabled,
"url": url,
"passage_profile_id": passage_profile_id,
"ready": bool(enabled and ping.get("ok")),
"health": ping,
}
def set_local_ner_settings(
*,
enabled: bool | None = None,
url: str | None = None,
passage_profile_id: str | None = None,
) -> dict:
from gliner_client import normalize_base_url
from providers import is_local_url
if enabled is not None:
_put_app_setting(GLINER_ENABLED_KEY, "1" if enabled else "0")
if url is not None:
chosen = normalize_base_url(url) if (url or "").strip() else GLINER_DEFAULT_URL
if not is_local_url(chosen):
raise ValueError("gliner_url_not_local")
_put_app_setting(GLINER_URL_KEY, chosen)
if passage_profile_id is not None:
token = (passage_profile_id or "").strip()
if token:
from provider_settings import get_profile
if not get_profile(token):
raise ValueError("passage_profile_missing")
_put_app_setting(PASSAGE_PROFILE_KEY, token)
return get_local_ner_settings()
def get_sense(profile_id: str, label: str) -> dict:
key = normalize_label(label) or (label or "").strip()
if not key:
@@ -376,21 +494,16 @@ def decision_for_mention(
end: int | None,
rendered: str = "",
) -> str:
"""identity / not_identity from prior review, or ask once for a new cue."""
"""identity / not_identity from a stored cue, otherwise ask.
A sense with only identity hits must not lock a new cue (Sushi day 21).
"""
if not label:
return _DECISION_ASK
cue = cue_for_mention(user_body, start, end, label, rendered)
learned = get_cue_decision(profile_id, label, cue)
if learned:
return learned
sense = get_sense(profile_id, label)
identity_shaped = cue in KINSHIP
if sense["ambiguous"]:
return _DECISION_ASK
if sense["identity_hits"] > 0 and not sense["non_identity_hits"]:
return _DECISION_IDENTITY
if sense["non_identity_hits"] > 0 and not sense["identity_hits"]:
return _DECISION_ASK if identity_shaped else _DECISION_NOT
return _DECISION_ASK
@@ -533,6 +646,193 @@ def supplement_kinship_candidates(
return extra_candidates, extra_mappings
def merge_gliner_candidates(
profile_id: str,
candidates: list[dict],
mappings: list[dict],
user_body: str,
rendered: str,
spans: list[dict] | None,
) -> tuple[list[dict], list[dict]]:
"""Add GLiNER spans from the user line. Learned cues still win afterwards."""
from gliner_client import gliner_identity_type, is_gliner_non_identity
extra_candidates = list(candidates)
extra_mappings = list(mappings)
if not spans:
return extra_candidates, extra_mappings
occupied: set[tuple[int, int, str]] = set()
for item in extra_candidates:
span = _resolve_user_span(
user_body,
item.get("user_start", item.get("start")),
item.get("user_end", item.get("end")),
item.get("text") or "",
rendered,
)
if span:
occupied.add((span[0], span[1], (item.get("text") or "").casefold()))
food_keys: set[tuple[int, int, str]] = set()
for raw in spans:
label = raw.get("text") or ""
kind = raw.get("label") or ""
try:
start, end = int(raw["start"]), int(raw["end"])
except (KeyError, TypeError, ValueError):
continue
if start < 0 or end > len(user_body) or end <= start:
continue
if user_body[start:end].casefold() != label.casefold():
resolved = _resolve_user_span(user_body, start, end, label, rendered)
if not resolved:
continue
start, end = resolved
label = user_body[start:end]
if not is_maskable_label(label):
continue
key = (start, end, label.casefold())
if is_gliner_non_identity(kind):
food_keys.add(key)
continue
entity_type = gliner_identity_type(kind)
if not entity_type or key in occupied:
continue
if decision_for_mention(profile_id, label, user_body, start, end, rendered) != _DECISION_ASK:
continue
occupied.add(key)
aligned = _align_user_span(rendered, user_body, start, end)
extra_candidates.append(
{
"id": str(uuid.uuid4()),
"text": label,
"entity_type": entity_type,
"start": aligned[0],
"end": aligned[1],
**_mention_fields(user_body, start, end, label, rendered),
"ambiguous": get_sense(profile_id, label)["ambiguous"],
"suggested": _DECISION_IDENTITY,
}
)
extra_mappings.append(
{
"token": _token_for_label(extra_mappings, label, entity_type),
"local_label": label,
"canonical_label": label,
"demask_label": label,
"entity_type": entity_type,
"source": "request_local",
"start": aligned[0],
"end": aligned[1],
"aliases": [],
"labels": [label],
}
)
for item in extra_candidates:
span = _resolve_user_span(
user_body,
item.get("user_start", item.get("start")),
item.get("user_end", item.get("end")),
item.get("text") or "",
rendered,
)
if not span:
continue
key = (span[0], span[1], (item.get("text") or "").casefold())
if key in food_keys and not attachment_kinship(user_body, span[0], span[1]):
item["suggested"] = _DECISION_NOT
for start, end, folded in food_keys:
if (start, end, folded) in occupied:
continue
label = user_body[start:end]
if decision_for_mention(profile_id, label, user_body, start, end, rendered) != _DECISION_ASK:
continue
occupied.add((start, end, folded))
aligned = _align_user_span(rendered, user_body, start, end)
extra_candidates.append(
{
"id": str(uuid.uuid4()),
"text": label,
"entity_type": "PERSON",
"start": aligned[0],
"end": aligned[1],
**_mention_fields(user_body, start, end, label, rendered),
"ambiguous": get_sense(profile_id, label)["ambiguous"],
"suggested": _DECISION_NOT,
}
)
extra_mappings.append(
{
"token": _token_for_label(extra_mappings, label),
"local_label": label,
"canonical_label": label,
"demask_label": label,
"entity_type": "PERSON",
"source": "request_local",
"start": aligned[0],
"end": aligned[1],
"aliases": [],
"labels": [label],
}
)
return extra_candidates, extra_mappings
def apply_gliner_if_enabled(
profile_id: str,
candidates: list[dict],
mappings: list[dict],
user_body: str,
rendered: str = "",
) -> tuple[list[dict], list[dict]]:
settings = get_local_ner_settings()
if not settings.get("enabled"):
return candidates, mappings
from gliner_client import extract_entities
spans = extract_entities(user_body, url=settings.get("url"))
if spans is None:
return candidates, mappings
return merge_gliner_candidates(profile_id, candidates, mappings, user_body, rendered, spans)
def passage_provider():
"""LLM profile for mini-passage decisions. Independent of the detect role."""
import os
from providers import ProviderConfig, is_local_url
from provider_settings import get_profile
profile_id = _app_setting(PASSAGE_PROFILE_KEY, "")
if not profile_id:
return None
profile = get_profile(profile_id)
if not profile:
return None
url = profile.get("url") or ""
model = profile.get("model") or ""
if not url or not model:
return None
local = is_local_url(url)
key = ""
if not local:
key = (os.environ.get("KANSHO_PROVIDER_KEY") or os.environ.get("KANSHO_DETECT_PROVIDER_KEY") or "").strip()
if not key:
return None
if not profile.get("zdr") or not profile.get("no_train"):
return None
return ProviderConfig(
role="passage",
name=profile.get("name") or ("ollama" if local else "openrouter"),
mode="http",
url=url,
model=model,
key=key,
local=local,
zdr=True if local else bool(profile.get("zdr")),
no_train=True if local else bool(profile.get("no_train")),
)
def build_candidates(profile_id: str, mappings: list[dict], user_body: str, rendered: str = "") -> list[dict]:
seen: set[tuple[int | None, int | None, str]] = set()
items: list[dict] = []
@@ -721,6 +1021,8 @@ def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | No
return {
"id": row["id"],
"candidates": payload.get("candidates") or [],
"kind": payload.get("kind") or "diff",
"purpose": payload.get("purpose") or "dialogue_turn",
}
@@ -756,11 +1058,11 @@ def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: li
def try_local_passage_decision(excerpt: str, label: str, examples: list[dict] | None = None) -> str | None:
"""Return identity, not_identity, or None if no local model or unusable answer."""
from providers import ProviderError, complete_chat, detect_provider
"""Return identity, not_identity, or None. Uses the admin passage profile, not Detect."""
from providers import ProviderError, complete_chat
config = detect_provider()
if not config or not config.local or config.mode != "http":
config = passage_provider()
if not config or config.mode != "http":
return None
few_shot = ""
lines = []
+618
View File
@@ -0,0 +1,618 @@
"""Student-first detect pipeline with optional teacher comparison.
Cues stay local. OpenRouter/GLiNER never create identities. Compact counters
carry no plaintext labels.
"""
from __future__ import annotations
import uuid
from dataclasses import dataclass, field
from detect_learning import (
MODE_LEARNING_MANUAL,
MODE_STUDENT,
_CUE_BARE,
_DECISION_ASK,
_DECISION_IDENTITY,
_DECISION_NOT,
_align_user_span,
_mention_fields,
_resolve_user_span,
_token_for_label,
_word_matches,
apply_learned_decisions,
auto_resolve_ambiguous,
cue_for_mention,
decision_for_mention,
get_cue_decision,
get_detect_operating_mode,
get_local_ner_settings,
get_sense,
is_bypass_mode,
is_learning_mode,
learning_stage,
merge_gliner_candidates,
record_cue,
supplement_kinship_candidates,
)
from entity_detect import DetectError, detect_personal_egress
from identity_store import ENTITY_TYPES, is_maskable_label
KIND_DIFF = "diff"
KIND_MANUAL = "manual"
KIND_CONFIRM_REMOTE = "confirm_remote"
PROTECTING_TYPES = set(ENTITY_TYPES)
FOOD_TYPES = frozenset(
{"FOOD", "DISH", "MEAL", "CUISINE", "FOODSTUFF", "INGREDIENT", "food", "dish"}
)
TYPE_MAP = {
"person": "PERSON",
"place": "PLACE",
"org": "ORG",
"organization": "ORG",
"project": "PROJECT",
"food": "FOOD",
"dish": "DISH",
}
@dataclass
class MentionSpan:
text: str
start: int
end: int
entity_type: str
protection: str
source: str
confirmed_cue: bool = False
cue: str = _CUE_BARE
def key(self) -> tuple[int, int, str]:
return (self.start, self.end, self.text.casefold())
@dataclass
class StudentResult:
mappings: list[dict] = field(default_factory=list)
candidates: list[dict] = field(default_factory=list)
spans: list[MentionSpan] = field(default_factory=list)
gliner_missing: bool = False
@dataclass
class TeacherResult:
spans: list[MentionSpan] = field(default_factory=list)
mappings: list[dict] = field(default_factory=list)
ok: bool = True
error: str = ""
@dataclass
class PipelineResult:
mappings: list[dict]
candidates: list[dict]
kind: str = KIND_DIFF
need_review: bool = False
need_remote_confirm: bool = False
counters: dict[str, int] = field(default_factory=dict)
mode: str = ""
def empty_counters() -> dict[str, int]:
return {
"diff_agree": 0,
"diff_only_student": 0,
"diff_only_teacher": 0,
"diff_type_conflict": 0,
"diff_new_identity": 0,
"diff_cue_reinforced": 0,
}
def map_entity_type(raw: str) -> str:
token = (raw or "").strip()
if token.upper() in PROTECTING_TYPES or token.upper() in {item.upper() for item in FOOD_TYPES}:
mapped = TYPE_MAP.get(token.lower())
return mapped or token.upper()
return TYPE_MAP.get(token.lower()) or token.upper()
def protects(entity_type: str) -> bool:
kind = map_entity_type(entity_type)
if kind in FOOD_TYPES or kind.upper() in {item.upper() for item in FOOD_TYPES}:
return False
return kind in PROTECTING_TYPES
def types_equivalent(left: str, right: str) -> bool:
a, b = map_entity_type(left), map_entity_type(right)
if a in FOOD_TYPES or b in FOOD_TYPES or a in {"FOOD", "DISH"} or b in {"FOOD", "DISH"}:
return a == b
if protects(a) and protects(b):
return True
return a == b
def _contains(outer: MentionSpan, inner: MentionSpan) -> bool:
return outer.start <= inner.start and inner.end <= outer.end and outer.end > outer.start
def same_mention(left: MentionSpan, right: MentionSpan) -> bool:
if left.text.casefold() == right.text.casefold() and left.start == right.start and left.end == right.end:
return True
if _contains(left, right) or _contains(right, left):
return True
return False
def _collapse_same_protection(spans: list[MentionSpan]) -> list[MentionSpan]:
if not spans:
return []
remaining = list(spans)
chosen: list[MentionSpan] = []
while remaining:
current = remaining.pop(0)
cluster = [current]
rest = []
for item in remaining:
if same_mention(current, item):
cluster.append(item)
else:
rest.append(item)
remaining = rest
protections = {item.protection for item in cluster}
if len(protections) > 1:
chosen.extend(cluster)
continue
chosen.append(max(cluster, key=lambda item: (item.end - item.start, -item.start)))
return chosen
def _user_span_for_mapping(mapping: dict, user_body: str, rendered: str = "") -> tuple[int, int] | None:
return _resolve_user_span(
user_body,
mapping.get("start"),
mapping.get("end"),
mapping.get("local_label") or "",
rendered,
)
def _mapping_from_user_span(
mappings: list[dict],
label: str,
user_body: str,
start: int,
end: int,
entity_type: str,
rendered: str,
) -> dict:
aligned = _align_user_span(rendered, user_body, start, end)
kind = entity_type if entity_type in ENTITY_TYPES else "PERSON"
return {
"token": _token_for_label(mappings, label, kind),
"local_label": label,
"canonical_label": label,
"demask_label": label,
"entity_type": kind,
"source": "request_local",
"start": aligned[0],
"end": aligned[1],
"aliases": [],
"labels": [label],
}
def run_student_path(profile_id: str, user_body: str, rendered: str = "") -> StudentResult:
mappings: list[dict] = []
candidates: list[dict] = []
spans: list[MentionSpan] = []
text = user_body or ""
for match in _word_matches(text):
label = match.group(0)
if not is_maskable_label(label):
continue
start, end = match.start(), match.end()
verdict = decision_for_mention(profile_id, label, text, start, end, rendered)
cue = cue_for_mention(text, start, end, label, rendered)
if verdict == _DECISION_IDENTITY:
mappings.append(_mapping_from_user_span(mappings, label, text, start, end, "PERSON", rendered))
spans.append(
MentionSpan(label, start, end, "PERSON", _DECISION_IDENTITY, "cue", True, cue)
)
elif verdict == _DECISION_NOT:
spans.append(
MentionSpan(label, start, end, "FOOD", _DECISION_NOT, "cue", True, cue)
)
elif get_sense(profile_id, label)["ambiguous"]:
aligned = _align_user_span(rendered, text, start, end)
candidates.append(
{
"id": str(uuid.uuid4()),
"text": label,
"entity_type": "PERSON",
"start": aligned[0],
"end": aligned[1],
**_mention_fields(text, start, end, label, rendered),
"ambiguous": True,
}
)
candidates, mappings = supplement_kinship_candidates(
profile_id, candidates, mappings, text, rendered
)
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
bound = {(span.start, span.end, span.text.casefold()) for span in spans}
for item in candidates:
resolved = _resolve_user_span(
text,
item.get("user_start", item.get("start")),
item.get("user_end", item.get("end")),
item.get("text") or "",
rendered,
)
if resolved:
bound.add((resolved[0], resolved[1], (item.get("text") or "").casefold()))
settings = get_local_ner_settings()
gliner_missing = False
if settings.get("enabled"):
from gliner_client import extract_entities
raw = extract_entities(text, url=settings.get("url"))
if raw is None:
gliner_missing = True
else:
usable = []
for item in raw:
try:
start, end = int(item["start"]), int(item["end"])
except (KeyError, TypeError, ValueError):
continue
label = (item.get("text") or "").casefold()
if any(
not (end <= left or start >= right) and folded == label
for left, right, folded in bound
) or any(left <= start and end <= right for left, right, _folded in bound):
continue
usable.append(item)
candidates, mappings = merge_gliner_candidates(
profile_id, candidates, mappings, text, rendered, usable
)
candidates, mappings = apply_learned_decisions(profile_id, candidates, mappings, text, rendered)
for item in candidates:
resolved = _resolve_user_span(
text,
item.get("user_start", item.get("start")),
item.get("user_end", item.get("end")),
item.get("text") or "",
rendered,
)
if not resolved:
continue
suggested = item.get("suggested") or _DECISION_IDENTITY
kind = item.get("entity_type") or "PERSON"
protection = _DECISION_NOT if suggested == _DECISION_NOT else _DECISION_IDENTITY
entity = "FOOD" if protection == _DECISION_NOT else kind
spans.append(
MentionSpan(
item.get("text") or "",
resolved[0],
resolved[1],
entity,
protection,
"candidate",
False,
item.get("cue") or _CUE_BARE,
)
)
return StudentResult(
mappings=mappings,
candidates=candidates,
spans=_collapse_same_protection(spans),
gliner_missing=gliner_missing,
)
def blank_cue_safe_forms(profile_id: str, user_body: str, rendered: str = "") -> str:
chars = list(user_body or "")
for match in _word_matches(user_body or ""):
label = match.group(0)
if not is_maskable_label(label):
continue
verdict = decision_for_mention(
profile_id, label, user_body, match.start(), match.end(), rendered
)
if verdict not in {_DECISION_IDENTITY, _DECISION_NOT}:
continue
for index in range(match.start(), match.end()):
chars[index] = " "
return "".join(chars)
def _teacher_span(label: str, start: int, end: int, entity_type: str) -> MentionSpan:
kind = map_entity_type(entity_type)
protection = _DECISION_IDENTITY if protects(kind) else _DECISION_NOT
return MentionSpan(label, start, end, kind, protection, "teacher", False)
def run_teacher(
profile_id: str,
user_body: str,
*,
stage: int = 1,
rendered: str = "",
) -> TeacherResult:
source = user_body or ""
if stage >= 2:
source = blank_cue_safe_forms(profile_id, user_body, rendered)
try:
outcome = detect_personal_egress(profile_id, source)
except DetectError as exc:
return TeacherResult(ok=False, error=exc.code)
spans: list[MentionSpan] = []
for mapping in outcome.mappings or []:
label = mapping.get("local_label") or ""
start, end = mapping.get("start"), mapping.get("end")
resolved = None
if start is not None and end is not None:
left, right = int(start), int(end)
if 0 <= left < right <= len(user_body) and user_body[left:right].casefold() == label.casefold():
resolved = (left, right)
if not resolved:
resolved = _resolve_user_span(user_body, start, end, label, user_body)
if not resolved:
continue
spans.append(
_teacher_span(label, resolved[0], resolved[1], mapping.get("entity_type") or "PERSON")
)
omitted = list((outcome.stats.omitted_non_identity_types or []) if outcome.stats else [])
for kind in omitted:
if map_entity_type(kind) in FOOD_TYPES or kind.upper() in {"FOOD", "DISH"}:
continue
return TeacherResult(spans=_collapse_same_protection(spans), mappings=list(outcome.mappings or []), ok=True)
def _candidate_from_span(
span: MentionSpan,
user_body: str,
rendered: str,
*,
suggested: str,
reason: str,
) -> dict:
aligned = _align_user_span(rendered, user_body, span.start, span.end)
kind = span.entity_type if span.entity_type in ENTITY_TYPES else "PERSON"
return {
"id": str(uuid.uuid4()),
"text": span.text,
"entity_type": kind,
"start": aligned[0],
"end": aligned[1],
**_mention_fields(user_body, span.start, span.end, span.text, rendered),
"ambiguous": False,
"suggested": suggested,
"reason": reason,
}
def compare_mentions(
profile_id: str,
user_body: str,
student_spans: list[MentionSpan],
teacher_spans: list[MentionSpan],
rendered: str = "",
) -> tuple[list[dict], dict[str, int]]:
counters = empty_counters()
review: list[dict] = []
used_teacher: set[int] = set()
student_list = _collapse_same_protection(student_spans)
teacher_list = _collapse_same_protection(teacher_spans)
def take_teacher(index: int) -> MentionSpan:
used_teacher.add(index)
return teacher_list[index]
for student in student_list:
match_index = next(
(
index
for index, teacher in enumerate(teacher_list)
if index not in used_teacher and same_mention(student, teacher)
),
None,
)
if match_index is None:
counters["diff_only_student"] += 1
if student.confirmed_cue:
continue
if student.protection == _DECISION_IDENTITY:
counters["diff_new_identity"] += 1
review.append(
_candidate_from_span(
student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_student"
)
)
continue
teacher = take_teacher(match_index)
if student.protection != teacher.protection or (
protects(student.entity_type) != protects(teacher.entity_type)
):
counters["diff_type_conflict"] += 1
review.append(
_candidate_from_span(
student,
user_body,
rendered,
suggested=student.protection,
reason="type_conflict",
)
)
continue
if types_equivalent(student.entity_type, teacher.entity_type) or (
student.protection == teacher.protection
):
counters["diff_agree"] += 1
if student.confirmed_cue:
record_cue(profile_id, student.text, student.cue, student.protection)
counters["diff_cue_reinforced"] += 1
elif student.protection == _DECISION_IDENTITY:
learned = get_cue_decision(profile_id, student.text, student.cue)
if learned == _DECISION_IDENTITY:
record_cue(profile_id, student.text, student.cue, _DECISION_IDENTITY)
counters["diff_cue_reinforced"] += 1
else:
counters["diff_new_identity"] += 1
review.append(
_candidate_from_span(
student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="new_identity"
)
)
for index, teacher in enumerate(teacher_list):
if index in used_teacher:
continue
counters["diff_only_teacher"] += 1
cue = cue_for_mention(user_body, teacher.start, teacher.end, teacher.text, rendered)
learned = get_cue_decision(profile_id, teacher.text, cue)
if learned:
continue
if teacher.protection == _DECISION_IDENTITY:
counters["diff_new_identity"] += 1
review.append(
_candidate_from_span(
teacher, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_teacher"
)
)
return review, counters
def _remote_confirm_candidate(user_body: str) -> dict:
snippet = (user_body or "")[:80]
return {
"id": str(uuid.uuid4()),
"text": "Klartext-Detect",
"entity_type": "PERSON",
"start": 0,
"end": 0,
"excerpt": snippet,
"highlight_start": None,
"highlight_end": None,
"user_start": None,
"user_end": None,
"cue": _CUE_BARE,
"ambiguous": False,
"suggested": _DECISION_NOT,
"reason": KIND_CONFIRM_REMOTE,
}
def prepare_detect_pipeline(profile_id: str, user_body: str, rendered: str = "") -> PipelineResult:
mode = get_detect_operating_mode()
counters = empty_counters()
if is_bypass_mode(mode):
return PipelineResult(mappings=[], candidates=[], kind=KIND_DIFF, mode=mode, counters=counters)
student = run_student_path(profile_id, user_body, rendered)
if mode == MODE_STUDENT:
if student.gliner_missing:
return PipelineResult(
mappings=student.mappings,
candidates=[_remote_confirm_candidate(user_body)],
kind=KIND_CONFIRM_REMOTE,
need_review=True,
need_remote_confirm=True,
counters=counters,
mode=mode,
)
return PipelineResult(
mappings=student.mappings,
candidates=[],
kind=KIND_DIFF,
counters=counters,
mode=mode,
)
if mode == MODE_LEARNING_MANUAL or student.gliner_missing:
return PipelineResult(
mappings=student.mappings,
candidates=student.candidates,
kind=KIND_MANUAL,
need_review=bool(student.candidates),
counters=counters,
mode=mode,
)
teacher = run_teacher(profile_id, user_body, stage=learning_stage(mode), rendered=rendered)
if not teacher.ok:
return PipelineResult(
mappings=student.mappings,
candidates=student.candidates,
kind=KIND_MANUAL,
need_review=bool(student.candidates),
counters=counters,
mode=mode,
)
review, counters = compare_mentions(
profile_id, user_body, student.spans, teacher.spans, rendered
)
merged = list(student.mappings)
seen = {
(
item.get("start"),
item.get("end"),
(item.get("local_label") or "").casefold(),
)
for item in merged
}
for mapping in teacher.mappings:
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
if key in seen:
continue
label = mapping.get("local_label") or ""
resolved = _resolve_user_span(
user_body, mapping.get("start"), mapping.get("end"), label, user_body
)
if resolved:
cue = cue_for_mention(user_body, resolved[0], resolved[1], label, rendered)
if get_cue_decision(profile_id, label, cue) == _DECISION_NOT:
continue
merged.append(mapping)
seen.add(key)
return PipelineResult(
mappings=merged,
candidates=review,
kind=KIND_DIFF,
need_review=bool(review),
counters=counters,
mode=mode,
)
def compact_diff_counters(counters: dict[str, int] | None) -> dict[str, int]:
payload = empty_counters()
for key in payload:
payload[key] = int((counters or {}).get(key) or 0)
return payload
def apply_remote_confirm(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
"""After the user allows plaintext detect, run teacher on the user line."""
approved = any((item.get("decision") or "").strip() == _DECISION_IDENTITY for item in decisions)
if not approved:
raise DetectError(
"remote_detect_declined",
"Klartext-Detect wurde nicht bestätigt.",
403,
)
user_body = pending.get("user_body") or ""
teacher = run_teacher(profile_id, user_body, stage=1)
if not teacher.ok:
raise DetectError(teacher.error or "detect_provider_unavailable", "Lehrer-Detect ist nicht erreichbar.")
merged = list(pending.get("mappings") or [])
seen = {
(item.get("start"), item.get("end"), (item.get("local_label") or "").casefold())
for item in merged
}
for mapping in teacher.mappings:
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
if key in seen:
continue
merged.append(mapping)
seen.add(key)
return merged
+37 -31
View File
@@ -8,22 +8,18 @@ from context_builder import assemble_text, build_internal_context, is_closing_tu
from conversation_signals import infer_signals
from debug_store import persist_engine_error, persist_step
from detect_learning import (
MODE_LEARNING,
apply_learned_decisions,
apply_review_decisions,
auto_resolve_ambiguous,
build_candidates,
confirm_known_identity_spans,
drop_pending,
get_detect_operating_mode,
is_bypass_mode,
load_pending,
save_pending,
supplement_kinship_candidates,
suppress_known_non_identity,
)
from detect_pipeline import KIND_CONFIRM_REMOTE, prepare_detect_pipeline
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError, detect_personal_egress
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError
from writing_profile_store import remember_dialogue_style
from profile_review import consider_dialogue
@@ -331,6 +327,7 @@ def _finish_turn(
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
call_traces: list[dict] = []
working = dict(assembled)
working["user_body"] = user.get("body") or working.get("user_body") or ""
try:
while calls < 2:
result = execute_prompt(
@@ -431,24 +428,22 @@ def _finish_turn(
}
def _review_pause_payload(review_id: str, candidates: list[dict], kind: str, purpose: str = "dialogue_turn") -> dict:
return {
"id": review_id,
"candidates": candidates,
"kind": kind,
"purpose": purpose,
}
def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled: dict, prompt: dict) -> dict | None:
preview = preview_prompt(prompt, assembled)
rendered = preview.get("rendered") or ""
try:
outcome = detect_personal_egress(profile_id, rendered)
except DetectError:
return None
user_body = user.get("body") or ""
mappings = list(outcome.mappings or [])
candidates = build_candidates(profile_id, mappings, user_body, rendered)
candidates, mappings = supplement_kinship_candidates(
profile_id, candidates, mappings, user_body, rendered
)
candidates, mappings = apply_learned_decisions(
profile_id, candidates, mappings, user_body, rendered
)
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
if not candidates:
result = prepare_detect_pipeline(profile_id, user_body, rendered)
mappings = list(result.mappings or [])
if not result.need_review and not result.candidates:
confirm_known_identity_spans(profile_id, mappings, user_body)
mappings = suppress_known_non_identity(profile_id, mappings, user_body)
return _finish_turn(
@@ -465,8 +460,11 @@ def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled
user.get("id") or "",
{
"mappings": mappings,
"candidates": candidates,
"candidates": result.candidates,
"user_body": user_body,
"kind": result.kind,
"counters": result.counters,
"purpose": "dialogue_turn",
},
)
return {
@@ -475,10 +473,7 @@ def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled
"assistant": None,
"calls": 0,
"messages": list_messages(profile_id, conversation_id),
"pending_mask_review": {
"id": review_id,
"candidates": candidates,
},
"pending_mask_review": _review_pause_payload(review_id, result.candidates, result.kind),
}
@@ -494,10 +489,11 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
)
assembled = assemble_text(context)
prompt = load_active_prompt("mvp.dialogue_turn")
if get_detect_operating_mode() == MODE_LEARNING:
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
if paused is not None:
return paused
if is_bypass_mode():
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
if paused is not None:
return paused
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
@@ -505,7 +501,17 @@ def continue_turn(profile_id: str, conversation_id: str, review_id: str, decisio
pending = load_pending(profile_id, review_id)
if not pending or pending.get("conversation_id") != conversation_id:
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
mappings = apply_review_decisions(profile_id, pending, decisions)
if pending.get("purpose") == "journal_generate":
raise EngineError("mask_review_missing", "Diese Prüfung gehört zum Journalentwurf.", 409)
if pending.get("kind") == KIND_CONFIRM_REMOTE:
try:
from detect_pipeline import apply_remote_confirm
mappings = apply_remote_confirm(profile_id, pending, decisions)
except DetectError as exc:
raise EngineError(exc.code, exc.message, getattr(exc, "status_code", 403)) from exc
else:
mappings = apply_review_decisions(profile_id, pending, decisions)
drop_pending(profile_id, review_id)
conversation = get_conversation(profile_id, conversation_id)
context = build_internal_context(
+3
View File
@@ -76,6 +76,7 @@ def execute_prompt(
budget=None,
diagnostics: dict[str, Any] | None = None,
precomputed_mappings: list[dict] | None = None,
detect_counters: dict | None = None,
) -> dict:
preview = preview_prompt(prompt, context)
feature_id = prompt.get("required_feature") or "ai_calls"
@@ -101,6 +102,7 @@ def execute_prompt(
payload={
"rendered": preview["rendered"],
"source_text": source_text,
"user_body": (ctx.get("user_body") or user_source_text(ctx.get("dialogue_context") or "")).strip(),
"privacy_tokens": preview["privacy_tokens"],
"prompt_slug": prompt.get("slug"),
"prompt_revision": prompt.get("seed_revision") or "",
@@ -109,6 +111,7 @@ def execute_prompt(
"budget": budget,
"diagnostics": diagnostics or {},
"precomputed_mappings": precomputed_mappings,
"detect_counters": detect_counters or {},
},
)
)
+121
View File
@@ -0,0 +1,121 @@
"""Stateless client for the LAN GLiNER spike. Fail-soft; never logs request text."""
from __future__ import annotations
import logging
from typing import Any
from urllib.parse import urlparse, urlunparse
import httpx
from providers import is_local_url
log = logging.getLogger("gliner_client")
DEFAULT_URL = "http://192.168.2.144:8090"
DEFAULT_LABELS = ("person", "place", "organization", "project", "dish", "food")
IDENTITY_LABELS = {
"person": "PERSON",
"place": "PLACE",
"organization": "ORG",
"project": "PROJECT",
}
NON_IDENTITY_LABELS = frozenset({"dish", "food"})
EXTRACT_TIMEOUT = 8.0
HEALTH_TIMEOUT = 2.0
def normalize_base_url(url: str | None) -> str:
raw = (url or DEFAULT_URL).strip() or DEFAULT_URL
parsed = urlparse(raw)
if not parsed.scheme or not parsed.netloc:
return DEFAULT_URL
path = (parsed.path or "").rstrip("/")
return urlunparse((parsed.scheme, parsed.netloc, path, "", "", ""))
def gliner_identity_type(label: str) -> str | None:
return IDENTITY_LABELS.get((label or "").strip().lower())
def is_gliner_non_identity(label: str) -> bool:
return (label or "").strip().lower() in NON_IDENTITY_LABELS
def health(url: str | None = None) -> dict[str, Any]:
"""Ping GLiNER. Never include request text. Fail-soft."""
base = normalize_base_url(url)
if not is_local_url(base):
return {"ok": False, "status": "rejected", "reason": "not_local"}
try:
response = httpx.get(f"{base}/health", timeout=HEALTH_TIMEOUT)
except httpx.HTTPError:
return {"ok": False, "status": "unreachable", "url": base}
if response.status_code >= 400:
return {"ok": False, "status": "http_error", "http_status": response.status_code}
try:
payload = response.json()
except ValueError:
return {"ok": False, "status": "invalid"}
if not isinstance(payload, dict):
return {"ok": False, "status": "invalid"}
status = (payload.get("status") or "").strip() or "ok"
return {
"ok": status == "ok",
"status": status,
"model": payload.get("model") or "",
"offline": bool(payload.get("offline")),
}
def extract_entities(text: str, url: str | None = None) -> list[dict] | None:
"""Return spans or None if the service cannot be used. Empty list is a valid miss."""
body = text or ""
if not body.strip():
return []
base = normalize_base_url(url)
if not is_local_url(base):
log.info("gliner skipped: url is not local")
return None
try:
response = httpx.post(
f"{base}/v1/extract",
json={"text": body, "labels": list(DEFAULT_LABELS)},
timeout=EXTRACT_TIMEOUT,
)
except httpx.HTTPError:
log.info("gliner extract unreachable chars=%s", len(body))
return None
if response.status_code >= 400:
log.info("gliner extract http=%s chars=%s", response.status_code, len(body))
return None
try:
payload = response.json()
except ValueError:
return None
raw = payload.get("spans") if isinstance(payload, dict) else None
if not isinstance(raw, list):
return None
spans = []
for item in raw:
if not isinstance(item, dict):
continue
try:
start = int(item["start"])
end = int(item["end"])
except (KeyError, TypeError, ValueError):
continue
label = str(item.get("text") or "")
kind = str(item.get("label") or "")
if start < 0 or end <= start or not label or not kind:
continue
spans.append(
{
"text": label,
"start": start,
"end": end,
"label": kind,
"score": item.get("score"),
}
)
log.info("gliner extract chars=%s spans=%s", len(body), len(spans))
return spans
+80
View File
@@ -5,8 +5,17 @@ import re
from context_builder import assemble_text, build_internal_context
from debug_store import persist_engine_error, persist_step
from detect_learning import (
apply_review_decisions,
drop_pending,
is_bypass_mode,
load_pending,
save_pending,
)
from detect_pipeline import KIND_CONFIRM_REMOTE, apply_remote_confirm, prepare_detect_pipeline
from dialogue_store import StoreError, list_conversations_for_day, list_messages
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
from entity_detect import DetectError
from journal_policy import require_explicit_generate, source_conversation_ids
from identity_store import is_maskable_label, list_mappings, mapping_spellings
from journal_body import clean_title
@@ -372,6 +381,7 @@ def generate_draft(
explicit: bool = True,
generation_selection: dict | None = None,
remember_generation_selection: bool = False,
precomputed_mappings: list[dict] | None = None,
) -> dict:
try:
draft = _generate_draft(
@@ -382,6 +392,7 @@ def generate_draft(
explicit=explicit,
generation_selection=generation_selection,
remember_generation_selection=remember_generation_selection,
precomputed_mappings=precomputed_mappings,
)
except EngineError as exc:
persist_engine_error(
@@ -394,6 +405,8 @@ def generate_draft(
extra={"conversation_ids": conversation_ids or []},
)
raise
if draft.get("pending_mask_review"):
return draft
persist_step(
profile_id,
purpose="journal_generate",
@@ -423,6 +436,7 @@ def _generate_draft(
explicit: bool = True,
generation_selection: dict | None = None,
remember_generation_selection: bool = False,
precomputed_mappings: list[dict] | None = None,
) -> dict:
require_explicit_generate(explicit)
day = get_day(profile_id, journal_day_id)
@@ -564,6 +578,44 @@ def _generate_draft(
)
rendered_preview = preview_prompt(narrate_prompt, assembled)
assert_template_resolved(rendered_preview.get("rendered") or "")
detect_mappings = precomputed_mappings
if detect_mappings is None and not is_bypass_mode():
user_text = "\n".join(source_user)
pipe = prepare_detect_pipeline(profile_id, user_text, "")
if (pipe.need_review or pipe.candidates) and selected:
review_id = save_pending(
profile_id,
selected[0],
"",
{
"mappings": pipe.mappings,
"candidates": pipe.candidates,
"user_body": user_text,
"kind": pipe.kind,
"counters": pipe.counters,
"purpose": "journal_generate",
"generate": {
"journal_day_id": journal_day_id,
"conversation_ids": selected,
"include_existing": include_existing,
"generation_selection": generation_selection,
"remember_generation_selection": remember_generation_selection,
},
},
)
return {
"pending_mask_review": {
"id": review_id,
"candidates": pipe.candidates,
"kind": pipe.kind,
"purpose": "journal_generate",
},
"run_log": run_log,
"trace": None,
}
detect_mappings = pipe.mappings
assembled = dict(assembled)
assembled["user_body"] = "\n".join(source_user)
narrate_result = execute_prompt(
narrate_prompt,
profile_id,
@@ -573,6 +625,7 @@ def _generate_draft(
max_tokens=narrate_budget.reserved_output_tokens,
disable_context_compression=True,
budget=narrate_budget,
precomputed_mappings=detect_mappings,
)
_stamp_log(run_log, "journal_generate", _log_items(narrate_result))
title, body = _split_title(narrate_result.get("content") or "")
@@ -745,3 +798,30 @@ def _generate_draft(
draft["run_log"] = run_log
draft["trace"] = bundle
return draft
def continue_journal_generate(profile_id: str, journal_day_id: str, review_id: str, decisions: list[dict]) -> dict:
pending = load_pending(profile_id, review_id)
if not pending or pending.get("purpose") != "journal_generate":
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
generate = pending.get("generate") or {}
if generate.get("journal_day_id") != journal_day_id:
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
if pending.get("kind") == KIND_CONFIRM_REMOTE:
try:
mappings = apply_remote_confirm(profile_id, pending, decisions)
except DetectError as exc:
raise EngineError(exc.code, exc.message, getattr(exc, "status_code", 403)) from exc
else:
mappings = apply_review_decisions(profile_id, pending, decisions)
drop_pending(profile_id, review_id)
return generate_draft(
profile_id,
journal_day_id,
conversation_ids=generate.get("conversation_ids"),
include_existing=bool(generate.get("include_existing")),
explicit=True,
generation_selection=generate.get("generation_selection"),
remember_generation_selection=bool(generate.get("remember_generation_selection")),
precomputed_mappings=mappings,
)
+39 -6
View File
@@ -98,6 +98,12 @@ COMPACT_DIAGNOSTIC_KEYS = (
"invalid_entity_type",
"omitted_non_identity_types",
"detect_cost_unknown",
"diff_agree",
"diff_only_student",
"diff_only_teacher",
"diff_type_conflict",
"diff_new_identity",
"diff_cue_reinforced",
"generation_selection",
"style_application",
)
@@ -1059,10 +1065,12 @@ def complete(request: GatewayRequest) -> GatewayResult:
try:
if precomputed is not None:
mappings = list(precomputed)
extra = dict((request.payload or {}).get("detect_counters") or {})
detect_stats = {
"detect_note": "precomputed_learning_review",
"full_detection_coverage": True,
"semantic_identity_guaranteed": False,
**{key: extra[key] for key in extra if key.startswith("diff_")},
}
local_identities = [
{
@@ -1075,12 +1083,37 @@ def complete(request: GatewayRequest) -> GatewayResult:
]
detect_name = "learning_review"
else:
outcome = detect_personal_egress(request.profile_id, rendered)
mappings = outcome.mappings
detect_stats = outcome.stats.public()
detect_name = outcome.stats.detect_provider
detect_note = outcome.stats.detect_note
local_identities = outcome.local_identities
from detect_learning import is_bypass_mode
from detect_pipeline import compact_diff_counters, prepare_detect_pipeline
user_body = str((request.payload or {}).get("user_body") or "").strip()
if request.profile_id and not is_bypass_mode() and user_body:
pipe = prepare_detect_pipeline(request.profile_id, user_body, rendered)
mappings = list(pipe.mappings or [])
detect_stats = {
"detect_note": "student_path" if not pipe.need_review else "student_path_unreviewed",
"full_detection_coverage": True,
"semantic_identity_guaranteed": False,
**compact_diff_counters(pipe.counters),
}
detect_name = "student_path"
detect_note = detect_stats["detect_note"]
local_identities = [
{
"local_label": item.get("local_label"),
"token": item.get("token"),
"entity_type": item.get("entity_type"),
"source": item.get("source"),
}
for item in mappings
]
else:
outcome = detect_personal_egress(request.profile_id, rendered)
mappings = outcome.mappings
detect_stats = outcome.stats.public()
detect_name = outcome.stats.detect_provider
detect_note = outcome.stats.detect_note
local_identities = outcome.local_identities
except DetectError as exc:
detect_stats = dict(exc.diagnostics or {})
_log_event(
+11
View File
@@ -285,6 +285,10 @@ def delete_profile(profile_id: str) -> None:
"profile_in_use",
f"Profil ist noch der Stufe {roles} zugeordnet. Zuerst ein anderes Profil wählen.",
)
from detect_learning import get_local_ner_settings, set_local_ner_settings
if get_local_ner_settings().get("passage_profile_id") == profile_id:
set_local_ner_settings(passage_profile_id="")
conn.execute("DELETE FROM llm_profiles WHERE id = ?", (profile_id,))
@@ -462,4 +466,11 @@ def public_status() -> dict:
"remote_plaintext_allowed": allows_remote_plaintext_detect(),
"ollama_url": OLLAMA_LAN_URL,
"detect_operating_mode": get_detect_operating_mode(),
"local_ner": _local_ner_status(),
}
def _local_ner_status() -> dict:
from detect_learning import get_local_ner_settings
return get_local_ner_settings()
+40 -1
View File
@@ -231,6 +231,12 @@ class DetectModeWrite(BaseModel):
mode: str
class LocalNerWrite(BaseModel):
enabled: bool | None = None
url: str | None = None
passage_profile_id: str | None = None
@router.put("/providers/detect-mode")
def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require_admin_dep)):
from detect_learning import set_detect_operating_mode
@@ -243,12 +249,45 @@ def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require
status_code=400,
detail={
"code": "invalid_detect_operating_mode",
"message": "Detect-Modus muss semantic oder learning sein.",
"message": "Detect-Modus muss student, learning_full, learning_reduced, learning_manual oder bypass sein.",
},
)
return provider_public_status()
@router.put("/providers/local-ner")
def admin_set_local_ner(body: LocalNerWrite, session: dict = Depends(require_admin_dep)):
from detect_learning import set_local_ner_settings
from provider_settings import public_status as provider_public_status
try:
set_local_ner_settings(
enabled=body.enabled,
url=body.url,
passage_profile_id=body.passage_profile_id,
)
except ValueError as exc:
code = str(exc)
if code == "gliner_url_not_local":
raise HTTPException(
status_code=400,
detail={
"code": code,
"message": "GLiNER-URL muss in der lokalen Trusted Zone liegen.",
},
) from exc
if code == "passage_profile_missing":
raise HTTPException(
status_code=400,
detail={"code": code, "message": "Das Klärungs-Profil fehlt."},
) from exc
raise HTTPException(
status_code=400,
detail={"code": "invalid_local_ner", "message": "Lokale NER-Einstellung ungültig."},
) from exc
return provider_public_status()
@router.get("/providers")
def admin_providers(session: dict = Depends(require_admin_dep)):
return public_status()
+17 -1
View File
@@ -10,7 +10,7 @@ from dialogue_store import StoreError, delete_conversation, get_conversation, li
from dialogue_turn import continue_turn, run_turn, visible_for_role
from privacy_gateway import GatewayRequest, inspect, public_error_detail
from engine import EngineError
from journal_generate import generate_draft
from journal_generate import continue_journal_generate, generate_draft
from journal_generation_policy import settings_payload
from journal_opening import maybe_open_journal_conversation
from journal_policy import PolicyError
@@ -353,6 +353,22 @@ def generate(journal_day_id: str, body: GenerateWrite, session: dict = Depends(r
_http(exc)
@router.post("/days/{journal_day_id}/generate/review")
def generate_review(journal_day_id: str, body: MaskReviewWrite, session: dict = Depends(require_auth)):
try:
return visible_for_role(
continue_journal_generate(
session["profile_id"],
journal_day_id,
body.review_id,
body.decisions,
),
session.get("role"),
)
except (StoreError, PolicyError, EngineError) as exc:
_http(exc)
@router.post("/entries")
def write_entry(body: EntryWrite, session: dict = Depends(require_auth)):
try:
+2
View File
@@ -36,6 +36,8 @@ UNIT_MODULES = frozenset(
"test_journal_shape",
"test_journal_body",
"test_writing_profile",
"test_gliner_client",
"test_detect_pipeline",
}
)
+129 -9
View File
@@ -18,12 +18,27 @@ os.environ["KANSHO_FAKE_DETECT"] = "1"
from fastapi.testclient import TestClient
from detect_learning import (
cue_for_mention,
decision_for_mention,
excerpt_view,
get_cue_decision,
get_detect_operating_mode,
get_sense,
merge_gliner_candidates,
passage_provider,
record_cue,
record_sense,
set_detect_operating_mode,
try_local_passage_decision,
)
from detect_pipeline import (
MentionSpan,
compact_diff_counters,
compare_mentions,
map_entity_type,
prepare_detect_pipeline,
same_mention,
)
from providers import ChatResult, ProviderConfig
from dialogue_turn import visible_for_role
from identity_store import list_confirmed_identities
from main import app
@@ -75,8 +90,28 @@ def main() -> None:
providers = client.get("/api/admin/providers", headers=headers)
expect(providers.status_code == 200, "providers status")
expect(providers.json()["detect_operating_mode"] == "semantic", "default detect mode is semantic")
expect(get_detect_operating_mode() == "semantic", "store default is semantic")
expect(providers.json()["detect_operating_mode"] == "bypass", "fake-env default is bypass")
expect(get_detect_operating_mode() == "bypass", "store default in tests is bypass")
ner = providers.json().get("local_ner") or {}
expect(ner.get("enabled") is False, "GLiNER is off by default")
detect_before = next(item for item in providers.json()["roles"] if item["role"] == "detect")
remote_gliner = client.put(
"/api/admin/providers/local-ner",
headers=headers,
json={"url": "https://openrouter.ai/api/v1"},
)
expect(remote_gliner.status_code == 400, "GLiNER URL must stay local")
enabled = client.put(
"/api/admin/providers/local-ner",
headers=headers,
json={"enabled": True, "url": "http://192.168.2.144:8090"},
)
status_ok(enabled, "enable local GLiNER")
expect(enabled.json()["local_ner"]["enabled"] is True, "GLiNER enabled flag persisted")
expect(enabled.json()["local_ner"]["url"] == "http://192.168.2.144:8090", "GLiNER LAN URL persisted")
detect_after = next(item for item in enabled.json()["roles"] if item["role"] == "detect")
expect(detect_after.get("url") == detect_before.get("url"), "OpenRouter detect URL is unchanged")
client.put("/api/admin/providers/local-ner", headers=headers, json={"enabled": False})
bad = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "wordlist"})
expect(bad.status_code == 400, "invalid detect mode is rejected")
@@ -87,13 +122,13 @@ def main() -> None:
headers=headers,
json={"body": "Anna kam vorbei."},
)
status_ok(semantic_turn, "semantic turn")
expect(not semantic_turn.json().get("pending_mask_review"), "semantic mode does not pause for review")
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "semantic turn gets an assistant")
status_ok(semantic_turn, "bypass turn")
expect(not semantic_turn.json().get("pending_mask_review"), "bypass mode does not pause for review")
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "bypass turn gets an assistant")
switched = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning"})
status_ok(switched, "switch to learning")
expect(switched.json()["detect_operating_mode"] == "learning", "learning mode persisted")
status_ok(switched, "switch to learning alias")
expect(switched.json()["detect_operating_mode"] == "learning_full", "learning alias stores learning_full")
conv = open_conv(client, headers)
paused = client.post(
@@ -174,7 +209,11 @@ def main() -> None:
json={"body": "Hanna aß mit uns."},
)
status_ok(hanna_again, "second Hanna turn")
expect(not hanna_again.json().get("pending_mask_review"), "known non-identity skips the popup")
hanna_conflict = hanna_again.json().get("pending_mask_review") or {}
expect(
bool(hanna_conflict.get("id")),
"teacher PERSON versus stored not_identity cue stays a visible type conflict",
)
record_sense(profile_id, "Clarissa", identity=True)
record_sense(profile_id, "Clarissa", identity=False)
@@ -311,8 +350,89 @@ def main() -> None:
status_ok(sushi_again, "repeat homonym sentence")
expect(not sushi_again.json().get("pending_mask_review"), "same reviewed sentence does not re-ask")
dish = "Ich aß Sushi."
dish_at = dish.index("Sushi")
food_spans = [{"text": "Sushi", "start": dish_at, "end": dish_at + 5, "label": "food", "score": 0.8}]
food_cands, _maps = merge_gliner_candidates(profile_id, [], [], dish, dish, food_spans)
expect(
any(item.get("text") == "Sushi" and item.get("suggested") == "not_identity" for item in food_cands),
"GLiNER food span suggests not_identity",
)
person_line = "Ich war mit meiner Frau Sushi unterwegs."
person_at = person_line.index("Sushi")
gliner_miss = [{"text": "Sushi", "start": person_at, "end": person_at + 5, "label": "food", "score": 0.9}]
merged, _maps = merge_gliner_candidates(profile_id, [], [], person_line, person_line, gliner_miss)
expect(
not any(item.get("suggested") == "not_identity" for item in merged),
"GLiNER food does not override Frau attachment",
)
expect(
decision_for_mention(profile_id, "Sushi", person_line, person_at, person_at + 5, person_line) == "identity",
"learned Frau cue wins over GLiNER food",
)
dish_line = "Ich aß Sushi."
dish_pos = dish_line.index("Sushi")
expect(
decision_for_mention(profile_id, "Sushi", dish_line, dish_pos, dish_pos + 5, dish_line) == "ask",
"Tag-21: identity hits plus new cue stay visible",
)
person = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "cue", True, "frau")
food = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "teacher", False, "aß")
expect(same_mention(person, food), "same offsets are the same mention")
wrapped = MentionSpan("Frau Sushi", 15, 25, "PERSON", "identity", "teacher", False, "frau")
expect(same_mention(person, wrapped), "contained Sushi matches Frau Sushi")
expect(map_entity_type("person") == "PERSON", "GLiNER person maps to PERSON")
review, counters = compare_mentions(profile_id, dish_line, [person], [food], dish_line)
expect(counters["diff_type_conflict"] >= 1, "FOOD versus PERSON is a type conflict")
expect(any(item.get("reason") == "type_conflict" for item in review), "type conflict is offered")
compact = compact_diff_counters(counters)
dumped = str(compact)
expect("Sushi" not in dumped and "Frau" not in dumped, "compact diff counters carry no plaintext")
expect(set(compact) == {
"diff_agree",
"diff_only_student",
"diff_only_teacher",
"diff_type_conflict",
"diff_new_identity",
"diff_cue_reinforced",
}, "compact counters are the agreed keys")
set_detect_operating_mode("learning_reduced")
expect(get_detect_operating_mode() == "learning_reduced", "stage 2 mode persists")
set_detect_operating_mode("learning_manual")
expect(get_detect_operating_mode() == "learning_manual", "manual mode persists")
set_detect_operating_mode("student")
pipe = prepare_detect_pipeline(profile_id, "Anna kam vorbei.", "Anna kam vorbei.")
expect(pipe.mode == "student", "student path is selectable")
expect(not pipe.need_review, "zielpfad does not open a teacher popup")
record_sense(profile_id, "Sushi", identity=True)
record_cue(profile_id, "Sushi", "frau", "identity", "meine Frau Sushi")
expect(
decision_for_mention(profile_id, "Sushi", "Ich aß Sushi.", dish_pos, dish_pos + 5) == "ask",
"Tag-21 remains ask after more identity hits",
)
fake_passage = ProviderConfig(
role="passage",
name="ollama",
mode="http",
url="http://192.168.2.144:11434/v1/chat/completions",
model="phi3:mini",
key="",
local=True,
zdr=True,
no_train=True,
)
with patch("detect_learning.passage_provider", return_value=fake_passage):
with patch("providers.complete_chat", return_value=ChatResult(content='{"decision":"not_identity"}')) as chat:
decided = try_local_passage_decision("Rohan Sushi essen", "Sushi")
expect(decided == "not_identity", "passage profile can decide without local Detect")
expect(chat.called, "passage uses complete_chat")
expect(passage_provider() is None, "no passage profile means no silent detect fallback")
back = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "semantic"})
expect(back.json()["detect_operating_mode"] == "semantic", "mode can return to semantic")
expect(back.json()["detect_operating_mode"] == "bypass", "semantic alias stores bypass")
full = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning_full"})
expect(full.json()["detect_operating_mode"] == "learning_full", "learning_full is accepted")
if __name__ == "__main__":
+68
View File
@@ -0,0 +1,68 @@
"""Unit tests for student/teacher comparison. No database."""
from __future__ import annotations
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
from unittest.mock import patch
from detect_learning import canonicalize_detect_mode, decision_for_mention
from detect_pipeline import (
MentionSpan,
compact_diff_counters,
map_entity_type,
protects,
same_mention,
types_equivalent,
)
def expect(ok: bool, message: str) -> None:
if not ok:
raise SystemExit(f"FAIL: {message}")
print(f"OK {message}")
def main() -> None:
expect(canonicalize_detect_mode("learning") == "learning_full", "learning aliases to learning_full")
expect(canonicalize_detect_mode("semantic") == "bypass", "semantic aliases to bypass")
expect(canonicalize_detect_mode("wordlist") is None, "unknown mode is rejected")
expect(map_entity_type("person") == "PERSON", "person maps to PERSON")
expect(map_entity_type("place") == "PLACE", "place maps to PLACE")
expect(protects("PERSON") and not protects("FOOD"), "PERSON protects, FOOD does not")
expect(types_equivalent("PERSON", "PLACE"), "similar protectable types are equivalent")
expect(not types_equivalent("FOOD", "PERSON"), "FOOD versus PERSON is never equivalent")
inner = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "cue", True, "frau")
outer = MentionSpan("Frau Sushi", 15, 25, "PERSON", "identity", "teacher", False, "frau")
expect(same_mention(inner, outer), "contained span is the same mention")
food = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "teacher", False, "aß")
expect(same_mention(inner, food), "same offsets stay one mention even when types differ")
compact = compact_diff_counters({"diff_agree": 2, "diff_type_conflict": 1, "label": "Sushi"})
expect(compact["diff_agree"] == 2 and compact["diff_type_conflict"] == 1, "known counters are kept")
expect("label" not in compact and "Sushi" not in str(compact), "compact counters drop plaintext")
with patch("detect_learning.get_cue_decision", return_value=None), patch(
"detect_learning.get_sense",
return_value={"identity_hits": 3, "non_identity_hits": 0, "ambiguous": False},
), patch("detect_learning.cue_for_mention", return_value="aß"):
expect(
decision_for_mention("p", "Sushi", "Ich aß Sushi.", 7, 12) == "ask",
"Tag-21: identity-only sense does not lock a new cue",
)
with patch("detect_learning.get_cue_decision", return_value="identity"), patch(
"detect_learning.cue_for_mention", return_value="frau"
):
expect(
decision_for_mention("p", "Sushi", "meine Frau Sushi", 11, 16) == "identity",
"stored cue remains binding",
)
print("detect pipeline unit ok")
if __name__ == "__main__":
main()
+1 -1
View File
@@ -89,7 +89,7 @@ def main() -> None:
expect(set(roles) == {"generate", "detect"}, "two provider roles")
expect(providers.json()["remote_plaintext_allowed"] is True, "test runtime allows remote detect")
expect(providers.json()["remote_plaintext_reason"] == "non_production", "tests are not production")
expect(providers.json().get("detect_operating_mode") == "semantic", "detect mode defaults to semantic")
expect(providers.json().get("detect_operating_mode") == "learning_full", "detect mode defaults to learning stage 1")
expect(roles["generate"]["ready"] is False, "generate fail-closed without key")
expect("sk-" not in providers.text, "provider status has no secret")
expect("KANSHO_PROVIDER_KEY=" not in providers.text, "env assignment not leaked")
+54
View File
@@ -0,0 +1,54 @@
"""GLiNER client: local URL only, fail-soft, no request text in logs."""
from __future__ import annotations
import os
import sys
from pathlib import Path
from unittest.mock import patch
import httpx
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
from gliner_client import DEFAULT_URL, extract_entities, health, normalize_base_url
def expect(ok: bool, message: str) -> None:
if not ok:
raise SystemExit(f"FAIL: {message}")
print(f"OK {message}")
class _Resp:
def __init__(self, status_code: int, payload):
self.status_code = status_code
self._payload = payload
def json(self):
return self._payload
def main() -> None:
expect(normalize_base_url("http://192.168.2.144:8090/") == DEFAULT_URL, "default GLiNER URL is normalized")
expect(extract_entities("Anna", url="https://openrouter.ai/api/v1") is None, "OpenRouter URL is rejected")
expect(health("https://openrouter.ai").get("reason") == "not_local", "health rejects non-local URL")
with patch("gliner_client.httpx.post", return_value=_Resp(200, {"spans": [{"text": "Anna", "start": 0, "end": 4, "label": "person", "score": 0.9}]})):
spans = extract_entities("Anna kam.", url="http://192.168.2.144:8090")
expect(spans == [{"text": "Anna", "start": 0, "end": 4, "label": "person", "score": 0.9}], "extract returns spans")
with patch("gliner_client.httpx.post", side_effect=httpx.ConnectError("down")):
expect(extract_entities("Anna kam.", url="http://192.168.2.144:8090") is None, "extract is fail-soft on errors")
with patch("gliner_client.httpx.get", return_value=_Resp(200, {"status": "ok", "model": "urchade/gliner_multi-v2.1", "offline": True})):
ping = health("http://192.168.2.144:8090")
expect(ping.get("ok") is True and ping.get("offline") is True, "health reports ready offline model")
print("gliner client: OK")
if __name__ == "__main__":
main()
+1 -1
View File
@@ -31,7 +31,7 @@ def expect(ok: bool, message: str) -> None:
def main() -> None:
expect(is_local_url("http://127.0.0.1:11434/v1/chat/completions"), "loopback is local")
expect(is_local_url("http://localhost:11434/v1/chat/completions"), "localhost is local")
expect(is_local_url("http://192.168.2.144:11434/v1/chat/completions"), "RFC1918 Ollama host is local")
expect(is_local_url("http://192.168.2.144:8090"), "RFC1918 GLiNER host is local")
expect(is_local_url("http://10.0.0.8:11434/v1/chat/completions"), "10/8 is local")
expect(is_local_url("http://ollama:11434/v1/chat/completions"), "compose hostname ollama is local")
expect(not is_local_url("https://openrouter.ai/api/v1/chat/completions"), "OpenRouter is not local")