Add transitional learning detect with in-dialog mask review.
Some checks failed
Deploy Development / deploy (push) Successful in 57s
Test Suite / pytest-backend (push) Failing after 2m57s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 16s

Senses grow from confirmations instead of a word list, so a later local pipeline can decide homonyms on a short passage. Default stays semantic. Local detect waits longer, and the reverse-proxy timeout is documented so Dev does not 504 first.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Lars 2026-09-09 08:34:43 +02:00
parent 59921fc5b5
commit fa1c11c33e
29 changed files with 1135 additions and 45 deletions

410
backend/detect_learning.py Normal file
View File

@ -0,0 +1,410 @@
"""Transitional learning detect: senses grow from dialogue confirmations.
Not a global word list. Detect still does not auto-activate identities.
Ambiguous spellings get a local passage call when a local detect provider exists.
"""
from __future__ import annotations
import json
import re
import uuid
from typing import Any
from db import get_db, row_to_dict
from identity_store import ENTITY_TYPES, confirm_identity, is_maskable_label, normalize_label
MODE_SEMANTIC = "semantic"
MODE_LEARNING = "learning"
MODES = (MODE_SEMANTIC, MODE_LEARNING)
SETTING_KEY = "detect_operating_mode"
_LETTER = r"A-Za-zÄÖÜäöüß"
WINDOW = 80
_DECISION_IDENTITY = "identity"
_DECISION_NOT = "not_identity"
def get_detect_operating_mode() -> str:
with get_db() as conn:
row = row_to_dict(
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
)
value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower()
return value if value in MODES else MODE_SEMANTIC
def set_detect_operating_mode(mode: str) -> str:
chosen = (mode or "").strip().lower()
if chosen not in MODES:
raise ValueError("invalid_detect_operating_mode")
with get_db() as conn:
conn.execute(
"""
INSERT INTO app_settings (key, value, updated)
VALUES (?, ?, datetime('now'))
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
""",
(SETTING_KEY, chosen),
)
return chosen
def get_sense(profile_id: str, label: str) -> dict:
key = normalize_label(label) or (label or "").strip()
if not key:
return {
"normalized_label": "",
"identity_hits": 0,
"non_identity_hits": 0,
"ambiguous": False,
}
with get_db() as conn:
row = row_to_dict(
conn.execute(
"""
SELECT normalized_label, identity_hits, non_identity_hits
FROM label_senses
WHERE profile_id = ? AND lower(normalized_label) = lower(?)
""",
(profile_id, key),
).fetchone()
)
if not row:
return {
"normalized_label": key,
"identity_hits": 0,
"non_identity_hits": 0,
"ambiguous": False,
}
identity = int(row.get("identity_hits") or 0)
other = int(row.get("non_identity_hits") or 0)
return {
"normalized_label": row.get("normalized_label") or key,
"identity_hits": identity,
"non_identity_hits": other,
"ambiguous": identity > 0 and other > 0,
}
def record_sense(profile_id: str, label: str, *, identity: bool) -> dict:
key = normalize_label(label) or (label or "").strip()
if not key:
raise ValueError("empty_label")
with get_db() as conn:
existing = row_to_dict(
conn.execute(
"""
SELECT normalized_label, identity_hits, non_identity_hits FROM label_senses
WHERE profile_id = ? AND lower(normalized_label) = lower(?)
""",
(profile_id, key),
).fetchone()
)
if existing:
key = existing.get("normalized_label") or key
identity_hits = int((existing or {}).get("identity_hits") or 0)
non_identity_hits = int((existing or {}).get("non_identity_hits") or 0)
if identity:
identity_hits += 1
else:
non_identity_hits += 1
conn.execute(
"""
INSERT INTO label_senses (
profile_id, normalized_label, identity_hits, non_identity_hits, updated
)
VALUES (?, ?, ?, ?, datetime('now'))
ON CONFLICT(profile_id, normalized_label) DO UPDATE SET
identity_hits = excluded.identity_hits,
non_identity_hits = excluded.non_identity_hits,
updated = datetime('now')
""",
(profile_id, key, identity_hits, non_identity_hits),
)
return get_sense(profile_id, key)
def _in_user_text(label: str, user_body: str) -> bool:
if not label or not user_body:
return False
return bool(
re.search(
rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
user_body,
re.IGNORECASE,
)
)
def _excerpt(user_body: str, start: int | None, end: int | None, label: str) -> str:
text = user_body or ""
if start is None or end is None or start < 0 or end > len(text):
match = re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text, re.IGNORECASE)
if not match:
return text[:WINDOW]
start, end = match.start(), match.end()
left = max(0, int(start) - WINDOW)
right = min(len(text), int(end) + WINDOW)
return text[left:right]
def needs_review(profile_id: str, mapping: dict, user_body: str) -> bool:
label = mapping.get("local_label") or ""
if mapping.get("source") == "confirmed_registry":
sense = get_sense(profile_id, label)
return bool(sense["ambiguous"]) and _in_user_text(label, user_body)
if mapping.get("source") != "request_local":
return False
if not is_maskable_label(label) or not _in_user_text(label, user_body):
return False
sense = get_sense(profile_id, label)
if sense["identity_hits"] > 0 and not sense["ambiguous"]:
return False
return True
def build_candidates(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
seen: set[tuple[int | None, int | None, str]] = set()
items: list[dict] = []
for mapping in mappings:
if not needs_review(profile_id, mapping, user_body):
continue
label = mapping.get("local_label") or ""
start = mapping.get("start")
end = mapping.get("end")
key = (start, end, label.casefold())
if key in seen:
continue
seen.add(key)
items.append(
{
"id": str(uuid.uuid4()),
"text": label,
"entity_type": mapping.get("entity_type") or "PERSON",
"start": start,
"end": end,
"excerpt": _excerpt(user_body, start, end, label),
"ambiguous": get_sense(profile_id, label)["ambiguous"],
}
)
return items
def suppress_known_non_identity(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
kept = []
for mapping in mappings:
label = mapping.get("local_label") or ""
sense = get_sense(profile_id, label)
if (
mapping.get("source") == "request_local"
and sense["non_identity_hits"] > 0
and not sense["identity_hits"]
and _in_user_text(label, user_body)
):
continue
kept.append(mapping)
return kept
def save_pending(
profile_id: str,
conversation_id: str,
user_message_id: str,
payload: dict,
) -> str:
review_id = str(uuid.uuid4())
with get_db() as conn:
conn.execute(
"DELETE FROM pending_mask_reviews WHERE profile_id = ? AND conversation_id = ?",
(profile_id, conversation_id),
)
conn.execute(
"""
INSERT INTO pending_mask_reviews (
id, profile_id, conversation_id, user_message_id, payload_json, created
)
VALUES (?, ?, ?, ?, ?, datetime('now'))
""",
(review_id, profile_id, conversation_id, user_message_id, json.dumps(payload, ensure_ascii=False)),
)
return review_id
def load_pending(profile_id: str, review_id: str) -> dict | None:
with get_db() as conn:
row = row_to_dict(
conn.execute(
"""
SELECT * FROM pending_mask_reviews
WHERE id = ? AND profile_id = ?
""",
(review_id, profile_id),
).fetchone()
)
if not row:
return None
payload = json.loads(row.get("payload_json") or "{}")
payload["id"] = row["id"]
payload["conversation_id"] = row["conversation_id"]
payload["user_message_id"] = row["user_message_id"]
return payload
def drop_pending(profile_id: str, review_id: str) -> None:
with get_db() as conn:
conn.execute(
"DELETE FROM pending_mask_reviews WHERE id = ? AND profile_id = ?",
(review_id, profile_id),
)
def apply_review_decisions(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
by_id = {item["id"]: item for item in pending.get("candidates") or []}
drop_keys: set[tuple] = set()
for raw in decisions:
candidate = by_id.get(raw.get("id") or "")
if not candidate:
continue
decision = (raw.get("decision") or "").strip()
label = candidate.get("text") or ""
kind = candidate.get("entity_type") if candidate.get("entity_type") in ENTITY_TYPES else "PERSON"
if decision == _DECISION_IDENTITY:
record_sense(profile_id, label, identity=True)
confirm_identity(profile_id, label, entity_type=kind)
elif decision == _DECISION_NOT:
record_sense(profile_id, label, identity=False)
drop_keys.add((candidate.get("start"), candidate.get("end"), label.casefold()))
mappings = []
for mapping in pending.get("mappings") or []:
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
if key in drop_keys:
continue
mappings.append(mapping)
return suppress_known_non_identity(profile_id, mappings, pending.get("user_body") or "")
def confirm_known_identity_spans(profile_id: str, mappings: list[dict], user_body: str) -> None:
for mapping in mappings:
label = mapping.get("local_label") or ""
sense = get_sense(profile_id, label)
if not (
sense["identity_hits"] > 0
and not sense["ambiguous"]
and _in_user_text(label, user_body)
):
continue
kind = mapping.get("entity_type") if mapping.get("entity_type") in ENTITY_TYPES else "PERSON"
try:
confirm_identity(profile_id, label, entity_type=kind)
except ValueError:
continue
def list_senses(profile_id: str) -> list[dict]:
with get_db() as conn:
rows = conn.execute(
"""
SELECT normalized_label, identity_hits, non_identity_hits
FROM label_senses WHERE profile_id = ?
ORDER BY lower(normalized_label)
""",
(profile_id,),
).fetchall()
items = []
for raw in rows:
row = row_to_dict(raw) or {}
identity = int(row.get("identity_hits") or 0)
other = int(row.get("non_identity_hits") or 0)
items.append(
{
"normalized_label": row.get("normalized_label") or "",
"identity_hits": identity,
"non_identity_hits": other,
"ambiguous": identity > 0 and other > 0,
}
)
return items
def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | None:
with get_db() as conn:
row = row_to_dict(
conn.execute(
"""
SELECT id, payload_json FROM pending_mask_reviews
WHERE profile_id = ? AND conversation_id = ?
""",
(profile_id, conversation_id),
).fetchone()
)
if not row:
return None
payload = json.loads(row.get("payload_json") or "{}")
return {
"id": row["id"],
"candidates": payload.get("candidates") or [],
}
def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: list[dict]) -> tuple[list[dict], list[dict]]:
"""Local passage LLM for already-ambiguous spellings. Unresolved stay for the popup."""
remaining = []
drop_keys: set[tuple] = set()
for item in candidates:
if not item.get("ambiguous"):
remaining.append(item)
continue
label = item.get("text") or ""
decision = try_local_passage_decision(item.get("excerpt") or "", label)
if decision == _DECISION_NOT:
record_sense(profile_id, label, identity=False)
drop_keys.add((item.get("start"), item.get("end"), label.casefold()))
elif decision == _DECISION_IDENTITY:
record_sense(profile_id, label, identity=True)
else:
remaining.append(item)
if not drop_keys:
return remaining, mappings
kept = []
for mapping in mappings:
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
if key in drop_keys:
continue
kept.append(mapping)
return remaining, kept
def try_local_passage_decision(excerpt: str, label: str) -> str | None:
"""Return identity, not_identity, or None if no local model or unusable answer."""
from providers import ProviderError, complete_chat, detect_provider
config = detect_provider()
if not config or not config.local or config.mode != "http":
return None
prompt = (
"Entscheide nur für die markierte Nennung in diesem kurzen Ausschnitt. "
"Ist sie eine schützenswerte Identität (Person, Ort, Organisation, privates Projekt) "
f"oder eine Sache/Allgemeinbedeutung? Wort: {label}\n"
f"Ausschnitt: {excerpt}\n"
'Antworte nur mit JSON {"decision":"identity"} oder {"decision":"not_identity"}.'
)
try:
result = complete_chat(
config,
[{"role": "user", "content": prompt}],
timeout=30.0,
max_tokens=32,
)
except ProviderError:
return None
match = re.search(r"\{.*\}", result.content or "", re.DOTALL)
if not match:
return None
try:
data = json.loads(match.group(0))
except json.JSONDecodeError:
return None
value = (data.get("decision") or "").strip()
if value in {_DECISION_IDENTITY, _DECISION_NOT}:
return value
return None

View File

@ -7,9 +7,21 @@ import re
from context_builder import assemble_text, build_internal_context, is_closing_turn
from conversation_signals import infer_signals
from debug_store import persist_engine_error, persist_step
from detect_learning import (
MODE_LEARNING,
apply_review_decisions,
auto_resolve_ambiguous,
build_candidates,
confirm_known_identity_spans,
drop_pending,
get_detect_operating_mode,
load_pending,
save_pending,
suppress_known_non_identity,
)
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
from engine import EngineError, execute_prompt, load_active_prompt
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError, detect_personal_egress
from writing_profile_store import remember_dialogue_style
from profile_review import consider_dialogue
@ -302,23 +314,21 @@ def visible_for_role(payload: dict, role: str | None) -> dict:
return cleaned
def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str | None = None) -> dict:
conversation = get_conversation(profile_id, conversation_id)
user = append_message(profile_id, conversation_id, body, role="user", message_id=message_id)
context = build_internal_context(
profile_id,
conversation_id=conversation_id,
space_id=conversation.get("space_id"),
journal_day_id=conversation.get("journal_day_id"),
purpose="dialogue_turn",
)
assembled = assemble_text(context)
prompt = load_active_prompt("mvp.dialogue_turn")
def _finish_turn(
profile_id: str,
conversation_id: str,
user: dict,
assembled: dict,
prompt: dict,
*,
precomputed_mappings: list[dict] | None = None,
) -> dict:
calls = 0
result = None
impulse = ""
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
call_traces: list[dict] = []
working = dict(assembled)
try:
while calls < 2:
result = execute_prompt(
@ -326,7 +336,8 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
profile_id,
purpose="dialogue_turn",
data_class="B",
context=assembled,
context=working,
precomputed_mappings=precomputed_mappings if calls == 0 else None,
)
calls += 1
if result.get("trace"):
@ -335,16 +346,16 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
if not content:
raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.")
impulse, decision = parse_turn_payload(content)
if not needs_repair(impulse, assembled):
if not needs_repair(impulse, working):
break
decision = {**decision, "guard": "impulse_rejected"}
assembled = dict(assembled)
assembled["dialogue_context"] = (
(assembled.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, assembled)
working = dict(working)
working["dialogue_context"] = (
(working.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, working)
)
if needs_repair(impulse, assembled):
if needs_repair(impulse, working):
parsed_ok = bool(decision.get("parsed"))
impulse = local_hold(last_user_text(assembled))
impulse = local_hold(last_user_text(working))
decision = {
"operation": "fortfuehren",
"label": OPERATIONS["fortfuehren"],
@ -369,7 +380,7 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
},
)
raise
impulse = local_hold(last_user_text(assembled))
impulse = local_hold(last_user_text(working))
decision = {
"operation": "fortfuehren",
"label": OPERATIONS["fortfuehren"],
@ -388,7 +399,7 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
infer_signals(user_bodies, decision.get("operation")),
)
remember_dialogue_style(profile_id)
consider_dialogue(profile_id, body)
consider_dialogue(profile_id, user.get("body") or "")
trace = result.get("trace") if result else None
persist_step(
profile_id,
@ -416,3 +427,99 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
"decision": decision,
"trace": trace,
}
def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled: dict, prompt: dict) -> dict | None:
preview = preview_prompt(prompt, assembled)
rendered = preview.get("rendered") or ""
try:
outcome = detect_personal_egress(profile_id, rendered)
except DetectError:
return None
user_body = user.get("body") or ""
mappings = list(outcome.mappings or [])
candidates = build_candidates(profile_id, mappings, user_body)
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
if not candidates:
confirm_known_identity_spans(profile_id, mappings, user_body)
mappings = suppress_known_non_identity(profile_id, mappings, user_body)
return _finish_turn(
profile_id,
conversation_id,
user,
assembled,
prompt,
precomputed_mappings=mappings,
)
review_id = save_pending(
profile_id,
conversation_id,
user.get("id") or "",
{
"mappings": mappings,
"candidates": candidates,
"user_body": user_body,
},
)
return {
"conversation": get_conversation(profile_id, conversation_id),
"user": user,
"assistant": None,
"calls": 0,
"messages": list_messages(profile_id, conversation_id),
"pending_mask_review": {
"id": review_id,
"candidates": candidates,
},
}
def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str | None = None) -> dict:
conversation = get_conversation(profile_id, conversation_id)
user = append_message(profile_id, conversation_id, body, role="user", message_id=message_id)
context = build_internal_context(
profile_id,
conversation_id=conversation_id,
space_id=conversation.get("space_id"),
journal_day_id=conversation.get("journal_day_id"),
purpose="dialogue_turn",
)
assembled = assemble_text(context)
prompt = load_active_prompt("mvp.dialogue_turn")
if get_detect_operating_mode() == MODE_LEARNING:
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
if paused is not None:
return paused
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
def continue_turn(profile_id: str, conversation_id: str, review_id: str, decisions: list[dict]) -> dict:
pending = load_pending(profile_id, review_id)
if not pending or pending.get("conversation_id") != conversation_id:
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
mappings = apply_review_decisions(profile_id, pending, decisions)
drop_pending(profile_id, review_id)
conversation = get_conversation(profile_id, conversation_id)
context = build_internal_context(
profile_id,
conversation_id=conversation_id,
space_id=conversation.get("space_id"),
journal_day_id=conversation.get("journal_day_id"),
purpose="dialogue_turn",
)
assembled = assemble_text(context)
prompt = load_active_prompt("mvp.dialogue_turn")
user = {"id": pending.get("user_message_id"), "body": pending.get("user_body") or ""}
messages = list_messages(profile_id, conversation_id)
for item in messages:
if item.get("id") == pending.get("user_message_id"):
user = item
break
return _finish_turn(
profile_id,
conversation_id,
user,
assembled,
prompt,
precomputed_mappings=mappings,
)

View File

@ -75,6 +75,7 @@ def execute_prompt(
disable_context_compression: bool = False,
budget=None,
diagnostics: dict[str, Any] | None = None,
precomputed_mappings: list[dict] | None = None,
) -> dict:
preview = preview_prompt(prompt, context)
feature_id = prompt.get("required_feature") or "ai_calls"
@ -107,6 +108,7 @@ def execute_prompt(
"disable_context_compression": disable_context_compression,
"budget": budget,
"diagnostics": diagnostics or {},
"precomputed_mappings": precomputed_mappings,
},
)
)

View File

@ -31,7 +31,9 @@ from providers import ChatResult, ProviderError, complete_chat, detect_provider
DETECT_CHUNK_CHARS = 4000
DETECT_CHUNK_OVERLAP = 250
DETECT_TIMEOUT = 90.0
DETECT_TIMEOUT_LOCAL = 300.0
DETECT_MAX_TOKENS = 1024
DETECT_MAX_TOKENS_LOCAL = 256
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
@ -573,6 +575,13 @@ def _add_usage(stats: DetectionStats, usage: dict | None) -> None:
pass
def detect_chat_limits(config) -> tuple[float, int]:
"""Remote detect stays at 90s/1024. Local CPU hosts need a longer wait and a shorter cap."""
if getattr(config, "local", False):
return DETECT_TIMEOUT_LOCAL, DETECT_MAX_TOKENS_LOCAL
return DETECT_TIMEOUT, DETECT_MAX_TOKENS
def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResult:
prompt = resolve_template(
_detect_prompt()["template"],
@ -580,11 +589,12 @@ def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResul
)
if schema_retry:
prompt = f"{prompt.rstrip()}\n\n{SCHEMA_RETRY_HINT}"
timeout, max_tokens = detect_chat_limits(config)
return complete_chat(
config,
[{"role": "user", "content": prompt}],
timeout=DETECT_TIMEOUT,
max_tokens=DETECT_MAX_TOKENS,
timeout=timeout,
max_tokens=max_tokens,
disable_context_compression=True,
)

View File

@ -0,0 +1,23 @@
-- Transitional learning detect: user-confirmed label senses, pending mask review.
-- Detect output still does not auto-activate identities.
CREATE TABLE IF NOT EXISTS label_senses (
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
normalized_label TEXT NOT NULL,
identity_hits INTEGER NOT NULL DEFAULT 0,
non_identity_hits INTEGER NOT NULL DEFAULT 0,
updated TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text,
PRIMARY KEY (profile_id, normalized_label)
);
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
id TEXT PRIMARY KEY,
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
user_message_id TEXT NOT NULL,
payload_json TEXT NOT NULL DEFAULT '{}',
created TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text
);
CREATE INDEX IF NOT EXISTS idx_pending_mask_reviews_conv
ON pending_mask_reviews (profile_id, conversation_id);

View File

@ -1055,7 +1055,26 @@ def complete(request: GatewayRequest) -> GatewayResult:
detect_note = None
detect_stats: dict[str, Any] = {}
local_identities: list[dict[str, Any]] = []
precomputed = request.payload.get("precomputed_mappings") if request.payload else None
try:
if precomputed is not None:
mappings = list(precomputed)
detect_stats = {
"detect_note": "precomputed_learning_review",
"full_detection_coverage": True,
"semantic_identity_guaranteed": False,
}
local_identities = [
{
"local_label": item.get("local_label"),
"token": item.get("token"),
"entity_type": item.get("entity_type"),
"source": item.get("source"),
}
for item in mappings
]
detect_name = "learning_review"
else:
outcome = detect_personal_egress(request.profile_id, rendered)
mappings = outcome.mappings
detect_stats = outcome.stats.public()

View File

@ -8,6 +8,7 @@ from pathlib import Path
from urllib.parse import urlparse
from db import get_db
from detect_learning import get_detect_operating_mode
from env_loader import (
allows_remote_plaintext_detect,
remote_plaintext_detect_reason,
@ -460,4 +461,5 @@ def public_status() -> dict:
"remote_plaintext_reason": remote_plaintext_detect_reason(),
"remote_plaintext_allowed": allows_remote_plaintext_detect(),
"ollama_url": OLLAMA_LAN_URL,
"detect_operating_mode": get_detect_operating_mode(),
}

View File

@ -20,6 +20,7 @@ from debug_store import (
settings_payload,
)
from dialogue_store import StoreError, get_conversation, inventory, list_conversations, list_derived_for_conversation, list_messages
from detect_learning import list_senses
from identity_store import (
confirm_identity,
confirm_review_proposal,
@ -226,6 +227,28 @@ class ActivateProfile(BaseModel):
profile_id: str
class DetectModeWrite(BaseModel):
mode: str
@router.put("/providers/detect-mode")
def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require_admin_dep)):
from detect_learning import set_detect_operating_mode
from provider_settings import public_status as provider_public_status
try:
set_detect_operating_mode(body.mode)
except ValueError:
raise HTTPException(
status_code=400,
detail={
"code": "invalid_detect_operating_mode",
"message": "Detect-Modus muss semantic oder learning sein.",
},
)
return provider_public_status()
@router.get("/providers")
def admin_providers(session: dict = Depends(require_admin_dep)):
return public_status()
@ -364,6 +387,7 @@ def admin_identities(session: dict = Depends(require_admin_dep)):
return {
"registry": list_registry(profile_id),
"proposals": list_review_proposals(profile_id, include_dismissed=True),
"senses": list_senses(profile_id),
"note": "Nur lokale bestätigte Registry und unbestätigte Vorschläge. Kein externer Egress.",
"backup": (
"Vor einer Bereinigung die lokale Datei backend/data/kansho.sqlite kopieren. "

View File

@ -7,7 +7,8 @@ from auth import require_auth
from context_builder import build_internal_context
from continuity import checkpoint_usage_session, close_usage_session
from derived_kinds import catalog
from dialogue_turn import run_turn, visible_for_role
from detect_learning import pending_for_conversation
from dialogue_turn import continue_turn, run_turn, visible_for_role
from engine import EngineError
from privacy_gateway import public_error_detail
from dialogue_store import (
@ -51,6 +52,11 @@ class MessageWrite(BaseModel):
id: str | None = None
class MaskReviewWrite(BaseModel):
review_id: str
decisions: list[dict] = Field(default_factory=list)
class ThreadWrite(BaseModel):
title: str = ""
status: str = "open"
@ -140,6 +146,9 @@ def get_one_conversation(conversation_id: str, session: dict = Depends(require_a
conv = get_conversation(session["profile_id"], conversation_id)
conv["messages"] = list_messages(session["profile_id"], conversation_id)
conv["derived"] = list_derived_for_conversation(session["profile_id"], conversation_id)
pending = pending_for_conversation(session["profile_id"], conversation_id)
if pending:
conv["pending_mask_review"] = pending
return conv
except StoreError as exc:
_http(exc)
@ -172,6 +181,17 @@ def conversation_turn(conversation_id: str, req: MessageWrite, session: dict = D
_http(exc)
@router.post("/conversations/{conversation_id}/turn/review")
def conversation_turn_review(conversation_id: str, req: MaskReviewWrite, session: dict = Depends(require_auth)):
try:
return visible_for_role(
continue_turn(session["profile_id"], conversation_id, req.review_id, req.decisions),
session.get("role"),
)
except (StoreError, EngineError) as exc:
_http(exc)
@router.get("/conversations/{conversation_id}/messages")
def get_messages(conversation_id: str, session: dict = Depends(require_auth)):
try:

View File

@ -5,8 +5,9 @@ from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from auth import require_auth
from detect_learning import pending_for_conversation
from dialogue_store import StoreError, delete_conversation, get_conversation, list_messages
from dialogue_turn import run_turn, visible_for_role
from dialogue_turn import continue_turn, run_turn, visible_for_role
from privacy_gateway import GatewayRequest, inspect, public_error_detail
from engine import EngineError
from journal_generate import generate_draft
@ -96,6 +97,11 @@ class TurnWrite(BaseModel):
id: str | None = None
class MaskReviewWrite(BaseModel):
review_id: str
decisions: list[dict] = Field(default_factory=list)
class GenerateWrite(BaseModel):
conversation_ids: list[str] | None = None
include_existing: bool = False
@ -286,7 +292,11 @@ def remove_conversation(conversation_id: str, session: dict = Depends(require_au
def read_conversation(conversation_id: str, session: dict = Depends(require_auth)):
try:
conv = get_conversation(session["profile_id"], conversation_id)
return {"conversation": conv, "messages": list_messages(session["profile_id"], conversation_id)}
payload = {"conversation": conv, "messages": list_messages(session["profile_id"], conversation_id)}
pending = pending_for_conversation(session["profile_id"], conversation_id)
if pending:
payload["pending_mask_review"] = pending
return payload
except StoreError as exc:
_http(exc)
@ -308,6 +318,17 @@ def conversation_turn(conversation_id: str, body: TurnWrite, session: dict = Dep
_http(exc)
@router.post("/conversations/{conversation_id}/turn/review")
def conversation_turn_review(conversation_id: str, body: MaskReviewWrite, session: dict = Depends(require_auth)):
try:
return visible_for_role(
continue_turn(session["profile_id"], conversation_id, body.review_id, body.decisions),
session.get("role"),
)
except (StoreError, EngineError) as exc:
_http(exc)
@router.get("/generation-settings")
def read_generation_settings(session: dict = Depends(require_auth)):
return settings_payload(session["profile_id"])

View File

@ -230,6 +230,27 @@ CREATE TABLE IF NOT EXISTS identity_review_proposals (
UNIQUE (profile_id, observed_label, entity_type)
);
CREATE TABLE IF NOT EXISTS label_senses (
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
normalized_label TEXT NOT NULL,
identity_hits INTEGER NOT NULL DEFAULT 0,
non_identity_hits INTEGER NOT NULL DEFAULT 0,
updated TEXT NOT NULL DEFAULT (datetime('now')),
PRIMARY KEY (profile_id, normalized_label)
);
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
id TEXT PRIMARY KEY,
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
user_message_id TEXT NOT NULL,
payload_json TEXT NOT NULL DEFAULT '{}',
created TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_pending_mask_reviews_conv
ON pending_mask_reviews (profile_id, conversation_id);
CREATE TABLE IF NOT EXISTS journal_days (
id TEXT PRIMARY KEY,
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,

View File

@ -43,6 +43,8 @@ TABLES = [
"re_grounding_events",
"identity_mappings",
"identity_review_proposals",
"label_senses",
"pending_mask_reviews",
"journal_days",
"journal_drafts",
"journal_entries",

View File

@ -0,0 +1,184 @@
"""Transitional learning detect: senses from dialogue review, admin mode switch.
Fake detect/provider only. Run from backend/: python tests/test_detect_learning.py
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
os.environ["KANSHO_FAKE_DETECT"] = "1"
from fastapi.testclient import TestClient
from detect_learning import get_detect_operating_mode, get_sense, record_sense
from dialogue_turn import visible_for_role
from identity_store import list_confirmed_identities
from main import app
from privacy_gateway import reset_debug
def expect(ok: bool, message: str) -> None:
if not ok:
raise SystemExit(f"FAIL: {message}")
print(f"OK {message}")
def status_ok(response, label: str) -> None:
if response.status_code != 200:
detail = response.json() if response.headers.get("content-type", "").startswith("application/json") else {}
code = detail.get("detail", {}).get("code") if isinstance(detail, dict) else None
expect(False, f"{label} HTTP {response.status_code} {code or ''}".strip())
expect(True, label)
def header(token: str) -> dict:
return {"X-Auth-Token": token}
def open_conv(client: TestClient, headers: dict) -> str:
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"})
day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": "2026-09-08"},
)
conv = client.post(
f"/api/journal/days/{day.json()['day']['id']}/conversations",
headers=headers,
json={"title": "Gespräch"},
)
return conv.json()["id"]
def main() -> None:
reset_debug()
with TestClient(app) as client:
setup = client.post(
"/api/auth/setup",
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
)
headers = header(setup.json()["token"])
profile_id = setup.json()["profile_id"]
providers = client.get("/api/admin/providers", headers=headers)
expect(providers.status_code == 200, "providers status")
expect(providers.json()["detect_operating_mode"] == "semantic", "default detect mode is semantic")
expect(get_detect_operating_mode() == "semantic", "store default is semantic")
bad = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "wordlist"})
expect(bad.status_code == 400, "invalid detect mode is rejected")
conv_semantic = open_conv(client, headers)
semantic_turn = client.post(
f"/api/journal/conversations/{conv_semantic}/turn",
headers=headers,
json={"body": "Anna kam vorbei."},
)
status_ok(semantic_turn, "semantic turn")
expect(not semantic_turn.json().get("pending_mask_review"), "semantic mode does not pause for review")
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "semantic turn gets an assistant")
switched = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning"})
status_ok(switched, "switch to learning")
expect(switched.json()["detect_operating_mode"] == "learning", "learning mode persisted")
conv = open_conv(client, headers)
paused = client.post(
f"/api/journal/conversations/{conv}/turn",
headers=headers,
json={"body": "Anna kam vorbei."},
)
status_ok(paused, "learning pause")
review = paused.json().get("pending_mask_review") or {}
expect(bool(review.get("id")), "learning mode returns a review id")
expect(paused.json().get("assistant") is None, "paused turn has no assistant")
candidates = review.get("candidates") or []
anna = next((item for item in candidates if item.get("text") == "Anna"), None)
expect(anna is not None, "Anna is a review candidate")
hidden = visible_for_role({"pending_mask_review": review, "trace": {"egress": "x"}}, "user")
expect("pending_mask_review" in hidden, "users still receive the mask review")
expect("trace" not in hidden, "users do not receive the detect trace")
reloaded = client.get(f"/api/journal/conversations/{conv}", headers=headers)
expect(reloaded.json().get("pending_mask_review", {}).get("id") == review["id"], "pending review survives reload")
missing = client.post(
f"/api/journal/conversations/{conv}/turn/review",
headers=headers,
json={"review_id": "missing", "decisions": []},
)
expect(missing.status_code == 404, "unknown review is gone")
identity = client.post(
f"/api/journal/conversations/{conv}/turn/review",
headers=headers,
json={
"review_id": review["id"],
"decisions": [{"id": anna["id"], "decision": "identity"}],
},
)
status_ok(identity, "identity review")
expect(not identity.json().get("pending_mask_review"), "review continue clears the pause")
expect(any(item.get("role") == "assistant" for item in identity.json().get("messages") or []), "continue produces an assistant")
confirmed = list_confirmed_identities(profile_id)
expect(any((item.get("canonical_label") or "").casefold() == "anna" for item in confirmed), "identity review confirms the registry")
sense = get_sense(profile_id, "Anna")
expect(sense["identity_hits"] >= 1 and not sense["ambiguous"], "Anna is identity-only")
second = client.post(
f"/api/journal/conversations/{conv}/turn",
headers=headers,
json={"body": "Anna hat später angerufen."},
)
status_ok(second, "second Anna turn")
expect(not second.json().get("pending_mask_review"), "identity-only spelling skips the popup")
food_conv = open_conv(client, headers)
food = client.post(
f"/api/journal/conversations/{food_conv}/turn",
headers=headers,
json={"body": "Hanna aß mit uns."},
)
status_ok(food, "hanna pause")
food_review = food.json().get("pending_mask_review") or {}
food_candidates = food_review.get("candidates") or []
hanna = next((item for item in food_candidates if item.get("text") == "Hanna"), None)
expect(hanna is not None, "Hanna is a review candidate")
declined = client.post(
f"/api/journal/conversations/{food_conv}/turn/review",
headers=headers,
json={
"review_id": food_review["id"],
"decisions": [{"id": hanna["id"], "decision": "not_identity"}],
},
)
status_ok(declined, "not_identity review")
expect(not any((item.get("canonical_label") or "").casefold() == "hanna" for item in list_confirmed_identities(profile_id)), "not_identity does not confirm Hanna")
expect(get_sense(profile_id, "Hanna")["non_identity_hits"] >= 1, "not_identity records a sense")
record_sense(profile_id, "Clarissa", identity=True)
record_sense(profile_id, "Clarissa", identity=False)
expect(get_sense(profile_id, "Clarissa")["ambiguous"], "both senses mark Clarissa as ambiguous")
amb_conv = open_conv(client, headers)
with patch("detect_learning.try_local_passage_decision", return_value="not_identity"):
amb = client.post(
f"/api/journal/conversations/{amb_conv}/turn",
headers=headers,
json={"body": "Clarissa lag auf dem Tisch."},
)
status_ok(amb, "ambiguous local passage")
expect(not amb.json().get("pending_mask_review"), "local passage can finish without a popup")
expect(any(item.get("role") == "assistant" for item in amb.json().get("messages") or []), "local passage still generates")
back = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "semantic"})
expect(back.json()["detect_operating_mode"] == "semantic", "mode can return to semantic")
if __name__ == "__main__":
main()

View File

@ -89,6 +89,7 @@ def main() -> None:
expect(set(roles) == {"generate", "detect"}, "two provider roles")
expect(providers.json()["remote_plaintext_allowed"] is True, "test runtime allows remote detect")
expect(providers.json()["remote_plaintext_reason"] == "non_production", "tests are not production")
expect(providers.json().get("detect_operating_mode") == "semantic", "detect mode defaults to semantic")
expect(roles["generate"]["ready"] is False, "generate fail-closed without key")
expect("sk-" not in providers.text, "provider status has no secret")
expect("KANSHO_PROVIDER_KEY=" not in providers.text, "env assignment not leaked")

View File

@ -12,7 +12,14 @@ os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
Path(tempfile.gettempdir()).mkdir(parents=True, exist_ok=True)
from providers import is_local_url
from entity_detect import (
DETECT_MAX_TOKENS,
DETECT_MAX_TOKENS_LOCAL,
DETECT_TIMEOUT,
DETECT_TIMEOUT_LOCAL,
detect_chat_limits,
)
from providers import ProviderConfig, is_local_url
def expect(ok: bool, message: str) -> None:
@ -29,6 +36,33 @@ def main() -> None:
expect(is_local_url("http://ollama:11434/v1/chat/completions"), "compose hostname ollama is local")
expect(not is_local_url("https://openrouter.ai/api/v1/chat/completions"), "OpenRouter is not local")
expect(not is_local_url("https://8.8.8.8/v1/chat/completions"), "public IP is not local")
remote = ProviderConfig(
role="detect",
name="openrouter",
mode="http",
url="https://openrouter.ai/api/v1/chat/completions",
model="openai/gpt-4.1-nano",
key="x",
local=False,
zdr=True,
no_train=True,
)
local = ProviderConfig(
role="detect",
name="ollama",
mode="http",
url="http://192.168.2.144:11434/v1/chat/completions",
model="phi3:mini",
key="",
local=True,
zdr=True,
no_train=True,
)
expect(detect_chat_limits(remote) == (DETECT_TIMEOUT, DETECT_MAX_TOKENS), "remote detect keeps 90s/1024")
expect(
detect_chat_limits(local) == (DETECT_TIMEOUT_LOCAL, DETECT_MAX_TOKENS_LOCAL),
"local detect waits 300s with a 256-token cap",
)
print("local url tests passed.")

View File

@ -49,6 +49,8 @@ cp .env.example .env
Host-nginx: `nginx/kansho.conf` und `nginx/kansho-dev.conf` nach `/etc/nginx/sites-available/`, dann `nginx/certbot-setup.sh`. DNS A/AAAA für `dev.kansho.jinkendo.de` und `kansho.jinkendo.de` auf den Reverse-Proxy. Bis TLS steht, bleibt der LAN-Zugriff über die Publish-Ports.
**Additiv 2026-09-08:** Live-TLS endet auf dem Synology-NAS `192.168.2.63` (DSM Reverse Proxy, nicht auf dem Kanshō-Pi). Pro Kanshō-Regel unter Erweitert Proxy-Lese- und -Sende-Timeout auf 600s. DSM-Default 60s liefert HTTP 504, während Detect/Generate auf dem Pi weiterlaufen. LAN `http://192.168.2.49:3096` umgeht diesen Hop (Frontend-Container bereits 600s). AdGuard-Rewrites bleiben auf `192.168.2.63`.
Gitea: Actions aktivieren. Derselbe Pi-Runner wie die Schwesterprodukte (`ubuntu-latest`).
Watchtower bleibt aus.

View File

@ -709,6 +709,18 @@ Das ist kein Admin-Feature-Flag und kein Umdeuten von `KANSHO_ENV=production` na
**Später prüfen:** Flag entfernen, sobald lokales Detect auf Prod läuft. Dann wieder Production ohne Klartext-Detect.
## 22.4 Übergang: Detect-Lernmodus (2026-09-08)
Additiv. Technische Abbildung: `../technical/privacy_gateway.md` §9.7.
**Entschieden (Übergang):** Doppeldeutigkeiten werden nicht als eigene Wortliste gepflegt. Sinne entstehen aus Bestätigungen im Dialog (Identität / nicht schützenswert). Beides bei derselben Schreibweise markiert sie als mehrdeutig.
**Entschieden (Übergang):** Im Admin konfigurierbarer Detector-Modus `semantic` (Default) oder `learning`. Lernmodus öffnet vor Generate ein Bestätigungs-Popup für Detect-Treffer im aktuellen Nutzersatz. Das ist mehr Arbeit am Anfang und kein Ersatz für semantische Detection.
**Entschieden (Übergang):** Eine Mini-Passage an ein internes Modell geht nur bei bereits mehrdeutiger Schreibweise und nur an ein lokales Detect. OpenRouter sieht diese Passage nicht. Fehlt das lokale Modell, bleibt das Popup.
**Nicht:** Gateway abschalten, Detect-Treffer auto-speichern, `Sushi_`/`Sushi+` im Nutzertext, Pattern-Wortliste als Wahrheit.
---
# 23. Externe Referenzquellen

View File

@ -55,6 +55,8 @@ Nicht übernehmen: Mitai-Admin für Körpertarife, Coupons, Training Types als K
**Additiv 2026-09-08 (LLM-Profile):** `GET/POST/PUT/DELETE /api/admin/llm-profiles`, `POST /api/admin/providers/{role}/activate`. Profile enthalten URL/Modell/ZDR, niemals Keys. LAN-Ollama (`192.168.2.144:11434`) ist lokal.
**Additiv 2026-09-08 (Detect-Lernmodus):** `PUT /api/admin/providers/detect-mode` mit `semantic` | `learning`. Statusfeld `detect_operating_mode`. Keine Wortlisten-Seite. Identitäten dürfen ein Badge „mehrdeutig“ aus Dialog-Sinnen zeigen. Dialog und Journal-Gespräch pausieren im Lernmodus mit Bestätigungs-Popup; Journal-Generate nicht.
## 4.2 Implementierungsstand (Dialog-Testspur)
**Additiv 2026-08-26:** Lokale Identitätsregistry unter `/admin/identities`. Detect-Vorschläge sind unbestätigt. Compact-Diagnose enthält Detect-Abdeckung, Chunks, Kosten und Laufzeit, aber keine Labels.

View File

@ -179,7 +179,7 @@ Editor speichert Markdown, nicht HTML. Medien-Token bleiben lokal. Dirty: In-App
| Schicht | Rolle | Call |
|---|---|---|
| Maskierung | `detect` | Vollständige semantische Detection des Generate-Egress. Externes Klartext-Detect in Development/Test; in Production default-off, optional Operator-Übergang `KANSHO_ALLOW_REMOTE_DETECT`. Ziel: lokales Detect-Modell. Kein Pattern-Fallback. |
| Maskierung | `detect` | Vollständige semantische Detection des Generate-Egress. Externes Klartext-Detect in Development/Test; in Production default-off, optional Operator-Übergang `KANSHO_ALLOW_REMOTE_DETECT`. Ziel: lokales Detect-Modell. Kein Pattern-Fallback. Admin-Modus `semantic` (Default) oder `learning` (Übergang, Bestätigung vor Generate). |
| Dialogzug | `generate` | Operation + Impuls |
| Journalentwurf | `generate` | Explizit, getrennt |

View File

@ -239,7 +239,7 @@ Bestätigte Registry-Zeilen und bestätigte Aliase werden im gesamten gerenderte
### Tests und Live-Qualität
Contract-Tests: `backend/tests/test_privacy_detect.py`, `backend/tests/test_identity_registry.py`, `backend/tests/test_privacy_response_integrity.py`, `backend/tests/test_detect_contract_retry.py`. Sie beweisen Schema, Fail-closed, span-genaue Maskierung und Datenfluss, nicht semantische Modellleistung. Opt-in: `python entity_detect_eval.py --live` mit synthetischen Sätzen und exakten erwarteten Spans. Ohne diesen Lauf bleibt die Live-Qualität unbestätigt. Das aktuell konfigurierte `openai/gpt-4.1-nano` gilt durch reale False-Positive-Vorschläge qualitativ nicht als zuverlässig bestätigt; das Modell wird deshalb nicht stillschweigend gewechselt.
Contract-Tests: `backend/tests/test_privacy_detect.py`, `backend/tests/test_identity_registry.py`, `backend/tests/test_privacy_response_integrity.py`, `backend/tests/test_detect_contract_retry.py`, `backend/tests/test_detect_learning.py`. Sie beweisen Schema, Fail-closed, span-genaue Maskierung, Lernmodus-Pause und Datenfluss, nicht semantische Modellleistung. Opt-in: `python entity_detect_eval.py --live` mit synthetischen Sätzen und exakten erwarteten Spans. Ohne diesen Lauf bleibt die Live-Qualität unbestätigt. Das aktuell konfigurierte `openai/gpt-4.1-nano` gilt durch reale False-Positive-Vorschläge qualitativ nicht als zuverlässig bestätigt; das Modell wird deshalb nicht stillschweigend gewechselt.
## 9.6 Übergang: `KANSHO_ALLOW_REMOTE_DETECT` (2026-09-08)
@ -251,7 +251,21 @@ Operator-Übergang bis Ollama: `KANSHO_ALLOW_REMOTE_DETECT=1` (Compose `.env`, C
**Additiv 2026-09-08 (LLM-Profile, LAN-Ollama):** `llm_profiles` speichert benannte URL/Modell-Presets ohne Keys. Jede Stufe (`generate` / `detect`) zeigt auf ein Profil; Wechsel kopiert die gespeicherten Felder, statt sie neu einzugeben. RFC1918-, Loopback- und `.local`-URLs gelten als lokale Trusted Zone. `http://192.168.2.144:11434/v1/chat/completions` ist damit lokales Detect, kein externes Klartext-Detect. Ollama muss auf dem Host auf `0.0.0.0:11434` lauschen; der Pi muss Port 11434 erreichen.
Status in `GET /api/admin/providers`: `remote_plaintext_allowed`, `remote_plaintext_reason` (`non_production` | `operator_override` | `blocked`), `profiles`. Modellauswahl: `GET /api/admin/providers/models` (OpenRouter-Katalog oder Ollama `/api/tags`, soft-fail).
**Additiv 2026-09-08 (lokales Detect, Timeouts):** CPU-Ollama braucht für denselben Detect-Prompt deutlich länger als OpenRouter. Remote-Detect bleibt 90s / 1024 Tokens. Lokales Detect wartet 300s und begrenzt auf 256 Tokens. Live-TLS endet auf dem Synology-Reverse-Proxy (`192.168.2.63`, DSM-Default 60s). Dort Proxy-Lese- und -Sende-Timeout 600s setzen, sonst sieht der Browser HTTP 504, während das Backend weiterläuft. Ein Dialogzug kann danach lokal einen Halte-Impuls gespeichert haben. Die Compose-Frontend-Nginx auf Port 3096 hat bereits 600s.
Status in `GET /api/admin/providers`: `remote_plaintext_allowed`, `remote_plaintext_reason` (`non_production` | `operator_override` | `blocked`), `profiles`, `detect_operating_mode` (`semantic` | `learning`). Modellauswahl: `GET /api/admin/providers/models` (OpenRouter-Katalog oder Ollama `/api/tags`, soft-fail).
## 9.7 Übergang: Detect-Lernmodus (2026-09-08)
Additiv. Fachliches Home: `../functional/guardrails.md` §22.4. Ersetzt weder semantische Detection noch die bestätigte Registry.
Default bleibt `semantic`: Detect läuft wie bisher, Generate folgt ohne Pause. `learning` ist ein Admin-Schalter (`PUT /api/admin/providers/detect-mode`), kein Gateway-Bypass und keine Wortlisten-UI.
Im Lernmodus untersucht Detect weiterhin den vollen gerenderten Generate-Egress. Bevor Generate startet, werden `request_local`-Spans im aktuellen Nutzersatz zur Bestätigung angeboten (Dialog und Journal-Gespräch, nicht Journal-Generate). „Identität“ bestätigt die lokale Registry und zählt einen Identitätssinn. „Nicht schützenswert“ maskiert diese Nennung nicht und zählt den anderen Sinn. Detect-Ausgabe allein speichert weiterhin keine aktive Identität.
Es gibt keine separat zu pflegende Doppeldeutigkeitsliste. Mehrdeutigkeit entsteht, wenn dieselbe Schreibweise beide Sinne hat. Nur dann darf ein **lokales** Detect-Modell eine Mini-Passage (Ausschnitt um die Nennung) entscheiden. Fehlt ein lokales Modell oder ist die Antwort unbrauchbar, bleibt das Popup. OpenRouter erhält diese Passage nicht.
Nach der Bestätigung läuft Generate mit den geprüften Mappings (`precomputed_learning_review`), ohne zweiten Detect-Pass. Compact-Diagnose enthält weiterhin keine Klartextlabels. Tests: `backend/tests/test_detect_learning.py`.
## 10. Offene Fragen
@ -259,6 +273,8 @@ Status in `GET /api/admin/providers`: `remote_plaintext_allowed`, `remote_plaint
**Additiv 2026-08-28:** Wie sollen neue, vom Detect verfehlte und lokal nicht bestätigte Identitäten vor dem Egress behandelt werden, ohne span-genaue Homonyme oder Allgemeinbegriffe dauerhaft zu maskieren? Optionen ohne Vorentscheidung: fail-closed vor Generate, Review der Detect-Vorschläge vor Egress, oder ein zweites lokales Verfahren. Keine Heuristik auf Verdacht.
**Additiv 2026-09-08:** Der Detect-Lernmodus ist eine Übergangslösung für die Review-Option, nicht der Endzustand. Semantische Detection bleibt Pflicht. Unbekannte Namen sind nicht deshalb harmlos, weil sie nicht auf einer Liste stehen.
## 11. Querverweise
- Fachlich: `../functional/guardrails.md`

View File

@ -85,6 +85,8 @@ merge --no-ff develop in main → push main → deploy-prod.yml
**Additiv 2026-09-07 (Qualitätssystem):** Backend-Runner ist pytest (`backend/pytest.ini`, `backend/tests/conftest.py`). Gitea startet die Suite erst nach erfolgreichem Dev-Deploy (`workflow_run` auf `Deploy Development`), nicht parallel zum Image-Build. Destruktive Tests bleiben auf `kansho_test`. Zusätzlich ein nicht-schreibender Smoke gegen die laufende Dev-API (`kansho_dev`, nur `/api/health`). Öffentliche URLs nach Host-Nginx/TLS: `https://dev.kansho.jinkendo.de` und `https://kansho.jinkendo.de`. LAN-Ports 3096/8096 und 3006/8005 bleiben die Compose-Publish-Ziele.
**Additiv 2026-09-08 (TLS-Hop):** DNS für `*.kansho.jinkendo.de` zeigt auf das Synology-NAS `192.168.2.63`. Der Pi hat kein Host-HTTPS auf 443. DSM Reverse Proxy: Proxy-Lese-/Sende-Timeout 600s; Default 60s bricht lokale Detect-Läufe mit HTTP 504 ab. Ziel bleibt der Pi (`3096`/`8096` Dev, `3006`/`8005` Prod). Repo-Dateien `nginx/kansho.conf` und `nginx/kansho-dev.conf` beschreiben denselben Timeout, falls TLS später auf dem Pi endet.
Kanshō-Repo liegt auf Gitea (`Lars/Kansho`). HTTPS-Push ist eingerichtet. Der Pi-Runner (`ubuntu-latest`) ist derselbe wie bei Mitai/Shinkan/Kairo.
## 5. Was Deploy nicht übernimmt

View File

@ -687,3 +687,11 @@ pre.code {
.runlog-list li.bad { border-color: #e3c0c0; }
.runlog-trace { margin-top: 0.8rem; }
.runlog-trace .trace-panel { border-top: 1px dashed var(--line); }
.runlog-modal fieldset.row-actions {
border: 0;
margin: 0.4rem 0 0;
padding: 0;
display: flex;
flex-wrap: wrap;
gap: 0.8rem;
}

View File

@ -0,0 +1,53 @@
export default function MaskReviewPanel({ review, busy, onSubmit }) {
if (!review?.candidates?.length) return null
return (
<div className="runlog-overlay" role="dialog" aria-labelledby="mask-review-title">
<div className="runlog-modal">
<div className="runlog-head">
<div>
<h2 id="mask-review-title">Maskierung prüfen</h2>
<p className="runlog-status">
Lernmodus: nur diese Nennung. Bestätigen legt die Bezeichnung lokal ab.
Nicht schützenswert verhindert die Maskierung. Beides bei demselben Wort markiert es als mehrdeutig.
</p>
</div>
</div>
<form
className="stack"
onSubmit={(event) => {
event.preventDefault()
const data = new FormData(event.currentTarget)
const decisions = review.candidates.map((item) => ({
id: item.id,
decision: data.get(`decision-${item.id}`) || 'identity'
}))
onSubmit(decisions)
}}
>
<ul className="stack">
{review.candidates.map((item) => (
<li key={item.id} className="row-item">
<div>
<strong>{item.text}</strong>
<span className="muted"> · {item.entity_type}{item.ambiguous ? ' · mehrdeutig' : ''}</span>
{item.excerpt && <p className="muted">{item.excerpt}</p>}
</div>
<fieldset className="row-actions">
<label className="check">
<input type="radio" name={`decision-${item.id}`} value="identity" defaultChecked />
Identität
</label>
<label className="check">
<input type="radio" name={`decision-${item.id}`} value="not_identity" />
Nicht schützenswert
</label>
</fieldset>
</li>
))}
</ul>
<button type="submit" disabled={busy}>{busy ? 'Übernimmt …' : 'Übernehmen und weiter'}</button>
</form>
</div>
</div>
)
}

View File

@ -104,6 +104,7 @@ export default function AdminIdentitiesPage() {
<p className="muted">
Nur lokale bestätigte Registry. Detect-Treffer werden nicht automatisch aktiv.
Unbestätigte Vorschläge gelten nicht als bekannte Identität. Kein externer Egress.
Mehrdeutigkeit entsteht aus Dialog-Bestätigungen, nicht aus einer Wortlisten-Seite.
</p>
{error && <p className="error">{error}</p>}
{notice && <p>{notice}</p>}
@ -120,9 +121,16 @@ export default function AdminIdentitiesPage() {
</tr>
</thead>
<tbody>
{(data?.registry || []).map((item) => (
{(data?.registry || []).map((item) => {
const sense = (data?.senses || []).find(
(entry) => (entry.normalized_label || '').toLowerCase() === (item.canonical_label || '').toLowerCase()
)
return (
<tr key={item.id}>
<td>{item.canonical_label}</td>
<td>
{item.canonical_label}
{sense?.ambiguous ? <span className="muted"> · mehrdeutig</span> : null}
</td>
<td>
<select
value={item.entity_type}
@ -150,7 +158,8 @@ export default function AdminIdentitiesPage() {
<button type="button" className="ghost" onClick={() => remove(item.id)}>Entfernen</button>
</td>
</tr>
))}
)
})}
</tbody>
</table>
</div>

View File

@ -152,6 +152,27 @@ export default function AdminProvidersPage() {
}
}
const saveDetectMode = async (mode) => {
setError('')
setNotice('')
setSaving('detect-mode')
try {
const payload = await api('/api/admin/providers/detect-mode', {
token: session.token,
method: 'PUT',
body: { mode }
})
applyPayload(payload)
setNotice(mode === 'learning'
? 'Lernmodus: Detect pausiert vor Generate, bis Maskierungen bestätigt sind.'
: 'Detect-Modus: semantisch, ohne Bestätigungs-Popup.')
} catch (err) {
setError(err.message)
} finally {
setSaving('')
}
}
const saveRole = async (role) => {
const form = forms[role]
if (!form) return
@ -255,6 +276,25 @@ export default function AdminProvidersPage() {
<div key={role.role} className="stack">
<h2>{role.title || role.role}</h2>
{role.task && <p className="muted">{role.task}</p>}
{role.role === 'detect' && (
<label>
Detect-Modus
<select
value={data.detect_operating_mode || 'semantic'}
onChange={(e) => saveDetectMode(e.target.value)}
disabled={saving === 'detect-mode'}
>
<option value="semantic">Semantisch (Default)</option>
<option value="learning">Lernmodus (Übergang)</option>
</select>
</label>
)}
{role.role === 'detect' && data.detect_operating_mode === 'learning' && (
<p className="muted">
Keine separate Wortliste. Bestätigungen im Dialog legen lokale Sinne an.
Mehrdeutige Nennungen gehen nur als Mini-Passage an ein lokales Detect-Modell.
</p>
)}
<dl className="meta">
<dt>Status</dt>
<dd>{statusLabel(view)}</dd>

View File

@ -1,6 +1,7 @@
import { useEffect, useState } from 'react'
import { Link } from 'react-router-dom'
import { api, apiDownload } from '../api.js'
import MaskReviewPanel from '../components/MaskReviewPanel.jsx'
import { useAuth } from '../context/AuthContext.jsx'
function convLabel(item, index) {
@ -18,6 +19,7 @@ export default function DialoguePage() {
const [busy, setBusy] = useState(false)
const [egress, setEgress] = useState(null)
const [debugOn, setDebugOn] = useState(false)
const [maskReview, setMaskReview] = useState(null)
const loadList = () =>
api('/api/dialogue/conversations', { token: session.token })
@ -33,6 +35,7 @@ export default function DialoguePage() {
const data = await api(`/api/dialogue/conversations/${id}`, { token: session.token })
setActiveId(id)
setMessages(data.messages || [])
setMaskReview(data.pending_mask_review || null)
} catch (err) {
setError(err.message)
}
@ -78,6 +81,7 @@ export default function DialoguePage() {
else {
setActiveId(null)
setMessages([])
setMaskReview(null)
}
} catch (err) {
setError(err.message)
@ -109,7 +113,12 @@ export default function DialoguePage() {
})
setDraft('')
setMessages(result.messages || [])
if (result.pending_mask_review) {
setMaskReview(result.pending_mask_review)
} else {
setMaskReview(null)
await loadList()
}
} catch (err) {
setError(err.message)
if (activeId) await openConversation(activeId)
@ -118,6 +127,26 @@ export default function DialoguePage() {
}
}
const submitMaskReview = async (decisions) => {
if (!activeId || !maskReview?.id) return
setError('')
setBusy(true)
try {
const result = await api(`/api/dialogue/conversations/${activeId}/turn/review`, {
token: session.token,
method: 'POST',
body: { review_id: maskReview.id, decisions }
})
setMaskReview(null)
setMessages(result.messages || [])
await loadList()
} catch (err) {
setError(err.message)
} finally {
setBusy(false)
}
}
return (
<section className="card dialogue-page">
<header className="dialogue-head">
@ -174,11 +203,12 @@ export default function DialoguePage() {
{activeId && (
<form className="dialogue-composer" onSubmit={send}>
<textarea rows={3} value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Erzählen …" />
<button type="submit" disabled={busy}>{busy ? 'Antwortet …' : 'Senden'}</button>
<button type="submit" disabled={busy || Boolean(maskReview)}>{busy ? 'Antwortet …' : 'Senden'}</button>
</form>
)}
</div>
</div>
<MaskReviewPanel review={maskReview} busy={busy} onSubmit={submitMaskReview} />
</section>
)
}

View File

@ -2,6 +2,7 @@ import { useEffect, useState } from 'react'
import { Link, useNavigate, useParams } from 'react-router-dom'
import { api, apiDownload } from '../api.js'
import DayScratch from '../components/DayScratch.jsx'
import MaskReviewPanel from '../components/MaskReviewPanel.jsx'
import RunLogPopup from '../components/RunLogPopup.jsx'
import { useAuth } from '../context/AuthContext.jsx'
import { entryTitle } from '../journal/document.js'
@ -58,6 +59,7 @@ export default function JournalDayPage() {
const [logStatus, setLogStatus] = useState('ok')
const [runLog, setRunLog] = useState([])
const [debugOn, setDebugOn] = useState(false)
const [maskReview, setMaskReview] = useState(null)
const loadDay = async (preferId) => {
const data = await api(`/api/journal/days/${dayId}`, { token: session.token })
@ -68,9 +70,11 @@ export default function JournalDayPage() {
const conv = await api(`/api/journal/conversations/${nextId}`, { token: session.token })
setActiveId(nextId)
setMessages(conv.messages || [])
setMaskReview(conv.pending_mask_review || null)
} else {
setActiveId(null)
setMessages([])
setMaskReview(null)
}
return data
}
@ -148,7 +152,12 @@ export default function JournalDayPage() {
})
setDraft('')
setMessages(result.messages || [])
if (result.pending_mask_review) {
setMaskReview(result.pending_mask_review)
} else {
setMaskReview(null)
await loadDay(activeId)
}
} catch (err) {
setError(err.message)
await loadDay(activeId)
@ -157,6 +166,26 @@ export default function JournalDayPage() {
}
}
const submitMaskReview = async (decisions) => {
if (!activeId || !maskReview?.id) return
setError('')
setBusy(true)
try {
const result = await api(`/api/journal/conversations/${activeId}/turn/review`, {
token: session.token,
method: 'POST',
body: { review_id: maskReview.id, decisions }
})
setMaskReview(null)
setMessages(result.messages || [])
await loadDay(activeId)
} catch (err) {
setError(err.message)
} finally {
setBusy(false)
}
}
const generate = async (conversationIds) => {
setError('')
setBusy(true)
@ -391,7 +420,7 @@ export default function JournalDayPage() {
onChange={(e) => setDraft(e.target.value)}
placeholder="Erzählen …"
/>
<button type="submit" disabled={busy}>{busy ? 'Antwortet …' : 'Senden'}</button>
<button type="submit" disabled={busy || Boolean(maskReview)}>{busy ? 'Antwortet …' : 'Senden'}</button>
</form>
)}
</div>
@ -403,6 +432,7 @@ export default function JournalDayPage() {
onError={setError}
/>
</div>
<MaskReviewPanel review={maskReview} busy={busy} onSubmit={submitMaskReview} />
<RunLogPopup
open={logOpen}
status={logStatus}

View File

@ -1,4 +1,6 @@
# Kanshō development vhost. Install as /etc/nginx/sites-available/kansho-dev.
# Live TLS terminates on the Synology Reverse Proxy at 192.168.2.63, not on the Pi.
# Set DSM proxy read/send timeout to 600s (default 60s returns HTTP 504).
server {
listen 80;
@ -46,5 +48,6 @@ server {
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_cache_bypass $http_upgrade;
proxy_read_timeout 600s;
}
}

View File

@ -1,6 +1,8 @@
# Kanshō Host nginx (outside Compose)
# Install as /etc/nginx/sites-available/kansho and symlink into sites-enabled.
# TLS via nginx/certbot-setup.sh. Reverse-proxy targets are the Pi publish ports.
# Live TLS currently terminates on the Synology Reverse Proxy at 192.168.2.63;
# set DSM proxy read/send timeout to 600s (default 60s returns HTTP 504).
server {
listen 80;
@ -50,5 +52,6 @@ server {
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_cache_bypass $http_upgrade;
proxy_read_timeout 600s;
}
}