Add transitional learning detect with in-dialog mask review.
Deploy Development / deploy (push) Successful in 57s
Test Suite / pytest-backend (push) Failing after 2m57s
Test Suite / smoke-dev (push) Successful in 0s
Test Suite / frontend-build (push) Successful in 16s

Senses grow from confirmations instead of a word list, so a later local pipeline can decide homonyms on a short passage. Default stays semantic. Local detect waits longer, and the reverse-proxy timeout is documented so Dev does not 504 first.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
LarsandCursor committed 2026-09-09 08:34:43 +02:00
1 parent 59921fc5b5
commit fa1c11c33e
29 files changed
+1135 -45

No files matched your search

+410
View File
@@ -0,0 +1,410 @@
"""Transitional learning detect: senses grow from dialogue confirmations.
Not a global word list. Detect still does not auto-activate identities.
Ambiguous spellings get a local passage call when a local detect provider exists.
"""
from __future__ import annotations
import json
import re
import uuid
from typing import Any
from db import get_db, row_to_dict
from identity_store import ENTITY_TYPES, confirm_identity, is_maskable_label, normalize_label
MODE_SEMANTIC = "semantic"
MODE_LEARNING = "learning"
MODES = (MODE_SEMANTIC, MODE_LEARNING)
SETTING_KEY = "detect_operating_mode"
_LETTER = r"A-Za-zÄÖÜäöüß"
WINDOW = 80
_DECISION_IDENTITY = "identity"
_DECISION_NOT = "not_identity"
def get_detect_operating_mode() -> str:
with get_db() as conn:
row = row_to_dict(
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
)
value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower()
return value if value in MODES else MODE_SEMANTIC
def set_detect_operating_mode(mode: str) -> str:
chosen = (mode or "").strip().lower()
if chosen not in MODES:
raise ValueError("invalid_detect_operating_mode")
with get_db() as conn:
conn.execute(
"""
INSERT INTO app_settings (key, value, updated)
VALUES (?, ?, datetime('now'))
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
""",
(SETTING_KEY, chosen),
)
return chosen
def get_sense(profile_id: str, label: str) -> dict:
key = normalize_label(label) or (label or "").strip()
if not key:
return {
"normalized_label": "",
"identity_hits": 0,
"non_identity_hits": 0,
"ambiguous": False,
}
with get_db() as conn:
row = row_to_dict(
conn.execute(
"""
SELECT normalized_label, identity_hits, non_identity_hits
FROM label_senses
WHERE profile_id = ? AND lower(normalized_label) = lower(?)
""",
(profile_id, key),
).fetchone()
)
if not row:
return {
"normalized_label": key,
"identity_hits": 0,
"non_identity_hits": 0,
"ambiguous": False,
}
identity = int(row.get("identity_hits") or 0)
other = int(row.get("non_identity_hits") or 0)
return {
"normalized_label": row.get("normalized_label") or key,
"identity_hits": identity,
"non_identity_hits": other,
"ambiguous": identity > 0 and other > 0,
}
def record_sense(profile_id: str, label: str, *, identity: bool) -> dict:
key = normalize_label(label) or (label or "").strip()
if not key:
raise ValueError("empty_label")
with get_db() as conn:
existing = row_to_dict(
conn.execute(
"""
SELECT normalized_label, identity_hits, non_identity_hits FROM label_senses
WHERE profile_id = ? AND lower(normalized_label) = lower(?)
""",
(profile_id, key),
).fetchone()
)
if existing:
key = existing.get("normalized_label") or key
identity_hits = int((existing or {}).get("identity_hits") or 0)
non_identity_hits = int((existing or {}).get("non_identity_hits") or 0)
if identity:
identity_hits += 1
else:
non_identity_hits += 1
conn.execute(
"""
INSERT INTO label_senses (
profile_id, normalized_label, identity_hits, non_identity_hits, updated
)
VALUES (?, ?, ?, ?, datetime('now'))
ON CONFLICT(profile_id, normalized_label) DO UPDATE SET
identity_hits = excluded.identity_hits,
non_identity_hits = excluded.non_identity_hits,
updated = datetime('now')
""",
(profile_id, key, identity_hits, non_identity_hits),
)
return get_sense(profile_id, key)
def _in_user_text(label: str, user_body: str) -> bool:
if not label or not user_body:
return False
return bool(
re.search(
rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
user_body,
re.IGNORECASE,
)
)
def _excerpt(user_body: str, start: int | None, end: int | None, label: str) -> str:
text = user_body or ""
if start is None or end is None or start < 0 or end > len(text):
match = re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text, re.IGNORECASE)
if not match:
return text[:WINDOW]
start, end = match.start(), match.end()
left = max(0, int(start) - WINDOW)
right = min(len(text), int(end) + WINDOW)
return text[left:right]
def needs_review(profile_id: str, mapping: dict, user_body: str) -> bool:
label = mapping.get("local_label") or ""
if mapping.get("source") == "confirmed_registry":
sense = get_sense(profile_id, label)
return bool(sense["ambiguous"]) and _in_user_text(label, user_body)
if mapping.get("source") != "request_local":
return False
if not is_maskable_label(label) or not _in_user_text(label, user_body):
return False
sense = get_sense(profile_id, label)
if sense["identity_hits"] > 0 and not sense["ambiguous"]:
return False
return True
def build_candidates(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
seen: set[tuple[int | None, int | None, str]] = set()
items: list[dict] = []
for mapping in mappings:
if not needs_review(profile_id, mapping, user_body):
continue
label = mapping.get("local_label") or ""
start = mapping.get("start")
end = mapping.get("end")
key = (start, end, label.casefold())
if key in seen:
continue
seen.add(key)
items.append(
{
"id": str(uuid.uuid4()),
"text": label,
"entity_type": mapping.get("entity_type") or "PERSON",
"start": start,
"end": end,
"excerpt": _excerpt(user_body, start, end, label),
"ambiguous": get_sense(profile_id, label)["ambiguous"],
}
)
return items
def suppress_known_non_identity(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
kept = []
for mapping in mappings:
label = mapping.get("local_label") or ""
sense = get_sense(profile_id, label)
if (
mapping.get("source") == "request_local"
and sense["non_identity_hits"] > 0
and not sense["identity_hits"]
and _in_user_text(label, user_body)
):
continue
kept.append(mapping)
return kept
def save_pending(
profile_id: str,
conversation_id: str,
user_message_id: str,
payload: dict,
) -> str:
review_id = str(uuid.uuid4())
with get_db() as conn:
conn.execute(
"DELETE FROM pending_mask_reviews WHERE profile_id = ? AND conversation_id = ?",
(profile_id, conversation_id),
)
conn.execute(
"""
INSERT INTO pending_mask_reviews (
id, profile_id, conversation_id, user_message_id, payload_json, created
)
VALUES (?, ?, ?, ?, ?, datetime('now'))
""",
(review_id, profile_id, conversation_id, user_message_id, json.dumps(payload, ensure_ascii=False)),
)
return review_id
def load_pending(profile_id: str, review_id: str) -> dict | None:
with get_db() as conn:
row = row_to_dict(
conn.execute(
"""
SELECT * FROM pending_mask_reviews
WHERE id = ? AND profile_id = ?
""",
(review_id, profile_id),
).fetchone()
)
if not row:
return None
payload = json.loads(row.get("payload_json") or "{}")
payload["id"] = row["id"]
payload["conversation_id"] = row["conversation_id"]
payload["user_message_id"] = row["user_message_id"]
return payload
def drop_pending(profile_id: str, review_id: str) -> None:
with get_db() as conn:
conn.execute(
"DELETE FROM pending_mask_reviews WHERE id = ? AND profile_id = ?",
(review_id, profile_id),
)
def apply_review_decisions(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
by_id = {item["id"]: item for item in pending.get("candidates") or []}
drop_keys: set[tuple] = set()
for raw in decisions:
candidate = by_id.get(raw.get("id") or "")
if not candidate:
continue
decision = (raw.get("decision") or "").strip()
label = candidate.get("text") or ""
kind = candidate.get("entity_type") if candidate.get("entity_type") in ENTITY_TYPES else "PERSON"
if decision == _DECISION_IDENTITY:
record_sense(profile_id, label, identity=True)
confirm_identity(profile_id, label, entity_type=kind)
elif decision == _DECISION_NOT:
record_sense(profile_id, label, identity=False)
drop_keys.add((candidate.get("start"), candidate.get("end"), label.casefold()))
mappings = []
for mapping in pending.get("mappings") or []:
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
if key in drop_keys:
continue
mappings.append(mapping)
return suppress_known_non_identity(profile_id, mappings, pending.get("user_body") or "")
def confirm_known_identity_spans(profile_id: str, mappings: list[dict], user_body: str) -> None:
for mapping in mappings:
label = mapping.get("local_label") or ""
sense = get_sense(profile_id, label)
if not (
sense["identity_hits"] > 0
and not sense["ambiguous"]
and _in_user_text(label, user_body)
):
continue
kind = mapping.get("entity_type") if mapping.get("entity_type") in ENTITY_TYPES else "PERSON"
try:
confirm_identity(profile_id, label, entity_type=kind)
except ValueError:
continue
def list_senses(profile_id: str) -> list[dict]:
with get_db() as conn:
rows = conn.execute(
"""
SELECT normalized_label, identity_hits, non_identity_hits
FROM label_senses WHERE profile_id = ?
ORDER BY lower(normalized_label)
""",
(profile_id,),
).fetchall()
items = []
for raw in rows:
row = row_to_dict(raw) or {}
identity = int(row.get("identity_hits") or 0)
other = int(row.get("non_identity_hits") or 0)
items.append(
{
"normalized_label": row.get("normalized_label") or "",
"identity_hits": identity,
"non_identity_hits": other,
"ambiguous": identity > 0 and other > 0,
}
)
return items
def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | None:
with get_db() as conn:
row = row_to_dict(
conn.execute(
"""
SELECT id, payload_json FROM pending_mask_reviews
WHERE profile_id = ? AND conversation_id = ?
""",
(profile_id, conversation_id),
).fetchone()
)
if not row:
return None
payload = json.loads(row.get("payload_json") or "{}")
return {
"id": row["id"],
"candidates": payload.get("candidates") or [],
}
def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: list[dict]) -> tuple[list[dict], list[dict]]:
"""Local passage LLM for already-ambiguous spellings. Unresolved stay for the popup."""
remaining = []
drop_keys: set[tuple] = set()
for item in candidates:
if not item.get("ambiguous"):
remaining.append(item)
continue
label = item.get("text") or ""
decision = try_local_passage_decision(item.get("excerpt") or "", label)
if decision == _DECISION_NOT:
record_sense(profile_id, label, identity=False)
drop_keys.add((item.get("start"), item.get("end"), label.casefold()))
elif decision == _DECISION_IDENTITY:
record_sense(profile_id, label, identity=True)
else:
remaining.append(item)
if not drop_keys:
return remaining, mappings
kept = []
for mapping in mappings:
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
if key in drop_keys:
continue
kept.append(mapping)
return remaining, kept
def try_local_passage_decision(excerpt: str, label: str) -> str | None:
"""Return identity, not_identity, or None if no local model or unusable answer."""
from providers import ProviderError, complete_chat, detect_provider
config = detect_provider()
if not config or not config.local or config.mode != "http":
return None
prompt = (
"Entscheide nur für die markierte Nennung in diesem kurzen Ausschnitt. "
"Ist sie eine schützenswerte Identität (Person, Ort, Organisation, privates Projekt) "
f"oder eine Sache/Allgemeinbedeutung? Wort: {label}\n"
f"Ausschnitt: {excerpt}\n"
'Antworte nur mit JSON {"decision":"identity"} oder {"decision":"not_identity"}.'
)
try:
result = complete_chat(
config,
[{"role": "user", "content": prompt}],
timeout=30.0,
max_tokens=32,
)
except ProviderError:
return None
match = re.search(r"\{.*\}", result.content or "", re.DOTALL)
if not match:
return None
try:
data = json.loads(match.group(0))
except json.JSONDecodeError:
return None
value = (data.get("decision") or "").strip()
if value in {_DECISION_IDENTITY, _DECISION_NOT}:
return value
return None
+130 -23
View File
@@ -7,9 +7,21 @@ import re
from context_builder import assemble_text, build_internal_context, is_closing_turn
from conversation_signals import infer_signals
from debug_store import persist_engine_error, persist_step
from detect_learning import (
MODE_LEARNING,
apply_review_decisions,
auto_resolve_ambiguous,
build_candidates,
confirm_known_identity_spans,
drop_pending,
get_detect_operating_mode,
load_pending,
save_pending,
suppress_known_non_identity,
)
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
from engine import EngineError, execute_prompt, load_active_prompt
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError, detect_personal_egress
from writing_profile_store import remember_dialogue_style
from profile_review import consider_dialogue
@@ -302,23 +314,21 @@ def visible_for_role(payload: dict, role: str | None) -> dict:
return cleaned
def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str | None = None) -> dict:
conversation = get_conversation(profile_id, conversation_id)
user = append_message(profile_id, conversation_id, body, role="user", message_id=message_id)
context = build_internal_context(
profile_id,
conversation_id=conversation_id,
space_id=conversation.get("space_id"),
journal_day_id=conversation.get("journal_day_id"),
purpose="dialogue_turn",
)
assembled = assemble_text(context)
prompt = load_active_prompt("mvp.dialogue_turn")
def _finish_turn(
profile_id: str,
conversation_id: str,
user: dict,
assembled: dict,
prompt: dict,
*,
precomputed_mappings: list[dict] | None = None,
) -> dict:
calls = 0
result = None
impulse = ""
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
call_traces: list[dict] = []
working = dict(assembled)
try:
while calls < 2:
result = execute_prompt(
@@ -326,7 +336,8 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
profile_id,
purpose="dialogue_turn",
data_class="B",
context=assembled,
context=working,
precomputed_mappings=precomputed_mappings if calls == 0 else None,
)
calls += 1
if result.get("trace"):
@@ -335,16 +346,16 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
if not content:
raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.")
impulse, decision = parse_turn_payload(content)
if not needs_repair(impulse, assembled):
if not needs_repair(impulse, working):
break
decision = {**decision, "guard": "impulse_rejected"}
assembled = dict(assembled)
assembled["dialogue_context"] = (
(assembled.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, assembled)
working = dict(working)
working["dialogue_context"] = (
(working.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, working)
)
if needs_repair(impulse, assembled):
if needs_repair(impulse, working):
parsed_ok = bool(decision.get("parsed"))
impulse = local_hold(last_user_text(assembled))
impulse = local_hold(last_user_text(working))
decision = {
"operation": "fortfuehren",
"label": OPERATIONS["fortfuehren"],
@@ -369,7 +380,7 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
},
)
raise
impulse = local_hold(last_user_text(assembled))
impulse = local_hold(last_user_text(working))
decision = {
"operation": "fortfuehren",
"label": OPERATIONS["fortfuehren"],
@@ -388,7 +399,7 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
infer_signals(user_bodies, decision.get("operation")),
)
remember_dialogue_style(profile_id)
consider_dialogue(profile_id, body)
consider_dialogue(profile_id, user.get("body") or "")
trace = result.get("trace") if result else None
persist_step(
profile_id,
@@ -416,3 +427,99 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
"decision": decision,
"trace": trace,
}
def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled: dict, prompt: dict) -> dict | None:
preview = preview_prompt(prompt, assembled)
rendered = preview.get("rendered") or ""
try:
outcome = detect_personal_egress(profile_id, rendered)
except DetectError:
return None
user_body = user.get("body") or ""
mappings = list(outcome.mappings or [])
candidates = build_candidates(profile_id, mappings, user_body)
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
if not candidates:
confirm_known_identity_spans(profile_id, mappings, user_body)
mappings = suppress_known_non_identity(profile_id, mappings, user_body)
return _finish_turn(
profile_id,
conversation_id,
user,
assembled,
prompt,
precomputed_mappings=mappings,
)
review_id = save_pending(
profile_id,
conversation_id,
user.get("id") or "",
{
"mappings": mappings,
"candidates": candidates,
"user_body": user_body,
},
)
return {
"conversation": get_conversation(profile_id, conversation_id),
"user": user,
"assistant": None,
"calls": 0,
"messages": list_messages(profile_id, conversation_id),
"pending_mask_review": {
"id": review_id,
"candidates": candidates,
},
}
def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str | None = None) -> dict:
conversation = get_conversation(profile_id, conversation_id)
user = append_message(profile_id, conversation_id, body, role="user", message_id=message_id)
context = build_internal_context(
profile_id,
conversation_id=conversation_id,
space_id=conversation.get("space_id"),
journal_day_id=conversation.get("journal_day_id"),
purpose="dialogue_turn",
)
assembled = assemble_text(context)
prompt = load_active_prompt("mvp.dialogue_turn")
if get_detect_operating_mode() == MODE_LEARNING:
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
if paused is not None:
return paused
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
def continue_turn(profile_id: str, conversation_id: str, review_id: str, decisions: list[dict]) -> dict:
pending = load_pending(profile_id, review_id)
if not pending or pending.get("conversation_id") != conversation_id:
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
mappings = apply_review_decisions(profile_id, pending, decisions)
drop_pending(profile_id, review_id)
conversation = get_conversation(profile_id, conversation_id)
context = build_internal_context(
profile_id,
conversation_id=conversation_id,
space_id=conversation.get("space_id"),
journal_day_id=conversation.get("journal_day_id"),
purpose="dialogue_turn",
)
assembled = assemble_text(context)
prompt = load_active_prompt("mvp.dialogue_turn")
user = {"id": pending.get("user_message_id"), "body": pending.get("user_body") or ""}
messages = list_messages(profile_id, conversation_id)
for item in messages:
if item.get("id") == pending.get("user_message_id"):
user = item
break
return _finish_turn(
profile_id,
conversation_id,
user,
assembled,
prompt,
precomputed_mappings=mappings,
)
+2
View File
@@ -75,6 +75,7 @@ def execute_prompt(
disable_context_compression: bool = False,
budget=None,
diagnostics: dict[str, Any] | None = None,
precomputed_mappings: list[dict] | None = None,
) -> dict:
preview = preview_prompt(prompt, context)
feature_id = prompt.get("required_feature") or "ai_calls"
@@ -107,6 +108,7 @@ def execute_prompt(
"disable_context_compression": disable_context_compression,
"budget": budget,
"diagnostics": diagnostics or {},
"precomputed_mappings": precomputed_mappings,
},
)
)
+12 -2
View File
@@ -31,7 +31,9 @@ from providers import ChatResult, ProviderError, complete_chat, detect_provider
DETECT_CHUNK_CHARS = 4000
DETECT_CHUNK_OVERLAP = 250
DETECT_TIMEOUT = 90.0
DETECT_TIMEOUT_LOCAL = 300.0
DETECT_MAX_TOKENS = 1024
DETECT_MAX_TOKENS_LOCAL = 256
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
@@ -573,6 +575,13 @@ def _add_usage(stats: DetectionStats, usage: dict | None) -> None:
pass
def detect_chat_limits(config) -> tuple[float, int]:
"""Remote detect stays at 90s/1024. Local CPU hosts need a longer wait and a shorter cap."""
if getattr(config, "local", False):
return DETECT_TIMEOUT_LOCAL, DETECT_MAX_TOKENS_LOCAL
return DETECT_TIMEOUT, DETECT_MAX_TOKENS
def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResult:
prompt = resolve_template(
_detect_prompt()["template"],
@@ -580,11 +589,12 @@ def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResul
)
if schema_retry:
prompt = f"{prompt.rstrip()}\n\n{SCHEMA_RETRY_HINT}"
timeout, max_tokens = detect_chat_limits(config)
return complete_chat(
config,
[{"role": "user", "content": prompt}],
timeout=DETECT_TIMEOUT,
max_tokens=DETECT_MAX_TOKENS,
timeout=timeout,
max_tokens=max_tokens,
disable_context_compression=True,
)
@@ -0,0 +1,23 @@
-- Transitional learning detect: user-confirmed label senses, pending mask review.
-- Detect output still does not auto-activate identities.
CREATE TABLE IF NOT EXISTS label_senses (
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
normalized_label TEXT NOT NULL,
identity_hits INTEGER NOT NULL DEFAULT 0,
non_identity_hits INTEGER NOT NULL DEFAULT 0,
updated TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text,
PRIMARY KEY (profile_id, normalized_label)
);
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
id TEXT PRIMARY KEY,
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
user_message_id TEXT NOT NULL,
payload_json TEXT NOT NULL DEFAULT '{}',
created TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text
);
CREATE INDEX IF NOT EXISTS idx_pending_mask_reviews_conv
ON pending_mask_reviews (profile_id, conversation_id);
+25 -6
View File
@@ -1055,13 +1055,32 @@ def complete(request: GatewayRequest) -> GatewayResult:
detect_note = None
detect_stats: dict[str, Any] = {}
local_identities: list[dict[str, Any]] = []
precomputed = request.payload.get("precomputed_mappings") if request.payload else None
try:
outcome = detect_personal_egress(request.profile_id, rendered)
mappings = outcome.mappings
detect_stats = outcome.stats.public()
detect_name = outcome.stats.detect_provider
detect_note = outcome.stats.detect_note
local_identities = outcome.local_identities
if precomputed is not None:
mappings = list(precomputed)
detect_stats = {
"detect_note": "precomputed_learning_review",
"full_detection_coverage": True,
"semantic_identity_guaranteed": False,
}
local_identities = [
{
"local_label": item.get("local_label"),
"token": item.get("token"),
"entity_type": item.get("entity_type"),
"source": item.get("source"),
}
for item in mappings
]
detect_name = "learning_review"
else:
outcome = detect_personal_egress(request.profile_id, rendered)
mappings = outcome.mappings
detect_stats = outcome.stats.public()
detect_name = outcome.stats.detect_provider
detect_note = outcome.stats.detect_note
local_identities = outcome.local_identities
except DetectError as exc:
detect_stats = dict(exc.diagnostics or {})
_log_event(
+2
View File
@@ -8,6 +8,7 @@ from pathlib import Path
from urllib.parse import urlparse
from db import get_db
from detect_learning import get_detect_operating_mode
from env_loader import (
allows_remote_plaintext_detect,
remote_plaintext_detect_reason,
@@ -460,4 +461,5 @@ def public_status() -> dict:
"remote_plaintext_reason": remote_plaintext_detect_reason(),
"remote_plaintext_allowed": allows_remote_plaintext_detect(),
"ollama_url": OLLAMA_LAN_URL,
"detect_operating_mode": get_detect_operating_mode(),
}
+24
View File
@@ -20,6 +20,7 @@ from debug_store import (
settings_payload,
)
from dialogue_store import StoreError, get_conversation, inventory, list_conversations, list_derived_for_conversation, list_messages
from detect_learning import list_senses
from identity_store import (
confirm_identity,
confirm_review_proposal,
@@ -226,6 +227,28 @@ class ActivateProfile(BaseModel):
profile_id: str
class DetectModeWrite(BaseModel):
mode: str
@router.put("/providers/detect-mode")
def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require_admin_dep)):
from detect_learning import set_detect_operating_mode
from provider_settings import public_status as provider_public_status
try:
set_detect_operating_mode(body.mode)
except ValueError:
raise HTTPException(
status_code=400,
detail={
"code": "invalid_detect_operating_mode",
"message": "Detect-Modus muss semantic oder learning sein.",
},
)
return provider_public_status()
@router.get("/providers")
def admin_providers(session: dict = Depends(require_admin_dep)):
return public_status()
@@ -364,6 +387,7 @@ def admin_identities(session: dict = Depends(require_admin_dep)):
return {
"registry": list_registry(profile_id),
"proposals": list_review_proposals(profile_id, include_dismissed=True),
"senses": list_senses(profile_id),
"note": "Nur lokale bestätigte Registry und unbestätigte Vorschläge. Kein externer Egress.",
"backup": (
"Vor einer Bereinigung die lokale Datei backend/data/kansho.sqlite kopieren. "
+21 -1
View File
@@ -7,7 +7,8 @@ from auth import require_auth
from context_builder import build_internal_context
from continuity import checkpoint_usage_session, close_usage_session
from derived_kinds import catalog
from dialogue_turn import run_turn, visible_for_role
from detect_learning import pending_for_conversation
from dialogue_turn import continue_turn, run_turn, visible_for_role
from engine import EngineError
from privacy_gateway import public_error_detail
from dialogue_store import (
@@ -51,6 +52,11 @@ class MessageWrite(BaseModel):
id: str | None = None
class MaskReviewWrite(BaseModel):
review_id: str
decisions: list[dict] = Field(default_factory=list)
class ThreadWrite(BaseModel):
title: str = ""
status: str = "open"
@@ -140,6 +146,9 @@ def get_one_conversation(conversation_id: str, session: dict = Depends(require_a
conv = get_conversation(session["profile_id"], conversation_id)
conv["messages"] = list_messages(session["profile_id"], conversation_id)
conv["derived"] = list_derived_for_conversation(session["profile_id"], conversation_id)
pending = pending_for_conversation(session["profile_id"], conversation_id)
if pending:
conv["pending_mask_review"] = pending
return conv
except StoreError as exc:
_http(exc)
@@ -172,6 +181,17 @@ def conversation_turn(conversation_id: str, req: MessageWrite, session: dict = D
_http(exc)
@router.post("/conversations/{conversation_id}/turn/review")
def conversation_turn_review(conversation_id: str, req: MaskReviewWrite, session: dict = Depends(require_auth)):
try:
return visible_for_role(
continue_turn(session["profile_id"], conversation_id, req.review_id, req.decisions),
session.get("role"),
)
except (StoreError, EngineError) as exc:
_http(exc)
@router.get("/conversations/{conversation_id}/messages")
def get_messages(conversation_id: str, session: dict = Depends(require_auth)):
try:
+23 -2
View File
@@ -5,8 +5,9 @@ from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from auth import require_auth
from detect_learning import pending_for_conversation
from dialogue_store import StoreError, delete_conversation, get_conversation, list_messages
from dialogue_turn import run_turn, visible_for_role
from dialogue_turn import continue_turn, run_turn, visible_for_role
from privacy_gateway import GatewayRequest, inspect, public_error_detail
from engine import EngineError
from journal_generate import generate_draft
@@ -96,6 +97,11 @@ class TurnWrite(BaseModel):
id: str | None = None
class MaskReviewWrite(BaseModel):
review_id: str
decisions: list[dict] = Field(default_factory=list)
class GenerateWrite(BaseModel):
conversation_ids: list[str] | None = None
include_existing: bool = False
@@ -286,7 +292,11 @@ def remove_conversation(conversation_id: str, session: dict = Depends(require_au
def read_conversation(conversation_id: str, session: dict = Depends(require_auth)):
try:
conv = get_conversation(session["profile_id"], conversation_id)
return {"conversation": conv, "messages": list_messages(session["profile_id"], conversation_id)}
payload = {"conversation": conv, "messages": list_messages(session["profile_id"], conversation_id)}
pending = pending_for_conversation(session["profile_id"], conversation_id)
if pending:
payload["pending_mask_review"] = pending
return payload
except StoreError as exc:
_http(exc)
@@ -308,6 +318,17 @@ def conversation_turn(conversation_id: str, body: TurnWrite, session: dict = Dep
_http(exc)
@router.post("/conversations/{conversation_id}/turn/review")
def conversation_turn_review(conversation_id: str, body: MaskReviewWrite, session: dict = Depends(require_auth)):
try:
return visible_for_role(
continue_turn(session["profile_id"], conversation_id, body.review_id, body.decisions),
session.get("role"),
)
except (StoreError, EngineError) as exc:
_http(exc)
@router.get("/generation-settings")
def read_generation_settings(session: dict = Depends(require_auth)):
return settings_payload(session["profile_id"])
+21
View File
@@ -230,6 +230,27 @@ CREATE TABLE IF NOT EXISTS identity_review_proposals (
UNIQUE (profile_id, observed_label, entity_type)
);
CREATE TABLE IF NOT EXISTS label_senses (
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
normalized_label TEXT NOT NULL,
identity_hits INTEGER NOT NULL DEFAULT 0,
non_identity_hits INTEGER NOT NULL DEFAULT 0,
updated TEXT NOT NULL DEFAULT (datetime('now')),
PRIMARY KEY (profile_id, normalized_label)
);
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
id TEXT PRIMARY KEY,
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
user_message_id TEXT NOT NULL,
payload_json TEXT NOT NULL DEFAULT '{}',
created TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_pending_mask_reviews_conv
ON pending_mask_reviews (profile_id, conversation_id);
CREATE TABLE IF NOT EXISTS journal_days (
id TEXT PRIMARY KEY,
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
+2
View File
@@ -43,6 +43,8 @@ TABLES = [
"re_grounding_events",
"identity_mappings",
"identity_review_proposals",
"label_senses",
"pending_mask_reviews",
"journal_days",
"journal_drafts",
"journal_entries",
+184
View File
@@ -0,0 +1,184 @@
"""Transitional learning detect: senses from dialogue review, admin mode switch.
Fake detect/provider only. Run from backend/: python tests/test_detect_learning.py
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
os.environ["KANSHO_FAKE_DETECT"] = "1"
from fastapi.testclient import TestClient
from detect_learning import get_detect_operating_mode, get_sense, record_sense
from dialogue_turn import visible_for_role
from identity_store import list_confirmed_identities
from main import app
from privacy_gateway import reset_debug
def expect(ok: bool, message: str) -> None:
if not ok:
raise SystemExit(f"FAIL: {message}")
print(f"OK {message}")
def status_ok(response, label: str) -> None:
if response.status_code != 200:
detail = response.json() if response.headers.get("content-type", "").startswith("application/json") else {}
code = detail.get("detail", {}).get("code") if isinstance(detail, dict) else None
expect(False, f"{label} HTTP {response.status_code} {code or ''}".strip())
expect(True, label)
def header(token: str) -> dict:
return {"X-Auth-Token": token}
def open_conv(client: TestClient, headers: dict) -> str:
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"})
day = client.post(
f"/api/journal/spaces/{space.json()['id']}/days",
headers=headers,
json={"calendar_date": "2026-09-08"},
)
conv = client.post(
f"/api/journal/days/{day.json()['day']['id']}/conversations",
headers=headers,
json={"title": "Gespräch"},
)
return conv.json()["id"]
def main() -> None:
reset_debug()
with TestClient(app) as client:
setup = client.post(
"/api/auth/setup",
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
)
headers = header(setup.json()["token"])
profile_id = setup.json()["profile_id"]
providers = client.get("/api/admin/providers", headers=headers)
expect(providers.status_code == 200, "providers status")
expect(providers.json()["detect_operating_mode"] == "semantic", "default detect mode is semantic")
expect(get_detect_operating_mode() == "semantic", "store default is semantic")
bad = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "wordlist"})
expect(bad.status_code == 400, "invalid detect mode is rejected")
conv_semantic = open_conv(client, headers)
semantic_turn = client.post(
f"/api/journal/conversations/{conv_semantic}/turn",
headers=headers,
json={"body": "Anna kam vorbei."},
)
status_ok(semantic_turn, "semantic turn")
expect(not semantic_turn.json().get("pending_mask_review"), "semantic mode does not pause for review")
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "semantic turn gets an assistant")
switched = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning"})
status_ok(switched, "switch to learning")
expect(switched.json()["detect_operating_mode"] == "learning", "learning mode persisted")
conv = open_conv(client, headers)
paused = client.post(
f"/api/journal/conversations/{conv}/turn",
headers=headers,
json={"body": "Anna kam vorbei."},
)
status_ok(paused, "learning pause")
review = paused.json().get("pending_mask_review") or {}
expect(bool(review.get("id")), "learning mode returns a review id")
expect(paused.json().get("assistant") is None, "paused turn has no assistant")
candidates = review.get("candidates") or []
anna = next((item for item in candidates if item.get("text") == "Anna"), None)
expect(anna is not None, "Anna is a review candidate")
hidden = visible_for_role({"pending_mask_review": review, "trace": {"egress": "x"}}, "user")
expect("pending_mask_review" in hidden, "users still receive the mask review")
expect("trace" not in hidden, "users do not receive the detect trace")
reloaded = client.get(f"/api/journal/conversations/{conv}", headers=headers)
expect(reloaded.json().get("pending_mask_review", {}).get("id") == review["id"], "pending review survives reload")
missing = client.post(
f"/api/journal/conversations/{conv}/turn/review",
headers=headers,
json={"review_id": "missing", "decisions": []},
)
expect(missing.status_code == 404, "unknown review is gone")
identity = client.post(
f"/api/journal/conversations/{conv}/turn/review",
headers=headers,
json={
"review_id": review["id"],
"decisions": [{"id": anna["id"], "decision": "identity"}],
},
)
status_ok(identity, "identity review")
expect(not identity.json().get("pending_mask_review"), "review continue clears the pause")
expect(any(item.get("role") == "assistant" for item in identity.json().get("messages") or []), "continue produces an assistant")
confirmed = list_confirmed_identities(profile_id)
expect(any((item.get("canonical_label") or "").casefold() == "anna" for item in confirmed), "identity review confirms the registry")
sense = get_sense(profile_id, "Anna")
expect(sense["identity_hits"] >= 1 and not sense["ambiguous"], "Anna is identity-only")
second = client.post(
f"/api/journal/conversations/{conv}/turn",
headers=headers,
json={"body": "Anna hat später angerufen."},
)
status_ok(second, "second Anna turn")
expect(not second.json().get("pending_mask_review"), "identity-only spelling skips the popup")
food_conv = open_conv(client, headers)
food = client.post(
f"/api/journal/conversations/{food_conv}/turn",
headers=headers,
json={"body": "Hanna aß mit uns."},
)
status_ok(food, "hanna pause")
food_review = food.json().get("pending_mask_review") or {}
food_candidates = food_review.get("candidates") or []
hanna = next((item for item in food_candidates if item.get("text") == "Hanna"), None)
expect(hanna is not None, "Hanna is a review candidate")
declined = client.post(
f"/api/journal/conversations/{food_conv}/turn/review",
headers=headers,
json={
"review_id": food_review["id"],
"decisions": [{"id": hanna["id"], "decision": "not_identity"}],
},
)
status_ok(declined, "not_identity review")
expect(not any((item.get("canonical_label") or "").casefold() == "hanna" for item in list_confirmed_identities(profile_id)), "not_identity does not confirm Hanna")
expect(get_sense(profile_id, "Hanna")["non_identity_hits"] >= 1, "not_identity records a sense")
record_sense(profile_id, "Clarissa", identity=True)
record_sense(profile_id, "Clarissa", identity=False)
expect(get_sense(profile_id, "Clarissa")["ambiguous"], "both senses mark Clarissa as ambiguous")
amb_conv = open_conv(client, headers)
with patch("detect_learning.try_local_passage_decision", return_value="not_identity"):
amb = client.post(
f"/api/journal/conversations/{amb_conv}/turn",
headers=headers,
json={"body": "Clarissa lag auf dem Tisch."},
)
status_ok(amb, "ambiguous local passage")
expect(not amb.json().get("pending_mask_review"), "local passage can finish without a popup")
expect(any(item.get("role") == "assistant" for item in amb.json().get("messages") or []), "local passage still generates")
back = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "semantic"})
expect(back.json()["detect_operating_mode"] == "semantic", "mode can return to semantic")
if __name__ == "__main__":
main()
+1
View File
@@ -89,6 +89,7 @@ def main() -> None:
expect(set(roles) == {"generate", "detect"}, "two provider roles")
expect(providers.json()["remote_plaintext_allowed"] is True, "test runtime allows remote detect")
expect(providers.json()["remote_plaintext_reason"] == "non_production", "tests are not production")
expect(providers.json().get("detect_operating_mode") == "semantic", "detect mode defaults to semantic")
expect(roles["generate"]["ready"] is False, "generate fail-closed without key")
expect("sk-" not in providers.text, "provider status has no secret")
expect("KANSHO_PROVIDER_KEY=" not in providers.text, "env assignment not leaked")
+35 -1
View File
@@ -12,7 +12,14 @@ os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
Path(tempfile.gettempdir()).mkdir(parents=True, exist_ok=True)
from providers import is_local_url
from entity_detect import (
DETECT_MAX_TOKENS,
DETECT_MAX_TOKENS_LOCAL,
DETECT_TIMEOUT,
DETECT_TIMEOUT_LOCAL,
detect_chat_limits,
)
from providers import ProviderConfig, is_local_url
def expect(ok: bool, message: str) -> None:
@@ -29,6 +36,33 @@ def main() -> None:
expect(is_local_url("http://ollama:11434/v1/chat/completions"), "compose hostname ollama is local")
expect(not is_local_url("https://openrouter.ai/api/v1/chat/completions"), "OpenRouter is not local")
expect(not is_local_url("https://8.8.8.8/v1/chat/completions"), "public IP is not local")
remote = ProviderConfig(
role="detect",
name="openrouter",
mode="http",
url="https://openrouter.ai/api/v1/chat/completions",
model="openai/gpt-4.1-nano",
key="x",
local=False,
zdr=True,
no_train=True,
)
local = ProviderConfig(
role="detect",
name="ollama",
mode="http",
url="http://192.168.2.144:11434/v1/chat/completions",
model="phi3:mini",
key="",
local=True,
zdr=True,
no_train=True,
)
expect(detect_chat_limits(remote) == (DETECT_TIMEOUT, DETECT_MAX_TOKENS), "remote detect keeps 90s/1024")
expect(
detect_chat_limits(local) == (DETECT_TIMEOUT_LOCAL, DETECT_MAX_TOKENS_LOCAL),
"local detect waits 300s with a 256-token cap",
)
print("local url tests passed.")