"""Transitional learning detect: senses grow from dialogue confirmations. Not a global word list. Detect still does not auto-activate identities. Ambiguous spellings get a local passage call when a local detect provider exists. """ from __future__ import annotations import json import re import uuid from typing import Any from db import get_db, row_to_dict from identity_store import ENTITY_TYPES, confirm_identity, is_maskable_label, normalize_label MODE_SEMANTIC = "semantic" MODE_LEARNING = "learning" MODES = (MODE_SEMANTIC, MODE_LEARNING) SETTING_KEY = "detect_operating_mode" _LETTER = r"A-Za-zÄÖÜäöüß" WINDOW = 80 _DECISION_IDENTITY = "identity" _DECISION_NOT = "not_identity" def get_detect_operating_mode() -> str: with get_db() as conn: row = row_to_dict( conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone() ) value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower() return value if value in MODES else MODE_SEMANTIC def set_detect_operating_mode(mode: str) -> str: chosen = (mode or "").strip().lower() if chosen not in MODES: raise ValueError("invalid_detect_operating_mode") with get_db() as conn: conn.execute( """ INSERT INTO app_settings (key, value, updated) VALUES (?, ?, datetime('now')) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now') """, (SETTING_KEY, chosen), ) return chosen def get_sense(profile_id: str, label: str) -> dict: key = normalize_label(label) or (label or "").strip() if not key: return { "normalized_label": "", "identity_hits": 0, "non_identity_hits": 0, "ambiguous": False, } with get_db() as conn: row = row_to_dict( conn.execute( """ SELECT normalized_label, identity_hits, non_identity_hits FROM label_senses WHERE profile_id = ? AND lower(normalized_label) = lower(?) """, (profile_id, key), ).fetchone() ) if not row: return { "normalized_label": key, "identity_hits": 0, "non_identity_hits": 0, "ambiguous": False, } identity = int(row.get("identity_hits") or 0) other = int(row.get("non_identity_hits") or 0) return { "normalized_label": row.get("normalized_label") or key, "identity_hits": identity, "non_identity_hits": other, "ambiguous": identity > 0 and other > 0, } def record_sense(profile_id: str, label: str, *, identity: bool) -> dict: key = normalize_label(label) or (label or "").strip() if not key: raise ValueError("empty_label") with get_db() as conn: existing = row_to_dict( conn.execute( """ SELECT normalized_label, identity_hits, non_identity_hits FROM label_senses WHERE profile_id = ? AND lower(normalized_label) = lower(?) """, (profile_id, key), ).fetchone() ) if existing: key = existing.get("normalized_label") or key identity_hits = int((existing or {}).get("identity_hits") or 0) non_identity_hits = int((existing or {}).get("non_identity_hits") or 0) if identity: identity_hits += 1 else: non_identity_hits += 1 conn.execute( """ INSERT INTO label_senses ( profile_id, normalized_label, identity_hits, non_identity_hits, updated ) VALUES (?, ?, ?, ?, datetime('now')) ON CONFLICT(profile_id, normalized_label) DO UPDATE SET identity_hits = excluded.identity_hits, non_identity_hits = excluded.non_identity_hits, updated = datetime('now') """, (profile_id, key, identity_hits, non_identity_hits), ) return get_sense(profile_id, key) def _in_user_text(label: str, user_body: str) -> bool: if not label or not user_body: return False return bool( re.search( rf"(? str: text = user_body or "" if start is None or end is None or start < 0 or end > len(text): match = re.search(rf"(? bool: label = mapping.get("local_label") or "" if mapping.get("source") == "confirmed_registry": sense = get_sense(profile_id, label) return bool(sense["ambiguous"]) and _in_user_text(label, user_body) if mapping.get("source") != "request_local": return False if not is_maskable_label(label) or not _in_user_text(label, user_body): return False sense = get_sense(profile_id, label) if sense["identity_hits"] > 0 and not sense["ambiguous"]: return False return True def build_candidates(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]: seen: set[tuple[int | None, int | None, str]] = set() items: list[dict] = [] for mapping in mappings: if not needs_review(profile_id, mapping, user_body): continue label = mapping.get("local_label") or "" start = mapping.get("start") end = mapping.get("end") key = (start, end, label.casefold()) if key in seen: continue seen.add(key) items.append( { "id": str(uuid.uuid4()), "text": label, "entity_type": mapping.get("entity_type") or "PERSON", "start": start, "end": end, "excerpt": _excerpt(user_body, start, end, label), "ambiguous": get_sense(profile_id, label)["ambiguous"], } ) return items def suppress_known_non_identity(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]: kept = [] for mapping in mappings: label = mapping.get("local_label") or "" sense = get_sense(profile_id, label) if ( mapping.get("source") == "request_local" and sense["non_identity_hits"] > 0 and not sense["identity_hits"] and _in_user_text(label, user_body) ): continue kept.append(mapping) return kept def save_pending( profile_id: str, conversation_id: str, user_message_id: str, payload: dict, ) -> str: review_id = str(uuid.uuid4()) with get_db() as conn: conn.execute( "DELETE FROM pending_mask_reviews WHERE profile_id = ? AND conversation_id = ?", (profile_id, conversation_id), ) conn.execute( """ INSERT INTO pending_mask_reviews ( id, profile_id, conversation_id, user_message_id, payload_json, created ) VALUES (?, ?, ?, ?, ?, datetime('now')) """, (review_id, profile_id, conversation_id, user_message_id, json.dumps(payload, ensure_ascii=False)), ) return review_id def load_pending(profile_id: str, review_id: str) -> dict | None: with get_db() as conn: row = row_to_dict( conn.execute( """ SELECT * FROM pending_mask_reviews WHERE id = ? AND profile_id = ? """, (review_id, profile_id), ).fetchone() ) if not row: return None payload = json.loads(row.get("payload_json") or "{}") payload["id"] = row["id"] payload["conversation_id"] = row["conversation_id"] payload["user_message_id"] = row["user_message_id"] return payload def drop_pending(profile_id: str, review_id: str) -> None: with get_db() as conn: conn.execute( "DELETE FROM pending_mask_reviews WHERE id = ? AND profile_id = ?", (review_id, profile_id), ) def apply_review_decisions(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]: by_id = {item["id"]: item for item in pending.get("candidates") or []} drop_keys: set[tuple] = set() for raw in decisions: candidate = by_id.get(raw.get("id") or "") if not candidate: continue decision = (raw.get("decision") or "").strip() label = candidate.get("text") or "" kind = candidate.get("entity_type") if candidate.get("entity_type") in ENTITY_TYPES else "PERSON" if decision == _DECISION_IDENTITY: record_sense(profile_id, label, identity=True) confirm_identity(profile_id, label, entity_type=kind) elif decision == _DECISION_NOT: record_sense(profile_id, label, identity=False) drop_keys.add((candidate.get("start"), candidate.get("end"), label.casefold())) mappings = [] for mapping in pending.get("mappings") or []: key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold()) if key in drop_keys: continue mappings.append(mapping) return suppress_known_non_identity(profile_id, mappings, pending.get("user_body") or "") def confirm_known_identity_spans(profile_id: str, mappings: list[dict], user_body: str) -> None: for mapping in mappings: label = mapping.get("local_label") or "" sense = get_sense(profile_id, label) if not ( sense["identity_hits"] > 0 and not sense["ambiguous"] and _in_user_text(label, user_body) ): continue kind = mapping.get("entity_type") if mapping.get("entity_type") in ENTITY_TYPES else "PERSON" try: confirm_identity(profile_id, label, entity_type=kind) except ValueError: continue def list_senses(profile_id: str) -> list[dict]: with get_db() as conn: rows = conn.execute( """ SELECT normalized_label, identity_hits, non_identity_hits FROM label_senses WHERE profile_id = ? ORDER BY lower(normalized_label) """, (profile_id,), ).fetchall() items = [] for raw in rows: row = row_to_dict(raw) or {} identity = int(row.get("identity_hits") or 0) other = int(row.get("non_identity_hits") or 0) items.append( { "normalized_label": row.get("normalized_label") or "", "identity_hits": identity, "non_identity_hits": other, "ambiguous": identity > 0 and other > 0, } ) return items def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | None: with get_db() as conn: row = row_to_dict( conn.execute( """ SELECT id, payload_json FROM pending_mask_reviews WHERE profile_id = ? AND conversation_id = ? """, (profile_id, conversation_id), ).fetchone() ) if not row: return None payload = json.loads(row.get("payload_json") or "{}") return { "id": row["id"], "candidates": payload.get("candidates") or [], } def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: list[dict]) -> tuple[list[dict], list[dict]]: """Local passage LLM for already-ambiguous spellings. Unresolved stay for the popup.""" remaining = [] drop_keys: set[tuple] = set() for item in candidates: if not item.get("ambiguous"): remaining.append(item) continue label = item.get("text") or "" decision = try_local_passage_decision(item.get("excerpt") or "", label) if decision == _DECISION_NOT: record_sense(profile_id, label, identity=False) drop_keys.add((item.get("start"), item.get("end"), label.casefold())) elif decision == _DECISION_IDENTITY: record_sense(profile_id, label, identity=True) else: remaining.append(item) if not drop_keys: return remaining, mappings kept = [] for mapping in mappings: key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold()) if key in drop_keys: continue kept.append(mapping) return remaining, kept def try_local_passage_decision(excerpt: str, label: str) -> str | None: """Return identity, not_identity, or None if no local model or unusable answer.""" from providers import ProviderError, complete_chat, detect_provider config = detect_provider() if not config or not config.local or config.mode != "http": return None prompt = ( "Entscheide nur für die markierte Nennung in diesem kurzen Ausschnitt. " "Ist sie eine schützenswerte Identität (Person, Ort, Organisation, privates Projekt) " f"oder eine Sache/Allgemeinbedeutung? Wort: {label}\n" f"Ausschnitt: {excerpt}\n" 'Antworte nur mit JSON {"decision":"identity"} oder {"decision":"not_identity"}.' ) try: result = complete_chat( config, [{"role": "user", "content": prompt}], timeout=30.0, max_tokens=32, ) except ProviderError: return None match = re.search(r"\{.*\}", result.content or "", re.DOTALL) if not match: return None try: data = json.loads(match.group(0)) except json.JSONDecodeError: return None value = (data.get("decision") or "").strip() if value in {_DECISION_IDENTITY, _DECISION_NOT}: return value return None