Kansho/backend/detect_learning.py

"""Transitional learning detect: senses grow from dialogue confirmations.

Not a global word list. Detect still does not auto-activate identities.
Ambiguous spellings get a local passage call when a local detect provider exists.
"""
from __future__ import annotations

import json
import re
import uuid
from typing import Any

from db import get_db, row_to_dict
from identity_store import KINSHIP, ENTITY_TYPES, confirm_identity, is_maskable_label, normalize_label

MODE_SEMANTIC = "semantic"
MODE_LEARNING = "learning"
MODES = (MODE_SEMANTIC, MODE_LEARNING)
SETTING_KEY = "detect_operating_mode"
_LETTER = r"A-Za-zÄÖÜäöüß"
WINDOW = 80

_DECISION_IDENTITY = "identity"
_DECISION_NOT = "not_identity"


def get_detect_operating_mode() -> str:
    with get_db() as conn:
        row = row_to_dict(
            conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
        )
    value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower()
    return value if value in MODES else MODE_SEMANTIC


def set_detect_operating_mode(mode: str) -> str:
    chosen = (mode or "").strip().lower()
    if chosen not in MODES:
        raise ValueError("invalid_detect_operating_mode")
    with get_db() as conn:
        conn.execute(
            """
            INSERT INTO app_settings (key, value, updated)
            VALUES (?, ?, datetime('now'))
            ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
            """,
            (SETTING_KEY, chosen),
        )
    return chosen


def get_sense(profile_id: str, label: str) -> dict:
    key = normalize_label(label) or (label or "").strip()
    if not key:
        return {
            "normalized_label": "",
            "identity_hits": 0,
            "non_identity_hits": 0,
            "ambiguous": False,
        }
    with get_db() as conn:
        row = row_to_dict(
            conn.execute(
                """
                SELECT normalized_label, identity_hits, non_identity_hits
                FROM label_senses
                WHERE profile_id = ? AND lower(normalized_label) = lower(?)
                """,
                (profile_id, key),
            ).fetchone()
        )
    if not row:
        return {
            "normalized_label": key,
            "identity_hits": 0,
            "non_identity_hits": 0,
            "ambiguous": False,
        }
    identity = int(row.get("identity_hits") or 0)
    other = int(row.get("non_identity_hits") or 0)
    return {
        "normalized_label": row.get("normalized_label") or key,
        "identity_hits": identity,
        "non_identity_hits": other,
        "ambiguous": identity > 0 and other > 0,
    }


def record_sense(profile_id: str, label: str, *, identity: bool) -> dict:
    key = normalize_label(label) or (label or "").strip()
    if not key:
        raise ValueError("empty_label")
    with get_db() as conn:
        existing = row_to_dict(
            conn.execute(
                """
                SELECT normalized_label, identity_hits, non_identity_hits FROM label_senses
                WHERE profile_id = ? AND lower(normalized_label) = lower(?)
                """,
                (profile_id, key),
            ).fetchone()
        )
        if existing:
            key = existing.get("normalized_label") or key
        identity_hits = int((existing or {}).get("identity_hits") or 0)
        non_identity_hits = int((existing or {}).get("non_identity_hits") or 0)
        if identity:
            identity_hits += 1
        else:
            non_identity_hits += 1
        conn.execute(
            """
            INSERT INTO label_senses (
              profile_id, normalized_label, identity_hits, non_identity_hits, updated
            )
            VALUES (?, ?, ?, ?, datetime('now'))
            ON CONFLICT(profile_id, normalized_label) DO UPDATE SET
              identity_hits = excluded.identity_hits,
              non_identity_hits = excluded.non_identity_hits,
              updated = datetime('now')
            """,
            (profile_id, key, identity_hits, non_identity_hits),
        )
    return get_sense(profile_id, key)


def _in_user_text(label: str, user_body: str) -> bool:
    if not label or not user_body:
        return False
    return bool(
        re.search(
            rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
            user_body,
            re.IGNORECASE,
        )
    )


def _excerpt(user_body: str, start: int | None, end: int | None, label: str) -> str:
    text = user_body or ""
    if start is None or end is None or start < 0 or end > len(text):
        match = re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text, re.IGNORECASE)
        if not match:
            return text[:WINDOW]
        start, end = match.start(), match.end()
    left = max(0, int(start) - WINDOW)
    right = min(len(text), int(end) + WINDOW)
    return text[left:right]


def needs_review(profile_id: str, mapping: dict, user_body: str) -> bool:
    label = mapping.get("local_label") or ""
    if mapping.get("source") == "confirmed_registry":
        sense = get_sense(profile_id, label)
        return bool(sense["ambiguous"]) and _in_user_text(label, user_body)
    if mapping.get("source") != "request_local":
        return False
    if not is_maskable_label(label) or not _in_user_text(label, user_body):
        return False
    sense = get_sense(profile_id, label)
    if sense["identity_hits"] > 0 and not sense["ambiguous"]:
        return False
    return True


def _word_matches(text: str):
    return list(re.finditer(rf"[{_LETTER}]+", text or ""))


def _prev_word(text: str, index: int) -> str:
    words = re.findall(rf"[{_LETTER}]+", text[:index])
    return (words[-1].lower() if words else "")


def kinship_governed_labels(user_body: str) -> set[str]:
    """Labels after Frau/Sohn/… in the current user line. Not a food word list."""
    found: set[str] = set()
    for match in _word_matches(user_body):
        label = match.group(0)
        if not is_maskable_label(label):
            continue
        if _prev_word(user_body, match.start()) in KINSHIP:
            found.add(label.casefold())
    return found


def _align_user_span(rendered: str, user_body: str, start: int, end: int) -> tuple[int, int]:
    if not user_body:
        return start, end
    pos = (rendered or "").rfind(user_body)
    if pos < 0:
        return start, end
    return pos + start, pos + end


def _token_for_label(mappings: list[dict], label: str, entity_type: str = "PERSON") -> str:
    needle = label.casefold()
    for mapping in mappings:
        if (mapping.get("local_label") or "").casefold() == needle and mapping.get("token"):
            return mapping["token"]
    return f"{entity_type}:L{uuid.uuid4().hex[:6].upper()}"


def supplement_kinship_candidates(
    profile_id: str,
    candidates: list[dict],
    mappings: list[dict],
    user_body: str,
    rendered: str = "",
) -> tuple[list[dict], list[dict]]:
    """If Detect misses Frau X / Sohn X, still offer those user-line mentions for review."""
    wanted = kinship_governed_labels(user_body)
    if not wanted:
        return candidates, mappings
    occupied: set[tuple[int, int, str]] = set()
    for item in candidates:
        label = (item.get("text") or "").casefold()
        excerpt = item.get("excerpt") or ""
        for match in _word_matches(user_body):
            if match.group(0).casefold() != label:
                continue
            snippet = _excerpt(user_body, match.start(), match.end(), match.group(0))
            if snippet == excerpt or match.group(0) in excerpt:
                occupied.add((match.start(), match.end(), label))
                break
    extra_mappings = list(mappings)
    extra_candidates = list(candidates)
    for match in _word_matches(user_body):
        label = match.group(0)
        if label.casefold() not in wanted:
            continue
        user_key = (match.start(), match.end(), label.casefold())
        if user_key in occupied:
            continue
        occupied.add(user_key)
        start, end = _align_user_span(rendered, user_body, match.start(), match.end())
        identity_shaped = _prev_word(user_body, match.start()) in KINSHIP
        extra_candidates.append(
            {
                "id": str(uuid.uuid4()),
                "text": label,
                "entity_type": "PERSON",
                "start": start,
                "end": end,
                "excerpt": _excerpt(user_body, match.start(), match.end(), label),
                "ambiguous": get_sense(profile_id, label)["ambiguous"],
                "suggested": _DECISION_IDENTITY if identity_shaped else _DECISION_NOT,
            }
        )
        extra_mappings.append(
            {
                "token": _token_for_label(extra_mappings, label),
                "local_label": label,
                "canonical_label": label,
                "demask_label": label,
                "entity_type": "PERSON",
                "source": "request_local",
                "start": start,
                "end": end,
                "aliases": [],
                "labels": [label],
            }
        )
    return extra_candidates, extra_mappings


def build_candidates(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
    seen: set[tuple[int | None, int | None, str]] = set()
    items: list[dict] = []
    for mapping in mappings:
        if not needs_review(profile_id, mapping, user_body):
            continue
        label = mapping.get("local_label") or ""
        start = mapping.get("start")
        end = mapping.get("end")
        key = (start, end, label.casefold())
        if key in seen:
            continue
        seen.add(key)
        items.append(
            {
                "id": str(uuid.uuid4()),
                "text": label,
                "entity_type": mapping.get("entity_type") or "PERSON",
                "start": start,
                "end": end,
                "excerpt": _excerpt(user_body, start, end, label),
                "ambiguous": get_sense(profile_id, label)["ambiguous"],
            }
        )
    return items


def suppress_known_non_identity(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
    kept = []
    for mapping in mappings:
        label = mapping.get("local_label") or ""
        sense = get_sense(profile_id, label)
        if (
            mapping.get("source") == "request_local"
            and sense["non_identity_hits"] > 0
            and not sense["identity_hits"]
            and _in_user_text(label, user_body)
        ):
            continue
        kept.append(mapping)
    return kept


def save_pending(
    profile_id: str,
    conversation_id: str,
    user_message_id: str,
    payload: dict,
) -> str:
    review_id = str(uuid.uuid4())
    with get_db() as conn:
        conn.execute(
            "DELETE FROM pending_mask_reviews WHERE profile_id = ? AND conversation_id = ?",
            (profile_id, conversation_id),
        )
        conn.execute(
            """
            INSERT INTO pending_mask_reviews (
              id, profile_id, conversation_id, user_message_id, payload_json, created
            )
            VALUES (?, ?, ?, ?, ?, datetime('now'))
            """,
            (review_id, profile_id, conversation_id, user_message_id, json.dumps(payload, ensure_ascii=False)),
        )
    return review_id


def load_pending(profile_id: str, review_id: str) -> dict | None:
    with get_db() as conn:
        row = row_to_dict(
            conn.execute(
                """
                SELECT * FROM pending_mask_reviews
                WHERE id = ? AND profile_id = ?
                """,
                (review_id, profile_id),
            ).fetchone()
        )
    if not row:
        return None
    payload = json.loads(row.get("payload_json") or "{}")
    payload["id"] = row["id"]
    payload["conversation_id"] = row["conversation_id"]
    payload["user_message_id"] = row["user_message_id"]
    return payload


def drop_pending(profile_id: str, review_id: str) -> None:
    with get_db() as conn:
        conn.execute(
            "DELETE FROM pending_mask_reviews WHERE id = ? AND profile_id = ?",
            (review_id, profile_id),
        )


def apply_review_decisions(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
    by_id = {item["id"]: item for item in pending.get("candidates") or []}
    drop_keys: set[tuple] = set()
    for raw in decisions:
        candidate = by_id.get(raw.get("id") or "")
        if not candidate:
            continue
        decision = (raw.get("decision") or "").strip()
        label = candidate.get("text") or ""
        kind = candidate.get("entity_type") if candidate.get("entity_type") in ENTITY_TYPES else "PERSON"
        if decision == _DECISION_IDENTITY:
            record_sense(profile_id, label, identity=True)
            confirm_identity(profile_id, label, entity_type=kind)
        elif decision == _DECISION_NOT:
            record_sense(profile_id, label, identity=False)
            drop_keys.add((candidate.get("start"), candidate.get("end"), label.casefold()))
    mappings = []
    for mapping in pending.get("mappings") or []:
        key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
        if key in drop_keys:
            continue
        mappings.append(mapping)
    return suppress_known_non_identity(profile_id, mappings, pending.get("user_body") or "")


def confirm_known_identity_spans(profile_id: str, mappings: list[dict], user_body: str) -> None:
    for mapping in mappings:
        label = mapping.get("local_label") or ""
        sense = get_sense(profile_id, label)
        if not (
            sense["identity_hits"] > 0
            and not sense["ambiguous"]
            and _in_user_text(label, user_body)
        ):
            continue
        kind = mapping.get("entity_type") if mapping.get("entity_type") in ENTITY_TYPES else "PERSON"
        try:
            confirm_identity(profile_id, label, entity_type=kind)
        except ValueError:
            continue


def list_senses(profile_id: str) -> list[dict]:
    with get_db() as conn:
        rows = conn.execute(
            """
            SELECT normalized_label, identity_hits, non_identity_hits
            FROM label_senses WHERE profile_id = ?
            ORDER BY lower(normalized_label)
            """,
            (profile_id,),
        ).fetchall()
    items = []
    for raw in rows:
        row = row_to_dict(raw) or {}
        identity = int(row.get("identity_hits") or 0)
        other = int(row.get("non_identity_hits") or 0)
        items.append(
            {
                "normalized_label": row.get("normalized_label") or "",
                "identity_hits": identity,
                "non_identity_hits": other,
                "ambiguous": identity > 0 and other > 0,
            }
        )
    return items


def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | None:
    with get_db() as conn:
        row = row_to_dict(
            conn.execute(
                """
                SELECT id, payload_json FROM pending_mask_reviews
                WHERE profile_id = ? AND conversation_id = ?
                """,
                (profile_id, conversation_id),
            ).fetchone()
        )
    if not row:
        return None
    payload = json.loads(row.get("payload_json") or "{}")
    return {
        "id": row["id"],
        "candidates": payload.get("candidates") or [],
    }


def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: list[dict]) -> tuple[list[dict], list[dict]]:
    """Local passage LLM for already-ambiguous spellings. Unresolved stay for the popup."""
    remaining = []
    drop_keys: set[tuple] = set()
    for item in candidates:
        if not item.get("ambiguous"):
            remaining.append(item)
            continue
        label = item.get("text") or ""
        decision = try_local_passage_decision(item.get("excerpt") or "", label)
        if decision == _DECISION_NOT:
            record_sense(profile_id, label, identity=False)
            drop_keys.add((item.get("start"), item.get("end"), label.casefold()))
        elif decision == _DECISION_IDENTITY:
            record_sense(profile_id, label, identity=True)
        else:
            remaining.append(item)
    if not drop_keys:
        return remaining, mappings
    kept = []
    for mapping in mappings:
        key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
        if key in drop_keys:
            continue
        kept.append(mapping)
    return remaining, kept


def try_local_passage_decision(excerpt: str, label: str) -> str | None:
    """Return identity, not_identity, or None if no local model or unusable answer."""
    from providers import ProviderError, complete_chat, detect_provider

    config = detect_provider()
    if not config or not config.local or config.mode != "http":
        return None
    prompt = (
        "Entscheide nur für die markierte Nennung in diesem kurzen Ausschnitt. "
        "Ist sie eine schützenswerte Identität (Person, Ort, Organisation, privates Projekt) "
        f"oder eine Sache/Allgemeinbedeutung? Wort: {label}\n"
        f"Ausschnitt: {excerpt}\n"
        'Antworte nur mit JSON {"decision":"identity"} oder {"decision":"not_identity"}.'
    )
    try:
        result = complete_chat(
            config,
            [{"role": "user", "content": prompt}],
            timeout=30.0,
            max_tokens=32,
        )
    except ProviderError:
        return None
    match = re.search(r"\{.*\}", result.content or "", re.DOTALL)
    if not match:
        return None
    try:
        data = json.loads(match.group(0))
    except json.JSONDecodeError:
        return None
    value = (data.get("decision") or "").strip()
    if value in {_DECISION_IDENTITY, _DECISION_NOT}:
        return value
    return None