Add transitional learning detect with in-dialog mask review.
Senses grow from confirmations instead of a word list, so a later local pipeline can decide homonyms on a short passage. Default stays semantic. Local detect waits longer, and the reverse-proxy timeout is documented so Dev does not 504 first. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
59921fc5b5
commit
fa1c11c33e
29 files changed
+1135
-45
No files matched your search
@@ -0,0 +1,410 @@
|
||||
"""Transitional learning detect: senses grow from dialogue confirmations.
|
||||
|
||||
Not a global word list. Detect still does not auto-activate identities.
|
||||
Ambiguous spellings get a local passage call when a local detect provider exists.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from db import get_db, row_to_dict
|
||||
from identity_store import ENTITY_TYPES, confirm_identity, is_maskable_label, normalize_label
|
||||
|
||||
MODE_SEMANTIC = "semantic"
|
||||
MODE_LEARNING = "learning"
|
||||
MODES = (MODE_SEMANTIC, MODE_LEARNING)
|
||||
SETTING_KEY = "detect_operating_mode"
|
||||
_LETTER = r"A-Za-zÄÖÜäöüß"
|
||||
WINDOW = 80
|
||||
|
||||
_DECISION_IDENTITY = "identity"
|
||||
_DECISION_NOT = "not_identity"
|
||||
|
||||
|
||||
def get_detect_operating_mode() -> str:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
|
||||
)
|
||||
value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower()
|
||||
return value if value in MODES else MODE_SEMANTIC
|
||||
|
||||
|
||||
def set_detect_operating_mode(mode: str) -> str:
|
||||
chosen = (mode or "").strip().lower()
|
||||
if chosen not in MODES:
|
||||
raise ValueError("invalid_detect_operating_mode")
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO app_settings (key, value, updated)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
|
||||
""",
|
||||
(SETTING_KEY, chosen),
|
||||
)
|
||||
return chosen
|
||||
|
||||
|
||||
def get_sense(profile_id: str, label: str) -> dict:
|
||||
key = normalize_label(label) or (label or "").strip()
|
||||
if not key:
|
||||
return {
|
||||
"normalized_label": "",
|
||||
"identity_hits": 0,
|
||||
"non_identity_hits": 0,
|
||||
"ambiguous": False,
|
||||
}
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT normalized_label, identity_hits, non_identity_hits
|
||||
FROM label_senses
|
||||
WHERE profile_id = ? AND lower(normalized_label) = lower(?)
|
||||
""",
|
||||
(profile_id, key),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
return {
|
||||
"normalized_label": key,
|
||||
"identity_hits": 0,
|
||||
"non_identity_hits": 0,
|
||||
"ambiguous": False,
|
||||
}
|
||||
identity = int(row.get("identity_hits") or 0)
|
||||
other = int(row.get("non_identity_hits") or 0)
|
||||
return {
|
||||
"normalized_label": row.get("normalized_label") or key,
|
||||
"identity_hits": identity,
|
||||
"non_identity_hits": other,
|
||||
"ambiguous": identity > 0 and other > 0,
|
||||
}
|
||||
|
||||
|
||||
def record_sense(profile_id: str, label: str, *, identity: bool) -> dict:
|
||||
key = normalize_label(label) or (label or "").strip()
|
||||
if not key:
|
||||
raise ValueError("empty_label")
|
||||
with get_db() as conn:
|
||||
existing = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT normalized_label, identity_hits, non_identity_hits FROM label_senses
|
||||
WHERE profile_id = ? AND lower(normalized_label) = lower(?)
|
||||
""",
|
||||
(profile_id, key),
|
||||
).fetchone()
|
||||
)
|
||||
if existing:
|
||||
key = existing.get("normalized_label") or key
|
||||
identity_hits = int((existing or {}).get("identity_hits") or 0)
|
||||
non_identity_hits = int((existing or {}).get("non_identity_hits") or 0)
|
||||
if identity:
|
||||
identity_hits += 1
|
||||
else:
|
||||
non_identity_hits += 1
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO label_senses (
|
||||
profile_id, normalized_label, identity_hits, non_identity_hits, updated
|
||||
)
|
||||
VALUES (?, ?, ?, ?, datetime('now'))
|
||||
ON CONFLICT(profile_id, normalized_label) DO UPDATE SET
|
||||
identity_hits = excluded.identity_hits,
|
||||
non_identity_hits = excluded.non_identity_hits,
|
||||
updated = datetime('now')
|
||||
""",
|
||||
(profile_id, key, identity_hits, non_identity_hits),
|
||||
)
|
||||
return get_sense(profile_id, key)
|
||||
|
||||
|
||||
def _in_user_text(label: str, user_body: str) -> bool:
|
||||
if not label or not user_body:
|
||||
return False
|
||||
return bool(
|
||||
re.search(
|
||||
rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
|
||||
user_body,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _excerpt(user_body: str, start: int | None, end: int | None, label: str) -> str:
|
||||
text = user_body or ""
|
||||
if start is None or end is None or start < 0 or end > len(text):
|
||||
match = re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text, re.IGNORECASE)
|
||||
if not match:
|
||||
return text[:WINDOW]
|
||||
start, end = match.start(), match.end()
|
||||
left = max(0, int(start) - WINDOW)
|
||||
right = min(len(text), int(end) + WINDOW)
|
||||
return text[left:right]
|
||||
|
||||
|
||||
def needs_review(profile_id: str, mapping: dict, user_body: str) -> bool:
|
||||
label = mapping.get("local_label") or ""
|
||||
if mapping.get("source") == "confirmed_registry":
|
||||
sense = get_sense(profile_id, label)
|
||||
return bool(sense["ambiguous"]) and _in_user_text(label, user_body)
|
||||
if mapping.get("source") != "request_local":
|
||||
return False
|
||||
if not is_maskable_label(label) or not _in_user_text(label, user_body):
|
||||
return False
|
||||
sense = get_sense(profile_id, label)
|
||||
if sense["identity_hits"] > 0 and not sense["ambiguous"]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def build_candidates(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
|
||||
seen: set[tuple[int | None, int | None, str]] = set()
|
||||
items: list[dict] = []
|
||||
for mapping in mappings:
|
||||
if not needs_review(profile_id, mapping, user_body):
|
||||
continue
|
||||
label = mapping.get("local_label") or ""
|
||||
start = mapping.get("start")
|
||||
end = mapping.get("end")
|
||||
key = (start, end, label.casefold())
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
items.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": mapping.get("entity_type") or "PERSON",
|
||||
"start": start,
|
||||
"end": end,
|
||||
"excerpt": _excerpt(user_body, start, end, label),
|
||||
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def suppress_known_non_identity(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
|
||||
kept = []
|
||||
for mapping in mappings:
|
||||
label = mapping.get("local_label") or ""
|
||||
sense = get_sense(profile_id, label)
|
||||
if (
|
||||
mapping.get("source") == "request_local"
|
||||
and sense["non_identity_hits"] > 0
|
||||
and not sense["identity_hits"]
|
||||
and _in_user_text(label, user_body)
|
||||
):
|
||||
continue
|
||||
kept.append(mapping)
|
||||
return kept
|
||||
|
||||
|
||||
def save_pending(
|
||||
profile_id: str,
|
||||
conversation_id: str,
|
||||
user_message_id: str,
|
||||
payload: dict,
|
||||
) -> str:
|
||||
review_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM pending_mask_reviews WHERE profile_id = ? AND conversation_id = ?",
|
||||
(profile_id, conversation_id),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO pending_mask_reviews (
|
||||
id, profile_id, conversation_id, user_message_id, payload_json, created
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, datetime('now'))
|
||||
""",
|
||||
(review_id, profile_id, conversation_id, user_message_id, json.dumps(payload, ensure_ascii=False)),
|
||||
)
|
||||
return review_id
|
||||
|
||||
|
||||
def load_pending(profile_id: str, review_id: str) -> dict | None:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM pending_mask_reviews
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(review_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
payload = json.loads(row.get("payload_json") or "{}")
|
||||
payload["id"] = row["id"]
|
||||
payload["conversation_id"] = row["conversation_id"]
|
||||
payload["user_message_id"] = row["user_message_id"]
|
||||
return payload
|
||||
|
||||
|
||||
def drop_pending(profile_id: str, review_id: str) -> None:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM pending_mask_reviews WHERE id = ? AND profile_id = ?",
|
||||
(review_id, profile_id),
|
||||
)
|
||||
|
||||
|
||||
def apply_review_decisions(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
|
||||
by_id = {item["id"]: item for item in pending.get("candidates") or []}
|
||||
drop_keys: set[tuple] = set()
|
||||
for raw in decisions:
|
||||
candidate = by_id.get(raw.get("id") or "")
|
||||
if not candidate:
|
||||
continue
|
||||
decision = (raw.get("decision") or "").strip()
|
||||
label = candidate.get("text") or ""
|
||||
kind = candidate.get("entity_type") if candidate.get("entity_type") in ENTITY_TYPES else "PERSON"
|
||||
if decision == _DECISION_IDENTITY:
|
||||
record_sense(profile_id, label, identity=True)
|
||||
confirm_identity(profile_id, label, entity_type=kind)
|
||||
elif decision == _DECISION_NOT:
|
||||
record_sense(profile_id, label, identity=False)
|
||||
drop_keys.add((candidate.get("start"), candidate.get("end"), label.casefold()))
|
||||
mappings = []
|
||||
for mapping in pending.get("mappings") or []:
|
||||
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||
if key in drop_keys:
|
||||
continue
|
||||
mappings.append(mapping)
|
||||
return suppress_known_non_identity(profile_id, mappings, pending.get("user_body") or "")
|
||||
|
||||
|
||||
def confirm_known_identity_spans(profile_id: str, mappings: list[dict], user_body: str) -> None:
|
||||
for mapping in mappings:
|
||||
label = mapping.get("local_label") or ""
|
||||
sense = get_sense(profile_id, label)
|
||||
if not (
|
||||
sense["identity_hits"] > 0
|
||||
and not sense["ambiguous"]
|
||||
and _in_user_text(label, user_body)
|
||||
):
|
||||
continue
|
||||
kind = mapping.get("entity_type") if mapping.get("entity_type") in ENTITY_TYPES else "PERSON"
|
||||
try:
|
||||
confirm_identity(profile_id, label, entity_type=kind)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
|
||||
def list_senses(profile_id: str) -> list[dict]:
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT normalized_label, identity_hits, non_identity_hits
|
||||
FROM label_senses WHERE profile_id = ?
|
||||
ORDER BY lower(normalized_label)
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
items = []
|
||||
for raw in rows:
|
||||
row = row_to_dict(raw) or {}
|
||||
identity = int(row.get("identity_hits") or 0)
|
||||
other = int(row.get("non_identity_hits") or 0)
|
||||
items.append(
|
||||
{
|
||||
"normalized_label": row.get("normalized_label") or "",
|
||||
"identity_hits": identity,
|
||||
"non_identity_hits": other,
|
||||
"ambiguous": identity > 0 and other > 0,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | None:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id, payload_json FROM pending_mask_reviews
|
||||
WHERE profile_id = ? AND conversation_id = ?
|
||||
""",
|
||||
(profile_id, conversation_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
payload = json.loads(row.get("payload_json") or "{}")
|
||||
return {
|
||||
"id": row["id"],
|
||||
"candidates": payload.get("candidates") or [],
|
||||
}
|
||||
|
||||
|
||||
def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: list[dict]) -> tuple[list[dict], list[dict]]:
|
||||
"""Local passage LLM for already-ambiguous spellings. Unresolved stay for the popup."""
|
||||
remaining = []
|
||||
drop_keys: set[tuple] = set()
|
||||
for item in candidates:
|
||||
if not item.get("ambiguous"):
|
||||
remaining.append(item)
|
||||
continue
|
||||
label = item.get("text") or ""
|
||||
decision = try_local_passage_decision(item.get("excerpt") or "", label)
|
||||
if decision == _DECISION_NOT:
|
||||
record_sense(profile_id, label, identity=False)
|
||||
drop_keys.add((item.get("start"), item.get("end"), label.casefold()))
|
||||
elif decision == _DECISION_IDENTITY:
|
||||
record_sense(profile_id, label, identity=True)
|
||||
else:
|
||||
remaining.append(item)
|
||||
if not drop_keys:
|
||||
return remaining, mappings
|
||||
kept = []
|
||||
for mapping in mappings:
|
||||
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||
if key in drop_keys:
|
||||
continue
|
||||
kept.append(mapping)
|
||||
return remaining, kept
|
||||
|
||||
|
||||
def try_local_passage_decision(excerpt: str, label: str) -> str | None:
|
||||
"""Return identity, not_identity, or None if no local model or unusable answer."""
|
||||
from providers import ProviderError, complete_chat, detect_provider
|
||||
|
||||
config = detect_provider()
|
||||
if not config or not config.local or config.mode != "http":
|
||||
return None
|
||||
prompt = (
|
||||
"Entscheide nur für die markierte Nennung in diesem kurzen Ausschnitt. "
|
||||
"Ist sie eine schützenswerte Identität (Person, Ort, Organisation, privates Projekt) "
|
||||
f"oder eine Sache/Allgemeinbedeutung? Wort: {label}\n"
|
||||
f"Ausschnitt: {excerpt}\n"
|
||||
'Antworte nur mit JSON {"decision":"identity"} oder {"decision":"not_identity"}.'
|
||||
)
|
||||
try:
|
||||
result = complete_chat(
|
||||
config,
|
||||
[{"role": "user", "content": prompt}],
|
||||
timeout=30.0,
|
||||
max_tokens=32,
|
||||
)
|
||||
except ProviderError:
|
||||
return None
|
||||
match = re.search(r"\{.*\}", result.content or "", re.DOTALL)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(match.group(0))
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
value = (data.get("decision") or "").strip()
|
||||
if value in {_DECISION_IDENTITY, _DECISION_NOT}:
|
||||
return value
|
||||
return None
|
||||
Reference in new issue
Block a user