Apply learned mask reviews by local cue and highlight the reviewed mention.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
3f7c0d71a5
commit
f7ffd28332
|
|
@ -22,6 +22,8 @@ WINDOW = 80
|
||||||
|
|
||||||
_DECISION_IDENTITY = "identity"
|
_DECISION_IDENTITY = "identity"
|
||||||
_DECISION_NOT = "not_identity"
|
_DECISION_NOT = "not_identity"
|
||||||
|
_DECISION_ASK = "ask"
|
||||||
|
_CUE_BARE = "_bare"
|
||||||
|
|
||||||
|
|
||||||
def get_detect_operating_mode() -> str:
|
def get_detect_operating_mode() -> str:
|
||||||
|
|
@ -136,31 +138,82 @@ def _in_user_text(label: str, user_body: str) -> bool:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _excerpt(user_body: str, start: int | None, end: int | None, label: str) -> str:
|
def _resolve_user_span(
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
label: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> tuple[int, int] | None:
|
||||||
text = user_body or ""
|
text = user_body or ""
|
||||||
if start is None or end is None or start < 0 or end > len(text):
|
needle = (label or "").casefold()
|
||||||
match = re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text, re.IGNORECASE)
|
if start is not None and end is not None:
|
||||||
if not match:
|
left, right = int(start), int(end)
|
||||||
return text[:WINDOW]
|
if 0 <= left < right <= len(text) and text[left:right].casefold() == needle:
|
||||||
start, end = match.start(), match.end()
|
return left, right
|
||||||
left = max(0, int(start) - WINDOW)
|
if rendered and text:
|
||||||
right = min(len(text), int(end) + WINDOW)
|
origin = rendered.rfind(text)
|
||||||
return text[left:right]
|
if origin >= 0:
|
||||||
|
user_left, user_right = left - origin, right - origin
|
||||||
|
if 0 <= user_left < user_right <= len(text) and text[user_left:user_right].casefold() == needle:
|
||||||
|
return user_left, user_right
|
||||||
|
if not needle:
|
||||||
|
return None
|
||||||
|
match = re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text, re.IGNORECASE)
|
||||||
|
if not match:
|
||||||
|
return None
|
||||||
|
return match.start(), match.end()
|
||||||
|
|
||||||
|
|
||||||
def needs_review(profile_id: str, mapping: dict, user_body: str) -> bool:
|
def excerpt_view(
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
label: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> dict:
|
||||||
|
text = user_body or ""
|
||||||
|
span = _resolve_user_span(text, start, end, label, rendered)
|
||||||
|
if not span:
|
||||||
|
snippet = text[:WINDOW]
|
||||||
|
return {"excerpt": snippet, "highlight_start": None, "highlight_end": None}
|
||||||
|
left_i, right_i = span
|
||||||
|
clip_left = max(0, left_i - WINDOW)
|
||||||
|
clip_right = min(len(text), right_i + WINDOW)
|
||||||
|
prefix = "…" if clip_left > 0 else ""
|
||||||
|
suffix = "…" if clip_right < len(text) else ""
|
||||||
|
snippet = text[clip_left:clip_right]
|
||||||
|
return {
|
||||||
|
"excerpt": f"{prefix}{snippet}{suffix}",
|
||||||
|
"highlight_start": left_i - clip_left + len(prefix),
|
||||||
|
"highlight_end": right_i - clip_left + len(prefix),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _excerpt(user_body: str, start: int | None, end: int | None, label: str) -> str:
|
||||||
|
return excerpt_view(user_body, start, end, label)["excerpt"]
|
||||||
|
|
||||||
|
|
||||||
|
def needs_review(profile_id: str, mapping: dict, user_body: str, rendered: str = "") -> bool:
|
||||||
label = mapping.get("local_label") or ""
|
label = mapping.get("local_label") or ""
|
||||||
if mapping.get("source") == "confirmed_registry":
|
if mapping.get("source") == "confirmed_registry":
|
||||||
sense = get_sense(profile_id, label)
|
return (
|
||||||
return bool(sense["ambiguous"]) and _in_user_text(label, user_body)
|
_in_user_text(label, user_body)
|
||||||
|
and decision_for_mention(
|
||||||
|
profile_id, label, user_body, mapping.get("start"), mapping.get("end"), rendered
|
||||||
|
)
|
||||||
|
== _DECISION_ASK
|
||||||
|
)
|
||||||
if mapping.get("source") != "request_local":
|
if mapping.get("source") != "request_local":
|
||||||
return False
|
return False
|
||||||
if not is_maskable_label(label) or not _in_user_text(label, user_body):
|
if not is_maskable_label(label) or not _in_user_text(label, user_body):
|
||||||
return False
|
return False
|
||||||
sense = get_sense(profile_id, label)
|
return (
|
||||||
if sense["identity_hits"] > 0 and not sense["ambiguous"]:
|
decision_for_mention(
|
||||||
return False
|
profile_id, label, user_body, mapping.get("start"), mapping.get("end"), rendered
|
||||||
return True
|
)
|
||||||
|
== _DECISION_ASK
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _word_matches(text: str):
|
def _word_matches(text: str):
|
||||||
|
|
@ -172,6 +225,143 @@ def _prev_word(text: str, index: int) -> str:
|
||||||
return (words[-1].lower() if words else "")
|
return (words[-1].lower() if words else "")
|
||||||
|
|
||||||
|
|
||||||
|
def cue_for_mention(
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
label: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> str:
|
||||||
|
span = _resolve_user_span(user_body, start, end, label, rendered)
|
||||||
|
if not span:
|
||||||
|
return _CUE_BARE
|
||||||
|
return _prev_word(user_body, span[0]) or _CUE_BARE
|
||||||
|
|
||||||
|
|
||||||
|
def _mention_fields(
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
label: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> dict:
|
||||||
|
span = _resolve_user_span(user_body, start, end, label, rendered)
|
||||||
|
return {
|
||||||
|
**excerpt_view(user_body, start, end, label, rendered),
|
||||||
|
"user_start": span[0] if span else None,
|
||||||
|
"user_end": span[1] if span else None,
|
||||||
|
"cue": (_prev_word(user_body, span[0]) or _CUE_BARE) if span else _CUE_BARE,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_cue_decision(profile_id: str, label: str, cue: str) -> str | None:
|
||||||
|
key = normalize_label(label) or (label or "").strip()
|
||||||
|
token = (cue or _CUE_BARE).strip().casefold() or _CUE_BARE
|
||||||
|
if not key:
|
||||||
|
return None
|
||||||
|
with get_db() as conn:
|
||||||
|
row = row_to_dict(
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT decision FROM label_sense_cues
|
||||||
|
WHERE profile_id = ? AND lower(normalized_label) = lower(?) AND cue = ?
|
||||||
|
""",
|
||||||
|
(profile_id, key, token),
|
||||||
|
).fetchone()
|
||||||
|
)
|
||||||
|
value = ((row or {}).get("decision") or "").strip()
|
||||||
|
return value if value in {_DECISION_IDENTITY, _DECISION_NOT} else None
|
||||||
|
|
||||||
|
|
||||||
|
def record_cue(profile_id: str, label: str, cue: str, decision: str) -> None:
|
||||||
|
key = normalize_label(label) or (label or "").strip()
|
||||||
|
token = (cue or _CUE_BARE).strip().casefold() or _CUE_BARE
|
||||||
|
if not key or decision not in {_DECISION_IDENTITY, _DECISION_NOT}:
|
||||||
|
return
|
||||||
|
with get_db() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO label_sense_cues (
|
||||||
|
profile_id, normalized_label, cue, decision, hits, updated
|
||||||
|
)
|
||||||
|
VALUES (?, ?, ?, ?, 1, datetime('now'))
|
||||||
|
ON CONFLICT(profile_id, normalized_label, cue) DO UPDATE SET
|
||||||
|
decision = excluded.decision,
|
||||||
|
hits = label_sense_cues.hits + 1,
|
||||||
|
updated = datetime('now')
|
||||||
|
""",
|
||||||
|
(profile_id, key, token, decision),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def decision_for_mention(
|
||||||
|
profile_id: str,
|
||||||
|
label: str,
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> str:
|
||||||
|
"""identity / not_identity from prior review, or ask once for a new cue."""
|
||||||
|
if not label:
|
||||||
|
return _DECISION_ASK
|
||||||
|
cue = cue_for_mention(user_body, start, end, label, rendered)
|
||||||
|
learned = get_cue_decision(profile_id, label, cue)
|
||||||
|
if learned:
|
||||||
|
return learned
|
||||||
|
sense = get_sense(profile_id, label)
|
||||||
|
identity_shaped = cue in KINSHIP
|
||||||
|
if sense["ambiguous"]:
|
||||||
|
return _DECISION_ASK
|
||||||
|
if sense["identity_hits"] > 0 and not sense["non_identity_hits"]:
|
||||||
|
return _DECISION_IDENTITY
|
||||||
|
if sense["non_identity_hits"] > 0 and not sense["identity_hits"]:
|
||||||
|
return _DECISION_ASK if identity_shaped else _DECISION_NOT
|
||||||
|
return _DECISION_ASK
|
||||||
|
|
||||||
|
|
||||||
|
def apply_learned_decisions(
|
||||||
|
profile_id: str,
|
||||||
|
candidates: list[dict],
|
||||||
|
mappings: list[dict],
|
||||||
|
user_body: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> tuple[list[dict], list[dict]]:
|
||||||
|
remaining = []
|
||||||
|
for item in candidates:
|
||||||
|
verdict = decision_for_mention(
|
||||||
|
profile_id,
|
||||||
|
item.get("text") or "",
|
||||||
|
user_body,
|
||||||
|
item.get("user_start", item.get("start")),
|
||||||
|
item.get("user_end", item.get("end")),
|
||||||
|
rendered,
|
||||||
|
)
|
||||||
|
if verdict == _DECISION_ASK:
|
||||||
|
remaining.append(item)
|
||||||
|
kept = []
|
||||||
|
for mapping in mappings:
|
||||||
|
label = mapping.get("local_label") or ""
|
||||||
|
verdict = decision_for_mention(
|
||||||
|
profile_id,
|
||||||
|
label,
|
||||||
|
user_body,
|
||||||
|
mapping.get("start"),
|
||||||
|
mapping.get("end"),
|
||||||
|
rendered,
|
||||||
|
)
|
||||||
|
if verdict == _DECISION_NOT:
|
||||||
|
continue
|
||||||
|
kept.append(mapping)
|
||||||
|
if verdict == _DECISION_IDENTITY:
|
||||||
|
kind = mapping.get("entity_type") if mapping.get("entity_type") in ENTITY_TYPES else "PERSON"
|
||||||
|
try:
|
||||||
|
confirm_identity(profile_id, label, entity_type=kind)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return remaining, kept
|
||||||
|
|
||||||
|
|
||||||
def kinship_governed_labels(user_body: str) -> set[str]:
|
def kinship_governed_labels(user_body: str) -> set[str]:
|
||||||
"""Labels after Frau/Sohn/… in the current user line. Not a food word list."""
|
"""Labels after Frau/Sohn/… in the current user line. Not a food word list."""
|
||||||
found: set[str] = set()
|
found: set[str] = set()
|
||||||
|
|
@ -214,15 +404,15 @@ def supplement_kinship_candidates(
|
||||||
return candidates, mappings
|
return candidates, mappings
|
||||||
occupied: set[tuple[int, int, str]] = set()
|
occupied: set[tuple[int, int, str]] = set()
|
||||||
for item in candidates:
|
for item in candidates:
|
||||||
label = (item.get("text") or "").casefold()
|
span = _resolve_user_span(
|
||||||
excerpt = item.get("excerpt") or ""
|
user_body,
|
||||||
for match in _word_matches(user_body):
|
item.get("start"),
|
||||||
if match.group(0).casefold() != label:
|
item.get("end"),
|
||||||
continue
|
item.get("text") or "",
|
||||||
snippet = _excerpt(user_body, match.start(), match.end(), match.group(0))
|
rendered,
|
||||||
if snippet == excerpt or match.group(0) in excerpt:
|
)
|
||||||
occupied.add((match.start(), match.end(), label))
|
if span:
|
||||||
break
|
occupied.add((span[0], span[1], (item.get("text") or "").casefold()))
|
||||||
extra_mappings = list(mappings)
|
extra_mappings = list(mappings)
|
||||||
extra_candidates = list(candidates)
|
extra_candidates = list(candidates)
|
||||||
for match in _word_matches(user_body):
|
for match in _word_matches(user_body):
|
||||||
|
|
@ -232,6 +422,11 @@ def supplement_kinship_candidates(
|
||||||
user_key = (match.start(), match.end(), label.casefold())
|
user_key = (match.start(), match.end(), label.casefold())
|
||||||
if user_key in occupied:
|
if user_key in occupied:
|
||||||
continue
|
continue
|
||||||
|
if (
|
||||||
|
decision_for_mention(profile_id, label, user_body, match.start(), match.end(), rendered)
|
||||||
|
!= _DECISION_ASK
|
||||||
|
):
|
||||||
|
continue
|
||||||
occupied.add(user_key)
|
occupied.add(user_key)
|
||||||
start, end = _align_user_span(rendered, user_body, match.start(), match.end())
|
start, end = _align_user_span(rendered, user_body, match.start(), match.end())
|
||||||
identity_shaped = _prev_word(user_body, match.start()) in KINSHIP
|
identity_shaped = _prev_word(user_body, match.start()) in KINSHIP
|
||||||
|
|
@ -242,7 +437,7 @@ def supplement_kinship_candidates(
|
||||||
"entity_type": "PERSON",
|
"entity_type": "PERSON",
|
||||||
"start": start,
|
"start": start,
|
||||||
"end": end,
|
"end": end,
|
||||||
"excerpt": _excerpt(user_body, match.start(), match.end(), label),
|
**_mention_fields(user_body, match.start(), match.end(), label, rendered),
|
||||||
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||||
"suggested": _DECISION_IDENTITY if identity_shaped else _DECISION_NOT,
|
"suggested": _DECISION_IDENTITY if identity_shaped else _DECISION_NOT,
|
||||||
}
|
}
|
||||||
|
|
@ -264,11 +459,11 @@ def supplement_kinship_candidates(
|
||||||
return extra_candidates, extra_mappings
|
return extra_candidates, extra_mappings
|
||||||
|
|
||||||
|
|
||||||
def build_candidates(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
|
def build_candidates(profile_id: str, mappings: list[dict], user_body: str, rendered: str = "") -> list[dict]:
|
||||||
seen: set[tuple[int | None, int | None, str]] = set()
|
seen: set[tuple[int | None, int | None, str]] = set()
|
||||||
items: list[dict] = []
|
items: list[dict] = []
|
||||||
for mapping in mappings:
|
for mapping in mappings:
|
||||||
if not needs_review(profile_id, mapping, user_body):
|
if not needs_review(profile_id, mapping, user_body, rendered):
|
||||||
continue
|
continue
|
||||||
label = mapping.get("local_label") or ""
|
label = mapping.get("local_label") or ""
|
||||||
start = mapping.get("start")
|
start = mapping.get("start")
|
||||||
|
|
@ -284,7 +479,7 @@ def build_candidates(profile_id: str, mappings: list[dict], user_body: str) -> l
|
||||||
"entity_type": mapping.get("entity_type") or "PERSON",
|
"entity_type": mapping.get("entity_type") or "PERSON",
|
||||||
"start": start,
|
"start": start,
|
||||||
"end": end,
|
"end": end,
|
||||||
"excerpt": _excerpt(user_body, start, end, label),
|
**_mention_fields(user_body, start, end, label, rendered),
|
||||||
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
@ -369,11 +564,19 @@ def apply_review_decisions(profile_id: str, pending: dict, decisions: list[dict]
|
||||||
decision = (raw.get("decision") or "").strip()
|
decision = (raw.get("decision") or "").strip()
|
||||||
label = candidate.get("text") or ""
|
label = candidate.get("text") or ""
|
||||||
kind = candidate.get("entity_type") if candidate.get("entity_type") in ENTITY_TYPES else "PERSON"
|
kind = candidate.get("entity_type") if candidate.get("entity_type") in ENTITY_TYPES else "PERSON"
|
||||||
|
cue = candidate.get("cue") or cue_for_mention(
|
||||||
|
pending.get("user_body") or "",
|
||||||
|
candidate.get("user_start", candidate.get("start")),
|
||||||
|
candidate.get("user_end", candidate.get("end")),
|
||||||
|
label,
|
||||||
|
)
|
||||||
if decision == _DECISION_IDENTITY:
|
if decision == _DECISION_IDENTITY:
|
||||||
record_sense(profile_id, label, identity=True)
|
record_sense(profile_id, label, identity=True)
|
||||||
|
record_cue(profile_id, label, cue, _DECISION_IDENTITY)
|
||||||
confirm_identity(profile_id, label, entity_type=kind)
|
confirm_identity(profile_id, label, entity_type=kind)
|
||||||
elif decision == _DECISION_NOT:
|
elif decision == _DECISION_NOT:
|
||||||
record_sense(profile_id, label, identity=False)
|
record_sense(profile_id, label, identity=False)
|
||||||
|
record_cue(profile_id, label, cue, _DECISION_NOT)
|
||||||
drop_keys.add((candidate.get("start"), candidate.get("end"), label.casefold()))
|
drop_keys.add((candidate.get("start"), candidate.get("end"), label.casefold()))
|
||||||
mappings = []
|
mappings = []
|
||||||
for mapping in pending.get("mappings") or []:
|
for mapping in pending.get("mappings") or []:
|
||||||
|
|
@ -459,9 +662,11 @@ def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: li
|
||||||
decision = try_local_passage_decision(item.get("excerpt") or "", label)
|
decision = try_local_passage_decision(item.get("excerpt") or "", label)
|
||||||
if decision == _DECISION_NOT:
|
if decision == _DECISION_NOT:
|
||||||
record_sense(profile_id, label, identity=False)
|
record_sense(profile_id, label, identity=False)
|
||||||
|
record_cue(profile_id, label, item.get("cue") or _CUE_BARE, _DECISION_NOT)
|
||||||
drop_keys.add((item.get("start"), item.get("end"), label.casefold()))
|
drop_keys.add((item.get("start"), item.get("end"), label.casefold()))
|
||||||
elif decision == _DECISION_IDENTITY:
|
elif decision == _DECISION_IDENTITY:
|
||||||
record_sense(profile_id, label, identity=True)
|
record_sense(profile_id, label, identity=True)
|
||||||
|
record_cue(profile_id, label, item.get("cue") or _CUE_BARE, _DECISION_IDENTITY)
|
||||||
else:
|
else:
|
||||||
remaining.append(item)
|
remaining.append(item)
|
||||||
if not drop_keys:
|
if not drop_keys:
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ from conversation_signals import infer_signals
|
||||||
from debug_store import persist_engine_error, persist_step
|
from debug_store import persist_engine_error, persist_step
|
||||||
from detect_learning import (
|
from detect_learning import (
|
||||||
MODE_LEARNING,
|
MODE_LEARNING,
|
||||||
|
apply_learned_decisions,
|
||||||
apply_review_decisions,
|
apply_review_decisions,
|
||||||
auto_resolve_ambiguous,
|
auto_resolve_ambiguous,
|
||||||
build_candidates,
|
build_candidates,
|
||||||
|
|
@ -439,10 +440,13 @@ def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled
|
||||||
return None
|
return None
|
||||||
user_body = user.get("body") or ""
|
user_body = user.get("body") or ""
|
||||||
mappings = list(outcome.mappings or [])
|
mappings = list(outcome.mappings or [])
|
||||||
candidates = build_candidates(profile_id, mappings, user_body)
|
candidates = build_candidates(profile_id, mappings, user_body, rendered)
|
||||||
candidates, mappings = supplement_kinship_candidates(
|
candidates, mappings = supplement_kinship_candidates(
|
||||||
profile_id, candidates, mappings, user_body, rendered
|
profile_id, candidates, mappings, user_body, rendered
|
||||||
)
|
)
|
||||||
|
candidates, mappings = apply_learned_decisions(
|
||||||
|
profile_id, candidates, mappings, user_body, rendered
|
||||||
|
)
|
||||||
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
|
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
|
||||||
if not candidates:
|
if not candidates:
|
||||||
confirm_known_identity_spans(profile_id, mappings, user_body)
|
confirm_known_identity_spans(profile_id, mappings, user_body)
|
||||||
|
|
|
||||||
12
backend/migrations/025_label_sense_cues.sql
Normal file
12
backend/migrations/025_label_sense_cues.sql
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
-- Remember mask-review decisions per local context cue (previous word).
|
||||||
|
-- Not a word list: cues come from the user line, e.g. Frau vs. a food mention.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS label_sense_cues (
|
||||||
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
normalized_label TEXT NOT NULL,
|
||||||
|
cue TEXT NOT NULL,
|
||||||
|
decision TEXT NOT NULL,
|
||||||
|
hits INTEGER NOT NULL DEFAULT 1,
|
||||||
|
updated TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text,
|
||||||
|
PRIMARY KEY (profile_id, normalized_label, cue)
|
||||||
|
);
|
||||||
|
|
@ -239,6 +239,16 @@ CREATE TABLE IF NOT EXISTS label_senses (
|
||||||
PRIMARY KEY (profile_id, normalized_label)
|
PRIMARY KEY (profile_id, normalized_label)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS label_sense_cues (
|
||||||
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
normalized_label TEXT NOT NULL,
|
||||||
|
cue TEXT NOT NULL,
|
||||||
|
decision TEXT NOT NULL,
|
||||||
|
hits INTEGER NOT NULL DEFAULT 1,
|
||||||
|
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
PRIMARY KEY (profile_id, normalized_label, cue)
|
||||||
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
|
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,7 @@ TABLES = [
|
||||||
"identity_mappings",
|
"identity_mappings",
|
||||||
"identity_review_proposals",
|
"identity_review_proposals",
|
||||||
"label_senses",
|
"label_senses",
|
||||||
|
"label_sense_cues",
|
||||||
"pending_mask_reviews",
|
"pending_mask_reviews",
|
||||||
"journal_days",
|
"journal_days",
|
||||||
"journal_drafts",
|
"journal_drafts",
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,13 @@ os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from detect_learning import get_detect_operating_mode, get_sense, record_sense
|
from detect_learning import (
|
||||||
|
excerpt_view,
|
||||||
|
get_cue_decision,
|
||||||
|
get_detect_operating_mode,
|
||||||
|
get_sense,
|
||||||
|
record_sense,
|
||||||
|
)
|
||||||
from dialogue_turn import visible_for_role
|
from dialogue_turn import visible_for_role
|
||||||
from identity_store import list_confirmed_identities
|
from identity_store import list_confirmed_identities
|
||||||
from main import app
|
from main import app
|
||||||
|
|
@ -161,6 +167,13 @@ def main() -> None:
|
||||||
status_ok(declined, "not_identity review")
|
status_ok(declined, "not_identity review")
|
||||||
expect(not any((item.get("canonical_label") or "").casefold() == "hanna" for item in list_confirmed_identities(profile_id)), "not_identity does not confirm Hanna")
|
expect(not any((item.get("canonical_label") or "").casefold() == "hanna" for item in list_confirmed_identities(profile_id)), "not_identity does not confirm Hanna")
|
||||||
expect(get_sense(profile_id, "Hanna")["non_identity_hits"] >= 1, "not_identity records a sense")
|
expect(get_sense(profile_id, "Hanna")["non_identity_hits"] >= 1, "not_identity records a sense")
|
||||||
|
hanna_again = client.post(
|
||||||
|
f"/api/journal/conversations/{food_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Hanna aß mit uns."},
|
||||||
|
)
|
||||||
|
status_ok(hanna_again, "second Hanna turn")
|
||||||
|
expect(not hanna_again.json().get("pending_mask_review"), "known non-identity skips the popup")
|
||||||
|
|
||||||
record_sense(profile_id, "Clarissa", identity=True)
|
record_sense(profile_id, "Clarissa", identity=True)
|
||||||
record_sense(profile_id, "Clarissa", identity=False)
|
record_sense(profile_id, "Clarissa", identity=False)
|
||||||
|
|
@ -180,12 +193,30 @@ def main() -> None:
|
||||||
kin = client.post(
|
kin = client.post(
|
||||||
f"/api/journal/conversations/{kin_conv}/turn",
|
f"/api/journal/conversations/{kin_conv}/turn",
|
||||||
headers=headers,
|
headers=headers,
|
||||||
json={"body": "Ich war mit meinem Sohn Rohan im Park."},
|
json={"body": "Ich war mit meinem Sohn Leon im Park."},
|
||||||
)
|
)
|
||||||
status_ok(kin, "kinship Rohan pause")
|
status_ok(kin, "kinship Leon pause")
|
||||||
kin_review = kin.json().get("pending_mask_review") or {}
|
kin_review = kin.json().get("pending_mask_review") or {}
|
||||||
kin_names = [item.get("text") for item in kin_review.get("candidates") or []]
|
kin_names = [item.get("text") for item in kin_review.get("candidates") or []]
|
||||||
expect("Rohan" in kin_names, "Sohn Rohan is offered even when Detect does not report it")
|
expect("Leon" in kin_names, "Sohn Leon is offered even when Detect does not report it")
|
||||||
|
leon = next((item for item in kin_review.get("candidates") or [] if item.get("text") == "Leon"), None)
|
||||||
|
expect(leon is not None, "Leon candidate id is present")
|
||||||
|
kin_done = client.post(
|
||||||
|
f"/api/journal/conversations/{kin_conv}/turn/review",
|
||||||
|
headers=headers,
|
||||||
|
json={
|
||||||
|
"review_id": kin_review["id"],
|
||||||
|
"decisions": [{"id": leon["id"], "decision": "identity"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
status_ok(kin_done, "Leon identity review")
|
||||||
|
kin_again = client.post(
|
||||||
|
f"/api/journal/conversations/{kin_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Ich war mit meinem Sohn Leon im Park."},
|
||||||
|
)
|
||||||
|
status_ok(kin_again, "second Leon turn")
|
||||||
|
expect(not kin_again.json().get("pending_mask_review"), "confirmed Sohn Leon skips the popup")
|
||||||
|
|
||||||
sushi_conv = open_conv(client, headers)
|
sushi_conv = open_conv(client, headers)
|
||||||
sushi = client.post(
|
sushi = client.post(
|
||||||
|
|
@ -203,6 +234,73 @@ def main() -> None:
|
||||||
expect("Rohan" in sushi_names, "Rohan from Sohn is a candidate")
|
expect("Rohan" in sushi_names, "Rohan from Sohn is a candidate")
|
||||||
expect("Sushi" in sushi_names, "Sushi from Frau is a candidate")
|
expect("Sushi" in sushi_names, "Sushi from Frau is a candidate")
|
||||||
expect("Restaurant" not in sushi_names, "plain Restaurant is not kinship-offered")
|
expect("Restaurant" not in sushi_names, "plain Restaurant is not kinship-offered")
|
||||||
|
sushi_hits = [
|
||||||
|
item
|
||||||
|
for item in (sushi.json().get("pending_mask_review") or {}).get("candidates") or []
|
||||||
|
if item.get("text") == "Sushi"
|
||||||
|
]
|
||||||
|
expect(len(sushi_hits) >= 2, "both Sushi mentions are offered")
|
||||||
|
expect(
|
||||||
|
sushi_hits[0].get("highlight_start") != sushi_hits[1].get("highlight_start"),
|
||||||
|
"the two Sushi mentions highlight different offsets",
|
||||||
|
)
|
||||||
|
for item in sushi_hits:
|
||||||
|
excerpt = item.get("excerpt") or ""
|
||||||
|
start = item.get("highlight_start")
|
||||||
|
end = item.get("highlight_end")
|
||||||
|
expect(excerpt[start:end] == "Sushi", "highlight covers the Sushi token")
|
||||||
|
|
||||||
|
homonym = (
|
||||||
|
"Gestern bin ich mit meiner Frau Sushi und meinem Sohn Rohan "
|
||||||
|
"Sushi essen gegangen."
|
||||||
|
)
|
||||||
|
first_at = homonym.index("Sushi")
|
||||||
|
second_at = homonym.rindex("Sushi")
|
||||||
|
first = excerpt_view(homonym, first_at, first_at + 5, "Sushi")
|
||||||
|
second = excerpt_view(homonym, second_at, second_at + 5, "Sushi")
|
||||||
|
expect(first["highlight_start"] < second["highlight_start"], "person Sushi sits left of dish Sushi")
|
||||||
|
expect(first["excerpt"][first["highlight_start"]:first["highlight_end"]] == "Sushi", "first highlight is Sushi")
|
||||||
|
expect(second["excerpt"][second["highlight_start"]:second["highlight_end"]] == "Sushi", "second highlight is Sushi")
|
||||||
|
prefix = "SYSTEM\n"
|
||||||
|
mapped = excerpt_view(
|
||||||
|
homonym,
|
||||||
|
len(prefix) + second_at,
|
||||||
|
len(prefix) + second_at + 5,
|
||||||
|
"Sushi",
|
||||||
|
prefix + homonym,
|
||||||
|
)
|
||||||
|
expect(mapped["highlight_start"] == second["highlight_start"], "rendered detect offsets map to the dish Sushi")
|
||||||
|
|
||||||
|
sushi_review = sushi.json().get("pending_mask_review") or {}
|
||||||
|
sushi_decisions = []
|
||||||
|
for item in sushi_review.get("candidates") or []:
|
||||||
|
label = item.get("text")
|
||||||
|
cue = item.get("cue")
|
||||||
|
if label == "Sushi" and cue != "frau":
|
||||||
|
sushi_decisions.append({"id": item["id"], "decision": "not_identity"})
|
||||||
|
else:
|
||||||
|
sushi_decisions.append({"id": item["id"], "decision": "identity"})
|
||||||
|
sushi_done = client.post(
|
||||||
|
f"/api/journal/conversations/{sushi_conv}/turn/review",
|
||||||
|
headers=headers,
|
||||||
|
json={"review_id": sushi_review["id"], "decisions": sushi_decisions},
|
||||||
|
)
|
||||||
|
status_ok(sushi_done, "homonym review")
|
||||||
|
expect(get_cue_decision(profile_id, "Sushi", "frau") == "identity", "Frau Sushi is stored as identity")
|
||||||
|
expect(get_cue_decision(profile_id, "Sushi", "rohan") == "not_identity", "dish Sushi after Rohan is stored")
|
||||||
|
expect(get_cue_decision(profile_id, "Rohan", "sohn") == "identity", "Sohn Rohan cue is stored")
|
||||||
|
sushi_again = client.post(
|
||||||
|
f"/api/journal/conversations/{sushi_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={
|
||||||
|
"body": (
|
||||||
|
"Gestern bin ich mit meiner Frau Sushi und meinem Sohn Rohan "
|
||||||
|
"Sushi essen gegangen. Das Restaurant hat mich dabei total beeindruckt."
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
status_ok(sushi_again, "repeat homonym sentence")
|
||||||
|
expect(not sushi_again.json().get("pending_mask_review"), "same reviewed sentence does not re-ask")
|
||||||
|
|
||||||
back = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "semantic"})
|
back = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "semantic"})
|
||||||
expect(back.json()["detect_operating_mode"] == "semantic", "mode can return to semantic")
|
expect(back.json()["detect_operating_mode"] == "semantic", "mode can return to semantic")
|
||||||
|
|
|
||||||
|
|
@ -719,6 +719,8 @@ Additiv. Technische Abbildung: `../technical/privacy_gateway.md` §9.7.
|
||||||
|
|
||||||
**Entschieden (Übergang):** Eine Mini-Passage an ein internes Modell geht nur bei bereits mehrdeutiger Schreibweise und nur an ein lokales Detect. OpenRouter sieht diese Passage nicht. Fehlt das lokale Modell, bleibt das Popup.
|
**Entschieden (Übergang):** Eine Mini-Passage an ein internes Modell geht nur bei bereits mehrdeutiger Schreibweise und nur an ein lokales Detect. OpenRouter sieht diese Passage nicht. Fehlt das lokale Modell, bleibt das Popup.
|
||||||
|
|
||||||
|
**Additiv 2026-09-10:** Eine bestätigte Nennung wird nicht erneut gefragt, wenn derselbe lokale Kontext wiederkehrt (vorheriges Wort, z. B. `Frau` gegenüber einer Speisenennung). Das ist kein Wortlisten-Editor. Unbekannte Kontexte derselben Schreibweise bleiben prüfpflichtig. Reines Zählen von Identität/nicht-schützenswert ohne diese Anwendung ist kein Lernen.
|
||||||
|
|
||||||
**Nicht:** Gateway abschalten, Detect-Treffer auto-speichern, `Sushi_`/`Sushi+` im Nutzertext, Pattern-Wortliste als Wahrheit.
|
**Nicht:** Gateway abschalten, Detect-Treffer auto-speichern, `Sushi_`/`Sushi+` im Nutzertext, Pattern-Wortliste als Wahrheit.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
|
||||||
|
|
@ -265,6 +265,8 @@ Im Lernmodus untersucht Detect weiterhin den vollen gerenderten Generate-Egress.
|
||||||
|
|
||||||
Es gibt keine separat zu pflegende Doppeldeutigkeitsliste. Mehrdeutigkeit entsteht, wenn dieselbe Schreibweise beide Sinne hat. Nur dann darf ein **lokales** Detect-Modell eine Mini-Passage (Ausschnitt um die Nennung) entscheiden. Fehlt ein lokales Modell oder ist die Antwort unbrauchbar, bleibt das Popup. OpenRouter erhält diese Passage nicht.
|
Es gibt keine separat zu pflegende Doppeldeutigkeitsliste. Mehrdeutigkeit entsteht, wenn dieselbe Schreibweise beide Sinne hat. Nur dann darf ein **lokales** Detect-Modell eine Mini-Passage (Ausschnitt um die Nennung) entscheiden. Fehlt ein lokales Modell oder ist die Antwort unbrauchbar, bleibt das Popup. OpenRouter erhält diese Passage nicht.
|
||||||
|
|
||||||
|
**Additiv 2026-09-10:** Bestätigungen speichern zusätzlich den lokalen Cue (Wort vor der Nennung, sonst `_bare`) in `label_sense_cues`. Derselbe Cue wird angewendet statt erneut gefragt. Verwandtschafts-Nachträge (`Frau`/`Sohn`/…) respektieren dieselbe Regel. Bekannte Nur-Identität und bekannte Nur-Allgemeinbedeutung werden nicht erneut gefragt; Verwandtschaft nach einer Nur-Allgemeinbedeutung bleibt einmal prüfpflichtig, damit ein Personen-Sinn entdeckt werden kann.
|
||||||
|
|
||||||
Nach der Bestätigung läuft Generate mit den geprüften Mappings (`precomputed_learning_review`), ohne zweiten Detect-Pass. Compact-Diagnose enthält weiterhin keine Klartextlabels. Tests: `backend/tests/test_detect_learning.py`.
|
Nach der Bestätigung läuft Generate mit den geprüften Mappings (`precomputed_learning_review`), ohne zweiten Detect-Pass. Compact-Diagnose enthält weiterhin keine Klartextlabels. Tests: `backend/tests/test_detect_learning.py`.
|
||||||
|
|
||||||
## 10. Offene Fragen
|
## 10. Offene Fragen
|
||||||
|
|
|
||||||
|
|
@ -695,3 +695,16 @@ pre.code {
|
||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
gap: 0.8rem;
|
gap: 0.8rem;
|
||||||
}
|
}
|
||||||
|
.mask-excerpt {
|
||||||
|
margin: 0.35rem 0 0;
|
||||||
|
color: var(--ink);
|
||||||
|
font-size: 0.95rem;
|
||||||
|
line-height: 1.45;
|
||||||
|
}
|
||||||
|
.mask-excerpt mark {
|
||||||
|
background: #f3e2b8;
|
||||||
|
color: inherit;
|
||||||
|
font-weight: 600;
|
||||||
|
padding: 0 0.12em;
|
||||||
|
border-radius: 3px;
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,28 @@
|
||||||
|
function HighlightedExcerpt({ excerpt, highlightStart, highlightEnd, label }) {
|
||||||
|
if (!excerpt) return null
|
||||||
|
const start = Number(highlightStart)
|
||||||
|
const end = Number(highlightEnd)
|
||||||
|
if (Number.isInteger(start) && Number.isInteger(end) && start >= 0 && end <= excerpt.length && end > start) {
|
||||||
|
return (
|
||||||
|
<p className="mask-excerpt">
|
||||||
|
{excerpt.slice(0, start)}
|
||||||
|
<mark>{excerpt.slice(start, end)}</mark>
|
||||||
|
{excerpt.slice(end)}
|
||||||
|
</p>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const needle = label || ''
|
||||||
|
const at = needle ? excerpt.toLowerCase().indexOf(needle.toLowerCase()) : -1
|
||||||
|
if (at < 0) return <p className="mask-excerpt">{excerpt}</p>
|
||||||
|
return (
|
||||||
|
<p className="mask-excerpt">
|
||||||
|
{excerpt.slice(0, at)}
|
||||||
|
<mark>{excerpt.slice(at, at + needle.length)}</mark>
|
||||||
|
{excerpt.slice(at + needle.length)}
|
||||||
|
</p>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
export default function MaskReviewPanel({ review, busy, onSubmit }) {
|
export default function MaskReviewPanel({ review, busy, onSubmit }) {
|
||||||
if (!review?.candidates?.length) return null
|
if (!review?.candidates?.length) return null
|
||||||
return (
|
return (
|
||||||
|
|
@ -7,7 +32,7 @@ export default function MaskReviewPanel({ review, busy, onSubmit }) {
|
||||||
<div>
|
<div>
|
||||||
<h2 id="mask-review-title">Maskierung prüfen</h2>
|
<h2 id="mask-review-title">Maskierung prüfen</h2>
|
||||||
<p className="runlog-status">
|
<p className="runlog-status">
|
||||||
Lernmodus: nur diese Nennung. Bestätigen legt die Bezeichnung lokal ab.
|
Lernmodus: nur die gelb markierte Nennung. Bestätigen legt die Bezeichnung lokal ab.
|
||||||
„Nicht schützenswert“ verhindert die Maskierung. Beides bei demselben Wort markiert es als mehrdeutig.
|
„Nicht schützenswert“ verhindert die Maskierung. Beides bei demselben Wort markiert es als mehrdeutig.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -30,7 +55,12 @@ export default function MaskReviewPanel({ review, busy, onSubmit }) {
|
||||||
<div>
|
<div>
|
||||||
<strong>{item.text}</strong>
|
<strong>{item.text}</strong>
|
||||||
<span className="muted"> · {item.entity_type}{item.ambiguous ? ' · mehrdeutig' : ''}</span>
|
<span className="muted"> · {item.entity_type}{item.ambiguous ? ' · mehrdeutig' : ''}</span>
|
||||||
{item.excerpt && <p className="muted">{item.excerpt}</p>}
|
<HighlightedExcerpt
|
||||||
|
excerpt={item.excerpt}
|
||||||
|
highlightStart={item.highlight_start}
|
||||||
|
highlightEnd={item.highlight_end}
|
||||||
|
label={item.text}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
<fieldset className="row-actions">
|
<fieldset className="row-actions">
|
||||||
<label className="check">
|
<label className="check">
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user