Put student-first detect learning ahead of the teacher call.
Identity-only senses no longer lock a new cue. Learning stages, journal-generate review, and additive docs follow the closed section 6. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
a65ea98b0c
commit
011cc9dd81
26 files changed
+1893
-89
No files matched your search
+322
-20
@@ -1,12 +1,13 @@
|
||||
"""Transitional learning detect: senses grow from dialogue confirmations.
|
||||
|
||||
Not a global word list. Detect still does not auto-activate identities.
|
||||
Ambiguous spellings get a local passage call when a local detect provider exists.
|
||||
Stored passages are the context a later local GLiNER/detect can use for pre-decision.
|
||||
Student path: cues, kinship, homonym passage, then GLiNER. Teacher is optional.
|
||||
Uncertain mentions go as a mini-passage to a chosen LLM profile, not to Detect.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
@@ -21,10 +22,31 @@ from identity_store import (
|
||||
normalize_label,
|
||||
)
|
||||
|
||||
MODE_STUDENT = "student"
|
||||
MODE_LEARNING_FULL = "learning_full"
|
||||
MODE_LEARNING_REDUCED = "learning_reduced"
|
||||
MODE_LEARNING_MANUAL = "learning_manual"
|
||||
MODE_BYPASS = "bypass"
|
||||
MODE_SEMANTIC = "semantic"
|
||||
MODE_LEARNING = "learning"
|
||||
MODES = (MODE_SEMANTIC, MODE_LEARNING)
|
||||
CANONICAL_MODES = (
|
||||
MODE_STUDENT,
|
||||
MODE_LEARNING_FULL,
|
||||
MODE_LEARNING_REDUCED,
|
||||
MODE_LEARNING_MANUAL,
|
||||
MODE_BYPASS,
|
||||
)
|
||||
MODE_ALIASES = {
|
||||
MODE_SEMANTIC: MODE_BYPASS,
|
||||
MODE_LEARNING: MODE_LEARNING_FULL,
|
||||
}
|
||||
MODES = CANONICAL_MODES + (MODE_SEMANTIC, MODE_LEARNING)
|
||||
LEARNING_MODES = {MODE_LEARNING_FULL, MODE_LEARNING_REDUCED, MODE_LEARNING_MANUAL}
|
||||
SETTING_KEY = "detect_operating_mode"
|
||||
GLINER_ENABLED_KEY = "gliner_enabled"
|
||||
GLINER_URL_KEY = "gliner_url"
|
||||
PASSAGE_PROFILE_KEY = "passage_profile_id"
|
||||
GLINER_DEFAULT_URL = "http://192.168.2.144:8090"
|
||||
_LETTER = r"A-Za-zÄÖÜäöüß"
|
||||
WINDOW = 80
|
||||
|
||||
@@ -34,18 +56,32 @@ _DECISION_ASK = "ask"
|
||||
_CUE_BARE = "_bare"
|
||||
|
||||
|
||||
def canonicalize_detect_mode(mode: str | None) -> str | None:
|
||||
chosen = (mode or "").strip().lower()
|
||||
if not chosen:
|
||||
return None
|
||||
mapped = MODE_ALIASES.get(chosen, chosen)
|
||||
return mapped if mapped in CANONICAL_MODES else None
|
||||
|
||||
|
||||
def _default_detect_mode() -> str:
|
||||
if os.environ.get("KANSHO_FAKE_PROVIDER") or os.environ.get("KANSHO_FAKE_DETECT"):
|
||||
return MODE_BYPASS
|
||||
return MODE_LEARNING_FULL
|
||||
|
||||
|
||||
def get_detect_operating_mode() -> str:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
|
||||
)
|
||||
value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower()
|
||||
return value if value in MODES else MODE_SEMANTIC
|
||||
stored = canonicalize_detect_mode((row or {}).get("value") or "")
|
||||
return stored or _default_detect_mode()
|
||||
|
||||
|
||||
def set_detect_operating_mode(mode: str) -> str:
|
||||
chosen = (mode or "").strip().lower()
|
||||
if chosen not in MODES:
|
||||
chosen = canonicalize_detect_mode(mode)
|
||||
if not chosen:
|
||||
raise ValueError("invalid_detect_operating_mode")
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
@@ -59,6 +95,88 @@ def set_detect_operating_mode(mode: str) -> str:
|
||||
return chosen
|
||||
|
||||
|
||||
def is_bypass_mode(mode: str | None = None) -> bool:
|
||||
return (mode or get_detect_operating_mode()) == MODE_BYPASS
|
||||
|
||||
|
||||
def is_learning_mode(mode: str | None = None) -> bool:
|
||||
return (mode or get_detect_operating_mode()) in LEARNING_MODES
|
||||
|
||||
|
||||
def learning_stage(mode: str | None = None) -> int:
|
||||
current = mode or get_detect_operating_mode()
|
||||
if current == MODE_LEARNING_REDUCED:
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
def _app_setting(key: str, default: str = "") -> str:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT value FROM app_settings WHERE key = ?", (key,)).fetchone()
|
||||
)
|
||||
if not row or row.get("value") is None:
|
||||
return default
|
||||
return str(row.get("value") or "").strip() or default
|
||||
|
||||
|
||||
def _put_app_setting(key: str, value: str) -> None:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO app_settings (key, value, updated)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
|
||||
""",
|
||||
(key, value),
|
||||
)
|
||||
|
||||
|
||||
def get_local_ner_settings() -> dict:
|
||||
from gliner_client import health, normalize_base_url
|
||||
from providers import is_local_url
|
||||
|
||||
enabled_raw = _app_setting(GLINER_ENABLED_KEY, "0").lower()
|
||||
enabled = enabled_raw in {"1", "true", "yes", "on"}
|
||||
url = normalize_base_url(_app_setting(GLINER_URL_KEY, GLINER_DEFAULT_URL) or GLINER_DEFAULT_URL)
|
||||
passage_profile_id = _app_setting(PASSAGE_PROFILE_KEY, "")
|
||||
ping = health(url) if enabled and is_local_url(url) else {"ok": False, "status": "off"}
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"url": url,
|
||||
"passage_profile_id": passage_profile_id,
|
||||
"ready": bool(enabled and ping.get("ok")),
|
||||
"health": ping,
|
||||
}
|
||||
|
||||
|
||||
def set_local_ner_settings(
|
||||
*,
|
||||
enabled: bool | None = None,
|
||||
url: str | None = None,
|
||||
passage_profile_id: str | None = None,
|
||||
) -> dict:
|
||||
from gliner_client import normalize_base_url
|
||||
from providers import is_local_url
|
||||
|
||||
if enabled is not None:
|
||||
_put_app_setting(GLINER_ENABLED_KEY, "1" if enabled else "0")
|
||||
if url is not None:
|
||||
chosen = normalize_base_url(url) if (url or "").strip() else GLINER_DEFAULT_URL
|
||||
if not is_local_url(chosen):
|
||||
raise ValueError("gliner_url_not_local")
|
||||
_put_app_setting(GLINER_URL_KEY, chosen)
|
||||
if passage_profile_id is not None:
|
||||
token = (passage_profile_id or "").strip()
|
||||
if token:
|
||||
from provider_settings import get_profile
|
||||
|
||||
if not get_profile(token):
|
||||
raise ValueError("passage_profile_missing")
|
||||
_put_app_setting(PASSAGE_PROFILE_KEY, token)
|
||||
return get_local_ner_settings()
|
||||
|
||||
|
||||
def get_sense(profile_id: str, label: str) -> dict:
|
||||
key = normalize_label(label) or (label or "").strip()
|
||||
if not key:
|
||||
@@ -376,21 +494,16 @@ def decision_for_mention(
|
||||
end: int | None,
|
||||
rendered: str = "",
|
||||
) -> str:
|
||||
"""identity / not_identity from prior review, or ask once for a new cue."""
|
||||
"""identity / not_identity from a stored cue, otherwise ask.
|
||||
|
||||
A sense with only identity hits must not lock a new cue (Sushi day 21).
|
||||
"""
|
||||
if not label:
|
||||
return _DECISION_ASK
|
||||
cue = cue_for_mention(user_body, start, end, label, rendered)
|
||||
learned = get_cue_decision(profile_id, label, cue)
|
||||
if learned:
|
||||
return learned
|
||||
sense = get_sense(profile_id, label)
|
||||
identity_shaped = cue in KINSHIP
|
||||
if sense["ambiguous"]:
|
||||
return _DECISION_ASK
|
||||
if sense["identity_hits"] > 0 and not sense["non_identity_hits"]:
|
||||
return _DECISION_IDENTITY
|
||||
if sense["non_identity_hits"] > 0 and not sense["identity_hits"]:
|
||||
return _DECISION_ASK if identity_shaped else _DECISION_NOT
|
||||
return _DECISION_ASK
|
||||
|
||||
|
||||
@@ -533,6 +646,193 @@ def supplement_kinship_candidates(
|
||||
return extra_candidates, extra_mappings
|
||||
|
||||
|
||||
def merge_gliner_candidates(
|
||||
profile_id: str,
|
||||
candidates: list[dict],
|
||||
mappings: list[dict],
|
||||
user_body: str,
|
||||
rendered: str,
|
||||
spans: list[dict] | None,
|
||||
) -> tuple[list[dict], list[dict]]:
|
||||
"""Add GLiNER spans from the user line. Learned cues still win afterwards."""
|
||||
from gliner_client import gliner_identity_type, is_gliner_non_identity
|
||||
|
||||
extra_candidates = list(candidates)
|
||||
extra_mappings = list(mappings)
|
||||
if not spans:
|
||||
return extra_candidates, extra_mappings
|
||||
occupied: set[tuple[int, int, str]] = set()
|
||||
for item in extra_candidates:
|
||||
span = _resolve_user_span(
|
||||
user_body,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if span:
|
||||
occupied.add((span[0], span[1], (item.get("text") or "").casefold()))
|
||||
food_keys: set[tuple[int, int, str]] = set()
|
||||
for raw in spans:
|
||||
label = raw.get("text") or ""
|
||||
kind = raw.get("label") or ""
|
||||
try:
|
||||
start, end = int(raw["start"]), int(raw["end"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
if start < 0 or end > len(user_body) or end <= start:
|
||||
continue
|
||||
if user_body[start:end].casefold() != label.casefold():
|
||||
resolved = _resolve_user_span(user_body, start, end, label, rendered)
|
||||
if not resolved:
|
||||
continue
|
||||
start, end = resolved
|
||||
label = user_body[start:end]
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
key = (start, end, label.casefold())
|
||||
if is_gliner_non_identity(kind):
|
||||
food_keys.add(key)
|
||||
continue
|
||||
entity_type = gliner_identity_type(kind)
|
||||
if not entity_type or key in occupied:
|
||||
continue
|
||||
if decision_for_mention(profile_id, label, user_body, start, end, rendered) != _DECISION_ASK:
|
||||
continue
|
||||
occupied.add(key)
|
||||
aligned = _align_user_span(rendered, user_body, start, end)
|
||||
extra_candidates.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": entity_type,
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(user_body, start, end, label, rendered),
|
||||
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||
"suggested": _DECISION_IDENTITY,
|
||||
}
|
||||
)
|
||||
extra_mappings.append(
|
||||
{
|
||||
"token": _token_for_label(extra_mappings, label, entity_type),
|
||||
"local_label": label,
|
||||
"canonical_label": label,
|
||||
"demask_label": label,
|
||||
"entity_type": entity_type,
|
||||
"source": "request_local",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
"aliases": [],
|
||||
"labels": [label],
|
||||
}
|
||||
)
|
||||
for item in extra_candidates:
|
||||
span = _resolve_user_span(
|
||||
user_body,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if not span:
|
||||
continue
|
||||
key = (span[0], span[1], (item.get("text") or "").casefold())
|
||||
if key in food_keys and not attachment_kinship(user_body, span[0], span[1]):
|
||||
item["suggested"] = _DECISION_NOT
|
||||
for start, end, folded in food_keys:
|
||||
if (start, end, folded) in occupied:
|
||||
continue
|
||||
label = user_body[start:end]
|
||||
if decision_for_mention(profile_id, label, user_body, start, end, rendered) != _DECISION_ASK:
|
||||
continue
|
||||
occupied.add((start, end, folded))
|
||||
aligned = _align_user_span(rendered, user_body, start, end)
|
||||
extra_candidates.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": "PERSON",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(user_body, start, end, label, rendered),
|
||||
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||
"suggested": _DECISION_NOT,
|
||||
}
|
||||
)
|
||||
extra_mappings.append(
|
||||
{
|
||||
"token": _token_for_label(extra_mappings, label),
|
||||
"local_label": label,
|
||||
"canonical_label": label,
|
||||
"demask_label": label,
|
||||
"entity_type": "PERSON",
|
||||
"source": "request_local",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
"aliases": [],
|
||||
"labels": [label],
|
||||
}
|
||||
)
|
||||
return extra_candidates, extra_mappings
|
||||
|
||||
|
||||
def apply_gliner_if_enabled(
|
||||
profile_id: str,
|
||||
candidates: list[dict],
|
||||
mappings: list[dict],
|
||||
user_body: str,
|
||||
rendered: str = "",
|
||||
) -> tuple[list[dict], list[dict]]:
|
||||
settings = get_local_ner_settings()
|
||||
if not settings.get("enabled"):
|
||||
return candidates, mappings
|
||||
from gliner_client import extract_entities
|
||||
|
||||
spans = extract_entities(user_body, url=settings.get("url"))
|
||||
if spans is None:
|
||||
return candidates, mappings
|
||||
return merge_gliner_candidates(profile_id, candidates, mappings, user_body, rendered, spans)
|
||||
|
||||
|
||||
def passage_provider():
|
||||
"""LLM profile for mini-passage decisions. Independent of the detect role."""
|
||||
import os
|
||||
|
||||
from providers import ProviderConfig, is_local_url
|
||||
from provider_settings import get_profile
|
||||
|
||||
profile_id = _app_setting(PASSAGE_PROFILE_KEY, "")
|
||||
if not profile_id:
|
||||
return None
|
||||
profile = get_profile(profile_id)
|
||||
if not profile:
|
||||
return None
|
||||
url = profile.get("url") or ""
|
||||
model = profile.get("model") or ""
|
||||
if not url or not model:
|
||||
return None
|
||||
local = is_local_url(url)
|
||||
key = ""
|
||||
if not local:
|
||||
key = (os.environ.get("KANSHO_PROVIDER_KEY") or os.environ.get("KANSHO_DETECT_PROVIDER_KEY") or "").strip()
|
||||
if not key:
|
||||
return None
|
||||
if not profile.get("zdr") or not profile.get("no_train"):
|
||||
return None
|
||||
return ProviderConfig(
|
||||
role="passage",
|
||||
name=profile.get("name") or ("ollama" if local else "openrouter"),
|
||||
mode="http",
|
||||
url=url,
|
||||
model=model,
|
||||
key=key,
|
||||
local=local,
|
||||
zdr=True if local else bool(profile.get("zdr")),
|
||||
no_train=True if local else bool(profile.get("no_train")),
|
||||
)
|
||||
|
||||
|
||||
def build_candidates(profile_id: str, mappings: list[dict], user_body: str, rendered: str = "") -> list[dict]:
|
||||
seen: set[tuple[int | None, int | None, str]] = set()
|
||||
items: list[dict] = []
|
||||
@@ -721,6 +1021,8 @@ def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | No
|
||||
return {
|
||||
"id": row["id"],
|
||||
"candidates": payload.get("candidates") or [],
|
||||
"kind": payload.get("kind") or "diff",
|
||||
"purpose": payload.get("purpose") or "dialogue_turn",
|
||||
}
|
||||
|
||||
|
||||
@@ -756,11 +1058,11 @@ def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: li
|
||||
|
||||
|
||||
def try_local_passage_decision(excerpt: str, label: str, examples: list[dict] | None = None) -> str | None:
|
||||
"""Return identity, not_identity, or None if no local model or unusable answer."""
|
||||
from providers import ProviderError, complete_chat, detect_provider
|
||||
"""Return identity, not_identity, or None. Uses the admin passage profile, not Detect."""
|
||||
from providers import ProviderError, complete_chat
|
||||
|
||||
config = detect_provider()
|
||||
if not config or not config.local or config.mode != "http":
|
||||
config = passage_provider()
|
||||
if not config or config.mode != "http":
|
||||
return None
|
||||
few_shot = ""
|
||||
lines = []
|
||||
|
||||
Reference in new issue
Block a user