Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a65ea98b0c | |||
| f7ffd28332 | |||
| 3f7c0d71a5 | |||
| cc43fc6409 | |||
| fa1c11c33e | |||
| 59921fc5b5 | |||
| bef422a429 |
|
|
@ -14,8 +14,8 @@ KANSHO_PROVIDER_KEY=
|
||||||
# KANSHO_PROVIDER_ZDR=1
|
# KANSHO_PROVIDER_ZDR=1
|
||||||
# KANSHO_PROVIDER_NO_TRAIN=1
|
# KANSHO_PROVIDER_NO_TRAIN=1
|
||||||
|
|
||||||
# Detect: Klartext lokal, in Development/Test, oder in Production mit KANSHO_ALLOW_REMOTE_DETECT.
|
# Detect: lokal (Ollama LAN) oder in Development/Test remote. Production remote nur mit KANSHO_ALLOW_REMOTE_DETECT.
|
||||||
# Leer = Generate-Key mitnutzen (nur wenn remote Detect erlaubt ist).
|
# URL/Modell liegen in LLM-Profilen (Admin → Schnittstellen), nicht in der .env.
|
||||||
KANSHO_DETECT_PROVIDER_KEY=
|
KANSHO_DETECT_PROVIDER_KEY=
|
||||||
# KANSHO_DETECT_PROVIDER_URL=https://openrouter.ai/api/v1/chat/completions
|
# KANSHO_DETECT_PROVIDER_URL=https://openrouter.ai/api/v1/chat/completions
|
||||||
# KANSHO_DETECT_PROVIDER_MODEL=openai/gpt-4.1-nano
|
# KANSHO_DETECT_PROVIDER_MODEL=openai/gpt-4.1-nano
|
||||||
|
|
|
||||||
|
|
@ -580,6 +580,7 @@ def init_db() -> None:
|
||||||
_seed_prompts(conn)
|
_seed_prompts(conn)
|
||||||
from provider_settings import seed_provider_settings
|
from provider_settings import seed_provider_settings
|
||||||
|
|
||||||
|
_ensure_columns(conn, "provider_settings", {"profile_id": "TEXT"})
|
||||||
seed_provider_settings(conn)
|
seed_provider_settings(conn)
|
||||||
_ensure_columns(conn, "profiles", _PROFILE_COLUMNS)
|
_ensure_columns(conn, "profiles", _PROFILE_COLUMNS)
|
||||||
_ensure_columns(conn, "conversations", _CONVERSATION_COLUMNS)
|
_ensure_columns(conn, "conversations", _CONVERSATION_COLUMNS)
|
||||||
|
|
@ -593,6 +594,7 @@ def init_db() -> None:
|
||||||
migrate_journal_source_refs(conn)
|
migrate_journal_source_refs(conn)
|
||||||
_migrate_writing_profile_shell(conn)
|
_migrate_writing_profile_shell(conn)
|
||||||
_ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS)
|
_ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS)
|
||||||
|
_ensure_columns(conn, "label_sense_cues", {"excerpt": "TEXT NOT NULL DEFAULT ''"})
|
||||||
from identity_store import migrate_legacy_identity_rows
|
from identity_store import migrate_legacy_identity_rows
|
||||||
|
|
||||||
migrate_legacy_identity_rows(conn)
|
migrate_legacy_identity_rows(conn)
|
||||||
|
|
|
||||||
802
backend/detect_learning.py
Normal file
802
backend/detect_learning.py
Normal file
|
|
@ -0,0 +1,802 @@
|
||||||
|
"""Transitional learning detect: senses grow from dialogue confirmations.
|
||||||
|
|
||||||
|
Not a global word list. Detect still does not auto-activate identities.
|
||||||
|
Ambiguous spellings get a local passage call when a local detect provider exists.
|
||||||
|
Stored passages are the context a later local GLiNER/detect can use for pre-decision.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import uuid
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from db import get_db, row_to_dict
|
||||||
|
from identity_store import (
|
||||||
|
DETERMINERS,
|
||||||
|
ENTITY_TYPES,
|
||||||
|
KINSHIP,
|
||||||
|
confirm_identity,
|
||||||
|
is_maskable_label,
|
||||||
|
normalize_label,
|
||||||
|
)
|
||||||
|
|
||||||
|
MODE_SEMANTIC = "semantic"
|
||||||
|
MODE_LEARNING = "learning"
|
||||||
|
MODES = (MODE_SEMANTIC, MODE_LEARNING)
|
||||||
|
SETTING_KEY = "detect_operating_mode"
|
||||||
|
_LETTER = r"A-Za-zÄÖÜäöüß"
|
||||||
|
WINDOW = 80
|
||||||
|
|
||||||
|
_DECISION_IDENTITY = "identity"
|
||||||
|
_DECISION_NOT = "not_identity"
|
||||||
|
_DECISION_ASK = "ask"
|
||||||
|
_CUE_BARE = "_bare"
|
||||||
|
|
||||||
|
|
||||||
|
def get_detect_operating_mode() -> str:
|
||||||
|
with get_db() as conn:
|
||||||
|
row = row_to_dict(
|
||||||
|
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
|
||||||
|
)
|
||||||
|
value = ((row or {}).get("value") or MODE_SEMANTIC).strip().lower()
|
||||||
|
return value if value in MODES else MODE_SEMANTIC
|
||||||
|
|
||||||
|
|
||||||
|
def set_detect_operating_mode(mode: str) -> str:
|
||||||
|
chosen = (mode or "").strip().lower()
|
||||||
|
if chosen not in MODES:
|
||||||
|
raise ValueError("invalid_detect_operating_mode")
|
||||||
|
with get_db() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO app_settings (key, value, updated)
|
||||||
|
VALUES (?, ?, datetime('now'))
|
||||||
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
|
||||||
|
""",
|
||||||
|
(SETTING_KEY, chosen),
|
||||||
|
)
|
||||||
|
return chosen
|
||||||
|
|
||||||
|
|
||||||
|
def get_sense(profile_id: str, label: str) -> dict:
|
||||||
|
key = normalize_label(label) or (label or "").strip()
|
||||||
|
if not key:
|
||||||
|
return {
|
||||||
|
"normalized_label": "",
|
||||||
|
"identity_hits": 0,
|
||||||
|
"non_identity_hits": 0,
|
||||||
|
"ambiguous": False,
|
||||||
|
}
|
||||||
|
with get_db() as conn:
|
||||||
|
row = row_to_dict(
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT normalized_label, identity_hits, non_identity_hits
|
||||||
|
FROM label_senses
|
||||||
|
WHERE profile_id = ? AND lower(normalized_label) = lower(?)
|
||||||
|
""",
|
||||||
|
(profile_id, key),
|
||||||
|
).fetchone()
|
||||||
|
)
|
||||||
|
if not row:
|
||||||
|
return {
|
||||||
|
"normalized_label": key,
|
||||||
|
"identity_hits": 0,
|
||||||
|
"non_identity_hits": 0,
|
||||||
|
"ambiguous": False,
|
||||||
|
}
|
||||||
|
identity = int(row.get("identity_hits") or 0)
|
||||||
|
other = int(row.get("non_identity_hits") or 0)
|
||||||
|
return {
|
||||||
|
"normalized_label": row.get("normalized_label") or key,
|
||||||
|
"identity_hits": identity,
|
||||||
|
"non_identity_hits": other,
|
||||||
|
"ambiguous": identity > 0 and other > 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def record_sense(profile_id: str, label: str, *, identity: bool) -> dict:
|
||||||
|
key = normalize_label(label) or (label or "").strip()
|
||||||
|
if not key:
|
||||||
|
raise ValueError("empty_label")
|
||||||
|
with get_db() as conn:
|
||||||
|
existing = row_to_dict(
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT normalized_label, identity_hits, non_identity_hits FROM label_senses
|
||||||
|
WHERE profile_id = ? AND lower(normalized_label) = lower(?)
|
||||||
|
""",
|
||||||
|
(profile_id, key),
|
||||||
|
).fetchone()
|
||||||
|
)
|
||||||
|
if existing:
|
||||||
|
key = existing.get("normalized_label") or key
|
||||||
|
identity_hits = int((existing or {}).get("identity_hits") or 0)
|
||||||
|
non_identity_hits = int((existing or {}).get("non_identity_hits") or 0)
|
||||||
|
if identity:
|
||||||
|
identity_hits += 1
|
||||||
|
else:
|
||||||
|
non_identity_hits += 1
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO label_senses (
|
||||||
|
profile_id, normalized_label, identity_hits, non_identity_hits, updated
|
||||||
|
)
|
||||||
|
VALUES (?, ?, ?, ?, datetime('now'))
|
||||||
|
ON CONFLICT(profile_id, normalized_label) DO UPDATE SET
|
||||||
|
identity_hits = excluded.identity_hits,
|
||||||
|
non_identity_hits = excluded.non_identity_hits,
|
||||||
|
updated = datetime('now')
|
||||||
|
""",
|
||||||
|
(profile_id, key, identity_hits, non_identity_hits),
|
||||||
|
)
|
||||||
|
return get_sense(profile_id, key)
|
||||||
|
|
||||||
|
|
||||||
|
def _in_user_text(label: str, user_body: str) -> bool:
|
||||||
|
if not label or not user_body:
|
||||||
|
return False
|
||||||
|
return bool(
|
||||||
|
re.search(
|
||||||
|
rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])",
|
||||||
|
user_body,
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_user_span(
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
label: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> tuple[int, int] | None:
|
||||||
|
text = user_body or ""
|
||||||
|
needle = (label or "").casefold()
|
||||||
|
if start is not None and end is not None:
|
||||||
|
left, right = int(start), int(end)
|
||||||
|
if 0 <= left < right <= len(text) and text[left:right].casefold() == needle:
|
||||||
|
return left, right
|
||||||
|
if rendered and text:
|
||||||
|
origin = rendered.rfind(text)
|
||||||
|
if origin >= 0:
|
||||||
|
user_left, user_right = left - origin, right - origin
|
||||||
|
if 0 <= user_left < user_right <= len(text) and text[user_left:user_right].casefold() == needle:
|
||||||
|
return user_left, user_right
|
||||||
|
if not needle:
|
||||||
|
return None
|
||||||
|
match = re.search(rf"(?<![{_LETTER}]){re.escape(label)}(?![{_LETTER}])", text, re.IGNORECASE)
|
||||||
|
if not match:
|
||||||
|
return None
|
||||||
|
return match.start(), match.end()
|
||||||
|
|
||||||
|
|
||||||
|
def excerpt_view(
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
label: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> dict:
|
||||||
|
text = user_body or ""
|
||||||
|
span = _resolve_user_span(text, start, end, label, rendered)
|
||||||
|
if not span:
|
||||||
|
snippet = text[:WINDOW]
|
||||||
|
return {"excerpt": snippet, "highlight_start": None, "highlight_end": None}
|
||||||
|
left_i, right_i = span
|
||||||
|
clip_left = max(0, left_i - WINDOW)
|
||||||
|
clip_right = min(len(text), right_i + WINDOW)
|
||||||
|
prefix = "…" if clip_left > 0 else ""
|
||||||
|
suffix = "…" if clip_right < len(text) else ""
|
||||||
|
snippet = text[clip_left:clip_right]
|
||||||
|
return {
|
||||||
|
"excerpt": f"{prefix}{snippet}{suffix}",
|
||||||
|
"highlight_start": left_i - clip_left + len(prefix),
|
||||||
|
"highlight_end": right_i - clip_left + len(prefix),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _excerpt(user_body: str, start: int | None, end: int | None, label: str) -> str:
|
||||||
|
return excerpt_view(user_body, start, end, label)["excerpt"]
|
||||||
|
|
||||||
|
|
||||||
|
def needs_review(profile_id: str, mapping: dict, user_body: str, rendered: str = "") -> bool:
|
||||||
|
label = mapping.get("local_label") or ""
|
||||||
|
if mapping.get("source") == "confirmed_registry":
|
||||||
|
return (
|
||||||
|
_in_user_text(label, user_body)
|
||||||
|
and decision_for_mention(
|
||||||
|
profile_id, label, user_body, mapping.get("start"), mapping.get("end"), rendered
|
||||||
|
)
|
||||||
|
== _DECISION_ASK
|
||||||
|
)
|
||||||
|
if mapping.get("source") != "request_local":
|
||||||
|
return False
|
||||||
|
if not is_maskable_label(label) or not _in_user_text(label, user_body):
|
||||||
|
return False
|
||||||
|
return (
|
||||||
|
decision_for_mention(
|
||||||
|
profile_id, label, user_body, mapping.get("start"), mapping.get("end"), rendered
|
||||||
|
)
|
||||||
|
== _DECISION_ASK
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _word_matches(text: str):
|
||||||
|
return list(re.finditer(rf"[{_LETTER}]+", text or ""))
|
||||||
|
|
||||||
|
|
||||||
|
def _prev_word(text: str, index: int) -> str:
|
||||||
|
words = re.findall(rf"[{_LETTER}]+", text[:index])
|
||||||
|
return (words[-1].lower() if words else "")
|
||||||
|
|
||||||
|
|
||||||
|
def _token_index_at(tokens, start: int) -> int | None:
|
||||||
|
for index, match in enumerate(tokens):
|
||||||
|
if match.start() <= start < match.end() or match.start() == start:
|
||||||
|
return index
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def attachment_kinship(text: str, start: int, end: int) -> str:
|
||||||
|
"""Kinship in the local NP: left of the mention or right apposition, not the whole sentence."""
|
||||||
|
tokens = _word_matches(text)
|
||||||
|
index = _token_index_at(tokens, start)
|
||||||
|
if index is None:
|
||||||
|
return ""
|
||||||
|
left = index - 1
|
||||||
|
while left >= 0 and tokens[left].group(0).lower() in DETERMINERS:
|
||||||
|
left -= 1
|
||||||
|
if left >= 0 and tokens[left].group(0).lower() in KINSHIP:
|
||||||
|
return tokens[left].group(0).lower()
|
||||||
|
right = index + 1
|
||||||
|
while right < len(tokens) and tokens[right].group(0).lower() in DETERMINERS:
|
||||||
|
right += 1
|
||||||
|
if right < len(tokens) and tokens[right].group(0).lower() in KINSHIP:
|
||||||
|
return tokens[right].group(0).lower()
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def cue_for_mention(
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
label: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> str:
|
||||||
|
span = _resolve_user_span(user_body, start, end, label, rendered)
|
||||||
|
if not span:
|
||||||
|
return _CUE_BARE
|
||||||
|
attached = attachment_kinship(user_body, span[0], span[1])
|
||||||
|
if attached:
|
||||||
|
return attached
|
||||||
|
return _prev_word(user_body, span[0]) or _CUE_BARE
|
||||||
|
|
||||||
|
|
||||||
|
def _mention_fields(
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
label: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> dict:
|
||||||
|
span = _resolve_user_span(user_body, start, end, label, rendered)
|
||||||
|
return {
|
||||||
|
**excerpt_view(user_body, start, end, label, rendered),
|
||||||
|
"user_start": span[0] if span else None,
|
||||||
|
"user_end": span[1] if span else None,
|
||||||
|
"cue": cue_for_mention(user_body, start, end, label, rendered),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_cue_decision(profile_id: str, label: str, cue: str) -> str | None:
|
||||||
|
key = normalize_label(label) or (label or "").strip()
|
||||||
|
token = (cue or _CUE_BARE).strip().casefold() or _CUE_BARE
|
||||||
|
if not key:
|
||||||
|
return None
|
||||||
|
with get_db() as conn:
|
||||||
|
row = row_to_dict(
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT decision FROM label_sense_cues
|
||||||
|
WHERE profile_id = ? AND lower(normalized_label) = lower(?) AND cue = ?
|
||||||
|
""",
|
||||||
|
(profile_id, key, token),
|
||||||
|
).fetchone()
|
||||||
|
)
|
||||||
|
value = ((row or {}).get("decision") or "").strip()
|
||||||
|
return value if value in {_DECISION_IDENTITY, _DECISION_NOT} else None
|
||||||
|
|
||||||
|
|
||||||
|
def record_cue(profile_id: str, label: str, cue: str, decision: str, excerpt: str = "") -> None:
|
||||||
|
key = normalize_label(label) or (label or "").strip()
|
||||||
|
token = (cue or _CUE_BARE).strip().casefold() or _CUE_BARE
|
||||||
|
snippet = (excerpt or "").strip()
|
||||||
|
if not key or decision not in {_DECISION_IDENTITY, _DECISION_NOT}:
|
||||||
|
return
|
||||||
|
with get_db() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO label_sense_cues (
|
||||||
|
profile_id, normalized_label, cue, decision, hits, excerpt, updated
|
||||||
|
)
|
||||||
|
VALUES (?, ?, ?, ?, 1, ?, datetime('now'))
|
||||||
|
ON CONFLICT(profile_id, normalized_label, cue) DO UPDATE SET
|
||||||
|
decision = excluded.decision,
|
||||||
|
hits = label_sense_cues.hits + 1,
|
||||||
|
excerpt = CASE
|
||||||
|
WHEN excluded.excerpt <> '' THEN excluded.excerpt
|
||||||
|
ELSE label_sense_cues.excerpt
|
||||||
|
END,
|
||||||
|
updated = datetime('now')
|
||||||
|
""",
|
||||||
|
(profile_id, key, token, decision, snippet),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def list_cue_examples(profile_id: str, label: str, limit: int = 6) -> list[dict]:
|
||||||
|
key = normalize_label(label) or (label or "").strip()
|
||||||
|
if not key:
|
||||||
|
return []
|
||||||
|
with get_db() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT cue, decision, excerpt, hits
|
||||||
|
FROM label_sense_cues
|
||||||
|
WHERE profile_id = ? AND lower(normalized_label) = lower(?)
|
||||||
|
ORDER BY hits DESC, updated DESC
|
||||||
|
""",
|
||||||
|
(profile_id, key),
|
||||||
|
).fetchall()
|
||||||
|
items = []
|
||||||
|
for raw in rows:
|
||||||
|
row = row_to_dict(raw) or {}
|
||||||
|
decision = (row.get("decision") or "").strip()
|
||||||
|
if decision not in {_DECISION_IDENTITY, _DECISION_NOT}:
|
||||||
|
continue
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"cue": row.get("cue") or _CUE_BARE,
|
||||||
|
"decision": decision,
|
||||||
|
"excerpt": (row.get("excerpt") or "").strip(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if len(items) >= limit:
|
||||||
|
break
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def decision_for_mention(
|
||||||
|
profile_id: str,
|
||||||
|
label: str,
|
||||||
|
user_body: str,
|
||||||
|
start: int | None,
|
||||||
|
end: int | None,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> str:
|
||||||
|
"""identity / not_identity from prior review, or ask once for a new cue."""
|
||||||
|
if not label:
|
||||||
|
return _DECISION_ASK
|
||||||
|
cue = cue_for_mention(user_body, start, end, label, rendered)
|
||||||
|
learned = get_cue_decision(profile_id, label, cue)
|
||||||
|
if learned:
|
||||||
|
return learned
|
||||||
|
sense = get_sense(profile_id, label)
|
||||||
|
identity_shaped = cue in KINSHIP
|
||||||
|
if sense["ambiguous"]:
|
||||||
|
return _DECISION_ASK
|
||||||
|
if sense["identity_hits"] > 0 and not sense["non_identity_hits"]:
|
||||||
|
return _DECISION_IDENTITY
|
||||||
|
if sense["non_identity_hits"] > 0 and not sense["identity_hits"]:
|
||||||
|
return _DECISION_ASK if identity_shaped else _DECISION_NOT
|
||||||
|
return _DECISION_ASK
|
||||||
|
|
||||||
|
|
||||||
|
def apply_learned_decisions(
|
||||||
|
profile_id: str,
|
||||||
|
candidates: list[dict],
|
||||||
|
mappings: list[dict],
|
||||||
|
user_body: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> tuple[list[dict], list[dict]]:
|
||||||
|
remaining = []
|
||||||
|
for item in candidates:
|
||||||
|
verdict = decision_for_mention(
|
||||||
|
profile_id,
|
||||||
|
item.get("text") or "",
|
||||||
|
user_body,
|
||||||
|
item.get("user_start", item.get("start")),
|
||||||
|
item.get("user_end", item.get("end")),
|
||||||
|
rendered,
|
||||||
|
)
|
||||||
|
if verdict == _DECISION_ASK:
|
||||||
|
remaining.append(item)
|
||||||
|
kept = []
|
||||||
|
for mapping in mappings:
|
||||||
|
label = mapping.get("local_label") or ""
|
||||||
|
verdict = decision_for_mention(
|
||||||
|
profile_id,
|
||||||
|
label,
|
||||||
|
user_body,
|
||||||
|
mapping.get("start"),
|
||||||
|
mapping.get("end"),
|
||||||
|
rendered,
|
||||||
|
)
|
||||||
|
if verdict == _DECISION_NOT:
|
||||||
|
continue
|
||||||
|
kept.append(mapping)
|
||||||
|
if verdict == _DECISION_IDENTITY:
|
||||||
|
kind = mapping.get("entity_type") if mapping.get("entity_type") in ENTITY_TYPES else "PERSON"
|
||||||
|
try:
|
||||||
|
confirm_identity(profile_id, label, entity_type=kind)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return remaining, kept
|
||||||
|
|
||||||
|
|
||||||
|
def kinship_governed_labels(user_body: str) -> set[str]:
|
||||||
|
"""Labels attached to Frau/Sohn/… in the current user line. Not a food word list."""
|
||||||
|
found: set[str] = set()
|
||||||
|
for match in _word_matches(user_body):
|
||||||
|
label = match.group(0)
|
||||||
|
if not is_maskable_label(label):
|
||||||
|
continue
|
||||||
|
if attachment_kinship(user_body, match.start(), match.end()):
|
||||||
|
found.add(label.casefold())
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def _align_user_span(rendered: str, user_body: str, start: int, end: int) -> tuple[int, int]:
|
||||||
|
if not user_body:
|
||||||
|
return start, end
|
||||||
|
pos = (rendered or "").rfind(user_body)
|
||||||
|
if pos < 0:
|
||||||
|
return start, end
|
||||||
|
return pos + start, pos + end
|
||||||
|
|
||||||
|
|
||||||
|
def _token_for_label(mappings: list[dict], label: str, entity_type: str = "PERSON") -> str:
|
||||||
|
needle = label.casefold()
|
||||||
|
for mapping in mappings:
|
||||||
|
if (mapping.get("local_label") or "").casefold() == needle and mapping.get("token"):
|
||||||
|
return mapping["token"]
|
||||||
|
return f"{entity_type}:L{uuid.uuid4().hex[:6].upper()}"
|
||||||
|
|
||||||
|
|
||||||
|
def supplement_kinship_candidates(
|
||||||
|
profile_id: str,
|
||||||
|
candidates: list[dict],
|
||||||
|
mappings: list[dict],
|
||||||
|
user_body: str,
|
||||||
|
rendered: str = "",
|
||||||
|
) -> tuple[list[dict], list[dict]]:
|
||||||
|
"""If Detect misses Frau X / Sohn X, still offer those user-line mentions for review."""
|
||||||
|
wanted = kinship_governed_labels(user_body)
|
||||||
|
if not wanted:
|
||||||
|
return candidates, mappings
|
||||||
|
occupied: set[tuple[int, int, str]] = set()
|
||||||
|
for item in candidates:
|
||||||
|
span = _resolve_user_span(
|
||||||
|
user_body,
|
||||||
|
item.get("start"),
|
||||||
|
item.get("end"),
|
||||||
|
item.get("text") or "",
|
||||||
|
rendered,
|
||||||
|
)
|
||||||
|
if span:
|
||||||
|
occupied.add((span[0], span[1], (item.get("text") or "").casefold()))
|
||||||
|
extra_mappings = list(mappings)
|
||||||
|
extra_candidates = list(candidates)
|
||||||
|
for match in _word_matches(user_body):
|
||||||
|
label = match.group(0)
|
||||||
|
if label.casefold() not in wanted:
|
||||||
|
continue
|
||||||
|
user_key = (match.start(), match.end(), label.casefold())
|
||||||
|
if user_key in occupied:
|
||||||
|
continue
|
||||||
|
if (
|
||||||
|
decision_for_mention(profile_id, label, user_body, match.start(), match.end(), rendered)
|
||||||
|
!= _DECISION_ASK
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
occupied.add(user_key)
|
||||||
|
start, end = _align_user_span(rendered, user_body, match.start(), match.end())
|
||||||
|
identity_shaped = bool(attachment_kinship(user_body, match.start(), match.end()))
|
||||||
|
extra_candidates.append(
|
||||||
|
{
|
||||||
|
"id": str(uuid.uuid4()),
|
||||||
|
"text": label,
|
||||||
|
"entity_type": "PERSON",
|
||||||
|
"start": start,
|
||||||
|
"end": end,
|
||||||
|
**_mention_fields(user_body, match.start(), match.end(), label, rendered),
|
||||||
|
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||||
|
"suggested": _DECISION_IDENTITY if identity_shaped else _DECISION_NOT,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
extra_mappings.append(
|
||||||
|
{
|
||||||
|
"token": _token_for_label(extra_mappings, label),
|
||||||
|
"local_label": label,
|
||||||
|
"canonical_label": label,
|
||||||
|
"demask_label": label,
|
||||||
|
"entity_type": "PERSON",
|
||||||
|
"source": "request_local",
|
||||||
|
"start": start,
|
||||||
|
"end": end,
|
||||||
|
"aliases": [],
|
||||||
|
"labels": [label],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return extra_candidates, extra_mappings
|
||||||
|
|
||||||
|
|
||||||
|
def build_candidates(profile_id: str, mappings: list[dict], user_body: str, rendered: str = "") -> list[dict]:
|
||||||
|
seen: set[tuple[int | None, int | None, str]] = set()
|
||||||
|
items: list[dict] = []
|
||||||
|
for mapping in mappings:
|
||||||
|
if not needs_review(profile_id, mapping, user_body, rendered):
|
||||||
|
continue
|
||||||
|
label = mapping.get("local_label") or ""
|
||||||
|
start = mapping.get("start")
|
||||||
|
end = mapping.get("end")
|
||||||
|
key = (start, end, label.casefold())
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"id": str(uuid.uuid4()),
|
||||||
|
"text": label,
|
||||||
|
"entity_type": mapping.get("entity_type") or "PERSON",
|
||||||
|
"start": start,
|
||||||
|
"end": end,
|
||||||
|
**_mention_fields(user_body, start, end, label, rendered),
|
||||||
|
"ambiguous": get_sense(profile_id, label)["ambiguous"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def suppress_known_non_identity(profile_id: str, mappings: list[dict], user_body: str) -> list[dict]:
|
||||||
|
kept = []
|
||||||
|
for mapping in mappings:
|
||||||
|
label = mapping.get("local_label") or ""
|
||||||
|
sense = get_sense(profile_id, label)
|
||||||
|
if (
|
||||||
|
mapping.get("source") == "request_local"
|
||||||
|
and sense["non_identity_hits"] > 0
|
||||||
|
and not sense["identity_hits"]
|
||||||
|
and _in_user_text(label, user_body)
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
kept.append(mapping)
|
||||||
|
return kept
|
||||||
|
|
||||||
|
|
||||||
|
def save_pending(
|
||||||
|
profile_id: str,
|
||||||
|
conversation_id: str,
|
||||||
|
user_message_id: str,
|
||||||
|
payload: dict,
|
||||||
|
) -> str:
|
||||||
|
review_id = str(uuid.uuid4())
|
||||||
|
with get_db() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM pending_mask_reviews WHERE profile_id = ? AND conversation_id = ?",
|
||||||
|
(profile_id, conversation_id),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO pending_mask_reviews (
|
||||||
|
id, profile_id, conversation_id, user_message_id, payload_json, created
|
||||||
|
)
|
||||||
|
VALUES (?, ?, ?, ?, ?, datetime('now'))
|
||||||
|
""",
|
||||||
|
(review_id, profile_id, conversation_id, user_message_id, json.dumps(payload, ensure_ascii=False)),
|
||||||
|
)
|
||||||
|
return review_id
|
||||||
|
|
||||||
|
|
||||||
|
def load_pending(profile_id: str, review_id: str) -> dict | None:
|
||||||
|
with get_db() as conn:
|
||||||
|
row = row_to_dict(
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT * FROM pending_mask_reviews
|
||||||
|
WHERE id = ? AND profile_id = ?
|
||||||
|
""",
|
||||||
|
(review_id, profile_id),
|
||||||
|
).fetchone()
|
||||||
|
)
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
payload = json.loads(row.get("payload_json") or "{}")
|
||||||
|
payload["id"] = row["id"]
|
||||||
|
payload["conversation_id"] = row["conversation_id"]
|
||||||
|
payload["user_message_id"] = row["user_message_id"]
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def drop_pending(profile_id: str, review_id: str) -> None:
|
||||||
|
with get_db() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM pending_mask_reviews WHERE id = ? AND profile_id = ?",
|
||||||
|
(review_id, profile_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_review_decisions(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
|
||||||
|
by_id = {item["id"]: item for item in pending.get("candidates") or []}
|
||||||
|
drop_keys: set[tuple] = set()
|
||||||
|
for raw in decisions:
|
||||||
|
candidate = by_id.get(raw.get("id") or "")
|
||||||
|
if not candidate:
|
||||||
|
continue
|
||||||
|
decision = (raw.get("decision") or "").strip()
|
||||||
|
label = candidate.get("text") or ""
|
||||||
|
kind = candidate.get("entity_type") if candidate.get("entity_type") in ENTITY_TYPES else "PERSON"
|
||||||
|
cue = candidate.get("cue") or cue_for_mention(
|
||||||
|
pending.get("user_body") or "",
|
||||||
|
candidate.get("user_start", candidate.get("start")),
|
||||||
|
candidate.get("user_end", candidate.get("end")),
|
||||||
|
label,
|
||||||
|
)
|
||||||
|
if decision == _DECISION_IDENTITY:
|
||||||
|
record_sense(profile_id, label, identity=True)
|
||||||
|
record_cue(profile_id, label, cue, _DECISION_IDENTITY, candidate.get("excerpt") or "")
|
||||||
|
confirm_identity(profile_id, label, entity_type=kind)
|
||||||
|
elif decision == _DECISION_NOT:
|
||||||
|
record_sense(profile_id, label, identity=False)
|
||||||
|
record_cue(profile_id, label, cue, _DECISION_NOT, candidate.get("excerpt") or "")
|
||||||
|
drop_keys.add((candidate.get("start"), candidate.get("end"), label.casefold()))
|
||||||
|
mappings = []
|
||||||
|
for mapping in pending.get("mappings") or []:
|
||||||
|
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||||
|
if key in drop_keys:
|
||||||
|
continue
|
||||||
|
mappings.append(mapping)
|
||||||
|
return suppress_known_non_identity(profile_id, mappings, pending.get("user_body") or "")
|
||||||
|
|
||||||
|
|
||||||
|
def confirm_known_identity_spans(profile_id: str, mappings: list[dict], user_body: str) -> None:
|
||||||
|
for mapping in mappings:
|
||||||
|
label = mapping.get("local_label") or ""
|
||||||
|
sense = get_sense(profile_id, label)
|
||||||
|
if not (
|
||||||
|
sense["identity_hits"] > 0
|
||||||
|
and not sense["ambiguous"]
|
||||||
|
and _in_user_text(label, user_body)
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
kind = mapping.get("entity_type") if mapping.get("entity_type") in ENTITY_TYPES else "PERSON"
|
||||||
|
try:
|
||||||
|
confirm_identity(profile_id, label, entity_type=kind)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
|
||||||
|
def list_senses(profile_id: str) -> list[dict]:
|
||||||
|
with get_db() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT normalized_label, identity_hits, non_identity_hits
|
||||||
|
FROM label_senses WHERE profile_id = ?
|
||||||
|
ORDER BY lower(normalized_label)
|
||||||
|
""",
|
||||||
|
(profile_id,),
|
||||||
|
).fetchall()
|
||||||
|
items = []
|
||||||
|
for raw in rows:
|
||||||
|
row = row_to_dict(raw) or {}
|
||||||
|
identity = int(row.get("identity_hits") or 0)
|
||||||
|
other = int(row.get("non_identity_hits") or 0)
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"normalized_label": row.get("normalized_label") or "",
|
||||||
|
"identity_hits": identity,
|
||||||
|
"non_identity_hits": other,
|
||||||
|
"ambiguous": identity > 0 and other > 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def pending_for_conversation(profile_id: str, conversation_id: str) -> dict | None:
|
||||||
|
with get_db() as conn:
|
||||||
|
row = row_to_dict(
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id, payload_json FROM pending_mask_reviews
|
||||||
|
WHERE profile_id = ? AND conversation_id = ?
|
||||||
|
""",
|
||||||
|
(profile_id, conversation_id),
|
||||||
|
).fetchone()
|
||||||
|
)
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
payload = json.loads(row.get("payload_json") or "{}")
|
||||||
|
return {
|
||||||
|
"id": row["id"],
|
||||||
|
"candidates": payload.get("candidates") or [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def auto_resolve_ambiguous(profile_id: str, candidates: list[dict], mappings: list[dict]) -> tuple[list[dict], list[dict]]:
|
||||||
|
"""Local passage LLM for already-ambiguous spellings. Unresolved stay for the popup."""
|
||||||
|
remaining = []
|
||||||
|
drop_keys: set[tuple] = set()
|
||||||
|
for item in candidates:
|
||||||
|
if not item.get("ambiguous"):
|
||||||
|
remaining.append(item)
|
||||||
|
continue
|
||||||
|
label = item.get("text") or ""
|
||||||
|
examples = list_cue_examples(profile_id, label)
|
||||||
|
decision = try_local_passage_decision(item.get("excerpt") or "", label, examples)
|
||||||
|
if decision == _DECISION_NOT:
|
||||||
|
record_sense(profile_id, label, identity=False)
|
||||||
|
record_cue(profile_id, label, item.get("cue") or _CUE_BARE, _DECISION_NOT, item.get("excerpt") or "")
|
||||||
|
drop_keys.add((item.get("start"), item.get("end"), label.casefold()))
|
||||||
|
elif decision == _DECISION_IDENTITY:
|
||||||
|
record_sense(profile_id, label, identity=True)
|
||||||
|
record_cue(profile_id, label, item.get("cue") or _CUE_BARE, _DECISION_IDENTITY, item.get("excerpt") or "")
|
||||||
|
else:
|
||||||
|
remaining.append(item)
|
||||||
|
if not drop_keys:
|
||||||
|
return remaining, mappings
|
||||||
|
kept = []
|
||||||
|
for mapping in mappings:
|
||||||
|
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||||
|
if key in drop_keys:
|
||||||
|
continue
|
||||||
|
kept.append(mapping)
|
||||||
|
return remaining, kept
|
||||||
|
|
||||||
|
|
||||||
|
def try_local_passage_decision(excerpt: str, label: str, examples: list[dict] | None = None) -> str | None:
|
||||||
|
"""Return identity, not_identity, or None if no local model or unusable answer."""
|
||||||
|
from providers import ProviderError, complete_chat, detect_provider
|
||||||
|
|
||||||
|
config = detect_provider()
|
||||||
|
if not config or not config.local or config.mode != "http":
|
||||||
|
return None
|
||||||
|
few_shot = ""
|
||||||
|
lines = []
|
||||||
|
for example in examples or []:
|
||||||
|
snippet = (example.get("excerpt") or "").strip()
|
||||||
|
decision = (example.get("decision") or "").strip()
|
||||||
|
if snippet and decision in {_DECISION_IDENTITY, _DECISION_NOT}:
|
||||||
|
lines.append(f'- "{snippet}" → {decision}')
|
||||||
|
if lines:
|
||||||
|
few_shot = "Bisherige lokale Bestätigungen als Beispiele, nicht als Wortliste:\n" + "\n".join(lines) + "\n"
|
||||||
|
prompt = (
|
||||||
|
f"{few_shot}"
|
||||||
|
"Entscheide nur für die markierte Nennung in diesem kurzen Ausschnitt. "
|
||||||
|
"Nutze den Satzkontext, nicht nur das Wort davor. "
|
||||||
|
"Ist sie eine schützenswerte Identität (Person, Ort, Organisation, privates Projekt) "
|
||||||
|
f"oder eine Sache/Allgemeinbedeutung? Wort: {label}\n"
|
||||||
|
f"Ausschnitt: {excerpt}\n"
|
||||||
|
'Antworte nur mit JSON {"decision":"identity"} oder {"decision":"not_identity"}.'
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = complete_chat(
|
||||||
|
config,
|
||||||
|
[{"role": "user", "content": prompt}],
|
||||||
|
timeout=30.0,
|
||||||
|
max_tokens=32,
|
||||||
|
)
|
||||||
|
except ProviderError:
|
||||||
|
return None
|
||||||
|
match = re.search(r"\{.*\}", result.content or "", re.DOTALL)
|
||||||
|
if not match:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
data = json.loads(match.group(0))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return None
|
||||||
|
value = (data.get("decision") or "").strip()
|
||||||
|
if value in {_DECISION_IDENTITY, _DECISION_NOT}:
|
||||||
|
return value
|
||||||
|
return None
|
||||||
|
|
@ -7,9 +7,23 @@ import re
|
||||||
from context_builder import assemble_text, build_internal_context, is_closing_turn
|
from context_builder import assemble_text, build_internal_context, is_closing_turn
|
||||||
from conversation_signals import infer_signals
|
from conversation_signals import infer_signals
|
||||||
from debug_store import persist_engine_error, persist_step
|
from debug_store import persist_engine_error, persist_step
|
||||||
|
from detect_learning import (
|
||||||
|
MODE_LEARNING,
|
||||||
|
apply_learned_decisions,
|
||||||
|
apply_review_decisions,
|
||||||
|
auto_resolve_ambiguous,
|
||||||
|
build_candidates,
|
||||||
|
confirm_known_identity_spans,
|
||||||
|
drop_pending,
|
||||||
|
get_detect_operating_mode,
|
||||||
|
load_pending,
|
||||||
|
save_pending,
|
||||||
|
supplement_kinship_candidates,
|
||||||
|
suppress_known_non_identity,
|
||||||
|
)
|
||||||
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
|
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
|
||||||
from engine import EngineError, execute_prompt, load_active_prompt
|
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
||||||
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES
|
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError, detect_personal_egress
|
||||||
from writing_profile_store import remember_dialogue_style
|
from writing_profile_store import remember_dialogue_style
|
||||||
from profile_review import consider_dialogue
|
from profile_review import consider_dialogue
|
||||||
|
|
||||||
|
|
@ -302,23 +316,21 @@ def visible_for_role(payload: dict, role: str | None) -> dict:
|
||||||
return cleaned
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str | None = None) -> dict:
|
def _finish_turn(
|
||||||
conversation = get_conversation(profile_id, conversation_id)
|
profile_id: str,
|
||||||
user = append_message(profile_id, conversation_id, body, role="user", message_id=message_id)
|
conversation_id: str,
|
||||||
context = build_internal_context(
|
user: dict,
|
||||||
profile_id,
|
assembled: dict,
|
||||||
conversation_id=conversation_id,
|
prompt: dict,
|
||||||
space_id=conversation.get("space_id"),
|
*,
|
||||||
journal_day_id=conversation.get("journal_day_id"),
|
precomputed_mappings: list[dict] | None = None,
|
||||||
purpose="dialogue_turn",
|
) -> dict:
|
||||||
)
|
|
||||||
assembled = assemble_text(context)
|
|
||||||
prompt = load_active_prompt("mvp.dialogue_turn")
|
|
||||||
calls = 0
|
calls = 0
|
||||||
result = None
|
result = None
|
||||||
impulse = ""
|
impulse = ""
|
||||||
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
|
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
|
||||||
call_traces: list[dict] = []
|
call_traces: list[dict] = []
|
||||||
|
working = dict(assembled)
|
||||||
try:
|
try:
|
||||||
while calls < 2:
|
while calls < 2:
|
||||||
result = execute_prompt(
|
result = execute_prompt(
|
||||||
|
|
@ -326,7 +338,8 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||||
profile_id,
|
profile_id,
|
||||||
purpose="dialogue_turn",
|
purpose="dialogue_turn",
|
||||||
data_class="B",
|
data_class="B",
|
||||||
context=assembled,
|
context=working,
|
||||||
|
precomputed_mappings=precomputed_mappings if calls == 0 else None,
|
||||||
)
|
)
|
||||||
calls += 1
|
calls += 1
|
||||||
if result.get("trace"):
|
if result.get("trace"):
|
||||||
|
|
@ -335,16 +348,16 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||||
if not content:
|
if not content:
|
||||||
raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.")
|
raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.")
|
||||||
impulse, decision = parse_turn_payload(content)
|
impulse, decision = parse_turn_payload(content)
|
||||||
if not needs_repair(impulse, assembled):
|
if not needs_repair(impulse, working):
|
||||||
break
|
break
|
||||||
decision = {**decision, "guard": "impulse_rejected"}
|
decision = {**decision, "guard": "impulse_rejected"}
|
||||||
assembled = dict(assembled)
|
working = dict(working)
|
||||||
assembled["dialogue_context"] = (
|
working["dialogue_context"] = (
|
||||||
(assembled.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, assembled)
|
(working.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, working)
|
||||||
)
|
)
|
||||||
if needs_repair(impulse, assembled):
|
if needs_repair(impulse, working):
|
||||||
parsed_ok = bool(decision.get("parsed"))
|
parsed_ok = bool(decision.get("parsed"))
|
||||||
impulse = local_hold(last_user_text(assembled))
|
impulse = local_hold(last_user_text(working))
|
||||||
decision = {
|
decision = {
|
||||||
"operation": "fortfuehren",
|
"operation": "fortfuehren",
|
||||||
"label": OPERATIONS["fortfuehren"],
|
"label": OPERATIONS["fortfuehren"],
|
||||||
|
|
@ -369,7 +382,7 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
raise
|
raise
|
||||||
impulse = local_hold(last_user_text(assembled))
|
impulse = local_hold(last_user_text(working))
|
||||||
decision = {
|
decision = {
|
||||||
"operation": "fortfuehren",
|
"operation": "fortfuehren",
|
||||||
"label": OPERATIONS["fortfuehren"],
|
"label": OPERATIONS["fortfuehren"],
|
||||||
|
|
@ -388,7 +401,7 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||||
infer_signals(user_bodies, decision.get("operation")),
|
infer_signals(user_bodies, decision.get("operation")),
|
||||||
)
|
)
|
||||||
remember_dialogue_style(profile_id)
|
remember_dialogue_style(profile_id)
|
||||||
consider_dialogue(profile_id, body)
|
consider_dialogue(profile_id, user.get("body") or "")
|
||||||
trace = result.get("trace") if result else None
|
trace = result.get("trace") if result else None
|
||||||
persist_step(
|
persist_step(
|
||||||
profile_id,
|
profile_id,
|
||||||
|
|
@ -416,3 +429,105 @@ def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str |
|
||||||
"decision": decision,
|
"decision": decision,
|
||||||
"trace": trace,
|
"trace": trace,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled: dict, prompt: dict) -> dict | None:
|
||||||
|
preview = preview_prompt(prompt, assembled)
|
||||||
|
rendered = preview.get("rendered") or ""
|
||||||
|
try:
|
||||||
|
outcome = detect_personal_egress(profile_id, rendered)
|
||||||
|
except DetectError:
|
||||||
|
return None
|
||||||
|
user_body = user.get("body") or ""
|
||||||
|
mappings = list(outcome.mappings or [])
|
||||||
|
candidates = build_candidates(profile_id, mappings, user_body, rendered)
|
||||||
|
candidates, mappings = supplement_kinship_candidates(
|
||||||
|
profile_id, candidates, mappings, user_body, rendered
|
||||||
|
)
|
||||||
|
candidates, mappings = apply_learned_decisions(
|
||||||
|
profile_id, candidates, mappings, user_body, rendered
|
||||||
|
)
|
||||||
|
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
|
||||||
|
if not candidates:
|
||||||
|
confirm_known_identity_spans(profile_id, mappings, user_body)
|
||||||
|
mappings = suppress_known_non_identity(profile_id, mappings, user_body)
|
||||||
|
return _finish_turn(
|
||||||
|
profile_id,
|
||||||
|
conversation_id,
|
||||||
|
user,
|
||||||
|
assembled,
|
||||||
|
prompt,
|
||||||
|
precomputed_mappings=mappings,
|
||||||
|
)
|
||||||
|
review_id = save_pending(
|
||||||
|
profile_id,
|
||||||
|
conversation_id,
|
||||||
|
user.get("id") or "",
|
||||||
|
{
|
||||||
|
"mappings": mappings,
|
||||||
|
"candidates": candidates,
|
||||||
|
"user_body": user_body,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"conversation": get_conversation(profile_id, conversation_id),
|
||||||
|
"user": user,
|
||||||
|
"assistant": None,
|
||||||
|
"calls": 0,
|
||||||
|
"messages": list_messages(profile_id, conversation_id),
|
||||||
|
"pending_mask_review": {
|
||||||
|
"id": review_id,
|
||||||
|
"candidates": candidates,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str | None = None) -> dict:
|
||||||
|
conversation = get_conversation(profile_id, conversation_id)
|
||||||
|
user = append_message(profile_id, conversation_id, body, role="user", message_id=message_id)
|
||||||
|
context = build_internal_context(
|
||||||
|
profile_id,
|
||||||
|
conversation_id=conversation_id,
|
||||||
|
space_id=conversation.get("space_id"),
|
||||||
|
journal_day_id=conversation.get("journal_day_id"),
|
||||||
|
purpose="dialogue_turn",
|
||||||
|
)
|
||||||
|
assembled = assemble_text(context)
|
||||||
|
prompt = load_active_prompt("mvp.dialogue_turn")
|
||||||
|
if get_detect_operating_mode() == MODE_LEARNING:
|
||||||
|
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
|
||||||
|
if paused is not None:
|
||||||
|
return paused
|
||||||
|
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
|
||||||
|
|
||||||
|
|
||||||
|
def continue_turn(profile_id: str, conversation_id: str, review_id: str, decisions: list[dict]) -> dict:
|
||||||
|
pending = load_pending(profile_id, review_id)
|
||||||
|
if not pending or pending.get("conversation_id") != conversation_id:
|
||||||
|
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||||
|
mappings = apply_review_decisions(profile_id, pending, decisions)
|
||||||
|
drop_pending(profile_id, review_id)
|
||||||
|
conversation = get_conversation(profile_id, conversation_id)
|
||||||
|
context = build_internal_context(
|
||||||
|
profile_id,
|
||||||
|
conversation_id=conversation_id,
|
||||||
|
space_id=conversation.get("space_id"),
|
||||||
|
journal_day_id=conversation.get("journal_day_id"),
|
||||||
|
purpose="dialogue_turn",
|
||||||
|
)
|
||||||
|
assembled = assemble_text(context)
|
||||||
|
prompt = load_active_prompt("mvp.dialogue_turn")
|
||||||
|
user = {"id": pending.get("user_message_id"), "body": pending.get("user_body") or ""}
|
||||||
|
messages = list_messages(profile_id, conversation_id)
|
||||||
|
for item in messages:
|
||||||
|
if item.get("id") == pending.get("user_message_id"):
|
||||||
|
user = item
|
||||||
|
break
|
||||||
|
return _finish_turn(
|
||||||
|
profile_id,
|
||||||
|
conversation_id,
|
||||||
|
user,
|
||||||
|
assembled,
|
||||||
|
prompt,
|
||||||
|
precomputed_mappings=mappings,
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -75,6 +75,7 @@ def execute_prompt(
|
||||||
disable_context_compression: bool = False,
|
disable_context_compression: bool = False,
|
||||||
budget=None,
|
budget=None,
|
||||||
diagnostics: dict[str, Any] | None = None,
|
diagnostics: dict[str, Any] | None = None,
|
||||||
|
precomputed_mappings: list[dict] | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
preview = preview_prompt(prompt, context)
|
preview = preview_prompt(prompt, context)
|
||||||
feature_id = prompt.get("required_feature") or "ai_calls"
|
feature_id = prompt.get("required_feature") or "ai_calls"
|
||||||
|
|
@ -107,6 +108,7 @@ def execute_prompt(
|
||||||
"disable_context_compression": disable_context_compression,
|
"disable_context_compression": disable_context_compression,
|
||||||
"budget": budget,
|
"budget": budget,
|
||||||
"diagnostics": diagnostics or {},
|
"diagnostics": diagnostics or {},
|
||||||
|
"precomputed_mappings": precomputed_mappings,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,9 @@ from providers import ChatResult, ProviderError, complete_chat, detect_provider
|
||||||
DETECT_CHUNK_CHARS = 4000
|
DETECT_CHUNK_CHARS = 4000
|
||||||
DETECT_CHUNK_OVERLAP = 250
|
DETECT_CHUNK_OVERLAP = 250
|
||||||
DETECT_TIMEOUT = 90.0
|
DETECT_TIMEOUT = 90.0
|
||||||
|
DETECT_TIMEOUT_LOCAL = 300.0
|
||||||
DETECT_MAX_TOKENS = 1024
|
DETECT_MAX_TOKENS = 1024
|
||||||
|
DETECT_MAX_TOKENS_LOCAL = 256
|
||||||
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
||||||
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
|
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
|
||||||
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
|
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
|
||||||
|
|
@ -573,6 +575,13 @@ def _add_usage(stats: DetectionStats, usage: dict | None) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def detect_chat_limits(config) -> tuple[float, int]:
|
||||||
|
"""Remote detect stays at 90s/1024. Local CPU hosts need a longer wait and a shorter cap."""
|
||||||
|
if getattr(config, "local", False):
|
||||||
|
return DETECT_TIMEOUT_LOCAL, DETECT_MAX_TOKENS_LOCAL
|
||||||
|
return DETECT_TIMEOUT, DETECT_MAX_TOKENS
|
||||||
|
|
||||||
|
|
||||||
def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResult:
|
def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResult:
|
||||||
prompt = resolve_template(
|
prompt = resolve_template(
|
||||||
_detect_prompt()["template"],
|
_detect_prompt()["template"],
|
||||||
|
|
@ -580,11 +589,12 @@ def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResul
|
||||||
)
|
)
|
||||||
if schema_retry:
|
if schema_retry:
|
||||||
prompt = f"{prompt.rstrip()}\n\n{SCHEMA_RETRY_HINT}"
|
prompt = f"{prompt.rstrip()}\n\n{SCHEMA_RETRY_HINT}"
|
||||||
|
timeout, max_tokens = detect_chat_limits(config)
|
||||||
return complete_chat(
|
return complete_chat(
|
||||||
config,
|
config,
|
||||||
[{"role": "user", "content": prompt}],
|
[{"role": "user", "content": prompt}],
|
||||||
timeout=DETECT_TIMEOUT,
|
timeout=timeout,
|
||||||
max_tokens=DETECT_MAX_TOKENS,
|
max_tokens=max_tokens,
|
||||||
disable_context_compression=True,
|
disable_context_compression=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
16
backend/migrations/023_llm_profiles.sql
Normal file
16
backend/migrations/023_llm_profiles.sql
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
-- LLM profiles: named, reusable generate/detect endpoint+model presets.
|
||||||
|
-- Keys stay in env. Active assignment remains provider_settings.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS llm_profiles (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL DEFAULT '',
|
||||||
|
url TEXT NOT NULL DEFAULT '',
|
||||||
|
model TEXT NOT NULL DEFAULT '',
|
||||||
|
zdr INTEGER NOT NULL DEFAULT 1,
|
||||||
|
no_train INTEGER NOT NULL DEFAULT 1,
|
||||||
|
created TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text,
|
||||||
|
updated TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text
|
||||||
|
);
|
||||||
|
|
||||||
|
ALTER TABLE provider_settings ADD COLUMN IF NOT EXISTS profile_id TEXT;
|
||||||
23
backend/migrations/024_detect_learning.sql
Normal file
23
backend/migrations/024_detect_learning.sql
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
-- Transitional learning detect: user-confirmed label senses, pending mask review.
|
||||||
|
-- Detect output still does not auto-activate identities.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS label_senses (
|
||||||
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
normalized_label TEXT NOT NULL,
|
||||||
|
identity_hits INTEGER NOT NULL DEFAULT 0,
|
||||||
|
non_identity_hits INTEGER NOT NULL DEFAULT 0,
|
||||||
|
updated TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text,
|
||||||
|
PRIMARY KEY (profile_id, normalized_label)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
|
||||||
|
user_message_id TEXT NOT NULL,
|
||||||
|
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||||
|
created TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_pending_mask_reviews_conv
|
||||||
|
ON pending_mask_reviews (profile_id, conversation_id);
|
||||||
12
backend/migrations/025_label_sense_cues.sql
Normal file
12
backend/migrations/025_label_sense_cues.sql
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
-- Remember mask-review decisions per local context cue (previous word).
|
||||||
|
-- Not a word list: cues come from the user line, e.g. Frau vs. a food mention.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS label_sense_cues (
|
||||||
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
normalized_label TEXT NOT NULL,
|
||||||
|
cue TEXT NOT NULL,
|
||||||
|
decision TEXT NOT NULL,
|
||||||
|
hits INTEGER NOT NULL DEFAULT 1,
|
||||||
|
updated TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text,
|
||||||
|
PRIMARY KEY (profile_id, normalized_label, cue)
|
||||||
|
);
|
||||||
2
backend/migrations/026_label_sense_cue_excerpt.sql
Normal file
2
backend/migrations/026_label_sense_cue_excerpt.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
-- Mini-passages beside each cue so local detect/GLiNER can pre-decide from context.
|
||||||
|
ALTER TABLE label_sense_cues ADD COLUMN IF NOT EXISTS excerpt TEXT NOT NULL DEFAULT '';
|
||||||
|
|
@ -263,17 +263,77 @@ def list_provider_models(config) -> list[dict]:
|
||||||
url = getattr(config, "url", "") or ""
|
url = getattr(config, "url", "") or ""
|
||||||
if not url:
|
if not url:
|
||||||
return []
|
return []
|
||||||
catalog_key = models_url(url)
|
key = getattr(config, "key", "") or ""
|
||||||
|
items = _list_openai_models(url, key)
|
||||||
|
if items:
|
||||||
|
return items
|
||||||
|
return _list_ollama_tags(url)
|
||||||
|
|
||||||
|
|
||||||
|
def list_models_for_url(url: str, key: str = "") -> list[dict]:
|
||||||
|
from providers import ProviderConfig, is_local_url
|
||||||
|
|
||||||
|
url = (url or "").strip()
|
||||||
|
if not url:
|
||||||
|
return []
|
||||||
|
local = is_local_url(url)
|
||||||
|
return list_provider_models(
|
||||||
|
ProviderConfig(
|
||||||
|
role="generate",
|
||||||
|
name="ollama" if local else "openrouter",
|
||||||
|
mode="http",
|
||||||
|
url=url,
|
||||||
|
model="",
|
||||||
|
key=key,
|
||||||
|
local=local,
|
||||||
|
zdr=True,
|
||||||
|
no_train=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _list_openai_models(chat_url: str, key: str) -> list[dict]:
|
||||||
|
catalog_key = models_url(chat_url)
|
||||||
try:
|
try:
|
||||||
rows = _request_models_payload(catalog_key, getattr(config, "key", "") or "")
|
rows = _request_models_payload(catalog_key, key)
|
||||||
except JournalBudgetError:
|
except JournalBudgetError:
|
||||||
return []
|
return []
|
||||||
|
return _ids_from_rows(rows, id_keys=("id",))
|
||||||
|
|
||||||
|
|
||||||
|
def _list_ollama_tags(chat_url: str) -> list[dict]:
|
||||||
|
parsed = urlparse(chat_url or "")
|
||||||
|
if not parsed.scheme or not parsed.netloc:
|
||||||
|
return []
|
||||||
|
tags_url = f"{parsed.scheme}://{parsed.netloc}/api/tags"
|
||||||
|
headers = {"Content-Type": "application/json"}
|
||||||
|
try:
|
||||||
|
response = httpx.get(tags_url, headers=headers, timeout=8.0)
|
||||||
|
except httpx.HTTPError:
|
||||||
|
return []
|
||||||
|
if response.status_code >= 400:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
payload = response.json()
|
||||||
|
except ValueError:
|
||||||
|
return []
|
||||||
|
rows = payload.get("models") if isinstance(payload, dict) else payload
|
||||||
|
if not isinstance(rows, list):
|
||||||
|
return []
|
||||||
|
return _ids_from_rows(rows, id_keys=("name", "model", "id"))
|
||||||
|
|
||||||
|
|
||||||
|
def _ids_from_rows(rows: list, *, id_keys: tuple[str, ...]) -> list[dict]:
|
||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
items: list[dict] = []
|
items: list[dict] = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
if not isinstance(row, dict):
|
if not isinstance(row, dict):
|
||||||
continue
|
continue
|
||||||
model_id = str(row.get("id") or "").strip()
|
model_id = ""
|
||||||
|
for key in id_keys:
|
||||||
|
model_id = str(row.get(key) or "").strip()
|
||||||
|
if model_id:
|
||||||
|
break
|
||||||
if not model_id or model_id in seen:
|
if not model_id or model_id in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(model_id)
|
seen.add(model_id)
|
||||||
|
|
|
||||||
|
|
@ -1055,7 +1055,26 @@ def complete(request: GatewayRequest) -> GatewayResult:
|
||||||
detect_note = None
|
detect_note = None
|
||||||
detect_stats: dict[str, Any] = {}
|
detect_stats: dict[str, Any] = {}
|
||||||
local_identities: list[dict[str, Any]] = []
|
local_identities: list[dict[str, Any]] = []
|
||||||
|
precomputed = request.payload.get("precomputed_mappings") if request.payload else None
|
||||||
try:
|
try:
|
||||||
|
if precomputed is not None:
|
||||||
|
mappings = list(precomputed)
|
||||||
|
detect_stats = {
|
||||||
|
"detect_note": "precomputed_learning_review",
|
||||||
|
"full_detection_coverage": True,
|
||||||
|
"semantic_identity_guaranteed": False,
|
||||||
|
}
|
||||||
|
local_identities = [
|
||||||
|
{
|
||||||
|
"local_label": item.get("local_label"),
|
||||||
|
"token": item.get("token"),
|
||||||
|
"entity_type": item.get("entity_type"),
|
||||||
|
"source": item.get("source"),
|
||||||
|
}
|
||||||
|
for item in mappings
|
||||||
|
]
|
||||||
|
detect_name = "learning_review"
|
||||||
|
else:
|
||||||
outcome = detect_personal_egress(request.profile_id, rendered)
|
outcome = detect_personal_egress(request.profile_id, rendered)
|
||||||
mappings = outcome.mappings
|
mappings = outcome.mappings
|
||||||
detect_stats = outcome.stats.public()
|
detect_stats = outcome.stats.public()
|
||||||
|
|
|
||||||
|
|
@ -3,10 +3,12 @@ from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import uuid
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from db import get_db
|
from db import get_db
|
||||||
|
from detect_learning import get_detect_operating_mode
|
||||||
from env_loader import (
|
from env_loader import (
|
||||||
allows_remote_plaintext_detect,
|
allows_remote_plaintext_detect,
|
||||||
remote_plaintext_detect_reason,
|
remote_plaintext_detect_reason,
|
||||||
|
|
@ -16,6 +18,8 @@ from env_loader import (
|
||||||
|
|
||||||
SEED_PATH = Path(__file__).resolve().parent / "config" / "providers.seed.json"
|
SEED_PATH = Path(__file__).resolve().parent / "config" / "providers.seed.json"
|
||||||
ROLES = ("generate", "detect")
|
ROLES = ("generate", "detect")
|
||||||
|
OLLAMA_LAN_ID = "llm-ollama-lan"
|
||||||
|
OLLAMA_LAN_URL = "http://192.168.2.144:11434/v1/chat/completions"
|
||||||
_KEY_ENV = {
|
_KEY_ENV = {
|
||||||
"generate": "KANSHO_PROVIDER_KEY",
|
"generate": "KANSHO_PROVIDER_KEY",
|
||||||
"detect": "KANSHO_DETECT_PROVIDER_KEY",
|
"detect": "KANSHO_DETECT_PROVIDER_KEY",
|
||||||
|
|
@ -28,7 +32,7 @@ ROLE_META = {
|
||||||
},
|
},
|
||||||
"detect": {
|
"detect": {
|
||||||
"title": "Maskierung",
|
"title": "Maskierung",
|
||||||
"task": "Eigener Detect-Endpunkt und eigenes Modell. Vollständige semantische Detection des persönlichen Egress. Ziel: lokales Detect-Modell. Übergang: externes Klartext-Detect nur mit Operator-Freigabe. Kein Pattern-Fallback.",
|
"task": "Eigener Detect-Endpunkt und eigenes Modell. Vollständige semantische Detection des persönlichen Egress. Ziel: lokales Detect-Modell (Ollama). Übergang: externes Klartext-Detect nur mit Operator-Freigabe. Kein Pattern-Fallback.",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -41,6 +45,38 @@ class SettingsError(Exception):
|
||||||
self.status_code = status_code
|
self.status_code = status_code
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_url(url: str) -> str:
|
||||||
|
url = (url or "").strip()
|
||||||
|
if not url:
|
||||||
|
return ""
|
||||||
|
parsed = urlparse(url)
|
||||||
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||||
|
raise SettingsError("provider_url_invalid", "URL muss http(s) sein.")
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
def _profile_row(conn, profile_id: str) -> dict | None:
|
||||||
|
row = conn.execute("SELECT * FROM llm_profiles WHERE id = ?", (profile_id,)).fetchone()
|
||||||
|
return dict(row) if row else None
|
||||||
|
|
||||||
|
|
||||||
|
def _public_profile(row: dict) -> dict:
|
||||||
|
from providers import is_local_url
|
||||||
|
|
||||||
|
url = row.get("url") or ""
|
||||||
|
return {
|
||||||
|
"id": row.get("id") or "",
|
||||||
|
"title": row.get("title") or "",
|
||||||
|
"name": row.get("name") or "",
|
||||||
|
"url": url,
|
||||||
|
"model": row.get("model") or "",
|
||||||
|
"zdr": bool(row.get("zdr", 1)),
|
||||||
|
"no_train": bool(row.get("no_train", 1)),
|
||||||
|
"local": is_local_url(url),
|
||||||
|
"updated": row.get("updated") or "",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def seed_provider_settings(conn) -> None:
|
def seed_provider_settings(conn) -> None:
|
||||||
items = json.loads(SEED_PATH.read_text(encoding="utf-8"))
|
items = json.loads(SEED_PATH.read_text(encoding="utf-8"))
|
||||||
for item in items:
|
for item in items:
|
||||||
|
|
@ -64,6 +100,233 @@ def seed_provider_settings(conn) -> None:
|
||||||
1 if item.get("no_train", True) else 0,
|
1 if item.get("no_train", True) else 0,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
seed_llm_profiles(conn)
|
||||||
|
|
||||||
|
|
||||||
|
def seed_llm_profiles(conn) -> None:
|
||||||
|
"""Named presets. Existing rows are not overwritten. Keys never stored."""
|
||||||
|
ollama = conn.execute("SELECT * FROM llm_profiles WHERE id = ?", (OLLAMA_LAN_ID,)).fetchone()
|
||||||
|
if not ollama:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO llm_profiles (id, title, name, url, model, zdr, no_train)
|
||||||
|
VALUES (?, ?, ?, ?, ?, 1, 1)
|
||||||
|
""",
|
||||||
|
(OLLAMA_LAN_ID, "Ollama (LAN)", "ollama", OLLAMA_LAN_URL, "phi3:mini"),
|
||||||
|
)
|
||||||
|
elif not (dict(ollama).get("model") or "").strip():
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE llm_profiles SET model = ?, updated = datetime('now') WHERE id = ?",
|
||||||
|
("phi3:mini", OLLAMA_LAN_ID),
|
||||||
|
)
|
||||||
|
for role in ROLES:
|
||||||
|
row = conn.execute("SELECT * FROM provider_settings WHERE role = ?", (role,)).fetchone()
|
||||||
|
if not row:
|
||||||
|
continue
|
||||||
|
data = dict(row)
|
||||||
|
profile_id = (data.get("profile_id") or "").strip()
|
||||||
|
if profile_id and _profile_row(conn, profile_id):
|
||||||
|
continue
|
||||||
|
url = (data.get("url") or "").strip()
|
||||||
|
model = (data.get("model") or "").strip()
|
||||||
|
if not url:
|
||||||
|
continue
|
||||||
|
match = conn.execute(
|
||||||
|
"SELECT id FROM llm_profiles WHERE url = ? AND model = ? ORDER BY created LIMIT 1",
|
||||||
|
(url, model),
|
||||||
|
).fetchone()
|
||||||
|
if match:
|
||||||
|
chosen = match["id"]
|
||||||
|
else:
|
||||||
|
chosen = f"llm-{role}-current"
|
||||||
|
if _profile_row(conn, chosen):
|
||||||
|
chosen = str(uuid.uuid4())
|
||||||
|
title = "Sprachmodell (aktuell)" if role == "generate" else "Maskierung (aktuell)"
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO llm_profiles (id, title, name, url, model, zdr, no_train)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
chosen,
|
||||||
|
title,
|
||||||
|
data.get("name") or "",
|
||||||
|
url,
|
||||||
|
model,
|
||||||
|
1 if data.get("zdr", 1) else 0,
|
||||||
|
1 if data.get("no_train", 1) else 0,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE provider_settings SET profile_id = ? WHERE role = ?",
|
||||||
|
(chosen, role),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def list_profiles() -> list[dict]:
|
||||||
|
with get_db() as conn:
|
||||||
|
rows = conn.execute("SELECT * FROM llm_profiles ORDER BY title, created").fetchall()
|
||||||
|
return [_public_profile(dict(row)) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def get_profile(profile_id: str) -> dict | None:
|
||||||
|
with get_db() as conn:
|
||||||
|
row = _profile_row(conn, profile_id)
|
||||||
|
return _public_profile(row) if row else None
|
||||||
|
|
||||||
|
|
||||||
|
def _write_profile(
|
||||||
|
conn,
|
||||||
|
profile_id: str,
|
||||||
|
*,
|
||||||
|
title: str,
|
||||||
|
name: str,
|
||||||
|
url: str,
|
||||||
|
model: str,
|
||||||
|
zdr: bool,
|
||||||
|
no_train: bool,
|
||||||
|
) -> dict:
|
||||||
|
title = (title or "").strip()
|
||||||
|
if not title:
|
||||||
|
raise SettingsError("profile_title_required", "Ein Profil braucht einen Namen.")
|
||||||
|
url = _validate_url(url)
|
||||||
|
if not url:
|
||||||
|
raise SettingsError("provider_url_required", "Ein Profil braucht eine URL.")
|
||||||
|
model = (model or "").strip()
|
||||||
|
name = (name or "").strip()
|
||||||
|
existing = _profile_row(conn, profile_id)
|
||||||
|
if existing:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE llm_profiles
|
||||||
|
SET title = ?, name = ?, url = ?, model = ?, zdr = ?, no_train = ?, updated = datetime('now')
|
||||||
|
WHERE id = ?
|
||||||
|
""",
|
||||||
|
(title, name, url, model, 1 if zdr else 0, 1 if no_train else 0, profile_id),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO llm_profiles (id, title, name, url, model, zdr, no_train)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(profile_id, title, name, url, model, 1 if zdr else 0, 1 if no_train else 0),
|
||||||
|
)
|
||||||
|
for role in ROLES:
|
||||||
|
active = conn.execute(
|
||||||
|
"SELECT role FROM provider_settings WHERE role = ? AND profile_id = ?",
|
||||||
|
(role, profile_id),
|
||||||
|
).fetchone()
|
||||||
|
if not active:
|
||||||
|
continue
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE provider_settings
|
||||||
|
SET name = ?, url = ?, model = ?, zdr = ?, no_train = ?, updated = datetime('now')
|
||||||
|
WHERE role = ?
|
||||||
|
""",
|
||||||
|
(name, url, model, 1 if zdr else 0, 1 if no_train else 0, role),
|
||||||
|
)
|
||||||
|
row = _profile_row(conn, profile_id)
|
||||||
|
return _public_profile(row or {})
|
||||||
|
|
||||||
|
|
||||||
|
def create_profile(*, title: str, name: str, url: str, model: str, zdr: bool, no_train: bool) -> dict:
|
||||||
|
profile_id = str(uuid.uuid4())
|
||||||
|
with get_db() as conn:
|
||||||
|
return _write_profile(
|
||||||
|
conn,
|
||||||
|
profile_id,
|
||||||
|
title=title,
|
||||||
|
name=name,
|
||||||
|
url=url,
|
||||||
|
model=model,
|
||||||
|
zdr=zdr,
|
||||||
|
no_train=no_train,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def update_profile(
|
||||||
|
profile_id: str,
|
||||||
|
*,
|
||||||
|
title: str,
|
||||||
|
name: str,
|
||||||
|
url: str,
|
||||||
|
model: str,
|
||||||
|
zdr: bool,
|
||||||
|
no_train: bool,
|
||||||
|
) -> dict:
|
||||||
|
with get_db() as conn:
|
||||||
|
if not _profile_row(conn, profile_id):
|
||||||
|
raise SettingsError("profile_missing", "Profil fehlt.", 404)
|
||||||
|
return _write_profile(
|
||||||
|
conn,
|
||||||
|
profile_id,
|
||||||
|
title=title,
|
||||||
|
name=name,
|
||||||
|
url=url,
|
||||||
|
model=model,
|
||||||
|
zdr=zdr,
|
||||||
|
no_train=no_train,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def delete_profile(profile_id: str) -> None:
|
||||||
|
with get_db() as conn:
|
||||||
|
if not _profile_row(conn, profile_id):
|
||||||
|
raise SettingsError("profile_missing", "Profil fehlt.", 404)
|
||||||
|
used = conn.execute(
|
||||||
|
"SELECT role FROM provider_settings WHERE profile_id = ?",
|
||||||
|
(profile_id,),
|
||||||
|
).fetchall()
|
||||||
|
if used:
|
||||||
|
roles = ", ".join(row["role"] for row in used)
|
||||||
|
raise SettingsError(
|
||||||
|
"profile_in_use",
|
||||||
|
f"Profil ist noch der Stufe {roles} zugeordnet. Zuerst ein anderes Profil wählen.",
|
||||||
|
)
|
||||||
|
conn.execute("DELETE FROM llm_profiles WHERE id = ?", (profile_id,))
|
||||||
|
|
||||||
|
|
||||||
|
def activate_profile(role: str, profile_id: str) -> dict:
|
||||||
|
if role not in ROLES:
|
||||||
|
raise SettingsError("unknown_provider_role", "Unbekannte Provider-Rolle.")
|
||||||
|
with get_db() as conn:
|
||||||
|
row = _profile_row(conn, profile_id)
|
||||||
|
if not row:
|
||||||
|
raise SettingsError("profile_missing", "Profil fehlt.", 404)
|
||||||
|
url = (row.get("url") or "").strip()
|
||||||
|
model = (row.get("model") or "").strip()
|
||||||
|
if role == "generate" and not url:
|
||||||
|
raise SettingsError("provider_url_required", "Das Sprachmodell braucht eine URL.")
|
||||||
|
if role == "detect" and not url:
|
||||||
|
raise SettingsError("provider_url_required", "Die Maskierung braucht eine URL.")
|
||||||
|
if not model:
|
||||||
|
raise SettingsError("provider_model_required", "Das Profil braucht ein Modell.")
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO provider_settings (role, name, url, model, zdr, no_train, profile_id, updated)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))
|
||||||
|
ON CONFLICT(role) DO UPDATE SET
|
||||||
|
name = excluded.name,
|
||||||
|
url = excluded.url,
|
||||||
|
model = excluded.model,
|
||||||
|
zdr = excluded.zdr,
|
||||||
|
no_train = excluded.no_train,
|
||||||
|
profile_id = excluded.profile_id,
|
||||||
|
updated = datetime('now')
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
role,
|
||||||
|
row.get("name") or "",
|
||||||
|
url,
|
||||||
|
model,
|
||||||
|
1 if row.get("zdr", 1) else 0,
|
||||||
|
1 if row.get("no_train", 1) else 0,
|
||||||
|
profile_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return get_setting(role) or {}
|
||||||
|
|
||||||
|
|
||||||
def get_setting(role: str) -> dict | None:
|
def get_setting(role: str) -> dict | None:
|
||||||
|
|
@ -74,34 +337,71 @@ def get_setting(role: str) -> dict | None:
|
||||||
return dict(row) if row else None
|
return dict(row) if row else None
|
||||||
|
|
||||||
|
|
||||||
def upsert_setting(role: str, *, name: str, url: str, model: str, zdr: bool, no_train: bool) -> dict:
|
def upsert_setting(
|
||||||
|
role: str,
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
url: str,
|
||||||
|
model: str,
|
||||||
|
zdr: bool,
|
||||||
|
no_train: bool,
|
||||||
|
profile_id: str | None = None,
|
||||||
|
save_as_title: str | None = None,
|
||||||
|
) -> dict:
|
||||||
if role not in ROLES:
|
if role not in ROLES:
|
||||||
raise SettingsError("unknown_provider_role", "Unbekannte Provider-Rolle.")
|
raise SettingsError("unknown_provider_role", "Unbekannte Provider-Rolle.")
|
||||||
url = (url or "").strip()
|
url = _validate_url(url)
|
||||||
model = (model or "").strip()
|
model = (model or "").strip()
|
||||||
name = (name or "").strip()
|
name = (name or "").strip()
|
||||||
if role == "generate" and not url:
|
if role == "generate" and not url:
|
||||||
raise SettingsError("provider_url_required", "Das Sprachmodell braucht eine URL.")
|
raise SettingsError("provider_url_required", "Das Sprachmodell braucht eine URL.")
|
||||||
if url:
|
|
||||||
parsed = urlparse(url)
|
|
||||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
||||||
raise SettingsError("provider_url_invalid", "URL muss http(s) sein.")
|
|
||||||
if role == "generate" and not model:
|
if role == "generate" and not model:
|
||||||
raise SettingsError("provider_model_required", "Das Sprachmodell braucht ein Modell.")
|
raise SettingsError("provider_model_required", "Das Sprachmodell braucht ein Modell.")
|
||||||
|
title = (save_as_title or "").strip()
|
||||||
with get_db() as conn:
|
with get_db() as conn:
|
||||||
|
current = conn.execute("SELECT * FROM provider_settings WHERE role = ?", (role,)).fetchone()
|
||||||
|
current_id = (dict(current).get("profile_id") if current else "") or ""
|
||||||
|
chosen = (profile_id or "").strip() or current_id
|
||||||
|
if title:
|
||||||
|
chosen = str(uuid.uuid4())
|
||||||
|
_write_profile(
|
||||||
|
conn,
|
||||||
|
chosen,
|
||||||
|
title=title,
|
||||||
|
name=name,
|
||||||
|
url=url,
|
||||||
|
model=model,
|
||||||
|
zdr=zdr,
|
||||||
|
no_train=no_train,
|
||||||
|
)
|
||||||
|
elif chosen:
|
||||||
|
existing = _profile_row(conn, chosen)
|
||||||
|
if not existing:
|
||||||
|
raise SettingsError("profile_missing", "Profil fehlt.", 404)
|
||||||
|
_write_profile(
|
||||||
|
conn,
|
||||||
|
chosen,
|
||||||
|
title=existing.get("title") or title or role,
|
||||||
|
name=name,
|
||||||
|
url=url,
|
||||||
|
model=model,
|
||||||
|
zdr=zdr,
|
||||||
|
no_train=no_train,
|
||||||
|
)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""
|
"""
|
||||||
INSERT INTO provider_settings (role, name, url, model, zdr, no_train, updated)
|
INSERT INTO provider_settings (role, name, url, model, zdr, no_train, profile_id, updated)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
|
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))
|
||||||
ON CONFLICT(role) DO UPDATE SET
|
ON CONFLICT(role) DO UPDATE SET
|
||||||
name = excluded.name,
|
name = excluded.name,
|
||||||
url = excluded.url,
|
url = excluded.url,
|
||||||
model = excluded.model,
|
model = excluded.model,
|
||||||
zdr = excluded.zdr,
|
zdr = excluded.zdr,
|
||||||
no_train = excluded.no_train,
|
no_train = excluded.no_train,
|
||||||
|
profile_id = excluded.profile_id,
|
||||||
updated = datetime('now')
|
updated = datetime('now')
|
||||||
""",
|
""",
|
||||||
(role, name, url, model, 1 if zdr else 0, 1 if no_train else 0),
|
(role, name, url, model, 1 if zdr else 0, 1 if no_train else 0, chosen or None),
|
||||||
)
|
)
|
||||||
return get_setting(role) or {}
|
return get_setting(role) or {}
|
||||||
|
|
||||||
|
|
@ -116,11 +416,11 @@ def set_role_key(role: str, key: str) -> None:
|
||||||
|
|
||||||
|
|
||||||
def public_role_status(role: str) -> dict:
|
def public_role_status(role: str) -> dict:
|
||||||
from providers import _is_local_url, detect_provider, generate_provider
|
from providers import detect_provider, generate_provider, is_local_url
|
||||||
|
|
||||||
row = get_setting(role) or {}
|
row = get_setting(role) or {}
|
||||||
config = generate_provider() if role == "generate" else detect_provider()
|
config = generate_provider() if role == "generate" else detect_provider()
|
||||||
local = _is_local_url(row.get("url") or "")
|
local = is_local_url(row.get("url") or "")
|
||||||
own_key = bool((os.environ.get(_KEY_ENV[role]) or "").strip())
|
own_key = bool((os.environ.get(_KEY_ENV[role]) or "").strip())
|
||||||
generate_key = bool((os.environ.get(_KEY_ENV["generate"]) or "").strip())
|
generate_key = bool((os.environ.get(_KEY_ENV["generate"]) or "").strip())
|
||||||
if own_key:
|
if own_key:
|
||||||
|
|
@ -139,6 +439,7 @@ def public_role_status(role: str) -> dict:
|
||||||
"model": row.get("model") or "",
|
"model": row.get("model") or "",
|
||||||
"zdr": bool(row.get("zdr", 1)),
|
"zdr": bool(row.get("zdr", 1)),
|
||||||
"no_train": bool(row.get("no_train", 1)),
|
"no_train": bool(row.get("no_train", 1)),
|
||||||
|
"profile_id": row.get("profile_id") or "",
|
||||||
"local": local,
|
"local": local,
|
||||||
"key_present": key_source != "none",
|
"key_present": key_source != "none",
|
||||||
"key_source": key_source,
|
"key_source": key_source,
|
||||||
|
|
@ -154,8 +455,11 @@ def public_role_status(role: str) -> dict:
|
||||||
def public_status() -> dict:
|
def public_status() -> dict:
|
||||||
return {
|
return {
|
||||||
"roles": [public_role_status(role) for role in ROLES],
|
"roles": [public_role_status(role) for role in ROLES],
|
||||||
|
"profiles": list_profiles(),
|
||||||
"runtime_env": runtime_env(),
|
"runtime_env": runtime_env(),
|
||||||
"note": "Zwei Stufen, zwei Modelle: Maskierung und Sprachmodell. Semantische Detection ist Pflicht vor Generate. Ziel bleibt lokales Detect. Externes Klartext-Detect in Production nur mit KANSHO_ALLOW_REMOTE_DETECT. Keys nur in .env.",
|
"note": "LLM-Profile speichern URL und Modell je Endpunkt. Maskierung und Sprachmodell wählen je ein Profil. Semantische Detection ist Pflicht vor Generate. LAN-Ollama gilt als lokal. Keys nur in .env.",
|
||||||
"remote_plaintext_reason": remote_plaintext_detect_reason(),
|
"remote_plaintext_reason": remote_plaintext_detect_reason(),
|
||||||
"remote_plaintext_allowed": allows_remote_plaintext_detect(),
|
"remote_plaintext_allowed": allows_remote_plaintext_detect(),
|
||||||
|
"ollama_url": OLLAMA_LAN_URL,
|
||||||
|
"detect_operating_mode": get_detect_operating_mode(),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
"""Two independently configured OpenAI-compatible providers: generate vs detect."""
|
"""Two independently configured OpenAI-compatible providers: generate vs detect."""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
@ -107,9 +108,27 @@ def _truthy(name: str) -> bool:
|
||||||
return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
|
return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
|
||||||
|
|
||||||
|
|
||||||
|
def is_local_url(url: str) -> bool:
|
||||||
|
"""Loopback, RFC1918/ULA and .local stay in the local trusted zone.
|
||||||
|
|
||||||
|
Homelab Ollama (e.g. 192.168.2.144) is local, not an external provider.
|
||||||
|
"""
|
||||||
|
host = (urlparse(url or "").hostname or "").lower()
|
||||||
|
if not host:
|
||||||
|
return False
|
||||||
|
if host in {"localhost", "127.0.0.1", "::1", "ollama"}:
|
||||||
|
return True
|
||||||
|
if host.endswith(".local"):
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
ip = ipaddress.ip_address(host)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
return bool(ip.is_loopback or ip.is_private or ip.is_link_local)
|
||||||
|
|
||||||
|
|
||||||
def _is_local_url(url: str) -> bool:
|
def _is_local_url(url: str) -> bool:
|
||||||
host = (urlparse(url).hostname or "").lower()
|
return is_local_url(url)
|
||||||
return host in {"localhost", "127.0.0.1", "::1", "ollama"}
|
|
||||||
|
|
||||||
|
|
||||||
def _setting(role: str) -> dict:
|
def _setting(role: str) -> dict:
|
||||||
|
|
@ -125,9 +144,9 @@ def _text(env_name: str, db_value: str | None, default: str = "") -> str:
|
||||||
env = (os.environ.get(env_name) or "").strip()
|
env = (os.environ.get(env_name) or "").strip()
|
||||||
if env:
|
if env:
|
||||||
return env
|
return env
|
||||||
if db_value:
|
if db_value is None:
|
||||||
return str(db_value).strip()
|
|
||||||
return default
|
return default
|
||||||
|
return str(db_value).strip()
|
||||||
|
|
||||||
|
|
||||||
def _flag(env_name: str, db_value, default: bool = True) -> bool:
|
def _flag(env_name: str, db_value, default: bool = True) -> bool:
|
||||||
|
|
@ -155,9 +174,11 @@ def generate_provider() -> ProviderConfig | None:
|
||||||
key = (os.environ.get("KANSHO_PROVIDER_KEY") or "").strip()
|
key = (os.environ.get("KANSHO_PROVIDER_KEY") or "").strip()
|
||||||
url = _text("KANSHO_PROVIDER_URL", row.get("url"), "https://openrouter.ai/api/v1/chat/completions")
|
url = _text("KANSHO_PROVIDER_URL", row.get("url"), "https://openrouter.ai/api/v1/chat/completions")
|
||||||
model = _text("KANSHO_PROVIDER_MODEL", row.get("model"), "openai/gpt-4o-mini")
|
model = _text("KANSHO_PROVIDER_MODEL", row.get("model"), "openai/gpt-4o-mini")
|
||||||
local = _is_local_url(url)
|
local = is_local_url(url)
|
||||||
zdr = local or _flag("KANSHO_PROVIDER_ZDR", row.get("zdr"), True)
|
zdr = local or _flag("KANSHO_PROVIDER_ZDR", row.get("zdr"), True)
|
||||||
no_train = local or _flag("KANSHO_PROVIDER_NO_TRAIN", row.get("no_train"), True)
|
no_train = local or _flag("KANSHO_PROVIDER_NO_TRAIN", row.get("no_train"), True)
|
||||||
|
if not url or not model:
|
||||||
|
return None
|
||||||
if not local and not key:
|
if not local and not key:
|
||||||
return None
|
return None
|
||||||
if not local and (not zdr or not no_train):
|
if not local and (not zdr or not no_train):
|
||||||
|
|
@ -193,8 +214,10 @@ def detect_provider() -> ProviderConfig | None:
|
||||||
if not url:
|
if not url:
|
||||||
return None
|
return None
|
||||||
model = _text("KANSHO_DETECT_PROVIDER_MODEL", row.get("model"), "openai/gpt-4.1-nano")
|
model = _text("KANSHO_DETECT_PROVIDER_MODEL", row.get("model"), "openai/gpt-4.1-nano")
|
||||||
|
if not model:
|
||||||
|
return None
|
||||||
key = (os.environ.get("KANSHO_DETECT_PROVIDER_KEY") or "").strip()
|
key = (os.environ.get("KANSHO_DETECT_PROVIDER_KEY") or "").strip()
|
||||||
local = _is_local_url(url)
|
local = is_local_url(url)
|
||||||
if not key and not local:
|
if not key and not local:
|
||||||
key = (os.environ.get("KANSHO_PROVIDER_KEY") or "").strip()
|
key = (os.environ.get("KANSHO_PROVIDER_KEY") or "").strip()
|
||||||
zdr = local or _flag("KANSHO_DETECT_ZDR", row.get("zdr"), True)
|
zdr = local or _flag("KANSHO_DETECT_ZDR", row.get("zdr"), True)
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query
|
||||||
from fastapi.responses import Response
|
from fastapi.responses import Response
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
from auth import require_admin_dep
|
from auth import require_admin_dep
|
||||||
from db import get_db
|
from db import get_db
|
||||||
|
|
@ -19,6 +20,7 @@ from debug_store import (
|
||||||
settings_payload,
|
settings_payload,
|
||||||
)
|
)
|
||||||
from dialogue_store import StoreError, get_conversation, inventory, list_conversations, list_derived_for_conversation, list_messages
|
from dialogue_store import StoreError, get_conversation, inventory, list_conversations, list_derived_for_conversation, list_messages
|
||||||
|
from detect_learning import list_senses
|
||||||
from identity_store import (
|
from identity_store import (
|
||||||
confirm_identity,
|
confirm_identity,
|
||||||
confirm_review_proposal,
|
confirm_review_proposal,
|
||||||
|
|
@ -30,7 +32,18 @@ from identity_store import (
|
||||||
list_review_proposals,
|
list_review_proposals,
|
||||||
update_identity,
|
update_identity,
|
||||||
)
|
)
|
||||||
from provider_settings import ROLES, SettingsError, public_status, set_role_key, upsert_setting
|
from provider_settings import (
|
||||||
|
ROLES,
|
||||||
|
SettingsError,
|
||||||
|
activate_profile,
|
||||||
|
create_profile,
|
||||||
|
delete_profile,
|
||||||
|
list_profiles,
|
||||||
|
public_status,
|
||||||
|
set_role_key,
|
||||||
|
update_profile,
|
||||||
|
upsert_setting,
|
||||||
|
)
|
||||||
from version import APP_VERSION, BUILD_DATE, MODULE_VERSIONS
|
from version import APP_VERSION, BUILD_DATE, MODULE_VERSIONS
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||||
|
|
@ -196,19 +209,63 @@ class ProviderUpdate(BaseModel):
|
||||||
model: str = ""
|
model: str = ""
|
||||||
zdr: bool = True
|
zdr: bool = True
|
||||||
no_train: bool = True
|
no_train: bool = True
|
||||||
|
profile_id: str | None = None
|
||||||
|
save_as_title: str | None = None
|
||||||
key: str | None = Field(default=None, description="Write-only. Never returned.")
|
key: str | None = Field(default=None, description="Write-only. Never returned.")
|
||||||
|
|
||||||
|
|
||||||
|
class LlmProfileWrite(BaseModel):
|
||||||
|
title: str
|
||||||
|
name: str = ""
|
||||||
|
url: str
|
||||||
|
model: str = ""
|
||||||
|
zdr: bool = True
|
||||||
|
no_train: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class ActivateProfile(BaseModel):
|
||||||
|
profile_id: str
|
||||||
|
|
||||||
|
|
||||||
|
class DetectModeWrite(BaseModel):
|
||||||
|
mode: str
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/providers/detect-mode")
|
||||||
|
def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require_admin_dep)):
|
||||||
|
from detect_learning import set_detect_operating_mode
|
||||||
|
from provider_settings import public_status as provider_public_status
|
||||||
|
|
||||||
|
try:
|
||||||
|
set_detect_operating_mode(body.mode)
|
||||||
|
except ValueError:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail={
|
||||||
|
"code": "invalid_detect_operating_mode",
|
||||||
|
"message": "Detect-Modus muss semantic oder learning sein.",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return provider_public_status()
|
||||||
|
|
||||||
|
|
||||||
@router.get("/providers")
|
@router.get("/providers")
|
||||||
def admin_providers(session: dict = Depends(require_admin_dep)):
|
def admin_providers(session: dict = Depends(require_admin_dep)):
|
||||||
return public_status()
|
return public_status()
|
||||||
|
|
||||||
|
|
||||||
@router.get("/providers/models")
|
@router.get("/providers/models")
|
||||||
def admin_provider_models(session: dict = Depends(require_admin_dep)):
|
def admin_provider_models(url: str | None = None, session: dict = Depends(require_admin_dep)):
|
||||||
from model_catalog import list_provider_models
|
from model_catalog import list_models_for_url, list_provider_models
|
||||||
from providers import detect_provider, generate_provider
|
from providers import detect_provider, generate_provider
|
||||||
|
|
||||||
|
if url:
|
||||||
|
key = ""
|
||||||
|
from providers import is_local_url
|
||||||
|
|
||||||
|
if not is_local_url(url):
|
||||||
|
key = (os.environ.get("KANSHO_PROVIDER_KEY") or "").strip()
|
||||||
|
return {"url": url, "models": list_models_for_url(url, key)}
|
||||||
return {
|
return {
|
||||||
"roles": {
|
"roles": {
|
||||||
"generate": list_provider_models(generate_provider()),
|
"generate": list_provider_models(generate_provider()),
|
||||||
|
|
@ -229,6 +286,8 @@ def admin_update_provider(role: str, body: ProviderUpdate, session: dict = Depen
|
||||||
model=body.model,
|
model=body.model,
|
||||||
zdr=body.zdr,
|
zdr=body.zdr,
|
||||||
no_train=body.no_train,
|
no_train=body.no_train,
|
||||||
|
profile_id=body.profile_id,
|
||||||
|
save_as_title=body.save_as_title,
|
||||||
)
|
)
|
||||||
if body.key is not None and body.key.strip():
|
if body.key is not None and body.key.strip():
|
||||||
set_role_key(role, body.key)
|
set_role_key(role, body.key)
|
||||||
|
|
@ -237,6 +296,64 @@ def admin_update_provider(role: str, body: ProviderUpdate, session: dict = Depen
|
||||||
return public_status()
|
return public_status()
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/providers/{role}/activate")
|
||||||
|
def admin_activate_provider(role: str, body: ActivateProfile, session: dict = Depends(require_admin_dep)):
|
||||||
|
if role not in ROLES:
|
||||||
|
raise HTTPException(status_code=404, detail={"code": "unknown_provider_role", "message": "Unbekannte Provider-Rolle."})
|
||||||
|
try:
|
||||||
|
activate_profile(role, body.profile_id)
|
||||||
|
except SettingsError as exc:
|
||||||
|
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||||
|
return public_status()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/llm-profiles")
|
||||||
|
def admin_list_profiles(session: dict = Depends(require_admin_dep)):
|
||||||
|
return {"profiles": list_profiles()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/llm-profiles")
|
||||||
|
def admin_create_profile(body: LlmProfileWrite, session: dict = Depends(require_admin_dep)):
|
||||||
|
try:
|
||||||
|
profile = create_profile(
|
||||||
|
title=body.title,
|
||||||
|
name=body.name,
|
||||||
|
url=body.url,
|
||||||
|
model=body.model,
|
||||||
|
zdr=body.zdr,
|
||||||
|
no_train=body.no_train,
|
||||||
|
)
|
||||||
|
except SettingsError as exc:
|
||||||
|
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||||
|
return {"profile": profile, **public_status()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/llm-profiles/{profile_id}")
|
||||||
|
def admin_update_profile(profile_id: str, body: LlmProfileWrite, session: dict = Depends(require_admin_dep)):
|
||||||
|
try:
|
||||||
|
profile = update_profile(
|
||||||
|
profile_id,
|
||||||
|
title=body.title,
|
||||||
|
name=body.name,
|
||||||
|
url=body.url,
|
||||||
|
model=body.model,
|
||||||
|
zdr=body.zdr,
|
||||||
|
no_train=body.no_train,
|
||||||
|
)
|
||||||
|
except SettingsError as exc:
|
||||||
|
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||||
|
return {"profile": profile, **public_status()}
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/llm-profiles/{profile_id}")
|
||||||
|
def admin_delete_profile(profile_id: str, session: dict = Depends(require_admin_dep)):
|
||||||
|
try:
|
||||||
|
delete_profile(profile_id)
|
||||||
|
except SettingsError as exc:
|
||||||
|
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||||
|
return public_status()
|
||||||
|
|
||||||
|
|
||||||
class IdentityWrite(BaseModel):
|
class IdentityWrite(BaseModel):
|
||||||
canonical_label: str = ""
|
canonical_label: str = ""
|
||||||
entity_type: str = "PERSON"
|
entity_type: str = "PERSON"
|
||||||
|
|
@ -270,6 +387,7 @@ def admin_identities(session: dict = Depends(require_admin_dep)):
|
||||||
return {
|
return {
|
||||||
"registry": list_registry(profile_id),
|
"registry": list_registry(profile_id),
|
||||||
"proposals": list_review_proposals(profile_id, include_dismissed=True),
|
"proposals": list_review_proposals(profile_id, include_dismissed=True),
|
||||||
|
"senses": list_senses(profile_id),
|
||||||
"note": "Nur lokale bestätigte Registry und unbestätigte Vorschläge. Kein externer Egress.",
|
"note": "Nur lokale bestätigte Registry und unbestätigte Vorschläge. Kein externer Egress.",
|
||||||
"backup": (
|
"backup": (
|
||||||
"Vor einer Bereinigung die lokale Datei backend/data/kansho.sqlite kopieren. "
|
"Vor einer Bereinigung die lokale Datei backend/data/kansho.sqlite kopieren. "
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,8 @@ from auth import require_auth
|
||||||
from context_builder import build_internal_context
|
from context_builder import build_internal_context
|
||||||
from continuity import checkpoint_usage_session, close_usage_session
|
from continuity import checkpoint_usage_session, close_usage_session
|
||||||
from derived_kinds import catalog
|
from derived_kinds import catalog
|
||||||
from dialogue_turn import run_turn, visible_for_role
|
from detect_learning import pending_for_conversation
|
||||||
|
from dialogue_turn import continue_turn, run_turn, visible_for_role
|
||||||
from engine import EngineError
|
from engine import EngineError
|
||||||
from privacy_gateway import public_error_detail
|
from privacy_gateway import public_error_detail
|
||||||
from dialogue_store import (
|
from dialogue_store import (
|
||||||
|
|
@ -51,6 +52,11 @@ class MessageWrite(BaseModel):
|
||||||
id: str | None = None
|
id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class MaskReviewWrite(BaseModel):
|
||||||
|
review_id: str
|
||||||
|
decisions: list[dict] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
class ThreadWrite(BaseModel):
|
class ThreadWrite(BaseModel):
|
||||||
title: str = ""
|
title: str = ""
|
||||||
status: str = "open"
|
status: str = "open"
|
||||||
|
|
@ -140,6 +146,9 @@ def get_one_conversation(conversation_id: str, session: dict = Depends(require_a
|
||||||
conv = get_conversation(session["profile_id"], conversation_id)
|
conv = get_conversation(session["profile_id"], conversation_id)
|
||||||
conv["messages"] = list_messages(session["profile_id"], conversation_id)
|
conv["messages"] = list_messages(session["profile_id"], conversation_id)
|
||||||
conv["derived"] = list_derived_for_conversation(session["profile_id"], conversation_id)
|
conv["derived"] = list_derived_for_conversation(session["profile_id"], conversation_id)
|
||||||
|
pending = pending_for_conversation(session["profile_id"], conversation_id)
|
||||||
|
if pending:
|
||||||
|
conv["pending_mask_review"] = pending
|
||||||
return conv
|
return conv
|
||||||
except StoreError as exc:
|
except StoreError as exc:
|
||||||
_http(exc)
|
_http(exc)
|
||||||
|
|
@ -172,6 +181,17 @@ def conversation_turn(conversation_id: str, req: MessageWrite, session: dict = D
|
||||||
_http(exc)
|
_http(exc)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/conversations/{conversation_id}/turn/review")
|
||||||
|
def conversation_turn_review(conversation_id: str, req: MaskReviewWrite, session: dict = Depends(require_auth)):
|
||||||
|
try:
|
||||||
|
return visible_for_role(
|
||||||
|
continue_turn(session["profile_id"], conversation_id, req.review_id, req.decisions),
|
||||||
|
session.get("role"),
|
||||||
|
)
|
||||||
|
except (StoreError, EngineError) as exc:
|
||||||
|
_http(exc)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/conversations/{conversation_id}/messages")
|
@router.get("/conversations/{conversation_id}/messages")
|
||||||
def get_messages(conversation_id: str, session: dict = Depends(require_auth)):
|
def get_messages(conversation_id: str, session: dict = Depends(require_auth)):
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,9 @@ from fastapi.responses import FileResponse
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from auth import require_auth
|
from auth import require_auth
|
||||||
|
from detect_learning import pending_for_conversation
|
||||||
from dialogue_store import StoreError, delete_conversation, get_conversation, list_messages
|
from dialogue_store import StoreError, delete_conversation, get_conversation, list_messages
|
||||||
from dialogue_turn import run_turn, visible_for_role
|
from dialogue_turn import continue_turn, run_turn, visible_for_role
|
||||||
from privacy_gateway import GatewayRequest, inspect, public_error_detail
|
from privacy_gateway import GatewayRequest, inspect, public_error_detail
|
||||||
from engine import EngineError
|
from engine import EngineError
|
||||||
from journal_generate import generate_draft
|
from journal_generate import generate_draft
|
||||||
|
|
@ -96,6 +97,11 @@ class TurnWrite(BaseModel):
|
||||||
id: str | None = None
|
id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class MaskReviewWrite(BaseModel):
|
||||||
|
review_id: str
|
||||||
|
decisions: list[dict] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
class GenerateWrite(BaseModel):
|
class GenerateWrite(BaseModel):
|
||||||
conversation_ids: list[str] | None = None
|
conversation_ids: list[str] | None = None
|
||||||
include_existing: bool = False
|
include_existing: bool = False
|
||||||
|
|
@ -286,7 +292,11 @@ def remove_conversation(conversation_id: str, session: dict = Depends(require_au
|
||||||
def read_conversation(conversation_id: str, session: dict = Depends(require_auth)):
|
def read_conversation(conversation_id: str, session: dict = Depends(require_auth)):
|
||||||
try:
|
try:
|
||||||
conv = get_conversation(session["profile_id"], conversation_id)
|
conv = get_conversation(session["profile_id"], conversation_id)
|
||||||
return {"conversation": conv, "messages": list_messages(session["profile_id"], conversation_id)}
|
payload = {"conversation": conv, "messages": list_messages(session["profile_id"], conversation_id)}
|
||||||
|
pending = pending_for_conversation(session["profile_id"], conversation_id)
|
||||||
|
if pending:
|
||||||
|
payload["pending_mask_review"] = pending
|
||||||
|
return payload
|
||||||
except StoreError as exc:
|
except StoreError as exc:
|
||||||
_http(exc)
|
_http(exc)
|
||||||
|
|
||||||
|
|
@ -308,6 +318,17 @@ def conversation_turn(conversation_id: str, body: TurnWrite, session: dict = Dep
|
||||||
_http(exc)
|
_http(exc)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/conversations/{conversation_id}/turn/review")
|
||||||
|
def conversation_turn_review(conversation_id: str, body: MaskReviewWrite, session: dict = Depends(require_auth)):
|
||||||
|
try:
|
||||||
|
return visible_for_role(
|
||||||
|
continue_turn(session["profile_id"], conversation_id, body.review_id, body.decisions),
|
||||||
|
session.get("role"),
|
||||||
|
)
|
||||||
|
except (StoreError, EngineError) as exc:
|
||||||
|
_http(exc)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/generation-settings")
|
@router.get("/generation-settings")
|
||||||
def read_generation_settings(session: dict = Depends(require_auth)):
|
def read_generation_settings(session: dict = Depends(require_auth)):
|
||||||
return settings_payload(session["profile_id"])
|
return settings_payload(session["profile_id"])
|
||||||
|
|
|
||||||
|
|
@ -230,6 +230,38 @@ CREATE TABLE IF NOT EXISTS identity_review_proposals (
|
||||||
UNIQUE (profile_id, observed_label, entity_type)
|
UNIQUE (profile_id, observed_label, entity_type)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS label_senses (
|
||||||
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
normalized_label TEXT NOT NULL,
|
||||||
|
identity_hits INTEGER NOT NULL DEFAULT 0,
|
||||||
|
non_identity_hits INTEGER NOT NULL DEFAULT 0,
|
||||||
|
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
PRIMARY KEY (profile_id, normalized_label)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS label_sense_cues (
|
||||||
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
normalized_label TEXT NOT NULL,
|
||||||
|
cue TEXT NOT NULL,
|
||||||
|
decision TEXT NOT NULL,
|
||||||
|
hits INTEGER NOT NULL DEFAULT 1,
|
||||||
|
excerpt TEXT NOT NULL DEFAULT '',
|
||||||
|
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
PRIMARY KEY (profile_id, normalized_label, cue)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
|
||||||
|
user_message_id TEXT NOT NULL,
|
||||||
|
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||||
|
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_pending_mask_reviews_conv
|
||||||
|
ON pending_mask_reviews (profile_id, conversation_id);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS journal_days (
|
CREATE TABLE IF NOT EXISTS journal_days (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||||
|
|
@ -455,6 +487,18 @@ CREATE TABLE IF NOT EXISTS writing_profile_versions (
|
||||||
UNIQUE (profile_id, seq)
|
UNIQUE (profile_id, seq)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS llm_profiles (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL DEFAULT '',
|
||||||
|
url TEXT NOT NULL DEFAULT '',
|
||||||
|
model TEXT NOT NULL DEFAULT '',
|
||||||
|
zdr INTEGER NOT NULL DEFAULT 1,
|
||||||
|
no_train INTEGER NOT NULL DEFAULT 1,
|
||||||
|
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS provider_settings (
|
CREATE TABLE IF NOT EXISTS provider_settings (
|
||||||
role TEXT PRIMARY KEY CHECK (role IN ('generate', 'detect')),
|
role TEXT PRIMARY KEY CHECK (role IN ('generate', 'detect')),
|
||||||
name TEXT NOT NULL DEFAULT '',
|
name TEXT NOT NULL DEFAULT '',
|
||||||
|
|
@ -462,6 +506,7 @@ CREATE TABLE IF NOT EXISTS provider_settings (
|
||||||
model TEXT NOT NULL DEFAULT '',
|
model TEXT NOT NULL DEFAULT '',
|
||||||
zdr INTEGER NOT NULL DEFAULT 1,
|
zdr INTEGER NOT NULL DEFAULT 1,
|
||||||
no_train INTEGER NOT NULL DEFAULT 1,
|
no_train INTEGER NOT NULL DEFAULT 1,
|
||||||
|
profile_id TEXT,
|
||||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -43,6 +43,9 @@ TABLES = [
|
||||||
"re_grounding_events",
|
"re_grounding_events",
|
||||||
"identity_mappings",
|
"identity_mappings",
|
||||||
"identity_review_proposals",
|
"identity_review_proposals",
|
||||||
|
"label_senses",
|
||||||
|
"label_sense_cues",
|
||||||
|
"pending_mask_reviews",
|
||||||
"journal_days",
|
"journal_days",
|
||||||
"journal_drafts",
|
"journal_drafts",
|
||||||
"journal_entries",
|
"journal_entries",
|
||||||
|
|
@ -62,6 +65,7 @@ TABLES = [
|
||||||
"writing_profile_evidence",
|
"writing_profile_evidence",
|
||||||
"writing_profile_reviews",
|
"writing_profile_reviews",
|
||||||
"writing_profile_versions",
|
"writing_profile_versions",
|
||||||
|
"llm_profiles",
|
||||||
"provider_settings",
|
"provider_settings",
|
||||||
"journal_generation_selection",
|
"journal_generation_selection",
|
||||||
"generation_guidelines",
|
"generation_guidelines",
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ UNIT_MODULES = frozenset(
|
||||||
"test_local_backup",
|
"test_local_backup",
|
||||||
"test_provenance",
|
"test_provenance",
|
||||||
"test_model_catalog",
|
"test_model_catalog",
|
||||||
|
"test_local_url",
|
||||||
"test_privacy_detect_eval",
|
"test_privacy_detect_eval",
|
||||||
"test_journal_shape",
|
"test_journal_shape",
|
||||||
"test_journal_body",
|
"test_journal_body",
|
||||||
|
|
|
||||||
319
backend/tests/test_detect_learning.py
Normal file
319
backend/tests/test_detect_learning.py
Normal file
|
|
@ -0,0 +1,319 @@
|
||||||
|
"""Transitional learning detect: senses from dialogue review, admin mode switch.
|
||||||
|
|
||||||
|
Fake detect/provider only. Run from backend/: python tests/test_detect_learning.py
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
|
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from detect_learning import (
|
||||||
|
cue_for_mention,
|
||||||
|
excerpt_view,
|
||||||
|
get_cue_decision,
|
||||||
|
get_detect_operating_mode,
|
||||||
|
get_sense,
|
||||||
|
record_sense,
|
||||||
|
)
|
||||||
|
from dialogue_turn import visible_for_role
|
||||||
|
from identity_store import list_confirmed_identities
|
||||||
|
from main import app
|
||||||
|
from privacy_gateway import reset_debug
|
||||||
|
|
||||||
|
|
||||||
|
def expect(ok: bool, message: str) -> None:
|
||||||
|
if not ok:
|
||||||
|
raise SystemExit(f"FAIL: {message}")
|
||||||
|
print(f"OK {message}")
|
||||||
|
|
||||||
|
|
||||||
|
def status_ok(response, label: str) -> None:
|
||||||
|
if response.status_code != 200:
|
||||||
|
detail = response.json() if response.headers.get("content-type", "").startswith("application/json") else {}
|
||||||
|
code = detail.get("detail", {}).get("code") if isinstance(detail, dict) else None
|
||||||
|
expect(False, f"{label} HTTP {response.status_code} {code or ''}".strip())
|
||||||
|
expect(True, label)
|
||||||
|
|
||||||
|
|
||||||
|
def header(token: str) -> dict:
|
||||||
|
return {"X-Auth-Token": token}
|
||||||
|
|
||||||
|
|
||||||
|
def open_conv(client: TestClient, headers: dict) -> str:
|
||||||
|
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"})
|
||||||
|
day = client.post(
|
||||||
|
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||||
|
headers=headers,
|
||||||
|
json={"calendar_date": "2026-09-08"},
|
||||||
|
)
|
||||||
|
conv = client.post(
|
||||||
|
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||||
|
headers=headers,
|
||||||
|
json={"title": "Gespräch"},
|
||||||
|
)
|
||||||
|
return conv.json()["id"]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
reset_debug()
|
||||||
|
with TestClient(app) as client:
|
||||||
|
setup = client.post(
|
||||||
|
"/api/auth/setup",
|
||||||
|
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||||
|
)
|
||||||
|
headers = header(setup.json()["token"])
|
||||||
|
profile_id = setup.json()["profile_id"]
|
||||||
|
|
||||||
|
providers = client.get("/api/admin/providers", headers=headers)
|
||||||
|
expect(providers.status_code == 200, "providers status")
|
||||||
|
expect(providers.json()["detect_operating_mode"] == "semantic", "default detect mode is semantic")
|
||||||
|
expect(get_detect_operating_mode() == "semantic", "store default is semantic")
|
||||||
|
|
||||||
|
bad = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "wordlist"})
|
||||||
|
expect(bad.status_code == 400, "invalid detect mode is rejected")
|
||||||
|
|
||||||
|
conv_semantic = open_conv(client, headers)
|
||||||
|
semantic_turn = client.post(
|
||||||
|
f"/api/journal/conversations/{conv_semantic}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Anna kam vorbei."},
|
||||||
|
)
|
||||||
|
status_ok(semantic_turn, "semantic turn")
|
||||||
|
expect(not semantic_turn.json().get("pending_mask_review"), "semantic mode does not pause for review")
|
||||||
|
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "semantic turn gets an assistant")
|
||||||
|
|
||||||
|
switched = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning"})
|
||||||
|
status_ok(switched, "switch to learning")
|
||||||
|
expect(switched.json()["detect_operating_mode"] == "learning", "learning mode persisted")
|
||||||
|
|
||||||
|
conv = open_conv(client, headers)
|
||||||
|
paused = client.post(
|
||||||
|
f"/api/journal/conversations/{conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Anna kam vorbei."},
|
||||||
|
)
|
||||||
|
status_ok(paused, "learning pause")
|
||||||
|
review = paused.json().get("pending_mask_review") or {}
|
||||||
|
expect(bool(review.get("id")), "learning mode returns a review id")
|
||||||
|
expect(paused.json().get("assistant") is None, "paused turn has no assistant")
|
||||||
|
candidates = review.get("candidates") or []
|
||||||
|
anna = next((item for item in candidates if item.get("text") == "Anna"), None)
|
||||||
|
expect(anna is not None, "Anna is a review candidate")
|
||||||
|
hidden = visible_for_role({"pending_mask_review": review, "trace": {"egress": "x"}}, "user")
|
||||||
|
expect("pending_mask_review" in hidden, "users still receive the mask review")
|
||||||
|
expect("trace" not in hidden, "users do not receive the detect trace")
|
||||||
|
|
||||||
|
reloaded = client.get(f"/api/journal/conversations/{conv}", headers=headers)
|
||||||
|
expect(reloaded.json().get("pending_mask_review", {}).get("id") == review["id"], "pending review survives reload")
|
||||||
|
|
||||||
|
missing = client.post(
|
||||||
|
f"/api/journal/conversations/{conv}/turn/review",
|
||||||
|
headers=headers,
|
||||||
|
json={"review_id": "missing", "decisions": []},
|
||||||
|
)
|
||||||
|
expect(missing.status_code == 404, "unknown review is gone")
|
||||||
|
|
||||||
|
identity = client.post(
|
||||||
|
f"/api/journal/conversations/{conv}/turn/review",
|
||||||
|
headers=headers,
|
||||||
|
json={
|
||||||
|
"review_id": review["id"],
|
||||||
|
"decisions": [{"id": anna["id"], "decision": "identity"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
status_ok(identity, "identity review")
|
||||||
|
expect(not identity.json().get("pending_mask_review"), "review continue clears the pause")
|
||||||
|
expect(any(item.get("role") == "assistant" for item in identity.json().get("messages") or []), "continue produces an assistant")
|
||||||
|
confirmed = list_confirmed_identities(profile_id)
|
||||||
|
expect(any((item.get("canonical_label") or "").casefold() == "anna" for item in confirmed), "identity review confirms the registry")
|
||||||
|
sense = get_sense(profile_id, "Anna")
|
||||||
|
expect(sense["identity_hits"] >= 1 and not sense["ambiguous"], "Anna is identity-only")
|
||||||
|
|
||||||
|
second = client.post(
|
||||||
|
f"/api/journal/conversations/{conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Anna hat später angerufen."},
|
||||||
|
)
|
||||||
|
status_ok(second, "second Anna turn")
|
||||||
|
expect(not second.json().get("pending_mask_review"), "identity-only spelling skips the popup")
|
||||||
|
|
||||||
|
food_conv = open_conv(client, headers)
|
||||||
|
food = client.post(
|
||||||
|
f"/api/journal/conversations/{food_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Hanna aß mit uns."},
|
||||||
|
)
|
||||||
|
status_ok(food, "hanna pause")
|
||||||
|
food_review = food.json().get("pending_mask_review") or {}
|
||||||
|
food_candidates = food_review.get("candidates") or []
|
||||||
|
hanna = next((item for item in food_candidates if item.get("text") == "Hanna"), None)
|
||||||
|
expect(hanna is not None, "Hanna is a review candidate")
|
||||||
|
declined = client.post(
|
||||||
|
f"/api/journal/conversations/{food_conv}/turn/review",
|
||||||
|
headers=headers,
|
||||||
|
json={
|
||||||
|
"review_id": food_review["id"],
|
||||||
|
"decisions": [{"id": hanna["id"], "decision": "not_identity"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
status_ok(declined, "not_identity review")
|
||||||
|
expect(not any((item.get("canonical_label") or "").casefold() == "hanna" for item in list_confirmed_identities(profile_id)), "not_identity does not confirm Hanna")
|
||||||
|
expect(get_sense(profile_id, "Hanna")["non_identity_hits"] >= 1, "not_identity records a sense")
|
||||||
|
hanna_again = client.post(
|
||||||
|
f"/api/journal/conversations/{food_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Hanna aß mit uns."},
|
||||||
|
)
|
||||||
|
status_ok(hanna_again, "second Hanna turn")
|
||||||
|
expect(not hanna_again.json().get("pending_mask_review"), "known non-identity skips the popup")
|
||||||
|
|
||||||
|
record_sense(profile_id, "Clarissa", identity=True)
|
||||||
|
record_sense(profile_id, "Clarissa", identity=False)
|
||||||
|
expect(get_sense(profile_id, "Clarissa")["ambiguous"], "both senses mark Clarissa as ambiguous")
|
||||||
|
amb_conv = open_conv(client, headers)
|
||||||
|
with patch("detect_learning.try_local_passage_decision", return_value="not_identity"):
|
||||||
|
amb = client.post(
|
||||||
|
f"/api/journal/conversations/{amb_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Clarissa lag auf dem Tisch."},
|
||||||
|
)
|
||||||
|
status_ok(amb, "ambiguous local passage")
|
||||||
|
expect(not amb.json().get("pending_mask_review"), "local passage can finish without a popup")
|
||||||
|
expect(any(item.get("role") == "assistant" for item in amb.json().get("messages") or []), "local passage still generates")
|
||||||
|
|
||||||
|
kin_conv = open_conv(client, headers)
|
||||||
|
kin = client.post(
|
||||||
|
f"/api/journal/conversations/{kin_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Ich war mit meinem Sohn Leon im Park."},
|
||||||
|
)
|
||||||
|
status_ok(kin, "kinship Leon pause")
|
||||||
|
kin_review = kin.json().get("pending_mask_review") or {}
|
||||||
|
kin_names = [item.get("text") for item in kin_review.get("candidates") or []]
|
||||||
|
expect("Leon" in kin_names, "Sohn Leon is offered even when Detect does not report it")
|
||||||
|
leon = next((item for item in kin_review.get("candidates") or [] if item.get("text") == "Leon"), None)
|
||||||
|
expect(leon is not None, "Leon candidate id is present")
|
||||||
|
kin_done = client.post(
|
||||||
|
f"/api/journal/conversations/{kin_conv}/turn/review",
|
||||||
|
headers=headers,
|
||||||
|
json={
|
||||||
|
"review_id": kin_review["id"],
|
||||||
|
"decisions": [{"id": leon["id"], "decision": "identity"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
status_ok(kin_done, "Leon identity review")
|
||||||
|
kin_again = client.post(
|
||||||
|
f"/api/journal/conversations/{kin_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={"body": "Ich war mit meinem Sohn Leon im Park."},
|
||||||
|
)
|
||||||
|
status_ok(kin_again, "second Leon turn")
|
||||||
|
expect(not kin_again.json().get("pending_mask_review"), "confirmed Sohn Leon skips the popup")
|
||||||
|
|
||||||
|
sushi_conv = open_conv(client, headers)
|
||||||
|
sushi = client.post(
|
||||||
|
f"/api/journal/conversations/{sushi_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={
|
||||||
|
"body": (
|
||||||
|
"Gestern bin ich mit meiner Frau Sushi und meinem Sohn Rohan "
|
||||||
|
"Sushi essen gegangen. Das Restaurant hat mich dabei total beeindruckt."
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
status_ok(sushi, "homonym sentence pause")
|
||||||
|
sushi_names = [item.get("text") for item in (sushi.json().get("pending_mask_review") or {}).get("candidates") or []]
|
||||||
|
expect("Rohan" in sushi_names, "Rohan from Sohn is a candidate")
|
||||||
|
expect("Sushi" in sushi_names, "Sushi from Frau is a candidate")
|
||||||
|
expect("Restaurant" not in sushi_names, "plain Restaurant is not kinship-offered")
|
||||||
|
sushi_hits = [
|
||||||
|
item
|
||||||
|
for item in (sushi.json().get("pending_mask_review") or {}).get("candidates") or []
|
||||||
|
if item.get("text") == "Sushi"
|
||||||
|
]
|
||||||
|
expect(len(sushi_hits) >= 2, "both Sushi mentions are offered")
|
||||||
|
expect(
|
||||||
|
sushi_hits[0].get("highlight_start") != sushi_hits[1].get("highlight_start"),
|
||||||
|
"the two Sushi mentions highlight different offsets",
|
||||||
|
)
|
||||||
|
for item in sushi_hits:
|
||||||
|
excerpt = item.get("excerpt") or ""
|
||||||
|
start = item.get("highlight_start")
|
||||||
|
end = item.get("highlight_end")
|
||||||
|
expect(excerpt[start:end] == "Sushi", "highlight covers the Sushi token")
|
||||||
|
|
||||||
|
homonym = (
|
||||||
|
"Gestern bin ich mit meiner Frau Sushi und meinem Sohn Rohan "
|
||||||
|
"Sushi essen gegangen."
|
||||||
|
)
|
||||||
|
first_at = homonym.index("Sushi")
|
||||||
|
second_at = homonym.rindex("Sushi")
|
||||||
|
first = excerpt_view(homonym, first_at, first_at + 5, "Sushi")
|
||||||
|
second = excerpt_view(homonym, second_at, second_at + 5, "Sushi")
|
||||||
|
expect(first["highlight_start"] < second["highlight_start"], "person Sushi sits left of dish Sushi")
|
||||||
|
expect(first["excerpt"][first["highlight_start"]:first["highlight_end"]] == "Sushi", "first highlight is Sushi")
|
||||||
|
expect(second["excerpt"][second["highlight_start"]:second["highlight_end"]] == "Sushi", "second highlight is Sushi")
|
||||||
|
prefix = "SYSTEM\n"
|
||||||
|
mapped = excerpt_view(
|
||||||
|
homonym,
|
||||||
|
len(prefix) + second_at,
|
||||||
|
len(prefix) + second_at + 5,
|
||||||
|
"Sushi",
|
||||||
|
prefix + homonym,
|
||||||
|
)
|
||||||
|
expect(mapped["highlight_start"] == second["highlight_start"], "rendered detect offsets map to the dish Sushi")
|
||||||
|
expect(cue_for_mention(homonym, first_at, first_at + 5, "Sushi") == "frau", "person Sushi attaches to Frau")
|
||||||
|
expect(cue_for_mention(homonym, second_at, second_at + 5, "Sushi") == "rohan", "dish Sushi stays distinct from Frau")
|
||||||
|
appos = "Sushi, meine Frau war erkältet."
|
||||||
|
appos_at = appos.index("Sushi")
|
||||||
|
expect(cue_for_mention(appos, appos_at, appos_at + 5, "Sushi") == "frau", "apposition Sushi, meine Frau binds Frau")
|
||||||
|
far = "Meine Frau und ich aßen Sushi."
|
||||||
|
far_at = far.rindex("Sushi")
|
||||||
|
expect(cue_for_mention(far, far_at, far_at + 5, "Sushi") != "frau", "Frau elsewhere in the sentence does not bind the dish")
|
||||||
|
|
||||||
|
sushi_review = sushi.json().get("pending_mask_review") or {}
|
||||||
|
sushi_decisions = []
|
||||||
|
for item in sushi_review.get("candidates") or []:
|
||||||
|
label = item.get("text")
|
||||||
|
cue = item.get("cue")
|
||||||
|
if label == "Sushi" and cue != "frau":
|
||||||
|
sushi_decisions.append({"id": item["id"], "decision": "not_identity"})
|
||||||
|
else:
|
||||||
|
sushi_decisions.append({"id": item["id"], "decision": "identity"})
|
||||||
|
sushi_done = client.post(
|
||||||
|
f"/api/journal/conversations/{sushi_conv}/turn/review",
|
||||||
|
headers=headers,
|
||||||
|
json={"review_id": sushi_review["id"], "decisions": sushi_decisions},
|
||||||
|
)
|
||||||
|
status_ok(sushi_done, "homonym review")
|
||||||
|
expect(get_cue_decision(profile_id, "Sushi", "frau") == "identity", "Frau Sushi is stored as identity")
|
||||||
|
expect(get_cue_decision(profile_id, "Sushi", "rohan") == "not_identity", "dish Sushi after Rohan is stored")
|
||||||
|
expect(get_cue_decision(profile_id, "Rohan", "sohn") == "identity", "Sohn Rohan cue is stored")
|
||||||
|
sushi_again = client.post(
|
||||||
|
f"/api/journal/conversations/{sushi_conv}/turn",
|
||||||
|
headers=headers,
|
||||||
|
json={
|
||||||
|
"body": (
|
||||||
|
"Gestern bin ich mit meiner Frau Sushi und meinem Sohn Rohan "
|
||||||
|
"Sushi essen gegangen. Das Restaurant hat mich dabei total beeindruckt."
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
status_ok(sushi_again, "repeat homonym sentence")
|
||||||
|
expect(not sushi_again.json().get("pending_mask_review"), "same reviewed sentence does not re-ask")
|
||||||
|
|
||||||
|
back = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "semantic"})
|
||||||
|
expect(back.json()["detect_operating_mode"] == "semantic", "mode can return to semantic")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -89,6 +89,7 @@ def main() -> None:
|
||||||
expect(set(roles) == {"generate", "detect"}, "two provider roles")
|
expect(set(roles) == {"generate", "detect"}, "two provider roles")
|
||||||
expect(providers.json()["remote_plaintext_allowed"] is True, "test runtime allows remote detect")
|
expect(providers.json()["remote_plaintext_allowed"] is True, "test runtime allows remote detect")
|
||||||
expect(providers.json()["remote_plaintext_reason"] == "non_production", "tests are not production")
|
expect(providers.json()["remote_plaintext_reason"] == "non_production", "tests are not production")
|
||||||
|
expect(providers.json().get("detect_operating_mode") == "semantic", "detect mode defaults to semantic")
|
||||||
expect(roles["generate"]["ready"] is False, "generate fail-closed without key")
|
expect(roles["generate"]["ready"] is False, "generate fail-closed without key")
|
||||||
expect("sk-" not in providers.text, "provider status has no secret")
|
expect("sk-" not in providers.text, "provider status has no secret")
|
||||||
expect("KANSHO_PROVIDER_KEY=" not in providers.text, "env assignment not leaked")
|
expect("KANSHO_PROVIDER_KEY=" not in providers.text, "env assignment not leaked")
|
||||||
|
|
@ -107,6 +108,37 @@ def main() -> None:
|
||||||
expect(saved.status_code == 200, f"save generate settings {saved.text}")
|
expect(saved.status_code == 200, f"save generate settings {saved.text}")
|
||||||
saved_roles = {item["role"]: item for item in saved.json()["roles"]}
|
saved_roles = {item["role"]: item for item in saved.json()["roles"]}
|
||||||
expect(saved_roles["generate"]["model"] == "openai/gpt-4o", "generate model persisted")
|
expect(saved_roles["generate"]["model"] == "openai/gpt-4o", "generate model persisted")
|
||||||
|
expect("profiles" in saved.json(), "llm profiles are listed")
|
||||||
|
titles = {item["title"] for item in saved.json()["profiles"]}
|
||||||
|
expect("Ollama (LAN)" in titles, "LAN Ollama profile is seeded")
|
||||||
|
ollama = next(item for item in saved.json()["profiles"] if item["title"] == "Ollama (LAN)")
|
||||||
|
expect(ollama["local"] is True, "LAN Ollama is a local trusted endpoint")
|
||||||
|
expect(ollama["url"].startswith("http://192.168.2.144:11434"), "Ollama URL is the agreed LAN host")
|
||||||
|
|
||||||
|
switched = client.put(
|
||||||
|
"/api/admin/llm-profiles/" + ollama["id"],
|
||||||
|
headers=headers,
|
||||||
|
json={
|
||||||
|
"title": "Ollama (LAN)",
|
||||||
|
"name": "ollama",
|
||||||
|
"url": ollama["url"],
|
||||||
|
"model": "llama3.1:latest",
|
||||||
|
"zdr": True,
|
||||||
|
"no_train": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
expect(switched.status_code == 200, f"save ollama model {switched.text}")
|
||||||
|
activated = client.post(
|
||||||
|
"/api/admin/providers/detect/activate",
|
||||||
|
headers=headers,
|
||||||
|
json={"profile_id": ollama["id"]},
|
||||||
|
)
|
||||||
|
expect(activated.status_code == 200, f"activate ollama detect {activated.text}")
|
||||||
|
detect_role = {item["role"]: item for item in activated.json()["roles"]}["detect"]
|
||||||
|
expect(detect_role["local"] is True, "detect uses local ollama after profile switch")
|
||||||
|
expect(detect_role["model"] == "llama3.1:latest", "detect keeps the saved ollama model")
|
||||||
|
generate_role = {item["role"]: item for item in activated.json()["roles"]}["generate"]
|
||||||
|
expect(generate_role["model"] == "openai/gpt-4o", "generate profile stays on its own model")
|
||||||
|
|
||||||
unknown = client.put(
|
unknown = client.put(
|
||||||
"/api/admin/providers/training",
|
"/api/admin/providers/training",
|
||||||
|
|
|
||||||
70
backend/tests/test_local_url.py
Normal file
70
backend/tests/test_local_url.py
Normal file
|
|
@ -0,0 +1,70 @@
|
||||||
|
"""Local trusted-zone URL tests. No database."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
|
||||||
|
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
|
||||||
|
Path(tempfile.gettempdir()).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
from entity_detect import (
|
||||||
|
DETECT_MAX_TOKENS,
|
||||||
|
DETECT_MAX_TOKENS_LOCAL,
|
||||||
|
DETECT_TIMEOUT,
|
||||||
|
DETECT_TIMEOUT_LOCAL,
|
||||||
|
detect_chat_limits,
|
||||||
|
)
|
||||||
|
from providers import ProviderConfig, is_local_url
|
||||||
|
|
||||||
|
|
||||||
|
def expect(ok: bool, message: str) -> None:
|
||||||
|
if not ok:
|
||||||
|
raise SystemExit(f"FAIL: {message}")
|
||||||
|
print(f"OK {message}")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
expect(is_local_url("http://127.0.0.1:11434/v1/chat/completions"), "loopback is local")
|
||||||
|
expect(is_local_url("http://localhost:11434/v1/chat/completions"), "localhost is local")
|
||||||
|
expect(is_local_url("http://192.168.2.144:11434/v1/chat/completions"), "RFC1918 Ollama host is local")
|
||||||
|
expect(is_local_url("http://10.0.0.8:11434/v1/chat/completions"), "10/8 is local")
|
||||||
|
expect(is_local_url("http://ollama:11434/v1/chat/completions"), "compose hostname ollama is local")
|
||||||
|
expect(not is_local_url("https://openrouter.ai/api/v1/chat/completions"), "OpenRouter is not local")
|
||||||
|
expect(not is_local_url("https://8.8.8.8/v1/chat/completions"), "public IP is not local")
|
||||||
|
remote = ProviderConfig(
|
||||||
|
role="detect",
|
||||||
|
name="openrouter",
|
||||||
|
mode="http",
|
||||||
|
url="https://openrouter.ai/api/v1/chat/completions",
|
||||||
|
model="openai/gpt-4.1-nano",
|
||||||
|
key="x",
|
||||||
|
local=False,
|
||||||
|
zdr=True,
|
||||||
|
no_train=True,
|
||||||
|
)
|
||||||
|
local = ProviderConfig(
|
||||||
|
role="detect",
|
||||||
|
name="ollama",
|
||||||
|
mode="http",
|
||||||
|
url="http://192.168.2.144:11434/v1/chat/completions",
|
||||||
|
model="phi3:mini",
|
||||||
|
key="",
|
||||||
|
local=True,
|
||||||
|
zdr=True,
|
||||||
|
no_train=True,
|
||||||
|
)
|
||||||
|
expect(detect_chat_limits(remote) == (DETECT_TIMEOUT, DETECT_MAX_TOKENS), "remote detect keeps 90s/1024")
|
||||||
|
expect(
|
||||||
|
detect_chat_limits(local) == (DETECT_TIMEOUT_LOCAL, DETECT_MAX_TOKENS_LOCAL),
|
||||||
|
"local detect waits 300s with a 256-token cap",
|
||||||
|
)
|
||||||
|
print("local url tests passed.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -210,6 +210,30 @@ def test_list_provider_models() -> None:
|
||||||
expect(list_provider_models(fake) == [], "fake provider has no catalog")
|
expect(list_provider_models(fake) == [], "fake provider has no catalog")
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_ollama_tags() -> None:
|
||||||
|
reset_catalog()
|
||||||
|
|
||||||
|
def fake_get(url, *args, **kwargs):
|
||||||
|
if str(url).endswith("/api/tags"):
|
||||||
|
return FakeResponse({"models": [{"name": "llama3.1:latest"}, {"name": "qwen2.5:7b"}]})
|
||||||
|
return FakeResponse({"nope": True})
|
||||||
|
|
||||||
|
config = ProviderConfig(
|
||||||
|
role="detect",
|
||||||
|
name="ollama",
|
||||||
|
mode="http",
|
||||||
|
url="http://192.168.2.144:11434/v1/chat/completions",
|
||||||
|
model="llama3.1:latest",
|
||||||
|
key="",
|
||||||
|
local=True,
|
||||||
|
zdr=True,
|
||||||
|
no_train=True,
|
||||||
|
)
|
||||||
|
with patch("model_catalog.httpx.get", fake_get):
|
||||||
|
items = list_provider_models(config)
|
||||||
|
expect([item["id"] for item in items] == ["llama3.1:latest", "qwen2.5:7b"], "ollama tags fill the picker")
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
test_reads_context_and_completion()
|
test_reads_context_and_completion()
|
||||||
test_model_not_in_catalog()
|
test_model_not_in_catalog()
|
||||||
|
|
@ -217,6 +241,7 @@ def main() -> None:
|
||||||
test_cache_hit_and_ttl_expiry()
|
test_cache_hit_and_ttl_expiry()
|
||||||
test_env_fallback_and_fail_closed()
|
test_env_fallback_and_fail_closed()
|
||||||
test_list_provider_models()
|
test_list_provider_models()
|
||||||
|
test_list_ollama_tags()
|
||||||
print("model catalog tests passed.")
|
print("model catalog tests passed.")
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -49,6 +49,8 @@ cp .env.example .env
|
||||||
|
|
||||||
Host-nginx: `nginx/kansho.conf` und `nginx/kansho-dev.conf` nach `/etc/nginx/sites-available/`, dann `nginx/certbot-setup.sh`. DNS A/AAAA für `dev.kansho.jinkendo.de` und `kansho.jinkendo.de` auf den Reverse-Proxy. Bis TLS steht, bleibt der LAN-Zugriff über die Publish-Ports.
|
Host-nginx: `nginx/kansho.conf` und `nginx/kansho-dev.conf` nach `/etc/nginx/sites-available/`, dann `nginx/certbot-setup.sh`. DNS A/AAAA für `dev.kansho.jinkendo.de` und `kansho.jinkendo.de` auf den Reverse-Proxy. Bis TLS steht, bleibt der LAN-Zugriff über die Publish-Ports.
|
||||||
|
|
||||||
|
**Additiv 2026-09-08:** Live-TLS endet auf dem Synology-NAS `192.168.2.63` (DSM Reverse Proxy, nicht auf dem Kanshō-Pi). Pro Kanshō-Regel unter Erweitert Proxy-Lese- und -Sende-Timeout auf 600s. DSM-Default 60s liefert HTTP 504, während Detect/Generate auf dem Pi weiterlaufen. LAN `http://192.168.2.49:3096` umgeht diesen Hop (Frontend-Container bereits 600s). AdGuard-Rewrites bleiben auf `192.168.2.63`.
|
||||||
|
|
||||||
Gitea: Actions aktivieren. Derselbe Pi-Runner wie die Schwesterprodukte (`ubuntu-latest`).
|
Gitea: Actions aktivieren. Derselbe Pi-Runner wie die Schwesterprodukte (`ubuntu-latest`).
|
||||||
|
|
||||||
Watchtower bleibt aus.
|
Watchtower bleibt aus.
|
||||||
|
|
|
||||||
|
|
@ -705,8 +705,24 @@ Das ist kein Admin-Feature-Flag und kein Umdeuten von `KANSHO_ENV=production` na
|
||||||
|
|
||||||
**Entschieden (Modellwahl):** Maskierung und Sprachmodell bleiben zwei getrennte Stufen. URL und Modell sind je Stufe in Admin → Schnittstellen wählbar. Das größere Modell gehört an Generate, nicht zwingend an Detect.
|
**Entschieden (Modellwahl):** Maskierung und Sprachmodell bleiben zwei getrennte Stufen. URL und Modell sind je Stufe in Admin → Schnittstellen wählbar. Das größere Modell gehört an Generate, nicht zwingend an Detect.
|
||||||
|
|
||||||
|
**Additiv 2026-09-08 (LLM-Profile / LAN-Ollama):** Benannte Profile speichern Endpunkt und Modell ohne Keys. Ein Wechsel lädt das gespeicherte Profil. RFC1918-Ollama (`192.168.2.144:11434`) liegt in der lokalen Trusted Zone; Detect dorthin ist kein externes Klartext-Detect.
|
||||||
|
|
||||||
**Später prüfen:** Flag entfernen, sobald lokales Detect auf Prod läuft. Dann wieder Production ohne Klartext-Detect.
|
**Später prüfen:** Flag entfernen, sobald lokales Detect auf Prod läuft. Dann wieder Production ohne Klartext-Detect.
|
||||||
|
|
||||||
|
## 22.4 Übergang: Detect-Lernmodus (2026-09-08)
|
||||||
|
|
||||||
|
Additiv. Technische Abbildung: `../technical/privacy_gateway.md` §9.7.
|
||||||
|
|
||||||
|
**Entschieden (Übergang):** Doppeldeutigkeiten werden nicht als eigene Wortliste gepflegt. Sinne entstehen aus Bestätigungen im Dialog (Identität / nicht schützenswert). Beides bei derselben Schreibweise markiert sie als mehrdeutig.
|
||||||
|
|
||||||
|
**Entschieden (Übergang):** Im Admin konfigurierbarer Detector-Modus `semantic` (Default) oder `learning`. Lernmodus öffnet vor Generate ein Bestätigungs-Popup für Detect-Treffer im aktuellen Nutzersatz. Das ist mehr Arbeit am Anfang und kein Ersatz für semantische Detection.
|
||||||
|
|
||||||
|
**Entschieden (Übergang):** Eine Mini-Passage an ein internes Modell geht nur bei bereits mehrdeutiger Schreibweise und nur an ein lokales Detect. OpenRouter sieht diese Passage nicht. Fehlt das lokale Modell, bleibt das Popup.
|
||||||
|
|
||||||
|
**Additiv 2026-09-10:** Eine bestätigte Nennung wird nicht erneut gefragt, wenn derselbe lokale Kontext wiederkehrt. Enge Verwandtschaftsbindung in der Nominalphrase (auch Apposition: `Sushi, meine Frau`) ist ein Hinweis, nicht die ganze Wahrheit. Der gespeicherte Ausschnitt ist der Kontext für eine lokale Vorentscheidung. Ein späteres lokales GLiNER/Detect sieht diese Mini-Passage plus bestätigte Beispiele; OpenRouter sieht sie nicht. Unbekannte Kontexte bleiben prüfpflichtig. Das ist kein Wortlisten-Editor.
|
||||||
|
|
||||||
|
**Nicht:** Gateway abschalten, Detect-Treffer auto-speichern, `Sushi_`/`Sushi+` im Nutzertext, Pattern-Wortliste als Wahrheit.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# 23. Externe Referenzquellen
|
# 23. Externe Referenzquellen
|
||||||
|
|
|
||||||
|
|
@ -53,6 +53,10 @@ Nicht übernehmen: Mitai-Admin für Körpertarife, Coupons, Training Types als K
|
||||||
|
|
||||||
**Additiv 2026-09-08:** Zwei Stufen mit je eigenem Modell. `GET /api/admin/providers/models` liefert den Katalog zur Auswahl (soft-fail). `remote_plaintext_reason` macht den Production-Übergang `KANSHO_ALLOW_REMOTE_DETECT` sichtbar, setzt ihn aber nicht.
|
**Additiv 2026-09-08:** Zwei Stufen mit je eigenem Modell. `GET /api/admin/providers/models` liefert den Katalog zur Auswahl (soft-fail). `remote_plaintext_reason` macht den Production-Übergang `KANSHO_ALLOW_REMOTE_DETECT` sichtbar, setzt ihn aber nicht.
|
||||||
|
|
||||||
|
**Additiv 2026-09-08 (LLM-Profile):** `GET/POST/PUT/DELETE /api/admin/llm-profiles`, `POST /api/admin/providers/{role}/activate`. Profile enthalten URL/Modell/ZDR, niemals Keys. LAN-Ollama (`192.168.2.144:11434`) ist lokal.
|
||||||
|
|
||||||
|
**Additiv 2026-09-08 (Detect-Lernmodus):** `PUT /api/admin/providers/detect-mode` mit `semantic` | `learning`. Statusfeld `detect_operating_mode`. Keine Wortlisten-Seite. Identitäten dürfen ein Badge „mehrdeutig“ aus Dialog-Sinnen zeigen. Dialog und Journal-Gespräch pausieren im Lernmodus mit Bestätigungs-Popup; Journal-Generate nicht.
|
||||||
|
|
||||||
## 4.2 Implementierungsstand (Dialog-Testspur)
|
## 4.2 Implementierungsstand (Dialog-Testspur)
|
||||||
|
|
||||||
**Additiv 2026-08-26:** Lokale Identitätsregistry unter `/admin/identities`. Detect-Vorschläge sind unbestätigt. Compact-Diagnose enthält Detect-Abdeckung, Chunks, Kosten und Laufzeit, aber keine Labels.
|
**Additiv 2026-08-26:** Lokale Identitätsregistry unter `/admin/identities`. Detect-Vorschläge sind unbestätigt. Compact-Diagnose enthält Detect-Abdeckung, Chunks, Kosten und Laufzeit, aber keine Labels.
|
||||||
|
|
|
||||||
|
|
@ -139,6 +139,8 @@ UI: `/admin/generation` mit getrennten Dimensionen, Karten und einem Editor nach
|
||||||
|
|
||||||
**Status: Code vorhanden.** `GET /api/admin/providers`, `PUT /api/admin/providers/{generate|detect}`. Session-Admin. Body darf einen write-only `key` enthalten; die Antwort enthält ihn nicht. Persistenz: URL/Modell/Flags in `provider_settings`, Key in `backend/.env`.
|
**Status: Code vorhanden.** `GET /api/admin/providers`, `PUT /api/admin/providers/{generate|detect}`. Session-Admin. Body darf einen write-only `key` enthalten; die Antwort enthält ihn nicht. Persistenz: URL/Modell/Flags in `provider_settings`, Key in `backend/.env`.
|
||||||
|
|
||||||
|
**Additiv 2026-09-08:** `llm_profiles`. `POST /api/admin/providers/{role}/activate`, CRUD unter `/api/admin/llm-profiles`. Keys weiterhin nicht in der DB.
|
||||||
|
|
||||||
## 5.4 Implementierungsstand (Debug-Persistenz)
|
## 5.4 Implementierungsstand (Debug-Persistenz)
|
||||||
|
|
||||||
**Additiv 2026-08-28.** Session-Admin. Default aus. Nur eigenes Profil in Liste/Export.
|
**Additiv 2026-08-28.** Session-Admin. Default aus. Nur eigenes Profil in Liste/Export.
|
||||||
|
|
|
||||||
|
|
@ -179,7 +179,7 @@ Editor speichert Markdown, nicht HTML. Medien-Token bleiben lokal. Dirty: In-App
|
||||||
|
|
||||||
| Schicht | Rolle | Call |
|
| Schicht | Rolle | Call |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Maskierung | `detect` | Vollständige semantische Detection des Generate-Egress. Externes Klartext-Detect in Development/Test; in Production default-off, optional Operator-Übergang `KANSHO_ALLOW_REMOTE_DETECT`. Ziel: lokales Detect-Modell. Kein Pattern-Fallback. |
|
| Maskierung | `detect` | Vollständige semantische Detection des Generate-Egress. Externes Klartext-Detect in Development/Test; in Production default-off, optional Operator-Übergang `KANSHO_ALLOW_REMOTE_DETECT`. Ziel: lokales Detect-Modell. Kein Pattern-Fallback. Admin-Modus `semantic` (Default) oder `learning` (Übergang, Bestätigung vor Generate). |
|
||||||
| Dialogzug | `generate` | Operation + Impuls |
|
| Dialogzug | `generate` | Operation + Impuls |
|
||||||
| Journalentwurf | `generate` | Explizit, getrennt |
|
| Journalentwurf | `generate` | Explizit, getrennt |
|
||||||
|
|
||||||
|
|
@ -258,7 +258,7 @@ Additiv zum Slice, 2026-08-25. Kein Target-Model-Vorbau.
|
||||||
- Provenance von Draft und Entry-Version ist relational (`journal_*_source_refs`); JSON-ID-Listen werden verlustfrei migriert. Source-Messages bleiben unangetastet.
|
- Provenance von Draft und Entry-Version ist relational (`journal_*_source_refs`); JSON-ID-Listen werden verlustfrei migriert. Source-Messages bleiben unangetastet.
|
||||||
- Context Builder spricht Selection-Specs; Recency bleibt die aktuelle Retrieval-Heuristik, nicht die fachliche API.
|
- Context Builder spricht Selection-Specs; Recency bleibt die aktuelle Retrieval-Heuristik, nicht die fachliche API.
|
||||||
- Dialogzug kann begrenzte jüngere Original-Conversations desselben Space sehen, ohne denselben Journal Day voll zu laden.
|
- Dialogzug kann begrenzte jüngere Original-Conversations desselben Space sehen, ohne denselben Journal Day voll zu laden.
|
||||||
- Externes Klartext-Detect ist kein Produktmodus (`KANSHO_ENV=production` ohne lokales Detect fail-closed). **Additiv 2026-09-08:** Operator darf mit `KANSHO_ALLOW_REMOTE_DETECT` den Übergang bis Ollama explizit öffnen; ohne Flag bleibt der Block. Lokales Detect bleibt austauschbare Rolle. Detect-Ausgabe persistiert keine aktive Identität.
|
- Externes Klartext-Detect ist kein Produktmodus (`KANSHO_ENV=production` ohne lokales Detect fail-closed). **Additiv 2026-09-08:** Operator darf mit `KANSHO_ALLOW_REMOTE_DETECT` den Übergang bis Ollama explizit öffnen; ohne Flag bleibt der Block. **Additiv 2026-09-08:** RFC1918-Ollama gilt als lokal; LLM-Profile speichern URL/Modell je Endpunkt. Lokales Detect bleibt austauschbare Rolle. Detect-Ausgabe persistiert keine aktive Identität.
|
||||||
- Response Validation blockiert Klartext-Identität vor Demask. Dialog-Egress maskiert den ganzen Prompt inkl. Opening/Space-Kontext; ein Leak bricht den Zug nicht leer ab. Journal-Generate bricht bei einem Leak nach Retry nicht ab: lokales Quellenartefakt bzw. lokaler Entwurf, die leckende Modellantwort wird nicht verwendet.
|
- Response Validation blockiert Klartext-Identität vor Demask. Dialog-Egress maskiert den ganzen Prompt inkl. Opening/Space-Kontext; ein Leak bricht den Zug nicht leer ab. Journal-Generate bricht bei einem Leak nach Retry nicht ab: lokales Quellenartefakt bzw. lokaler Entwurf, die leckende Modellantwort wird nicht verwendet.
|
||||||
- Konsolidierung nutzt lokale Signale, nicht `count >= 2`.
|
- Konsolidierung nutzt lokale Signale, nicht `count >= 2`.
|
||||||
- `/dialog` ist Admin-Harness, nicht Produkt-IA.
|
- `/dialog` ist Admin-Harness, nicht Produkt-IA.
|
||||||
|
|
|
||||||
|
|
@ -124,6 +124,7 @@ Ziel wie Mitai: **Einfachheit in der Nutzer-UX, Differenzierung in administrierb
|
||||||
- UI-Katalog: welche Karten/Nav-Beiträge sichtbar, Layout-Overrides
|
- UI-Katalog: welche Karten/Nav-Beiträge sichtbar, Layout-Overrides
|
||||||
- System-Prompts vs. Instanz-Anpassungen (Reset auf Default)
|
- System-Prompts vs. Instanz-Anpassungen (Reset auf Default)
|
||||||
- Provider-URL, Modell und ZDR/No-Train für Generate und Detect (`provider_settings`, Admin Schnittstellen)
|
- Provider-URL, Modell und ZDR/No-Train für Generate und Detect (`provider_settings`, Admin Schnittstellen)
|
||||||
|
- **Additiv 2026-09-08:** benannte LLM-Profile (`llm_profiles`); aktive Zuordnung je Stufe. Secrets bleiben Env.
|
||||||
- später: Privacy-Profile (Strict/Balanced), sobald fachlich konkret
|
- später: Privacy-Profile (Strict/Balanced), sobald fachlich konkret
|
||||||
|
|
||||||
### 4.2 Code / Env (nicht pro Klick im Admin)
|
### 4.2 Code / Env (nicht pro Klick im Admin)
|
||||||
|
|
|
||||||
|
|
@ -239,7 +239,7 @@ Bestätigte Registry-Zeilen und bestätigte Aliase werden im gesamten gerenderte
|
||||||
|
|
||||||
### Tests und Live-Qualität
|
### Tests und Live-Qualität
|
||||||
|
|
||||||
Contract-Tests: `backend/tests/test_privacy_detect.py`, `backend/tests/test_identity_registry.py`, `backend/tests/test_privacy_response_integrity.py`, `backend/tests/test_detect_contract_retry.py`. Sie beweisen Schema, Fail-closed, span-genaue Maskierung und Datenfluss, nicht semantische Modellleistung. Opt-in: `python entity_detect_eval.py --live` mit synthetischen Sätzen und exakten erwarteten Spans. Ohne diesen Lauf bleibt die Live-Qualität unbestätigt. Das aktuell konfigurierte `openai/gpt-4.1-nano` gilt durch reale False-Positive-Vorschläge qualitativ nicht als zuverlässig bestätigt; das Modell wird deshalb nicht stillschweigend gewechselt.
|
Contract-Tests: `backend/tests/test_privacy_detect.py`, `backend/tests/test_identity_registry.py`, `backend/tests/test_privacy_response_integrity.py`, `backend/tests/test_detect_contract_retry.py`, `backend/tests/test_detect_learning.py`. Sie beweisen Schema, Fail-closed, span-genaue Maskierung, Lernmodus-Pause und Datenfluss, nicht semantische Modellleistung. Opt-in: `python entity_detect_eval.py --live` mit synthetischen Sätzen und exakten erwarteten Spans. Ohne diesen Lauf bleibt die Live-Qualität unbestätigt. Das aktuell konfigurierte `openai/gpt-4.1-nano` gilt durch reale False-Positive-Vorschläge qualitativ nicht als zuverlässig bestätigt; das Modell wird deshalb nicht stillschweigend gewechselt.
|
||||||
|
|
||||||
## 9.6 Übergang: `KANSHO_ALLOW_REMOTE_DETECT` (2026-09-08)
|
## 9.6 Übergang: `KANSHO_ALLOW_REMOTE_DETECT` (2026-09-08)
|
||||||
|
|
||||||
|
|
@ -249,7 +249,25 @@ Default in `KANSHO_ENV=production`: externes Klartext-Detect bleibt fail-closed
|
||||||
|
|
||||||
Operator-Übergang bis Ollama: `KANSHO_ALLOW_REMOTE_DETECT=1` (Compose `.env`, Container neu anlegen). Das Gateway, die Pflicht-Detection vor Generate, ZDR/No-Train für Generate und die Key-Trennung bleiben. Admin kann das Flag nicht setzen. URL und Modell je Stufe bleiben in `provider_settings`.
|
Operator-Übergang bis Ollama: `KANSHO_ALLOW_REMOTE_DETECT=1` (Compose `.env`, Container neu anlegen). Das Gateway, die Pflicht-Detection vor Generate, ZDR/No-Train für Generate und die Key-Trennung bleiben. Admin kann das Flag nicht setzen. URL und Modell je Stufe bleiben in `provider_settings`.
|
||||||
|
|
||||||
Status in `GET /api/admin/providers`: `remote_plaintext_allowed`, `remote_plaintext_reason` (`non_production` | `operator_override` | `blocked`). Modellauswahl: `GET /api/admin/providers/models` (Katalog, soft-fail).
|
**Additiv 2026-09-08 (LLM-Profile, LAN-Ollama):** `llm_profiles` speichert benannte URL/Modell-Presets ohne Keys. Jede Stufe (`generate` / `detect`) zeigt auf ein Profil; Wechsel kopiert die gespeicherten Felder, statt sie neu einzugeben. RFC1918-, Loopback- und `.local`-URLs gelten als lokale Trusted Zone. `http://192.168.2.144:11434/v1/chat/completions` ist damit lokales Detect, kein externes Klartext-Detect. Ollama muss auf dem Host auf `0.0.0.0:11434` lauschen; der Pi muss Port 11434 erreichen.
|
||||||
|
|
||||||
|
**Additiv 2026-09-08 (lokales Detect, Timeouts):** CPU-Ollama braucht für denselben Detect-Prompt deutlich länger als OpenRouter. Remote-Detect bleibt 90s / 1024 Tokens. Lokales Detect wartet 300s und begrenzt auf 256 Tokens. Live-TLS endet auf dem Synology-Reverse-Proxy (`192.168.2.63`, DSM-Default 60s). Dort Proxy-Lese- und -Sende-Timeout 600s setzen, sonst sieht der Browser HTTP 504, während das Backend weiterläuft. Ein Dialogzug kann danach lokal einen Halte-Impuls gespeichert haben. Die Compose-Frontend-Nginx auf Port 3096 hat bereits 600s.
|
||||||
|
|
||||||
|
Status in `GET /api/admin/providers`: `remote_plaintext_allowed`, `remote_plaintext_reason` (`non_production` | `operator_override` | `blocked`), `profiles`, `detect_operating_mode` (`semantic` | `learning`). Modellauswahl: `GET /api/admin/providers/models` (OpenRouter-Katalog oder Ollama `/api/tags`, soft-fail).
|
||||||
|
|
||||||
|
## 9.7 Übergang: Detect-Lernmodus (2026-09-08)
|
||||||
|
|
||||||
|
Additiv. Fachliches Home: `../functional/guardrails.md` §22.4. Ersetzt weder semantische Detection noch die bestätigte Registry.
|
||||||
|
|
||||||
|
Default bleibt `semantic`: Detect läuft wie bisher, Generate folgt ohne Pause. `learning` ist ein Admin-Schalter (`PUT /api/admin/providers/detect-mode`), kein Gateway-Bypass und keine Wortlisten-UI.
|
||||||
|
|
||||||
|
Im Lernmodus untersucht Detect weiterhin den vollen gerenderten Generate-Egress. Bevor Generate startet, werden `request_local`-Spans im aktuellen Nutzersatz zur Bestätigung angeboten (Dialog und Journal-Gespräch, nicht Journal-Generate). Zusätzlich: Nennungen nach Verwandtschaftswörtern der bestehenden Identitätsregel (`Frau`, `Sohn`, …) im Nutzersatz, auch wenn Detect sie als Gericht weglässt oder nicht meldet. Andere Großschreibung (`Restaurant`) wird dadurch nicht angeboten. „Identität“ bestätigt die lokale Registry und zählt einen Identitätssinn. „Nicht schützenswert“ maskiert diese Nennung nicht und zählt den anderen Sinn. Detect-Ausgabe allein speichert weiterhin keine aktive Identität.
|
||||||
|
|
||||||
|
Es gibt keine separat zu pflegende Doppeldeutigkeitsliste. Mehrdeutigkeit entsteht, wenn dieselbe Schreibweise beide Sinne hat. Nur dann darf ein **lokales** Detect-Modell eine Mini-Passage (Ausschnitt um die Nennung) entscheiden. Fehlt ein lokales Modell oder ist die Antwort unbrauchbar, bleibt das Popup. OpenRouter erhält diese Passage nicht.
|
||||||
|
|
||||||
|
**Additiv 2026-09-10:** Bestätigungen speichern den lokalen Cue und den Ausschnitt in `label_sense_cues`. Cue ist die enge Bindung in der Phrase (`Frau Sushi`, `Sushi, meine Frau`), sonst das Wort unmittelbar davor — nicht jedes Verwandtschaftswort irgendwo im Satz, damit Homonyme (`Frau Sushi` / Speise) getrennt bleiben. Der Ausschnitt geht nur an ein **lokales** Detect/GLiNER als Mini-Passage plus bestätigte Beispiele; das ist die Vorentscheidung. GLiNER ist dafür der bevorzugte lokale Weg, in diesem Slice noch nicht verdrahtet. Fehlt das lokale Modell, bleibt das Popup. OpenRouter erhält diese Passage nicht.
|
||||||
|
|
||||||
|
Nach der Bestätigung läuft Generate mit den geprüften Mappings (`precomputed_learning_review`), ohne zweiten Detect-Pass. Compact-Diagnose enthält weiterhin keine Klartextlabels. Tests: `backend/tests/test_detect_learning.py`.
|
||||||
|
|
||||||
## 10. Offene Fragen
|
## 10. Offene Fragen
|
||||||
|
|
||||||
|
|
@ -257,6 +275,8 @@ Status in `GET /api/admin/providers`: `remote_plaintext_allowed`, `remote_plaint
|
||||||
|
|
||||||
**Additiv 2026-08-28:** Wie sollen neue, vom Detect verfehlte und lokal nicht bestätigte Identitäten vor dem Egress behandelt werden, ohne span-genaue Homonyme oder Allgemeinbegriffe dauerhaft zu maskieren? Optionen ohne Vorentscheidung: fail-closed vor Generate, Review der Detect-Vorschläge vor Egress, oder ein zweites lokales Verfahren. Keine Heuristik auf Verdacht.
|
**Additiv 2026-08-28:** Wie sollen neue, vom Detect verfehlte und lokal nicht bestätigte Identitäten vor dem Egress behandelt werden, ohne span-genaue Homonyme oder Allgemeinbegriffe dauerhaft zu maskieren? Optionen ohne Vorentscheidung: fail-closed vor Generate, Review der Detect-Vorschläge vor Egress, oder ein zweites lokales Verfahren. Keine Heuristik auf Verdacht.
|
||||||
|
|
||||||
|
**Additiv 2026-09-08:** Der Detect-Lernmodus ist eine Übergangslösung für die Review-Option, nicht der Endzustand. Semantische Detection bleibt Pflicht. Unbekannte Namen sind nicht deshalb harmlos, weil sie nicht auf einer Liste stehen.
|
||||||
|
|
||||||
## 11. Querverweise
|
## 11. Querverweise
|
||||||
|
|
||||||
- Fachlich: `../functional/guardrails.md`
|
- Fachlich: `../functional/guardrails.md`
|
||||||
|
|
|
||||||
|
|
@ -85,6 +85,8 @@ merge --no-ff develop in main → push main → deploy-prod.yml
|
||||||
|
|
||||||
**Additiv 2026-09-07 (Qualitätssystem):** Backend-Runner ist pytest (`backend/pytest.ini`, `backend/tests/conftest.py`). Gitea startet die Suite erst nach erfolgreichem Dev-Deploy (`workflow_run` auf `Deploy Development`), nicht parallel zum Image-Build. Destruktive Tests bleiben auf `kansho_test`. Zusätzlich ein nicht-schreibender Smoke gegen die laufende Dev-API (`kansho_dev`, nur `/api/health`). Öffentliche URLs nach Host-Nginx/TLS: `https://dev.kansho.jinkendo.de` und `https://kansho.jinkendo.de`. LAN-Ports 3096/8096 und 3006/8005 bleiben die Compose-Publish-Ziele.
|
**Additiv 2026-09-07 (Qualitätssystem):** Backend-Runner ist pytest (`backend/pytest.ini`, `backend/tests/conftest.py`). Gitea startet die Suite erst nach erfolgreichem Dev-Deploy (`workflow_run` auf `Deploy Development`), nicht parallel zum Image-Build. Destruktive Tests bleiben auf `kansho_test`. Zusätzlich ein nicht-schreibender Smoke gegen die laufende Dev-API (`kansho_dev`, nur `/api/health`). Öffentliche URLs nach Host-Nginx/TLS: `https://dev.kansho.jinkendo.de` und `https://kansho.jinkendo.de`. LAN-Ports 3096/8096 und 3006/8005 bleiben die Compose-Publish-Ziele.
|
||||||
|
|
||||||
|
**Additiv 2026-09-08 (TLS-Hop):** DNS für `*.kansho.jinkendo.de` zeigt auf das Synology-NAS `192.168.2.63`. Der Pi hat kein Host-HTTPS auf 443. DSM Reverse Proxy: Proxy-Lese-/Sende-Timeout 600s; Default 60s bricht lokale Detect-Läufe mit HTTP 504 ab. Ziel bleibt der Pi (`3096`/`8096` Dev, `3006`/`8005` Prod). Repo-Dateien `nginx/kansho.conf` und `nginx/kansho-dev.conf` beschreiben denselben Timeout, falls TLS später auf dem Pi endet.
|
||||||
|
|
||||||
Kanshō-Repo liegt auf Gitea (`Lars/Kansho`). HTTPS-Push ist eingerichtet. Der Pi-Runner (`ubuntu-latest`) ist derselbe wie bei Mitai/Shinkan/Kairo.
|
Kanshō-Repo liegt auf Gitea (`Lars/Kansho`). HTTPS-Push ist eingerichtet. Der Pi-Runner (`ubuntu-latest`) ist derselbe wie bei Mitai/Shinkan/Kairo.
|
||||||
|
|
||||||
## 5. Was Deploy nicht übernimmt
|
## 5. Was Deploy nicht übernimmt
|
||||||
|
|
|
||||||
|
|
@ -687,3 +687,24 @@ pre.code {
|
||||||
.runlog-list li.bad { border-color: #e3c0c0; }
|
.runlog-list li.bad { border-color: #e3c0c0; }
|
||||||
.runlog-trace { margin-top: 0.8rem; }
|
.runlog-trace { margin-top: 0.8rem; }
|
||||||
.runlog-trace .trace-panel { border-top: 1px dashed var(--line); }
|
.runlog-trace .trace-panel { border-top: 1px dashed var(--line); }
|
||||||
|
.runlog-modal fieldset.row-actions {
|
||||||
|
border: 0;
|
||||||
|
margin: 0.4rem 0 0;
|
||||||
|
padding: 0;
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 0.8rem;
|
||||||
|
}
|
||||||
|
.mask-excerpt {
|
||||||
|
margin: 0.35rem 0 0;
|
||||||
|
color: var(--ink);
|
||||||
|
font-size: 0.95rem;
|
||||||
|
line-height: 1.45;
|
||||||
|
}
|
||||||
|
.mask-excerpt mark {
|
||||||
|
background: #f3e2b8;
|
||||||
|
color: inherit;
|
||||||
|
font-weight: 600;
|
||||||
|
padding: 0 0.12em;
|
||||||
|
border-radius: 3px;
|
||||||
|
}
|
||||||
|
|
|
||||||
83
frontend/src/components/MaskReviewPanel.jsx
Normal file
83
frontend/src/components/MaskReviewPanel.jsx
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
function HighlightedExcerpt({ excerpt, highlightStart, highlightEnd, label }) {
|
||||||
|
if (!excerpt) return null
|
||||||
|
const start = Number(highlightStart)
|
||||||
|
const end = Number(highlightEnd)
|
||||||
|
if (Number.isInteger(start) && Number.isInteger(end) && start >= 0 && end <= excerpt.length && end > start) {
|
||||||
|
return (
|
||||||
|
<p className="mask-excerpt">
|
||||||
|
{excerpt.slice(0, start)}
|
||||||
|
<mark>{excerpt.slice(start, end)}</mark>
|
||||||
|
{excerpt.slice(end)}
|
||||||
|
</p>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const needle = label || ''
|
||||||
|
const at = needle ? excerpt.toLowerCase().indexOf(needle.toLowerCase()) : -1
|
||||||
|
if (at < 0) return <p className="mask-excerpt">{excerpt}</p>
|
||||||
|
return (
|
||||||
|
<p className="mask-excerpt">
|
||||||
|
{excerpt.slice(0, at)}
|
||||||
|
<mark>{excerpt.slice(at, at + needle.length)}</mark>
|
||||||
|
{excerpt.slice(at + needle.length)}
|
||||||
|
</p>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function MaskReviewPanel({ review, busy, onSubmit }) {
|
||||||
|
if (!review?.candidates?.length) return null
|
||||||
|
return (
|
||||||
|
<div className="runlog-overlay" role="dialog" aria-labelledby="mask-review-title">
|
||||||
|
<div className="runlog-modal">
|
||||||
|
<div className="runlog-head">
|
||||||
|
<div>
|
||||||
|
<h2 id="mask-review-title">Maskierung prüfen</h2>
|
||||||
|
<p className="runlog-status">
|
||||||
|
Lernmodus: nur die gelb markierte Nennung. Bestätigen legt die Bezeichnung lokal ab.
|
||||||
|
„Nicht schützenswert“ verhindert die Maskierung. Beides bei demselben Wort markiert es als mehrdeutig.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<form
|
||||||
|
className="stack"
|
||||||
|
onSubmit={(event) => {
|
||||||
|
event.preventDefault()
|
||||||
|
const data = new FormData(event.currentTarget)
|
||||||
|
const decisions = review.candidates.map((item) => ({
|
||||||
|
id: item.id,
|
||||||
|
decision: data.get(`decision-${item.id}`) || 'identity'
|
||||||
|
}))
|
||||||
|
onSubmit(decisions)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<ul className="stack">
|
||||||
|
{review.candidates.map((item) => (
|
||||||
|
<li key={item.id} className="row-item">
|
||||||
|
<div>
|
||||||
|
<strong>{item.text}</strong>
|
||||||
|
<span className="muted"> · {item.entity_type}{item.ambiguous ? ' · mehrdeutig' : ''}</span>
|
||||||
|
<HighlightedExcerpt
|
||||||
|
excerpt={item.excerpt}
|
||||||
|
highlightStart={item.highlight_start}
|
||||||
|
highlightEnd={item.highlight_end}
|
||||||
|
label={item.text}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<fieldset className="row-actions">
|
||||||
|
<label className="check">
|
||||||
|
<input type="radio" name={`decision-${item.id}`} value="identity" defaultChecked={item.suggested !== 'not_identity'} />
|
||||||
|
Identität
|
||||||
|
</label>
|
||||||
|
<label className="check">
|
||||||
|
<input type="radio" name={`decision-${item.id}`} value="not_identity" defaultChecked={item.suggested === 'not_identity'} />
|
||||||
|
Nicht schützenswert
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
<button type="submit" disabled={busy}>{busy ? 'Übernimmt …' : 'Übernehmen und weiter'}</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
@ -104,6 +104,7 @@ export default function AdminIdentitiesPage() {
|
||||||
<p className="muted">
|
<p className="muted">
|
||||||
Nur lokale bestätigte Registry. Detect-Treffer werden nicht automatisch aktiv.
|
Nur lokale bestätigte Registry. Detect-Treffer werden nicht automatisch aktiv.
|
||||||
Unbestätigte Vorschläge gelten nicht als bekannte Identität. Kein externer Egress.
|
Unbestätigte Vorschläge gelten nicht als bekannte Identität. Kein externer Egress.
|
||||||
|
Mehrdeutigkeit entsteht aus Dialog-Bestätigungen, nicht aus einer Wortlisten-Seite.
|
||||||
</p>
|
</p>
|
||||||
{error && <p className="error">{error}</p>}
|
{error && <p className="error">{error}</p>}
|
||||||
{notice && <p>{notice}</p>}
|
{notice && <p>{notice}</p>}
|
||||||
|
|
@ -120,9 +121,16 @@ export default function AdminIdentitiesPage() {
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{(data?.registry || []).map((item) => (
|
{(data?.registry || []).map((item) => {
|
||||||
|
const sense = (data?.senses || []).find(
|
||||||
|
(entry) => (entry.normalized_label || '').toLowerCase() === (item.canonical_label || '').toLowerCase()
|
||||||
|
)
|
||||||
|
return (
|
||||||
<tr key={item.id}>
|
<tr key={item.id}>
|
||||||
<td>{item.canonical_label}</td>
|
<td>
|
||||||
|
{item.canonical_label}
|
||||||
|
{sense?.ambiguous ? <span className="muted"> · mehrdeutig</span> : null}
|
||||||
|
</td>
|
||||||
<td>
|
<td>
|
||||||
<select
|
<select
|
||||||
value={item.entity_type}
|
value={item.entity_type}
|
||||||
|
|
@ -150,7 +158,8 @@ export default function AdminIdentitiesPage() {
|
||||||
<button type="button" className="ghost" onClick={() => remove(item.id)}>Entfernen</button>
|
<button type="button" className="ghost" onClick={() => remove(item.id)}>Entfernen</button>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
))}
|
)
|
||||||
|
})}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { useEffect, useState } from 'react'
|
import { useEffect, useMemo, useState } from 'react'
|
||||||
import { api } from '../api.js'
|
import { api } from '../api.js'
|
||||||
import { useAuth } from '../context/AuthContext.jsx'
|
import { useAuth } from '../context/AuthContext.jsx'
|
||||||
|
|
||||||
|
|
@ -9,13 +9,20 @@ const emptyRole = () => ({
|
||||||
model: '',
|
model: '',
|
||||||
zdr: true,
|
zdr: true,
|
||||||
no_train: true,
|
no_train: true,
|
||||||
key: ''
|
profile_id: '',
|
||||||
|
key: '',
|
||||||
|
saveAsTitle: ''
|
||||||
})
|
})
|
||||||
|
|
||||||
const FALLBACK_MODELS = {
|
const emptyDraft = () => ({
|
||||||
generate: ['openai/gpt-5.4', 'openai/gpt-4o', 'openai/gpt-4o-mini', 'anthropic/claude-sonnet-4'],
|
id: '',
|
||||||
detect: ['openai/gpt-4.1-nano', 'openai/gpt-4o-mini', 'llama3.1']
|
title: '',
|
||||||
}
|
name: '',
|
||||||
|
url: 'http://192.168.2.144:11434/v1/chat/completions',
|
||||||
|
model: '',
|
||||||
|
zdr: true,
|
||||||
|
no_train: true
|
||||||
|
})
|
||||||
|
|
||||||
function statusLabel(role) {
|
function statusLabel(role) {
|
||||||
if (role.role === 'detect' && !role.url) return 'nicht konfiguriert, fail-closed'
|
if (role.role === 'detect' && !role.url) return 'nicht konfiguriert, fail-closed'
|
||||||
|
|
@ -29,16 +36,21 @@ function statusLabel(role) {
|
||||||
if (role.ready) return 'bereit'
|
if (role.ready) return 'bereit'
|
||||||
if (!role.key_present && !role.local) return 'Key fehlt'
|
if (!role.key_present && !role.local) return 'Key fehlt'
|
||||||
if (!role.url) return 'URL fehlt'
|
if (!role.url) return 'URL fehlt'
|
||||||
|
if (role.local && !role.model) return 'Modell fehlt'
|
||||||
return 'nicht bereit'
|
return 'nicht bereit'
|
||||||
}
|
}
|
||||||
|
|
||||||
function detectBanner(data) {
|
function detectBanner(data) {
|
||||||
if (!data) return ''
|
if (!data) return ''
|
||||||
|
const detect = (data.roles || []).find((role) => role.role === 'detect')
|
||||||
|
if (detect?.local) {
|
||||||
|
return 'Maskierung läuft lokal (LAN/Ollama). Generate sieht nur den maskierten Kontext.'
|
||||||
|
}
|
||||||
if (data.runtime_env === 'production' && data.remote_plaintext_reason === 'operator_override') {
|
if (data.runtime_env === 'production' && data.remote_plaintext_reason === 'operator_override') {
|
||||||
return 'Production erlaubt derzeit externes Klartext-Detect (KANSHO_ALLOW_REMOTE_DETECT). Das Gateway bleibt aktiv; Generate sieht nur maskierten Kontext. Ziel bleibt ein lokales Detect-Modell.'
|
return 'Production erlaubt derzeit externes Klartext-Detect (KANSHO_ALLOW_REMOTE_DETECT). Ziel bleibt ein lokales Detect-Modell.'
|
||||||
}
|
}
|
||||||
if (data.runtime_env === 'production' && data.remote_plaintext_allowed === false) {
|
if (data.runtime_env === 'production' && data.remote_plaintext_allowed === false) {
|
||||||
return 'Production blockiert externes Klartext-Detect. Maskierung braucht eine lokale URL oder die Operator-Freigabe, sonst entsteht kein Impuls.'
|
return 'Production blockiert externes Klartext-Detect. Maskierung braucht ein lokales Profil oder die Operator-Freigabe.'
|
||||||
}
|
}
|
||||||
return ''
|
return ''
|
||||||
}
|
}
|
||||||
|
|
@ -47,17 +59,16 @@ export default function AdminProvidersPage() {
|
||||||
const { session } = useAuth()
|
const { session } = useAuth()
|
||||||
const [data, setData] = useState(null)
|
const [data, setData] = useState(null)
|
||||||
const [forms, setForms] = useState({})
|
const [forms, setForms] = useState({})
|
||||||
const [catalog, setCatalog] = useState({ generate: [], detect: [] })
|
const [draft, setDraft] = useState(emptyDraft())
|
||||||
|
const [catalog, setCatalog] = useState([])
|
||||||
const [error, setError] = useState('')
|
const [error, setError] = useState('')
|
||||||
const [notice, setNotice] = useState('')
|
const [notice, setNotice] = useState('')
|
||||||
const [saving, setSaving] = useState('')
|
const [saving, setSaving] = useState('')
|
||||||
|
|
||||||
const load = () =>
|
const applyPayload = (payload) => {
|
||||||
api('/api/admin/providers', { token: session.token })
|
|
||||||
.then((payload) => {
|
|
||||||
setData(payload)
|
setData(payload)
|
||||||
const next = {}
|
const next = {}
|
||||||
for (const role of payload.roles) {
|
for (const role of payload.roles || []) {
|
||||||
next[role.role] = {
|
next[role.role] = {
|
||||||
role: role.role,
|
role: role.role,
|
||||||
name: role.name || '',
|
name: role.name || '',
|
||||||
|
|
@ -65,35 +76,104 @@ export default function AdminProvidersPage() {
|
||||||
model: role.model || '',
|
model: role.model || '',
|
||||||
zdr: Boolean(role.zdr),
|
zdr: Boolean(role.zdr),
|
||||||
no_train: Boolean(role.no_train),
|
no_train: Boolean(role.no_train),
|
||||||
key: ''
|
profile_id: role.profile_id || '',
|
||||||
|
key: '',
|
||||||
|
saveAsTitle: ''
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
setForms(next)
|
setForms(next)
|
||||||
})
|
}
|
||||||
|
|
||||||
|
const load = () =>
|
||||||
|
api('/api/admin/providers', { token: session.token })
|
||||||
|
.then(applyPayload)
|
||||||
.catch((e) => setError(e.message))
|
.catch((e) => setError(e.message))
|
||||||
|
|
||||||
const loadCatalog = () =>
|
const loadModels = (url) => {
|
||||||
api('/api/admin/providers/models', { token: session.token })
|
if (!url) {
|
||||||
.then((payload) => {
|
setCatalog([])
|
||||||
setCatalog({
|
return
|
||||||
generate: payload.roles?.generate || [],
|
}
|
||||||
detect: payload.roles?.detect || []
|
api(`/api/admin/providers/models?url=${encodeURIComponent(url)}`, { token: session.token })
|
||||||
})
|
.then((payload) => setCatalog(payload.models || payload.roles?.generate || []))
|
||||||
})
|
.catch(() => setCatalog([]))
|
||||||
.catch(() => {
|
}
|
||||||
setCatalog({ generate: [], detect: [] })
|
|
||||||
})
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => { load() }, [session.token])
|
||||||
load()
|
useEffect(() => { loadModels(draft.url) }, [draft.url, session.token])
|
||||||
loadCatalog()
|
|
||||||
}, [session.token])
|
|
||||||
|
|
||||||
const update = (role, field, value) => {
|
const profiles = data?.profiles || []
|
||||||
|
const usedIds = useMemo(
|
||||||
|
() => new Set((data?.roles || []).map((role) => role.profile_id).filter(Boolean)),
|
||||||
|
[data]
|
||||||
|
)
|
||||||
|
|
||||||
|
const updateRole = (role, field, value) => {
|
||||||
setForms((prev) => ({ ...prev, [role]: { ...(prev[role] || emptyRole()), [field]: value } }))
|
setForms((prev) => ({ ...prev, [role]: { ...(prev[role] || emptyRole()), [field]: value } }))
|
||||||
}
|
}
|
||||||
|
|
||||||
const save = async (role) => {
|
const applyRole = async (role, profileId) => {
|
||||||
|
const profile = profiles.find((item) => item.id === profileId)
|
||||||
|
if (!profile) return
|
||||||
|
setError('')
|
||||||
|
setNotice('')
|
||||||
|
setForms((prev) => ({
|
||||||
|
...prev,
|
||||||
|
[role]: {
|
||||||
|
...(prev[role] || emptyRole()),
|
||||||
|
role,
|
||||||
|
name: profile.name || '',
|
||||||
|
url: profile.url || '',
|
||||||
|
model: profile.model || '',
|
||||||
|
zdr: Boolean(profile.zdr),
|
||||||
|
no_train: Boolean(profile.no_train),
|
||||||
|
profile_id: profile.id,
|
||||||
|
key: '',
|
||||||
|
saveAsTitle: ''
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
if (!profile.model) {
|
||||||
|
setError('Das Profil braucht ein Modell, bevor es einer Stufe zugeordnet werden kann.')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
setSaving(`activate-${role}`)
|
||||||
|
try {
|
||||||
|
const payload = await api(`/api/admin/providers/${role}/activate`, {
|
||||||
|
token: session.token,
|
||||||
|
method: 'POST',
|
||||||
|
body: { profile_id: profileId }
|
||||||
|
})
|
||||||
|
applyPayload(payload)
|
||||||
|
setNotice(`${role === 'detect' ? 'Maskierung' : 'Sprachmodell'}: ${profile.title} ist aktiv.`)
|
||||||
|
} catch (err) {
|
||||||
|
setError(err.message)
|
||||||
|
} finally {
|
||||||
|
setSaving('')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const saveDetectMode = async (mode) => {
|
||||||
|
setError('')
|
||||||
|
setNotice('')
|
||||||
|
setSaving('detect-mode')
|
||||||
|
try {
|
||||||
|
const payload = await api('/api/admin/providers/detect-mode', {
|
||||||
|
token: session.token,
|
||||||
|
method: 'PUT',
|
||||||
|
body: { mode }
|
||||||
|
})
|
||||||
|
applyPayload(payload)
|
||||||
|
setNotice(mode === 'learning'
|
||||||
|
? 'Lernmodus: Detect pausiert vor Generate, bis Maskierungen bestätigt sind.'
|
||||||
|
: 'Detect-Modus: semantisch, ohne Bestätigungs-Popup.')
|
||||||
|
} catch (err) {
|
||||||
|
setError(err.message)
|
||||||
|
} finally {
|
||||||
|
setSaving('')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const saveRole = async (role) => {
|
||||||
const form = forms[role]
|
const form = forms[role]
|
||||||
if (!form) return
|
if (!form) return
|
||||||
setError('')
|
setError('')
|
||||||
|
|
@ -105,7 +185,9 @@ export default function AdminProvidersPage() {
|
||||||
url: form.url,
|
url: form.url,
|
||||||
model: form.model,
|
model: form.model,
|
||||||
zdr: form.zdr,
|
zdr: form.zdr,
|
||||||
no_train: form.no_train
|
no_train: form.no_train,
|
||||||
|
profile_id: form.profile_id || undefined,
|
||||||
|
save_as_title: form.saveAsTitle.trim() || undefined
|
||||||
}
|
}
|
||||||
if (form.key.trim()) body.key = form.key.trim()
|
if (form.key.trim()) body.key = form.key.trim()
|
||||||
const payload = await api(`/api/admin/providers/${role}`, {
|
const payload = await api(`/api/admin/providers/${role}`, {
|
||||||
|
|
@ -113,10 +195,8 @@ export default function AdminProvidersPage() {
|
||||||
method: 'PUT',
|
method: 'PUT',
|
||||||
body
|
body
|
||||||
})
|
})
|
||||||
setData(payload)
|
applyPayload(payload)
|
||||||
setForms((prev) => ({ ...prev, [role]: { ...prev[role], key: '' } }))
|
setNotice(form.saveAsTitle.trim() ? 'Neues Profil gespeichert und dieser Stufe zugeordnet.' : 'Stufe gespeichert. Zugeordnetes Profil wurde mitaktualisiert.')
|
||||||
setNotice(form.key.trim() ? `${role}: gespeichert, Key in .env geschrieben.` : `${role}: gespeichert.`)
|
|
||||||
loadCatalog()
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err.message)
|
setError(err.message)
|
||||||
} finally {
|
} finally {
|
||||||
|
|
@ -124,10 +204,50 @@ export default function AdminProvidersPage() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const modelOptions = (role) => {
|
const saveDraft = async () => {
|
||||||
const fromCatalog = catalog[role] || []
|
setError('')
|
||||||
if (fromCatalog.length) return fromCatalog
|
setNotice('')
|
||||||
return (FALLBACK_MODELS[role] || []).map((id) => ({ id, name: id }))
|
setSaving('draft')
|
||||||
|
try {
|
||||||
|
const body = {
|
||||||
|
title: draft.title,
|
||||||
|
name: draft.name,
|
||||||
|
url: draft.url,
|
||||||
|
model: draft.model,
|
||||||
|
zdr: draft.zdr,
|
||||||
|
no_train: draft.no_train
|
||||||
|
}
|
||||||
|
const payload = draft.id
|
||||||
|
? await api(`/api/admin/llm-profiles/${draft.id}`, { token: session.token, method: 'PUT', body })
|
||||||
|
: await api('/api/admin/llm-profiles', { token: session.token, method: 'POST', body })
|
||||||
|
applyPayload(payload)
|
||||||
|
const saved = payload.profile || profiles.find((item) => item.id === draft.id)
|
||||||
|
setDraft(saved ? { ...saved } : emptyDraft())
|
||||||
|
setNotice(draft.id ? 'Profil aktualisiert.' : 'Profil angelegt. Du kannst es den Stufen zuordnen, ohne die Felder neu einzugeben.')
|
||||||
|
} catch (err) {
|
||||||
|
setError(err.message)
|
||||||
|
} finally {
|
||||||
|
setSaving('')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const removeProfile = async (profileId) => {
|
||||||
|
setError('')
|
||||||
|
setNotice('')
|
||||||
|
setSaving(`delete-${profileId}`)
|
||||||
|
try {
|
||||||
|
const payload = await api(`/api/admin/llm-profiles/${profileId}`, {
|
||||||
|
token: session.token,
|
||||||
|
method: 'DELETE'
|
||||||
|
})
|
||||||
|
applyPayload(payload)
|
||||||
|
if (draft.id === profileId) setDraft(emptyDraft())
|
||||||
|
setNotice('Profil gelöscht.')
|
||||||
|
} catch (err) {
|
||||||
|
setError(err.message)
|
||||||
|
} finally {
|
||||||
|
setSaving('')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const banner = detectBanner(data)
|
const banner = detectBanner(data)
|
||||||
|
|
@ -136,59 +256,188 @@ export default function AdminProvidersPage() {
|
||||||
<section className="card">
|
<section className="card">
|
||||||
<h1>Schnittstellen</h1>
|
<h1>Schnittstellen</h1>
|
||||||
<p className="muted">
|
<p className="muted">
|
||||||
Zwei Stufen, zwei eigene Modelle: Maskierung prüft den persönlichen Text, das Sprachmodell schreibt Dialogzug und Journalentwurf.
|
LLM-Profile merken URL, Modell und Policy. Maskierung und Sprachmodell wählen je ein Profil.
|
||||||
Pro Stufe URL und Modell getrennt wählen. Ohne Detection kein Generate. Der Secret-Key wird nie angezeigt und nie in der Datenbank gespeichert.
|
Beim Wechsel musst du die Daten nicht neu eintippen. Keys bleiben in der Umgebung, nicht im Profil.
|
||||||
</p>
|
</p>
|
||||||
{data && (
|
{data && <p className="muted">Laufzeit: {data.runtime_env}</p>}
|
||||||
<p className="muted">Laufzeit: {data.runtime_env}</p>
|
|
||||||
)}
|
|
||||||
{banner && <p>{banner}</p>}
|
{banner && <p>{banner}</p>}
|
||||||
{error && <p className="error">{error}</p>}
|
{error && <p className="error">{error}</p>}
|
||||||
{notice && <p>{notice}</p>}
|
{notice && <p>{notice}</p>}
|
||||||
|
|
||||||
{data && (
|
{data && (
|
||||||
<>
|
<>
|
||||||
{data.roles.map((role) => {
|
{data.roles.map((role) => {
|
||||||
const form = forms[role.role] || emptyRole()
|
const form = forms[role.role] || emptyRole()
|
||||||
const options = modelOptions(role.role)
|
const selected = profiles.find((item) => item.id === form.profile_id)
|
||||||
const listId = `kansho-models-${role.role}`
|
const view = selected
|
||||||
|
? { ...role, ...form, local: selected.local, ready: selected.local ? Boolean(form.url && form.model) : role.ready }
|
||||||
|
: role
|
||||||
return (
|
return (
|
||||||
<form
|
<div key={role.role} className="stack">
|
||||||
key={role.role}
|
|
||||||
className="stack"
|
|
||||||
onSubmit={(e) => { e.preventDefault(); save(role.role) }}
|
|
||||||
>
|
|
||||||
<h2>{role.title || role.role}</h2>
|
<h2>{role.title || role.role}</h2>
|
||||||
{role.task && <p className="muted">{role.task}</p>}
|
{role.task && <p className="muted">{role.task}</p>}
|
||||||
|
{role.role === 'detect' && (
|
||||||
|
<label>
|
||||||
|
Detect-Modus
|
||||||
|
<select
|
||||||
|
value={data.detect_operating_mode || 'semantic'}
|
||||||
|
onChange={(e) => saveDetectMode(e.target.value)}
|
||||||
|
disabled={saving === 'detect-mode'}
|
||||||
|
>
|
||||||
|
<option value="semantic">Semantisch (Default)</option>
|
||||||
|
<option value="learning">Lernmodus (Übergang)</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
)}
|
||||||
|
{role.role === 'detect' && data.detect_operating_mode === 'learning' && (
|
||||||
|
<p className="muted">
|
||||||
|
Keine separate Wortliste. Bestätigungen im Dialog legen lokale Sinne an.
|
||||||
|
Mehrdeutige Nennungen gehen nur als Mini-Passage an ein lokales Detect-Modell.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
<dl className="meta">
|
<dl className="meta">
|
||||||
<dt>Status</dt>
|
<dt>Status</dt>
|
||||||
<dd>{statusLabel(role)}</dd>
|
<dd>{statusLabel(view)}</dd>
|
||||||
<dt>Key</dt>
|
<dt>Key</dt>
|
||||||
<dd>{role.key_present ? (role.key_source === 'generate' ? 'über Sprachmodell' : 'gesetzt') : 'fehlt'}</dd>
|
<dd>{view.key_present ? (view.key_source === 'generate' ? 'über Sprachmodell' : 'gesetzt') : (view.local ? 'nicht nötig (lokal)' : 'fehlt')}</dd>
|
||||||
<dt>Modus</dt>
|
<dt>Modus</dt>
|
||||||
<dd>{role.local ? 'lokal' : 'extern'} · {role.mode}</dd>
|
<dd>{view.local ? 'lokal' : 'extern'} · {view.mode}</dd>
|
||||||
</dl>
|
</dl>
|
||||||
|
<label>
|
||||||
|
Aktives Profil
|
||||||
|
<select
|
||||||
|
value={form.profile_id}
|
||||||
|
onChange={(e) => {
|
||||||
|
if (!e.target.value) return
|
||||||
|
applyRole(role.role, e.target.value)
|
||||||
|
}}
|
||||||
|
disabled={saving.startsWith('activate-') || !profiles.length}
|
||||||
|
>
|
||||||
|
<option value="">Profil wählen …</option>
|
||||||
|
{profiles.map((profile) => (
|
||||||
|
<option key={profile.id} value={profile.id}>
|
||||||
|
{profile.title} ({profile.local ? 'lokal' : 'extern'}{profile.model ? ` · ${profile.model}` : ''})
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<form
|
||||||
|
className="stack"
|
||||||
|
onSubmit={(e) => { e.preventDefault(); saveRole(role.role) }}
|
||||||
|
>
|
||||||
<label>
|
<label>
|
||||||
Name
|
Name
|
||||||
<input value={form.name} onChange={(e) => update(role.role, 'name', e.target.value)} />
|
<input value={form.name} onChange={(e) => updateRole(role.role, 'name', e.target.value)} />
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
URL
|
||||||
|
<input value={form.url} onChange={(e) => updateRole(role.role, 'url', e.target.value)} />
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
Modell für diese Stufe
|
||||||
|
<input value={form.model} onChange={(e) => updateRole(role.role, 'model', e.target.value)} />
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
Als neues Profil speichern (optional)
|
||||||
|
<input
|
||||||
|
value={form.saveAsTitle}
|
||||||
|
onChange={(e) => updateRole(role.role, 'saveAsTitle', e.target.value)}
|
||||||
|
placeholder="z. B. Ollama Detect"
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<label className="check">
|
||||||
|
<input type="checkbox" checked={form.zdr} onChange={(e) => updateRole(role.role, 'zdr', e.target.checked)} />
|
||||||
|
Zero Data Retention
|
||||||
|
</label>
|
||||||
|
<label className="check">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={form.no_train}
|
||||||
|
onChange={(e) => updateRole(role.role, 'no_train', e.target.checked)}
|
||||||
|
/>
|
||||||
|
kein Provider-Training
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
API-Key {view.key_present ? '(leer lassen, um den bestehenden zu behalten)' : '(für externe URL)'}
|
||||||
|
<input
|
||||||
|
type="password"
|
||||||
|
autoComplete="off"
|
||||||
|
value={form.key}
|
||||||
|
onChange={(e) => updateRole(role.role, 'key', e.target.value)}
|
||||||
|
placeholder={view.local ? 'lokal ohne Key' : (role.role === 'detect' ? 'leer = Key des Sprachmodells' : '')}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<button type="submit" disabled={saving === role.role}>
|
||||||
|
{saving === role.role ? 'Speichert …' : 'Stufe speichern'}
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
|
||||||
|
<h2>Gespeicherte Profile</h2>
|
||||||
|
<p className="muted">Ein Profil kann Maskierung, Sprachmodell oder beiden Stufen dienen. Löschen geht nur, wenn es gerade keiner Stufe zugeordnet ist.</p>
|
||||||
|
<ul className="stack">
|
||||||
|
{profiles.map((profile) => (
|
||||||
|
<li key={profile.id} className="row-item">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="ghost"
|
||||||
|
onClick={() => setDraft({ ...profile })}
|
||||||
|
>
|
||||||
|
{profile.title}
|
||||||
|
<span className="muted"> · {profile.local ? 'lokal' : 'extern'} · {profile.model || 'kein Modell'} · {profile.url}</span>
|
||||||
|
</button>
|
||||||
|
<div className="row-actions">
|
||||||
|
<button type="button" className="ghost" onClick={() => setDraft({ ...profile })}>Bearbeiten</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="ghost"
|
||||||
|
disabled={usedIds.has(profile.id) || saving === `delete-${profile.id}`}
|
||||||
|
onClick={() => removeProfile(profile.id)}
|
||||||
|
>
|
||||||
|
Löschen
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<form className="stack" onSubmit={(e) => { e.preventDefault(); saveDraft() }}>
|
||||||
|
<h2>{draft.id ? 'Profil bearbeiten' : 'Neues Profil'}</h2>
|
||||||
|
<label>
|
||||||
|
Profilname
|
||||||
|
<input
|
||||||
|
value={draft.title}
|
||||||
|
onChange={(e) => setDraft((prev) => ({ ...prev, title: e.target.value }))}
|
||||||
|
placeholder="z. B. Ollama (LAN)"
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
Anbietername
|
||||||
|
<input
|
||||||
|
value={draft.name}
|
||||||
|
onChange={(e) => setDraft((prev) => ({ ...prev, name: e.target.value }))}
|
||||||
|
placeholder="ollama oder openrouter"
|
||||||
|
/>
|
||||||
</label>
|
</label>
|
||||||
<label>
|
<label>
|
||||||
URL
|
URL
|
||||||
<input
|
<input
|
||||||
value={form.url}
|
value={draft.url}
|
||||||
onChange={(e) => update(role.role, 'url', e.target.value)}
|
onChange={(e) => setDraft((prev) => ({ ...prev, url: e.target.value }))}
|
||||||
placeholder={role.role === 'detect' ? 'http://127.0.0.1:11434/v1/chat/completions' : ''}
|
|
||||||
/>
|
/>
|
||||||
</label>
|
</label>
|
||||||
<label>
|
<label>
|
||||||
Modell für diese Stufe
|
Modell
|
||||||
<input
|
<input
|
||||||
list={listId}
|
list="kansho-profile-models"
|
||||||
value={form.model}
|
value={draft.model}
|
||||||
onChange={(e) => update(role.role, 'model', e.target.value)}
|
onChange={(e) => setDraft((prev) => ({ ...prev, model: e.target.value }))}
|
||||||
placeholder={role.role === 'detect' ? 'openai/gpt-4.1-nano oder lokales Ollama-Modell' : 'openai/gpt-5.4'}
|
placeholder="Ollama-Tag oder OpenRouter-ID"
|
||||||
/>
|
/>
|
||||||
<datalist id={listId}>
|
<datalist id="kansho-profile-models">
|
||||||
{options.map((item) => (
|
{catalog.map((item) => (
|
||||||
<option key={item.id} value={item.id}>
|
<option key={item.id} value={item.id}>
|
||||||
{item.name && item.name !== item.id ? item.name : item.id}
|
{item.name && item.name !== item.id ? item.name : item.id}
|
||||||
</option>
|
</option>
|
||||||
|
|
@ -196,34 +445,33 @@ export default function AdminProvidersPage() {
|
||||||
</datalist>
|
</datalist>
|
||||||
</label>
|
</label>
|
||||||
<label className="check">
|
<label className="check">
|
||||||
<input type="checkbox" checked={form.zdr} onChange={(e) => update(role.role, 'zdr', e.target.checked)} />
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={draft.zdr}
|
||||||
|
onChange={(e) => setDraft((prev) => ({ ...prev, zdr: e.target.checked }))}
|
||||||
|
/>
|
||||||
Zero Data Retention
|
Zero Data Retention
|
||||||
</label>
|
</label>
|
||||||
<label className="check">
|
<label className="check">
|
||||||
<input
|
<input
|
||||||
type="checkbox"
|
type="checkbox"
|
||||||
checked={form.no_train}
|
checked={draft.no_train}
|
||||||
onChange={(e) => update(role.role, 'no_train', e.target.checked)}
|
onChange={(e) => setDraft((prev) => ({ ...prev, no_train: e.target.checked }))}
|
||||||
/>
|
/>
|
||||||
kein Provider-Training
|
kein Provider-Training
|
||||||
</label>
|
</label>
|
||||||
<label>
|
<div className="row-actions">
|
||||||
API-Key {role.key_present ? '(leer lassen, um den bestehenden zu behalten)' : '(pflichtig für externe URL)'}
|
<button type="submit" disabled={saving === 'draft'}>
|
||||||
<input
|
{saving === 'draft' ? 'Speichert …' : (draft.id ? 'Profil speichern' : 'Profil anlegen')}
|
||||||
type="password"
|
|
||||||
autoComplete="off"
|
|
||||||
value={form.key}
|
|
||||||
onChange={(e) => update(role.role, 'key', e.target.value)}
|
|
||||||
placeholder={role.role === 'detect' ? 'leer = Key des Sprachmodells' : ''}
|
|
||||||
/>
|
|
||||||
</label>
|
|
||||||
<button type="submit" disabled={saving === role.role}>
|
|
||||||
{saving === role.role ? 'Speichert …' : 'Speichern'}
|
|
||||||
</button>
|
</button>
|
||||||
|
{draft.id && (
|
||||||
|
<button type="button" className="ghost" onClick={() => setDraft(emptyDraft())}>
|
||||||
|
Neu
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
</form>
|
</form>
|
||||||
)
|
<p className="muted">Ohne ZDR und ohne Trainingsverbot bleibt ein externes Sprachmodell fail-closed. Gateway lässt sich darüber nicht umgehen. Ollama auf der LAN-Adresse gilt als lokal.</p>
|
||||||
})}
|
|
||||||
<p className="muted">Ohne ZDR und ohne Trainingsverbot bleibt das Sprachmodell fail-closed. Maskierung wählt keine Operation. Gateway lässt sich darüber nicht umgehen.</p>
|
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</section>
|
</section>
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import { useEffect, useState } from 'react'
|
import { useEffect, useState } from 'react'
|
||||||
import { Link } from 'react-router-dom'
|
import { Link } from 'react-router-dom'
|
||||||
import { api, apiDownload } from '../api.js'
|
import { api, apiDownload } from '../api.js'
|
||||||
|
import MaskReviewPanel from '../components/MaskReviewPanel.jsx'
|
||||||
import { useAuth } from '../context/AuthContext.jsx'
|
import { useAuth } from '../context/AuthContext.jsx'
|
||||||
|
|
||||||
function convLabel(item, index) {
|
function convLabel(item, index) {
|
||||||
|
|
@ -18,6 +19,7 @@ export default function DialoguePage() {
|
||||||
const [busy, setBusy] = useState(false)
|
const [busy, setBusy] = useState(false)
|
||||||
const [egress, setEgress] = useState(null)
|
const [egress, setEgress] = useState(null)
|
||||||
const [debugOn, setDebugOn] = useState(false)
|
const [debugOn, setDebugOn] = useState(false)
|
||||||
|
const [maskReview, setMaskReview] = useState(null)
|
||||||
|
|
||||||
const loadList = () =>
|
const loadList = () =>
|
||||||
api('/api/dialogue/conversations', { token: session.token })
|
api('/api/dialogue/conversations', { token: session.token })
|
||||||
|
|
@ -33,6 +35,7 @@ export default function DialoguePage() {
|
||||||
const data = await api(`/api/dialogue/conversations/${id}`, { token: session.token })
|
const data = await api(`/api/dialogue/conversations/${id}`, { token: session.token })
|
||||||
setActiveId(id)
|
setActiveId(id)
|
||||||
setMessages(data.messages || [])
|
setMessages(data.messages || [])
|
||||||
|
setMaskReview(data.pending_mask_review || null)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err.message)
|
setError(err.message)
|
||||||
}
|
}
|
||||||
|
|
@ -78,6 +81,7 @@ export default function DialoguePage() {
|
||||||
else {
|
else {
|
||||||
setActiveId(null)
|
setActiveId(null)
|
||||||
setMessages([])
|
setMessages([])
|
||||||
|
setMaskReview(null)
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err.message)
|
setError(err.message)
|
||||||
|
|
@ -109,7 +113,12 @@ export default function DialoguePage() {
|
||||||
})
|
})
|
||||||
setDraft('')
|
setDraft('')
|
||||||
setMessages(result.messages || [])
|
setMessages(result.messages || [])
|
||||||
|
if (result.pending_mask_review) {
|
||||||
|
setMaskReview(result.pending_mask_review)
|
||||||
|
} else {
|
||||||
|
setMaskReview(null)
|
||||||
await loadList()
|
await loadList()
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err.message)
|
setError(err.message)
|
||||||
if (activeId) await openConversation(activeId)
|
if (activeId) await openConversation(activeId)
|
||||||
|
|
@ -118,6 +127,26 @@ export default function DialoguePage() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const submitMaskReview = async (decisions) => {
|
||||||
|
if (!activeId || !maskReview?.id) return
|
||||||
|
setError('')
|
||||||
|
setBusy(true)
|
||||||
|
try {
|
||||||
|
const result = await api(`/api/dialogue/conversations/${activeId}/turn/review`, {
|
||||||
|
token: session.token,
|
||||||
|
method: 'POST',
|
||||||
|
body: { review_id: maskReview.id, decisions }
|
||||||
|
})
|
||||||
|
setMaskReview(null)
|
||||||
|
setMessages(result.messages || [])
|
||||||
|
await loadList()
|
||||||
|
} catch (err) {
|
||||||
|
setError(err.message)
|
||||||
|
} finally {
|
||||||
|
setBusy(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<section className="card dialogue-page">
|
<section className="card dialogue-page">
|
||||||
<header className="dialogue-head">
|
<header className="dialogue-head">
|
||||||
|
|
@ -174,11 +203,12 @@ export default function DialoguePage() {
|
||||||
{activeId && (
|
{activeId && (
|
||||||
<form className="dialogue-composer" onSubmit={send}>
|
<form className="dialogue-composer" onSubmit={send}>
|
||||||
<textarea rows={3} value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Erzählen …" />
|
<textarea rows={3} value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Erzählen …" />
|
||||||
<button type="submit" disabled={busy}>{busy ? 'Antwortet …' : 'Senden'}</button>
|
<button type="submit" disabled={busy || Boolean(maskReview)}>{busy ? 'Antwortet …' : 'Senden'}</button>
|
||||||
</form>
|
</form>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<MaskReviewPanel review={maskReview} busy={busy} onSubmit={submitMaskReview} />
|
||||||
</section>
|
</section>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import { useEffect, useState } from 'react'
|
||||||
import { Link, useNavigate, useParams } from 'react-router-dom'
|
import { Link, useNavigate, useParams } from 'react-router-dom'
|
||||||
import { api, apiDownload } from '../api.js'
|
import { api, apiDownload } from '../api.js'
|
||||||
import DayScratch from '../components/DayScratch.jsx'
|
import DayScratch from '../components/DayScratch.jsx'
|
||||||
|
import MaskReviewPanel from '../components/MaskReviewPanel.jsx'
|
||||||
import RunLogPopup from '../components/RunLogPopup.jsx'
|
import RunLogPopup from '../components/RunLogPopup.jsx'
|
||||||
import { useAuth } from '../context/AuthContext.jsx'
|
import { useAuth } from '../context/AuthContext.jsx'
|
||||||
import { entryTitle } from '../journal/document.js'
|
import { entryTitle } from '../journal/document.js'
|
||||||
|
|
@ -58,6 +59,7 @@ export default function JournalDayPage() {
|
||||||
const [logStatus, setLogStatus] = useState('ok')
|
const [logStatus, setLogStatus] = useState('ok')
|
||||||
const [runLog, setRunLog] = useState([])
|
const [runLog, setRunLog] = useState([])
|
||||||
const [debugOn, setDebugOn] = useState(false)
|
const [debugOn, setDebugOn] = useState(false)
|
||||||
|
const [maskReview, setMaskReview] = useState(null)
|
||||||
|
|
||||||
const loadDay = async (preferId) => {
|
const loadDay = async (preferId) => {
|
||||||
const data = await api(`/api/journal/days/${dayId}`, { token: session.token })
|
const data = await api(`/api/journal/days/${dayId}`, { token: session.token })
|
||||||
|
|
@ -68,9 +70,11 @@ export default function JournalDayPage() {
|
||||||
const conv = await api(`/api/journal/conversations/${nextId}`, { token: session.token })
|
const conv = await api(`/api/journal/conversations/${nextId}`, { token: session.token })
|
||||||
setActiveId(nextId)
|
setActiveId(nextId)
|
||||||
setMessages(conv.messages || [])
|
setMessages(conv.messages || [])
|
||||||
|
setMaskReview(conv.pending_mask_review || null)
|
||||||
} else {
|
} else {
|
||||||
setActiveId(null)
|
setActiveId(null)
|
||||||
setMessages([])
|
setMessages([])
|
||||||
|
setMaskReview(null)
|
||||||
}
|
}
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
@ -148,7 +152,12 @@ export default function JournalDayPage() {
|
||||||
})
|
})
|
||||||
setDraft('')
|
setDraft('')
|
||||||
setMessages(result.messages || [])
|
setMessages(result.messages || [])
|
||||||
|
if (result.pending_mask_review) {
|
||||||
|
setMaskReview(result.pending_mask_review)
|
||||||
|
} else {
|
||||||
|
setMaskReview(null)
|
||||||
await loadDay(activeId)
|
await loadDay(activeId)
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err.message)
|
setError(err.message)
|
||||||
await loadDay(activeId)
|
await loadDay(activeId)
|
||||||
|
|
@ -157,6 +166,26 @@ export default function JournalDayPage() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const submitMaskReview = async (decisions) => {
|
||||||
|
if (!activeId || !maskReview?.id) return
|
||||||
|
setError('')
|
||||||
|
setBusy(true)
|
||||||
|
try {
|
||||||
|
const result = await api(`/api/journal/conversations/${activeId}/turn/review`, {
|
||||||
|
token: session.token,
|
||||||
|
method: 'POST',
|
||||||
|
body: { review_id: maskReview.id, decisions }
|
||||||
|
})
|
||||||
|
setMaskReview(null)
|
||||||
|
setMessages(result.messages || [])
|
||||||
|
await loadDay(activeId)
|
||||||
|
} catch (err) {
|
||||||
|
setError(err.message)
|
||||||
|
} finally {
|
||||||
|
setBusy(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const generate = async (conversationIds) => {
|
const generate = async (conversationIds) => {
|
||||||
setError('')
|
setError('')
|
||||||
setBusy(true)
|
setBusy(true)
|
||||||
|
|
@ -391,7 +420,7 @@ export default function JournalDayPage() {
|
||||||
onChange={(e) => setDraft(e.target.value)}
|
onChange={(e) => setDraft(e.target.value)}
|
||||||
placeholder="Erzählen …"
|
placeholder="Erzählen …"
|
||||||
/>
|
/>
|
||||||
<button type="submit" disabled={busy}>{busy ? 'Antwortet …' : 'Senden'}</button>
|
<button type="submit" disabled={busy || Boolean(maskReview)}>{busy ? 'Antwortet …' : 'Senden'}</button>
|
||||||
</form>
|
</form>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -403,6 +432,7 @@ export default function JournalDayPage() {
|
||||||
onError={setError}
|
onError={setError}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
<MaskReviewPanel review={maskReview} busy={busy} onSubmit={submitMaskReview} />
|
||||||
<RunLogPopup
|
<RunLogPopup
|
||||||
open={logOpen}
|
open={logOpen}
|
||||||
status={logStatus}
|
status={logStatus}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
# Kanshō development vhost. Install as /etc/nginx/sites-available/kansho-dev.
|
# Kanshō development vhost. Install as /etc/nginx/sites-available/kansho-dev.
|
||||||
|
# Live TLS terminates on the Synology Reverse Proxy at 192.168.2.63, not on the Pi.
|
||||||
|
# Set DSM proxy read/send timeout to 600s (default 60s returns HTTP 504).
|
||||||
|
|
||||||
server {
|
server {
|
||||||
listen 80;
|
listen 80;
|
||||||
|
|
@ -46,5 +48,6 @@ server {
|
||||||
proxy_set_header Upgrade $http_upgrade;
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
proxy_set_header Connection "upgrade";
|
proxy_set_header Connection "upgrade";
|
||||||
proxy_cache_bypass $http_upgrade;
|
proxy_cache_bypass $http_upgrade;
|
||||||
|
proxy_read_timeout 600s;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
# Kanshō – Host nginx (outside Compose)
|
# Kanshō – Host nginx (outside Compose)
|
||||||
# Install as /etc/nginx/sites-available/kansho and symlink into sites-enabled.
|
# Install as /etc/nginx/sites-available/kansho and symlink into sites-enabled.
|
||||||
# TLS via nginx/certbot-setup.sh. Reverse-proxy targets are the Pi publish ports.
|
# TLS via nginx/certbot-setup.sh. Reverse-proxy targets are the Pi publish ports.
|
||||||
|
# Live TLS currently terminates on the Synology Reverse Proxy at 192.168.2.63;
|
||||||
|
# set DSM proxy read/send timeout to 600s (default 60s returns HTTP 504).
|
||||||
|
|
||||||
server {
|
server {
|
||||||
listen 80;
|
listen 80;
|
||||||
|
|
@ -50,5 +52,6 @@ server {
|
||||||
proxy_set_header Upgrade $http_upgrade;
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
proxy_set_header Connection "upgrade";
|
proxy_set_header Connection "upgrade";
|
||||||
proxy_cache_bypass $http_upgrade;
|
proxy_cache_bypass $http_upgrade;
|
||||||
|
proxy_read_timeout 600s;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user