Compare commits
35
Commits
d62673e48b
...
develop
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
89d149684e | ||
|
|
0277039b07 | ||
|
|
011cc9dd81 | ||
|
|
a65ea98b0c | ||
|
|
f7ffd28332 | ||
|
|
3f7c0d71a5 | ||
|
|
cc43fc6409 | ||
|
|
fa1c11c33e | ||
|
|
59921fc5b5 | ||
|
|
bef422a429 | ||
|
|
af6fe55577 | ||
|
|
dd7858347c | ||
|
|
79e4fd0d4b | ||
|
|
3c0213ae74 | ||
|
|
9d79e3e92f | ||
|
|
6ea36f7d1f | ||
|
|
36b9a2e424 | ||
|
|
709370da93 | ||
|
|
0ef3e3bb7a | ||
|
|
ea38528ecf | ||
|
|
9d5befdde9 | ||
|
|
07479f6a9f | ||
|
|
32acdc220b | ||
|
|
326a9a44c8 | ||
|
|
29e3d0ff74 | ||
|
|
ec0f8bad55 | ||
|
|
e42f751fdc | ||
|
|
be11f82f84 | ||
|
|
4f37991ad1 | ||
|
|
f17b3083e8 | ||
|
|
53b1eff833 | ||
|
|
60f0d2830d | ||
|
|
fd7102e3b3 | ||
|
|
a8caeb0dd4 | ||
|
|
662eb78b60 |
No files matched your search
@@ -0,0 +1,16 @@
|
||||
.git
|
||||
.gitea
|
||||
**/.venv
|
||||
**/__pycache__
|
||||
**/*.pyc
|
||||
backend/data
|
||||
backend/.env
|
||||
frontend/node_modules
|
||||
frontend/dist
|
||||
local-backups
|
||||
transfer
|
||||
tmp
|
||||
temp
|
||||
docs
|
||||
*.md
|
||||
!frontend/README.md
|
||||
@@ -0,0 +1,37 @@
|
||||
# Compose secrets on the Pi. Copy to /home/lars/docker/kansho/.env and
|
||||
# /home/lars/docker/kansho-dev/.env. Never commit the filled file.
|
||||
# Local Windows development keeps using backend/.env (SQLite, ports 5188/8018).
|
||||
|
||||
# ─── DEV (docker-compose.dev-env.yml) ────────────────────────────────────────
|
||||
# DB_NAME=kansho_dev
|
||||
# DB_USER=kansho_dev
|
||||
# DB_PASSWORD=dev_password_change_me
|
||||
# APP_URL=https://dev.kansho.jinkendo.de
|
||||
# ALLOWED_ORIGINS=https://dev.kansho.jinkendo.de,http://192.168.2.49:3096
|
||||
# KANSHO_ENV=development
|
||||
# KANSHO_ALLOW_REMOTE_DETECT=
|
||||
# KANSHO_FRONTEND_PORT=3096
|
||||
# KANSHO_BACKEND_PORT=8096
|
||||
# KANSHO_PROVIDER_KEY=
|
||||
# KANSHO_DETECT_PROVIDER_KEY=
|
||||
|
||||
# ─── PROD (docker-compose.yml) ───────────────────────────────────────────────
|
||||
DB_NAME=kansho
|
||||
DB_USER=kansho
|
||||
DB_PASSWORD=CHANGE_ME_SECURE_PASSWORD
|
||||
APP_URL=https://kansho.jinkendo.de
|
||||
ALLOWED_ORIGINS=https://kansho.jinkendo.de
|
||||
KANSHO_ENV=production
|
||||
# Transitional until local detect (Ollama). Empty = blocked in production.
|
||||
KANSHO_ALLOW_REMOTE_DETECT=
|
||||
KANSHO_FRONTEND_PORT=3006
|
||||
KANSHO_BACKEND_PORT=8005
|
||||
KANSHO_PROVIDER_KEY=
|
||||
KANSHO_DETECT_PROVIDER_KEY=
|
||||
|
||||
# Betriebsmail. Leer = kein Versand (production/test: Fehler, development: Link nur im Server-Log).
|
||||
# SMTP_HOST=
|
||||
# SMTP_PORT=587
|
||||
# SMTP_USER=
|
||||
# SMTP_PASSWORD=
|
||||
# SMTP_FROM=kansho@example.test
|
||||
@@ -0,0 +1,40 @@
|
||||
name: Deploy Development
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [develop]
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deploy Kanshō to development
|
||||
run: |
|
||||
set -e
|
||||
echo "=== Deploying Kanshō to DEVELOPMENT ==="
|
||||
REPO="http://192.168.2.144:3000/Lars/Kansho.git"
|
||||
TARGET="/home/lars/docker/kansho-dev"
|
||||
mkdir -p "$TARGET"
|
||||
cd "$TARGET"
|
||||
if [ ! -d .git ]; then
|
||||
git clone -b develop "$REPO" .
|
||||
else
|
||||
git fetch origin develop
|
||||
git checkout develop
|
||||
git reset --hard origin/develop
|
||||
fi
|
||||
docker compose -f docker-compose.dev-env.yml build --no-cache backend frontend
|
||||
if ! docker compose -f docker-compose.dev-env.yml up -d --wait; then
|
||||
echo "compose up --wait failed — backend logs:"
|
||||
docker compose -f docker-compose.dev-env.yml logs backend --tail 150 || true
|
||||
docker compose -f docker-compose.dev-env.yml ps || true
|
||||
exit 1
|
||||
fi
|
||||
if ! curl -sf http://localhost:8096/api/health; then
|
||||
echo "DEV API not reachable — backend logs:"
|
||||
docker compose -f docker-compose.dev-env.yml logs backend --tail 150 || true
|
||||
exit 1
|
||||
fi
|
||||
echo "DEV API /api/health OK"
|
||||
curl -sf http://localhost:3096/api/health && echo "DEV frontend proxy /api/health OK"
|
||||
echo "=== Kanshō DEV deploy complete ==="
|
||||
@@ -0,0 +1,40 @@
|
||||
name: Deploy Production
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deploy Kanshō to production
|
||||
run: |
|
||||
set -e
|
||||
echo "=== Deploying Kanshō to PRODUCTION ==="
|
||||
REPO="http://192.168.2.144:3000/Lars/Kansho.git"
|
||||
TARGET="/home/lars/docker/kansho"
|
||||
mkdir -p "$TARGET"
|
||||
cd "$TARGET"
|
||||
if [ ! -d .git ]; then
|
||||
git clone -b main "$REPO" .
|
||||
else
|
||||
git fetch origin main
|
||||
git checkout main
|
||||
git reset --hard origin/main
|
||||
fi
|
||||
docker compose build --no-cache backend frontend
|
||||
if ! docker compose up -d --wait; then
|
||||
echo "compose up --wait failed — backend logs:"
|
||||
docker compose logs backend --tail 150 || true
|
||||
docker compose ps || true
|
||||
exit 1
|
||||
fi
|
||||
if ! curl -sf http://localhost:8005/api/health; then
|
||||
echo "PROD API not reachable — backend logs:"
|
||||
docker compose logs backend --tail 150 || true
|
||||
exit 1
|
||||
fi
|
||||
echo "PROD API /api/health OK"
|
||||
curl -sf http://localhost:3006/api/health && echo "PROD frontend proxy /api/health OK"
|
||||
echo "=== Kanshō PROD deploy complete ==="
|
||||
@@ -0,0 +1,90 @@
|
||||
name: Test Suite
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [develop]
|
||||
workflow_run:
|
||||
workflows: ["Deploy Development"]
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
pytest-backend:
|
||||
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: pytest against Dev Postgres (kansho_test)
|
||||
run: |
|
||||
set -e
|
||||
APP_DIR="/home/lars/docker/kansho-dev"
|
||||
COMPOSE_FILE="docker-compose.dev-env.yml"
|
||||
echo "pytest uses Dev Postgres (${APP_DIR}); event=${{ github.event_name }}"
|
||||
cd "$APP_DIR"
|
||||
for i in $(seq 1 60); do
|
||||
if [ -f "$APP_DIR/backend/pytest.ini" ] && docker compose -f "$COMPOSE_FILE" exec -T backend true 2>/dev/null; then
|
||||
echo "backend and pytest.ini ready (attempt $i)"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 60 ]; then
|
||||
echo "timeout waiting for Dev checkout/backend"
|
||||
docker compose -f "$COMPOSE_FILE" ps || true
|
||||
ls -la "$APP_DIR/backend" || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
PGUSER="$(docker compose -f "$COMPOSE_FILE" exec -T postgres printenv POSTGRES_USER | tr -d '\r')"
|
||||
echo "test database kansho_test on the existing Dev Postgres (owner ${PGUSER})"
|
||||
docker compose -f "$COMPOSE_FILE" exec -T postgres \
|
||||
psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'kansho_test' AND pid <> pg_backend_pid();"
|
||||
docker compose -f "$COMPOSE_FILE" exec -T postgres \
|
||||
psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "DROP DATABASE IF EXISTS kansho_test;"
|
||||
docker compose -f "$COMPOSE_FILE" exec -T postgres \
|
||||
psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "CREATE DATABASE kansho_test OWNER ${PGUSER};"
|
||||
docker compose -f "$COMPOSE_FILE" run --rm --no-deps \
|
||||
-v "${APP_DIR}:/src:ro" \
|
||||
-w /src/backend \
|
||||
-e KANSHO_DB_BACKEND=postgres \
|
||||
-e DB_NAME=kansho_test \
|
||||
-e KANSHO_PROVIDER_KEY= \
|
||||
-e KANSHO_DETECT_PROVIDER_KEY= \
|
||||
-e KANSHO_ALLOW_REMOTE_DETECT= \
|
||||
-e PYTHONUTF8=1 \
|
||||
backend sh -lc '
|
||||
set -e
|
||||
unset KANSHO_DB_PATH KANSHO_FAKE_PROVIDER KANSHO_FAKE_DETECT
|
||||
unset KANSHO_PROVIDER_KEY KANSHO_DETECT_PROVIDER_KEY KANSHO_ALLOW_REMOTE_DETECT
|
||||
export KANSHO_PROVIDER_KEY=
|
||||
export KANSHO_DETECT_PROVIDER_KEY=
|
||||
pip install -q -r requirements-dev.txt
|
||||
python -m pytest tests -m "not slow" -ra --tb=short
|
||||
'
|
||||
|
||||
smoke-dev:
|
||||
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Non-destructive smoke against the running Dev app
|
||||
run: |
|
||||
set -e
|
||||
echo "live Dev API (kansho_dev), read-only health"
|
||||
curl -sf http://localhost:8096/api/health
|
||||
echo
|
||||
curl -sf http://localhost:3096/api/health
|
||||
echo
|
||||
echo "LAN smoke OK. Public URL after TLS: https://dev.kansho.jinkendo.de"
|
||||
|
||||
frontend-build:
|
||||
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Frontend production build on Dev checkout
|
||||
run: |
|
||||
set -e
|
||||
APP_DIR="/home/lars/docker/kansho-dev"
|
||||
cd "$APP_DIR/frontend"
|
||||
npm ci
|
||||
npm run build
|
||||
+6
-1
@@ -38,9 +38,14 @@ Thumbs.db
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
|
||||
# Test / local data
|
||||
# Local data (never commit)
|
||||
tmp/
|
||||
temp/
|
||||
data/local/
|
||||
backend/data/
|
||||
local-backups/
|
||||
*.kansho-backup.zip
|
||||
*.db
|
||||
|
||||
# Personal journal zip was a one-time Gitea transport. Do not re-add.
|
||||
transfer/
|
||||
@@ -8,7 +8,7 @@ Kanshō ist **kein** Tagebuch mit KI, keine Meditations-App und kein generischer
|
||||
|
||||
## Aktueller Stand
|
||||
|
||||
Fachliche Konzeption und technische Rahmenarchitektur plus ein **lokales Produktrahmen-Gerüst** (`frontend/`, `backend/`). Das Gerüst ist Auth, PWA-Shell und Privacy-Gateway-Stub – noch kein Reflexions-MVP.
|
||||
Fachliche Konzeption, technische Rahmenarchitektur und der erste **vertikale MVP-Slice** (dialoggeführtes Journal) auf dem lokalen Produktrahmen (`frontend/`, `backend/`). Identity bleibt lokal; persönlicher LLM-Egress nur über das Privacy Gateway.
|
||||
|
||||
| Bereich | Stand |
|
||||
|---|---|
|
||||
@@ -16,10 +16,12 @@ Fachliche Konzeption und technische Rahmenarchitektur plus ein **lokales Produkt
|
||||
| Technische Form | entschieden: PWA, Mobile First, Offline, Transkription |
|
||||
| Fachliche Doku | Baseline in `docs/architecture/functional/` |
|
||||
| Technische Doku | vorläufiger Rahmen in `docs/architecture/technical/` |
|
||||
| App | lokaler Produktrahmen, Fachkonzept nicht eingefroren |
|
||||
| App | MVP-Journal-Slice auf dem Produktrahmen; Dialog-IA und Target-Modell nicht eingefroren |
|
||||
|
||||
Remote: [gitea.stommer.de/Lars/Kansho](https://gitea.stommer.de/Lars/Kansho.git)
|
||||
|
||||
Die Urlaubs-Laptop-Instanz lief ohne Docker auf SQLite. Heimtransfer, Backup und späterer Server-/Postgres-Pfad: `docs/architecture/technical/environment_handover.md`.
|
||||
|
||||
## Dokumentation zuerst laden
|
||||
|
||||
Nicht alle Kapitel gleichzeitig in den Kontext ziehen. Der Index beschreibt sinnvolle Bundles:
|
||||
@@ -53,13 +55,46 @@ cd frontend
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Kanshō nutzt eigene lokale Ports, nicht die Vite-/FastAPI-Defaults: Frontend **5188**, Backend **8018** (`strictPort`, kein Ausweichen auf 5173/5174). Dann http://localhost:5188 – erster Start legt das Admin-Profil an. Frame-Test:
|
||||
Kanshō nutzt eigene lokale Ports, nicht die Vite-/FastAPI-Defaults: Frontend **5188**, Backend **8018** (`strictPort`, kein Ausweichen auf 5173/5174). Dann http://localhost:5188 – erster Start legt das Admin-Profil an.
|
||||
|
||||
Frame-Test und lokaler Abnahmelauf:
|
||||
|
||||
```powershell
|
||||
.\scripts\test-mvp.ps1
|
||||
```
|
||||
|
||||
Der Lauf setzt UTF-8, leere Provider-Keys und isolierte Temp-Daten. Er ändert nicht `backend/data/kansho.sqlite` und nicht die echten Medien. Fail-closed bleibt prüfbar; Suites, die einen Fake-Provider brauchen, setzen ihn selbst. Einzelne Suites bleiben möglich:
|
||||
|
||||
```powershell
|
||||
cd backend
|
||||
$env:PYTHONUTF8 = "1"
|
||||
.\.venv\Scripts\python tests\test_frame.py
|
||||
.\.venv\Scripts\python tests\test_mvp_journal.py
|
||||
```
|
||||
|
||||
Zwei Provider. URL, Modell und ZDR/No-Train unter **Admin → Schnittstellen**. Der Secret-Key nur in `backend/.env` (Vorlage: `backend/.env.example`, wird nicht committet) oder ebenfalls dort eintragen, nie in der Datenbank.
|
||||
|
||||
Das Sprachmodell (Dialogzug und Journalentwurf) bleibt fail-closed ohne Key plus ZDR und Trainingsverbot. Maskierung ist lokal per Muster; ein Detect-LLM ist optional und sieht Klartext, deshalb klein oder später Ollama.
|
||||
|
||||
Tests: `KANSHO_FAKE_PROVIDER=1`, optional `KANSHO_FAKE_DETECT=1`. Zusätzlich `tests/test_privacy_detect.py`.
|
||||
|
||||
Lokales Backup der Urlaubs-Testdaten (persönlich, unverschlüsselt, ohne `.env` oder Keys):
|
||||
|
||||
```powershell
|
||||
.\scripts\backup-local.ps1 create
|
||||
.\scripts\backup-local.ps1 restore -Archive .\local-backups\<datei>.zip -Confirm
|
||||
# bestehenden Stand ersetzen, nach automatischem Sicherheitsbackup:
|
||||
.\scripts\backup-local.ps1 restore -Archive .\local-backups\<datei>.zip -Confirm -Replace
|
||||
```
|
||||
|
||||
Restore bestätigt ausdrücklich, legt vorher ein Sicherheitsbackup an und überschreibt nie still. Backend währenddessen beenden.
|
||||
|
||||
## Transfer auf die Heim-Umgebung
|
||||
|
||||
Persönliche Daten liegen **nicht** beim Provider. Der einmalige Laptop-Transport lief über das selbst gehostete Gitea. Nach dem Restore auf dem Heimrechner `transfer/` aus dem Arbeitsbaum entfernen (Historie behält das Zip).
|
||||
|
||||
Docker/Postgres, Gitea-Deploy und Cutover: `docs/DEPLOYMENT.md` und `docs/architecture/technical/runtime_and_deploy.md`.
|
||||
|
||||
## Lokal weiterarbeiten
|
||||
|
||||
```powershell
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# Secrets only. URL, Modell und ZDR/No-Train: Admin → Schnittstellen.
|
||||
# Env überschreibt die Admin-Werte, wenn gesetzt.
|
||||
|
||||
# development | test | production
|
||||
# production: externes Klartext-Detect default-off. Ziel bleibt lokales Detect.
|
||||
# Übergang bis Ollama: KANSHO_ALLOW_REMOTE_DETECT=1 (Operator, kein Admin-Flag).
|
||||
# KANSHO_ENV=development
|
||||
# KANSHO_ALLOW_REMOTE_DETECT=
|
||||
|
||||
# Generate: nur maskierter Kontext. Extern: Key Pflicht.
|
||||
KANSHO_PROVIDER_KEY=
|
||||
# KANSHO_PROVIDER_URL=https://openrouter.ai/api/v1/chat/completions
|
||||
# KANSHO_PROVIDER_MODEL=openai/gpt-4o-mini
|
||||
# KANSHO_PROVIDER_ZDR=1
|
||||
# KANSHO_PROVIDER_NO_TRAIN=1
|
||||
|
||||
# Detect: lokal (Ollama LAN) oder in Development/Test remote. Production remote nur mit KANSHO_ALLOW_REMOTE_DETECT.
|
||||
# URL/Modell liegen in LLM-Profilen (Admin → Schnittstellen), nicht in der .env.
|
||||
KANSHO_DETECT_PROVIDER_KEY=
|
||||
# KANSHO_DETECT_PROVIDER_URL=https://openrouter.ai/api/v1/chat/completions
|
||||
# KANSHO_DETECT_PROVIDER_MODEL=openai/gpt-4.1-nano
|
||||
# KANSHO_DETECT_ZDR=1
|
||||
# KANSHO_DETECT_NO_TRAIN=1
|
||||
|
||||
# Nur Tests
|
||||
# KANSHO_FAKE_PROVIDER=1
|
||||
# KANSHO_FAKE_DETECT=1
|
||||
|
||||
# Docker/Server only. Local Windows stays SQLite (default).
|
||||
# KANSHO_DB_BACKEND=postgres
|
||||
# DB_HOST=postgres
|
||||
# DB_NAME=kansho
|
||||
# DB_USER=kansho
|
||||
# DB_PASSWORD=
|
||||
|
||||
# Betriebsmail für Passwort-Reset und Adressbestätigung. Ohne SMTP schlägt der Versand fehl.
|
||||
# In development wird der Link nur ins Server-Log geschrieben.
|
||||
# SMTP_HOST=
|
||||
# SMTP_PORT=587
|
||||
# SMTP_USER=
|
||||
# SMTP_PASSWORD=
|
||||
# SMTP_FROM=kansho@example.test
|
||||
# APP_URL=http://localhost:5188
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
ENV PIP_DEFAULT_TIMEOUT=120
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
RUN chmod +x /app/startup.sh \
|
||||
&& mkdir -p /app/data/media
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["/app/startup.sh"]
|
||||
@@ -0,0 +1,468 @@
|
||||
"""Login identifiers, address verification, and one-time reset links."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from auth import hash_password
|
||||
from db import row_to_dict
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9._]{3,32}$")
|
||||
PUBLIC_COLUMNS = (
|
||||
"id",
|
||||
"email",
|
||||
"name",
|
||||
"role",
|
||||
"status",
|
||||
"tier_id",
|
||||
"created",
|
||||
"username",
|
||||
"secondary_email",
|
||||
"email_verified_at",
|
||||
"secondary_email_verified_at",
|
||||
"pending_email",
|
||||
"pending_secondary_email",
|
||||
)
|
||||
FORGOT_MESSAGE = (
|
||||
"Wenn ein Konto zu dieser Angabe existiert, wurde ein Link an die verifizierten Adressen gesendet."
|
||||
)
|
||||
RESET_MINUTES = 60
|
||||
VERIFY_HOURS = 24
|
||||
LINK_LIMIT = 3
|
||||
|
||||
|
||||
def utc_stamp(moment: datetime | None = None) -> str:
|
||||
current = moment or datetime.utcnow()
|
||||
return current.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
|
||||
def app_base_url() -> str:
|
||||
return (os.environ.get("APP_URL") or "http://localhost:5188").rstrip("/")
|
||||
|
||||
|
||||
def profile_view(row: dict | None) -> dict | None:
|
||||
if not row:
|
||||
return None
|
||||
data = {key: row.get(key) for key in PUBLIC_COLUMNS}
|
||||
data["email_verified"] = bool(row.get("email_verified_at"))
|
||||
data["secondary_email_verified"] = bool(row.get("secondary_email_verified_at"))
|
||||
return data
|
||||
|
||||
|
||||
def normalize_email(value: str) -> str:
|
||||
email = (value or "").strip().lower()
|
||||
if "@" not in email or email.startswith("@") or email.endswith("@") or " " in email or email.count("@") != 1:
|
||||
raise HTTPException(400, "E-Mail-Adresse ist ungültig")
|
||||
local, domain = email.split("@", 1)
|
||||
if not local or "." not in domain:
|
||||
raise HTTPException(400, "E-Mail-Adresse ist ungültig")
|
||||
return email
|
||||
|
||||
|
||||
def normalize_username(value: str) -> str:
|
||||
username = (value or "").strip().lower()
|
||||
if not USERNAME_RE.match(username):
|
||||
raise HTTPException(400, "Benutzername: 3–32 Zeichen, Buchstaben, Ziffern, Punkt oder Unterstrich")
|
||||
return username
|
||||
|
||||
|
||||
def find_by_login(conn, identifier: str) -> dict | None:
|
||||
key = (identifier or "").strip().lower()
|
||||
if not key:
|
||||
return None
|
||||
return row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM profiles
|
||||
WHERE lower(email) = ? OR lower(COALESCE(username, '')) = ?
|
||||
""",
|
||||
(key, key),
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
|
||||
def _owner_of_address(conn, email: str, exclude_id: str | None) -> dict | None:
|
||||
return row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id FROM profiles
|
||||
WHERE (
|
||||
lower(email) = ?
|
||||
OR lower(COALESCE(secondary_email, '')) = ?
|
||||
OR lower(COALESCE(pending_email, '')) = ?
|
||||
OR lower(COALESCE(pending_secondary_email, '')) = ?
|
||||
)
|
||||
AND id != COALESCE(?, '')
|
||||
""",
|
||||
(email, email, email, email, exclude_id or ""),
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
|
||||
def assert_address_free(conn, email: str, exclude_id: str | None) -> None:
|
||||
if _owner_of_address(conn, email, exclude_id):
|
||||
raise HTTPException(409, "E-Mail ist bereits vergeben")
|
||||
|
||||
|
||||
def assert_username_free(conn, username: str, exclude_id: str | None) -> None:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id FROM profiles
|
||||
WHERE lower(COALESCE(username, '')) = ? AND id != COALESCE(?, '')
|
||||
""",
|
||||
(username, exclude_id or ""),
|
||||
).fetchone()
|
||||
if row:
|
||||
raise HTTPException(409, "Benutzername ist bereits vergeben")
|
||||
|
||||
|
||||
def _recent_count(conn, table: str, profile_id: str) -> int:
|
||||
cutoff = utc_stamp(datetime.utcnow() - timedelta(hours=1))
|
||||
if table not in {"password_reset_tokens", "email_verification_tokens"}:
|
||||
raise HTTPException(500, "Unbekannte Token-Tabelle")
|
||||
row = conn.execute(
|
||||
f"SELECT COUNT(*) AS n FROM {table} WHERE profile_id = ? AND created > ?",
|
||||
(profile_id, cutoff),
|
||||
).fetchone()
|
||||
return int(row["n"])
|
||||
|
||||
|
||||
def issue_password_reset(conn, profile_id: str) -> str | None:
|
||||
if _recent_count(conn, "password_reset_tokens", profile_id) >= LINK_LIMIT:
|
||||
return None
|
||||
raw = secrets.token_urlsafe(32)
|
||||
now = datetime.utcnow()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO password_reset_tokens (id, profile_id, token_hash, expires_at, created)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
str(uuid.uuid4()),
|
||||
profile_id,
|
||||
hash_token(raw),
|
||||
utc_stamp(now + timedelta(minutes=RESET_MINUTES)),
|
||||
utc_stamp(now),
|
||||
),
|
||||
)
|
||||
return raw
|
||||
|
||||
|
||||
def issue_verification(conn, profile_id: str, purpose: str, email: str) -> str | None:
|
||||
if purpose not in {"primary", "secondary"}:
|
||||
raise HTTPException(400, "Unbekannter Verifikationszweck")
|
||||
if _recent_count(conn, "email_verification_tokens", profile_id) >= LINK_LIMIT:
|
||||
return None
|
||||
raw = secrets.token_urlsafe(32)
|
||||
now = datetime.utcnow()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO email_verification_tokens
|
||||
(id, profile_id, purpose, email, token_hash, expires_at, created)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
str(uuid.uuid4()),
|
||||
profile_id,
|
||||
purpose,
|
||||
email,
|
||||
hash_token(raw),
|
||||
utc_stamp(now + timedelta(hours=VERIFY_HOURS)),
|
||||
utc_stamp(now),
|
||||
),
|
||||
)
|
||||
return raw
|
||||
|
||||
|
||||
def reset_link(token: str) -> str:
|
||||
return f"{app_base_url()}/reset-password?token={token}"
|
||||
|
||||
|
||||
def verify_link(token: str) -> str:
|
||||
return f"{app_base_url()}/verify-email?token={token}"
|
||||
|
||||
|
||||
def reset_message(token: str) -> tuple[str, str]:
|
||||
body = (
|
||||
"Über diesen Link setzt du das Passwort für Kanshō zurück.\n\n"
|
||||
f"{reset_link(token)}\n\n"
|
||||
"Der Link gilt 60 Minuten und kann nur einmal verwendet werden. "
|
||||
"Wenn du das nicht angefordert hast, ignoriere diese Nachricht."
|
||||
)
|
||||
return "Kanshō – Passwort zurücksetzen", body
|
||||
|
||||
|
||||
def verify_message(token: str) -> tuple[str, str]:
|
||||
body = (
|
||||
"Über diesen Link bestätigst du eine E-Mail-Adresse für Kanshō.\n\n"
|
||||
f"{verify_link(token)}\n\n"
|
||||
"Der Link gilt 24 Stunden und kann nur einmal verwendet werden."
|
||||
)
|
||||
return "Kanshō – E-Mail-Adresse bestätigen", body
|
||||
|
||||
|
||||
def change_notice() -> tuple[str, str]:
|
||||
body = (
|
||||
"Für dieses Kanshō-Konto wurde eine Änderung der E-Mail-Adresse angefordert. "
|
||||
"Wenn du das nicht warst, setze das Passwort zurück."
|
||||
)
|
||||
return "Kanshō – Änderung der E-Mail-Adresse", body
|
||||
|
||||
|
||||
def verified_reset_recipients(profile: dict) -> list[str]:
|
||||
recipients: list[str] = []
|
||||
if profile.get("email_verified_at") and profile.get("email"):
|
||||
recipients.append(profile["email"])
|
||||
if profile.get("secondary_email_verified_at") and profile.get("secondary_email"):
|
||||
recipients.append(profile["secondary_email"])
|
||||
return recipients
|
||||
|
||||
|
||||
def admin_reset_recipients(profile: dict) -> list[str]:
|
||||
recipients: list[str] = []
|
||||
if profile.get("email"):
|
||||
recipients.append(profile["email"])
|
||||
if profile.get("secondary_email_verified_at") and profile.get("secondary_email"):
|
||||
recipients.append(profile["secondary_email"])
|
||||
return recipients
|
||||
|
||||
|
||||
def notice_recipients(profile: dict) -> list[str]:
|
||||
return verified_reset_recipients(profile)
|
||||
|
||||
|
||||
def load_profile(conn, profile_id: str) -> dict | None:
|
||||
return row_to_dict(conn.execute("SELECT * FROM profiles WHERE id = ?", (profile_id,)).fetchone())
|
||||
|
||||
|
||||
def consume_password_reset(conn, token: str, password: str) -> None:
|
||||
if len(password or "") < 4:
|
||||
raise HTTPException(400, "Passwort muss mind. 4 Zeichen haben")
|
||||
now = utc_stamp()
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM password_reset_tokens
|
||||
WHERE token_hash = ? AND used_at IS NULL AND expires_at > ?
|
||||
""",
|
||||
(hash_token(token.strip()), now),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
raise HTTPException(400, "Ungültiger oder abgelaufener Link")
|
||||
conn.execute(
|
||||
"UPDATE profiles SET password_hash = ? WHERE id = ?",
|
||||
(hash_password(password), row["profile_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE password_reset_tokens SET used_at = ? WHERE id = ?",
|
||||
(now, row["id"]),
|
||||
)
|
||||
conn.execute("DELETE FROM sessions WHERE profile_id = ?", (row["profile_id"],))
|
||||
|
||||
|
||||
def consume_verification(conn, token: str) -> dict:
|
||||
now = utc_stamp()
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM email_verification_tokens
|
||||
WHERE token_hash = ? AND used_at IS NULL AND expires_at > ?
|
||||
""",
|
||||
(hash_token(token.strip()), now),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
raise HTTPException(400, "Ungültiger oder abgelaufener Link")
|
||||
profile = load_profile(conn, row["profile_id"])
|
||||
if not profile:
|
||||
raise HTTPException(400, "Ungültiger oder abgelaufener Link")
|
||||
purpose = row["purpose"]
|
||||
address = row["email"]
|
||||
if purpose == "primary":
|
||||
pending = (profile.get("pending_email") or "").strip().lower()
|
||||
current = (profile.get("email") or "").strip().lower()
|
||||
if pending:
|
||||
if address != pending:
|
||||
raise HTTPException(400, "Ungültiger oder abgelaufener Link")
|
||||
assert_address_free(conn, address, profile["id"])
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE profiles
|
||||
SET email = ?, email_verified_at = ?, pending_email = NULL
|
||||
WHERE id = ?
|
||||
""",
|
||||
(address, now, profile["id"]),
|
||||
)
|
||||
conn.execute("DELETE FROM sessions WHERE profile_id = ?", (profile["id"],))
|
||||
elif address == current:
|
||||
conn.execute(
|
||||
"UPDATE profiles SET email_verified_at = ? WHERE id = ?",
|
||||
(now, profile["id"]),
|
||||
)
|
||||
else:
|
||||
raise HTTPException(400, "Ungültiger oder abgelaufener Link")
|
||||
elif purpose == "secondary":
|
||||
pending = (profile.get("pending_secondary_email") or "").strip().lower()
|
||||
current = (profile.get("secondary_email") or "").strip().lower()
|
||||
if pending:
|
||||
if address != pending:
|
||||
raise HTTPException(400, "Ungültiger oder abgelaufener Link")
|
||||
assert_address_free(conn, address, profile["id"])
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE profiles
|
||||
SET secondary_email = ?, secondary_email_verified_at = ?, pending_secondary_email = NULL
|
||||
WHERE id = ?
|
||||
""",
|
||||
(address, now, profile["id"]),
|
||||
)
|
||||
elif current and address == current:
|
||||
conn.execute(
|
||||
"UPDATE profiles SET secondary_email_verified_at = ? WHERE id = ?",
|
||||
(now, profile["id"]),
|
||||
)
|
||||
else:
|
||||
raise HTTPException(400, "Ungültiger oder abgelaufener Link")
|
||||
else:
|
||||
raise HTTPException(400, "Ungültiger oder abgelaufener Link")
|
||||
conn.execute(
|
||||
"UPDATE email_verification_tokens SET used_at = ? WHERE id = ?",
|
||||
(now, row["id"]),
|
||||
)
|
||||
updated = load_profile(conn, profile["id"])
|
||||
viewed = profile_view(updated)
|
||||
if viewed is None:
|
||||
raise HTTPException(400, "Ungültiger oder abgelaufener Link")
|
||||
return viewed
|
||||
|
||||
|
||||
def _queue_verification(conn, profile: dict, purpose: str, address: str) -> tuple[str, str] | None:
|
||||
raw = issue_verification(conn, profile["id"], purpose, address)
|
||||
if not raw:
|
||||
return None
|
||||
return verify_message(raw)
|
||||
|
||||
|
||||
def apply_identifier_edits(
|
||||
conn,
|
||||
profile: dict,
|
||||
*,
|
||||
username: str | None = None,
|
||||
email: str | None = None,
|
||||
secondary_email: str | None = None,
|
||||
username_set: bool = False,
|
||||
email_set: bool = False,
|
||||
secondary_set: bool = False,
|
||||
) -> list[tuple[list[str], str, str]]:
|
||||
"""Apply username and address edits. Returns mails as (recipients, subject, body)."""
|
||||
jobs: list[tuple[list[str], str, str]] = []
|
||||
if username_set:
|
||||
if username is None or username.strip() == "":
|
||||
conn.execute("UPDATE profiles SET username = NULL WHERE id = ?", (profile["id"],))
|
||||
else:
|
||||
cleaned = normalize_username(username)
|
||||
assert_username_free(conn, cleaned, profile["id"])
|
||||
conn.execute("UPDATE profiles SET username = ? WHERE id = ?", (cleaned, profile["id"]))
|
||||
profile = load_profile(conn, profile["id"]) or profile
|
||||
if email_set and email is not None:
|
||||
cleaned = normalize_email(email)
|
||||
current = (profile.get("email") or "").strip().lower()
|
||||
secondary = (profile.get("secondary_email") or "").strip().lower()
|
||||
pending_secondary = (profile.get("pending_secondary_email") or "").strip().lower()
|
||||
if cleaned == current:
|
||||
pass
|
||||
else:
|
||||
if cleaned == secondary or cleaned == pending_secondary:
|
||||
raise HTTPException(409, "E-Mail ist bereits vergeben")
|
||||
assert_address_free(conn, cleaned, profile["id"])
|
||||
conn.execute("UPDATE profiles SET pending_email = ? WHERE id = ?", (cleaned, profile["id"]))
|
||||
queued = _queue_verification(conn, profile, "primary", cleaned)
|
||||
if queued:
|
||||
subject, body = queued
|
||||
jobs.append(([cleaned], subject, body))
|
||||
notice = notice_recipients(profile)
|
||||
notice = [item for item in notice if item != cleaned]
|
||||
if notice:
|
||||
jobs.append((notice, *change_notice()))
|
||||
else:
|
||||
raise HTTPException(429, "Zu viele Bestätigungslinks. Bitte später erneut versuchen.")
|
||||
profile = load_profile(conn, profile["id"]) or profile
|
||||
if secondary_set:
|
||||
if secondary_email is None or secondary_email.strip() == "":
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE profiles
|
||||
SET secondary_email = NULL,
|
||||
secondary_email_verified_at = NULL,
|
||||
pending_secondary_email = NULL
|
||||
WHERE id = ?
|
||||
""",
|
||||
(profile["id"],),
|
||||
)
|
||||
else:
|
||||
cleaned = normalize_email(secondary_email)
|
||||
current_primary = (profile.get("email") or "").strip().lower()
|
||||
pending_primary = (profile.get("pending_email") or "").strip().lower()
|
||||
current_secondary = (profile.get("secondary_email") or "").strip().lower()
|
||||
if cleaned == current_primary or cleaned == pending_primary:
|
||||
raise HTTPException(409, "E-Mail ist bereits vergeben")
|
||||
if cleaned == current_secondary and profile.get("secondary_email_verified_at") and not profile.get("pending_secondary_email"):
|
||||
pass
|
||||
else:
|
||||
assert_address_free(conn, cleaned, profile["id"])
|
||||
conn.execute(
|
||||
"UPDATE profiles SET pending_secondary_email = ? WHERE id = ?",
|
||||
(cleaned, profile["id"]),
|
||||
)
|
||||
queued = _queue_verification(conn, profile, "secondary", cleaned)
|
||||
if queued:
|
||||
subject, body = queued
|
||||
jobs.append(([cleaned], subject, body))
|
||||
notice = [item for item in notice_recipients(profile) if item != cleaned]
|
||||
if notice:
|
||||
jobs.append((notice, *change_notice()))
|
||||
else:
|
||||
raise HTTPException(429, "Zu viele Bestätigungslinks. Bitte später erneut versuchen.")
|
||||
return jobs
|
||||
|
||||
|
||||
def deliver_jobs(jobs: list[tuple[list[str], str, str]]) -> None:
|
||||
from mailer import send_mail
|
||||
|
||||
for recipients, subject, body in jobs:
|
||||
send_mail(recipients, subject, body)
|
||||
|
||||
|
||||
def resend_verification(conn, profile: dict, purpose: str) -> tuple[list[str], str, str]:
|
||||
if purpose == "primary":
|
||||
target = (profile.get("pending_email") or "").strip().lower()
|
||||
if not target and not profile.get("email_verified_at"):
|
||||
target = (profile.get("email") or "").strip().lower()
|
||||
if not target or (profile.get("email_verified_at") and not profile.get("pending_email")):
|
||||
raise HTTPException(400, "Keine offene Bestätigung für die primäre E-Mail")
|
||||
elif purpose == "secondary":
|
||||
target = (profile.get("pending_secondary_email") or "").strip().lower()
|
||||
if not target and profile.get("secondary_email") and not profile.get("secondary_email_verified_at"):
|
||||
target = profile["secondary_email"].strip().lower()
|
||||
if not target or (
|
||||
profile.get("secondary_email_verified_at") and not profile.get("pending_secondary_email")
|
||||
):
|
||||
raise HTTPException(400, "Keine offene Bestätigung für die zweite E-Mail")
|
||||
else:
|
||||
raise HTTPException(400, "Unbekannter Verifikationszweck")
|
||||
queued = _queue_verification(conn, profile, purpose, target)
|
||||
if not queued:
|
||||
raise HTTPException(429, "Zu viele Bestätigungslinks. Bitte später erneut versuchen.")
|
||||
subject, body = queued
|
||||
return [target], subject, body
|
||||
@@ -0,0 +1,234 @@
|
||||
{
|
||||
"seed_revision": "2026-08-29-voice-legacy-immutable-v1",
|
||||
"purpose": "journal_generate",
|
||||
"max_instruction_chars": 1000,
|
||||
"max_label_chars": 80,
|
||||
"max_summary_chars": 160,
|
||||
"slots": {
|
||||
"transformation": {
|
||||
"instruction_key": "transformation_instructions",
|
||||
"variants": [
|
||||
{
|
||||
"id": "journal-generate-transformation-correction",
|
||||
"guideline_key": "correction",
|
||||
"label": "behutsam",
|
||||
"summary": "Nur Korrektur von Sprache und Zeichensetzung.",
|
||||
"instruction": "Korrigiere nur Rechtschreibung, Grammatik und Zeichensetzung. Behalte Satzbau, Reihenfolge und Formulierungen."
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-transformation-copyedit",
|
||||
"guideline_key": "copyedit",
|
||||
"label": "redaktionell",
|
||||
"summary": "Glättet Übergänge und behält die vorhandene Struktur.",
|
||||
"instruction": "Glätte holprige Stellen und verbessere Übergänge. Behalte die vorhandene Satz- und Absatzstruktur weitgehend."
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-transformation-reshape",
|
||||
"guideline_key": "reshape",
|
||||
"label": "spürbar",
|
||||
"summary": "Formuliert schwache Stellen neu, wenn die Lesbarkeit gewinnt.",
|
||||
"instruction": "Formuliere schwache oder fehlerhafte Stellen eigenständig neu. Die Grundstruktur darf sich ändern, wenn die Lesbarkeit gewinnt."
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-transformation-substantial",
|
||||
"guideline_key": "substantial",
|
||||
"label": "substanziell",
|
||||
"summary": "Überarbeitet Sätze und Absätze eigenständig.",
|
||||
"instruction": "Überarbeite den Text substanziell. Formuliere Sätze und Absätze eigenständig neu, wenn dadurch Lesbarkeit, Rhythmus oder Wirkung gewinnen. Bewahre nicht automatisch die Quellsyntax.",
|
||||
"is_default": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"detail": {
|
||||
"instruction_key": "detail_instructions",
|
||||
"variants": [
|
||||
{
|
||||
"id": "journal-generate-detail-compact",
|
||||
"guideline_key": "compact",
|
||||
"label": "kompakt",
|
||||
"summary": "Nur die tragenden belegten Ereignisse.",
|
||||
"instruction": "Wähle nur die tragenden belegten Ereignisse. Lass nebensächliche Einzelheiten weg."
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-detail-selected",
|
||||
"guideline_key": "selected",
|
||||
"label": "ausgewählt",
|
||||
"summary": "Wichtige Ereignisse und wenige markante Details.",
|
||||
"instruction": "Behalte die wichtigen belegten Ereignisse und wenige markante Details."
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-detail-broad",
|
||||
"guideline_key": "broad",
|
||||
"label": "weitgehend",
|
||||
"summary": "Die meisten belegten Ereignisse und einmaligen Details.",
|
||||
"instruction": "Behalte die meisten belegten Ereignisse und einmaligen Details. Kürze nur Offensichtliches."
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-detail-complete",
|
||||
"guideline_key": "complete",
|
||||
"label": "vollständig",
|
||||
"summary": "Alle belegten Ereignisse; nur echte Wiederholungen entfernen.",
|
||||
"instruction": "Erhalte sämtliche belegten Ereignisse und einmaligen Details. Entferne nur echte Wiederholungen.",
|
||||
"is_default": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"voice": {
|
||||
"instruction_key": "voice_instructions",
|
||||
"variants": [
|
||||
{
|
||||
"id": "journal-generate-voice-neutral",
|
||||
"guideline_key": "neutral",
|
||||
"label": "Ohne persönliches Profil",
|
||||
"summary": "Natürlicher, neutraler Journalstil ohne Writing Profile und ohne Stilbeispiele.",
|
||||
"instruction": "Schreibe in einem natürlichen, neutralen Journalstil. Der aktuelle Tagesdialog ist nur Inhaltsquelle, nicht Stilvorlage.",
|
||||
"style_context": {
|
||||
"include_core": false,
|
||||
"include_facet": false,
|
||||
"include_traits": false,
|
||||
"include_style_examples": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-voice-light",
|
||||
"guideline_key": "light",
|
||||
"label": "Persönliche Stimme dezent",
|
||||
"summary": "Zurückhaltende Orientierung am Schreibkern, ohne Facet, Traits oder Beispiele.",
|
||||
"instruction": "Orientiere dich zurückhaltend am mitgegebenen Schreibkern. Übernimm daraus keine Ereignisse, Personen oder Bewertungen.",
|
||||
"style_context": {
|
||||
"include_core": true,
|
||||
"include_facet": false,
|
||||
"include_traits": false,
|
||||
"include_style_examples": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-voice-noticeable",
|
||||
"guideline_key": "noticeable",
|
||||
"label": "spürbar",
|
||||
"summary": "Writing Profile spürbar, ohne in den Vordergrund zu treten.",
|
||||
"instruction": "Wende das Writing Profile spürbar an, ohne es in den Vordergrund zu stellen.",
|
||||
"style_context": {
|
||||
"include_core": true,
|
||||
"include_facet": true,
|
||||
"include_traits": true,
|
||||
"include_style_examples": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-voice-clear",
|
||||
"guideline_key": "clear",
|
||||
"label": "Persönliche Stimme deutlich",
|
||||
"summary": "Deutliche Orientierung an Tonlage, Rhythmus und Wortwahl des Profils.",
|
||||
"instruction": "Wende Tonlage, Rhythmus und Wortwahl des mitgegebenen Writing Profiles deutlich an, ohne den Text künstlich zu literarisieren. Übernimm daraus keine Ereignisse, Personen oder Bewertungen.",
|
||||
"is_default": true,
|
||||
"style_context": {
|
||||
"include_core": true,
|
||||
"include_facet": true,
|
||||
"include_traits": true,
|
||||
"include_style_examples": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-voice-with-examples",
|
||||
"guideline_key": "with_examples",
|
||||
"label": "Persönliche Stimme mit Beispielen",
|
||||
"summary": "Deutliche Profilanwendung; historische Texte nur als sprachliche Referenz.",
|
||||
"instruction": "Wende Tonlage, Rhythmus und Wortwahl des mitgegebenen Writing Profiles deutlich an. STYLE_EXAMPLES sind ausschließlich sprachliche Referenz; ihre Ereignisse, Personen, Orte und Bewertungen gehören nicht zum heutigen Tag.",
|
||||
"style_context": {
|
||||
"include_core": true,
|
||||
"include_facet": true,
|
||||
"include_traits": true,
|
||||
"include_style_examples": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-voice-legacy-neutral",
|
||||
"guideline_key": "legacy_neutral",
|
||||
"label": "Legacy – vor Stilkontextsteuerung · neutral",
|
||||
"summary": "Rekonstruierte frühere Semantik: Neutraler Journalstil, Writing Profile nur als leise Tendenz.",
|
||||
"instruction": "Schreibe in einem neutralen Journalstil. Das Writing Profile höchstens als leise Tendenz.",
|
||||
"style_context": {
|
||||
"include_core": true,
|
||||
"include_facet": true,
|
||||
"include_traits": true,
|
||||
"include_style_examples": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-voice-legacy-light",
|
||||
"guideline_key": "legacy_light",
|
||||
"label": "Legacy – vor Stilkontextsteuerung · dezent",
|
||||
"summary": "Rekonstruierte frühere Semantik: Rhythmus und Wortwahl des Writing Profiles zurückhaltend.",
|
||||
"instruction": "Nimm Rhythmus und Wortwahl des Writing Profiles zurückhaltend auf.",
|
||||
"style_context": {
|
||||
"include_core": true,
|
||||
"include_facet": true,
|
||||
"include_traits": true,
|
||||
"include_style_examples": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-voice-legacy-noticeable",
|
||||
"guideline_key": "legacy_noticeable",
|
||||
"label": "Legacy – vor Stilkontextsteuerung · spürbar",
|
||||
"summary": "Rekonstruierte frühere Semantik: Writing Profile spürbar, ohne in den Vordergrund zu treten.",
|
||||
"instruction": "Wende das Writing Profile spürbar an, ohne es in den Vordergrund zu stellen.",
|
||||
"style_context": {
|
||||
"include_core": true,
|
||||
"include_facet": true,
|
||||
"include_traits": true,
|
||||
"include_style_examples": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-voice-legacy-clear",
|
||||
"guideline_key": "legacy_clear",
|
||||
"label": "Legacy – vor Stilkontextsteuerung · deutlich",
|
||||
"summary": "Rekonstruierte frühere Semantik: Rhythmus, Wortwahl und Reflexionsdichte des Writing Profiles deutlich.",
|
||||
"instruction": "Wende Rhythmus, Wortwahl und Reflexionsdichte des Writing Profiles deutlich an, ohne den Text künstlich zu literarisieren.",
|
||||
"style_context": {
|
||||
"include_core": true,
|
||||
"include_facet": true,
|
||||
"include_traits": true,
|
||||
"include_style_examples": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"narrative": {
|
||||
"instruction_key": "narrative_instructions",
|
||||
"variants": [
|
||||
{
|
||||
"id": "journal-generate-narrative-chronicle",
|
||||
"guideline_key": "chronicle",
|
||||
"label": "Chronik",
|
||||
"summary": "Sachlich in der Reihenfolge der Quellen.",
|
||||
"instruction": "Erzähle sachlich in der Reihenfolge der Quellen. Gewichte nichts extra."
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-narrative-structured",
|
||||
"guideline_key": "structured",
|
||||
"label": "gegliedert",
|
||||
"summary": "Klar gegliedert, ohne besondere Momente extra herauszustellen.",
|
||||
"instruction": "Gliedere den Tag klar. Besondere Momente nicht extra herausstellen."
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-narrative-weighted",
|
||||
"guideline_key": "weighted",
|
||||
"label": "gewichtet",
|
||||
"summary": "Belegte Kontraste und besondere Momente erkennbar.",
|
||||
"instruction": "Arbeite belegte Kontraste und besondere Momente erkennbar heraus. Erzeuge daraus keine neue Dramatik, Bewertung oder Kausalität.",
|
||||
"is_default": true
|
||||
},
|
||||
{
|
||||
"id": "journal-generate-narrative-emphasized",
|
||||
"guideline_key": "emphasized",
|
||||
"label": "betont",
|
||||
"summary": "Belegte Kontraste und Höhepunkte im Vordergrund.",
|
||||
"instruction": "Stelle belegte Kontraste und Höhepunkte deutlich in den Vordergrund. Erzeuge daraus keine neue Dramatik, Bewertung oder Kausalität."
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1 +1,54 @@
|
||||
[]
|
||||
[
|
||||
{
|
||||
"id": "mvp-dialogue-turn",
|
||||
"slug": "mvp.dialogue_turn",
|
||||
"name": "MVP Dialogzug",
|
||||
"description": "Ein maskierter Call: Operation und Impuls zusammen. Keine Maskierung, kein Journalentwurf, keine Intent-Engines.",
|
||||
"category": "mvp",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"template": "Du sitzt mit [[SELF]] im Reflection Space {{space_title}} und bist Kanshō: jemand, der zugehört hat. Warm, genau, auf Augenhöhe. Ein Mensch würde dich für einen Vertrauten halten, nicht für ein Programm.\n\n{{register_hint}}\n\n{{opening_hint}}\n\n{{interaction_hint}}\n\nDie gesprochene Zeile (impulse) klingt wie mündliches Deutsch in einem Gespräch: flüssig, nicht steif, nicht literarisch, nicht wie eine App. Kurze Sätze sind in Ordnung. Kein „also“-Referat, kein „interessant“, kein „danke, dass du das teilst“, kein „lass uns“, kein „zusammengefasst“, keine Operationsnamen, kein JSON, keine Registerworte.\n\nNur user:-Zeilen sind Erlebtes. assistant:-Zeilen sind frühere Impulse, keine Tatsachen.\nErfinde nichts: keinen nächsten Ort, kein Gefühl, keine Motivation. Keine Tatsachen, Gefühle oder Motivationen erfinden. Ein Plan ist kein Vollzug. Nicht erwähnt ist nicht geschehen.\n\nLänge und Zahl der Anschlussimpulse stehen in der Interaction-Angabe oben, nicht als anonyme Produktregel. Leite einen echten Gedanken oder eine knappe Frage aus dem Gerade-Gesagten ab. Zusammenfassung des Zugs ist nicht der Default.\nNacherzählen nur, um einen Widerspruch oder Logikbruch zu klären.\n\nIntern eine Operation wählen, sie nicht aussprechen:\n- fortfuehren: am offenen Faden bleiben; bei Schluss den Schluss halten\n- konkretisieren: eine wirklich fehlende, tragende Angabe\n- plan_aufgreifen: ein genanntes Vorhaben als Vorhaben\n- abweichung: eine selbst markierte Veränderung\n- erleben_vertiefen: nur wenn [[SELF]] Wahrnehmung oder Wirkung öffnet\n- bedeutung: einen erkennbaren Punkt, ohne Tiefe zu erzwingen\n\nBei geschlossener Szene: keine Frage, keinen Bogen. Nicht den nächsten Vollzug setzen.\n\nKontext:\n{{dialogue_context}}\n\nNur JSON:\n{\"operation\":\"fortfuehren|konkretisieren|plan_aufgreifen|abweichung|erleben_vertiefen|bedeutung\",\"impulse\":\"gesprochene Zeile wie ein Vertrauter\"}\n"
|
||||
},
|
||||
{
|
||||
"id": "mvp-journal-generate",
|
||||
"slug": "mvp.journal_generate",
|
||||
"name": "MVP Journalentwurf",
|
||||
"description": "Stufe 2: persönliche Narration mit journalspezifischer Generation Policy. Explizit ausgelöst. Nicht der Dialogzug.",
|
||||
"category": "mvp",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"template": "Du redigierst einen persönlichen Tagebucheintrag in der Ich-Form.\n\nAUFGABE\nFormuliere aus CURRENT_DAY_SOURCES einen lesenswerten, eigenständigen Journaltext. Die Quellen bestimmen, was geschehen ist. Ein mitgegebenes WRITING_PROFILE bestimmt, wie es erzählt wird; fehlt es, gilt nur die Stimmanweisung. Quellsyntax, Rechtschreibfehler und Dialogstruktur sind keine Ausgabevorlage.\n\nBEARBEITUNG\n{{transformation_instructions}}\n{{detail_instructions}}\n{{voice_instructions}}\n{{narrative_instructions}}\n\nINHALTSTREUE\nVerändere keine Ereignisse, Beteiligten, Orte, Zeiten oder zeitlichen Abläufe. Erhalte Verneinungen, Unsicherheiten, Korrekturen, ausdrücklich genannte Gefühle und Bewertungen sowie den Unterschied zwischen Plan und Vollzug. Ergänze keine neuen Tatsachen, Ursachen, Motive, Gefühle oder Hintergrundinformationen. Übergänge dürfen verbinden, aber nichts erklären, was die Quellen nicht erklären. Bei unklaren Satzfragmenten nicht raten, sondern neutral umformulieren oder nur den unverständlichen Teil weglassen.\n\nSTIL\nKorrigiere Rechtschreibung, Grammatik und Zeichensetzung. Verbessere Satzbau, Rhythmus, Übergänge und Absätze. Fasse echte Wiederholungen zusammen. Formuliere schwache oder fehlerhafte Stellen eigenständig neu. Gute persönliche Formulierungen dürfen erhalten bleiben. Der aktuelle Tagesdialog ist keine Stilvorlage.\nDie Quellen können aus Fließtext, Stichpunkten und Satzfragmenten bestehen. Behandle jede Passage entsprechend ihrer Form: Redigiere vorhandenen Fließtext, verwandle Stichpunkte und Fragmente in vollständige Prosa und verbinde beides zu einem einheitlichen Eintrag. Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus.\n\nDATENSCHUTZ\nÜbernimm vorhandene Platzhalter wie [[PERSON:01]] unverändert. Erzeuge keine neuen Platzhalter und schreibe keine Klarnamen an ihre Stelle.\n\nKONTEXT\n{{writing_profile}}{{style_examples}}\nCURRENT_DAY_SOURCES\n{{reconstruction}}\n\nEXISTING_TEXT\n{{existing_text}}\n\nAUSGABE\nErste Zeile: eine kurze passende Überschrift ohne neue Tatsachen. Danach zusammenhängende, natürlich gegliederte Absätze. Keine Aufzählung und keine Meta-Erklärung.\n",
|
||||
"seed_revision": "2026-08-29-voice-legacy-immutable-v1"
|
||||
},
|
||||
{
|
||||
"id": "mvp-journal-reconstruct",
|
||||
"slug": "mvp.journal_reconstruct",
|
||||
"name": "MVP Journal Rekonstruktion",
|
||||
"description": "Stufe 1: inhaltliche Rekonstruktion des Tagesdialogs. Keine Narration, kein Writing Profile.",
|
||||
"category": "mvp",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"template": "Referenziere die lokal vergebenen Quellen. Du entscheidest nicht, welche ausgewählten Nutzerpassagen verloren gehen dürfen. Du lieferst keine autoritative Semantik.\n\nJede nichtleere Nutzerzeile hat eine lokale Quellen-ID in eckigen Klammern: [u1], [u2], … in der ursprünglichen Reihenfolge. Assistant-Zeilen haben [a1], [a2], … und sind nur Antwortkontext, niemals Inhaltsquelle oder Evidence.\n\nJeder Claim ist ausschließlich ein wörtlicher Ausschnitt aus genau der genannten Nutzerzeile. Keine Paraphrasen. Keine Assistentenformulierungen. kind-Werte und contradictions/uncertainties/plan_changes/corrections sind unverbindliche Hinweise, keine festgestellten Tatsachen. Eine Evidence legitimiert keine zusätzliche Behauptung.\n\nsource_order ist die lokale Nutzerreihenfolge. chronology darf davon nur abweichen, wenn Uhrzeiten in der jeweils zugeordneten Nutzerzeile nachweisbar sind. Relative Angaben wie später oder danach nicht zum Umsortieren verwenden, wenn die Reihenfolge nicht aus bestätigten Uhrzeiten folgt. Gleiche oder mehrdeutige Uhrzeiten in Quellenreihenfolge lassen.\n\nAntworte ausschließlich mit JSON:\n{\"source_order\":[\"u1\",\"u2\"],\"chronology\":[{\"source_id\":\"u1\",\"source\":\"user\",\"time\":null,\"claims\":[{\"kind\":\"event\",\"evidence\":\"wörtlicher Ausschnitt aus genau dieser Nutzerzeile\"}]}],\"contradictions\":[],\"uncertainties\":[],\"plan_changes\":[],\"corrections\":[]}\n\nsource_order muss jede Nutzer-ID genau einmal in der lokalen Reihenfolge enthalten. chronology muss jede Nutzer-ID genau einmal abdecken. source ist Pflicht und immer genau user. time nur als reines Zeitstoken aus derselben Nutzerzeile (6:00, 06:00, 6:00 Uhr). Assistant-IDs und jedes andere source sind verboten. Keine events-Listen, keine allgemeine Evidence für mehrere Behauptungen. Du darfst den kanonischen Quellentext nicht ersetzen oder kürzen.\n\nDialog:\n{{dialogue_context}}\n"
|
||||
},
|
||||
{
|
||||
"id": "mvp-entity-detect",
|
||||
"slug": "mvp.entity_detect",
|
||||
"name": "MVP Maskierung",
|
||||
"description": "Semantische Detection von Identitäten im persönlichen Egress. Spans und Typen, keine Tokens. Keine Dialogoperation, kein Journalentwurf.",
|
||||
"category": "privacy",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"seed_revision": "2026-09-08-detect-homonym-v1",
|
||||
"template": "Untersuche den gesamten Text semantisch. Entscheide kontextabhängig, nicht nach Großschreibung oder Namensähnlichkeit allein.\n\nSchützenswert ist eine konkrete Bezeichnung oder Information, durch die eine natürliche Person, ein genauer persönlicher Ort, eine Organisation, ein privates Projekt oder ein anderer im Produktrahmen definierter Identifikator erkennbar werden kann. Allgemeine Gegenstände, Tätigkeiten, Lebensmittel, Räume und Beschreibungen sind keine Identitäten, sofern sie im konkreten Satz nicht als Eigenname oder identifizierende Information verwendet werden.\n\nErlaubte entity_type-Werte ausschließlich: PERSON, PLACE, ORG, PROJECT.\nEin Lebensmittel oder Gericht ist keine Entität. Nicht als FOOD oder einen anderen Typ melden, sondern weglassen. Derselbe Wortlaut kann im selben Text Person und Gericht sein; nur die identifizierende Nennung gehört in entities.\n\nKontrast, keine Wortliste:\n- «Sushi kam ins Wohnzimmer» kann eine Person bezeichnen.\n- «Ich aß Sushi» bezeichnet ein Gericht.\n- «Ich arbeitete am privaten Projekt Aurora» kann ein schützenswertes Projekt bezeichnen.\n- «Ich ging auf den Balkon» bezeichnet keinen Eigennamen.\n- «Ich traf Anna» bezeichnet eine Person.\n\nNur das identifizierende Wort oder die identifizierende Wortgruppe, nicht den ganzen Satz. Offsets beziehen sich ausschließlich auf den gelieferten Text nach «Text:», 0-basiert, end ausschließlich. text muss ein exakter Substring dieses Texts sein, nicht aus den Beispielen oben. Wenn Offsets unsicher sind, kopiere trotzdem das identifizierende Wort zeichengetreu.\n\nKeine Tokens, keine Platzhalter, keine kanonischen Namen, keine Aliase, keine zusätzlichen Felder.\n\nAntworte nur mit JSON. Form: {\"entities\":[{\"start\":<zahl>,\"end\":<zahl>,\"text\":\"<exakter Substring>\",\"entity_type\":\"PERSON\"}]}\nNichts schützenswert: {\"entities\":[]}\n\nText:\n{{source_text}}\n"
|
||||
},
|
||||
{
|
||||
"id": "mvp-profile-review",
|
||||
"slug": "mvp.profile_review",
|
||||
"name": "MVP Writing-Profile-Review",
|
||||
"description": "Gebündelte semantische Review. Kein neues Profil, nur Änderungsvorschläge. Explizit ausgelöst, nicht nach jedem Dialog.",
|
||||
"category": "mvp",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"template": "Du prüfst das Writing Profile von [[SELF]] anhand des ProfileAnalysisPackage.\n\nLokale Heuristiken sind Messhilfe, keine Traits. Existing-before-New: bestätigen, präzisieren, Scope ändern, zusammenführen oder aufteilen, erst dann neu anlegen. Neue Facet nur bei semantischer Notwendigkeit.\nUrlaubstagebücher belegen primär autobiographical_journal. Core nur soweit unterschiedliche Quellenarten das tragen.\nWenn du aus diesem Chat weitere authentische Nutzertexte kennst, darfst du sie als evidence_basis external_chat_history nutzen. Keine erfundenen Kanshō-Source-IDs, keine erfundenen Beispiele.\nRepresentative Examples nur aus echten Nutzertexten, nie aus Trigger- oder UI-Texten.\n\nAntworte ausschließlich mit JSON gemäß expected_result. kind: kansho.profile_analysis_result. Actions: keep, update, add, remove, reclassify, merge, split.\n\nPaket:\n{{review_package}}\n"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,18 @@
|
||||
[
|
||||
{
|
||||
"role": "generate",
|
||||
"name": "openrouter",
|
||||
"url": "https://openrouter.ai/api/v1/chat/completions",
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"zdr": true,
|
||||
"no_train": true
|
||||
},
|
||||
{
|
||||
"role": "detect",
|
||||
"name": "openrouter",
|
||||
"url": "https://openrouter.ai/api/v1/chat/completions",
|
||||
"model": "openai/gpt-4.1-nano",
|
||||
"zdr": true,
|
||||
"no_train": true
|
||||
}
|
||||
]
|
||||
+272
-4
@@ -1,14 +1,282 @@
|
||||
"""Internal context for later Gateway use. Never sends originals to a provider."""
|
||||
"""Internal context for later Gateway use. Never sends originals to a provider.
|
||||
|
||||
Selection goes through retrieval specs. Recency/SQL limits live in retrieval.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from data_layer import read
|
||||
from dialogue_store import get_conversation
|
||||
from retrieval import retrieve
|
||||
from journal_body import plain_text
|
||||
|
||||
PURPOSES = {"dialogue_turn", "journal_generate", "journal_reconstruct"}
|
||||
CLOSURE = re.compile(
|
||||
r"\bauf den heimweg\b|"
|
||||
r"\bheimweg\b|"
|
||||
r"\bauf den rückweg\b|"
|
||||
r"\bdamit (?:war|ist) der tag\b|"
|
||||
r"\bdas war (?:der|so ein|unser) tag\b|"
|
||||
r"\bende des tages\b|"
|
||||
r"\bgenug für heute\b|"
|
||||
r"\bso endete\b|"
|
||||
r"müde.{0,80}(?:heimweg|rückweg)|"
|
||||
r"(?:eindr[uü]cke|erschöpft).{0,80}(?:heimweg|rückweg)|"
|
||||
r"machten uns.{0,40}auf den (?:heimweg|rückweg)|"
|
||||
r"\bbald ins bett\b|"
|
||||
r"\blegten uns.{0,40}ins bett\b|"
|
||||
r"\bgleich ins bett\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def build_internal_context(profile_id: str, conversation_id: str) -> dict:
|
||||
messages = read("conversation_messages", profile_id=profile_id, context={"conversation_id": conversation_id})
|
||||
def is_closing_turn(text: str) -> bool:
|
||||
return bool(CLOSURE.search((text or "").strip()))
|
||||
|
||||
|
||||
def infer_register(user_bodies: list[str]) -> str:
|
||||
last = (user_bodies[-1] if user_bodies else "").strip()
|
||||
if not last:
|
||||
return "Noch kein Erzählstand. Warte auf den ersten Satz, setze keine Szene."
|
||||
if is_closing_turn(last):
|
||||
return (
|
||||
"Die Szene oder der Abschnitt ist zu Ende. Halte den Schluss. "
|
||||
"Keine Frage, keinen Bogen, keine neue Station."
|
||||
)
|
||||
words = last.split()
|
||||
emotion = re.search(r"aufgeregt|traurig|berührt|ängst|wütend|freude|ruhig|unsicher", last, re.I)
|
||||
plan = re.search(r"geplant|wollen|werde|sollten|vorhaben", last, re.I)
|
||||
dense = len(words) >= 40 or last.count(".") + last.count("!") >= 3
|
||||
if dense:
|
||||
return (
|
||||
"[[SELF]] erzählt gerade ausführlich. Am letzten erzählten Punkt bleiben. "
|
||||
"Nicht nacherzählen. Nacherzählen nur bei Widerspruch oder Logikbruch. "
|
||||
"Keine erfundene Empfindung, keine erfundene nächste Szene."
|
||||
)
|
||||
if emotion:
|
||||
return (
|
||||
"[[SELF]] hat Erleben geöffnet. Bei Bedeutung oder Wirkung bleiben, "
|
||||
"die schon benannte Empfindung nicht nachplappern."
|
||||
)
|
||||
if plan:
|
||||
return "Ein Vorhaben ist genannt. Als Plan behandeln, nicht als vollzogen."
|
||||
return "Kurz und natürlich antworten, am offenen Punkt, wie jemand der zugehört hat."
|
||||
|
||||
|
||||
def build_internal_context(
|
||||
profile_id: str,
|
||||
conversation_id: str | None = None,
|
||||
space_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
purpose: str = "dialogue_turn",
|
||||
include_space_recency: bool = True,
|
||||
include_existing: bool = False,
|
||||
conversation_ids: list[str] | None = None,
|
||||
existing_text: str = "",
|
||||
reconstruction: str = "",
|
||||
day_spec: dict | None = None,
|
||||
writing_profile: str | None = None,
|
||||
style_examples: str = "",
|
||||
transformation_instructions: str = "",
|
||||
detail_instructions: str = "",
|
||||
voice_instructions: str = "",
|
||||
narrative_instructions: str = "",
|
||||
) -> dict:
|
||||
purpose = purpose if purpose in PURPOSES else "dialogue_turn"
|
||||
if conversation_id and not space_id:
|
||||
conv = get_conversation(profile_id, conversation_id)
|
||||
space_id = conv.get("space_id")
|
||||
journal_day_id = journal_day_id or conv.get("journal_day_id")
|
||||
|
||||
items: list[dict] = []
|
||||
space_title = ""
|
||||
if space_id:
|
||||
from dialogue_store import get_space
|
||||
|
||||
try:
|
||||
space_title = get_space(profile_id, space_id).get("title") or ""
|
||||
except Exception:
|
||||
space_title = ""
|
||||
items.append({"type": "space", "space_id": space_id, "title": space_title})
|
||||
|
||||
selected_ids = list(conversation_ids or [])
|
||||
if conversation_id and conversation_id not in selected_ids and purpose == "dialogue_turn":
|
||||
selected_ids.append(conversation_id)
|
||||
|
||||
day_messages = []
|
||||
if journal_day_id and purpose != "journal_generate":
|
||||
spec = {
|
||||
"kind": "day_messages",
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_ids": selected_ids or None,
|
||||
}
|
||||
if day_spec:
|
||||
spec.update({key: value for key, value in day_spec.items() if key != "kind"})
|
||||
day_messages = retrieve(profile_id, spec)
|
||||
items.append({"type": "day_messages", "messages": day_messages})
|
||||
elif conversation_id and purpose == "dialogue_turn":
|
||||
day_messages = read("conversation_messages", profile_id=profile_id, context={"conversation_id": conversation_id})
|
||||
items.append({"type": "day_messages", "messages": day_messages})
|
||||
|
||||
if purpose == "dialogue_turn" and space_id and include_space_recency:
|
||||
prior = retrieve(
|
||||
profile_id,
|
||||
{
|
||||
"kind": "space_entries",
|
||||
"space_id": space_id,
|
||||
"exclude_day_id": journal_day_id,
|
||||
},
|
||||
)
|
||||
items.append(
|
||||
{
|
||||
"type": "prior_entries",
|
||||
"entries": [
|
||||
{
|
||||
"entry_id": item["entry_id"],
|
||||
"calendar_date": item.get("calendar_date"),
|
||||
"excerpt": item.get("excerpt") or plain_text(item.get("body") or ""),
|
||||
}
|
||||
for item in prior
|
||||
],
|
||||
}
|
||||
)
|
||||
recent_sources = retrieve(
|
||||
profile_id,
|
||||
{
|
||||
"kind": "space_recent_sources",
|
||||
"space_id": space_id,
|
||||
"exclude_conversation_id": conversation_id,
|
||||
"exclude_journal_day_id": journal_day_id,
|
||||
},
|
||||
)
|
||||
items.append({"type": "space_recent_sources", "sources": recent_sources})
|
||||
|
||||
if purpose == "journal_generate":
|
||||
if writing_profile is None:
|
||||
from writing_profile_store import compile_task_brief
|
||||
|
||||
brief = compile_task_brief(profile_id, "journal_generate")
|
||||
else:
|
||||
brief = writing_profile
|
||||
items.append({"type": "writing_profile", "compiled_brief": brief})
|
||||
items.append({"type": "style_examples", "body": style_examples or ""})
|
||||
items.append({"type": "transformation_instructions", "text": transformation_instructions or ""})
|
||||
items.append({"type": "detail_instructions", "text": detail_instructions or ""})
|
||||
items.append({"type": "voice_instructions", "text": voice_instructions or ""})
|
||||
items.append({"type": "narrative_instructions", "text": narrative_instructions or ""})
|
||||
if reconstruction:
|
||||
items.append({"type": "reconstruction", "body": reconstruction})
|
||||
if include_existing and existing_text:
|
||||
items.append({"type": "existing_text", "body": plain_text(existing_text)})
|
||||
elif purpose == "dialogue_turn":
|
||||
items.append({"type": "interaction_hint", "text": ""})
|
||||
|
||||
return {
|
||||
"profile_id": profile_id,
|
||||
"conversation_id": conversation_id,
|
||||
"messages": messages,
|
||||
"space_id": space_id,
|
||||
"space_title": space_title,
|
||||
"journal_day_id": journal_day_id,
|
||||
"purpose": purpose,
|
||||
"items": items,
|
||||
"messages": day_messages,
|
||||
"egress": False,
|
||||
"note": "Volltext bleibt lokal. Minimierung und Egress nur über das Privacy Gateway.",
|
||||
}
|
||||
|
||||
|
||||
def assemble_text(context: dict) -> dict[str, str]:
|
||||
dialogue_parts: list[str] = []
|
||||
prior_parts: list[str] = []
|
||||
source_parts: list[str] = []
|
||||
writing_profile = ""
|
||||
interaction_hint = ""
|
||||
existing_text = ""
|
||||
reconstruction = ""
|
||||
style_examples = ""
|
||||
transformation_instructions = ""
|
||||
detail_instructions = ""
|
||||
voice_instructions = ""
|
||||
narrative_instructions = ""
|
||||
user_bodies: list[str] = []
|
||||
purpose = context.get("purpose") or ""
|
||||
for item in context.get("items") or []:
|
||||
kind = item.get("type")
|
||||
if kind == "day_messages":
|
||||
day_messages = item.get("messages") or []
|
||||
if purpose == "journal_reconstruct":
|
||||
from journal_reconstruct import assign_source_ids
|
||||
|
||||
day_messages = assign_source_ids(day_messages)
|
||||
for message in day_messages:
|
||||
role = message.get("role") or "user"
|
||||
body = message.get("body") or ""
|
||||
source_id = message.get("source_id")
|
||||
if purpose == "journal_reconstruct" and source_id:
|
||||
dialogue_parts.append(f"[{source_id}] {role}: {body}")
|
||||
else:
|
||||
dialogue_parts.append(f"{role}: {body}")
|
||||
if role == "user":
|
||||
user_bodies.append(body)
|
||||
elif kind == "prior_entries":
|
||||
for entry in item.get("entries") or []:
|
||||
date = entry.get("calendar_date") or ""
|
||||
excerpt = entry.get("excerpt") or ""
|
||||
if excerpt:
|
||||
prior_parts.append(f"{date}: {excerpt}")
|
||||
elif kind == "space_recent_sources":
|
||||
for source in item.get("sources") or []:
|
||||
excerpt = source.get("excerpt") or ""
|
||||
if excerpt:
|
||||
source_parts.append(excerpt)
|
||||
elif kind == "writing_profile":
|
||||
writing_profile = item.get("compiled_brief") or ""
|
||||
elif kind == "interaction_hint":
|
||||
interaction_hint = item.get("text") or ""
|
||||
elif kind == "existing_text":
|
||||
existing_text = item.get("body") or ""
|
||||
elif kind == "reconstruction":
|
||||
reconstruction = item.get("body") or ""
|
||||
elif kind == "style_examples":
|
||||
style_examples = item.get("body") or ""
|
||||
elif kind == "transformation_instructions":
|
||||
transformation_instructions = item.get("text") or ""
|
||||
elif kind == "detail_instructions":
|
||||
detail_instructions = item.get("text") or ""
|
||||
elif kind == "voice_instructions":
|
||||
voice_instructions = item.get("text") or ""
|
||||
elif kind == "narrative_instructions":
|
||||
narrative_instructions = item.get("text") or ""
|
||||
elif kind == "opening":
|
||||
pass
|
||||
opening_hint = ""
|
||||
if purpose == "dialogue_turn":
|
||||
from conversation_signals import infer_signals
|
||||
from interaction_profile_store import assemble_interaction_hint
|
||||
|
||||
interaction_hint = assemble_interaction_hint(
|
||||
context.get("profile_id"),
|
||||
infer_signals(user_bodies),
|
||||
)
|
||||
if prior_parts:
|
||||
dialogue_parts.append("Frühere Einträge im Space (nur Hinweise, keine Gewissheit):")
|
||||
dialogue_parts.extend(prior_parts)
|
||||
if source_parts and purpose == "dialogue_turn":
|
||||
dialogue_parts.append("Frühere Originalgespräche im Space (Ausschnitt, keine Gewissheit):")
|
||||
dialogue_parts.extend(source_parts)
|
||||
return {
|
||||
"dialogue_context": "\n".join(dialogue_parts).strip(),
|
||||
"writing_profile": writing_profile,
|
||||
"style_examples": style_examples,
|
||||
"transformation_instructions": transformation_instructions,
|
||||
"detail_instructions": detail_instructions,
|
||||
"voice_instructions": voice_instructions,
|
||||
"narrative_instructions": narrative_instructions,
|
||||
"interaction_hint": interaction_hint,
|
||||
"existing_text": existing_text,
|
||||
"reconstruction": reconstruction,
|
||||
"space_title": context.get("space_title") or "",
|
||||
"register_hint": infer_register(user_bodies),
|
||||
"opening_hint": opening_hint,
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Local operational signals for a conversation. No extra LLM call, not a Writing Profile."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
EMOTION = re.compile(
|
||||
r"aufgeregt|traurig|berührt|ängst|wütend|freude|ruhig|unsicher|gefühl|tiefer",
|
||||
re.I,
|
||||
)
|
||||
PLAN = re.compile(r"geplant|wollen|werde|sollten|vorhaben", re.I)
|
||||
CHRONICLE = re.compile(
|
||||
r"\b(heute|gestern|morgens|mittags|abends|danach|vorher|später|zuerst|einkauf|"
|
||||
r"spaziergang|markt|chronik)\b",
|
||||
re.I,
|
||||
)
|
||||
CLOCK = re.compile(r"\b\d{1,2}:\d{2}\b|\b\d{1,2}\s*Uhr\b", re.I)
|
||||
STOP = {
|
||||
"dann", "noch", "schon", "wieder", "gegen", "waren", "wurde", "haben", "hatte",
|
||||
"unter", "über", "nach", "beim", "eine", "einem", "einer", "dieser", "dieses",
|
||||
"auch", "aber", "dass", "wenn", "sich", "uns", "euch", "mein", "dein", "sein",
|
||||
"heute", "dann", "nach", "noch",
|
||||
}
|
||||
|
||||
DEEP_OPS = {"erleben_vertiefen", "bedeutung"}
|
||||
PLAN_OPS = {"plan_aufgreifen"}
|
||||
|
||||
|
||||
def _words(text: str) -> list[str]:
|
||||
return [
|
||||
word
|
||||
for word in re.findall(r"[a-zäöüß]{4,}", (text or "").lower())
|
||||
if word not in STOP
|
||||
]
|
||||
|
||||
|
||||
def infer_signals(user_bodies: list[str], last_operation: str | None = None) -> dict:
|
||||
"""Derive operational dialogue state. Not a Writing Profile or Interaction Preference."""
|
||||
bodies = [item.strip() for item in user_bodies if (item or "").strip()]
|
||||
blob = " ".join(bodies)
|
||||
last = bodies[-1] if bodies else ""
|
||||
op = (last_operation or "").strip().lower()
|
||||
emotion = bool(EMOTION.search(blob))
|
||||
plan = bool(PLAN.search(blob))
|
||||
chronicle = bool(CHRONICLE.search(blob) or CLOCK.search(blob))
|
||||
if emotion:
|
||||
mode, depth = "reflective", "deep"
|
||||
elif plan:
|
||||
mode, depth = "plan", "surface"
|
||||
elif chronicle:
|
||||
mode, depth = "chronicle", "surface"
|
||||
elif op in DEEP_OPS:
|
||||
mode, depth = "reflective", "deep"
|
||||
elif op in PLAN_OPS:
|
||||
mode, depth = "plan", "surface"
|
||||
else:
|
||||
mode, depth = "open", "surface"
|
||||
counts: dict[str, int] = {}
|
||||
for word in _words(blob):
|
||||
counts[word] = counts.get(word, 0) + 1
|
||||
focus = " ".join(
|
||||
item[0] for item in sorted(counts.items(), key=lambda pair: (-pair[1], pair[0]))[:8]
|
||||
)
|
||||
last_words = last.split()
|
||||
long_story = len(last_words) >= 40 or last.count(".") + last.count("!") >= 3
|
||||
if emotion and (long_story or depth == "deep"):
|
||||
intensity = "high"
|
||||
elif emotion:
|
||||
intensity = "medium"
|
||||
else:
|
||||
intensity = "low"
|
||||
return {
|
||||
"narrative_mode": mode,
|
||||
"reflection_depth": depth,
|
||||
"current_focus": focus,
|
||||
"emotional_intensity": intensity,
|
||||
"long_story": 1 if long_story else 0,
|
||||
}
|
||||
|
||||
|
||||
def _focus_words(row: dict) -> set[str]:
|
||||
return {item for item in str(row.get("current_focus") or "").split() if item}
|
||||
|
||||
|
||||
def compatible(left: dict, right: dict) -> bool:
|
||||
left_mode = (left.get("narrative_mode") or "open").strip() or "open"
|
||||
right_mode = (right.get("narrative_mode") or "open").strip() or "open"
|
||||
if left_mode == "open" or right_mode == "open":
|
||||
return False
|
||||
if left_mode != right_mode:
|
||||
return False
|
||||
left_depth = (left.get("reflection_depth") or "surface").strip() or "surface"
|
||||
right_depth = (right.get("reflection_depth") or "surface").strip() or "surface"
|
||||
if left_depth != right_depth:
|
||||
return False
|
||||
left_focus = _focus_words(left)
|
||||
right_focus = _focus_words(right)
|
||||
if len(left_focus) >= 2 and len(right_focus) >= 2 and not (left_focus & right_focus):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def similar_enough(conversations: list[dict]) -> bool:
|
||||
usable = [item for item in conversations if item]
|
||||
if len(usable) < 2:
|
||||
return False
|
||||
for index, left in enumerate(usable):
|
||||
for right in usable[index + 1 :]:
|
||||
if not compatible(left, right):
|
||||
return False
|
||||
return True
|
||||
+532
-23
@@ -1,4 +1,8 @@
|
||||
"""SQLite persistence for the local frame. PostgreSQL remains the later target."""
|
||||
"""Persistence for the local frame.
|
||||
|
||||
SQLite is the Windows/test engine. PostgreSQL 16 is the Docker/server engine.
|
||||
Switch at connect time via KANSHO_DB_BACKEND / DB_HOST; stores keep SQLite-shaped SQL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
@@ -6,18 +10,81 @@ import os
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DATA_DIR = Path(__file__).resolve().parent / "data"
|
||||
from sql_compat import (
|
||||
adapt_sql,
|
||||
postgres_connect_kwargs,
|
||||
rewrite_catalog_sql,
|
||||
split_sql,
|
||||
sqlite_schema_to_postgres,
|
||||
use_postgres,
|
||||
)
|
||||
|
||||
_env_data = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
DATA_DIR = Path(_env_data) if _env_data else Path(__file__).resolve().parent / "data"
|
||||
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
||||
SEED_PATH = Path(__file__).resolve().parent / "config" / "platform_seed.json"
|
||||
PROMPTS_SEED_PATH = Path(__file__).resolve().parent / "config" / "prompts.seed.json"
|
||||
_env_db = os.environ.get("KANSHO_DB_PATH")
|
||||
DB_PATH = Path(_env_db) if _env_db else DATA_DIR / "kansho.sqlite"
|
||||
|
||||
SQLITE_HISTORY_MIGRATIONS = (
|
||||
"001_frame",
|
||||
"002_platform",
|
||||
"003_dialogue_memory",
|
||||
"004_mvp_journal",
|
||||
"005_provider_settings",
|
||||
"006_writing_profile_dialogue_style",
|
||||
"007_journal_day_scratch",
|
||||
"008_journal_source_refs",
|
||||
"009_conversation_signals",
|
||||
"010_profile_governance",
|
||||
"011_profile_review",
|
||||
"012_profile_shell",
|
||||
"013_journal_generate_narration",
|
||||
"014_identity_registry",
|
||||
"015_journal_generation_settings",
|
||||
"016_generation_instruction_fragments",
|
||||
"017_generation_guidelines",
|
||||
"018_debug_runs",
|
||||
"019_debug_run_placement",
|
||||
"020_voice_style_context",
|
||||
"021_voice_legacy_immutable",
|
||||
)
|
||||
|
||||
_PROFILE_COLUMNS = {
|
||||
"status": "TEXT NOT NULL DEFAULT 'active'",
|
||||
"tier_id": "TEXT NOT NULL DEFAULT 'local'",
|
||||
"username": "TEXT",
|
||||
"secondary_email": "TEXT",
|
||||
"email_verified_at": "TEXT",
|
||||
"secondary_email_verified_at": "TEXT",
|
||||
"pending_email": "TEXT",
|
||||
"pending_secondary_email": "TEXT",
|
||||
}
|
||||
_ACCOUNT_INDEXES = (
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS profiles_username_lower_idx
|
||||
ON profiles (lower(username))
|
||||
WHERE username IS NOT NULL AND username <> ''
|
||||
""",
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS profiles_secondary_email_lower_idx
|
||||
ON profiles (lower(secondary_email))
|
||||
WHERE secondary_email IS NOT NULL AND secondary_email <> ''
|
||||
""",
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS profiles_pending_email_lower_idx
|
||||
ON profiles (lower(pending_email))
|
||||
WHERE pending_email IS NOT NULL AND pending_email <> ''
|
||||
""",
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS profiles_pending_secondary_email_lower_idx
|
||||
ON profiles (lower(pending_secondary_email))
|
||||
WHERE pending_secondary_email IS NOT NULL AND pending_secondary_email <> ''
|
||||
""",
|
||||
)
|
||||
_PROMPT_COLUMNS = {
|
||||
"description": "TEXT NOT NULL DEFAULT ''",
|
||||
"category": "TEXT NOT NULL DEFAULT 'uncategorized'",
|
||||
@@ -27,12 +94,122 @@ _PROMPT_COLUMNS = {
|
||||
"output_schema_json": "TEXT",
|
||||
"required_feature": "TEXT NOT NULL DEFAULT 'ai_calls'",
|
||||
"default_template": "TEXT NOT NULL DEFAULT ''",
|
||||
"seed_revision": "TEXT NOT NULL DEFAULT ''",
|
||||
"sort_order": "INTEGER NOT NULL DEFAULT 0",
|
||||
"updated": "TEXT NOT NULL DEFAULT (datetime('now'))",
|
||||
}
|
||||
_CONVERSATION_COLUMNS = {
|
||||
"space_id": "TEXT",
|
||||
"journal_day_id": "TEXT",
|
||||
"narrative_mode": "TEXT NOT NULL DEFAULT ''",
|
||||
"reflection_depth": "TEXT NOT NULL DEFAULT ''",
|
||||
"current_focus": "TEXT NOT NULL DEFAULT ''",
|
||||
"emotional_intensity": "TEXT NOT NULL DEFAULT ''",
|
||||
"long_story": "INTEGER NOT NULL DEFAULT 0",
|
||||
}
|
||||
_WRITING_PROFILE_COLUMNS = {
|
||||
"governance": "TEXT NOT NULL DEFAULT 'learning'",
|
||||
"version": "INTEGER NOT NULL DEFAULT 0",
|
||||
"review_ready": "INTEGER NOT NULL DEFAULT 0",
|
||||
"last_reviewed": "TEXT",
|
||||
"lifecycle": "TEXT NOT NULL DEFAULT 'uninitialized'",
|
||||
}
|
||||
_WRITING_SOURCE_COLUMNS = {
|
||||
"occurred_at": "TEXT",
|
||||
"context_hint": "TEXT NOT NULL DEFAULT ''",
|
||||
}
|
||||
_WRITING_SUGGESTION_COLUMNS = {
|
||||
"trait_slug": "TEXT NOT NULL DEFAULT ''",
|
||||
"action": "TEXT NOT NULL DEFAULT 'update'",
|
||||
"payload_json": "TEXT NOT NULL DEFAULT '{}'",
|
||||
}
|
||||
_WRITING_VERSION_COLUMNS = {
|
||||
"lifecycle": "TEXT NOT NULL DEFAULT ''",
|
||||
"traits_json": "TEXT NOT NULL DEFAULT '[]'",
|
||||
}
|
||||
_JOURNAL_DAY_COLUMNS = {
|
||||
"scratch_json": "TEXT NOT NULL DEFAULT '[]'",
|
||||
}
|
||||
_JOURNAL_DRAFT_COLUMNS = {
|
||||
"generation_snapshot": "TEXT NOT NULL DEFAULT '{}'",
|
||||
}
|
||||
_GUIDELINE_COLUMNS = {
|
||||
"style_context_json": "TEXT NOT NULL DEFAULT '{}'",
|
||||
}
|
||||
_DEBUG_RUN_COLUMNS = {
|
||||
"space_id": "TEXT",
|
||||
"journal_day_id": "TEXT",
|
||||
"space_title": "TEXT NOT NULL DEFAULT ''",
|
||||
"calendar_date": "TEXT NOT NULL DEFAULT ''",
|
||||
"conversation_title": "TEXT NOT NULL DEFAULT ''",
|
||||
}
|
||||
_IDENTITY_MAPPING_COLUMNS = {
|
||||
"canonical_label": "TEXT NOT NULL DEFAULT ''",
|
||||
"entity_type": "TEXT NOT NULL DEFAULT 'PERSON'",
|
||||
"status": "TEXT NOT NULL DEFAULT 'legacy_review_required'",
|
||||
"origin": "TEXT NOT NULL DEFAULT 'legacy_auto'",
|
||||
"aliases_json": "TEXT NOT NULL DEFAULT '[]'",
|
||||
# SQLite ALTER TABLE cannot use datetime('now'); backfill in migrate_legacy_identity_rows.
|
||||
"updated": "TEXT NOT NULL DEFAULT ''",
|
||||
"confirmed_at": "TEXT",
|
||||
}
|
||||
|
||||
|
||||
def _connect() -> sqlite3.Connection:
|
||||
class PostgresCompat:
|
||||
"""sqlite3-like execute/commit surface over psycopg, with SQL translation."""
|
||||
|
||||
def __init__(self, raw: Any):
|
||||
self._raw = raw
|
||||
|
||||
def execute(self, sql: str, params: Any = None):
|
||||
special = rewrite_catalog_sql(sql)
|
||||
if special:
|
||||
adapted, extra = special
|
||||
args = extra if extra is not None else params
|
||||
else:
|
||||
adapted = adapt_sql(sql)
|
||||
args = params
|
||||
if args is None:
|
||||
return self._raw.execute(adapted)
|
||||
return self._raw.execute(adapted, args)
|
||||
|
||||
def executemany(self, sql: str, seq_of_params: Any):
|
||||
adapted = adapt_sql(sql)
|
||||
cursor = None
|
||||
for params in seq_of_params:
|
||||
cursor = self._raw.execute(adapted, params)
|
||||
return cursor
|
||||
|
||||
def executescript(self, script: str):
|
||||
for statement in split_sql(sqlite_schema_to_postgres(script)):
|
||||
self._raw.execute(statement)
|
||||
return self
|
||||
|
||||
def commit(self):
|
||||
self._raw.commit()
|
||||
|
||||
def rollback(self):
|
||||
self._raw.rollback()
|
||||
|
||||
def close(self):
|
||||
self._raw.close()
|
||||
|
||||
|
||||
def _connect_postgres() -> PostgresCompat:
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
kwargs = postgres_connect_kwargs()
|
||||
if "conninfo" in kwargs:
|
||||
raw = psycopg.connect(kwargs["conninfo"], row_factory=dict_row)
|
||||
else:
|
||||
raw = psycopg.connect(row_factory=dict_row, **kwargs)
|
||||
return PostgresCompat(raw)
|
||||
|
||||
|
||||
def _connect():
|
||||
if use_postgres():
|
||||
return _connect_postgres()
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(DB_PATH, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
@@ -53,32 +230,53 @@ def get_db():
|
||||
conn.close()
|
||||
|
||||
|
||||
def row_to_dict(row: sqlite3.Row | None) -> dict | None:
|
||||
def row_to_dict(row: Any) -> dict | None:
|
||||
if row is None:
|
||||
return None
|
||||
return dict(row)
|
||||
|
||||
|
||||
def _column_names(conn: sqlite3.Connection, table: str) -> set[str]:
|
||||
def _column_names(conn: Any, table: str) -> set[str]:
|
||||
rows = conn.execute(f"PRAGMA table_info({table})").fetchall()
|
||||
return {row["name"] for row in rows}
|
||||
|
||||
|
||||
def _ensure_columns(conn: sqlite3.Connection, table: str, columns: dict[str, str]) -> None:
|
||||
def _ensure_columns(conn: Any, table: str, columns: dict[str, str]) -> None:
|
||||
existing = _column_names(conn, table)
|
||||
for name, ddl in columns.items():
|
||||
if name not in existing:
|
||||
conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}")
|
||||
|
||||
|
||||
def _mark(conn: sqlite3.Connection, migration_id: str) -> None:
|
||||
def _mark(conn: Any, migration_id: str) -> None:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO schema_migrations (id) VALUES (?)",
|
||||
(migration_id,),
|
||||
)
|
||||
|
||||
|
||||
def _seed_platform(conn: sqlite3.Connection) -> None:
|
||||
def _ensure_account_identifiers(conn: Any) -> None:
|
||||
"""Add login identifiers on existing SQLite files and grandfather current emails once."""
|
||||
_ensure_columns(conn, "profiles", _PROFILE_COLUMNS)
|
||||
for statement in _ACCOUNT_INDEXES:
|
||||
conn.execute(statement)
|
||||
row = conn.execute(
|
||||
"SELECT id FROM schema_migrations WHERE id = ?",
|
||||
("027_account_identifiers",),
|
||||
).fetchone()
|
||||
if row:
|
||||
return
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE profiles
|
||||
SET email_verified_at = COALESCE(created, datetime('now'))
|
||||
WHERE email_verified_at IS NULL OR email_verified_at = ''
|
||||
"""
|
||||
)
|
||||
_mark(conn, "027_account_identifiers")
|
||||
|
||||
|
||||
def _seed_platform(conn: Any) -> None:
|
||||
seed = json.loads(SEED_PATH.read_text(encoding="utf-8"))
|
||||
for tier in seed.get("tiers", []):
|
||||
conn.execute(
|
||||
@@ -115,39 +313,350 @@ def _seed_platform(conn: sqlite3.Connection) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _seed_prompts(conn: sqlite3.Connection) -> None:
|
||||
"""Prompts come from JSON/DB, never from Python string literals."""
|
||||
def _seed_prompts(conn: Any) -> None:
|
||||
"""Prompts come from JSON/DB, never from Python string literals.
|
||||
|
||||
Untouched system prompts (template == default_template) receive the seed.
|
||||
Independently edited prompts keep their template; only default_template and
|
||||
seed_revision are refreshed so Reset-to-default stays possible.
|
||||
"""
|
||||
items = json.loads(PROMPTS_SEED_PATH.read_text(encoding="utf-8"))
|
||||
for item in items:
|
||||
template = item.get("template") or ""
|
||||
revision = item.get("seed_revision") or ""
|
||||
existing = row_to_dict(
|
||||
conn.execute("SELECT * FROM ai_prompts WHERE slug = ?", (item["slug"],)).fetchone()
|
||||
)
|
||||
if existing is None:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO ai_prompts
|
||||
(id, slug, name, description, category, prompt_type, template,
|
||||
required_feature, is_system_default, default_template, seed_revision)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
|
||||
""",
|
||||
(
|
||||
item["id"],
|
||||
item["slug"],
|
||||
item["name"],
|
||||
item.get("description") or "",
|
||||
item.get("category") or "uncategorized",
|
||||
item.get("prompt_type") or "base",
|
||||
template,
|
||||
item.get("required_feature") or "ai_calls",
|
||||
template,
|
||||
revision,
|
||||
),
|
||||
)
|
||||
continue
|
||||
untouched = (existing.get("template") or "") == (existing.get("default_template") or "")
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO ai_prompts
|
||||
(id, slug, name, description, category, prompt_type, template,
|
||||
required_feature, is_system_default, default_template)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)
|
||||
UPDATE ai_prompts
|
||||
SET name = ?, description = ?, category = ?, prompt_type = ?,
|
||||
required_feature = ?, is_system_default = 1, default_template = ?,
|
||||
seed_revision = ?,
|
||||
template = CASE WHEN ? = 1 THEN ? ELSE template END,
|
||||
updated = datetime('now')
|
||||
WHERE slug = ?
|
||||
""",
|
||||
(
|
||||
item["id"],
|
||||
item["slug"],
|
||||
item["name"],
|
||||
item.get("description") or "",
|
||||
item.get("category") or "uncategorized",
|
||||
item.get("prompt_type") or "base",
|
||||
item.get("template") or "",
|
||||
item.get("required_feature") or "ai_calls",
|
||||
item.get("template") or "",
|
||||
template,
|
||||
revision,
|
||||
1 if untouched else 0,
|
||||
template,
|
||||
item["slug"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _parse_legacy_ids(raw: str | None) -> list[str]:
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
return [str(item) for item in data if item]
|
||||
|
||||
|
||||
def _insert_source_refs(
|
||||
conn: Any,
|
||||
table: str,
|
||||
owner_col: str,
|
||||
owner_id: str,
|
||||
profile_id: str,
|
||||
conversation_ids: list[str],
|
||||
message_ids: list[str],
|
||||
) -> None:
|
||||
conn.execute(f"DELETE FROM {table} WHERE {owner_col} = ?", (owner_id,))
|
||||
for index, source_id in enumerate(conversation_ids):
|
||||
conn.execute(
|
||||
f"""
|
||||
INSERT OR IGNORE INTO {table}
|
||||
({owner_col}, profile_id, source_kind, source_id, sort_order)
|
||||
VALUES (?, ?, 'conversation', ?, ?)
|
||||
""",
|
||||
(owner_id, profile_id, source_id, index),
|
||||
)
|
||||
for index, source_id in enumerate(message_ids):
|
||||
conn.execute(
|
||||
f"""
|
||||
INSERT OR IGNORE INTO {table}
|
||||
({owner_col}, profile_id, source_kind, source_id, sort_order)
|
||||
VALUES (?, ?, 'message', ?, ?)
|
||||
""",
|
||||
(owner_id, profile_id, source_id, index),
|
||||
)
|
||||
|
||||
|
||||
def migrate_journal_source_refs(conn: Any) -> None:
|
||||
"""Copy JSON id lists into relational source-ref tables without touching Source."""
|
||||
existing = {
|
||||
row["name"]
|
||||
for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
||||
).fetchall()
|
||||
}
|
||||
if "journal_draft_source_refs" not in existing or "journal_entry_version_source_refs" not in existing:
|
||||
return
|
||||
for row in conn.execute(
|
||||
"SELECT id, profile_id, source_conversation_ids, source_message_ids FROM journal_drafts"
|
||||
).fetchall():
|
||||
already = conn.execute(
|
||||
"SELECT 1 FROM journal_draft_source_refs WHERE draft_id = ? LIMIT 1",
|
||||
(row["id"],),
|
||||
).fetchone()
|
||||
if already:
|
||||
continue
|
||||
_insert_source_refs(
|
||||
conn,
|
||||
"journal_draft_source_refs",
|
||||
"draft_id",
|
||||
row["id"],
|
||||
row["profile_id"],
|
||||
_parse_legacy_ids(row["source_conversation_ids"]),
|
||||
_parse_legacy_ids(row["source_message_ids"]),
|
||||
)
|
||||
for row in conn.execute(
|
||||
"SELECT id, profile_id, source_conversation_ids, source_message_ids FROM journal_entry_versions"
|
||||
).fetchall():
|
||||
already = conn.execute(
|
||||
"SELECT 1 FROM journal_entry_version_source_refs WHERE version_id = ? LIMIT 1",
|
||||
(row["id"],),
|
||||
).fetchone()
|
||||
if already:
|
||||
continue
|
||||
_insert_source_refs(
|
||||
conn,
|
||||
"journal_entry_version_source_refs",
|
||||
"version_id",
|
||||
row["id"],
|
||||
row["profile_id"],
|
||||
_parse_legacy_ids(row["source_conversation_ids"]),
|
||||
_parse_legacy_ids(row["source_message_ids"]),
|
||||
)
|
||||
|
||||
|
||||
def _migrate_writing_profile_dialogue_style(conn: Any) -> None:
|
||||
"""Existing DBs keep the old CHECK; recreate so dialogue_style is a valid source kind."""
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'writing_profile_sources'"
|
||||
).fetchone()
|
||||
)
|
||||
current = (row or {}).get("sql") or ""
|
||||
if "dialogue_style" in current:
|
||||
return
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE writing_profile_sources_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('imported_text', 'journal_entry', 'dialogue_style')),
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
entry_id TEXT REFERENCES journal_entries(id),
|
||||
weight REAL NOT NULL DEFAULT 1,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
INSERT INTO writing_profile_sources_new
|
||||
(id, profile_id, kind, body, entry_id, weight, created)
|
||||
SELECT id, profile_id, kind, body, entry_id, weight, created
|
||||
FROM writing_profile_sources;
|
||||
DROP TABLE writing_profile_sources;
|
||||
ALTER TABLE writing_profile_sources_new RENAME TO writing_profile_sources;
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _migrate_writing_profile_shell(conn: Any) -> None:
|
||||
"""Layers stay; inferred count-facets are not the profile. Manual style keys become traits."""
|
||||
from writing_profile_schema import LEGACY_STYLE_KEYS, coerce_slug, facet_layer, normalize_facet_key
|
||||
|
||||
facet_sql = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'writing_profile_facets'"
|
||||
).fetchone()
|
||||
)
|
||||
current = (facet_sql or {}).get("sql") or ""
|
||||
if current and ("inferred" in current or "layer" not in current):
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE writing_profile_facets_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
facet_key TEXT NOT NULL,
|
||||
layer TEXT NOT NULL DEFAULT 'context',
|
||||
value TEXT NOT NULL DEFAULT '',
|
||||
evidence TEXT NOT NULL DEFAULT '',
|
||||
origin TEXT NOT NULL DEFAULT 'manual',
|
||||
locked INTEGER NOT NULL DEFAULT 0,
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (profile_id, facet_key)
|
||||
);
|
||||
INSERT INTO writing_profile_facets_new
|
||||
(id, profile_id, facet_key, layer, value, evidence, origin, locked, updated)
|
||||
SELECT id, profile_id, facet_key, 'context', value, evidence,
|
||||
CASE WHEN origin IN ('manual', 'imported', 'accepted_suggestion', 'initial_build')
|
||||
THEN origin ELSE 'manual' END,
|
||||
locked, updated
|
||||
FROM writing_profile_facets
|
||||
WHERE origin IS NOT NULL AND origin != 'inferred';
|
||||
DROP TABLE writing_profile_facets;
|
||||
ALTER TABLE writing_profile_facets_new RENAME TO writing_profile_facets;
|
||||
"""
|
||||
)
|
||||
names = _column_names(conn, "writing_profile_facets")
|
||||
if "layer" not in names:
|
||||
conn.execute("ALTER TABLE writing_profile_facets ADD COLUMN layer TEXT NOT NULL DEFAULT 'context'")
|
||||
rows = [
|
||||
dict(item)
|
||||
for item in conn.execute("SELECT * FROM writing_profile_facets").fetchall()
|
||||
]
|
||||
for item in rows:
|
||||
key = normalize_facet_key(item.get("facet_key") or "")
|
||||
origin = item.get("origin") or "manual"
|
||||
if (item.get("facet_key") or "") in LEGACY_STYLE_KEYS:
|
||||
slug = coerce_slug(item.get("facet_key") or "trait")
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO writing_profile_traits
|
||||
(id, profile_id, facet_key, slug, label, statement, origin, locked, status)
|
||||
VALUES (?, ?, 'core', ?, ?, ?, ?, ?, 'active')
|
||||
""",
|
||||
(
|
||||
item["id"],
|
||||
item["profile_id"],
|
||||
slug,
|
||||
item.get("facet_key") or slug,
|
||||
item.get("value") or "",
|
||||
origin if origin in {"manual", "imported", "accepted_suggestion", "initial_build"} else "manual",
|
||||
int(item.get("locked") or 0),
|
||||
),
|
||||
)
|
||||
conn.execute("DELETE FROM writing_profile_facets WHERE id = ?", (item["id"],))
|
||||
continue
|
||||
if key != (item.get("facet_key") or ""):
|
||||
conn.execute(
|
||||
"UPDATE writing_profile_facets SET facet_key = ? WHERE id = ?",
|
||||
(key, item["id"]),
|
||||
)
|
||||
layer = facet_layer(key)
|
||||
conn.execute("UPDATE writing_profile_facets SET layer = ? WHERE id = ?", (layer, item["id"]))
|
||||
for row in conn.execute("SELECT profile_id FROM writing_profiles").fetchall():
|
||||
profile_id = row["profile_id"]
|
||||
remaining = conn.execute(
|
||||
"SELECT 1 FROM writing_profile_facets WHERE profile_id = ? LIMIT 1",
|
||||
(profile_id,),
|
||||
).fetchone()
|
||||
traits = conn.execute(
|
||||
"SELECT 1 FROM writing_profile_traits WHERE profile_id = ? AND status = 'active' LIMIT 1",
|
||||
(profile_id,),
|
||||
).fetchone()
|
||||
if remaining or traits:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE writing_profiles
|
||||
SET lifecycle = 'confirmed'
|
||||
WHERE profile_id = ? AND lifecycle = 'uninitialized'
|
||||
""",
|
||||
(profile_id,),
|
||||
)
|
||||
|
||||
|
||||
def _seed_runtime(conn: Any) -> None:
|
||||
_seed_platform(conn)
|
||||
_seed_prompts(conn)
|
||||
from provider_settings import seed_provider_settings
|
||||
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
|
||||
|
||||
seed_provider_settings(conn)
|
||||
seed_generation_instructions(conn)
|
||||
backfill_missing_settings(conn)
|
||||
seed_generation_instructions(conn)
|
||||
seed_generation_instructions(conn)
|
||||
|
||||
|
||||
def refresh_runtime_seed() -> None:
|
||||
"""Re-seed catalog rows after a data truncate. Schema must already exist."""
|
||||
with get_db() as conn:
|
||||
_seed_runtime(conn)
|
||||
from writing_profile_store import bootstrap_from_existing
|
||||
|
||||
bootstrap_from_existing()
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
if use_postgres():
|
||||
from db_init import ensure_postgres_ready
|
||||
|
||||
ensure_postgres_ready()
|
||||
refresh_runtime_seed()
|
||||
return
|
||||
|
||||
schema = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||
with get_db() as conn:
|
||||
conn.executescript(schema)
|
||||
_seed_platform(conn)
|
||||
_seed_prompts(conn)
|
||||
_ensure_columns(conn, "profiles", _PROFILE_COLUMNS)
|
||||
_ensure_columns(conn, "ai_prompts", _PROMPT_COLUMNS)
|
||||
_mark(conn, "001_frame")
|
||||
_mark(conn, "002_platform")
|
||||
_mark(conn, "003_dialogue_memory")
|
||||
_seed_prompts(conn)
|
||||
from provider_settings import seed_provider_settings
|
||||
|
||||
_ensure_columns(conn, "provider_settings", {"profile_id": "TEXT"})
|
||||
seed_provider_settings(conn)
|
||||
_ensure_account_identifiers(conn)
|
||||
_ensure_columns(conn, "conversations", _CONVERSATION_COLUMNS)
|
||||
_ensure_columns(conn, "journal_days", _JOURNAL_DAY_COLUMNS)
|
||||
_ensure_columns(conn, "journal_drafts", _JOURNAL_DRAFT_COLUMNS)
|
||||
_ensure_columns(conn, "writing_profiles", _WRITING_PROFILE_COLUMNS)
|
||||
_migrate_writing_profile_dialogue_style(conn)
|
||||
_ensure_columns(conn, "writing_profile_sources", _WRITING_SOURCE_COLUMNS)
|
||||
_ensure_columns(conn, "writing_profile_suggestions", _WRITING_SUGGESTION_COLUMNS)
|
||||
_ensure_columns(conn, "writing_profile_versions", _WRITING_VERSION_COLUMNS)
|
||||
migrate_journal_source_refs(conn)
|
||||
_migrate_writing_profile_shell(conn)
|
||||
_ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS)
|
||||
_ensure_columns(conn, "label_sense_cues", {"excerpt": "TEXT NOT NULL DEFAULT ''"})
|
||||
from identity_store import migrate_legacy_identity_rows
|
||||
|
||||
migrate_legacy_identity_rows(conn)
|
||||
for migration_id in SQLITE_HISTORY_MIGRATIONS:
|
||||
_mark(conn, migration_id)
|
||||
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
|
||||
|
||||
_ensure_columns(conn, "generation_guidelines", _GUIDELINE_COLUMNS)
|
||||
seed_generation_instructions(conn)
|
||||
backfill_missing_settings(conn)
|
||||
_ensure_columns(conn, "debug_runs", _DEBUG_RUN_COLUMNS)
|
||||
seed_generation_instructions(conn)
|
||||
seed_generation_instructions(conn)
|
||||
from writing_profile_store import bootstrap_from_existing
|
||||
|
||||
bootstrap_from_existing()
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wait for PostgreSQL, load schema.sql, apply numbered SQL migrations. Fail-fast."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from sql_compat import postgres_connect_kwargs, split_sql, sqlite_schema_to_postgres, use_postgres
|
||||
|
||||
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parent / "migrations"
|
||||
SQLITE_HISTORY_MIGRATIONS = (
|
||||
"001_frame",
|
||||
"002_platform",
|
||||
"003_dialogue_memory",
|
||||
"004_mvp_journal",
|
||||
"005_provider_settings",
|
||||
"006_writing_profile_dialogue_style",
|
||||
"007_journal_day_scratch",
|
||||
"008_journal_source_refs",
|
||||
"009_conversation_signals",
|
||||
"010_profile_governance",
|
||||
"011_profile_review",
|
||||
"012_profile_shell",
|
||||
"013_journal_generate_narration",
|
||||
"014_identity_registry",
|
||||
"015_journal_generation_settings",
|
||||
"016_generation_instruction_fragments",
|
||||
"017_generation_guidelines",
|
||||
"018_debug_runs",
|
||||
"019_debug_run_placement",
|
||||
"020_voice_style_context",
|
||||
"021_voice_legacy_immutable",
|
||||
)
|
||||
_LEADING_DIGITS = re.compile(r"^(\d{3})_.*\.sql$")
|
||||
_schema_ready = False
|
||||
|
||||
|
||||
def mark_schema_dirty() -> None:
|
||||
"""Call after DROP SCHEMA so the next init_db reloads schema.sql."""
|
||||
global _schema_ready
|
||||
_schema_ready = False
|
||||
|
||||
|
||||
def _connect_raw():
|
||||
import psycopg
|
||||
|
||||
kwargs = postgres_connect_kwargs()
|
||||
if "conninfo" in kwargs:
|
||||
return psycopg.connect(kwargs["conninfo"], autocommit=False)
|
||||
return psycopg.connect(autocommit=False, **kwargs)
|
||||
|
||||
|
||||
def wait_for_postgres(max_retries: int = 30) -> None:
|
||||
print("Checking PostgreSQL connection...")
|
||||
last_error = None
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
conn = _connect_raw()
|
||||
conn.close()
|
||||
print("PostgreSQL ready")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 — fail-fast after retries
|
||||
last_error = exc
|
||||
print(f" waiting ({attempt}/{max_retries})")
|
||||
time.sleep(2)
|
||||
print(f"PostgreSQL not ready: {last_error}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _table_exists(conn, name: str) -> bool:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
""",
|
||||
(name,),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _load_schema(conn) -> None:
|
||||
sql = sqlite_schema_to_postgres(SCHEMA_PATH.read_text(encoding="utf-8"))
|
||||
for statement in split_sql(sql):
|
||||
conn.execute(statement)
|
||||
print("Schema loaded from schema.sql (Postgres dialect)")
|
||||
|
||||
|
||||
def _applied_ids(conn) -> set[str]:
|
||||
rows = conn.execute("SELECT id FROM schema_migrations").fetchall()
|
||||
return {row[0] for row in rows}
|
||||
|
||||
|
||||
def _record(conn, migration_id: str) -> None:
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (id) VALUES (%s) ON CONFLICT (id) DO NOTHING",
|
||||
(migration_id,),
|
||||
)
|
||||
|
||||
|
||||
def _migration_files() -> list[tuple[str, Path]]:
|
||||
if not MIGRATIONS_DIR.is_dir():
|
||||
return []
|
||||
rows: list[tuple[str, Path]] = []
|
||||
for path in sorted(MIGRATIONS_DIR.iterdir()):
|
||||
match = _LEADING_DIGITS.match(path.name)
|
||||
if not match or path.suffix != ".sql":
|
||||
continue
|
||||
rows.append((path.stem, path))
|
||||
return rows
|
||||
|
||||
|
||||
def apply_schema_and_migrations() -> None:
|
||||
conn = _connect_raw()
|
||||
try:
|
||||
if not _table_exists(conn, "schema_migrations"):
|
||||
_load_schema(conn)
|
||||
conn.commit()
|
||||
applied = _applied_ids(conn)
|
||||
if "001_frame" not in applied:
|
||||
for migration_id in SQLITE_HISTORY_MIGRATIONS:
|
||||
_record(conn, migration_id)
|
||||
conn.commit()
|
||||
print("Recorded SQLite history 001-021 as applied (greenfield Postgres schema)")
|
||||
applied = _applied_ids(conn)
|
||||
for stem, path in _migration_files():
|
||||
if stem in applied:
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
for statement in split_sql(sql):
|
||||
conn.execute(statement)
|
||||
_record(conn, stem)
|
||||
conn.commit()
|
||||
print(f"Applied migration {stem}")
|
||||
except Exception as exc:
|
||||
conn.rollback()
|
||||
print(f"Database initialization failed: {exc}")
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def ensure_postgres_ready() -> None:
|
||||
global _schema_ready
|
||||
if not use_postgres():
|
||||
return
|
||||
if _schema_ready:
|
||||
return
|
||||
wait_for_postgres()
|
||||
try:
|
||||
apply_schema_and_migrations()
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
_schema_ready = True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.environ.setdefault("KANSHO_DB_BACKEND", "postgres")
|
||||
ensure_postgres_ready()
|
||||
print("db_init complete")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,884 @@
|
||||
"""Opt-in local development debug persist. Mapping labels and secrets stay out."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
import uuid
|
||||
|
||||
from db import get_db, row_to_dict
|
||||
from version import APP_VERSION, BUILD_DATE
|
||||
|
||||
SETTING_KEY = "debug.persist_traces"
|
||||
EXPORT_KIND = "kansho.debug_export"
|
||||
EXPORT_VERSION = 2
|
||||
UNASSIGNED_SPACE_TITLE = "Ohne Space"
|
||||
UNASSIGNED_DAY_LABEL = "Ohne Tag"
|
||||
FORBIDDEN_KEYS = frozenset(
|
||||
{
|
||||
"local_label",
|
||||
"demask_label",
|
||||
"mapping_table",
|
||||
"replacements",
|
||||
"restore_by_token",
|
||||
"password",
|
||||
"password_hash",
|
||||
"key",
|
||||
"api_key",
|
||||
"secret",
|
||||
"token",
|
||||
}
|
||||
)
|
||||
LAYER_FOR_PURPOSE = {
|
||||
"dialogue_turn": "dialogzug",
|
||||
"journal_generate": "journalentwurf",
|
||||
"local_source_artifact": "journalquellen",
|
||||
"journal_reconstruct": "journalrekonstruktion",
|
||||
"profile_review": "profilreview",
|
||||
}
|
||||
SANITIZE_MAX_DEPTH = 80
|
||||
LOGGER = logging.getLogger("kansho.debug")
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def persist_enabled() -> bool:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT value FROM app_settings WHERE key = ?", (SETTING_KEY,)).fetchone()
|
||||
)
|
||||
return (row or {}).get("value") == "1"
|
||||
|
||||
|
||||
def set_persist_enabled(enabled: bool) -> dict:
|
||||
value = "1" if enabled else "0"
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO app_settings (key, value, updated)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = datetime('now')
|
||||
""",
|
||||
(SETTING_KEY, value),
|
||||
)
|
||||
return settings_payload()
|
||||
|
||||
|
||||
def _is_admin(profile_id: str) -> bool:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT role FROM profiles WHERE id = ?", (profile_id,)).fetchone()
|
||||
)
|
||||
return (row or {}).get("role") == "admin"
|
||||
|
||||
|
||||
def should_persist(profile_id: str | None) -> bool:
|
||||
if not profile_id:
|
||||
return False
|
||||
return persist_enabled() and _is_admin(profile_id)
|
||||
|
||||
|
||||
def sanitize(value: Any, *, _stack: set[int] | None = None, _depth: int = 0) -> Any:
|
||||
if _depth > SANITIZE_MAX_DEPTH:
|
||||
return None
|
||||
if isinstance(value, (dict, list, tuple)):
|
||||
stack = _stack if _stack is not None else set()
|
||||
ident = id(value)
|
||||
if ident in stack:
|
||||
return None
|
||||
stack.add(ident)
|
||||
try:
|
||||
if isinstance(value, dict):
|
||||
cleaned = {}
|
||||
for key, item in value.items():
|
||||
if key in FORBIDDEN_KEYS:
|
||||
continue
|
||||
cleaned[key] = sanitize(item, _stack=stack, _depth=_depth + 1)
|
||||
return cleaned
|
||||
return [sanitize(item, _stack=stack, _depth=_depth + 1) for item in value]
|
||||
finally:
|
||||
stack.discard(ident)
|
||||
return value
|
||||
|
||||
|
||||
def settings_payload(profile_id: str | None = None) -> dict:
|
||||
enabled = persist_enabled()
|
||||
count = 0
|
||||
if profile_id:
|
||||
with get_db() as conn:
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM debug_runs WHERE profile_id = ?",
|
||||
(profile_id,),
|
||||
).fetchone()["n"]
|
||||
return {
|
||||
"persist_enabled": enabled,
|
||||
"run_count": count,
|
||||
"note": (
|
||||
"Entwicklungsoption. Speichert die Admin-Testspur lokal je Dialogschritt. "
|
||||
"Zuordnung Space / Tag / Gespräch. Mapping-Tabelle, Secrets und Klarnamen-Labels "
|
||||
"bleiben ausgeschlossen. Nur Aktionen des Admin-Profils. Default aus."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def resolve_placement(
|
||||
profile_id: str,
|
||||
*,
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
space_id: str | None = None,
|
||||
) -> dict:
|
||||
place = {
|
||||
"space_id": space_id or None,
|
||||
"journal_day_id": journal_day_id or None,
|
||||
"conversation_id": conversation_id or None,
|
||||
"space_title": "",
|
||||
"calendar_date": "",
|
||||
"conversation_title": "",
|
||||
}
|
||||
with get_db() as conn:
|
||||
if conversation_id:
|
||||
conv = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id, title, space_id, journal_day_id
|
||||
FROM conversations
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(conversation_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if conv:
|
||||
place["conversation_id"] = conv["id"]
|
||||
place["conversation_title"] = conv.get("title") or ""
|
||||
place["space_id"] = place["space_id"] or conv.get("space_id")
|
||||
place["journal_day_id"] = place["journal_day_id"] or conv.get("journal_day_id")
|
||||
if place["journal_day_id"]:
|
||||
day = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id, calendar_date, space_id
|
||||
FROM journal_days
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(place["journal_day_id"], profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if day:
|
||||
place["calendar_date"] = day.get("calendar_date") or ""
|
||||
place["space_id"] = place["space_id"] or day.get("space_id")
|
||||
if place["space_id"]:
|
||||
space = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id, title FROM spaces
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(place["space_id"], profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if space:
|
||||
place["space_title"] = space.get("title") or ""
|
||||
return place
|
||||
|
||||
|
||||
def persist_step(
|
||||
profile_id: str,
|
||||
*,
|
||||
purpose: str,
|
||||
status: str = "ok",
|
||||
layer: str = "",
|
||||
subject_type: str = "",
|
||||
subject_id: str = "",
|
||||
conversation_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
space_id: str | None = None,
|
||||
decision: dict | None = None,
|
||||
trace: dict | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> str | None:
|
||||
if not should_persist(profile_id):
|
||||
return None
|
||||
if not journal_day_id and subject_type == "journal_day" and subject_id:
|
||||
journal_day_id = subject_id
|
||||
place = resolve_placement(
|
||||
profile_id,
|
||||
conversation_id=conversation_id,
|
||||
journal_day_id=journal_day_id,
|
||||
space_id=space_id,
|
||||
)
|
||||
payload = sanitize(
|
||||
{
|
||||
"decision": decision,
|
||||
"trace": trace,
|
||||
**(extra or {}),
|
||||
}
|
||||
)
|
||||
run_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO debug_runs (
|
||||
id, profile_id, purpose, layer, status,
|
||||
subject_type, subject_id, conversation_id, message_id,
|
||||
space_id, journal_day_id, space_title, calendar_date, conversation_title,
|
||||
payload_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
run_id,
|
||||
profile_id,
|
||||
purpose,
|
||||
layer or LAYER_FOR_PURPOSE.get(purpose, ""),
|
||||
status,
|
||||
subject_type or "",
|
||||
subject_id or "",
|
||||
place["conversation_id"],
|
||||
message_id,
|
||||
place["space_id"],
|
||||
place["journal_day_id"],
|
||||
place["space_title"],
|
||||
place["calendar_date"],
|
||||
place["conversation_title"],
|
||||
json.dumps(payload, ensure_ascii=False, default=str),
|
||||
),
|
||||
)
|
||||
return run_id
|
||||
|
||||
|
||||
def persist_engine_error(
|
||||
profile_id: str,
|
||||
*,
|
||||
purpose: str,
|
||||
exc,
|
||||
layer: str = "",
|
||||
subject_type: str = "",
|
||||
subject_id: str = "",
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> str | None:
|
||||
try:
|
||||
diag = dict(getattr(exc, "diagnostics", None) or {})
|
||||
merged = dict(extra or {})
|
||||
merged.setdefault("code", getattr(exc, "code", None))
|
||||
merged.setdefault("message", getattr(exc, "message", None) or str(exc))
|
||||
log = diag.get("log")
|
||||
if log is not None:
|
||||
merged.setdefault("run_log", list(log) if isinstance(log, list) else log)
|
||||
trace = diag.get("trace") if isinstance(diag.get("trace"), dict) else {}
|
||||
try:
|
||||
from privacy_gateway import compact_diagnostics, public_cost_report
|
||||
|
||||
compact = compact_diagnostics({**diag, **trace, **merged})
|
||||
for key, value in compact.items():
|
||||
merged.setdefault(key, value)
|
||||
merged.setdefault("cost_report", public_cost_report({**diag, **merged}))
|
||||
except Exception:
|
||||
LOGGER.exception("Kompakte Fehlerdiagnose konnte nicht erzeugt werden.")
|
||||
for key in (
|
||||
"generation_selection",
|
||||
"style_application",
|
||||
"abort_reason",
|
||||
"detect_attempts",
|
||||
"detect_passes",
|
||||
"detect_partial_discarded",
|
||||
"contract_violation",
|
||||
"invalid_entity_type",
|
||||
"omitted_non_identity_types",
|
||||
"generate_called",
|
||||
):
|
||||
if merged.get(key) is None:
|
||||
if diag.get(key) is not None:
|
||||
merged[key] = diag.get(key)
|
||||
elif trace.get(key) is not None:
|
||||
merged[key] = trace.get(key)
|
||||
merged.setdefault("generate_called", False)
|
||||
return persist_step(
|
||||
profile_id,
|
||||
purpose=purpose,
|
||||
status="error",
|
||||
layer=layer,
|
||||
subject_type=subject_type,
|
||||
subject_id=subject_id,
|
||||
conversation_id=conversation_id,
|
||||
journal_day_id=journal_day_id,
|
||||
trace=trace or None,
|
||||
extra=merged,
|
||||
)
|
||||
except Exception:
|
||||
LOGGER.exception(
|
||||
"Diagnosepersistenz fehlgeschlagen; ursprünglicher Fehler bleibt %s",
|
||||
getattr(exc, "code", type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_payload(raw: str | None) -> dict:
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _row_public(row: dict, *, include_payload: bool = True) -> dict:
|
||||
payload = _parse_payload(row.get("payload_json"))
|
||||
item = {
|
||||
"id": row["id"],
|
||||
"created": row["created"],
|
||||
"purpose": row["purpose"],
|
||||
"layer": row["layer"],
|
||||
"status": row["status"],
|
||||
"subject_type": row.get("subject_type") or "",
|
||||
"subject_id": row.get("subject_id") or "",
|
||||
"conversation_id": row.get("conversation_id"),
|
||||
"message_id": row.get("message_id"),
|
||||
"space_id": row.get("space_id"),
|
||||
"journal_day_id": row.get("journal_day_id"),
|
||||
"space_title": row.get("space_title") or "",
|
||||
"calendar_date": row.get("calendar_date") or "",
|
||||
"conversation_title": row.get("conversation_title") or "",
|
||||
"decision": payload.get("decision"),
|
||||
"summary": _summary(row, payload),
|
||||
}
|
||||
if include_payload:
|
||||
item["payload"] = payload
|
||||
item["trace"] = payload.get("trace")
|
||||
item["run_log"] = payload.get("run_log") or ((payload.get("trace") or {}).get("log"))
|
||||
return item
|
||||
|
||||
|
||||
def _summary(row: dict, payload: dict) -> str:
|
||||
decision = payload.get("decision") or {}
|
||||
label = decision.get("label") or decision.get("operation") or ""
|
||||
purpose = row.get("purpose") or ""
|
||||
status = row.get("status") or ""
|
||||
parts = [purpose]
|
||||
if label:
|
||||
parts.append(str(label))
|
||||
if status and status != "ok":
|
||||
parts.append(status)
|
||||
code = payload.get("code")
|
||||
if code:
|
||||
parts.append(str(code))
|
||||
return " · ".join(parts)
|
||||
|
||||
|
||||
def list_runs(profile_id: str, *, limit: int = 200) -> list[dict]:
|
||||
capped = max(1, min(int(limit or 200), 500))
|
||||
with get_db() as conn:
|
||||
rows = [
|
||||
dict(item)
|
||||
for item in conn.execute(
|
||||
"""
|
||||
SELECT * FROM debug_runs
|
||||
WHERE profile_id = ?
|
||||
ORDER BY created DESC, id DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(profile_id, capped),
|
||||
).fetchall()
|
||||
]
|
||||
return [_row_public(row, include_payload=False) for row in rows]
|
||||
|
||||
|
||||
def latest_run(
|
||||
profile_id: str,
|
||||
*,
|
||||
journal_day_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
) -> dict | None:
|
||||
clauses = ["profile_id = ?"]
|
||||
args: list[Any] = [profile_id]
|
||||
if journal_day_id:
|
||||
clauses.append("journal_day_id = ?")
|
||||
args.append(journal_day_id)
|
||||
if purpose:
|
||||
clauses.append("purpose = ?")
|
||||
args.append(purpose)
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
f"""
|
||||
SELECT * FROM debug_runs
|
||||
WHERE {' AND '.join(clauses)}
|
||||
ORDER BY created DESC, id DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
args,
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
items = _fill_placement(profile_id, [_row_public(row, include_payload=True)])
|
||||
return items[0] if items else None
|
||||
|
||||
|
||||
def get_run(profile_id: str, run_id: str) -> dict | None:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT * FROM debug_runs WHERE id = ? AND profile_id = ?",
|
||||
(run_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
items = _fill_placement(profile_id, [_row_public(row, include_payload=True)])
|
||||
return items[0]
|
||||
|
||||
|
||||
def delete_run(profile_id: str, run_id: str) -> bool:
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
"DELETE FROM debug_runs WHERE id = ? AND profile_id = ?",
|
||||
(run_id, profile_id),
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
def clear_runs(profile_id: str) -> int:
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute("DELETE FROM debug_runs WHERE profile_id = ?", (profile_id,))
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def list_runs_full(profile_id: str) -> list[dict]:
|
||||
with get_db() as conn:
|
||||
rows = [
|
||||
dict(item)
|
||||
for item in conn.execute(
|
||||
"""
|
||||
SELECT * FROM debug_runs
|
||||
WHERE profile_id = ?
|
||||
ORDER BY created ASC, id ASC
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
]
|
||||
items = [_row_public(row, include_payload=True) for row in rows]
|
||||
return _fill_placement(profile_id, items)
|
||||
|
||||
|
||||
def _fill_placement(profile_id: str, items: list[dict]) -> list[dict]:
|
||||
missing_conv = [
|
||||
item.get("conversation_id")
|
||||
for item in items
|
||||
if item.get("conversation_id") and not item.get("space_id") and not item.get("journal_day_id")
|
||||
]
|
||||
missing_days = [
|
||||
item.get("journal_day_id")
|
||||
for item in items
|
||||
if item.get("journal_day_id") and not item.get("space_id")
|
||||
]
|
||||
conv_ids = list(dict.fromkeys(missing_conv))
|
||||
day_ids = list(dict.fromkeys(missing_days))
|
||||
convs: dict[str, dict] = {}
|
||||
days: dict[str, dict] = {}
|
||||
spaces: dict[str, dict] = {}
|
||||
with get_db() as conn:
|
||||
if conv_ids:
|
||||
marks = ",".join("?" * len(conv_ids))
|
||||
for row in conn.execute(
|
||||
f"SELECT id, title, space_id, journal_day_id FROM conversations WHERE profile_id = ? AND id IN ({marks})",
|
||||
[profile_id, *conv_ids],
|
||||
):
|
||||
convs[row["id"]] = dict(row)
|
||||
if row["journal_day_id"]:
|
||||
day_ids.append(row["journal_day_id"])
|
||||
day_ids = list(dict.fromkeys(day_ids))
|
||||
if day_ids:
|
||||
marks = ",".join("?" * len(day_ids))
|
||||
for row in conn.execute(
|
||||
f"SELECT id, calendar_date, space_id FROM journal_days WHERE profile_id = ? AND id IN ({marks})",
|
||||
[profile_id, *day_ids],
|
||||
):
|
||||
days[row["id"]] = dict(row)
|
||||
space_ids = [
|
||||
item
|
||||
for item in (
|
||||
[row.get("space_id") for row in convs.values()]
|
||||
+ [row.get("space_id") for row in days.values()]
|
||||
+ [item.get("space_id") for item in items]
|
||||
)
|
||||
if item
|
||||
]
|
||||
space_ids = list(dict.fromkeys(space_ids))
|
||||
if space_ids:
|
||||
marks = ",".join("?" * len(space_ids))
|
||||
for row in conn.execute(
|
||||
f"SELECT id, title FROM spaces WHERE profile_id = ? AND id IN ({marks})",
|
||||
[profile_id, *space_ids],
|
||||
):
|
||||
spaces[row["id"]] = dict(row)
|
||||
for item in items:
|
||||
conv = convs.get(item.get("conversation_id") or "")
|
||||
if conv:
|
||||
item["conversation_title"] = item.get("conversation_title") or conv.get("title") or ""
|
||||
item["space_id"] = item.get("space_id") or conv.get("space_id")
|
||||
item["journal_day_id"] = item.get("journal_day_id") or conv.get("journal_day_id")
|
||||
day = days.get(item.get("journal_day_id") or "")
|
||||
if day:
|
||||
item["calendar_date"] = item.get("calendar_date") or day.get("calendar_date") or ""
|
||||
item["space_id"] = item.get("space_id") or day.get("space_id")
|
||||
space = spaces.get(item.get("space_id") or "")
|
||||
if space:
|
||||
item["space_title"] = item.get("space_title") or space.get("title") or ""
|
||||
return items
|
||||
|
||||
|
||||
def _export_run(item: dict) -> dict:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"created": item["created"],
|
||||
"purpose": item["purpose"],
|
||||
"layer": item["layer"],
|
||||
"status": item["status"],
|
||||
"subject_type": item["subject_type"],
|
||||
"subject_id": item["subject_id"],
|
||||
"conversation_id": item.get("conversation_id"),
|
||||
"message_id": item.get("message_id"),
|
||||
"space_id": item.get("space_id"),
|
||||
"journal_day_id": item.get("journal_day_id"),
|
||||
"space_title": item.get("space_title") or "",
|
||||
"calendar_date": item.get("calendar_date") or "",
|
||||
"conversation_title": item.get("conversation_title") or "",
|
||||
"summary": item["summary"],
|
||||
**(item.get("payload") or {}),
|
||||
}
|
||||
|
||||
|
||||
def _step_summary(item: dict) -> dict:
|
||||
return {
|
||||
"id": item["id"],
|
||||
"created": item["created"],
|
||||
"purpose": item["purpose"],
|
||||
"layer": item["layer"],
|
||||
"status": item["status"],
|
||||
"summary": item["summary"],
|
||||
"message_id": item.get("message_id"),
|
||||
}
|
||||
|
||||
|
||||
def _filter_runs(
|
||||
runs: list[dict],
|
||||
*,
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
space_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
) -> list[dict]:
|
||||
selected = runs
|
||||
if conversation_id:
|
||||
selected = [
|
||||
item
|
||||
for item in selected
|
||||
if item.get("conversation_id") == conversation_id and item.get("purpose") != "journal_generate"
|
||||
]
|
||||
elif journal_day_id:
|
||||
selected = [item for item in selected if item.get("journal_day_id") == journal_day_id]
|
||||
if space_id:
|
||||
selected = [item for item in selected if item.get("space_id") == space_id]
|
||||
if purpose:
|
||||
selected = [item for item in selected if item.get("purpose") == purpose]
|
||||
return selected
|
||||
|
||||
|
||||
def group_tree(runs: list[dict], *, include_payload: bool = False) -> list[dict]:
|
||||
spaces_map: dict[str, dict] = {}
|
||||
space_order: list[str] = []
|
||||
for run in runs:
|
||||
exported = _export_run(run) if include_payload else _step_summary(run)
|
||||
sid = run.get("space_id") or ""
|
||||
space_key = sid or "_unassigned"
|
||||
if space_key not in spaces_map:
|
||||
spaces_map[space_key] = {
|
||||
"id": sid or None,
|
||||
"title": run.get("space_title") or (UNASSIGNED_SPACE_TITLE if not sid else ""),
|
||||
"days_map": {},
|
||||
"day_order": [],
|
||||
}
|
||||
space_order.append(space_key)
|
||||
space = spaces_map[space_key]
|
||||
if run.get("space_title") and not space["title"]:
|
||||
space["title"] = run.get("space_title")
|
||||
did = run.get("journal_day_id") or ""
|
||||
day_key = did or "_unassigned"
|
||||
if day_key not in space["days_map"]:
|
||||
space["days_map"][day_key] = {
|
||||
"id": did or None,
|
||||
"calendar_date": run.get("calendar_date") or (UNASSIGNED_DAY_LABEL if not did else ""),
|
||||
"conversations_map": {},
|
||||
"conversation_order": [],
|
||||
"generates": [],
|
||||
}
|
||||
space["day_order"].append(day_key)
|
||||
day = space["days_map"][day_key]
|
||||
if run.get("calendar_date") and (not day["calendar_date"] or day["calendar_date"] == UNASSIGNED_DAY_LABEL):
|
||||
day["calendar_date"] = run.get("calendar_date")
|
||||
if run.get("purpose") == "journal_generate":
|
||||
day["generates"].append(exported)
|
||||
continue
|
||||
cid = run.get("conversation_id") or ""
|
||||
conv_key = cid or "_none"
|
||||
if conv_key not in day["conversations_map"]:
|
||||
day["conversations_map"][conv_key] = {
|
||||
"id": cid or None,
|
||||
"title": run.get("conversation_title") or "Gespräch",
|
||||
"steps": [],
|
||||
}
|
||||
day["conversation_order"].append(conv_key)
|
||||
conv = day["conversations_map"][conv_key]
|
||||
if run.get("conversation_title"):
|
||||
conv["title"] = run.get("conversation_title")
|
||||
conv["steps"].append(exported)
|
||||
spaces = []
|
||||
for space_key in space_order:
|
||||
space = spaces_map[space_key]
|
||||
days = []
|
||||
for day_key in space["day_order"]:
|
||||
day = space["days_map"][day_key]
|
||||
conversations = []
|
||||
for conv_key in day["conversation_order"]:
|
||||
conv = day["conversations_map"][conv_key]
|
||||
conversations.append(
|
||||
{
|
||||
"id": conv["id"],
|
||||
"title": conv["title"] or "Gespräch",
|
||||
"step_count": len(conv["steps"]),
|
||||
"steps": conv["steps"],
|
||||
}
|
||||
)
|
||||
days.append(
|
||||
{
|
||||
"id": day["id"],
|
||||
"calendar_date": day["calendar_date"] or UNASSIGNED_DAY_LABEL,
|
||||
"conversations": conversations,
|
||||
"generates": day["generates"],
|
||||
}
|
||||
)
|
||||
spaces.append({"id": space["id"], "title": space["title"] or UNASSIGNED_SPACE_TITLE, "days": days})
|
||||
return spaces
|
||||
|
||||
|
||||
def debug_tree(profile_id: str) -> dict:
|
||||
runs = list_runs_full(profile_id)
|
||||
return {"spaces": group_tree(runs, include_payload=False), "run_count": len(runs)}
|
||||
|
||||
|
||||
def export_document(
|
||||
profile_id: str,
|
||||
*,
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
) -> dict:
|
||||
runs = list_runs_full(profile_id)
|
||||
selected = _filter_runs(
|
||||
runs,
|
||||
conversation_id=conversation_id,
|
||||
journal_day_id=journal_day_id,
|
||||
purpose=purpose,
|
||||
)
|
||||
scope = "all"
|
||||
if purpose == "journal_generate":
|
||||
scope = "journal_generate"
|
||||
elif conversation_id:
|
||||
scope = "conversation"
|
||||
elif journal_day_id:
|
||||
scope = "day"
|
||||
spaces = group_tree(selected, include_payload=True)
|
||||
first_space = spaces[0] if spaces else None
|
||||
first_day = ((first_space or {}).get("days") or [None])[0] if first_space else None
|
||||
first_conv = ((first_day or {}).get("conversations") or [None])[0] if first_day else None
|
||||
return {
|
||||
"kind": EXPORT_KIND,
|
||||
"version": EXPORT_VERSION,
|
||||
"exported_at": _now(),
|
||||
"app_version": APP_VERSION,
|
||||
"build_date": BUILD_DATE,
|
||||
"scope": scope,
|
||||
"note": (
|
||||
"Lokale Entwicklungsdiagnose, gegliedert nach Space / Tag / Gespräch / Schritt. "
|
||||
"Mapping-Tabelle und Secrets sind ausgeschlossen. "
|
||||
"Enthält Klartext-Vorlagen (intern) und maskierten Egress des eigenen Admin-Profils. "
|
||||
"Nicht an Provider senden, außer bewusst zur lokalen Auswertung."
|
||||
),
|
||||
"persist_enabled": persist_enabled(),
|
||||
"run_count": len(selected),
|
||||
"space": {"id": (first_space or {}).get("id"), "title": (first_space or {}).get("title")} if scope != "all" else None,
|
||||
"day": (
|
||||
{"id": (first_day or {}).get("id"), "calendar_date": (first_day or {}).get("calendar_date")}
|
||||
if scope in {"conversation", "day", "journal_generate"}
|
||||
else None
|
||||
),
|
||||
"conversation": (
|
||||
{"id": (first_conv or {}).get("id"), "title": (first_conv or {}).get("title")}
|
||||
if scope == "conversation"
|
||||
else None
|
||||
),
|
||||
"spaces": spaces,
|
||||
"runs": [_export_run(item) for item in selected],
|
||||
}
|
||||
|
||||
|
||||
def _fence(text: str | None) -> str:
|
||||
body = (text or "").replace("```", "'''")
|
||||
return f"```\n{body}\n```" if body.strip() else "_leer_"
|
||||
|
||||
|
||||
def _kv(title: str, value: Any) -> str:
|
||||
if value is None or value == "":
|
||||
return f"- {title}: nicht verfügbar"
|
||||
if isinstance(value, (dict, list)):
|
||||
return f"- {title}: `{json.dumps(value, ensure_ascii=False)}`"
|
||||
return f"- {title}: {value}"
|
||||
|
||||
|
||||
def _render_trace_markdown(trace: dict | None, heading: str) -> list[str]:
|
||||
if not trace:
|
||||
return [f"### {heading}", "", "_keine Testspur_", ""]
|
||||
lines = [f"### {heading}", ""]
|
||||
lines.append(_kv("Zweck", trace.get("purpose")))
|
||||
lines.append(_kv("Schicht", trace.get("layer")))
|
||||
lines.append(_kv("Prompt", trace.get("prompt_slug")))
|
||||
lines.append(_kv("Prompt-Revision", trace.get("prompt_revision")))
|
||||
lines.append(_kv("Provider", trace.get("provider")))
|
||||
lines.append(_kv("Modell", trace.get("model")))
|
||||
lines.append(_kv("Detect", trace.get("detect_note") or trace.get("detect_provider")))
|
||||
lines.append(_kv("Detect-Modell", trace.get("detect_model")))
|
||||
lines.append(_kv("Abbruchgrund", trace.get("abort_reason")))
|
||||
lines.append(_kv("Modelltext übernommen", trace.get("model_text_accepted")))
|
||||
budget = trace.get("budget") or {}
|
||||
if budget:
|
||||
lines.append(_kv("Prompt-Tokens", budget.get("prompt_tokens")))
|
||||
lines.append(_kv("Completion-Tokens", budget.get("completion_tokens")))
|
||||
lines.append(_kv("Kosten", budget.get("cost")))
|
||||
lines.append(_kv("Generate-ms", budget.get("generate_ms") or trace.get("generate_ms")))
|
||||
lines.extend(["", "#### Intern (Klartext-Vorlage, nicht gesendet)", "", _fence(trace.get("intern")), ""])
|
||||
lines.extend(["#### Egress (maskiert, gesendet)", "", _fence(trace.get("egress")), ""])
|
||||
lines.extend(["#### Maskierung, Eingabe", "", _fence(trace.get("mask_input")), ""])
|
||||
lines.extend(["#### Modellantwort roh", "", _fence(trace.get("raw")), ""])
|
||||
lines.extend(["#### Lokal demaskiert", "", _fence(trace.get("reply")), ""])
|
||||
stored = "\n\n".join(part for part in (trace.get("stored_title"), trace.get("stored_body")) if part)
|
||||
if stored:
|
||||
lines.extend(["#### Gespeicherter Entwurf", "", _fence(stored), ""])
|
||||
if trace.get("source_preview"):
|
||||
lines.extend(["#### Lokale Quellenansicht", "", _fence(trace.get("source_preview")), ""])
|
||||
return lines
|
||||
|
||||
|
||||
def render_markdown(document: dict) -> str:
|
||||
lines = [
|
||||
"# Kanshō Debug-Export",
|
||||
"",
|
||||
_kv("kind", document.get("kind")),
|
||||
_kv("version", document.get("version")),
|
||||
_kv("scope", document.get("scope")),
|
||||
_kv("exported_at", document.get("exported_at")),
|
||||
_kv("app_version", document.get("app_version")),
|
||||
_kv("build_date", document.get("build_date")),
|
||||
_kv("run_count", document.get("run_count")),
|
||||
_kv("persist_enabled", document.get("persist_enabled")),
|
||||
"",
|
||||
document.get("note") or "",
|
||||
"",
|
||||
"Gliederung: Space / Tag / Gespräch / Schritt. Die Schritte eines Gesprächs sind chronologisch.",
|
||||
"",
|
||||
]
|
||||
if (document.get("space") or {}).get("title"):
|
||||
lines.append(_kv("Space", (document.get("space") or {}).get("title")))
|
||||
if (document.get("day") or {}).get("calendar_date"):
|
||||
lines.append(_kv("Tag", (document.get("day") or {}).get("calendar_date")))
|
||||
if (document.get("conversation") or {}).get("title"):
|
||||
lines.append(_kv("Gespräch", (document.get("conversation") or {}).get("title")))
|
||||
lines.append("")
|
||||
spaces = document.get("spaces") or []
|
||||
if spaces:
|
||||
for space in spaces:
|
||||
lines.extend(["---", "", f"# Space: {space.get('title') or UNASSIGNED_SPACE_TITLE}", ""])
|
||||
for day in space.get("days") or []:
|
||||
lines.extend([f"## Tag: {day.get('calendar_date') or UNASSIGNED_DAY_LABEL}", ""])
|
||||
for conv in day.get("conversations") or []:
|
||||
lines.extend([f"### Gespräch: {conv.get('title') or 'Gespräch'}", ""])
|
||||
for index, run in enumerate(conv.get("steps") or [], start=1):
|
||||
lines.extend(_render_run_markdown(run, index))
|
||||
for index, run in enumerate(day.get("generates") or [], start=1):
|
||||
lines.extend([f"### Journalentwurf {index}", ""])
|
||||
lines.extend(_render_run_markdown(run, index))
|
||||
else:
|
||||
for index, run in enumerate(document.get("runs") or [], start=1):
|
||||
lines.extend(_render_run_markdown(run, index))
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _render_run_markdown(run: dict, index: int) -> list[str]:
|
||||
created = run.get("created") or ""
|
||||
purpose = run.get("purpose") or ""
|
||||
status = run.get("status") or ""
|
||||
lines = [
|
||||
"---",
|
||||
"",
|
||||
f"#### {index}. {purpose} · {created} · {status}",
|
||||
"",
|
||||
_kv("id", run.get("id")),
|
||||
_kv("layer", run.get("layer")),
|
||||
_kv("conversation_id", run.get("conversation_id")),
|
||||
_kv("code", run.get("code")),
|
||||
]
|
||||
decision = run.get("decision")
|
||||
if decision:
|
||||
lines.append(_kv("Operation", decision.get("label") or decision.get("operation")))
|
||||
if decision.get("guard"):
|
||||
lines.append(_kv("Guard", decision.get("guard")))
|
||||
user = run.get("user") or {}
|
||||
assistant = run.get("assistant") or {}
|
||||
if user.get("body"):
|
||||
lines.extend(["", "##### Nutzer", "", _fence(user.get("body")), ""])
|
||||
if assistant.get("body"):
|
||||
lines.extend(["##### Antwort", "", _fence(assistant.get("body")), ""])
|
||||
traces = []
|
||||
if run.get("call_traces"):
|
||||
traces = list(run.get("call_traces") or [])
|
||||
elif run.get("trace"):
|
||||
nested = (run.get("trace") or {}).get("stages")
|
||||
traces = list(nested) if nested else [run.get("trace")]
|
||||
if not traces and run.get("trace"):
|
||||
traces = [run.get("trace")]
|
||||
for offset, stage in enumerate(traces):
|
||||
title = f"Schritt {offset + 1}"
|
||||
if isinstance(stage, dict) and stage.get("purpose"):
|
||||
title = f"Schritt {offset + 1}: {stage.get('purpose')}"
|
||||
lines.extend(_render_trace_markdown(stage if isinstance(stage, dict) else None, title))
|
||||
log = run.get("run_log") or ((run.get("trace") or {}).get("log") if isinstance(run.get("trace"), dict) else None)
|
||||
if log:
|
||||
lines.extend(
|
||||
[
|
||||
"##### Kompaktes Laufprotokoll (ohne Promptkörper)",
|
||||
"",
|
||||
_fence(json.dumps(log, ensure_ascii=False, indent=2)),
|
||||
"",
|
||||
]
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def export_filename(fmt: str, document: dict | None = None) -> str:
|
||||
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
||||
suffix = "md" if fmt == "markdown" else "json"
|
||||
scope = (document or {}).get("scope") or "all"
|
||||
if scope == "conversation":
|
||||
return f"kansho-debug-conversation-{stamp}.{suffix}"
|
||||
if scope == "journal_generate":
|
||||
return f"kansho-debug-journal-{stamp}.{suffix}"
|
||||
if scope == "day":
|
||||
return f"kansho-debug-day-{stamp}.{suffix}"
|
||||
return f"kansho-debug-{stamp}.{suffix}"
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,620 @@
|
||||
"""Student-first detect pipeline with optional teacher comparison.
|
||||
|
||||
Cues stay local. OpenRouter/GLiNER never create identities. Compact counters
|
||||
carry no plaintext labels.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from detect_learning import (
|
||||
MODE_LEARNING_MANUAL,
|
||||
MODE_STUDENT,
|
||||
_CUE_BARE,
|
||||
_DECISION_ASK,
|
||||
_DECISION_IDENTITY,
|
||||
_DECISION_NOT,
|
||||
_align_user_span,
|
||||
_mention_fields,
|
||||
_resolve_user_span,
|
||||
_token_for_label,
|
||||
_word_matches,
|
||||
apply_learned_decisions,
|
||||
auto_resolve_ambiguous,
|
||||
cue_for_mention,
|
||||
decision_for_mention,
|
||||
get_cue_decision,
|
||||
get_detect_operating_mode,
|
||||
get_local_ner_settings,
|
||||
get_sense,
|
||||
is_bypass_mode,
|
||||
is_learning_mode,
|
||||
learning_stage,
|
||||
merge_gliner_candidates,
|
||||
record_cue,
|
||||
supplement_kinship_candidates,
|
||||
)
|
||||
from entity_detect import DetectError, detect_personal_egress
|
||||
from identity_store import ENTITY_TYPES, is_maskable_label
|
||||
|
||||
KIND_DIFF = "diff"
|
||||
KIND_MANUAL = "manual"
|
||||
KIND_CONFIRM_REMOTE = "confirm_remote"
|
||||
|
||||
PROTECTING_TYPES = set(ENTITY_TYPES)
|
||||
FOOD_TYPES = frozenset(
|
||||
{"FOOD", "DISH", "MEAL", "CUISINE", "FOODSTUFF", "INGREDIENT", "food", "dish"}
|
||||
)
|
||||
TYPE_MAP = {
|
||||
"person": "PERSON",
|
||||
"place": "PLACE",
|
||||
"org": "ORG",
|
||||
"organization": "ORG",
|
||||
"project": "PROJECT",
|
||||
"food": "FOOD",
|
||||
"dish": "DISH",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MentionSpan:
|
||||
text: str
|
||||
start: int
|
||||
end: int
|
||||
entity_type: str
|
||||
protection: str
|
||||
source: str
|
||||
confirmed_cue: bool = False
|
||||
cue: str = _CUE_BARE
|
||||
|
||||
def key(self) -> tuple[int, int, str]:
|
||||
return (self.start, self.end, self.text.casefold())
|
||||
|
||||
|
||||
@dataclass
|
||||
class StudentResult:
|
||||
mappings: list[dict] = field(default_factory=list)
|
||||
candidates: list[dict] = field(default_factory=list)
|
||||
spans: list[MentionSpan] = field(default_factory=list)
|
||||
gliner_missing: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class TeacherResult:
|
||||
spans: list[MentionSpan] = field(default_factory=list)
|
||||
mappings: list[dict] = field(default_factory=list)
|
||||
ok: bool = True
|
||||
error: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineResult:
|
||||
mappings: list[dict]
|
||||
candidates: list[dict]
|
||||
kind: str = KIND_DIFF
|
||||
need_review: bool = False
|
||||
need_remote_confirm: bool = False
|
||||
counters: dict[str, int] = field(default_factory=dict)
|
||||
mode: str = ""
|
||||
|
||||
|
||||
def empty_counters() -> dict[str, int]:
|
||||
return {
|
||||
"diff_agree": 0,
|
||||
"diff_only_student": 0,
|
||||
"diff_only_teacher": 0,
|
||||
"diff_type_conflict": 0,
|
||||
"diff_new_identity": 0,
|
||||
"diff_cue_reinforced": 0,
|
||||
}
|
||||
|
||||
|
||||
def map_entity_type(raw: str) -> str:
|
||||
token = (raw or "").strip()
|
||||
if token.upper() in PROTECTING_TYPES or token.upper() in {item.upper() for item in FOOD_TYPES}:
|
||||
mapped = TYPE_MAP.get(token.lower())
|
||||
return mapped or token.upper()
|
||||
return TYPE_MAP.get(token.lower()) or token.upper()
|
||||
|
||||
|
||||
def protects(entity_type: str) -> bool:
|
||||
kind = map_entity_type(entity_type)
|
||||
if kind in FOOD_TYPES or kind.upper() in {item.upper() for item in FOOD_TYPES}:
|
||||
return False
|
||||
return kind in PROTECTING_TYPES
|
||||
|
||||
|
||||
def types_equivalent(left: str, right: str) -> bool:
|
||||
a, b = map_entity_type(left), map_entity_type(right)
|
||||
if a in FOOD_TYPES or b in FOOD_TYPES or a in {"FOOD", "DISH"} or b in {"FOOD", "DISH"}:
|
||||
return a == b
|
||||
if protects(a) and protects(b):
|
||||
return True
|
||||
return a == b
|
||||
|
||||
|
||||
def _contains(outer: MentionSpan, inner: MentionSpan) -> bool:
|
||||
return outer.start <= inner.start and inner.end <= outer.end and outer.end > outer.start
|
||||
|
||||
|
||||
def same_mention(left: MentionSpan, right: MentionSpan) -> bool:
|
||||
if left.text.casefold() == right.text.casefold() and left.start == right.start and left.end == right.end:
|
||||
return True
|
||||
if _contains(left, right) or _contains(right, left):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _collapse_same_protection(spans: list[MentionSpan]) -> list[MentionSpan]:
|
||||
if not spans:
|
||||
return []
|
||||
remaining = list(spans)
|
||||
chosen: list[MentionSpan] = []
|
||||
while remaining:
|
||||
current = remaining.pop(0)
|
||||
cluster = [current]
|
||||
rest = []
|
||||
for item in remaining:
|
||||
if same_mention(current, item):
|
||||
cluster.append(item)
|
||||
else:
|
||||
rest.append(item)
|
||||
remaining = rest
|
||||
protections = {item.protection for item in cluster}
|
||||
if len(protections) > 1:
|
||||
chosen.extend(cluster)
|
||||
continue
|
||||
chosen.append(max(cluster, key=lambda item: (item.end - item.start, -item.start)))
|
||||
return chosen
|
||||
|
||||
|
||||
def _user_span_for_mapping(mapping: dict, user_body: str, rendered: str = "") -> tuple[int, int] | None:
|
||||
return _resolve_user_span(
|
||||
user_body,
|
||||
mapping.get("start"),
|
||||
mapping.get("end"),
|
||||
mapping.get("local_label") or "",
|
||||
rendered,
|
||||
)
|
||||
|
||||
|
||||
def _mapping_from_user_span(
|
||||
mappings: list[dict],
|
||||
label: str,
|
||||
user_body: str,
|
||||
start: int,
|
||||
end: int,
|
||||
entity_type: str,
|
||||
rendered: str,
|
||||
) -> dict:
|
||||
aligned = _align_user_span(rendered, user_body, start, end)
|
||||
kind = entity_type if entity_type in ENTITY_TYPES else "PERSON"
|
||||
return {
|
||||
"token": _token_for_label(mappings, label, kind),
|
||||
"local_label": label,
|
||||
"canonical_label": label,
|
||||
"demask_label": label,
|
||||
"entity_type": kind,
|
||||
"source": "request_local",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
"aliases": [],
|
||||
"labels": [label],
|
||||
}
|
||||
|
||||
|
||||
def run_student_path(profile_id: str, user_body: str, rendered: str = "") -> StudentResult:
|
||||
mappings: list[dict] = []
|
||||
candidates: list[dict] = []
|
||||
spans: list[MentionSpan] = []
|
||||
text = user_body or ""
|
||||
for match in _word_matches(text):
|
||||
label = match.group(0)
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
start, end = match.start(), match.end()
|
||||
verdict = decision_for_mention(profile_id, label, text, start, end, rendered)
|
||||
cue = cue_for_mention(text, start, end, label, rendered)
|
||||
if verdict == _DECISION_IDENTITY:
|
||||
mappings.append(_mapping_from_user_span(mappings, label, text, start, end, "PERSON", rendered))
|
||||
spans.append(
|
||||
MentionSpan(label, start, end, "PERSON", _DECISION_IDENTITY, "cue", True, cue)
|
||||
)
|
||||
elif verdict == _DECISION_NOT:
|
||||
spans.append(
|
||||
MentionSpan(label, start, end, "FOOD", _DECISION_NOT, "cue", True, cue)
|
||||
)
|
||||
elif get_sense(profile_id, label)["ambiguous"]:
|
||||
aligned = _align_user_span(rendered, text, start, end)
|
||||
candidates.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": label,
|
||||
"entity_type": "PERSON",
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(text, start, end, label, rendered),
|
||||
"ambiguous": True,
|
||||
}
|
||||
)
|
||||
candidates, mappings = supplement_kinship_candidates(
|
||||
profile_id, candidates, mappings, text, rendered
|
||||
)
|
||||
candidates, mappings = auto_resolve_ambiguous(profile_id, candidates, mappings)
|
||||
bound = {(span.start, span.end, span.text.casefold()) for span in spans}
|
||||
for item in candidates:
|
||||
resolved = _resolve_user_span(
|
||||
text,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if resolved:
|
||||
bound.add((resolved[0], resolved[1], (item.get("text") or "").casefold()))
|
||||
settings = get_local_ner_settings()
|
||||
gliner_missing = False
|
||||
if settings.get("enabled"):
|
||||
from gliner_client import extract_entities
|
||||
|
||||
raw = extract_entities(text, url=settings.get("url"))
|
||||
if raw is None:
|
||||
gliner_missing = True
|
||||
else:
|
||||
usable = []
|
||||
for item in raw:
|
||||
try:
|
||||
start, end = int(item["start"]), int(item["end"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
label = (item.get("text") or "").casefold()
|
||||
if any(
|
||||
not (end <= left or start >= right) and folded == label
|
||||
for left, right, folded in bound
|
||||
) or any(left <= start and end <= right for left, right, _folded in bound):
|
||||
continue
|
||||
usable.append(item)
|
||||
candidates, mappings = merge_gliner_candidates(
|
||||
profile_id, candidates, mappings, text, rendered, usable
|
||||
)
|
||||
candidates, mappings = apply_learned_decisions(profile_id, candidates, mappings, text, rendered)
|
||||
for item in candidates:
|
||||
resolved = _resolve_user_span(
|
||||
text,
|
||||
item.get("user_start", item.get("start")),
|
||||
item.get("user_end", item.get("end")),
|
||||
item.get("text") or "",
|
||||
rendered,
|
||||
)
|
||||
if not resolved:
|
||||
continue
|
||||
suggested = item.get("suggested") or _DECISION_IDENTITY
|
||||
kind = item.get("entity_type") or "PERSON"
|
||||
protection = _DECISION_NOT if suggested == _DECISION_NOT else _DECISION_IDENTITY
|
||||
entity = "FOOD" if protection == _DECISION_NOT else kind
|
||||
spans.append(
|
||||
MentionSpan(
|
||||
item.get("text") or "",
|
||||
resolved[0],
|
||||
resolved[1],
|
||||
entity,
|
||||
protection,
|
||||
"candidate",
|
||||
False,
|
||||
item.get("cue") or _CUE_BARE,
|
||||
)
|
||||
)
|
||||
return StudentResult(
|
||||
mappings=mappings,
|
||||
candidates=candidates,
|
||||
spans=_collapse_same_protection(spans),
|
||||
gliner_missing=gliner_missing,
|
||||
)
|
||||
|
||||
|
||||
def blank_cue_safe_forms(profile_id: str, user_body: str, rendered: str = "") -> str:
|
||||
chars = list(user_body or "")
|
||||
for match in _word_matches(user_body or ""):
|
||||
label = match.group(0)
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
verdict = decision_for_mention(
|
||||
profile_id, label, user_body, match.start(), match.end(), rendered
|
||||
)
|
||||
if verdict not in {_DECISION_IDENTITY, _DECISION_NOT}:
|
||||
continue
|
||||
for index in range(match.start(), match.end()):
|
||||
chars[index] = " "
|
||||
return "".join(chars)
|
||||
|
||||
|
||||
def _teacher_span(label: str, start: int, end: int, entity_type: str) -> MentionSpan:
|
||||
kind = map_entity_type(entity_type)
|
||||
protection = _DECISION_IDENTITY if protects(kind) else _DECISION_NOT
|
||||
return MentionSpan(label, start, end, kind, protection, "teacher", False)
|
||||
|
||||
|
||||
def run_teacher(
|
||||
profile_id: str,
|
||||
user_body: str,
|
||||
*,
|
||||
stage: int = 1,
|
||||
rendered: str = "",
|
||||
) -> TeacherResult:
|
||||
source = user_body or ""
|
||||
if stage >= 2:
|
||||
source = blank_cue_safe_forms(profile_id, user_body, rendered)
|
||||
try:
|
||||
outcome = detect_personal_egress(profile_id, source)
|
||||
except DetectError as exc:
|
||||
return TeacherResult(ok=False, error=exc.code)
|
||||
spans: list[MentionSpan] = []
|
||||
for mapping in outcome.mappings or []:
|
||||
label = mapping.get("local_label") or ""
|
||||
start, end = mapping.get("start"), mapping.get("end")
|
||||
resolved = None
|
||||
if start is not None and end is not None:
|
||||
left, right = int(start), int(end)
|
||||
if 0 <= left < right <= len(user_body) and user_body[left:right].casefold() == label.casefold():
|
||||
resolved = (left, right)
|
||||
if not resolved:
|
||||
resolved = _resolve_user_span(user_body, start, end, label, user_body)
|
||||
if not resolved:
|
||||
continue
|
||||
spans.append(
|
||||
_teacher_span(label, resolved[0], resolved[1], mapping.get("entity_type") or "PERSON")
|
||||
)
|
||||
omitted = list((outcome.stats.omitted_non_identity_types or []) if outcome.stats else [])
|
||||
for kind in omitted:
|
||||
if map_entity_type(kind) in FOOD_TYPES or kind.upper() in {"FOOD", "DISH"}:
|
||||
continue
|
||||
return TeacherResult(spans=_collapse_same_protection(spans), mappings=list(outcome.mappings or []), ok=True)
|
||||
|
||||
|
||||
def _candidate_from_span(
|
||||
span: MentionSpan,
|
||||
user_body: str,
|
||||
rendered: str,
|
||||
*,
|
||||
suggested: str,
|
||||
reason: str,
|
||||
) -> dict:
|
||||
aligned = _align_user_span(rendered, user_body, span.start, span.end)
|
||||
kind = span.entity_type if span.entity_type in ENTITY_TYPES else "PERSON"
|
||||
return {
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": span.text,
|
||||
"entity_type": kind,
|
||||
"start": aligned[0],
|
||||
"end": aligned[1],
|
||||
**_mention_fields(user_body, span.start, span.end, span.text, rendered),
|
||||
"ambiguous": False,
|
||||
"suggested": suggested,
|
||||
"reason": reason,
|
||||
}
|
||||
|
||||
|
||||
def compare_mentions(
|
||||
profile_id: str,
|
||||
user_body: str,
|
||||
student_spans: list[MentionSpan],
|
||||
teacher_spans: list[MentionSpan],
|
||||
rendered: str = "",
|
||||
) -> tuple[list[dict], dict[str, int]]:
|
||||
counters = empty_counters()
|
||||
review: list[dict] = []
|
||||
used_teacher: set[int] = set()
|
||||
student_list = _collapse_same_protection(student_spans)
|
||||
teacher_list = _collapse_same_protection(teacher_spans)
|
||||
|
||||
def take_teacher(index: int) -> MentionSpan:
|
||||
used_teacher.add(index)
|
||||
return teacher_list[index]
|
||||
|
||||
for student in student_list:
|
||||
match_index = next(
|
||||
(
|
||||
index
|
||||
for index, teacher in enumerate(teacher_list)
|
||||
if index not in used_teacher and same_mention(student, teacher)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if match_index is None:
|
||||
counters["diff_only_student"] += 1
|
||||
if student.confirmed_cue:
|
||||
continue
|
||||
if student.protection == _DECISION_IDENTITY:
|
||||
counters["diff_new_identity"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_student"
|
||||
)
|
||||
)
|
||||
continue
|
||||
teacher = take_teacher(match_index)
|
||||
if student.protection != teacher.protection or (
|
||||
protects(student.entity_type) != protects(teacher.entity_type)
|
||||
):
|
||||
counters["diff_type_conflict"] += 1
|
||||
if student.confirmed_cue:
|
||||
continue
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
student,
|
||||
user_body,
|
||||
rendered,
|
||||
suggested=student.protection,
|
||||
reason="type_conflict",
|
||||
)
|
||||
)
|
||||
continue
|
||||
if types_equivalent(student.entity_type, teacher.entity_type) or (
|
||||
student.protection == teacher.protection
|
||||
):
|
||||
counters["diff_agree"] += 1
|
||||
if student.confirmed_cue:
|
||||
record_cue(profile_id, student.text, student.cue, student.protection)
|
||||
counters["diff_cue_reinforced"] += 1
|
||||
elif student.protection == _DECISION_IDENTITY:
|
||||
learned = get_cue_decision(profile_id, student.text, student.cue)
|
||||
if learned == _DECISION_IDENTITY:
|
||||
record_cue(profile_id, student.text, student.cue, _DECISION_IDENTITY)
|
||||
counters["diff_cue_reinforced"] += 1
|
||||
else:
|
||||
counters["diff_new_identity"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
student, user_body, rendered, suggested=_DECISION_IDENTITY, reason="new_identity"
|
||||
)
|
||||
)
|
||||
for index, teacher in enumerate(teacher_list):
|
||||
if index in used_teacher:
|
||||
continue
|
||||
counters["diff_only_teacher"] += 1
|
||||
cue = cue_for_mention(user_body, teacher.start, teacher.end, teacher.text, rendered)
|
||||
learned = get_cue_decision(profile_id, teacher.text, cue)
|
||||
if learned:
|
||||
continue
|
||||
if teacher.protection == _DECISION_IDENTITY:
|
||||
counters["diff_new_identity"] += 1
|
||||
review.append(
|
||||
_candidate_from_span(
|
||||
teacher, user_body, rendered, suggested=_DECISION_IDENTITY, reason="only_teacher"
|
||||
)
|
||||
)
|
||||
return review, counters
|
||||
|
||||
|
||||
def _remote_confirm_candidate(user_body: str) -> dict:
|
||||
snippet = (user_body or "")[:80]
|
||||
return {
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": "Klartext-Detect",
|
||||
"entity_type": "PERSON",
|
||||
"start": 0,
|
||||
"end": 0,
|
||||
"excerpt": snippet,
|
||||
"highlight_start": None,
|
||||
"highlight_end": None,
|
||||
"user_start": None,
|
||||
"user_end": None,
|
||||
"cue": _CUE_BARE,
|
||||
"ambiguous": False,
|
||||
"suggested": _DECISION_NOT,
|
||||
"reason": KIND_CONFIRM_REMOTE,
|
||||
}
|
||||
|
||||
|
||||
def prepare_detect_pipeline(profile_id: str, user_body: str, rendered: str = "") -> PipelineResult:
|
||||
mode = get_detect_operating_mode()
|
||||
counters = empty_counters()
|
||||
if is_bypass_mode(mode):
|
||||
return PipelineResult(mappings=[], candidates=[], kind=KIND_DIFF, mode=mode, counters=counters)
|
||||
student = run_student_path(profile_id, user_body, rendered)
|
||||
if mode == MODE_STUDENT:
|
||||
if student.gliner_missing:
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=[_remote_confirm_candidate(user_body)],
|
||||
kind=KIND_CONFIRM_REMOTE,
|
||||
need_review=True,
|
||||
need_remote_confirm=True,
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=[],
|
||||
kind=KIND_DIFF,
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
if mode == MODE_LEARNING_MANUAL or student.gliner_missing:
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=student.candidates,
|
||||
kind=KIND_MANUAL,
|
||||
need_review=bool(student.candidates),
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
teacher = run_teacher(profile_id, user_body, stage=learning_stage(mode), rendered=rendered)
|
||||
if not teacher.ok:
|
||||
return PipelineResult(
|
||||
mappings=student.mappings,
|
||||
candidates=student.candidates,
|
||||
kind=KIND_MANUAL,
|
||||
need_review=bool(student.candidates),
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
review, counters = compare_mentions(
|
||||
profile_id, user_body, student.spans, teacher.spans, rendered
|
||||
)
|
||||
merged = list(student.mappings)
|
||||
seen = {
|
||||
(
|
||||
item.get("start"),
|
||||
item.get("end"),
|
||||
(item.get("local_label") or "").casefold(),
|
||||
)
|
||||
for item in merged
|
||||
}
|
||||
for mapping in teacher.mappings:
|
||||
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||
if key in seen:
|
||||
continue
|
||||
label = mapping.get("local_label") or ""
|
||||
resolved = _resolve_user_span(
|
||||
user_body, mapping.get("start"), mapping.get("end"), label, user_body
|
||||
)
|
||||
if resolved:
|
||||
cue = cue_for_mention(user_body, resolved[0], resolved[1], label, rendered)
|
||||
if get_cue_decision(profile_id, label, cue) == _DECISION_NOT:
|
||||
continue
|
||||
merged.append(mapping)
|
||||
seen.add(key)
|
||||
return PipelineResult(
|
||||
mappings=merged,
|
||||
candidates=review,
|
||||
kind=KIND_DIFF,
|
||||
need_review=bool(review),
|
||||
counters=counters,
|
||||
mode=mode,
|
||||
)
|
||||
|
||||
|
||||
def compact_diff_counters(counters: dict[str, int] | None) -> dict[str, int]:
|
||||
payload = empty_counters()
|
||||
for key in payload:
|
||||
payload[key] = int((counters or {}).get(key) or 0)
|
||||
return payload
|
||||
|
||||
|
||||
def apply_remote_confirm(profile_id: str, pending: dict, decisions: list[dict]) -> list[dict]:
|
||||
"""After the user allows plaintext detect, run teacher on the user line."""
|
||||
approved = any((item.get("decision") or "").strip() == _DECISION_IDENTITY for item in decisions)
|
||||
if not approved:
|
||||
raise DetectError(
|
||||
"remote_detect_declined",
|
||||
"Klartext-Detect wurde nicht bestätigt.",
|
||||
403,
|
||||
)
|
||||
user_body = pending.get("user_body") or ""
|
||||
teacher = run_teacher(profile_id, user_body, stage=1)
|
||||
if not teacher.ok:
|
||||
raise DetectError(teacher.error or "detect_provider_unavailable", "Lehrer-Detect ist nicht erreichbar.")
|
||||
merged = list(pending.get("mappings") or [])
|
||||
seen = {
|
||||
(item.get("start"), item.get("end"), (item.get("local_label") or "").casefold())
|
||||
for item in merged
|
||||
}
|
||||
for mapping in teacher.mappings:
|
||||
key = (mapping.get("start"), mapping.get("end"), (mapping.get("local_label") or "").casefold())
|
||||
if key in seen:
|
||||
continue
|
||||
merged.append(mapping)
|
||||
seen.add(key)
|
||||
return merged
|
||||
+127
-4
@@ -81,19 +81,36 @@ def thread_snapshot(profile_id: str) -> list[dict]:
|
||||
return [row_to_dict(row) for row in rows]
|
||||
|
||||
|
||||
def create_conversation(profile_id: str, usage_session_id: str | None = None, title: str = "") -> dict:
|
||||
def create_conversation(
|
||||
profile_id: str,
|
||||
usage_session_id: str | None = None,
|
||||
title: str = "",
|
||||
space_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
) -> dict:
|
||||
conversation_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
if usage_session_id:
|
||||
session = _owned(conn, "usage_sessions", usage_session_id, profile_id)
|
||||
if not session:
|
||||
raise StoreError("not_found", "Nutzungssitzung nicht gefunden", 404)
|
||||
if space_id and not _owned(conn, "spaces", space_id, profile_id):
|
||||
raise StoreError("not_found", "Reflection Space nicht gefunden", 404)
|
||||
if journal_day_id:
|
||||
day = _owned(conn, "journal_days", journal_day_id, profile_id)
|
||||
if not day:
|
||||
raise StoreError("not_found", "Journal Day nicht gefunden", 404)
|
||||
if space_id and day["space_id"] != space_id:
|
||||
raise StoreError("assignment_mismatch", "Journal Day gehört zu einem anderen Space")
|
||||
space_id = space_id or day["space_id"]
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO conversations (id, profile_id, usage_session_id, title)
|
||||
VALUES (?, ?, ?, ?)
|
||||
INSERT INTO conversations
|
||||
(id, profile_id, usage_session_id, title, space_id, journal_day_id,
|
||||
narrative_mode, reflection_depth, current_focus)
|
||||
VALUES (?, ?, ?, ?, ?, ?, '', '', '')
|
||||
""",
|
||||
(conversation_id, profile_id, usage_session_id, title or ""),
|
||||
(conversation_id, profile_id, usage_session_id, title or "", space_id, journal_day_id),
|
||||
)
|
||||
return row_to_dict(conn.execute("SELECT * FROM conversations WHERE id = ?", (conversation_id,)).fetchone())
|
||||
|
||||
@@ -106,6 +123,29 @@ def get_conversation(profile_id: str, conversation_id: str) -> dict:
|
||||
return conv
|
||||
|
||||
|
||||
def update_conversation_signals(profile_id: str, conversation_id: str, signals: dict) -> dict:
|
||||
get_conversation(profile_id, conversation_id)
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE conversations
|
||||
SET narrative_mode = ?, reflection_depth = ?, current_focus = ?,
|
||||
emotional_intensity = ?, long_story = ?, updated = datetime('now')
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(
|
||||
(signals.get("narrative_mode") or "")[:40],
|
||||
(signals.get("reflection_depth") or "")[:40],
|
||||
(signals.get("current_focus") or "")[:200],
|
||||
(signals.get("emotional_intensity") or "")[:40],
|
||||
1 if signals.get("long_story") else 0,
|
||||
conversation_id,
|
||||
profile_id,
|
||||
),
|
||||
)
|
||||
return get_conversation(profile_id, conversation_id)
|
||||
|
||||
|
||||
def list_conversations(profile_id: str) -> list[dict]:
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
@@ -223,6 +263,89 @@ def create_space(profile_id: str, title: str = "", visibility: str = "internal")
|
||||
return row_to_dict(conn.execute("SELECT * FROM spaces WHERE id = ?", (space_id,)).fetchone())
|
||||
|
||||
|
||||
def get_space(profile_id: str, space_id: str) -> dict:
|
||||
with get_db() as conn:
|
||||
space = _owned(conn, "spaces", space_id, profile_id)
|
||||
if not space:
|
||||
raise StoreError("not_found", "Reflection Space nicht gefunden", 404)
|
||||
return space
|
||||
|
||||
|
||||
def list_spaces(profile_id: str, visibility: str | None = "user") -> list[dict]:
|
||||
with get_db() as conn:
|
||||
if visibility:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM spaces
|
||||
WHERE profile_id = ? AND visibility = ?
|
||||
ORDER BY updated DESC, created DESC
|
||||
""",
|
||||
(profile_id, visibility),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM spaces WHERE profile_id = ? ORDER BY updated DESC, created DESC",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
return [row_to_dict(row) for row in rows]
|
||||
|
||||
|
||||
def rename_space(profile_id: str, space_id: str, title: str) -> dict:
|
||||
if not (title or "").strip():
|
||||
raise StoreError("empty_title", "Space-Name darf nicht leer sein")
|
||||
with get_db() as conn:
|
||||
if not _owned(conn, "spaces", space_id, profile_id):
|
||||
raise StoreError("not_found", "Reflection Space nicht gefunden", 404)
|
||||
conn.execute(
|
||||
"UPDATE spaces SET title = ?, updated = datetime('now') WHERE id = ? AND profile_id = ?",
|
||||
(title.strip(), space_id, profile_id),
|
||||
)
|
||||
return row_to_dict(
|
||||
conn.execute("SELECT * FROM spaces WHERE id = ? AND profile_id = ?", (space_id, profile_id)).fetchone()
|
||||
)
|
||||
|
||||
|
||||
def delete_conversation(profile_id: str, conversation_id: str) -> dict:
|
||||
with get_db() as conn:
|
||||
conv = _owned(conn, "conversations", conversation_id, profile_id)
|
||||
if not conv:
|
||||
raise StoreError("not_found", "Conversation nicht gefunden", 404)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE handoffs SET source_conversation_id = NULL
|
||||
WHERE profile_id = ? AND source_conversation_id = ?
|
||||
""",
|
||||
(profile_id, conversation_id),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
DELETE FROM derived_records
|
||||
WHERE profile_id = ? AND subject_type = 'conversation' AND subject_id = ?
|
||||
""",
|
||||
(profile_id, conversation_id),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM conversations WHERE id = ? AND profile_id = ?",
|
||||
(conversation_id, profile_id),
|
||||
)
|
||||
return {"id": conversation_id, "deleted": True}
|
||||
|
||||
|
||||
def list_conversations_for_day(profile_id: str, journal_day_id: str) -> list[dict]:
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.*,
|
||||
(SELECT COUNT(*) FROM messages m WHERE m.conversation_id = c.id) AS message_count
|
||||
FROM conversations c
|
||||
WHERE c.profile_id = ? AND c.journal_day_id = ?
|
||||
ORDER BY c.created, c.id
|
||||
""",
|
||||
(profile_id, journal_day_id),
|
||||
).fetchall()
|
||||
return [row_to_dict(row) for row in rows]
|
||||
|
||||
|
||||
def link_thread_space(profile_id: str, thread_id: str, space_id: str, confidence: float | None = None) -> dict:
|
||||
with get_db() as conn:
|
||||
if not _owned(conn, "threads", thread_id, profile_id):
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
"""One generative call per user turn. Source messages are persisted before and after the call."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from context_builder import assemble_text, build_internal_context, is_closing_turn
|
||||
from conversation_signals import infer_signals
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from detect_learning import (
|
||||
apply_review_decisions,
|
||||
confirm_known_identity_spans,
|
||||
drop_pending,
|
||||
is_bypass_mode,
|
||||
load_pending,
|
||||
save_pending,
|
||||
suppress_known_non_identity,
|
||||
)
|
||||
from detect_pipeline import KIND_CONFIRM_REMOTE, prepare_detect_pipeline
|
||||
from dialogue_store import append_message, get_conversation, list_messages, update_conversation_signals
|
||||
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
||||
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES, DetectError
|
||||
from writing_profile_store import remember_dialogue_style
|
||||
from profile_review import consider_dialogue
|
||||
|
||||
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
||||
PLOT_CONTINUATION = re.compile(
|
||||
r"^\s*und\s+dann\b|"
|
||||
r"ihr\s+seid\s+dann|"
|
||||
r"seid\s+(?:ihr|du)\s+(?:dann\s+)?(?:zur|zum|in|auf|nach)|"
|
||||
r"\b(?:aufgebrochen|losgegangen|losgefahren)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
INTERVIEW_OPEN = re.compile(
|
||||
r"^\s*(?:was|wie|warum|weshalb|wieso|erzähl(?:st|t)?|und\s+was)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
DANN_PROBE = re.compile(r"\b(?:was|wie)\b.+\bdann\b|\bdann\b.+\b(?:gezeigt|geschehen|passiert)\b", re.IGNORECASE)
|
||||
COMPLETION_ASK = re.compile(
|
||||
r"^\s*ob\s+(?:ihr|du|wir)\b|"
|
||||
r"\blosgekommen\b|"
|
||||
r"\brechtzeitig\b.+\b(?:seid|habt|wart|gekommen)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
FINITE_VERB = re.compile(
|
||||
r"\b(?:ist|war|sind|waren|hat|hatte|wird|wurde|bleibt|liegt|steht|"
|
||||
r"kam|ging|kann|muss|soll|will)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
UNEARNED_COMPLETION = re.compile(
|
||||
r"\b(?:holtet|geholt|kauftet|gekauft|aßet|gegessen|unternahmt|"
|
||||
r"angekommen|aufgebrochen|losgegangen|losgefahren|losgekommen|weitergegangen)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
NEXT_BEAT_Q = re.compile(
|
||||
r"^\s*also\s+(?:seid ihr|habt ihr|bist du)\b|"
|
||||
r"(?:seid ihr|habt ihr|bist du)\s+dann\b|"
|
||||
r"(?:seid ihr|habt ihr|bist du).+\b(?:gesprungen|gerannt|rüber|geschafft|losgekommen)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
MACHINE_TELL = re.compile(
|
||||
r"\bals ki\b|"
|
||||
r"ich bin (?:eine? )?(?:ki|sprachmodell|assistent)\b|"
|
||||
r"danke,? dass du (?:das )?(?:teilst|erzählst)|"
|
||||
r"lass uns (?:das )?(?:gemeinsam|mal)|"
|
||||
r"\bzusammengefasst\b|"
|
||||
r"ich höre (?:da )?heraus|"
|
||||
r"^\s*interessant\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
UNEARNED_STANCE = re.compile(
|
||||
r"fühlte\s+sich|an(?:ge)?fühlt|"
|
||||
r"schmeckte|roch\b|klang\b|"
|
||||
r"anders als(?:\s+erwartet|\s+gedacht|\s+geplant)?|"
|
||||
r"sicher anders|"
|
||||
r"\bmusste noch\b|\bmuss noch\b|"
|
||||
r"für die nächste[n]?\b|"
|
||||
r"gehörte uns\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
CONTENT_STOP = {
|
||||
"dann", "noch", "schon", "wieder", "gegen",
|
||||
"waren", "wurde", "haben", "hatte", "lagen", "bereit", "durch",
|
||||
"unter", "über", "nach", "beim", "eine", "einem", "einer",
|
||||
"dieser", "dieses", "auch", "aber", "dass", "wenn", "dann",
|
||||
"sich", "uns", "euch", "mein", "dein", "sein", "ihre",
|
||||
}
|
||||
OPERATIONS = {
|
||||
"fortfuehren": "Fortführen",
|
||||
"konkretisieren": "Konkretisieren",
|
||||
"plan_aufgreifen": "Plan oder Erwartung aufgreifen",
|
||||
"abweichung": "Abweichung erkunden",
|
||||
"erleben_vertiefen": "Erleben vertiefen",
|
||||
"bedeutung": "Bedeutung erkunden",
|
||||
}
|
||||
|
||||
|
||||
def parse_turn_payload(content: str) -> tuple[str, dict]:
|
||||
text = (content or "").strip()
|
||||
blob = text
|
||||
if blob.startswith("```"):
|
||||
blob = re.sub(r"^```(?:json)?\s*|\s*```$", "", blob, flags=re.IGNORECASE | re.DOTALL)
|
||||
match = JSON_BLOCK.search(blob)
|
||||
if match:
|
||||
try:
|
||||
data = json.loads(match.group(0))
|
||||
except json.JSONDecodeError:
|
||||
data = None
|
||||
if isinstance(data, dict):
|
||||
operation = str(data.get("operation") or "").strip().lower()
|
||||
impulse = str(data.get("impulse") or "").strip()
|
||||
if impulse:
|
||||
label = OPERATIONS.get(operation)
|
||||
return impulse, {
|
||||
"operation": operation if label else "unparsed",
|
||||
"label": label or "nicht erkannt",
|
||||
"parsed": bool(label),
|
||||
}
|
||||
return text, {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
|
||||
|
||||
|
||||
def is_plot_continuation(impulse: str) -> bool:
|
||||
text = (impulse or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
return bool(PLOT_CONTINUATION.search(text))
|
||||
|
||||
|
||||
def is_interview_question(impulse: str) -> bool:
|
||||
return bool(INTERVIEW_OPEN.search((impulse or "").strip()))
|
||||
|
||||
|
||||
def is_unearned_completion(impulse: str) -> bool:
|
||||
return bool(UNEARNED_COMPLETION.search(impulse or ""))
|
||||
|
||||
|
||||
def is_unearned_stance(impulse: str) -> bool:
|
||||
return bool(UNEARNED_STANCE.search(impulse or ""))
|
||||
|
||||
|
||||
def is_machine_tell(impulse: str) -> bool:
|
||||
return bool(MACHINE_TELL.search(impulse or ""))
|
||||
|
||||
|
||||
def last_user_text(assembled: dict | None) -> str:
|
||||
ctx = (assembled or {}).get("dialogue_context") or ""
|
||||
lines = [line[5:].strip() for line in ctx.splitlines() if line.startswith("user:")]
|
||||
return lines[-1] if lines else ""
|
||||
|
||||
|
||||
def earlier_user_text(assembled: dict | None) -> str:
|
||||
ctx = (assembled or {}).get("dialogue_context") or ""
|
||||
lines = [line[5:].strip() for line in ctx.splitlines() if line.startswith("user:")]
|
||||
return " ".join(lines[:-1])
|
||||
|
||||
|
||||
def user_closed_day(assembled: dict | None) -> bool:
|
||||
return is_closing_turn(last_user_text(assembled))
|
||||
|
||||
|
||||
def is_day_arc_recap(impulse: str, assembled: dict | None) -> bool:
|
||||
last = last_user_text(assembled)
|
||||
earlier = earlier_user_text(assembled)
|
||||
if not last or not earlier:
|
||||
return False
|
||||
last_words = set(content_words(last))
|
||||
earlier_only = set(content_words(earlier)) - last_words
|
||||
if len(earlier_only) < 6:
|
||||
return False
|
||||
foreign = {word for word in content_words(impulse) if word in earlier_only}
|
||||
return len(foreign) >= 3
|
||||
|
||||
|
||||
def is_reopen_after_close(impulse: str, assembled: dict | None) -> bool:
|
||||
if not user_closed_day(assembled):
|
||||
return False
|
||||
if "?" in (impulse or ""):
|
||||
return True
|
||||
return is_day_arc_recap(impulse, assembled)
|
||||
|
||||
|
||||
def content_words(text: str) -> list[str]:
|
||||
return [
|
||||
word
|
||||
for word in re.findall(r"[a-zäöüß]{4,}", (text or "").lower())
|
||||
if word not in CONTENT_STOP
|
||||
]
|
||||
|
||||
|
||||
def is_echo(impulse: str, source: str) -> bool:
|
||||
src = set(content_words(source))
|
||||
imp = content_words(impulse)
|
||||
if len(imp) < 3 or len(src) < 3:
|
||||
return False
|
||||
hits = 0
|
||||
for word in imp:
|
||||
if word in src or any(len(item) >= 5 and word.startswith(item[:5]) for item in src):
|
||||
hits += 1
|
||||
return hits / len(imp) >= 0.5
|
||||
|
||||
|
||||
def is_verbless_echo(impulse: str, source: str) -> bool:
|
||||
text = (impulse or "").strip()
|
||||
if not text or "?" in text or FINITE_VERB.search(text):
|
||||
return False
|
||||
src = set(content_words(source))
|
||||
imp = content_words(text)
|
||||
if not imp or not src:
|
||||
return False
|
||||
return all(
|
||||
word in src or any(len(item) >= 5 and word.startswith(item[:5]) for item in src)
|
||||
for word in imp
|
||||
)
|
||||
|
||||
|
||||
def is_dann_probe(impulse: str) -> bool:
|
||||
return bool(DANN_PROBE.search(impulse or ""))
|
||||
|
||||
|
||||
def is_completion_ask(impulse: str) -> bool:
|
||||
return bool(COMPLETION_ASK.search(impulse or ""))
|
||||
|
||||
|
||||
def last_user_is_narrative(assembled: dict | None) -> bool:
|
||||
return len(last_user_text(assembled).split()) >= 15
|
||||
|
||||
|
||||
def is_next_beat_question(impulse: str) -> bool:
|
||||
return bool(NEXT_BEAT_Q.search(impulse or ""))
|
||||
|
||||
|
||||
def is_recap_then_ask(impulse: str, assembled: dict | None = None) -> bool:
|
||||
text = (impulse or "").strip()
|
||||
if "?" not in text or not last_user_is_narrative(assembled):
|
||||
return False
|
||||
statement = text.split("?", 1)[0].strip()
|
||||
if len(content_words(statement)) < 6:
|
||||
return False
|
||||
return is_echo(statement, last_user_text(assembled))
|
||||
|
||||
|
||||
def is_pure_recap(impulse: str, assembled: dict | None = None) -> bool:
|
||||
text = (impulse or "").strip()
|
||||
if not text or "?" in text or not last_user_is_narrative(assembled):
|
||||
return False
|
||||
last = last_user_text(assembled)
|
||||
return is_echo(text, last) or is_verbless_echo(text, last)
|
||||
|
||||
|
||||
def local_hold(last_user: str) -> str:
|
||||
if is_closing_turn(last_user):
|
||||
return "Dann ist das der Schluss."
|
||||
if len((last_user or "").split()) >= 15:
|
||||
return "Erzähl bitte weiter."
|
||||
return "Ich bin gespannt, wie es weitergeht."
|
||||
|
||||
|
||||
def repair_note(impulse: str, assembled: dict | None) -> str:
|
||||
if is_reopen_after_close(impulse, assembled):
|
||||
return (
|
||||
"Korrektur: Der Tag oder die Szene ist geschlossen. "
|
||||
"Halte den Schluss. Keine Frage, keinen Tagesbogen."
|
||||
)
|
||||
if is_unearned_stance(impulse):
|
||||
return (
|
||||
"Korrektur: Keine erfundene Empfindung und keinen Vergleich mit ungenannten anderen Tagen. "
|
||||
"Bleib bei dem, was [[SELF]] selbst gesagt hat."
|
||||
)
|
||||
if is_machine_tell(impulse):
|
||||
return (
|
||||
"Die gesprochene Zeile klingt wie ein Mensch im Gespräch, "
|
||||
"nicht wie ein System oder eine Zusammenfassung."
|
||||
)
|
||||
if is_next_beat_question(impulse) or is_recap_then_ask(impulse, assembled) or is_pure_recap(impulse, assembled):
|
||||
return (
|
||||
"Korrektur: Nicht nacherzählen. Ein kurzer Impuls oder Denkanstoß am letzten Faden. "
|
||||
"Nacherzählen nur, um einen Widerspruch oder Logikbruch zu klären."
|
||||
)
|
||||
return (
|
||||
"Korrektur: Der vorige Impuls hat den nächsten Vollzug gesetzt "
|
||||
"oder danach gefragt. Bleib Vertrauter. Kein Also-seid-ihr-dann. "
|
||||
"Erzähle den Tag nicht weiter."
|
||||
)
|
||||
|
||||
|
||||
def needs_repair(impulse: str, assembled: dict | None = None) -> bool:
|
||||
return (
|
||||
is_plot_continuation(impulse)
|
||||
or is_unearned_completion(impulse)
|
||||
or is_unearned_stance(impulse)
|
||||
or is_next_beat_question(impulse)
|
||||
or is_recap_then_ask(impulse, assembled)
|
||||
or is_pure_recap(impulse, assembled)
|
||||
or is_machine_tell(impulse)
|
||||
or is_reopen_after_close(impulse, assembled)
|
||||
)
|
||||
|
||||
|
||||
def visible_for_role(payload: dict, role: str | None) -> dict:
|
||||
if role == "admin":
|
||||
return payload
|
||||
cleaned = dict(payload)
|
||||
cleaned.pop("trace", None)
|
||||
cleaned.pop("decision", None)
|
||||
opening = cleaned.get("opening")
|
||||
if isinstance(opening, dict):
|
||||
opening = dict(opening)
|
||||
opening.pop("trace", None)
|
||||
opening.pop("decision", None)
|
||||
opening.pop("opening_context", None)
|
||||
cleaned["opening"] = opening
|
||||
return cleaned
|
||||
|
||||
|
||||
def _finish_turn(
|
||||
profile_id: str,
|
||||
conversation_id: str,
|
||||
user: dict,
|
||||
assembled: dict,
|
||||
prompt: dict,
|
||||
*,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
) -> dict:
|
||||
calls = 0
|
||||
result = None
|
||||
impulse = ""
|
||||
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
|
||||
call_traces: list[dict] = []
|
||||
working = dict(assembled)
|
||||
working["user_body"] = user.get("body") or working.get("user_body") or ""
|
||||
try:
|
||||
while calls < 2:
|
||||
result = execute_prompt(
|
||||
prompt,
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
data_class="B",
|
||||
context=working,
|
||||
precomputed_mappings=precomputed_mappings if calls == 0 else None,
|
||||
)
|
||||
calls += 1
|
||||
if result.get("trace"):
|
||||
call_traces.append(result.get("trace"))
|
||||
content = (result.get("content") or "").strip()
|
||||
if not content:
|
||||
raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.")
|
||||
impulse, decision = parse_turn_payload(content)
|
||||
if not needs_repair(impulse, working):
|
||||
break
|
||||
decision = {**decision, "guard": "impulse_rejected"}
|
||||
working = dict(working)
|
||||
working["dialogue_context"] = (
|
||||
(working.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, working)
|
||||
)
|
||||
if needs_repair(impulse, working):
|
||||
parsed_ok = bool(decision.get("parsed"))
|
||||
impulse = local_hold(last_user_text(working))
|
||||
decision = {
|
||||
"operation": "fortfuehren",
|
||||
"label": OPERATIONS["fortfuehren"],
|
||||
"parsed": parsed_ok,
|
||||
"guard": "local_fallback",
|
||||
}
|
||||
except EngineError as exc:
|
||||
if result and result.get("trace") and result.get("trace") not in call_traces:
|
||||
call_traces.append(result.get("trace"))
|
||||
if exc.code != "response_validation_failed" and exc.code not in DETECT_DIALOGUE_FALLBACK_CODES:
|
||||
persist_engine_error(
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
exc=exc,
|
||||
subject_type="conversation",
|
||||
subject_id=conversation_id,
|
||||
conversation_id=conversation_id,
|
||||
extra={
|
||||
"calls": calls,
|
||||
"call_traces": call_traces,
|
||||
"user": {"id": user.get("id"), "body": user.get("body")},
|
||||
},
|
||||
)
|
||||
raise
|
||||
impulse = local_hold(last_user_text(working))
|
||||
decision = {
|
||||
"operation": "fortfuehren",
|
||||
"label": OPERATIONS["fortfuehren"],
|
||||
"parsed": False,
|
||||
"guard": "identity_leak_blocked" if exc.code == "response_validation_failed" else "detect_blocked",
|
||||
}
|
||||
assistant = append_message(profile_id, conversation_id, impulse, role="assistant")
|
||||
user_bodies = [
|
||||
item.get("body") or ""
|
||||
for item in list_messages(profile_id, conversation_id)
|
||||
if item.get("role") == "user"
|
||||
]
|
||||
update_conversation_signals(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
infer_signals(user_bodies, decision.get("operation")),
|
||||
)
|
||||
remember_dialogue_style(profile_id)
|
||||
consider_dialogue(profile_id, user.get("body") or "")
|
||||
trace = result.get("trace") if result else None
|
||||
persist_step(
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
status="ok",
|
||||
subject_type="conversation",
|
||||
subject_id=conversation_id,
|
||||
conversation_id=conversation_id,
|
||||
message_id=assistant.get("id"),
|
||||
decision=decision,
|
||||
trace=trace,
|
||||
extra={
|
||||
"calls": calls,
|
||||
"call_traces": call_traces,
|
||||
"user": {"id": user.get("id"), "body": user.get("body")},
|
||||
"assistant": {"id": assistant.get("id"), "body": assistant.get("body")},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"conversation": get_conversation(profile_id, conversation_id),
|
||||
"user": user,
|
||||
"assistant": assistant,
|
||||
"calls": calls,
|
||||
"messages": list_messages(profile_id, conversation_id),
|
||||
"decision": decision,
|
||||
"trace": trace,
|
||||
}
|
||||
|
||||
|
||||
def _review_pause_payload(review_id: str, candidates: list[dict], kind: str, purpose: str = "dialogue_turn") -> dict:
|
||||
return {
|
||||
"id": review_id,
|
||||
"candidates": candidates,
|
||||
"kind": kind,
|
||||
"purpose": purpose,
|
||||
}
|
||||
|
||||
|
||||
def _learning_pause(profile_id: str, conversation_id: str, user: dict, assembled: dict, prompt: dict) -> dict | None:
|
||||
preview = preview_prompt(prompt, assembled)
|
||||
rendered = preview.get("rendered") or ""
|
||||
user_body = user.get("body") or ""
|
||||
result = prepare_detect_pipeline(profile_id, user_body, rendered)
|
||||
mappings = list(result.mappings or [])
|
||||
if not result.need_review and not result.candidates:
|
||||
confirm_known_identity_spans(profile_id, mappings, user_body)
|
||||
mappings = suppress_known_non_identity(profile_id, mappings, user_body)
|
||||
return _finish_turn(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
user,
|
||||
assembled,
|
||||
prompt,
|
||||
precomputed_mappings=mappings,
|
||||
)
|
||||
review_id = save_pending(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
user.get("id") or "",
|
||||
{
|
||||
"mappings": mappings,
|
||||
"candidates": result.candidates,
|
||||
"user_body": user_body,
|
||||
"kind": result.kind,
|
||||
"counters": result.counters,
|
||||
"purpose": "dialogue_turn",
|
||||
},
|
||||
)
|
||||
return {
|
||||
"conversation": get_conversation(profile_id, conversation_id),
|
||||
"user": user,
|
||||
"assistant": None,
|
||||
"calls": 0,
|
||||
"messages": list_messages(profile_id, conversation_id),
|
||||
"pending_mask_review": _review_pause_payload(review_id, result.candidates, result.kind),
|
||||
}
|
||||
|
||||
|
||||
def run_turn(profile_id: str, conversation_id: str, body: str, message_id: str | None = None) -> dict:
|
||||
conversation = get_conversation(profile_id, conversation_id)
|
||||
user = append_message(profile_id, conversation_id, body, role="user", message_id=message_id)
|
||||
context = build_internal_context(
|
||||
profile_id,
|
||||
conversation_id=conversation_id,
|
||||
space_id=conversation.get("space_id"),
|
||||
journal_day_id=conversation.get("journal_day_id"),
|
||||
purpose="dialogue_turn",
|
||||
)
|
||||
assembled = assemble_text(context)
|
||||
prompt = load_active_prompt("mvp.dialogue_turn")
|
||||
if is_bypass_mode():
|
||||
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
|
||||
paused = _learning_pause(profile_id, conversation_id, user, assembled, prompt)
|
||||
if paused is not None:
|
||||
return paused
|
||||
return _finish_turn(profile_id, conversation_id, user, assembled, prompt)
|
||||
|
||||
|
||||
def continue_turn(profile_id: str, conversation_id: str, review_id: str, decisions: list[dict]) -> dict:
|
||||
pending = load_pending(profile_id, review_id)
|
||||
if not pending or pending.get("conversation_id") != conversation_id:
|
||||
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||
if pending.get("purpose") == "journal_generate":
|
||||
raise EngineError("mask_review_missing", "Diese Prüfung gehört zum Journalentwurf.", 409)
|
||||
if pending.get("kind") == KIND_CONFIRM_REMOTE:
|
||||
try:
|
||||
from detect_pipeline import apply_remote_confirm
|
||||
|
||||
mappings = apply_remote_confirm(profile_id, pending, decisions)
|
||||
except DetectError as exc:
|
||||
raise EngineError(exc.code, exc.message, getattr(exc, "status_code", 403)) from exc
|
||||
else:
|
||||
mappings = apply_review_decisions(profile_id, pending, decisions)
|
||||
drop_pending(profile_id, review_id)
|
||||
conversation = get_conversation(profile_id, conversation_id)
|
||||
context = build_internal_context(
|
||||
profile_id,
|
||||
conversation_id=conversation_id,
|
||||
space_id=conversation.get("space_id"),
|
||||
journal_day_id=conversation.get("journal_day_id"),
|
||||
purpose="dialogue_turn",
|
||||
)
|
||||
assembled = assemble_text(context)
|
||||
prompt = load_active_prompt("mvp.dialogue_turn")
|
||||
user = {"id": pending.get("user_message_id"), "body": pending.get("user_body") or ""}
|
||||
messages = list_messages(profile_id, conversation_id)
|
||||
for item in messages:
|
||||
if item.get("id") == pending.get("user_message_id"):
|
||||
user = item
|
||||
break
|
||||
return _finish_turn(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
user,
|
||||
assembled,
|
||||
prompt,
|
||||
precomputed_mappings=mappings,
|
||||
)
|
||||
+79
-7
@@ -3,19 +3,36 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import placeholder_mvp # noqa: F401
|
||||
import placeholder_system # noqa: F401 registers system keys
|
||||
import privacy_placeholders
|
||||
from entitlements import EntitlementError, check_feature_access
|
||||
from db import get_db, row_to_dict
|
||||
from entitlements import EntitlementError, check_feature_access, increment_feature_usage
|
||||
from placeholders import PlaceholderError, resolve_template
|
||||
from privacy_gateway import GatewayRequest, PrivacyGatewayError, complete
|
||||
from journal_reconstruct import parse_dialogue_line
|
||||
|
||||
|
||||
class EngineError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 400):
|
||||
def __init__(self, code: str, message: str, status_code: int = 400, diagnostics: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
self.diagnostics = diagnostics or {}
|
||||
|
||||
|
||||
def load_active_prompt(slug: str) -> dict:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT * FROM ai_prompts WHERE slug = ? AND active = 1",
|
||||
(slug,),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
raise EngineError("prompt_missing", f"Prompt fehlt oder ist inaktiv: {slug}", 404)
|
||||
return row
|
||||
|
||||
|
||||
def preview_prompt(prompt: dict, context: dict[str, Any] | None = None) -> dict:
|
||||
@@ -36,7 +53,31 @@ def preview_prompt(prompt: dict, context: dict[str, Any] | None = None) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def execute_prompt(prompt: dict, profile_id: str, purpose: str, data_class: str, context: dict[str, Any] | None = None) -> dict:
|
||||
def user_source_text(dialogue_context: str) -> str:
|
||||
parts = []
|
||||
for line in (dialogue_context or "").splitlines():
|
||||
parsed = parse_dialogue_line(line)
|
||||
if parsed and parsed["role"] == "user":
|
||||
parts.append(parsed["body"])
|
||||
elif line.startswith("user:"):
|
||||
parts.append(line[5:].lstrip())
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def execute_prompt(
|
||||
prompt: dict,
|
||||
profile_id: str,
|
||||
purpose: str,
|
||||
data_class: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
*,
|
||||
max_tokens: int | None = None,
|
||||
disable_context_compression: bool = False,
|
||||
budget=None,
|
||||
diagnostics: dict[str, Any] | None = None,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
detect_counters: dict | None = None,
|
||||
) -> dict:
|
||||
preview = preview_prompt(prompt, context)
|
||||
feature_id = prompt.get("required_feature") or "ai_calls"
|
||||
try:
|
||||
@@ -44,14 +85,45 @@ def execute_prompt(prompt: dict, profile_id: str, purpose: str, data_class: str,
|
||||
except EntitlementError as exc:
|
||||
raise EngineError(exc.code, exc.message, exc.status_code) from exc
|
||||
try:
|
||||
complete(
|
||||
ctx = context or {}
|
||||
source_text = preview["rendered"]
|
||||
if purpose == "profile_review" and ctx.get("source_text"):
|
||||
source_text = "\n".join(
|
||||
part for part in (ctx.get("source_text"), ctx.get("review_package"), source_text) if part
|
||||
)
|
||||
elif not (source_text or "").strip():
|
||||
source_text = user_source_text(ctx.get("dialogue_context") or "")
|
||||
result = complete(
|
||||
GatewayRequest(
|
||||
prompt_id=prompt["id"],
|
||||
purpose=purpose,
|
||||
data_class=data_class,
|
||||
payload={"template_chars": len(preview["rendered"]), "privacy_tokens": preview["privacy_tokens"]},
|
||||
profile_id=profile_id,
|
||||
payload={
|
||||
"rendered": preview["rendered"],
|
||||
"source_text": source_text,
|
||||
"user_body": (ctx.get("user_body") or user_source_text(ctx.get("dialogue_context") or "")).strip(),
|
||||
"privacy_tokens": preview["privacy_tokens"],
|
||||
"prompt_slug": prompt.get("slug"),
|
||||
"prompt_revision": prompt.get("seed_revision") or "",
|
||||
"max_tokens": max_tokens,
|
||||
"disable_context_compression": disable_context_compression,
|
||||
"budget": budget,
|
||||
"diagnostics": diagnostics or {},
|
||||
"precomputed_mappings": precomputed_mappings,
|
||||
"detect_counters": detect_counters or {},
|
||||
},
|
||||
)
|
||||
)
|
||||
except PrivacyGatewayError as exc:
|
||||
raise EngineError(exc.code, exc.message, exc.status_code) from exc
|
||||
raise EngineError("provider_missing", "Gateway erlaubte einen Aufruf ohne konfigurierten Provider", 500)
|
||||
raise EngineError(exc.code, exc.message, exc.status_code, getattr(exc, "diagnostics", None)) from exc
|
||||
increment_feature_usage(profile_id, feature_id)
|
||||
return {
|
||||
**preview,
|
||||
"llm": True,
|
||||
"content": result.content,
|
||||
"provider": result.provider,
|
||||
"trace": result.trace,
|
||||
"diagnostics": result.diagnostics,
|
||||
"local_identities": result.local_identities,
|
||||
}
|
||||
@@ -91,7 +91,7 @@ def increment_feature_usage(profile_id: str, feature_id: str) -> None:
|
||||
INSERT INTO user_feature_usage (profile_id, feature_id, period_key, used)
|
||||
VALUES (?, ?, ?, 1)
|
||||
ON CONFLICT(profile_id, feature_id, period_key)
|
||||
DO UPDATE SET used = used + 1
|
||||
DO UPDATE SET used = user_feature_usage.used + 1
|
||||
""",
|
||||
(profile_id, feature_id, period),
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,257 @@
|
||||
"""Opt-in live semantic detect comparison. Not part of production generate.
|
||||
|
||||
Usage from backend/:
|
||||
python entity_detect_eval.py
|
||||
python entity_detect_eval.py --live
|
||||
|
||||
Synthetic sentences only. No personal data. Live quality stays unconfirmed
|
||||
until an explicit --live run succeeds against gold spans.
|
||||
Fake mode proves the scoring contract, not live semantic quality.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
|
||||
def _span(text: str, needle: str, entity_type: str, *, occurrence: int = 0) -> dict:
|
||||
start = -1
|
||||
found = -1
|
||||
while True:
|
||||
start = text.find(needle, start + 1)
|
||||
if start < 0:
|
||||
raise ValueError(f"needle {needle!r} occurrence {occurrence} missing in {text!r}")
|
||||
found += 1
|
||||
if found == occurrence:
|
||||
return {
|
||||
"start": start,
|
||||
"end": start + len(needle),
|
||||
"text": needle,
|
||||
"entity_type": entity_type,
|
||||
}
|
||||
|
||||
|
||||
CASES = (
|
||||
{
|
||||
"id": "person_vs_food",
|
||||
"text": "Ich aß Sushi. Sushi kam später.",
|
||||
"expected": [_span("Ich aß Sushi. Sushi kam später.", "Sushi", "PERSON", occurrence=1)],
|
||||
"note": "Person versus Lebensmittel: nur die identifizierende Nennung.",
|
||||
},
|
||||
{
|
||||
"id": "project_vs_activity",
|
||||
"text": "Ich arbeitete am privaten Projekt Aurora. Danach arbeitete ich.",
|
||||
"expected": [_span("Ich arbeitete am privaten Projekt Aurora. Danach arbeitete ich.", "Aurora", "PROJECT")],
|
||||
"note": "Privates Projekt versus allgemeine Tätigkeit.",
|
||||
},
|
||||
{
|
||||
"id": "place_vs_room",
|
||||
"text": "Ich war in Hamburg. Später saß ich im Wohnzimmer.",
|
||||
"expected": [_span("Ich war in Hamburg. Später saß ich im Wohnzimmer.", "Hamburg", "PLACE")],
|
||||
"note": "Genauer Eigenort versus allgemeiner Raum.",
|
||||
},
|
||||
{
|
||||
"id": "org_vs_noun",
|
||||
"text": "Ich sprach mit der Firma Nordwerk. Das Notizbuch blieb liegen.",
|
||||
"expected": [_span("Ich sprach mit der Firma Nordwerk. Das Notizbuch blieb liegen.", "Nordwerk", "ORG")],
|
||||
"note": "Organisation versus allgemeines Substantiv.",
|
||||
},
|
||||
{
|
||||
"id": "person_vs_group",
|
||||
"text": "Anna kam vorbei. Die Nachbarn spielten draußen.",
|
||||
"expected": [_span("Anna kam vorbei. Die Nachbarn spielten draußen.", "Anna", "PERSON")],
|
||||
"note": "Personenname versus generische Personengruppe.",
|
||||
},
|
||||
{
|
||||
"id": "animal_vs_org",
|
||||
"text": "Die Seehunde schwammen nah am Ufer.",
|
||||
"expected": [],
|
||||
"note": "Tierbezeichnung versus Organisation oder Person.",
|
||||
},
|
||||
{
|
||||
"id": "typo_common",
|
||||
"text": "Ich gieng zum Laden.",
|
||||
"expected": [],
|
||||
"note": "Tippfehler eines Allgemeinbegriffs ist keine Identität.",
|
||||
},
|
||||
{
|
||||
"id": "same_word_two_roles",
|
||||
"text": "Ich aß Sushi. Sushi kam ins Zimmer.",
|
||||
"expected": [_span("Ich aß Sushi. Sushi kam ins Zimmer.", "Sushi", "PERSON", occurrence=1)],
|
||||
"note": "Identischer Wortlaut in zwei semantischen Rollen.",
|
||||
},
|
||||
{
|
||||
"id": "multipart_place",
|
||||
"text": "Ich war in Hamburg und sprach mit der Organisation Nordwerk.",
|
||||
"expected": [
|
||||
_span("Ich war in Hamburg und sprach mit der Organisation Nordwerk.", "Hamburg", "PLACE"),
|
||||
_span("Ich war in Hamburg und sprach mit der Organisation Nordwerk.", "Nordwerk", "ORG"),
|
||||
],
|
||||
"note": "Mehrteiliger Ort plus Organisation, keine Ganzsatz-Spans.",
|
||||
},
|
||||
{
|
||||
"id": "no_sentence_span",
|
||||
"text": "Ich traf Anna am Nachmittag.",
|
||||
"expected": [_span("Ich traf Anna am Nachmittag.", "Anna", "PERSON")],
|
||||
"note": "Nur der Name, kein Ganzsatz- oder Satzfragment-Span.",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _key(item: dict) -> tuple[int, int, str, str]:
|
||||
return (
|
||||
int(item.get("start") or -1),
|
||||
int(item.get("end") or -1),
|
||||
str(item.get("text") or ""),
|
||||
str(item.get("entity_type") or "").upper(),
|
||||
)
|
||||
|
||||
|
||||
def _offset_key(item: dict) -> tuple[int, int, str]:
|
||||
return (int(item.get("start") or -1), int(item.get("end") or -1), str(item.get("text") or ""))
|
||||
|
||||
|
||||
def score_spans(predicted: list[dict], expected: list[dict]) -> dict:
|
||||
pred = [_key(item) for item in predicted]
|
||||
gold = [_key(item) for item in expected]
|
||||
pred_off = [_offset_key(item) for item in predicted]
|
||||
gold_off = [_offset_key(item) for item in expected]
|
||||
found = [item for item in gold if item in pred]
|
||||
unexpected = [item for item in pred if item not in gold]
|
||||
missing = [item for item in gold if item not in pred]
|
||||
wrong_type = []
|
||||
wrong_offset = []
|
||||
for item in predicted:
|
||||
matches_text = [
|
||||
gold_item
|
||||
for gold_item in expected
|
||||
if (gold_item.get("text") or "") == (item.get("text") or "")
|
||||
]
|
||||
if not matches_text:
|
||||
continue
|
||||
if _key(item) in gold:
|
||||
continue
|
||||
same_offsets = any(_offset_key(item) == _offset_key(gold_item) for gold_item in matches_text)
|
||||
if same_offsets:
|
||||
wrong_type.append(_key(item))
|
||||
else:
|
||||
wrong_offset.append(_key(item))
|
||||
tp = len(found)
|
||||
fp = len(unexpected)
|
||||
fn = len(missing)
|
||||
precision = tp / (tp + fp) if (tp + fp) else 1.0
|
||||
recall = tp / (tp + fn) if (tp + fn) else 1.0
|
||||
return {
|
||||
"expected_found": tp,
|
||||
"expected_count": len(gold),
|
||||
"unexpected": fp,
|
||||
"missing": fn,
|
||||
"wrong_type": len(wrong_type),
|
||||
"wrong_offset": len(wrong_offset),
|
||||
"precision": round(precision, 4),
|
||||
"recall": round(recall, 4),
|
||||
"sentence_or_fragment_span": any(
|
||||
(item.get("end") or 0) - (item.get("start") or 0) > max(len(item.get("text") or ""), 0)
|
||||
and " " in (item.get("text") or "")
|
||||
for item in predicted
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _summarize(entities: list[dict], expected: list[dict]) -> dict:
|
||||
types = sorted({(item.get("entity_type") or "").upper() for item in entities})
|
||||
return {
|
||||
"count": len(entities),
|
||||
"types": types,
|
||||
"has_labels": False,
|
||||
"metrics": score_spans(entities, expected),
|
||||
}
|
||||
|
||||
|
||||
def run_fake() -> dict:
|
||||
from entity_detect import _contract_fake_spans
|
||||
|
||||
rows = []
|
||||
for case in CASES:
|
||||
entities = _contract_fake_spans(case["text"])
|
||||
rows.append(
|
||||
{
|
||||
"id": case["id"],
|
||||
"note": case["note"],
|
||||
"entities": _summarize(entities, case["expected"]),
|
||||
"mode": "fake",
|
||||
}
|
||||
)
|
||||
return {
|
||||
"mode": "fake",
|
||||
"live_quality": "unconfirmed",
|
||||
"detect_model_unconfirmed": "openai/gpt-4.1-nano",
|
||||
"cases": rows,
|
||||
}
|
||||
|
||||
|
||||
def run_live() -> dict:
|
||||
if os.environ.get("KANSHO_FAKE_DETECT"):
|
||||
raise SystemExit("Live-Detect verweigert, solange KANSHO_FAKE_DETECT gesetzt ist.")
|
||||
from entity_detect import detect_personal_egress
|
||||
|
||||
rows = []
|
||||
for case in CASES:
|
||||
started = time.perf_counter()
|
||||
outcome = detect_personal_egress(None, case["text"])
|
||||
elapsed = int((time.perf_counter() - started) * 1000)
|
||||
predicted = [
|
||||
{
|
||||
"start": item.get("start"),
|
||||
"end": item.get("end"),
|
||||
"text": item.get("local_label") or item.get("text"),
|
||||
"entity_type": item.get("entity_type"),
|
||||
}
|
||||
for item in outcome.mappings
|
||||
]
|
||||
rows.append(
|
||||
{
|
||||
"id": case["id"],
|
||||
"note": case["note"],
|
||||
"metrics": score_spans(predicted, case["expected"]),
|
||||
"request_local_hits": outcome.stats.request_local_hits,
|
||||
"detect_calls": outcome.stats.detect_calls,
|
||||
"detect_ms": elapsed,
|
||||
"coverage": outcome.stats.full_detection_coverage,
|
||||
"detect_tokens": outcome.stats.total_tokens,
|
||||
"detect_cost": outcome.stats.cost,
|
||||
"detect_model": outcome.stats.detect_model,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"mode": "live",
|
||||
"live_quality": "unconfirmed",
|
||||
"detect_model_unconfirmed": "openai/gpt-4.1-nano",
|
||||
"cases": rows,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Synthetic detect evaluation. No personal data.")
|
||||
parser.add_argument("--live", action="store_true", help="Call the configured detect provider.")
|
||||
args = parser.parse_args()
|
||||
payload = run_live() if args.live else run_fake()
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
print(
|
||||
"Live-Qualität: unbestätigt. Das konfigurierte openai/gpt-4.1-nano "
|
||||
"gilt durch reale False-Positive-Vorschläge nicht als zuverlässig bestätigt."
|
||||
)
|
||||
if payload["mode"] != "live":
|
||||
print("Explizit live: python entity_detect_eval.py --live")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Load backend/.env into os.environ without overwriting a real process environment."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
ENV_PATH = Path(__file__).resolve().parent / ".env"
|
||||
|
||||
|
||||
def load_env_file(path: Path | None = None) -> None:
|
||||
target = path or ENV_PATH
|
||||
if not target.is_file():
|
||||
return
|
||||
for raw in target.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
if not key or key in os.environ:
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
||||
value = value[1:-1]
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def runtime_env() -> str:
|
||||
"""development/test may use remote plaintext detect. production must not, unless overridden."""
|
||||
raw = (os.environ.get("KANSHO_ENV") or "").strip().lower()
|
||||
if raw in {"production", "prod"}:
|
||||
return "production"
|
||||
if raw in {"test", "testing"}:
|
||||
return "test"
|
||||
if raw in {"development", "dev"}:
|
||||
return "development"
|
||||
return "development"
|
||||
|
||||
|
||||
def env_flag(name: str) -> bool:
|
||||
raw = (os.environ.get(name) or "").strip().lower()
|
||||
return raw in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def remote_plaintext_detect_reason() -> str:
|
||||
"""Why remote detect is allowed or blocked. Never logs secrets."""
|
||||
if runtime_env() != "production":
|
||||
return "non_production"
|
||||
if env_flag("KANSHO_ALLOW_REMOTE_DETECT"):
|
||||
return "operator_override"
|
||||
return "blocked"
|
||||
|
||||
|
||||
def allows_remote_plaintext_detect() -> bool:
|
||||
"""Remote plaintext detect is default-off in production.
|
||||
|
||||
Endbetrieb remains local detect. Until Ollama is connected, an operator
|
||||
may set KANSHO_ALLOW_REMOTE_DETECT. That does not disable the gateway.
|
||||
"""
|
||||
return remote_plaintext_detect_reason() != "blocked"
|
||||
|
||||
|
||||
def upsert_env_value(name: str, value: str, path: Path | None = None) -> None:
|
||||
"""Write one key into backend/.env and the current process. Never log the value."""
|
||||
target = path or ENV_PATH
|
||||
name = name.strip()
|
||||
if not name:
|
||||
raise ValueError("empty env name")
|
||||
lines: list[str] = []
|
||||
if target.is_file():
|
||||
lines = target.read_text(encoding="utf-8").splitlines()
|
||||
found = False
|
||||
updated: list[str] = []
|
||||
prefix = f"{name}="
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped.startswith(prefix) or stripped.startswith(f"{name} ="):
|
||||
updated.append(f"{name}={value}")
|
||||
found = True
|
||||
else:
|
||||
updated.append(line)
|
||||
if not found:
|
||||
if updated and updated[-1] != "":
|
||||
updated.append("")
|
||||
updated.append(f"{name}={value}")
|
||||
target.write_text("\n".join(updated) + "\n", encoding="utf-8")
|
||||
os.environ[name] = value
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Stateless client for the LAN GLiNER spike. Fail-soft; never logs request text."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
import httpx
|
||||
|
||||
from providers import is_local_url
|
||||
|
||||
log = logging.getLogger("gliner_client")
|
||||
|
||||
DEFAULT_URL = "http://192.168.2.144:8090"
|
||||
DEFAULT_LABELS = ("person", "place", "organization", "project", "dish", "food")
|
||||
IDENTITY_LABELS = {
|
||||
"person": "PERSON",
|
||||
"place": "PLACE",
|
||||
"organization": "ORG",
|
||||
"project": "PROJECT",
|
||||
}
|
||||
NON_IDENTITY_LABELS = frozenset({"dish", "food"})
|
||||
EXTRACT_TIMEOUT = 8.0
|
||||
HEALTH_TIMEOUT = 2.0
|
||||
|
||||
|
||||
def normalize_base_url(url: str | None) -> str:
|
||||
raw = (url or DEFAULT_URL).strip() or DEFAULT_URL
|
||||
parsed = urlparse(raw)
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
return DEFAULT_URL
|
||||
path = (parsed.path or "").rstrip("/")
|
||||
return urlunparse((parsed.scheme, parsed.netloc, path, "", "", ""))
|
||||
|
||||
|
||||
def gliner_identity_type(label: str) -> str | None:
|
||||
return IDENTITY_LABELS.get((label or "").strip().lower())
|
||||
|
||||
|
||||
def is_gliner_non_identity(label: str) -> bool:
|
||||
return (label or "").strip().lower() in NON_IDENTITY_LABELS
|
||||
|
||||
|
||||
def health(url: str | None = None) -> dict[str, Any]:
|
||||
"""Ping GLiNER. Never include request text. Fail-soft."""
|
||||
base = normalize_base_url(url)
|
||||
if not is_local_url(base):
|
||||
return {"ok": False, "status": "rejected", "reason": "not_local"}
|
||||
try:
|
||||
response = httpx.get(f"{base}/health", timeout=HEALTH_TIMEOUT)
|
||||
except httpx.HTTPError:
|
||||
return {"ok": False, "status": "unreachable", "url": base}
|
||||
if response.status_code >= 400:
|
||||
return {"ok": False, "status": "http_error", "http_status": response.status_code}
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return {"ok": False, "status": "invalid"}
|
||||
if not isinstance(payload, dict):
|
||||
return {"ok": False, "status": "invalid"}
|
||||
status = (payload.get("status") or "").strip() or "ok"
|
||||
return {
|
||||
"ok": status == "ok",
|
||||
"status": status,
|
||||
"model": payload.get("model") or "",
|
||||
"offline": bool(payload.get("offline")),
|
||||
}
|
||||
|
||||
|
||||
def extract_entities(text: str, url: str | None = None) -> list[dict] | None:
|
||||
"""Return spans or None if the service cannot be used. Empty list is a valid miss."""
|
||||
body = text or ""
|
||||
if not body.strip():
|
||||
return []
|
||||
base = normalize_base_url(url)
|
||||
if not is_local_url(base):
|
||||
log.info("gliner skipped: url is not local")
|
||||
return None
|
||||
try:
|
||||
response = httpx.post(
|
||||
f"{base}/v1/extract",
|
||||
json={"text": body, "labels": list(DEFAULT_LABELS)},
|
||||
timeout=EXTRACT_TIMEOUT,
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
log.info("gliner extract unreachable chars=%s", len(body))
|
||||
return None
|
||||
if response.status_code >= 400:
|
||||
log.info("gliner extract http=%s chars=%s", response.status_code, len(body))
|
||||
return None
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return None
|
||||
raw = payload.get("spans") if isinstance(payload, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
return None
|
||||
spans = []
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
try:
|
||||
start = int(item["start"])
|
||||
end = int(item["end"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
label = str(item.get("text") or "")
|
||||
kind = str(item.get("label") or "")
|
||||
if start < 0 or end <= start or not label or not kind:
|
||||
continue
|
||||
spans.append(
|
||||
{
|
||||
"text": label,
|
||||
"start": start,
|
||||
"end": end,
|
||||
"label": kind,
|
||||
"score": item.get("score"),
|
||||
}
|
||||
)
|
||||
log.info("gliner extract chars=%s spans=%s", len(body), len(spans))
|
||||
return spans
|
||||
@@ -0,0 +1,642 @@
|
||||
"""Confirmed identity registry. Class A; never part of an external prompt.
|
||||
|
||||
Detect output is not stored here. Request-local detections live only on the
|
||||
request manifest. Review proposals are unconfirmed and never used for masking.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from db import get_db, row_to_dict
|
||||
|
||||
TOKEN_RE = re.compile(
|
||||
r"^(SELF|PERSON:[A-Z][A-Z0-9_]{0,24}|PLACE:[A-Z][A-Z0-9_]{0,24}|"
|
||||
r"ORG:[A-Z][A-Z0-9_]{0,24}|PROJECT:[A-Z][A-Z0-9_]{0,24})$"
|
||||
)
|
||||
MIN_LABEL_LEN = 4
|
||||
TIME_RE = re.compile(r"\d|:\d|\buhr\b|\bminuten\b|\bstunde\b", re.IGNORECASE)
|
||||
DETERMINERS = {
|
||||
"der", "die", "das", "den", "dem", "des",
|
||||
"ein", "eine", "einen", "einem", "eines",
|
||||
"mein", "meine", "meinen", "meinem", "meiner", "meines",
|
||||
"dein", "deine", "sein", "seine", "ihr", "ihre",
|
||||
"unser", "unsere", "euer", "eure",
|
||||
}
|
||||
KINSHIP = {"frau", "herr", "mann", "kind", "tochter", "sohn", "partner", "partnerin"}
|
||||
STRUCTURAL_UNMASKABLE = {
|
||||
"ich", "du", "er", "sie", "es", "wir", "ihr", "mich", "mir", "mein", "meine",
|
||||
"dir", "dich", "uns", "euch", "sein", "seine", "ihre",
|
||||
"heute", "gestern", "morgen", "user", "assistant", "self", "kontext",
|
||||
}
|
||||
RESERVED_TOKEN_SUFFIXES = {"KURZ", "NAME", "TEXT", "LABEL", "EXAMPLE", "FOO", "BAR"}
|
||||
|
||||
ENTITY_TYPES = ("PERSON", "PLACE", "ORG", "PROJECT")
|
||||
STATUS_CONFIRMED = "confirmed"
|
||||
STATUS_INACTIVE = "inactive"
|
||||
STATUS_LEGACY = "legacy_review_required"
|
||||
ORIGIN_USER_CONFIRMED = "user_confirmed"
|
||||
ORIGIN_LOCAL = "local_authoritative"
|
||||
ORIGIN_LEGACY = "legacy_auto"
|
||||
PROPOSAL_UNCONFIRMED = "unconfirmed"
|
||||
PROPOSAL_DISMISSED = "dismissed"
|
||||
PROPOSAL_ORIGIN = "detect_proposal"
|
||||
|
||||
# Backward-compatible alias. Not a detection stopword list.
|
||||
UNMASKABLE = STRUCTURAL_UNMASKABLE
|
||||
|
||||
|
||||
def normalize_label(label: str) -> str:
|
||||
words = re.findall(r"[0-9A-Za-zÀ-žÄÖÜäöüß.:]+", label or "", re.UNICODE)
|
||||
keep = [
|
||||
word
|
||||
for word in words
|
||||
if word.lower() not in DETERMINERS and word.lower() not in KINSHIP
|
||||
]
|
||||
return " ".join(keep).strip()
|
||||
|
||||
|
||||
def is_maskable_label(label: str) -> bool:
|
||||
"""Structural gate only: pronouns, roles, clocks. Not a food/weather word list."""
|
||||
core = normalize_label(label)
|
||||
if len(core) < MIN_LABEL_LEN:
|
||||
return False
|
||||
if core.lower() in STRUCTURAL_UNMASKABLE:
|
||||
return False
|
||||
if TIME_RE.search(core):
|
||||
return False
|
||||
if re.fullmatch(r"user|assistant|self", core, re.IGNORECASE):
|
||||
return False
|
||||
if not re.search(r"[A-Za-zÀ-žÄÖÜäöüß]", core):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_registry_maskable_label(label: str) -> bool:
|
||||
"""Confirmed names may be shorter than the detect gate. Not a semantic word list."""
|
||||
core = (label or "").strip()
|
||||
if len(core) < 2:
|
||||
return False
|
||||
if core.lower() in STRUCTURAL_UNMASKABLE:
|
||||
return False
|
||||
if TIME_RE.search(core):
|
||||
return False
|
||||
if re.fullmatch(r"user|assistant|self", core, re.IGNORECASE):
|
||||
return False
|
||||
if not re.search(r"[A-Za-zÀ-žÄÖÜäöüß]", core):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _token_suffix(token: str) -> str:
|
||||
raw = (token or "").upper()
|
||||
return raw.split(":")[-1] if ":" in token else raw
|
||||
|
||||
|
||||
def normalize_token(token: str | None) -> str | None:
|
||||
raw = (token or "").strip().upper().replace(" ", "_")
|
||||
raw = raw[2:-2] if raw.startswith("[[") and raw.endswith("]]") else raw
|
||||
if not TOKEN_RE.match(raw):
|
||||
return None
|
||||
if raw.count("_") > 1:
|
||||
return None
|
||||
if _token_suffix(raw) in RESERVED_TOKEN_SUFFIXES:
|
||||
return None
|
||||
return raw
|
||||
|
||||
|
||||
def normalize_entity_type(value: str | None, *, default: str = "PERSON") -> str | None:
|
||||
raw = (value or "").strip().upper()
|
||||
if not raw:
|
||||
raw = default
|
||||
if raw not in ENTITY_TYPES:
|
||||
return None
|
||||
return raw
|
||||
|
||||
|
||||
def parse_aliases(raw) -> list[str]:
|
||||
if isinstance(raw, list):
|
||||
items = raw
|
||||
else:
|
||||
try:
|
||||
items = json.loads(raw or "[]")
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
items = []
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for item in items:
|
||||
label = (item or "").strip()
|
||||
if not label or label.casefold() in seen or not is_registry_maskable_label(label):
|
||||
continue
|
||||
seen.add(label.casefold())
|
||||
result.append(label)
|
||||
return result
|
||||
|
||||
|
||||
def _row_public(row: dict | None) -> dict | None:
|
||||
if not row:
|
||||
return None
|
||||
canonical = (row.get("canonical_label") or row.get("local_label") or "").strip()
|
||||
aliases = [
|
||||
item
|
||||
for item in parse_aliases(row.get("aliases_json"))
|
||||
if item.casefold() != canonical.casefold()
|
||||
]
|
||||
return {
|
||||
"id": row.get("id"),
|
||||
"token": row.get("token"),
|
||||
"local_label": canonical,
|
||||
"canonical_label": canonical,
|
||||
"entity_type": (row.get("entity_type") or "PERSON").upper(),
|
||||
"status": row.get("status") or STATUS_LEGACY,
|
||||
"origin": row.get("origin") or ORIGIN_LEGACY,
|
||||
"aliases": aliases,
|
||||
"created": row.get("created"),
|
||||
"updated": row.get("updated"),
|
||||
"confirmed_at": row.get("confirmed_at"),
|
||||
}
|
||||
|
||||
|
||||
def _select_all(profile_id: str) -> list[dict]:
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM identity_mappings
|
||||
WHERE profile_id = ?
|
||||
ORDER BY created
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
return [_row_public(row_to_dict(row)) for row in rows]
|
||||
|
||||
|
||||
def list_registry(profile_id: str, *, include_inactive: bool = True) -> list[dict]:
|
||||
rows = [item for item in _select_all(profile_id) if item]
|
||||
if include_inactive:
|
||||
return rows
|
||||
return [item for item in rows if item.get("status") == STATUS_CONFIRMED]
|
||||
|
||||
|
||||
def list_confirmed_identities(profile_id: str) -> list[dict]:
|
||||
return [item for item in _select_all(profile_id) if item and item.get("status") == STATUS_CONFIRMED]
|
||||
|
||||
|
||||
def list_mappings(profile_id: str) -> list[dict]:
|
||||
"""Confirmed identities only. Legacy and inactive rows are not a masking dictionary."""
|
||||
return list_confirmed_identities(profile_id)
|
||||
|
||||
|
||||
def confirmed_match_labels(item: dict) -> list[str]:
|
||||
labels = []
|
||||
canonical = (item.get("canonical_label") or item.get("local_label") or "").strip()
|
||||
if canonical:
|
||||
labels.append(canonical)
|
||||
for alias in item.get("aliases") or []:
|
||||
if alias and alias.casefold() != canonical.casefold():
|
||||
labels.append(alias)
|
||||
return labels
|
||||
|
||||
|
||||
def mapping_spellings(item: dict) -> list[str]:
|
||||
"""All known writings of one mapping: canonical, aliases, observed label, demask form."""
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
|
||||
def add(raw) -> None:
|
||||
label = (raw or "").strip()
|
||||
if not label or label.casefold() in seen or not is_registry_maskable_label(label):
|
||||
return
|
||||
seen.add(label.casefold())
|
||||
result.append(label)
|
||||
|
||||
add(item.get("local_label"))
|
||||
add(item.get("canonical_label"))
|
||||
add(item.get("demask_label"))
|
||||
for alias in item.get("aliases") or []:
|
||||
add(alias)
|
||||
for label in item.get("labels") or []:
|
||||
add(label)
|
||||
return result
|
||||
|
||||
|
||||
def masking_rows_from_confirmed(profile_id: str) -> list[dict]:
|
||||
"""One masking row per confirmed canonical label or alias. Demask uses canonical."""
|
||||
rows: list[dict] = []
|
||||
for item in list_confirmed_identities(profile_id):
|
||||
token = (item.get("token") or "").strip()
|
||||
canonical = (item.get("canonical_label") or "").strip()
|
||||
entity_type = (item.get("entity_type") or "PERSON").upper()
|
||||
if not token or not canonical:
|
||||
continue
|
||||
for label in confirmed_match_labels(item):
|
||||
if not is_registry_maskable_label(label):
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"token": token,
|
||||
"local_label": label,
|
||||
"canonical_label": canonical,
|
||||
"demask_label": canonical,
|
||||
"entity_type": entity_type,
|
||||
"source": "confirmed_registry",
|
||||
"aliases": [item for item in confirmed_match_labels(item) if item.casefold() != label.casefold()],
|
||||
"labels": confirmed_match_labels(item),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def find_existing_mapping(existing: list[dict], label: str) -> dict | None:
|
||||
core = normalize_label(label).casefold()
|
||||
if not core:
|
||||
return None
|
||||
for item in existing:
|
||||
candidates = [item.get("canonical_label") or "", item.get("local_label") or ""]
|
||||
candidates.extend(item.get("aliases") or [])
|
||||
for stored in candidates:
|
||||
stored_core = normalize_label(stored).casefold()
|
||||
if stored.casefold() == label.casefold() or (stored_core and stored_core == core):
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def _next_generic(existing: list[dict], prefix: str) -> str:
|
||||
used = set()
|
||||
for item in existing:
|
||||
token = (item.get("token") or "").upper()
|
||||
if token.startswith(prefix):
|
||||
suffix = token[len(prefix) :]
|
||||
if suffix.isdigit():
|
||||
used.add(int(suffix))
|
||||
n = 1
|
||||
while n in used:
|
||||
n += 1
|
||||
return f"{prefix}{n:02d}"
|
||||
|
||||
|
||||
def _prefix_for_type(entity_type: str) -> str:
|
||||
return f"{entity_type}:"
|
||||
|
||||
|
||||
def confirm_identity(
|
||||
profile_id: str,
|
||||
canonical_label: str,
|
||||
*,
|
||||
entity_type: str = "PERSON",
|
||||
token: str | None = None,
|
||||
aliases: list[str] | None = None,
|
||||
origin: str = ORIGIN_USER_CONFIRMED,
|
||||
identity_id: str | None = None,
|
||||
) -> dict:
|
||||
label = normalize_label(canonical_label) or (canonical_label or "").strip()
|
||||
if not label or not is_registry_maskable_label(label):
|
||||
raise ValueError("empty_label")
|
||||
kind = normalize_entity_type(entity_type)
|
||||
if not kind:
|
||||
raise ValueError("invalid_entity_type")
|
||||
alias_list = parse_aliases(aliases or [])
|
||||
alias_list = [item for item in alias_list if item.casefold() != label.casefold()]
|
||||
existing = _select_all(profile_id)
|
||||
found = None
|
||||
if identity_id:
|
||||
found = next((item for item in existing if item.get("id") == identity_id), None)
|
||||
if found is None:
|
||||
found = find_existing_mapping(existing, label)
|
||||
chosen_token = normalize_token(token)
|
||||
if found:
|
||||
chosen_token = found["token"]
|
||||
elif chosen_token and any((item.get("token") or "").upper() == chosen_token for item in existing):
|
||||
chosen_token = None
|
||||
if not chosen_token:
|
||||
chosen_token = _next_generic(existing, _prefix_for_type(kind))
|
||||
row_id = (found or {}).get("id") or str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO identity_mappings (
|
||||
id, profile_id, token, local_label, canonical_label, entity_type,
|
||||
status, origin, aliases_json, confirmed_at, updated
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
token = excluded.token,
|
||||
local_label = excluded.local_label,
|
||||
canonical_label = excluded.canonical_label,
|
||||
entity_type = excluded.entity_type,
|
||||
status = excluded.status,
|
||||
origin = excluded.origin,
|
||||
aliases_json = excluded.aliases_json,
|
||||
confirmed_at = datetime('now'),
|
||||
updated = datetime('now')
|
||||
""",
|
||||
(
|
||||
row_id,
|
||||
profile_id,
|
||||
chosen_token,
|
||||
label,
|
||||
label,
|
||||
kind,
|
||||
STATUS_CONFIRMED,
|
||||
origin if origin in {ORIGIN_USER_CONFIRMED, ORIGIN_LOCAL} else ORIGIN_USER_CONFIRMED,
|
||||
json.dumps(alias_list, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
return next(item for item in _select_all(profile_id) if item.get("id") == row_id)
|
||||
|
||||
|
||||
def remember_mapping(profile_id: str, local_label: str, suggested_token: str | None = None) -> dict:
|
||||
"""Explicit local confirmation helper for tests and admin. Not the detect path."""
|
||||
token = normalize_token(suggested_token)
|
||||
entity_type = "PERSON"
|
||||
if token and ":" in token:
|
||||
entity_type = token.split(":", 1)[0]
|
||||
return confirm_identity(
|
||||
profile_id,
|
||||
local_label,
|
||||
entity_type=entity_type if entity_type in ENTITY_TYPES else "PERSON",
|
||||
token=suggested_token,
|
||||
origin=ORIGIN_LOCAL,
|
||||
)
|
||||
|
||||
|
||||
def update_identity(
|
||||
profile_id: str,
|
||||
identity_id: str,
|
||||
*,
|
||||
canonical_label: str | None = None,
|
||||
entity_type: str | None = None,
|
||||
aliases: list[str] | None = None,
|
||||
status: str | None = None,
|
||||
) -> dict:
|
||||
rows = _select_all(profile_id)
|
||||
found = next((item for item in rows if item.get("id") == identity_id), None)
|
||||
if not found:
|
||||
raise ValueError("identity_missing")
|
||||
label = found["canonical_label"]
|
||||
if canonical_label is not None:
|
||||
label = normalize_label(canonical_label) or canonical_label.strip()
|
||||
if not label or not is_registry_maskable_label(label):
|
||||
raise ValueError("empty_label")
|
||||
kind = found["entity_type"]
|
||||
if entity_type is not None:
|
||||
kind = normalize_entity_type(entity_type)
|
||||
if not kind:
|
||||
raise ValueError("invalid_entity_type")
|
||||
alias_list = found.get("aliases") or []
|
||||
if aliases is not None:
|
||||
alias_list = [item for item in parse_aliases(aliases) if item.casefold() != label.casefold()]
|
||||
next_status = found["status"]
|
||||
if status is not None:
|
||||
if status not in {STATUS_CONFIRMED, STATUS_INACTIVE, STATUS_LEGACY}:
|
||||
raise ValueError("invalid_status")
|
||||
next_status = status
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE identity_mappings
|
||||
SET local_label = ?, canonical_label = ?, entity_type = ?,
|
||||
aliases_json = ?, status = ?,
|
||||
confirmed_at = CASE WHEN ? = 'confirmed' THEN datetime('now') ELSE confirmed_at END,
|
||||
origin = CASE WHEN ? = 'confirmed' AND origin = 'legacy_auto' THEN 'user_confirmed' ELSE origin END,
|
||||
updated = datetime('now')
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(
|
||||
label,
|
||||
label,
|
||||
kind,
|
||||
json.dumps(alias_list, ensure_ascii=False),
|
||||
next_status,
|
||||
next_status,
|
||||
next_status,
|
||||
identity_id,
|
||||
profile_id,
|
||||
),
|
||||
)
|
||||
return next(item for item in _select_all(profile_id) if item.get("id") == identity_id)
|
||||
|
||||
|
||||
def deactivate_identity(profile_id: str, identity_id: str) -> dict:
|
||||
return update_identity(profile_id, identity_id, status=STATUS_INACTIVE)
|
||||
|
||||
|
||||
def delete_identity(profile_id: str, identity_id: str) -> None:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM identity_mappings WHERE id = ? AND profile_id = ?",
|
||||
(identity_id, profile_id),
|
||||
)
|
||||
|
||||
|
||||
def list_review_proposals(profile_id: str, *, include_dismissed: bool = False) -> list[dict]:
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM identity_review_proposals
|
||||
WHERE profile_id = ?
|
||||
ORDER BY last_seen DESC
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
result = []
|
||||
for row in rows:
|
||||
item = row_to_dict(row)
|
||||
if not include_dismissed and item.get("status") != PROPOSAL_UNCONFIRMED:
|
||||
continue
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def record_review_proposal(profile_id: str, observed_label: str, entity_type: str) -> None:
|
||||
"""Local unconfirmed note only. Never used for masking or later detection skip."""
|
||||
label = (observed_label or "").strip()
|
||||
kind = normalize_entity_type(entity_type)
|
||||
if not profile_id or not label or not kind or not is_maskable_label(label):
|
||||
return
|
||||
if find_existing_mapping(list_confirmed_identities(profile_id), label):
|
||||
return
|
||||
with get_db() as conn:
|
||||
existing = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id FROM identity_review_proposals
|
||||
WHERE profile_id = ? AND observed_label = ? AND entity_type = ?
|
||||
""",
|
||||
(profile_id, label, kind),
|
||||
).fetchone()
|
||||
)
|
||||
if existing:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE identity_review_proposals
|
||||
SET last_seen = datetime('now'), status = CASE
|
||||
WHEN status = 'dismissed' THEN status ELSE 'unconfirmed' END
|
||||
WHERE id = ?
|
||||
""",
|
||||
(existing["id"],),
|
||||
)
|
||||
return
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO identity_review_proposals
|
||||
(id, profile_id, observed_label, entity_type, status, origin)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(str(uuid.uuid4()), profile_id, label, kind, PROPOSAL_UNCONFIRMED, PROPOSAL_ORIGIN),
|
||||
)
|
||||
|
||||
|
||||
def confirm_review_proposal(profile_id: str, proposal_id: str, **overrides) -> dict:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT * FROM identity_review_proposals WHERE id = ? AND profile_id = ?",
|
||||
(proposal_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
raise ValueError("proposal_missing")
|
||||
confirmed = confirm_identity(
|
||||
profile_id,
|
||||
overrides.get("canonical_label") or row["observed_label"],
|
||||
entity_type=overrides.get("entity_type") or row["entity_type"],
|
||||
aliases=overrides.get("aliases"),
|
||||
origin=ORIGIN_USER_CONFIRMED,
|
||||
)
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM identity_review_proposals WHERE id = ? AND profile_id = ?",
|
||||
(proposal_id, profile_id),
|
||||
)
|
||||
return confirmed
|
||||
|
||||
|
||||
def dismiss_review_proposal(profile_id: str, proposal_id: str) -> None:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE identity_review_proposals
|
||||
SET status = ?, last_seen = datetime('now')
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(PROPOSAL_DISMISSED, proposal_id, profile_id),
|
||||
)
|
||||
|
||||
|
||||
def delete_review_proposal(profile_id: str, proposal_id: str) -> None:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM identity_review_proposals WHERE id = ? AND profile_id = ?",
|
||||
(proposal_id, profile_id),
|
||||
)
|
||||
|
||||
|
||||
def purge_unmaskable(profile_id: str) -> int:
|
||||
removed = 0
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, local_label, canonical_label FROM identity_mappings WHERE profile_id = ?",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
label = row["canonical_label"] or row["local_label"]
|
||||
if is_registry_maskable_label(label):
|
||||
continue
|
||||
conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],))
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
|
||||
def coalesce_mappings(profile_id: str) -> int:
|
||||
"""One confirmed/legacy row per normalized canonical name."""
|
||||
merged = 0
|
||||
with get_db() as conn:
|
||||
rows = [
|
||||
row_to_dict(row)
|
||||
for row in conn.execute(
|
||||
"""
|
||||
SELECT id, token, local_label, canonical_label, entity_type, status, origin, created
|
||||
FROM identity_mappings
|
||||
WHERE profile_id = ?
|
||||
ORDER BY created
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
]
|
||||
groups: dict[str, list[dict]] = {}
|
||||
for row in rows:
|
||||
core = normalize_label(row.get("canonical_label") or row.get("local_label") or "")
|
||||
if not core or not is_registry_maskable_label(core):
|
||||
conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],))
|
||||
merged += 1
|
||||
continue
|
||||
groups.setdefault(core.casefold(), []).append({**row, "core": core})
|
||||
for items in groups.values():
|
||||
keeper = min(
|
||||
items,
|
||||
key=lambda row: (
|
||||
0 if (row.get("status") == STATUS_CONFIRMED) else 1,
|
||||
0 if _token_suffix(row.get("token") or "").isdigit() else 1,
|
||||
row.get("created") or "",
|
||||
row.get("id") or "",
|
||||
),
|
||||
)
|
||||
core = items[0]["core"]
|
||||
if (keeper.get("canonical_label") or keeper.get("local_label") or "") != core:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE identity_mappings
|
||||
SET local_label = ?, canonical_label = ?, updated = datetime('now')
|
||||
WHERE id = ?
|
||||
""",
|
||||
(core, core, keeper["id"]),
|
||||
)
|
||||
for row in items:
|
||||
if row["id"] == keeper["id"]:
|
||||
continue
|
||||
conn.execute("DELETE FROM identity_mappings WHERE id = ?", (row["id"],))
|
||||
merged += 1
|
||||
return merged
|
||||
|
||||
|
||||
def migrate_legacy_identity_rows(conn) -> int:
|
||||
"""Mark pre-existing auto-mappings as review-required. Never auto-confirm them."""
|
||||
names = {row["name"] for row in conn.execute("PRAGMA table_info(identity_mappings)").fetchall()}
|
||||
if "status" not in names:
|
||||
return 0
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE identity_mappings
|
||||
SET updated = COALESCE(NULLIF(updated, ''), created)
|
||||
WHERE updated IS NULL OR updated = ''
|
||||
"""
|
||||
)
|
||||
marked = 0
|
||||
rows = conn.execute(
|
||||
"SELECT id, local_label, canonical_label, status, origin FROM identity_mappings"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
canonical = (row["canonical_label"] or row["local_label"] or "").strip()
|
||||
status = row["status"] or ""
|
||||
origin = row["origin"] or ""
|
||||
updates: list[str] = []
|
||||
values: list[str] = []
|
||||
if not (row["canonical_label"] or "").strip() and canonical:
|
||||
updates.append("canonical_label = ?")
|
||||
values.append(canonical)
|
||||
if not status or (status == STATUS_CONFIRMED and origin == ORIGIN_LEGACY):
|
||||
updates.append("status = ?")
|
||||
values.append(STATUS_LEGACY)
|
||||
if not origin:
|
||||
updates.append("origin = ?")
|
||||
values.append(ORIGIN_LEGACY)
|
||||
if not updates:
|
||||
continue
|
||||
values.append(row["id"])
|
||||
conn.execute(
|
||||
f"UPDATE identity_mappings SET {', '.join(updates)}, updated = datetime('now') WHERE id = ?",
|
||||
values,
|
||||
)
|
||||
marked += 1
|
||||
return marked
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Labeled product defaults for dialogue interaction. Not a personality model.
|
||||
|
||||
These are visible, overridable defaults — not silent personal facts inferred from
|
||||
the mere fact that a dialogue happened.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
DEFAULTS = (
|
||||
{
|
||||
"scope": "global",
|
||||
"pref_key": "reply_length",
|
||||
"title": "Antwortlänge",
|
||||
"value": "Ein bis drei Sätze.",
|
||||
},
|
||||
{
|
||||
"scope": "global",
|
||||
"pref_key": "follow_up",
|
||||
"title": "Anschlussimpuls",
|
||||
"value": "Nur einen dominanten Anschlussimpuls setzen, nicht mehrere Fragen stapeln.",
|
||||
},
|
||||
{
|
||||
"scope": "autobiographical",
|
||||
"pref_key": "listening",
|
||||
"title": "Zuhören bei Erzählung",
|
||||
"value": "Bei längeren autobiografischen Erzählungen zunächst stärker zuhören und wenig unterbrechen.",
|
||||
},
|
||||
)
|
||||
|
||||
SCOPES = ("global", "autobiographical")
|
||||
PREF_KEYS = tuple(item["pref_key"] for item in DEFAULTS)
|
||||
GOVERNANCE = {"learning", "advising", "frozen"}
|
||||
DEFAULT_GOVERNANCE = "advising"
|
||||
@@ -0,0 +1,423 @@
|
||||
"""Interaction profile: stable preferences, never learned from silence.
|
||||
|
||||
Strong sources only: explicit instruction, explicit feedback, manual setting,
|
||||
accepted suggestion. Observations may be stored as suggestions, never auto-applied
|
||||
from a dialogue turn.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from db import get_db, row_to_dict
|
||||
from dialogue_store import StoreError
|
||||
from interaction_defaults import DEFAULT_GOVERNANCE, DEFAULTS, GOVERNANCE, PREF_KEYS, SCOPES
|
||||
from profile_documents import (
|
||||
FORMAT_VERSION,
|
||||
KIND_INTERACTION,
|
||||
ORIGIN_INTERACTION,
|
||||
exported_at,
|
||||
interaction_payload,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"GOVERNANCE",
|
||||
"SCOPES",
|
||||
"get_profile",
|
||||
"set_governance",
|
||||
"upsert_preference",
|
||||
"accept_suggestion",
|
||||
"reject_suggestion",
|
||||
"propose_observation",
|
||||
"assemble_interaction_hint",
|
||||
"export_document",
|
||||
"restore_document",
|
||||
]
|
||||
|
||||
|
||||
def ensure_profile(profile_id: str) -> dict:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute("SELECT * FROM interaction_profiles WHERE profile_id = ?", (profile_id,)).fetchone()
|
||||
)
|
||||
if row:
|
||||
return row
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO interaction_profiles (profile_id, governance)
|
||||
VALUES (?, ?)
|
||||
""",
|
||||
(profile_id, DEFAULT_GOVERNANCE),
|
||||
)
|
||||
return row_to_dict(
|
||||
conn.execute("SELECT * FROM interaction_profiles WHERE profile_id = ?", (profile_id,)).fetchone()
|
||||
)
|
||||
|
||||
|
||||
def get_profile(profile_id: str) -> dict:
|
||||
row = ensure_profile(profile_id)
|
||||
with get_db() as conn:
|
||||
prefs = [
|
||||
row_to_dict(item)
|
||||
for item in conn.execute(
|
||||
"""
|
||||
SELECT * FROM interaction_preferences
|
||||
WHERE profile_id = ?
|
||||
ORDER BY CASE scope WHEN 'global' THEN 0 ELSE 1 END, pref_key
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
]
|
||||
suggestions = [
|
||||
row_to_dict(item)
|
||||
for item in conn.execute(
|
||||
"""
|
||||
SELECT * FROM interaction_suggestions
|
||||
WHERE profile_id = ? AND status = 'pending'
|
||||
ORDER BY created DESC
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
]
|
||||
for item in prefs:
|
||||
item["locked"] = bool(item.get("locked"))
|
||||
item["title"] = _title(item.get("scope") or "global", item.get("pref_key") or "")
|
||||
defaults = [
|
||||
{
|
||||
**item,
|
||||
"source": "product_default",
|
||||
"overridden": any(
|
||||
pref.get("scope") == item["scope"] and pref.get("pref_key") == item["pref_key"]
|
||||
for pref in prefs
|
||||
),
|
||||
}
|
||||
for item in DEFAULTS
|
||||
]
|
||||
return {
|
||||
"profile_id": profile_id,
|
||||
"governance": (row or {}).get("governance") or DEFAULT_GOVERNANCE,
|
||||
"updated": (row or {}).get("updated"),
|
||||
"preferences": prefs,
|
||||
"defaults": defaults,
|
||||
"suggestions": suggestions,
|
||||
"note": (
|
||||
"Persönliche Interaction-Präferenzen entstehen nur durch ausdrückliche Angabe, "
|
||||
"Feedback, manuelle Einstellung oder angenommene Vorschläge. "
|
||||
"Ein Dialog ohne Widerspruch ist kein Lernsignal."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def set_governance(profile_id: str, governance: str) -> dict:
|
||||
mode = (governance or "").strip()
|
||||
if mode not in GOVERNANCE:
|
||||
raise StoreError("invalid_governance", "governance muss learning, advising oder frozen sein")
|
||||
ensure_profile(profile_id)
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE interaction_profiles SET governance = ?, updated = datetime('now') WHERE profile_id = ?",
|
||||
(mode, profile_id),
|
||||
)
|
||||
return get_profile(profile_id)
|
||||
|
||||
|
||||
def upsert_preference(
|
||||
profile_id: str,
|
||||
*,
|
||||
scope: str,
|
||||
pref_key: str,
|
||||
value: str,
|
||||
origin: str = "manual",
|
||||
locked: bool | None = None,
|
||||
evidence: str = "manuelle Einstellung",
|
||||
) -> dict:
|
||||
if scope not in SCOPES:
|
||||
raise StoreError("unknown_scope", "Unbekannter Interaction-Scope.")
|
||||
if pref_key not in PREF_KEYS:
|
||||
raise StoreError("unknown_pref", "Unbekannte Interaction-Präferenz.")
|
||||
if origin not in {"explicit", "manual", "accepted_suggestion"}:
|
||||
raise StoreError("invalid_origin", "Nur explizite, manuelle oder angenommene Herkunft.")
|
||||
ensure_profile(profile_id)
|
||||
with get_db() as conn:
|
||||
current = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM interaction_preferences
|
||||
WHERE profile_id = ? AND scope = ? AND pref_key = ?
|
||||
""",
|
||||
(profile_id, scope, pref_key),
|
||||
).fetchone()
|
||||
)
|
||||
next_locked = int(locked) if locked is not None else int((current or {}).get("locked") or 0)
|
||||
if current:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE interaction_preferences
|
||||
SET value = ?, evidence = ?, origin = ?, locked = ?, updated = datetime('now')
|
||||
WHERE id = ?
|
||||
""",
|
||||
(value, evidence, origin, next_locked, current["id"]),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO interaction_preferences
|
||||
(id, profile_id, scope, pref_key, value, evidence, origin, locked)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(str(uuid.uuid4()), profile_id, scope, pref_key, value, evidence, origin, next_locked),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE interaction_profiles SET updated = datetime('now') WHERE profile_id = ?",
|
||||
(profile_id,),
|
||||
)
|
||||
return get_profile(profile_id)
|
||||
|
||||
|
||||
def propose_observation(
|
||||
profile_id: str,
|
||||
*,
|
||||
scope: str,
|
||||
pref_key: str,
|
||||
proposed_value: str,
|
||||
evidence: str,
|
||||
) -> dict | None:
|
||||
"""Candidate only. Never applied here. Not called from the dialogue turn."""
|
||||
if scope not in SCOPES or pref_key not in PREF_KEYS:
|
||||
return None
|
||||
ensure_profile(profile_id)
|
||||
row = ensure_profile(profile_id)
|
||||
if (row.get("governance") or DEFAULT_GOVERNANCE) == "frozen":
|
||||
return None
|
||||
with get_db() as conn:
|
||||
current = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT locked FROM interaction_preferences
|
||||
WHERE profile_id = ? AND scope = ? AND pref_key = ?
|
||||
""",
|
||||
(profile_id, scope, pref_key),
|
||||
).fetchone()
|
||||
)
|
||||
if current and int(current.get("locked") or 0):
|
||||
return None
|
||||
pending = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT id, proposed_value FROM interaction_suggestions
|
||||
WHERE profile_id = ? AND scope = ? AND pref_key = ? AND status = 'pending'
|
||||
ORDER BY created DESC LIMIT 1
|
||||
""",
|
||||
(profile_id, scope, pref_key),
|
||||
).fetchone()
|
||||
)
|
||||
if pending and (pending.get("proposed_value") or "") == proposed_value:
|
||||
return get_profile(profile_id)
|
||||
if pending:
|
||||
conn.execute(
|
||||
"UPDATE interaction_suggestions SET status = 'rejected', resolved = datetime('now') WHERE id = ?",
|
||||
(pending["id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO interaction_suggestions
|
||||
(id, profile_id, scope, pref_key, proposed_value, evidence, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'pending')
|
||||
""",
|
||||
(str(uuid.uuid4()), profile_id, scope, pref_key, proposed_value, evidence),
|
||||
)
|
||||
return get_profile(profile_id)
|
||||
|
||||
|
||||
def accept_suggestion(profile_id: str, suggestion_id: str) -> dict:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM interaction_suggestions
|
||||
WHERE id = ? AND profile_id = ? AND status = 'pending'
|
||||
""",
|
||||
(suggestion_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
raise StoreError("not_found", "Vorschlag nicht gefunden", 404)
|
||||
upsert_preference(
|
||||
profile_id,
|
||||
scope=row.get("scope") or "global",
|
||||
pref_key=row["pref_key"],
|
||||
value=row.get("proposed_value") or "",
|
||||
origin="accepted_suggestion",
|
||||
evidence=row.get("evidence") or "angenommener Vorschlag",
|
||||
)
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE interaction_suggestions SET status = 'accepted', resolved = datetime('now') WHERE id = ?",
|
||||
(suggestion_id,),
|
||||
)
|
||||
return get_profile(profile_id)
|
||||
|
||||
|
||||
def reject_suggestion(profile_id: str, suggestion_id: str) -> dict:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM interaction_suggestions
|
||||
WHERE id = ? AND profile_id = ? AND status = 'pending'
|
||||
""",
|
||||
(suggestion_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
raise StoreError("not_found", "Vorschlag nicht gefunden", 404)
|
||||
conn.execute(
|
||||
"UPDATE interaction_suggestions SET status = 'rejected', resolved = datetime('now') WHERE id = ?",
|
||||
(suggestion_id,),
|
||||
)
|
||||
return get_profile(profile_id)
|
||||
|
||||
|
||||
def export_document(profile_id: str) -> dict:
|
||||
profile = get_profile(profile_id)
|
||||
return {
|
||||
"kind": KIND_INTERACTION,
|
||||
"format_version": FORMAT_VERSION,
|
||||
"exported_at": exported_at(),
|
||||
"governance": profile.get("governance") or DEFAULT_GOVERNANCE,
|
||||
"preferences": [
|
||||
{
|
||||
"scope": item.get("scope") or "global",
|
||||
"pref_key": item.get("pref_key"),
|
||||
"value": item.get("value") or "",
|
||||
"evidence": item.get("evidence") or "",
|
||||
"origin": item.get("origin") or "manual",
|
||||
"locked": bool(item.get("locked")),
|
||||
}
|
||||
for item in profile.get("preferences") or []
|
||||
if item.get("pref_key")
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def restore_document(profile_id: str, document: dict) -> dict:
|
||||
payload = interaction_payload(document)
|
||||
governance = (payload.get("governance") or DEFAULT_GOVERNANCE).strip()
|
||||
if governance not in GOVERNANCE:
|
||||
raise StoreError("invalid_governance", "governance muss learning, advising oder frozen sein")
|
||||
prefs = payload.get("preferences")
|
||||
if prefs is None:
|
||||
raise StoreError("invalid_document", "Interaction-Profile-Dokument braucht preferences.")
|
||||
if not isinstance(prefs, list):
|
||||
raise StoreError("invalid_document", "preferences muss eine Liste sein.")
|
||||
prepared: list[dict] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for item in prefs:
|
||||
if not isinstance(item, dict):
|
||||
raise StoreError("invalid_document", "Jede Präferenz muss ein Objekt sein.")
|
||||
scope = (item.get("scope") or "global").strip()
|
||||
pref_key = (item.get("pref_key") or "").strip()
|
||||
if scope not in SCOPES:
|
||||
raise StoreError("unknown_scope", f"Unbekannter Interaction-Scope: {scope}")
|
||||
if pref_key not in PREF_KEYS:
|
||||
raise StoreError("unknown_pref", f"Unbekannte Interaction-Präferenz: {pref_key or '—'}")
|
||||
key = (scope, pref_key)
|
||||
if key in seen:
|
||||
raise StoreError("duplicate_pref", f"Präferenz {scope}/{pref_key} ist doppelt.")
|
||||
origin = (item.get("origin") or "manual").strip()
|
||||
if origin not in ORIGIN_INTERACTION:
|
||||
origin = "manual"
|
||||
seen.add(key)
|
||||
prepared.append(
|
||||
{
|
||||
"scope": scope,
|
||||
"pref_key": pref_key,
|
||||
"value": item.get("value") or "",
|
||||
"evidence": item.get("evidence") or "aus Profil-Dokument",
|
||||
"origin": origin,
|
||||
"locked": 1 if item.get("locked") else 0,
|
||||
}
|
||||
)
|
||||
ensure_profile(profile_id)
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE interaction_profiles SET governance = ?, updated = datetime('now') WHERE profile_id = ?",
|
||||
(governance, profile_id),
|
||||
)
|
||||
conn.execute("DELETE FROM interaction_preferences WHERE profile_id = ?", (profile_id,))
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE interaction_suggestions
|
||||
SET status = 'rejected', resolved = datetime('now')
|
||||
WHERE profile_id = ? AND status = 'pending'
|
||||
""",
|
||||
(profile_id,),
|
||||
)
|
||||
for item in prepared:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO interaction_preferences
|
||||
(id, profile_id, scope, pref_key, value, evidence, origin, locked)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
str(uuid.uuid4()),
|
||||
profile_id,
|
||||
item["scope"],
|
||||
item["pref_key"],
|
||||
item["value"],
|
||||
item["evidence"],
|
||||
item["origin"],
|
||||
item["locked"],
|
||||
),
|
||||
)
|
||||
return get_profile(profile_id)
|
||||
|
||||
|
||||
def assemble_interaction_hint(profile_id: str | None, state: dict | None = None) -> str:
|
||||
"""Visible hint for the dialogue turn. Product defaults plus personal overrides plus current state."""
|
||||
prefs: dict[tuple[str, str], dict] = {}
|
||||
if profile_id:
|
||||
profile = get_profile(profile_id)
|
||||
for item in profile.get("preferences") or []:
|
||||
prefs[(item.get("scope") or "global", item.get("pref_key") or "")] = item
|
||||
lines = [
|
||||
"Interaction (sichtbar, überschreibbar; Nicht-Widersprechen ist kein Lernsignal):",
|
||||
]
|
||||
for item in DEFAULTS:
|
||||
key = (item["scope"], item["pref_key"])
|
||||
personal = prefs.get(key)
|
||||
if item["scope"] != "global":
|
||||
continue
|
||||
value = (personal.get("value") if personal else None) or item["value"]
|
||||
source = "persönlich" if personal else "Produktdefault"
|
||||
lines.append(f"- {item['title']}: {value} [{source}]")
|
||||
auto = next((item for item in DEFAULTS if item["pref_key"] == "listening"), None)
|
||||
if auto:
|
||||
personal = prefs.get(("autobiographical", "listening"))
|
||||
value = (personal.get("value") if personal else None) or auto["value"]
|
||||
source = "persönlich" if personal else "Produktdefault"
|
||||
if state and (state.get("long_story") or (state.get("narrative_mode") or "") == "chronicle"):
|
||||
lines.append(f"- {auto['title']}: {value} [{source}]")
|
||||
elif personal:
|
||||
lines.append(f"- {auto['title']}: {value} [{source}]")
|
||||
if state:
|
||||
mode = (state.get("narrative_mode") or "open").strip() or "open"
|
||||
depth = (state.get("reflection_depth") or "surface").strip() or "surface"
|
||||
intensity = (state.get("emotional_intensity") or "low").strip() or "low"
|
||||
long_story = "ja" if state.get("long_story") else "nein"
|
||||
focus = (state.get("current_focus") or "").strip()
|
||||
lines.append(
|
||||
"Aktueller Dialogue State (operativ, kein Profil, ändert sich von Zug zu Zug): "
|
||||
f"narrative_mode={mode}; reflection_depth={depth}; "
|
||||
f"emotional_intensity={intensity}; long_story={long_story}."
|
||||
)
|
||||
if focus:
|
||||
lines.append(f"current_focus={focus}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _title(scope: str, pref_key: str) -> str:
|
||||
for item in DEFAULTS:
|
||||
if item["scope"] == scope and item["pref_key"] == pref_key:
|
||||
return item["title"]
|
||||
return pref_key
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Journal entry body as Markdown. Media stays in media_assets; the body only references it.
|
||||
|
||||
Canonical embed: 
|
||||
Legacy [[media:<uuid>|caption]] is still read and normalised on save.
|
||||
|
||||
Tokens never go to the Privacy Gateway; captions may, as user wording.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
UUID = r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
|
||||
UUID_RE = re.compile(rf"^{UUID}$")
|
||||
MD_MEDIA_RE = re.compile(rf"!\[([^\]]*)\]\(kansho-media:({UUID})\)")
|
||||
LEGACY_MEDIA_RE = re.compile(rf"\[\[media:({UUID})(?:\|([^\]]*))?\]\]")
|
||||
HEADING_PREFIX = re.compile(r"^#{1,6}\s+")
|
||||
|
||||
|
||||
def clean_title(raw: str) -> str:
|
||||
"""First line only, without Markdown heading marks."""
|
||||
line = (raw or "").strip().splitlines()[0].strip() if raw else ""
|
||||
return HEADING_PREFIX.sub("", line).strip()
|
||||
|
||||
|
||||
def media_token(media_id: str, caption: str = "") -> str:
|
||||
cap = (caption or "").replace("]", "").replace("\n", " ").strip()
|
||||
return f""
|
||||
|
||||
|
||||
def media_ids(body: str) -> list[str]:
|
||||
seen: list[str] = []
|
||||
for match in MD_MEDIA_RE.finditer(body or ""):
|
||||
media_id = match.group(2)
|
||||
if media_id not in seen:
|
||||
seen.append(media_id)
|
||||
for match in LEGACY_MEDIA_RE.finditer(body or ""):
|
||||
media_id = match.group(1)
|
||||
if media_id not in seen:
|
||||
seen.append(media_id)
|
||||
return seen
|
||||
|
||||
|
||||
def to_markdown(body: str) -> str:
|
||||
"""Legacy media tokens become Markdown image syntax. Already-Markdown is unchanged."""
|
||||
|
||||
def repl(match: re.Match) -> str:
|
||||
return media_token(match.group(1), match.group(2) or "")
|
||||
|
||||
return LEGACY_MEDIA_RE.sub(repl, body or "")
|
||||
|
||||
|
||||
def plain_text(body: str) -> str:
|
||||
"""User wording only: no media URLs, no asset IDs, no Markdown markup. Captions stay."""
|
||||
text = to_markdown(body)
|
||||
|
||||
def caption(match: re.Match) -> str:
|
||||
return (match.group(1) or "").strip()
|
||||
|
||||
text = MD_MEDIA_RE.sub(caption, text)
|
||||
text = re.sub(r"<u>(.*?)</u>", r"\1", text, flags=re.I | re.S)
|
||||
text = re.sub(r"^#{1,6}\s+", "", text, flags=re.M)
|
||||
text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
|
||||
text = re.sub(r"__(.+?)__", r"\1", text)
|
||||
text = re.sub(r"(?<!\*)\*(?!\*)(.+?)(?<!\*)\*(?!\*)", r"\1", text)
|
||||
text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.M)
|
||||
text = re.sub(r"^\s*\d+\.\s+", "", text, flags=re.M)
|
||||
text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)
|
||||
text = re.sub(r"`([^`]+)`", r"\1", text)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
@@ -0,0 +1,515 @@
|
||||
"""Journal-adapter editorial policy. Not a general provenance or privacy rule.
|
||||
|
||||
Fact fidelity is not wording fidelity. Mixed prose, notes and fragments in the
|
||||
same day are the normal case and are handled in one generate call. Historical
|
||||
texts are style references, never today's facts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
from journal_body import plain_text
|
||||
from journal_generation_policy import (
|
||||
EMPTY_STYLE_CONTEXT,
|
||||
STYLE_CONTEXT_KEYS,
|
||||
normalize_style_context,
|
||||
)
|
||||
from writing_profile_schema import is_meta_style_text, recency_weight
|
||||
from writing_profile_store import (
|
||||
clip_field,
|
||||
get_profile,
|
||||
has_confirmed_profile,
|
||||
list_style_sources,
|
||||
)
|
||||
|
||||
STYLE_EXAMPLE_MAX = 2
|
||||
STYLE_EXAMPLE_CHARS = 900
|
||||
MIN_EXAMPLE_CHARS = 40
|
||||
|
||||
GENERATE_SEED_REVISION = "2026-08-29-voice-legacy-immutable-v1"
|
||||
|
||||
EMPTY_STYLE_EXAMPLES = (
|
||||
"Keine historischen Stilbeispiele. Es gilt nur WRITING_PROFILE "
|
||||
"(oder der neutrale Journalstil)."
|
||||
)
|
||||
|
||||
|
||||
def lexical_similarity(left: str, right: str) -> float:
|
||||
"""Diagnostic only. Must not reject a draft or trigger a retry."""
|
||||
a = re.sub(r"\s+", " ", (left or "").strip().lower())
|
||||
b = re.sub(r"\s+", " ", (right or "").strip().lower())
|
||||
if not a or not b:
|
||||
return 0.0
|
||||
return round(SequenceMatcher(None, a, b).ratio(), 3)
|
||||
|
||||
|
||||
_DANGLING_DETERMINER = re.compile(
|
||||
r"(?i)\b(?:der|die|das|des|dem|den|ein|eine|einem|einen|einer)\s*$"
|
||||
)
|
||||
_DETERMINER_BEFORE_FINITE = re.compile(
|
||||
r"(?i)\b(?:des|dem|den|der|die|das|ein|eine|einem|einen|einer)\s+"
|
||||
r"(?:kann|können|konnte|muss|müssen|will|wollen|soll|sollen|"
|
||||
r"ist|sind|war|waren|wird|werden|hat|haben|hatte|"
|
||||
r"geht|gehen|ging|kam|kommen|kommt|gelangen|gelangte)\b"
|
||||
)
|
||||
|
||||
|
||||
def incomplete_syntax_markers(text: str) -> int:
|
||||
"""Diagnostic count of dangling determiners or unpunctuated long clauses.
|
||||
|
||||
Does not rewrite text and must not trigger a retry.
|
||||
"""
|
||||
body = (text or "").strip()
|
||||
if not body:
|
||||
return 0
|
||||
count = 0
|
||||
clauses = [part.strip() for part in re.split(r"(?<=[.!?])\s+|\n+", body) if part.strip()]
|
||||
for clause in clauses:
|
||||
bare = clause.rstrip(".!?…\"»'")
|
||||
if _DANGLING_DETERMINER.search(bare):
|
||||
count += 1
|
||||
if _DETERMINER_BEFORE_FINITE.search(clause):
|
||||
count += 1
|
||||
for para in re.split(r"\n\s*\n", body):
|
||||
chunk = para.strip()
|
||||
if len(chunk.split()) >= 8 and not re.search(r"[.!?]", chunk):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def style_example_diagnostics(examples: list[dict]) -> dict:
|
||||
return {
|
||||
"count": len(examples or []),
|
||||
"kinds": [item.get("kind") or "style" for item in (examples or [])],
|
||||
"chars": sum(len(item.get("excerpt") or "") for item in (examples or [])),
|
||||
}
|
||||
|
||||
|
||||
def writing_profile_trace(
|
||||
profile_id: str,
|
||||
*,
|
||||
style_context: dict | None = None,
|
||||
brief: str | None = None,
|
||||
trait_count: int | None = None,
|
||||
) -> dict:
|
||||
"""Presence metadata only. No profile text, no labels.
|
||||
|
||||
When style_context or a compiled brief is supplied, the flags describe the
|
||||
effective provider input rather than the stored profile inventory.
|
||||
"""
|
||||
from writing_profile_store import (
|
||||
NEUTRAL_JOURNAL_STYLE,
|
||||
compile_task_brief,
|
||||
get_profile,
|
||||
has_confirmed_profile,
|
||||
)
|
||||
|
||||
confirmed = has_confirmed_profile(profile_id)
|
||||
profile = get_profile(profile_id)
|
||||
context = normalize_style_context(style_context, slot="voice") if style_context is not None else None
|
||||
core = ((profile.get("core") or {}).get("value") or "").strip()
|
||||
facet = next(
|
||||
(
|
||||
item
|
||||
for item in profile.get("facets") or []
|
||||
if item.get("facet_key") == "autobiographical_journal" and (item.get("value") or "").strip()
|
||||
),
|
||||
None,
|
||||
)
|
||||
traits = [
|
||||
item
|
||||
for item in profile.get("traits") or []
|
||||
if item.get("status") == "active" and (item.get("statement") or "").strip()
|
||||
]
|
||||
if brief is None:
|
||||
brief = compile_task_brief(
|
||||
profile_id,
|
||||
"journal_generate",
|
||||
include_core=True if context is None else context["include_core"],
|
||||
include_facet=True if context is None else context["include_facet"],
|
||||
include_traits=True if context is None else context["include_traits"],
|
||||
)
|
||||
has_core = "Core:" in (brief or "")
|
||||
has_facet = "Facet-Delta:" in (brief or "") or "Autobiografisches Journaling" in (brief or "")
|
||||
if trait_count is None:
|
||||
trait_count = sum(1 for line in (brief or "").splitlines() if line.startswith("- "))
|
||||
present = bool(brief and brief != NEUTRAL_JOURNAL_STYLE)
|
||||
return {
|
||||
"confirmed": confirmed,
|
||||
"present": present,
|
||||
"neutral_fallback": brief == NEUTRAL_JOURNAL_STYLE,
|
||||
"has_core": has_core,
|
||||
"has_facet": has_facet,
|
||||
"trait_count": int(trait_count or 0),
|
||||
"brief_chars": len(brief or ""),
|
||||
"inventory_has_core": bool(confirmed and core),
|
||||
"inventory_has_facet": bool(confirmed and facet),
|
||||
"inventory_trait_count": len(traits) if confirmed else 0,
|
||||
}
|
||||
|
||||
|
||||
def narration_sources_text(artifact: dict) -> str:
|
||||
"""Present attested day facts to the model. Not a wording template, not JSON."""
|
||||
parts: list[str] = []
|
||||
for item in artifact.get("sources") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
text = (item.get("text") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
source_id = (item.get("source_id") or "").strip()
|
||||
prefix = f"[{source_id}]\n" if source_id else ""
|
||||
parts.append(prefix + text)
|
||||
if parts:
|
||||
return "\n\n".join(parts)
|
||||
from journal_reconstruct import claim_texts
|
||||
|
||||
fallback = [str(part).strip() for part in claim_texts(artifact) if str(part).strip()]
|
||||
return "\n\n".join(fallback)
|
||||
|
||||
|
||||
def format_style_examples(examples: list[dict]) -> str:
|
||||
if not examples:
|
||||
return EMPTY_STYLE_EXAMPLES
|
||||
lines = [
|
||||
"Nur Ton, Rhythmus und sprachliche Entscheidungen. "
|
||||
"Ereignisse, Personen, Orte und Bewertungen aus diesen Beispielen "
|
||||
"sind keine Tatsachen des heutigen Eintrags und dürfen nicht übernommen werden."
|
||||
]
|
||||
for index, item in enumerate(examples, start=1):
|
||||
kind = item.get("kind") or "style"
|
||||
when = (item.get("occurred_at") or "")[:10]
|
||||
header = f"Beispiel {index} ({kind}" + (f", {when}" if when else "") + "):"
|
||||
lines.append(header)
|
||||
lines.append((item.get("excerpt") or "").strip())
|
||||
return "\n".join(part for part in lines if part).strip()
|
||||
|
||||
|
||||
def wrap_style_block(title: str, body: str) -> str:
|
||||
text = (body or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
return f"{title}\n{text}\n"
|
||||
|
||||
|
||||
def _estimated_tokens(text: str) -> int:
|
||||
from prompt_budget import estimate_tokens
|
||||
|
||||
body = text or ""
|
||||
if not body:
|
||||
return 0
|
||||
return estimate_tokens(body)
|
||||
|
||||
|
||||
def compile_journal_style_application(
|
||||
profile_id: str,
|
||||
style_context: dict | None,
|
||||
*,
|
||||
exclude_dates: list[str] | None = None,
|
||||
exclude_entry_ids: list[str] | None = None,
|
||||
) -> dict:
|
||||
"""Select only the style sources released by the voice guideline.
|
||||
|
||||
Returns prompt fragments and an effective trace. Empty sections stay empty
|
||||
so the prompt builder can omit their headings.
|
||||
"""
|
||||
from writing_profile_store import (
|
||||
NEUTRAL_JOURNAL_STYLE,
|
||||
compile_task_brief,
|
||||
has_confirmed_profile,
|
||||
)
|
||||
|
||||
context = normalize_style_context(style_context or EMPTY_STYLE_CONTEXT, slot="voice")
|
||||
confirmed = has_confirmed_profile(profile_id)
|
||||
omitted: list[dict] = []
|
||||
requested = {key: bool(context.get(key)) for key in STYLE_CONTEXT_KEYS}
|
||||
|
||||
def omit(part: str, reason: str) -> None:
|
||||
omitted.append({"part": part, "reason": reason})
|
||||
|
||||
if not requested["include_core"]:
|
||||
omit("core", "not_selected")
|
||||
if not requested["include_facet"]:
|
||||
omit("facet", "not_selected")
|
||||
if not requested["include_traits"]:
|
||||
omit("traits", "not_selected")
|
||||
if not requested["include_style_examples"]:
|
||||
omit("style_examples", "not_selected")
|
||||
|
||||
wants_profile = any(requested[key] for key in ("include_core", "include_facet", "include_traits"))
|
||||
brief = compile_task_brief(
|
||||
profile_id,
|
||||
"journal_generate",
|
||||
include_core=requested["include_core"],
|
||||
include_facet=requested["include_facet"],
|
||||
include_traits=requested["include_traits"],
|
||||
)
|
||||
if wants_profile and not confirmed:
|
||||
for part, flag in (
|
||||
("core", "include_core"),
|
||||
("facet", "include_facet"),
|
||||
("traits", "include_traits"),
|
||||
):
|
||||
if requested[flag] and not any(item["part"] == part and item["reason"] == "unconfirmed" for item in omitted):
|
||||
omit(part, "unconfirmed")
|
||||
elif confirmed:
|
||||
if requested["include_core"] and "Core:" not in (brief or ""):
|
||||
omit("core", "empty")
|
||||
if requested["include_facet"] and "Facet-Delta:" not in (brief or "") and "Autobiografisches Journaling" not in (brief or ""):
|
||||
omit("facet", "empty")
|
||||
if requested["include_traits"] and not any(line.startswith("- ") for line in (brief or "").splitlines()):
|
||||
omit("traits", "empty")
|
||||
|
||||
example_rows: list[dict] = []
|
||||
examples_text = ""
|
||||
if requested["include_style_examples"]:
|
||||
example_rows = select_journal_style_examples(
|
||||
profile_id,
|
||||
exclude_dates=exclude_dates,
|
||||
exclude_entry_ids=exclude_entry_ids,
|
||||
)
|
||||
if example_rows:
|
||||
examples_text = format_style_examples(example_rows)
|
||||
else:
|
||||
omit("style_examples", "empty")
|
||||
|
||||
profile_block = wrap_style_block("WRITING_PROFILE", brief)
|
||||
examples_block = wrap_style_block("STYLE_EXAMPLES", examples_text)
|
||||
trait_count = sum(1 for line in (brief or "").splitlines() if line.startswith("- "))
|
||||
core_included = "Core:" in (brief or "")
|
||||
facet_included = "Facet-Delta:" in (brief or "") or "Autobiografisches Journaling" in (brief or "")
|
||||
return {
|
||||
"style_context": context,
|
||||
"brief": brief,
|
||||
"profile_block": profile_block,
|
||||
"style_examples": examples_text,
|
||||
"examples_block": examples_block,
|
||||
"example_rows": example_rows,
|
||||
"omitted": omitted,
|
||||
"requested": requested,
|
||||
"effective": {
|
||||
"include_core": core_included,
|
||||
"include_facet": facet_included,
|
||||
"include_traits": trait_count > 0,
|
||||
"include_style_examples": bool(example_rows),
|
||||
"trait_count": trait_count,
|
||||
"style_example_count": len(example_rows),
|
||||
"neutral_fallback": brief == NEUTRAL_JOURNAL_STYLE,
|
||||
"blocks": [
|
||||
{
|
||||
"key": "writing_profile",
|
||||
"included": bool(profile_block),
|
||||
"chars": len(profile_block),
|
||||
"estimated_tokens": _estimated_tokens(profile_block),
|
||||
},
|
||||
{
|
||||
"key": "core",
|
||||
"included": core_included,
|
||||
"chars": len(brief.split("Autobiografisches Journaling")[0]) if core_included else 0,
|
||||
"estimated_tokens": _estimated_tokens(brief) if core_included and not facet_included and trait_count == 0 else 0,
|
||||
},
|
||||
{
|
||||
"key": "facet",
|
||||
"included": facet_included,
|
||||
"chars": 0,
|
||||
"estimated_tokens": 0,
|
||||
},
|
||||
{
|
||||
"key": "traits",
|
||||
"included": trait_count > 0,
|
||||
"count": trait_count,
|
||||
"chars": 0,
|
||||
"estimated_tokens": 0,
|
||||
},
|
||||
{
|
||||
"key": "style_examples",
|
||||
"included": bool(examples_block),
|
||||
"count": len(example_rows),
|
||||
"chars": len(examples_block),
|
||||
"estimated_tokens": _estimated_tokens(examples_block),
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def style_application_trace(
|
||||
compiled_policy,
|
||||
bundle: dict,
|
||||
*,
|
||||
dropped: list[str] | None = None,
|
||||
) -> dict:
|
||||
dropped = list(dropped or [])
|
||||
omitted = [dict(item) for item in bundle.get("omitted") or []]
|
||||
effective = dict(bundle.get("effective") or {})
|
||||
example_rows = list(bundle.get("example_rows") or [])
|
||||
profile_block = bundle.get("profile_block") or ""
|
||||
examples_block = bundle.get("examples_block") or ""
|
||||
if "style_examples" in dropped:
|
||||
examples_block = ""
|
||||
example_rows = []
|
||||
effective["include_style_examples"] = False
|
||||
effective["style_example_count"] = 0
|
||||
if not any(item.get("part") == "style_examples" and item.get("reason") == "budget" for item in omitted):
|
||||
omitted.append({"part": "style_examples", "reason": "budget"})
|
||||
for block in effective.get("blocks") or []:
|
||||
if block.get("key") == "style_examples":
|
||||
block["included"] = False
|
||||
block["count"] = 0
|
||||
block["chars"] = 0
|
||||
block["estimated_tokens"] = 0
|
||||
brief = bundle.get("brief") or ""
|
||||
core_chars = 0
|
||||
facet_chars = 0
|
||||
trait_chars = 0
|
||||
if "Core:" in brief:
|
||||
core_part = brief.split("Autobiografisches Journaling")[0]
|
||||
core_chars = len(core_part.strip())
|
||||
if "Autobiografisches Journaling" in brief:
|
||||
rest = brief.split("Autobiografisches Journaling", 1)[1]
|
||||
facet_part = rest.split("\n- ", 1)[0]
|
||||
facet_chars = len(("Autobiografisches Journaling" + facet_part).strip())
|
||||
trait_lines = [line for line in brief.splitlines() if line.startswith("- ")]
|
||||
trait_chars = sum(len(line) for line in trait_lines)
|
||||
blocks = [
|
||||
{
|
||||
"key": "writing_profile",
|
||||
"included": bool(profile_block.strip()),
|
||||
"chars": len(profile_block),
|
||||
"estimated_tokens": _estimated_tokens(profile_block),
|
||||
},
|
||||
{
|
||||
"key": "core",
|
||||
"included": "Core:" in brief,
|
||||
"chars": core_chars,
|
||||
"estimated_tokens": _estimated_tokens(brief.split("\n", 1)[0] if "Core:" in brief else ""),
|
||||
},
|
||||
{
|
||||
"key": "facet",
|
||||
"included": "Autobiografisches Journaling" in brief,
|
||||
"chars": facet_chars,
|
||||
"estimated_tokens": _estimated_tokens("Autobiografisches Journaling" + brief.split("Autobiografisches Journaling", 1)[1].split("\n- ", 1)[0] if "Autobiografisches Journaling" in brief else ""),
|
||||
},
|
||||
{
|
||||
"key": "traits",
|
||||
"included": bool(trait_lines),
|
||||
"count": len(trait_lines),
|
||||
"chars": trait_chars,
|
||||
"estimated_tokens": _estimated_tokens("\n".join(trait_lines)),
|
||||
},
|
||||
{
|
||||
"key": "style_examples",
|
||||
"included": bool(examples_block.strip()),
|
||||
"count": len(example_rows),
|
||||
"chars": len(examples_block),
|
||||
"estimated_tokens": _estimated_tokens(examples_block),
|
||||
},
|
||||
]
|
||||
requested = bundle.get("requested") or dict(EMPTY_STYLE_CONTEXT)
|
||||
return {
|
||||
"id": compiled_policy.ids.get("voice") if compiled_policy else "",
|
||||
"key": compiled_policy.keys.get("voice") if compiled_policy else "",
|
||||
"label": compiled_policy.labels.get("voice") if compiled_policy else "",
|
||||
"revision": compiled_policy.revisions.get("voice") if compiled_policy else 1,
|
||||
"cloned_from": (compiled_policy.cloned_from.get("voice") if compiled_policy else "") or "",
|
||||
"requested": requested,
|
||||
"effective": {
|
||||
"include_core": any(block["key"] == "core" and block["included"] for block in blocks),
|
||||
"include_facet": any(block["key"] == "facet" and block["included"] for block in blocks),
|
||||
"trait_count": len(trait_lines),
|
||||
"style_example_count": len(example_rows),
|
||||
"neutral_fallback": bool((bundle.get("effective") or {}).get("neutral_fallback")),
|
||||
"blocks": blocks,
|
||||
},
|
||||
"omitted": omitted,
|
||||
"dropped_optional_blocks": dropped,
|
||||
}
|
||||
|
||||
|
||||
def _content_digest(text: str) -> str:
|
||||
body = re.sub(r"\s+", " ", plain_text(text or "")).strip().lower()
|
||||
return hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _eligible_excerpt(body: str, limit: int) -> str:
|
||||
text = plain_text(body or "").strip()
|
||||
if len(text) < MIN_EXAMPLE_CHARS or is_meta_style_text(text):
|
||||
return ""
|
||||
return clip_field(text, limit)
|
||||
|
||||
|
||||
def select_journal_style_examples(
|
||||
profile_id: str,
|
||||
*,
|
||||
exclude_dates: list[str] | None = None,
|
||||
exclude_entry_ids: list[str] | None = None,
|
||||
max_n: int = STYLE_EXAMPLE_MAX,
|
||||
max_chars: int = STYLE_EXAMPLE_CHARS,
|
||||
) -> list[dict]:
|
||||
"""Final accepted journal texts first, then imports, then trait excerpts.
|
||||
|
||||
Current-day sources are excluded. Dialogue is never a style authority.
|
||||
"""
|
||||
excluded_dates = {(item or "")[:10] for item in (exclude_dates or []) if item}
|
||||
excluded_ids = {item for item in (exclude_entry_ids or []) if item}
|
||||
ranked = list_style_sources(profile_id)
|
||||
buckets = [
|
||||
("journal_entry", ranked.get("journal_entry") or []),
|
||||
("imported_text", ranked.get("imported_text") or []),
|
||||
]
|
||||
picked: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def consider(kind: str, item: dict, excerpt: str) -> None:
|
||||
if len(picked) >= max_n or not excerpt:
|
||||
return
|
||||
digest = _content_digest(excerpt)
|
||||
if digest in seen:
|
||||
return
|
||||
seen.add(digest)
|
||||
picked.append(
|
||||
{
|
||||
"kind": kind,
|
||||
"excerpt": excerpt,
|
||||
"occurred_at": item.get("occurred_at") or item.get("created"),
|
||||
"entry_id": item.get("entry_id"),
|
||||
"weight": item.get("weight") or recency_weight(item.get("occurred_at")),
|
||||
}
|
||||
)
|
||||
|
||||
for kind, rows in buckets:
|
||||
ordered = sorted(
|
||||
rows,
|
||||
key=lambda row: (
|
||||
-float(row.get("weight") or 0),
|
||||
-recency_weight(row.get("occurred_at") or row.get("created")),
|
||||
),
|
||||
)
|
||||
for item in ordered:
|
||||
if len(picked) >= max_n:
|
||||
return picked
|
||||
entry_id = (item.get("entry_id") or "").strip()
|
||||
if entry_id and entry_id in excluded_ids:
|
||||
continue
|
||||
when = (item.get("occurred_at") or "")[:10]
|
||||
if when and when in excluded_dates:
|
||||
continue
|
||||
consider(kind, item, _eligible_excerpt(item.get("body") or "", max_chars))
|
||||
|
||||
if picked:
|
||||
return picked
|
||||
|
||||
profile = get_profile(profile_id)
|
||||
if not has_confirmed_profile(profile_id):
|
||||
return picked
|
||||
for trait in profile.get("traits") or []:
|
||||
if len(picked) >= max_n:
|
||||
break
|
||||
for ref in trait.get("exemplars") or []:
|
||||
excerpt = _eligible_excerpt(ref.get("excerpt") or "", max_chars)
|
||||
consider("trait_exemplar", ref, excerpt)
|
||||
if len(picked) >= max_n:
|
||||
break
|
||||
return picked
|
||||
@@ -0,0 +1,467 @@
|
||||
"""Opt-in journal quality comparison. Not part of production generate.
|
||||
|
||||
Usage from backend/:
|
||||
python journal_eval.py # synthetic, fake provider, contract only
|
||||
python journal_eval.py --live --profile-id <id>
|
||||
python journal_eval.py --profile-ab # two synthetic writing profiles, prompts only unless --live
|
||||
|
||||
Never writes private texts into the repository. Live quality stays unconfirmed
|
||||
until an explicit --live run succeeds. The harness never declares a winner.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from journal_editorial import lexical_similarity
|
||||
|
||||
VARIANT_BASELINE = "baseline_simple"
|
||||
VARIANT_PREVIOUS = "kansho_previous"
|
||||
VARIANT_CURRENT = "kansho_current"
|
||||
|
||||
BASELINE_TEMPLATE = (
|
||||
"Erstelle aus diesen Angaben einen ansprechenden persönlichen Tagebucheintrag. "
|
||||
"Bewahre alle Tatsachen und Unsicherheiten, erfinde nichts, korrigiere Sprache "
|
||||
"und schreibe im bereitgestellten persönlichen Stil.\n\n"
|
||||
"Persönlicher Stil:\n{{writing_profile}}\n\n"
|
||||
"Angaben:\n{{reconstruction}}\n"
|
||||
)
|
||||
|
||||
PREVIOUS_TEMPLATE = (
|
||||
"Schreibe einen eigenständigen, gut lesbaren Tagebucheintrag in der Ich-Form von [[SELF]].\n"
|
||||
"Verwende ausschließlich die verifizierten Informationen aus sources[].text. "
|
||||
"Der Quellwortlaut ist keine Ausgabevorlage.\n"
|
||||
"Keine Dialogabschrift. Unsicherheiten im Wortlaut erhalten.\n"
|
||||
"Writing Profile:\n{{writing_profile}}\n\n"
|
||||
"Verified Artifact:\n{{reconstruction}}\n"
|
||||
)
|
||||
|
||||
PROFILE_A = (
|
||||
"Core: kurze Sätze, trockener Schnitt, wenig Adjektive, kaum Reflexion. "
|
||||
"Wortwahl nüchtern, Rhythmus abgehackt."
|
||||
)
|
||||
PROFILE_B = (
|
||||
"Core: längere, ruhig fließende Sätze, beobachtend, leise Reflexion am Satzende. "
|
||||
"Wortwahl behutsam, Rhythmus getragen."
|
||||
)
|
||||
|
||||
FIXTURES: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "prose_typos",
|
||||
"class": "already_narrative_with_errors",
|
||||
"source": (
|
||||
"ich bin dan zum markt gegangen und da war es zimlich voll. "
|
||||
"vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. "
|
||||
"die rote tasche lag im auto."
|
||||
),
|
||||
"typos": ("zimlich", "dan zum"),
|
||||
"must_keep": ("vielleicht", "nicht gemacht", "rote tasche"),
|
||||
"must_not_invent": ("traurig", "weil"),
|
||||
},
|
||||
{
|
||||
"id": "notes_fragments",
|
||||
"class": "bullet_points_and_fragments",
|
||||
"source": "markt\nkirschen kaufen\nspäter hafen",
|
||||
"typos": (),
|
||||
"must_keep": ("markt", "kirschen", "hafen"),
|
||||
"must_not_invent": ("glücklich", "weil"),
|
||||
},
|
||||
{
|
||||
"id": "plan_vs_done",
|
||||
"class": "plan_versus_completion",
|
||||
"source": "Ich wollte um elf zum Hafen. Stattdessen blieb ich zu Hause. Den Brief habe ich nicht abgeschickt.",
|
||||
"typos": (),
|
||||
"must_keep": ("wollte", "blieb", "nicht abgeschickt"),
|
||||
"must_not_invent": ("geschickt", "bin zum hafen"),
|
||||
},
|
||||
{
|
||||
"id": "negation_uncertainty",
|
||||
"class": "negation_and_uncertainty",
|
||||
"source": "Vielleicht kommt der Techniker. Ich bin unsicher wegen des Tees. Den Kuchen habe ich nicht gebacken.",
|
||||
"typos": (),
|
||||
"must_keep": ("vielleicht", "unsicher", "nicht gebacken"),
|
||||
"must_not_invent": ("sicher", "gebacken"),
|
||||
},
|
||||
{
|
||||
"id": "correction",
|
||||
"class": "correction_of_earlier_claim",
|
||||
"source": "Zuerst dachte ich, der Markt sei um neun. Später korrigierte ich mich: er war schon um acht zu Ende.",
|
||||
"typos": (),
|
||||
"must_keep": ("zuerst", "korrigierte", "acht"),
|
||||
"must_not_invent": ("neun zu ende",),
|
||||
},
|
||||
{
|
||||
"id": "imprecise_time",
|
||||
"class": "imprecise_time",
|
||||
"source": "Gegen sechs bin ich aufgewacht. Irgendwann vorm Mittag war ich am Markt.",
|
||||
"typos": (),
|
||||
"must_keep": ("gegen sechs", "irgendwann"),
|
||||
"must_not_invent": ("genau 6:00", "12:00"),
|
||||
},
|
||||
{
|
||||
"id": "outstanding_event",
|
||||
"class": "outstanding_event_among_everyday",
|
||||
"source": (
|
||||
"gegen 6 uhr aufgewacht\n"
|
||||
"morgenroutine mit sprache, notizen und tee\n"
|
||||
"unsichere teepraeferenz\n"
|
||||
"techniker sollte um 11 uhr kommen\n"
|
||||
"kueche aufgeraeumt\n"
|
||||
"gegen 9 uhr gefruehstueckt\n"
|
||||
"kinder erst gegen 10 oder 11 uhr aufgestanden\n"
|
||||
"normaler strandtag\n"
|
||||
"ploetzlich seehunde im wasser gesehen"
|
||||
),
|
||||
"typos": ("praeferenz", "aufgeraeumt", "gefruehstueckt"),
|
||||
"must_keep": ("normaler strandtag", "seehunde", "unsichere"),
|
||||
"must_not_invent": ("gluecklich", "schicksal"),
|
||||
"weight_tokens": ("seehunde", "normaler"),
|
||||
},
|
||||
{
|
||||
"id": "recurring_people",
|
||||
"class": "recurring_people_and_projects",
|
||||
"source": (
|
||||
"Am Vormittag sprach ich mit [[PERSON:01]] über Projekt [[PROJECT:01]]. "
|
||||
"Später half [[PERSON:01]] beim Aufräumen. [[PROJECT:01]] blieb liegen."
|
||||
),
|
||||
"typos": (),
|
||||
"must_keep": ("[[PERSON:01]]", "[[PROJECT:01]]", "blieb liegen"),
|
||||
"must_not_invent": ("[[PERSON:02]]",),
|
||||
},
|
||||
{
|
||||
"id": "incomplete_clause",
|
||||
"class": "incomplete_source_rebuildable",
|
||||
"source": "ich gieng zum laden. danach sprach ich mit dem und kam zurück. es war kald.",
|
||||
"typos": ("gieng", "kald"),
|
||||
"must_keep": ("laden", "kam zurück"),
|
||||
"must_not_invent": ("nachbarn", "freund"),
|
||||
"incomplete_ok": False,
|
||||
},
|
||||
]
|
||||
|
||||
SYNTHETIC_PROSE = FIXTURES[0]["source"]
|
||||
SYNTHETIC_NOTES = FIXTURES[1]["source"]
|
||||
SYNTHETIC_TYPOS = FIXTURES[0]["typos"]
|
||||
|
||||
|
||||
def _words(text: str) -> list[str]:
|
||||
return re.findall(r"[A-Za-zÄÖÜäöüß]+", (text or "").lower())
|
||||
|
||||
|
||||
def _sentences(text: str) -> list[str]:
|
||||
parts = [item.strip() for item in re.split(r"(?<=[.!?])\s+", text or "") if item.strip()]
|
||||
return parts or ([text.strip()] if (text or "").strip() else [])
|
||||
|
||||
|
||||
def score_output(source: str, output: str, *, profile: str = "", typos: tuple[str, ...] = ()) -> dict[str, Any]:
|
||||
"""Diagnostic scores. Not a production gate and not a live-quality certificate."""
|
||||
body = output or ""
|
||||
src_words = set(_words(source))
|
||||
out_words = set(_words(body))
|
||||
sentences = _sentences(body)
|
||||
paragraphs = [item for item in re.split(r"\n\s*\n", body) if item.strip()]
|
||||
avg_len = round(sum(len(item.split()) for item in sentences) / max(1, len(sentences)), 2)
|
||||
transitions = len(re.findall(r"(?i)\b(?:danach|später|dann|zuerst|schließlich|bis|plötzlich)\b", body))
|
||||
profile_words = set(_words(profile))
|
||||
style_overlap = round(len(out_words & profile_words) / max(1, len(profile_words)), 3) if profile_words else 0.0
|
||||
kept = round(len(src_words & out_words) / max(1, len(src_words)), 3)
|
||||
extra = sorted(out_words - src_words - profile_words)
|
||||
lost = sorted(src_words - out_words)
|
||||
remaining_typos = [item for item in typos if item.lower() in body.lower()]
|
||||
from journal_editorial import incomplete_syntax_markers
|
||||
|
||||
return {
|
||||
"spelling_typos_remaining": remaining_typos,
|
||||
"spelling_typos_fixed": [item for item in typos if item.lower() not in body.lower()],
|
||||
"readability_sentence_count": len(sentences),
|
||||
"readability_avg_sentence_words": avg_len,
|
||||
"readability_paragraphs": max(1, len(paragraphs)),
|
||||
"transitions": transitions,
|
||||
"style_token_overlap": style_overlap,
|
||||
"fact_token_keep": kept,
|
||||
"new_info_tokens": extra[:24],
|
||||
"lost_info_tokens": lost[:24],
|
||||
"lexical_similarity": lexical_similarity(source, body),
|
||||
"incomplete_syntax": incomplete_syntax_markers(body),
|
||||
"unresolved_privacy_tokens": bool(re.search(r"\[\[\s*(?:…|\.{2,})\s*\]\]", body)),
|
||||
}
|
||||
|
||||
|
||||
def variant_templates() -> dict[str, str]:
|
||||
from engine import load_active_prompt
|
||||
|
||||
current = (load_active_prompt("mvp.journal_generate") or {}).get("template") or ""
|
||||
return {
|
||||
VARIANT_BASELINE: BASELINE_TEMPLATE,
|
||||
VARIANT_PREVIOUS: PREVIOUS_TEMPLATE,
|
||||
VARIANT_CURRENT: current,
|
||||
}
|
||||
|
||||
|
||||
def fixture_context(source: str, *, writing_profile: str | None = None) -> dict[str, str]:
|
||||
artifact = {
|
||||
"kind": "verified_artifact",
|
||||
"coverage": "all_selected_sources",
|
||||
"sources": [{"source_id": "u1", "role": "user", "text": source}],
|
||||
}
|
||||
from journal_editorial import narration_sources_text
|
||||
from journal_generation_policy import compile_selection, default_selection_ids
|
||||
from writing_profile_store import NEUTRAL_JOURNAL_STYLE
|
||||
|
||||
compiled = compile_selection(default_selection_ids())
|
||||
return {
|
||||
"reconstruction": narration_sources_text(artifact) or source,
|
||||
"writing_profile": writing_profile or NEUTRAL_JOURNAL_STYLE,
|
||||
"style_examples": "Keine historischen Stilbeispiele.",
|
||||
**compiled.instructions,
|
||||
"existing_text": "",
|
||||
"space_title": "Eval",
|
||||
}
|
||||
|
||||
|
||||
def synthetic_context(source: str) -> dict[str, str]:
|
||||
return fixture_context(source)
|
||||
|
||||
|
||||
def run_variant(
|
||||
name: str,
|
||||
template: str,
|
||||
context: dict[str, str],
|
||||
profile_id: str,
|
||||
*,
|
||||
live: bool,
|
||||
) -> dict[str, Any]:
|
||||
from engine import execute_prompt
|
||||
from model_catalog import resolve_generate_metadata
|
||||
from prompt_budget import plan_journal_budget
|
||||
from providers import generate_provider
|
||||
|
||||
prompt = {
|
||||
"id": f"eval-{name}",
|
||||
"slug": "mvp.journal_generate" if name == VARIANT_CURRENT else f"eval.{name}",
|
||||
"seed_revision": "eval",
|
||||
"prompt_type": "base",
|
||||
"required_feature": "ai_calls",
|
||||
"template": template,
|
||||
}
|
||||
started = time.perf_counter()
|
||||
config = generate_provider()
|
||||
window = resolve_generate_metadata(config) if config else None
|
||||
budget = plan_journal_budget(window, purpose="journal_generate") if window else None
|
||||
result = execute_prompt(
|
||||
prompt,
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
data_class="B",
|
||||
context=context,
|
||||
max_tokens=budget.reserved_output_tokens if budget else 1024,
|
||||
disable_context_compression=True,
|
||||
budget=budget,
|
||||
)
|
||||
elapsed_ms = int((time.perf_counter() - started) * 1000)
|
||||
diag = result.get("diagnostics") or {}
|
||||
return {
|
||||
"variant": name,
|
||||
"live": live,
|
||||
"content": result.get("content") or "",
|
||||
"prompt_tokens": diag.get("prompt_tokens"),
|
||||
"completion_tokens": diag.get("completion_tokens"),
|
||||
"total_tokens": diag.get("total_tokens"),
|
||||
"cost": diag.get("cost"),
|
||||
"runtime_ms": elapsed_ms,
|
||||
"generate_ms": diag.get("generate_ms"),
|
||||
"model": (result.get("trace") or {}).get("model") or diag.get("actual_model"),
|
||||
"prompt_revision": diag.get("prompt_revision"),
|
||||
"fake_provider": not live,
|
||||
"rendered_contains_profile": (context.get("writing_profile") or "")[:40]
|
||||
in ((result.get("trace") or {}).get("intern") or ""),
|
||||
}
|
||||
|
||||
|
||||
def _offline_row(name: str, template: str, context: dict[str, str]) -> dict[str, Any]:
|
||||
from placeholders import resolve_template
|
||||
from privacy_gateway import _fake_complete
|
||||
|
||||
rendered = resolve_template(template, context)
|
||||
started = time.perf_counter()
|
||||
content = _fake_complete("journal_generate", rendered)
|
||||
return {
|
||||
"variant": name,
|
||||
"live": False,
|
||||
"content": content,
|
||||
"prompt_tokens": None,
|
||||
"completion_tokens": None,
|
||||
"total_tokens": None,
|
||||
"cost": None,
|
||||
"runtime_ms": int((time.perf_counter() - started) * 1000),
|
||||
"model": "fake",
|
||||
"fake_provider": True,
|
||||
"rendered": rendered,
|
||||
"rendered_contains_profile": (context.get("writing_profile") or "")[:24] in rendered,
|
||||
}
|
||||
|
||||
|
||||
def _human_blind_pair(left: dict, right: dict) -> dict[str, Any]:
|
||||
"""Side-by-side without naming a winner. Mapping stays in the machine report."""
|
||||
return {
|
||||
"prompt_1": {"label": "Prompt 1", "text": left.get("content") or ""},
|
||||
"prompt_2": {"label": "Prompt 2", "text": right.get("content") or ""},
|
||||
"hidden_mapping": {
|
||||
"prompt_1": left.get("variant"),
|
||||
"prompt_2": right.get("variant"),
|
||||
},
|
||||
"instruction": (
|
||||
"Blind bewerten: Faktentreue, Unsicherheit, Sprache, Lesefluss, "
|
||||
"Gewichtung, Profiltreue, Eigenständigkeit. Keine automatische Siegeraussage."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def compare_synthetic(
|
||||
*,
|
||||
live: bool = False,
|
||||
profile_id: str | None = None,
|
||||
profile_ab: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
import placeholder_mvp # noqa: F401
|
||||
|
||||
templates = variant_templates()
|
||||
fixtures_out = []
|
||||
for fixture in FIXTURES:
|
||||
context = fixture_context(fixture["source"])
|
||||
rows = []
|
||||
for name, template in templates.items():
|
||||
if live:
|
||||
if not profile_id:
|
||||
raise SystemExit("--live requires --profile-id")
|
||||
row = run_variant(name, template, context, profile_id=profile_id, live=True)
|
||||
else:
|
||||
row = _offline_row(name, template, context)
|
||||
row["scores"] = score_output(
|
||||
fixture["source"],
|
||||
row["content"],
|
||||
profile=context.get("writing_profile") or "",
|
||||
typos=tuple(fixture.get("typos") or ()),
|
||||
)
|
||||
rows.append(row)
|
||||
current = next((item for item in rows if item["variant"] == VARIANT_CURRENT), rows[-1])
|
||||
baseline = next((item for item in rows if item["variant"] == VARIANT_BASELINE), rows[0])
|
||||
fixtures_out.append(
|
||||
{
|
||||
"id": fixture["id"],
|
||||
"class": fixture["class"],
|
||||
"source": fixture["source"],
|
||||
"variants": rows,
|
||||
"human_blind": _human_blind_pair(current, baseline),
|
||||
}
|
||||
)
|
||||
|
||||
profile_ab_report = None
|
||||
if profile_ab:
|
||||
source = next(item["source"] for item in FIXTURES if item["id"] == "outstanding_event")
|
||||
ctx_a = fixture_context(source, writing_profile=PROFILE_A)
|
||||
ctx_b = fixture_context(source, writing_profile=PROFILE_B)
|
||||
current_template = templates[VARIANT_CURRENT]
|
||||
if live:
|
||||
if not profile_id:
|
||||
raise SystemExit("--live requires --profile-id")
|
||||
row_a = run_variant("profile_a", current_template, ctx_a, profile_id=profile_id, live=True)
|
||||
row_b = run_variant("profile_b", current_template, ctx_b, profile_id=profile_id, live=True)
|
||||
else:
|
||||
row_a = _offline_row("profile_a", current_template, ctx_a)
|
||||
row_b = _offline_row("profile_b", current_template, ctx_b)
|
||||
rendered_a = row_a.get("rendered") or ""
|
||||
rendered_b = row_b.get("rendered") or ""
|
||||
profile_ab_report = {
|
||||
"same_facts": ctx_a["reconstruction"] == ctx_b["reconstruction"],
|
||||
"prompts_differ": (PROFILE_A in rendered_a and PROFILE_B in rendered_b and PROFILE_A not in rendered_b)
|
||||
if not live
|
||||
else row_a.get("content") != row_b.get("content"),
|
||||
"profile_a_in_prompt": PROFILE_A[:20] in rendered_a if not live else None,
|
||||
"profile_b_in_prompt": PROFILE_B[:20] in rendered_b if not live else None,
|
||||
"human_blind": _human_blind_pair(row_a, row_b),
|
||||
"note": (
|
||||
"Offline: beweist verschiedene Stilvorgaben im gerenderten Prompt, nicht Live-Prosa."
|
||||
if not live
|
||||
else "Live-A/B über das Privacy Gateway. Ob Rhythmus und Wortwahl divergieren, bewertet ein Mensch."
|
||||
),
|
||||
}
|
||||
|
||||
return {
|
||||
"live": live,
|
||||
"live_quality_confirmed": False,
|
||||
"winner_declared": False,
|
||||
"note": (
|
||||
"Fake-Provider-Lauf: beweist den Vergleichsvertrag, nicht echte Modellprosa."
|
||||
if not live
|
||||
else "Live-Lauf über das Privacy Gateway. Qualitative Bewertung bleibt manuell. Keine automatische Siegeraussage."
|
||||
),
|
||||
"source_kind": "synthetic",
|
||||
"fixtures": fixtures_out,
|
||||
"profile_ab": profile_ab_report,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Journal narration quality comparison (opt-in).")
|
||||
parser.add_argument("--live", action="store_true", help="Call the configured generate provider. Costs money.")
|
||||
parser.add_argument("--profile-id", help="Required with --live. Uses the Privacy Gateway.")
|
||||
parser.add_argument("--profile-ab", action="store_true", help="Compare two synthetic writing profiles.")
|
||||
parser.add_argument("--json", action="store_true", help="Print JSON instead of text.")
|
||||
args = parser.parse_args(argv)
|
||||
if args.live:
|
||||
import os
|
||||
|
||||
if (os.environ.get("KANSHO_FAKE_PROVIDER") or "").strip() in {"1", "true", "yes"}:
|
||||
print("Refusing --live while KANSHO_FAKE_PROVIDER is set.", file=sys.stderr)
|
||||
return 2
|
||||
if not args.profile_id:
|
||||
print("--live requires --profile-id", file=sys.stderr)
|
||||
return 2
|
||||
else:
|
||||
import os
|
||||
|
||||
os.environ.setdefault("KANSHO_FAKE_PROVIDER", "1")
|
||||
os.environ.setdefault("KANSHO_FAKE_DETECT", "1")
|
||||
from db import init_db
|
||||
|
||||
init_db()
|
||||
report = compare_synthetic(live=bool(args.live), profile_id=args.profile_id, profile_ab=bool(args.profile_ab))
|
||||
if args.json:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
print(report["note"])
|
||||
if not report["live"]:
|
||||
print("Live-Qualität: noch nicht bestätigt. Keine Siegeraussage.")
|
||||
for fixture in report["fixtures"]:
|
||||
print(f"\n[{fixture['id']} / {fixture['class']}]")
|
||||
for item in fixture["variants"]:
|
||||
scores = item["scores"]
|
||||
print(
|
||||
f" {item['variant']}: similarity={scores['lexical_similarity']} "
|
||||
f"keep={scores['fact_token_keep']} typos_left={scores['spelling_typos_remaining']} "
|
||||
f"incomplete={scores['incomplete_syntax']} tokens={item.get('total_tokens')} "
|
||||
f"cost={item.get('cost')} ms={item['runtime_ms']}"
|
||||
)
|
||||
blind = fixture["human_blind"]
|
||||
print(" Blindvergleich: Prompt 1 vs Prompt 2 (Mapping nur im JSON).")
|
||||
print(f" {blind['instruction']}")
|
||||
if report.get("profile_ab"):
|
||||
ab = report["profile_ab"]
|
||||
print("\n[profile_ab]")
|
||||
print(f" same_facts={ab['same_facts']} prompts_differ={ab['prompts_differ']}")
|
||||
print(f" {ab['note']}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,827 @@
|
||||
"""Explicit journal draft generation. Never overwrites the current user entry."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from detect_learning import (
|
||||
apply_review_decisions,
|
||||
drop_pending,
|
||||
is_bypass_mode,
|
||||
load_pending,
|
||||
save_pending,
|
||||
)
|
||||
from detect_pipeline import KIND_CONFIRM_REMOTE, apply_remote_confirm, prepare_detect_pipeline
|
||||
from dialogue_store import StoreError, list_conversations_for_day, list_messages
|
||||
from engine import EngineError, execute_prompt, load_active_prompt, preview_prompt
|
||||
from entity_detect import DetectError
|
||||
from journal_policy import require_explicit_generate, source_conversation_ids
|
||||
from identity_store import is_maskable_label, list_mappings, mapping_spellings
|
||||
from journal_body import clean_title
|
||||
from journal_editorial import (
|
||||
compile_journal_style_application,
|
||||
incomplete_syntax_markers,
|
||||
lexical_similarity,
|
||||
narration_sources_text,
|
||||
style_application_trace,
|
||||
style_example_diagnostics,
|
||||
writing_profile_trace,
|
||||
)
|
||||
from journal_generation_policy import (
|
||||
assert_journal_prompt_contract,
|
||||
assert_template_resolved,
|
||||
compile_selection,
|
||||
draft_snapshot,
|
||||
mark_guidelines_used,
|
||||
policy_trace,
|
||||
prompt_without_orphaned_style_example_rule,
|
||||
resolve_run_selection,
|
||||
)
|
||||
from journal_reconstruct import (
|
||||
assign_source_ids,
|
||||
claim_texts,
|
||||
local_verified_artifact,
|
||||
reconstruction_text,
|
||||
)
|
||||
from journal_shape import shape_journal
|
||||
from journal_store import current_draft, current_entries, get_day, insert_draft
|
||||
from model_catalog import resolve_generate_metadata
|
||||
from prompt_budget import (
|
||||
JournalBudgetError,
|
||||
assert_input_fits,
|
||||
plan_journal_budget,
|
||||
)
|
||||
from privacy_gateway import (
|
||||
GENERIC_PLACEHOLDER_INNER,
|
||||
canonical_token,
|
||||
compact_diagnostics,
|
||||
identity_label_pattern,
|
||||
identity_occurrence_count,
|
||||
is_identity_mention,
|
||||
)
|
||||
from providers import generate_provider
|
||||
from retrieval import retrieve
|
||||
from writing_profile_store import remember_dialogue_style
|
||||
|
||||
JOURNAL_NOT_ACCEPTED = "journal_generation_not_accepted"
|
||||
JOURNAL_NOT_ACCEPTED_MESSAGE = "Generierung nicht übernommen."
|
||||
|
||||
|
||||
IDENTITY_PLACEHOLDER = re.compile(
|
||||
r"\[\[\s*(?:SELF|PERSON:[^\]]+|PLACE:[^\]]+|ORG:[^\]]+|PROJECT:[^\]]+|…|\.{2,})\s*\]\]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _message_ids(profile_id: str, conversation_ids: list[str]) -> list[str]:
|
||||
ids: list[str] = []
|
||||
for conversation_id in conversation_ids:
|
||||
for message in list_messages(profile_id, conversation_id):
|
||||
ids.append(message["id"])
|
||||
return ids
|
||||
|
||||
|
||||
def _split_title(content: str) -> tuple[str, str]:
|
||||
text = (content or "").strip()
|
||||
if not text:
|
||||
return "", ""
|
||||
lines = text.splitlines()
|
||||
title = clean_title(lines[0])
|
||||
body = "\n".join(lines[1:]).strip() if len(lines) > 1 else text
|
||||
if not title or len(title) > 80:
|
||||
return "", text
|
||||
return title, body or text
|
||||
|
||||
|
||||
def _local_narration(reconstruction: dict) -> tuple[str, str]:
|
||||
"""Local source preview only. Never stored as a generated journal draft."""
|
||||
parts = [str(part).strip() for part in claim_texts(reconstruction) if str(part).strip()]
|
||||
body = "\n\n".join(parts)
|
||||
return "Ein Tag", body
|
||||
|
||||
|
||||
def unattested_journal_content(
|
||||
text: str,
|
||||
source_user: list[str],
|
||||
mappings: list[dict],
|
||||
active_tokens: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Journal-only: historical names not in the sources are unattested facts, not a privacy leak.
|
||||
|
||||
Title and body are checked together. Placeholders must stay consistent across both.
|
||||
Attestation is token-based: any attested spelling of a token (canonical or alias)
|
||||
allows every known spelling of that same token after local demasking.
|
||||
"""
|
||||
body = text or ""
|
||||
allowed = {canonical_token(token).upper().replace(" ", "") for token in (active_tokens or [])}
|
||||
allowed.add("SELF")
|
||||
for match in IDENTITY_PLACEHOLDER.finditer(body):
|
||||
token = canonical_token(match.group(0)).upper().replace(" ", "")
|
||||
if token == "SELF":
|
||||
continue
|
||||
if token in GENERIC_PLACEHOLDER_INNER or token not in allowed:
|
||||
return "unattested_placeholder"
|
||||
sources = "\n".join(source_user or [])
|
||||
spellings_by_token: dict[str, list[str]] = {}
|
||||
for item in mappings or []:
|
||||
token = canonical_token(item.get("token") or "").upper().replace(" ", "")
|
||||
if not token:
|
||||
continue
|
||||
bucket = spellings_by_token.setdefault(token, [])
|
||||
seen = {label.casefold() for label in bucket}
|
||||
for label in mapping_spellings(item):
|
||||
if not label or label.casefold() in seen:
|
||||
continue
|
||||
seen.add(label.casefold())
|
||||
bucket.append(label)
|
||||
attested: set[str] = set()
|
||||
for token, labels in spellings_by_token.items():
|
||||
for label in labels:
|
||||
if identity_occurrence_count(sources, label, token) > 0:
|
||||
attested.add(token)
|
||||
break
|
||||
for token, labels in spellings_by_token.items():
|
||||
if token in attested:
|
||||
continue
|
||||
for label in labels:
|
||||
if not is_maskable_label(label):
|
||||
continue
|
||||
for match in identity_label_pattern(label).finditer(body):
|
||||
if is_identity_mention(body, match.start(), match.end(), token):
|
||||
return "unattested_identity"
|
||||
return None
|
||||
|
||||
|
||||
def _reject_generation(
|
||||
*,
|
||||
reason: str,
|
||||
diagnostics: dict | None = None,
|
||||
source_preview: str = "",
|
||||
extra: dict | None = None,
|
||||
) -> None:
|
||||
diag = dict(diagnostics or {})
|
||||
trace = dict(diag.get("trace") or {})
|
||||
extra = dict(extra or {})
|
||||
trace.update(
|
||||
{
|
||||
"purpose": "journal_generate",
|
||||
"prompt_slug": extra.get("prompt_slug") or trace.get("prompt_slug") or "mvp.journal_generate",
|
||||
"model_text_accepted": False,
|
||||
"narration_source": "not_accepted",
|
||||
"abort_reason": reason,
|
||||
"source_preview": source_preview,
|
||||
**extra,
|
||||
}
|
||||
)
|
||||
diag["trace"] = trace
|
||||
diag["log"] = extra.get("log") or diag.get("log") or trace.get("log") or []
|
||||
diag["model_text_accepted"] = False
|
||||
diag["abort_reason"] = reason
|
||||
diag["provenance_decision"] = extra.get("provenance_decision") or diag.get("provenance_decision")
|
||||
raise EngineError(
|
||||
JOURNAL_NOT_ACCEPTED,
|
||||
JOURNAL_NOT_ACCEPTED_MESSAGE,
|
||||
409,
|
||||
diagnostics=diag,
|
||||
)
|
||||
|
||||
|
||||
def _compose_journal_trace(
|
||||
reconstruct_result: dict,
|
||||
narrate_result: dict,
|
||||
*,
|
||||
run_log: list[dict],
|
||||
narrate_prompt: dict,
|
||||
profile_meta: dict,
|
||||
style_meta: dict,
|
||||
dropped: list[str],
|
||||
extra: dict | None = None,
|
||||
) -> dict:
|
||||
reconstruct_trace = _stage_trace(reconstruct_result, "local_source_artifact")
|
||||
reconstruct_trace.setdefault("purpose", "local_source_artifact")
|
||||
reconstruct_trace.setdefault("status", "local_ok")
|
||||
reconstruct_trace.setdefault("stage1", "local_ok")
|
||||
narrate_trace = _stage_trace(narrate_result, "journal_generate")
|
||||
narrate_trace.setdefault("prompt_slug", narrate_prompt.get("slug") or "mvp.journal_generate")
|
||||
narrate_trace["prompt_revision"] = (
|
||||
narrate_trace.get("prompt_revision")
|
||||
or narrate_prompt.get("seed_revision")
|
||||
or ""
|
||||
)
|
||||
narrate_trace["writing_profile"] = profile_meta
|
||||
narrate_trace["style_examples"] = {**style_meta, "dropped": "style_examples" in dropped}
|
||||
narrate_trace["dropped_optional_blocks"] = dropped
|
||||
if extra and extra.get("style_application"):
|
||||
narrate_trace["style_application"] = extra["style_application"]
|
||||
diag = narrate_result.get("diagnostics") or {}
|
||||
if diag.get("generate_ms") is not None:
|
||||
narrate_trace["generate_ms"] = diag.get("generate_ms")
|
||||
if diag.get("prompt_revision"):
|
||||
narrate_trace["prompt_revision"] = diag.get("prompt_revision")
|
||||
if diag.get("generate_calls") is not None:
|
||||
narrate_trace.setdefault("generate_calls", diag.get("generate_calls"))
|
||||
if diag.get("model"):
|
||||
narrate_trace.setdefault("model", diag.get("model"))
|
||||
if diag.get("provider"):
|
||||
narrate_trace.setdefault("provider", diag.get("provider"))
|
||||
if diag.get("completion_tokens") is not None:
|
||||
narrate_trace.setdefault("completion_tokens", diag.get("completion_tokens"))
|
||||
if extra:
|
||||
narrate_trace.update(extra)
|
||||
return {
|
||||
**narrate_trace,
|
||||
"prompt_revision": narrate_trace.get("prompt_revision"),
|
||||
"writing_profile": profile_meta,
|
||||
"style_examples": narrate_trace.get("style_examples"),
|
||||
"style_application": narrate_trace.get("style_application") or (extra or {}).get("style_application"),
|
||||
"dropped_optional_blocks": dropped,
|
||||
"log": [dict(item) if isinstance(item, dict) else item for item in run_log],
|
||||
"stages": [reconstruct_trace, narrate_trace],
|
||||
}
|
||||
|
||||
|
||||
def _log_items(source: dict | BaseException | None) -> list[dict]:
|
||||
if source is None:
|
||||
return []
|
||||
if isinstance(source, BaseException):
|
||||
diag = getattr(source, "diagnostics", None) or {}
|
||||
return list(diag.get("log") or [])
|
||||
if not isinstance(source, dict):
|
||||
return []
|
||||
if source.get("log"):
|
||||
return list(source["log"])
|
||||
trace = source.get("trace") or {}
|
||||
if trace.get("log"):
|
||||
return list(trace["log"])
|
||||
diag = source.get("diagnostics") or {}
|
||||
return list(diag.get("log") or [])
|
||||
|
||||
|
||||
def _stamp_log(run_log: list[dict], stage: str, items: list[dict]) -> None:
|
||||
for item in items:
|
||||
row = dict(item)
|
||||
row.setdefault("stage", stage)
|
||||
run_log.append(row)
|
||||
|
||||
|
||||
def _event(run_log: list[dict], stage: str, kind: str, **fields) -> None:
|
||||
row = {"stage": stage, "kind": kind}
|
||||
for key, value in fields.items():
|
||||
if value is not None and value != "":
|
||||
row[key] = value
|
||||
run_log.append(row)
|
||||
|
||||
|
||||
def _raise_budget(exc: JournalBudgetError) -> None:
|
||||
raise EngineError(exc.code, exc.message, exc.status_code, exc.diagnostics) from exc
|
||||
|
||||
|
||||
def _stage_trace(result: dict, fallback_purpose: str) -> dict:
|
||||
source = result.get("trace")
|
||||
trace = dict(source) if isinstance(source, dict) else {}
|
||||
budget_src = result.get("diagnostics")
|
||||
if not isinstance(budget_src, dict):
|
||||
budget_src = trace.get("budget") if isinstance(trace.get("budget"), dict) else None
|
||||
if isinstance(budget_src, dict):
|
||||
trace["budget"] = compact_diagnostics(budget_src)
|
||||
else:
|
||||
trace["budget"] = None
|
||||
if "log" in trace and isinstance(trace["log"], list):
|
||||
trace["log"] = [dict(item) if isinstance(item, dict) else item for item in trace["log"]]
|
||||
trace["purpose"] = trace.get("purpose") or fallback_purpose
|
||||
return trace
|
||||
|
||||
|
||||
def _local_stage_trace(artifact: dict, *, source_count: int) -> dict:
|
||||
return {
|
||||
"purpose": "local_source_artifact",
|
||||
"layer": "journalquellen",
|
||||
"provider": None,
|
||||
"model": None,
|
||||
"coverage": artifact.get("coverage") or "all_selected_sources",
|
||||
"status": "local_ok",
|
||||
"stage1": "local_ok",
|
||||
"source_count": source_count,
|
||||
"intern": reconstruction_text(artifact),
|
||||
"budget": {
|
||||
"purpose": "local_source_artifact",
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
"cost": 0,
|
||||
"budget_ok": True,
|
||||
"status": "local",
|
||||
"context_compression": "not_applicable",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _existing_text(profile_id: str, journal_day_id: str) -> str:
|
||||
"""Saved user entries are the Fassung. A leftover draft is only used if none exist."""
|
||||
entries = current_entries(profile_id, journal_day_id)
|
||||
bodies = [(item.get("body") or "").strip() for item in entries if (item.get("body") or "").strip()]
|
||||
if bodies:
|
||||
return "\n\n".join(bodies)
|
||||
draft = current_draft(profile_id, journal_day_id)
|
||||
if draft:
|
||||
return (draft.get("body") or "").strip()
|
||||
return ""
|
||||
|
||||
|
||||
def pack_narration_context(
|
||||
prompt: dict,
|
||||
budget,
|
||||
assembled: dict[str, str],
|
||||
*,
|
||||
style_examples: str,
|
||||
existing_text: str,
|
||||
include_existing: bool,
|
||||
) -> tuple[dict[str, str], list[str]]:
|
||||
"""Drop optional blocks locally if they overflow. Never drop day sources or the profile.
|
||||
|
||||
No extra model call. Order: try all, then drop style examples, then existing text.
|
||||
"""
|
||||
dropped: list[str] = []
|
||||
attempts = [
|
||||
(style_examples, existing_text if include_existing else ""),
|
||||
("", existing_text if include_existing else ""),
|
||||
("", ""),
|
||||
]
|
||||
last_error: JournalBudgetError | None = None
|
||||
for examples, existing in attempts:
|
||||
candidate = dict(assembled)
|
||||
candidate["style_examples"] = examples
|
||||
candidate["existing_text"] = existing
|
||||
fit_prompt = prompt_without_orphaned_style_example_rule(
|
||||
prompt,
|
||||
include_style_examples=bool((examples or "").strip()),
|
||||
)
|
||||
rendered = preview_prompt(fit_prompt, candidate)["rendered"]
|
||||
try:
|
||||
assert_input_fits(budget, rendered)
|
||||
if not examples and style_examples:
|
||||
dropped.append("style_examples")
|
||||
if include_existing and existing_text and not existing:
|
||||
dropped.append("existing_text")
|
||||
return candidate, dropped
|
||||
except JournalBudgetError as exc:
|
||||
last_error = exc
|
||||
continue
|
||||
if last_error:
|
||||
raise last_error
|
||||
return assembled, dropped
|
||||
|
||||
|
||||
def generate_draft(
|
||||
profile_id: str,
|
||||
journal_day_id: str,
|
||||
conversation_ids: list[str] | None = None,
|
||||
include_existing: bool = False,
|
||||
explicit: bool = True,
|
||||
generation_selection: dict | None = None,
|
||||
remember_generation_selection: bool = False,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
) -> dict:
|
||||
try:
|
||||
draft = _generate_draft(
|
||||
profile_id,
|
||||
journal_day_id,
|
||||
conversation_ids=conversation_ids,
|
||||
include_existing=include_existing,
|
||||
explicit=explicit,
|
||||
generation_selection=generation_selection,
|
||||
remember_generation_selection=remember_generation_selection,
|
||||
precomputed_mappings=precomputed_mappings,
|
||||
)
|
||||
except EngineError as exc:
|
||||
persist_engine_error(
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
exc=exc,
|
||||
subject_type="journal_day",
|
||||
subject_id=journal_day_id,
|
||||
journal_day_id=journal_day_id,
|
||||
extra={"conversation_ids": conversation_ids or []},
|
||||
)
|
||||
raise
|
||||
if draft.get("pending_mask_review"):
|
||||
return draft
|
||||
persist_step(
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
status="ok",
|
||||
subject_type="journal_day",
|
||||
subject_id=journal_day_id,
|
||||
journal_day_id=journal_day_id,
|
||||
conversation_id=(conversation_ids[0] if conversation_ids else None),
|
||||
decision=None,
|
||||
trace=draft.get("trace"),
|
||||
extra={
|
||||
"draft_id": draft.get("id"),
|
||||
"run_log": draft.get("run_log"),
|
||||
"conversation_ids": conversation_ids or [],
|
||||
"stored_title": draft.get("title"),
|
||||
"stored_body": draft.get("body"),
|
||||
},
|
||||
)
|
||||
return draft
|
||||
|
||||
|
||||
def _generate_draft(
|
||||
profile_id: str,
|
||||
journal_day_id: str,
|
||||
conversation_ids: list[str] | None = None,
|
||||
include_existing: bool = False,
|
||||
explicit: bool = True,
|
||||
generation_selection: dict | None = None,
|
||||
remember_generation_selection: bool = False,
|
||||
precomputed_mappings: list[dict] | None = None,
|
||||
) -> dict:
|
||||
require_explicit_generate(explicit)
|
||||
day = get_day(profile_id, journal_day_id)
|
||||
day_conversations = list_conversations_for_day(profile_id, journal_day_id)
|
||||
selected = source_conversation_ids(
|
||||
conversation_ids,
|
||||
[item["id"] for item in day_conversations],
|
||||
)
|
||||
remember_dialogue_style(profile_id, exclude_conversation_ids=selected)
|
||||
existing_text = _existing_text(profile_id, journal_day_id) if include_existing else ""
|
||||
selection_ids, persist_meta = resolve_run_selection(
|
||||
profile_id,
|
||||
generation_selection,
|
||||
remember_generation_selection,
|
||||
)
|
||||
narrate_prompt = load_active_prompt("mvp.journal_generate")
|
||||
assert_journal_prompt_contract(narrate_prompt.get("template") or "")
|
||||
|
||||
config = generate_provider()
|
||||
if not config:
|
||||
raise EngineError(
|
||||
"no_egress_provider_configured",
|
||||
"Persönlicher KI-Aufruf wurde vom Privacy Gateway blockiert. "
|
||||
"Es ist kein Egress-Provider konfiguriert.",
|
||||
503,
|
||||
)
|
||||
try:
|
||||
window = resolve_generate_metadata(config)
|
||||
narrate_budget = plan_journal_budget(window, purpose="journal_generate")
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
|
||||
try:
|
||||
source_messages = retrieve(
|
||||
profile_id,
|
||||
{
|
||||
"kind": "day_messages",
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_ids": selected,
|
||||
"overflow": "abort",
|
||||
},
|
||||
)
|
||||
reconstruction = local_verified_artifact(source_messages)
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
source_user = [
|
||||
message.get("body") or ""
|
||||
for message in assign_source_ids(source_messages)
|
||||
if message.get("role") == "user" and (message.get("body") or "").strip()
|
||||
]
|
||||
compiled_policy = compile_selection(selection_ids)
|
||||
policy_meta = policy_trace(
|
||||
compiled_policy,
|
||||
source=persist_meta["source"],
|
||||
remembered=persist_meta["remembered"],
|
||||
)
|
||||
style_bundle = compile_journal_style_application(
|
||||
profile_id,
|
||||
compiled_policy.style_context,
|
||||
exclude_dates=[day.get("calendar_date") or ""],
|
||||
)
|
||||
profile_meta = writing_profile_trace(
|
||||
profile_id,
|
||||
style_context=compiled_policy.style_context,
|
||||
brief=style_bundle.get("brief") or "",
|
||||
trait_count=(style_bundle.get("effective") or {}).get("trait_count"),
|
||||
)
|
||||
style_example_rows = list(style_bundle.get("example_rows") or [])
|
||||
style_examples = style_bundle.get("examples_block") or ""
|
||||
writing_profile = style_bundle.get("profile_block") or ""
|
||||
style_meta = style_example_diagnostics(style_example_rows)
|
||||
run_log: list[dict] = []
|
||||
_event(
|
||||
run_log,
|
||||
"local_source_artifact",
|
||||
"stage1_result",
|
||||
stage1="local_ok",
|
||||
coverage=reconstruction.get("coverage") or "all_selected_sources",
|
||||
status="local_ok",
|
||||
source_count=len(source_user),
|
||||
writing_profile_present=profile_meta.get("present"),
|
||||
style_example_count=style_meta.get("count"),
|
||||
generation_selection_source=policy_meta.get("source"),
|
||||
generation_selection_keys=",".join(
|
||||
f"{slot}:{policy_meta.get('keys', {}).get(slot)}"
|
||||
for slot in ("transformation", "detail", "voice", "narrative")
|
||||
),
|
||||
)
|
||||
reconstruct_result = {
|
||||
"trace": _local_stage_trace(reconstruction, source_count=len(source_user)),
|
||||
"content": reconstruction_text(reconstruction),
|
||||
}
|
||||
|
||||
narrate_context = build_internal_context(
|
||||
profile_id,
|
||||
space_id=day["space_id"],
|
||||
journal_day_id=journal_day_id,
|
||||
purpose="journal_generate",
|
||||
include_existing=include_existing,
|
||||
conversation_ids=selected,
|
||||
existing_text=existing_text,
|
||||
conversation_id=selected[0] if selected else None,
|
||||
reconstruction=narration_sources_text(reconstruction),
|
||||
writing_profile=writing_profile,
|
||||
style_examples=style_examples,
|
||||
transformation_instructions=compiled_policy.instructions["transformation_instructions"],
|
||||
detail_instructions=compiled_policy.instructions["detail_instructions"],
|
||||
voice_instructions=compiled_policy.instructions["voice_instructions"],
|
||||
narrative_instructions=compiled_policy.instructions["narrative_instructions"],
|
||||
)
|
||||
assembled = assemble_text(narrate_context)
|
||||
try:
|
||||
assembled, dropped = pack_narration_context(
|
||||
narrate_prompt,
|
||||
narrate_budget,
|
||||
assembled,
|
||||
style_examples=style_examples,
|
||||
existing_text=existing_text,
|
||||
include_existing=include_existing,
|
||||
)
|
||||
except JournalBudgetError as exc:
|
||||
_raise_budget(exc)
|
||||
if dropped:
|
||||
_event(run_log, "journal_generate", "budget_pack", dropped=",".join(dropped))
|
||||
style_app = style_application_trace(compiled_policy, style_bundle, dropped=dropped)
|
||||
if "style_examples" in dropped:
|
||||
style_meta = {**style_example_diagnostics([]), "dropped": True, "dropped_reason": "budget"}
|
||||
else:
|
||||
style_meta = {**style_meta, "dropped": False}
|
||||
profile_meta["brief_chars"] = len((assembled.get("writing_profile") or "").strip())
|
||||
mappings = list_mappings(profile_id)
|
||||
preview_title, preview_body = _local_narration(reconstruction)
|
||||
source_preview = f"{preview_title}\n\n{preview_body}".strip()
|
||||
narrate_result: dict = {}
|
||||
try:
|
||||
narrate_prompt = prompt_without_orphaned_style_example_rule(
|
||||
narrate_prompt,
|
||||
include_style_examples=bool((assembled.get("style_examples") or "").strip()),
|
||||
)
|
||||
rendered_preview = preview_prompt(narrate_prompt, assembled)
|
||||
assert_template_resolved(rendered_preview.get("rendered") or "")
|
||||
detect_mappings = precomputed_mappings
|
||||
if detect_mappings is None and not is_bypass_mode():
|
||||
user_text = "\n".join(source_user)
|
||||
pipe = prepare_detect_pipeline(profile_id, user_text, "")
|
||||
if (pipe.need_review or pipe.candidates) and selected:
|
||||
review_id = save_pending(
|
||||
profile_id,
|
||||
selected[0],
|
||||
"",
|
||||
{
|
||||
"mappings": pipe.mappings,
|
||||
"candidates": pipe.candidates,
|
||||
"user_body": user_text,
|
||||
"kind": pipe.kind,
|
||||
"counters": pipe.counters,
|
||||
"purpose": "journal_generate",
|
||||
"generate": {
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_ids": selected,
|
||||
"include_existing": include_existing,
|
||||
"generation_selection": generation_selection,
|
||||
"remember_generation_selection": remember_generation_selection,
|
||||
},
|
||||
},
|
||||
)
|
||||
return {
|
||||
"pending_mask_review": {
|
||||
"id": review_id,
|
||||
"candidates": pipe.candidates,
|
||||
"kind": pipe.kind,
|
||||
"purpose": "journal_generate",
|
||||
},
|
||||
"run_log": run_log,
|
||||
"trace": None,
|
||||
}
|
||||
detect_mappings = pipe.mappings
|
||||
assembled = dict(assembled)
|
||||
assembled["user_body"] = "\n".join(source_user)
|
||||
narrate_result = execute_prompt(
|
||||
narrate_prompt,
|
||||
profile_id,
|
||||
purpose="journal_generate",
|
||||
data_class="B",
|
||||
context=assembled,
|
||||
max_tokens=narrate_budget.reserved_output_tokens,
|
||||
disable_context_compression=True,
|
||||
budget=narrate_budget,
|
||||
precomputed_mappings=detect_mappings,
|
||||
)
|
||||
_stamp_log(run_log, "journal_generate", _log_items(narrate_result))
|
||||
title, body = _split_title(narrate_result.get("content") or "")
|
||||
seen = {(item.get("local_label") or "").casefold() for item in mappings}
|
||||
for item in narrate_result.get("local_identities") or []:
|
||||
label = (item.get("local_label") or "").strip()
|
||||
if not label or label.casefold() in seen:
|
||||
continue
|
||||
mappings.append(item)
|
||||
seen.add(label.casefold())
|
||||
active_tokens = list((narrate_result.get("diagnostics") or {}).get("active_tokens") or [])
|
||||
combined = f"{title}\n\n{body}".strip()
|
||||
unattested = unattested_journal_content(combined, source_user, mappings, active_tokens)
|
||||
if unattested:
|
||||
_event(
|
||||
run_log,
|
||||
"journal_generate",
|
||||
"narration_result",
|
||||
status="not_accepted",
|
||||
reason=unattested,
|
||||
)
|
||||
bundle = _compose_journal_trace(
|
||||
reconstruct_result,
|
||||
narrate_result,
|
||||
run_log=run_log,
|
||||
narrate_prompt=narrate_prompt,
|
||||
profile_meta=profile_meta,
|
||||
style_meta=style_meta,
|
||||
dropped=dropped,
|
||||
extra={
|
||||
"narration_source": "not_accepted",
|
||||
"model_text_accepted": False,
|
||||
"provenance_decision": unattested,
|
||||
"abort_reason": unattested,
|
||||
"source_preview": source_preview,
|
||||
"generation_selection": policy_meta,
|
||||
"style_application": style_app,
|
||||
},
|
||||
)
|
||||
_reject_generation(
|
||||
reason=unattested,
|
||||
diagnostics={
|
||||
**(narrate_result.get("diagnostics") or {}),
|
||||
"trace": bundle,
|
||||
"log": run_log,
|
||||
"provenance_decision": unattested,
|
||||
},
|
||||
source_preview=source_preview,
|
||||
extra=bundle,
|
||||
)
|
||||
_event(
|
||||
run_log,
|
||||
"journal_generate",
|
||||
"narration_result",
|
||||
status="model",
|
||||
lexical_similarity=lexical_similarity("\n".join(source_user), body),
|
||||
incomplete_syntax=incomplete_syntax_markers(body),
|
||||
writing_profile_present=profile_meta.get("present"),
|
||||
)
|
||||
except EngineError as exc:
|
||||
if exc.code == JOURNAL_NOT_ACCEPTED:
|
||||
raise
|
||||
_stamp_log(run_log, "journal_generate", _log_items(exc))
|
||||
diag = dict(exc.diagnostics or {})
|
||||
generate_called = bool(diag.get("generate_called") or diag.get("generate_calls"))
|
||||
failed_result = {
|
||||
"diagnostics": diag,
|
||||
"trace": dict(diag.get("trace") or {}),
|
||||
"content": "",
|
||||
}
|
||||
bundle = _compose_journal_trace(
|
||||
reconstruct_result,
|
||||
failed_result,
|
||||
run_log=run_log,
|
||||
narrate_prompt=narrate_prompt,
|
||||
profile_meta=profile_meta,
|
||||
style_meta=style_meta,
|
||||
dropped=dropped,
|
||||
extra={
|
||||
"narration_source": "not_accepted",
|
||||
"model_text_accepted": False,
|
||||
"abort_reason": exc.code,
|
||||
"source_preview": source_preview,
|
||||
"generation_selection": policy_meta,
|
||||
"style_application": style_app,
|
||||
},
|
||||
)
|
||||
diag["log"] = run_log
|
||||
diag["trace"] = bundle
|
||||
if not generate_called:
|
||||
raise EngineError(exc.code, exc.message, exc.status_code, diag) from exc
|
||||
_event(
|
||||
run_log,
|
||||
"journal_generate",
|
||||
"narration_result",
|
||||
status="not_accepted",
|
||||
reason=exc.code,
|
||||
)
|
||||
_reject_generation(
|
||||
reason=exc.code,
|
||||
diagnostics=diag,
|
||||
source_preview=source_preview,
|
||||
extra=bundle,
|
||||
)
|
||||
person_labels = [
|
||||
(item.get("local_label") or "").strip()
|
||||
for item in mappings
|
||||
if (item.get("local_label") or "").strip()
|
||||
and str(item.get("token") or "").upper().lstrip("[").startswith("PERSON:")
|
||||
]
|
||||
title, body = shape_journal(
|
||||
title,
|
||||
body,
|
||||
source_user,
|
||||
person_labels=person_labels,
|
||||
source="model",
|
||||
)
|
||||
stored = f"{title}\n\n{body}".strip() if title else (body or "").strip()
|
||||
before_entries = {item["id"]: item.get("current_version_id") for item in current_entries(profile_id, journal_day_id)}
|
||||
model = (
|
||||
((narrate_result.get("diagnostics") or {}).get("actual_model"))
|
||||
or ((narrate_result.get("trace") or {}).get("model"))
|
||||
or narrate_result.get("provider")
|
||||
or ""
|
||||
)
|
||||
snapshot = draft_snapshot(compiled_policy, prompt=narrate_prompt, model=str(model or ""))
|
||||
mark_guidelines_used(
|
||||
[
|
||||
compiled_policy.ids["transformation"],
|
||||
compiled_policy.ids["detail"],
|
||||
compiled_policy.ids["voice"],
|
||||
compiled_policy.ids["narrative"],
|
||||
]
|
||||
)
|
||||
draft = insert_draft(
|
||||
profile_id,
|
||||
journal_day_id,
|
||||
title=title,
|
||||
body=body,
|
||||
source_conversation_ids=selected,
|
||||
source_message_ids=_message_ids(profile_id, selected),
|
||||
generation_snapshot=snapshot,
|
||||
)
|
||||
after_entries = current_entries(profile_id, journal_day_id)
|
||||
for item in after_entries:
|
||||
previous = before_entries.get(item["id"])
|
||||
if previous is not None and previous != item.get("current_version_id"):
|
||||
raise StoreError("policy_violation", "Generate darf die Nutzerfassung nicht verändern", 500)
|
||||
bundle = _compose_journal_trace(
|
||||
reconstruct_result,
|
||||
narrate_result,
|
||||
run_log=run_log,
|
||||
narrate_prompt=narrate_prompt,
|
||||
profile_meta=profile_meta,
|
||||
style_meta=style_meta,
|
||||
dropped=dropped,
|
||||
extra={
|
||||
"narration_source": "model",
|
||||
"model_text_accepted": True,
|
||||
"provenance_decision": "accepted",
|
||||
"lexical_similarity": lexical_similarity("\n".join(source_user), body),
|
||||
"incomplete_syntax": incomplete_syntax_markers(body),
|
||||
"generation_selection": policy_meta,
|
||||
"style_application": style_app,
|
||||
"stored_title": title,
|
||||
"stored_body": body,
|
||||
"reply": stored,
|
||||
},
|
||||
)
|
||||
draft["run_log"] = run_log
|
||||
draft["trace"] = bundle
|
||||
return draft
|
||||
|
||||
|
||||
def continue_journal_generate(profile_id: str, journal_day_id: str, review_id: str, decisions: list[dict]) -> dict:
|
||||
pending = load_pending(profile_id, review_id)
|
||||
if not pending or pending.get("purpose") != "journal_generate":
|
||||
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||
generate = pending.get("generate") or {}
|
||||
if generate.get("journal_day_id") != journal_day_id:
|
||||
raise EngineError("mask_review_missing", "Die Maskierungsprüfung ist nicht mehr gültig.", 404)
|
||||
if pending.get("kind") == KIND_CONFIRM_REMOTE:
|
||||
try:
|
||||
mappings = apply_remote_confirm(profile_id, pending, decisions)
|
||||
except DetectError as exc:
|
||||
raise EngineError(exc.code, exc.message, getattr(exc, "status_code", 403)) from exc
|
||||
else:
|
||||
mappings = apply_review_decisions(profile_id, pending, decisions)
|
||||
drop_pending(profile_id, review_id)
|
||||
return generate_draft(
|
||||
profile_id,
|
||||
journal_day_id,
|
||||
conversation_ids=generate.get("conversation_ids"),
|
||||
include_existing=bool(generate.get("include_existing")),
|
||||
explicit=True,
|
||||
generation_selection=generate.get("generation_selection"),
|
||||
remember_generation_selection=bool(generate.get("remember_generation_selection")),
|
||||
precomputed_mappings=mappings,
|
||||
)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,373 @@
|
||||
"""Journal-specific first impulse. Not a generic continuation or intent engine.
|
||||
|
||||
Uses the existing Privacy Gateway, dialogue prompt, guards and request-scoped
|
||||
trace. Attested plans come only from yesterday's user-role dialogue. Open day
|
||||
points stay on the same journal day. Saved journal entries are completed
|
||||
narrative, never an opening plan. Assistant text is conversation context, never
|
||||
a user fact. Recency is not a pattern.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, timedelta
|
||||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
from debug_store import persist_engine_error, persist_step
|
||||
from dialogue_store import StoreError, append_message, get_conversation, list_conversations_for_day, list_messages
|
||||
from dialogue_turn import OPERATIONS, needs_repair, parse_turn_payload, repair_note
|
||||
from engine import EngineError, execute_prompt, load_active_prompt
|
||||
from entity_detect import DETECT_DIALOGUE_FALLBACK_CODES
|
||||
from journal_store import get_day
|
||||
from retrieval import retrieve
|
||||
|
||||
NEUTRAL_OPENING = "Wenn du magst, fang einfach an – ich höre zu."
|
||||
OPENING_POLICY = (
|
||||
"Dies ist der erste Impuls dieses Gesprächs. Es gibt hier noch keine Nutzerzeile. "
|
||||
"Nur belegte Vorhaben aus dem user-Dialog des unmittelbar vorigen Kalendertags "
|
||||
"oder offene user-Punkte desselben Journal Day aufgreifen. "
|
||||
"Ein gespeicherter Journal-Eintrag ist abgeschlossene Erzählung, kein offenes Vorhaben. "
|
||||
"Ein Recency-Treffer ist kein bewiesenes wiederkehrendes Muster. "
|
||||
"Nicht erwähnt ist nicht geschehen. Ein Plan ist kein Vollzug. "
|
||||
"Assistententext ist Gesprächskontext, kein Nutzerfakt. "
|
||||
"Keine Behauptung, ein Vorhaben sei ausgeführt worden. "
|
||||
"Keine bereits erzählte, abgeschlossene Episode eines früheren Tags als offenen Faden führen."
|
||||
)
|
||||
|
||||
FUTURE_INTENT = re.compile(
|
||||
r"(?:"
|
||||
r"\b("
|
||||
r"morgen|übermorgen|"
|
||||
r"wollen(?: wir| sie)?|will(?:st)?|werde|werden wir|"
|
||||
r"vorhaben|geplant|"
|
||||
r"steht(?: heute| morgen)? an|"
|
||||
r"nächste[nrs]?\s+(?:woche|monat|tag|tage)"
|
||||
r")\b"
|
||||
r"|(?:habe|haben|hat|habt)\s+vor\b"
|
||||
r")",
|
||||
re.I,
|
||||
)
|
||||
UNEARNED_PATTERN = re.compile(
|
||||
r"\b(die letzten tage|häufig|immer|jedes mal|wiederkehr|typischerweise|muster)\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def parse_calendar_date(value: str | None) -> date | None:
|
||||
raw = (value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def is_previous_calendar_day(source: date | None, current: date | None) -> bool:
|
||||
if source is None or current is None:
|
||||
return False
|
||||
return source == current - timedelta(days=1)
|
||||
|
||||
|
||||
def is_attested_plan(text: str) -> bool:
|
||||
"""User wording that marks a plan or intention. Not past completion, not overlap."""
|
||||
body = (text or "").strip()
|
||||
if not body:
|
||||
return False
|
||||
return bool(FUTURE_INTENT.search(body))
|
||||
|
||||
|
||||
def attested_plans_from_user_texts(texts: list[str]) -> list[str]:
|
||||
plans: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for text in texts:
|
||||
body = (text or "").strip()
|
||||
if not body or body in seen or not is_attested_plan(body):
|
||||
continue
|
||||
seen.add(body)
|
||||
plans.append(body)
|
||||
return plans
|
||||
|
||||
|
||||
def _user_bodies(messages: list[dict]) -> list[str]:
|
||||
return [
|
||||
(item.get("body") or "").strip()
|
||||
for item in messages
|
||||
if item.get("role") == "user" and (item.get("body") or "").strip()
|
||||
]
|
||||
|
||||
|
||||
def _assistant_bodies(messages: list[dict]) -> list[str]:
|
||||
return [
|
||||
(item.get("body") or "").strip()
|
||||
for item in messages
|
||||
if item.get("role") == "assistant" and (item.get("body") or "").strip()
|
||||
]
|
||||
|
||||
|
||||
def collect_opening_context(profile_id: str, conversation_id: str) -> dict:
|
||||
conversation = get_conversation(profile_id, conversation_id)
|
||||
space_id = conversation.get("space_id")
|
||||
journal_day_id = conversation.get("journal_day_id")
|
||||
if not space_id or not journal_day_id:
|
||||
raise StoreError("not_journal_conversation", "Erster Impuls nur für einen Journal-Dialog.")
|
||||
day = get_day(profile_id, journal_day_id)
|
||||
current_date = parse_calendar_date(day.get("calendar_date"))
|
||||
|
||||
day_conversations = list_conversations_for_day(profile_id, journal_day_id)
|
||||
open_points: list[str] = []
|
||||
day_assistant: list[str] = []
|
||||
for item in day_conversations:
|
||||
messages = list_messages(profile_id, item["id"])
|
||||
if item["id"] == conversation_id:
|
||||
continue
|
||||
open_points.extend(_user_bodies(messages))
|
||||
day_assistant.extend(_assistant_bodies(messages))
|
||||
|
||||
prior_entries = retrieve(
|
||||
profile_id,
|
||||
{"kind": "space_entries", "space_id": space_id, "exclude_day_id": journal_day_id},
|
||||
)
|
||||
recent_sources = retrieve(
|
||||
profile_id,
|
||||
{
|
||||
"kind": "space_recent_sources",
|
||||
"space_id": space_id,
|
||||
"exclude_conversation_id": conversation_id,
|
||||
"exclude_journal_day_id": journal_day_id,
|
||||
},
|
||||
)
|
||||
recency_user: list[str] = []
|
||||
yesterday_user: list[str] = []
|
||||
for entry in prior_entries:
|
||||
excerpt = (entry.get("excerpt") or "").strip()
|
||||
if excerpt:
|
||||
recency_user.append(excerpt)
|
||||
for source in recent_sources:
|
||||
excerpt = (source.get("excerpt") or "").strip()
|
||||
if excerpt:
|
||||
recency_user.append(excerpt)
|
||||
if excerpt and is_previous_calendar_day(
|
||||
parse_calendar_date(source.get("calendar_date")),
|
||||
current_date,
|
||||
):
|
||||
yesterday_user.append(excerpt)
|
||||
|
||||
plans = attested_plans_from_user_texts(yesterday_user)
|
||||
has_relevant = bool(plans or open_points)
|
||||
return {
|
||||
"space_id": space_id,
|
||||
"journal_day_id": journal_day_id,
|
||||
"conversation_id": conversation_id,
|
||||
"user_texts": [*open_points, *yesterday_user],
|
||||
"assistant_texts": day_assistant,
|
||||
"attested_plans": plans,
|
||||
"open_day_points": open_points,
|
||||
"recency_excerpts": recency_user,
|
||||
"has_relevant_context": has_relevant,
|
||||
"recency_is_not_pattern": True,
|
||||
"assistant_is_not_user_fact": True,
|
||||
}
|
||||
|
||||
|
||||
def format_opening_hint(facts: dict) -> str:
|
||||
lines = [OPENING_POLICY]
|
||||
plans = facts.get("attested_plans") or []
|
||||
points = facts.get("open_day_points") or []
|
||||
if plans:
|
||||
lines.append("Belegte Vorhaben (nur user-Quellen):")
|
||||
lines.extend(f"- {item}" for item in plans)
|
||||
else:
|
||||
lines.append("Belegte Vorhaben: keine.")
|
||||
if points:
|
||||
lines.append("Offene Punkte dieses Journal Day (nur user):")
|
||||
lines.extend(f"- {item}" for item in points)
|
||||
else:
|
||||
lines.append("Offene Punkte dieses Journal Day: keine.")
|
||||
lines.append(
|
||||
"Recency-Ausschnitte dürfen den Impuls nicht als Muster oder als Vollzug begründen."
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _store_opening(profile_id: str, conversation_id: str, impulse: str, extra: dict) -> dict:
|
||||
assistant = append_message(profile_id, conversation_id, impulse, role="assistant")
|
||||
payload = {
|
||||
"conversation": get_conversation(profile_id, conversation_id),
|
||||
"assistant": assistant,
|
||||
"messages": list_messages(profile_id, conversation_id),
|
||||
**extra,
|
||||
}
|
||||
persist_step(
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
status="ok",
|
||||
layer="dialogzug",
|
||||
subject_type="conversation",
|
||||
subject_id=conversation_id,
|
||||
conversation_id=conversation_id,
|
||||
message_id=assistant.get("id"),
|
||||
decision=extra.get("decision"),
|
||||
trace=extra.get("trace"),
|
||||
extra={
|
||||
"kind": extra.get("kind"),
|
||||
"calls": extra.get("calls"),
|
||||
"opened": extra.get("opened"),
|
||||
"assistant": {"id": assistant.get("id"), "body": assistant.get("body")},
|
||||
},
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def start_journal_opening(profile_id: str, conversation_id: str) -> dict:
|
||||
"""Create the first Kanshō line. Invalid model output does not write a message."""
|
||||
existing = list_messages(profile_id, conversation_id)
|
||||
if existing:
|
||||
return {
|
||||
"opened": False,
|
||||
"reason": "already_started",
|
||||
"conversation": get_conversation(profile_id, conversation_id),
|
||||
"messages": existing,
|
||||
}
|
||||
facts = collect_opening_context(profile_id, conversation_id)
|
||||
if not facts["has_relevant_context"]:
|
||||
return _store_opening(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
NEUTRAL_OPENING,
|
||||
{
|
||||
"opened": True,
|
||||
"kind": "local_neutral",
|
||||
"calls": 0,
|
||||
"opening_context": facts,
|
||||
"decision": {
|
||||
"operation": "fortfuehren",
|
||||
"label": OPERATIONS["fortfuehren"],
|
||||
"parsed": True,
|
||||
"guard": "local_neutral_opening",
|
||||
},
|
||||
"trace": None,
|
||||
},
|
||||
)
|
||||
|
||||
conversation = get_conversation(profile_id, conversation_id)
|
||||
day_ids = [item["id"] for item in list_conversations_for_day(profile_id, conversation["journal_day_id"])]
|
||||
context = build_internal_context(
|
||||
profile_id,
|
||||
conversation_id=conversation_id,
|
||||
space_id=conversation.get("space_id"),
|
||||
journal_day_id=conversation.get("journal_day_id"),
|
||||
purpose="dialogue_turn",
|
||||
conversation_ids=day_ids,
|
||||
include_space_recency=False,
|
||||
)
|
||||
assembled = assemble_text(context)
|
||||
assembled = dict(assembled)
|
||||
assembled["opening_hint"] = format_opening_hint(facts)
|
||||
assembled["register_hint"] = (
|
||||
"Erster Impuls. Noch keine Nutzerzeile in diesem Gespräch. "
|
||||
"Nur Vorhaben aus dem user-Dialog von gestern oder offene user-Punkte dieses Tags. "
|
||||
"Kein Muster aus Recency. Keine abgeschlossene frühere Episode wieder öffnen."
|
||||
)
|
||||
prompt = load_active_prompt("mvp.dialogue_turn")
|
||||
calls = 0
|
||||
result = None
|
||||
impulse = ""
|
||||
decision: dict = {"operation": "unparsed", "label": "nicht erkannt", "parsed": False}
|
||||
try:
|
||||
while calls < 2:
|
||||
result = execute_prompt(
|
||||
prompt,
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
data_class="B",
|
||||
context=assembled,
|
||||
)
|
||||
calls += 1
|
||||
content = (result.get("content") or "").strip()
|
||||
if not content:
|
||||
raise EngineError("empty_provider_response", "Der Provider lieferte keine Antwort.")
|
||||
impulse, decision = parse_turn_payload(content)
|
||||
if UNEARNED_PATTERN.search(impulse or ""):
|
||||
decision = {**decision, "guard": "pattern_rejected"}
|
||||
assembled = dict(assembled)
|
||||
assembled["dialogue_context"] = (
|
||||
(assembled.get("dialogue_context") or "")
|
||||
+ "\n\nKorrektur: Kein Recency-Treffer als wiederkehrendes Muster."
|
||||
)
|
||||
continue
|
||||
if not needs_repair(impulse, assembled):
|
||||
break
|
||||
decision = {**decision, "guard": "impulse_rejected"}
|
||||
assembled = dict(assembled)
|
||||
assembled["dialogue_context"] = (
|
||||
(assembled.get("dialogue_context") or "") + "\n\n" + repair_note(impulse, assembled)
|
||||
)
|
||||
if needs_repair(impulse, assembled) or UNEARNED_PATTERN.search(impulse or "") or not impulse:
|
||||
impulse = NEUTRAL_OPENING
|
||||
decision = {
|
||||
"operation": "fortfuehren",
|
||||
"label": OPERATIONS["fortfuehren"],
|
||||
"parsed": bool(decision.get("parsed")),
|
||||
"guard": "local_fallback",
|
||||
}
|
||||
except EngineError as exc:
|
||||
if exc.code != "response_validation_failed" and exc.code not in DETECT_DIALOGUE_FALLBACK_CODES:
|
||||
persist_engine_error(
|
||||
profile_id,
|
||||
purpose="dialogue_turn",
|
||||
exc=exc,
|
||||
subject_type="conversation",
|
||||
subject_id=conversation_id,
|
||||
conversation_id=conversation_id,
|
||||
extra={"kind": "opening_failed", "calls": calls},
|
||||
)
|
||||
raise
|
||||
return _store_opening(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
NEUTRAL_OPENING,
|
||||
{
|
||||
"opened": True,
|
||||
"kind": "local_neutral",
|
||||
"calls": calls,
|
||||
"opening_context": facts,
|
||||
"decision": {
|
||||
"operation": "fortfuehren",
|
||||
"label": OPERATIONS["fortfuehren"],
|
||||
"parsed": False,
|
||||
"guard": "identity_leak_blocked" if exc.code == "response_validation_failed" else "detect_blocked",
|
||||
},
|
||||
"trace": result.get("trace") if result else None,
|
||||
},
|
||||
)
|
||||
return _store_opening(
|
||||
profile_id,
|
||||
conversation_id,
|
||||
impulse,
|
||||
{
|
||||
"opened": True,
|
||||
"kind": "model",
|
||||
"calls": calls,
|
||||
"opening_context": facts,
|
||||
"decision": decision,
|
||||
"trace": result.get("trace") if result else None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def maybe_open_journal_conversation(profile_id: str, conversation: dict) -> dict:
|
||||
payload = dict(conversation)
|
||||
try:
|
||||
opening = start_journal_opening(profile_id, conversation["id"])
|
||||
except EngineError as exc:
|
||||
payload["opening"] = {
|
||||
"opened": False,
|
||||
"reason": exc.code,
|
||||
"message": exc.message,
|
||||
"kind": "failed_closed",
|
||||
}
|
||||
payload["messages"] = list_messages(profile_id, conversation["id"])
|
||||
return payload
|
||||
payload["opening"] = opening
|
||||
payload["messages"] = opening.get("messages") or []
|
||||
return payload
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Small write/decision boundary for the journal MVP. Not a policy engine."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from conversation_signals import similar_enough
|
||||
|
||||
|
||||
class PolicyError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 400):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
ENTRY_ORIGINS = {"user_edit", "accepted_draft", "restore"}
|
||||
|
||||
|
||||
def require_explicit_generate(explicit: bool) -> None:
|
||||
if not explicit:
|
||||
raise PolicyError("generate_not_explicit", "Journal Drafts entstehen nur auf ausdrückliche Aktion.")
|
||||
|
||||
|
||||
def consolidation_offer(conversations: list[dict] | int) -> bool:
|
||||
"""Offer merge only when conversations look narratively compatible. User decides."""
|
||||
if isinstance(conversations, int):
|
||||
return False
|
||||
return similar_enough(conversations)
|
||||
|
||||
|
||||
def source_conversation_ids(explicit_ids: list[str] | None, day_conversation_ids: list[str]) -> list[str]:
|
||||
"""Missing selection never silently merges multiple conversations."""
|
||||
known = [item for item in day_conversation_ids if item]
|
||||
if explicit_ids:
|
||||
chosen = [item for item in explicit_ids if item in known]
|
||||
if not chosen:
|
||||
raise PolicyError("unknown_conversation", "Keine der angegebenen Conversations gehört zu diesem Tag.")
|
||||
return chosen
|
||||
if not known:
|
||||
raise PolicyError("no_source_dialogue", "Ohne Dialog gibt es nichts zu erzeugen.")
|
||||
return [known[-1]]
|
||||
|
||||
|
||||
def draft_must_not_touch_entry() -> None:
|
||||
return None
|
||||
|
||||
|
||||
def require_origin(origin: str) -> str:
|
||||
if origin not in ENTRY_ORIGINS:
|
||||
raise PolicyError("invalid_origin", "origin muss user_edit, accepted_draft oder restore sein")
|
||||
return origin
|
||||
@@ -0,0 +1,529 @@
|
||||
"""Journal adapter over the intent-neutral provenance layer. No egress.
|
||||
|
||||
Journal policy: selected user sources, full canonical rehydration, assistant never
|
||||
a content source, source-local clocks, reorder only with confirmed times.
|
||||
Semantic labels from the model are not verified content.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from prompt_budget import ERROR_NO_USER_SOURCES, ERROR_RECONSTRUCTION_INVALID, JournalBudgetError
|
||||
from provenance import (
|
||||
COVERAGE_ALL_SELECTED_SOURCES,
|
||||
EvidenceRef,
|
||||
ProvenanceError,
|
||||
SourceRecord,
|
||||
SourceRegistry,
|
||||
VerificationPolicy,
|
||||
texts_from_payload,
|
||||
verify,
|
||||
)
|
||||
|
||||
JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
||||
TIME_RE = re.compile(r"\b\d{1,2}:\d{2}\b|\b\d{1,2}\s*Uhr\b", re.I)
|
||||
CLOCK_STAMP_RE = re.compile(r"^(?:\d{1,2}:\d{2}(?:\s*Uhr)?|\d{1,2}\s*Uhr)$", re.I)
|
||||
DIALOGUE_LINE = re.compile(
|
||||
r"^(?:\[(?P<source_id>u\d+|a\d+)\]\s*)?(?P<role>user|assistant):\s*(?P<body>.*)$"
|
||||
)
|
||||
USER_SOURCE_ID = re.compile(r"^u\d+$")
|
||||
ASSISTANT_SOURCE_ID = re.compile(r"^a\d+$")
|
||||
REQUIRED_TOP = ("source_order", "chronology")
|
||||
LEGACY_FACT_KEYS = (
|
||||
"events",
|
||||
"people",
|
||||
"places",
|
||||
"perceptions",
|
||||
"feelings_stated",
|
||||
"evaluations",
|
||||
"quotes",
|
||||
"uncertainties",
|
||||
"plan_changes",
|
||||
"corrections",
|
||||
"evidence",
|
||||
"text",
|
||||
)
|
||||
SOURCED_LIST_KEYS = ("contradictions", "uncertainties", "plan_changes", "corrections")
|
||||
ALLOWED_CLAIM_KEYS = frozenset({"kind", "evidence"})
|
||||
ALLOWED_ITEM_KEYS = frozenset({"source_id", "source", "time", "claims"})
|
||||
ALLOWED_SOURCED_KEYS = frozenset({"source_id", "evidence"})
|
||||
CONTENT_ROLE = "user"
|
||||
JOURNAL_ALLOWED_ROLES = frozenset({CONTENT_ROLE})
|
||||
|
||||
|
||||
def _fail(reason: str, **extra: Any) -> None:
|
||||
raise JournalBudgetError(
|
||||
ERROR_RECONSTRUCTION_INVALID,
|
||||
diagnostics={"reason": reason, **extra},
|
||||
)
|
||||
|
||||
|
||||
def parse_dialogue_line(line: str) -> dict[str, str] | None:
|
||||
match = DIALOGUE_LINE.match((line or "").rstrip("\n"))
|
||||
if not match:
|
||||
return None
|
||||
return {
|
||||
"source_id": match.group("source_id") or "",
|
||||
"role": match.group("role"),
|
||||
"body": match.group("body") or "",
|
||||
}
|
||||
|
||||
|
||||
def is_dialogue_role_line(line: str) -> bool:
|
||||
return parse_dialogue_line(line) is not None
|
||||
|
||||
|
||||
def assign_source_ids(messages: list[dict]) -> list[dict]:
|
||||
"""Local, stable IDs in encounter order. User u1…; assistant a1…."""
|
||||
labeled: list[dict] = []
|
||||
user_n = 0
|
||||
assistant_n = 0
|
||||
for message in messages or []:
|
||||
item = dict(message)
|
||||
role = item.get("role") or "user"
|
||||
body = (item.get("body") or "").strip()
|
||||
if role == "user" and body:
|
||||
user_n += 1
|
||||
item["source_id"] = f"u{user_n}"
|
||||
elif role == "assistant" and body:
|
||||
assistant_n += 1
|
||||
item["source_id"] = f"a{assistant_n}"
|
||||
else:
|
||||
item["source_id"] = None
|
||||
labeled.append(item)
|
||||
return labeled
|
||||
|
||||
|
||||
def expected_user_source_ids(messages: list[dict]) -> list[str]:
|
||||
return [
|
||||
item["source_id"]
|
||||
for item in assign_source_ids(messages)
|
||||
if item.get("role") == "user" and item.get("source_id")
|
||||
]
|
||||
|
||||
|
||||
def parse_reconstruction(raw: str) -> dict[str, Any]:
|
||||
text = (raw or "").strip()
|
||||
if text.startswith("```"):
|
||||
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL)
|
||||
match = JSON_BLOCK.search(text)
|
||||
if not match:
|
||||
_fail("not_json")
|
||||
try:
|
||||
data = json.loads(match.group(0))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise JournalBudgetError(
|
||||
ERROR_RECONSTRUCTION_INVALID,
|
||||
diagnostics={"reason": "json_parse"},
|
||||
) from exc
|
||||
if not isinstance(data, dict):
|
||||
_fail("not_object")
|
||||
return data
|
||||
|
||||
|
||||
def parse_clock(stamp: str) -> tuple[int, int] | None:
|
||||
text = (stamp or "").strip()
|
||||
match = re.search(r"(\d{1,2}):(\d{2})", text)
|
||||
if match:
|
||||
hour, minute = int(match.group(1)), int(match.group(2))
|
||||
if 0 <= hour <= 23 and 0 <= minute <= 59:
|
||||
return hour, minute
|
||||
return None
|
||||
match = re.search(r"(\d{1,2})\s*Uhr", text, re.I)
|
||||
if match:
|
||||
hour = int(match.group(1))
|
||||
if 0 <= hour <= 23:
|
||||
return hour, 0
|
||||
return None
|
||||
|
||||
|
||||
def clocks_in_text(text: str) -> set[tuple[int, int]]:
|
||||
found: set[tuple[int, int]] = set()
|
||||
for match in TIME_RE.finditer(text or ""):
|
||||
parsed = parse_clock(match.group(0))
|
||||
if parsed:
|
||||
found.add(parsed)
|
||||
return found
|
||||
|
||||
|
||||
def clock_in_source(stamp: str, source_body: str) -> bool:
|
||||
parsed = parse_clock(stamp)
|
||||
if parsed is None:
|
||||
return False
|
||||
return parsed in clocks_in_text(source_body)
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return [value]
|
||||
|
||||
|
||||
def _legacy_payload(raw: dict[str, Any]) -> bool:
|
||||
for key in LEGACY_FACT_KEYS:
|
||||
values = [str(part).strip() for part in _as_list(raw.get(key)) if str(part).strip()]
|
||||
if values:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _normalize_claim(raw: Any, *, index: int, claim_index: int) -> dict[str, str]:
|
||||
if not isinstance(raw, dict):
|
||||
_fail("claim_shape", index=index, claim_index=claim_index)
|
||||
extra = {key: raw.get(key) for key in raw.keys() if key not in ALLOWED_CLAIM_KEYS}
|
||||
if any(value not in (None, "", [], {}) for value in extra.values()):
|
||||
_fail("unverified_claim", index=index, claim_index=claim_index)
|
||||
evidence = str(raw.get("evidence") or "").strip()
|
||||
if not evidence:
|
||||
_fail("evidence_missing", index=index, claim_index=claim_index)
|
||||
kind = raw.get("kind")
|
||||
return {"kind": "" if kind is None else str(kind), "evidence": evidence}
|
||||
|
||||
|
||||
def _normalize_sourced_item(raw: Any, *, key: str, index: int) -> dict[str, str]:
|
||||
if isinstance(raw, str) and raw.strip():
|
||||
_fail("unverified_claim", key=key, index=index)
|
||||
if not isinstance(raw, dict):
|
||||
_fail("sourced_item_shape", key=key, index=index)
|
||||
extra = {item: raw.get(item) for item in raw.keys() if item not in ALLOWED_SOURCED_KEYS}
|
||||
if any(value not in (None, "", [], {}) for value in extra.values()):
|
||||
_fail("unverified_claim", key=key, index=index)
|
||||
source_id = str(raw.get("source_id") or "").strip()
|
||||
evidence = str(raw.get("evidence") or "").strip()
|
||||
if not source_id:
|
||||
_fail("missing_source_id", key=key, index=index)
|
||||
if not evidence:
|
||||
_fail("evidence_missing", key=key, index=index)
|
||||
return {"source_id": source_id, "evidence": evidence}
|
||||
|
||||
|
||||
def _confirmed_minutes(stamp: str | None, cited: str) -> int | None:
|
||||
if not stamp:
|
||||
return None
|
||||
parsed = parse_clock(stamp)
|
||||
if parsed is None or parsed not in clocks_in_text(cited):
|
||||
return None
|
||||
return parsed[0] * 60 + parsed[1]
|
||||
|
||||
|
||||
def _reorder_allowed(
|
||||
items: list[dict[str, Any]],
|
||||
expected_ids: list[str],
|
||||
user_by_id: dict[str, str],
|
||||
) -> bool:
|
||||
chrono_ids = [item["source_id"] for item in items]
|
||||
if chrono_ids == expected_ids:
|
||||
return True
|
||||
minutes: list[int] = []
|
||||
for item in items:
|
||||
value = _confirmed_minutes(item.get("time"), user_by_id.get(item["source_id"]) or "")
|
||||
if value is None:
|
||||
return False
|
||||
minutes.append(value)
|
||||
for index in range(len(items) - 1):
|
||||
if minutes[index] > minutes[index + 1]:
|
||||
return False
|
||||
if minutes[index] == minutes[index + 1]:
|
||||
left = expected_ids.index(items[index]["source_id"])
|
||||
right = expected_ids.index(items[index + 1]["source_id"])
|
||||
if left > right:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _registry_from_messages(labeled: list[dict]) -> SourceRegistry:
|
||||
registry = SourceRegistry()
|
||||
for item in labeled:
|
||||
source_id = item.get("source_id")
|
||||
role = item.get("role") or ""
|
||||
body = item.get("body") or ""
|
||||
if not source_id or not body.strip():
|
||||
continue
|
||||
registry.add(
|
||||
SourceRecord(
|
||||
source_id=source_id,
|
||||
role=role,
|
||||
text=body,
|
||||
metadata={"conversation_id": item.get("conversation_id")},
|
||||
)
|
||||
)
|
||||
return registry
|
||||
|
||||
|
||||
def _map_provenance(exc: ProvenanceError) -> None:
|
||||
reason = exc.reason
|
||||
details = dict(exc.details or {})
|
||||
if reason == "unknown_source":
|
||||
_fail("unknown_source_id", **details)
|
||||
if reason == "role_not_allowed":
|
||||
role = details.get("role")
|
||||
source_id = str(details.get("source_id") or "")
|
||||
if role != CONTENT_ROLE or ASSISTANT_SOURCE_ID.match(source_id):
|
||||
_fail("assistant_as_fact", **details)
|
||||
_fail("invalid_source", **details)
|
||||
if reason in {"unverified_excerpt", "canonical_overwrite"}:
|
||||
_fail("unverified_claim", **details)
|
||||
if reason == "wrong_source":
|
||||
_fail("wrong_source", **details)
|
||||
if reason == "duplicate_source_id":
|
||||
_fail("duplicate_source_id", **details)
|
||||
if reason == "missing_source_id":
|
||||
_fail("missing_source_id", **details)
|
||||
_fail(reason, **details)
|
||||
|
||||
|
||||
def _journal_policy(expected_ids: list[str], chronology_ids: list[str]) -> VerificationPolicy:
|
||||
return VerificationPolicy(
|
||||
allowed_roles=JOURNAL_ALLOWED_ROLES,
|
||||
coverage=COVERAGE_ALL_SELECTED_SOURCES,
|
||||
selected_ids=tuple(chronology_ids),
|
||||
source_order=tuple(expected_ids),
|
||||
include_unverified_annotations=False,
|
||||
)
|
||||
|
||||
|
||||
def validate_reconstruction(
|
||||
data: dict[str, Any],
|
||||
*,
|
||||
messages: list[dict] | None = None,
|
||||
user_bodies: list[str] | None = None,
|
||||
assistant_bodies: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
for key in REQUIRED_TOP:
|
||||
if key not in data:
|
||||
_fail("missing_key", key=key)
|
||||
chronology = data.get("chronology")
|
||||
source_order_raw = data.get("source_order")
|
||||
if not isinstance(chronology, list):
|
||||
_fail("chronology_missing")
|
||||
if not isinstance(source_order_raw, list):
|
||||
_fail("source_order_shape")
|
||||
|
||||
if messages is not None:
|
||||
labeled = assign_source_ids(messages)
|
||||
else:
|
||||
labeled = []
|
||||
for index, body in enumerate(user_bodies or [], start=1):
|
||||
if (body or "").strip():
|
||||
labeled.append({"role": "user", "body": body, "source_id": f"u{index}"})
|
||||
for index, body in enumerate(assistant_bodies or [], start=1):
|
||||
if (body or "").strip():
|
||||
labeled.append({"role": "assistant", "body": body, "source_id": f"a{index}"})
|
||||
|
||||
expected_ids = [
|
||||
item["source_id"] for item in labeled if item.get("role") == CONTENT_ROLE and item.get("source_id")
|
||||
]
|
||||
user_by_id = {
|
||||
item["source_id"]: item.get("body") or ""
|
||||
for item in labeled
|
||||
if item.get("role") == CONTENT_ROLE and item.get("source_id")
|
||||
}
|
||||
source_order = [str(part).strip() for part in source_order_raw]
|
||||
if source_order != expected_ids:
|
||||
_fail("source_order_mismatch", expected=expected_ids, got=source_order)
|
||||
|
||||
items: list[dict[str, Any]] = []
|
||||
evidence_refs: list[EvidenceRef] = []
|
||||
seen: list[str] = []
|
||||
for index, raw in enumerate(chronology):
|
||||
if not isinstance(raw, dict):
|
||||
_fail("chronology_item_shape", index=index)
|
||||
if _legacy_payload(raw):
|
||||
_fail("unverified_claim", index=index)
|
||||
extra_keys = [
|
||||
key for key in raw.keys() if key not in ALLOWED_ITEM_KEYS and raw.get(key) not in (None, "", [], {})
|
||||
]
|
||||
if extra_keys:
|
||||
_fail("unverified_claim", index=index, keys=extra_keys)
|
||||
if "source" not in raw or raw.get("source") is None:
|
||||
source = None
|
||||
elif not isinstance(raw.get("source"), str):
|
||||
_fail("invalid_source", source=raw.get("source"), index=index)
|
||||
elif raw.get("source") == "":
|
||||
source = None
|
||||
else:
|
||||
source = raw.get("source")
|
||||
claims_raw = raw.get("claims")
|
||||
if claims_raw is None:
|
||||
claims_raw = []
|
||||
if not isinstance(claims_raw, list):
|
||||
_fail("claims_shape", index=index)
|
||||
source_id = str(raw.get("source_id") or "").strip()
|
||||
if not source_id:
|
||||
_fail("missing_source_id", index=index)
|
||||
if ASSISTANT_SOURCE_ID.match(source_id):
|
||||
_fail("assistant_as_fact", source_id=source_id, index=index)
|
||||
if source_id not in user_by_id:
|
||||
_fail("unknown_source_id", source_id=source_id, index=index)
|
||||
if source_id in seen:
|
||||
_fail("duplicate_source_id", source_id=source_id, index=index)
|
||||
seen.append(source_id)
|
||||
if source is None:
|
||||
_fail("missing_source", source_id=source_id, index=index)
|
||||
if source != CONTENT_ROLE:
|
||||
_fail("invalid_source", source=source, source_id=source_id, index=index)
|
||||
claims = [
|
||||
_normalize_claim(claim, index=index, claim_index=claim_index)
|
||||
for claim_index, claim in enumerate(claims_raw)
|
||||
]
|
||||
if not claims:
|
||||
_fail("empty_event", source_id=source_id, index=index)
|
||||
stamp = raw.get("time")
|
||||
if isinstance(stamp, str):
|
||||
stamp = stamp.strip() or None
|
||||
elif stamp is None:
|
||||
stamp = None
|
||||
else:
|
||||
stamp = str(stamp).strip() or None
|
||||
cited = user_by_id[source_id]
|
||||
if stamp:
|
||||
if (
|
||||
not CLOCK_STAMP_RE.match(stamp)
|
||||
or parse_clock(stamp) is None
|
||||
or not clock_in_source(stamp, cited)
|
||||
):
|
||||
_fail("time_source_mismatch", source_id=source_id, time=stamp, index=index)
|
||||
items.append({"source_id": source_id, "time": stamp, "claims": claims})
|
||||
for claim in claims:
|
||||
evidence_refs.append(EvidenceRef(source_id, claim["evidence"]))
|
||||
|
||||
missing = [sid for sid in expected_ids if sid not in seen]
|
||||
if missing:
|
||||
_fail("incomplete_coverage", missing=missing)
|
||||
if expected_ids and not items:
|
||||
_fail("empty_chronology")
|
||||
|
||||
if not _reorder_allowed(items, expected_ids, user_by_id):
|
||||
_fail(
|
||||
"reorder_unjustified",
|
||||
expected=expected_ids,
|
||||
chronology=[item["source_id"] for item in items],
|
||||
)
|
||||
|
||||
for key in SOURCED_LIST_KEYS:
|
||||
raw_list = data.get(key)
|
||||
if raw_list is None:
|
||||
continue
|
||||
if not isinstance(raw_list, list):
|
||||
_fail("sourced_list_shape", key=key)
|
||||
for index, raw in enumerate(raw_list):
|
||||
row = _normalize_sourced_item(raw, key=key, index=index)
|
||||
if ASSISTANT_SOURCE_ID.match(row["source_id"]):
|
||||
_fail("assistant_as_fact", source_id=row["source_id"], key=key)
|
||||
evidence_refs.append(EvidenceRef(row["source_id"], row["evidence"]))
|
||||
|
||||
registry = _registry_from_messages(labeled)
|
||||
policy = _journal_policy(expected_ids, [item["source_id"] for item in items])
|
||||
try:
|
||||
artifact = verify(registry, policy=policy, evidence=evidence_refs)
|
||||
except ProvenanceError as exc:
|
||||
_map_provenance(exc)
|
||||
return artifact.to_payload()
|
||||
|
||||
|
||||
def local_verified_artifact(messages: list[dict]) -> dict[str, Any]:
|
||||
"""Authoritative coverage from local user sources. No model JSON required."""
|
||||
labeled = assign_source_ids(messages)
|
||||
users = [
|
||||
item
|
||||
for item in labeled
|
||||
if item.get("role") == CONTENT_ROLE and item.get("source_id") and (item.get("body") or "").strip()
|
||||
]
|
||||
if not users:
|
||||
raise JournalBudgetError(
|
||||
ERROR_NO_USER_SOURCES,
|
||||
diagnostics={"reason": "no_user_sources"},
|
||||
)
|
||||
expected_ids = [item["source_id"] for item in users]
|
||||
registry = _registry_from_messages(labeled)
|
||||
evidence = [EvidenceRef(item["source_id"], (item.get("body") or "").strip()) for item in users]
|
||||
policy = _journal_policy(expected_ids, expected_ids)
|
||||
try:
|
||||
artifact = verify(registry, policy=policy, evidence=evidence)
|
||||
except ProvenanceError as exc:
|
||||
_map_provenance(exc)
|
||||
return artifact.to_payload()
|
||||
|
||||
|
||||
def reconstruction_from_model(raw_content: str, messages: list[dict]) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Stage 1 model is untrusted. Invalid output never drops local user sources."""
|
||||
try:
|
||||
parsed = parse_reconstruction(raw_content)
|
||||
artifact = validate_reconstruction(parsed, messages=messages)
|
||||
return artifact, {"stage1": "model_accepted"}
|
||||
except JournalBudgetError as exc:
|
||||
if exc.code == ERROR_NO_USER_SOURCES:
|
||||
raise
|
||||
artifact = local_verified_artifact(messages)
|
||||
return artifact, {
|
||||
"stage1": "local_fallback",
|
||||
"model_rejected": exc.code,
|
||||
"reason": (exc.diagnostics or {}).get("reason") or exc.code,
|
||||
}
|
||||
|
||||
|
||||
def reconstruction_text(data: dict[str, Any]) -> str:
|
||||
return json.dumps(data, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def _claim_for_body(body: str) -> list[dict[str, str]]:
|
||||
claims = [{"kind": "event", "evidence": body}]
|
||||
if TIME_RE.search(body):
|
||||
stamp = TIME_RE.search(body).group(0)
|
||||
claims.append({"kind": "time", "evidence": stamp})
|
||||
return claims
|
||||
|
||||
|
||||
def fake_reconstruction(rendered: str) -> str:
|
||||
"""Deterministic stage-1 stand-in. Claims remain untrusted; the adapter rehydrates."""
|
||||
chronology = []
|
||||
source_order: list[str] = []
|
||||
user_n = 0
|
||||
for line in (rendered or "").splitlines():
|
||||
parsed = parse_dialogue_line(line)
|
||||
if parsed is None and line.startswith("user:"):
|
||||
parsed = {"role": "user", "body": line[5:].strip(), "source_id": ""}
|
||||
if not parsed or parsed["role"] != "user":
|
||||
continue
|
||||
body = (parsed["body"] or "").strip()
|
||||
if not body:
|
||||
continue
|
||||
user_n += 1
|
||||
source_id = parsed["source_id"] if USER_SOURCE_ID.match(parsed["source_id"] or "") else f"u{user_n}"
|
||||
source_order.append(source_id)
|
||||
times = TIME_RE.findall(body)
|
||||
chronology.append(
|
||||
{
|
||||
"source_id": source_id,
|
||||
"source": "user",
|
||||
"time": times[0] if times else None,
|
||||
"claims": _claim_for_body(body),
|
||||
}
|
||||
)
|
||||
payload = {
|
||||
"source_order": source_order,
|
||||
"chronology": chronology,
|
||||
"contradictions": [],
|
||||
"uncertainties": [],
|
||||
"plan_changes": [],
|
||||
"corrections": [],
|
||||
}
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def claim_texts(data: dict[str, Any]) -> list[str]:
|
||||
parts = texts_from_payload(data)
|
||||
if parts:
|
||||
return parts
|
||||
collected: list[str] = []
|
||||
for item in data.get("chronology") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
for claim in item.get("claims") or []:
|
||||
if isinstance(claim, dict) and claim.get("evidence"):
|
||||
collected.append(str(claim["evidence"]))
|
||||
return collected
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Local journal shape after the model. Does not invent or rewrite content."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
NEW_PHASE = re.compile(
|
||||
r"^(?:Heute|Gestern|Morgens|Mittags|Abends|Nachts|Danach|Später|Dann|Zuerst|"
|
||||
r"Schließlich|Irgendwann|Unterwegs|Zurück|"
|
||||
r"Um\s+\d|Nach\s+(?:ca\.|einer|einem|dem|der|kurzer|weiteren)|"
|
||||
r"Als\s+(?:wir|ich)\b)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def _strip_roles(text: str) -> str:
|
||||
lines = [re.sub(r"^(?:user|assistant):\s*", "", line, flags=re.I) for line in (text or "").splitlines()]
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _norm(text: str) -> str:
|
||||
return re.sub(r"[^\w\s]+", " ", re.sub(r"\s+", " ", _strip_roles(text).lower())).strip()
|
||||
|
||||
|
||||
def _turns(user_bodies: list[str]) -> list[str]:
|
||||
return [re.sub(r"\s+", " ", item).strip() for item in user_bodies if (item or "").strip()]
|
||||
|
||||
|
||||
def is_verbatim_join(body: str, user_bodies: list[str]) -> bool:
|
||||
"""Diagnostic helper. Must not be used to overwrite accepted model text."""
|
||||
turns = _turns(user_bodies)
|
||||
if not turns or not (body or "").strip():
|
||||
return False
|
||||
got = _norm(body)
|
||||
return got == _norm(" ".join(turns)) or got == _norm("\n\n".join(turns))
|
||||
|
||||
|
||||
def paragraphize(body: str) -> str:
|
||||
"""Optional local preview helper. Must not rewrite an accepted model draft."""
|
||||
text = (body or "").strip()
|
||||
if "\n\n" in text:
|
||||
return text
|
||||
sentences = [item.strip() for item in re.split(r"(?<=[.!?])\s+", text) if item.strip()]
|
||||
if len(sentences) < 4:
|
||||
return text
|
||||
chunks: list[str] = []
|
||||
current: list[str] = []
|
||||
for sentence in sentences:
|
||||
split = bool(current) and (NEW_PHASE.match(sentence) or len(current) >= 3)
|
||||
if split:
|
||||
chunks.append(" ".join(current))
|
||||
current = [sentence]
|
||||
else:
|
||||
current.append(sentence)
|
||||
if current:
|
||||
chunks.append(" ".join(current))
|
||||
return "\n\n".join(chunks)
|
||||
|
||||
|
||||
def shape_journal(
|
||||
title: str,
|
||||
body: str,
|
||||
user_bodies: list[str],
|
||||
person_labels: list[str] | None = None,
|
||||
*,
|
||||
source: str = "model",
|
||||
) -> tuple[str, str]:
|
||||
"""Safe technical trim only. Never replaces names, pronouns, or accepted wording."""
|
||||
del user_bodies, person_labels
|
||||
heading = (title or "").strip()
|
||||
text = _strip_roles(body or "").strip()
|
||||
if source == "fallback" and "\n\n" not in text:
|
||||
text = paragraphize(text)
|
||||
return heading, text
|
||||
@@ -0,0 +1,790 @@
|
||||
"""Journal Day, Draft, Entry and Space product store. Source messages stay in dialogue_store."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from dialogue_store import (
|
||||
StoreError,
|
||||
create_conversation,
|
||||
create_space,
|
||||
get_space,
|
||||
list_conversations_for_day,
|
||||
list_spaces,
|
||||
rename_space,
|
||||
start_usage_session,
|
||||
)
|
||||
from db import _insert_source_refs, get_db, row_to_dict
|
||||
from journal_policy import consolidation_offer, require_origin
|
||||
from journal_body import clean_title, to_markdown
|
||||
|
||||
DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
SCRATCH_MAX_ITEMS = 40
|
||||
SCRATCH_MAX_TEXT = 400
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def _parse_ids(raw: str | None) -> list[str]:
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
return [str(item) for item in data]
|
||||
|
||||
|
||||
def _load_source_ids(conn, table: str, owner_col: str, owner_id: str, row: dict | None) -> tuple[list[str], list[str]]:
|
||||
refs = conn.execute(
|
||||
f"""
|
||||
SELECT source_kind, source_id, sort_order
|
||||
FROM {table}
|
||||
WHERE {owner_col} = ?
|
||||
ORDER BY source_kind, sort_order
|
||||
""",
|
||||
(owner_id,),
|
||||
).fetchall()
|
||||
conversations: list[str] = []
|
||||
messages: list[str] = []
|
||||
for item in refs:
|
||||
kind = item["source_kind"]
|
||||
source_id = item["source_id"]
|
||||
if kind == "conversation":
|
||||
conversations.append(source_id)
|
||||
elif kind == "message":
|
||||
messages.append(source_id)
|
||||
if conversations or messages:
|
||||
return conversations, messages
|
||||
row = row or {}
|
||||
return _parse_ids(row.get("source_conversation_ids")), _parse_ids(row.get("source_message_ids"))
|
||||
|
||||
|
||||
def _attach_sources(conn, table: str, owner_col: str, owner_id: str, row: dict | None) -> dict | None:
|
||||
if not row:
|
||||
return None
|
||||
conversations, messages = _load_source_ids(conn, table, owner_col, owner_id, row)
|
||||
row["source_conversation_ids"] = conversations
|
||||
row["source_message_ids"] = messages
|
||||
row["title"] = clean_title(row.get("title") or "")
|
||||
return row
|
||||
|
||||
|
||||
def _owned(conn, table: str, record_id: str, profile_id: str) -> dict | None:
|
||||
return row_to_dict(
|
||||
conn.execute(
|
||||
f"SELECT * FROM {table} WHERE id = ? AND profile_id = ?",
|
||||
(record_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
|
||||
|
||||
def _decode_draft(conn, row: dict | None) -> dict | None:
|
||||
if not row:
|
||||
return None
|
||||
payload = _attach_sources(conn, "journal_draft_source_refs", "draft_id", row["id"], row)
|
||||
raw = payload.get("generation_snapshot") or "{}"
|
||||
try:
|
||||
snapshot = json.loads(raw) if isinstance(raw, str) else (raw or {})
|
||||
except json.JSONDecodeError:
|
||||
snapshot = {}
|
||||
if not isinstance(snapshot, dict):
|
||||
snapshot = {}
|
||||
payload["generation_snapshot"] = snapshot
|
||||
from journal_generation_policy import snapshot_summary
|
||||
|
||||
payload["generation_summary"] = snapshot_summary(snapshot)
|
||||
return payload
|
||||
|
||||
|
||||
def _decode_version(conn, row: dict | None) -> dict | None:
|
||||
if not row:
|
||||
return None
|
||||
owner_id = row.get("current_version_id") or row.get("id")
|
||||
return _attach_sources(conn, "journal_entry_version_source_refs", "version_id", owner_id, row)
|
||||
|
||||
|
||||
def create_user_space(profile_id: str, title: str) -> dict:
|
||||
if not (title or "").strip():
|
||||
raise StoreError("empty_title", "Space-Name darf nicht leer sein")
|
||||
return create_space(profile_id, title.strip(), visibility="user")
|
||||
|
||||
|
||||
def list_user_spaces(profile_id: str) -> list[dict]:
|
||||
return list_spaces(profile_id, visibility="user")
|
||||
|
||||
|
||||
def get_user_space(profile_id: str, space_id: str) -> dict:
|
||||
space = get_space(profile_id, space_id)
|
||||
if space.get("visibility") != "user":
|
||||
raise StoreError("not_found", "Reflection Space nicht gefunden", 404)
|
||||
return space
|
||||
|
||||
|
||||
def rename_user_space(profile_id: str, space_id: str, title: str) -> dict:
|
||||
get_user_space(profile_id, space_id)
|
||||
return rename_space(profile_id, space_id, title)
|
||||
|
||||
|
||||
def get_or_create_day(profile_id: str, space_id: str, calendar_date: str) -> dict:
|
||||
get_user_space(profile_id, space_id)
|
||||
if not DATE_RE.match(calendar_date or ""):
|
||||
raise StoreError("invalid_date", "calendar_date muss YYYY-MM-DD sein")
|
||||
with get_db() as conn:
|
||||
existing = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM journal_days
|
||||
WHERE profile_id = ? AND space_id = ? AND calendar_date = ?
|
||||
""",
|
||||
(profile_id, space_id, calendar_date),
|
||||
).fetchone()
|
||||
)
|
||||
if existing:
|
||||
return existing
|
||||
day_id = str(uuid.uuid4())
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO journal_days (id, profile_id, space_id, calendar_date)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(day_id, profile_id, space_id, calendar_date),
|
||||
)
|
||||
return row_to_dict(conn.execute("SELECT * FROM journal_days WHERE id = ?", (day_id,)).fetchone())
|
||||
|
||||
|
||||
def get_day(profile_id: str, journal_day_id: str) -> dict:
|
||||
with get_db() as conn:
|
||||
day = _owned(conn, "journal_days", journal_day_id, profile_id)
|
||||
if not day:
|
||||
raise StoreError("not_found", "Journal Day nicht gefunden", 404)
|
||||
return day
|
||||
|
||||
|
||||
def list_days(profile_id: str, space_id: str) -> list[dict]:
|
||||
get_user_space(profile_id, space_id)
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT d.*,
|
||||
(SELECT COUNT(*) FROM conversations c
|
||||
WHERE c.journal_day_id = d.id AND c.profile_id = d.profile_id) AS conversation_count,
|
||||
(SELECT COUNT(*) FROM journal_entries e
|
||||
WHERE e.journal_day_id = d.id AND e.profile_id = d.profile_id AND e.deleted_at IS NULL
|
||||
) AS entry_count
|
||||
FROM journal_days d
|
||||
WHERE d.profile_id = ? AND d.space_id = ?
|
||||
ORDER BY d.calendar_date DESC
|
||||
""",
|
||||
(profile_id, space_id),
|
||||
).fetchall()
|
||||
result = []
|
||||
for row in rows:
|
||||
item = row_to_dict(row)
|
||||
entry = current_entries(profile_id, item["id"])
|
||||
item["entry_title"] = (entry[-1].get("title") if entry else "") or ""
|
||||
item["entry_id"] = entry[-1]["id"] if entry else None
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def create_day_conversation(profile_id: str, journal_day_id: str, title: str = "") -> dict:
|
||||
day = get_day(profile_id, journal_day_id)
|
||||
usage = start_usage_session(profile_id, intent="journal")
|
||||
return create_conversation(
|
||||
profile_id,
|
||||
usage_session_id=usage["id"],
|
||||
title=title or "Gespräch",
|
||||
space_id=day["space_id"],
|
||||
journal_day_id=day["id"],
|
||||
)
|
||||
|
||||
|
||||
def current_draft(profile_id: str, journal_day_id: str) -> dict | None:
|
||||
get_day(profile_id, journal_day_id)
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM journal_drafts
|
||||
WHERE profile_id = ? AND journal_day_id = ? AND superseded_at IS NULL
|
||||
ORDER BY created DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(profile_id, journal_day_id),
|
||||
).fetchone()
|
||||
)
|
||||
return _decode_draft(conn, row)
|
||||
|
||||
|
||||
def insert_draft(
|
||||
profile_id: str,
|
||||
journal_day_id: str,
|
||||
title: str,
|
||||
body: str,
|
||||
source_conversation_ids: list[str],
|
||||
source_message_ids: list[str],
|
||||
generation_snapshot: dict | None = None,
|
||||
) -> dict:
|
||||
get_day(profile_id, journal_day_id)
|
||||
draft_id = str(uuid.uuid4())
|
||||
as_of = _now()
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE journal_drafts
|
||||
SET superseded_at = ?
|
||||
WHERE profile_id = ? AND journal_day_id = ? AND superseded_at IS NULL
|
||||
""",
|
||||
(as_of, profile_id, journal_day_id),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO journal_drafts
|
||||
(id, profile_id, journal_day_id, title, body,
|
||||
source_conversation_ids, source_message_ids, as_of, generation_snapshot)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
draft_id,
|
||||
profile_id,
|
||||
journal_day_id,
|
||||
clean_title(title),
|
||||
body or "",
|
||||
"[]",
|
||||
"[]",
|
||||
as_of,
|
||||
json.dumps(generation_snapshot or {}, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
_insert_source_refs(
|
||||
conn,
|
||||
"journal_draft_source_refs",
|
||||
"draft_id",
|
||||
draft_id,
|
||||
profile_id,
|
||||
source_conversation_ids or [],
|
||||
source_message_ids or [],
|
||||
)
|
||||
row = row_to_dict(conn.execute("SELECT * FROM journal_drafts WHERE id = ?", (draft_id,)).fetchone())
|
||||
return _decode_draft(conn, row)
|
||||
|
||||
|
||||
def current_entries(profile_id: str, journal_day_id: str) -> list[dict]:
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT e.*, v.title, v.body, v.origin, v.source_conversation_ids, v.source_message_ids, v.created AS version_created
|
||||
FROM journal_entries e
|
||||
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
||||
WHERE e.profile_id = ? AND e.journal_day_id = ? AND e.deleted_at IS NULL
|
||||
ORDER BY e.created, e.id
|
||||
""",
|
||||
(profile_id, journal_day_id),
|
||||
).fetchall()
|
||||
return [_decode_version(conn, row_to_dict(row)) for row in rows]
|
||||
|
||||
|
||||
def get_entry(profile_id: str, entry_id: str, include_deleted: bool = False) -> dict:
|
||||
with get_db() as conn:
|
||||
entry = _owned(conn, "journal_entries", entry_id, profile_id)
|
||||
if not entry:
|
||||
raise StoreError("not_found", "Journal Entry nicht gefunden", 404)
|
||||
if entry.get("deleted_at") and not include_deleted:
|
||||
raise StoreError("not_found", "Journal Entry nicht gefunden", 404)
|
||||
version = None
|
||||
if entry.get("current_version_id"):
|
||||
version = _decode_version(
|
||||
conn,
|
||||
row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT * FROM journal_entry_versions WHERE id = ? AND profile_id = ?",
|
||||
(entry["current_version_id"], profile_id),
|
||||
).fetchone()
|
||||
),
|
||||
)
|
||||
entry["version"] = version
|
||||
if version:
|
||||
entry["title"] = version.get("title") or ""
|
||||
entry["body"] = version.get("body") or ""
|
||||
entry["origin"] = version.get("origin")
|
||||
entry["source_conversation_ids"] = version.get("source_conversation_ids") or []
|
||||
entry["source_message_ids"] = version.get("source_message_ids") or []
|
||||
return entry
|
||||
|
||||
|
||||
def list_versions(profile_id: str, entry_id: str) -> list[dict]:
|
||||
get_entry(profile_id, entry_id, include_deleted=True)
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM journal_entry_versions
|
||||
WHERE profile_id = ? AND entry_id = ?
|
||||
ORDER BY created
|
||||
""",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
return [_decode_version(conn, row_to_dict(row)) for row in rows]
|
||||
|
||||
|
||||
def save_entry(
|
||||
profile_id: str,
|
||||
journal_day_id: str,
|
||||
title: str,
|
||||
body: str,
|
||||
origin: str,
|
||||
entry_id: str | None = None,
|
||||
source_conversation_ids: list[str] | None = None,
|
||||
source_message_ids: list[str] | None = None,
|
||||
) -> dict:
|
||||
require_origin(origin)
|
||||
day = get_day(profile_id, journal_day_id)
|
||||
version_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
if entry_id:
|
||||
entry = _owned(conn, "journal_entries", entry_id, profile_id)
|
||||
if not entry or entry.get("deleted_at"):
|
||||
raise StoreError("not_found", "Journal Entry nicht gefunden", 404)
|
||||
if entry["journal_day_id"] != journal_day_id:
|
||||
raise StoreError("assignment_mismatch", "Entry gehört zu einem anderen Journal Day")
|
||||
else:
|
||||
entry_id = str(uuid.uuid4())
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO journal_entries (id, profile_id, journal_day_id, space_id)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(entry_id, profile_id, journal_day_id, day["space_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO journal_entry_versions
|
||||
(id, profile_id, entry_id, title, body, source_conversation_ids, source_message_ids, origin, created)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
version_id,
|
||||
profile_id,
|
||||
entry_id,
|
||||
clean_title(title),
|
||||
to_markdown(body or ""),
|
||||
"[]",
|
||||
"[]",
|
||||
origin,
|
||||
_now(),
|
||||
),
|
||||
)
|
||||
_insert_source_refs(
|
||||
conn,
|
||||
"journal_entry_version_source_refs",
|
||||
"version_id",
|
||||
version_id,
|
||||
profile_id,
|
||||
source_conversation_ids or [],
|
||||
source_message_ids or [],
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE journal_entries
|
||||
SET current_version_id = ?, updated = datetime('now'), deleted_at = NULL
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(version_id, entry_id, profile_id),
|
||||
)
|
||||
return get_entry(profile_id, entry_id)
|
||||
|
||||
|
||||
def restore_entry(profile_id: str, entry_id: str, version_id: str) -> dict:
|
||||
entry = get_entry(profile_id, entry_id)
|
||||
with get_db() as conn:
|
||||
version = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM journal_entry_versions
|
||||
WHERE id = ? AND entry_id = ? AND profile_id = ?
|
||||
""",
|
||||
(version_id, entry_id, profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not version:
|
||||
raise StoreError("not_found", "Version nicht gefunden", 404)
|
||||
decoded = _decode_version(conn, version)
|
||||
return save_entry(
|
||||
profile_id,
|
||||
entry["journal_day_id"],
|
||||
decoded.get("title") or "",
|
||||
decoded.get("body") or "",
|
||||
origin="restore",
|
||||
entry_id=entry_id,
|
||||
source_conversation_ids=decoded.get("source_conversation_ids") or [],
|
||||
source_message_ids=decoded.get("source_message_ids") or [],
|
||||
)
|
||||
|
||||
|
||||
def soft_delete_entry(profile_id: str, entry_id: str) -> dict:
|
||||
get_entry(profile_id, entry_id)
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE journal_entries SET deleted_at = ?, updated = datetime('now') WHERE id = ? AND profile_id = ?",
|
||||
(_now(), entry_id, profile_id),
|
||||
)
|
||||
return {"id": entry_id, "deleted": True}
|
||||
|
||||
|
||||
def _parse_scratch(raw) -> list[dict]:
|
||||
if isinstance(raw, list):
|
||||
data = raw
|
||||
else:
|
||||
try:
|
||||
data = json.loads(raw or "[]")
|
||||
except json.JSONDecodeError:
|
||||
data = []
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
items: list[dict] = []
|
||||
for item in data[:SCRATCH_MAX_ITEMS]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
text = str(item.get("text") or "")[:SCRATCH_MAX_TEXT]
|
||||
items.append(
|
||||
{
|
||||
"id": str(item.get("id") or uuid.uuid4()),
|
||||
"text": text,
|
||||
"done": bool(item.get("done")),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def save_day_scratch(profile_id: str, journal_day_id: str, items: list[dict]) -> list[dict]:
|
||||
get_day(profile_id, journal_day_id)
|
||||
cleaned = _parse_scratch(items)
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE journal_days SET scratch_json = ?, updated = datetime('now') WHERE id = ? AND profile_id = ?",
|
||||
(json.dumps(cleaned, ensure_ascii=False), journal_day_id, profile_id),
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
def day_payload(profile_id: str, journal_day_id: str) -> dict:
|
||||
day = get_day(profile_id, journal_day_id)
|
||||
space = get_user_space(profile_id, day["space_id"])
|
||||
conversations = list_conversations_for_day(profile_id, journal_day_id)
|
||||
return {
|
||||
"day": {key: value for key, value in day.items() if key != "scratch_json"},
|
||||
"space": space,
|
||||
"conversations": conversations,
|
||||
"current_draft": current_draft(profile_id, journal_day_id),
|
||||
"entries": current_entries(profile_id, journal_day_id),
|
||||
"scratch": _parse_scratch(day.get("scratch_json")),
|
||||
"consolidation_offer": consolidation_offer(conversations),
|
||||
}
|
||||
|
||||
|
||||
def _entry_list_row(conn, row) -> dict:
|
||||
item = row_to_dict(row)
|
||||
item["title"] = clean_title(item.get("title") or "")
|
||||
item["version_count"] = int(item.get("version_count") or 0)
|
||||
return item
|
||||
|
||||
|
||||
def list_space_entries(profile_id: str, space_id: str) -> list[dict]:
|
||||
"""Chronological journal entries of a space. Deleted entries are excluded."""
|
||||
get_user_space(profile_id, space_id)
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT e.id, e.profile_id, e.space_id, e.journal_day_id, e.current_version_id,
|
||||
e.created, e.updated, e.deleted_at,
|
||||
d.calendar_date, v.title, v.origin,
|
||||
(SELECT COUNT(*) FROM journal_entry_versions ver
|
||||
WHERE ver.entry_id = e.id AND ver.profile_id = e.profile_id) AS version_count,
|
||||
(SELECT r.source_id FROM journal_entry_version_source_refs r
|
||||
WHERE r.version_id = e.current_version_id AND r.source_kind = 'conversation'
|
||||
ORDER BY r.sort_order LIMIT 1) AS source_conversation_id
|
||||
FROM journal_entries e
|
||||
JOIN journal_days d ON d.id = e.journal_day_id
|
||||
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
||||
WHERE e.profile_id = ? AND e.space_id = ? AND e.deleted_at IS NULL
|
||||
ORDER BY d.calendar_date DESC, e.created ASC, e.id ASC
|
||||
""",
|
||||
(profile_id, space_id),
|
||||
).fetchall()
|
||||
return [_entry_list_row(conn, row) for row in rows]
|
||||
|
||||
|
||||
def list_deleted_entries(profile_id: str, space_id: str | None = None) -> list[dict]:
|
||||
if space_id:
|
||||
get_user_space(profile_id, space_id)
|
||||
params: list = [profile_id]
|
||||
space_sql = ""
|
||||
if space_id:
|
||||
space_sql = "AND e.space_id = ?"
|
||||
params.append(space_id)
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT e.id, e.profile_id, e.space_id, e.journal_day_id, e.current_version_id,
|
||||
e.created, e.updated, e.deleted_at,
|
||||
d.calendar_date, v.title, v.origin,
|
||||
(SELECT COUNT(*) FROM journal_entry_versions ver
|
||||
WHERE ver.entry_id = e.id AND ver.profile_id = e.profile_id) AS version_count,
|
||||
(SELECT r.source_id FROM journal_entry_version_source_refs r
|
||||
WHERE r.version_id = e.current_version_id AND r.source_kind = 'conversation'
|
||||
ORDER BY r.sort_order LIMIT 1) AS source_conversation_id
|
||||
FROM journal_entries e
|
||||
JOIN journal_days d ON d.id = e.journal_day_id
|
||||
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
||||
WHERE e.profile_id = ? {space_sql} AND e.deleted_at IS NOT NULL
|
||||
ORDER BY e.deleted_at DESC, e.id ASC
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
return [_entry_list_row(conn, row) for row in rows]
|
||||
|
||||
|
||||
def undelete_entry(profile_id: str, entry_id: str) -> dict:
|
||||
"""Return a soft-deleted entry to the space list. Not a version restore."""
|
||||
entry = get_entry(profile_id, entry_id, include_deleted=True)
|
||||
if not entry.get("deleted_at"):
|
||||
raise StoreError("not_deleted", "Eintrag ist nicht im Papierkorb.")
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE journal_entries
|
||||
SET deleted_at = NULL, updated = datetime('now')
|
||||
WHERE id = ? AND profile_id = ?
|
||||
""",
|
||||
(entry_id, profile_id),
|
||||
)
|
||||
return get_entry(profile_id, entry_id)
|
||||
|
||||
|
||||
def continuable_journal_day(profile_id: str) -> dict | None:
|
||||
"""Most recently active journal day that already has a stored message."""
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT d.id AS journal_day_id, d.space_id, d.calendar_date,
|
||||
s.title AS space_title, c.id AS conversation_id, m.created AS last_message_at
|
||||
FROM messages m
|
||||
JOIN conversations c ON c.id = m.conversation_id AND c.profile_id = m.profile_id
|
||||
JOIN journal_days d ON d.id = c.journal_day_id AND d.profile_id = m.profile_id
|
||||
JOIN spaces s ON s.id = d.space_id AND s.profile_id = m.profile_id
|
||||
WHERE m.profile_id = ? AND s.visibility = 'user'
|
||||
ORDER BY m.created DESC, m.seq DESC, m.id DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchone()
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
"space_id": row["space_id"],
|
||||
"space_title": row.get("space_title") or "",
|
||||
"journal_day_id": row["journal_day_id"],
|
||||
"calendar_date": row["calendar_date"],
|
||||
"conversation_id": row["conversation_id"],
|
||||
"last_message_at": row.get("last_message_at"),
|
||||
"reason": "recent_dialogue",
|
||||
}
|
||||
|
||||
|
||||
def start_payload(profile_id: str) -> dict:
|
||||
spaces = list_user_spaces(profile_id)
|
||||
return {
|
||||
"continuable": continuable_journal_day(profile_id),
|
||||
"has_spaces": bool(spaces),
|
||||
"space_count": len(spaces),
|
||||
"spaces": [{"id": item["id"], "title": item.get("title") or ""} for item in spaces],
|
||||
}
|
||||
|
||||
|
||||
def _bodies_referencing_media(conn, profile_id: str, media_id: str, exclude_entry_id: str) -> list[str]:
|
||||
from journal_body import media_ids
|
||||
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT v.entry_id, v.body
|
||||
FROM journal_entry_versions v
|
||||
WHERE v.profile_id = ? AND v.entry_id != ?
|
||||
""",
|
||||
(profile_id, exclude_entry_id),
|
||||
).fetchall()
|
||||
owners: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for row in rows:
|
||||
if media_id in media_ids(row["body"] or "") and row["entry_id"] not in seen:
|
||||
seen.add(row["entry_id"])
|
||||
owners.append(row["entry_id"])
|
||||
return owners
|
||||
|
||||
|
||||
def inventory_entry_for_purge(profile_id: str, entry_id: str) -> dict:
|
||||
entry = get_entry(profile_id, entry_id, include_deleted=True)
|
||||
with get_db() as conn:
|
||||
versions = [
|
||||
row_to_dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT id, origin, created FROM journal_entry_versions WHERE profile_id = ? AND entry_id = ? ORDER BY created",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
]
|
||||
version_ids = [item["id"] for item in versions]
|
||||
source_refs = 0
|
||||
if version_ids:
|
||||
placeholders = ",".join("?" * len(version_ids))
|
||||
source_refs = conn.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS n FROM journal_entry_version_source_refs
|
||||
WHERE version_id IN ({placeholders}) AND profile_id = ?
|
||||
""",
|
||||
(*version_ids, profile_id),
|
||||
).fetchone()["n"]
|
||||
writing_sources = [
|
||||
row_to_dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT id, kind FROM writing_profile_sources WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
]
|
||||
evidence = [
|
||||
row_to_dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM writing_profile_evidence WHERE profile_id = ? AND source_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
]
|
||||
media_rows = [
|
||||
row_to_dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT * FROM media_assets WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
]
|
||||
media = []
|
||||
from media_store import media_root
|
||||
|
||||
root = media_root()
|
||||
for row in media_rows:
|
||||
owners = _bodies_referencing_media(conn, profile_id, row["id"], entry_id)
|
||||
path = root / row["rel_path"]
|
||||
media.append(
|
||||
{
|
||||
"id": row["id"],
|
||||
"rel_path": row["rel_path"],
|
||||
"path_exists": path.is_file(),
|
||||
"referenced_by_entries": owners,
|
||||
"keep_file": bool(owners),
|
||||
"reassign_to": owners[0] if owners else None,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"entry": {"id": entry["id"], "space_id": entry["space_id"], "deleted_at": entry.get("deleted_at")},
|
||||
"versions": versions,
|
||||
"source_ref_count": int(source_refs or 0),
|
||||
"writing_profile_sources": writing_sources,
|
||||
"writing_profile_evidence": evidence,
|
||||
"media": media,
|
||||
"conversations_untouched": True,
|
||||
}
|
||||
|
||||
|
||||
def purge_entry(profile_id: str, entry_id: str, *, confirm: bool) -> dict:
|
||||
"""Irreversible delete after explicit confirm. Source dialogues stay."""
|
||||
if not confirm:
|
||||
raise StoreError("confirm_required", "Endgültiges Löschen braucht eine ausdrückliche Bestätigung.")
|
||||
entry = get_entry(profile_id, entry_id, include_deleted=True)
|
||||
if not entry.get("deleted_at"):
|
||||
raise StoreError("not_in_trash", "Endgültiges Löschen nur aus dem Papierkorb.")
|
||||
inventory = inventory_entry_for_purge(profile_id, entry_id)
|
||||
from media_store import media_root
|
||||
|
||||
root = media_root()
|
||||
files_to_delete: list[Path] = []
|
||||
try:
|
||||
with get_db() as conn:
|
||||
for item in inventory["media"]:
|
||||
if item.get("keep_file") and item.get("reassign_to"):
|
||||
other = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT id FROM journal_entries WHERE id = ? AND profile_id = ?",
|
||||
(item["reassign_to"], profile_id),
|
||||
).fetchone()
|
||||
)
|
||||
if not other:
|
||||
raise StoreError("purge_conflict", "Geteiltes Medium hat keinen sicheren neuen Träger.")
|
||||
conn.execute(
|
||||
"UPDATE media_assets SET entry_id = ? WHERE id = ? AND profile_id = ?",
|
||||
(item["reassign_to"], item["id"], profile_id),
|
||||
)
|
||||
elif not item.get("keep_file"):
|
||||
files_to_delete.append(root / item["rel_path"])
|
||||
conn.execute(
|
||||
"DELETE FROM writing_profile_sources WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM writing_profile_evidence WHERE profile_id = ? AND source_id = ?",
|
||||
(profile_id, entry_id),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM journal_entries WHERE id = ? AND profile_id = ?",
|
||||
(entry_id, profile_id),
|
||||
)
|
||||
leftover = conn.execute(
|
||||
"SELECT id FROM journal_entries WHERE id = ? AND profile_id = ?",
|
||||
(entry_id, profile_id),
|
||||
).fetchone()
|
||||
if leftover:
|
||||
raise StoreError("purge_failed", "Eintrag konnte nicht vollständig gelöscht werden.", 500)
|
||||
leftover_media = conn.execute(
|
||||
"SELECT id FROM media_assets WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchone()
|
||||
if leftover_media:
|
||||
raise StoreError("purge_failed", "Medienverweise konnten nicht aufgelöst werden.", 500)
|
||||
leftover_sources = conn.execute(
|
||||
"SELECT id FROM writing_profile_sources WHERE profile_id = ? AND entry_id = ?",
|
||||
(profile_id, entry_id),
|
||||
).fetchone()
|
||||
if leftover_sources:
|
||||
raise StoreError("purge_failed", "Writing-Profile-Quellen konnten nicht gelöscht werden.", 500)
|
||||
except StoreError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise StoreError("purge_failed", "Endgültiges Löschen wurde abgebrochen, nichts wurde entfernt.", 500) from exc
|
||||
|
||||
removed_files: list[str] = []
|
||||
missing_files: list[str] = []
|
||||
for path in files_to_delete:
|
||||
if not path.is_file():
|
||||
missing_files.append(str(path))
|
||||
continue
|
||||
try:
|
||||
path.unlink()
|
||||
removed_files.append(str(path))
|
||||
except OSError as exc:
|
||||
raise StoreError(
|
||||
"purge_failed",
|
||||
"Datenbank entfernt, Mediendatei konnte nicht gelöscht werden. Vorgang fail-closed gemeldet.",
|
||||
500,
|
||||
) from exc
|
||||
return {
|
||||
"id": entry_id,
|
||||
"purged": True,
|
||||
"inventory": inventory,
|
||||
"removed_files": len(removed_files),
|
||||
"missing_files": missing_files,
|
||||
"conversations_untouched": True,
|
||||
}
|
||||
@@ -0,0 +1,395 @@
|
||||
"""Consistent local backup and restore for the holiday test phase.
|
||||
|
||||
Uses the SQLite backup API (not a live-file copy). Includes journal media.
|
||||
Never packs .env, provider keys, or logs. The archive can contain personal
|
||||
unencrypted journal data.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
FORMAT_KIND = "kansho.local_backup"
|
||||
FORMAT_VERSION = 1
|
||||
DB_NAME = "kansho.sqlite"
|
||||
MANIFEST_NAME = "manifest.json"
|
||||
README_NAME = "README.txt"
|
||||
MEDIA_PREFIX = "media/"
|
||||
DEFAULT_DIR_NAME = "local-backups"
|
||||
PERSONAL_DATA_WARNING = (
|
||||
"Dieses Archiv kann persönliche, unverschlüsselte Journalinhalte und Medien enthalten. "
|
||||
"Es enthält keine .env-Dateien, Provider-Keys oder Logs, ist aber selbst nicht verschlüsselt."
|
||||
)
|
||||
|
||||
EXCLUDE_NAMES = {".env", ".env.example"}
|
||||
EXCLUDE_SUFFIXES = {".log", ".pem", ".key"}
|
||||
EXCLUDE_DIR_NAMES = {"logs", "__pycache__"}
|
||||
|
||||
|
||||
class BackupError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 400, details: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
self.details = details or {}
|
||||
|
||||
|
||||
def _now_stamp() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def default_backup_dir() -> Path:
|
||||
env = (os.environ.get("KANSHO_BACKUP_DIR") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
return _repo_root() / DEFAULT_DIR_NAME
|
||||
|
||||
|
||||
def default_db_path() -> Path:
|
||||
env = (os.environ.get("KANSHO_DB_PATH") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
if data_dir:
|
||||
return Path(data_dir) / "kansho.sqlite"
|
||||
return Path(__file__).resolve().parent / "data" / "kansho.sqlite"
|
||||
|
||||
|
||||
def default_media_root() -> Path:
|
||||
env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
if data_dir:
|
||||
return Path(data_dir) / "media"
|
||||
return Path(__file__).resolve().parent / "data" / "media"
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _is_excluded(path: Path) -> bool:
|
||||
if path.name in EXCLUDE_NAMES:
|
||||
return True
|
||||
if path.suffix.lower() in EXCLUDE_SUFFIXES:
|
||||
return True
|
||||
return any(part in EXCLUDE_DIR_NAMES for part in path.parts)
|
||||
|
||||
|
||||
def list_media_files(media_root: Path) -> list[Path]:
|
||||
if not media_root.exists():
|
||||
return []
|
||||
files: list[Path] = []
|
||||
for path in sorted(media_root.rglob("*")):
|
||||
if not path.is_file() or _is_excluded(path):
|
||||
continue
|
||||
files.append(path)
|
||||
return files
|
||||
|
||||
|
||||
def sqlite_snapshot(src: Path, dest: Path) -> None:
|
||||
if not src.exists():
|
||||
raise BackupError("db_missing", f"Keine Datenbank unter {src}")
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
source = sqlite3.connect(str(src), timeout=5)
|
||||
try:
|
||||
target = sqlite3.connect(str(dest))
|
||||
try:
|
||||
source.backup(target)
|
||||
finally:
|
||||
target.close()
|
||||
except sqlite3.OperationalError as exc:
|
||||
raise BackupError("db_unavailable", f"SQLite-Snapshot fehlgeschlagen: {exc}") from exc
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
|
||||
def assert_db_idle(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
conn = sqlite3.connect(str(path), timeout=1)
|
||||
try:
|
||||
conn.execute("BEGIN EXCLUSIVE")
|
||||
conn.rollback()
|
||||
except sqlite3.OperationalError as exc:
|
||||
raise BackupError(
|
||||
"db_in_use",
|
||||
"Die Datenbank ist geöffnet. Backend beenden und Restore erneut ausführen.",
|
||||
) from exc
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def build_manifest(created: str, db_file: Path, media_files: list[tuple[str, Path]]) -> dict:
|
||||
checksums = {DB_NAME: sha256_file(db_file)}
|
||||
media_entries = []
|
||||
for rel, path in media_files:
|
||||
checksums[f"{MEDIA_PREFIX}{rel}"] = sha256_file(path)
|
||||
media_entries.append({"path": rel, "bytes": path.stat().st_size})
|
||||
return {
|
||||
"kind": FORMAT_KIND,
|
||||
"format_version": FORMAT_VERSION,
|
||||
"created": created,
|
||||
"warning": PERSONAL_DATA_WARNING,
|
||||
"contains": ["sqlite", "media"],
|
||||
"excludes": [".env", "provider keys", "logs"],
|
||||
"parts": {
|
||||
"sqlite": {"path": DB_NAME, "bytes": db_file.stat().st_size},
|
||||
"media": media_entries,
|
||||
},
|
||||
"checksums": checksums,
|
||||
}
|
||||
|
||||
|
||||
def create_backup(
|
||||
dest: Path | None = None,
|
||||
*,
|
||||
db_path: Path | None = None,
|
||||
media_root: Path | None = None,
|
||||
) -> dict:
|
||||
db_path = Path(db_path or default_db_path())
|
||||
media_root = Path(media_root or default_media_root())
|
||||
if dest is None:
|
||||
dest = default_backup_dir() / f"kansho-{_now_stamp()}.zip"
|
||||
dest = Path(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
created = _now_iso()
|
||||
with tempfile.TemporaryDirectory(prefix="kansho-backup-") as raw:
|
||||
tmp = Path(raw)
|
||||
snapshot = tmp / DB_NAME
|
||||
sqlite_snapshot(db_path, snapshot)
|
||||
media_pairs: list[tuple[str, Path]] = []
|
||||
for path in list_media_files(media_root):
|
||||
rel = path.relative_to(media_root).as_posix()
|
||||
media_pairs.append((rel, path))
|
||||
manifest = build_manifest(created, snapshot, media_pairs)
|
||||
staging = tmp / "archive"
|
||||
staging.mkdir()
|
||||
shutil.copy2(snapshot, staging / DB_NAME)
|
||||
(staging / README_NAME).write_text(PERSONAL_DATA_WARNING + "\n", encoding="utf-8")
|
||||
(staging / MANIFEST_NAME).write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
media_dir = staging / "media"
|
||||
for rel, path in media_pairs:
|
||||
target = media_dir / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, target)
|
||||
if dest.exists():
|
||||
raise BackupError("backup_exists", f"Zieldatei existiert bereits: {dest}")
|
||||
with zipfile.ZipFile(dest, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
for path in staging.rglob("*"):
|
||||
if path.is_file():
|
||||
zf.write(path, path.relative_to(staging).as_posix())
|
||||
return {"path": str(dest), "manifest": manifest}
|
||||
|
||||
|
||||
def _read_zip_text(zf: zipfile.ZipFile, name: str) -> str:
|
||||
try:
|
||||
return zf.read(name).decode("utf-8")
|
||||
except KeyError as exc:
|
||||
raise BackupError("archive_invalid", f"Archivteil fehlt: {name}") from exc
|
||||
|
||||
|
||||
def verify_archive(archive: Path) -> dict:
|
||||
archive = Path(archive)
|
||||
if not archive.is_file():
|
||||
raise BackupError("archive_missing", f"Archiv nicht gefunden: {archive}")
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
names = set(zf.namelist())
|
||||
if MANIFEST_NAME not in names:
|
||||
raise BackupError("archive_invalid", "Manifest fehlt.")
|
||||
try:
|
||||
manifest = json.loads(_read_zip_text(zf, MANIFEST_NAME))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise BackupError("archive_invalid", "Manifest ist kein JSON.") from exc
|
||||
if manifest.get("kind") != FORMAT_KIND:
|
||||
raise BackupError("archive_invalid", "Unbekanntes Backup-Format.")
|
||||
if int(manifest.get("format_version") or 0) != FORMAT_VERSION:
|
||||
raise BackupError("archive_unsupported", "Backup-Formatversion wird nicht unterstützt.")
|
||||
checksums = manifest.get("checksums") or {}
|
||||
if not isinstance(checksums, dict) or DB_NAME not in checksums:
|
||||
raise BackupError("archive_invalid", "Checksummen fehlen.")
|
||||
extra = sorted(
|
||||
name
|
||||
for name in names
|
||||
if not name.endswith("/")
|
||||
and name not in {MANIFEST_NAME, README_NAME}
|
||||
and name not in checksums
|
||||
)
|
||||
missing = sorted(key for key in checksums if key not in names)
|
||||
if missing:
|
||||
raise BackupError(
|
||||
"checksum_mismatch",
|
||||
"Im Archiv fehlen deklarierte Dateien.",
|
||||
details={"missing": missing},
|
||||
)
|
||||
with tempfile.TemporaryDirectory(prefix="kansho-verify-") as raw:
|
||||
tmp = Path(raw)
|
||||
for name, expected in checksums.items():
|
||||
target = tmp / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(name) as src, target.open("wb") as dest:
|
||||
shutil.copyfileobj(src, dest)
|
||||
actual = sha256_file(target)
|
||||
if actual != expected:
|
||||
raise BackupError(
|
||||
"checksum_mismatch",
|
||||
f"Checksumme stimmt nicht: {name}",
|
||||
details={"path": name},
|
||||
)
|
||||
return {
|
||||
"manifest": manifest,
|
||||
"skipped_extra": extra,
|
||||
"names": sorted(names),
|
||||
}
|
||||
|
||||
|
||||
def restore_backup(
|
||||
archive: Path,
|
||||
*,
|
||||
db_path: Path | None = None,
|
||||
media_root: Path | None = None,
|
||||
confirm: bool = False,
|
||||
allow_overwrite: bool = False,
|
||||
safety_dir: Path | None = None,
|
||||
) -> dict:
|
||||
if not confirm:
|
||||
raise BackupError(
|
||||
"confirm_required",
|
||||
"Restore überschreibt lokale Daten nur mit ausdrücklicher Bestätigung (--confirm).",
|
||||
)
|
||||
db_path = Path(db_path or default_db_path())
|
||||
media_root = Path(media_root or default_media_root())
|
||||
verified = verify_archive(archive)
|
||||
current_exists = db_path.exists() or (media_root.exists() and any(media_root.rglob("*")))
|
||||
if current_exists and not allow_overwrite:
|
||||
raise BackupError(
|
||||
"would_overwrite",
|
||||
"Ein aktueller Datenbestand existiert. Restore nicht still. "
|
||||
"Mit Bestätigung und --replace nach Sicherheitsbackup fortsetzen.",
|
||||
)
|
||||
assert_db_idle(db_path)
|
||||
safety_path = None
|
||||
if current_exists:
|
||||
safety_dir = Path(safety_dir or default_backup_dir())
|
||||
safety_path = safety_dir / f"pre-restore-{_now_stamp()}.zip"
|
||||
create_backup(safety_path, db_path=db_path, media_root=media_root)
|
||||
with tempfile.TemporaryDirectory(prefix="kansho-restore-") as raw:
|
||||
extracted = Path(raw) / "extracted"
|
||||
extracted.mkdir()
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
checksums = verified["manifest"]["checksums"]
|
||||
for name in checksums:
|
||||
target = extracted / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(name) as src, target.open("wb") as dest:
|
||||
shutil.copyfileobj(src, dest)
|
||||
if sha256_file(target) != checksums[name]:
|
||||
raise BackupError("checksum_mismatch", f"Checksumme nach Entpacken ungültig: {name}")
|
||||
new_db = extracted / DB_NAME
|
||||
new_media = extracted / "media"
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_db = db_path.with_suffix(db_path.suffix + ".restore-tmp")
|
||||
if tmp_db.exists():
|
||||
tmp_db.unlink()
|
||||
shutil.copy2(new_db, tmp_db)
|
||||
os.replace(tmp_db, db_path)
|
||||
incoming_root = Path(raw) / "media-incoming"
|
||||
incoming_root.mkdir()
|
||||
if new_media.exists():
|
||||
for path in new_media.rglob("*"):
|
||||
if path.is_file():
|
||||
rel = path.relative_to(new_media)
|
||||
dest = incoming_root / rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, dest)
|
||||
swap = media_root.with_name(media_root.name + ".restore-old")
|
||||
if swap.exists():
|
||||
shutil.rmtree(swap)
|
||||
if media_root.exists():
|
||||
media_root.rename(swap)
|
||||
incoming_root.rename(media_root)
|
||||
if swap.exists():
|
||||
shutil.rmtree(swap, ignore_errors=True)
|
||||
return {
|
||||
"restored": str(archive),
|
||||
"db_path": str(db_path),
|
||||
"media_root": str(media_root),
|
||||
"safety_backup": str(safety_path) if safety_path else None,
|
||||
"skipped_extra": verified["skipped_extra"],
|
||||
"warning": PERSONAL_DATA_WARNING,
|
||||
}
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Lokales Kanshō-Backup (SQLite-Snapshot + Medien). Persönliche unverschlüsselte Daten."
|
||||
)
|
||||
sub = parser.add_subparsers(dest="action", required=True)
|
||||
create = sub.add_parser("create", help="Konsistentes lokales Backup erzeugen")
|
||||
create.add_argument("--out", help="Zieldatei. Standard: local-backups/kansho-<zeit>.zip")
|
||||
restore = sub.add_parser("restore", help="Backup prüfen und einspielen")
|
||||
restore.add_argument("archive", help="Backup-Archiv")
|
||||
restore.add_argument("--confirm", action="store_true", help="Ausdrückliche Bestätigung, sonst Abbruch")
|
||||
restore.add_argument("--replace", action="store_true", help="Nach Sicherheitsbackup aktuellen Stand ersetzen")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
try:
|
||||
if args.action == "create":
|
||||
result = create_backup(Path(args.out) if args.out else None)
|
||||
print(PERSONAL_DATA_WARNING)
|
||||
print(f"Backup: {result['path']}")
|
||||
print(f"Teile: sqlite + {len(result['manifest']['parts']['media'])} Mediendateien")
|
||||
return 0
|
||||
result = restore_backup(
|
||||
Path(args.archive),
|
||||
confirm=bool(args.confirm),
|
||||
allow_overwrite=bool(args.replace),
|
||||
)
|
||||
print(PERSONAL_DATA_WARNING)
|
||||
print(f"Restore: {result['db_path']}")
|
||||
if result["safety_backup"]:
|
||||
print(f"Sicherheitsbackup: {result['safety_backup']}")
|
||||
if result["skipped_extra"]:
|
||||
print("Zusätzliche Archivdateien ignoriert: " + ", ".join(result["skipped_extra"]))
|
||||
return 0
|
||||
except BackupError as exc:
|
||||
print(f"ERROR {exc.code}: {exc.message}", file=sys.stderr)
|
||||
if exc.details:
|
||||
print(json.dumps(exc.details, ensure_ascii=False), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Operational mail for account links. No dialogue content, no model egress."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import smtplib
|
||||
from email.message import EmailMessage
|
||||
|
||||
from env_loader import env_flag, runtime_env
|
||||
|
||||
log = logging.getLogger("kansho.mail")
|
||||
|
||||
OUTBOX: list[dict] = []
|
||||
|
||||
|
||||
class MailNotConfigured(RuntimeError):
|
||||
"""SMTP is required and neither configured nor in the development log path."""
|
||||
|
||||
|
||||
def clear_outbox() -> None:
|
||||
OUTBOX.clear()
|
||||
|
||||
|
||||
def smtp_configured() -> bool:
|
||||
return bool((os.environ.get("SMTP_HOST") or "").strip() and (os.environ.get("SMTP_FROM") or "").strip())
|
||||
|
||||
|
||||
def send_mail(recipients: list[str], subject: str, body: str) -> None:
|
||||
cleaned: list[str] = []
|
||||
for item in recipients:
|
||||
address = (item or "").strip().lower()
|
||||
if address and address not in cleaned:
|
||||
cleaned.append(address)
|
||||
if not cleaned:
|
||||
raise MailNotConfigured("Mailversand ist nicht konfiguriert")
|
||||
if env_flag("KANSHO_FAKE_MAIL"):
|
||||
OUTBOX.append({"to": cleaned, "subject": subject, "body": body})
|
||||
return
|
||||
host = (os.environ.get("SMTP_HOST") or "").strip()
|
||||
sender = (os.environ.get("SMTP_FROM") or "").strip()
|
||||
if not host or not sender:
|
||||
if runtime_env() == "development":
|
||||
log.warning("Kansho mail (development, SMTP fehlt): %s", body)
|
||||
return
|
||||
raise MailNotConfigured("Mailversand ist nicht konfiguriert")
|
||||
port = int((os.environ.get("SMTP_PORT") or "587").strip() or "587")
|
||||
user = (os.environ.get("SMTP_USER") or "").strip()
|
||||
password = os.environ.get("SMTP_PASSWORD") or ""
|
||||
message = EmailMessage()
|
||||
message["Subject"] = subject
|
||||
message["From"] = sender
|
||||
message["To"] = ", ".join(cleaned)
|
||||
message.set_content(body)
|
||||
if port == 465:
|
||||
with smtplib.SMTP_SSL(host, port, timeout=20) as smtp:
|
||||
if user:
|
||||
smtp.login(user, password)
|
||||
smtp.send_message(message)
|
||||
return
|
||||
with smtplib.SMTP(host, port, timeout=20) as smtp:
|
||||
smtp.starttls()
|
||||
if user:
|
||||
smtp.login(user, password)
|
||||
smtp.send_message(message)
|
||||
+31
-2
@@ -1,16 +1,43 @@
|
||||
from env_loader import load_env_file
|
||||
|
||||
load_env_file()
|
||||
|
||||
import os
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from db import init_db
|
||||
import data_layer_dialogue # noqa: F401
|
||||
from routers import admin, auth, dialogue, placeholders, prompts, subscription, users
|
||||
from routers import admin, auth, dialogue, generation_instructions, journal, placeholders, prompts, subscription, users
|
||||
from version import APP_VERSION
|
||||
|
||||
|
||||
def allowed_origins() -> list[str]:
|
||||
origins = [
|
||||
"http://localhost:5188",
|
||||
"http://127.0.0.1:5188",
|
||||
"https://kansho.jinkendo.de",
|
||||
"https://dev.kansho.jinkendo.de",
|
||||
]
|
||||
extra = (os.environ.get("ALLOWED_ORIGINS") or os.environ.get("KANSHO_ALLOWED_ORIGINS") or "").strip()
|
||||
if extra:
|
||||
origins.extend(item.strip() for item in extra.split(",") if item.strip())
|
||||
app_url = (os.environ.get("APP_URL") or "").strip().rstrip("/")
|
||||
if app_url:
|
||||
origins.append(app_url)
|
||||
seen: list[str] = []
|
||||
for item in origins:
|
||||
if item not in seen:
|
||||
seen.append(item)
|
||||
return seen
|
||||
|
||||
|
||||
app = FastAPI(title="Kanshō", version=APP_VERSION)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["http://localhost:5188", "http://127.0.0.1:5188"],
|
||||
allow_origins=allowed_origins(),
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
@@ -19,10 +46,12 @@ app.add_middleware(
|
||||
app.include_router(auth.router)
|
||||
app.include_router(users.router)
|
||||
app.include_router(dialogue.router)
|
||||
app.include_router(journal.router)
|
||||
app.include_router(prompts.router)
|
||||
app.include_router(placeholders.router)
|
||||
app.include_router(subscription.router)
|
||||
app.include_router(admin.router)
|
||||
app.include_router(generation_instructions.router)
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Local image and video attachments for journal entries.
|
||||
|
||||
Files never leave the trusted zone. Images have EXIF/text chunks stripped.
|
||||
Video containers are stored as-is; GPS in video metadata is not stripped yet.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from db import DATA_DIR, get_db, row_to_dict
|
||||
from dialogue_store import StoreError
|
||||
from journal_store import get_entry
|
||||
|
||||
IMAGE_TYPES = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/jpg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/gif": ".gif",
|
||||
"image/webp": ".webp",
|
||||
}
|
||||
VIDEO_TYPES = {
|
||||
"video/mp4": ".mp4",
|
||||
"video/webm": ".webm",
|
||||
"video/quicktime": ".mov",
|
||||
}
|
||||
ALLOWED_TYPES = {**IMAGE_TYPES, **VIDEO_TYPES}
|
||||
IMAGE_MAX_BYTES = 8 * 1024 * 1024
|
||||
VIDEO_MAX_BYTES = 64 * 1024 * 1024
|
||||
MAX_BYTES = IMAGE_MAX_BYTES # tests and older imports
|
||||
|
||||
|
||||
def media_root() -> Path:
|
||||
env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
return DATA_DIR / "media"
|
||||
|
||||
|
||||
MEDIA_ROOT = DATA_DIR / "media" # default; call media_root() at use time
|
||||
|
||||
|
||||
def _owned(conn, table: str, record_id: str, profile_id: str) -> dict | None:
|
||||
return row_to_dict(
|
||||
conn.execute(f"SELECT * FROM {table} WHERE id = ? AND profile_id = ?", (record_id, profile_id)).fetchone()
|
||||
)
|
||||
|
||||
|
||||
def media_kind(content_type: str) -> str:
|
||||
kind = (content_type or "").lower()
|
||||
if kind in VIDEO_TYPES or kind.startswith("video/"):
|
||||
return "video"
|
||||
return "image"
|
||||
|
||||
|
||||
def public_media(row: dict | None) -> dict | None:
|
||||
if not row:
|
||||
return None
|
||||
out = dict(row)
|
||||
out["kind"] = media_kind(row.get("content_type") or "")
|
||||
return out
|
||||
|
||||
|
||||
def sniff_content_type(data: bytes, declared: str) -> str:
|
||||
declared = (declared or "").lower().split(";")[0].strip()
|
||||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return "image/png"
|
||||
if data[:3] == b"\xff\xd8\xff":
|
||||
return "image/jpeg"
|
||||
if data[:6] in {b"GIF87a", b"GIF89a"}:
|
||||
return "image/gif"
|
||||
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
if len(data) >= 12 and data[4:8] == b"ftyp":
|
||||
brand = data[8:12]
|
||||
if brand in {b"qt ", b"moov"}:
|
||||
return "video/quicktime"
|
||||
return "video/mp4"
|
||||
if data[:4] == b"\x1a\x45\xdf\xa3":
|
||||
return "video/webm"
|
||||
if declared in ALLOWED_TYPES:
|
||||
return declared
|
||||
return declared
|
||||
|
||||
|
||||
def strip_jpeg_exif(data: bytes) -> bytes:
|
||||
if len(data) < 4 or data[0:2] != b"\xff\xd8":
|
||||
return data
|
||||
out = bytearray(b"\xff\xd8")
|
||||
i = 2
|
||||
while i + 1 < len(data):
|
||||
if data[i] != 0xFF:
|
||||
out.extend(data[i:])
|
||||
break
|
||||
marker = data[i + 1]
|
||||
if marker == 0xDA:
|
||||
out.extend(data[i:])
|
||||
break
|
||||
if marker == 0xD9:
|
||||
out.extend(data[i : i + 2])
|
||||
break
|
||||
if marker in {0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0x01}:
|
||||
out.extend(data[i : i + 2])
|
||||
i += 2
|
||||
continue
|
||||
if i + 4 > len(data):
|
||||
break
|
||||
length = int.from_bytes(data[i + 2 : i + 4], "big")
|
||||
block_end = i + 2 + length
|
||||
if marker == 0xE1 and data[i + 4 : i + 10] == b"Exif\x00\x00":
|
||||
i = block_end
|
||||
continue
|
||||
out.extend(data[i:block_end])
|
||||
i = block_end
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def strip_png_metadata(data: bytes) -> bytes:
|
||||
signature = b"\x89PNG\r\n\x1a\n"
|
||||
if not data.startswith(signature):
|
||||
return data
|
||||
out = bytearray(signature)
|
||||
drop = {b"eXIf", b"eXif", b"tEXt", b"zTXt", b"iTXt", b"tIME"}
|
||||
i = 8
|
||||
while i + 12 <= len(data):
|
||||
length = int.from_bytes(data[i : i + 4], "big")
|
||||
chunk_type = data[i + 4 : i + 8]
|
||||
end = i + 12 + length
|
||||
if chunk_type not in drop:
|
||||
out.extend(data[i:end])
|
||||
if chunk_type == b"IEND":
|
||||
break
|
||||
i = end
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def prepare_image(data: bytes, content_type: str) -> bytes:
|
||||
if content_type in {"image/jpeg", "image/jpg"}:
|
||||
return strip_jpeg_exif(data)
|
||||
if content_type == "image/png":
|
||||
return strip_png_metadata(data)
|
||||
return data
|
||||
|
||||
|
||||
def save_media(profile_id: str, entry_id: str, filename: str, content_type: str, data: bytes) -> dict:
|
||||
entry = get_entry(profile_id, entry_id)
|
||||
if not data:
|
||||
raise StoreError("empty_media", "Datei ist leer")
|
||||
kind = sniff_content_type(data, content_type)
|
||||
if kind not in ALLOWED_TYPES:
|
||||
raise StoreError("unsupported_media", "Nur Bilder (JPEG, PNG, GIF, WebP) und Videos (MP4, WebM, MOV) sind erlaubt")
|
||||
is_video = kind in VIDEO_TYPES
|
||||
limit = VIDEO_MAX_BYTES if is_video else IMAGE_MAX_BYTES
|
||||
if len(data) > limit:
|
||||
label = "Video" if is_video else "Bild"
|
||||
mb = limit // (1024 * 1024)
|
||||
raise StoreError("media_too_large", f"{label} ist größer als {mb} MB")
|
||||
payload = data if is_video else prepare_image(data, kind)
|
||||
asset_id = str(uuid.uuid4())
|
||||
ext = ALLOWED_TYPES[kind]
|
||||
rel_path = f"{profile_id}/{asset_id}{ext}"
|
||||
dest = media_root() / rel_path
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(payload)
|
||||
prefix = "video" if is_video else "image"
|
||||
safe_name = Path(filename or f"{prefix}{ext}").name
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO media_assets
|
||||
(id, profile_id, entry_id, entry_version_id, filename, content_type, rel_path)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(asset_id, profile_id, entry_id, entry.get("current_version_id"), safe_name, kind, rel_path),
|
||||
)
|
||||
row = row_to_dict(conn.execute("SELECT * FROM media_assets WHERE id = ?", (asset_id,)).fetchone())
|
||||
return public_media(row)
|
||||
|
||||
|
||||
def save_image(profile_id: str, entry_id: str, filename: str, content_type: str, data: bytes) -> dict:
|
||||
return save_media(profile_id, entry_id, filename, content_type, data)
|
||||
|
||||
|
||||
def list_media(profile_id: str, entry_id: str, *, include_deleted: bool = False) -> list[dict]:
|
||||
get_entry(profile_id, entry_id, include_deleted=include_deleted)
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM media_assets
|
||||
WHERE profile_id = ? AND entry_id = ?
|
||||
ORDER BY created
|
||||
""",
|
||||
(profile_id, entry_id),
|
||||
).fetchall()
|
||||
return [public_media(row_to_dict(row)) for row in rows]
|
||||
|
||||
|
||||
def get_media(profile_id: str, media_id: str) -> tuple[dict, Path]:
|
||||
with get_db() as conn:
|
||||
row = _owned(conn, "media_assets", media_id, profile_id)
|
||||
if not row:
|
||||
raise StoreError("not_found", "Medium nicht gefunden", 404)
|
||||
path = media_root() / row["rel_path"]
|
||||
if not path.is_file():
|
||||
raise StoreError("not_found", "Mediendatei fehlt", 404)
|
||||
return public_media(row), path
|
||||
|
||||
|
||||
def delete_media(profile_id: str, media_id: str) -> dict:
|
||||
row, path = get_media(profile_id, media_id)
|
||||
with get_db() as conn:
|
||||
conn.execute("DELETE FROM media_assets WHERE id = ? AND profile_id = ?", (media_id, profile_id))
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
return {"id": media_id, "deleted": True}
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Postgres runtime baseline for Kanshō.
|
||||
-- Greenfield schema is backend/schema.sql translated at startup
|
||||
-- (datetime('now') → CURRENT_TIMESTAMP::text). Historical SQLite
|
||||
-- inline migrations 001-021 are recorded as applied, not replayed.
|
||||
-- Future incremental DDL belongs in 023_*.sql and later.
|
||||
|
||||
SELECT 1;
|
||||
@@ -0,0 +1,16 @@
|
||||
-- LLM profiles: named, reusable generate/detect endpoint+model presets.
|
||||
-- Keys stay in env. Active assignment remains provider_settings.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS llm_profiles (
|
||||
id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
url TEXT NOT NULL DEFAULT '',
|
||||
model TEXT NOT NULL DEFAULT '',
|
||||
zdr INTEGER NOT NULL DEFAULT 1,
|
||||
no_train INTEGER NOT NULL DEFAULT 1,
|
||||
created TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text,
|
||||
updated TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text
|
||||
);
|
||||
|
||||
ALTER TABLE provider_settings ADD COLUMN IF NOT EXISTS profile_id TEXT;
|
||||
@@ -0,0 +1,23 @@
|
||||
-- Transitional learning detect: user-confirmed label senses, pending mask review.
|
||||
-- Detect output still does not auto-activate identities.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS label_senses (
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
normalized_label TEXT NOT NULL,
|
||||
identity_hits INTEGER NOT NULL DEFAULT 0,
|
||||
non_identity_hits INTEGER NOT NULL DEFAULT 0,
|
||||
updated TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text,
|
||||
PRIMARY KEY (profile_id, normalized_label)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
|
||||
user_message_id TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||
created TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pending_mask_reviews_conv
|
||||
ON pending_mask_reviews (profile_id, conversation_id);
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Remember mask-review decisions per local context cue (previous word).
|
||||
-- Not a word list: cues come from the user line, e.g. Frau vs. a food mention.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS label_sense_cues (
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
normalized_label TEXT NOT NULL,
|
||||
cue TEXT NOT NULL,
|
||||
decision TEXT NOT NULL,
|
||||
hits INTEGER NOT NULL DEFAULT 1,
|
||||
updated TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text,
|
||||
PRIMARY KEY (profile_id, normalized_label, cue)
|
||||
);
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Mini-passages beside each cue so local detect/GLiNER can pre-decide from context.
|
||||
ALTER TABLE label_sense_cues ADD COLUMN IF NOT EXISTS excerpt TEXT NOT NULL DEFAULT '';
|
||||
@@ -0,0 +1,49 @@
|
||||
-- Account identifiers, verified addresses, and one-time mail tokens.
|
||||
-- Existing primary addresses are grandfathered as verified once.
|
||||
|
||||
ALTER TABLE profiles ADD COLUMN IF NOT EXISTS username TEXT;
|
||||
ALTER TABLE profiles ADD COLUMN IF NOT EXISTS secondary_email TEXT;
|
||||
ALTER TABLE profiles ADD COLUMN IF NOT EXISTS email_verified_at TEXT;
|
||||
ALTER TABLE profiles ADD COLUMN IF NOT EXISTS secondary_email_verified_at TEXT;
|
||||
ALTER TABLE profiles ADD COLUMN IF NOT EXISTS pending_email TEXT;
|
||||
ALTER TABLE profiles ADD COLUMN IF NOT EXISTS pending_secondary_email TEXT;
|
||||
|
||||
UPDATE profiles
|
||||
SET email_verified_at = COALESCE(NULLIF(created, ''), CURRENT_TIMESTAMP::text)
|
||||
WHERE email_verified_at IS NULL OR email_verified_at = '';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS password_reset_tokens (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
expires_at TEXT NOT NULL,
|
||||
used_at TEXT,
|
||||
created TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS email_verification_tokens (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
purpose TEXT NOT NULL CHECK (purpose IN ('primary', 'secondary')),
|
||||
email TEXT NOT NULL,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
expires_at TEXT NOT NULL,
|
||||
used_at TEXT,
|
||||
created TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP::text
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS profiles_username_lower_idx
|
||||
ON profiles (lower(username))
|
||||
WHERE username IS NOT NULL AND username <> '';
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS profiles_secondary_email_lower_idx
|
||||
ON profiles (lower(secondary_email))
|
||||
WHERE secondary_email IS NOT NULL AND secondary_email <> '';
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS profiles_pending_email_lower_idx
|
||||
ON profiles (lower(pending_email))
|
||||
WHERE pending_email IS NOT NULL AND pending_email <> '';
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS profiles_pending_secondary_email_lower_idx
|
||||
ON profiles (lower(pending_secondary_email))
|
||||
WHERE pending_secondary_email IS NOT NULL AND pending_secondary_email <> '';
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Cached model window metadata for journal budget checks.
|
||||
|
||||
OpenRouter exposes context_length and top_provider.max_completion_tokens via
|
||||
GET /api/v1/models. The catalog is cached so journal calls do not refetch on
|
||||
every generate. Missing metadata fails closed for journal generation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from prompt_budget import (
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
JOURNAL_TARGET_COMPLETION_TOKENS,
|
||||
JournalBudgetError,
|
||||
ModelWindow,
|
||||
_int_env,
|
||||
)
|
||||
|
||||
DEFAULT_TTL_SECONDS = 3600
|
||||
FAKE_CONTEXT = 32_768
|
||||
_lock = threading.Lock()
|
||||
# url -> (fetched_at, {model_id: ModelWindow fields as dict})
|
||||
_catalog: dict[str, tuple[float, dict[str, dict]]] = {}
|
||||
_overrides: dict[str, ModelWindow] = {}
|
||||
|
||||
|
||||
def catalog_ttl() -> int:
|
||||
raw = (os.environ.get("KANSHO_MODEL_CATALOG_TTL_SECONDS") or "").strip()
|
||||
if not raw:
|
||||
return DEFAULT_TTL_SECONDS
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return DEFAULT_TTL_SECONDS
|
||||
return max(60, value)
|
||||
|
||||
|
||||
def reset_catalog() -> None:
|
||||
with _lock:
|
||||
_catalog.clear()
|
||||
_overrides.clear()
|
||||
|
||||
|
||||
def set_metadata_override(model: str, window: ModelWindow) -> None:
|
||||
with _lock:
|
||||
_overrides[model] = window
|
||||
|
||||
|
||||
def is_openrouter_url(url: str) -> bool:
|
||||
host = (urlparse(url or "").hostname or "").lower()
|
||||
return host.endswith("openrouter.ai")
|
||||
|
||||
|
||||
def models_url(chat_url: str) -> str:
|
||||
raw = (chat_url or "").rstrip("/")
|
||||
if raw.endswith("/chat/completions"):
|
||||
return raw[: -len("/chat/completions")] + "/models"
|
||||
if is_openrouter_url(raw):
|
||||
return "https://openrouter.ai/api/v1/models"
|
||||
return raw + "/models" if raw else "https://openrouter.ai/api/v1/models"
|
||||
|
||||
|
||||
def _from_env(model: str, *, provider: str | None, source: str) -> ModelWindow | None:
|
||||
context = _int_env("KANSHO_PROVIDER_CONTEXT_LENGTH", 0)
|
||||
completion = _int_env("KANSHO_PROVIDER_MAX_COMPLETION_TOKENS", 0)
|
||||
if context <= 0:
|
||||
return None
|
||||
return ModelWindow(
|
||||
model=model,
|
||||
context_length=context,
|
||||
max_completion_tokens=completion or JOURNAL_TARGET_COMPLETION_TOKENS,
|
||||
source=source,
|
||||
provider=provider,
|
||||
cached=False,
|
||||
)
|
||||
|
||||
|
||||
def _parse_model_row(row: dict) -> ModelWindow | None:
|
||||
model_id = str(row.get("id") or "").strip()
|
||||
if not model_id:
|
||||
return None
|
||||
top = row.get("top_provider") if isinstance(row.get("top_provider"), dict) else {}
|
||||
context = row.get("context_length") or top.get("context_length")
|
||||
completion = top.get("max_completion_tokens")
|
||||
if completion is None:
|
||||
limits = row.get("per_request_limits") if isinstance(row.get("per_request_limits"), dict) else {}
|
||||
completion = limits.get("completion_tokens") or limits.get("max_tokens")
|
||||
try:
|
||||
context_n = int(context)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
try:
|
||||
completion_n = int(completion) if completion is not None else None
|
||||
except (TypeError, ValueError):
|
||||
completion_n = None
|
||||
if context_n <= 0:
|
||||
return None
|
||||
return ModelWindow(
|
||||
model=model_id,
|
||||
context_length=context_n,
|
||||
max_completion_tokens=completion_n,
|
||||
source="openrouter_models",
|
||||
provider="openrouter",
|
||||
cached=True,
|
||||
)
|
||||
|
||||
|
||||
def _request_models_payload(url: str, key: str) -> list:
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if key:
|
||||
headers["Authorization"] = f"Bearer {key}"
|
||||
try:
|
||||
response = httpx.get(url, headers=headers, timeout=15.0)
|
||||
except httpx.HTTPError as exc:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "reason": "unreachable"},
|
||||
) from exc
|
||||
if response.status_code >= 400:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "http_status": response.status_code},
|
||||
)
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "reason": "invalid_json"},
|
||||
) from exc
|
||||
rows = payload.get("data") if isinstance(payload, dict) else payload
|
||||
if not isinstance(rows, list):
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "reason": "unexpected_shape"},
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _fetch_catalog(url: str, key: str) -> dict[str, dict]:
|
||||
rows = _request_models_payload(url, key)
|
||||
parsed: dict[str, dict] = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
window = _parse_model_row(row)
|
||||
if window:
|
||||
parsed[window.model] = {
|
||||
"model": window.model,
|
||||
"context_length": window.context_length,
|
||||
"max_completion_tokens": window.max_completion_tokens,
|
||||
"source": window.source,
|
||||
"provider": window.provider,
|
||||
}
|
||||
if not parsed:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"models_url": url, "reason": "empty_catalog"},
|
||||
)
|
||||
return parsed
|
||||
|
||||
|
||||
def _window_from_cache(fields: dict, *, model: str) -> ModelWindow:
|
||||
return ModelWindow(
|
||||
model=model,
|
||||
context_length=int(fields["context_length"]),
|
||||
max_completion_tokens=fields.get("max_completion_tokens"),
|
||||
source=fields.get("source") or "openrouter_models",
|
||||
provider=fields.get("provider"),
|
||||
cached=True,
|
||||
)
|
||||
|
||||
|
||||
def resolve_generate_metadata(config, *, now: float | None = None) -> ModelWindow:
|
||||
"""Return a window for the generate provider. Fail closed if unknown."""
|
||||
model = getattr(config, "model", None) or ""
|
||||
with _lock:
|
||||
override = _overrides.get(model)
|
||||
if override:
|
||||
return override
|
||||
if getattr(config, "mode", None) == "fake" or model == "fake":
|
||||
return ModelWindow(
|
||||
model=model or "fake",
|
||||
context_length=FAKE_CONTEXT,
|
||||
max_completion_tokens=JOURNAL_TARGET_COMPLETION_TOKENS,
|
||||
source="fake",
|
||||
provider="fake",
|
||||
cached=False,
|
||||
)
|
||||
env_window = _from_env(model, provider=getattr(config, "name", None), source="env")
|
||||
if getattr(config, "local", False):
|
||||
if env_window:
|
||||
return env_window
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={
|
||||
"model": model,
|
||||
"provider": getattr(config, "name", None),
|
||||
"reason": "local_context_unconfigured",
|
||||
},
|
||||
)
|
||||
url = getattr(config, "url", "") or ""
|
||||
if not is_openrouter_url(url):
|
||||
if env_window:
|
||||
return env_window
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={
|
||||
"model": model,
|
||||
"provider": getattr(config, "name", None),
|
||||
"reason": "non_openrouter_unconfigured",
|
||||
},
|
||||
)
|
||||
catalog_key = models_url(url)
|
||||
clock = time.time() if now is None else now
|
||||
ttl = catalog_ttl()
|
||||
with _lock:
|
||||
cached = _catalog.get(catalog_key)
|
||||
if cached and clock - cached[0] < ttl and model in cached[1]:
|
||||
return _window_from_cache(cached[1][model], model=model)
|
||||
stale = cached
|
||||
try:
|
||||
fetched = _fetch_catalog(catalog_key, getattr(config, "key", "") or "")
|
||||
except JournalBudgetError:
|
||||
if stale and model in stale[1]:
|
||||
window = _window_from_cache(stale[1][model], model=model)
|
||||
window.source = "openrouter_models_stale"
|
||||
return window
|
||||
if env_window:
|
||||
env_window.source = "env_fallback"
|
||||
return env_window
|
||||
raise
|
||||
with _lock:
|
||||
_catalog[catalog_key] = (clock, fetched)
|
||||
if model not in fetched:
|
||||
if env_window:
|
||||
env_window.source = "env_fallback"
|
||||
return env_window
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_METADATA_UNKNOWN,
|
||||
status_code=503,
|
||||
diagnostics={"model": model, "reason": "model_not_in_catalog"},
|
||||
)
|
||||
return _window_from_cache(fetched[model], model=model)
|
||||
|
||||
|
||||
def list_provider_models(config) -> list[dict]:
|
||||
"""Model ids for the admin picker. Soft-fail; free-text remains valid."""
|
||||
if not config or getattr(config, "mode", None) == "fake":
|
||||
return []
|
||||
url = getattr(config, "url", "") or ""
|
||||
if not url:
|
||||
return []
|
||||
key = getattr(config, "key", "") or ""
|
||||
items = _list_openai_models(url, key)
|
||||
if items:
|
||||
return items
|
||||
return _list_ollama_tags(url)
|
||||
|
||||
|
||||
def list_models_for_url(url: str, key: str = "") -> list[dict]:
|
||||
from providers import ProviderConfig, is_local_url
|
||||
|
||||
url = (url or "").strip()
|
||||
if not url:
|
||||
return []
|
||||
local = is_local_url(url)
|
||||
return list_provider_models(
|
||||
ProviderConfig(
|
||||
role="generate",
|
||||
name="ollama" if local else "openrouter",
|
||||
mode="http",
|
||||
url=url,
|
||||
model="",
|
||||
key=key,
|
||||
local=local,
|
||||
zdr=True,
|
||||
no_train=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _list_openai_models(chat_url: str, key: str) -> list[dict]:
|
||||
catalog_key = models_url(chat_url)
|
||||
try:
|
||||
rows = _request_models_payload(catalog_key, key)
|
||||
except JournalBudgetError:
|
||||
return []
|
||||
return _ids_from_rows(rows, id_keys=("id",))
|
||||
|
||||
|
||||
def _list_ollama_tags(chat_url: str) -> list[dict]:
|
||||
parsed = urlparse(chat_url or "")
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
return []
|
||||
tags_url = f"{parsed.scheme}://{parsed.netloc}/api/tags"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
try:
|
||||
response = httpx.get(tags_url, headers=headers, timeout=8.0)
|
||||
except httpx.HTTPError:
|
||||
return []
|
||||
if response.status_code >= 400:
|
||||
return []
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return []
|
||||
rows = payload.get("models") if isinstance(payload, dict) else payload
|
||||
if not isinstance(rows, list):
|
||||
return []
|
||||
return _ids_from_rows(rows, id_keys=("name", "model", "id"))
|
||||
|
||||
|
||||
def _ids_from_rows(rows: list, *, id_keys: tuple[str, ...]) -> list[dict]:
|
||||
seen: set[str] = set()
|
||||
items: list[dict] = []
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
model_id = ""
|
||||
for key in id_keys:
|
||||
model_id = str(row.get(key) or "").strip()
|
||||
if model_id:
|
||||
break
|
||||
if not model_id or model_id in seen:
|
||||
continue
|
||||
seen.add(model_id)
|
||||
name = str(row.get("name") or model_id).strip() or model_id
|
||||
items.append({"id": model_id, "name": name})
|
||||
items.sort(key=lambda item: item["id"])
|
||||
return items
|
||||
@@ -0,0 +1,138 @@
|
||||
"""MVP context keys. Values come from the Context Builder, never from Python prompt strings."""
|
||||
from __future__ import annotations
|
||||
|
||||
from placeholders import Placeholder, register
|
||||
|
||||
|
||||
def _from_ctx(key: str):
|
||||
return lambda ctx: (ctx or {}).get(key) or ""
|
||||
|
||||
|
||||
register(
|
||||
Placeholder(
|
||||
key="dialogue_context",
|
||||
description="Vom Context Builder zusammengestellter Dialog- und Space-Kontext.",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("dialogue_context"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="writing_profile",
|
||||
description="Aktueller Writing-Profile-Brief für Journal-Artefakte. Kein KI-Draft, keine Dialogsteuerung.",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("writing_profile"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="style_examples",
|
||||
description="Historische Stilreferenzen. Keine Tatsachen des aktuellen Tages.",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("style_examples"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="transformation_instructions",
|
||||
description="Kompilierte Bearbeitungsstärke. Semantische Ausgabeeinstellung, keine Modelltemperatur.",
|
||||
data_class="C",
|
||||
resolver=_from_ctx("transformation_instructions"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="detail_instructions",
|
||||
description="Kompilierte Detailerhaltung. Semantische Ausgabeeinstellung, keine Modelltemperatur.",
|
||||
data_class="C",
|
||||
resolver=_from_ctx("detail_instructions"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="voice_instructions",
|
||||
description="Kompilierte Stimmanweisung aus dem Generation-Policy-Compiler.",
|
||||
data_class="C",
|
||||
resolver=_from_ctx("voice_instructions"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="narrative_instructions",
|
||||
description="Kompilierte Erzählgestaltung aus dem Generation-Policy-Compiler.",
|
||||
data_class="C",
|
||||
resolver=_from_ctx("narrative_instructions"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="interaction_hint",
|
||||
description="Interaction Profile plus aktueller Dialogue State. Kein Writing Profile, keine Ableitung aus Nicht-Widersprechen.",
|
||||
data_class="C",
|
||||
resolver=_from_ctx("interaction_hint"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="space_title",
|
||||
description="Titel des aktuellen Reflection Space.",
|
||||
data_class="C",
|
||||
resolver=_from_ctx("space_title"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="existing_text",
|
||||
description="Bestehender Draft oder Entry, nur bei ausdrücklichem Regenerieren.",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("existing_text"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="source_text",
|
||||
description="Klartext für Entity Detection. Verlässt die Trusted Zone nur über den Detect-Provider.",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("source_text"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="known_labels",
|
||||
description="Legacy-Platzhalter. Der aktuelle Detect-Prompt nutzt keine Ausnahmeliste bekannter Labels.",
|
||||
data_class="C",
|
||||
resolver=_from_ctx("known_labels"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="register_hint",
|
||||
description="Lokales Erzählregister für den Dialogzug. Kein zweiter Modellaufruf.",
|
||||
data_class="C",
|
||||
resolver=_from_ctx("register_hint"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="opening_hint",
|
||||
description="Journalspezifischer erster Impuls. Leer bei späteren Zügen. Keine Intent-Engine.",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("opening_hint"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="reconstruction",
|
||||
description="Validierte inhaltliche Rekonstruktion der Stufe 1. Kein vollständiger Tagesdialog.",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("reconstruction"),
|
||||
)
|
||||
)
|
||||
register(
|
||||
Placeholder(
|
||||
key="review_package",
|
||||
description="Gebündeltes Profile-Review-Paket (derselbe Vertrag für API und Copy/Paste).",
|
||||
data_class="B",
|
||||
resolver=_from_ctx("review_package"),
|
||||
)
|
||||
)
|
||||
+1414
-16
File diff suppressed because it is too large.
Load diff
@@ -7,7 +7,7 @@ from placeholders import PlaceholderError
|
||||
|
||||
PRIVACY_PATTERN = re.compile(r"\[\[([A-Z][A-Z0-9_]*(?::[A-Z0-9_]+)?)\]\]")
|
||||
KNOWN_EXACT = {"SELF"}
|
||||
KNOWN_PREFIXES = ("PERSON:", "PLACE:")
|
||||
KNOWN_PREFIXES = ("PERSON:", "PLACE:", "ORG:", "PROJECT:")
|
||||
|
||||
|
||||
def catalog() -> list[dict]:
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Shared Writing-Profile analysis contract for API and Copy/Paste.
|
||||
|
||||
Not a second profile model. Builds a bounded ProfileAnalysisPackage and a
|
||||
human-readable paste prompt. External chat history is evidence provenance,
|
||||
not a Kanshō source.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from writing_profile_schema import (
|
||||
EVIDENCE_BASIS,
|
||||
EXISTING_BEFORE_NEW,
|
||||
is_meta_style_text,
|
||||
recency_role,
|
||||
)
|
||||
|
||||
KIND_PACKAGE = "kansho.profile_analysis_package"
|
||||
KIND_RESULT = "kansho.profile_analysis_result"
|
||||
KIND_PACKAGE_LEGACY = "kansho.profile_review_package"
|
||||
KIND_RESULT_LEGACY = "kansho.profile_review_result"
|
||||
KIND_PACKAGES = {KIND_PACKAGE, KIND_PACKAGE_LEGACY}
|
||||
KIND_RESULTS = {KIND_RESULT, KIND_RESULT_LEGACY}
|
||||
FORMAT_VERSION = 1
|
||||
PACKAGE_CHAR_BUDGET = int(os.environ.get("KANSHO_PROFILE_PACKAGE_CHARS") or "12000")
|
||||
CORPUS_EXCERPT = int(os.environ.get("KANSHO_PROFILE_EXCERPT_CHARS") or "360")
|
||||
MAX_CURRENT = 5
|
||||
MAX_LONG_TERM = 3
|
||||
MAX_PENDING = 6
|
||||
MAX_INITIAL = 14
|
||||
|
||||
|
||||
def package_budget() -> int:
|
||||
return max(4000, PACKAGE_CHAR_BUDGET)
|
||||
|
||||
|
||||
def expected_result_schema(mode: str) -> dict:
|
||||
change = {
|
||||
"layer": "core|facet|trait",
|
||||
"key": "core|autobiographical_journal|datengetriebener-slug",
|
||||
"slug": "datengetriebener Trait-Slug",
|
||||
"facet_key": "core|autobiographical_journal|…",
|
||||
"action": "keep|update|add|remove|reclassify|merge|split",
|
||||
"proposed_value": "Aussage",
|
||||
"label": "optional",
|
||||
"rationale": "kurz, semantisch",
|
||||
"evidence_basis": list(EVIDENCE_BASIS),
|
||||
"evidence_ids": ["nur echte Kanshō-IDs aus diesem Paket, sonst weglassen"],
|
||||
"exemplars": [
|
||||
{
|
||||
"excerpt": "echter Originalausschnitt",
|
||||
"occurred_at": "YYYY-MM-DD",
|
||||
"source_id": "nur bei kansho_sources",
|
||||
"evidence_basis": "kansho_sources|external_chat_history",
|
||||
}
|
||||
],
|
||||
"merge_slugs": ["optional"],
|
||||
"split_into": [{"slug": "neu", "statement": "…", "facet_key": "…"}],
|
||||
}
|
||||
result = {
|
||||
"kind": KIND_RESULT,
|
||||
"format_version": FORMAT_VERSION,
|
||||
"target": "writing",
|
||||
"mode": mode,
|
||||
"evidence_basis": list(EVIDENCE_BASIS),
|
||||
"uncertainties": ["was die Evidence nicht trägt"],
|
||||
"changes": [change],
|
||||
}
|
||||
if mode == "initial_build":
|
||||
result["profile"] = {
|
||||
"core": {"value": "nur soweit unterschiedliche Quellenarten das tragen", "traits": []},
|
||||
"facets": [
|
||||
{
|
||||
"key": "autobiographical_journal",
|
||||
"value": "Facet-Delta gegenüber dem Core",
|
||||
"traits": [],
|
||||
}
|
||||
],
|
||||
"traits": [
|
||||
{
|
||||
"slug": "datengetrieben",
|
||||
"facet_key": "autobiographical_journal",
|
||||
"statement": "semantisches Merkmal",
|
||||
"exemplars": [{"excerpt": "Originalzitat", "evidence_basis": "kansho_sources"}],
|
||||
}
|
||||
],
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def select_corpus_items(items: list[dict], *, mode: str) -> list[dict]:
|
||||
real = [item for item in items if not is_meta_style_text(item.get("body") or item.get("excerpt"))]
|
||||
current = [
|
||||
item
|
||||
for item in real
|
||||
if (item.get("recency_role") or recency_role(item.get("occurred_at"))) == "current_expression"
|
||||
]
|
||||
long_term = [
|
||||
item
|
||||
for item in real
|
||||
if (item.get("recency_role") or recency_role(item.get("occurred_at"))) == "long_term"
|
||||
]
|
||||
picked: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(item: dict) -> None:
|
||||
item_id = item.get("id") or ""
|
||||
if item_id and item_id in seen:
|
||||
return
|
||||
if item_id:
|
||||
seen.add(item_id)
|
||||
picked.append(item)
|
||||
|
||||
for item in current[:MAX_CURRENT]:
|
||||
add(item)
|
||||
for item in list(reversed(long_term))[:MAX_LONG_TERM]:
|
||||
add(item)
|
||||
by_hint: dict[str, list[dict]] = {}
|
||||
for item in real:
|
||||
by_hint.setdefault(item.get("facet_hint") or item.get("context_hint") or "unspecified", []).append(item)
|
||||
for group in by_hint.values():
|
||||
add(group[0])
|
||||
limit = MAX_INITIAL if mode == "initial_build" else MAX_CURRENT + MAX_LONG_TERM + 4
|
||||
if mode == "initial_build":
|
||||
for item in real:
|
||||
if len(picked) >= limit:
|
||||
break
|
||||
add(item)
|
||||
return _trim_budget(picked)
|
||||
|
||||
|
||||
def _trim_budget(items: list[dict]) -> list[dict]:
|
||||
budget = package_budget()
|
||||
kept: list[dict] = []
|
||||
used = 0
|
||||
for item in items:
|
||||
excerpt = (item.get("excerpt") or item.get("body") or "")[:CORPUS_EXCERPT]
|
||||
cost = len(excerpt)
|
||||
if kept and used + cost > budget:
|
||||
break
|
||||
kept.append(item)
|
||||
used += cost
|
||||
return kept or items[:2]
|
||||
|
||||
|
||||
def filter_style_evidence(items: list[dict]) -> list[dict]:
|
||||
cleaned = []
|
||||
for item in items:
|
||||
excerpt = item.get("excerpt") or item.get("body") or ""
|
||||
if is_meta_style_text(excerpt):
|
||||
continue
|
||||
cleaned.append(item)
|
||||
return cleaned[:MAX_PENDING]
|
||||
|
||||
|
||||
def semantic_task(mode: str) -> str:
|
||||
shared = (
|
||||
"Lokale Heuristiken (Wortzählungen, Signalwörter, Uhrzeiten) sind Messhilfe, keine fertigen Stilurteile. "
|
||||
"Leite keine Traits allein aus Zählungen ab. Keine Persönlichkeitsdiagnose, keine psychologischen Zuschreibungen, "
|
||||
"keine erfundenen Stilmerkmale, keine erfundenen Quellen, keine erfundenen Kanshō-Source-IDs, keine erfundenen Beispiele.\n\n"
|
||||
"Analysiere semantisch:\n"
|
||||
"- Was macht den persönlichen Schreibstil tatsächlich charakteristisch?\n"
|
||||
"- Welche Merkmale sind stabilerer Core?\n"
|
||||
"- Welche Merkmale sind kontextspezifische Facet-Deltas (z. B. autobiographical_journal)?\n"
|
||||
"- Welche bestehenden Traits werden bestätigt, sind zu grob, redundant oder falsch gescoped?\n"
|
||||
"- Welche fehlenden Merkmale sind über mehrere Texte hinweg belastbar?\n"
|
||||
"- Welche Merkmale haben sich über die Zeit verändert?\n"
|
||||
"- Welche Aussagen sind wegen zu wenig Evidence noch unsicher?\n\n"
|
||||
"Zeit: Ältere Texte können Langzeitmerkmale belegen. Neuere Texte wiegen stärker für die aktuelle Ausprägung. "
|
||||
"Nicht alle Jahre gleich gewichten.\n"
|
||||
"Besteht das Korpus vor allem aus Urlaubstagebüchern, sind starke Aussagen zur Facet autobiographical_journal zulässig. "
|
||||
"Einen globalen Core nur so weit, wie unterschiedliche Quellenarten das tragen; Generalisierbarkeit sonst als unsicher markieren.\n\n"
|
||||
"Existing-before-New:\n- " + "\n- ".join(EXISTING_BEFORE_NEW) + "\n"
|
||||
"Keine komplett neue Taxonomie nur weil ein anderes Raster eleganter wirkt.\n\n"
|
||||
"Representative Examples müssen echte Ausschnitte aus authentischen Nutzertexten sein. "
|
||||
"Unzulässig als Stilbeispiel: technische Triggertexte, UI-Texte, Meta-Beschreibungen einer Review, Statistiktexte, KI-Drafts.\n\n"
|
||||
"Wenn du aus dem bisherigen Verlauf dieses Chats bereits weitere authentische Texte des Nutzers kennst, "
|
||||
"darfst du dieses Wissen ergänzend für die semantische Analyse des Schreibstils verwenden. "
|
||||
"Das ist externe Chat-Historie, kein Kanshō-Source-Layer. Solche Evidenz als evidence_basis: external_chat_history kennzeichnen. "
|
||||
"Keine internen Source-IDs dafür erfinden. Später kann der Nutzer echte Texte nach Kanshō importieren, wenn interne Provenance gewünscht ist."
|
||||
)
|
||||
if mode == "initial_build":
|
||||
return (
|
||||
"Modus initial_build: Erzeuge ein vollständiges Profile Proposal (Core, optionale Facets, dynamische Traits, "
|
||||
"Facet-Deltas gegenüber dem Core, Representative Exemplars, Evidence-Basis, Unsicherheiten, Strukturänderungen). "
|
||||
"Bestehende Struktur zuerst berücksichtigen. Du darfst bei guter Begründung Traits, Core/Facet-Zuordnung und Facets ändern.\n\n"
|
||||
+ shared
|
||||
)
|
||||
return (
|
||||
"Modus review: Vergleiche neue Evidence mit dem bestehenden Profil. "
|
||||
"Schlage primär inkrementelle Änderungen vor (keep, update, add, remove, reclassify, merge, split). "
|
||||
"Erzeuge nicht bei jeder Review ein komplett neues Profil.\n\n"
|
||||
+ shared
|
||||
)
|
||||
|
||||
|
||||
def render_paste_prompt(package: dict) -> str:
|
||||
mode = package.get("mode") or "review"
|
||||
task = semantic_task(mode)
|
||||
empty = package.get("kansho_evidence_empty")
|
||||
empty_note = (
|
||||
"\nIn diesem Paket sind keine oder nur wenige Kanshō-Quellen. "
|
||||
"Du darfst, wenn der Chat authentische Nutzertexte kennt, external_chat_history nutzen und das kennzeichnen.\n"
|
||||
if empty
|
||||
else "\n"
|
||||
)
|
||||
body = json.dumps(package, ensure_ascii=False, indent=2)
|
||||
return (
|
||||
"Kanshō Writing-Profile-Analyse\n\n"
|
||||
"Du hilfst, den persönlichen Schreibstil einer Person semantisch zu beschreiben. "
|
||||
"Kanshō ist ein Reflexionsbegleiter; du brauchst kein internes Datenmodell zu kennen. "
|
||||
"Das JSON unten ist das ProfileAnalysisPackage (gemeinsamer Vertrag für Copy/Paste und API).\n\n"
|
||||
f"{task}{empty_note}\n"
|
||||
"Antworte ausschließlich mit einem JSON-Objekt gemäß expected_result. "
|
||||
"Kein Fließtext davor oder danach. kind muss kansho.profile_analysis_result sein.\n\n"
|
||||
"ProfileAnalysisPackage:\n"
|
||||
f"{body}\n"
|
||||
)
|
||||
|
||||
|
||||
def package_note(mode: str) -> str:
|
||||
if mode == "initial_build":
|
||||
return (
|
||||
"Copy/Paste ist in der Testphase ein vollwertiger Ausführungsweg. "
|
||||
"Das Paket ist eine repräsentative Auswahl, keine unlimitierte Volltextsammlung. "
|
||||
"Der aktuelle Brief ist kein Exportformat."
|
||||
)
|
||||
return (
|
||||
"Inkrementelle Review gegen das bestehende Profil. "
|
||||
"Copy/Paste und API nutzen dasselbe Package. Der aktuelle Brief ist kein Importformat."
|
||||
)
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Portable JSON documents for Writing and Interaction profiles. Not a journal backup."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from dialogue_store import StoreError
|
||||
|
||||
FORMAT_VERSION = 1
|
||||
KIND_WRITING = "kansho.writing_profile"
|
||||
KIND_INTERACTION = "kansho.interaction_profile"
|
||||
KIND_BUNDLE = "kansho.profiles"
|
||||
ORIGIN_WRITING = {"inferred", "manual", "imported", "accepted_suggestion", "initial_build"}
|
||||
ORIGIN_INTERACTION = {"explicit", "manual", "accepted_suggestion"}
|
||||
|
||||
|
||||
def exported_at() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def as_document(raw) -> dict:
|
||||
if isinstance(raw, str):
|
||||
import json
|
||||
|
||||
try:
|
||||
raw = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise StoreError("invalid_document", "JSON ist ungültig.") from exc
|
||||
if not isinstance(raw, dict):
|
||||
raise StoreError("invalid_document", "Profil-Dokument muss ein JSON-Objekt sein.")
|
||||
version = raw.get("format_version", FORMAT_VERSION)
|
||||
try:
|
||||
version = int(version)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise StoreError("unsupported_format", "format_version fehlt oder ist ungültig.") from exc
|
||||
if version != FORMAT_VERSION:
|
||||
raise StoreError("unsupported_format", f"format_version {version} wird nicht unterstützt.")
|
||||
return raw
|
||||
|
||||
|
||||
def writing_payload(document: dict) -> dict:
|
||||
data = as_document(document)
|
||||
kind = (data.get("kind") or "").strip()
|
||||
if kind == KIND_BUNDLE:
|
||||
nested = data.get("writing")
|
||||
if not isinstance(nested, dict):
|
||||
raise StoreError("invalid_document", "Sammeldokument enthält kein Writing Profile.")
|
||||
return nested
|
||||
if kind != KIND_WRITING:
|
||||
raise StoreError("invalid_document", "Das ist kein Writing-Profile-Dokument.")
|
||||
return data
|
||||
|
||||
|
||||
def interaction_payload(document: dict) -> dict:
|
||||
data = as_document(document)
|
||||
kind = (data.get("kind") or "").strip()
|
||||
if kind == KIND_BUNDLE:
|
||||
nested = data.get("interaction")
|
||||
if not isinstance(nested, dict):
|
||||
raise StoreError("invalid_document", "Sammeldokument enthält kein Interaction Profile.")
|
||||
return nested
|
||||
if kind != KIND_INTERACTION:
|
||||
raise StoreError("invalid_document", "Das ist kein Interaction-Profile-Dokument.")
|
||||
return data
|
||||
|
||||
|
||||
def export_bundle(profile_id: str) -> dict:
|
||||
from interaction_profile_store import export_document as export_interaction
|
||||
from writing_profile_store import export_document as export_writing
|
||||
|
||||
writing = export_writing(profile_id)
|
||||
interaction = export_interaction(profile_id)
|
||||
writing.pop("kind", None)
|
||||
writing.pop("format_version", None)
|
||||
writing.pop("exported_at", None)
|
||||
interaction.pop("kind", None)
|
||||
interaction.pop("format_version", None)
|
||||
interaction.pop("exported_at", None)
|
||||
return {
|
||||
"kind": KIND_BUNDLE,
|
||||
"format_version": FORMAT_VERSION,
|
||||
"exported_at": exported_at(),
|
||||
"writing": writing,
|
||||
"interaction": interaction,
|
||||
}
|
||||
|
||||
|
||||
def restore_bundle(profile_id: str, document: dict) -> dict:
|
||||
data = as_document(document)
|
||||
kind = (data.get("kind") or "").strip()
|
||||
if kind != KIND_BUNDLE:
|
||||
raise StoreError("invalid_document", "Das ist kein Sammeldokument beider Profile.")
|
||||
from interaction_profile_store import restore_document as restore_interaction
|
||||
from writing_profile_store import restore_document as restore_writing
|
||||
|
||||
writing = restore_writing(profile_id, data)
|
||||
interaction = restore_interaction(profile_id, data)
|
||||
return {"writing": writing, "interaction": interaction}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,311 @@
|
||||
"""Conservative prompt budget for personal journal generation.
|
||||
|
||||
Token counts are estimates, not tokenizer-accurate. Kanshō overestimates input
|
||||
so a request is refused before it can overflow a context window. There is no
|
||||
silent middle-out deletion of the day.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
# Journal generation requires a wide window so a full day can be reconstructed
|
||||
# without dropping middle events. 32_768 is the product default, not a claim
|
||||
# that every model tokenizer uses this unit identically.
|
||||
JOURNAL_MIN_CONTEXT_TOKENS = 32_768
|
||||
JOURNAL_TARGET_COMPLETION_TOKENS = 4_096
|
||||
# Overestimate: German mixed with prompt English is typically closer to 3–4
|
||||
# characters per token. Two characters per token is deliberately conservative.
|
||||
DEFAULT_CHARS_PER_TOKEN = 2.0
|
||||
DEFAULT_SAFETY_MARGIN = 0.15
|
||||
CHAT_FORMAT_OVERHEAD_TOKENS = 64
|
||||
JOURNAL_PURPOSES = {"journal_generate", "journal_reconstruct"}
|
||||
|
||||
ERROR_MODEL_CONTEXT_TOO_SMALL = "model_context_too_small"
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED = "prompt_budget_exceeded"
|
||||
ERROR_MODEL_METADATA_UNKNOWN = "model_metadata_unknown"
|
||||
ERROR_RECONSTRUCTION_INVALID = "reconstruction_invalid"
|
||||
ERROR_NO_USER_SOURCES = "no_user_sources"
|
||||
ERROR_OUTPUT_LIMIT_UNSUPPORTED = "output_limit_unsupported"
|
||||
ERROR_PROVIDER_CONTEXT_LENGTH = "provider_context_length_rejected"
|
||||
|
||||
USER_MESSAGES = {
|
||||
ERROR_MODEL_CONTEXT_TOO_SMALL: (
|
||||
"Das gewählte Modell hat ein zu kleines Kontextfenster für die "
|
||||
"Journalgenerierung. Es sind mindestens 32.000 Tokens erforderlich. "
|
||||
"Es wurde kein anderes Modell gewählt."
|
||||
),
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED: (
|
||||
"Der Tagesdialog ist für eine sichere Verarbeitung zu umfangreich. "
|
||||
"Kanshō hat nichts stillschweigend aus der Mitte entfernt. "
|
||||
"Bitte wähle weniger Gespräche oder ein Modell mit größerem Kontext."
|
||||
),
|
||||
ERROR_MODEL_METADATA_UNKNOWN: (
|
||||
"Die Kontextgrenzen des Modells konnten nicht sicher bestimmt werden. "
|
||||
"Der Aufruf wurde nicht gesendet."
|
||||
),
|
||||
ERROR_RECONSTRUCTION_INVALID: (
|
||||
"Die inhaltliche Rekonstruktion war unvollständig oder ungültig. "
|
||||
"Es wurde kein Journalentwurf erzeugt."
|
||||
),
|
||||
ERROR_NO_USER_SOURCES: (
|
||||
"In den ausgewählten Gesprächen gibt es noch keinen Nutzertext. "
|
||||
"Es wurde kein Journalentwurf erzeugt."
|
||||
),
|
||||
ERROR_OUTPUT_LIMIT_UNSUPPORTED: (
|
||||
"Das Modell unterstützt die für die Journalgenerierung reservierte "
|
||||
"Ausgabelänge nicht. Der Aufruf wurde nicht gesendet."
|
||||
),
|
||||
ERROR_PROVIDER_CONTEXT_LENGTH: (
|
||||
"Der Anbieter hat die Anfrage wegen der Kontextlänge abgelehnt. "
|
||||
"Kanshō hat den Tagesdialog nicht automatisch gekürzt."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class JournalBudgetError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
code: str,
|
||||
message: str | None = None,
|
||||
status_code: int = 422,
|
||||
diagnostics: dict[str, Any] | None = None,
|
||||
):
|
||||
super().__init__(message or USER_MESSAGES.get(code) or code)
|
||||
self.code = code
|
||||
self.message = message or USER_MESSAGES.get(code) or code
|
||||
self.status_code = status_code
|
||||
self.diagnostics = diagnostics or {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelWindow:
|
||||
model: str
|
||||
context_length: int
|
||||
max_completion_tokens: int | None
|
||||
source: str
|
||||
provider: str | None = None
|
||||
cached: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class JournalBudget:
|
||||
model: str
|
||||
purpose: str
|
||||
effective_context_window: int
|
||||
reserved_output_tokens: int
|
||||
safety_margin: float
|
||||
estimated_input_tokens: int = 0
|
||||
available_input_tokens: int = 0
|
||||
chars_per_token: float = DEFAULT_CHARS_PER_TOKEN
|
||||
budget_ok: bool = True
|
||||
abort_reason: str | None = None
|
||||
context_compression: str = "disabled"
|
||||
extras: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_diagnostics(self) -> dict[str, Any]:
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"purpose": self.purpose,
|
||||
"estimated_input_tokens": self.estimated_input_tokens,
|
||||
"effective_context_window": self.effective_context_window,
|
||||
"reserved_output_tokens": self.reserved_output_tokens,
|
||||
"safety_margin": self.safety_margin,
|
||||
"available_input_tokens": self.available_input_tokens,
|
||||
"chars_per_token": self.chars_per_token,
|
||||
"budget_ok": self.budget_ok,
|
||||
"context_compression": self.context_compression,
|
||||
"abort_reason": self.abort_reason,
|
||||
"estimation": "conservative_char_ratio",
|
||||
}
|
||||
payload.update(self.extras)
|
||||
return payload
|
||||
|
||||
|
||||
def _float_env(name: str, default: float) -> float:
|
||||
raw = (os.environ.get(name) or "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
def _int_env(name: str, default: int) -> int:
|
||||
raw = (os.environ.get(name) or "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
def chars_per_token() -> float:
|
||||
return _float_env("KANSHO_TOKEN_CHARS_PER_TOKEN", DEFAULT_CHARS_PER_TOKEN)
|
||||
|
||||
|
||||
def safety_margin() -> float:
|
||||
return min(0.4, max(0.05, _float_env("KANSHO_JOURNAL_SAFETY_MARGIN", DEFAULT_SAFETY_MARGIN)))
|
||||
|
||||
|
||||
def min_context_tokens() -> int:
|
||||
return _int_env("KANSHO_JOURNAL_MIN_CONTEXT_TOKENS", JOURNAL_MIN_CONTEXT_TOKENS)
|
||||
|
||||
|
||||
def target_completion_tokens() -> int:
|
||||
return _int_env("KANSHO_JOURNAL_MAX_COMPLETION_TOKENS", JOURNAL_TARGET_COMPLETION_TOKENS)
|
||||
|
||||
|
||||
def day_message_safety_cap() -> int:
|
||||
return _int_env("KANSHO_JOURNAL_DAY_MAX_MESSAGES", 500)
|
||||
|
||||
|
||||
def estimate_tokens(text: str, *, extra: int = 0) -> int:
|
||||
"""Conservative character ratio. Not a model tokenizer."""
|
||||
n = len(text or "")
|
||||
if n <= 0:
|
||||
return extra
|
||||
return max(1, math.ceil(n / chars_per_token())) + extra
|
||||
|
||||
|
||||
def usable_context_tokens(window: ModelWindow) -> int:
|
||||
margin = safety_margin()
|
||||
usable = math.floor(window.context_length * (1.0 - margin))
|
||||
return max(0, usable)
|
||||
|
||||
|
||||
def reserved_output_tokens(window: ModelWindow) -> int:
|
||||
wanted = target_completion_tokens()
|
||||
supported = window.max_completion_tokens
|
||||
if supported is None or supported <= 0:
|
||||
raise JournalBudgetError(
|
||||
ERROR_OUTPUT_LIMIT_UNSUPPORTED,
|
||||
diagnostics={"model": window.model, "max_completion_tokens": supported},
|
||||
)
|
||||
reserved = min(wanted, int(supported))
|
||||
if reserved < 256:
|
||||
raise JournalBudgetError(
|
||||
ERROR_OUTPUT_LIMIT_UNSUPPORTED,
|
||||
diagnostics={
|
||||
"model": window.model,
|
||||
"max_completion_tokens": supported,
|
||||
"reserved_output_tokens": reserved,
|
||||
},
|
||||
)
|
||||
return reserved
|
||||
|
||||
|
||||
def plan_journal_budget(window: ModelWindow, *, purpose: str) -> JournalBudget:
|
||||
required = min_context_tokens()
|
||||
if window.context_length < required:
|
||||
raise JournalBudgetError(
|
||||
ERROR_MODEL_CONTEXT_TOO_SMALL,
|
||||
diagnostics={
|
||||
"model": window.model,
|
||||
"effective_context_window": window.context_length,
|
||||
"required_context_window": required,
|
||||
"source": window.source,
|
||||
},
|
||||
)
|
||||
reserved = reserved_output_tokens(window)
|
||||
usable = usable_context_tokens(window)
|
||||
available = usable - reserved - CHAT_FORMAT_OVERHEAD_TOKENS
|
||||
if available < 512:
|
||||
raise JournalBudgetError(
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
diagnostics={
|
||||
"model": window.model,
|
||||
"effective_context_window": window.context_length,
|
||||
"reserved_output_tokens": reserved,
|
||||
"available_input_tokens": available,
|
||||
},
|
||||
)
|
||||
compression = "disabled" if purpose in JOURNAL_PURPOSES else "provider_default"
|
||||
return JournalBudget(
|
||||
model=window.model,
|
||||
purpose=purpose,
|
||||
effective_context_window=window.context_length,
|
||||
reserved_output_tokens=reserved,
|
||||
safety_margin=safety_margin(),
|
||||
available_input_tokens=available,
|
||||
chars_per_token=chars_per_token(),
|
||||
context_compression=compression,
|
||||
extras={"metadata_source": window.source, "cached_metadata": window.cached},
|
||||
)
|
||||
|
||||
|
||||
def assert_input_fits(budget: JournalBudget, text: str) -> JournalBudget:
|
||||
estimated = estimate_tokens(text, extra=CHAT_FORMAT_OVERHEAD_TOKENS)
|
||||
budget.estimated_input_tokens = estimated
|
||||
total = estimated + budget.reserved_output_tokens
|
||||
ceiling = math.floor(budget.effective_context_window * (1.0 - budget.safety_margin))
|
||||
if estimated > budget.available_input_tokens or total > ceiling:
|
||||
budget.budget_ok = False
|
||||
budget.abort_reason = ERROR_PROMPT_BUDGET_EXCEEDED
|
||||
raise JournalBudgetError(
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
diagnostics={
|
||||
**budget.as_diagnostics(),
|
||||
"estimated_input_tokens": estimated,
|
||||
"required_tokens": total,
|
||||
"usable_tokens": ceiling,
|
||||
},
|
||||
)
|
||||
budget.budget_ok = True
|
||||
budget.abort_reason = None
|
||||
return budget
|
||||
|
||||
|
||||
def merge_usage(diagnostics: dict[str, Any], usage: dict[str, Any] | None, model: str | None = None) -> dict[str, Any]:
|
||||
payload = dict(diagnostics)
|
||||
data = usage or {}
|
||||
if model:
|
||||
payload["actual_model"] = model
|
||||
payload["prompt_tokens"] = data.get("prompt_tokens")
|
||||
payload["completion_tokens"] = data.get("completion_tokens")
|
||||
payload["total_tokens"] = data.get("total_tokens")
|
||||
cost = data.get("cost")
|
||||
if cost is None:
|
||||
cost = data.get("total_cost")
|
||||
payload["cost"] = cost
|
||||
return payload
|
||||
|
||||
|
||||
def sum_usages(parts: list[dict[str, Any] | None]) -> dict[str, Any]:
|
||||
prompt = 0
|
||||
completion = 0
|
||||
total = 0
|
||||
cost = 0.0
|
||||
saw_tokens = False
|
||||
saw_cost = False
|
||||
for item in parts:
|
||||
data = item or {}
|
||||
if data.get("prompt_tokens") is not None:
|
||||
prompt += int(data.get("prompt_tokens") or 0)
|
||||
saw_tokens = True
|
||||
if data.get("completion_tokens") is not None:
|
||||
completion += int(data.get("completion_tokens") or 0)
|
||||
saw_tokens = True
|
||||
if data.get("total_tokens") is not None:
|
||||
total += int(data.get("total_tokens") or 0)
|
||||
saw_tokens = True
|
||||
extra = data.get("cost")
|
||||
if extra is None:
|
||||
extra = data.get("total_cost")
|
||||
if extra is not None:
|
||||
cost += float(extra)
|
||||
saw_cost = True
|
||||
payload: dict[str, Any] = {}
|
||||
if saw_tokens:
|
||||
payload["prompt_tokens"] = prompt
|
||||
payload["completion_tokens"] = completion
|
||||
payload["total_tokens"] = total or (prompt + completion)
|
||||
if saw_cost:
|
||||
payload["cost"] = cost
|
||||
return payload
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Bind German 3rd-person pronouns to the last named person. Local; no new Detect tokens."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
PLACEHOLDER = re.compile(r"\[\[(PERSON:[^\]]+)\]\]")
|
||||
PIECE = re.compile(r"\[\[PERSON:[^\]]+\]\]|[A-Za-zÄÖÜäöüß]+")
|
||||
|
||||
FEM_CUES = {
|
||||
"frau", "partnerin", "tochter", "mutter", "schwester", "freundin",
|
||||
"oma", "tante",
|
||||
}
|
||||
MASC_CUES = {
|
||||
"herr", "mann", "partner", "sohn", "vater", "bruder", "freund",
|
||||
"opa", "onkel",
|
||||
}
|
||||
FEM_PRON = {"sie", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres"}
|
||||
MASC_PRON = {"er", "ihn", "ihm", "seine", "seinem", "seinen", "seiner", "seines"}
|
||||
IHR_ADDRESS = {
|
||||
"seid", "habt", "wart", "könnt", "müsst", "sollt", "wollt", "dürft",
|
||||
"hättet", "wäret", "werdet",
|
||||
}
|
||||
PLURAL_VERB = {
|
||||
"sind", "waren", "haben", "hatten", "werden", "wurden",
|
||||
"können", "müssen", "wollen", "sollen",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class BindState:
|
||||
last: str | None = None
|
||||
last_f: str | None = None
|
||||
last_m: str | None = None
|
||||
|
||||
|
||||
def _gender_from_prev(pieces: list[re.Match], index: int) -> str | None:
|
||||
seen: list[str] = []
|
||||
pos = index - 1
|
||||
while pos >= 0 and len(seen) < 3:
|
||||
raw = pieces[pos].group(0)
|
||||
if PLACEHOLDER.fullmatch(raw):
|
||||
break
|
||||
seen.append(raw.lower())
|
||||
pos -= 1
|
||||
window = set(seen)
|
||||
if window & FEM_CUES:
|
||||
return "f"
|
||||
if window & MASC_CUES:
|
||||
return "m"
|
||||
return None
|
||||
|
||||
|
||||
def _referent(state: BindState, kind: str) -> str | None:
|
||||
if kind == "f" and state.last_f:
|
||||
return state.last_f
|
||||
if kind == "m" and state.last_m:
|
||||
return state.last_m
|
||||
return state.last
|
||||
|
||||
|
||||
def _is_address_ihr(prev: str, nxt: str) -> bool:
|
||||
if nxt in IHR_ADDRESS or prev in IHR_ADDRESS:
|
||||
return True
|
||||
if prev.endswith("t") and len(prev) >= 4:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def bind_pronouns(text: str, state: BindState | None = None) -> tuple[str, BindState]:
|
||||
state = state or BindState()
|
||||
pieces = list(PIECE.finditer(text or ""))
|
||||
replacements: list[tuple[int, int, str]] = []
|
||||
for i, match in enumerate(pieces):
|
||||
raw = match.group(0)
|
||||
person = PLACEHOLDER.fullmatch(raw)
|
||||
if person:
|
||||
token = person.group(1)
|
||||
state.last = token
|
||||
gender = _gender_from_prev(pieces, i)
|
||||
if gender == "f":
|
||||
state.last_f = token
|
||||
elif gender == "m":
|
||||
state.last_m = token
|
||||
continue
|
||||
low = raw.lower()
|
||||
prev = pieces[i - 1].group(0).lower() if i else ""
|
||||
nxt = pieces[i + 1].group(0).lower() if i + 1 < len(pieces) else ""
|
||||
if low == "ihr" and _is_address_ihr(prev, nxt):
|
||||
continue
|
||||
if low == "sie" and nxt in PLURAL_VERB:
|
||||
continue
|
||||
kind = "f" if low in FEM_PRON else "m" if low in MASC_PRON else ""
|
||||
if not kind:
|
||||
continue
|
||||
ref = _referent(state, kind)
|
||||
if not ref:
|
||||
continue
|
||||
replacements.append((match.start(), match.end(), f"[[{ref}]]"))
|
||||
if not replacements:
|
||||
return text or "", state
|
||||
out = text or ""
|
||||
for start, end, repl in reversed(replacements):
|
||||
out = out[:start] + repl + out[end:]
|
||||
return out, state
|
||||
|
||||
|
||||
def bind_user_lines(text: str) -> str:
|
||||
"""Only user: lines; state carries across those lines. Instructions stay untouched."""
|
||||
state = BindState()
|
||||
chunks: list[str] = []
|
||||
for line in (text or "").splitlines(keepends=True):
|
||||
ended = line.endswith("\n")
|
||||
body = line[:-1] if ended else line
|
||||
if body.startswith("user:"):
|
||||
bound, state = bind_pronouns(body[5:], state)
|
||||
chunks.append("user:" + bound + ("\n" if ended else ""))
|
||||
else:
|
||||
chunks.append(line)
|
||||
return "".join(chunks)
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Intent-neutral provenance verification. Model output is untrusted input.
|
||||
|
||||
Canonical source texts come only from a locally built SourceRegistry.
|
||||
Downstream stages consume a locally materialized VerifiedArtifact.
|
||||
This module has no journal, dialogue, or product-intent vocabulary.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping
|
||||
|
||||
COVERAGE_ALL_SELECTED_SOURCES = "all_selected_sources"
|
||||
COVERAGE_SELECTED_EVIDENCE = "selected_evidence"
|
||||
COVERAGE_MODES = frozenset({COVERAGE_ALL_SELECTED_SOURCES, COVERAGE_SELECTED_EVIDENCE})
|
||||
ARTIFACT_KIND = "verified_artifact"
|
||||
|
||||
|
||||
class ProvenanceError(Exception):
|
||||
"""Fail-closed verification error. Domain mapping happens at the caller boundary."""
|
||||
|
||||
def __init__(self, reason: str, **details: Any):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
self.details = details
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceRecord:
|
||||
source_id: str
|
||||
role: str
|
||||
text: str
|
||||
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class SourceRegistry:
|
||||
"""Local authority for original content. Model text never replaces these records."""
|
||||
|
||||
def __init__(self, sources: list[SourceRecord] | None = None):
|
||||
self._sources: dict[str, SourceRecord] = {}
|
||||
self._order: list[str] = []
|
||||
for source in sources or []:
|
||||
self.add(source)
|
||||
|
||||
def add(self, source: SourceRecord) -> None:
|
||||
source_id = (source.source_id or "").strip()
|
||||
if not source_id:
|
||||
raise ProvenanceError("missing_source_id")
|
||||
if source_id in self._sources:
|
||||
raise ProvenanceError("duplicate_source_id", source_id=source_id)
|
||||
self._sources[source_id] = SourceRecord(
|
||||
source_id=source_id,
|
||||
role=source.role,
|
||||
text=source.text,
|
||||
metadata=dict(source.metadata or {}),
|
||||
)
|
||||
self._order.append(source_id)
|
||||
|
||||
def get(self, source_id: str) -> SourceRecord | None:
|
||||
return self._sources.get(source_id)
|
||||
|
||||
def order(self) -> list[str]:
|
||||
return list(self._order)
|
||||
|
||||
def items(self) -> list[SourceRecord]:
|
||||
return [self._sources[source_id] for source_id in self._order]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvidenceRef:
|
||||
source_id: str
|
||||
excerpt: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerifiedEvidence:
|
||||
source_id: str
|
||||
excerpt: str
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerificationPolicy:
|
||||
allowed_roles: frozenset[str]
|
||||
coverage: str = COVERAGE_ALL_SELECTED_SOURCES
|
||||
selected_ids: tuple[str, ...] = ()
|
||||
source_order: tuple[str, ...] = ()
|
||||
include_unverified_annotations: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerifiedArtifact:
|
||||
coverage: str
|
||||
source_order: tuple[str, ...]
|
||||
sources: tuple[dict[str, str], ...]
|
||||
evidence: tuple[VerifiedEvidence, ...]
|
||||
annotations_unverified: tuple[dict[str, Any], ...] = ()
|
||||
|
||||
def texts(self) -> list[str]:
|
||||
return [item.get("text") or "" for item in self.sources if item.get("text")]
|
||||
|
||||
def to_payload(self) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"kind": ARTIFACT_KIND,
|
||||
"coverage": self.coverage,
|
||||
"source_order": list(self.source_order),
|
||||
"order": [item.get("source_id") for item in self.sources],
|
||||
"sources": [dict(item) for item in self.sources],
|
||||
"evidence": [
|
||||
{
|
||||
"source_id": item.source_id,
|
||||
"excerpt": item.excerpt,
|
||||
"start": item.start,
|
||||
"end": item.end,
|
||||
}
|
||||
for item in self.evidence
|
||||
],
|
||||
}
|
||||
if self.annotations_unverified:
|
||||
payload["annotations_unverified"] = [dict(item) for item in self.annotations_unverified]
|
||||
return payload
|
||||
|
||||
|
||||
def locate_excerpt(text: str, excerpt: str) -> tuple[int, int] | None:
|
||||
"""Local span from a confirmed literal excerpt. Model offsets are ignored."""
|
||||
if not excerpt:
|
||||
return None
|
||||
start = (text or "").find(excerpt)
|
||||
if start < 0:
|
||||
return None
|
||||
return start, start + len(excerpt)
|
||||
|
||||
|
||||
def texts_from_payload(data: Mapping[str, Any] | None) -> list[str]:
|
||||
if not isinstance(data, Mapping):
|
||||
return []
|
||||
parts: list[str] = []
|
||||
for item in data.get("sources") or []:
|
||||
if isinstance(item, Mapping) and item.get("text"):
|
||||
parts.append(str(item["text"]))
|
||||
return parts
|
||||
|
||||
|
||||
def verify_evidence(
|
||||
registry: SourceRegistry,
|
||||
ref: EvidenceRef,
|
||||
policy: VerificationPolicy,
|
||||
) -> VerifiedEvidence:
|
||||
source_id = (ref.source_id or "").strip()
|
||||
excerpt = ref.excerpt or ""
|
||||
if not source_id:
|
||||
raise ProvenanceError("missing_source_id")
|
||||
if not excerpt.strip():
|
||||
raise ProvenanceError("unverified_excerpt", source_id=source_id)
|
||||
if policy.selected_ids and source_id not in policy.selected_ids:
|
||||
raise ProvenanceError("unknown_source", source_id=source_id)
|
||||
source = registry.get(source_id)
|
||||
if source is None:
|
||||
raise ProvenanceError("unknown_source", source_id=source_id)
|
||||
if source.role not in policy.allowed_roles:
|
||||
raise ProvenanceError("role_not_allowed", source_id=source_id, role=source.role)
|
||||
located = locate_excerpt(source.text, excerpt)
|
||||
if located is not None:
|
||||
return VerifiedEvidence(source_id, excerpt, located[0], located[1])
|
||||
for other in registry.items():
|
||||
if other.source_id == source_id or excerpt not in (other.text or ""):
|
||||
continue
|
||||
if other.role not in policy.allowed_roles:
|
||||
raise ProvenanceError("role_not_allowed", source_id=other.source_id, role=other.role)
|
||||
raise ProvenanceError("wrong_source", source_id=source_id, excerpt=excerpt[:120])
|
||||
raise ProvenanceError("unverified_excerpt", source_id=source_id, excerpt=excerpt[:120])
|
||||
|
||||
|
||||
def _selected_records(registry: SourceRegistry, policy: VerificationPolicy) -> list[SourceRecord]:
|
||||
if policy.coverage not in COVERAGE_MODES:
|
||||
raise ProvenanceError("unknown_coverage", coverage=policy.coverage)
|
||||
selected = tuple(item for item in policy.selected_ids if item)
|
||||
if not selected:
|
||||
raise ProvenanceError("selection_empty")
|
||||
if len(set(selected)) != len(selected):
|
||||
raise ProvenanceError("duplicate_source_id", ids=list(selected))
|
||||
records: list[SourceRecord] = []
|
||||
for source_id in selected:
|
||||
source = registry.get(source_id)
|
||||
if source is None:
|
||||
raise ProvenanceError("unknown_source", source_id=source_id)
|
||||
if source.role not in policy.allowed_roles:
|
||||
raise ProvenanceError("role_not_allowed", source_id=source_id, role=source.role)
|
||||
records.append(source)
|
||||
return records
|
||||
|
||||
|
||||
def verify(
|
||||
registry: SourceRegistry,
|
||||
*,
|
||||
policy: VerificationPolicy,
|
||||
evidence: list[EvidenceRef] | None = None,
|
||||
annotations: list[dict[str, Any]] | None = None,
|
||||
proposed_texts: Mapping[str, str] | None = None,
|
||||
) -> VerifiedArtifact:
|
||||
"""Materialize a VerifiedArtifact from local sources plus untrusted references."""
|
||||
selected = _selected_records(registry, policy)
|
||||
for source in selected:
|
||||
proposed = (proposed_texts or {}).get(source.source_id)
|
||||
if proposed is not None and proposed != source.text:
|
||||
raise ProvenanceError("canonical_overwrite", source_id=source.source_id)
|
||||
|
||||
verified: list[VerifiedEvidence] = []
|
||||
for ref in evidence or []:
|
||||
verified.append(verify_evidence(registry, ref, policy))
|
||||
|
||||
if policy.coverage == COVERAGE_ALL_SELECTED_SOURCES:
|
||||
sources = tuple(
|
||||
{"source_id": source.source_id, "role": source.role, "text": source.text} for source in selected
|
||||
)
|
||||
else:
|
||||
grouped: dict[str, list[str]] = {source.source_id: [] for source in selected}
|
||||
for item in verified:
|
||||
grouped[item.source_id].append(item.excerpt)
|
||||
sources = tuple(
|
||||
{
|
||||
"source_id": source.source_id,
|
||||
"role": source.role,
|
||||
"text": "\n".join(grouped[source.source_id]),
|
||||
}
|
||||
for source in selected
|
||||
)
|
||||
|
||||
unverified: tuple[dict[str, Any], ...] = ()
|
||||
if policy.include_unverified_annotations and annotations:
|
||||
unverified = tuple(dict(item) for item in annotations)
|
||||
|
||||
encounter = tuple(item for item in policy.source_order if item) or tuple(
|
||||
source.source_id for source in selected
|
||||
)
|
||||
return VerifiedArtifact(
|
||||
coverage=policy.coverage,
|
||||
source_order=encounter,
|
||||
sources=sources,
|
||||
evidence=tuple(verified),
|
||||
annotations_unverified=unverified,
|
||||
)
|
||||
@@ -0,0 +1,476 @@
|
||||
"""Admin-configurable generate/detect endpoints. Keys stay in env, never in the DB."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from db import get_db
|
||||
from detect_learning import get_detect_operating_mode
|
||||
from env_loader import (
|
||||
allows_remote_plaintext_detect,
|
||||
remote_plaintext_detect_reason,
|
||||
runtime_env,
|
||||
upsert_env_value,
|
||||
)
|
||||
|
||||
SEED_PATH = Path(__file__).resolve().parent / "config" / "providers.seed.json"
|
||||
ROLES = ("generate", "detect")
|
||||
OLLAMA_LAN_ID = "llm-ollama-lan"
|
||||
OLLAMA_LAN_URL = "http://192.168.2.144:11434/v1/chat/completions"
|
||||
_KEY_ENV = {
|
||||
"generate": "KANSHO_PROVIDER_KEY",
|
||||
"detect": "KANSHO_DETECT_PROVIDER_KEY",
|
||||
}
|
||||
# Interne Rollen bleiben generate/detect. Fachlich: Sprachmodell vs. Maskierung.
|
||||
ROLE_META = {
|
||||
"generate": {
|
||||
"title": "Sprachmodell",
|
||||
"task": "Dialogzug und Journalentwurf. Nur maskierter Kontext. Wählt die Operation im Dialogzug mit.",
|
||||
},
|
||||
"detect": {
|
||||
"title": "Maskierung",
|
||||
"task": "Eigener Detect-Endpunkt und eigenes Modell. Vollständige semantische Detection des persönlichen Egress. Ziel: lokales Detect-Modell (Ollama). Übergang: externes Klartext-Detect nur mit Operator-Freigabe. Kein Pattern-Fallback.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class SettingsError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 400):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def _validate_url(url: str) -> str:
|
||||
url = (url or "").strip()
|
||||
if not url:
|
||||
return ""
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise SettingsError("provider_url_invalid", "URL muss http(s) sein.")
|
||||
return url
|
||||
|
||||
|
||||
def _profile_row(conn, profile_id: str) -> dict | None:
|
||||
row = conn.execute("SELECT * FROM llm_profiles WHERE id = ?", (profile_id,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def _public_profile(row: dict) -> dict:
|
||||
from providers import is_local_url
|
||||
|
||||
url = row.get("url") or ""
|
||||
return {
|
||||
"id": row.get("id") or "",
|
||||
"title": row.get("title") or "",
|
||||
"name": row.get("name") or "",
|
||||
"url": url,
|
||||
"model": row.get("model") or "",
|
||||
"zdr": bool(row.get("zdr", 1)),
|
||||
"no_train": bool(row.get("no_train", 1)),
|
||||
"local": is_local_url(url),
|
||||
"updated": row.get("updated") or "",
|
||||
}
|
||||
|
||||
|
||||
def seed_provider_settings(conn) -> None:
|
||||
items = json.loads(SEED_PATH.read_text(encoding="utf-8"))
|
||||
for item in items:
|
||||
role = item["role"]
|
||||
if role not in ROLES:
|
||||
continue
|
||||
existing = conn.execute("SELECT role FROM provider_settings WHERE role = ?", (role,)).fetchone()
|
||||
if existing:
|
||||
continue
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO provider_settings (role, name, url, model, zdr, no_train)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
role,
|
||||
item.get("name") or "",
|
||||
item.get("url") or "",
|
||||
item.get("model") or "",
|
||||
1 if item.get("zdr", True) else 0,
|
||||
1 if item.get("no_train", True) else 0,
|
||||
),
|
||||
)
|
||||
seed_llm_profiles(conn)
|
||||
|
||||
|
||||
def seed_llm_profiles(conn) -> None:
|
||||
"""Named presets. Existing rows are not overwritten. Keys never stored."""
|
||||
ollama = conn.execute("SELECT * FROM llm_profiles WHERE id = ?", (OLLAMA_LAN_ID,)).fetchone()
|
||||
if not ollama:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO llm_profiles (id, title, name, url, model, zdr, no_train)
|
||||
VALUES (?, ?, ?, ?, ?, 1, 1)
|
||||
""",
|
||||
(OLLAMA_LAN_ID, "Ollama (LAN)", "ollama", OLLAMA_LAN_URL, "phi3:mini"),
|
||||
)
|
||||
elif not (dict(ollama).get("model") or "").strip():
|
||||
conn.execute(
|
||||
"UPDATE llm_profiles SET model = ?, updated = datetime('now') WHERE id = ?",
|
||||
("phi3:mini", OLLAMA_LAN_ID),
|
||||
)
|
||||
for role in ROLES:
|
||||
row = conn.execute("SELECT * FROM provider_settings WHERE role = ?", (role,)).fetchone()
|
||||
if not row:
|
||||
continue
|
||||
data = dict(row)
|
||||
profile_id = (data.get("profile_id") or "").strip()
|
||||
if profile_id and _profile_row(conn, profile_id):
|
||||
continue
|
||||
url = (data.get("url") or "").strip()
|
||||
model = (data.get("model") or "").strip()
|
||||
if not url:
|
||||
continue
|
||||
match = conn.execute(
|
||||
"SELECT id FROM llm_profiles WHERE url = ? AND model = ? ORDER BY created LIMIT 1",
|
||||
(url, model),
|
||||
).fetchone()
|
||||
if match:
|
||||
chosen = match["id"]
|
||||
else:
|
||||
chosen = f"llm-{role}-current"
|
||||
if _profile_row(conn, chosen):
|
||||
chosen = str(uuid.uuid4())
|
||||
title = "Sprachmodell (aktuell)" if role == "generate" else "Maskierung (aktuell)"
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO llm_profiles (id, title, name, url, model, zdr, no_train)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
chosen,
|
||||
title,
|
||||
data.get("name") or "",
|
||||
url,
|
||||
model,
|
||||
1 if data.get("zdr", 1) else 0,
|
||||
1 if data.get("no_train", 1) else 0,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE provider_settings SET profile_id = ? WHERE role = ?",
|
||||
(chosen, role),
|
||||
)
|
||||
|
||||
|
||||
def list_profiles() -> list[dict]:
|
||||
with get_db() as conn:
|
||||
rows = conn.execute("SELECT * FROM llm_profiles ORDER BY title, created").fetchall()
|
||||
return [_public_profile(dict(row)) for row in rows]
|
||||
|
||||
|
||||
def get_profile(profile_id: str) -> dict | None:
|
||||
with get_db() as conn:
|
||||
row = _profile_row(conn, profile_id)
|
||||
return _public_profile(row) if row else None
|
||||
|
||||
|
||||
def _write_profile(
|
||||
conn,
|
||||
profile_id: str,
|
||||
*,
|
||||
title: str,
|
||||
name: str,
|
||||
url: str,
|
||||
model: str,
|
||||
zdr: bool,
|
||||
no_train: bool,
|
||||
) -> dict:
|
||||
title = (title or "").strip()
|
||||
if not title:
|
||||
raise SettingsError("profile_title_required", "Ein Profil braucht einen Namen.")
|
||||
url = _validate_url(url)
|
||||
if not url:
|
||||
raise SettingsError("provider_url_required", "Ein Profil braucht eine URL.")
|
||||
model = (model or "").strip()
|
||||
name = (name or "").strip()
|
||||
existing = _profile_row(conn, profile_id)
|
||||
if existing:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE llm_profiles
|
||||
SET title = ?, name = ?, url = ?, model = ?, zdr = ?, no_train = ?, updated = datetime('now')
|
||||
WHERE id = ?
|
||||
""",
|
||||
(title, name, url, model, 1 if zdr else 0, 1 if no_train else 0, profile_id),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO llm_profiles (id, title, name, url, model, zdr, no_train)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(profile_id, title, name, url, model, 1 if zdr else 0, 1 if no_train else 0),
|
||||
)
|
||||
for role in ROLES:
|
||||
active = conn.execute(
|
||||
"SELECT role FROM provider_settings WHERE role = ? AND profile_id = ?",
|
||||
(role, profile_id),
|
||||
).fetchone()
|
||||
if not active:
|
||||
continue
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE provider_settings
|
||||
SET name = ?, url = ?, model = ?, zdr = ?, no_train = ?, updated = datetime('now')
|
||||
WHERE role = ?
|
||||
""",
|
||||
(name, url, model, 1 if zdr else 0, 1 if no_train else 0, role),
|
||||
)
|
||||
row = _profile_row(conn, profile_id)
|
||||
return _public_profile(row or {})
|
||||
|
||||
|
||||
def create_profile(*, title: str, name: str, url: str, model: str, zdr: bool, no_train: bool) -> dict:
|
||||
profile_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
return _write_profile(
|
||||
conn,
|
||||
profile_id,
|
||||
title=title,
|
||||
name=name,
|
||||
url=url,
|
||||
model=model,
|
||||
zdr=zdr,
|
||||
no_train=no_train,
|
||||
)
|
||||
|
||||
|
||||
def update_profile(
|
||||
profile_id: str,
|
||||
*,
|
||||
title: str,
|
||||
name: str,
|
||||
url: str,
|
||||
model: str,
|
||||
zdr: bool,
|
||||
no_train: bool,
|
||||
) -> dict:
|
||||
with get_db() as conn:
|
||||
if not _profile_row(conn, profile_id):
|
||||
raise SettingsError("profile_missing", "Profil fehlt.", 404)
|
||||
return _write_profile(
|
||||
conn,
|
||||
profile_id,
|
||||
title=title,
|
||||
name=name,
|
||||
url=url,
|
||||
model=model,
|
||||
zdr=zdr,
|
||||
no_train=no_train,
|
||||
)
|
||||
|
||||
|
||||
def delete_profile(profile_id: str) -> None:
|
||||
with get_db() as conn:
|
||||
if not _profile_row(conn, profile_id):
|
||||
raise SettingsError("profile_missing", "Profil fehlt.", 404)
|
||||
used = conn.execute(
|
||||
"SELECT role FROM provider_settings WHERE profile_id = ?",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
if used:
|
||||
roles = ", ".join(row["role"] for row in used)
|
||||
raise SettingsError(
|
||||
"profile_in_use",
|
||||
f"Profil ist noch der Stufe {roles} zugeordnet. Zuerst ein anderes Profil wählen.",
|
||||
)
|
||||
from detect_learning import get_local_ner_settings, set_local_ner_settings
|
||||
|
||||
if get_local_ner_settings().get("passage_profile_id") == profile_id:
|
||||
set_local_ner_settings(passage_profile_id="")
|
||||
conn.execute("DELETE FROM llm_profiles WHERE id = ?", (profile_id,))
|
||||
|
||||
|
||||
def activate_profile(role: str, profile_id: str) -> dict:
|
||||
if role not in ROLES:
|
||||
raise SettingsError("unknown_provider_role", "Unbekannte Provider-Rolle.")
|
||||
with get_db() as conn:
|
||||
row = _profile_row(conn, profile_id)
|
||||
if not row:
|
||||
raise SettingsError("profile_missing", "Profil fehlt.", 404)
|
||||
url = (row.get("url") or "").strip()
|
||||
model = (row.get("model") or "").strip()
|
||||
if role == "generate" and not url:
|
||||
raise SettingsError("provider_url_required", "Das Sprachmodell braucht eine URL.")
|
||||
if role == "detect" and not url:
|
||||
raise SettingsError("provider_url_required", "Die Maskierung braucht eine URL.")
|
||||
if not model:
|
||||
raise SettingsError("provider_model_required", "Das Profil braucht ein Modell.")
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO provider_settings (role, name, url, model, zdr, no_train, profile_id, updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))
|
||||
ON CONFLICT(role) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
url = excluded.url,
|
||||
model = excluded.model,
|
||||
zdr = excluded.zdr,
|
||||
no_train = excluded.no_train,
|
||||
profile_id = excluded.profile_id,
|
||||
updated = datetime('now')
|
||||
""",
|
||||
(
|
||||
role,
|
||||
row.get("name") or "",
|
||||
url,
|
||||
model,
|
||||
1 if row.get("zdr", 1) else 0,
|
||||
1 if row.get("no_train", 1) else 0,
|
||||
profile_id,
|
||||
),
|
||||
)
|
||||
return get_setting(role) or {}
|
||||
|
||||
|
||||
def get_setting(role: str) -> dict | None:
|
||||
if role not in ROLES:
|
||||
return None
|
||||
with get_db() as conn:
|
||||
row = conn.execute("SELECT * FROM provider_settings WHERE role = ?", (role,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def upsert_setting(
|
||||
role: str,
|
||||
*,
|
||||
name: str,
|
||||
url: str,
|
||||
model: str,
|
||||
zdr: bool,
|
||||
no_train: bool,
|
||||
profile_id: str | None = None,
|
||||
save_as_title: str | None = None,
|
||||
) -> dict:
|
||||
if role not in ROLES:
|
||||
raise SettingsError("unknown_provider_role", "Unbekannte Provider-Rolle.")
|
||||
url = _validate_url(url)
|
||||
model = (model or "").strip()
|
||||
name = (name or "").strip()
|
||||
if role == "generate" and not url:
|
||||
raise SettingsError("provider_url_required", "Das Sprachmodell braucht eine URL.")
|
||||
if role == "generate" and not model:
|
||||
raise SettingsError("provider_model_required", "Das Sprachmodell braucht ein Modell.")
|
||||
title = (save_as_title or "").strip()
|
||||
with get_db() as conn:
|
||||
current = conn.execute("SELECT * FROM provider_settings WHERE role = ?", (role,)).fetchone()
|
||||
current_id = (dict(current).get("profile_id") if current else "") or ""
|
||||
chosen = (profile_id or "").strip() or current_id
|
||||
if title:
|
||||
chosen = str(uuid.uuid4())
|
||||
_write_profile(
|
||||
conn,
|
||||
chosen,
|
||||
title=title,
|
||||
name=name,
|
||||
url=url,
|
||||
model=model,
|
||||
zdr=zdr,
|
||||
no_train=no_train,
|
||||
)
|
||||
elif chosen:
|
||||
existing = _profile_row(conn, chosen)
|
||||
if not existing:
|
||||
raise SettingsError("profile_missing", "Profil fehlt.", 404)
|
||||
_write_profile(
|
||||
conn,
|
||||
chosen,
|
||||
title=existing.get("title") or title or role,
|
||||
name=name,
|
||||
url=url,
|
||||
model=model,
|
||||
zdr=zdr,
|
||||
no_train=no_train,
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO provider_settings (role, name, url, model, zdr, no_train, profile_id, updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))
|
||||
ON CONFLICT(role) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
url = excluded.url,
|
||||
model = excluded.model,
|
||||
zdr = excluded.zdr,
|
||||
no_train = excluded.no_train,
|
||||
profile_id = excluded.profile_id,
|
||||
updated = datetime('now')
|
||||
""",
|
||||
(role, name, url, model, 1 if zdr else 0, 1 if no_train else 0, chosen or None),
|
||||
)
|
||||
return get_setting(role) or {}
|
||||
|
||||
|
||||
def set_role_key(role: str, key: str) -> None:
|
||||
if role not in ROLES:
|
||||
raise SettingsError("unknown_provider_role", "Unbekannte Provider-Rolle.")
|
||||
value = (key or "").strip()
|
||||
if not value:
|
||||
raise SettingsError("provider_key_empty", "Leerer Key wird nicht gespeichert.")
|
||||
upsert_env_value(_KEY_ENV[role], value)
|
||||
|
||||
|
||||
def public_role_status(role: str) -> dict:
|
||||
from providers import detect_provider, generate_provider, is_local_url
|
||||
|
||||
row = get_setting(role) or {}
|
||||
config = generate_provider() if role == "generate" else detect_provider()
|
||||
local = is_local_url(row.get("url") or "")
|
||||
own_key = bool((os.environ.get(_KEY_ENV[role]) or "").strip())
|
||||
generate_key = bool((os.environ.get(_KEY_ENV["generate"]) or "").strip())
|
||||
if own_key:
|
||||
key_source = "own"
|
||||
elif role == "detect" and generate_key and not local:
|
||||
key_source = "generate"
|
||||
else:
|
||||
key_source = "none"
|
||||
meta = ROLE_META.get(role, {})
|
||||
return {
|
||||
"role": role,
|
||||
"title": meta.get("title") or role,
|
||||
"task": meta.get("task") or "",
|
||||
"name": row.get("name") or (config.name if config else ""),
|
||||
"url": row.get("url") or "",
|
||||
"model": row.get("model") or "",
|
||||
"zdr": bool(row.get("zdr", 1)),
|
||||
"no_train": bool(row.get("no_train", 1)),
|
||||
"profile_id": row.get("profile_id") or "",
|
||||
"local": local,
|
||||
"key_present": key_source != "none",
|
||||
"key_source": key_source,
|
||||
"ready": config is not None,
|
||||
"mode": config.mode if config else "unconfigured",
|
||||
"remote_plaintext_detect": bool(config and not config.local and config.mode == "http" and role == "detect"),
|
||||
"remote_plaintext_allowed": allows_remote_plaintext_detect() if role == "detect" else None,
|
||||
"remote_plaintext_reason": remote_plaintext_detect_reason() if role == "detect" else None,
|
||||
"updated": row.get("updated") or "",
|
||||
}
|
||||
|
||||
|
||||
def public_status() -> dict:
|
||||
return {
|
||||
"roles": [public_role_status(role) for role in ROLES],
|
||||
"profiles": list_profiles(),
|
||||
"runtime_env": runtime_env(),
|
||||
"note": "LLM-Profile speichern URL und Modell je Endpunkt. Maskierung und Sprachmodell wählen je ein Profil. Semantische Detection ist Pflicht vor Generate. LAN-Ollama gilt als lokal. Keys nur in .env.",
|
||||
"remote_plaintext_reason": remote_plaintext_detect_reason(),
|
||||
"remote_plaintext_allowed": allows_remote_plaintext_detect(),
|
||||
"ollama_url": OLLAMA_LAN_URL,
|
||||
"detect_operating_mode": get_detect_operating_mode(),
|
||||
"local_ner": _local_ner_status(),
|
||||
}
|
||||
|
||||
|
||||
def _local_ner_status() -> dict:
|
||||
from detect_learning import get_local_ner_settings
|
||||
|
||||
return get_local_ner_settings()
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Two independently configured OpenAI-compatible providers: generate vs detect."""
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from env_loader import load_env_file
|
||||
|
||||
load_env_file()
|
||||
|
||||
CONTEXT_LENGTH_REJECT = re.compile(
|
||||
r"context.?length|maximum context|prompt is too long|too many tokens|"
|
||||
r"context window|max context|exceeds? (?:the )?(?:maximum|context)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
class ProviderError(Exception):
|
||||
def __init__(self, code: str, message: str, status_code: int = 503, diagnostics: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
self.diagnostics = diagnostics or {}
|
||||
|
||||
|
||||
PROVIDER_MESSAGE_MAX = 240
|
||||
_PROVIDER_USAGE_KEYS = ("prompt_tokens", "completion_tokens", "total_tokens", "cost", "total_cost")
|
||||
|
||||
|
||||
def provider_error_diagnostics(status_code: int, body: str, *, extra: dict | None = None) -> dict:
|
||||
"""Compact provider failure fields. No prompt body, no mapping labels."""
|
||||
diagnostics: dict = {"http_status": status_code}
|
||||
if extra:
|
||||
diagnostics.update({key: value for key, value in extra.items() if value is not None})
|
||||
parsed = None
|
||||
try:
|
||||
parsed = json.loads(body or "")
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
parsed = None
|
||||
if isinstance(parsed, dict):
|
||||
err = parsed.get("error")
|
||||
if isinstance(err, str) and err.strip():
|
||||
diagnostics["provider_message"] = err.strip()[:PROVIDER_MESSAGE_MAX]
|
||||
elif isinstance(err, dict):
|
||||
msg = str(err.get("message") or err.get("msg") or "").strip()
|
||||
if msg:
|
||||
diagnostics["provider_message"] = msg[:PROVIDER_MESSAGE_MAX]
|
||||
code = err.get("code") or err.get("type")
|
||||
if code:
|
||||
diagnostics["provider_code"] = str(code)[:80]
|
||||
usage = parsed.get("usage")
|
||||
if isinstance(usage, dict):
|
||||
kept = {key: usage[key] for key in _PROVIDER_USAGE_KEYS if usage.get(key) is not None}
|
||||
if kept:
|
||||
diagnostics["usage"] = kept
|
||||
if kept.get("cost") is not None:
|
||||
diagnostics["cost"] = kept["cost"]
|
||||
elif kept.get("total_cost") is not None:
|
||||
diagnostics["cost"] = kept["total_cost"]
|
||||
return diagnostics
|
||||
|
||||
|
||||
def _reject_message(role: str, status_code: int, diagnostics: dict) -> str:
|
||||
base = f"{role}-Provider hat die Anfrage abgelehnt"
|
||||
if status_code:
|
||||
base += f" (HTTP {status_code})"
|
||||
snippet = (diagnostics.get("provider_message") or "").strip()
|
||||
if snippet:
|
||||
return f"{base}: {snippet}"
|
||||
return f"{base}."
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderConfig:
|
||||
role: str
|
||||
name: str
|
||||
mode: str
|
||||
url: str
|
||||
model: str
|
||||
key: str
|
||||
local: bool
|
||||
zdr: bool
|
||||
no_train: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatResult:
|
||||
content: str
|
||||
model: str | None = None
|
||||
usage: dict = field(default_factory=dict)
|
||||
context_compression: str = "not_applicable"
|
||||
finish_reason: str | None = None
|
||||
|
||||
|
||||
def is_openrouter(config: ProviderConfig) -> bool:
|
||||
host = (urlparse(config.url or "").hostname or "").lower()
|
||||
return host.endswith("openrouter.ai") or (config.name or "").lower() == "openrouter"
|
||||
|
||||
|
||||
def _truthy(name: str) -> bool:
|
||||
return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def is_local_url(url: str) -> bool:
|
||||
"""Loopback, RFC1918/ULA and .local stay in the local trusted zone.
|
||||
|
||||
Homelab Ollama (e.g. 192.168.2.144) is local, not an external provider.
|
||||
"""
|
||||
host = (urlparse(url or "").hostname or "").lower()
|
||||
if not host:
|
||||
return False
|
||||
if host in {"localhost", "127.0.0.1", "::1", "ollama"}:
|
||||
return True
|
||||
if host.endswith(".local"):
|
||||
return True
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
return False
|
||||
return bool(ip.is_loopback or ip.is_private or ip.is_link_local)
|
||||
|
||||
|
||||
def _is_local_url(url: str) -> bool:
|
||||
return is_local_url(url)
|
||||
|
||||
|
||||
def _setting(role: str) -> dict:
|
||||
try:
|
||||
from provider_settings import get_setting
|
||||
|
||||
return get_setting(role) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _text(env_name: str, db_value: str | None, default: str = "") -> str:
|
||||
env = (os.environ.get(env_name) or "").strip()
|
||||
if env:
|
||||
return env
|
||||
if db_value is None:
|
||||
return default
|
||||
return str(db_value).strip()
|
||||
|
||||
|
||||
def _flag(env_name: str, db_value, default: bool = True) -> bool:
|
||||
if (os.environ.get(env_name) or "").strip():
|
||||
return _truthy(env_name)
|
||||
if db_value is None:
|
||||
return default
|
||||
return bool(int(db_value) if not isinstance(db_value, bool) else db_value)
|
||||
|
||||
|
||||
def generate_provider() -> ProviderConfig | None:
|
||||
if _truthy("KANSHO_FAKE_PROVIDER"):
|
||||
return ProviderConfig(
|
||||
role="generate",
|
||||
name="fake",
|
||||
mode="fake",
|
||||
url="",
|
||||
model="fake",
|
||||
key="",
|
||||
local=True,
|
||||
zdr=True,
|
||||
no_train=True,
|
||||
)
|
||||
row = _setting("generate")
|
||||
key = (os.environ.get("KANSHO_PROVIDER_KEY") or "").strip()
|
||||
url = _text("KANSHO_PROVIDER_URL", row.get("url"), "https://openrouter.ai/api/v1/chat/completions")
|
||||
model = _text("KANSHO_PROVIDER_MODEL", row.get("model"), "openai/gpt-4o-mini")
|
||||
local = is_local_url(url)
|
||||
zdr = local or _flag("KANSHO_PROVIDER_ZDR", row.get("zdr"), True)
|
||||
no_train = local or _flag("KANSHO_PROVIDER_NO_TRAIN", row.get("no_train"), True)
|
||||
if not url or not model:
|
||||
return None
|
||||
if not local and not key:
|
||||
return None
|
||||
if not local and (not zdr or not no_train):
|
||||
return None
|
||||
return ProviderConfig(
|
||||
role="generate",
|
||||
name=row.get("name") or ("ollama" if local else "openrouter"),
|
||||
mode="http",
|
||||
url=url,
|
||||
model=model,
|
||||
key=key,
|
||||
local=local,
|
||||
zdr=zdr,
|
||||
no_train=no_train,
|
||||
)
|
||||
|
||||
|
||||
def detect_provider() -> ProviderConfig | None:
|
||||
if _truthy("KANSHO_FAKE_DETECT"):
|
||||
return ProviderConfig(
|
||||
role="detect",
|
||||
name="fake-detect",
|
||||
mode="fake",
|
||||
url="",
|
||||
model="fake",
|
||||
key="",
|
||||
local=True,
|
||||
zdr=True,
|
||||
no_train=True,
|
||||
)
|
||||
row = _setting("detect")
|
||||
url = _text("KANSHO_DETECT_PROVIDER_URL", row.get("url"), "")
|
||||
if not url:
|
||||
return None
|
||||
model = _text("KANSHO_DETECT_PROVIDER_MODEL", row.get("model"), "openai/gpt-4.1-nano")
|
||||
if not model:
|
||||
return None
|
||||
key = (os.environ.get("KANSHO_DETECT_PROVIDER_KEY") or "").strip()
|
||||
local = is_local_url(url)
|
||||
if not key and not local:
|
||||
key = (os.environ.get("KANSHO_PROVIDER_KEY") or "").strip()
|
||||
zdr = local or _flag("KANSHO_DETECT_ZDR", row.get("zdr"), True)
|
||||
no_train = local or _flag("KANSHO_DETECT_NO_TRAIN", row.get("no_train"), True)
|
||||
if not local and not key:
|
||||
return None
|
||||
if not local and (not zdr or not no_train):
|
||||
return None
|
||||
return ProviderConfig(
|
||||
role="detect",
|
||||
name=row.get("name") or ("ollama" if local else "openrouter"),
|
||||
mode="http",
|
||||
url=url,
|
||||
model=model,
|
||||
key=key,
|
||||
local=local,
|
||||
zdr=zdr,
|
||||
no_train=no_train,
|
||||
)
|
||||
|
||||
|
||||
def complete_chat(
|
||||
config: ProviderConfig,
|
||||
messages: list[dict],
|
||||
*,
|
||||
timeout: float,
|
||||
max_tokens: int | None = None,
|
||||
disable_context_compression: bool = False,
|
||||
) -> ChatResult:
|
||||
payload: dict = {"model": config.model, "messages": messages}
|
||||
if max_tokens is not None:
|
||||
payload["max_tokens"] = max_tokens
|
||||
compression = "not_applicable"
|
||||
if not config.local:
|
||||
payload["provider"] = {"data_collection": "deny"}
|
||||
if disable_context_compression and is_openrouter(config) and not config.local:
|
||||
payload["plugins"] = [{"id": "context-compression", "enabled": False}]
|
||||
compression = "disabled"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if config.key:
|
||||
headers["Authorization"] = f"Bearer {config.key}"
|
||||
if not config.local:
|
||||
headers["HTTP-Referer"] = "https://kansho.local"
|
||||
headers["X-Title"] = "Kansho"
|
||||
try:
|
||||
response = httpx.post(config.url, json=payload, headers=headers, timeout=timeout)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise ProviderError(
|
||||
"provider_timeout",
|
||||
f"{config.role}-Provider hat nicht innerhalb von {int(timeout)} Sekunden geantwortet.",
|
||||
504,
|
||||
diagnostics={"timeout_s": timeout, "generate_requested": True},
|
||||
) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise ProviderError(
|
||||
"provider_unreachable",
|
||||
f"{config.role}-Provider ist nicht erreichbar.",
|
||||
503,
|
||||
diagnostics={"generate_requested": True},
|
||||
) from exc
|
||||
if response.status_code >= 400:
|
||||
body = ""
|
||||
try:
|
||||
body = response.text or ""
|
||||
except Exception:
|
||||
body = ""
|
||||
diagnostics = provider_error_diagnostics(
|
||||
response.status_code,
|
||||
body,
|
||||
extra={"context_compression": compression, "generate_requested": True},
|
||||
)
|
||||
if CONTEXT_LENGTH_REJECT.search(body):
|
||||
raise ProviderError(
|
||||
"provider_context_length_rejected",
|
||||
"Der Anbieter hat die Anfrage wegen der Kontextlänge abgelehnt. "
|
||||
"Kanshō hat den Tagesdialog nicht automatisch gekürzt.",
|
||||
502,
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
raise ProviderError(
|
||||
"provider_rejected",
|
||||
_reject_message(config.role, response.status_code, diagnostics),
|
||||
502,
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
data = response.json()
|
||||
try:
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ProviderError("provider_shape", f"Unerwartete {config.role}-Antwort.") from exc
|
||||
usage = data.get("usage") if isinstance(data.get("usage"), dict) else {}
|
||||
finish_reason = None
|
||||
try:
|
||||
finish_reason = data["choices"][0].get("finish_reason")
|
||||
except (KeyError, IndexError, TypeError, AttributeError):
|
||||
finish_reason = None
|
||||
return ChatResult(
|
||||
content=content or "",
|
||||
model=data.get("model") or config.model,
|
||||
usage=usage,
|
||||
context_compression=compression,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
@@ -0,0 +1,12 @@
|
||||
[pytest]
|
||||
testpaths = tests
|
||||
# Default python collector is disabled; tests/conftest.py collects each
|
||||
# test_*.py as one case via main(). That keeps the existing scripts until
|
||||
# they are split into native pytest functions.
|
||||
python_files = __pytest_default_collector_disabled__.py
|
||||
python_functions = test_*
|
||||
addopts = -ra --tb=short
|
||||
markers =
|
||||
unit: no Postgres; safe on Windows SQLite checkouts
|
||||
integration: API/store tests on kansho_test (never kansho_dev)
|
||||
slow: long-running; omitted in the default Gitea command
|
||||
@@ -0,0 +1,3 @@
|
||||
# CI / local test runner. Install on top of requirements.txt:
|
||||
# pip install -r requirements.txt -r requirements-dev.txt
|
||||
pytest>=8.0
|
||||
@@ -2,3 +2,5 @@ fastapi==0.115.6
|
||||
uvicorn[standard]==0.34.0
|
||||
bcrypt==4.2.1
|
||||
httpx==0.28.1
|
||||
python-multipart==0.0.20
|
||||
psycopg[binary]==3.2.6
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Swappable retrieval interface. Specs name what to select; recency/SQL is implementation."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from db import get_db, row_to_dict
|
||||
from journal_body import plain_text
|
||||
from prompt_budget import (
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
JournalBudgetError,
|
||||
day_message_safety_cap,
|
||||
estimate_tokens,
|
||||
)
|
||||
|
||||
# Implementation caps. Not part of the selection-spec contract.
|
||||
_DAY_MESSAGE_CAP = 80
|
||||
_SPACE_ENTRY_LIMIT = 5
|
||||
_SPACE_ENTRY_EXCERPT_CHARS = 400
|
||||
_SPACE_SOURCE_CONV_LIMIT = 3
|
||||
_SPACE_SOURCE_EXCERPT_CHARS = 400
|
||||
_SPACE_SOURCE_MESSAGE_CAP = 8
|
||||
|
||||
|
||||
def retrieve(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||
kind = spec.get("kind")
|
||||
if kind == "day_messages":
|
||||
return _day_messages(profile_id, spec)
|
||||
if kind == "space_entries":
|
||||
return _space_entries(profile_id, spec)
|
||||
if kind == "space_recent_sources":
|
||||
return _space_recent_sources(profile_id, spec)
|
||||
if kind == "writing_profile":
|
||||
return _writing_profile(profile_id)
|
||||
return []
|
||||
|
||||
|
||||
def format_day_messages(messages: list[dict], *, with_source_ids: bool = False) -> str:
|
||||
parts = []
|
||||
for message in messages:
|
||||
role = message.get("role") or "user"
|
||||
body = message.get("body") or ""
|
||||
source_id = message.get("source_id")
|
||||
if with_source_ids and source_id:
|
||||
parts.append(f"[{source_id}] {role}: {body}")
|
||||
else:
|
||||
parts.append(f"{role}: {body}")
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
def pair_user_priority_messages(messages: list[dict]) -> list[dict]:
|
||||
"""Keep every user message. Assistant context stays inside the same conversation."""
|
||||
selected: list[dict] = []
|
||||
pending_assistant = None
|
||||
pending_conversation = None
|
||||
for message in messages:
|
||||
conversation_id = message.get("conversation_id")
|
||||
role = message.get("role") or "user"
|
||||
if pending_assistant is not None and conversation_id != pending_conversation:
|
||||
pending_assistant = None
|
||||
pending_conversation = None
|
||||
if role == "assistant":
|
||||
pending_assistant = message
|
||||
pending_conversation = conversation_id
|
||||
continue
|
||||
if role == "user":
|
||||
if pending_assistant is not None and pending_conversation == conversation_id:
|
||||
selected.append(pending_assistant)
|
||||
selected.append(message)
|
||||
pending_assistant = None
|
||||
pending_conversation = None
|
||||
continue
|
||||
pending_assistant = None
|
||||
pending_conversation = None
|
||||
return selected
|
||||
|
||||
|
||||
def _day_messages(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||
journal_day_id = spec.get("journal_day_id")
|
||||
conversation_ids = spec.get("conversation_ids") or []
|
||||
if not journal_day_id:
|
||||
return []
|
||||
overflow = spec.get("overflow") or "limit"
|
||||
if overflow == "abort":
|
||||
cap = int(spec.get("message_cap") or day_message_safety_cap())
|
||||
fetch_limit = cap + 1
|
||||
else:
|
||||
cap = int(spec.get("message_cap") or _DAY_MESSAGE_CAP)
|
||||
fetch_limit = cap
|
||||
with get_db() as conn:
|
||||
if conversation_ids:
|
||||
placeholders = ",".join("?" * len(conversation_ids))
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT m.*, c.title AS conversation_title
|
||||
FROM messages m
|
||||
JOIN conversations c ON c.id = m.conversation_id
|
||||
WHERE m.profile_id = ? AND c.journal_day_id = ? AND m.conversation_id IN ({placeholders})
|
||||
ORDER BY c.created, c.id, m.seq, m.id
|
||||
LIMIT ?
|
||||
""",
|
||||
(profile_id, journal_day_id, *conversation_ids, fetch_limit),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT m.*, c.title AS conversation_title
|
||||
FROM messages m
|
||||
JOIN conversations c ON c.id = m.conversation_id
|
||||
WHERE m.profile_id = ? AND c.journal_day_id = ?
|
||||
ORDER BY c.created, c.id, m.seq, m.id
|
||||
LIMIT ?
|
||||
""",
|
||||
(profile_id, journal_day_id, fetch_limit),
|
||||
).fetchall()
|
||||
messages = [row_to_dict(row) for row in rows]
|
||||
if overflow == "abort":
|
||||
if len(messages) > cap:
|
||||
raise JournalBudgetError(
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
diagnostics={"reason": "day_message_cap", "message_cap": cap, "fetched": len(messages)},
|
||||
)
|
||||
from journal_reconstruct import assign_source_ids
|
||||
|
||||
paired = assign_source_ids(pair_user_priority_messages(messages))
|
||||
max_tokens = spec.get("max_estimated_tokens")
|
||||
if max_tokens is not None:
|
||||
used = estimate_tokens(format_day_messages(paired, with_source_ids=True))
|
||||
if used > int(max_tokens):
|
||||
raise JournalBudgetError(
|
||||
ERROR_PROMPT_BUDGET_EXCEEDED,
|
||||
diagnostics={
|
||||
"reason": "day_dialogue_tokens",
|
||||
"estimated_tokens": used,
|
||||
"available_tokens": int(max_tokens),
|
||||
"message_count": len(paired),
|
||||
},
|
||||
)
|
||||
return paired
|
||||
return messages
|
||||
|
||||
|
||||
def _space_entries(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||
space_id = spec.get("space_id")
|
||||
exclude_day_id = spec.get("exclude_day_id")
|
||||
if not space_id:
|
||||
return []
|
||||
params: list[Any] = [profile_id, space_id]
|
||||
exclude_sql = ""
|
||||
if exclude_day_id:
|
||||
exclude_sql = "AND e.journal_day_id != ?"
|
||||
params.append(exclude_day_id)
|
||||
params.append(_SPACE_ENTRY_LIMIT)
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT e.id AS entry_id, e.journal_day_id, d.calendar_date, v.title, v.body, v.origin
|
||||
FROM journal_entries e
|
||||
JOIN journal_days d ON d.id = e.journal_day_id
|
||||
JOIN journal_entry_versions v ON v.id = e.current_version_id
|
||||
WHERE e.profile_id = ? AND e.space_id = ? AND e.deleted_at IS NULL {exclude_sql}
|
||||
ORDER BY d.calendar_date DESC, e.updated DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
result = []
|
||||
for row in rows:
|
||||
item = row_to_dict(row)
|
||||
item["excerpt"] = plain_text(item.get("body") or "")[:_SPACE_ENTRY_EXCERPT_CHARS]
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def _space_recent_sources(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||
"""Recent original conversations in the same space. Recency is the current heuristic."""
|
||||
space_id = spec.get("space_id")
|
||||
exclude_conversation_id = spec.get("exclude_conversation_id")
|
||||
exclude_journal_day_id = spec.get("exclude_journal_day_id")
|
||||
if not space_id:
|
||||
return []
|
||||
params: list[Any] = [profile_id, space_id]
|
||||
extra = ""
|
||||
if exclude_conversation_id:
|
||||
extra += " AND c.id != ?"
|
||||
params.append(exclude_conversation_id)
|
||||
if exclude_journal_day_id:
|
||||
extra += " AND (c.journal_day_id IS NULL OR c.journal_day_id != ?)"
|
||||
params.append(exclude_journal_day_id)
|
||||
params.append(_SPACE_SOURCE_CONV_LIMIT)
|
||||
with get_db() as conn:
|
||||
conversations = conn.execute(
|
||||
f"""
|
||||
SELECT c.id, c.updated, c.journal_day_id, d.calendar_date
|
||||
FROM conversations c
|
||||
LEFT JOIN journal_days d ON d.id = c.journal_day_id AND d.profile_id = c.profile_id
|
||||
WHERE c.profile_id = ? AND c.space_id = ? {extra}
|
||||
ORDER BY c.updated DESC, c.created DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
result = []
|
||||
for conv in conversations:
|
||||
messages = conn.execute(
|
||||
"""
|
||||
SELECT body FROM messages
|
||||
WHERE profile_id = ? AND conversation_id = ? AND role = 'user'
|
||||
ORDER BY seq
|
||||
LIMIT ?
|
||||
""",
|
||||
(profile_id, conv["id"], _SPACE_SOURCE_MESSAGE_CAP),
|
||||
).fetchall()
|
||||
excerpt = " ".join((row["body"] or "").strip() for row in messages if (row["body"] or "").strip())
|
||||
excerpt = excerpt[:_SPACE_SOURCE_EXCERPT_CHARS]
|
||||
if not excerpt:
|
||||
continue
|
||||
result.append(
|
||||
{
|
||||
"conversation_id": conv["id"],
|
||||
"journal_day_id": conv["journal_day_id"],
|
||||
"calendar_date": conv["calendar_date"],
|
||||
"excerpt": excerpt,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _writing_profile(profile_id: str) -> list[dict]:
|
||||
with get_db() as conn:
|
||||
row = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT compiled_brief, updated FROM writing_profiles WHERE profile_id = ?",
|
||||
(profile_id,),
|
||||
).fetchone()
|
||||
)
|
||||
sources = [
|
||||
row_to_dict(item)
|
||||
for item in conn.execute(
|
||||
"""
|
||||
SELECT id, kind, body, entry_id, weight, created
|
||||
FROM writing_profile_sources
|
||||
WHERE profile_id = ?
|
||||
ORDER BY CASE kind
|
||||
WHEN 'journal_entry' THEN 0
|
||||
WHEN 'imported_text' THEN 1
|
||||
ELSE 2
|
||||
END, created DESC
|
||||
""",
|
||||
(profile_id,),
|
||||
).fetchall()
|
||||
]
|
||||
if not row and not sources:
|
||||
return []
|
||||
return [{"compiled_brief": (row or {}).get("compiled_brief") or "", "sources": sources}]
|
||||
+450
-1
@@ -1,8 +1,49 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field
|
||||
import json
|
||||
import os
|
||||
|
||||
from auth import require_admin_dep
|
||||
from db import get_db
|
||||
from debug_store import (
|
||||
clear_runs,
|
||||
debug_tree,
|
||||
delete_run,
|
||||
export_document,
|
||||
export_filename,
|
||||
get_run,
|
||||
latest_run,
|
||||
list_runs,
|
||||
render_markdown,
|
||||
set_persist_enabled,
|
||||
settings_payload,
|
||||
)
|
||||
from dialogue_store import StoreError, get_conversation, inventory, list_conversations, list_derived_for_conversation, list_messages
|
||||
from detect_learning import list_senses
|
||||
from identity_store import (
|
||||
confirm_identity,
|
||||
confirm_review_proposal,
|
||||
deactivate_identity,
|
||||
delete_identity,
|
||||
delete_review_proposal,
|
||||
dismiss_review_proposal,
|
||||
list_registry,
|
||||
list_review_proposals,
|
||||
update_identity,
|
||||
)
|
||||
from provider_settings import (
|
||||
ROLES,
|
||||
SettingsError,
|
||||
activate_profile,
|
||||
create_profile,
|
||||
delete_profile,
|
||||
list_profiles,
|
||||
public_status,
|
||||
set_role_key,
|
||||
update_profile,
|
||||
upsert_setting,
|
||||
)
|
||||
from version import APP_VERSION, BUILD_DATE, MODULE_VERSIONS
|
||||
|
||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
@@ -15,6 +56,8 @@ def admin_health(session: dict = Depends(require_admin_dep)):
|
||||
prompts = conn.execute("SELECT COUNT(*) AS n FROM ai_prompts").fetchone()["n"]
|
||||
features = conn.execute("SELECT COUNT(*) AS n FROM features").fetchone()["n"]
|
||||
dialogue = inventory()
|
||||
providers = public_status()
|
||||
debug = settings_payload(session["profile_id"])
|
||||
return {
|
||||
"ok": True,
|
||||
"service": "kansho",
|
||||
@@ -26,6 +69,12 @@ def admin_health(session: dict = Depends(require_admin_dep)):
|
||||
"prompts": prompts,
|
||||
"features": features,
|
||||
"dialogue": dialogue,
|
||||
"debug_runs": debug["run_count"],
|
||||
"debug_persist_enabled": debug["persist_enabled"],
|
||||
"providers": {
|
||||
role["role"]: {"ready": role["ready"], "key_present": role["key_present"]}
|
||||
for role in providers["roles"]
|
||||
},
|
||||
},
|
||||
"session": {
|
||||
"profile_id": session["profile_id"],
|
||||
@@ -54,3 +103,403 @@ def admin_conversation(conversation_id: str, session: dict = Depends(require_adm
|
||||
}
|
||||
except StoreError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
|
||||
|
||||
class DebugToggle(BaseModel):
|
||||
persist_enabled: bool
|
||||
|
||||
|
||||
@router.get("/debug")
|
||||
def admin_debug(session: dict = Depends(require_admin_dep)):
|
||||
return settings_payload(session["profile_id"])
|
||||
|
||||
|
||||
@router.put("/debug")
|
||||
def admin_update_debug(body: DebugToggle, session: dict = Depends(require_admin_dep)):
|
||||
payload = set_persist_enabled(body.persist_enabled)
|
||||
payload["run_count"] = settings_payload(session["profile_id"])["run_count"]
|
||||
return payload
|
||||
|
||||
|
||||
@router.get("/debug/tree")
|
||||
def admin_debug_tree(session: dict = Depends(require_admin_dep)):
|
||||
return debug_tree(session["profile_id"])
|
||||
|
||||
|
||||
@router.get("/debug/runs")
|
||||
def admin_debug_runs(limit: int = Query(default=200, ge=1, le=500), session: dict = Depends(require_admin_dep)):
|
||||
return {"runs": list_runs(session["profile_id"], limit=limit)}
|
||||
|
||||
|
||||
@router.get("/debug/latest")
|
||||
def admin_debug_latest(
|
||||
journal_day_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
session: dict = Depends(require_admin_dep),
|
||||
):
|
||||
return {
|
||||
"run": latest_run(
|
||||
session["profile_id"],
|
||||
journal_day_id=journal_day_id,
|
||||
purpose=purpose,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@router.get("/debug/export")
|
||||
def admin_debug_export(
|
||||
format: str = Query(default="json"),
|
||||
conversation_id: str | None = None,
|
||||
journal_day_id: str | None = None,
|
||||
purpose: str | None = None,
|
||||
session: dict = Depends(require_admin_dep),
|
||||
):
|
||||
fmt = (format or "json").strip().lower()
|
||||
if fmt not in {"json", "markdown"}:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "invalid_debug_export_format", "message": "Format ist json oder markdown."},
|
||||
)
|
||||
document = export_document(
|
||||
session["profile_id"],
|
||||
conversation_id=conversation_id,
|
||||
journal_day_id=journal_day_id,
|
||||
purpose=purpose,
|
||||
)
|
||||
filename = export_filename(fmt, document)
|
||||
if fmt == "markdown":
|
||||
body = render_markdown(document)
|
||||
return Response(
|
||||
content=body.encode("utf-8"),
|
||||
media_type="text/markdown; charset=utf-8",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
body = json.dumps(document, ensure_ascii=False, indent=2)
|
||||
return Response(
|
||||
content=body.encode("utf-8"),
|
||||
media_type="application/json; charset=utf-8",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/debug/runs/clear")
|
||||
def admin_clear_debug_runs(session: dict = Depends(require_admin_dep)):
|
||||
deleted = clear_runs(session["profile_id"])
|
||||
return {"ok": True, "deleted": deleted}
|
||||
|
||||
|
||||
@router.get("/debug/runs/{run_id}")
|
||||
def admin_debug_run(run_id: str, session: dict = Depends(require_admin_dep)):
|
||||
item = get_run(session["profile_id"], run_id)
|
||||
if not item:
|
||||
raise HTTPException(status_code=404, detail={"code": "debug_run_missing", "message": "Debug-Lauf fehlt."})
|
||||
return item
|
||||
|
||||
|
||||
@router.delete("/debug/runs/{run_id}")
|
||||
def admin_delete_debug_run(run_id: str, session: dict = Depends(require_admin_dep)):
|
||||
if not delete_run(session["profile_id"], run_id):
|
||||
raise HTTPException(status_code=404, detail={"code": "debug_run_missing", "message": "Debug-Lauf fehlt."})
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
class ProviderUpdate(BaseModel):
|
||||
name: str = ""
|
||||
url: str = ""
|
||||
model: str = ""
|
||||
zdr: bool = True
|
||||
no_train: bool = True
|
||||
profile_id: str | None = None
|
||||
save_as_title: str | None = None
|
||||
key: str | None = Field(default=None, description="Write-only. Never returned.")
|
||||
|
||||
|
||||
class LlmProfileWrite(BaseModel):
|
||||
title: str
|
||||
name: str = ""
|
||||
url: str
|
||||
model: str = ""
|
||||
zdr: bool = True
|
||||
no_train: bool = True
|
||||
|
||||
|
||||
class ActivateProfile(BaseModel):
|
||||
profile_id: str
|
||||
|
||||
|
||||
class DetectModeWrite(BaseModel):
|
||||
mode: str
|
||||
|
||||
|
||||
class LocalNerWrite(BaseModel):
|
||||
enabled: bool | None = None
|
||||
url: str | None = None
|
||||
passage_profile_id: str | None = None
|
||||
|
||||
|
||||
@router.put("/providers/detect-mode")
|
||||
def admin_set_detect_mode(body: DetectModeWrite, session: dict = Depends(require_admin_dep)):
|
||||
from detect_learning import set_detect_operating_mode
|
||||
from provider_settings import public_status as provider_public_status
|
||||
|
||||
try:
|
||||
set_detect_operating_mode(body.mode)
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "invalid_detect_operating_mode",
|
||||
"message": "Detect-Modus muss student, learning_full, learning_reduced, learning_manual oder bypass sein.",
|
||||
},
|
||||
)
|
||||
return provider_public_status()
|
||||
|
||||
|
||||
@router.put("/providers/local-ner")
|
||||
def admin_set_local_ner(body: LocalNerWrite, session: dict = Depends(require_admin_dep)):
|
||||
from detect_learning import set_local_ner_settings
|
||||
from provider_settings import public_status as provider_public_status
|
||||
|
||||
try:
|
||||
set_local_ner_settings(
|
||||
enabled=body.enabled,
|
||||
url=body.url,
|
||||
passage_profile_id=body.passage_profile_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
code = str(exc)
|
||||
if code == "gliner_url_not_local":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": code,
|
||||
"message": "GLiNER-URL muss in der lokalen Trusted Zone liegen.",
|
||||
},
|
||||
) from exc
|
||||
if code == "passage_profile_missing":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": code, "message": "Das Klärungs-Profil fehlt."},
|
||||
) from exc
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "invalid_local_ner", "message": "Lokale NER-Einstellung ungültig."},
|
||||
) from exc
|
||||
return provider_public_status()
|
||||
|
||||
|
||||
@router.get("/providers")
|
||||
def admin_providers(session: dict = Depends(require_admin_dep)):
|
||||
return public_status()
|
||||
|
||||
|
||||
@router.get("/providers/models")
|
||||
def admin_provider_models(url: str | None = None, session: dict = Depends(require_admin_dep)):
|
||||
from model_catalog import list_models_for_url, list_provider_models
|
||||
from providers import detect_provider, generate_provider
|
||||
|
||||
if url:
|
||||
key = ""
|
||||
from providers import is_local_url
|
||||
|
||||
if not is_local_url(url):
|
||||
key = (os.environ.get("KANSHO_PROVIDER_KEY") or "").strip()
|
||||
return {"url": url, "models": list_models_for_url(url, key)}
|
||||
return {
|
||||
"roles": {
|
||||
"generate": list_provider_models(generate_provider()),
|
||||
"detect": list_provider_models(detect_provider()),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@router.put("/providers/{role}")
|
||||
def admin_update_provider(role: str, body: ProviderUpdate, session: dict = Depends(require_admin_dep)):
|
||||
if role not in ROLES:
|
||||
raise HTTPException(status_code=404, detail={"code": "unknown_provider_role", "message": "Unbekannte Provider-Rolle."})
|
||||
try:
|
||||
upsert_setting(
|
||||
role,
|
||||
name=body.name,
|
||||
url=body.url,
|
||||
model=body.model,
|
||||
zdr=body.zdr,
|
||||
no_train=body.no_train,
|
||||
profile_id=body.profile_id,
|
||||
save_as_title=body.save_as_title,
|
||||
)
|
||||
if body.key is not None and body.key.strip():
|
||||
set_role_key(role, body.key)
|
||||
except SettingsError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
return public_status()
|
||||
|
||||
|
||||
@router.post("/providers/{role}/activate")
|
||||
def admin_activate_provider(role: str, body: ActivateProfile, session: dict = Depends(require_admin_dep)):
|
||||
if role not in ROLES:
|
||||
raise HTTPException(status_code=404, detail={"code": "unknown_provider_role", "message": "Unbekannte Provider-Rolle."})
|
||||
try:
|
||||
activate_profile(role, body.profile_id)
|
||||
except SettingsError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
return public_status()
|
||||
|
||||
|
||||
@router.get("/llm-profiles")
|
||||
def admin_list_profiles(session: dict = Depends(require_admin_dep)):
|
||||
return {"profiles": list_profiles()}
|
||||
|
||||
|
||||
@router.post("/llm-profiles")
|
||||
def admin_create_profile(body: LlmProfileWrite, session: dict = Depends(require_admin_dep)):
|
||||
try:
|
||||
profile = create_profile(
|
||||
title=body.title,
|
||||
name=body.name,
|
||||
url=body.url,
|
||||
model=body.model,
|
||||
zdr=body.zdr,
|
||||
no_train=body.no_train,
|
||||
)
|
||||
except SettingsError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
return {"profile": profile, **public_status()}
|
||||
|
||||
|
||||
@router.put("/llm-profiles/{profile_id}")
|
||||
def admin_update_profile(profile_id: str, body: LlmProfileWrite, session: dict = Depends(require_admin_dep)):
|
||||
try:
|
||||
profile = update_profile(
|
||||
profile_id,
|
||||
title=body.title,
|
||||
name=body.name,
|
||||
url=body.url,
|
||||
model=body.model,
|
||||
zdr=body.zdr,
|
||||
no_train=body.no_train,
|
||||
)
|
||||
except SettingsError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
return {"profile": profile, **public_status()}
|
||||
|
||||
|
||||
@router.delete("/llm-profiles/{profile_id}")
|
||||
def admin_delete_profile(profile_id: str, session: dict = Depends(require_admin_dep)):
|
||||
try:
|
||||
delete_profile(profile_id)
|
||||
except SettingsError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
return public_status()
|
||||
|
||||
|
||||
class IdentityWrite(BaseModel):
|
||||
canonical_label: str = ""
|
||||
entity_type: str = "PERSON"
|
||||
aliases: list[str] = Field(default_factory=list)
|
||||
token: str | None = None
|
||||
status: str | None = None
|
||||
|
||||
|
||||
class IdentityPatch(BaseModel):
|
||||
canonical_label: str | None = None
|
||||
entity_type: str | None = None
|
||||
aliases: list[str] | None = None
|
||||
status: str | None = None
|
||||
|
||||
|
||||
def _identity_error(exc: ValueError) -> HTTPException:
|
||||
code = str(exc)
|
||||
messages = {
|
||||
"empty_label": "Bezeichnung ist leer oder nicht maskierbar.",
|
||||
"invalid_entity_type": "Unzulässiger Entitätstyp.",
|
||||
"invalid_status": "Unzulässiger Status.",
|
||||
"identity_missing": "Registry-Eintrag fehlt.",
|
||||
"proposal_missing": "Vorschlag fehlt.",
|
||||
}
|
||||
return HTTPException(status_code=400, detail={"code": code, "message": messages.get(code, "Ungültige Identitätsangabe.")})
|
||||
|
||||
|
||||
@router.get("/identities")
|
||||
def admin_identities(session: dict = Depends(require_admin_dep)):
|
||||
profile_id = session["profile_id"]
|
||||
return {
|
||||
"registry": list_registry(profile_id),
|
||||
"proposals": list_review_proposals(profile_id, include_dismissed=True),
|
||||
"senses": list_senses(profile_id),
|
||||
"note": "Nur lokale bestätigte Registry und unbestätigte Vorschläge. Kein externer Egress.",
|
||||
"backup": (
|
||||
"Vor einer Bereinigung die lokale Datei backend/data/kansho.sqlite kopieren. "
|
||||
"Wiederherstellung: Anwendung beenden und die Kopie zurücklegen."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/identities")
|
||||
def admin_create_identity(body: IdentityWrite, session: dict = Depends(require_admin_dep)):
|
||||
try:
|
||||
return confirm_identity(
|
||||
session["profile_id"],
|
||||
body.canonical_label,
|
||||
entity_type=body.entity_type,
|
||||
token=body.token,
|
||||
aliases=body.aliases,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise _identity_error(exc) from exc
|
||||
|
||||
|
||||
@router.patch("/identities/{identity_id}")
|
||||
def admin_update_identity(identity_id: str, body: IdentityPatch, session: dict = Depends(require_admin_dep)):
|
||||
try:
|
||||
return update_identity(
|
||||
session["profile_id"],
|
||||
identity_id,
|
||||
canonical_label=body.canonical_label,
|
||||
entity_type=body.entity_type,
|
||||
aliases=body.aliases,
|
||||
status=body.status,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise _identity_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/identities/{identity_id}/deactivate")
|
||||
def admin_deactivate_identity(identity_id: str, session: dict = Depends(require_admin_dep)):
|
||||
try:
|
||||
return deactivate_identity(session["profile_id"], identity_id)
|
||||
except ValueError as exc:
|
||||
raise _identity_error(exc) from exc
|
||||
|
||||
|
||||
@router.delete("/identities/{identity_id}")
|
||||
def admin_delete_identity(identity_id: str, session: dict = Depends(require_admin_dep)):
|
||||
delete_identity(session["profile_id"], identity_id)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/identities/proposals/{proposal_id}/confirm")
|
||||
def admin_confirm_proposal(proposal_id: str, body: IdentityPatch | None = None, session: dict = Depends(require_admin_dep)):
|
||||
payload = body or IdentityPatch()
|
||||
try:
|
||||
return confirm_review_proposal(
|
||||
session["profile_id"],
|
||||
proposal_id,
|
||||
canonical_label=payload.canonical_label,
|
||||
entity_type=payload.entity_type,
|
||||
aliases=payload.aliases,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise _identity_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/identities/proposals/{proposal_id}/dismiss")
|
||||
def admin_dismiss_proposal(proposal_id: str, session: dict = Depends(require_admin_dep)):
|
||||
dismiss_review_proposal(session["profile_id"], proposal_id)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.delete("/identities/proposals/{proposal_id}")
|
||||
def admin_delete_proposal(proposal_id: str, session: dict = Depends(require_admin_dep)):
|
||||
delete_review_proposal(session["profile_id"], proposal_id)
|
||||
return {"ok": True}
|
||||
+97
-11
@@ -5,8 +5,25 @@ import uuid
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from account_access import (
|
||||
FORGOT_MESSAGE,
|
||||
consume_password_reset,
|
||||
consume_verification,
|
||||
deliver_jobs,
|
||||
find_by_login,
|
||||
issue_password_reset,
|
||||
issue_verification,
|
||||
load_profile,
|
||||
normalize_email,
|
||||
profile_view,
|
||||
resend_verification,
|
||||
reset_message,
|
||||
verified_reset_recipients,
|
||||
verify_message,
|
||||
)
|
||||
from auth import create_session, hash_password, require_auth, verify_password
|
||||
from db import get_db, row_to_dict
|
||||
from db import get_db
|
||||
from mailer import MailNotConfigured
|
||||
from version import APP_VERSION
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["auth"])
|
||||
@@ -23,6 +40,23 @@ class SetupRequest(BaseModel):
|
||||
password: str = Field(min_length=4)
|
||||
|
||||
|
||||
class ForgotRequest(BaseModel):
|
||||
email: str = Field(min_length=1)
|
||||
|
||||
|
||||
class ResetPasswordRequest(BaseModel):
|
||||
token: str = Field(min_length=10)
|
||||
password: str = Field(min_length=4)
|
||||
|
||||
|
||||
class VerifyEmailRequest(BaseModel):
|
||||
token: str = Field(min_length=10)
|
||||
|
||||
|
||||
class ResendVerificationRequest(BaseModel):
|
||||
purpose: str
|
||||
|
||||
|
||||
def _profile_count() -> int:
|
||||
with get_db() as conn:
|
||||
row = conn.execute("SELECT COUNT(*) AS n FROM profiles").fetchone()
|
||||
@@ -43,15 +77,22 @@ def auth_status():
|
||||
def setup(req: SetupRequest):
|
||||
if _profile_count() > 0:
|
||||
raise HTTPException(409, "Setup ist bereits abgeschlossen")
|
||||
email = normalize_email(req.email)
|
||||
profile_id = str(uuid.uuid4())
|
||||
jobs: list[tuple[list[str], str, str]] = []
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO profiles (id, email, name, password_hash, role)
|
||||
VALUES (?, ?, ?, ?, 'admin')
|
||||
""",
|
||||
(profile_id, req.email.lower().strip(), req.name.strip(), hash_password(req.password)),
|
||||
(profile_id, email, req.name.strip(), hash_password(req.password)),
|
||||
)
|
||||
raw = issue_verification(conn, profile_id, "primary", email)
|
||||
if raw:
|
||||
subject, body = verify_message(raw)
|
||||
jobs.append(([email], subject, body))
|
||||
_send_or_log_failure(jobs)
|
||||
token, expires = create_session(profile_id)
|
||||
return {
|
||||
"token": token,
|
||||
@@ -62,15 +103,20 @@ def setup(req: SetupRequest):
|
||||
}
|
||||
|
||||
|
||||
def _send_or_log_failure(jobs: list[tuple[list[str], str, str]]) -> bool:
|
||||
if not jobs:
|
||||
return True
|
||||
try:
|
||||
deliver_jobs(jobs)
|
||||
except MailNotConfigured:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
def login(req: LoginRequest):
|
||||
with get_db() as conn:
|
||||
prof = row_to_dict(
|
||||
conn.execute(
|
||||
"SELECT * FROM profiles WHERE email = ?",
|
||||
(req.email.lower().strip(),),
|
||||
).fetchone()
|
||||
)
|
||||
prof = find_by_login(conn, req.email)
|
||||
if not prof or not verify_password(req.password, prof["password_hash"]):
|
||||
raise HTTPException(401, "Ungültige Zugangsdaten")
|
||||
if (prof.get("status") or "active") != "active":
|
||||
@@ -95,9 +141,49 @@ def logout(session: dict = Depends(require_auth)):
|
||||
@router.get("/me")
|
||||
def me(session: dict = Depends(require_auth)):
|
||||
with get_db() as conn:
|
||||
prof = row_to_dict(
|
||||
conn.execute("SELECT id, email, name, role, status, tier_id FROM profiles WHERE id = ?", (session["profile_id"],)).fetchone()
|
||||
)
|
||||
prof = profile_view(load_profile(conn, session["profile_id"]))
|
||||
if not prof:
|
||||
raise HTTPException(401, "Profil nicht gefunden")
|
||||
return prof
|
||||
|
||||
|
||||
@router.post("/forgot-password")
|
||||
def forgot_password(req: ForgotRequest):
|
||||
jobs: list[tuple[list[str], str, str]] = []
|
||||
with get_db() as conn:
|
||||
prof = find_by_login(conn, req.email)
|
||||
if prof and (prof.get("status") or "active") == "active":
|
||||
recipients = verified_reset_recipients(prof)
|
||||
if recipients:
|
||||
raw = issue_password_reset(conn, prof["id"])
|
||||
if raw:
|
||||
subject, body = reset_message(raw)
|
||||
jobs.append((recipients, subject, body))
|
||||
_send_or_log_failure(jobs)
|
||||
return {"ok": True, "message": FORGOT_MESSAGE}
|
||||
|
||||
|
||||
@router.post("/reset-password")
|
||||
def reset_password(req: ResetPasswordRequest):
|
||||
with get_db() as conn:
|
||||
consume_password_reset(conn, req.token, req.password)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/verify-email")
|
||||
def verify_email(req: VerifyEmailRequest):
|
||||
with get_db() as conn:
|
||||
return consume_verification(conn, req.token)
|
||||
|
||||
|
||||
@router.post("/resend-verification")
|
||||
def resend(req: ResendVerificationRequest, session: dict = Depends(require_auth)):
|
||||
with get_db() as conn:
|
||||
profile = load_profile(conn, session["profile_id"])
|
||||
if not profile:
|
||||
raise HTTPException(401, "Profil nicht gefunden")
|
||||
recipients, subject, body = resend_verification(conn, profile, req.purpose)
|
||||
jobs = [(recipients, subject, body)]
|
||||
if not _send_or_log_failure(jobs):
|
||||
raise HTTPException(503, "Mailversand ist nicht konfiguriert")
|
||||
return {"ok": True}
|
||||
@@ -7,10 +7,15 @@ from auth import require_auth
|
||||
from context_builder import build_internal_context
|
||||
from continuity import checkpoint_usage_session, close_usage_session
|
||||
from derived_kinds import catalog
|
||||
from detect_learning import pending_for_conversation
|
||||
from dialogue_turn import continue_turn, run_turn, visible_for_role
|
||||
from engine import EngineError
|
||||
from privacy_gateway import public_error_detail
|
||||
from dialogue_store import (
|
||||
StoreError,
|
||||
append_message,
|
||||
create_conversation,
|
||||
delete_conversation,
|
||||
create_handoff,
|
||||
create_space,
|
||||
create_thread,
|
||||
@@ -27,8 +32,8 @@ from dialogue_store import (
|
||||
router = APIRouter(prefix="/api/dialogue", tags=["dialogue"])
|
||||
|
||||
|
||||
def _http(exc: StoreError):
|
||||
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": exc.message}) from exc
|
||||
def _http(exc: StoreError | EngineError):
|
||||
raise HTTPException(status_code=exc.status_code, detail=public_error_detail(exc)) from exc
|
||||
|
||||
|
||||
class SessionWrite(BaseModel):
|
||||
@@ -38,6 +43,7 @@ class SessionWrite(BaseModel):
|
||||
class ConversationWrite(BaseModel):
|
||||
usage_session_id: str | None = None
|
||||
title: str = ""
|
||||
space_id: str | None = None
|
||||
|
||||
|
||||
class MessageWrite(BaseModel):
|
||||
@@ -46,6 +52,11 @@ class MessageWrite(BaseModel):
|
||||
id: str | None = None
|
||||
|
||||
|
||||
class MaskReviewWrite(BaseModel):
|
||||
review_id: str
|
||||
decisions: list[dict] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ThreadWrite(BaseModel):
|
||||
title: str = ""
|
||||
status: str = "open"
|
||||
@@ -102,7 +113,12 @@ def checkpoint_session(usage_session_id: str, session: dict = Depends(require_au
|
||||
@router.post("/conversations")
|
||||
def post_conversation(req: ConversationWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return create_conversation(session["profile_id"], req.usage_session_id, req.title)
|
||||
return create_conversation(
|
||||
session["profile_id"],
|
||||
req.usage_session_id,
|
||||
req.title,
|
||||
space_id=req.space_id,
|
||||
)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
@@ -112,12 +128,27 @@ def get_conversations(session: dict = Depends(require_auth)):
|
||||
return list_conversations(session["profile_id"])
|
||||
|
||||
|
||||
@router.delete("/conversations/{conversation_id}")
|
||||
def remove_conversation(conversation_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
from writing_profile_store import remember_dialogue_style
|
||||
|
||||
result = delete_conversation(session["profile_id"], conversation_id)
|
||||
remember_dialogue_style(session["profile_id"])
|
||||
return result
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/conversations/{conversation_id}")
|
||||
def get_one_conversation(conversation_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
conv = get_conversation(session["profile_id"], conversation_id)
|
||||
conv["messages"] = list_messages(session["profile_id"], conversation_id)
|
||||
conv["derived"] = list_derived_for_conversation(session["profile_id"], conversation_id)
|
||||
pending = pending_for_conversation(session["profile_id"], conversation_id)
|
||||
if pending:
|
||||
conv["pending_mask_review"] = pending
|
||||
return conv
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
@@ -139,6 +170,28 @@ def post_message(conversation_id: str, req: MessageWrite, session: dict = Depend
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/conversations/{conversation_id}/turn")
|
||||
def conversation_turn(conversation_id: str, req: MessageWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return visible_for_role(
|
||||
run_turn(session["profile_id"], conversation_id, req.body, message_id=req.id),
|
||||
session.get("role"),
|
||||
)
|
||||
except (StoreError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/conversations/{conversation_id}/turn/review")
|
||||
def conversation_turn_review(conversation_id: str, req: MaskReviewWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return visible_for_role(
|
||||
continue_turn(session["profile_id"], conversation_id, req.review_id, req.decisions),
|
||||
session.get("role"),
|
||||
)
|
||||
except (StoreError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/conversations/{conversation_id}/messages")
|
||||
def get_messages(conversation_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from auth import require_admin_dep
|
||||
from journal_generation_policy import (
|
||||
PURPOSE_JOURNAL,
|
||||
CatalogError,
|
||||
GenerationPolicyError,
|
||||
archive_guideline,
|
||||
clone_guideline,
|
||||
create_guideline,
|
||||
default_selection_ids,
|
||||
delete_guideline,
|
||||
get_guideline,
|
||||
overview_payload,
|
||||
preview_selection,
|
||||
publish_guideline,
|
||||
reset_seed_drafts,
|
||||
set_default_guideline,
|
||||
update_guideline,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/admin/generation-instructions", tags=["admin"])
|
||||
|
||||
|
||||
class GuidelineWrite(BaseModel):
|
||||
slot: str | None = None
|
||||
guideline_key: str | None = None
|
||||
label: str | None = None
|
||||
summary: str | None = None
|
||||
instruction: str | None = None
|
||||
style_context: dict | None = None
|
||||
sort_order: int | None = None
|
||||
|
||||
|
||||
class PreviewRequest(BaseModel):
|
||||
generation_selection: dict | None = None
|
||||
|
||||
|
||||
def _http(exc: GenerationPolicyError):
|
||||
raise HTTPException(
|
||||
status_code=exc.status_code,
|
||||
detail={"code": exc.code, "message": exc.message},
|
||||
) from exc
|
||||
|
||||
|
||||
def _purpose(purpose: str) -> str:
|
||||
if purpose != PURPOSE_JOURNAL:
|
||||
raise HTTPException(404, "Unbekannter Generation-Purpose")
|
||||
return purpose
|
||||
|
||||
|
||||
@router.get("/{purpose}")
|
||||
def list_overview(purpose: str, session: dict = Depends(require_admin_dep)):
|
||||
return overview_payload(_purpose(purpose))
|
||||
|
||||
|
||||
@router.post("/{purpose}/reset")
|
||||
def reset_items(purpose: str, session: dict = Depends(require_admin_dep)):
|
||||
try:
|
||||
return reset_seed_drafts(_purpose(purpose))
|
||||
except CatalogError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/{purpose}/preview")
|
||||
def preview_items(purpose: str, body: PreviewRequest, session: dict = Depends(require_admin_dep)):
|
||||
checked = _purpose(purpose)
|
||||
selection = body.generation_selection or default_selection_ids(checked)
|
||||
try:
|
||||
return preview_selection(selection, purpose=checked)
|
||||
except GenerationPolicyError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/{purpose}")
|
||||
def create_item(purpose: str, body: GuidelineWrite, session: dict = Depends(require_admin_dep)):
|
||||
try:
|
||||
return create_guideline(_purpose(purpose), body.slot or "", body.model_dump())
|
||||
except CatalogError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/{purpose}/{guideline_id}")
|
||||
def read_item(purpose: str, guideline_id: str, session: dict = Depends(require_admin_dep)):
|
||||
_purpose(purpose)
|
||||
try:
|
||||
item = get_guideline(guideline_id, include_instruction=True)
|
||||
except CatalogError as exc:
|
||||
_http(exc)
|
||||
if item.get("purpose") != purpose:
|
||||
raise HTTPException(404, "Ausprägung nicht gefunden.")
|
||||
return item
|
||||
|
||||
|
||||
@router.put("/{purpose}/{guideline_id}")
|
||||
def write_item(purpose: str, guideline_id: str, body: GuidelineWrite, session: dict = Depends(require_admin_dep)):
|
||||
_purpose(purpose)
|
||||
try:
|
||||
payload = {key: value for key, value in body.model_dump().items() if value is not None}
|
||||
payload.pop("slot", None)
|
||||
return update_guideline(guideline_id, payload)
|
||||
except CatalogError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/{purpose}/{guideline_id}/clone")
|
||||
def clone_item(purpose: str, guideline_id: str, session: dict = Depends(require_admin_dep)):
|
||||
_purpose(purpose)
|
||||
try:
|
||||
return clone_guideline(guideline_id)
|
||||
except CatalogError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/{purpose}/{guideline_id}/publish")
|
||||
def publish_item(purpose: str, guideline_id: str, session: dict = Depends(require_admin_dep)):
|
||||
_purpose(purpose)
|
||||
try:
|
||||
return publish_guideline(guideline_id)
|
||||
except CatalogError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/{purpose}/{guideline_id}/archive")
|
||||
def archive_item(purpose: str, guideline_id: str, session: dict = Depends(require_admin_dep)):
|
||||
_purpose(purpose)
|
||||
try:
|
||||
return archive_guideline(guideline_id)
|
||||
except CatalogError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/{purpose}/{guideline_id}/default")
|
||||
def default_item(purpose: str, guideline_id: str, session: dict = Depends(require_admin_dep)):
|
||||
_purpose(purpose)
|
||||
try:
|
||||
return set_default_guideline(guideline_id)
|
||||
except CatalogError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.delete("/{purpose}/{guideline_id}")
|
||||
def delete_item(purpose: str, guideline_id: str, session: dict = Depends(require_admin_dep)):
|
||||
_purpose(purpose)
|
||||
try:
|
||||
delete_guideline(guideline_id)
|
||||
return {"ok": True}
|
||||
except CatalogError as exc:
|
||||
_http(exc)
|
||||
@@ -0,0 +1,733 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from auth import require_auth
|
||||
from detect_learning import pending_for_conversation
|
||||
from dialogue_store import StoreError, delete_conversation, get_conversation, list_messages
|
||||
from dialogue_turn import continue_turn, run_turn, visible_for_role
|
||||
from privacy_gateway import GatewayRequest, inspect, public_error_detail
|
||||
from engine import EngineError
|
||||
from journal_generate import continue_journal_generate, generate_draft
|
||||
from journal_generation_policy import settings_payload
|
||||
from journal_opening import maybe_open_journal_conversation
|
||||
from journal_policy import PolicyError
|
||||
from journal_store import (
|
||||
create_day_conversation,
|
||||
create_user_space,
|
||||
day_payload,
|
||||
get_day,
|
||||
get_entry,
|
||||
get_or_create_day,
|
||||
get_user_space,
|
||||
list_days,
|
||||
list_deleted_entries,
|
||||
list_space_entries,
|
||||
list_user_spaces,
|
||||
list_versions,
|
||||
purge_entry,
|
||||
rename_user_space,
|
||||
restore_entry,
|
||||
save_day_scratch,
|
||||
save_entry,
|
||||
soft_delete_entry,
|
||||
start_payload,
|
||||
undelete_entry,
|
||||
)
|
||||
from media_store import delete_media, get_media, list_media, save_media
|
||||
from writing_profile_store import (
|
||||
accept_suggestion as accept_writing_suggestion,
|
||||
export_document as export_writing_document,
|
||||
get_profile,
|
||||
import_corpus,
|
||||
list_corpus,
|
||||
rebuild_brief,
|
||||
reject_suggestion as reject_writing_suggestion,
|
||||
remember_dialogue_style,
|
||||
remember_journal_entry,
|
||||
restore_document as restore_writing_document,
|
||||
set_governance as set_writing_governance,
|
||||
update_facet,
|
||||
update_trait,
|
||||
)
|
||||
from profile_review import (
|
||||
ingest_import,
|
||||
ingest_journal,
|
||||
import_result as import_review_result,
|
||||
open_paste_review,
|
||||
review_status,
|
||||
run_api_review,
|
||||
accept_proposal,
|
||||
reject_proposal,
|
||||
)
|
||||
from interaction_profile_store import (
|
||||
accept_suggestion as accept_interaction_suggestion,
|
||||
export_document as export_interaction_document,
|
||||
get_profile as get_interaction_profile,
|
||||
reject_suggestion as reject_interaction_suggestion,
|
||||
restore_document as restore_interaction_document,
|
||||
set_governance as set_interaction_governance,
|
||||
upsert_preference,
|
||||
)
|
||||
from profile_documents import export_bundle, restore_bundle
|
||||
|
||||
router = APIRouter(prefix="/api/journal", tags=["journal"])
|
||||
|
||||
|
||||
def _http(exc: StoreError | PolicyError | EngineError):
|
||||
raise HTTPException(status_code=exc.status_code, detail=public_error_detail(exc)) from exc
|
||||
|
||||
|
||||
class SpaceWrite(BaseModel):
|
||||
title: str = Field(min_length=1)
|
||||
|
||||
|
||||
class DayWrite(BaseModel):
|
||||
calendar_date: str = Field(min_length=10, max_length=10)
|
||||
|
||||
|
||||
class ConversationWrite(BaseModel):
|
||||
title: str = ""
|
||||
|
||||
|
||||
class TurnWrite(BaseModel):
|
||||
body: str = Field(min_length=1)
|
||||
id: str | None = None
|
||||
|
||||
|
||||
class MaskReviewWrite(BaseModel):
|
||||
review_id: str
|
||||
decisions: list[dict] = Field(default_factory=list)
|
||||
|
||||
|
||||
class GenerateWrite(BaseModel):
|
||||
conversation_ids: list[str] | None = None
|
||||
include_existing: bool = False
|
||||
generation_selection: dict | None = None
|
||||
remember_generation_selection: bool = False
|
||||
|
||||
|
||||
class EntryWrite(BaseModel):
|
||||
journal_day_id: str
|
||||
title: str = ""
|
||||
body: str = ""
|
||||
entry_id: str | None = None
|
||||
origin: str = "user_edit"
|
||||
source_conversation_ids: list[str] = Field(default_factory=list)
|
||||
source_message_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class RestoreWrite(BaseModel):
|
||||
version_id: str
|
||||
|
||||
|
||||
class PurgeWrite(BaseModel):
|
||||
confirm: bool = False
|
||||
|
||||
|
||||
class ScratchItem(BaseModel):
|
||||
id: str | None = None
|
||||
text: str = ""
|
||||
done: bool = False
|
||||
|
||||
|
||||
class ScratchWrite(BaseModel):
|
||||
items: list[ScratchItem] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ImportWrite(BaseModel):
|
||||
body: str = Field(min_length=1)
|
||||
occurred_at: str | None = None
|
||||
context_hint: str | None = None
|
||||
|
||||
|
||||
class CorpusWrite(BaseModel):
|
||||
items: list[dict]
|
||||
|
||||
|
||||
class TraitWrite(BaseModel):
|
||||
statement: str | None = None
|
||||
locked: bool | None = None
|
||||
facet_key: str | None = None
|
||||
label: str | None = None
|
||||
|
||||
|
||||
class GovernanceWrite(BaseModel):
|
||||
governance: str
|
||||
|
||||
|
||||
class FacetWrite(BaseModel):
|
||||
value: str | None = None
|
||||
locked: bool | None = None
|
||||
|
||||
|
||||
class PreferenceWrite(BaseModel):
|
||||
scope: str = "global"
|
||||
pref_key: str
|
||||
value: str
|
||||
locked: bool | None = None
|
||||
origin: str = "manual"
|
||||
|
||||
|
||||
class ReviewImportWrite(BaseModel):
|
||||
result: dict | str
|
||||
review_id: str | None = None
|
||||
|
||||
|
||||
class ReviewAcceptWrite(BaseModel):
|
||||
review_id: str | None = None
|
||||
indexes: list[int] | None = None
|
||||
baseline: bool = False
|
||||
|
||||
|
||||
@router.get("/continuity")
|
||||
def continuity(session: dict = Depends(require_auth)):
|
||||
return start_payload(session["profile_id"])
|
||||
|
||||
|
||||
@router.get("/spaces")
|
||||
def spaces(session: dict = Depends(require_auth)):
|
||||
return list_user_spaces(session["profile_id"])
|
||||
|
||||
|
||||
@router.post("/spaces")
|
||||
def create_space(body: SpaceWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return create_user_space(session["profile_id"], body.title)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/spaces/{space_id}")
|
||||
def space_detail(space_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
space = get_user_space(session["profile_id"], space_id)
|
||||
return {
|
||||
"space": space,
|
||||
"days": list_days(session["profile_id"], space_id),
|
||||
"entries": list_space_entries(session["profile_id"], space_id),
|
||||
}
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/spaces/{space_id}/entries")
|
||||
def space_entries(space_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return list_space_entries(session["profile_id"], space_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/spaces/{space_id}/trash")
|
||||
def space_trash(space_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return list_deleted_entries(session["profile_id"], space_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.patch("/spaces/{space_id}")
|
||||
def space_rename(space_id: str, body: SpaceWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return rename_user_space(session["profile_id"], space_id, body.title)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/spaces/{space_id}/days")
|
||||
def open_day(space_id: str, body: DayWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
day = get_or_create_day(session["profile_id"], space_id, body.calendar_date)
|
||||
return day_payload(session["profile_id"], day["id"])
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/days/{journal_day_id}")
|
||||
def read_day(journal_day_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return day_payload(session["profile_id"], journal_day_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.patch("/days/{journal_day_id}/scratch")
|
||||
def write_scratch(journal_day_id: str, body: ScratchWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return {
|
||||
"scratch": save_day_scratch(
|
||||
session["profile_id"],
|
||||
journal_day_id,
|
||||
[{"id": item.id, "text": item.text, "done": item.done} for item in body.items],
|
||||
)
|
||||
}
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/days/{journal_day_id}/conversations")
|
||||
def start_conversation(journal_day_id: str, body: ConversationWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
conversation = create_day_conversation(session["profile_id"], journal_day_id, body.title)
|
||||
opened = maybe_open_journal_conversation(session["profile_id"], conversation)
|
||||
return visible_for_role(opened, session.get("role"))
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.delete("/conversations/{conversation_id}")
|
||||
def remove_conversation(conversation_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
result = delete_conversation(session["profile_id"], conversation_id)
|
||||
remember_dialogue_style(session["profile_id"])
|
||||
return result
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/conversations/{conversation_id}")
|
||||
def read_conversation(conversation_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
conv = get_conversation(session["profile_id"], conversation_id)
|
||||
payload = {"conversation": conv, "messages": list_messages(session["profile_id"], conversation_id)}
|
||||
pending = pending_for_conversation(session["profile_id"], conversation_id)
|
||||
if pending:
|
||||
payload["pending_mask_review"] = pending
|
||||
return payload
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/egress-status")
|
||||
def egress_status(session: dict = Depends(require_auth)):
|
||||
result = inspect(GatewayRequest(prompt_id=None, purpose="dialogue_turn", data_class="B"))
|
||||
return {"ready": result.allowed, "reason": result.reason, "provider": result.provider}
|
||||
|
||||
|
||||
@router.post("/conversations/{conversation_id}/turn")
|
||||
def conversation_turn(conversation_id: str, body: TurnWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return visible_for_role(
|
||||
run_turn(session["profile_id"], conversation_id, body.body, message_id=body.id),
|
||||
session.get("role"),
|
||||
)
|
||||
except (StoreError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/conversations/{conversation_id}/turn/review")
|
||||
def conversation_turn_review(conversation_id: str, body: MaskReviewWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return visible_for_role(
|
||||
continue_turn(session["profile_id"], conversation_id, body.review_id, body.decisions),
|
||||
session.get("role"),
|
||||
)
|
||||
except (StoreError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/generation-settings")
|
||||
def read_generation_settings(session: dict = Depends(require_auth)):
|
||||
return settings_payload(session["profile_id"])
|
||||
|
||||
|
||||
@router.post("/days/{journal_day_id}/generate")
|
||||
def generate(journal_day_id: str, body: GenerateWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return visible_for_role(
|
||||
generate_draft(
|
||||
session["profile_id"],
|
||||
journal_day_id,
|
||||
conversation_ids=body.conversation_ids,
|
||||
include_existing=body.include_existing,
|
||||
explicit=True,
|
||||
generation_selection=body.generation_selection,
|
||||
remember_generation_selection=body.remember_generation_selection,
|
||||
),
|
||||
session.get("role"),
|
||||
)
|
||||
except (StoreError, PolicyError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/days/{journal_day_id}/generate/review")
|
||||
def generate_review(journal_day_id: str, body: MaskReviewWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return visible_for_role(
|
||||
continue_journal_generate(
|
||||
session["profile_id"],
|
||||
journal_day_id,
|
||||
body.review_id,
|
||||
body.decisions,
|
||||
),
|
||||
session.get("role"),
|
||||
)
|
||||
except (StoreError, PolicyError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/entries")
|
||||
def write_entry(body: EntryWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
get_day(session["profile_id"], body.journal_day_id)
|
||||
entry = save_entry(
|
||||
session["profile_id"],
|
||||
body.journal_day_id,
|
||||
body.title,
|
||||
body.body,
|
||||
origin=body.origin,
|
||||
entry_id=body.entry_id,
|
||||
source_conversation_ids=body.source_conversation_ids,
|
||||
source_message_ids=body.source_message_ids,
|
||||
)
|
||||
remember_journal_entry(
|
||||
session["profile_id"],
|
||||
entry["id"],
|
||||
entry.get("body") or "",
|
||||
origin=body.origin,
|
||||
)
|
||||
ingest_journal(
|
||||
session["profile_id"],
|
||||
entry["id"],
|
||||
entry.get("body") or "",
|
||||
origin=body.origin,
|
||||
)
|
||||
return entry
|
||||
except (StoreError, PolicyError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/entries/{entry_id}")
|
||||
def read_entry(entry_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
entry = get_entry(session["profile_id"], entry_id)
|
||||
entry["media"] = list_media(session["profile_id"], entry_id)
|
||||
entry["versions"] = list_versions(session["profile_id"], entry_id)
|
||||
return entry
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/entries/{entry_id}/versions")
|
||||
def entry_versions(entry_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return list_versions(session["profile_id"], entry_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/entries/{entry_id}/restore")
|
||||
def entry_restore(entry_id: str, body: RestoreWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
entry = restore_entry(session["profile_id"], entry_id, body.version_id)
|
||||
remember_journal_entry(
|
||||
session["profile_id"],
|
||||
entry["id"],
|
||||
entry.get("body") or "",
|
||||
origin=entry.get("origin"),
|
||||
)
|
||||
ingest_journal(
|
||||
session["profile_id"],
|
||||
entry["id"],
|
||||
entry.get("body") or "",
|
||||
origin=entry.get("origin"),
|
||||
)
|
||||
return entry
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/entries/{entry_id}/undelete")
|
||||
def entry_undelete(entry_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return undelete_entry(session["profile_id"], entry_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/entries/{entry_id}/purge")
|
||||
def entry_purge(entry_id: str, body: PurgeWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return purge_entry(session["profile_id"], entry_id, confirm=body.confirm)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.delete("/entries/{entry_id}")
|
||||
def entry_delete(entry_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return soft_delete_entry(session["profile_id"], entry_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/entries/{entry_id}/media")
|
||||
async def entry_media(entry_id: str, file: UploadFile = File(...), session: dict = Depends(require_auth)):
|
||||
try:
|
||||
data = await file.read()
|
||||
return save_media(
|
||||
session["profile_id"],
|
||||
entry_id,
|
||||
file.filename or "media",
|
||||
file.content_type or "",
|
||||
data,
|
||||
)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/media/{media_id}")
|
||||
def read_media(media_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
row, path = get_media(session["profile_id"], media_id)
|
||||
return FileResponse(path, media_type=row["content_type"], filename=row["filename"])
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.delete("/media/{media_id}")
|
||||
def remove_media(media_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return delete_media(session["profile_id"], media_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/writing-profile")
|
||||
def writing_profile(session: dict = Depends(require_auth)):
|
||||
return get_profile(session["profile_id"])
|
||||
|
||||
|
||||
@router.patch("/writing-profile")
|
||||
def writing_patch(body: GovernanceWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return set_writing_governance(session["profile_id"], body.governance)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.patch("/writing-profile/facets/{facet_key}")
|
||||
def writing_facet(facet_key: str, body: FacetWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return update_facet(
|
||||
session["profile_id"],
|
||||
facet_key,
|
||||
value=body.value,
|
||||
locked=body.locked,
|
||||
)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.patch("/writing-profile/traits/{slug}")
|
||||
def writing_trait(slug: str, body: TraitWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return update_trait(
|
||||
session["profile_id"],
|
||||
slug,
|
||||
statement=body.statement,
|
||||
locked=body.locked,
|
||||
facet_key=body.facet_key,
|
||||
label=body.label,
|
||||
)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/suggestions/{suggestion_id}/accept")
|
||||
def writing_accept(suggestion_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return accept_writing_suggestion(session["profile_id"], suggestion_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/suggestions/{suggestion_id}/reject")
|
||||
def writing_reject(suggestion_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return reject_writing_suggestion(session["profile_id"], suggestion_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/import")
|
||||
def writing_import(body: ImportWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return ingest_import(
|
||||
session["profile_id"],
|
||||
body.body,
|
||||
occurred_at=body.occurred_at,
|
||||
context_hint=body.context_hint,
|
||||
)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/corpus")
|
||||
def writing_corpus(body: CorpusWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return import_corpus(session["profile_id"], body.items)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/writing-profile/corpus")
|
||||
def writing_corpus_list(session: dict = Depends(require_auth)):
|
||||
return {"items": list_corpus(session["profile_id"])}
|
||||
|
||||
|
||||
@router.post("/writing-profile/rebuild")
|
||||
def writing_rebuild(session: dict = Depends(require_auth)):
|
||||
return rebuild_brief(session["profile_id"])
|
||||
|
||||
|
||||
@router.get("/writing-profile/review")
|
||||
def writing_review_status(session: dict = Depends(require_auth)):
|
||||
return review_status(session["profile_id"])
|
||||
|
||||
|
||||
@router.post("/writing-profile/review/paste")
|
||||
def writing_review_paste(session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return open_paste_review(session["profile_id"])
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/initial-build/paste")
|
||||
def writing_initial_build_paste(session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return open_paste_review(session["profile_id"], mode="initial_build")
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/review/api")
|
||||
def writing_review_api(session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return run_api_review(session["profile_id"])
|
||||
except (StoreError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/initial-build/api")
|
||||
def writing_initial_build_api(session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return run_api_review(session["profile_id"], mode="initial_build")
|
||||
except (StoreError, EngineError) as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/review/import")
|
||||
def writing_review_import(body: ReviewImportWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return import_review_result(session["profile_id"], body.result, review_id=body.review_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/review/accept")
|
||||
def writing_review_accept(body: ReviewAcceptWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return accept_proposal(
|
||||
session["profile_id"],
|
||||
body.review_id,
|
||||
indexes=body.indexes,
|
||||
baseline=body.baseline,
|
||||
)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/writing-profile/review/reject")
|
||||
def writing_review_reject(body: ReviewAcceptWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return reject_proposal(session["profile_id"], body.review_id, indexes=body.indexes)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/writing-profile/export")
|
||||
def writing_export(session: dict = Depends(require_auth)):
|
||||
return export_writing_document(session["profile_id"])
|
||||
|
||||
|
||||
@router.post("/writing-profile/restore")
|
||||
def writing_restore(body: dict, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return restore_writing_document(session["profile_id"], body)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/interaction-profile")
|
||||
def interaction_profile(session: dict = Depends(require_auth)):
|
||||
return get_interaction_profile(session["profile_id"])
|
||||
|
||||
|
||||
@router.patch("/interaction-profile")
|
||||
def interaction_patch(body: GovernanceWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return set_interaction_governance(session["profile_id"], body.governance)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.patch("/interaction-profile/preferences")
|
||||
def interaction_pref(body: PreferenceWrite, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return upsert_preference(
|
||||
session["profile_id"],
|
||||
scope=body.scope,
|
||||
pref_key=body.pref_key,
|
||||
value=body.value,
|
||||
origin=body.origin if body.origin in {"explicit", "manual"} else "manual",
|
||||
locked=body.locked,
|
||||
)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/interaction-profile/suggestions/{suggestion_id}/accept")
|
||||
def interaction_accept(suggestion_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return accept_interaction_suggestion(session["profile_id"], suggestion_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.post("/interaction-profile/suggestions/{suggestion_id}/reject")
|
||||
def interaction_reject(suggestion_id: str, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return reject_interaction_suggestion(session["profile_id"], suggestion_id)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/interaction-profile/export")
|
||||
def interaction_export(session: dict = Depends(require_auth)):
|
||||
return export_interaction_document(session["profile_id"])
|
||||
|
||||
|
||||
@router.post("/interaction-profile/restore")
|
||||
def interaction_restore(body: dict, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return restore_interaction_document(session["profile_id"], body)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
|
||||
|
||||
@router.get("/profiles/export")
|
||||
def profiles_export(session: dict = Depends(require_auth)):
|
||||
return export_bundle(session["profile_id"])
|
||||
|
||||
|
||||
@router.post("/profiles/restore")
|
||||
def profiles_restore(body: dict, session: dict = Depends(require_auth)):
|
||||
try:
|
||||
return restore_bundle(session["profile_id"], body)
|
||||
except StoreError as exc:
|
||||
_http(exc)
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from auth import require_auth
|
||||
import placeholder_mvp # noqa: F401
|
||||
import placeholder_system # noqa: F401
|
||||
from placeholders import list_placeholders
|
||||
from privacy_placeholders import catalog as privacy_catalog
|
||||
|
||||
@@ -117,6 +117,19 @@ def update_prompt(prompt_id: str, req: PromptWrite, session: dict = Depends(requ
|
||||
current = row_to_dict(conn.execute("SELECT id FROM ai_prompts WHERE id = ?", (prompt_id,)).fetchone())
|
||||
if not current:
|
||||
raise HTTPException(404, "Prompt nicht gefunden")
|
||||
if req.prompt_type == "base" and req.template:
|
||||
try:
|
||||
preview_prompt(
|
||||
{
|
||||
"id": prompt_id,
|
||||
"slug": req.slug.strip(),
|
||||
"prompt_type": req.prompt_type,
|
||||
"template": req.template,
|
||||
},
|
||||
{},
|
||||
)
|
||||
except EngineError as exc:
|
||||
_http(exc)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE ai_prompts
|
||||
|
||||
+132
-17
@@ -5,19 +5,34 @@ import uuid
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from account_access import (
|
||||
admin_reset_recipients,
|
||||
apply_identifier_edits,
|
||||
assert_address_free,
|
||||
assert_username_free,
|
||||
deliver_jobs,
|
||||
issue_password_reset,
|
||||
issue_verification,
|
||||
load_profile,
|
||||
normalize_email,
|
||||
normalize_username,
|
||||
profile_view,
|
||||
reset_message,
|
||||
verify_message,
|
||||
)
|
||||
from auth import hash_password, require_admin_dep, require_auth
|
||||
from db import get_db, row_to_dict
|
||||
from db import get_db
|
||||
from mailer import MailNotConfigured
|
||||
|
||||
router = APIRouter(prefix="/api/users", tags=["users"])
|
||||
|
||||
PUBLIC_FIELDS = "id, email, name, role, status, tier_id, created"
|
||||
|
||||
|
||||
class CreateUserRequest(BaseModel):
|
||||
email: str
|
||||
name: str = Field(min_length=1)
|
||||
password: str = Field(min_length=4)
|
||||
role: str = "user"
|
||||
username: str | None = None
|
||||
|
||||
|
||||
class PatchUserRequest(BaseModel):
|
||||
@@ -25,6 +40,16 @@ class PatchUserRequest(BaseModel):
|
||||
role: str | None = None
|
||||
status: str | None = None
|
||||
password: str | None = None
|
||||
username: str | None = None
|
||||
email: str | None = None
|
||||
secondary_email: str | None = None
|
||||
|
||||
|
||||
class SelfPatchRequest(BaseModel):
|
||||
username: str | None = None
|
||||
email: str | None = None
|
||||
secondary_email: str | None = None
|
||||
password: str | None = None
|
||||
|
||||
|
||||
def _active_admin_count(conn, exclude_id: str | None = None) -> int:
|
||||
@@ -40,45 +65,104 @@ def _active_admin_count(conn, exclude_id: str | None = None) -> int:
|
||||
return int(row["n"])
|
||||
|
||||
|
||||
def _send_jobs(jobs: list[tuple[list[str], str, str]]) -> bool:
|
||||
if not jobs:
|
||||
return True
|
||||
try:
|
||||
deliver_jobs(jobs)
|
||||
except MailNotConfigured:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_users(session: dict = Depends(require_admin_dep)):
|
||||
with get_db() as conn:
|
||||
rows = conn.execute(f"SELECT {PUBLIC_FIELDS} FROM profiles ORDER BY created").fetchall()
|
||||
return [row_to_dict(row) for row in rows]
|
||||
rows = conn.execute("SELECT * FROM profiles ORDER BY created").fetchall()
|
||||
return [profile_view(dict(row)) for row in rows]
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_user(req: CreateUserRequest, session: dict = Depends(require_admin_dep)):
|
||||
if req.role not in {"user", "admin"}:
|
||||
raise HTTPException(400, "Rolle muss user oder admin sein")
|
||||
email = normalize_email(req.email)
|
||||
username = normalize_username(req.username) if req.username and req.username.strip() else None
|
||||
profile_id = str(uuid.uuid4())
|
||||
jobs: list[tuple[list[str], str, str]] = []
|
||||
try:
|
||||
with get_db() as conn:
|
||||
assert_address_free(conn, email, None)
|
||||
if username:
|
||||
assert_username_free(conn, username, None)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO profiles (id, email, name, password_hash, role, status, tier_id)
|
||||
VALUES (?, ?, ?, ?, ?, 'active', 'local')
|
||||
INSERT INTO profiles (id, email, name, password_hash, role, status, tier_id, username)
|
||||
VALUES (?, ?, ?, ?, ?, 'active', 'local', ?)
|
||||
""",
|
||||
(
|
||||
profile_id,
|
||||
req.email.lower().strip(),
|
||||
email,
|
||||
req.name.strip(),
|
||||
hash_password(req.password),
|
||||
req.role,
|
||||
username,
|
||||
),
|
||||
)
|
||||
prof = row_to_dict(conn.execute(f"SELECT {PUBLIC_FIELDS} FROM profiles WHERE id = ?", (profile_id,)).fetchone())
|
||||
raw = issue_verification(conn, profile_id, "primary", email)
|
||||
if raw:
|
||||
subject, body = verify_message(raw)
|
||||
jobs.append(([email], subject, body))
|
||||
prof = profile_view(load_profile(conn, profile_id))
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if "UNIQUE" in str(exc).upper():
|
||||
raise HTTPException(409, "E-Mail ist bereits vergeben") from exc
|
||||
raise
|
||||
if not _send_jobs(jobs):
|
||||
prof = dict(prof or {})
|
||||
prof["mail_sent"] = False
|
||||
return prof
|
||||
|
||||
|
||||
@router.patch("/me")
|
||||
def patch_me(req: SelfPatchRequest, session: dict = Depends(require_auth)):
|
||||
provided = req.model_fields_set
|
||||
jobs: list[tuple[list[str], str, str]] = []
|
||||
with get_db() as conn:
|
||||
current = load_profile(conn, session["profile_id"])
|
||||
if not current:
|
||||
raise HTTPException(401, "Profil nicht gefunden")
|
||||
if req.password:
|
||||
if len(req.password) < 4:
|
||||
raise HTTPException(400, "Passwort muss mind. 4 Zeichen haben")
|
||||
conn.execute(
|
||||
"UPDATE profiles SET password_hash = ? WHERE id = ?",
|
||||
(hash_password(req.password), current["id"]),
|
||||
)
|
||||
jobs = apply_identifier_edits(
|
||||
conn,
|
||||
current,
|
||||
username=req.username,
|
||||
email=req.email,
|
||||
secondary_email=req.secondary_email,
|
||||
username_set="username" in provided,
|
||||
email_set="email" in provided,
|
||||
secondary_set="secondary_email" in provided,
|
||||
)
|
||||
prof = profile_view(load_profile(conn, current["id"]))
|
||||
if not _send_jobs(jobs):
|
||||
raise HTTPException(503, "Die Adresse ist vorgemerkt, der Mailversand ist nicht konfiguriert.")
|
||||
return prof
|
||||
|
||||
|
||||
@router.patch("/{profile_id}")
|
||||
def patch_user(profile_id: str, req: PatchUserRequest, session: dict = Depends(require_admin_dep)):
|
||||
provided = req.model_fields_set
|
||||
jobs: list[tuple[list[str], str, str]] = []
|
||||
with get_db() as conn:
|
||||
current = row_to_dict(conn.execute("SELECT * FROM profiles WHERE id = ?", (profile_id,)).fetchone())
|
||||
current = load_profile(conn, profile_id)
|
||||
if not current:
|
||||
raise HTTPException(404, "Nutzer nicht gefunden")
|
||||
name = req.name.strip() if req.name is not None else current["name"]
|
||||
@@ -105,18 +189,49 @@ def patch_user(profile_id: str, req: PatchUserRequest, session: dict = Depends(r
|
||||
)
|
||||
if status == "disabled":
|
||||
conn.execute("DELETE FROM sessions WHERE profile_id = ?", (profile_id,))
|
||||
return row_to_dict(conn.execute(f"SELECT {PUBLIC_FIELDS} FROM profiles WHERE id = ?", (profile_id,)).fetchone())
|
||||
current = load_profile(conn, profile_id) or current
|
||||
jobs = apply_identifier_edits(
|
||||
conn,
|
||||
current,
|
||||
username=req.username,
|
||||
email=req.email,
|
||||
secondary_email=req.secondary_email,
|
||||
username_set="username" in provided,
|
||||
email_set="email" in provided,
|
||||
secondary_set="secondary_email" in provided,
|
||||
)
|
||||
prof = profile_view(load_profile(conn, profile_id))
|
||||
if not _send_jobs(jobs):
|
||||
raise HTTPException(503, "Die Adresse ist vorgemerkt, der Mailversand ist nicht konfiguriert.")
|
||||
return prof
|
||||
|
||||
|
||||
@router.post("/{profile_id}/password-reset")
|
||||
def admin_password_reset(profile_id: str, session: dict = Depends(require_admin_dep)):
|
||||
if profile_id == session["profile_id"]:
|
||||
raise HTTPException(400, "Den eigenen Reset startest du über die Anmeldung")
|
||||
jobs: list[tuple[list[str], str, str]] = []
|
||||
with get_db() as conn:
|
||||
profile = load_profile(conn, profile_id)
|
||||
if not profile:
|
||||
raise HTTPException(404, "Nutzer nicht gefunden")
|
||||
recipients = admin_reset_recipients(profile)
|
||||
if not recipients:
|
||||
raise HTTPException(400, "Für dieses Konto ist keine E-Mail hinterlegt")
|
||||
raw = issue_password_reset(conn, profile_id)
|
||||
if not raw:
|
||||
raise HTTPException(429, "Zu viele Reset-Links. Bitte später erneut versuchen.")
|
||||
subject, body = reset_message(raw)
|
||||
jobs.append((recipients, subject, body))
|
||||
if not _send_jobs(jobs):
|
||||
raise HTTPException(503, "Mailversand ist nicht konfiguriert")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
def my_account(session: dict = Depends(require_auth)):
|
||||
with get_db() as conn:
|
||||
prof = row_to_dict(
|
||||
conn.execute(
|
||||
f"SELECT {PUBLIC_FIELDS} FROM profiles WHERE id = ?",
|
||||
(session["profile_id"],),
|
||||
).fetchone()
|
||||
)
|
||||
prof = profile_view(load_profile(conn, session["profile_id"]))
|
||||
if not prof:
|
||||
raise HTTPException(401, "Profil nicht gefunden")
|
||||
return prof
|
||||
@@ -40,6 +40,12 @@ CREATE TABLE IF NOT EXISTS profiles (
|
||||
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'disabled')),
|
||||
tier_id TEXT NOT NULL DEFAULT 'local' REFERENCES tiers(id),
|
||||
session_days INTEGER NOT NULL DEFAULT 30,
|
||||
username TEXT,
|
||||
secondary_email TEXT,
|
||||
email_verified_at TEXT,
|
||||
secondary_email_verified_at TEXT,
|
||||
pending_email TEXT,
|
||||
pending_secondary_email TEXT,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
@@ -50,6 +56,26 @@ CREATE TABLE IF NOT EXISTS sessions (
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS password_reset_tokens (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
expires_at TEXT NOT NULL,
|
||||
used_at TEXT,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS email_verification_tokens (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
purpose TEXT NOT NULL CHECK (purpose IN ('primary', 'secondary')),
|
||||
email TEXT NOT NULL,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
expires_at TEXT NOT NULL,
|
||||
used_at TEXT,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_feature_restrictions (
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
feature_id TEXT NOT NULL REFERENCES features(id) ON DELETE CASCADE,
|
||||
@@ -81,6 +107,7 @@ CREATE TABLE IF NOT EXISTS ai_prompts (
|
||||
active INTEGER NOT NULL DEFAULT 1,
|
||||
is_system_default INTEGER NOT NULL DEFAULT 0,
|
||||
default_template TEXT NOT NULL DEFAULT '',
|
||||
seed_revision TEXT NOT NULL DEFAULT '',
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
@@ -100,6 +127,13 @@ CREATE TABLE IF NOT EXISTS conversations (
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
usage_session_id TEXT REFERENCES usage_sessions(id),
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
space_id TEXT,
|
||||
journal_day_id TEXT,
|
||||
narrative_mode TEXT NOT NULL DEFAULT '',
|
||||
reflection_depth TEXT NOT NULL DEFAULT '',
|
||||
current_focus TEXT NOT NULL DEFAULT '',
|
||||
emotional_intensity TEXT NOT NULL DEFAULT '',
|
||||
long_story INTEGER NOT NULL DEFAULT 0,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
@@ -199,6 +233,367 @@ CREATE TABLE IF NOT EXISTS identity_mappings (
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
token TEXT NOT NULL,
|
||||
local_label TEXT NOT NULL DEFAULT '',
|
||||
canonical_label TEXT NOT NULL DEFAULT '',
|
||||
entity_type TEXT NOT NULL DEFAULT 'PERSON',
|
||||
status TEXT NOT NULL DEFAULT 'legacy_review_required',
|
||||
origin TEXT NOT NULL DEFAULT 'legacy_auto',
|
||||
aliases_json TEXT NOT NULL DEFAULT '[]',
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
confirmed_at TEXT,
|
||||
UNIQUE (profile_id, token)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS identity_review_proposals (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
observed_label TEXT NOT NULL DEFAULT '',
|
||||
entity_type TEXT NOT NULL DEFAULT 'PERSON',
|
||||
status TEXT NOT NULL DEFAULT 'unconfirmed',
|
||||
origin TEXT NOT NULL DEFAULT 'detect_proposal',
|
||||
last_seen TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (profile_id, observed_label, entity_type)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS label_senses (
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
normalized_label TEXT NOT NULL,
|
||||
identity_hits INTEGER NOT NULL DEFAULT 0,
|
||||
non_identity_hits INTEGER NOT NULL DEFAULT 0,
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
PRIMARY KEY (profile_id, normalized_label)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS label_sense_cues (
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
normalized_label TEXT NOT NULL,
|
||||
cue TEXT NOT NULL,
|
||||
decision TEXT NOT NULL,
|
||||
hits INTEGER NOT NULL DEFAULT 1,
|
||||
excerpt TEXT NOT NULL DEFAULT '',
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
PRIMARY KEY (profile_id, normalized_label, cue)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pending_mask_reviews (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
|
||||
user_message_id TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pending_mask_reviews_conv
|
||||
ON pending_mask_reviews (profile_id, conversation_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS journal_days (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
space_id TEXT NOT NULL REFERENCES spaces(id) ON DELETE CASCADE,
|
||||
calendar_date TEXT NOT NULL,
|
||||
scratch_json TEXT NOT NULL DEFAULT '[]',
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (space_id, calendar_date)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS journal_drafts (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
journal_day_id TEXT NOT NULL REFERENCES journal_days(id) ON DELETE CASCADE,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
source_conversation_ids TEXT NOT NULL DEFAULT '[]',
|
||||
source_message_ids TEXT NOT NULL DEFAULT '[]',
|
||||
as_of TEXT NOT NULL,
|
||||
superseded_at TEXT,
|
||||
generation_snapshot TEXT NOT NULL DEFAULT '{}',
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS journal_entries (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
journal_day_id TEXT NOT NULL REFERENCES journal_days(id) ON DELETE CASCADE,
|
||||
space_id TEXT NOT NULL REFERENCES spaces(id) ON DELETE CASCADE,
|
||||
current_version_id TEXT,
|
||||
deleted_at TEXT,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS journal_entry_versions (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
entry_id TEXT NOT NULL REFERENCES journal_entries(id) ON DELETE CASCADE,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
source_conversation_ids TEXT NOT NULL DEFAULT '[]',
|
||||
source_message_ids TEXT NOT NULL DEFAULT '[]',
|
||||
origin TEXT NOT NULL CHECK (origin IN ('user_edit', 'accepted_draft', 'restore')),
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS journal_draft_source_refs (
|
||||
draft_id TEXT NOT NULL REFERENCES journal_drafts(id) ON DELETE CASCADE,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
source_kind TEXT NOT NULL CHECK (source_kind IN ('conversation', 'message')),
|
||||
source_id TEXT NOT NULL,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (draft_id, source_kind, source_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS journal_entry_version_source_refs (
|
||||
version_id TEXT NOT NULL REFERENCES journal_entry_versions(id) ON DELETE CASCADE,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
source_kind TEXT NOT NULL CHECK (source_kind IN ('conversation', 'message')),
|
||||
source_id TEXT NOT NULL,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (version_id, source_kind, source_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS media_assets (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
entry_id TEXT NOT NULL REFERENCES journal_entries(id) ON DELETE CASCADE,
|
||||
entry_version_id TEXT REFERENCES journal_entry_versions(id),
|
||||
filename TEXT NOT NULL DEFAULT '',
|
||||
content_type TEXT NOT NULL,
|
||||
rel_path TEXT NOT NULL,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS writing_profiles (
|
||||
profile_id TEXT PRIMARY KEY REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
compiled_brief TEXT NOT NULL DEFAULT '',
|
||||
governance TEXT NOT NULL DEFAULT 'learning' CHECK (governance IN ('learning', 'advising', 'frozen')),
|
||||
lifecycle TEXT NOT NULL DEFAULT 'uninitialized' CHECK (lifecycle IN ('uninitialized', 'initial_pending', 'confirmed')),
|
||||
version INTEGER NOT NULL DEFAULT 0,
|
||||
review_ready INTEGER NOT NULL DEFAULT 0,
|
||||
last_reviewed TEXT,
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS writing_profile_sources (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('imported_text', 'journal_entry', 'dialogue_style')),
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
entry_id TEXT REFERENCES journal_entries(id),
|
||||
weight REAL NOT NULL DEFAULT 1,
|
||||
occurred_at TEXT,
|
||||
context_hint TEXT NOT NULL DEFAULT '',
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS writing_profile_facets (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
facet_key TEXT NOT NULL,
|
||||
layer TEXT NOT NULL DEFAULT 'context' CHECK (layer IN ('core', 'context', 'output')),
|
||||
value TEXT NOT NULL DEFAULT '',
|
||||
evidence TEXT NOT NULL DEFAULT '',
|
||||
origin TEXT NOT NULL DEFAULT 'manual' CHECK (origin IN ('manual', 'imported', 'accepted_suggestion', 'initial_build')),
|
||||
locked INTEGER NOT NULL DEFAULT 0,
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (profile_id, facet_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS writing_profile_traits (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
facet_key TEXT NOT NULL,
|
||||
slug TEXT NOT NULL,
|
||||
label TEXT NOT NULL DEFAULT '',
|
||||
statement TEXT NOT NULL DEFAULT '',
|
||||
origin TEXT NOT NULL DEFAULT 'manual' CHECK (origin IN ('manual', 'imported', 'accepted_suggestion', 'initial_build')),
|
||||
locked INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'merged', 'split', 'retired')),
|
||||
observed_from TEXT,
|
||||
observed_to TEXT,
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (profile_id, slug)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS writing_profile_trait_refs (
|
||||
id TEXT PRIMARY KEY,
|
||||
trait_id TEXT NOT NULL REFERENCES writing_profile_traits(id) ON DELETE CASCADE,
|
||||
source_id TEXT,
|
||||
role TEXT NOT NULL CHECK (role IN ('evidence', 'exemplar')),
|
||||
excerpt TEXT NOT NULL DEFAULT '',
|
||||
occurred_at TEXT,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS writing_profile_suggestions (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
facet_key TEXT NOT NULL DEFAULT '',
|
||||
trait_slug TEXT NOT NULL DEFAULT '',
|
||||
action TEXT NOT NULL DEFAULT 'update',
|
||||
proposed_value TEXT NOT NULL DEFAULT '',
|
||||
evidence TEXT NOT NULL DEFAULT '',
|
||||
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'accepted', 'rejected')),
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
resolved TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS interaction_profiles (
|
||||
profile_id TEXT PRIMARY KEY REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
governance TEXT NOT NULL DEFAULT 'advising' CHECK (governance IN ('learning', 'advising', 'frozen')),
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS interaction_preferences (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
pref_key TEXT NOT NULL,
|
||||
value TEXT NOT NULL DEFAULT '',
|
||||
evidence TEXT NOT NULL DEFAULT '',
|
||||
origin TEXT NOT NULL DEFAULT 'manual' CHECK (origin IN ('explicit', 'manual', 'accepted_suggestion')),
|
||||
locked INTEGER NOT NULL DEFAULT 0,
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (profile_id, scope, pref_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS interaction_suggestions (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
pref_key TEXT NOT NULL,
|
||||
proposed_value TEXT NOT NULL DEFAULT '',
|
||||
evidence TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'accepted', 'rejected')),
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
resolved TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS writing_profile_evidence (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
trigger TEXT NOT NULL CHECK (trigger IN (
|
||||
'user_edit_of_draft', 'heuristic_drift', 'dialogue_sample', 'periodic_batch', 'explicit_review'
|
||||
)),
|
||||
source_kind TEXT NOT NULL DEFAULT '',
|
||||
source_id TEXT,
|
||||
excerpt TEXT NOT NULL DEFAULT '',
|
||||
local_signals_json TEXT NOT NULL DEFAULT '{}',
|
||||
example_for_json TEXT NOT NULL DEFAULT '[]',
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'bundled', 'consumed', 'dismissed')),
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS writing_profile_reviews (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
target TEXT NOT NULL DEFAULT 'writing' CHECK (target IN ('writing', 'interaction')),
|
||||
channel TEXT NOT NULL CHECK (channel IN ('api', 'paste')),
|
||||
status TEXT NOT NULL DEFAULT 'open' CHECK (status IN ('open', 'proposed', 'applied', 'dismissed')),
|
||||
package_json TEXT NOT NULL DEFAULT '{}',
|
||||
result_json TEXT,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
resolved TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS writing_profile_versions (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
seq INTEGER NOT NULL,
|
||||
governance TEXT NOT NULL DEFAULT '',
|
||||
lifecycle TEXT NOT NULL DEFAULT '',
|
||||
facets_json TEXT NOT NULL DEFAULT '[]',
|
||||
traits_json TEXT NOT NULL DEFAULT '[]',
|
||||
compiled_brief TEXT NOT NULL DEFAULT '',
|
||||
cause TEXT NOT NULL DEFAULT '',
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (profile_id, seq)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS llm_profiles (
|
||||
id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
url TEXT NOT NULL DEFAULT '',
|
||||
model TEXT NOT NULL DEFAULT '',
|
||||
zdr INTEGER NOT NULL DEFAULT 1,
|
||||
no_train INTEGER NOT NULL DEFAULT 1,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS provider_settings (
|
||||
role TEXT PRIMARY KEY CHECK (role IN ('generate', 'detect')),
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
url TEXT NOT NULL DEFAULT '',
|
||||
model TEXT NOT NULL DEFAULT '',
|
||||
zdr INTEGER NOT NULL DEFAULT 1,
|
||||
no_train INTEGER NOT NULL DEFAULT 1,
|
||||
profile_id TEXT,
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS journal_generation_selection (
|
||||
profile_id TEXT PRIMARY KEY REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
transformation_id TEXT NOT NULL,
|
||||
detail_id TEXT NOT NULL,
|
||||
voice_id TEXT NOT NULL,
|
||||
narrative_id TEXT NOT NULL,
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS generation_guidelines (
|
||||
id TEXT PRIMARY KEY,
|
||||
purpose TEXT NOT NULL,
|
||||
slot TEXT NOT NULL,
|
||||
guideline_key TEXT NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
instruction TEXT NOT NULL,
|
||||
style_context_json TEXT NOT NULL DEFAULT '{}',
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('draft', 'active', 'archived')),
|
||||
revision INTEGER NOT NULL DEFAULT 1,
|
||||
cloned_from TEXT,
|
||||
is_default INTEGER NOT NULL DEFAULT 0,
|
||||
is_system_seed INTEGER NOT NULL DEFAULT 0,
|
||||
seed_id TEXT NOT NULL DEFAULT '',
|
||||
seed_revision TEXT NOT NULL DEFAULT '',
|
||||
used_at TEXT,
|
||||
created TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS app_settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL DEFAULT '',
|
||||
updated TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS debug_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE,
|
||||
purpose TEXT NOT NULL,
|
||||
layer TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'ok',
|
||||
subject_type TEXT NOT NULL DEFAULT '',
|
||||
subject_id TEXT NOT NULL DEFAULT '',
|
||||
conversation_id TEXT,
|
||||
message_id TEXT,
|
||||
space_id TEXT,
|
||||
journal_day_id TEXT,
|
||||
space_title TEXT NOT NULL DEFAULT '',
|
||||
calendar_date TEXT NOT NULL DEFAULT '',
|
||||
conversation_title TEXT NOT NULL DEFAULT '',
|
||||
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||
created TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_debug_runs_profile_created
|
||||
ON debug_runs (profile_id, created DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_debug_runs_conversation
|
||||
ON debug_runs (profile_id, conversation_id, created);
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Translate SQLite-oriented SQL so the same stores can run against PostgreSQL.
|
||||
|
||||
Windows without Docker still uses SQLite for the running app. Docker/Server and
|
||||
the test suite use PostgreSQL (`KANSHO_DB_BACKEND=postgres`). Identity, keys and
|
||||
journal bodies are not logged here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
_INSERT_OR_IGNORE = re.compile(r"INSERT\s+OR\s+IGNORE\s+INTO", re.IGNORECASE)
|
||||
_ROWID_COL = re.compile(r"\.rowid\b", re.IGNORECASE)
|
||||
_ON_CONFLICT_PAREN = re.compile(r"ON\s+CONFLICT\s*\(", re.IGNORECASE)
|
||||
_PRAGMA_TABLE = re.compile(
|
||||
r"^\s*PRAGMA\s+table_info\(\s*['\"]?(\w+)['\"]?\s*\)\s*;?\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SQLITE_MASTER_TABLES = re.compile(
|
||||
r"^\s*SELECT\s+name\s+FROM\s+sqlite_master\s+WHERE\s+type\s*=\s*'table'\s*;?\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SQLITE_MASTER_SQL = re.compile(
|
||||
r"^\s*SELECT\s+sql\s+FROM\s+sqlite_master\s+WHERE\s+type\s*=\s*'table'\s+AND\s+name\s*=\s*\?\s*;?\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def use_postgres() -> bool:
|
||||
"""Decide the engine at connect time, not at import time.
|
||||
|
||||
Explicit KANSHO_DB_BACKEND wins. KANSHO_DB_PATH without an explicit postgres
|
||||
backend keeps the Windows local app on SQLite. Tests set postgres and DB_NAME
|
||||
kansho_test on the Dev instance, never kansho_dev.
|
||||
"""
|
||||
raw = (os.environ.get("KANSHO_DB_BACKEND") or "").strip().lower()
|
||||
if raw in {"sqlite", "sqlite3"}:
|
||||
return False
|
||||
if raw in {"postgres", "postgresql", "pg"}:
|
||||
return True
|
||||
if (os.environ.get("KANSHO_DB_PATH") or "").strip():
|
||||
return False
|
||||
url = (os.environ.get("DATABASE_URL") or "").strip().lower()
|
||||
if url.startswith("postgres"):
|
||||
return True
|
||||
if (os.environ.get("DB_HOST") or "").strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def postgres_connect_kwargs() -> dict:
|
||||
url = (os.environ.get("DATABASE_URL") or "").strip()
|
||||
if url:
|
||||
return {"conninfo": url}
|
||||
host = (os.environ.get("DB_HOST") or "postgres").strip() or "postgres"
|
||||
port = int((os.environ.get("DB_PORT") or "5432").strip() or "5432")
|
||||
dbname = (os.environ.get("DB_NAME") or "kansho").strip() or "kansho"
|
||||
user = (os.environ.get("DB_USER") or "kansho").strip() or "kansho"
|
||||
password = os.environ.get("DB_PASSWORD") or ""
|
||||
return {
|
||||
"host": host,
|
||||
"port": port,
|
||||
"dbname": dbname,
|
||||
"user": user,
|
||||
"password": password,
|
||||
}
|
||||
|
||||
|
||||
def sqlite_schema_to_postgres(sql: str) -> str:
|
||||
return sql.replace("datetime('now')", "CURRENT_TIMESTAMP::text")
|
||||
|
||||
|
||||
def replace_placeholders(sql: str) -> str:
|
||||
"""Replace SQLite `?` placeholders with psycopg `%s`, ignoring quoted text."""
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
in_single = False
|
||||
in_double = False
|
||||
while i < len(sql):
|
||||
ch = sql[i]
|
||||
if ch == "'" and not in_double:
|
||||
if in_single and i + 1 < len(sql) and sql[i + 1] == "'":
|
||||
out.append("''")
|
||||
i += 2
|
||||
continue
|
||||
in_single = not in_single
|
||||
out.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == '"' and not in_single:
|
||||
in_double = not in_double
|
||||
out.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == "?" and not in_single and not in_double:
|
||||
out.append("%s")
|
||||
i += 1
|
||||
continue
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def adapt_sql(sql: str) -> str:
|
||||
"""Make a SQLite-shaped statement runnable on PostgreSQL."""
|
||||
sql = sqlite_schema_to_postgres(sql)
|
||||
sql = _ROWID_COL.sub(".id", sql)
|
||||
sql = _ON_CONFLICT_PAREN.sub("ON CONFLICT (", sql)
|
||||
used_ignore = bool(_INSERT_OR_IGNORE.search(sql))
|
||||
sql = _INSERT_OR_IGNORE.sub("INSERT INTO", sql)
|
||||
if used_ignore and re.search(r"ON\s+CONFLICT", sql, re.IGNORECASE) is None:
|
||||
stripped = sql.rstrip()
|
||||
ended = stripped.endswith(";")
|
||||
if ended:
|
||||
stripped = stripped[:-1].rstrip()
|
||||
stripped = f"{stripped} ON CONFLICT DO NOTHING"
|
||||
sql = stripped + (";" if ended else "")
|
||||
return replace_placeholders(sql)
|
||||
|
||||
|
||||
def rewrite_catalog_sql(sql: str) -> tuple[str, tuple | None] | None:
|
||||
"""Rewrite SQLite catalog queries. Returns (sql, extra_params) or None."""
|
||||
match = _PRAGMA_TABLE.match(sql)
|
||||
if match:
|
||||
return (
|
||||
"""
|
||||
SELECT column_name AS name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
ORDER BY ordinal_position
|
||||
""",
|
||||
(match.group(1).lower(),),
|
||||
)
|
||||
if _SQLITE_MASTER_TABLES.match(sql):
|
||||
return (
|
||||
"""
|
||||
SELECT tablename AS name
|
||||
FROM pg_catalog.pg_tables
|
||||
WHERE schemaname = 'public'
|
||||
""",
|
||||
None,
|
||||
)
|
||||
if _SQLITE_MASTER_SQL.match(sql):
|
||||
return ("SELECT NULL AS sql WHERE FALSE", None)
|
||||
return None
|
||||
|
||||
|
||||
def split_sql(script: str) -> list[str]:
|
||||
"""Split a SQL script into statements, respecting quotes."""
|
||||
statements: list[str] = []
|
||||
buf: list[str] = []
|
||||
in_single = False
|
||||
in_double = False
|
||||
i = 0
|
||||
while i < len(script):
|
||||
ch = script[i]
|
||||
if ch == "'" and not in_double:
|
||||
if in_single and i + 1 < len(script) and script[i + 1] == "'":
|
||||
buf.append("''")
|
||||
i += 2
|
||||
continue
|
||||
in_single = not in_single
|
||||
buf.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == '"' and not in_single:
|
||||
in_double = not in_double
|
||||
buf.append(ch)
|
||||
i += 1
|
||||
continue
|
||||
if ch == ";" and not in_single and not in_double:
|
||||
stmt = "".join(buf).strip()
|
||||
if stmt:
|
||||
statements.append(stmt)
|
||||
buf = []
|
||||
i += 1
|
||||
continue
|
||||
buf.append(ch)
|
||||
i += 1
|
||||
tail = "".join(buf).strip()
|
||||
if tail:
|
||||
statements.append(tail)
|
||||
return statements
|
||||
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-shot SQLite → PostgreSQL copy for the laptop/home journal instance.
|
||||
|
||||
Probe on a copy, never on the only backup. Does not send data to providers.
|
||||
Media files are copied separately into the destination media root.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from sql_compat import postgres_connect_kwargs
|
||||
|
||||
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
||||
DEFAULT_SQLITE = Path(__file__).resolve().parent / "data" / "kansho.sqlite"
|
||||
DEFAULT_MEDIA = Path(__file__).resolve().parent / "data" / "media"
|
||||
|
||||
# Parent tables first — matches schema.sql CREATE order.
|
||||
TABLES = [
|
||||
"schema_migrations",
|
||||
"tiers",
|
||||
"features",
|
||||
"tier_limits",
|
||||
"profiles",
|
||||
"sessions",
|
||||
"user_feature_restrictions",
|
||||
"user_feature_usage",
|
||||
"ai_prompts",
|
||||
"usage_sessions",
|
||||
"conversations",
|
||||
"messages",
|
||||
"threads",
|
||||
"conversation_threads",
|
||||
"spaces",
|
||||
"thread_spaces",
|
||||
"derived_records",
|
||||
"handoffs",
|
||||
"external_refs",
|
||||
"re_grounding_events",
|
||||
"identity_mappings",
|
||||
"identity_review_proposals",
|
||||
"label_senses",
|
||||
"label_sense_cues",
|
||||
"pending_mask_reviews",
|
||||
"journal_days",
|
||||
"journal_drafts",
|
||||
"journal_entries",
|
||||
"journal_entry_versions",
|
||||
"journal_draft_source_refs",
|
||||
"journal_entry_version_source_refs",
|
||||
"media_assets",
|
||||
"writing_profiles",
|
||||
"writing_profile_sources",
|
||||
"writing_profile_facets",
|
||||
"writing_profile_traits",
|
||||
"writing_profile_trait_refs",
|
||||
"writing_profile_suggestions",
|
||||
"interaction_profiles",
|
||||
"interaction_preferences",
|
||||
"interaction_suggestions",
|
||||
"writing_profile_evidence",
|
||||
"writing_profile_reviews",
|
||||
"writing_profile_versions",
|
||||
"llm_profiles",
|
||||
"provider_settings",
|
||||
"journal_generation_selection",
|
||||
"generation_guidelines",
|
||||
"app_settings",
|
||||
"debug_runs",
|
||||
]
|
||||
|
||||
|
||||
def _sqlite_path() -> Path:
|
||||
env = (os.environ.get("KANSHO_DB_PATH") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
if data_dir:
|
||||
return Path(data_dir) / "kansho.sqlite"
|
||||
return DEFAULT_SQLITE
|
||||
|
||||
|
||||
def _media_root() -> Path:
|
||||
env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip()
|
||||
if env:
|
||||
return Path(env)
|
||||
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||
if data_dir:
|
||||
return Path(data_dir) / "media"
|
||||
return DEFAULT_MEDIA
|
||||
|
||||
|
||||
def _connect_pg():
|
||||
import psycopg
|
||||
|
||||
kwargs = postgres_connect_kwargs()
|
||||
if "conninfo" in kwargs:
|
||||
return psycopg.connect(kwargs["conninfo"])
|
||||
return psycopg.connect(**kwargs)
|
||||
|
||||
|
||||
def _pg_columns(conn, table: str) -> list[str]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
ORDER BY ordinal_position
|
||||
""",
|
||||
(table,),
|
||||
).fetchall()
|
||||
return [row[0] for row in rows]
|
||||
|
||||
|
||||
def _sqlite_rows(sqlite_path: Path, table: str) -> list[dict]:
|
||||
conn = sqlite3.connect(str(sqlite_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
rows = conn.execute(f"SELECT * FROM {table}").fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
except sqlite3.OperationalError:
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _copy_media(src: Path, dest: Path) -> int:
|
||||
if not src.is_dir():
|
||||
print(f"No media directory at {src}")
|
||||
return 0
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
count = 0
|
||||
for path in src.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
relative = path.relative_to(src)
|
||||
target = dest / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, target)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def migrate(sqlite_path: Path, media_src: Path, media_dest: Path, replace: bool) -> dict[str, tuple[int, int]]:
|
||||
if not sqlite_path.is_file():
|
||||
raise SystemExit(f"SQLite not found: {sqlite_path}")
|
||||
|
||||
pg = _connect_pg()
|
||||
stats: dict[str, tuple[int, int]] = {}
|
||||
try:
|
||||
existing = pg.execute("SELECT COUNT(*) FROM profiles").fetchone()[0]
|
||||
seeds = pg.execute("SELECT COUNT(*) FROM tiers").fetchone()[0]
|
||||
if existing and not replace:
|
||||
raise SystemExit(
|
||||
f"PostgreSQL already has {existing} profiles. Pass --replace after a probe copy, not on the only backup."
|
||||
)
|
||||
if seeds and not replace:
|
||||
raise SystemExit(
|
||||
"PostgreSQL already has platform seeds from startup. "
|
||||
"Pass --replace to load a SQLite snapshot onto this instance (no user profiles yet is OK)."
|
||||
)
|
||||
if replace:
|
||||
print(f"Replacing Postgres snapshot (profiles={existing}, seed_tiers={seeds})")
|
||||
for table in reversed(TABLES):
|
||||
pg.execute(f"TRUNCATE TABLE {table} CASCADE")
|
||||
|
||||
for table in TABLES:
|
||||
rows = _sqlite_rows(sqlite_path, table)
|
||||
columns = _pg_columns(pg, table)
|
||||
if not columns:
|
||||
print(f" skip {table} (missing in Postgres schema)")
|
||||
stats[table] = (len(rows), 0)
|
||||
continue
|
||||
if not rows:
|
||||
print(f" {table}: empty")
|
||||
stats[table] = (0, 0)
|
||||
continue
|
||||
usable = [col for col in columns if col in rows[0]]
|
||||
placeholders = ", ".join(["%s"] * len(usable))
|
||||
col_sql = ", ".join(usable)
|
||||
sql = f"INSERT INTO {table} ({col_sql}) VALUES ({placeholders})"
|
||||
values = [tuple(row.get(col) for col in usable) for row in rows]
|
||||
with pg.cursor() as cur:
|
||||
cur.executemany(sql, values)
|
||||
count = pg.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
||||
print(f" {table}: {len(rows)} → {count}")
|
||||
stats[table] = (len(rows), int(count))
|
||||
pg.execute(
|
||||
"""
|
||||
UPDATE profiles
|
||||
SET email_verified_at = COALESCE(NULLIF(created, ''), CURRENT_TIMESTAMP::text)
|
||||
WHERE email_verified_at IS NULL OR email_verified_at = ''
|
||||
"""
|
||||
)
|
||||
pg.execute(
|
||||
"INSERT INTO schema_migrations (id) VALUES (%s) ON CONFLICT (id) DO NOTHING",
|
||||
("022_postgres_runtime_baseline",),
|
||||
)
|
||||
pg.commit()
|
||||
except Exception:
|
||||
pg.rollback()
|
||||
raise
|
||||
finally:
|
||||
pg.close()
|
||||
|
||||
copied = _copy_media(media_src, media_dest)
|
||||
print(f"Media files copied: {copied} → {media_dest}")
|
||||
return stats
|
||||
|
||||
|
||||
def verify(stats: dict[str, tuple[int, int]]) -> None:
|
||||
failed = False
|
||||
print("\nVerification")
|
||||
for table, (src, dest) in stats.items():
|
||||
if table == "schema_migrations" and dest == src + 1:
|
||||
print(f" ok {table:36} sqlite={src:5} postgres={dest:5} (incl. 022)")
|
||||
continue
|
||||
mark = "ok" if src == dest else "MISMATCH"
|
||||
if src != dest:
|
||||
failed = True
|
||||
print(f" {mark:8} {table:36} sqlite={src:5} postgres={dest:5}")
|
||||
if failed:
|
||||
raise SystemExit("Row counts do not match")
|
||||
print("Row counts match")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Copy Kanshō SQLite data into PostgreSQL")
|
||||
parser.add_argument("--sqlite", type=Path, default=_sqlite_path())
|
||||
parser.add_argument("--media-src", type=Path, default=_media_root())
|
||||
parser.add_argument("--media-dest", type=Path, default=Path(os.environ.get("KANSHO_MEDIA_DEST") or "/app/data/media"))
|
||||
parser.add_argument("--confirm", action="store_true", help="Required. Refuse a silent copy.")
|
||||
parser.add_argument("--replace", action="store_true", help="Truncate Postgres tables first")
|
||||
args = parser.parse_args()
|
||||
if not args.confirm:
|
||||
raise SystemExit("Refusing to copy personal data without --confirm")
|
||||
print(f"SQLite source: {args.sqlite}")
|
||||
print(f"Media source: {args.media_src}")
|
||||
print(f"Media dest: {args.media_dest}")
|
||||
stats = migrate(args.sqlite, args.media_src, args.media_dest, args.replace)
|
||||
verify(stats)
|
||||
print("Import complete. Check entry/message counts and one media GET before Prod.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
echo "=== Kanshō backend startup ==="
|
||||
python /app/db_init.py
|
||||
echo "=== starting uvicorn ==="
|
||||
exec uvicorn main:app --host 0.0.0.0 --port 8000
|
||||
@@ -0,0 +1,100 @@
|
||||
"""pytest runner for the existing test_*.py scripts.
|
||||
|
||||
Each file stays one case (`main()`) until it is split into native pytest
|
||||
functions. Schema is loaded once per session; each integration file only
|
||||
truncates data so kansho_dev is never the target and CI does not reload
|
||||
schema.sql twenty times.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from tests.harness import ( # noqa: E402
|
||||
apply_module_env,
|
||||
is_postgres_test_database,
|
||||
prepare_test_env,
|
||||
reset_postgres_data,
|
||||
reset_postgres_schema,
|
||||
)
|
||||
|
||||
UNIT_MODULES = frozenset(
|
||||
{
|
||||
"test_sql_compat",
|
||||
"test_local_backup",
|
||||
"test_provenance",
|
||||
"test_model_catalog",
|
||||
"test_local_url",
|
||||
"test_privacy_detect_eval",
|
||||
"test_journal_shape",
|
||||
"test_journal_body",
|
||||
"test_writing_profile",
|
||||
"test_gliner_client",
|
||||
"test_detect_pipeline",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def pytest_configure(config: pytest.Config) -> None:
|
||||
prepare_test_env()
|
||||
|
||||
|
||||
def pytest_collect_file(file_path: Path, parent: pytest.Collector):
|
||||
if file_path.suffix == ".py" and file_path.name.startswith("test_"):
|
||||
return KanshoScriptModule.from_parent(parent, path=file_path)
|
||||
return None
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
|
||||
for item in items:
|
||||
stem = Path(str(item.path)).stem
|
||||
marker = pytest.mark.unit if stem in UNIT_MODULES else pytest.mark.integration
|
||||
item.add_marker(marker)
|
||||
|
||||
|
||||
class KanshoScriptModule(pytest.Module):
|
||||
def collect(self):
|
||||
module = self.obj
|
||||
main = getattr(module, "main", None)
|
||||
if main is None or not callable(main):
|
||||
pytest.fail(f"{self.path.name} has no callable main()")
|
||||
yield pytest.Function.from_parent(self, name="test_main", callobj=main)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def _session_postgres():
|
||||
if not is_postgres_test_database():
|
||||
pytest.skip(
|
||||
"integration requires KANSHO_DB_BACKEND=postgres and DB_NAME ending _test "
|
||||
"(Gitea: kansho_test on the Dev Postgres, never kansho_dev)"
|
||||
)
|
||||
reset_postgres_schema()
|
||||
from db import init_db
|
||||
|
||||
init_db()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def _module_backend(request: pytest.FixtureRequest):
|
||||
source = Path(str(request.path))
|
||||
stem = source.stem
|
||||
saved = dict(os.environ)
|
||||
try:
|
||||
apply_module_env(source)
|
||||
if stem in UNIT_MODULES:
|
||||
yield
|
||||
return
|
||||
request.getfixturevalue("_session_postgres")
|
||||
reset_postgres_data()
|
||||
yield
|
||||
finally:
|
||||
os.environ.clear()
|
||||
os.environ.update(saved)
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Postgres test engine on the existing Dev instance.
|
||||
|
||||
Two Compose Postgres instances exist: Dev and Prod. Tests use the Dev
|
||||
instance, database `kansho_test` beside `kansho_dev`. That is not a third
|
||||
server. The suite drops and recreates `public` on the test database only,
|
||||
so personal Dev/Prod journals are not wiped.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
LIVE_DB_NAMES = frozenset(
|
||||
{
|
||||
"kansho",
|
||||
"kansho_dev",
|
||||
"postgres",
|
||||
"template0",
|
||||
"template1",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _backend() -> str:
|
||||
return (os.environ.get("KANSHO_DB_BACKEND") or "").strip().lower()
|
||||
|
||||
|
||||
def _db_name() -> str:
|
||||
return (os.environ.get("DB_NAME") or "").strip()
|
||||
|
||||
|
||||
def is_postgres_test_database() -> bool:
|
||||
if _backend() not in {"postgres", "postgresql", "pg"}:
|
||||
return False
|
||||
name = _db_name().lower()
|
||||
if not name or name in LIVE_DB_NAMES:
|
||||
return False
|
||||
return name.endswith("_test")
|
||||
|
||||
|
||||
def prepare_test_env() -> None:
|
||||
"""Fail-closed defaults. Do not keep Compose provider keys or leftover fakes."""
|
||||
os.environ["KANSHO_PROVIDER_KEY"] = ""
|
||||
os.environ["KANSHO_DETECT_PROVIDER_KEY"] = ""
|
||||
os.environ.pop("KANSHO_DB_PATH", None)
|
||||
os.environ.pop("KANSHO_FAKE_PROVIDER", None)
|
||||
os.environ.pop("KANSHO_FAKE_DETECT", None)
|
||||
os.environ.pop("KANSHO_ALLOW_REMOTE_DETECT", None)
|
||||
|
||||
|
||||
def apply_module_env(source) -> None:
|
||||
"""Restore per-file env so pytest does not leak fakes/keys across scripts."""
|
||||
from pathlib import Path
|
||||
|
||||
prepare_test_env()
|
||||
text = Path(source).read_text(encoding="utf-8")
|
||||
if 'os.environ["KANSHO_FAKE_PROVIDER"] = "1"' in text or "os.environ['KANSHO_FAKE_PROVIDER'] = '1'" in text:
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
if (
|
||||
'os.environ["KANSHO_FAKE_DETECT"] = "1"' in text
|
||||
or "os.environ['KANSHO_FAKE_DETECT'] = '1'" in text
|
||||
or 'os.environ.setdefault("KANSHO_FAKE_DETECT", "1")' in text
|
||||
):
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
|
||||
|
||||
def _require_test_database() -> None:
|
||||
if is_postgres_test_database():
|
||||
return
|
||||
backend = _backend()
|
||||
if backend not in {"postgres", "postgresql", "pg"}:
|
||||
raise SystemExit(
|
||||
"Backend-Tests laufen gegen PostgreSQL (KANSHO_DB_BACKEND=postgres), "
|
||||
"dieselbe Engine wie Dev und Prod. SQLite-Testdateien sind abgelöst. "
|
||||
"Gitea nutzt auf der Dev-Postgres die Datenbank kansho_test "
|
||||
"(neben kansho_dev, keine dritte Instanz)."
|
||||
)
|
||||
name = _db_name().lower()
|
||||
if not name:
|
||||
raise SystemExit(
|
||||
"DB_NAME fehlt. Tests nutzen auf der Dev-Postgres die Datenbank kansho_test, "
|
||||
"nicht kansho_dev."
|
||||
)
|
||||
raise SystemExit(
|
||||
f"DB_NAME={_db_name()!r} ist für Tests gesperrt. "
|
||||
"Die Suite setzt das Schema zurück und darf daher nicht kansho oder kansho_dev treffen."
|
||||
)
|
||||
|
||||
|
||||
def reset_postgres_schema() -> None:
|
||||
import psycopg
|
||||
|
||||
from db_init import mark_schema_dirty
|
||||
from sql_compat import postgres_connect_kwargs
|
||||
|
||||
kwargs = postgres_connect_kwargs()
|
||||
if "conninfo" in kwargs:
|
||||
conn = psycopg.connect(kwargs["conninfo"], autocommit=True)
|
||||
else:
|
||||
conn = psycopg.connect(autocommit=True, **kwargs)
|
||||
try:
|
||||
conn.execute("DROP SCHEMA IF EXISTS public CASCADE")
|
||||
conn.execute("CREATE SCHEMA public")
|
||||
conn.execute("GRANT ALL ON SCHEMA public TO PUBLIC")
|
||||
conn.execute("GRANT ALL ON SCHEMA public TO CURRENT_USER")
|
||||
finally:
|
||||
conn.close()
|
||||
mark_schema_dirty()
|
||||
|
||||
|
||||
def reset_postgres_data() -> None:
|
||||
"""Clear rows on kansho_test without reloading schema.sql."""
|
||||
import psycopg
|
||||
|
||||
from sql_compat import postgres_connect_kwargs
|
||||
|
||||
kwargs = postgres_connect_kwargs()
|
||||
if "conninfo" in kwargs:
|
||||
conn = psycopg.connect(kwargs["conninfo"], autocommit=True)
|
||||
else:
|
||||
conn = psycopg.connect(autocommit=True, **kwargs)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT tablename FROM pg_tables
|
||||
WHERE schemaname = 'public' AND tablename <> 'schema_migrations'
|
||||
"""
|
||||
).fetchall()
|
||||
names = [row[0] for row in rows]
|
||||
if names:
|
||||
joined = ", ".join('"' + name.replace('"', '""') + '"' for name in names)
|
||||
conn.execute(f"TRUNCATE TABLE {joined} RESTART IDENTITY CASCADE")
|
||||
finally:
|
||||
conn.close()
|
||||
from db import refresh_runtime_seed
|
||||
|
||||
refresh_runtime_seed()
|
||||
|
||||
|
||||
def configure_test_engine() -> None:
|
||||
"""Reset kansho_test. pytest uses the fixture in tests/conftest.py instead."""
|
||||
prepare_test_env()
|
||||
_require_test_database()
|
||||
try:
|
||||
reset_postgres_schema()
|
||||
except Exception as exc: # noqa: BLE001 — fail before the suite touches a live DB
|
||||
print(f"Postgres-Testschema konnte nicht zurückgesetzt werden: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1) from exc
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Account identifiers, reset links, and address verification."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_PROVIDER_KEY"] = ""
|
||||
os.environ["KANSHO_DETECT_PROVIDER_KEY"] = ""
|
||||
os.environ["KANSHO_FAKE_MAIL"] = "1"
|
||||
os.environ["KANSHO_ENV"] = "test"
|
||||
os.environ["APP_URL"] = "http://localhost:5188"
|
||||
if (os.environ.get("KANSHO_DB_BACKEND") or "").strip().lower() not in {"postgres", "postgresql", "pg"}:
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.mkdtemp(prefix="kansho-account-")) / "account.sqlite")
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from main import app
|
||||
|
||||
from db import get_db
|
||||
from mailer import OUTBOX, clear_outbox
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def auth_header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def mail_tokens() -> list[str]:
|
||||
found: list[str] = []
|
||||
for item in OUTBOX:
|
||||
found.extend(re.findall(r"token=([A-Za-z0-9_\-]+)", item["body"]))
|
||||
return found
|
||||
|
||||
|
||||
def main() -> None:
|
||||
clear_outbox()
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
expect(setup.status_code == 200, f"setup {setup.text}")
|
||||
token = setup.json()["token"]
|
||||
headers = auth_header(token)
|
||||
verify_tokens = mail_tokens()
|
||||
expect(len(verify_tokens) == 1, "setup sends one verification link")
|
||||
verified = client.post("/api/auth/verify-email", json={"token": verify_tokens[0]})
|
||||
expect(verified.status_code == 200 and verified.json()["email_verified"] is True, "admin email verified")
|
||||
clear_outbox()
|
||||
|
||||
me = client.get("/api/auth/me", headers=headers)
|
||||
expect(me.status_code == 200, "session survives initial verification")
|
||||
admin_id = me.json()["id"]
|
||||
|
||||
blocked = client.patch(
|
||||
f"/api/users/{admin_id}",
|
||||
headers=headers,
|
||||
json={"status": "disabled"},
|
||||
)
|
||||
expect(blocked.status_code == 400, f"last admin cannot be disabled {blocked.text}")
|
||||
|
||||
created = client.post(
|
||||
"/api/users",
|
||||
headers=headers,
|
||||
json={
|
||||
"email": "ute@example.test",
|
||||
"name": "Ute",
|
||||
"password": "user-pass",
|
||||
"role": "user",
|
||||
"username": "ute.user",
|
||||
},
|
||||
)
|
||||
expect(created.status_code == 200 and created.json()["username"] == "ute.user", f"create with username {created.text}")
|
||||
expect(created.json()["email_verified"] is False, "new primary email starts unverified")
|
||||
clear_outbox()
|
||||
|
||||
by_name = client.post("/api/auth/login", json={"email": "Ute.User", "password": "user-pass"})
|
||||
expect(by_name.status_code == 200, f"login by username {by_name.text}")
|
||||
|
||||
unknown = client.post("/api/auth/forgot-password", json={"email": "missing@example.test"})
|
||||
known = client.post("/api/auth/forgot-password", json={"email": "lars@example.test"})
|
||||
expect(unknown.status_code == 200 and known.status_code == 200, "forgot password always 200")
|
||||
expect(unknown.json() == known.json(), "forgot password does not reveal whether the account exists")
|
||||
expect(len(mail_tokens()) == 1, "reset mail only for the verified account")
|
||||
reset_token = mail_tokens()[0]
|
||||
clear_outbox()
|
||||
|
||||
changed = client.post(
|
||||
"/api/auth/reset-password",
|
||||
json={"token": reset_token, "password": "new-pass"},
|
||||
)
|
||||
expect(changed.status_code == 200, f"reset password {changed.text}")
|
||||
stale = client.get("/api/auth/me", headers=headers)
|
||||
expect(stale.status_code == 401, "reset deletes sessions")
|
||||
old_password = client.post("/api/auth/login", json={"email": "lars@example.test", "password": "test-pass"})
|
||||
expect(old_password.status_code == 401, "old password rejected")
|
||||
relogin = client.post("/api/auth/login", json={"email": "lars@example.test", "password": "new-pass"})
|
||||
expect(relogin.status_code == 200, "login with reset password")
|
||||
headers = auth_header(relogin.json()["token"])
|
||||
again = client.post("/api/auth/reset-password", json={"token": reset_token, "password": "other-pass"})
|
||||
expect(again.status_code == 400, "reset token is single use")
|
||||
|
||||
client.post("/api/auth/forgot-password", json={"email": "lars@example.test"})
|
||||
expired_token = mail_tokens()[0]
|
||||
clear_outbox()
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE password_reset_tokens SET expires_at = ? WHERE token_hash IS NOT NULL",
|
||||
("2000-01-01 00:00:00",),
|
||||
)
|
||||
expired = client.post(
|
||||
"/api/auth/reset-password",
|
||||
json={"token": expired_token, "password": "newer-pass"},
|
||||
)
|
||||
expect(expired.status_code == 400, "expired reset token is rejected")
|
||||
|
||||
clear_outbox()
|
||||
renamed = client.patch(
|
||||
"/api/users/me",
|
||||
headers=headers,
|
||||
json={"email": "lars.new@example.test"},
|
||||
)
|
||||
expect(renamed.status_code == 200, f"email change queued {renamed.text}")
|
||||
expect(renamed.json()["email"] == "lars@example.test", "login email unchanged until verification")
|
||||
expect(renamed.json()["pending_email"] == "lars.new@example.test", "pending email stored")
|
||||
still = client.post("/api/auth/login", json={"email": "lars@example.test", "password": "new-pass"})
|
||||
expect(still.status_code == 200, "old email still logs in")
|
||||
confirm = client.post("/api/auth/verify-email", json={"token": mail_tokens()[0]})
|
||||
expect(confirm.status_code == 200 and confirm.json()["email"] == "lars.new@example.test", "verified address becomes login")
|
||||
gone = client.post("/api/auth/login", json={"email": "lars@example.test", "password": "new-pass"})
|
||||
expect(gone.status_code == 401, "previous email stops working")
|
||||
moved = client.post("/api/auth/login", json={"email": "lars.new@example.test", "password": "new-pass"})
|
||||
expect(moved.status_code == 200, "new email logs in")
|
||||
headers = auth_header(moved.json()["token"])
|
||||
|
||||
clear_outbox()
|
||||
other = client.post(
|
||||
f"/api/users/{created.json()['id']}/password-reset",
|
||||
headers=headers,
|
||||
)
|
||||
expect(other.status_code == 200, f"admin reset for another user {other.text}")
|
||||
expect(len(mail_tokens()) == 1, "admin reset sends one link")
|
||||
self_reset = client.post(f"/api/users/{moved.json()['profile_id']}/password-reset", headers=headers)
|
||||
expect(self_reset.status_code == 400, "admin cannot reset the own account via the admin endpoint")
|
||||
|
||||
os.environ.pop("KANSHO_FAKE_MAIL", None)
|
||||
os.environ.pop("SMTP_HOST", None)
|
||||
os.environ.pop("SMTP_FROM", None)
|
||||
os.environ["KANSHO_ENV"] = "test"
|
||||
failed = client.post(
|
||||
f"/api/users/{created.json()['id']}/password-reset",
|
||||
headers=headers,
|
||||
)
|
||||
expect(failed.status_code == 503, f"admin reset without SMTP fails closed {failed.text}")
|
||||
|
||||
secondary_login = client.post(
|
||||
"/api/auth/login",
|
||||
json={"email": "ute.second@example.test", "password": "user-pass"},
|
||||
)
|
||||
expect(secondary_login.status_code == 401, "unknown address is not a login")
|
||||
|
||||
print("Account access tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Architecture correction pass. Run from backend/: python tests/test_architecture_correction.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from db import get_db, migrate_journal_source_refs
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"})
|
||||
expect(space.status_code == 200, "space created")
|
||||
space_id = space.json()["id"]
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space_id}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-20"},
|
||||
)
|
||||
day_id = day.json()["day"]["id"]
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day_id}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Quelle"},
|
||||
)
|
||||
conv_id = conv.json()["id"]
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv_id}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute war der Markt voll. Ich habe Kirschen gekauft."},
|
||||
)
|
||||
expect(turn.status_code == 200, "source turn")
|
||||
gen = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv_id]},
|
||||
)
|
||||
expect(gen.status_code == 200, "draft from conversation")
|
||||
draft = gen.json()
|
||||
expect(draft["source_conversation_ids"] == [conv_id], "draft reconstructs conversation ids")
|
||||
expect(len(draft["source_message_ids"]) >= 2, "draft reconstructs message ids")
|
||||
|
||||
with get_db() as conn:
|
||||
refs = conn.execute(
|
||||
"SELECT source_kind, source_id FROM journal_draft_source_refs WHERE draft_id = ?",
|
||||
(draft["id"],),
|
||||
).fetchall()
|
||||
kinds = {row["source_kind"] for row in refs}
|
||||
expect("conversation" in kinds and "message" in kinds, "draft provenance is relational")
|
||||
|
||||
saved = client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={
|
||||
"journal_day_id": day_id,
|
||||
"title": draft.get("title") or "Markttag",
|
||||
"body": draft.get("body") or "Kirschen",
|
||||
"origin": "accepted_draft",
|
||||
"entry_id": None,
|
||||
"source_conversation_ids": draft["source_conversation_ids"],
|
||||
"source_message_ids": draft["source_message_ids"],
|
||||
},
|
||||
)
|
||||
expect(saved.status_code == 200, f"save entry {saved.text}")
|
||||
entry = saved.json()
|
||||
expect(entry["source_conversation_ids"] == [conv_id], "entry version reconstructs conversation ids")
|
||||
expect(entry["source_message_ids"] == draft["source_message_ids"], "entry version reconstructs message ids")
|
||||
|
||||
versions = client.get(f"/api/journal/entries/{entry['id']}/versions", headers=headers)
|
||||
expect(versions.status_code == 200, "list versions")
|
||||
expect(versions.json()[0]["source_conversation_ids"] == [conv_id], "version list keeps source links")
|
||||
|
||||
legacy_draft_id = str(uuid.uuid4())
|
||||
legacy_version_id = str(uuid.uuid4())
|
||||
legacy_entry_id = str(uuid.uuid4())
|
||||
mid = draft["source_message_ids"][0]
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO journal_drafts
|
||||
(id, profile_id, journal_day_id, title, body,
|
||||
source_conversation_ids, source_message_ids, as_of, superseded_at)
|
||||
VALUES (?, ?, ?, 'Legacy', 'alt', ?, ?, datetime('now'), datetime('now'))
|
||||
""",
|
||||
(legacy_draft_id, profile_id, day_id, json.dumps([conv_id]), json.dumps([mid])),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO journal_entries (id, profile_id, journal_day_id, space_id)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(legacy_entry_id, profile_id, day_id, space_id),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO journal_entry_versions
|
||||
(id, profile_id, entry_id, title, body, source_conversation_ids, source_message_ids, origin)
|
||||
VALUES (?, ?, ?, 'Legacy', 'alt', ?, ?, 'user_edit')
|
||||
""",
|
||||
(legacy_version_id, profile_id, legacy_entry_id, json.dumps([conv_id]), json.dumps([mid])),
|
||||
)
|
||||
migrate_journal_source_refs(conn)
|
||||
draft_refs = [
|
||||
(row["source_kind"], row["source_id"])
|
||||
for row in conn.execute(
|
||||
"SELECT source_kind, source_id FROM journal_draft_source_refs WHERE draft_id = ? ORDER BY sort_order",
|
||||
(legacy_draft_id,),
|
||||
).fetchall()
|
||||
]
|
||||
version_refs = [
|
||||
(row["source_kind"], row["source_id"])
|
||||
for row in conn.execute(
|
||||
"SELECT source_kind, source_id FROM journal_entry_version_source_refs WHERE version_id = ? ORDER BY sort_order",
|
||||
(legacy_version_id,),
|
||||
).fetchall()
|
||||
]
|
||||
expect(("conversation", conv_id) in draft_refs, "legacy draft conversation migrated")
|
||||
expect(("message", mid) in draft_refs, "legacy draft message migrated")
|
||||
expect(("conversation", conv_id) in version_refs, "legacy version conversation migrated")
|
||||
expect(("message", mid) in version_refs, "legacy version message migrated")
|
||||
|
||||
usage = client.post("/api/dialogue/sessions", headers=headers, json={"intent": "space-only"})
|
||||
space_conv = client.post(
|
||||
"/api/dialogue/conversations",
|
||||
headers=headers,
|
||||
json={"usage_session_id": usage.json()["id"], "title": "Plan", "space_id": space_id},
|
||||
)
|
||||
expect(space_conv.status_code == 200, f"space conversation without day {space_conv.text}")
|
||||
expect(space_conv.json()["space_id"] == space_id, "conversation assigned to space")
|
||||
expect(not space_conv.json().get("journal_day_id"), "conversation has no journal day")
|
||||
plan_turn = client.post(
|
||||
f"/api/dialogue/conversations/{space_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Wir wollen morgen zum Hafen, das ist geplant."},
|
||||
)
|
||||
expect(plan_turn.status_code == 200, "space-only turn works without journal day")
|
||||
|
||||
later = client.post(
|
||||
f"/api/journal/spaces/{space_id}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-21"},
|
||||
)
|
||||
later_id = later.json()["day"]["id"]
|
||||
later_conv = client.post(
|
||||
f"/api/journal/days/{later_id}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Heute"},
|
||||
)
|
||||
reset_debug()
|
||||
later_turn = client.post(
|
||||
f"/api/journal/conversations/{later_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute früh aufgestanden."},
|
||||
)
|
||||
expect(later_turn.status_code == 200, "later day turn")
|
||||
rendered = (later_turn.json().get("trace") or {}).get("intern") or ""
|
||||
expect("Hafen" in rendered, "earlier space source can supply a known plan")
|
||||
expect("Heute früh aufgestanden" in rendered, "current conversation remains primary")
|
||||
|
||||
sibling = client.post(
|
||||
f"/api/journal/days/{later_id}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Anderes"},
|
||||
)
|
||||
reset_debug()
|
||||
sibling_turn = client.post(
|
||||
f"/api/journal/conversations/{sibling.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Nur ein kurzer Einkauf."},
|
||||
)
|
||||
expect(sibling_turn.status_code == 200, "sibling turn")
|
||||
sibling_rendered = (sibling_turn.json().get("trace") or {}).get("intern") or ""
|
||||
expect("Heute früh aufgestanden" not in sibling_rendered, "same-day sibling is not fully loaded")
|
||||
expect("Nur ein kurzer Einkauf" in sibling_rendered, "current sibling conversation is primary")
|
||||
|
||||
from context_builder import assemble_text, build_internal_context
|
||||
|
||||
ctx = build_internal_context(profile_id, conversation_id=later_conv.json()["id"], purpose="dialogue_turn")
|
||||
assembled = assemble_text(ctx)
|
||||
expect(not assembled.get("writing_profile"), "dialogue turn has no writing profile brief")
|
||||
expect("interaction_hint" in assembled, "dialogue turn exposes interaction hint slot")
|
||||
expect(assembled.get("interaction_hint"), "interaction hint is filled from labeled defaults")
|
||||
expect("Produktdefault" in (assembled.get("interaction_hint") or ""), "defaults are labeled, not anonymous")
|
||||
expect(any(item.get("type") == "interaction_hint" for item in ctx["items"]), "interaction slot is a context item")
|
||||
expect(
|
||||
not any(item.get("type") == "writing_profile" for item in ctx["items"]),
|
||||
"writing profile is not a dialogue kernel dependency",
|
||||
)
|
||||
expect("space_recent_sources" in {item.get("type") for item in ctx["items"]}, "space source spec is selected")
|
||||
|
||||
journal_ctx = build_internal_context(
|
||||
profile_id,
|
||||
conversation_id=conv_id,
|
||||
journal_day_id=day_id,
|
||||
space_id=space_id,
|
||||
purpose="journal_generate",
|
||||
conversation_ids=[conv_id],
|
||||
)
|
||||
expect(
|
||||
any(item.get("type") == "writing_profile" for item in journal_ctx["items"]),
|
||||
"journal generate still uses writing profile",
|
||||
)
|
||||
expect(
|
||||
not any(item.get("type") == "space_recent_sources" for item in journal_ctx["items"]),
|
||||
"journal generate does not pull space conversation excerpts",
|
||||
)
|
||||
|
||||
from conversation_signals import infer_signals, similar_enough
|
||||
from journal_policy import consolidation_offer
|
||||
|
||||
chronicle = infer_signals(["Heute Vormittag zum Markt, danach Kirschen gekauft."])
|
||||
deep = infer_signals(["Tiefer: der Spaziergang hat mich berührt."])
|
||||
expect(chronicle["narrative_mode"] == "chronicle", "chronicle mode from user text")
|
||||
expect(deep["narrative_mode"] == "reflective", "reflective mode from user text")
|
||||
expect(not similar_enough([chronicle, deep]), "different modes are not similar")
|
||||
expect(consolidation_offer([chronicle, deep]) is False, "policy does not offer unlike modes")
|
||||
twin = infer_signals(["Nach dem Markt noch Brot geholt, dann heimgegangen."])
|
||||
expect(similar_enough([chronicle, twin]), "two chronicles can be offered")
|
||||
|
||||
nav = Path(ROOT).parent / "frontend" / "src" / "config" / "appNav.js"
|
||||
nav_text = nav.read_text(encoding="utf-8")
|
||||
expect("/dialog" not in nav_text, "/dialog is not in normal product navigation")
|
||||
expect("'/journal'" in nav_text, "journal remains in product navigation")
|
||||
|
||||
print("architecture correction tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,307 @@
|
||||
"""Opt-in debug persist for the admin test spur. Run from backend/: python tests/test_debug_persist.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO = ROOT.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-media")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from identity_store import remember_mapping
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def open_space_day(client, headers, title="Alltag", date="2026-08-28"):
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": title})
|
||||
expect(space.status_code == 200, f"create space {space.status_code}")
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": date},
|
||||
)
|
||||
expect(day.status_code == 200, f"open day {day.status_code}")
|
||||
return space.json(), day.json()
|
||||
|
||||
|
||||
def blob(data) -> str:
|
||||
return json.dumps(data, ensure_ascii=False)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
ui = (REPO / "frontend" / "src" / "pages" / "AdminDebugPage.jsx").read_text(encoding="utf-8")
|
||||
expect("Debug persistieren" in ui, "admin page has the persist toggle")
|
||||
expect("Space / Tag / Gespräche" in ui, "admin page uses space/day/conversation structure")
|
||||
expect("Gespräch-JSON" in ui, "admin page can download one conversation")
|
||||
expect("JSON herunterladen" not in ui, "admin page no longer uses a flat download label as primary")
|
||||
nav = (REPO / "frontend" / "src" / "config" / "adminNav.js").read_text(encoding="utf-8")
|
||||
expect("/admin/debug" in nav, "admin nav lists debug")
|
||||
dialogue_ui = (REPO / "frontend" / "src" / "pages" / "DialoguePage.jsx").read_text(encoding="utf-8")
|
||||
expect("CallTrace" not in dialogue_ui, "dialogue page no longer shows the live test spur")
|
||||
expect("Debug herunterladen" in dialogue_ui, "dialogue page can download the conversation debug log")
|
||||
day_ui = (REPO / "frontend" / "src" / "pages" / "JournalDayPage.jsx").read_text(encoding="utf-8")
|
||||
expect("import CallTrace" not in day_ui, "journal day does not render CallTrace under the thread")
|
||||
expect("Debug herunterladen" in day_ui, "journal day can download the conversation debug log")
|
||||
editor_ui = (REPO / "frontend" / "src" / "pages" / "JournalEditorPage.jsx").read_text(encoding="utf-8")
|
||||
expect("import CallTrace" in editor_ui, "journal editor still shows the generate trace")
|
||||
expect("<h2>Verlauf</h2>" in editor_ui, "journal editor names the generate trace")
|
||||
expect("Debug herunterladen" in editor_ui, "journal editor can download the generate debug log")
|
||||
expect("purpose=journal_generate" in editor_ui, "journal editor downloads the generate scope")
|
||||
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
expect(setup.status_code == 200, f"setup {setup.status_code}")
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
|
||||
other = client.post(
|
||||
"/api/users",
|
||||
headers=headers,
|
||||
json={"email": "ute@example.test", "name": "Ute", "password": "user-pass", "role": "user"},
|
||||
)
|
||||
expect(other.status_code == 200, "second profile")
|
||||
other_headers = header(
|
||||
client.post("/api/auth/login", json={"email": "ute@example.test", "password": "user-pass"}).json()["token"]
|
||||
)
|
||||
|
||||
settings = client.get("/api/admin/debug", headers=headers)
|
||||
expect(settings.status_code == 200, "admin can read debug settings")
|
||||
expect(settings.json()["persist_enabled"] is False, "persist is off by default")
|
||||
expect(settings.json()["run_count"] == 0, "no runs at start")
|
||||
|
||||
denied = client.get("/api/admin/debug", headers=other_headers)
|
||||
expect(denied.status_code == 403, "non-admin cannot read debug settings")
|
||||
|
||||
space, day = open_space_day(client, headers)
|
||||
day_id = day["day"]["id"]
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day_id}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Gespräch"},
|
||||
)
|
||||
expect(conv.status_code == 200, f"start conversation {conv.status_code}")
|
||||
conv_id = conv.json()["id"]
|
||||
reset_debug()
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv_id}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute war der Markt voll. Ich habe Kirschen gekauft."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn while persist off {turn.status_code}")
|
||||
listed = client.get("/api/admin/debug/runs", headers=headers)
|
||||
expect(listed.json()["runs"] == [], "off means no persisted steps")
|
||||
|
||||
enabled = client.put("/api/admin/debug", headers=headers, json={"persist_enabled": True})
|
||||
expect(enabled.status_code == 200, "enable persist")
|
||||
expect(enabled.json()["persist_enabled"] is True, "persist flag stored")
|
||||
|
||||
remember_mapping(profile_id, "Anna")
|
||||
reset_debug()
|
||||
first = client.post(
|
||||
f"/api/journal/conversations/{conv_id}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Anna war am Stand. Die Kirschen waren reif."},
|
||||
)
|
||||
expect(first.status_code == 200, f"first persisted turn {first.status_code}")
|
||||
reset_debug()
|
||||
second = client.post(
|
||||
f"/api/journal/conversations/{conv_id}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Später noch der Hafen, ungefähr 18:00 Uhr."},
|
||||
)
|
||||
expect(second.status_code == 200, f"second persisted turn {second.status_code}")
|
||||
|
||||
listed = client.get("/api/admin/debug/runs", headers=headers)
|
||||
expect(listed.status_code == 200, "list runs")
|
||||
runs = listed.json()["runs"]
|
||||
expect(len(runs) >= 2, "each turn is stored")
|
||||
expect(all(item.get("purpose") == "dialogue_turn" for item in runs[:2]), "listed steps are dialogue turns")
|
||||
|
||||
newest = client.get(f"/api/admin/debug/runs/{runs[0]['id']}", headers=headers)
|
||||
expect(newest.status_code == 200, "read run detail")
|
||||
payload = newest.json().get("payload") or {}
|
||||
stored = blob(payload)
|
||||
expect("local_label" not in stored, "mapping labels are not persisted")
|
||||
expect("mapping_table" not in stored, "mapping table is not persisted")
|
||||
expect("replacements" not in stored, "replacement objects are not persisted")
|
||||
details = [
|
||||
client.get(f"/api/admin/debug/runs/{item['id']}", headers=headers).json()
|
||||
for item in runs[:2]
|
||||
]
|
||||
expect(
|
||||
any("Hafen" in blob(item.get("payload") or {}) for item in details),
|
||||
"intern keeps the step text",
|
||||
)
|
||||
hafen = next(item.get("payload") or {} for item in details if "Hafen" in blob(item.get("payload") or {}))
|
||||
expect((hafen.get("trace") or {}).get("egress") or (hafen.get("call_traces")), "egress of the step is stored")
|
||||
expect((hafen.get("user") or {}).get("body"), "user line of the step is stored")
|
||||
expect((hafen.get("assistant") or {}).get("body"), "assistant line of the step is stored")
|
||||
expect(hafen.get("call_traces") or (hafen.get("trace") or {}).get("egress"), "gateway calls of the step are stored")
|
||||
|
||||
user_space = client.post("/api/journal/spaces", headers=other_headers, json={"title": "Utes Tag"})
|
||||
user_day = client.post(
|
||||
f"/api/journal/spaces/{user_space.json()['id']}/days",
|
||||
headers=other_headers,
|
||||
json={"calendar_date": "2026-08-28"},
|
||||
)
|
||||
user_conv = client.post(
|
||||
f"/api/journal/days/{user_day.json()['day']['id']}/conversations",
|
||||
headers=other_headers,
|
||||
json={"title": "Gespräch"},
|
||||
)
|
||||
user_turn = client.post(
|
||||
f"/api/journal/conversations/{user_conv.json()['id']}/turn",
|
||||
headers=other_headers,
|
||||
json={"body": "Utes privater Satz bleibt ohne Debug-Speicher."},
|
||||
)
|
||||
expect(user_turn.status_code == 200, "non-admin can still talk")
|
||||
after_user = client.get("/api/admin/debug/runs", headers=headers).json()["runs"]
|
||||
expect(len(after_user) == len(runs), "non-admin turns are not persisted")
|
||||
expect(
|
||||
not any("Utes privater Satz" in blob(item) for item in after_user),
|
||||
"admin list does not contain the other profile",
|
||||
)
|
||||
|
||||
reset_debug()
|
||||
gen = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv_id]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"generate {gen.status_code}")
|
||||
after_gen = client.get("/api/admin/debug/runs", headers=headers).json()["runs"]
|
||||
expect(any(item.get("purpose") == "journal_generate" for item in after_gen), "journal generate is a stored step")
|
||||
|
||||
exported = client.get("/api/admin/debug/export?format=json", headers=headers)
|
||||
expect(exported.status_code == 200, "json export")
|
||||
expect("attachment" in (exported.headers.get("content-disposition") or ""), "json is a download")
|
||||
document = exported.json()
|
||||
expect(document.get("kind") == "kansho.debug_export", "export names the contract")
|
||||
expect(document.get("version") == 2, "export version is nested tree")
|
||||
expect(document.get("scope") == "all", "full export is unscoped")
|
||||
expect(document.get("run_count") == len(document.get("runs") or []), "run_count matches runs")
|
||||
expect(len(document.get("runs") or []) >= 3, "export contains the dialogue and generate steps")
|
||||
expect((document.get("spaces") or []), "full export has space tree")
|
||||
space = (document.get("spaces") or [{}])[0]
|
||||
expect(space.get("title") == "Alltag", "tree names the space")
|
||||
day_node = (space.get("days") or [{}])[0]
|
||||
expect(day_node.get("calendar_date") == "2026-08-28", "tree names the day")
|
||||
conv_node = (day_node.get("conversations") or [{}])[0]
|
||||
expect(len(conv_node.get("steps") or []) >= 2, "tree keeps dialogue steps on the conversation")
|
||||
expect(day_node.get("generates"), "journal generate sits on the day")
|
||||
expect("local_label" not in blob(document), "export has no mapping labels")
|
||||
created = [item.get("created") for item in document["runs"]]
|
||||
expect(created == sorted(created), "export is chronological")
|
||||
|
||||
tree = client.get("/api/admin/debug/tree", headers=headers)
|
||||
expect(tree.status_code == 200, "tree endpoint")
|
||||
tree_space = (tree.json().get("spaces") or [{}])[0]
|
||||
expect(tree_space.get("title") == "Alltag", "live tree names the space")
|
||||
tree_conv = ((tree_space.get("days") or [{}])[0].get("conversations") or [{}])[0]
|
||||
expect(tree_conv.get("id") == conv_id, "live tree points at the conversation")
|
||||
expect(tree_conv.get("step_count") >= 2, "live tree counts steps")
|
||||
|
||||
conv_export = client.get(
|
||||
f"/api/admin/debug/export?format=json&conversation_id={conv_id}",
|
||||
headers=headers,
|
||||
)
|
||||
expect(conv_export.status_code == 200, "conversation export")
|
||||
conv_doc = conv_export.json()
|
||||
expect(conv_doc.get("scope") == "conversation", "conversation export is scoped")
|
||||
expect(conv_doc.get("conversation", {}).get("id") == conv_id, "conversation export names the dialogue")
|
||||
expect(
|
||||
all(item.get("purpose") == "dialogue_turn" for item in conv_doc.get("runs") or []),
|
||||
"conversation export is the dialogue steps, not the day generate",
|
||||
)
|
||||
expect(len(conv_doc.get("runs") or []) >= 2, "conversation export has both turns")
|
||||
expect("conversation" in (conv_export.headers.get("content-disposition") or ""), "conversation filename is distinct")
|
||||
|
||||
latest = client.get(
|
||||
f"/api/admin/debug/latest?journal_day_id={day_id}&purpose=journal_generate",
|
||||
headers=headers,
|
||||
)
|
||||
expect(latest.status_code == 200, "latest generate endpoint")
|
||||
latest_run = latest.json().get("run") or {}
|
||||
expect(latest_run.get("purpose") == "journal_generate", "latest run is the journal generate")
|
||||
expect(latest_run.get("journal_day_id") == day_id, "latest generate belongs to the day")
|
||||
expect((latest_run.get("trace") or {}).get("purpose") == "journal_generate" or (latest_run.get("trace") or {}).get("stages"), "latest generate keeps the trace")
|
||||
|
||||
gen_export = client.get(
|
||||
f"/api/admin/debug/export?format=json&journal_day_id={day_id}&purpose=journal_generate",
|
||||
headers=headers,
|
||||
)
|
||||
expect(gen_export.status_code == 200, "generate export")
|
||||
gen_doc = gen_export.json()
|
||||
expect(gen_doc.get("scope") == "journal_generate", "generate export is scoped")
|
||||
expect(
|
||||
all(item.get("purpose") == "journal_generate" for item in gen_doc.get("runs") or []),
|
||||
"generate export is only the journal draft steps",
|
||||
)
|
||||
expect(len(gen_doc.get("runs") or []) >= 1, "generate export has the draft run")
|
||||
expect("journal" in (gen_export.headers.get("content-disposition") or ""), "generate filename is distinct")
|
||||
|
||||
markdown = client.get("/api/admin/debug/export?format=markdown", headers=headers)
|
||||
expect(markdown.status_code == 200, "markdown export")
|
||||
text = markdown.text
|
||||
expect("Kanshō Debug-Export" in text, "markdown has a title")
|
||||
expect("Intern (Klartext-Vorlage, nicht gesendet)" in text, "markdown has intern section")
|
||||
expect("Egress (maskiert, gesendet)" in text, "markdown has egress section")
|
||||
expect("dialogue_turn" in text, "markdown lists dialogue steps")
|
||||
expect("journal_generate" in text, "markdown lists generate steps")
|
||||
|
||||
bad = client.get("/api/admin/debug/export?format=csv", headers=headers)
|
||||
expect(bad.status_code == 400, "unknown export format is rejected")
|
||||
|
||||
disabled = client.put("/api/admin/debug", headers=headers, json={"persist_enabled": False})
|
||||
expect(disabled.json()["persist_enabled"] is False, "persist can be turned off")
|
||||
count_before = len(client.get("/api/admin/debug/runs", headers=headers).json()["runs"])
|
||||
reset_debug()
|
||||
later = client.post(
|
||||
f"/api/journal/conversations/{conv_id}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Noch ein Satz ohne Persistenz."},
|
||||
)
|
||||
expect(later.status_code == 200, "turn after disable")
|
||||
count_after = len(client.get("/api/admin/debug/runs", headers=headers).json()["runs"])
|
||||
expect(count_after == count_before, "disable stops new writes and keeps history")
|
||||
|
||||
run_id = client.get("/api/admin/debug/runs", headers=headers).json()["runs"][0]["id"]
|
||||
removed = client.delete(f"/api/admin/debug/runs/{run_id}", headers=headers)
|
||||
expect(removed.status_code == 200, "delete one run")
|
||||
cleared = client.post("/api/admin/debug/runs/clear", headers=headers)
|
||||
expect(cleared.status_code == 200, "clear remaining runs")
|
||||
expect(client.get("/api/admin/debug/runs", headers=headers).json()["runs"] == [], "history empty after clear")
|
||||
|
||||
health = client.get("/api/admin/health", headers=headers)
|
||||
expect(health.json()["inventory"]["debug_persist_enabled"] is False, "health shows persist off")
|
||||
expect(health.json()["inventory"]["debug_runs"] == 0, "health shows empty debug count")
|
||||
|
||||
print("All debug persist tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,503 @@
|
||||
"""Bounded detect-contract retry, acyclic diagnostics, persist must not hide EngineError.
|
||||
|
||||
Run from backend/: python tests/test_detect_contract_retry.py
|
||||
Fake detect/provider only. No live calls.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from debug_store import persist_engine_error, sanitize
|
||||
from engine import EngineError
|
||||
from entity_detect import (
|
||||
DetectError,
|
||||
detect_personal_egress,
|
||||
install_test_detect_failure,
|
||||
install_test_detect_script,
|
||||
reset_detect_test_hooks,
|
||||
split_detect_chunks,
|
||||
validate_detected_entity,
|
||||
)
|
||||
from journal_generate import _compose_journal_trace, _stage_trace
|
||||
from main import app
|
||||
from privacy_gateway import GatewayRequest, PrivacyGatewayError, complete, reset_debug
|
||||
from providers import ChatResult
|
||||
|
||||
|
||||
USAGE_A = {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13, "cost": 0.001}
|
||||
USAGE_B = {"prompt_tokens": 8, "completion_tokens": 2, "total_tokens": 10, "cost": 0.0005}
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def _ids(obj, stack=None) -> bool:
|
||||
"""True if obj is acyclic."""
|
||||
if not isinstance(obj, (dict, list, tuple)):
|
||||
return True
|
||||
stack = stack or set()
|
||||
ident = id(obj)
|
||||
if ident in stack:
|
||||
return False
|
||||
stack.add(ident)
|
||||
try:
|
||||
items = obj.values() if isinstance(obj, dict) else obj
|
||||
return all(_ids(item, stack) for item in items)
|
||||
finally:
|
||||
stack.discard(ident)
|
||||
|
||||
|
||||
def _budget_is_not_diagnostics(payload: dict) -> bool:
|
||||
diag = payload.get("diagnostics") if isinstance(payload.get("diagnostics"), dict) else payload
|
||||
trace = diag.get("trace") if isinstance(diag.get("trace"), dict) else {}
|
||||
budget = trace.get("budget")
|
||||
if budget is diag:
|
||||
return False
|
||||
for stage in trace.get("stages") or []:
|
||||
if isinstance(stage, dict) and stage.get("budget") is diag:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _run(profile_id: str, rendered: str, purpose: str = "dialogue_turn"):
|
||||
return complete(
|
||||
GatewayRequest(
|
||||
prompt_id="detect-retry-test",
|
||||
purpose=purpose,
|
||||
data_class="B",
|
||||
profile_id=profile_id,
|
||||
payload={"rendered": rendered, "source_text": rendered},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _chat(entities: list[dict], usage: dict) -> ChatResult:
|
||||
return ChatResult(
|
||||
content=json.dumps({"entities": entities}),
|
||||
model="fake-detect",
|
||||
usage=usage,
|
||||
finish_reason="stop",
|
||||
)
|
||||
|
||||
|
||||
def _invalid_type():
|
||||
return [{"start": 0, "end": 1, "text": "x", "entity_type": "HUMAN"}]
|
||||
|
||||
|
||||
def _extra_field():
|
||||
return [{"start": 0, "end": 1, "text": "x", "entity_type": "PERSON", "score": 0.9}]
|
||||
|
||||
|
||||
def _missing_field():
|
||||
return [{"start": 0, "text": "x", "entity_type": "PERSON"}]
|
||||
|
||||
|
||||
def _bad_offset(chunk_text: str):
|
||||
return [{"start": 0, "end": 5, "text": "ZZZXQ", "entity_type": "PERSON"}]
|
||||
|
||||
|
||||
def open_space_day(client, headers, title="Alltag", date="2026-08-29"):
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": title})
|
||||
expect(space.status_code == 200, f"create space {space.status_code}")
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": date},
|
||||
)
|
||||
expect(day.status_code == 200, f"open day {day.status_code}")
|
||||
return space.json(), day.json()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
reset_detect_test_hooks()
|
||||
|
||||
sample = "Ich traf Anna."
|
||||
try:
|
||||
validate_detected_entity(
|
||||
{"start": 0, "end": 4, "text": "Anna", "entity_type": "HUMAN"},
|
||||
sample,
|
||||
0,
|
||||
)
|
||||
typed = False
|
||||
except DetectError as exc:
|
||||
typed = exc.code == "detect_invalid_output"
|
||||
expect((exc.diagnostics or {}).get("invalid_entity_type") == "HUMAN", "unknown type is recorded without the span text")
|
||||
expect((exc.diagnostics or {}).get("contract_violation") == "unknown_entity_type", "unknown type has a violation category")
|
||||
expect("Anna" not in json.dumps(exc.diagnostics or {}), "span plaintext is not stored on the type violation")
|
||||
expect(typed, "unknown entity_type is a contract error")
|
||||
|
||||
cyclic = {}
|
||||
nested = {"budget": cyclic, "stages": [cyclic]}
|
||||
cyclic["trace"] = nested
|
||||
cyclic["local_label"] = "secret-name"
|
||||
cleaned = sanitize(cyclic)
|
||||
expect(cleaned is not None, "cyclic sanitize returns a value")
|
||||
expect(_ids(cleaned), "sanitized cyclic input is acyclic")
|
||||
expect("local_label" not in json.dumps(cleaned), "sanitize still drops mapping labels")
|
||||
expect(json.dumps(cleaned, default=str), "sanitized cyclic input is JSON-serializable")
|
||||
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
expect(setup.status_code == 200, "setup")
|
||||
token = setup.json()["token"]
|
||||
headers = header(token)
|
||||
profile_id = setup.json()["profile_id"]
|
||||
|
||||
calls = []
|
||||
|
||||
def retry_then_ok(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
||||
usage = USAGE_A if attempt == 1 else USAGE_B
|
||||
calls.append((attempt, chunk_index, schema_retry))
|
||||
if attempt == 1:
|
||||
return _chat(_invalid_type(), usage)
|
||||
return _chat([], usage)
|
||||
|
||||
long_body = (
|
||||
"WRITING_PROFILE\nkurze trockene Saetze. " * 12
|
||||
+ "STYLE_EXAMPLES\nDer Abend blieb ruhig. " * 12
|
||||
+ "CURRENT_DAY_SOURCES\nHeute der Markt, danach der Hafen. "
|
||||
)
|
||||
with patch("entity_detect.DETECT_CHUNK_CHARS", 90), patch("entity_detect.DETECT_CHUNK_OVERLAP", 12):
|
||||
chunks = split_detect_chunks(long_body, chunk_chars=90, overlap=12)
|
||||
expect(len(chunks) >= 2, "long profile+examples text is chunked")
|
||||
install_test_detect_script(retry_then_ok)
|
||||
try:
|
||||
outcome = detect_personal_egress(profile_id, long_body)
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
attempts = {item[0] for item in calls}
|
||||
expect(attempts == {1, 2}, "invalid first pass starts a second full pass")
|
||||
first_chunks = [item[1] for item in calls if item[0] == 1]
|
||||
second_chunks = [item[1] for item in calls if item[0] == 2]
|
||||
expect(second_chunks == list(range(len(chunks))), "second pass re-checks every chunk")
|
||||
expect(first_chunks == [0], "failed first pass stops at the invalid chunk and discards partials")
|
||||
expect(any(item[2] for item in calls if item[0] == 2), "second pass is marked as schema retry")
|
||||
expect(not any(item[2] for item in calls if item[0] == 1), "first pass has no schema retry hint")
|
||||
expect(outcome.stats.detect_partial_discarded is True, "partial first-pass spans are discarded")
|
||||
expect(outcome.stats.detect_passes == 2, "successful retry records two passes")
|
||||
expect(outcome.stats.detect_calls == len(calls), "detect calls sum both passes")
|
||||
expect(outcome.stats.cost_known is True, "injected usage keeps known detect cost")
|
||||
expected_cost = USAGE_A["cost"] * len(first_chunks) + USAGE_B["cost"] * len(second_chunks)
|
||||
expect(abs(outcome.stats.cost - expected_cost) < 1e-9, "detect costs of both passes are aggregated")
|
||||
expect(
|
||||
abs(
|
||||
outcome.stats.prompt_tokens
|
||||
- (USAGE_A["prompt_tokens"] * len(first_chunks) + USAGE_B["prompt_tokens"] * len(second_chunks))
|
||||
)
|
||||
< 1e-9,
|
||||
"detect tokens of both passes are aggregated",
|
||||
)
|
||||
expect(outcome.stats.full_detection_coverage is True, "valid second pass has full coverage")
|
||||
expect(outcome.stats.detect_ms >= 0, "retry records duration")
|
||||
|
||||
generate_n = {"n": 0}
|
||||
|
||||
def count_generate(messages, policy):
|
||||
generate_n["n"] += 1
|
||||
from privacy_gateway import _fake_complete
|
||||
|
||||
return ChatResult(content=_fake_complete(policy.get("purpose") or "", messages[0].get("content") or ""), model="fake")
|
||||
|
||||
install_test_detect_script(retry_then_ok)
|
||||
generate_n["n"] = 0
|
||||
with patch("privacy_gateway.complete_model", count_generate):
|
||||
try:
|
||||
ran = _run(profile_id, "Heute der Markt und danach der Hafen.")
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(ran.trace.get("generate_called") is True, "valid retry allows exactly one narration call")
|
||||
expect(generate_n["n"] == 1, "narration runs once after a recovered detect pass")
|
||||
expect(_ids(ran.diagnostics or {}), "successful diagnostics are acyclic")
|
||||
expect(_ids(ran.trace or {}), "successful trace is acyclic")
|
||||
|
||||
def always_invalid(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
||||
return _chat(_invalid_type(), USAGE_A)
|
||||
|
||||
generate_n["n"] = 0
|
||||
install_test_detect_script(always_invalid)
|
||||
blocked = None
|
||||
with patch("privacy_gateway.complete_model", count_generate):
|
||||
try:
|
||||
_run(profile_id, "Heute der Markt und danach der Hafen.")
|
||||
except PrivacyGatewayError as exc:
|
||||
blocked = exc
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(blocked is not None and blocked.code == "detect_invalid_output", "two invalid passes fail closed")
|
||||
expect(generate_n["n"] == 0, "invalid detect never calls narration")
|
||||
expect(blocked.diagnostics.get("generate_called") is False, "final abort keeps generate_called false")
|
||||
expect(blocked.diagnostics.get("detect_passes") == 2, "both invalid passes are counted")
|
||||
expect(blocked.diagnostics.get("contract_violation") == "unknown_entity_type", "final abort keeps the violation category")
|
||||
expect(blocked.diagnostics.get("invalid_entity_type") == "HUMAN", "delivered type is stored without span text")
|
||||
expect(blocked.diagnostics.get("detect_partial_discarded") is True, "failed retry discarded the first pass")
|
||||
expect(_ids(blocked.diagnostics), "failed diagnostics are acyclic")
|
||||
expect("HUMAN" in json.dumps(blocked.diagnostics), "invalid type remains visible")
|
||||
expect("secret" not in json.dumps(blocked.diagnostics).lower(), "no extra identity payload")
|
||||
expect(blocked.status_code != 500, "detect abort is not a generic 500")
|
||||
|
||||
def extra_then_fail(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
||||
return _chat(_extra_field(), USAGE_A)
|
||||
|
||||
install_test_detect_script(extra_then_fail)
|
||||
extra_err = None
|
||||
try:
|
||||
detect_personal_egress(profile_id, "Heute der Markt.")
|
||||
except DetectError as exc:
|
||||
extra_err = exc
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(extra_err is not None and extra_err.diagnostics.get("contract_violation") == "extra_fields", "extra fields fail closed after retry")
|
||||
|
||||
def missing_then_fail(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
||||
return _chat(_missing_field(), USAGE_A)
|
||||
|
||||
install_test_detect_script(missing_then_fail)
|
||||
missing_err = None
|
||||
try:
|
||||
detect_personal_egress(profile_id, "Heute der Markt.")
|
||||
except DetectError as exc:
|
||||
missing_err = exc
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(missing_err is not None and missing_err.diagnostics.get("contract_violation") == "missing_fields", "missing fields fail closed after retry")
|
||||
|
||||
def offset_then_fail(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
||||
return _chat(_bad_offset(chunk_text), USAGE_A)
|
||||
|
||||
install_test_detect_script(offset_then_fail)
|
||||
offset_err = None
|
||||
try:
|
||||
detect_personal_egress(profile_id, "Heute der Markt.")
|
||||
except DetectError as exc:
|
||||
offset_err = exc
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(offset_err is not None and offset_err.diagnostics.get("contract_violation") == "unusable_offsets", "unusable offsets fail closed after retry")
|
||||
|
||||
net_calls = []
|
||||
|
||||
def network_fail(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
||||
net_calls.append(attempt)
|
||||
raise DetectError("detect_chunk_failed", "chunk failed")
|
||||
|
||||
install_test_detect_script(network_fail)
|
||||
net_err = None
|
||||
try:
|
||||
detect_personal_egress(profile_id, "Heute der Markt.")
|
||||
except DetectError as exc:
|
||||
net_err = exc
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(net_err is not None and net_err.code == "detect_chunk_failed", "provider/chunk failure stays distinct")
|
||||
expect(net_calls == [1], "network-like chunk failure is not doubled by the contract retry")
|
||||
|
||||
install_test_detect_failure(DetectError("detect_chunk_failed", "chunk failed"))
|
||||
immediate = None
|
||||
try:
|
||||
detect_personal_egress(profile_id, "Heute der Markt.")
|
||||
except DetectError as exc:
|
||||
immediate = exc
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(immediate is not None and immediate.diagnostics.get("detect_passes") == 0, "injected provider failure does not start a contract retry")
|
||||
|
||||
mixed = []
|
||||
|
||||
def worker(kind: str):
|
||||
reset_debug()
|
||||
seen = []
|
||||
|
||||
def script(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
||||
seen.append((kind, attempt))
|
||||
if attempt == 1:
|
||||
entity_type = "HUMAN" if kind == "alpha" else "ANIMAL"
|
||||
return _chat([{"start": 0, "end": 1, "text": "x", "entity_type": entity_type}], USAGE_A)
|
||||
return _chat([], USAGE_B)
|
||||
|
||||
install_test_detect_script(script)
|
||||
try:
|
||||
out = detect_personal_egress(profile_id, f"Heute der Markt {kind}.")
|
||||
mixed.append((kind, tuple(seen), out.stats.invalid_entity_type, out.stats.detect_calls, id(out.stats)))
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
one = pool.submit(worker, "alpha")
|
||||
two = pool.submit(worker, "beta")
|
||||
one.result()
|
||||
two.result()
|
||||
kinds = {item[0] for item in mixed}
|
||||
expect(kinds == {"alpha", "beta"}, "parallel detect retries both complete")
|
||||
by_kind = {item[0]: item for item in mixed}
|
||||
expect(
|
||||
all(kind == row[0] for row in by_kind.values() for kind, _attempt in row[1]),
|
||||
"parallel scripts stay on their request",
|
||||
)
|
||||
expect(by_kind["alpha"][2] in (None, "HUMAN"), "alpha does not keep beta's type")
|
||||
expect(by_kind["beta"][2] in (None, "ANIMAL"), "beta does not keep alpha's type")
|
||||
expect(by_kind["alpha"][4] != by_kind["beta"][4], "parallel stats objects are distinct")
|
||||
|
||||
failed = {
|
||||
"diagnostics": {"generate_called": False, "detect_cost": 0.001, "cost": None, "purpose": "journal_generate"},
|
||||
"trace": {"purpose": "journal_generate", "generate_called": False},
|
||||
"content": "",
|
||||
}
|
||||
staged = _stage_trace(failed, "journal_generate")
|
||||
expect(staged.get("budget") is not failed["diagnostics"], "stage budget is a snapshot, not the live diagnostics object")
|
||||
failed["diagnostics"]["trace"] = {"budget": staged.get("budget")}
|
||||
expect(_ids(failed), "attaching the snapshot does not create a diagnostics cycle")
|
||||
composed = _compose_journal_trace(
|
||||
{"trace": {"purpose": "local_source_artifact"}, "diagnostics": {"prompt_tokens": 0}},
|
||||
failed,
|
||||
run_log=[{"kind": "detect", "status": "error"}],
|
||||
narrate_prompt={"slug": "mvp.journal_generate", "seed_revision": "test"},
|
||||
profile_meta={"present": False},
|
||||
style_meta={"count": 0},
|
||||
dropped=[],
|
||||
extra={"generation_selection": {"voice_id": "journal-generate-voice-clear"}, "style_application": {"include_core": False}},
|
||||
)
|
||||
failed["diagnostics"]["trace"] = composed
|
||||
expect(_ids(failed["diagnostics"]), "composed journal diagnostics stay acyclic")
|
||||
expect(_budget_is_not_diagnostics(failed), "composed budget is not a back-reference")
|
||||
expect(json.dumps(sanitize(failed), default=str), "composed failure is serializable")
|
||||
|
||||
space, day = open_space_day(client, headers)
|
||||
day_id = day["day"]["id"]
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day_id}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Gespräch"},
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute war der Markt voll und danach der Hafen ruhig."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn {turn.status_code}")
|
||||
|
||||
enabled = client.put("/api/admin/debug", headers=headers, json={"persist_enabled": True})
|
||||
expect(enabled.status_code == 200 and enabled.json()["persist_enabled"] is True, "persist enabled")
|
||||
before = client.get("/api/admin/debug/runs", headers=headers).json()["runs"]
|
||||
|
||||
install_test_detect_script(always_invalid)
|
||||
try:
|
||||
gen = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(gen.status_code == 503, f"journal detect abort status {gen.status_code}")
|
||||
detail = gen.json()["detail"]
|
||||
expect(detail["code"] == "detect_invalid_output", "API keeps the detect contract code")
|
||||
expect(detail["code"] != "error" and gen.status_code != 500, "detect abort is not HTTP 500")
|
||||
expect("Angaben" in (detail.get("message") or ""), "API uses the user-facing detect message")
|
||||
expect(detail.get("cost_report"), "API includes a cost report")
|
||||
expect(detail["cost_report"].get("generate_called") is False, "cost report says generate was not called")
|
||||
expect(
|
||||
(detail.get("diagnostics") or {}).get("generate_called") is False,
|
||||
"diagnostics keep generate_called false",
|
||||
)
|
||||
expect(_ids(detail), "API error payload is acyclic")
|
||||
expect(_budget_is_not_diagnostics(detail), "API error budget is not a diagnostics alias")
|
||||
after = client.get("/api/admin/debug/runs", headers=headers).json()["runs"]
|
||||
error_runs = [
|
||||
item
|
||||
for item in after
|
||||
if item.get("purpose") == "journal_generate" and item.get("status") == "error"
|
||||
]
|
||||
expect(len(error_runs) == 1, "failed journal request is stored exactly once")
|
||||
run = error_runs[0]
|
||||
expect(run.get("journal_day_id") == day_id, "error run is attached to the journal day")
|
||||
detail_run = client.get(f"/api/admin/debug/runs/{run['id']}", headers=headers)
|
||||
expect(detail_run.status_code == 200, f"error run detail {detail_run.status_code}")
|
||||
payload = detail_run.json().get("payload") or {}
|
||||
expect(payload.get("code") == "detect_invalid_output", "stored run keeps the detect code")
|
||||
expect(payload.get("generate_called") is False, "stored run records generate_called false")
|
||||
expect(payload.get("detect_passes") == 2, "stored run records both detect passes")
|
||||
expect(payload.get("detect_partial_discarded") is True, "stored run records discarded partials")
|
||||
expect(payload.get("contract_violation") == "unknown_entity_type", "stored run records the violation")
|
||||
expect(payload.get("invalid_entity_type") == "HUMAN", "stored run records the delivered type")
|
||||
expect(payload.get("generation_selection") or (payload.get("trace") or {}).get("generation_selection"), "stored run keeps generation selection")
|
||||
expect("local_label" not in json.dumps(payload), "stored run has no mapping labels")
|
||||
exported = client.get(
|
||||
f"/api/admin/debug/export?format=json&journal_day_id={day_id}&purpose=journal_generate",
|
||||
headers=headers,
|
||||
)
|
||||
expect(exported.status_code == 200, f"error run export {exported.status_code}")
|
||||
document = exported.json()
|
||||
encoded = json.dumps(document)
|
||||
expect(encoded, "exported error run is JSON-serializable")
|
||||
expect("detect_invalid_output" in encoded, "export contains the detect code")
|
||||
|
||||
hidden = EngineError("detect_invalid_output", "Persönliche Angaben konnten nicht sicher zugeordnet werden.", 503)
|
||||
with patch("debug_store.persist_step", side_effect=RuntimeError("persist boom")):
|
||||
persist_engine_error(profile_id, purpose="journal_generate", exc=hidden, journal_day_id=day_id)
|
||||
install_test_detect_script(always_invalid)
|
||||
try:
|
||||
with patch("debug_store.persist_step", side_effect=RuntimeError("persist boom")):
|
||||
gen_hidden = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
finally:
|
||||
reset_detect_test_hooks()
|
||||
expect(gen_hidden.status_code == 503, f"persist failure keeps detect status {gen_hidden.status_code}")
|
||||
expect(gen_hidden.json()["detail"]["code"] == "detect_invalid_output", "persist failure does not replace EngineError")
|
||||
expect(gen_hidden.status_code != 500, "persist failure is not HTTP 500")
|
||||
|
||||
reset_detect_test_hooks()
|
||||
control_space, control_day = open_space_day(client, headers, "Kontrolle", "2026-08-28")
|
||||
control_conv = client.post(
|
||||
f"/api/journal/days/{control_day['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Gespräch"},
|
||||
)
|
||||
control_turn = client.post(
|
||||
f"/api/journal/conversations/{control_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute Tee auf dem Balkon, danach nur der Wind."},
|
||||
)
|
||||
expect(control_turn.status_code == 200, "control turn")
|
||||
control = client.post(
|
||||
f"/api/journal/days/{control_day['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [control_conv.json()["id"]]},
|
||||
)
|
||||
expect(control.status_code == 200, f"control generate without profile {control.status_code}")
|
||||
expect((control.json().get("trace") or {}).get("generate_called") is True, "control run still narrates once")
|
||||
expect(_ids(control.json().get("trace") or {}), "control trace is acyclic")
|
||||
|
||||
client.put("/api/admin/debug", headers=headers, json={"persist_enabled": False})
|
||||
|
||||
print("ALL TESTS PASSED")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,446 @@
|
||||
"""Transitional learning detect: senses from dialogue review, admin mode switch.
|
||||
|
||||
Fake detect/provider only. Run from backend/: python tests/test_detect_learning.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from detect_learning import (
|
||||
cue_for_mention,
|
||||
decision_for_mention,
|
||||
excerpt_view,
|
||||
get_cue_decision,
|
||||
get_detect_operating_mode,
|
||||
get_sense,
|
||||
merge_gliner_candidates,
|
||||
passage_provider,
|
||||
record_cue,
|
||||
record_sense,
|
||||
set_detect_operating_mode,
|
||||
try_local_passage_decision,
|
||||
)
|
||||
from detect_pipeline import (
|
||||
MentionSpan,
|
||||
compact_diff_counters,
|
||||
compare_mentions,
|
||||
map_entity_type,
|
||||
prepare_detect_pipeline,
|
||||
same_mention,
|
||||
)
|
||||
from providers import ChatResult, ProviderConfig
|
||||
from dialogue_turn import visible_for_role
|
||||
from identity_store import list_confirmed_identities
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def status_ok(response, label: str) -> None:
|
||||
if response.status_code != 200:
|
||||
detail = response.json() if response.headers.get("content-type", "").startswith("application/json") else {}
|
||||
code = detail.get("detail", {}).get("code") if isinstance(detail, dict) else None
|
||||
expect(False, f"{label} HTTP {response.status_code} {code or ''}".strip())
|
||||
expect(True, label)
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def open_conv(client: TestClient, headers: dict) -> str:
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Alltag"})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-09-08"},
|
||||
)
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Gespräch"},
|
||||
)
|
||||
return conv.json()["id"]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
|
||||
providers = client.get("/api/admin/providers", headers=headers)
|
||||
expect(providers.status_code == 200, "providers status")
|
||||
expect(providers.json()["detect_operating_mode"] == "bypass", "fake-env default is bypass")
|
||||
expect(get_detect_operating_mode() == "bypass", "store default in tests is bypass")
|
||||
ner = providers.json().get("local_ner") or {}
|
||||
expect(ner.get("enabled") is False, "GLiNER is off by default")
|
||||
detect_before = next(item for item in providers.json()["roles"] if item["role"] == "detect")
|
||||
remote_gliner = client.put(
|
||||
"/api/admin/providers/local-ner",
|
||||
headers=headers,
|
||||
json={"url": "https://openrouter.ai/api/v1"},
|
||||
)
|
||||
expect(remote_gliner.status_code == 400, "GLiNER URL must stay local")
|
||||
enabled = client.put(
|
||||
"/api/admin/providers/local-ner",
|
||||
headers=headers,
|
||||
json={"enabled": True, "url": "http://192.168.2.144:8090"},
|
||||
)
|
||||
status_ok(enabled, "enable local GLiNER")
|
||||
expect(enabled.json()["local_ner"]["enabled"] is True, "GLiNER enabled flag persisted")
|
||||
expect(enabled.json()["local_ner"]["url"] == "http://192.168.2.144:8090", "GLiNER LAN URL persisted")
|
||||
detect_after = next(item for item in enabled.json()["roles"] if item["role"] == "detect")
|
||||
expect(detect_after.get("url") == detect_before.get("url"), "OpenRouter detect URL is unchanged")
|
||||
client.put("/api/admin/providers/local-ner", headers=headers, json={"enabled": False})
|
||||
|
||||
bad = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "wordlist"})
|
||||
expect(bad.status_code == 400, "invalid detect mode is rejected")
|
||||
|
||||
conv_semantic = open_conv(client, headers)
|
||||
semantic_turn = client.post(
|
||||
f"/api/journal/conversations/{conv_semantic}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Anna kam vorbei."},
|
||||
)
|
||||
status_ok(semantic_turn, "bypass turn")
|
||||
expect(not semantic_turn.json().get("pending_mask_review"), "bypass mode does not pause for review")
|
||||
expect(any(item.get("role") == "assistant" for item in semantic_turn.json().get("messages") or []), "bypass turn gets an assistant")
|
||||
|
||||
switched = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning"})
|
||||
status_ok(switched, "switch to learning alias")
|
||||
expect(switched.json()["detect_operating_mode"] == "learning_full", "learning alias stores learning_full")
|
||||
|
||||
conv = open_conv(client, headers)
|
||||
paused = client.post(
|
||||
f"/api/journal/conversations/{conv}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Anna kam vorbei."},
|
||||
)
|
||||
status_ok(paused, "learning pause")
|
||||
review = paused.json().get("pending_mask_review") or {}
|
||||
expect(bool(review.get("id")), "learning mode returns a review id")
|
||||
expect(paused.json().get("assistant") is None, "paused turn has no assistant")
|
||||
candidates = review.get("candidates") or []
|
||||
anna = next((item for item in candidates if item.get("text") == "Anna"), None)
|
||||
expect(anna is not None, "Anna is a review candidate")
|
||||
hidden = visible_for_role({"pending_mask_review": review, "trace": {"egress": "x"}}, "user")
|
||||
expect("pending_mask_review" in hidden, "users still receive the mask review")
|
||||
expect("trace" not in hidden, "users do not receive the detect trace")
|
||||
|
||||
reloaded = client.get(f"/api/journal/conversations/{conv}", headers=headers)
|
||||
expect(reloaded.json().get("pending_mask_review", {}).get("id") == review["id"], "pending review survives reload")
|
||||
|
||||
missing = client.post(
|
||||
f"/api/journal/conversations/{conv}/turn/review",
|
||||
headers=headers,
|
||||
json={"review_id": "missing", "decisions": []},
|
||||
)
|
||||
expect(missing.status_code == 404, "unknown review is gone")
|
||||
|
||||
identity = client.post(
|
||||
f"/api/journal/conversations/{conv}/turn/review",
|
||||
headers=headers,
|
||||
json={
|
||||
"review_id": review["id"],
|
||||
"decisions": [{"id": anna["id"], "decision": "identity"}],
|
||||
},
|
||||
)
|
||||
status_ok(identity, "identity review")
|
||||
expect(not identity.json().get("pending_mask_review"), "review continue clears the pause")
|
||||
expect(any(item.get("role") == "assistant" for item in identity.json().get("messages") or []), "continue produces an assistant")
|
||||
confirmed = list_confirmed_identities(profile_id)
|
||||
expect(any((item.get("canonical_label") or "").casefold() == "anna" for item in confirmed), "identity review confirms the registry")
|
||||
sense = get_sense(profile_id, "Anna")
|
||||
expect(sense["identity_hits"] >= 1 and not sense["ambiguous"], "Anna is identity-only")
|
||||
|
||||
second = client.post(
|
||||
f"/api/journal/conversations/{conv}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Anna hat später angerufen."},
|
||||
)
|
||||
status_ok(second, "second Anna turn")
|
||||
expect(not second.json().get("pending_mask_review"), "identity-only spelling skips the popup")
|
||||
|
||||
food_conv = open_conv(client, headers)
|
||||
food = client.post(
|
||||
f"/api/journal/conversations/{food_conv}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Hanna aß mit uns."},
|
||||
)
|
||||
status_ok(food, "hanna pause")
|
||||
food_review = food.json().get("pending_mask_review") or {}
|
||||
food_candidates = food_review.get("candidates") or []
|
||||
hanna = next((item for item in food_candidates if item.get("text") == "Hanna"), None)
|
||||
expect(hanna is not None, "Hanna is a review candidate")
|
||||
declined = client.post(
|
||||
f"/api/journal/conversations/{food_conv}/turn/review",
|
||||
headers=headers,
|
||||
json={
|
||||
"review_id": food_review["id"],
|
||||
"decisions": [{"id": hanna["id"], "decision": "not_identity"}],
|
||||
},
|
||||
)
|
||||
status_ok(declined, "not_identity review")
|
||||
expect(not any((item.get("canonical_label") or "").casefold() == "hanna" for item in list_confirmed_identities(profile_id)), "not_identity does not confirm Hanna")
|
||||
expect(get_sense(profile_id, "Hanna")["non_identity_hits"] >= 1, "not_identity records a sense")
|
||||
hanna_again = client.post(
|
||||
f"/api/journal/conversations/{food_conv}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Hanna aß mit uns."},
|
||||
)
|
||||
status_ok(hanna_again, "second Hanna turn")
|
||||
expect(
|
||||
not hanna_again.json().get("pending_mask_review"),
|
||||
"stored Hanna cue is not re-asked against teacher PERSON",
|
||||
)
|
||||
|
||||
record_sense(profile_id, "Clarissa", identity=True)
|
||||
record_sense(profile_id, "Clarissa", identity=False)
|
||||
expect(get_sense(profile_id, "Clarissa")["ambiguous"], "both senses mark Clarissa as ambiguous")
|
||||
amb_conv = open_conv(client, headers)
|
||||
with patch("detect_learning.try_local_passage_decision", return_value="not_identity"):
|
||||
amb = client.post(
|
||||
f"/api/journal/conversations/{amb_conv}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Clarissa lag auf dem Tisch."},
|
||||
)
|
||||
status_ok(amb, "ambiguous local passage")
|
||||
expect(not amb.json().get("pending_mask_review"), "local passage can finish without a popup")
|
||||
expect(any(item.get("role") == "assistant" for item in amb.json().get("messages") or []), "local passage still generates")
|
||||
|
||||
kin_conv = open_conv(client, headers)
|
||||
kin = client.post(
|
||||
f"/api/journal/conversations/{kin_conv}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Ich war mit meinem Sohn Leon im Park."},
|
||||
)
|
||||
status_ok(kin, "kinship Leon pause")
|
||||
kin_review = kin.json().get("pending_mask_review") or {}
|
||||
kin_names = [item.get("text") for item in kin_review.get("candidates") or []]
|
||||
expect("Leon" in kin_names, "Sohn Leon is offered even when Detect does not report it")
|
||||
leon = next((item for item in kin_review.get("candidates") or [] if item.get("text") == "Leon"), None)
|
||||
expect(leon is not None, "Leon candidate id is present")
|
||||
kin_done = client.post(
|
||||
f"/api/journal/conversations/{kin_conv}/turn/review",
|
||||
headers=headers,
|
||||
json={
|
||||
"review_id": kin_review["id"],
|
||||
"decisions": [{"id": leon["id"], "decision": "identity"}],
|
||||
},
|
||||
)
|
||||
status_ok(kin_done, "Leon identity review")
|
||||
kin_again = client.post(
|
||||
f"/api/journal/conversations/{kin_conv}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Ich war mit meinem Sohn Leon im Park."},
|
||||
)
|
||||
status_ok(kin_again, "second Leon turn")
|
||||
expect(not kin_again.json().get("pending_mask_review"), "confirmed Sohn Leon skips the popup")
|
||||
|
||||
sushi_conv = open_conv(client, headers)
|
||||
sushi = client.post(
|
||||
f"/api/journal/conversations/{sushi_conv}/turn",
|
||||
headers=headers,
|
||||
json={
|
||||
"body": (
|
||||
"Gestern bin ich mit meiner Frau Sushi und meinem Sohn Rohan "
|
||||
"Sushi essen gegangen. Das Restaurant hat mich dabei total beeindruckt."
|
||||
)
|
||||
},
|
||||
)
|
||||
status_ok(sushi, "homonym sentence pause")
|
||||
sushi_names = [item.get("text") for item in (sushi.json().get("pending_mask_review") or {}).get("candidates") or []]
|
||||
expect("Rohan" in sushi_names, "Rohan from Sohn is a candidate")
|
||||
expect("Sushi" in sushi_names, "Sushi from Frau is a candidate")
|
||||
expect("Restaurant" not in sushi_names, "plain Restaurant is not kinship-offered")
|
||||
sushi_hits = [
|
||||
item
|
||||
for item in (sushi.json().get("pending_mask_review") or {}).get("candidates") or []
|
||||
if item.get("text") == "Sushi"
|
||||
]
|
||||
expect(len(sushi_hits) >= 2, "both Sushi mentions are offered")
|
||||
expect(
|
||||
sushi_hits[0].get("highlight_start") != sushi_hits[1].get("highlight_start"),
|
||||
"the two Sushi mentions highlight different offsets",
|
||||
)
|
||||
for item in sushi_hits:
|
||||
excerpt = item.get("excerpt") or ""
|
||||
start = item.get("highlight_start")
|
||||
end = item.get("highlight_end")
|
||||
expect(excerpt[start:end] == "Sushi", "highlight covers the Sushi token")
|
||||
|
||||
homonym = (
|
||||
"Gestern bin ich mit meiner Frau Sushi und meinem Sohn Rohan "
|
||||
"Sushi essen gegangen."
|
||||
)
|
||||
first_at = homonym.index("Sushi")
|
||||
second_at = homonym.rindex("Sushi")
|
||||
first = excerpt_view(homonym, first_at, first_at + 5, "Sushi")
|
||||
second = excerpt_view(homonym, second_at, second_at + 5, "Sushi")
|
||||
expect(first["highlight_start"] < second["highlight_start"], "person Sushi sits left of dish Sushi")
|
||||
expect(first["excerpt"][first["highlight_start"]:first["highlight_end"]] == "Sushi", "first highlight is Sushi")
|
||||
expect(second["excerpt"][second["highlight_start"]:second["highlight_end"]] == "Sushi", "second highlight is Sushi")
|
||||
prefix = "SYSTEM\n"
|
||||
mapped = excerpt_view(
|
||||
homonym,
|
||||
len(prefix) + second_at,
|
||||
len(prefix) + second_at + 5,
|
||||
"Sushi",
|
||||
prefix + homonym,
|
||||
)
|
||||
expect(mapped["highlight_start"] == second["highlight_start"], "rendered detect offsets map to the dish Sushi")
|
||||
expect(cue_for_mention(homonym, first_at, first_at + 5, "Sushi") == "frau", "person Sushi attaches to Frau")
|
||||
expect(cue_for_mention(homonym, second_at, second_at + 5, "Sushi") == "rohan", "dish Sushi stays distinct from Frau")
|
||||
appos = "Sushi, meine Frau war erkältet."
|
||||
appos_at = appos.index("Sushi")
|
||||
expect(cue_for_mention(appos, appos_at, appos_at + 5, "Sushi") == "frau", "apposition Sushi, meine Frau binds Frau")
|
||||
far = "Meine Frau und ich aßen Sushi."
|
||||
far_at = far.rindex("Sushi")
|
||||
expect(cue_for_mention(far, far_at, far_at + 5, "Sushi") != "frau", "Frau elsewhere in the sentence does not bind the dish")
|
||||
|
||||
sushi_review = sushi.json().get("pending_mask_review") or {}
|
||||
sushi_decisions = []
|
||||
for item in sushi_review.get("candidates") or []:
|
||||
label = item.get("text")
|
||||
cue = item.get("cue")
|
||||
if label == "Sushi" and cue != "frau":
|
||||
sushi_decisions.append({"id": item["id"], "decision": "not_identity"})
|
||||
else:
|
||||
sushi_decisions.append({"id": item["id"], "decision": "identity"})
|
||||
sushi_done = client.post(
|
||||
f"/api/journal/conversations/{sushi_conv}/turn/review",
|
||||
headers=headers,
|
||||
json={"review_id": sushi_review["id"], "decisions": sushi_decisions},
|
||||
)
|
||||
status_ok(sushi_done, "homonym review")
|
||||
expect(get_cue_decision(profile_id, "Sushi", "frau") == "identity", "Frau Sushi is stored as identity")
|
||||
expect(get_cue_decision(profile_id, "Sushi", "rohan") == "not_identity", "dish Sushi after Rohan is stored")
|
||||
expect(get_cue_decision(profile_id, "Rohan", "sohn") == "identity", "Sohn Rohan cue is stored")
|
||||
sushi_again = client.post(
|
||||
f"/api/journal/conversations/{sushi_conv}/turn",
|
||||
headers=headers,
|
||||
json={
|
||||
"body": (
|
||||
"Gestern bin ich mit meiner Frau Sushi und meinem Sohn Rohan "
|
||||
"Sushi essen gegangen. Das Restaurant hat mich dabei total beeindruckt."
|
||||
)
|
||||
},
|
||||
)
|
||||
status_ok(sushi_again, "repeat homonym sentence")
|
||||
expect(not sushi_again.json().get("pending_mask_review"), "same reviewed sentence does not re-ask")
|
||||
|
||||
dish = "Ich aß Sushi."
|
||||
dish_at = dish.index("Sushi")
|
||||
food_spans = [{"text": "Sushi", "start": dish_at, "end": dish_at + 5, "label": "food", "score": 0.8}]
|
||||
food_cands, _maps = merge_gliner_candidates(profile_id, [], [], dish, dish, food_spans)
|
||||
expect(
|
||||
any(item.get("text") == "Sushi" and item.get("suggested") == "not_identity" for item in food_cands),
|
||||
"GLiNER food span suggests not_identity",
|
||||
)
|
||||
person_line = "Ich war mit meiner Frau Sushi unterwegs."
|
||||
person_at = person_line.index("Sushi")
|
||||
gliner_miss = [{"text": "Sushi", "start": person_at, "end": person_at + 5, "label": "food", "score": 0.9}]
|
||||
merged, _maps = merge_gliner_candidates(profile_id, [], [], person_line, person_line, gliner_miss)
|
||||
expect(
|
||||
not any(item.get("suggested") == "not_identity" for item in merged),
|
||||
"GLiNER food does not override Frau attachment",
|
||||
)
|
||||
expect(
|
||||
decision_for_mention(profile_id, "Sushi", person_line, person_at, person_at + 5, person_line) == "identity",
|
||||
"learned Frau cue wins over GLiNER food",
|
||||
)
|
||||
dish_line = "Ich aß Sushi."
|
||||
dish_pos = dish_line.index("Sushi")
|
||||
expect(
|
||||
decision_for_mention(profile_id, "Sushi", dish_line, dish_pos, dish_pos + 5, dish_line) == "ask",
|
||||
"Tag-21: identity hits plus new cue stay visible",
|
||||
)
|
||||
person = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "cue", True, "frau")
|
||||
food = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "teacher", False, "aß")
|
||||
expect(same_mention(person, food), "same offsets are the same mention")
|
||||
wrapped = MentionSpan("Frau Sushi", 15, 25, "PERSON", "identity", "teacher", False, "frau")
|
||||
expect(same_mention(person, wrapped), "contained Sushi matches Frau Sushi")
|
||||
expect(map_entity_type("person") == "PERSON", "GLiNER person maps to PERSON")
|
||||
guessed = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "candidate", False, "aß")
|
||||
review, counters = compare_mentions(profile_id, dish_line, [guessed], [food], dish_line)
|
||||
expect(counters["diff_type_conflict"] >= 1, "FOOD versus PERSON is a type conflict")
|
||||
expect(any(item.get("reason") == "type_conflict" for item in review), "unconfirmed type conflict is offered")
|
||||
confirmed_dish = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "cue", True, "rohan")
|
||||
teacher_person = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "teacher", False, "rohan")
|
||||
kept, kept_counters = compare_mentions(
|
||||
profile_id, dish_line, [confirmed_dish], [teacher_person], dish_line
|
||||
)
|
||||
expect(kept_counters["diff_type_conflict"] >= 1, "confirmed cue still counts the type conflict")
|
||||
expect(not kept, "confirmed cue is not re-asked on type conflict")
|
||||
compact = compact_diff_counters(counters)
|
||||
dumped = str(compact)
|
||||
expect("Sushi" not in dumped and "Frau" not in dumped, "compact diff counters carry no plaintext")
|
||||
expect(set(compact) == {
|
||||
"diff_agree",
|
||||
"diff_only_student",
|
||||
"diff_only_teacher",
|
||||
"diff_type_conflict",
|
||||
"diff_new_identity",
|
||||
"diff_cue_reinforced",
|
||||
}, "compact counters are the agreed keys")
|
||||
set_detect_operating_mode("learning_reduced")
|
||||
expect(get_detect_operating_mode() == "learning_reduced", "stage 2 mode persists")
|
||||
set_detect_operating_mode("learning_manual")
|
||||
expect(get_detect_operating_mode() == "learning_manual", "manual mode persists")
|
||||
set_detect_operating_mode("student")
|
||||
pipe = prepare_detect_pipeline(profile_id, "Anna kam vorbei.", "Anna kam vorbei.")
|
||||
expect(pipe.mode == "student", "student path is selectable")
|
||||
expect(not pipe.need_review, "zielpfad does not open a teacher popup")
|
||||
record_sense(profile_id, "Sushi", identity=True)
|
||||
record_cue(profile_id, "Sushi", "frau", "identity", "meine Frau Sushi")
|
||||
expect(
|
||||
decision_for_mention(profile_id, "Sushi", "Ich aß Sushi.", dish_pos, dish_pos + 5) == "ask",
|
||||
"Tag-21 remains ask after more identity hits",
|
||||
)
|
||||
|
||||
fake_passage = ProviderConfig(
|
||||
role="passage",
|
||||
name="ollama",
|
||||
mode="http",
|
||||
url="http://192.168.2.144:11434/v1/chat/completions",
|
||||
model="phi3:mini",
|
||||
key="",
|
||||
local=True,
|
||||
zdr=True,
|
||||
no_train=True,
|
||||
)
|
||||
with patch("detect_learning.passage_provider", return_value=fake_passage):
|
||||
with patch("providers.complete_chat", return_value=ChatResult(content='{"decision":"not_identity"}')) as chat:
|
||||
decided = try_local_passage_decision("Rohan Sushi essen", "Sushi")
|
||||
expect(decided == "not_identity", "passage profile can decide without local Detect")
|
||||
expect(chat.called, "passage uses complete_chat")
|
||||
expect(passage_provider() is None, "no passage profile means no silent detect fallback")
|
||||
|
||||
back = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "semantic"})
|
||||
expect(back.json()["detect_operating_mode"] == "bypass", "semantic alias stores bypass")
|
||||
full = client.put("/api/admin/providers/detect-mode", headers=headers, json={"mode": "learning_full"})
|
||||
expect(full.json()["detect_operating_mode"] == "learning_full", "learning_full is accepted")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Unit tests for student/teacher comparison. No database."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
|
||||
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from detect_learning import canonicalize_detect_mode, decision_for_mention
|
||||
from detect_pipeline import (
|
||||
MentionSpan,
|
||||
compact_diff_counters,
|
||||
compare_mentions,
|
||||
map_entity_type,
|
||||
protects,
|
||||
same_mention,
|
||||
types_equivalent,
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
expect(canonicalize_detect_mode("learning") == "learning_full", "learning aliases to learning_full")
|
||||
expect(canonicalize_detect_mode("semantic") == "bypass", "semantic aliases to bypass")
|
||||
expect(canonicalize_detect_mode("wordlist") is None, "unknown mode is rejected")
|
||||
expect(map_entity_type("person") == "PERSON", "person maps to PERSON")
|
||||
expect(map_entity_type("place") == "PLACE", "place maps to PLACE")
|
||||
expect(protects("PERSON") and not protects("FOOD"), "PERSON protects, FOOD does not")
|
||||
expect(types_equivalent("PERSON", "PLACE"), "similar protectable types are equivalent")
|
||||
expect(not types_equivalent("FOOD", "PERSON"), "FOOD versus PERSON is never equivalent")
|
||||
guessed = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "candidate", False, "aß")
|
||||
teacher_food = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "teacher", False, "aß")
|
||||
offered, offered_counters = compare_mentions("p", "Ich aß Sushi.", [guessed], [teacher_food])
|
||||
expect(offered_counters["diff_type_conflict"] == 1, "unconfirmed FOOD versus PERSON is counted")
|
||||
expect(any(item.get("reason") == "type_conflict" for item in offered), "unconfirmed type conflict is offered")
|
||||
confirmed = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "cue", True, "rohan")
|
||||
teacher_person = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "teacher", False, "rohan")
|
||||
kept, kept_counters = compare_mentions("p", "Rohan Sushi essen", [confirmed], [teacher_person])
|
||||
expect(kept_counters["diff_type_conflict"] == 1, "confirmed cue still counts FOOD versus PERSON")
|
||||
expect(not kept, "confirmed cue is not re-asked on type conflict")
|
||||
inner = MentionSpan("Sushi", 20, 25, "PERSON", "identity", "cue", True, "frau")
|
||||
outer = MentionSpan("Frau Sushi", 15, 25, "PERSON", "identity", "teacher", False, "frau")
|
||||
expect(same_mention(inner, outer), "contained span is the same mention")
|
||||
food = MentionSpan("Sushi", 20, 25, "FOOD", "not_identity", "teacher", False, "aß")
|
||||
expect(same_mention(inner, food), "same offsets stay one mention even when types differ")
|
||||
compact = compact_diff_counters({"diff_agree": 2, "diff_type_conflict": 1, "label": "Sushi"})
|
||||
expect(compact["diff_agree"] == 2 and compact["diff_type_conflict"] == 1, "known counters are kept")
|
||||
expect("label" not in compact and "Sushi" not in str(compact), "compact counters drop plaintext")
|
||||
with patch("detect_learning.get_cue_decision", return_value=None), patch(
|
||||
"detect_learning.get_sense",
|
||||
return_value={"identity_hits": 3, "non_identity_hits": 0, "ambiguous": False},
|
||||
), patch("detect_learning.cue_for_mention", return_value="aß"):
|
||||
expect(
|
||||
decision_for_mention("p", "Sushi", "Ich aß Sushi.", 7, 12) == "ask",
|
||||
"Tag-21: identity-only sense does not lock a new cue",
|
||||
)
|
||||
with patch("detect_learning.get_cue_decision", return_value="identity"), patch(
|
||||
"detect_learning.cue_for_mention", return_value="frau"
|
||||
):
|
||||
expect(
|
||||
decision_for_mention("p", "Sushi", "meine Frau Sushi", 11, 16) == "identity",
|
||||
"stored cue remains binding",
|
||||
)
|
||||
print("detect pipeline unit ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -9,8 +9,6 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-dialogue-test.sqlite")
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from main import app
|
||||
|
||||
+109
-5
@@ -10,8 +10,8 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-frame-test.sqlite")
|
||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
||||
os.environ["KANSHO_PROVIDER_KEY"] = ""
|
||||
os.environ["KANSHO_DETECT_PROVIDER_KEY"] = ""
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from main import app
|
||||
@@ -29,7 +29,8 @@ def auth_header(token: str) -> dict:
|
||||
|
||||
def main() -> None:
|
||||
seed_prompts = json.loads((ROOT / "config" / "prompts.seed.json").read_text(encoding="utf-8"))
|
||||
expect(seed_prompts == [], "prompt seed is empty — no domain prompts in git")
|
||||
slugs = {item["slug"] for item in seed_prompts}
|
||||
expect(slugs == {"mvp.dialogue_turn", "mvp.journal_generate", "mvp.journal_reconstruct", "mvp.entity_detect", "mvp.profile_review"}, "mvp prompts seeded")
|
||||
|
||||
with TestClient(app) as client:
|
||||
health = client.get("/api/health")
|
||||
@@ -75,7 +76,88 @@ def main() -> None:
|
||||
|
||||
admin = client.get("/api/admin/health", headers=headers)
|
||||
expect(admin.status_code == 200, "admin health")
|
||||
expect(admin.json()["inventory"]["prompts"] == 0, "no seeded prompts")
|
||||
expect(admin.json()["inventory"]["prompts"] == 5, "five seeded prompts")
|
||||
expect(admin.json()["inventory"]["debug_persist_enabled"] is False, "debug persist off by default")
|
||||
expect(admin.json()["inventory"]["debug_runs"] == 0, "no debug runs at start")
|
||||
|
||||
debug = client.get("/api/admin/debug", headers=headers)
|
||||
expect(debug.status_code == 200 and debug.json()["persist_enabled"] is False, "debug settings default off")
|
||||
|
||||
providers = client.get("/api/admin/providers", headers=headers)
|
||||
expect(providers.status_code == 200, "admin providers")
|
||||
roles = {item["role"]: item for item in providers.json()["roles"]}
|
||||
expect(set(roles) == {"generate", "detect"}, "two provider roles")
|
||||
expect(providers.json()["remote_plaintext_allowed"] is True, "test runtime allows remote detect")
|
||||
expect(providers.json()["remote_plaintext_reason"] == "non_production", "tests are not production")
|
||||
expect(providers.json().get("detect_operating_mode") == "learning_full", "detect mode defaults to learning stage 1")
|
||||
expect(roles["generate"]["ready"] is False, "generate fail-closed without key")
|
||||
expect("sk-" not in providers.text, "provider status has no secret")
|
||||
expect("KANSHO_PROVIDER_KEY=" not in providers.text, "env assignment not leaked")
|
||||
|
||||
saved = client.put(
|
||||
"/api/admin/providers/generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"name": "openrouter",
|
||||
"url": "https://openrouter.ai/api/v1/chat/completions",
|
||||
"model": "openai/gpt-4o",
|
||||
"zdr": True,
|
||||
"no_train": True,
|
||||
},
|
||||
)
|
||||
expect(saved.status_code == 200, f"save generate settings {saved.text}")
|
||||
saved_roles = {item["role"]: item for item in saved.json()["roles"]}
|
||||
expect(saved_roles["generate"]["model"] == "openai/gpt-4o", "generate model persisted")
|
||||
expect("profiles" in saved.json(), "llm profiles are listed")
|
||||
titles = {item["title"] for item in saved.json()["profiles"]}
|
||||
expect("Ollama (LAN)" in titles, "LAN Ollama profile is seeded")
|
||||
ollama = next(item for item in saved.json()["profiles"] if item["title"] == "Ollama (LAN)")
|
||||
expect(ollama["local"] is True, "LAN Ollama is a local trusted endpoint")
|
||||
expect(ollama["url"].startswith("http://192.168.2.144:11434"), "Ollama URL is the agreed LAN host")
|
||||
|
||||
switched = client.put(
|
||||
"/api/admin/llm-profiles/" + ollama["id"],
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Ollama (LAN)",
|
||||
"name": "ollama",
|
||||
"url": ollama["url"],
|
||||
"model": "llama3.1:latest",
|
||||
"zdr": True,
|
||||
"no_train": True,
|
||||
},
|
||||
)
|
||||
expect(switched.status_code == 200, f"save ollama model {switched.text}")
|
||||
activated = client.post(
|
||||
"/api/admin/providers/detect/activate",
|
||||
headers=headers,
|
||||
json={"profile_id": ollama["id"]},
|
||||
)
|
||||
expect(activated.status_code == 200, f"activate ollama detect {activated.text}")
|
||||
detect_role = {item["role"]: item for item in activated.json()["roles"]}["detect"]
|
||||
expect(detect_role["local"] is True, "detect uses local ollama after profile switch")
|
||||
expect(detect_role["model"] == "llama3.1:latest", "detect keeps the saved ollama model")
|
||||
generate_role = {item["role"]: item for item in activated.json()["roles"]}["generate"]
|
||||
expect(generate_role["model"] == "openai/gpt-4o", "generate profile stays on its own model")
|
||||
|
||||
unknown = client.put(
|
||||
"/api/admin/providers/training",
|
||||
headers=headers,
|
||||
json={"url": "https://example.test", "model": "x"},
|
||||
)
|
||||
expect(unknown.status_code == 404, "unknown provider role")
|
||||
|
||||
os.environ["KANSHO_PROVIDER_KEY"] = "sk-test-not-a-real-key"
|
||||
try:
|
||||
from providers import generate_provider, detect_provider
|
||||
|
||||
expect(generate_provider() is not None, "generate ready after key")
|
||||
expect(detect_provider() is not None, "detect reuses generate key")
|
||||
ready = client.get("/api/admin/providers", headers=headers)
|
||||
expect("sk-test-not-a-real-key" not in ready.text, "secret never in status")
|
||||
expect(ready.json()["roles"][0]["ready"] is True, "status ready after key")
|
||||
finally:
|
||||
os.environ.pop("KANSHO_PROVIDER_KEY", None)
|
||||
|
||||
missing = client.post(
|
||||
"/api/prompts/execute",
|
||||
@@ -163,7 +245,29 @@ def main() -> None:
|
||||
placeholders = client.get("/api/placeholders", headers=user_headers)
|
||||
expect(placeholders.status_code == 200, "placeholder catalog")
|
||||
keys = {item["key"] for item in placeholders.json()["context"]}
|
||||
expect(keys == {"now"}, "only system context key now")
|
||||
expect(
|
||||
keys
|
||||
== {
|
||||
"now",
|
||||
"dialogue_context",
|
||||
"existing_text",
|
||||
"known_labels",
|
||||
"register_hint",
|
||||
"opening_hint",
|
||||
"reconstruction",
|
||||
"review_package",
|
||||
"source_text",
|
||||
"space_title",
|
||||
"writing_profile",
|
||||
"style_examples",
|
||||
"transformation_instructions",
|
||||
"detail_instructions",
|
||||
"voice_instructions",
|
||||
"narrative_instructions",
|
||||
"interaction_hint",
|
||||
},
|
||||
"system and mvp context keys",
|
||||
)
|
||||
|
||||
last_admin = client.patch(
|
||||
f"/api/users/{me.json()['id']}",
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""GLiNER client: local URL only, fail-soft, no request text in logs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
|
||||
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
|
||||
|
||||
from gliner_client import DEFAULT_URL, extract_entities, health, normalize_base_url
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status_code: int, payload):
|
||||
self.status_code = status_code
|
||||
self._payload = payload
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
def main() -> None:
|
||||
expect(normalize_base_url("http://192.168.2.144:8090/") == DEFAULT_URL, "default GLiNER URL is normalized")
|
||||
expect(extract_entities("Anna", url="https://openrouter.ai/api/v1") is None, "OpenRouter URL is rejected")
|
||||
expect(health("https://openrouter.ai").get("reason") == "not_local", "health rejects non-local URL")
|
||||
|
||||
with patch("gliner_client.httpx.post", return_value=_Resp(200, {"spans": [{"text": "Anna", "start": 0, "end": 4, "label": "person", "score": 0.9}]})):
|
||||
spans = extract_entities("Anna kam.", url="http://192.168.2.144:8090")
|
||||
expect(spans == [{"text": "Anna", "start": 0, "end": 4, "label": "person", "score": 0.9}], "extract returns spans")
|
||||
|
||||
with patch("gliner_client.httpx.post", side_effect=httpx.ConnectError("down")):
|
||||
expect(extract_entities("Anna kam.", url="http://192.168.2.144:8090") is None, "extract is fail-soft on errors")
|
||||
|
||||
with patch("gliner_client.httpx.get", return_value=_Resp(200, {"status": "ok", "model": "urchade/gliner_multi-v2.1", "offline": True})):
|
||||
ping = health("http://192.168.2.144:8090")
|
||||
expect(ping.get("ok") is True and ping.get("offline") is True, "health reports ready offline model")
|
||||
print("gliner client: OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Admin identity registry stays local. Run from backend/: python tests/test_identity_registry.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from identity_store import STATUS_LEGACY, list_confirmed_identities, list_registry
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
reset_debug()
|
||||
import sqlite3
|
||||
from db import _IDENTITY_MAPPING_COLUMNS, _ensure_columns
|
||||
from identity_store import migrate_legacy_identity_rows as migrate_legacy
|
||||
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE identity_mappings (
|
||||
id TEXT PRIMARY KEY,
|
||||
profile_id TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
local_label TEXT NOT NULL DEFAULT '',
|
||||
created TEXT NOT NULL DEFAULT '2026-08-01 00:00:00'
|
||||
);
|
||||
INSERT INTO identity_mappings (id, profile_id, token, local_label)
|
||||
VALUES ('m1', 'p1', 'PERSON:01', 'Altbau');
|
||||
"""
|
||||
)
|
||||
_ensure_columns(conn, "identity_mappings", _IDENTITY_MAPPING_COLUMNS)
|
||||
migrate_legacy(conn)
|
||||
legacy = dict(conn.execute("SELECT * FROM identity_mappings WHERE id = 'm1'").fetchone())
|
||||
expect(legacy["status"] == STATUS_LEGACY, "existing rows become legacy_review_required")
|
||||
expect(legacy["origin"] == "legacy_auto", "existing rows keep legacy origin")
|
||||
expect(legacy["canonical_label"] == "Altbau", "canonical_label is copied from local_label")
|
||||
expect(bool(legacy["updated"]), "updated is populated after ALTER")
|
||||
conn.close()
|
||||
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
headers = {"X-Auth-Token": setup.json()["token"]}
|
||||
profile_id = setup.json()["profile_id"]
|
||||
|
||||
from db import get_db
|
||||
import uuid
|
||||
|
||||
legacy_id = str(uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO identity_mappings
|
||||
(id, profile_id, token, local_label, canonical_label, entity_type, status, origin)
|
||||
VALUES (?, ?, ?, ?, ?, 'PERSON', ?, 'legacy_auto')
|
||||
""",
|
||||
(legacy_id, profile_id, "PERSON:70", "Altbau", "Altbau", STATUS_LEGACY),
|
||||
)
|
||||
expect(
|
||||
not any(item["id"] == legacy_id for item in list_confirmed_identities(profile_id)),
|
||||
"legacy row is not confirmed",
|
||||
)
|
||||
expect(any(item["id"] == legacy_id for item in list_registry(profile_id)), "legacy row remains reviewable")
|
||||
|
||||
patched = client.patch(
|
||||
f"/api/admin/identities/{legacy_id}",
|
||||
headers=headers,
|
||||
json={"status": "confirmed", "entity_type": "PLACE", "canonical_label": "Altbau"},
|
||||
)
|
||||
expect(patched.status_code == 200, patched.text)
|
||||
expect(patched.json()["status"] == "confirmed", "admin can confirm a legacy row")
|
||||
expect(patched.json()["entity_type"] == "PLACE", "admin can correct the type")
|
||||
|
||||
created = client.post(
|
||||
"/api/admin/identities",
|
||||
headers=headers,
|
||||
json={"canonical_label": "Nordwerk", "entity_type": "ORG", "aliases": ["Nordwerk AG"]},
|
||||
)
|
||||
expect(created.status_code == 200, created.text)
|
||||
identity_id = created.json()["id"]
|
||||
aliases = client.patch(
|
||||
f"/api/admin/identities/{identity_id}",
|
||||
headers=headers,
|
||||
json={"aliases": ["Nordwerk AG", "Nordwerk"]},
|
||||
)
|
||||
expect(aliases.status_code == 200, aliases.text)
|
||||
expect("Nordwerk AG" in (aliases.json().get("aliases") or []), "confirmed alias can be stored")
|
||||
|
||||
calls = {"n": 0}
|
||||
|
||||
def boom(*_args, **_kwargs):
|
||||
calls["n"] += 1
|
||||
raise AssertionError("no egress")
|
||||
|
||||
with patch("entity_detect.complete_chat", boom), patch("providers.complete_chat", boom):
|
||||
gone = client.delete(f"/api/admin/identities/{identity_id}", headers=headers)
|
||||
expect(gone.status_code == 200, "admin delete stays local")
|
||||
expect(calls["n"] == 0, "registry actions do not call detect or generate")
|
||||
expect(not any(item["id"] == identity_id for item in list_registry(profile_id)), "deleted row is gone")
|
||||
|
||||
print("All identity registry tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Journal body as Markdown. Run from backend/: python tests/test_journal_body.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from journal_body import media_ids, media_token, plain_text, to_markdown
|
||||
|
||||
MEDIA_A = "11111111-1111-4111-8111-111111111111"
|
||||
MEDIA_B = "22222222-2222-4222-8222-222222222222"
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
raw = (
|
||||
"Heute war der Markt voll.\n\n"
|
||||
"- Kirschen\n"
|
||||
"- Brot\n\n"
|
||||
f"\n\n"
|
||||
"Später der **Hafen**."
|
||||
)
|
||||
expect(media_ids(raw) == [MEDIA_A], "markdown media ids")
|
||||
text = plain_text(raw)
|
||||
expect(MEDIA_A not in text, "plain text drops asset id")
|
||||
expect("kansho-media" not in text, "plain text drops media url")
|
||||
expect("Stand am Eingang" in text and "Kirschen" in text and "Hafen" in text, "wording kept")
|
||||
expect("**" not in text, "markup stripped")
|
||||
|
||||
legacy = f"Anfang\n\n[[media:{MEDIA_A}|Stand]]\n\nMitte\n\n[[media:{MEDIA_B}]]"
|
||||
converted = to_markdown(legacy)
|
||||
expect(media_token(MEDIA_A, "Stand") in converted, "legacy becomes markdown")
|
||||
expect(media_ids(legacy) == [MEDIA_A, MEDIA_B], "legacy ids")
|
||||
expect(MEDIA_A not in plain_text(legacy), "legacy ids stripped in plain text")
|
||||
expect(plain_text("nur text") == "nur text", "plain body unchanged")
|
||||
expect(media_token(MEDIA_A, "Kai") == f"", "token shape")
|
||||
from journal_body import clean_title
|
||||
|
||||
expect(clean_title("# ein langer Tag") == "ein langer Tag", "markdown heading stripped from title")
|
||||
expect(clean_title("## Morgen\nRest") == "Morgen", "first line only")
|
||||
print("journal body ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,584 @@
|
||||
"""Deterministic journal editorial contract. Fake provider proves data flow, not live prose."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from db import get_db, init_db
|
||||
from engine import load_active_prompt
|
||||
from identity_store import remember_mapping
|
||||
from journal_editorial import (
|
||||
GENERATE_SEED_REVISION,
|
||||
format_style_examples,
|
||||
incomplete_syntax_markers,
|
||||
lexical_similarity,
|
||||
narration_sources_text,
|
||||
select_journal_style_examples,
|
||||
)
|
||||
from journal_generate import pack_narration_context, unattested_journal_content
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
from prompt_budget import JournalBudget
|
||||
from providers import ChatResult
|
||||
from writing_profile_store import (
|
||||
NEUTRAL_JOURNAL_STYLE,
|
||||
compile_task_brief,
|
||||
import_text,
|
||||
set_lifecycle,
|
||||
update_facet,
|
||||
)
|
||||
|
||||
|
||||
SEED_REVISION = GENERATE_SEED_REVISION
|
||||
MIXED_SOURCES_INSTRUCTION = (
|
||||
"Die Quellen können aus Fließtext, Stichpunkten und Satzfragmenten bestehen. "
|
||||
"Behandle jede Passage entsprechend ihrer Form: Redigiere vorhandenen Fließtext, "
|
||||
"verwandle Stichpunkte und Fragmente in vollständige Prosa und verbinde beides zu einem einheitlichen Eintrag. "
|
||||
"Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus."
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def test_syntax_diagnostics() -> None:
|
||||
expect(incomplete_syntax_markers("Danach sprach ich mit dem.") > 0, "dangling determiner is diagnostic")
|
||||
expect(incomplete_syntax_markers("Danach sprach ich und kam zurück.") == 0, "complete sentence is not flagged")
|
||||
|
||||
|
||||
def test_prompt_contract() -> None:
|
||||
init_db()
|
||||
prompt = load_active_prompt("mvp.journal_generate")
|
||||
text = prompt.get("template") or ""
|
||||
expect("INHALTSTREUE" in text, "prompt keeps hard content rules")
|
||||
expect("CURRENT_DAY_SOURCES" in text, "prompt labels current-day facts")
|
||||
expect("WRITING_PROFILE" in text, "prompt labels the writing profile")
|
||||
expect("{{transformation_instructions}}" in text, "prompt injects compiled transformation instructions")
|
||||
expect("{{detail_instructions}}" in text, "prompt injects compiled detail instructions")
|
||||
expect("{{voice_instructions}}" in text, "prompt injects compiled voice instructions")
|
||||
expect("{{narrative_instructions}}" in text, "prompt injects compiled narrative instructions")
|
||||
expect("{{source_mode_instructions}}" not in text, "prompt has no source-mode placeholder")
|
||||
expect(MIXED_SOURCES_INSTRUCTION in text, "prompt contains the mixed-source instruction")
|
||||
expect("{{writing_profile}}" in text, "prompt injects writing profile")
|
||||
expect("{{style_examples}}" in text, "prompt injects style examples")
|
||||
expect("{{reconstruction}}" in text, "prompt injects current-day sources")
|
||||
expect("{{existing_text}}" in text, "prompt injects existing text")
|
||||
expect("[[…" not in text and "[[..." not in text, "prompt must not teach ellipsis placeholders")
|
||||
expect("ich gieng zum laden" not in text, "synthetic prose_edit example is gone")
|
||||
expect("nachbarhund im garten" not in text, "synthetic notes_to_journal example is gone")
|
||||
expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone")
|
||||
expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone")
|
||||
expect("EDITORIAL_MODE" not in text, "source mode is no longer a labeled user setting")
|
||||
expect("Faktentreue ist nicht Wortlauttreue" not in text, "old redundant fidelity lecture is gone")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect(row["seed_revision"] == SEED_REVISION, "system prompt revision is stored")
|
||||
expect(row["template"] == row["default_template"], "untouched install uses the seeded template")
|
||||
|
||||
|
||||
def test_sources_and_examples_are_separated() -> None:
|
||||
artifact = {
|
||||
"kind": "verified_artifact",
|
||||
"coverage": "all_selected_sources",
|
||||
"sources": [{"source_id": "u1", "role": "user", "text": "Heute nur der Markt."}],
|
||||
}
|
||||
presented = narration_sources_text(artifact)
|
||||
expect("[u1]" in presented and "Heute nur der Markt." in presented, "day facts are labeled source blocks")
|
||||
expect("kind" not in presented, "narration does not dump artifact JSON")
|
||||
formatted = format_style_examples(
|
||||
[{"kind": "journal_entry", "excerpt": "Ruhige Sätze, trockener Schnitt.", "occurred_at": "2026-07-01"}]
|
||||
)
|
||||
expect("nicht übernommen" in formatted, "style examples forbid copying their facts")
|
||||
expect("Ruhige Sätze" in formatted, "selected example text is present")
|
||||
|
||||
|
||||
def test_unattested_covers_title_and_body() -> None:
|
||||
mappings = [{"local_label": "Maren", "token": "PERSON:01"}, {"local_label": "Hanna", "token": "PERSON:99"}]
|
||||
sources = ["Heute war ich mit Maren am Markt."]
|
||||
expect(
|
||||
unattested_journal_content("Markttag\n\nIch ging mit Maren zum Markt.", sources, mappings, ["PERSON:01"]) is None,
|
||||
"attested person in title and body is allowed",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Hanna am Hafen\n\nIch ging zum Markt.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_identity",
|
||||
"unattested person in the title is rejected",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("[[PERSON:99]]\n\nIch ging zum Markt.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_placeholder",
|
||||
"inactive placeholder in the title is unattested",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("[[PERSON:01]] am Markt\n\n[[PERSON:01]] kaufte Kirschen.", sources, mappings, ["PERSON:01"])
|
||||
is None,
|
||||
"the same attested placeholder may appear in title and body",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Um 6:30 Uhr kam [[...]] ins Wohnzimmer.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_placeholder",
|
||||
"ellipsis placeholder leftover is unattested",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Um 6:30 Uhr kam [[…]] ins Wohnzimmer.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_placeholder",
|
||||
"unicode ellipsis placeholder leftover is unattested",
|
||||
)
|
||||
|
||||
|
||||
def test_budget_pack_drops_examples_first() -> None:
|
||||
init_db()
|
||||
prompt = {
|
||||
"id": "eval-pack",
|
||||
"slug": "mvp.journal_generate",
|
||||
"prompt_type": "base",
|
||||
"template": "PROFILE\n{{writing_profile}}\nEX\n{{style_examples}}\nDAY\n{{reconstruction}}\nOLD\n{{existing_text}}\n",
|
||||
}
|
||||
assembled = {
|
||||
"writing_profile": "Core: kurze Sätze.",
|
||||
"reconstruction": "Heute Markt.",
|
||||
"style_examples": "",
|
||||
"existing_text": "",
|
||||
}
|
||||
huge_examples = "Stil " + ("Beispielwort " * 400)
|
||||
existing = "Bestehende Fassung " + ("alt " * 40)
|
||||
budget = JournalBudget(
|
||||
model="fake",
|
||||
purpose="journal_generate",
|
||||
effective_context_window=32_768,
|
||||
reserved_output_tokens=256,
|
||||
safety_margin=0.15,
|
||||
available_input_tokens=estimate_cap(assembled, prompt, extra=80),
|
||||
chars_per_token=2.0,
|
||||
)
|
||||
packed, dropped = pack_narration_context(
|
||||
prompt,
|
||||
budget,
|
||||
assembled,
|
||||
style_examples=huge_examples,
|
||||
existing_text=existing,
|
||||
include_existing=True,
|
||||
)
|
||||
expect("style_examples" in dropped, "style examples are dropped before day sources")
|
||||
expect("Heute Markt." in packed["reconstruction"], "day sources stay")
|
||||
expect("kurze Sätze" in packed["writing_profile"], "writing profile stays")
|
||||
|
||||
|
||||
def estimate_cap(assembled: dict, prompt: dict, extra: int) -> int:
|
||||
from engine import preview_prompt
|
||||
from prompt_budget import estimate_tokens
|
||||
|
||||
base = dict(assembled)
|
||||
rendered = preview_prompt(prompt, base)["rendered"]
|
||||
return estimate_tokens(rendered) + extra
|
||||
|
||||
|
||||
def intern_of(payload: dict) -> str:
|
||||
for stage in (payload.get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
return stage.get("intern") or ""
|
||||
return (payload.get("trace") or {}).get("intern") or ""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_syntax_diagnostics()
|
||||
test_prompt_contract()
|
||||
test_sources_and_examples_are_separated()
|
||||
test_unattested_covers_title_and_body()
|
||||
test_budget_pack_drops_examples_first()
|
||||
expect(lexical_similarity("a b c", "a b c") > 0.9, "similarity helper is diagnostic")
|
||||
expect(NEUTRAL_JOURNAL_STYLE.startswith("Neutraler Journalstil"), "neutral fallback exists")
|
||||
|
||||
reset_debug()
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "ada@example.test", "name": "Ada", "password": "test-pass"},
|
||||
)
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
expect(compile_task_brief(profile_id) == NEUTRAL_JOURNAL_STYLE, "unconfirmed profile is not a style authority")
|
||||
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Editorial"})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-26"},
|
||||
)
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Tag"},
|
||||
)
|
||||
prose_body = (
|
||||
"ich bin dan zum markt gegangen und da war es zimlich voll. "
|
||||
"vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. "
|
||||
"die rote tasche lag im auto. ich bin dan zum markt gegangen."
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": prose_body},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn {turn.status_code}")
|
||||
gen = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"generate {gen.status_code}")
|
||||
intern = intern_of(gen.json())
|
||||
expect(MIXED_SOURCES_INSTRUCTION in intern, "unified mixed-source instruction reaches the model")
|
||||
expect("CURRENT_DAY_SOURCES" in intern, "day facts are labeled")
|
||||
expect("\nSTYLE_EXAMPLES\n" not in intern, "default voice does not attach a style-example section")
|
||||
expect("STYLE_EXAMPLES" not in intern, "default voice has no orphaned STYLE_EXAMPLES reference")
|
||||
expect("WRITING_PROFILE" in intern, "writing profile heading or instruction remains")
|
||||
expect("Überarbeite den Text substanziell" in intern, "default transformation policy reaches the prompt")
|
||||
expect("Erhalte sämtliche belegten Ereignisse" in intern, "default detail policy reaches the prompt")
|
||||
expect("Neutraler Journalstil" in intern, "without a confirmed profile the neutral fallback is used")
|
||||
expect("zimlich" in intern.split("\nCURRENT_DAY_SOURCES\n")[-1], "today remains content, including typos")
|
||||
expect("zimlich" not in intern.split("\nCURRENT_DAY_SOURCES\n")[0], "today is not a style authority")
|
||||
expect(sum(1 for item in (gen.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "normal path is one generate call")
|
||||
expect("editorial_mode" not in (gen.json().get("trace") or {}), "trace has no abandoned editorial mode")
|
||||
expect((gen.json().get("trace") or {}).get("narration_source") == "model", "accepted model path is recorded")
|
||||
expect((gen.json().get("trace") or {}).get("prompt_revision") == SEED_REVISION, "prompt revision is in the admin trace")
|
||||
expect((gen.json().get("trace") or {}).get("writing_profile", {}).get("neutral_fallback") is True, "unconfirmed profile is visible as fallback")
|
||||
expect((gen.json().get("trace") or {}).get("style_examples", {}).get("count") == 0, "no historical examples yet")
|
||||
expect((gen.json().get("trace") or {}).get("dropped_optional_blocks") == [], "nothing dropped on a small day")
|
||||
|
||||
notes_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-25"},
|
||||
)
|
||||
notes_conv = client.post(
|
||||
f"/api/journal/days/{notes_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Notizen"},
|
||||
)
|
||||
client.post(
|
||||
f"/api/journal/conversations/{notes_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "markt\nkirschen\nspäter hafen"},
|
||||
)
|
||||
notes_gen = client.post(
|
||||
f"/api/journal/days/{notes_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [notes_conv.json()["id"]]},
|
||||
)
|
||||
expect(notes_gen.status_code == 200, f"notes generate {notes_gen.text}")
|
||||
notes_intern = intern_of(notes_gen.json())
|
||||
expect(MIXED_SOURCES_INSTRUCTION in notes_intern, "notes use the same mixed-source instruction")
|
||||
expect("markt" in notes_intern.lower() and "kirschen" in notes_intern.lower(), "notes sources remain complete")
|
||||
|
||||
import_text(
|
||||
profile_id,
|
||||
"Ich schreibe in kurzen, ruhigen Sätzen und lasse den Tag stehen. Der Hafen blieb hinter der Fähre.",
|
||||
occurred_at="2026-07-01",
|
||||
)
|
||||
examples = select_journal_style_examples(profile_id, exclude_dates=["2026-08-26"])
|
||||
expect(examples and "ruhigen Sätzen" in examples[0]["excerpt"], "imported text is a style example when no finals exist")
|
||||
expect(all("zimlich" not in (item.get("excerpt") or "") for item in examples), "current day is not selected as style")
|
||||
|
||||
saved = client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={
|
||||
"journal_day_id": notes_day.json()["day"]["id"],
|
||||
"title": "Eigene Fassung",
|
||||
"body": "Heute blieb ich beim klaren Schnitt und schrieb den Markt in eigenen Worten, ohne Pathos.",
|
||||
"origin": "user_edit",
|
||||
"source_conversation_ids": [notes_conv.json()["id"]],
|
||||
},
|
||||
)
|
||||
expect(saved.status_code == 200, f"save {saved.text}")
|
||||
ranked = select_journal_style_examples(profile_id, exclude_dates=["2026-08-26"])
|
||||
expect(ranked and ranked[0]["kind"] == "journal_entry", "final journal text outranks imports")
|
||||
expect("klaren Schnitt" in ranked[0]["excerpt"], "user-edited journal text is the preferred example")
|
||||
expect(len(ranked) <= 2, "at most two style examples")
|
||||
|
||||
update_facet(profile_id, "core", value="Kurze Sätze, trockener Schnitt, keine Pathoswolken.")
|
||||
expect("Core:" not in compile_task_brief(profile_id), "unconfirmed core is not a style authority")
|
||||
set_lifecycle(profile_id, "confirmed")
|
||||
with_core = compile_task_brief(profile_id)
|
||||
expect("Core:" in with_core and "trockener Schnitt" in with_core, "confirmed core reaches the task brief")
|
||||
expect("Neutraler Journalstil" not in with_core, "confirmed profile replaces the neutral fallback")
|
||||
|
||||
remember_mapping(profile_id, "Hanna", "PERSON:99")
|
||||
|
||||
def rewritten(_messages, _policy):
|
||||
return ChatResult(
|
||||
content=(
|
||||
"Markttag\n\n"
|
||||
"Ich ging dann zum Markt, auf dem es ziemlich voll war. "
|
||||
"Vielleicht bleibe ich kürzer. "
|
||||
"Brot holen wollte ich noch, habe es aber nicht gemacht. "
|
||||
"Die rote Tasche lag im Auto."
|
||||
),
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", rewritten):
|
||||
second = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(second.status_code == 200, f"rewritten generate {second.text}")
|
||||
body = second.json().get("body") or ""
|
||||
expect("ziemlich voll" in body, "spelling is corrected")
|
||||
expect("zimlich" not in body, "source typo is not kept")
|
||||
expect("Ich ging dann zum Markt" in body, "good rewritten wording may stay")
|
||||
expect("Vielleicht bleibe ich kürzer" in body, "uncertainty stays uncertainty")
|
||||
expect("wollte" in body.lower() and "nicht gemacht" in body, "plan is not turned into completion")
|
||||
expect("nicht gemacht" in body, "negation is kept")
|
||||
expect("rote Tasche" in body, "unique attested detail is kept")
|
||||
expect(body.lower().count("markt") <= 2, "repetition is allowed to be reduced")
|
||||
expect("traurig" not in body.lower() and "weil" not in body.lower(), "no invented feeling or cause in the patched rewrite")
|
||||
intern2 = intern_of(second.json())
|
||||
expect("trockener Schnitt" in intern2.split("\nCURRENT_DAY_SOURCES\n")[0], "confirmed writing profile reaches generate")
|
||||
expect("Hafen blieb hinter der Fähre" not in body, "historical style facts are not copied into today")
|
||||
expect(
|
||||
sum(1 for item in (second.json().get("run_log") or []) if item.get("kind") == "model_call") == 1,
|
||||
"high similarity does not trigger a retry",
|
||||
)
|
||||
expect((second.json().get("trace") or {}).get("lexical_similarity") is not None, "similarity is a diagnostic only")
|
||||
|
||||
def notes_prose(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Notizen\n\nAm Markt holte ich Kirschen. Später war ich am Hafen.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", notes_prose):
|
||||
notes_out = client.post(
|
||||
f"/api/journal/days/{notes_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [notes_conv.json()["id"]]},
|
||||
)
|
||||
notes_body = notes_out.json().get("body") or ""
|
||||
expect("Am Markt holte ich Kirschen" in notes_body, "notes become connected prose")
|
||||
expect("kirschen\nspäter" not in notes_body.lower(), "notes are not concatenated as fragments")
|
||||
|
||||
broken_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-24"},
|
||||
)
|
||||
broken_conv = client.post(
|
||||
f"/api/journal/days/{broken_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Bruch"},
|
||||
)
|
||||
client.post(
|
||||
f"/api/journal/conversations/{broken_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "ich gieng zum laden. danach sprach ich mit dem und kam zurück. es war kald."},
|
||||
)
|
||||
|
||||
def rebuilt(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Ladengang\n\nIch ging zum Laden. Danach sprach ich und kam zurück. Es war kalt.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", rebuilt):
|
||||
broken_out = client.post(
|
||||
f"/api/journal/days/{broken_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [broken_conv.json()["id"]]},
|
||||
)
|
||||
broken_body = broken_out.json().get("body") or ""
|
||||
expect("gieng" not in broken_body.lower(), "general spelling error is not kept")
|
||||
expect("kald" not in broken_body.lower(), "second general spelling error is not kept")
|
||||
expect("mit dem" not in broken_body.lower(), "dangling determiner is not kept")
|
||||
expect("kam zurück" in broken_body, "attested continuation stays")
|
||||
expect(incomplete_syntax_markers(broken_body) == 0, "patched rewrite has no incomplete syntax")
|
||||
expect((broken_out.json().get("trace") or {}).get("incomplete_syntax") == 0, "incomplete syntax is a diagnostic")
|
||||
|
||||
weight_day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-23"},
|
||||
)
|
||||
weight_conv = client.post(
|
||||
f"/api/journal/days/{weight_day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Notizen"},
|
||||
)
|
||||
client.post(
|
||||
f"/api/journal/conversations/{weight_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "morgens tee\nspäter markt\ngegen abend unerwartet der nachbarhund im garten"},
|
||||
)
|
||||
|
||||
def weighted(_messages, _policy):
|
||||
return ChatResult(
|
||||
content=(
|
||||
"Nachbarhund\n\n"
|
||||
"Morgens trank ich Tee, später war ich am Markt. "
|
||||
"Es war ein gewöhnlicher Tag – bis gegen Abend unerwartet der Nachbarhund im Garten war."
|
||||
),
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", weighted):
|
||||
weight_out = client.post(
|
||||
f"/api/journal/days/{weight_day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [weight_conv.json()["id"]]},
|
||||
)
|
||||
weight_intern = intern_of(weight_out.json())
|
||||
expect(MIXED_SOURCES_INSTRUCTION in weight_intern, "fragment notes still use the unified instruction")
|
||||
weight_body = weight_out.json().get("body") or ""
|
||||
expect("bis gegen Abend" in weight_body or "unerwartet" in weight_body, "attested standout may be weighted")
|
||||
expect("morgens tee\nspäter markt" not in weight_body.lower(), "notes are not a concatenated list")
|
||||
expect("glücklich" not in weight_body.lower() and "weil" not in weight_body.lower(), "weighting does not invent feeling or cause")
|
||||
|
||||
update_facet(profile_id, "core", value="Lange, ruhig fließende Sätze, behutsame Wortwahl, leise Reflexion.")
|
||||
set_lifecycle(profile_id, "confirmed")
|
||||
brief_b = compile_task_brief(profile_id)
|
||||
expect("fließende Sätze" in brief_b, "second confirmed core compiles")
|
||||
with patch("privacy_gateway.complete_model", rewritten):
|
||||
third = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
intern3 = intern_of(third.json())
|
||||
style3 = intern3.split("\nCURRENT_DAY_SOURCES\n")[0]
|
||||
expect("fließende Sätze" in style3, "second writing profile reaches the rendered prompt")
|
||||
expect("trockener Schnitt" not in style3, "replaced core is not still the style authority")
|
||||
expect((third.json().get("trace") or {}).get("writing_profile", {}).get("has_core") is True, "core presence is in the trace")
|
||||
expect((third.json().get("trace") or {}).get("writing_profile", {}).get("present") is True, "confirmed profile is marked present")
|
||||
expect(sum(1 for item in (third.json().get("run_log") or []) if item.get("kind") == "model_call") == 1, "profile A/B still uses one generate call")
|
||||
|
||||
kept_body = (
|
||||
"Die Kinder verstanden kein Wort. Später fragte ich, "
|
||||
"ob die Kinder überhaupt Deutsch verstehen. Die Kinder lachten."
|
||||
)
|
||||
|
||||
def keep_groups(_messages, _policy):
|
||||
return ChatResult(
|
||||
content=f"Markttag\n\n{kept_body}",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", keep_groups):
|
||||
kept = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(kept.status_code == 200, f"accepted model text {kept.text}")
|
||||
expect(kept.json().get("title") == "Markttag", "title split stays technical")
|
||||
expect(kept.json().get("body") == kept_body, "stored body equals demasked model body")
|
||||
expect("Er" not in (kept.json().get("body") or "").split(), "generic group is not rewritten to a pronoun")
|
||||
stored = f"{kept.json().get('title')}\n\n{kept.json().get('body')}".strip()
|
||||
expect((kept.json().get("trace") or {}).get("reply") == stored, "trace reply is the stored draft")
|
||||
expect((kept.json().get("trace") or {}).get("stored_body") == kept_body, "trace stores the same body")
|
||||
expect((kept.json().get("trace") or {}).get("generate_calls") == 1, "accepted text still uses one generate call")
|
||||
|
||||
def mixed_surface(_messages, _policy):
|
||||
body = (
|
||||
"Die Gruppe verstand kein Wort. "
|
||||
"Später die Gruppe wieder. "
|
||||
"die Gruppen standen abseits."
|
||||
)
|
||||
return ChatResult(
|
||||
content=f"Markttag\n\n{body}",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", mixed_surface):
|
||||
mixed = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(mixed.status_code == 200, f"mixed surface generate {mixed.text}")
|
||||
mixed_body = mixed.json().get("body") or ""
|
||||
expect(mixed_body.startswith("Die Gruppe verstand"), "journal stores sentence-initial model casing")
|
||||
expect("Später die Gruppe wieder" in mixed_body, "journal keeps mid-sentence model casing")
|
||||
expect("die Gruppen standen" in mixed_body, "journal keeps unmatched plural surface form")
|
||||
mixed_stored = f"{mixed.json().get('title')}\n\n{mixed_body}".strip()
|
||||
expect((mixed.json().get("trace") or {}).get("reply") == mixed_stored, "journal trace reply is the stored draft")
|
||||
expect((mixed.json().get("trace") or {}).get("stored_body") == mixed_body, "journal stored_body matches the draft")
|
||||
expect("Die Gruppe verstand" in ((mixed.json().get("trace") or {}).get("raw") or ""), "journal trace raw keeps provider casing")
|
||||
expect((mixed.json().get("trace") or {}).get("generate_calls") == 1, "mixed surface still uses one generate call")
|
||||
|
||||
def invent_title(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Hanna am Hafen\n\nIch ging zum Markt, auf dem es ziemlich voll war.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", invent_title):
|
||||
blocked = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(blocked.status_code == 409, f"unattested title {blocked.text}")
|
||||
detail = blocked.json().get("detail") or {}
|
||||
expect(detail.get("code") == "journal_generation_not_accepted", "unattested title is not stored as a draft")
|
||||
expect(detail.get("message") == "Generierung nicht übernommen.", "API names the rejection")
|
||||
log = (detail.get("diagnostics") or {}).get("log") or []
|
||||
expect(any(item.get("reason") == "unattested_identity" for item in log), "title identity is a provenance reject")
|
||||
expect((detail.get("diagnostics") or {}).get("trace", {}).get("model_text_accepted") is False, "model text is not accepted")
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE ai_prompts SET template = 'CUSTOM JOURNAL PROMPT' WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
)
|
||||
init_db()
|
||||
custom = load_active_prompt("mvp.journal_generate")
|
||||
expect(custom["template"] == "CUSTOM JOURNAL PROMPT", "independently edited prompt is not overwritten")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect("INHALTSTREUE" in (row["default_template"] or ""), "default template still tracks the seed")
|
||||
expect(row["seed_revision"] == SEED_REVISION, "revision updates even when template is custom")
|
||||
|
||||
print("journal editorial tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Evaluation harness contract. Does not prove live model quality."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
|
||||
from db import init_db
|
||||
from journal_editorial import GENERATE_SEED_REVISION
|
||||
from journal_eval import (
|
||||
FIXTURES,
|
||||
PROFILE_A,
|
||||
PROFILE_B,
|
||||
SYNTHETIC_PROSE,
|
||||
SYNTHETIC_TYPOS,
|
||||
VARIANT_BASELINE,
|
||||
VARIANT_CURRENT,
|
||||
VARIANT_PREVIOUS,
|
||||
compare_synthetic,
|
||||
fixture_context,
|
||||
score_output,
|
||||
variant_templates,
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_db()
|
||||
templates = variant_templates()
|
||||
expect(set(templates) == {VARIANT_BASELINE, VARIANT_PREVIOUS, VARIANT_CURRENT}, "three comparison variants exist")
|
||||
expect("Erstelle aus diesen Angaben einen ansprechenden persönlichen Tagebucheintrag" in templates[VARIANT_BASELINE], "baseline is a simple rewrite prompt")
|
||||
expect("CURRENT_DAY_SOURCES" in templates[VARIANT_CURRENT], "current variant uses the production prompt")
|
||||
expect("INHALTSTREUE" in templates[VARIANT_CURRENT], "current variant has the new contract")
|
||||
expect("{{transformation_instructions}}" in templates[VARIANT_CURRENT], "current variant compiles transformation policy")
|
||||
expect("{{source_mode_instructions}}" not in templates[VARIANT_CURRENT], "current variant has no source-mode placeholder")
|
||||
expect("Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus." in templates[VARIANT_CURRENT], "current variant has mixed-source instruction")
|
||||
expect("ich gieng zum laden" not in templates[VARIANT_CURRENT], "current variant has no synthetic examples")
|
||||
expect(GENERATE_SEED_REVISION == "2026-08-29-voice-legacy-immutable-v1", "eval tracks the seeded revision constant")
|
||||
|
||||
required_classes = {
|
||||
"already_narrative_with_errors",
|
||||
"bullet_points_and_fragments",
|
||||
"plan_versus_completion",
|
||||
"negation_and_uncertainty",
|
||||
"correction_of_earlier_claim",
|
||||
"imprecise_time",
|
||||
"outstanding_event_among_everyday",
|
||||
"recurring_people_and_projects",
|
||||
"incomplete_source_rebuildable",
|
||||
}
|
||||
got = {item["class"] for item in FIXTURES}
|
||||
expect(required_classes <= got, f"all required fixture classes exist, missing {required_classes - got}")
|
||||
expect(all("example.test" not in item["source"].lower() for item in FIXTURES), "fixtures stay synthetic")
|
||||
|
||||
context = fixture_context(SYNTHETIC_PROSE)
|
||||
expect("source_mode_instructions" not in context, "eval context has no source-mode instruction")
|
||||
expect("editorial_mode" not in context, "eval context has no editorial mode")
|
||||
copied = score_output(SYNTHETIC_PROSE, SYNTHETIC_PROSE, typos=SYNTHETIC_TYPOS)
|
||||
expect(copied["lexical_similarity"] == 1.0, "identical text is similarity 1")
|
||||
expect(copied["spelling_typos_remaining"] == list(SYNTHETIC_TYPOS) or "zimlich" in copied["spelling_typos_remaining"], "copy keeps typos")
|
||||
improved = score_output(
|
||||
SYNTHETIC_PROSE,
|
||||
"Ich ging dann zum Markt, auf dem es ziemlich voll war. Vielleicht bleibe ich kürzer.",
|
||||
typos=SYNTHETIC_TYPOS,
|
||||
)
|
||||
expect("zimlich" in improved["spelling_typos_fixed"], "metrics record typo fixes")
|
||||
expect(improved["lexical_similarity"] < 1.0, "rewrite is not identical")
|
||||
expect("markt" in [item.lower() for item in improved["lost_info_tokens"]] or improved["fact_token_keep"] > 0.2, "fact keep is scored")
|
||||
expect("private" not in str(improved).lower(), "synthetic scores contain no private fixtures")
|
||||
|
||||
notes = next(item for item in FIXTURES if item["id"] == "notes_fragments")
|
||||
expect("markt" in notes["source"].lower(), "notes fixture stays notes-shaped")
|
||||
incomplete = next(item for item in FIXTURES if item["id"] == "incomplete_clause")
|
||||
expect("danach sprach ich mit dem" in incomplete["source"].lower(), "incomplete fixture stays incomplete prose")
|
||||
|
||||
report = compare_synthetic(live=False)
|
||||
expect(report["live"] is False, "default eval is offline")
|
||||
expect(report["live_quality_confirmed"] is False, "offline run does not confirm live quality")
|
||||
expect(report["winner_declared"] is False, "harness does not declare a winner")
|
||||
expect(len(report["fixtures"]) == len(FIXTURES), "offline report covers every fixture")
|
||||
first = report["fixtures"][0]
|
||||
expect("human_blind" in first and "prompt_1" in first["human_blind"], "each fixture has a blind pair")
|
||||
expect(first["human_blind"]["hidden_mapping"]["prompt_1"] == VARIANT_CURRENT, "mapping stays machine-side")
|
||||
expect("Kanshō habe gewonnen" not in report["note"], "no victory claim")
|
||||
expect(any(item["variant"] == VARIANT_CURRENT and item["fake_provider"] for item in first["variants"]), "offline current variant is fake")
|
||||
|
||||
ab = compare_synthetic(live=False, profile_ab=True)
|
||||
expect(ab["profile_ab"]["same_facts"] is True, "profile A/B keeps facts identical")
|
||||
expect(ab["profile_ab"]["prompts_differ"] is True, "rendered prompts contain different style briefs")
|
||||
expect(PROFILE_A[:20] != PROFILE_B[:20], "synthetic profiles are distinct")
|
||||
expect("Live-Prosa" in (ab["profile_ab"]["note"] or ""), "offline A/B does not claim live prose")
|
||||
print("journal eval tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,320 @@
|
||||
"""Journal narration contract: facts stay, wording may change."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from db import get_db, init_db
|
||||
from engine import load_active_prompt
|
||||
from identity_store import remember_mapping
|
||||
from journal_generate import unattested_journal_content
|
||||
from journal_shape import shape_journal
|
||||
from main import app
|
||||
from privacy_gateway import reset_debug
|
||||
from providers import ChatResult
|
||||
from writing_profile_store import (
|
||||
NEUTRAL_JOURNAL_STYLE,
|
||||
compile_task_brief,
|
||||
get_profile,
|
||||
import_text,
|
||||
set_lifecycle,
|
||||
update_facet,
|
||||
)
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def test_active_prompt_contract() -> None:
|
||||
init_db()
|
||||
prompt = load_active_prompt("mvp.journal_generate")
|
||||
text = prompt.get("template") or ""
|
||||
expect("INHALTSTREUE" in text, "active prompt keeps hard content rules")
|
||||
expect("CURRENT_DAY_SOURCES" in text, "active prompt labels current-day facts")
|
||||
expect("{{style_examples}}" in text, "active prompt still has the style-examples slot")
|
||||
expect("Rechtschreibung" in text or "korrigieren" in text, "active prompt allows spelling and grammar fixes")
|
||||
expect("Verneinungen" in text and "Unsicherheiten" in text, "active prompt keeps semantic uncertainty")
|
||||
expect("keine neuen tatsachen" in text.lower(), "active prompt still forbids new facts")
|
||||
expect("Plan und Vollzug" in text, "active prompt keeps plan versus completion")
|
||||
expect("[[…" not in text and "[[..." not in text, "active prompt must not teach ellipsis placeholders")
|
||||
expect("Nur den verifizierten Nutzerwortlaut" not in text, "old wording-as-output rule is gone")
|
||||
expect("Unsicherheiten im Wortlaut" not in text, "old keep-uncertainty-in-wording rule is gone")
|
||||
expect("{{transformation_instructions}}" in text, "active prompt uses compiled transformation instructions")
|
||||
expect("{{source_mode_instructions}}" not in text, "active prompt has no source-mode placeholder")
|
||||
expect("Zwinge nicht den gesamten Tag in einen einzigen Quellenmodus." in text, "active prompt has mixed-source instruction")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect(row["seed_revision"] == "2026-08-29-voice-legacy-immutable-v1", "system prompt revision is stored")
|
||||
expect(row["template"] == row["default_template"], "untouched install uses the seeded template")
|
||||
|
||||
|
||||
def test_shape_keeps_rewritten_model_text() -> None:
|
||||
source = ["ich bin dan zum markt gegangen und da war es zimlich voll"]
|
||||
model = "Ich ging dann zum Markt, auf dem es ziemlich voll war."
|
||||
_, body = shape_journal("Markttag", model, source)
|
||||
expect(body == model, "shape_journal keeps the corrected model text")
|
||||
expect("zimlich" not in body, "source typo is not restored")
|
||||
|
||||
|
||||
def test_shape_does_not_restore_source_form() -> None:
|
||||
source = ["ich ging zum markt. danach der hafen."]
|
||||
_, punct = shape_journal("Tag", "Ich ging zum Markt. Danach der Hafen.", source)
|
||||
expect(punct == "Ich ging zum Markt. Danach der Hafen.", "corrected punctuation is not reset")
|
||||
_, casing = shape_journal("Tag", "Am Markt war es voll.", ["am markt war es voll"])
|
||||
expect(casing == "Am Markt war es voll.", "corrected capitalization is not reset")
|
||||
rewritten = "Später am Hafen packte ich die rote Tasche."
|
||||
_, structure = shape_journal("Tag", rewritten, ["danach hafen. rote tasche eingepackt."])
|
||||
expect(structure == rewritten, "changed sentence structure is not reset")
|
||||
fallback_parts = ["Zeile eins Markt.", "Zeile zwei Hafen und rote Tasche."]
|
||||
_, fallback = shape_journal("Ein Tag", "\n\n".join(fallback_parts), fallback_parts, source="fallback")
|
||||
expect("Markt" in fallback and "Hafen" in fallback and "rote Tasche" in fallback, "fallback keeps every unique user source")
|
||||
expect(fallback == "\n\n".join(fallback_parts) or "Markt" in fallback, "fallback path stays source-based")
|
||||
model_kept = "Ich war am Markt und später am Hafen."
|
||||
_, model = shape_journal("Tag", model_kept, fallback_parts, source="model")
|
||||
expect(model == model_kept, "model path is not mixed with the fallback join")
|
||||
|
||||
|
||||
def test_unattested_identity_is_journal_not_privacy() -> None:
|
||||
mappings = [
|
||||
{"local_label": "Anna", "token": "PERSON:01"},
|
||||
{"local_label": "Clarissa", "token": "PERSON:99"},
|
||||
]
|
||||
sources = ["Heute war ich mit Anna am Markt."]
|
||||
expect(
|
||||
unattested_journal_content("Ich ging mit Anna zum Markt.", sources, mappings, ["PERSON:01"]) is None,
|
||||
"attested person in rewritten prose is allowed",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Clarissa kaufte Kirschen am Markt.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_identity",
|
||||
"invented person as subject of a verb is still unattested",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("[[PERSON:99]] stand am Markt.", sources, mappings, ["PERSON:01"])
|
||||
== "unattested_placeholder",
|
||||
"inactive placeholder is unattested content",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content("Heute nur von Sushi essen erzählt.", sources, [{"local_label": "Sushi", "token": "PERSON:01"}], [])
|
||||
is None,
|
||||
"food homonym of an unused mapping is not unattested identity",
|
||||
)
|
||||
expect(
|
||||
unattested_journal_content(
|
||||
"Clarissa kam ins Wohnzimmer.",
|
||||
["Heute war Sushi im Wohnzimmer."],
|
||||
[
|
||||
{
|
||||
"local_label": "Clarissa",
|
||||
"canonical_label": "Clarissa",
|
||||
"token": "PERSON:01",
|
||||
"aliases": ["Sushi"],
|
||||
"demask_label": "Clarissa",
|
||||
}
|
||||
],
|
||||
["PERSON:01"],
|
||||
)
|
||||
is None,
|
||||
"confirmed alias in the source attests the canonical demasked spelling",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_active_prompt_contract()
|
||||
test_shape_keeps_rewritten_model_text()
|
||||
test_shape_does_not_restore_source_form()
|
||||
test_unattested_identity_is_journal_not_privacy()
|
||||
expect(NEUTRAL_JOURNAL_STYLE.startswith("Neutraler Journalstil"), "neutral fallback exists")
|
||||
|
||||
reset_debug()
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"},
|
||||
)
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
empty_brief = compile_task_brief(profile_id, "journal_generate")
|
||||
expect(empty_brief == NEUTRAL_JOURNAL_STYLE, "empty profile uses the neutral journal voice")
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Narration"})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-26"},
|
||||
)
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Tag"},
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "ich bin dan zum markt gegangen und da war es zimlich voll. vielleicht bleibe ich kürzer. ich wollte noch brot holen, hab es aber nicht gemacht. die rote tasche lag im auto. ich bin dan zum markt gegangen."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn {turn.status_code}")
|
||||
before = get_profile(profile_id)
|
||||
gen = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(gen.status_code == 200, f"generate {gen.text}")
|
||||
intern = ""
|
||||
for stage in (gen.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
intern = stage.get("intern") or ""
|
||||
style = intern.split("\nCURRENT_DAY_SOURCES\n")[0]
|
||||
expect("Neutraler Journalstil" in style, "neutral style reaches the generate prompt")
|
||||
expect("Erzählmerkmale" not in style, "current day dialogue is not a style brief")
|
||||
expect("zimlich" not in style, "today's typo is not a style exemplar")
|
||||
expect("INHALTSTREUE" in intern, "runtime intern uses the new narration contract")
|
||||
expect("CURRENT_DAY_SOURCES" in intern, "runtime intern labels current-day facts")
|
||||
expect("Nur den verifizierten Nutzerwortlaut" not in intern, "old output-wording rule is not in the runtime prompt")
|
||||
expect("Unsicherheiten im Wortlaut" not in intern, "old uncertainty-in-wording rule is not in the runtime prompt")
|
||||
after = get_profile(profile_id)
|
||||
expect(after.get("version") == before.get("version"), "generate does not create a new profile version")
|
||||
expect((after.get("suggestions") or []) == (before.get("suggestions") or []), "generate does not add suggestions")
|
||||
|
||||
import_text(profile_id, "Ich schreibe in kurzen, ruhigen Sätzen und lasse den Tag stehen.")
|
||||
from journal_editorial import select_journal_style_examples
|
||||
|
||||
imported_examples = select_journal_style_examples(profile_id)
|
||||
expect(
|
||||
imported_examples and "kurzen, ruhigen Sätzen" in imported_examples[0]["excerpt"],
|
||||
"imported own text is a style source",
|
||||
)
|
||||
imported_brief = compile_task_brief(profile_id, "journal_generate")
|
||||
expect(imported_brief == NEUTRAL_JOURNAL_STYLE, "unconfirmed profile does not become the brief")
|
||||
expect("zimlich" not in " ".join(item["excerpt"] for item in imported_examples), "today's dialogue is not mixed into the imported style")
|
||||
|
||||
saved = client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={
|
||||
"journal_day_id": day.json()["day"]["id"],
|
||||
"title": "Eigene Fassung",
|
||||
"body": "Heute blieb ich beim klaren Schnitt und schrieb den Markt in eigenen Worten.",
|
||||
"origin": "user_edit",
|
||||
"source_conversation_ids": [conv.json()["id"]],
|
||||
},
|
||||
)
|
||||
expect(saved.status_code == 200, f"save {saved.text}")
|
||||
entry_examples = select_journal_style_examples(profile_id)
|
||||
expect(
|
||||
any("klaren Schnitt" in (item.get("excerpt") or "") for item in entry_examples),
|
||||
"final user-edited journal text is a style source",
|
||||
)
|
||||
expect(all("zimlich" not in (item.get("excerpt") or "") for item in entry_examples), "current day dialogue is not a positive style reference")
|
||||
|
||||
update_facet(profile_id, "core", value="Kurze Sätze, trockener Schnitt, keine Pathoswolken.")
|
||||
set_lifecycle(profile_id, "confirmed")
|
||||
with_core = compile_task_brief(profile_id, "journal_generate")
|
||||
expect("Core:" in with_core and "trockener Schnitt" in with_core, "confirmed core reaches the task brief")
|
||||
expect("Neutraler Journalstil" not in with_core, "confirmed profile replaces the neutral fallback")
|
||||
|
||||
remember_mapping(profile_id, "Clarissa", "PERSON:99")
|
||||
|
||||
def rewritten(_messages, _policy):
|
||||
return ChatResult(
|
||||
content=(
|
||||
"Markttag\n\n"
|
||||
"Ich ging dann zum Markt, auf dem es ziemlich voll war. "
|
||||
"Vielleicht bleibe ich kürzer. "
|
||||
"Brot holen wollte ich noch, habe es aber nicht gemacht. "
|
||||
"Die rote Tasche lag im Auto."
|
||||
),
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", rewritten):
|
||||
second = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(second.status_code == 200, f"rewritten generate {second.text}")
|
||||
body = second.json().get("body") or ""
|
||||
expect("ziemlich voll" in body, "corrected spelling of the model text is kept")
|
||||
expect("zimlich" not in body, "source typo is not restored after a real model rewrite")
|
||||
expect("Vielleicht bleibe ich kürzer" in body, "semantic uncertainty is kept")
|
||||
expect("wollte" in body.lower() and "nicht gemacht" in body, "plan versus non-fulfillment stays")
|
||||
expect("rote Tasche" in body, "unique attested detail is kept after rewrite")
|
||||
expect(body.lower().count("markt") <= 2, "repeated source lines are not pasted twice")
|
||||
expect(
|
||||
any(item.get("status") == "model" for item in (second.json().get("run_log") or [])),
|
||||
"accepted model path is recorded as model",
|
||||
)
|
||||
intern2 = ""
|
||||
for stage in (second.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
intern2 = stage.get("intern") or ""
|
||||
expect("trockener Schnitt" in intern2.split("\nCURRENT_DAY_SOURCES\n")[0], "confirmed writing profile reaches generate")
|
||||
|
||||
def invent(_messages, _policy):
|
||||
return ChatResult(
|
||||
content="Clarissa blieb den ganzen Nachmittag am Hafen.",
|
||||
model="fake",
|
||||
usage={},
|
||||
context_compression="disabled",
|
||||
)
|
||||
|
||||
with patch("privacy_gateway.complete_model", invent):
|
||||
blocked = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv.json()["id"]]},
|
||||
)
|
||||
expect(blocked.status_code == 409, f"unattested generate {blocked.text}")
|
||||
detail = blocked.json().get("detail") or {}
|
||||
expect(detail.get("code") == "journal_generation_not_accepted", "unattested person is not stored as a draft")
|
||||
expect(detail.get("message") == "Generierung nicht übernommen.", "API names the rejection")
|
||||
log = (detail.get("diagnostics") or {}).get("log") or []
|
||||
expect(any(item.get("reason") == "unattested_identity" for item in log), "unattested person is a provenance reject")
|
||||
expect(any(item.get("status") == "not_accepted" for item in log), "reject path is distinct from the model path")
|
||||
expect("Clarissa" not in json.dumps((detail.get("diagnostics") or {}).get("log") or []), "clear labels stay out of compact logs")
|
||||
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE ai_prompts SET template = 'CUSTOM JOURNAL PROMPT' WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
)
|
||||
init_db()
|
||||
custom = load_active_prompt("mvp.journal_generate")
|
||||
expect(custom["template"] == "CUSTOM JOURNAL PROMPT", "independently edited prompt is not overwritten")
|
||||
with get_db() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT default_template, seed_revision FROM ai_prompts WHERE slug = ?",
|
||||
("mvp.journal_generate",),
|
||||
).fetchone()
|
||||
expect("INHALTSTREUE" in (row["default_template"] or ""), "default template still tracks the seed")
|
||||
expect(row["seed_revision"] == "2026-08-29-voice-legacy-immutable-v1", "revision updates even when template is custom")
|
||||
|
||||
print("journal narration tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,304 @@
|
||||
"""First journal impulse invariants. Run from backend/: python tests/test_journal_opening.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from datetime import date
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-opening-media")
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from main import app
|
||||
from journal_opening import (
|
||||
NEUTRAL_OPENING,
|
||||
attested_plans_from_user_texts,
|
||||
collect_opening_context,
|
||||
is_attested_plan,
|
||||
is_previous_calendar_day,
|
||||
parse_calendar_date,
|
||||
)
|
||||
from journal_reconstruct import assign_source_ids
|
||||
from retrieval import pair_user_priority_messages
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def open_space_day(client, headers, title, date):
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": title})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": date},
|
||||
)
|
||||
return space.json(), day.json()
|
||||
|
||||
|
||||
def start_conv(client, headers, day_id):
|
||||
return client.post(f"/api/journal/days/{day_id}/conversations", headers=headers, json={"title": "Gespräch"})
|
||||
|
||||
|
||||
def main() -> None:
|
||||
expect(is_attested_plan("Morgen wollen wir nach Cres fahren."), "future intent is a plan")
|
||||
expect(is_attested_plan("Wir haben vor, später an den Hafen zu gehen."), "vorhaben is a plan")
|
||||
expect(not is_attested_plan("Heute war der Hafen ruhig. Kirschen am Stand."), "past report is not a plan")
|
||||
expect(not is_attested_plan("Wir standen vor dem Hafen und aßen Kirschen."), "vor without haben is not a plan")
|
||||
expect(not is_attested_plan("Hafen Kirschen Wind"), "word overlap is not a plan")
|
||||
expect(parse_calendar_date("2026-08-29") == date(2026, 8, 29), "calendar date parses")
|
||||
expect(is_previous_calendar_day(date(2026, 8, 29), date(2026, 8, 30)), "yesterday is previous")
|
||||
expect(not is_previous_calendar_day(date(2026, 8, 29), date(2026, 9, 3)), "older than yesterday is not previous")
|
||||
plans = attested_plans_from_user_texts(
|
||||
["Heute war der Hafen ruhig.", "Morgen wollen wir nach Cres fahren.", "Hafen Kirschen"]
|
||||
)
|
||||
expect(len(plans) == 1 and "Cres" in plans[0], "only attested plan sentences are kept")
|
||||
expect(
|
||||
not any(is_attested_plan(text) for text in ["Was davon möchtest du festhalten?", "Die Fahrt nach Cres war schön."]),
|
||||
"assistant-like completion is not treated as a user plan by the helper",
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
setup = client.post("/api/auth/setup", json={"email": "lars@example.test", "name": "Lars", "password": "test-pass"})
|
||||
headers = header(setup.json()["token"])
|
||||
other = client.post(
|
||||
"/api/users",
|
||||
headers=headers,
|
||||
json={"email": "ute@example.test", "name": "Ute", "password": "user-pass", "role": "user"},
|
||||
)
|
||||
expect(other.status_code == 200, "second profile")
|
||||
other_headers = header(
|
||||
client.post("/api/auth/login", json={"email": "ute@example.test", "password": "user-pass"}).json()["token"]
|
||||
)
|
||||
|
||||
empty_space, empty_day = open_space_day(client, headers, "Leer", "2026-08-21")
|
||||
empty = start_conv(client, headers, empty_day["day"]["id"])
|
||||
expect(empty.status_code == 200, f"empty opening {empty.text}")
|
||||
expect(empty.json().get("opening", {}).get("kind") == "local_neutral", "no context uses local neutral")
|
||||
expect(empty.json()["opening"]["assistant"]["body"] == NEUTRAL_OPENING, "neutral impulse stored")
|
||||
expect(empty.json()["opening"]["calls"] == 0, "neutral opening does not call a model")
|
||||
empty_msgs = empty.json().get("messages") or []
|
||||
expect(len(empty_msgs) == 1 and empty_msgs[0]["role"] == "assistant", "neutral opening is assistant-first")
|
||||
|
||||
plan_space, plan_day = open_space_day(client, headers, "Urlaub", "2026-08-01")
|
||||
plan_conv = start_conv(client, headers, plan_day["day"]["id"])
|
||||
client.post(
|
||||
f"/api/journal/conversations/{plan_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Morgen wollen wir nach Cres fahren. Die Fähre ist schon bedacht."},
|
||||
)
|
||||
later = client.post(
|
||||
f"/api/journal/spaces/{plan_space['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-02"},
|
||||
)
|
||||
planned = start_conv(client, headers, later.json()["day"]["id"])
|
||||
facts = collect_opening_context(setup.json()["profile_id"], planned.json()["id"])
|
||||
expect(facts["has_relevant_context"], "attested plan counts as relevant context")
|
||||
expect(any("Cres" in item and is_attested_plan(item) for item in facts["attested_plans"]), "plan comes from user wording")
|
||||
expect(all(item in facts["user_texts"] for item in facts["attested_plans"]), "plans are a subset of user texts")
|
||||
expect(planned.json().get("opening", {}).get("kind") == "model", "attested plan uses the dialogue path")
|
||||
intern = (planned.json().get("opening") or {}).get("trace") or {}
|
||||
intern_text = intern.get("intern") or ""
|
||||
expect("Belegte Vorhaben" in intern_text, "opening policy reaches the prompt")
|
||||
expect("kein bewiesenes wiederkehrendes Muster" in intern_text or "kein bewiesenes" in intern_text, "pattern rule is in the opening hint")
|
||||
|
||||
thread_space, thread_day = open_space_day(client, headers, "Tagfaden", "2026-08-10")
|
||||
first = start_conv(client, headers, thread_day["day"]["id"])
|
||||
client.post(
|
||||
f"/api/journal/conversations/{first.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Morgens am Markt, danach noch offen, wie der Nachmittag wird."},
|
||||
)
|
||||
second = start_conv(client, headers, thread_day["day"]["id"])
|
||||
thread_facts = collect_opening_context(setup.json()["profile_id"], second.json()["id"])
|
||||
expect(thread_facts["open_day_points"], "same-day user text is an open point")
|
||||
expect(
|
||||
any("Markt" in item or "Nachmittag" in item for item in thread_facts["open_day_points"]),
|
||||
"open point is the day's user wording",
|
||||
)
|
||||
expect(second.json().get("opening", {}).get("kind") == "model", "open day thread uses the dialogue path")
|
||||
|
||||
overlap_space, overlap_day = open_space_day(client, headers, "Overlap", "2026-08-03")
|
||||
overlap_conv = start_conv(client, headers, overlap_day["day"]["id"])
|
||||
client.post(
|
||||
f"/api/journal/conversations/{overlap_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Heute war der Hafen ruhig. Kirschen am Stand, Wind von See."},
|
||||
)
|
||||
client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={
|
||||
"journal_day_id": overlap_day["day"]["id"],
|
||||
"title": "Hafen",
|
||||
"body": "Heute war der Hafen ruhig. Kirschen am Stand.",
|
||||
"origin": "user_edit",
|
||||
},
|
||||
)
|
||||
later_overlap = client.post(
|
||||
f"/api/journal/spaces/{overlap_space['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-04"},
|
||||
)
|
||||
overlap_open = start_conv(client, headers, later_overlap.json()["day"]["id"])
|
||||
overlap_facts = collect_opening_context(setup.json()["profile_id"], overlap_open.json()["id"])
|
||||
expect(not overlap_facts["attested_plans"], "random overlap is not an attested plan")
|
||||
expect(not overlap_facts["has_relevant_context"], "recency without plan or open day is not relevant opening context")
|
||||
expect(overlap_open.json().get("opening", {}).get("kind") == "local_neutral", "overlap falls back to neutral")
|
||||
expect("Muster" not in (overlap_open.json().get("opening") or {}).get("assistant", {}).get("body", ""), "neutral impulse does not claim a pattern")
|
||||
|
||||
done_space, done_day = open_space_day(client, headers, "Abgeschlossen", "2026-08-20")
|
||||
done_conv = start_conv(client, headers, done_day["day"]["id"])
|
||||
client.post(
|
||||
f"/api/journal/conversations/{done_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Nachmittag am Strand, abends Eis, dann schlafen."},
|
||||
)
|
||||
client.post(
|
||||
"/api/journal/entries",
|
||||
headers=headers,
|
||||
json={
|
||||
"journal_day_id": done_day["day"]["id"],
|
||||
"title": "Hitze",
|
||||
"body": (
|
||||
"Schon um 6 Uhr schlug mir eine drückende Luft entgegen. "
|
||||
"Ich hatte überlegt, die Roller auf den nächsten Tag zu schieben. "
|
||||
"Wir haben sie dann doch geholt und sind den Tag gefahren."
|
||||
),
|
||||
"origin": "user_edit",
|
||||
},
|
||||
)
|
||||
later_done = client.post(
|
||||
f"/api/journal/spaces/{done_space['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-21"},
|
||||
)
|
||||
done_open = start_conv(client, headers, later_done.json()["day"]["id"])
|
||||
done_facts = collect_opening_context(setup.json()["profile_id"], done_open.json()["id"])
|
||||
expect(not done_facts["attested_plans"], "completed journal entry is not an opening plan")
|
||||
expect(not done_facts["has_relevant_context"], "a closed earlier day does not keep the next opening relevant")
|
||||
expect(done_open.json().get("opening", {}).get("kind") == "local_neutral", "completed story falls back to neutral")
|
||||
|
||||
stale = client.post(
|
||||
f"/api/journal/spaces/{plan_space['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-04"},
|
||||
)
|
||||
stale_open = start_conv(client, headers, stale.json()["day"]["id"])
|
||||
stale_facts = collect_opening_context(setup.json()["profile_id"], stale_open.json()["id"])
|
||||
expect(not any("Cres" in item for item in stale_facts["attested_plans"]), "plan from two days ago is no longer an opening mandate")
|
||||
expect(stale_open.json().get("opening", {}).get("kind") == "local_neutral", "stale plan uses local neutral")
|
||||
|
||||
asst_space, asst_day = open_space_day(client, headers, "Assistent", "2026-08-05")
|
||||
asst_conv = start_conv(client, headers, asst_day["day"]["id"])
|
||||
from dialogue_store import append_message
|
||||
|
||||
append_message(
|
||||
setup.json()["profile_id"],
|
||||
asst_conv.json()["id"],
|
||||
"Morgen wollen wir nach Cres fahren.",
|
||||
role="assistant",
|
||||
)
|
||||
next_asst = client.post(
|
||||
f"/api/journal/spaces/{asst_space['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-06"},
|
||||
)
|
||||
asst_open = start_conv(client, headers, next_asst.json()["day"]["id"])
|
||||
asst_facts = collect_opening_context(setup.json()["profile_id"], asst_open.json()["id"])
|
||||
expect(not asst_facts["attested_plans"], "assistant wording is not a user plan")
|
||||
expect("Cres" not in " ".join(asst_facts["attested_plans"]), "assistant plan sentence stays out of attested plans")
|
||||
expect(asst_facts["assistant_is_not_user_fact"], "adapter marks assistant as non-fact")
|
||||
|
||||
pair_space, pair_day = open_space_day(client, headers, "Paarung", "2026-08-07")
|
||||
pair_conv = start_conv(client, headers, pair_day["day"]["id"])
|
||||
expect((pair_conv.json().get("messages") or [])[0]["role"] == "assistant", "assistant-first stored")
|
||||
user_turn = client.post(
|
||||
f"/api/journal/conversations/{pair_conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": "Nach der Fähre kamen wir spät an. Anna kaufte Brot."},
|
||||
)
|
||||
expect(user_turn.status_code == 200, "user turn after opening")
|
||||
msgs = user_turn.json()["messages"]
|
||||
users = [item for item in msgs if item["role"] == "user"]
|
||||
expect(len(users) == 1, "exactly one user source after opening plus turn")
|
||||
paired = pair_user_priority_messages(msgs)
|
||||
paired_users = [item for item in paired if item["role"] == "user"]
|
||||
expect([item["body"] for item in paired_users] == [item["body"] for item in users], "pairing keeps every user source")
|
||||
labeled = assign_source_ids(paired)
|
||||
expect(
|
||||
[item["source_id"] for item in labeled if item["role"] == "user"] == ["u1"],
|
||||
"first user source is u1 even after an opening assistant",
|
||||
)
|
||||
draft = client.post(
|
||||
f"/api/journal/days/{pair_day['day']['id']}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [pair_conv.json()["id"]]},
|
||||
)
|
||||
expect(draft.status_code == 200, f"generate after opening {draft.text}")
|
||||
reconstruct = None
|
||||
for stage in (draft.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "local_source_artifact":
|
||||
reconstruct = stage
|
||||
intern = (reconstruct or {}).get("intern") or ""
|
||||
expect("Anna kaufte Brot" in intern, "user source reaches local artifact")
|
||||
expect("u1" in intern, "user source id assigned")
|
||||
narrate = None
|
||||
for stage in (draft.json().get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
narrate = stage
|
||||
narrate_intern = (narrate or {}).get("intern") or ""
|
||||
expect("CURRENT_DAY_SOURCES" in narrate_intern or "Anna kaufte Brot" in narrate_intern, "stage 2 consumes day sources")
|
||||
expect("Nach der Fähre kamen wir spät an" in intern, "user wording not dropped after opening")
|
||||
expect(users[0]["body"].startswith("Nach der Fähre"), "user source is the user message, not the opening")
|
||||
|
||||
stolen = client.post(
|
||||
f"/api/journal/days/{empty_day['day']['id']}/conversations",
|
||||
headers=other_headers,
|
||||
json={"title": "fremd"},
|
||||
)
|
||||
expect(stolen.status_code == 404, "foreign profile cannot open a conversation")
|
||||
stolen_turn = client.post(
|
||||
f"/api/journal/conversations/{empty.json()['id']}/turn",
|
||||
headers=other_headers,
|
||||
json={"body": "Hallo"},
|
||||
)
|
||||
expect(stolen_turn.status_code == 404, "foreign profile cannot turn into the opening conversation")
|
||||
user_login_space = client.post("/api/journal/spaces", headers=other_headers, json={"title": "Ute"})
|
||||
user_day = client.post(
|
||||
f"/api/journal/spaces/{user_login_space.json()['id']}/days",
|
||||
headers=other_headers,
|
||||
json={"calendar_date": "2026-08-21"},
|
||||
)
|
||||
user_open = start_conv(client, other_headers, user_day.json()["day"]["id"])
|
||||
expect(user_open.status_code == 200, "non-admin can start their own day")
|
||||
expect("trace" not in (user_open.json().get("opening") or {}), "non-admin opening has no trace")
|
||||
expect("opening_context" not in (user_open.json().get("opening") or {}), "non-admin opening has no opening_context")
|
||||
expect("trace" not in user_open.json(), "non-admin response has no top-level trace")
|
||||
|
||||
start = client.get("/api/journal/continuity", headers=headers)
|
||||
expect(start.status_code == 200 and start.json().get("continuable"), "continuable day for the author")
|
||||
expect(start.json()["continuable"]["space_id"] == start.json()["continuable"]["space_id"], "continuity names a space")
|
||||
other_start = client.get("/api/journal/continuity", headers=other_headers)
|
||||
expect(other_start.json()["continuable"]["space_id"] == user_login_space.json()["id"], "continuity is profile-scoped")
|
||||
stolen_cont = client.get(f"/api/journal/spaces/{plan_space['id']}", headers=other_headers)
|
||||
expect(stolen_cont.status_code == 404, "continuity space stays isolated")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Local journal shaping. Run from backend/: python tests/test_journal_shape.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from journal_shape import is_verbatim_join, paragraphize, shape_journal
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
turns = [
|
||||
"Heute war der Markt voll. Ich bin früh gegangen und habe Kirschen gekauft.",
|
||||
"Später zwangen wir die Kinder aus dem Bett und packten Schwimmsachen ein.",
|
||||
"Im Hafen liefen wir zuerst zur falschen Seite.",
|
||||
]
|
||||
joined = " ".join(turns)
|
||||
expect(is_verbatim_join(joined, turns), "space-joined user turns are a paste")
|
||||
expect(not is_verbatim_join("Am Markt kaufte ich Kirschen und ging dann weiter.", turns), "rewritten prose is kept")
|
||||
|
||||
title, body = shape_journal("Markttag", joined, turns)
|
||||
expect(title == "Markttag", "title stays")
|
||||
expect("Kirschen" in body and "Hafen" in body, "model wording is kept")
|
||||
expect(body != "\n\n".join(turns), "accepted model text is not replaced by source turns")
|
||||
|
||||
source_typo = ["ich bin dan zum markt gegangen und da war es zimlich voll"]
|
||||
rewritten = "Ich ging dann zum Markt, auf dem es ziemlich voll war."
|
||||
_, kept_rewrite = shape_journal("Markttag", rewritten, source_typo)
|
||||
expect("ziemlich" in kept_rewrite, "corrected spelling stays")
|
||||
expect("zimlich" not in kept_rewrite, "source typo is not restored")
|
||||
expect(kept_rewrite.startswith("Ich ging"), "corrected sentence structure stays")
|
||||
expect("Markt" in kept_rewrite, "capitalization stays corrected")
|
||||
|
||||
_, punct = shape_journal("Tag", "Ich ging zum Markt.", ["ich ging zum markt"])
|
||||
expect(punct == "Ich ging zum Markt.", "corrected punctuation and casing are not reset")
|
||||
|
||||
fallback_body = "\n\n".join(turns)
|
||||
_, fallback = shape_journal("Ein Tag", fallback_body, turns, source="fallback")
|
||||
expect("Kirschen" in fallback and "Hafen" in fallback, "fallback still contains every user source")
|
||||
expect("Markt" in fallback, "fallback keeps first source")
|
||||
|
||||
blob = (
|
||||
"Heute war der Markt voll. Ich bin früh gegangen. "
|
||||
"Danach haben wir gepackt. Die Kinder kamen spät. "
|
||||
"Im Hafen liefen wir falsch. Sushi rief zur anderen Seite. "
|
||||
"Schließlich fanden wir das Boot. Mario begrüßte uns."
|
||||
)
|
||||
phased = paragraphize(blob)
|
||||
expect("\n\n" in phased, "a long single block gets phase breaks")
|
||||
expect("Markt" in phased and "Boot" in phased, "paragraphize does not drop words")
|
||||
|
||||
kept = "Am Morgen der Markt.\n\nSpäter das Boot, noch gehetzt."
|
||||
expect(shape_journal("Tag", kept, turns)[1] == kept, "already shaped text is left alone")
|
||||
|
||||
repeated = (
|
||||
"Die Kinder verstanden kein Wort. Später fragte ich, "
|
||||
"ob die Kinder überhaupt Deutsch verstehen. Die Kinder lachten."
|
||||
)
|
||||
_, unchanged = shape_journal(
|
||||
"Tag",
|
||||
repeated,
|
||||
["Die Kinder verstanden kein Wort."],
|
||||
person_labels=["Die Kinder", "Kinder"],
|
||||
)
|
||||
expect(unchanged == repeated, "repeated generic person terms stay unchanged")
|
||||
expect("Er" not in unchanged.split(), "generic group is not turned into a pronoun")
|
||||
expect(unchanged.count("Kinder") == 3, "every generic mention remains")
|
||||
|
||||
named = (
|
||||
"Maren kam zum Markt. Später rief Maren an. "
|
||||
"Ob Maren noch käme, wusste ich nicht. Maren blieb weg."
|
||||
)
|
||||
_, named_out = shape_journal("Tag", named, named.split(". "), person_labels=["Maren"])
|
||||
expect(named_out == named, "accepted person names are not replaced by pronouns")
|
||||
expect("Er " not in named_out and "Sie " not in named_out, "no guessed pronoun rewrite")
|
||||
|
||||
print("journal shape ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,373 @@
|
||||
"""Style-application variants control which writing-profile parts reach generate."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from db import get_db, init_db
|
||||
from journal_editorial import (
|
||||
compile_journal_style_application,
|
||||
style_application_trace,
|
||||
)
|
||||
from journal_generation_policy import (
|
||||
clone_guideline,
|
||||
compile_selection,
|
||||
default_selection_ids,
|
||||
get_guideline,
|
||||
save_selection,
|
||||
seed_generation_instructions,
|
||||
)
|
||||
from journal_generate import pack_narration_context
|
||||
from main import app
|
||||
from privacy_gateway import install_test_recorder, reset_debug
|
||||
from prompt_budget import JournalBudget
|
||||
from writing_profile_store import (
|
||||
compile_task_brief,
|
||||
import_text,
|
||||
set_lifecycle,
|
||||
update_facet,
|
||||
upsert_trait,
|
||||
)
|
||||
|
||||
|
||||
CORE_MARK = "ZXCORE_MARKER kurze trockene Saetze"
|
||||
FACET_MARK = "ZXFACET_MARKER ruhige Abendreflexion"
|
||||
TRAIT_MARK = "ZXTRAIT_MARKER keine rhythmischen Aufzaehlungen"
|
||||
EXAMPLE_MARK = "ZXEXAMPLE_MARKER Der Hafen blieb hinter der Faehre"
|
||||
TODAY_MARK = "ZXTODAY_MARKER am heutigen Markt"
|
||||
|
||||
|
||||
def expect(ok: bool, message: str) -> None:
|
||||
if not ok:
|
||||
raise SystemExit(f"FAIL: {message}")
|
||||
print(f"OK {message}")
|
||||
|
||||
|
||||
def header(token: str) -> dict:
|
||||
return {"X-Auth-Token": token}
|
||||
|
||||
|
||||
def intern_of(payload: dict) -> str:
|
||||
for stage in (payload.get("trace") or {}).get("stages") or []:
|
||||
if stage.get("purpose") == "journal_generate":
|
||||
return stage.get("intern") or ""
|
||||
return (payload.get("trace") or {}).get("intern") or ""
|
||||
|
||||
|
||||
def style_region(intern: str) -> str:
|
||||
return (intern or "").split("\nCURRENT_DAY_SOURCES\n")[0]
|
||||
|
||||
|
||||
def has_heading(intern: str, title: str) -> bool:
|
||||
body = intern or ""
|
||||
return f"\n{title}\n" in body or body.startswith(f"{title}\n")
|
||||
|
||||
|
||||
def seed_id(slot: str, key: str) -> str:
|
||||
from journal_generation_policy import load_seed_document
|
||||
|
||||
seed = load_seed_document()
|
||||
for item in (seed.get("slots") or {}).get(slot, {}).get("variants") or []:
|
||||
if item.get("guideline_key") == key:
|
||||
return item["id"]
|
||||
raise SystemExit(f"FAIL: missing seed {slot}/{key}")
|
||||
|
||||
|
||||
def selection_of(voice_key: str) -> dict[str, str]:
|
||||
ids = default_selection_ids()
|
||||
ids["voice_id"] = seed_id("voice", voice_key)
|
||||
return ids
|
||||
|
||||
|
||||
def ensure_local_profile(profile_id: str = "style-context-local") -> str:
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO profiles (id, email, name, password_hash, role)
|
||||
VALUES (?, ?, ?, ?, 'user')
|
||||
""",
|
||||
(profile_id, f"{profile_id}@example.test", "Style", "x"),
|
||||
)
|
||||
return profile_id
|
||||
|
||||
|
||||
def install_profile(profile_id: str) -> None:
|
||||
update_facet(profile_id, "core", value=CORE_MARK)
|
||||
update_facet(profile_id, "autobiographical_journal", value=FACET_MARK)
|
||||
upsert_trait(
|
||||
profile_id,
|
||||
slug="journal-rhythm",
|
||||
facet_key="autobiographical_journal",
|
||||
label="Rhythmus",
|
||||
statement=TRAIT_MARK,
|
||||
origin="manual",
|
||||
)
|
||||
set_lifecycle(profile_id, "confirmed")
|
||||
import_text(
|
||||
profile_id,
|
||||
EXAMPLE_MARK + " in knappen ruhigen Saetzen ohne Pathoswolken.",
|
||||
occurred_at="2026-07-01",
|
||||
)
|
||||
|
||||
|
||||
def test_compile_flags() -> None:
|
||||
init_db()
|
||||
profile_id = ensure_local_profile()
|
||||
install_profile(profile_id)
|
||||
full = compile_task_brief(profile_id)
|
||||
expect(CORE_MARK in full and FACET_MARK in full and TRAIT_MARK in full, "full brief keeps selected parts")
|
||||
core_only = compile_task_brief(
|
||||
profile_id,
|
||||
include_core=True,
|
||||
include_facet=False,
|
||||
include_traits=False,
|
||||
)
|
||||
expect(CORE_MARK in core_only, "core-only brief keeps core")
|
||||
expect(FACET_MARK not in core_only, "core-only brief omits facet")
|
||||
expect(TRAIT_MARK not in core_only, "core-only brief omits traits")
|
||||
none = compile_task_brief(
|
||||
profile_id,
|
||||
include_core=False,
|
||||
include_facet=False,
|
||||
include_traits=False,
|
||||
)
|
||||
expect(none == "", "no selected profile parts yield an empty brief")
|
||||
bundle = compile_journal_style_application(profile_id, compile_selection(selection_of("neutral")).style_context)
|
||||
expect(bundle["profile_block"] == "", "profile-free omits the wrapped profile block")
|
||||
expect(bundle["examples_block"] == "", "profile-free omits the wrapped examples block")
|
||||
|
||||
|
||||
def test_budget_omission_in_effective_trace() -> None:
|
||||
init_db()
|
||||
prompt = {
|
||||
"id": "style-pack",
|
||||
"slug": "mvp.journal_generate",
|
||||
"prompt_type": "base",
|
||||
"template": "{{writing_profile}}{{style_examples}}DAY\n{{reconstruction}}\nOLD\n{{existing_text}}\n",
|
||||
}
|
||||
assembled = {
|
||||
"writing_profile": "WRITING_PROFILE\nCore: kurze Sätze.\n",
|
||||
"reconstruction": "Heute Markt.",
|
||||
"style_examples": "",
|
||||
"existing_text": "",
|
||||
}
|
||||
huge = "Stil " + ("Beispielwort " * 400)
|
||||
from engine import preview_prompt
|
||||
from prompt_budget import estimate_tokens
|
||||
|
||||
rendered = preview_prompt(prompt, assembled)["rendered"]
|
||||
budget = JournalBudget(
|
||||
model="fake",
|
||||
purpose="journal_generate",
|
||||
effective_context_window=32_768,
|
||||
reserved_output_tokens=256,
|
||||
safety_margin=0.15,
|
||||
available_input_tokens=estimate_tokens(rendered) + 80,
|
||||
chars_per_token=2.0,
|
||||
)
|
||||
packed, dropped = pack_narration_context(
|
||||
prompt,
|
||||
budget,
|
||||
assembled,
|
||||
style_examples=huge,
|
||||
existing_text="Bestehende Fassung",
|
||||
include_existing=True,
|
||||
)
|
||||
expect("style_examples" in dropped, "budget drops style examples first")
|
||||
expect("Beispielwort" not in (packed.get("style_examples") or ""), "dropped examples leave the packed prompt")
|
||||
compiled = compile_selection(selection_of("with_examples"))
|
||||
bundle = {
|
||||
"requested": dict(compiled.style_context),
|
||||
"brief": "Core: kurze Sätze.",
|
||||
"profile_block": assembled["writing_profile"],
|
||||
"examples_block": huge,
|
||||
"example_rows": [{"kind": "imported_text", "excerpt": huge}],
|
||||
"omitted": [],
|
||||
"effective": {"neutral_fallback": False, "trait_count": 0},
|
||||
}
|
||||
trace = style_application_trace(compiled, bundle, dropped=dropped)
|
||||
expect(trace["effective"]["style_example_count"] == 0, "effective trace has no examples after budget drop")
|
||||
expect(
|
||||
any(item.get("part") == "style_examples" and item.get("reason") == "budget" for item in trace["omitted"]),
|
||||
"budget drop is recorded on the effective trace",
|
||||
)
|
||||
expect(trace["id"] == seed_id("voice", "with_examples"), "trace keeps the requested voice id")
|
||||
expect(trace["revision"] == 1, "trace keeps the requested revision")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
test_compile_flags()
|
||||
test_budget_omission_in_effective_trace()
|
||||
|
||||
from tests.harness import reset_postgres_data
|
||||
|
||||
reset_postgres_data()
|
||||
reset_debug()
|
||||
with TestClient(app) as client:
|
||||
setup = client.post(
|
||||
"/api/auth/setup",
|
||||
json={"email": "voice@example.test", "name": "Ada", "password": "test-pass"},
|
||||
)
|
||||
headers = header(setup.json()["token"])
|
||||
profile_id = setup.json()["profile_id"]
|
||||
install_profile(profile_id)
|
||||
|
||||
space = client.post("/api/journal/spaces", headers=headers, json={"title": "Stil"})
|
||||
day = client.post(
|
||||
f"/api/journal/spaces/{space.json()['id']}/days",
|
||||
headers=headers,
|
||||
json={"calendar_date": "2026-08-29"},
|
||||
)
|
||||
conv = client.post(
|
||||
f"/api/journal/days/{day.json()['day']['id']}/conversations",
|
||||
headers=headers,
|
||||
json={"title": "Tag"},
|
||||
)
|
||||
turn = client.post(
|
||||
f"/api/journal/conversations/{conv.json()['id']}/turn",
|
||||
headers=headers,
|
||||
json={"body": f"Ich war am Markt. {TODAY_MARK}. Vielleicht bleibe ich kürzer."},
|
||||
)
|
||||
expect(turn.status_code == 200, f"turn {turn.status_code}")
|
||||
day_id = day.json()["day"]["id"]
|
||||
conv_id = conv.json()["id"]
|
||||
|
||||
def generate(voice_key: str) -> dict:
|
||||
reset_debug()
|
||||
recorder = install_test_recorder()
|
||||
result = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={
|
||||
"conversation_ids": [conv_id],
|
||||
"generation_selection": selection_of(voice_key),
|
||||
"remember_generation_selection": False,
|
||||
},
|
||||
)
|
||||
expect(result.status_code == 200, f"{voice_key} generate {result.status_code}")
|
||||
payload = result.json()
|
||||
intern = intern_of(payload)
|
||||
style = style_region(intern)
|
||||
app_trace = (payload.get("trace") or {}).get("style_application") or {}
|
||||
calls = [item.get("purpose") for item in recorder]
|
||||
expect(calls == ["journal_generate"], f"{voice_key} still uses exactly one narration call")
|
||||
expect(app_trace.get("id") == seed_id("voice", voice_key), f"{voice_key} id reaches the trace")
|
||||
expect(app_trace.get("key") == voice_key, f"{voice_key} key reaches the trace")
|
||||
expect(app_trace.get("revision") == 1, f"{voice_key} revision reaches the trace")
|
||||
expect(
|
||||
(payload.get("trace") or {}).get("generation_selection", {}).get("ids", {}).get("voice")
|
||||
== seed_id("voice", voice_key),
|
||||
f"{voice_key} selection id matches prompt/trace",
|
||||
)
|
||||
expect(TODAY_MARK not in style, f"{voice_key} does not use today as a style example")
|
||||
expect(TODAY_MARK in intern.split("\nCURRENT_DAY_SOURCES\n")[-1], f"{voice_key} keeps today as content")
|
||||
expect("keine neuen Tatsachen" in intern, f"{voice_key} keeps fact fidelity")
|
||||
expect("[[PERSON:01]]" in intern, f"{voice_key} keeps privacy placeholders")
|
||||
return {"payload": payload, "intern": intern, "style": style, "trace": app_trace}
|
||||
|
||||
free = generate("neutral")
|
||||
expect(not has_heading(free["intern"], "WRITING_PROFILE"), "profile-free omits the profile heading")
|
||||
expect(not has_heading(free["intern"], "STYLE_EXAMPLES"), "profile-free omits the examples heading")
|
||||
expect("STYLE_EXAMPLES" not in free["intern"], "profile-free has no orphaned STYLE_EXAMPLES reference")
|
||||
expect(CORE_MARK not in free["intern"], "profile-free sends no core")
|
||||
expect(FACET_MARK not in free["intern"], "profile-free sends no facet")
|
||||
expect(TRAIT_MARK not in free["intern"], "profile-free sends no traits")
|
||||
expect(EXAMPLE_MARK not in free["intern"], "profile-free sends no style examples")
|
||||
expect(free["trace"]["effective"]["include_core"] is False, "profile-free effective core is no")
|
||||
expect(free["trace"]["effective"]["style_example_count"] == 0, "profile-free effective examples are zero")
|
||||
|
||||
light = generate("light")
|
||||
expect(CORE_MARK in light["style"], "light sends core")
|
||||
expect(FACET_MARK not in light["intern"], "light does not send facet even indirectly")
|
||||
expect(TRAIT_MARK not in light["intern"], "light does not send traits even indirectly")
|
||||
expect(EXAMPLE_MARK not in light["intern"], "light does not send examples even indirectly")
|
||||
expect(has_heading(light["intern"], "WRITING_PROFILE"), "light keeps a profile heading")
|
||||
expect(not has_heading(light["intern"], "STYLE_EXAMPLES"), "light omits the examples heading")
|
||||
expect("STYLE_EXAMPLES" not in light["intern"], "light has no orphaned STYLE_EXAMPLES reference")
|
||||
expect(light["trace"]["effective"]["include_core"] is True, "light effective core is yes")
|
||||
expect(light["trace"]["effective"]["include_facet"] is False, "light effective facet is no")
|
||||
expect(light["trace"]["effective"]["trait_count"] == 0, "light effective traits are zero")
|
||||
|
||||
clear = generate("clear")
|
||||
expect(CORE_MARK in clear["style"], "clear sends core")
|
||||
expect(FACET_MARK in clear["style"], "clear sends the relevant facet")
|
||||
expect(TRAIT_MARK in clear["style"], "clear sends traits")
|
||||
expect(EXAMPLE_MARK not in clear["intern"], "clear does not send examples")
|
||||
expect(not has_heading(clear["intern"], "STYLE_EXAMPLES"), "clear omits the examples heading")
|
||||
expect("STYLE_EXAMPLES" not in clear["intern"], "clear has no orphaned STYLE_EXAMPLES reference")
|
||||
expect(clear["trace"]["effective"]["include_core"] is True, "clear effective core is yes")
|
||||
expect(clear["trace"]["effective"]["include_facet"] is True, "clear effective facet is yes")
|
||||
expect(clear["trace"]["effective"]["trait_count"] >= 1, "clear effective traits are counted")
|
||||
expect(clear["trace"]["effective"]["style_example_count"] == 0, "clear effective examples are zero")
|
||||
|
||||
examples = generate("with_examples")
|
||||
expect(CORE_MARK in examples["style"], "with-examples sends core")
|
||||
expect(FACET_MARK in examples["style"], "with-examples sends facet")
|
||||
expect(TRAIT_MARK in examples["style"], "with-examples sends traits")
|
||||
expect(EXAMPLE_MARK in examples["style"], "with-examples sends style examples")
|
||||
expect(has_heading(examples["intern"], "STYLE_EXAMPLES"), "with-examples labels the examples block")
|
||||
expect("ausschließlich als Stilreferenz" in examples["intern"] or "sprachliche" in examples["style"], "examples stay style-only")
|
||||
expect("keine Tatsachen des heutigen" in examples["style"] or "nicht übernommen" in examples["style"], "example facts are forbidden")
|
||||
expect(examples["trace"]["effective"]["style_example_count"] >= 1, "with-examples effective count is used")
|
||||
expect(EXAMPLE_MARK not in (examples["payload"].get("body") or ""), "example facts are not copied into today's draft")
|
||||
|
||||
settings = client.get("/api/journal/generation-settings", headers=headers).json()
|
||||
voice_options = settings["options"]["voice"]
|
||||
free_opt = next(item for item in voice_options if item["id"] == seed_id("voice", "neutral"))
|
||||
expect(free_opt["style_context"]["include_core"] is False, "generation-settings expose style context without prompt text")
|
||||
expect("instruction" not in free_opt, "generation-settings still hide prompt wording")
|
||||
|
||||
save_selection(profile_id, selection_of("neutral"))
|
||||
remembered = generate("noticeable")
|
||||
expect(CORE_MARK in remembered["style"], "legacy noticeable remains selectable")
|
||||
profile_run = client.post(
|
||||
f"/api/journal/days/{day_id}/generate",
|
||||
headers=headers,
|
||||
json={"conversation_ids": [conv_id]},
|
||||
)
|
||||
expect(profile_run.status_code == 200, f"stored selection generate {profile_run.status_code}")
|
||||
stored_intern = intern_of(profile_run.json())
|
||||
expect(not has_heading(stored_intern, "WRITING_PROFILE"), "stored profile-free selection stays profile-free")
|
||||
expect(CORE_MARK not in stored_intern, "stored profile-free selection still omits core")
|
||||
|
||||
cloned = clone_guideline(seed_id("voice", "clear"))
|
||||
original_instruction = get_guideline(seed_id("voice", "clear"), include_instruction=True)["instruction"]
|
||||
with get_db() as conn:
|
||||
seed_generation_instructions(conn)
|
||||
expect(
|
||||
get_guideline(seed_id("voice", "clear"), include_instruction=True)["instruction"] == original_instruction,
|
||||
"seed refresh does not replace the published original with a clone",
|
||||
)
|
||||
expect(
|
||||
get_guideline(cloned["id"], include_instruction=True)["style_context"]["include_core"] is True,
|
||||
"cloned voice keeps the structured style context",
|
||||
)
|
||||
|
||||
def compile_one(key: str) -> str:
|
||||
bundle = compile_journal_style_application(profile_id, compile_selection(selection_of(key)).style_context)
|
||||
return (bundle.get("brief") or "") + "\n" + (bundle.get("style_examples") or "")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
first = pool.submit(compile_one, "neutral")
|
||||
second = pool.submit(compile_one, "with_examples")
|
||||
free_text = first.result()
|
||||
example_text = second.result()
|
||||
expect(CORE_MARK not in free_text, "parallel profile-free compile stays empty")
|
||||
expect(CORE_MARK in example_text and EXAMPLE_MARK in example_text, "parallel with-examples compile keeps all parts")
|
||||
expect(EXAMPLE_MARK not in free_text, "parallel requests do not mix style configurations")
|
||||
|
||||
print("journal style context tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loaded 100 of 210 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user