Compare commits
11 Commits
32acdc220b
...
dd7858347c
| Author | SHA1 | Date | |
|---|---|---|---|
| dd7858347c | |||
| 79e4fd0d4b | |||
| 3c0213ae74 | |||
| 9d79e3e92f | |||
| 6ea36f7d1f | |||
| 36b9a2e424 | |||
| 709370da93 | |||
| 0ef3e3bb7a | |||
| ea38528ecf | |||
| 9d5befdde9 | |||
| 07479f6a9f |
16
.dockerignore
Normal file
16
.dockerignore
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
.git
|
||||||
|
.gitea
|
||||||
|
**/.venv
|
||||||
|
**/__pycache__
|
||||||
|
**/*.pyc
|
||||||
|
backend/data
|
||||||
|
backend/.env
|
||||||
|
frontend/node_modules
|
||||||
|
frontend/dist
|
||||||
|
local-backups
|
||||||
|
transfer
|
||||||
|
tmp
|
||||||
|
temp
|
||||||
|
docs
|
||||||
|
*.md
|
||||||
|
!frontend/README.md
|
||||||
27
.env.example
Normal file
27
.env.example
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
# Compose secrets on the Pi. Copy to /home/lars/docker/kansho/.env and
|
||||||
|
# /home/lars/docker/kansho-dev/.env. Never commit the filled file.
|
||||||
|
# Local Windows development keeps using backend/.env (SQLite, ports 5188/8018).
|
||||||
|
|
||||||
|
# ─── DEV (docker-compose.dev-env.yml) ────────────────────────────────────────
|
||||||
|
# DB_NAME=kansho_dev
|
||||||
|
# DB_USER=kansho_dev
|
||||||
|
# DB_PASSWORD=dev_password_change_me
|
||||||
|
# APP_URL=https://dev.kansho.jinkendo.de
|
||||||
|
# ALLOWED_ORIGINS=https://dev.kansho.jinkendo.de,http://192.168.2.49:3096
|
||||||
|
# KANSHO_ENV=development
|
||||||
|
# KANSHO_FRONTEND_PORT=3096
|
||||||
|
# KANSHO_BACKEND_PORT=8096
|
||||||
|
# KANSHO_PROVIDER_KEY=
|
||||||
|
# KANSHO_DETECT_PROVIDER_KEY=
|
||||||
|
|
||||||
|
# ─── PROD (docker-compose.yml) ───────────────────────────────────────────────
|
||||||
|
DB_NAME=kansho
|
||||||
|
DB_USER=kansho
|
||||||
|
DB_PASSWORD=CHANGE_ME_SECURE_PASSWORD
|
||||||
|
APP_URL=https://kansho.jinkendo.de
|
||||||
|
ALLOWED_ORIGINS=https://kansho.jinkendo.de
|
||||||
|
KANSHO_ENV=production
|
||||||
|
KANSHO_FRONTEND_PORT=3006
|
||||||
|
KANSHO_BACKEND_PORT=8005
|
||||||
|
KANSHO_PROVIDER_KEY=
|
||||||
|
KANSHO_DETECT_PROVIDER_KEY=
|
||||||
40
.gitea/workflows/deploy-dev.yml
Normal file
40
.gitea/workflows/deploy-dev.yml
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
name: Deploy Development
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [develop]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Deploy Kanshō to development
|
||||||
|
run: |
|
||||||
|
set -e
|
||||||
|
echo "=== Deploying Kanshō to DEVELOPMENT ==="
|
||||||
|
REPO="http://192.168.2.144:3000/Lars/Kansho.git"
|
||||||
|
TARGET="/home/lars/docker/kansho-dev"
|
||||||
|
mkdir -p "$TARGET"
|
||||||
|
cd "$TARGET"
|
||||||
|
if [ ! -d .git ]; then
|
||||||
|
git clone -b develop "$REPO" .
|
||||||
|
else
|
||||||
|
git fetch origin develop
|
||||||
|
git checkout develop
|
||||||
|
git reset --hard origin/develop
|
||||||
|
fi
|
||||||
|
docker compose -f docker-compose.dev-env.yml build --no-cache backend frontend
|
||||||
|
if ! docker compose -f docker-compose.dev-env.yml up -d --wait; then
|
||||||
|
echo "compose up --wait failed — backend logs:"
|
||||||
|
docker compose -f docker-compose.dev-env.yml logs backend --tail 150 || true
|
||||||
|
docker compose -f docker-compose.dev-env.yml ps || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! curl -sf http://localhost:8096/api/health; then
|
||||||
|
echo "DEV API not reachable — backend logs:"
|
||||||
|
docker compose -f docker-compose.dev-env.yml logs backend --tail 150 || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "DEV API /api/health OK"
|
||||||
|
curl -sf http://localhost:3096/api/health && echo "DEV frontend proxy /api/health OK"
|
||||||
|
echo "=== Kanshō DEV deploy complete ==="
|
||||||
40
.gitea/workflows/deploy-prod.yml
Normal file
40
.gitea/workflows/deploy-prod.yml
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
name: Deploy Production
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Deploy Kanshō to production
|
||||||
|
run: |
|
||||||
|
set -e
|
||||||
|
echo "=== Deploying Kanshō to PRODUCTION ==="
|
||||||
|
REPO="http://192.168.2.144:3000/Lars/Kansho.git"
|
||||||
|
TARGET="/home/lars/docker/kansho"
|
||||||
|
mkdir -p "$TARGET"
|
||||||
|
cd "$TARGET"
|
||||||
|
if [ ! -d .git ]; then
|
||||||
|
git clone -b main "$REPO" .
|
||||||
|
else
|
||||||
|
git fetch origin main
|
||||||
|
git checkout main
|
||||||
|
git reset --hard origin/main
|
||||||
|
fi
|
||||||
|
docker compose build --no-cache backend frontend
|
||||||
|
if ! docker compose up -d --wait; then
|
||||||
|
echo "compose up --wait failed — backend logs:"
|
||||||
|
docker compose logs backend --tail 150 || true
|
||||||
|
docker compose ps || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! curl -sf http://localhost:8005/api/health; then
|
||||||
|
echo "PROD API not reachable — backend logs:"
|
||||||
|
docker compose logs backend --tail 150 || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "PROD API /api/health OK"
|
||||||
|
curl -sf http://localhost:3006/api/health && echo "PROD frontend proxy /api/health OK"
|
||||||
|
echo "=== Kanshō PROD deploy complete ==="
|
||||||
89
.gitea/workflows/test.yml
Normal file
89
.gitea/workflows/test.yml
Normal file
|
|
@ -0,0 +1,89 @@
|
||||||
|
name: Test Suite
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [develop]
|
||||||
|
workflow_run:
|
||||||
|
workflows: ["Deploy Development"]
|
||||||
|
types: [completed]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
pytest-backend:
|
||||||
|
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: pytest against Dev Postgres (kansho_test)
|
||||||
|
run: |
|
||||||
|
set -e
|
||||||
|
APP_DIR="/home/lars/docker/kansho-dev"
|
||||||
|
COMPOSE_FILE="docker-compose.dev-env.yml"
|
||||||
|
echo "pytest uses Dev Postgres (${APP_DIR}); event=${{ github.event_name }}"
|
||||||
|
cd "$APP_DIR"
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
if [ -f "$APP_DIR/backend/pytest.ini" ] && docker compose -f "$COMPOSE_FILE" exec -T backend true 2>/dev/null; then
|
||||||
|
echo "backend and pytest.ini ready (attempt $i)"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [ "$i" -eq 60 ]; then
|
||||||
|
echo "timeout waiting for Dev checkout/backend"
|
||||||
|
docker compose -f "$COMPOSE_FILE" ps || true
|
||||||
|
ls -la "$APP_DIR/backend" || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
PGUSER="$(docker compose -f "$COMPOSE_FILE" exec -T postgres printenv POSTGRES_USER | tr -d '\r')"
|
||||||
|
echo "test database kansho_test on the existing Dev Postgres (owner ${PGUSER})"
|
||||||
|
docker compose -f "$COMPOSE_FILE" exec -T postgres \
|
||||||
|
psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \
|
||||||
|
-c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'kansho_test' AND pid <> pg_backend_pid();"
|
||||||
|
docker compose -f "$COMPOSE_FILE" exec -T postgres \
|
||||||
|
psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \
|
||||||
|
-c "DROP DATABASE IF EXISTS kansho_test;"
|
||||||
|
docker compose -f "$COMPOSE_FILE" exec -T postgres \
|
||||||
|
psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \
|
||||||
|
-c "CREATE DATABASE kansho_test OWNER ${PGUSER};"
|
||||||
|
docker compose -f "$COMPOSE_FILE" run --rm --no-deps \
|
||||||
|
-v "${APP_DIR}:/src:ro" \
|
||||||
|
-w /src/backend \
|
||||||
|
-e KANSHO_DB_BACKEND=postgres \
|
||||||
|
-e DB_NAME=kansho_test \
|
||||||
|
-e KANSHO_PROVIDER_KEY= \
|
||||||
|
-e KANSHO_DETECT_PROVIDER_KEY= \
|
||||||
|
-e PYTHONUTF8=1 \
|
||||||
|
backend sh -lc '
|
||||||
|
set -e
|
||||||
|
unset KANSHO_DB_PATH KANSHO_FAKE_PROVIDER KANSHO_FAKE_DETECT
|
||||||
|
unset KANSHO_PROVIDER_KEY KANSHO_DETECT_PROVIDER_KEY
|
||||||
|
export KANSHO_PROVIDER_KEY=
|
||||||
|
export KANSHO_DETECT_PROVIDER_KEY=
|
||||||
|
pip install -q -r requirements-dev.txt
|
||||||
|
python -m pytest tests -m "not slow" -ra --tb=short
|
||||||
|
'
|
||||||
|
|
||||||
|
smoke-dev:
|
||||||
|
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Non-destructive smoke against the running Dev app
|
||||||
|
run: |
|
||||||
|
set -e
|
||||||
|
echo "live Dev API (kansho_dev), read-only health"
|
||||||
|
curl -sf http://localhost:8096/api/health
|
||||||
|
echo
|
||||||
|
curl -sf http://localhost:3096/api/health
|
||||||
|
echo
|
||||||
|
echo "LAN smoke OK. Public URL after TLS: https://dev.kansho.jinkendo.de"
|
||||||
|
|
||||||
|
frontend-build:
|
||||||
|
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Frontend production build on Dev checkout
|
||||||
|
run: |
|
||||||
|
set -e
|
||||||
|
APP_DIR="/home/lars/docker/kansho-dev"
|
||||||
|
cd "$APP_DIR/frontend"
|
||||||
|
npm ci
|
||||||
|
npm run build
|
||||||
8
.gitignore
vendored
8
.gitignore
vendored
|
|
@ -38,7 +38,7 @@ Thumbs.db
|
||||||
*.py[cod]
|
*.py[cod]
|
||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
|
|
||||||
# Test / local data
|
# Local data (never commit)
|
||||||
tmp/
|
tmp/
|
||||||
temp/
|
temp/
|
||||||
data/local/
|
data/local/
|
||||||
|
|
@ -47,7 +47,5 @@ local-backups/
|
||||||
*.kansho-backup.zip
|
*.kansho-backup.zip
|
||||||
*.db
|
*.db
|
||||||
|
|
||||||
# One-time self-hosted Gitea transport of the laptop backup (2026-09-07).
|
# Personal journal zip was a one-time Gitea transport. Do not re-add.
|
||||||
# Not a product datastore. Remove transfer/ after restore.
|
transfer/
|
||||||
!transfer/
|
|
||||||
!transfer/*.zip
|
|
||||||
|
|
|
||||||
|
|
@ -91,14 +91,9 @@ Restore bestätigt ausdrücklich, legt vorher ein Sicherheitsbackup an und über
|
||||||
|
|
||||||
## Transfer auf die Heim-Umgebung
|
## Transfer auf die Heim-Umgebung
|
||||||
|
|
||||||
Persönliche Daten liegen **nicht** beim Provider. Der einmalige Laptop-Transport (kein Stick/NAS auf diesem Gerät) läuft über das selbst gehostete Gitea:
|
Persönliche Daten liegen **nicht** beim Provider. Der einmalige Laptop-Transport lief über das selbst gehostete Gitea. Nach dem Restore auf dem Heimrechner `transfer/` aus dem Arbeitsbaum entfernen (Historie behält das Zip).
|
||||||
|
|
||||||
1. `git pull` muss `transfer/kansho-laptop-20260907.zip` enthalten.
|
Docker/Postgres, Gitea-Deploy und Cutover: `docs/DEPLOYMENT.md` und `docs/architecture/technical/runtime_and_deploy.md`.
|
||||||
2. `.\scripts\dev-setup.ps1`, `backend/.env` aus der Example plus Keys.
|
|
||||||
3. Restore: `.\scripts\backup-local.ps1 restore -Archive .\transfer\kansho-laptop-20260907.zip -Confirm -Replace`
|
|
||||||
4. Docker/Postgres sind Welle 2, nicht der erste Restore.
|
|
||||||
|
|
||||||
Details: `docs/architecture/technical/environment_handover.md`, `transfer/README.md`.
|
|
||||||
|
|
||||||
## Lokal weiterarbeiten
|
## Lokal weiterarbeiten
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -22,3 +22,11 @@ KANSHO_DETECT_PROVIDER_KEY=
|
||||||
# Nur Tests
|
# Nur Tests
|
||||||
# KANSHO_FAKE_PROVIDER=1
|
# KANSHO_FAKE_PROVIDER=1
|
||||||
# KANSHO_FAKE_DETECT=1
|
# KANSHO_FAKE_DETECT=1
|
||||||
|
|
||||||
|
# Docker/Server only. Local Windows stays SQLite (default).
|
||||||
|
# KANSHO_DB_BACKEND=postgres
|
||||||
|
# DB_HOST=postgres
|
||||||
|
# DB_NAME=kansho
|
||||||
|
# DB_USER=kansho
|
||||||
|
# DB_PASSWORD=
|
||||||
|
|
||||||
|
|
|
||||||
18
backend/Dockerfile
Normal file
18
backend/Dockerfile
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
FROM python:3.12-slim
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
ENV PIP_DEFAULT_TIMEOUT=120
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
RUN chmod +x /app/startup.sh \
|
||||||
|
&& mkdir -p /app/data/media
|
||||||
|
|
||||||
|
EXPOSE 8000
|
||||||
|
|
||||||
|
CMD ["/app/startup.sh"]
|
||||||
|
|
@ -38,8 +38,8 @@
|
||||||
"category": "privacy",
|
"category": "privacy",
|
||||||
"prompt_type": "base",
|
"prompt_type": "base",
|
||||||
"required_feature": "ai_calls",
|
"required_feature": "ai_calls",
|
||||||
"seed_revision": "2026-08-27-detect-ground-v1",
|
"seed_revision": "2026-09-08-detect-homonym-v1",
|
||||||
"template": "Untersuche den gesamten Text semantisch. Entscheide kontextabhängig, nicht nach Großschreibung oder Namensähnlichkeit allein.\n\nSchützenswert ist eine konkrete Bezeichnung oder Information, durch die eine natürliche Person, ein genauer persönlicher Ort, eine Organisation, ein privates Projekt oder ein anderer im Produktrahmen definierter Identifikator erkennbar werden kann. Allgemeine Gegenstände, Tätigkeiten, Lebensmittel, Räume und Beschreibungen sind keine Identitäten, sofern sie im konkreten Satz nicht als Eigenname oder identifizierende Information verwendet werden.\n\nErlaubte entity_type-Werte ausschließlich: PERSON, PLACE, ORG, PROJECT.\n\nKontrast, keine Wortliste:\n- «Sushi kam ins Wohnzimmer» kann eine Person bezeichnen.\n- «Ich aß Sushi» bezeichnet ein Gericht.\n- «Ich arbeitete am privaten Projekt Aurora» kann ein schützenswertes Projekt bezeichnen.\n- «Ich ging auf den Balkon» bezeichnet keinen Eigennamen.\n- «Ich traf Anna» bezeichnet eine Person.\n\nNur das identifizierende Wort oder die identifizierende Wortgruppe, nicht den ganzen Satz. Offsets beziehen sich ausschließlich auf den gelieferten Text nach «Text:», 0-basiert, end ausschließlich. text muss ein exakter Substring dieses Texts sein, nicht aus den Beispielen oben. Wenn Offsets unsicher sind, kopiere trotzdem das identifizierende Wort zeichengetreu.\n\nKeine Tokens, keine Platzhalter, keine kanonischen Namen, keine Aliase, keine zusätzlichen Felder.\n\nAntworte nur mit JSON. Form: {\"entities\":[{\"start\":<zahl>,\"end\":<zahl>,\"text\":\"<exakter Substring>\",\"entity_type\":\"PERSON\"}]}\nNichts schützenswert: {\"entities\":[]}\n\nText:\n{{source_text}}\n"
|
"template": "Untersuche den gesamten Text semantisch. Entscheide kontextabhängig, nicht nach Großschreibung oder Namensähnlichkeit allein.\n\nSchützenswert ist eine konkrete Bezeichnung oder Information, durch die eine natürliche Person, ein genauer persönlicher Ort, eine Organisation, ein privates Projekt oder ein anderer im Produktrahmen definierter Identifikator erkennbar werden kann. Allgemeine Gegenstände, Tätigkeiten, Lebensmittel, Räume und Beschreibungen sind keine Identitäten, sofern sie im konkreten Satz nicht als Eigenname oder identifizierende Information verwendet werden.\n\nErlaubte entity_type-Werte ausschließlich: PERSON, PLACE, ORG, PROJECT.\nEin Lebensmittel oder Gericht ist keine Entität. Nicht als FOOD oder einen anderen Typ melden, sondern weglassen. Derselbe Wortlaut kann im selben Text Person und Gericht sein; nur die identifizierende Nennung gehört in entities.\n\nKontrast, keine Wortliste:\n- «Sushi kam ins Wohnzimmer» kann eine Person bezeichnen.\n- «Ich aß Sushi» bezeichnet ein Gericht.\n- «Ich arbeitete am privaten Projekt Aurora» kann ein schützenswertes Projekt bezeichnen.\n- «Ich ging auf den Balkon» bezeichnet keinen Eigennamen.\n- «Ich traf Anna» bezeichnet eine Person.\n\nNur das identifizierende Wort oder die identifizierende Wortgruppe, nicht den ganzen Satz. Offsets beziehen sich ausschließlich auf den gelieferten Text nach «Text:», 0-basiert, end ausschließlich. text muss ein exakter Substring dieses Texts sein, nicht aus den Beispielen oben. Wenn Offsets unsicher sind, kopiere trotzdem das identifizierende Wort zeichengetreu.\n\nKeine Tokens, keine Platzhalter, keine kanonischen Namen, keine Aliase, keine zusätzlichen Felder.\n\nAntworte nur mit JSON. Form: {\"entities\":[{\"start\":<zahl>,\"end\":<zahl>,\"text\":\"<exakter Substring>\",\"entity_type\":\"PERSON\"}]}\nNichts schützenswert: {\"entities\":[]}\n\nText:\n{{source_text}}\n"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "mvp-profile-review",
|
"id": "mvp-profile-review",
|
||||||
|
|
|
||||||
173
backend/db.py
173
backend/db.py
|
|
@ -1,4 +1,8 @@
|
||||||
"""SQLite persistence for the local frame. PostgreSQL remains the later target."""
|
"""Persistence for the local frame.
|
||||||
|
|
||||||
|
SQLite is the Windows/test engine. PostgreSQL 16 is the Docker/server engine.
|
||||||
|
Switch at connect time via KANSHO_DB_BACKEND / DB_HOST; stores keep SQLite-shaped SQL.
|
||||||
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
@ -6,14 +10,49 @@ import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
DATA_DIR = Path(__file__).resolve().parent / "data"
|
from sql_compat import (
|
||||||
|
adapt_sql,
|
||||||
|
postgres_connect_kwargs,
|
||||||
|
rewrite_catalog_sql,
|
||||||
|
split_sql,
|
||||||
|
sqlite_schema_to_postgres,
|
||||||
|
use_postgres,
|
||||||
|
)
|
||||||
|
|
||||||
|
_env_data = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||||
|
DATA_DIR = Path(_env_data) if _env_data else Path(__file__).resolve().parent / "data"
|
||||||
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
||||||
SEED_PATH = Path(__file__).resolve().parent / "config" / "platform_seed.json"
|
SEED_PATH = Path(__file__).resolve().parent / "config" / "platform_seed.json"
|
||||||
PROMPTS_SEED_PATH = Path(__file__).resolve().parent / "config" / "prompts.seed.json"
|
PROMPTS_SEED_PATH = Path(__file__).resolve().parent / "config" / "prompts.seed.json"
|
||||||
_env_db = os.environ.get("KANSHO_DB_PATH")
|
_env_db = os.environ.get("KANSHO_DB_PATH")
|
||||||
DB_PATH = Path(_env_db) if _env_db else DATA_DIR / "kansho.sqlite"
|
DB_PATH = Path(_env_db) if _env_db else DATA_DIR / "kansho.sqlite"
|
||||||
|
|
||||||
|
SQLITE_HISTORY_MIGRATIONS = (
|
||||||
|
"001_frame",
|
||||||
|
"002_platform",
|
||||||
|
"003_dialogue_memory",
|
||||||
|
"004_mvp_journal",
|
||||||
|
"005_provider_settings",
|
||||||
|
"006_writing_profile_dialogue_style",
|
||||||
|
"007_journal_day_scratch",
|
||||||
|
"008_journal_source_refs",
|
||||||
|
"009_conversation_signals",
|
||||||
|
"010_profile_governance",
|
||||||
|
"011_profile_review",
|
||||||
|
"012_profile_shell",
|
||||||
|
"013_journal_generate_narration",
|
||||||
|
"014_identity_registry",
|
||||||
|
"015_journal_generation_settings",
|
||||||
|
"016_generation_instruction_fragments",
|
||||||
|
"017_generation_guidelines",
|
||||||
|
"018_debug_runs",
|
||||||
|
"019_debug_run_placement",
|
||||||
|
"020_voice_style_context",
|
||||||
|
"021_voice_legacy_immutable",
|
||||||
|
)
|
||||||
|
|
||||||
_PROFILE_COLUMNS = {
|
_PROFILE_COLUMNS = {
|
||||||
"status": "TEXT NOT NULL DEFAULT 'active'",
|
"status": "TEXT NOT NULL DEFAULT 'active'",
|
||||||
"tier_id": "TEXT NOT NULL DEFAULT 'local'",
|
"tier_id": "TEXT NOT NULL DEFAULT 'local'",
|
||||||
|
|
@ -88,7 +127,61 @@ _IDENTITY_MAPPING_COLUMNS = {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _connect() -> sqlite3.Connection:
|
class PostgresCompat:
|
||||||
|
"""sqlite3-like execute/commit surface over psycopg, with SQL translation."""
|
||||||
|
|
||||||
|
def __init__(self, raw: Any):
|
||||||
|
self._raw = raw
|
||||||
|
|
||||||
|
def execute(self, sql: str, params: Any = None):
|
||||||
|
special = rewrite_catalog_sql(sql)
|
||||||
|
if special:
|
||||||
|
adapted, extra = special
|
||||||
|
args = extra if extra is not None else params
|
||||||
|
else:
|
||||||
|
adapted = adapt_sql(sql)
|
||||||
|
args = params
|
||||||
|
if args is None:
|
||||||
|
return self._raw.execute(adapted)
|
||||||
|
return self._raw.execute(adapted, args)
|
||||||
|
|
||||||
|
def executemany(self, sql: str, seq_of_params: Any):
|
||||||
|
adapted = adapt_sql(sql)
|
||||||
|
cursor = None
|
||||||
|
for params in seq_of_params:
|
||||||
|
cursor = self._raw.execute(adapted, params)
|
||||||
|
return cursor
|
||||||
|
|
||||||
|
def executescript(self, script: str):
|
||||||
|
for statement in split_sql(sqlite_schema_to_postgres(script)):
|
||||||
|
self._raw.execute(statement)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def commit(self):
|
||||||
|
self._raw.commit()
|
||||||
|
|
||||||
|
def rollback(self):
|
||||||
|
self._raw.rollback()
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self._raw.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _connect_postgres() -> PostgresCompat:
|
||||||
|
import psycopg
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
|
||||||
|
kwargs = postgres_connect_kwargs()
|
||||||
|
if "conninfo" in kwargs:
|
||||||
|
raw = psycopg.connect(kwargs["conninfo"], row_factory=dict_row)
|
||||||
|
else:
|
||||||
|
raw = psycopg.connect(row_factory=dict_row, **kwargs)
|
||||||
|
return PostgresCompat(raw)
|
||||||
|
|
||||||
|
|
||||||
|
def _connect():
|
||||||
|
if use_postgres():
|
||||||
|
return _connect_postgres()
|
||||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
conn = sqlite3.connect(DB_PATH, timeout=30)
|
conn = sqlite3.connect(DB_PATH, timeout=30)
|
||||||
conn.row_factory = sqlite3.Row
|
conn.row_factory = sqlite3.Row
|
||||||
|
|
@ -109,32 +202,32 @@ def get_db():
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
def row_to_dict(row: sqlite3.Row | None) -> dict | None:
|
def row_to_dict(row: Any) -> dict | None:
|
||||||
if row is None:
|
if row is None:
|
||||||
return None
|
return None
|
||||||
return dict(row)
|
return dict(row)
|
||||||
|
|
||||||
|
|
||||||
def _column_names(conn: sqlite3.Connection, table: str) -> set[str]:
|
def _column_names(conn: Any, table: str) -> set[str]:
|
||||||
rows = conn.execute(f"PRAGMA table_info({table})").fetchall()
|
rows = conn.execute(f"PRAGMA table_info({table})").fetchall()
|
||||||
return {row["name"] for row in rows}
|
return {row["name"] for row in rows}
|
||||||
|
|
||||||
|
|
||||||
def _ensure_columns(conn: sqlite3.Connection, table: str, columns: dict[str, str]) -> None:
|
def _ensure_columns(conn: Any, table: str, columns: dict[str, str]) -> None:
|
||||||
existing = _column_names(conn, table)
|
existing = _column_names(conn, table)
|
||||||
for name, ddl in columns.items():
|
for name, ddl in columns.items():
|
||||||
if name not in existing:
|
if name not in existing:
|
||||||
conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}")
|
conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}")
|
||||||
|
|
||||||
|
|
||||||
def _mark(conn: sqlite3.Connection, migration_id: str) -> None:
|
def _mark(conn: Any, migration_id: str) -> None:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR IGNORE INTO schema_migrations (id) VALUES (?)",
|
"INSERT OR IGNORE INTO schema_migrations (id) VALUES (?)",
|
||||||
(migration_id,),
|
(migration_id,),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _seed_platform(conn: sqlite3.Connection) -> None:
|
def _seed_platform(conn: Any) -> None:
|
||||||
seed = json.loads(SEED_PATH.read_text(encoding="utf-8"))
|
seed = json.loads(SEED_PATH.read_text(encoding="utf-8"))
|
||||||
for tier in seed.get("tiers", []):
|
for tier in seed.get("tiers", []):
|
||||||
conn.execute(
|
conn.execute(
|
||||||
|
|
@ -171,7 +264,7 @@ def _seed_platform(conn: sqlite3.Connection) -> None:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _seed_prompts(conn: sqlite3.Connection) -> None:
|
def _seed_prompts(conn: Any) -> None:
|
||||||
"""Prompts come from JSON/DB, never from Python string literals.
|
"""Prompts come from JSON/DB, never from Python string literals.
|
||||||
|
|
||||||
Untouched system prompts (template == default_template) receive the seed.
|
Untouched system prompts (template == default_template) receive the seed.
|
||||||
|
|
@ -214,7 +307,7 @@ def _seed_prompts(conn: sqlite3.Connection) -> None:
|
||||||
SET name = ?, description = ?, category = ?, prompt_type = ?,
|
SET name = ?, description = ?, category = ?, prompt_type = ?,
|
||||||
required_feature = ?, is_system_default = 1, default_template = ?,
|
required_feature = ?, is_system_default = 1, default_template = ?,
|
||||||
seed_revision = ?,
|
seed_revision = ?,
|
||||||
template = CASE WHEN ? THEN ? ELSE template END,
|
template = CASE WHEN ? = 1 THEN ? ELSE template END,
|
||||||
updated = datetime('now')
|
updated = datetime('now')
|
||||||
WHERE slug = ?
|
WHERE slug = ?
|
||||||
""",
|
""",
|
||||||
|
|
@ -246,7 +339,7 @@ def _parse_legacy_ids(raw: str | None) -> list[str]:
|
||||||
|
|
||||||
|
|
||||||
def _insert_source_refs(
|
def _insert_source_refs(
|
||||||
conn: sqlite3.Connection,
|
conn: Any,
|
||||||
table: str,
|
table: str,
|
||||||
owner_col: str,
|
owner_col: str,
|
||||||
owner_id: str,
|
owner_id: str,
|
||||||
|
|
@ -275,7 +368,7 @@ def _insert_source_refs(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def migrate_journal_source_refs(conn: sqlite3.Connection) -> None:
|
def migrate_journal_source_refs(conn: Any) -> None:
|
||||||
"""Copy JSON id lists into relational source-ref tables without touching Source."""
|
"""Copy JSON id lists into relational source-ref tables without touching Source."""
|
||||||
existing = {
|
existing = {
|
||||||
row["name"]
|
row["name"]
|
||||||
|
|
@ -323,7 +416,7 @@ def migrate_journal_source_refs(conn: sqlite3.Connection) -> None:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _migrate_writing_profile_dialogue_style(conn: sqlite3.Connection) -> None:
|
def _migrate_writing_profile_dialogue_style(conn: Any) -> None:
|
||||||
"""Existing DBs keep the old CHECK; recreate so dialogue_style is a valid source kind."""
|
"""Existing DBs keep the old CHECK; recreate so dialogue_style is a valid source kind."""
|
||||||
row = row_to_dict(
|
row = row_to_dict(
|
||||||
conn.execute(
|
conn.execute(
|
||||||
|
|
@ -354,7 +447,7 @@ def _migrate_writing_profile_dialogue_style(conn: sqlite3.Connection) -> None:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _migrate_writing_profile_shell(conn: sqlite3.Connection) -> None:
|
def _migrate_writing_profile_shell(conn: Any) -> None:
|
||||||
"""Layers stay; inferred count-facets are not the profile. Manual style keys become traits."""
|
"""Layers stay; inferred count-facets are not the profile. Manual style keys become traits."""
|
||||||
from writing_profile_schema import LEGACY_STYLE_KEYS, coerce_slug, facet_layer, normalize_facet_key
|
from writing_profile_schema import LEGACY_STYLE_KEYS, coerce_slug, facet_layer, normalize_facet_key
|
||||||
|
|
||||||
|
|
@ -449,7 +542,36 @@ def _migrate_writing_profile_shell(conn: sqlite3.Connection) -> None:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_runtime(conn: Any) -> None:
|
||||||
|
_seed_platform(conn)
|
||||||
|
_seed_prompts(conn)
|
||||||
|
from provider_settings import seed_provider_settings
|
||||||
|
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
|
||||||
|
|
||||||
|
seed_provider_settings(conn)
|
||||||
|
seed_generation_instructions(conn)
|
||||||
|
backfill_missing_settings(conn)
|
||||||
|
seed_generation_instructions(conn)
|
||||||
|
seed_generation_instructions(conn)
|
||||||
|
|
||||||
|
|
||||||
|
def refresh_runtime_seed() -> None:
|
||||||
|
"""Re-seed catalog rows after a data truncate. Schema must already exist."""
|
||||||
|
with get_db() as conn:
|
||||||
|
_seed_runtime(conn)
|
||||||
|
from writing_profile_store import bootstrap_from_existing
|
||||||
|
|
||||||
|
bootstrap_from_existing()
|
||||||
|
|
||||||
|
|
||||||
def init_db() -> None:
|
def init_db() -> None:
|
||||||
|
if use_postgres():
|
||||||
|
from db_init import ensure_postgres_ready
|
||||||
|
|
||||||
|
ensure_postgres_ready()
|
||||||
|
refresh_runtime_seed()
|
||||||
|
return
|
||||||
|
|
||||||
schema = SCHEMA_PATH.read_text(encoding="utf-8")
|
schema = SCHEMA_PATH.read_text(encoding="utf-8")
|
||||||
with get_db() as conn:
|
with get_db() as conn:
|
||||||
conn.executescript(schema)
|
conn.executescript(schema)
|
||||||
|
|
@ -474,35 +596,16 @@ def init_db() -> None:
|
||||||
from identity_store import migrate_legacy_identity_rows
|
from identity_store import migrate_legacy_identity_rows
|
||||||
|
|
||||||
migrate_legacy_identity_rows(conn)
|
migrate_legacy_identity_rows(conn)
|
||||||
_mark(conn, "001_frame")
|
for migration_id in SQLITE_HISTORY_MIGRATIONS:
|
||||||
_mark(conn, "002_platform")
|
_mark(conn, migration_id)
|
||||||
_mark(conn, "003_dialogue_memory")
|
|
||||||
_mark(conn, "004_mvp_journal")
|
|
||||||
_mark(conn, "005_provider_settings")
|
|
||||||
_mark(conn, "006_writing_profile_dialogue_style")
|
|
||||||
_mark(conn, "007_journal_day_scratch")
|
|
||||||
_mark(conn, "008_journal_source_refs")
|
|
||||||
_mark(conn, "009_conversation_signals")
|
|
||||||
_mark(conn, "010_profile_governance")
|
|
||||||
_mark(conn, "011_profile_review")
|
|
||||||
_mark(conn, "012_profile_shell")
|
|
||||||
_mark(conn, "013_journal_generate_narration")
|
|
||||||
_mark(conn, "014_identity_registry")
|
|
||||||
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
|
from journal_generation_policy import backfill_missing_settings, seed_generation_instructions
|
||||||
|
|
||||||
_ensure_columns(conn, "generation_guidelines", _GUIDELINE_COLUMNS)
|
_ensure_columns(conn, "generation_guidelines", _GUIDELINE_COLUMNS)
|
||||||
seed_generation_instructions(conn)
|
seed_generation_instructions(conn)
|
||||||
backfill_missing_settings(conn)
|
backfill_missing_settings(conn)
|
||||||
_mark(conn, "015_journal_generation_settings")
|
|
||||||
_mark(conn, "016_generation_instruction_fragments")
|
|
||||||
_mark(conn, "017_generation_guidelines")
|
|
||||||
_mark(conn, "018_debug_runs")
|
|
||||||
_ensure_columns(conn, "debug_runs", _DEBUG_RUN_COLUMNS)
|
_ensure_columns(conn, "debug_runs", _DEBUG_RUN_COLUMNS)
|
||||||
_mark(conn, "019_debug_run_placement")
|
|
||||||
seed_generation_instructions(conn)
|
seed_generation_instructions(conn)
|
||||||
_mark(conn, "020_voice_style_context")
|
|
||||||
seed_generation_instructions(conn)
|
seed_generation_instructions(conn)
|
||||||
_mark(conn, "021_voice_legacy_immutable")
|
|
||||||
from writing_profile_store import bootstrap_from_existing
|
from writing_profile_store import bootstrap_from_existing
|
||||||
|
|
||||||
bootstrap_from_existing()
|
bootstrap_from_existing()
|
||||||
|
|
|
||||||
167
backend/db_init.py
Normal file
167
backend/db_init.py
Normal file
|
|
@ -0,0 +1,167 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Wait for PostgreSQL, load schema.sql, apply numbered SQL migrations. Fail-fast."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sql_compat import postgres_connect_kwargs, split_sql, sqlite_schema_to_postgres, use_postgres
|
||||||
|
|
||||||
|
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
||||||
|
MIGRATIONS_DIR = Path(__file__).resolve().parent / "migrations"
|
||||||
|
SQLITE_HISTORY_MIGRATIONS = (
|
||||||
|
"001_frame",
|
||||||
|
"002_platform",
|
||||||
|
"003_dialogue_memory",
|
||||||
|
"004_mvp_journal",
|
||||||
|
"005_provider_settings",
|
||||||
|
"006_writing_profile_dialogue_style",
|
||||||
|
"007_journal_day_scratch",
|
||||||
|
"008_journal_source_refs",
|
||||||
|
"009_conversation_signals",
|
||||||
|
"010_profile_governance",
|
||||||
|
"011_profile_review",
|
||||||
|
"012_profile_shell",
|
||||||
|
"013_journal_generate_narration",
|
||||||
|
"014_identity_registry",
|
||||||
|
"015_journal_generation_settings",
|
||||||
|
"016_generation_instruction_fragments",
|
||||||
|
"017_generation_guidelines",
|
||||||
|
"018_debug_runs",
|
||||||
|
"019_debug_run_placement",
|
||||||
|
"020_voice_style_context",
|
||||||
|
"021_voice_legacy_immutable",
|
||||||
|
)
|
||||||
|
_LEADING_DIGITS = re.compile(r"^(\d{3})_.*\.sql$")
|
||||||
|
_schema_ready = False
|
||||||
|
|
||||||
|
|
||||||
|
def mark_schema_dirty() -> None:
|
||||||
|
"""Call after DROP SCHEMA so the next init_db reloads schema.sql."""
|
||||||
|
global _schema_ready
|
||||||
|
_schema_ready = False
|
||||||
|
|
||||||
|
|
||||||
|
def _connect_raw():
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
kwargs = postgres_connect_kwargs()
|
||||||
|
if "conninfo" in kwargs:
|
||||||
|
return psycopg.connect(kwargs["conninfo"], autocommit=False)
|
||||||
|
return psycopg.connect(autocommit=False, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_postgres(max_retries: int = 30) -> None:
|
||||||
|
print("Checking PostgreSQL connection...")
|
||||||
|
last_error = None
|
||||||
|
for attempt in range(1, max_retries + 1):
|
||||||
|
try:
|
||||||
|
conn = _connect_raw()
|
||||||
|
conn.close()
|
||||||
|
print("PostgreSQL ready")
|
||||||
|
return
|
||||||
|
except Exception as exc: # noqa: BLE001 — fail-fast after retries
|
||||||
|
last_error = exc
|
||||||
|
print(f" waiting ({attempt}/{max_retries})")
|
||||||
|
time.sleep(2)
|
||||||
|
print(f"PostgreSQL not ready: {last_error}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def _table_exists(conn, name: str) -> bool:
|
||||||
|
row = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT 1 FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public' AND table_name = %s
|
||||||
|
""",
|
||||||
|
(name,),
|
||||||
|
).fetchone()
|
||||||
|
return row is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _load_schema(conn) -> None:
|
||||||
|
sql = sqlite_schema_to_postgres(SCHEMA_PATH.read_text(encoding="utf-8"))
|
||||||
|
for statement in split_sql(sql):
|
||||||
|
conn.execute(statement)
|
||||||
|
print("Schema loaded from schema.sql (Postgres dialect)")
|
||||||
|
|
||||||
|
|
||||||
|
def _applied_ids(conn) -> set[str]:
|
||||||
|
rows = conn.execute("SELECT id FROM schema_migrations").fetchall()
|
||||||
|
return {row[0] for row in rows}
|
||||||
|
|
||||||
|
|
||||||
|
def _record(conn, migration_id: str) -> None:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO schema_migrations (id) VALUES (%s) ON CONFLICT (id) DO NOTHING",
|
||||||
|
(migration_id,),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _migration_files() -> list[tuple[str, Path]]:
|
||||||
|
if not MIGRATIONS_DIR.is_dir():
|
||||||
|
return []
|
||||||
|
rows: list[tuple[str, Path]] = []
|
||||||
|
for path in sorted(MIGRATIONS_DIR.iterdir()):
|
||||||
|
match = _LEADING_DIGITS.match(path.name)
|
||||||
|
if not match or path.suffix != ".sql":
|
||||||
|
continue
|
||||||
|
rows.append((path.stem, path))
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def apply_schema_and_migrations() -> None:
|
||||||
|
conn = _connect_raw()
|
||||||
|
try:
|
||||||
|
if not _table_exists(conn, "schema_migrations"):
|
||||||
|
_load_schema(conn)
|
||||||
|
conn.commit()
|
||||||
|
applied = _applied_ids(conn)
|
||||||
|
if "001_frame" not in applied:
|
||||||
|
for migration_id in SQLITE_HISTORY_MIGRATIONS:
|
||||||
|
_record(conn, migration_id)
|
||||||
|
conn.commit()
|
||||||
|
print("Recorded SQLite history 001-021 as applied (greenfield Postgres schema)")
|
||||||
|
applied = _applied_ids(conn)
|
||||||
|
for stem, path in _migration_files():
|
||||||
|
if stem in applied:
|
||||||
|
continue
|
||||||
|
sql = path.read_text(encoding="utf-8")
|
||||||
|
for statement in split_sql(sql):
|
||||||
|
conn.execute(statement)
|
||||||
|
_record(conn, stem)
|
||||||
|
conn.commit()
|
||||||
|
print(f"Applied migration {stem}")
|
||||||
|
except Exception as exc:
|
||||||
|
conn.rollback()
|
||||||
|
print(f"Database initialization failed: {exc}")
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_postgres_ready() -> None:
|
||||||
|
global _schema_ready
|
||||||
|
if not use_postgres():
|
||||||
|
return
|
||||||
|
if _schema_ready:
|
||||||
|
return
|
||||||
|
wait_for_postgres()
|
||||||
|
try:
|
||||||
|
apply_schema_and_migrations()
|
||||||
|
except Exception:
|
||||||
|
sys.exit(1)
|
||||||
|
_schema_ready = True
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
os.environ.setdefault("KANSHO_DB_BACKEND", "postgres")
|
||||||
|
ensure_postgres_ready()
|
||||||
|
print("db_init complete")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -289,6 +289,7 @@ def persist_engine_error(
|
||||||
"detect_partial_discarded",
|
"detect_partial_discarded",
|
||||||
"contract_violation",
|
"contract_violation",
|
||||||
"invalid_entity_type",
|
"invalid_entity_type",
|
||||||
|
"omitted_non_identity_types",
|
||||||
"generate_called",
|
"generate_called",
|
||||||
):
|
):
|
||||||
if merged.get(key) is None:
|
if merged.get(key) is None:
|
||||||
|
|
|
||||||
|
|
@ -339,7 +339,7 @@ def list_conversations_for_day(profile_id: str, journal_day_id: str) -> list[dic
|
||||||
(SELECT COUNT(*) FROM messages m WHERE m.conversation_id = c.id) AS message_count
|
(SELECT COUNT(*) FROM messages m WHERE m.conversation_id = c.id) AS message_count
|
||||||
FROM conversations c
|
FROM conversations c
|
||||||
WHERE c.profile_id = ? AND c.journal_day_id = ?
|
WHERE c.profile_id = ? AND c.journal_day_id = ?
|
||||||
ORDER BY c.created, c.rowid
|
ORDER BY c.created, c.id
|
||||||
""",
|
""",
|
||||||
(profile_id, journal_day_id),
|
(profile_id, journal_day_id),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
|
|
||||||
|
|
@ -91,7 +91,7 @@ def increment_feature_usage(profile_id: str, feature_id: str) -> None:
|
||||||
INSERT INTO user_feature_usage (profile_id, feature_id, period_key, used)
|
INSERT INTO user_feature_usage (profile_id, feature_id, period_key, used)
|
||||||
VALUES (?, ?, ?, 1)
|
VALUES (?, ?, ?, 1)
|
||||||
ON CONFLICT(profile_id, feature_id, period_key)
|
ON CONFLICT(profile_id, feature_id, period_key)
|
||||||
DO UPDATE SET used = used + 1
|
DO UPDATE SET used = user_feature_usage.used + 1
|
||||||
""",
|
""",
|
||||||
(profile_id, feature_id, period),
|
(profile_id, feature_id, period),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,16 @@ JSON_BLOCK = re.compile(r"\{.*\}", re.DOTALL)
|
||||||
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
|
ALLOWED_ENTITY_FIELDS = frozenset({"start", "end", "text", "entity_type"})
|
||||||
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
|
ALLOWED_ROOT_FIELDS = frozenset({"entities"})
|
||||||
TYPE_PRIORITY = {"PERSON": 0, "PROJECT": 1, "ORG": 2, "PLACE": 3}
|
TYPE_PRIORITY = {"PERSON": 0, "PROJECT": 1, "ORG": 2, "PLACE": 3}
|
||||||
|
NON_IDENTITY_ENTITY_TYPES = frozenset(
|
||||||
|
{
|
||||||
|
"FOOD",
|
||||||
|
"DISH",
|
||||||
|
"MEAL",
|
||||||
|
"CUISINE",
|
||||||
|
"FOODSTUFF",
|
||||||
|
"INGREDIENT",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
ERROR_DETECT_UNAVAILABLE = "detect_provider_unavailable"
|
ERROR_DETECT_UNAVAILABLE = "detect_provider_unavailable"
|
||||||
ERROR_DETECT_INCOMPLETE = "detect_incomplete"
|
ERROR_DETECT_INCOMPLETE = "detect_incomplete"
|
||||||
|
|
@ -55,6 +65,7 @@ SCHEMA_RETRY_HINT = (
|
||||||
"Antworte ausschließlich mit dem geforderten JSON-Objekt. "
|
"Antworte ausschließlich mit dem geforderten JSON-Objekt. "
|
||||||
"Nur die Felder start, end, text und entity_type sind zulässig. "
|
"Nur die Felder start, end, text und entity_type sind zulässig. "
|
||||||
"Nur entity_type-Werte PERSON, PLACE, ORG, PROJECT. "
|
"Nur entity_type-Werte PERSON, PLACE, ORG, PROJECT. "
|
||||||
|
"Lebensmittel und Gerichte weglassen, nicht als FOOD oder anderen Typ melden. "
|
||||||
"Offsets sind 0-basiert und end ausschließlich, bezogen auf den Text nach «Text:». "
|
"Offsets sind 0-basiert und end ausschließlich, bezogen auf den Text nach «Text:». "
|
||||||
"text muss ein exakter Substring dieses Texts sein. "
|
"text muss ein exakter Substring dieses Texts sein. "
|
||||||
"Keine zusätzlichen Felder, keine Tokens, keine Platzhalter, keine Erklärungen."
|
"Keine zusätzlichen Felder, keine Tokens, keine Platzhalter, keine Erklärungen."
|
||||||
|
|
@ -143,6 +154,7 @@ class DetectionStats:
|
||||||
detect_attempts: list[dict[str, Any]] = field(default_factory=list)
|
detect_attempts: list[dict[str, Any]] = field(default_factory=list)
|
||||||
contract_violation: str | None = None
|
contract_violation: str | None = None
|
||||||
invalid_entity_type: str | None = None
|
invalid_entity_type: str | None = None
|
||||||
|
omitted_non_identity_types: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
def public(self) -> dict[str, Any]:
|
def public(self) -> dict[str, Any]:
|
||||||
payload = {
|
payload = {
|
||||||
|
|
@ -179,6 +191,8 @@ class DetectionStats:
|
||||||
payload["contract_violation"] = self.contract_violation
|
payload["contract_violation"] = self.contract_violation
|
||||||
if self.invalid_entity_type:
|
if self.invalid_entity_type:
|
||||||
payload["invalid_entity_type"] = self.invalid_entity_type
|
payload["invalid_entity_type"] = self.invalid_entity_type
|
||||||
|
if self.omitted_non_identity_types:
|
||||||
|
payload["omitted_non_identity_types"] = list(self.omitted_non_identity_types)
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -433,9 +447,56 @@ def _ground_span(chunk_text: str, start: int, end: int, text: str) -> tuple[int,
|
||||||
return best[0], best[1], source[best[0] : best[1]]
|
return best[0], best[1], source[best[0] : best[1]]
|
||||||
|
|
||||||
|
|
||||||
|
def _delivered_entity_type(raw: Any) -> str:
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
return ""
|
||||||
|
return re.sub(r"[^A-Z0-9_:-]", "", raw.strip().upper())[:40]
|
||||||
|
|
||||||
|
|
||||||
|
def _identity_mention(chunk_text: str, start: int, end: int) -> bool:
|
||||||
|
from privacy_gateway import is_identity_mention
|
||||||
|
|
||||||
|
return is_identity_mention(chunk_text, start, end, "PERSON:00")
|
||||||
|
|
||||||
|
|
||||||
|
def _span_from_non_identity(item: dict, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
|
||||||
|
"""Food/dish types are not identities. Keep only a person-shaped mention of the same word."""
|
||||||
|
text = item.get("text")
|
||||||
|
if not isinstance(text, str) or not text.strip():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
start = int(item.get("start") or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
start = 0
|
||||||
|
try:
|
||||||
|
end = int(item.get("end") or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
end = 0
|
||||||
|
grounded = _ground_span(chunk_text, start, end, text)
|
||||||
|
if not grounded:
|
||||||
|
return None
|
||||||
|
left, right, local_text = grounded
|
||||||
|
if not _identity_mention(chunk_text, left, right):
|
||||||
|
return None
|
||||||
|
return DetectedSpan(
|
||||||
|
start=left,
|
||||||
|
end=right,
|
||||||
|
text=local_text,
|
||||||
|
entity_type="PERSON",
|
||||||
|
chunk_index=chunk_index,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
|
def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> DetectedSpan | None:
|
||||||
if not isinstance(item, dict):
|
if not isinstance(item, dict):
|
||||||
raise _contract_error("Detect-Entity muss ein Objekt sein.", violation="invalid_entity")
|
raise _contract_error("Detect-Entity muss ein Objekt sein.", violation="invalid_entity")
|
||||||
|
if "token" in item or "placeholder" in item:
|
||||||
|
raise _contract_error("Detect darf keine Tokens festlegen.", violation="extra_fields")
|
||||||
|
raw_type = item.get("entity_type")
|
||||||
|
delivered = _delivered_entity_type(raw_type)
|
||||||
|
kind = normalize_entity_type(raw_type if isinstance(raw_type, str) else "", default="")
|
||||||
|
if not kind and delivered in NON_IDENTITY_ENTITY_TYPES:
|
||||||
|
return _span_from_non_identity(item, chunk_text, chunk_index)
|
||||||
extra = set(item.keys()) - ALLOWED_ENTITY_FIELDS
|
extra = set(item.keys()) - ALLOWED_ENTITY_FIELDS
|
||||||
if extra:
|
if extra:
|
||||||
raise _contract_error("Detect-Entity enthält unerwartete Felder.", violation="extra_fields")
|
raise _contract_error("Detect-Entity enthält unerwartete Felder.", violation="extra_fields")
|
||||||
|
|
@ -455,16 +516,12 @@ def validate_detected_entity(item: Any, chunk_text: str, chunk_index: int) -> De
|
||||||
raise _contract_error("Detect-text fehlt.", violation="missing_fields")
|
raise _contract_error("Detect-text fehlt.", violation="missing_fields")
|
||||||
if not text.strip():
|
if not text.strip():
|
||||||
raise _contract_error("Detect-text ist leer.", violation="invalid_text")
|
raise _contract_error("Detect-text ist leer.", violation="invalid_text")
|
||||||
if "token" in item or "placeholder" in item:
|
|
||||||
raise _contract_error("Detect darf keine Tokens festlegen.", violation="extra_fields")
|
|
||||||
raw_type = item.get("entity_type")
|
|
||||||
kind = normalize_entity_type(raw_type if isinstance(raw_type, str) else "", default="")
|
|
||||||
if not kind:
|
if not kind:
|
||||||
delivered = raw_type if isinstance(raw_type, str) else ""
|
delivered_raw = raw_type if isinstance(raw_type, str) else ""
|
||||||
raise _contract_error(
|
raise _contract_error(
|
||||||
"Detect-entity_type ist nicht erlaubt.",
|
"Detect-entity_type ist nicht erlaubt.",
|
||||||
violation="unknown_entity_type",
|
violation="unknown_entity_type",
|
||||||
invalid_entity_type=delivered,
|
invalid_entity_type=delivered_raw,
|
||||||
)
|
)
|
||||||
grounded = _ground_span(chunk_text, start, end, text)
|
grounded = _ground_span(chunk_text, start, end, text)
|
||||||
if not grounded:
|
if not grounded:
|
||||||
|
|
@ -532,7 +589,19 @@ def _llm_chunk(config, excerpt: str, *, schema_retry: bool = False) -> ChatResul
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int) -> list[DetectedSpan]:
|
def _note_omitted_type(stats: DetectionStats | None, delivered: str) -> None:
|
||||||
|
if not stats or not delivered or delivered not in NON_IDENTITY_ENTITY_TYPES:
|
||||||
|
return
|
||||||
|
if delivered not in stats.omitted_non_identity_types:
|
||||||
|
stats.omitted_non_identity_types.append(delivered)
|
||||||
|
|
||||||
|
|
||||||
|
def _entities_from_result(
|
||||||
|
result: ChatResult,
|
||||||
|
chunk_text: str,
|
||||||
|
chunk_index: int,
|
||||||
|
stats: DetectionStats | None = None,
|
||||||
|
) -> list[DetectedSpan]:
|
||||||
if (result.finish_reason or "").lower() in {"length", "max_tokens"}:
|
if (result.finish_reason or "").lower() in {"length", "max_tokens"}:
|
||||||
raise DetectError(
|
raise DetectError(
|
||||||
ERROR_DETECT_TRUNCATED,
|
ERROR_DETECT_TRUNCATED,
|
||||||
|
|
@ -543,7 +612,10 @@ def _entities_from_result(result: ChatResult, chunk_text: str, chunk_index: int)
|
||||||
spans: list[DetectedSpan] = []
|
spans: list[DetectedSpan] = []
|
||||||
for item in data.get("entities") or []:
|
for item in data.get("entities") or []:
|
||||||
span = validate_detected_entity(item, chunk_text, chunk_index)
|
span = validate_detected_entity(item, chunk_text, chunk_index)
|
||||||
if span is not None:
|
if span is None:
|
||||||
|
delivered = _delivered_entity_type(item.get("entity_type") if isinstance(item, dict) else "")
|
||||||
|
_note_omitted_type(stats, delivered)
|
||||||
|
continue
|
||||||
spans.append(span)
|
spans.append(span)
|
||||||
return spans
|
return spans
|
||||||
|
|
||||||
|
|
@ -674,6 +746,9 @@ def _merge_pass_stats(overall: DetectionStats, pass_stats: DetectionStats) -> No
|
||||||
if pass_stats.cost_known:
|
if pass_stats.cost_known:
|
||||||
overall.cost_known = True
|
overall.cost_known = True
|
||||||
overall.cost += pass_stats.cost
|
overall.cost += pass_stats.cost
|
||||||
|
for kind in pass_stats.omitted_non_identity_types:
|
||||||
|
if kind not in overall.omitted_non_identity_types:
|
||||||
|
overall.omitted_non_identity_types.append(kind)
|
||||||
|
|
||||||
|
|
||||||
def _pass_snapshot(
|
def _pass_snapshot(
|
||||||
|
|
@ -706,6 +781,8 @@ def _pass_snapshot(
|
||||||
row["contract_violation"] = pass_stats.contract_violation
|
row["contract_violation"] = pass_stats.contract_violation
|
||||||
if pass_stats.invalid_entity_type:
|
if pass_stats.invalid_entity_type:
|
||||||
row["invalid_entity_type"] = pass_stats.invalid_entity_type
|
row["invalid_entity_type"] = pass_stats.invalid_entity_type
|
||||||
|
if pass_stats.omitted_non_identity_types:
|
||||||
|
row["omitted_non_identity_types"] = list(pass_stats.omitted_non_identity_types)
|
||||||
return row
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -822,7 +899,7 @@ def _run_detection_pass(
|
||||||
)
|
)
|
||||||
stats.detect_calls += 1
|
stats.detect_calls += 1
|
||||||
_add_usage(stats, result.usage)
|
_add_usage(stats, result.usage)
|
||||||
spans = _entities_from_result(result, chunk_text, index)
|
spans = _entities_from_result(result, chunk_text, index, stats)
|
||||||
collected.extend(_to_global(span, offset) for span in spans)
|
collected.extend(_to_global(span, offset) for span in spans)
|
||||||
stats.chunks_ok += 1
|
stats.chunks_ok += 1
|
||||||
if stats.chunks_ok != stats.chunk_count:
|
if stats.chunks_ok != stats.chunk_count:
|
||||||
|
|
|
||||||
|
|
@ -285,7 +285,7 @@ def current_entries(profile_id: str, journal_day_id: str) -> list[dict]:
|
||||||
FROM journal_entries e
|
FROM journal_entries e
|
||||||
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
||||||
WHERE e.profile_id = ? AND e.journal_day_id = ? AND e.deleted_at IS NULL
|
WHERE e.profile_id = ? AND e.journal_day_id = ? AND e.deleted_at IS NULL
|
||||||
ORDER BY e.created, e.rowid
|
ORDER BY e.created, e.id
|
||||||
""",
|
""",
|
||||||
(profile_id, journal_day_id),
|
(profile_id, journal_day_id),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
|
@ -514,7 +514,7 @@ def list_space_entries(profile_id: str, space_id: str) -> list[dict]:
|
||||||
JOIN journal_days d ON d.id = e.journal_day_id
|
JOIN journal_days d ON d.id = e.journal_day_id
|
||||||
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
LEFT JOIN journal_entry_versions v ON v.id = e.current_version_id
|
||||||
WHERE e.profile_id = ? AND e.space_id = ? AND e.deleted_at IS NULL
|
WHERE e.profile_id = ? AND e.space_id = ? AND e.deleted_at IS NULL
|
||||||
ORDER BY d.calendar_date DESC, e.created ASC, e.rowid ASC
|
ORDER BY d.calendar_date DESC, e.created ASC, e.id ASC
|
||||||
""",
|
""",
|
||||||
(profile_id, space_id),
|
(profile_id, space_id),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,8 @@ from env_loader import load_env_file
|
||||||
|
|
||||||
load_env_file()
|
load_env_file()
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
|
|
@ -10,11 +12,32 @@ import data_layer_dialogue # noqa: F401
|
||||||
from routers import admin, auth, dialogue, generation_instructions, journal, placeholders, prompts, subscription, users
|
from routers import admin, auth, dialogue, generation_instructions, journal, placeholders, prompts, subscription, users
|
||||||
from version import APP_VERSION
|
from version import APP_VERSION
|
||||||
|
|
||||||
|
|
||||||
|
def allowed_origins() -> list[str]:
|
||||||
|
origins = [
|
||||||
|
"http://localhost:5188",
|
||||||
|
"http://127.0.0.1:5188",
|
||||||
|
"https://kansho.jinkendo.de",
|
||||||
|
"https://dev.kansho.jinkendo.de",
|
||||||
|
]
|
||||||
|
extra = (os.environ.get("ALLOWED_ORIGINS") or os.environ.get("KANSHO_ALLOWED_ORIGINS") or "").strip()
|
||||||
|
if extra:
|
||||||
|
origins.extend(item.strip() for item in extra.split(",") if item.strip())
|
||||||
|
app_url = (os.environ.get("APP_URL") or "").strip().rstrip("/")
|
||||||
|
if app_url:
|
||||||
|
origins.append(app_url)
|
||||||
|
seen: list[str] = []
|
||||||
|
for item in origins:
|
||||||
|
if item not in seen:
|
||||||
|
seen.append(item)
|
||||||
|
return seen
|
||||||
|
|
||||||
|
|
||||||
app = FastAPI(title="Kanshō", version=APP_VERSION)
|
app = FastAPI(title="Kanshō", version=APP_VERSION)
|
||||||
|
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
CORSMiddleware,
|
CORSMiddleware,
|
||||||
allow_origins=["http://localhost:5188", "http://127.0.0.1:5188"],
|
allow_origins=allowed_origins(),
|
||||||
allow_credentials=True,
|
allow_credentials=True,
|
||||||
allow_methods=["*"],
|
allow_methods=["*"],
|
||||||
allow_headers=["*"],
|
allow_headers=["*"],
|
||||||
|
|
|
||||||
7
backend/migrations/022_postgres_runtime_baseline.sql
Normal file
7
backend/migrations/022_postgres_runtime_baseline.sql
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
-- Postgres runtime baseline for Kanshō.
|
||||||
|
-- Greenfield schema is backend/schema.sql translated at startup
|
||||||
|
-- (datetime('now') → CURRENT_TIMESTAMP::text). Historical SQLite
|
||||||
|
-- inline migrations 001-021 are recorded as applied, not replayed.
|
||||||
|
-- Future incremental DDL belongs in 023_*.sql and later.
|
||||||
|
|
||||||
|
SELECT 1;
|
||||||
|
|
@ -96,6 +96,7 @@ COMPACT_DIAGNOSTIC_KEYS = (
|
||||||
"detect_attempts",
|
"detect_attempts",
|
||||||
"contract_violation",
|
"contract_violation",
|
||||||
"invalid_entity_type",
|
"invalid_entity_type",
|
||||||
|
"omitted_non_identity_types",
|
||||||
"detect_cost_unknown",
|
"detect_cost_unknown",
|
||||||
"generation_selection",
|
"generation_selection",
|
||||||
"style_application",
|
"style_application",
|
||||||
|
|
|
||||||
12
backend/pytest.ini
Normal file
12
backend/pytest.ini
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
[pytest]
|
||||||
|
testpaths = tests
|
||||||
|
# Default python collector is disabled; tests/conftest.py collects each
|
||||||
|
# test_*.py as one case via main(). That keeps the existing scripts until
|
||||||
|
# they are split into native pytest functions.
|
||||||
|
python_files = __pytest_default_collector_disabled__.py
|
||||||
|
python_functions = test_*
|
||||||
|
addopts = -ra --tb=short
|
||||||
|
markers =
|
||||||
|
unit: no Postgres; safe on Windows SQLite checkouts
|
||||||
|
integration: API/store tests on kansho_test (never kansho_dev)
|
||||||
|
slow: long-running; omitted in the default Gitea command
|
||||||
3
backend/requirements-dev.txt
Normal file
3
backend/requirements-dev.txt
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
# CI / local test runner. Install on top of requirements.txt:
|
||||||
|
# pip install -r requirements.txt -r requirements-dev.txt
|
||||||
|
pytest>=8.0
|
||||||
|
|
@ -3,3 +3,4 @@ uvicorn[standard]==0.34.0
|
||||||
bcrypt==4.2.1
|
bcrypt==4.2.1
|
||||||
httpx==0.28.1
|
httpx==0.28.1
|
||||||
python-multipart==0.0.20
|
python-multipart==0.0.20
|
||||||
|
psycopg[binary]==3.2.6
|
||||||
|
|
|
||||||
|
|
@ -95,7 +95,7 @@ def _day_messages(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||||
FROM messages m
|
FROM messages m
|
||||||
JOIN conversations c ON c.id = m.conversation_id
|
JOIN conversations c ON c.id = m.conversation_id
|
||||||
WHERE m.profile_id = ? AND c.journal_day_id = ? AND m.conversation_id IN ({placeholders})
|
WHERE m.profile_id = ? AND c.journal_day_id = ? AND m.conversation_id IN ({placeholders})
|
||||||
ORDER BY c.created, c.rowid, m.seq, m.id
|
ORDER BY c.created, c.id, m.seq, m.id
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
""",
|
""",
|
||||||
(profile_id, journal_day_id, *conversation_ids, fetch_limit),
|
(profile_id, journal_day_id, *conversation_ids, fetch_limit),
|
||||||
|
|
@ -107,7 +107,7 @@ def _day_messages(profile_id: str, spec: dict[str, Any]) -> list[dict]:
|
||||||
FROM messages m
|
FROM messages m
|
||||||
JOIN conversations c ON c.id = m.conversation_id
|
JOIN conversations c ON c.id = m.conversation_id
|
||||||
WHERE m.profile_id = ? AND c.journal_day_id = ?
|
WHERE m.profile_id = ? AND c.journal_day_id = ?
|
||||||
ORDER BY c.created, c.rowid, m.seq, m.id
|
ORDER BY c.created, c.id, m.seq, m.id
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
""",
|
""",
|
||||||
(profile_id, journal_day_id, fetch_limit),
|
(profile_id, journal_day_id, fetch_limit),
|
||||||
|
|
|
||||||
183
backend/sql_compat.py
Normal file
183
backend/sql_compat.py
Normal file
|
|
@ -0,0 +1,183 @@
|
||||||
|
"""Translate SQLite-oriented SQL so the same stores can run against PostgreSQL.
|
||||||
|
|
||||||
|
Windows without Docker still uses SQLite for the running app. Docker/Server and
|
||||||
|
the test suite use PostgreSQL (`KANSHO_DB_BACKEND=postgres`). Identity, keys and
|
||||||
|
journal bodies are not logged here.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
|
||||||
|
_INSERT_OR_IGNORE = re.compile(r"INSERT\s+OR\s+IGNORE\s+INTO", re.IGNORECASE)
|
||||||
|
_ROWID_COL = re.compile(r"\.rowid\b", re.IGNORECASE)
|
||||||
|
_ON_CONFLICT_PAREN = re.compile(r"ON\s+CONFLICT\s*\(", re.IGNORECASE)
|
||||||
|
_PRAGMA_TABLE = re.compile(
|
||||||
|
r"^\s*PRAGMA\s+table_info\(\s*['\"]?(\w+)['\"]?\s*\)\s*;?\s*$",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_SQLITE_MASTER_TABLES = re.compile(
|
||||||
|
r"^\s*SELECT\s+name\s+FROM\s+sqlite_master\s+WHERE\s+type\s*=\s*'table'\s*;?\s*$",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_SQLITE_MASTER_SQL = re.compile(
|
||||||
|
r"^\s*SELECT\s+sql\s+FROM\s+sqlite_master\s+WHERE\s+type\s*=\s*'table'\s+AND\s+name\s*=\s*\?\s*;?\s*$",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def use_postgres() -> bool:
|
||||||
|
"""Decide the engine at connect time, not at import time.
|
||||||
|
|
||||||
|
Explicit KANSHO_DB_BACKEND wins. KANSHO_DB_PATH without an explicit postgres
|
||||||
|
backend keeps the Windows local app on SQLite. Tests set postgres and DB_NAME
|
||||||
|
kansho_test on the Dev instance, never kansho_dev.
|
||||||
|
"""
|
||||||
|
raw = (os.environ.get("KANSHO_DB_BACKEND") or "").strip().lower()
|
||||||
|
if raw in {"sqlite", "sqlite3"}:
|
||||||
|
return False
|
||||||
|
if raw in {"postgres", "postgresql", "pg"}:
|
||||||
|
return True
|
||||||
|
if (os.environ.get("KANSHO_DB_PATH") or "").strip():
|
||||||
|
return False
|
||||||
|
url = (os.environ.get("DATABASE_URL") or "").strip().lower()
|
||||||
|
if url.startswith("postgres"):
|
||||||
|
return True
|
||||||
|
if (os.environ.get("DB_HOST") or "").strip():
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_connect_kwargs() -> dict:
|
||||||
|
url = (os.environ.get("DATABASE_URL") or "").strip()
|
||||||
|
if url:
|
||||||
|
return {"conninfo": url}
|
||||||
|
host = (os.environ.get("DB_HOST") or "postgres").strip() or "postgres"
|
||||||
|
port = int((os.environ.get("DB_PORT") or "5432").strip() or "5432")
|
||||||
|
dbname = (os.environ.get("DB_NAME") or "kansho").strip() or "kansho"
|
||||||
|
user = (os.environ.get("DB_USER") or "kansho").strip() or "kansho"
|
||||||
|
password = os.environ.get("DB_PASSWORD") or ""
|
||||||
|
return {
|
||||||
|
"host": host,
|
||||||
|
"port": port,
|
||||||
|
"dbname": dbname,
|
||||||
|
"user": user,
|
||||||
|
"password": password,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def sqlite_schema_to_postgres(sql: str) -> str:
|
||||||
|
return sql.replace("datetime('now')", "CURRENT_TIMESTAMP::text")
|
||||||
|
|
||||||
|
|
||||||
|
def replace_placeholders(sql: str) -> str:
|
||||||
|
"""Replace SQLite `?` placeholders with psycopg `%s`, ignoring quoted text."""
|
||||||
|
out: list[str] = []
|
||||||
|
i = 0
|
||||||
|
in_single = False
|
||||||
|
in_double = False
|
||||||
|
while i < len(sql):
|
||||||
|
ch = sql[i]
|
||||||
|
if ch == "'" and not in_double:
|
||||||
|
if in_single and i + 1 < len(sql) and sql[i + 1] == "'":
|
||||||
|
out.append("''")
|
||||||
|
i += 2
|
||||||
|
continue
|
||||||
|
in_single = not in_single
|
||||||
|
out.append(ch)
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
if ch == '"' and not in_single:
|
||||||
|
in_double = not in_double
|
||||||
|
out.append(ch)
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
if ch == "?" and not in_single and not in_double:
|
||||||
|
out.append("%s")
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
out.append(ch)
|
||||||
|
i += 1
|
||||||
|
return "".join(out)
|
||||||
|
|
||||||
|
|
||||||
|
def adapt_sql(sql: str) -> str:
|
||||||
|
"""Make a SQLite-shaped statement runnable on PostgreSQL."""
|
||||||
|
sql = sqlite_schema_to_postgres(sql)
|
||||||
|
sql = _ROWID_COL.sub(".id", sql)
|
||||||
|
sql = _ON_CONFLICT_PAREN.sub("ON CONFLICT (", sql)
|
||||||
|
used_ignore = bool(_INSERT_OR_IGNORE.search(sql))
|
||||||
|
sql = _INSERT_OR_IGNORE.sub("INSERT INTO", sql)
|
||||||
|
if used_ignore and re.search(r"ON\s+CONFLICT", sql, re.IGNORECASE) is None:
|
||||||
|
stripped = sql.rstrip()
|
||||||
|
ended = stripped.endswith(";")
|
||||||
|
if ended:
|
||||||
|
stripped = stripped[:-1].rstrip()
|
||||||
|
stripped = f"{stripped} ON CONFLICT DO NOTHING"
|
||||||
|
sql = stripped + (";" if ended else "")
|
||||||
|
return replace_placeholders(sql)
|
||||||
|
|
||||||
|
|
||||||
|
def rewrite_catalog_sql(sql: str) -> tuple[str, tuple | None] | None:
|
||||||
|
"""Rewrite SQLite catalog queries. Returns (sql, extra_params) or None."""
|
||||||
|
match = _PRAGMA_TABLE.match(sql)
|
||||||
|
if match:
|
||||||
|
return (
|
||||||
|
"""
|
||||||
|
SELECT column_name AS name
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public' AND table_name = %s
|
||||||
|
ORDER BY ordinal_position
|
||||||
|
""",
|
||||||
|
(match.group(1).lower(),),
|
||||||
|
)
|
||||||
|
if _SQLITE_MASTER_TABLES.match(sql):
|
||||||
|
return (
|
||||||
|
"""
|
||||||
|
SELECT tablename AS name
|
||||||
|
FROM pg_catalog.pg_tables
|
||||||
|
WHERE schemaname = 'public'
|
||||||
|
""",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if _SQLITE_MASTER_SQL.match(sql):
|
||||||
|
return ("SELECT NULL AS sql WHERE FALSE", None)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def split_sql(script: str) -> list[str]:
|
||||||
|
"""Split a SQL script into statements, respecting quotes."""
|
||||||
|
statements: list[str] = []
|
||||||
|
buf: list[str] = []
|
||||||
|
in_single = False
|
||||||
|
in_double = False
|
||||||
|
i = 0
|
||||||
|
while i < len(script):
|
||||||
|
ch = script[i]
|
||||||
|
if ch == "'" and not in_double:
|
||||||
|
if in_single and i + 1 < len(script) and script[i + 1] == "'":
|
||||||
|
buf.append("''")
|
||||||
|
i += 2
|
||||||
|
continue
|
||||||
|
in_single = not in_single
|
||||||
|
buf.append(ch)
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
if ch == '"' and not in_single:
|
||||||
|
in_double = not in_double
|
||||||
|
buf.append(ch)
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
if ch == ";" and not in_single and not in_double:
|
||||||
|
stmt = "".join(buf).strip()
|
||||||
|
if stmt:
|
||||||
|
statements.append(stmt)
|
||||||
|
buf = []
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
buf.append(ch)
|
||||||
|
i += 1
|
||||||
|
tail = "".join(buf).strip()
|
||||||
|
if tail:
|
||||||
|
statements.append(tail)
|
||||||
|
return statements
|
||||||
239
backend/sqlite_to_postgres.py
Normal file
239
backend/sqlite_to_postgres.py
Normal file
|
|
@ -0,0 +1,239 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""One-shot SQLite → PostgreSQL copy for the laptop/home journal instance.
|
||||||
|
|
||||||
|
Probe on a copy, never on the only backup. Does not send data to providers.
|
||||||
|
Media files are copied separately into the destination media root.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sql_compat import postgres_connect_kwargs
|
||||||
|
|
||||||
|
SCHEMA_PATH = Path(__file__).resolve().parent / "schema.sql"
|
||||||
|
DEFAULT_SQLITE = Path(__file__).resolve().parent / "data" / "kansho.sqlite"
|
||||||
|
DEFAULT_MEDIA = Path(__file__).resolve().parent / "data" / "media"
|
||||||
|
|
||||||
|
# Parent tables first — matches schema.sql CREATE order.
|
||||||
|
TABLES = [
|
||||||
|
"schema_migrations",
|
||||||
|
"tiers",
|
||||||
|
"features",
|
||||||
|
"tier_limits",
|
||||||
|
"profiles",
|
||||||
|
"sessions",
|
||||||
|
"user_feature_restrictions",
|
||||||
|
"user_feature_usage",
|
||||||
|
"ai_prompts",
|
||||||
|
"usage_sessions",
|
||||||
|
"conversations",
|
||||||
|
"messages",
|
||||||
|
"threads",
|
||||||
|
"conversation_threads",
|
||||||
|
"spaces",
|
||||||
|
"thread_spaces",
|
||||||
|
"derived_records",
|
||||||
|
"handoffs",
|
||||||
|
"external_refs",
|
||||||
|
"re_grounding_events",
|
||||||
|
"identity_mappings",
|
||||||
|
"identity_review_proposals",
|
||||||
|
"journal_days",
|
||||||
|
"journal_drafts",
|
||||||
|
"journal_entries",
|
||||||
|
"journal_entry_versions",
|
||||||
|
"journal_draft_source_refs",
|
||||||
|
"journal_entry_version_source_refs",
|
||||||
|
"media_assets",
|
||||||
|
"writing_profiles",
|
||||||
|
"writing_profile_sources",
|
||||||
|
"writing_profile_facets",
|
||||||
|
"writing_profile_traits",
|
||||||
|
"writing_profile_trait_refs",
|
||||||
|
"writing_profile_suggestions",
|
||||||
|
"interaction_profiles",
|
||||||
|
"interaction_preferences",
|
||||||
|
"interaction_suggestions",
|
||||||
|
"writing_profile_evidence",
|
||||||
|
"writing_profile_reviews",
|
||||||
|
"writing_profile_versions",
|
||||||
|
"provider_settings",
|
||||||
|
"journal_generation_selection",
|
||||||
|
"generation_guidelines",
|
||||||
|
"app_settings",
|
||||||
|
"debug_runs",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _sqlite_path() -> Path:
|
||||||
|
env = (os.environ.get("KANSHO_DB_PATH") or "").strip()
|
||||||
|
if env:
|
||||||
|
return Path(env)
|
||||||
|
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||||
|
if data_dir:
|
||||||
|
return Path(data_dir) / "kansho.sqlite"
|
||||||
|
return DEFAULT_SQLITE
|
||||||
|
|
||||||
|
|
||||||
|
def _media_root() -> Path:
|
||||||
|
env = (os.environ.get("KANSHO_MEDIA_ROOT") or "").strip()
|
||||||
|
if env:
|
||||||
|
return Path(env)
|
||||||
|
data_dir = (os.environ.get("KANSHO_DATA_DIR") or "").strip()
|
||||||
|
if data_dir:
|
||||||
|
return Path(data_dir) / "media"
|
||||||
|
return DEFAULT_MEDIA
|
||||||
|
|
||||||
|
|
||||||
|
def _connect_pg():
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
kwargs = postgres_connect_kwargs()
|
||||||
|
if "conninfo" in kwargs:
|
||||||
|
return psycopg.connect(kwargs["conninfo"])
|
||||||
|
return psycopg.connect(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def _pg_columns(conn, table: str) -> list[str]:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT column_name
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public' AND table_name = %s
|
||||||
|
ORDER BY ordinal_position
|
||||||
|
""",
|
||||||
|
(table,),
|
||||||
|
).fetchall()
|
||||||
|
return [row[0] for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def _sqlite_rows(sqlite_path: Path, table: str) -> list[dict]:
|
||||||
|
conn = sqlite3.connect(str(sqlite_path))
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
try:
|
||||||
|
rows = conn.execute(f"SELECT * FROM {table}").fetchall()
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
except sqlite3.OperationalError:
|
||||||
|
return []
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _copy_media(src: Path, dest: Path) -> int:
|
||||||
|
if not src.is_dir():
|
||||||
|
print(f"No media directory at {src}")
|
||||||
|
return 0
|
||||||
|
dest.mkdir(parents=True, exist_ok=True)
|
||||||
|
count = 0
|
||||||
|
for path in src.rglob("*"):
|
||||||
|
if not path.is_file():
|
||||||
|
continue
|
||||||
|
relative = path.relative_to(src)
|
||||||
|
target = dest / relative
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(path, target)
|
||||||
|
count += 1
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
def migrate(sqlite_path: Path, media_src: Path, media_dest: Path, replace: bool) -> dict[str, tuple[int, int]]:
|
||||||
|
if not sqlite_path.is_file():
|
||||||
|
raise SystemExit(f"SQLite not found: {sqlite_path}")
|
||||||
|
|
||||||
|
pg = _connect_pg()
|
||||||
|
stats: dict[str, tuple[int, int]] = {}
|
||||||
|
try:
|
||||||
|
existing = pg.execute("SELECT COUNT(*) FROM profiles").fetchone()[0]
|
||||||
|
seeds = pg.execute("SELECT COUNT(*) FROM tiers").fetchone()[0]
|
||||||
|
if existing and not replace:
|
||||||
|
raise SystemExit(
|
||||||
|
f"PostgreSQL already has {existing} profiles. Pass --replace after a probe copy, not on the only backup."
|
||||||
|
)
|
||||||
|
if seeds and not replace:
|
||||||
|
raise SystemExit(
|
||||||
|
"PostgreSQL already has platform seeds from startup. "
|
||||||
|
"Pass --replace to load a SQLite snapshot onto this instance (no user profiles yet is OK)."
|
||||||
|
)
|
||||||
|
if replace:
|
||||||
|
print(f"Replacing Postgres snapshot (profiles={existing}, seed_tiers={seeds})")
|
||||||
|
for table in reversed(TABLES):
|
||||||
|
pg.execute(f"TRUNCATE TABLE {table} CASCADE")
|
||||||
|
|
||||||
|
for table in TABLES:
|
||||||
|
rows = _sqlite_rows(sqlite_path, table)
|
||||||
|
columns = _pg_columns(pg, table)
|
||||||
|
if not columns:
|
||||||
|
print(f" skip {table} (missing in Postgres schema)")
|
||||||
|
stats[table] = (len(rows), 0)
|
||||||
|
continue
|
||||||
|
if not rows:
|
||||||
|
print(f" {table}: empty")
|
||||||
|
stats[table] = (0, 0)
|
||||||
|
continue
|
||||||
|
usable = [col for col in columns if col in rows[0]]
|
||||||
|
placeholders = ", ".join(["%s"] * len(usable))
|
||||||
|
col_sql = ", ".join(usable)
|
||||||
|
sql = f"INSERT INTO {table} ({col_sql}) VALUES ({placeholders})"
|
||||||
|
values = [tuple(row.get(col) for col in usable) for row in rows]
|
||||||
|
with pg.cursor() as cur:
|
||||||
|
cur.executemany(sql, values)
|
||||||
|
count = pg.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
||||||
|
print(f" {table}: {len(rows)} → {count}")
|
||||||
|
stats[table] = (len(rows), int(count))
|
||||||
|
pg.execute(
|
||||||
|
"INSERT INTO schema_migrations (id) VALUES (%s) ON CONFLICT (id) DO NOTHING",
|
||||||
|
("022_postgres_runtime_baseline",),
|
||||||
|
)
|
||||||
|
pg.commit()
|
||||||
|
except Exception:
|
||||||
|
pg.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
pg.close()
|
||||||
|
|
||||||
|
copied = _copy_media(media_src, media_dest)
|
||||||
|
print(f"Media files copied: {copied} → {media_dest}")
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def verify(stats: dict[str, tuple[int, int]]) -> None:
|
||||||
|
failed = False
|
||||||
|
print("\nVerification")
|
||||||
|
for table, (src, dest) in stats.items():
|
||||||
|
if table == "schema_migrations" and dest == src + 1:
|
||||||
|
print(f" ok {table:36} sqlite={src:5} postgres={dest:5} (incl. 022)")
|
||||||
|
continue
|
||||||
|
mark = "ok" if src == dest else "MISMATCH"
|
||||||
|
if src != dest:
|
||||||
|
failed = True
|
||||||
|
print(f" {mark:8} {table:36} sqlite={src:5} postgres={dest:5}")
|
||||||
|
if failed:
|
||||||
|
raise SystemExit("Row counts do not match")
|
||||||
|
print("Row counts match")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description="Copy Kanshō SQLite data into PostgreSQL")
|
||||||
|
parser.add_argument("--sqlite", type=Path, default=_sqlite_path())
|
||||||
|
parser.add_argument("--media-src", type=Path, default=_media_root())
|
||||||
|
parser.add_argument("--media-dest", type=Path, default=Path(os.environ.get("KANSHO_MEDIA_DEST") or "/app/data/media"))
|
||||||
|
parser.add_argument("--confirm", action="store_true", help="Required. Refuse a silent copy.")
|
||||||
|
parser.add_argument("--replace", action="store_true", help="Truncate Postgres tables first")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if not args.confirm:
|
||||||
|
raise SystemExit("Refusing to copy personal data without --confirm")
|
||||||
|
print(f"SQLite source: {args.sqlite}")
|
||||||
|
print(f"Media source: {args.media_src}")
|
||||||
|
print(f"Media dest: {args.media_dest}")
|
||||||
|
stats = migrate(args.sqlite, args.media_src, args.media_dest, args.replace)
|
||||||
|
verify(stats)
|
||||||
|
print("Import complete. Check entry/message counts and one media GET before Prod.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
7
backend/startup.sh
Normal file
7
backend/startup.sh
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
#!/bin/sh
|
||||||
|
set -e
|
||||||
|
|
||||||
|
echo "=== Kanshō backend startup ==="
|
||||||
|
python /app/db_init.py
|
||||||
|
echo "=== starting uvicorn ==="
|
||||||
|
exec uvicorn main:app --host 0.0.0.0 --port 8000
|
||||||
97
backend/tests/conftest.py
Normal file
97
backend/tests/conftest.py
Normal file
|
|
@ -0,0 +1,97 @@
|
||||||
|
"""pytest runner for the existing test_*.py scripts.
|
||||||
|
|
||||||
|
Each file stays one case (`main()`) until it is split into native pytest
|
||||||
|
functions. Schema is loaded once per session; each integration file only
|
||||||
|
truncates data so kansho_dev is never the target and CI does not reload
|
||||||
|
schema.sql twenty times.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from tests.harness import ( # noqa: E402
|
||||||
|
apply_module_env,
|
||||||
|
is_postgres_test_database,
|
||||||
|
prepare_test_env,
|
||||||
|
reset_postgres_data,
|
||||||
|
reset_postgres_schema,
|
||||||
|
)
|
||||||
|
|
||||||
|
UNIT_MODULES = frozenset(
|
||||||
|
{
|
||||||
|
"test_sql_compat",
|
||||||
|
"test_local_backup",
|
||||||
|
"test_provenance",
|
||||||
|
"test_model_catalog",
|
||||||
|
"test_privacy_detect_eval",
|
||||||
|
"test_journal_shape",
|
||||||
|
"test_journal_body",
|
||||||
|
"test_writing_profile",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_configure(config: pytest.Config) -> None:
|
||||||
|
prepare_test_env()
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_collect_file(file_path: Path, parent: pytest.Collector):
|
||||||
|
if file_path.suffix == ".py" and file_path.name.startswith("test_"):
|
||||||
|
return KanshoScriptModule.from_parent(parent, path=file_path)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
|
||||||
|
for item in items:
|
||||||
|
stem = Path(str(item.path)).stem
|
||||||
|
marker = pytest.mark.unit if stem in UNIT_MODULES else pytest.mark.integration
|
||||||
|
item.add_marker(marker)
|
||||||
|
|
||||||
|
|
||||||
|
class KanshoScriptModule(pytest.Module):
|
||||||
|
def collect(self):
|
||||||
|
module = self.obj
|
||||||
|
main = getattr(module, "main", None)
|
||||||
|
if main is None or not callable(main):
|
||||||
|
pytest.fail(f"{self.path.name} has no callable main()")
|
||||||
|
yield pytest.Function.from_parent(self, name="test_main", callobj=main)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def _session_postgres():
|
||||||
|
if not is_postgres_test_database():
|
||||||
|
pytest.skip(
|
||||||
|
"integration requires KANSHO_DB_BACKEND=postgres and DB_NAME ending _test "
|
||||||
|
"(Gitea: kansho_test on the Dev Postgres, never kansho_dev)"
|
||||||
|
)
|
||||||
|
reset_postgres_schema()
|
||||||
|
from db import init_db
|
||||||
|
|
||||||
|
init_db()
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def _module_backend(request: pytest.FixtureRequest):
|
||||||
|
source = Path(str(request.path))
|
||||||
|
stem = source.stem
|
||||||
|
saved = dict(os.environ)
|
||||||
|
try:
|
||||||
|
apply_module_env(source)
|
||||||
|
if stem in UNIT_MODULES:
|
||||||
|
yield
|
||||||
|
return
|
||||||
|
request.getfixturevalue("_session_postgres")
|
||||||
|
reset_postgres_data()
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
os.environ.clear()
|
||||||
|
os.environ.update(saved)
|
||||||
147
backend/tests/harness.py
Normal file
147
backend/tests/harness.py
Normal file
|
|
@ -0,0 +1,147 @@
|
||||||
|
"""Postgres test engine on the existing Dev instance.
|
||||||
|
|
||||||
|
Two Compose Postgres instances exist: Dev and Prod. Tests use the Dev
|
||||||
|
instance, database `kansho_test` beside `kansho_dev`. That is not a third
|
||||||
|
server. The suite drops and recreates `public` on the test database only,
|
||||||
|
so personal Dev/Prod journals are not wiped.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
LIVE_DB_NAMES = frozenset(
|
||||||
|
{
|
||||||
|
"kansho",
|
||||||
|
"kansho_dev",
|
||||||
|
"postgres",
|
||||||
|
"template0",
|
||||||
|
"template1",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _backend() -> str:
|
||||||
|
return (os.environ.get("KANSHO_DB_BACKEND") or "").strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _db_name() -> str:
|
||||||
|
return (os.environ.get("DB_NAME") or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def is_postgres_test_database() -> bool:
|
||||||
|
if _backend() not in {"postgres", "postgresql", "pg"}:
|
||||||
|
return False
|
||||||
|
name = _db_name().lower()
|
||||||
|
if not name or name in LIVE_DB_NAMES:
|
||||||
|
return False
|
||||||
|
return name.endswith("_test")
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_test_env() -> None:
|
||||||
|
"""Fail-closed defaults. Do not keep Compose provider keys or leftover fakes."""
|
||||||
|
os.environ["KANSHO_PROVIDER_KEY"] = ""
|
||||||
|
os.environ["KANSHO_DETECT_PROVIDER_KEY"] = ""
|
||||||
|
os.environ.pop("KANSHO_DB_PATH", None)
|
||||||
|
os.environ.pop("KANSHO_FAKE_PROVIDER", None)
|
||||||
|
os.environ.pop("KANSHO_FAKE_DETECT", None)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_module_env(source) -> None:
|
||||||
|
"""Restore per-file env so pytest does not leak fakes/keys across scripts."""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
prepare_test_env()
|
||||||
|
text = Path(source).read_text(encoding="utf-8")
|
||||||
|
if 'os.environ["KANSHO_FAKE_PROVIDER"] = "1"' in text or "os.environ['KANSHO_FAKE_PROVIDER'] = '1'" in text:
|
||||||
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
|
if (
|
||||||
|
'os.environ["KANSHO_FAKE_DETECT"] = "1"' in text
|
||||||
|
or "os.environ['KANSHO_FAKE_DETECT'] = '1'" in text
|
||||||
|
or 'os.environ.setdefault("KANSHO_FAKE_DETECT", "1")' in text
|
||||||
|
):
|
||||||
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
|
|
||||||
|
|
||||||
|
def _require_test_database() -> None:
|
||||||
|
if is_postgres_test_database():
|
||||||
|
return
|
||||||
|
backend = _backend()
|
||||||
|
if backend not in {"postgres", "postgresql", "pg"}:
|
||||||
|
raise SystemExit(
|
||||||
|
"Backend-Tests laufen gegen PostgreSQL (KANSHO_DB_BACKEND=postgres), "
|
||||||
|
"dieselbe Engine wie Dev und Prod. SQLite-Testdateien sind abgelöst. "
|
||||||
|
"Gitea nutzt auf der Dev-Postgres die Datenbank kansho_test "
|
||||||
|
"(neben kansho_dev, keine dritte Instanz)."
|
||||||
|
)
|
||||||
|
name = _db_name().lower()
|
||||||
|
if not name:
|
||||||
|
raise SystemExit(
|
||||||
|
"DB_NAME fehlt. Tests nutzen auf der Dev-Postgres die Datenbank kansho_test, "
|
||||||
|
"nicht kansho_dev."
|
||||||
|
)
|
||||||
|
raise SystemExit(
|
||||||
|
f"DB_NAME={_db_name()!r} ist für Tests gesperrt. "
|
||||||
|
"Die Suite setzt das Schema zurück und darf daher nicht kansho oder kansho_dev treffen."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def reset_postgres_schema() -> None:
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
from db_init import mark_schema_dirty
|
||||||
|
from sql_compat import postgres_connect_kwargs
|
||||||
|
|
||||||
|
kwargs = postgres_connect_kwargs()
|
||||||
|
if "conninfo" in kwargs:
|
||||||
|
conn = psycopg.connect(kwargs["conninfo"], autocommit=True)
|
||||||
|
else:
|
||||||
|
conn = psycopg.connect(autocommit=True, **kwargs)
|
||||||
|
try:
|
||||||
|
conn.execute("DROP SCHEMA IF EXISTS public CASCADE")
|
||||||
|
conn.execute("CREATE SCHEMA public")
|
||||||
|
conn.execute("GRANT ALL ON SCHEMA public TO PUBLIC")
|
||||||
|
conn.execute("GRANT ALL ON SCHEMA public TO CURRENT_USER")
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
mark_schema_dirty()
|
||||||
|
|
||||||
|
|
||||||
|
def reset_postgres_data() -> None:
|
||||||
|
"""Clear rows on kansho_test without reloading schema.sql."""
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
from sql_compat import postgres_connect_kwargs
|
||||||
|
|
||||||
|
kwargs = postgres_connect_kwargs()
|
||||||
|
if "conninfo" in kwargs:
|
||||||
|
conn = psycopg.connect(kwargs["conninfo"], autocommit=True)
|
||||||
|
else:
|
||||||
|
conn = psycopg.connect(autocommit=True, **kwargs)
|
||||||
|
try:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT tablename FROM pg_tables
|
||||||
|
WHERE schemaname = 'public' AND tablename <> 'schema_migrations'
|
||||||
|
"""
|
||||||
|
).fetchall()
|
||||||
|
names = [row[0] for row in rows]
|
||||||
|
if names:
|
||||||
|
joined = ", ".join('"' + name.replace('"', '""') + '"' for name in names)
|
||||||
|
conn.execute(f"TRUNCATE TABLE {joined} RESTART IDENTITY CASCADE")
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
from db import refresh_runtime_seed
|
||||||
|
|
||||||
|
refresh_runtime_seed()
|
||||||
|
|
||||||
|
|
||||||
|
def configure_test_engine() -> None:
|
||||||
|
"""Reset kansho_test. pytest uses the fixture in tests/conftest.py instead."""
|
||||||
|
prepare_test_env()
|
||||||
|
_require_test_database()
|
||||||
|
try:
|
||||||
|
reset_postgres_schema()
|
||||||
|
except Exception as exc: # noqa: BLE001 — fail before the suite touches a live DB
|
||||||
|
print(f"Postgres-Testschema konnte nicht zurückgesetzt werden: {exc}", file=sys.stderr)
|
||||||
|
raise SystemExit(1) from exc
|
||||||
|
|
@ -11,10 +11,8 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-arch-correction-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from db import get_db, migrate_journal_source_refs
|
from db import get_db, migrate_journal_source_refs
|
||||||
|
|
|
||||||
|
|
@ -11,11 +11,9 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||||
REPO = ROOT.parent
|
REPO = ROOT.parent
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-test.sqlite")
|
|
||||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-media")
|
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-media")
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
|
||||||
|
|
@ -16,10 +16,8 @@ from unittest.mock import patch
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-detect-contract-retry-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from debug_store import persist_engine_error, sanitize
|
from debug_store import persist_engine_error, sanitize
|
||||||
|
|
@ -103,7 +101,7 @@ def _chat(entities: list[dict], usage: dict) -> ChatResult:
|
||||||
|
|
||||||
|
|
||||||
def _invalid_type():
|
def _invalid_type():
|
||||||
return [{"start": 0, "end": 1, "text": "x", "entity_type": "FOOD"}]
|
return [{"start": 0, "end": 1, "text": "x", "entity_type": "HUMAN"}]
|
||||||
|
|
||||||
|
|
||||||
def _extra_field():
|
def _extra_field():
|
||||||
|
|
@ -137,14 +135,14 @@ def main() -> None:
|
||||||
sample = "Ich traf Anna."
|
sample = "Ich traf Anna."
|
||||||
try:
|
try:
|
||||||
validate_detected_entity(
|
validate_detected_entity(
|
||||||
{"start": 0, "end": 4, "text": "Anna", "entity_type": "FOOD"},
|
{"start": 0, "end": 4, "text": "Anna", "entity_type": "HUMAN"},
|
||||||
sample,
|
sample,
|
||||||
0,
|
0,
|
||||||
)
|
)
|
||||||
typed = False
|
typed = False
|
||||||
except DetectError as exc:
|
except DetectError as exc:
|
||||||
typed = exc.code == "detect_invalid_output"
|
typed = exc.code == "detect_invalid_output"
|
||||||
expect((exc.diagnostics or {}).get("invalid_entity_type") == "FOOD", "unknown type is recorded without the span text")
|
expect((exc.diagnostics or {}).get("invalid_entity_type") == "HUMAN", "unknown type is recorded without the span text")
|
||||||
expect((exc.diagnostics or {}).get("contract_violation") == "unknown_entity_type", "unknown type has a violation category")
|
expect((exc.diagnostics or {}).get("contract_violation") == "unknown_entity_type", "unknown type has a violation category")
|
||||||
expect("Anna" not in json.dumps(exc.diagnostics or {}), "span plaintext is not stored on the type violation")
|
expect("Anna" not in json.dumps(exc.diagnostics or {}), "span plaintext is not stored on the type violation")
|
||||||
expect(typed, "unknown entity_type is a contract error")
|
expect(typed, "unknown entity_type is a contract error")
|
||||||
|
|
@ -254,10 +252,10 @@ def main() -> None:
|
||||||
expect(blocked.diagnostics.get("generate_called") is False, "final abort keeps generate_called false")
|
expect(blocked.diagnostics.get("generate_called") is False, "final abort keeps generate_called false")
|
||||||
expect(blocked.diagnostics.get("detect_passes") == 2, "both invalid passes are counted")
|
expect(blocked.diagnostics.get("detect_passes") == 2, "both invalid passes are counted")
|
||||||
expect(blocked.diagnostics.get("contract_violation") == "unknown_entity_type", "final abort keeps the violation category")
|
expect(blocked.diagnostics.get("contract_violation") == "unknown_entity_type", "final abort keeps the violation category")
|
||||||
expect(blocked.diagnostics.get("invalid_entity_type") == "FOOD", "delivered type is stored without span text")
|
expect(blocked.diagnostics.get("invalid_entity_type") == "HUMAN", "delivered type is stored without span text")
|
||||||
expect(blocked.diagnostics.get("detect_partial_discarded") is True, "failed retry discarded the first pass")
|
expect(blocked.diagnostics.get("detect_partial_discarded") is True, "failed retry discarded the first pass")
|
||||||
expect(_ids(blocked.diagnostics), "failed diagnostics are acyclic")
|
expect(_ids(blocked.diagnostics), "failed diagnostics are acyclic")
|
||||||
expect("FOOD" in json.dumps(blocked.diagnostics), "invalid type remains visible")
|
expect("HUMAN" in json.dumps(blocked.diagnostics), "invalid type remains visible")
|
||||||
expect("secret" not in json.dumps(blocked.diagnostics).lower(), "no extra identity payload")
|
expect("secret" not in json.dumps(blocked.diagnostics).lower(), "no extra identity payload")
|
||||||
expect(blocked.status_code != 500, "detect abort is not a generic 500")
|
expect(blocked.status_code != 500, "detect abort is not a generic 500")
|
||||||
|
|
||||||
|
|
@ -336,7 +334,7 @@ def main() -> None:
|
||||||
def script(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
def script(attempt, chunk_index, chunk_text, offset, schema_retry=False):
|
||||||
seen.append((kind, attempt))
|
seen.append((kind, attempt))
|
||||||
if attempt == 1:
|
if attempt == 1:
|
||||||
entity_type = "FOOD" if kind == "alpha" else "ANIMAL"
|
entity_type = "HUMAN" if kind == "alpha" else "ANIMAL"
|
||||||
return _chat([{"start": 0, "end": 1, "text": "x", "entity_type": entity_type}], USAGE_A)
|
return _chat([{"start": 0, "end": 1, "text": "x", "entity_type": entity_type}], USAGE_A)
|
||||||
return _chat([], USAGE_B)
|
return _chat([], USAGE_B)
|
||||||
|
|
||||||
|
|
@ -359,7 +357,7 @@ def main() -> None:
|
||||||
all(kind == row[0] for row in by_kind.values() for kind, _attempt in row[1]),
|
all(kind == row[0] for row in by_kind.values() for kind, _attempt in row[1]),
|
||||||
"parallel scripts stay on their request",
|
"parallel scripts stay on their request",
|
||||||
)
|
)
|
||||||
expect(by_kind["alpha"][2] in (None, "FOOD"), "alpha does not keep beta's type")
|
expect(by_kind["alpha"][2] in (None, "HUMAN"), "alpha does not keep beta's type")
|
||||||
expect(by_kind["beta"][2] in (None, "ANIMAL"), "beta does not keep alpha's type")
|
expect(by_kind["beta"][2] in (None, "ANIMAL"), "beta does not keep alpha's type")
|
||||||
expect(by_kind["alpha"][4] != by_kind["beta"][4], "parallel stats objects are distinct")
|
expect(by_kind["alpha"][4] != by_kind["beta"][4], "parallel stats objects are distinct")
|
||||||
|
|
||||||
|
|
@ -444,7 +442,7 @@ def main() -> None:
|
||||||
expect(payload.get("detect_passes") == 2, "stored run records both detect passes")
|
expect(payload.get("detect_passes") == 2, "stored run records both detect passes")
|
||||||
expect(payload.get("detect_partial_discarded") is True, "stored run records discarded partials")
|
expect(payload.get("detect_partial_discarded") is True, "stored run records discarded partials")
|
||||||
expect(payload.get("contract_violation") == "unknown_entity_type", "stored run records the violation")
|
expect(payload.get("contract_violation") == "unknown_entity_type", "stored run records the violation")
|
||||||
expect(payload.get("invalid_entity_type") == "FOOD", "stored run records the delivered type")
|
expect(payload.get("invalid_entity_type") == "HUMAN", "stored run records the delivered type")
|
||||||
expect(payload.get("generation_selection") or (payload.get("trace") or {}).get("generation_selection"), "stored run keeps generation selection")
|
expect(payload.get("generation_selection") or (payload.get("trace") or {}).get("generation_selection"), "stored run keeps generation selection")
|
||||||
expect("local_label" not in json.dumps(payload), "stored run has no mapping labels")
|
expect("local_label" not in json.dumps(payload), "stored run has no mapping labels")
|
||||||
exported = client.get(
|
exported = client.get(
|
||||||
|
|
|
||||||
|
|
@ -9,8 +9,6 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-dialogue-test.sqlite")
|
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from main import app
|
from main import app
|
||||||
|
|
|
||||||
|
|
@ -10,10 +10,8 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-frame-test.sqlite")
|
|
||||||
os.environ["KANSHO_PROVIDER_KEY"] = ""
|
os.environ["KANSHO_PROVIDER_KEY"] = ""
|
||||||
os.environ["KANSHO_DETECT_PROVIDER_KEY"] = ""
|
os.environ["KANSHO_DETECT_PROVIDER_KEY"] = ""
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from main import app
|
from main import app
|
||||||
|
|
|
||||||
|
|
@ -10,10 +10,8 @@ from unittest.mock import patch
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-identity-registry-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from identity_store import STATUS_LEGACY, list_confirmed_identities, list_registry
|
from identity_store import STATUS_LEGACY, list_confirmed_identities, list_registry
|
||||||
|
|
|
||||||
|
|
@ -11,10 +11,8 @@ from unittest.mock import patch
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-budget-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from main import app
|
from main import app
|
||||||
|
|
|
||||||
|
|
@ -10,10 +10,8 @@ from unittest.mock import patch
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-editorial-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from db import get_db, init_db
|
from db import get_db, init_db
|
||||||
|
|
|
||||||
|
|
@ -9,10 +9,8 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-eval-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from db import init_db
|
from db import init_db
|
||||||
from journal_editorial import GENERATE_SEED_REVISION
|
from journal_editorial import GENERATE_SEED_REVISION
|
||||||
|
|
|
||||||
|
|
@ -12,10 +12,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||||
FRONTEND = ROOT.parent / "frontend"
|
FRONTEND = ROOT.parent / "frontend"
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-guidelines-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from db import get_db, init_db
|
from db import get_db, init_db
|
||||||
|
|
|
||||||
|
|
@ -11,10 +11,8 @@ from unittest.mock import patch
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-narration-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from db import get_db, init_db
|
from db import get_db, init_db
|
||||||
|
|
|
||||||
|
|
@ -11,11 +11,9 @@ from datetime import date
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-opening-test.sqlite")
|
|
||||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-opening-media")
|
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-opening-media")
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
|
||||||
|
|
@ -10,10 +10,8 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-style-context-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from db import get_db, init_db
|
from db import get_db, init_db
|
||||||
|
|
@ -211,8 +209,9 @@ def main() -> None:
|
||||||
test_compile_flags()
|
test_compile_flags()
|
||||||
test_budget_omission_in_effective_trace()
|
test_budget_omission_in_effective_trace()
|
||||||
|
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
from tests.harness import reset_postgres_data
|
||||||
init_db()
|
|
||||||
|
reset_postgres_data()
|
||||||
reset_debug()
|
reset_debug()
|
||||||
with TestClient(app) as client:
|
with TestClient(app) as client:
|
||||||
setup = client.post(
|
setup = client.post(
|
||||||
|
|
|
||||||
|
|
@ -10,10 +10,8 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-legacy-immutable-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from db import get_db, init_db
|
from db import get_db, init_db
|
||||||
|
|
@ -270,8 +268,9 @@ def test_later_semantic_seed_creates_successor() -> None:
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_generate_trace_and_prompt() -> None:
|
def test_legacy_generate_trace_and_prompt() -> None:
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
from tests.harness import reset_postgres_data
|
||||||
init_db()
|
|
||||||
|
reset_postgres_data()
|
||||||
client = TestClient(app)
|
client = TestClient(app)
|
||||||
setup = client.post(
|
setup = client.post(
|
||||||
"/api/auth/setup",
|
"/api/auth/setup",
|
||||||
|
|
|
||||||
|
|
@ -9,11 +9,9 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-trash-test.sqlite")
|
|
||||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-trash-media")
|
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-trash-media")
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
|
||||||
|
|
@ -10,8 +10,9 @@ from unittest.mock import patch
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-model-catalog-test.sqlite")
|
os.environ.setdefault("KANSHO_PROVIDER_KEY", "")
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "")
|
||||||
|
Path(tempfile.gettempdir()).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
from model_catalog import (
|
from model_catalog import (
|
||||||
ModelWindow,
|
ModelWindow,
|
||||||
|
|
|
||||||
|
|
@ -9,11 +9,9 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-mvp-journal-test.sqlite")
|
|
||||||
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-mvp-journal-media")
|
os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-mvp-journal-media")
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
|
||||||
|
|
@ -15,10 +15,8 @@ from unittest.mock import patch
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-detect-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from identity_store import (
|
from identity_store import (
|
||||||
|
|
@ -115,6 +113,73 @@ def main() -> None:
|
||||||
expect("[[PERSON:" in (person_sushi.trace.get("egress") or ""), "same word can be a person in another sentence")
|
expect("[[PERSON:" in (person_sushi.trace.get("egress") or ""), "same word can be a person in another sentence")
|
||||||
expect("Sushi" not in (person_sushi.trace.get("egress") or "").replace("[[PERSON:", ""), "person Sushi is masked")
|
expect("Sushi" not in (person_sushi.trace.get("egress") or "").replace("[[PERSON:", ""), "person Sushi is masked")
|
||||||
|
|
||||||
|
mixed_homonym = "Ich aß Sushi. Meine Frau Sushi kam später."
|
||||||
|
food_at = mixed_homonym.find("Sushi")
|
||||||
|
person_at = mixed_homonym.find("Sushi", food_at + 1)
|
||||||
|
dish = validate_detected_entity(
|
||||||
|
{"start": food_at, "end": food_at + 5, "text": "Sushi", "entity_type": "FOOD"},
|
||||||
|
mixed_homonym,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
expect(dish is None, "FOOD on a dish mention is omitted, not a contract error")
|
||||||
|
person_from_food = validate_detected_entity(
|
||||||
|
{"start": person_at, "end": person_at + 5, "text": "Sushi", "entity_type": "FOOD"},
|
||||||
|
mixed_homonym,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
expect(
|
||||||
|
person_from_food is not None and person_from_food.entity_type == "PERSON",
|
||||||
|
"FOOD on a kinship mention stays PERSON",
|
||||||
|
)
|
||||||
|
arrived = "Sushi kam später."
|
||||||
|
arrived_span = validate_detected_entity(
|
||||||
|
{"start": 0, "end": 5, "text": "Sushi", "entity_type": "DISH"},
|
||||||
|
arrived,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
expect(
|
||||||
|
arrived_span is not None and arrived_span.entity_type == "PERSON",
|
||||||
|
"DISH on a person subject stays PERSON",
|
||||||
|
)
|
||||||
|
noted = validate_detected_entity(
|
||||||
|
{
|
||||||
|
"start": food_at,
|
||||||
|
"end": food_at + 5,
|
||||||
|
"text": "Sushi",
|
||||||
|
"entity_type": "FOOD",
|
||||||
|
"note": "Gericht",
|
||||||
|
},
|
||||||
|
mixed_homonym,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
expect(noted is None, "extra fields on a FOOD dish do not fail the pass")
|
||||||
|
install_test_spans(
|
||||||
|
[
|
||||||
|
{"start": food_at, "end": food_at + 5, "text": "Sushi", "entity_type": "FOOD"},
|
||||||
|
{"start": person_at, "end": person_at + 5, "text": "Sushi", "entity_type": "FOOD"},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
mixed_run = _run(profile_id, mixed_homonym)
|
||||||
|
mixed_egress = mixed_run.trace.get("egress") or ""
|
||||||
|
expect(mixed_run.allowed, "FOOD homonym does not block generate")
|
||||||
|
expect("aß Sushi" in mixed_egress, "reconstructed dish stays unmasked")
|
||||||
|
expect("[[PERSON:" in mixed_egress, "person-shaped FOOD span is masked")
|
||||||
|
expect(
|
||||||
|
not mixed_run.diagnostics.get("contract_violation"),
|
||||||
|
"food type is not a contract violation",
|
||||||
|
)
|
||||||
|
expect(
|
||||||
|
not mixed_run.diagnostics.get("invalid_entity_type"),
|
||||||
|
"food type is not recorded as invalid",
|
||||||
|
)
|
||||||
|
expect(
|
||||||
|
"FOOD" in (mixed_run.diagnostics.get("omitted_non_identity_types") or []),
|
||||||
|
"omitted food type is recorded without span text",
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
reset_detect_test_hooks()
|
||||||
|
|
||||||
mixed = _run(
|
mixed = _run(
|
||||||
profile_id,
|
profile_id,
|
||||||
"Ich traf Anna. Ich arbeitete am privaten Projekt Aurora. "
|
"Ich traf Anna. Ich arbeitete am privaten Projekt Aurora. "
|
||||||
|
|
@ -472,7 +537,7 @@ def main() -> None:
|
||||||
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
"SELECT seed_revision, template, default_template FROM ai_prompts WHERE slug = ?",
|
||||||
("mvp.entity_detect",),
|
("mvp.entity_detect",),
|
||||||
).fetchone()
|
).fetchone()
|
||||||
expect(row["seed_revision"] == "2026-08-27-detect-ground-v1", "detect prompt revision is stored")
|
expect(row["seed_revision"] == "2026-09-08-detect-homonym-v1", "detect prompt revision is stored")
|
||||||
expect("{{known_labels}}" not in (row["template"] or ""), "new detect prompt has no known_labels skip list")
|
expect("{{known_labels}}" not in (row["template"] or ""), "new detect prompt has no known_labels skip list")
|
||||||
expect("Zwiebeln" not in (row["template"] or ""), "detect prompt has no food word list")
|
expect("Zwiebeln" not in (row["template"] or ""), "detect prompt has no food word list")
|
||||||
|
|
||||||
|
|
@ -490,7 +555,7 @@ def main() -> None:
|
||||||
("mvp.entity_detect",),
|
("mvp.entity_detect",),
|
||||||
).fetchone()
|
).fetchone()
|
||||||
expect(custom["template"] == "CUSTOM DETECT {{source_text}}", "user-edited detect prompt is not overwritten")
|
expect(custom["template"] == "CUSTOM DETECT {{source_text}}", "user-edited detect prompt is not overwritten")
|
||||||
expect(custom["seed_revision"] == "2026-08-27-detect-ground-v1", "revision still updates the default")
|
expect(custom["seed_revision"] == "2026-09-08-detect-homonym-v1", "revision still updates the default")
|
||||||
|
|
||||||
remote = ProviderConfig(
|
remote = ProviderConfig(
|
||||||
role="detect",
|
role="detect",
|
||||||
|
|
|
||||||
|
|
@ -13,10 +13,8 @@ from unittest.mock import patch
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-manifest-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from entity_detect import install_test_spans, reset_detect_test_hooks
|
from entity_detect import install_test_spans, reset_detect_test_hooks
|
||||||
|
|
|
||||||
|
|
@ -11,10 +11,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||||
REPO = ROOT.parent
|
REPO = ROOT.parent
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-response-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from identity_store import list_mappings, remember_mapping
|
from identity_store import list_mappings, remember_mapping
|
||||||
|
|
|
||||||
|
|
@ -9,10 +9,8 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-profile-governance-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from main import app
|
from main import app
|
||||||
|
|
|
||||||
|
|
@ -9,10 +9,8 @@ from pathlib import Path
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-profile-review-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from main import app
|
from main import app
|
||||||
|
|
|
||||||
|
|
@ -16,10 +16,8 @@ import httpx
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-profile-review-errors-test.sqlite")
|
|
||||||
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
os.environ["KANSHO_FAKE_PROVIDER"] = "1"
|
||||||
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
os.environ["KANSHO_FAKE_DETECT"] = "1"
|
||||||
Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from engine import EngineError
|
from engine import EngineError
|
||||||
|
|
|
||||||
|
|
@ -1,20 +1,13 @@
|
||||||
"""Intent-neutral provenance verification. Run from backend/: python tests/test_provenance.py"""
|
"""Intent-neutral provenance verification. Run from backend/: python tests/test_provenance.py"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
|
||||||
import threading
|
import threading
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
os.environ.setdefault(
|
|
||||||
"KANSHO_DB_PATH",
|
|
||||||
str(Path(tempfile.gettempdir()) / "kansho-provenance-test.sqlite"),
|
|
||||||
)
|
|
||||||
|
|
||||||
from provenance import (
|
from provenance import (
|
||||||
COVERAGE_ALL_SELECTED_SOURCES,
|
COVERAGE_ALL_SELECTED_SOURCES,
|
||||||
COVERAGE_SELECTED_EVIDENCE,
|
COVERAGE_SELECTED_EVIDENCE,
|
||||||
|
|
|
||||||
82
backend/tests/test_sql_compat.py
Normal file
82
backend/tests/test_sql_compat.py
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
"""SQL dialect adapter for the SQLite/Postgres dual backend."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from sql_compat import adapt_sql, split_sql, sqlite_schema_to_postgres, use_postgres
|
||||||
|
|
||||||
|
|
||||||
|
def expect(ok: bool, message: str) -> None:
|
||||||
|
if not ok:
|
||||||
|
raise SystemExit(f"FAIL: {message}")
|
||||||
|
print(f"OK {message}")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
os.environ.pop("KANSHO_DB_BACKEND", None)
|
||||||
|
os.environ.pop("KANSHO_DB_PATH", None)
|
||||||
|
os.environ.pop("DB_HOST", None)
|
||||||
|
os.environ.pop("DATABASE_URL", None)
|
||||||
|
expect(use_postgres() is False, "default engine is sqlite")
|
||||||
|
|
||||||
|
os.environ["KANSHO_DB_PATH"] = "/tmp/isolated.sqlite"
|
||||||
|
os.environ["DB_HOST"] = "postgres"
|
||||||
|
expect(use_postgres() is False, "KANSHO_DB_PATH without backend keeps Windows local on sqlite")
|
||||||
|
os.environ["KANSHO_DB_BACKEND"] = "postgres"
|
||||||
|
expect(use_postgres() is True, "explicit postgres backend wins")
|
||||||
|
os.environ["KANSHO_DB_BACKEND"] = "sqlite"
|
||||||
|
expect(use_postgres() is False, "explicit sqlite backend wins")
|
||||||
|
os.environ.pop("KANSHO_DB_BACKEND", None)
|
||||||
|
os.environ.pop("KANSHO_DB_PATH", None)
|
||||||
|
expect(use_postgres() is True, "DB_HOST alone selects postgres")
|
||||||
|
os.environ.pop("DB_HOST", None)
|
||||||
|
|
||||||
|
translated = sqlite_schema_to_postgres("created TEXT NOT NULL DEFAULT (datetime('now'))")
|
||||||
|
expect("CURRENT_TIMESTAMP::text" in translated, "schema datetime default")
|
||||||
|
expect("datetime('now')" not in translated, "no sqlite datetime left in schema")
|
||||||
|
|
||||||
|
ignore = adapt_sql("INSERT OR IGNORE INTO tiers (id) VALUES (?)")
|
||||||
|
expect("INSERT INTO tiers" in ignore, "insert or ignore becomes insert")
|
||||||
|
expect("ON CONFLICT DO NOTHING" in ignore, "conflict do nothing appended")
|
||||||
|
expect("%s" in ignore and "?" not in ignore, "placeholders converted")
|
||||||
|
|
||||||
|
upsert = adapt_sql(
|
||||||
|
"INSERT INTO app_settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value"
|
||||||
|
)
|
||||||
|
expect("ON CONFLICT (key)" in upsert, "on conflict parenthesis spacing")
|
||||||
|
expect(upsert.count("ON CONFLICT") == 1, "existing on conflict not duplicated")
|
||||||
|
expect("%s" in upsert, "upsert placeholders")
|
||||||
|
|
||||||
|
quoted = adapt_sql("SELECT '?' AS q, name FROM profiles WHERE id = ?")
|
||||||
|
expect("'?'" in quoted, "question mark inside quotes kept")
|
||||||
|
expect(quoted.endswith("%s") or quoted.rstrip().endswith("%s"), "trailing placeholder converted")
|
||||||
|
|
||||||
|
case_sql = adapt_sql("UPDATE t SET x = CASE WHEN ? = 1 THEN ? ELSE x END")
|
||||||
|
expect("CASE WHEN %s = 1 THEN %s ELSE x END" in case_sql, "boolean-safe case placeholder")
|
||||||
|
|
||||||
|
rowid_sql = adapt_sql("SELECT e.* FROM journal_entries e ORDER BY e.created, e.rowid")
|
||||||
|
expect("e.rowid" not in rowid_sql, "sqlite rowid removed for postgres")
|
||||||
|
expect("ORDER BY e.created, e.id" in rowid_sql, "rowid becomes primary id")
|
||||||
|
|
||||||
|
parts = split_sql("CREATE TABLE a (id TEXT); CREATE TABLE b (id TEXT);")
|
||||||
|
expect(parts == ["CREATE TABLE a (id TEXT)", "CREATE TABLE b (id TEXT)"], "split two statements")
|
||||||
|
schema = (ROOT / "schema.sql").read_text(encoding="utf-8")
|
||||||
|
pg_schema = sqlite_schema_to_postgres(schema)
|
||||||
|
expect("datetime('now')" not in pg_schema, "translated schema has no sqlite datetime")
|
||||||
|
expect("CURRENT_TIMESTAMP::text" in pg_schema, "translated schema uses timestamp text default")
|
||||||
|
expect(len(split_sql(pg_schema)) > 40, "schema splits into many statements")
|
||||||
|
from sqlite_to_postgres import TABLES
|
||||||
|
|
||||||
|
tables = re.findall(r"CREATE TABLE IF NOT EXISTS (\w+)", schema)
|
||||||
|
expect(tables == list(TABLES), "import table order matches schema.sql")
|
||||||
|
print("sql compat: OK")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
84
docker-compose.dev-env.yml
Normal file
84
docker-compose.dev-env.yml
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
# Server development stack: /home/lars/docker/kansho-dev
|
||||||
|
# No fixed container_name — Compose prefixes with the project name.
|
||||||
|
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: "${DB_NAME:-kansho_dev}"
|
||||||
|
POSTGRES_USER: "${DB_USER:-kansho_dev}"
|
||||||
|
POSTGRES_PASSWORD: "${DB_PASSWORD:-dev_password_change_me}"
|
||||||
|
volumes:
|
||||||
|
- dev-kansho-db-data:/var/lib/postgresql/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-kansho_dev} -d ${DB_NAME:-kansho_dev}"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 10
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- dev-kansho-network
|
||||||
|
|
||||||
|
backend:
|
||||||
|
build:
|
||||||
|
context: ./backend
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
environment:
|
||||||
|
KANSHO_DB_BACKEND: postgres
|
||||||
|
KANSHO_ENV: "${KANSHO_ENV:-development}"
|
||||||
|
KANSHO_DATA_DIR: /app/data
|
||||||
|
KANSHO_MEDIA_ROOT: /app/data/media
|
||||||
|
DB_HOST: postgres
|
||||||
|
DB_PORT: "5432"
|
||||||
|
DB_NAME: "${DB_NAME:-kansho_dev}"
|
||||||
|
DB_USER: "${DB_USER:-kansho_dev}"
|
||||||
|
DB_PASSWORD: "${DB_PASSWORD:-dev_password_change_me}"
|
||||||
|
APP_URL: "${APP_URL:-https://dev.kansho.jinkendo.de}"
|
||||||
|
ALLOWED_ORIGINS: "${ALLOWED_ORIGINS:-https://dev.kansho.jinkendo.de,http://192.168.2.49:3096,http://localhost:3096}"
|
||||||
|
KANSHO_PROVIDER_KEY: "${KANSHO_PROVIDER_KEY:-}"
|
||||||
|
KANSHO_DETECT_PROVIDER_KEY: "${KANSHO_DETECT_PROVIDER_KEY:-}"
|
||||||
|
volumes:
|
||||||
|
- dev-kansho-media:/app/data/media
|
||||||
|
ports:
|
||||||
|
- "${KANSHO_BACKEND_PORT:-8096}:8000"
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
[
|
||||||
|
"CMD",
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/health')",
|
||||||
|
]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 18
|
||||||
|
start_period: 90s
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- dev-kansho-network
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
build:
|
||||||
|
context: ./frontend
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
args:
|
||||||
|
VITE_API_URL: ""
|
||||||
|
ports:
|
||||||
|
- "${KANSHO_FRONTEND_PORT:-3096}:80"
|
||||||
|
depends_on:
|
||||||
|
backend:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- dev-kansho-network
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
dev-kansho-db-data:
|
||||||
|
dev-kansho-media:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
dev-kansho-network:
|
||||||
|
driver: bridge
|
||||||
87
docker-compose.yml
Normal file
87
docker-compose.yml
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
# Production stack on the Raspberry Pi: /home/lars/docker/kansho
|
||||||
|
# Secrets live in the host .env next to this file, never in Git.
|
||||||
|
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16-alpine
|
||||||
|
container_name: kansho-db-prod
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: "${DB_NAME:-kansho}"
|
||||||
|
POSTGRES_USER: "${DB_USER:-kansho}"
|
||||||
|
POSTGRES_PASSWORD: "${DB_PASSWORD:?set DB_PASSWORD in .env}"
|
||||||
|
volumes:
|
||||||
|
- kansho-db-data:/var/lib/postgresql/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-kansho} -d ${DB_NAME:-kansho}"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 10
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- kansho-network
|
||||||
|
|
||||||
|
backend:
|
||||||
|
build:
|
||||||
|
context: ./backend
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
container_name: kansho-api
|
||||||
|
environment:
|
||||||
|
KANSHO_DB_BACKEND: postgres
|
||||||
|
KANSHO_ENV: "${KANSHO_ENV:-production}"
|
||||||
|
KANSHO_DATA_DIR: /app/data
|
||||||
|
KANSHO_MEDIA_ROOT: /app/data/media
|
||||||
|
DB_HOST: postgres
|
||||||
|
DB_PORT: "5432"
|
||||||
|
DB_NAME: "${DB_NAME:-kansho}"
|
||||||
|
DB_USER: "${DB_USER:-kansho}"
|
||||||
|
DB_PASSWORD: "${DB_PASSWORD:?set DB_PASSWORD in .env}"
|
||||||
|
APP_URL: "${APP_URL:-https://kansho.jinkendo.de}"
|
||||||
|
ALLOWED_ORIGINS: "${ALLOWED_ORIGINS:-https://kansho.jinkendo.de}"
|
||||||
|
KANSHO_PROVIDER_KEY: "${KANSHO_PROVIDER_KEY:-}"
|
||||||
|
KANSHO_DETECT_PROVIDER_KEY: "${KANSHO_DETECT_PROVIDER_KEY:-}"
|
||||||
|
volumes:
|
||||||
|
- kansho-media:/app/data/media
|
||||||
|
ports:
|
||||||
|
- "${KANSHO_BACKEND_PORT:-8005}:8000"
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
[
|
||||||
|
"CMD",
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/health')",
|
||||||
|
]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 12
|
||||||
|
start_period: 40s
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- kansho-network
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
build:
|
||||||
|
context: ./frontend
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
args:
|
||||||
|
VITE_API_URL: ""
|
||||||
|
container_name: kansho-ui
|
||||||
|
ports:
|
||||||
|
- "${KANSHO_FRONTEND_PORT:-3006}:80"
|
||||||
|
depends_on:
|
||||||
|
backend:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- kansho-network
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
kansho-db-data:
|
||||||
|
kansho-media:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
kansho-network:
|
||||||
|
driver: bridge
|
||||||
135
docs/DEPLOYMENT.md
Normal file
135
docs/DEPLOYMENT.md
Normal file
|
|
@ -0,0 +1,135 @@
|
||||||
|
# Deployment – Kanshō
|
||||||
|
|
||||||
|
**Stand:** 2026-09-07
|
||||||
|
**Server:** Raspberry Pi 5 (`192.168.2.49`) — gleicher Host wie Mitai/Shinkan/Kairo, **eigene** Compose-Projekte
|
||||||
|
**Runner:** Gitea Actions (`/home/lars/gitea-runner/`)
|
||||||
|
**Repo:** `http://192.168.2.144:3000/Lars/Kansho.git`
|
||||||
|
|
||||||
|
Kanonische Runtime-Entscheidungen: `docs/architecture/technical/runtime_and_deploy.md`.
|
||||||
|
Datenübernahme SQLite → Postgres: `backend/sqlite_to_postgres.py` (Probe auf Kopie, `--confirm`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Port- und Pfad-Übersicht
|
||||||
|
|
||||||
|
| | Production | Development |
|
||||||
|
|---|------------|-------------|
|
||||||
|
| **Git-Branch** | `main` | `develop` |
|
||||||
|
| **Server-Verzeichnis** | `/home/lars/docker/kansho` | `/home/lars/docker/kansho-dev` |
|
||||||
|
| **Frontend-Port** | 3006 | 3096 |
|
||||||
|
| **Backend-Port** | 8005 | 8096 |
|
||||||
|
| **PostgreSQL** | nur Compose-Netz, DB `kansho` | nur Compose-Netz, DB `kansho_dev` |
|
||||||
|
| **Domain** | kansho.jinkendo.de | dev.kansho.jinkendo.de |
|
||||||
|
| **Öffentliche URL** | https://kansho.jinkendo.de | https://dev.kansho.jinkendo.de |
|
||||||
|
| **Compose** | `docker-compose.yml` | `docker-compose.dev-env.yml` |
|
||||||
|
|
||||||
|
Prod-Frontend ist **3006**, nicht 3005: auf dem Pi lauscht Bookstack bereits auf `0.0.0.0:3005`. Dev 3096/8096 und Prod-API 8005 waren frei.
|
||||||
|
|
||||||
|
Lokal ohne Docker bleibt Windows: Frontend **5188**, Backend **8018**, SQLite unter `backend/data/`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Einmalige Server-Einrichtung
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mkdir -p /home/lars/docker/kansho /home/lars/docker/kansho-dev
|
||||||
|
|
||||||
|
cd /home/lars/docker/kansho-dev
|
||||||
|
git clone http://192.168.2.144:3000/Lars/Kansho.git .
|
||||||
|
git checkout develop
|
||||||
|
cp .env.example .env
|
||||||
|
# DB_PASSWORD und Provider-Keys setzen
|
||||||
|
|
||||||
|
cd /home/lars/docker/kansho
|
||||||
|
git clone http://192.168.2.144:3000/Lars/Kansho.git .
|
||||||
|
git checkout main
|
||||||
|
cp .env.example .env
|
||||||
|
# Prod-DB_PASSWORD und Provider-Keys setzen
|
||||||
|
```
|
||||||
|
|
||||||
|
Host-nginx: `nginx/kansho.conf` und `nginx/kansho-dev.conf` nach `/etc/nginx/sites-available/`, dann `nginx/certbot-setup.sh`. DNS A/AAAA für `dev.kansho.jinkendo.de` und `kansho.jinkendo.de` auf den Reverse-Proxy. Bis TLS steht, bleibt der LAN-Zugriff über die Publish-Ports.
|
||||||
|
|
||||||
|
Gitea: Actions aktivieren. Derselbe Pi-Runner wie die Schwesterprodukte (`ubuntu-latest`).
|
||||||
|
|
||||||
|
Watchtower bleibt aus.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Gitea Actions
|
||||||
|
|
||||||
|
| Workflow | Trigger | Zweck |
|
||||||
|
|----------|---------|--------|
|
||||||
|
| `deploy-dev.yml` | Push `develop` | Deploy nach `/home/lars/docker/kansho-dev`, Health `localhost:8096` |
|
||||||
|
| `deploy-prod.yml` | Push `main` | Deploy nach `/home/lars/docker/kansho`, Health `localhost:8005` |
|
||||||
|
| `test.yml` | nach erfolgreichem `Deploy Development` | pytest auf `kansho_test`, nicht-schreibender Smoke gegen die laufende Dev-API, Frontend-Build. Keine Live-Provider-Keys. |
|
||||||
|
|
||||||
|
Prod nur über Merge `develop` → `main`, kein direkter Prod-Schreibzugriff.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Datenübernahme (persönlich, Klasse A/B)
|
||||||
|
|
||||||
|
1. Phase-A-SQLite auf dem Heimrechner ist führend, bis der Dev-Import grün ist.
|
||||||
|
2. Schema auf der Zielinstanz durch normalen Container-Start anlegen (leeres Postgres).
|
||||||
|
3. Import auf **Dev zuerst**:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Zip oder laufende SQLite + Medien in den Dev-Container legen, dann:
|
||||||
|
docker compose -f docker-compose.dev-env.yml exec -T backend \
|
||||||
|
python sqlite_to_postgres.py --confirm \
|
||||||
|
--sqlite /tmp/kansho.sqlite \
|
||||||
|
--media-src /tmp/media \
|
||||||
|
--media-dest /app/data/media
|
||||||
|
```
|
||||||
|
|
||||||
|
Windows-Vorbereitung (Kopie, nicht das einzige Backup):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\scripts\backup-local.ps1 create
|
||||||
|
# Archiv prüfen, dann SQLite+Medien auf den Pi kopieren — nicht nach Gitea.
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Abnahme Dev: Profilanzahl, Journal-Days/Entries, Messages, ein Medien-GET, Gateway fail-closed, `KANSHO_ENV=production` ohne Klartext-Detect.
|
||||||
|
5. Erst dann derselbe Import nach Prod (`--confirm`, leeres Volume). `--replace` nur auf einer bewussten Kopie.
|
||||||
|
|
||||||
|
`debug.persist_traces` nach dem Import prüfen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Postgres-Backup (Server)
|
||||||
|
|
||||||
|
Kein Ersatz durch `backup-local.ps1` (das bleibt der SQLite-Weg).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Dump
|
||||||
|
cd /home/lars/docker/kansho
|
||||||
|
docker compose exec -T postgres pg_dump -U kansho kansho > "/home/lars/backups/kansho-$(date -u +%Y%m%d).sql"
|
||||||
|
|
||||||
|
# Medienvolume
|
||||||
|
docker run --rm -v kansho_kansho-media:/src -v /home/lars/backups:/dst alpine \
|
||||||
|
tar -C /src -czf /dst/kansho-media-$(date -u +%Y%m%d).tgz .
|
||||||
|
```
|
||||||
|
|
||||||
|
Restore-Übung vor dem ersten Prod-Dialogbestand nach dem Cutover wiederholen. Dump und Medien nicht an Provider und nicht in öffentliche Remotes.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Manuelles Deploy
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Development
|
||||||
|
cd /home/lars/docker/kansho-dev
|
||||||
|
git fetch origin develop && git reset --hard origin/develop
|
||||||
|
docker compose -f docker-compose.dev-env.yml build --no-cache backend frontend
|
||||||
|
docker compose -f docker-compose.dev-env.yml up -d --wait
|
||||||
|
curl -sf http://localhost:8096/api/health
|
||||||
|
|
||||||
|
# Production
|
||||||
|
cd /home/lars/docker/kansho
|
||||||
|
git fetch origin main && git reset --hard origin/main
|
||||||
|
docker compose build --no-cache backend frontend
|
||||||
|
docker compose up -d --wait
|
||||||
|
curl -sf http://localhost:8005/api/health
|
||||||
|
```
|
||||||
|
|
||||||
|
Laptop-Checkout nach erfolgreichem Heim-Restore und Dev-Import nur noch Archiv. Keine parallelen Schreibzugriffe auf denselben persönlichen Bestand.
|
||||||
|
|
@ -79,6 +79,16 @@ Kein Thread-State-Machine-Kapitel in v0.1. Die fachlich genannten Zustände (`Op
|
||||||
|
|
||||||
Entsprechen den Interview-Leitfragen Phase F1/F4 und werden hier nicht vorab beantwortet: kanonische IDs, Backlinks, welche Strukturen nur in Obsidian existieren, Schema für Reflection Memory und Knowledge Delta.
|
Entsprechen den Interview-Leitfragen Phase F1/F4 und werden hier nicht vorab beantwortet: kanonische IDs, Backlinks, welche Strukturen nur in Obsidian existieren, Schema für Reflection Memory und Knowledge Delta.
|
||||||
|
|
||||||
|
## 6a. Persistenzgrenzen Server vs. lokal (2026-09-07)
|
||||||
|
|
||||||
|
Additiv, ohne das Fachschema vorzuziehen:
|
||||||
|
|
||||||
|
- Windows-Checkout: SQLite unter `backend/data/` (gitignoriert) für die lokale App ohne Docker.
|
||||||
|
- Test-Suite und Gitea: pytest auf derselben Dev-Postgres-Instanz, Datenbank `kansho_test` neben `kansho_dev` (keine dritte Instanz). Nicht `kansho` / `kansho_dev`, weil die Suite das Schema zurücksetzt. Live-Smoke trifft nur `/api/health` der laufenden Dev-App.
|
||||||
|
- Docker auf dem Pi: PostgreSQL 16, eigene Instanz je Umgebung, Medien im Compose-Volume `kansho-media` / `dev-kansho-media`.
|
||||||
|
- Identity-Mappings bleiben Klasse A in der Local Trusted Zone (jetzt: Pi-Postgres, nicht Provider).
|
||||||
|
- Kein Shared-Schema mit Mitai.
|
||||||
|
|
||||||
## 7. Querverweise
|
## 7. Querverweise
|
||||||
|
|
||||||
- `memory_storage_and_offline.md`, `integrations_technical.md`, `privacy_gateway.md`, `platform_extensibility.md`
|
- `memory_storage_and_offline.md`, `integrations_technical.md`, `privacy_gateway.md`, `platform_extensibility.md`
|
||||||
|
|
|
||||||
|
|
@ -137,7 +137,7 @@ Kanonisches Home **wie** der Slice gebaut ist: `mvp_implementation.md`. Fachlich
|
||||||
6. `../../work_orders/home_environment_setup.md`
|
6. `../../work_orders/home_environment_setup.md`
|
||||||
7. fachlich: `../functional/guardrails.md` (Datenklassen beim Kopieren von Backup und `.env`)
|
7. fachlich: `../functional/guardrails.md` (Datenklassen beim Kopieren von Backup und `.env`)
|
||||||
|
|
||||||
Arbeitsaufträge liegen unter `docs/work_orders/`. Sie ersetzen keine kanonischen Kapitel.
|
Arbeitsaufträge liegen unter `docs/work_orders/`. Sie ersetzen keine kanonischen Kapitel. Betrieb auf dem Pi: `docs/DEPLOYMENT.md`.
|
||||||
|
|
||||||
## 4. Ladeprinzip
|
## 4. Ladeprinzip
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -155,13 +155,16 @@ Entschieden (nicht neu verhandeln):
|
||||||
- Kein Auto-Rollback der Datenbank
|
- Kein Auto-Rollback der Datenbank
|
||||||
- Kanshō-Ports **nicht** Mitai `3002`/`8002`/`3099`/`8099` kopieren
|
- Kanshō-Ports **nicht** Mitai `3002`/`8002`/`3099`/`8099` kopieren
|
||||||
|
|
||||||
Offen, vor Compose festlegen:
|
Offen, vor Compose festlegen — **beantwortet 2026-09-07:**
|
||||||
|
|
||||||
1. Läuft Kanshō auf demselben Pi wie Mitai, als **eigenes** Compose-Projekt?
|
1. Läuft Kanshō auf demselben Pi wie Mitai, als **eigenes** Compose-Projekt? **Ja.** `/home/lars/docker/kansho` und `kansho-dev`.
|
||||||
2. Eigenes Postgres (bevorzugte Richtung: ja, eigene Instanz/DB `kansho`, kein Shared-Schema mit Mitai)?
|
2. Eigenes Postgres? **Ja**, `kansho` / `kansho_dev`, kein Shared-Schema.
|
||||||
3. Dev-Domain / Prod-Domain / Host-Pfade (Hypothese: `dev.kansho.jinkendo.de` / `kansho.jinkendo.de`)?
|
3. Dev-Domain / Prod-Domain / Host-Pfade? **`dev.kansho.jinkendo.de` / `kansho.jinkendo.de`**, Ports 3096/8096 und 3006/8005. Lokal bleiben 5188/8018. Prod-UI nicht 3005 (Bookstack auf dem Pi).
|
||||||
4. Dev-Ports auf dem Pi, lokal bleiben 5188/8018 für den Windows-Checkout ohne Docker.
|
4. Welle 2 SQLite im Volume oder direkt Postgres? **Direkt Postgres.** SQLite bleibt lokal und für Tests.
|
||||||
5. Ob Welle 2 zuerst Compose **mit SQLite-Volume** als Zwischenstand nutzt oder direkt Postgres + Migrationsadapter.
|
|
||||||
|
**Additiv 2026-09-07 (Test-Engine):** Zwei Postgres-Instanzen bleiben Dev und Prod. Die Gitea-Suite läuft auf der Dev-Postgres in der Datenbank `kansho_test` (neben `kansho_dev`), nicht gegen SQLite und nicht gegen das persönliche `kansho_dev`-Journal. Die Suite setzt dort das Schema zurück. SQLite bleibt die Windows-App ohne Docker und das lokale Backup-Format.
|
||||||
|
|
||||||
|
**Additiv 2026-09-07 (pytest + öffentliche URLs):** Kanonischer Runner ist `python -m pytest tests` nach Dev-Deploy. `scripts/test-mvp.ps1` wrappt pytest lokal. Produkt-URLs nach Nginx/TLS: `https://dev.kansho.jinkendo.de` und `https://kansho.jinkendo.de`.
|
||||||
|
|
||||||
Empfohlene Reihenfolge in Welle 2:
|
Empfohlene Reihenfolge in Welle 2:
|
||||||
|
|
||||||
|
|
@ -173,6 +176,8 @@ Empfohlene Reihenfolge in Welle 2:
|
||||||
6. Gitea-Workflows erst, wenn Runner-Pfade existieren (`runtime_and_deploy.md` §4).
|
6. Gitea-Workflows erst, wenn Runner-Pfade existieren (`runtime_and_deploy.md` §4).
|
||||||
7. Watchtower später prüfen, nicht in diesem Transfer.
|
7. Watchtower später prüfen, nicht in diesem Transfer.
|
||||||
|
|
||||||
|
Diese Schritte sind im Repository angelegt (`docker-compose*.yml`, `.gitea/workflows/`, `backend/db_init.py`, `backend/sqlite_to_postgres.py`). Pi-Bootstrap und Cutover: `docs/DEPLOYMENT.md`.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 7. Datenschutz beim Transfer
|
## 7. Datenschutz beim Transfer
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ Lokal, eine Instanz, ein Profil-Nutzer nach Setup:
|
||||||
|
|
||||||
Proxy: Vite leitet `/api` an 8018. CORS erlaubt `localhost:5188`.
|
Proxy: Vite leitet `/api` an 8018. CORS erlaubt `localhost:5188`.
|
||||||
|
|
||||||
**Abweichung vom technischen Zielrahmen:** `product_frame_and_stack.md` nennt PostgreSQL. Der Slice läuft auf SQLite. Lokal zulässig; Prod-Pfad offen. Transfer der Urlaubsinstanz: zuerst SQLite-Restore auf dem Heimrechner, danach Compose/Postgres. Siehe `environment_handover.md` und `runtime_and_deploy.md` §7.2.
|
**Abweichung vom technischen Zielrahmen (lokal):** `product_frame_and_stack.md` nennt PostgreSQL. Der Slice auf dem Windows-Checkout bleibt SQLite (`5188`/`8018`). **Additiv 2026-09-07:** Docker/Server nutzt PostgreSQL 16 (`KANSHO_DB_BACKEND=postgres`). Stores behalten SQLite-förmiges SQL; `sql_compat.py` übersetzt zur Laufzeit. Nummerierte Dateien ab `backend/migrations/022_*.sql`. Siehe `runtime_and_deploy.md` und `docs/DEPLOYMENT.md`. **Additiv (Qualitätssystem 2026-09-07):** Kanonischer Backend-Runner ist pytest im Dev-Checkout nach erfolgreichem Deploy. `python -m pytest tests -m "not slow"` gegen `kansho_test`. `scripts/test-mvp.ps1` ist nur noch der lokale Wrapper (Unit ohne Postgres, Integration nur mit `DB_NAME=kansho_test`). SQLite-Testdateien sind abgelöst; `test_local_backup.py` prüft weiter das Windows-Backup-Format.
|
||||||
|
|
||||||
Health: `GET /api/health`.
|
Health: `GET /api/health`.
|
||||||
|
|
||||||
|
|
@ -101,6 +101,8 @@ Prompts in der DB, nicht im Anwendungscode: `mvp.dialogue_turn`, `mvp.journal_re
|
||||||
|
|
||||||
Schema: `backend/schema.sql`. Isolation über `profile_id`.
|
Schema: `backend/schema.sql`. Isolation über `profile_id`.
|
||||||
|
|
||||||
|
**Additiv 2026-09-07:** Dieselbe `schema.sql` ist Greenfield für Postgres (Dialekt nur `datetime('now')` → `CURRENT_TIMESTAMP::text` in `db_init.py`). SQLite-History 001–021 wird auf leerem Postgres als applied erfasst, nicht als ALTER nachgespielt. Persönliche Daten kommen nicht über den Alltagspfad, sondern über `sqlite_to_postgres.py`.
|
||||||
|
|
||||||
**Rahmen / Layer 0:** `profiles`, Auth-`sessions`, `usage_sessions`, `conversations`, `messages`, Thread-/Space-Hüllen, `derived_records` (ungenutzt für Journal-Writes), `identity_mappings`, Prompt-/Feature-/Provider-Tabellen. **Additiv 2026-08-28:** `app_settings` (Instanzschalter, derzeit `debug.persist_traces`) und `debug_runs` (Admin-Testspur je Schritt, Profil-isoliert, keine Mapping-Tabelle).
|
**Rahmen / Layer 0:** `profiles`, Auth-`sessions`, `usage_sessions`, `conversations`, `messages`, Thread-/Space-Hüllen, `derived_records` (ungenutzt für Journal-Writes), `identity_mappings`, Prompt-/Feature-/Provider-Tabellen. **Additiv 2026-08-28:** `app_settings` (Instanzschalter, derzeit `debug.persist_traces`) und `debug_runs` (Admin-Testspur je Schritt, Profil-isoliert, keine Mapping-Tabelle).
|
||||||
|
|
||||||
**Journal-Slice:**
|
**Journal-Slice:**
|
||||||
|
|
@ -221,10 +223,12 @@ Gateway-Verfahren und offener Security Layer: `privacy_gateway.md` §9.1–9.2 u
|
||||||
| Erster Journal-Impuls | `backend/tests/test_journal_opening.py` |
|
| Erster Journal-Impuls | `backend/tests/test_journal_opening.py` |
|
||||||
| Writing-Profile-Hülle / Initial Build | `backend/tests/test_profile_review.py`, `backend/tests/test_profile_governance.py` |
|
| Writing-Profile-Hülle / Initial Build | `backend/tests/test_profile_review.py`, `backend/tests/test_profile_governance.py` |
|
||||||
| Markdown-Runde | `frontend/src/journal/document.test.js` |
|
| Markdown-Runde | `frontend/src/journal/document.test.js` |
|
||||||
| Lokaler Abnahmelauf | `scripts/test-mvp.ps1` |
|
| Lokaler Wrapper | `scripts/test-mvp.ps1` (pytest, kein kanonischer Runner) |
|
||||||
|
|
||||||
Tests beweisen API-Verträge und Invarianten, nicht Dialogqualität.
|
Tests beweisen API-Verträge und Invarianten, nicht Dialogqualität.
|
||||||
|
|
||||||
|
**Additiv 2026-09-07:** pytest ist der Backend-Runner (`backend/pytest.ini`). Gitea führt `python -m pytest tests -m "not slow"` nach erfolgreichem Dev-Deploy auf `kansho_test` aus. `scripts/test-mvp.ps1` wrappt denselben Aufruf lokal. Unit-Marker laufen ohne Postgres; Integration nie gegen `kansho_dev`. Schema einmal pro Lauf, dazwischen nur Daten-Truncate — nicht 22× `schema.sql`.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# 9. Technische Abweichungen und nicht gebaut
|
# 9. Technische Abweichungen und nicht gebaut
|
||||||
|
|
@ -275,7 +279,7 @@ Additiv zum Slice, 2026-08-25. Kein Target-Model-Vorbau.
|
||||||
|
|
||||||
Additiv zum Slice. Fachliche Abnahme: `../functional/mvp_freeze_candidate.md`.
|
Additiv zum Slice. Fachliche Abnahme: `../functional/mvp_freeze_candidate.md`.
|
||||||
|
|
||||||
- Lokaler Testeinstieg `scripts/test-mvp.ps1` (UTF-8, isolierte Temp-Daten, keine Live-Keys; Fake nur in den Suites, die ihn setzen).
|
- Lokaler Wrapper `scripts/test-mvp.ps1` (pytest; UTF-8, isolierte Temp-Daten, keine Live-Keys; Fake nur in den Suites, die ihn setzen). Kanonisch: Gitea pytest auf `kansho_test`.
|
||||||
- Backup: `backend/local_backup.py`, Einstieg `scripts/backup-local.ps1`. SQLite-Backup-API, Medien, Manifest, keine Secrets. Restore bestätigt, prüft Checksummen, legt Sicherheitsbackup an, bricht bei geöffneter DB ab.
|
- Backup: `backend/local_backup.py`, Einstieg `scripts/backup-local.ps1`. SQLite-Backup-API, Medien, Manifest, keine Secrets. Restore bestätigt, prüft Checksummen, legt Sicherheitsbackup an, bricht bei geöffneter DB ab.
|
||||||
- Papierkorb: `GET /spaces/{id}/entries`, `GET /spaces/{id}/trash`, `POST /entries/{id}/undelete`, `POST /entries/{id}/purge`.
|
- Papierkorb: `GET /spaces/{id}/entries`, `GET /spaces/{id}/trash`, `POST /entries/{id}/undelete`, `POST /entries/{id}/purge`.
|
||||||
- Erster Impuls: `journal_opening.py`, derselbe Gateway-Pfad, `{{opening_hint}}`. **Additiv 2026-09-03:** Vorhaben nur aus dem user-Dialog des vorigen Kalendertags (`calendar_date` an `space_recent_sources`); Journal-Entries und ältere Recency sind kein Opening-Mandat. Der Opening-Call setzt `include_space_recency=False`. Tests: abgeschlossener Entry mit „nächsten Tag“ und Plan von vorgestern → `local_neutral`; „Morgen wollen wir …“ nur am Folgetag → Modellpfad. **Additiv 2026-09-07:** Prefix-Recency (5×400 Entries, 3×400 Source-Conversations) bleibt im *laufenden* Dialogzug; das ist kein Opening-Mandat. Ein Nachsatz mit Morgenuhrzeit darf den Folgetag nicht über den Entry-Anfang steuern.
|
- Erster Impuls: `journal_opening.py`, derselbe Gateway-Pfad, `{{opening_hint}}`. **Additiv 2026-09-03:** Vorhaben nur aus dem user-Dialog des vorigen Kalendertags (`calendar_date` an `space_recent_sources`); Journal-Entries und ältere Recency sind kein Opening-Mandat. Der Opening-Call setzt `include_space_recency=False`. Tests: abgeschlossener Entry mit „nächsten Tag“ und Plan von vorgestern → `local_neutral`; „Morgen wollen wir …“ nur am Folgetag → Modellpfad. **Additiv 2026-09-07:** Prefix-Recency (5×400 Entries, 3×400 Source-Conversations) bleibt im *laufenden* Dialogzug; das ist kein Opening-Mandat. Ein Nachsatz mit Morgenuhrzeit darf den Folgetag nicht über den Entry-Anfang steuern.
|
||||||
|
|
|
||||||
|
|
@ -207,6 +207,8 @@ Erlaubte Typen: `PERSON`, `PLACE`, `ORG`, `PROJECT`. `PROJECT` ist additiv entsc
|
||||||
|
|
||||||
**Additiv 2026-08-29 (Detect-Vertrag, ein vollständiger Neuversuch):** Eine formal ungültige Detector-Antwort (unbekannter `entity_type`, Extra-/Fehlfelder, ungültiges JSON, nicht verankerbarer Text, Abbruch/`finish_reason=length`, unvollständige Chunk-Abdeckung) verwirft den gesamten bisherigen request-lokalen Pass. Es folgt genau ein neuer vollständiger Pass über alle Chunks mit einer allgemeinen Schema-Korrekturanweisung, ohne textspezifische Namen oder Fehlertypen. Misslingt auch der zweite Pass: fail-closed, kein Generate, kein Journalentwurf. Netzwerk- und Timeout-Fehler (`detect_chunk_failed`) werden nicht durch denselben Vertrag-Retry verdoppelt. Ungültige Entitäten werden nicht still ignoriert, nicht auf erlaubte Typen umgedeutet und nicht durch Pattern-Fallback ersetzt. Compact-Fehlerdiagnose: `detect_passes`, Versuche mit Chunk-/Aufruf-/Coverage-Zahlen, aggregierte Detect-Tokens/Kosten/Dauer, `detect_partial_discarded`, `contract_violation`, bei unbekanntem Typ den gelieferten `entity_type` ohne Klartextspan, `generate_called: false`. Unbekannte Providerkosten bleiben `detect_cost_unknown`. Diagnoseobjekte sind azyklisch (`trace.budget` ist ein Snapshot, kein Live-`diagnostics`). Persistenzfehler ersetzen den ursprünglichen EngineError nicht.
|
**Additiv 2026-08-29 (Detect-Vertrag, ein vollständiger Neuversuch):** Eine formal ungültige Detector-Antwort (unbekannter `entity_type`, Extra-/Fehlfelder, ungültiges JSON, nicht verankerbarer Text, Abbruch/`finish_reason=length`, unvollständige Chunk-Abdeckung) verwirft den gesamten bisherigen request-lokalen Pass. Es folgt genau ein neuer vollständiger Pass über alle Chunks mit einer allgemeinen Schema-Korrekturanweisung, ohne textspezifische Namen oder Fehlertypen. Misslingt auch der zweite Pass: fail-closed, kein Generate, kein Journalentwurf. Netzwerk- und Timeout-Fehler (`detect_chunk_failed`) werden nicht durch denselben Vertrag-Retry verdoppelt. Ungültige Entitäten werden nicht still ignoriert, nicht auf erlaubte Typen umgedeutet und nicht durch Pattern-Fallback ersetzt. Compact-Fehlerdiagnose: `detect_passes`, Versuche mit Chunk-/Aufruf-/Coverage-Zahlen, aggregierte Detect-Tokens/Kosten/Dauer, `detect_partial_discarded`, `contract_violation`, bei unbekanntem Typ den gelieferten `entity_type` ohne Klartextspan, `generate_called: false`. Unbekannte Providerkosten bleiben `detect_cost_unknown`. Diagnoseobjekte sind azyklisch (`trace.budget` ist ein Snapshot, kein Live-`diagnostics`). Persistenzfehler ersetzen den ursprünglichen EngineError nicht.
|
||||||
|
|
||||||
|
**Additiv 2026-09-08 (Lebensmittel-Homonym, kein Vertragsbruch):** Explizit nicht-identifizierende Detect-Typen (`FOOD`, `DISH`, `MEAL`, `CUISINE`, `FOODSTUFF`, `INGREDIENT`) sind keine Schemaverletzung. Sie werden nicht auf einen erlaubten Typ umgedeutet und nicht maskiert, sofern die lokale Homonymregel denselben Wortlaut als Sache behandelt (`Sushi essen`, `aß Sushi`). Steht dieselbe Meldung in einem Personenkontext (`Frau Sushi`, `Sushi kam`), gilt lokal die bestehende Identitätsregel und der Span wird als `PERSON` behalten. Andere unbekannte Typen (`HUMAN`, `NAME`, `WIFE`, …) bleiben fail-closed. Extrafelder, Tokens, unbrauchbare Offsets und unvollständige Chunks bleiben unverändert fail-closed. Compact-Diagnose darf `omitted_non_identity_types` ohne Klartextspan enthalten.
|
||||||
|
|
||||||
Schemaverletzungen, Abbruch und unvollständige Chunks bleiben fail-closed. Im Dialogzug erzeugt ein solcher Fail-closed lokal einen Halte-Impuls, ohne den Generate-Provider zu rufen. Der Nutzer sieht keine interne Substring-Diagnose.
|
Schemaverletzungen, Abbruch und unvollständige Chunks bleiben fail-closed. Im Dialogzug erzeugt ein solcher Fail-closed lokal einen Halte-Impuls, ohne den Generate-Provider zu rufen. Der Nutzer sieht keine interne Substring-Diagnose.
|
||||||
|
|
||||||
### RequestDetectionManifest vs. ConfirmedIdentityRegistry
|
### RequestDetectionManifest vs. ConfirmedIdentityRegistry
|
||||||
|
|
|
||||||
|
|
@ -110,14 +110,14 @@ Mitai-Prompt-Engine-Muster (ein Executor, Registry, Admin-Konfiguration) wird ü
|
||||||
| Referenz | Mitai-Rahmen weitgehend | entschieden |
|
| Referenz | Mitai-Rahmen weitgehend | entschieden |
|
||||||
| Fachliche IA / Startroute | nicht durch den Rahmen vorwegnehmen | offen (Fach-Arbeitsstand) |
|
| Fachliche IA / Startroute | nicht durch den Rahmen vorwegnehmen | offen (Fach-Arbeitsstand) |
|
||||||
| Mandantenmodell | keines | entschieden |
|
| Mandantenmodell | keines | entschieden |
|
||||||
| Konkrete Ports/Domains | nicht aus Mitai kopieren | offen |
|
| Konkrete Ports/Domains | lokal 5188/8018; Server 3006/8005 und 3096/8096; `kansho.jinkendo.de` | entschieden 2026-09-07 |
|
||||||
| Gemeinsame UI-Bibliothek der Familie | eigene Kopie des Musters, kein Shared-Package vorausgesetzt | bevorzugte Richtung |
|
| Gemeinsame UI-Bibliothek der Familie | eigene Kopie des Musters, kein Shared-Package vorausgesetzt | bevorzugte Richtung |
|
||||||
|
|
||||||
## 9. Offene Fragen
|
## 9. Offene Fragen
|
||||||
|
|
||||||
1. Soll der Rahmen später in ein gemeinsames Jinkendo-Paket extrahiert werden oder als kopiertes Muster in Kanshō leben?
|
1. Soll der Rahmen später in ein gemeinsames Jinkendo-Paket extrahiert werden oder als kopiertes Muster in Kanshō leben?
|
||||||
2. Welche Domain und welche Host-Ports gelten für Dev/Prod?
|
2. ~~Welche Domain und welche Host-Ports gelten für Dev/Prod?~~ **Entschieden 2026-09-07:** `kansho.jinkendo.de` / `dev.kansho.jinkendo.de`, Ports 3006/8005 und 3096/8096. Prod-Frontend nicht 3005 (Bookstack).
|
||||||
3. Wird Nginx als eigener Container beibehalten oder das Vite-Preview nur für lokale Entwicklung genutzt?
|
3. ~~Wird Nginx als eigener Container beibehalten oder das Vite-Preview nur für lokale Entwicklung genutzt?~~ **Entschieden:** Frontend-Nginx im Compose-Container (Prod/Dev-Server); Vite nur lokal.
|
||||||
|
|
||||||
## 10. Querverweise
|
## 10. Querverweise
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ Muster der Familie: zwei Umgebungen, zwei Branches.
|
||||||
| Development | `develop` | Auto-Deploy nach Push |
|
| Development | `develop` | Auto-Deploy nach Push |
|
||||||
| Production | `main` | Auto-Deploy nach Merge |
|
| Production | `main` | Auto-Deploy nach Merge |
|
||||||
|
|
||||||
**Status: entschieden** als Betriebsmuster. Konkrete Domains, Host-Pfade und **Produktions-Ports** sind **offen** und werden nicht aus Mitai (`3002`/`8002`, `3099`/`8099`, `bodytrack/`) kopiert.
|
**Status: entschieden** als Betriebsmuster. Host, Pfade, Domains und Ports sind seit 2026-09-07 festgelegt (gleicher Raspberry Pi wie die Schwesterprodukte, eigene Compose-Projekte, keine Mitai-`bodytrack/`-Pfade).
|
||||||
|
|
||||||
Lokale Entwicklung (ohne Docker) verwendet eigene Ports, nicht die Vite-/FastAPI-Defaults und nicht die Ports anderer lokaler Repos:
|
Lokale Entwicklung (ohne Docker) verwendet eigene Ports, nicht die Vite-/FastAPI-Defaults und nicht die Ports anderer lokaler Repos:
|
||||||
|
|
||||||
|
|
@ -31,7 +31,19 @@ Lokale Entwicklung (ohne Docker) verwendet eigene Ports, nicht die Vite-/FastAPI
|
||||||
|
|
||||||
Nicht verwenden: 5173, 5174, 4000, 4001, 8000.
|
Nicht verwenden: 5173, 5174, 4000, 4001, 8000.
|
||||||
|
|
||||||
Hypothese für spätere Benennung: `kansho.jinkendo.de` / `dev.kansho.jinkendo.de`, analog zur Foundation-Tabelle. Nicht festgelegt.
|
Server (Raspberry Pi 5, `192.168.2.49`):
|
||||||
|
|
||||||
|
| | Development | Production |
|
||||||
|
|---|---|---|
|
||||||
|
| Branch | `develop` | `main` |
|
||||||
|
| Host-Pfad | `/home/lars/docker/kansho-dev` | `/home/lars/docker/kansho` |
|
||||||
|
| Domain | `dev.kansho.jinkendo.de` | `kansho.jinkendo.de` |
|
||||||
|
| Frontend-Port | 3096 | 3006 |
|
||||||
|
| Backend-Port | 8096 | 8005 |
|
||||||
|
| Postgres | eigene Instanz `kansho_dev`, nicht nach außen | eigene Instanz `kansho`, nicht nach außen |
|
||||||
|
| Compose | `docker-compose.dev-env.yml` | `docker-compose.yml` |
|
||||||
|
|
||||||
|
Operative Schritte: `docs/DEPLOYMENT.md`. Gitea intern: `http://192.168.2.144:3000/Lars/Kansho.git`.
|
||||||
|
|
||||||
## 2. Container
|
## 2. Container
|
||||||
|
|
||||||
|
|
@ -65,13 +77,15 @@ Mitai-Pipeline als Vorlage:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
push develop → deploy-dev.yml → compose build/up → Healthcheck
|
push develop → deploy-dev.yml → compose build/up → Healthcheck
|
||||||
push/PR nach Tests → test.yml (pytest, Frontend-Build)
|
successful Dev deploy → test.yml (pytest on kansho_test, Live-Smoke /api/health, Frontend-Build)
|
||||||
merge in main → deploy-prod.yml
|
merge in main → deploy-prod.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
**Status: bevorzugte Richtung.** Workflow-Dateien erst anlegen, wenn Code und Runner-Pfade existieren.
|
**Status: entschieden.** Workflows liegen unter `.gitea/workflows/` (`deploy-dev.yml`, `deploy-prod.yml`, `test.yml`). Muster analog Kairo: `git reset --hard`, `build --no-cache`, Health `GET /api/health`.
|
||||||
|
|
||||||
Kanshō-Repo liegt bereits auf Gitea (`Lars/Kansho`). HTTPS-Push ist eingerichtet.
|
**Additiv 2026-09-07 (Qualitätssystem):** Backend-Runner ist pytest (`backend/pytest.ini`, `backend/tests/conftest.py`). Gitea startet die Suite erst nach erfolgreichem Dev-Deploy (`workflow_run` auf `Deploy Development`), nicht parallel zum Image-Build. Destruktive Tests bleiben auf `kansho_test`. Zusätzlich ein nicht-schreibender Smoke gegen die laufende Dev-API (`kansho_dev`, nur `/api/health`). Öffentliche URLs nach Host-Nginx/TLS: `https://dev.kansho.jinkendo.de` und `https://kansho.jinkendo.de`. LAN-Ports 3096/8096 und 3006/8005 bleiben die Compose-Publish-Ziele.
|
||||||
|
|
||||||
|
Kanshō-Repo liegt auf Gitea (`Lars/Kansho`). HTTPS-Push ist eingerichtet. Der Pi-Runner (`ubuntu-latest`) ist derselbe wie bei Mitai/Shinkan/Kairo.
|
||||||
|
|
||||||
## 5. Was Deploy nicht übernimmt
|
## 5. Was Deploy nicht übernimmt
|
||||||
|
|
||||||
|
|
@ -86,17 +100,21 @@ Kanshō-Repo liegt bereits auf Gitea (`Lars/Kansho`). HTTPS-Push ist eingerichte
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Compose + Postgres + nummerierte SQL-Migrationen | ja | entschieden |
|
| Compose + Postgres + nummerierte SQL-Migrationen | ja | entschieden |
|
||||||
| develop → Dev, main → Prod | ja | entschieden |
|
| develop → Dev, main → Prod | ja | entschieden |
|
||||||
| Ports/Domains/Hostpfade | Prod offen; lokal Frontend 5188 / Backend 8018 | lokal festgelegt, Prod offen |
|
| Ports/Domains/Hostpfade | lokal 5188/8018; Server 3096/8096 und 3006/8005; `*.kansho.jinkendo.de` | entschieden 2026-09-07 |
|
||||||
| Gitea Workflows | Mitai-Muster | bevorzugte Richtung |
|
| Host | gleicher Raspberry Pi wie Mitai/Shinkan/Kairo, eigene Compose-Projekte `kansho` / `kansho-dev` | entschieden 2026-09-07 |
|
||||||
|
| Postgres | eigene Instanz je Umgebung (`kansho` / `kansho_dev`), kein Shared-Schema | entschieden 2026-09-07 |
|
||||||
|
| Gitea Workflows | `.gitea/workflows/` analog Kairo | entschieden 2026-09-07 |
|
||||||
| Auto-Rollback | nein | verworfen |
|
| Auto-Rollback | nein | verworfen |
|
||||||
| Urlaubs-Laptop ohne Docker/SQLite | Ist bis Transfer Welle 1 | dokumentiert 2026-09-07 |
|
| Dual-Backend | SQLite nur Windows-App ohne Docker; Server Dev+Prod PostgreSQL. pytest auf der Dev-Instanz in DB `kansho_test` neben `kansho_dev` (keine dritte Instanz) | entschieden 2026-09-07, pytest 2026-09-07 |
|
||||||
|
| Öffentliche URLs | `https://dev.kansho.jinkendo.de` / `https://kansho.jinkendo.de` (Host-Nginx + Let's Encrypt; Einrichtung parallel) | entschieden 2026-09-07 |
|
||||||
|
| Urlaubs-Laptop ohne Docker/SQLite | Phase A auf dem Heimrechner restored | dokumentiert 2026-09-07 |
|
||||||
| Transfer vor Postgres | SQLite-Restore auf dem Heimrechner zuerst | entschieden als Reihenfolge |
|
| Transfer vor Postgres | SQLite-Restore auf dem Heimrechner zuerst | entschieden als Reihenfolge |
|
||||||
|
|
||||||
## 7. Offene Fragen
|
## 7. Offene Fragen
|
||||||
|
|
||||||
1. Läuft Kanshō auf demselben Raspberry-Pi-Host wie Mitai, mit eigenen Compose-Projekten?
|
1. ~~Läuft Kanshō auf demselben Raspberry-Pi-Host wie Mitai, mit eigenen Compose-Projekten?~~ **Entschieden:** ja, Pi `192.168.2.49`, Pfade `/home/lars/docker/kansho` und `kansho-dev`.
|
||||||
2. Gemeinsames oder separates Postgres?
|
2. ~~Gemeinsames oder separates Postgres?~~ **Entschieden:** eigene Instanz je Umgebung, kein Shared-Schema mit Mitai.
|
||||||
3. Backup-Rhythmus und Restore-Übung vor erstem persönlichen Dialogdatenbestand. Die Urlaubs-SQLite-Restore auf dem Heimrechner (Welle 1) ist diese Übung für den bestehenden Datenbestand; Produktions-Backup für Postgres bleibt offen.
|
3. Backup-Rhythmus nach dem Prod-Cutover. Die Urlaubs-SQLite-Restore auf dem Heimrechner (Welle 1) ist die Übung für den bestehenden Datenbestand. Postgres-Dump plus Medien: `docs/DEPLOYMENT.md` und `scripts/backup-postgres.sh`. Erste Restore-Übung auf Dev vor dem Prod-Import.
|
||||||
|
|
||||||
### 7.1 Lokales Backup für die Urlaubs-Testphase (2026-08-25)
|
### 7.1 Lokales Backup für die Urlaubs-Testphase (2026-08-25)
|
||||||
|
|
||||||
|
|
@ -111,15 +129,15 @@ Kein Ersatz für Produktions-Backup, Docker oder Verschlüsselung.
|
||||||
|
|
||||||
Die Urlaubsinstanz lief ohne Docker auf SQLite (`backend/data/kansho.sqlite`, Ports 5188/8018).
|
Die Urlaubsinstanz lief ohne Docker auf SQLite (`backend/data/kansho.sqlite`, Ports 5188/8018).
|
||||||
|
|
||||||
**Reihenfolge entschieden:** zuerst Kontinuität (Clone + SQLite-Restore + Smoke), danach Betriebsrahmen (Compose, Postgres 16, Gitea-Runner). Postgres bleibt Ziel; `backend/db.py` ist weiterhin SQLite. Ein direkter Sprung „Laptop-SQLite nach Prod-Postgres“ ohne Zwischen-Restore ist verworfen.
|
**Reihenfolge entschieden:** zuerst Kontinuität (Clone + SQLite-Restore + Smoke), danach Betriebsrahmen (Compose, Postgres 16, Gitea-Runner). Dual-Backend: SQLite nur für die Windows-App ohne Docker; Postgres im Container (zwei Instanzen: Dev `kansho_dev`, Prod `kansho`). Die Test-Suite nutzt dieselbe Dev-Postgres, Datenbank `kansho_test` neben `kansho_dev`, weil sie das Schema zurücksetzt. Ein direkter Sprung „Laptop-SQLite nach Prod-Postgres“ ohne Zwischen-Restore ist verworfen. Import: `sqlite_to_postgres.py --confirm`, zuerst Dev, dann Prod.
|
||||||
|
|
||||||
**Additiv 2026-09-07 (Transportweg):** Auf dem Urlaubs-Laptop dürfen weder USB-Stick noch NAS verbunden werden. Der einzige Weg für Datenbank und persönliche Journaldaten ist das **selbst gehostete** Gitea (`gitea.stommer.de`, nur eigene Infrastruktur). Das ist eine einmalige Transportentscheidung, kein Produktregelwechsel: Provider, OpenRouter und öffentliche Remotes bleiben ausgeschlossen. Provider-Keys und `backend/.env` bleiben **außerhalb** Git.
|
**Additiv 2026-09-07 (Transportweg):** Auf dem Urlaubs-Laptop dürfen weder USB-Stick noch NAS verbunden werden. Der einzige Weg für Datenbank und persönliche Journaldaten ist das **selbst gehostete** Gitea (`gitea.stommer.de`, nur eigene Infrastruktur). Das ist eine einmalige Transportentscheidung, kein Produktregelwechsel: Provider, OpenRouter und öffentliche Remotes bleiben ausgeschlossen. Provider-Keys und `backend/.env` bleiben **außerhalb** Git.
|
||||||
|
|
||||||
Aktuelles Transportartefakt: `transfer/kansho-laptop-20260907.zip` (SHA256 `EBF7455D5688588E8C6E50C05D16200CE191D79FB7CEACDBA9C25460B7497B07`). Nach erfolgreichem Restore auf dem Zielrechner das Verzeichnis `transfer/` aus dem Arbeitsbaum entfernen.
|
Transportartefakt 2026-09-07: `transfer/kansho-laptop-20260907.zip` (SHA256 `EBF7455D5688588E8C6E50C05D16200CE191D79FB7CEACDBA9C25460B7497B07`). Nach erfolgreichem Restore auf dem Heimrechner das Verzeichnis `transfer/` aus dem Arbeitsbaum entfernen. Die Git-Historie behält das Zip.
|
||||||
|
|
||||||
Kanonisches Session-Handover: `environment_handover.md`. Aufträge: `../../work_orders/laptop_closeout.md`, `../../work_orders/home_environment_setup.md`.
|
Kanonisches Session-Handover: `environment_handover.md`. Aufträge: `../../work_orders/laptop_closeout.md`, `../../work_orders/home_environment_setup.md`. Betrieb: `docs/DEPLOYMENT.md`.
|
||||||
|
|
||||||
Noch nicht im Repository: Dockerfiles, Compose, `.gitea/workflows/`, Postgres-Adapter. Mitai unter `C:\dev\mitai` bleibt Muster, nicht Quelle für Ports oder `bodytrack/`-Pfade.
|
Compose, Dockerfiles, `.gitea/workflows/` und der Postgres-Adapter liegen im Repository. Mitai unter `C:\Dev\mitai-jinkendo` bleibt Muster, nicht Quelle für Ports oder `bodytrack/`-Pfade.
|
||||||
|
|
||||||
## 8. Querverweise
|
## 8. Querverweise
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -156,3 +156,9 @@ Nach Phase A zwingend, nach B/C erneut:
|
||||||
3. Ob Daten SQLite oder Postgres führen
|
3. Ob Daten SQLite oder Postgres führen
|
||||||
4. Restore-Nachweis (Anzahlen, ein Medienbeispiel ohne Inhalt zu zitieren)
|
4. Restore-Nachweis (Anzahlen, ein Medienbeispiel ohne Inhalt zu zitieren)
|
||||||
5. Offene Punkte für die nächste Session
|
5. Offene Punkte für die nächste Session
|
||||||
|
|
||||||
|
## Stand 2026-09-07 (Heimrechner)
|
||||||
|
|
||||||
|
Phase A: Restore aus `transfer/kansho-laptop-20260907.zip` nach `backend/data/` (1 Profil, 2 Spaces inkl. `Kroatien 2026`, 17 Days/Entries, 20 Conversations, 379 Messages, 70 Mediendateien). Health `GET /api/health` ok. `backend/.env` neu aus Example (Keys vom Nutzer nachzutragen). `transfer/` danach aus dem Tree genommen.
|
||||||
|
|
||||||
|
Phase B–D im Repository: Compose, Dual-Backend, Workflows, `docs/DEPLOYMENT.md`. Pi-Verzeichnisse und Host-nginx sind einmalig auf `192.168.2.49` anzulegen. Persönlicher Postgres-Import zuerst nach Dev, dann Prod.
|
||||||
|
|
|
||||||
15
frontend/Dockerfile
Normal file
15
frontend/Dockerfile
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
FROM node:20-alpine AS build
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
COPY package.json package-lock.json ./
|
||||||
|
RUN npm ci
|
||||||
|
COPY . .
|
||||||
|
ARG VITE_API_URL=
|
||||||
|
ENV VITE_API_URL=$VITE_API_URL
|
||||||
|
RUN npm run build
|
||||||
|
|
||||||
|
FROM nginx:alpine
|
||||||
|
COPY --from=build /app/dist /usr/share/nginx/html
|
||||||
|
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||||
|
EXPOSE 80
|
||||||
|
CMD ["nginx", "-g", "daemon off;"]
|
||||||
26
frontend/nginx.conf
Normal file
26
frontend/nginx.conf
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name localhost;
|
||||||
|
root /usr/share/nginx/html;
|
||||||
|
index index.html;
|
||||||
|
|
||||||
|
resolver 127.0.0.11 valid=10s ipv6=off;
|
||||||
|
client_max_body_size 80m;
|
||||||
|
|
||||||
|
location ^~ /api/ {
|
||||||
|
set $docker_backend_svc backend;
|
||||||
|
proxy_pass http://$docker_backend_svc:8000$request_uri;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_connect_timeout 60s;
|
||||||
|
proxy_send_timeout 600s;
|
||||||
|
proxy_read_timeout 600s;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
try_files $uri $uri/ /index.html;
|
||||||
|
}
|
||||||
|
}
|
||||||
23
nginx/certbot-setup.sh
Normal file
23
nginx/certbot-setup.sh
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Let's Encrypt for kansho.jinkendo.de and dev.kansho.jinkendo.de
|
||||||
|
# Prerequisites: host nginx installed, ports 80/443 reachable, DNS A/AAAA in place.
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
EMAIL="${CERTBOT_EMAIL:-lars@stommer.de}"
|
||||||
|
|
||||||
|
echo "=== Let's Encrypt for Kanshō ==="
|
||||||
|
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y certbot python3-certbot-nginx
|
||||||
|
|
||||||
|
sudo certbot --nginx \
|
||||||
|
-d kansho.jinkendo.de \
|
||||||
|
-d dev.kansho.jinkendo.de \
|
||||||
|
--email "$EMAIL" \
|
||||||
|
--agree-tos \
|
||||||
|
--non-interactive \
|
||||||
|
--redirect
|
||||||
|
|
||||||
|
echo "Check renewal: sudo certbot renew --dry-run"
|
||||||
|
echo "Timer: sudo systemctl status certbot.timer"
|
||||||
50
nginx/kansho-dev.conf
Normal file
50
nginx/kansho-dev.conf
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
# Kanshō development vhost. Install as /etc/nginx/sites-available/kansho-dev.
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name dev.kansho.jinkendo.de;
|
||||||
|
|
||||||
|
location /.well-known/acme-challenge/ {
|
||||||
|
root /var/www/certbot;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
return 301 https://$host$request_uri;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 443 ssl http2;
|
||||||
|
server_name dev.kansho.jinkendo.de;
|
||||||
|
|
||||||
|
ssl_certificate /etc/letsencrypt/live/dev.kansho.jinkendo.de/fullchain.pem;
|
||||||
|
ssl_certificate_key /etc/letsencrypt/live/dev.kansho.jinkendo.de/privkey.pem;
|
||||||
|
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||||
|
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||||
|
|
||||||
|
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||||
|
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
|
||||||
|
client_max_body_size 80m;
|
||||||
|
|
||||||
|
location /api/ {
|
||||||
|
proxy_pass http://127.0.0.1:8096;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_read_timeout 600s;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://127.0.0.1:3096;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
proxy_cache_bypass $http_upgrade;
|
||||||
|
}
|
||||||
|
}
|
||||||
54
nginx/kansho.conf
Normal file
54
nginx/kansho.conf
Normal file
|
|
@ -0,0 +1,54 @@
|
||||||
|
# Kanshō – Host nginx (outside Compose)
|
||||||
|
# Install as /etc/nginx/sites-available/kansho and symlink into sites-enabled.
|
||||||
|
# TLS via nginx/certbot-setup.sh. Reverse-proxy targets are the Pi publish ports.
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name kansho.jinkendo.de;
|
||||||
|
|
||||||
|
location /.well-known/acme-challenge/ {
|
||||||
|
root /var/www/certbot;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
return 301 https://$host$request_uri;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 443 ssl http2;
|
||||||
|
server_name kansho.jinkendo.de;
|
||||||
|
|
||||||
|
ssl_certificate /etc/letsencrypt/live/kansho.jinkendo.de/fullchain.pem;
|
||||||
|
ssl_certificate_key /etc/letsencrypt/live/kansho.jinkendo.de/privkey.pem;
|
||||||
|
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||||
|
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||||
|
|
||||||
|
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||||
|
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
|
||||||
|
client_max_body_size 80m;
|
||||||
|
gzip on;
|
||||||
|
gzip_types text/plain text/css application/json application/javascript text/xml application/xml;
|
||||||
|
|
||||||
|
location /api/ {
|
||||||
|
proxy_pass http://127.0.0.1:8005;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_read_timeout 600s;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://127.0.0.1:3006;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
proxy_cache_bypass $http_upgrade;
|
||||||
|
}
|
||||||
|
}
|
||||||
25
scripts/backup-postgres.sh
Normal file
25
scripts/backup-postgres.sh
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
#!/bin/sh
|
||||||
|
# Postgres dump + note for the media volume. Run on the Pi next to Compose.
|
||||||
|
# Usage: ./scripts/backup-postgres.sh [prod|dev]
|
||||||
|
set -e
|
||||||
|
ENV_NAME="${1:-prod}"
|
||||||
|
STAMP=$(date -u +%Y%m%d-%H%M%S)
|
||||||
|
BACKUP_ROOT="${KANSHO_PG_BACKUP_DIR:-$HOME/backups/kansho}"
|
||||||
|
mkdir -p "$BACKUP_ROOT"
|
||||||
|
|
||||||
|
if [ "$ENV_NAME" = "dev" ]; then
|
||||||
|
APP_DIR="${KANSHO_DEV_DIR:-/home/lars/docker/kansho-dev}"
|
||||||
|
COMPOSE="docker compose -f docker-compose.dev-env.yml"
|
||||||
|
DB_USER="${DB_USER:-kansho_dev}"
|
||||||
|
DB_NAME="${DB_NAME:-kansho_dev}"
|
||||||
|
else
|
||||||
|
APP_DIR="${KANSHO_PROD_DIR:-/home/lars/docker/kansho}"
|
||||||
|
COMPOSE="docker compose"
|
||||||
|
DB_USER="${DB_USER:-kansho}"
|
||||||
|
DB_NAME="${DB_NAME:-kansho}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$APP_DIR"
|
||||||
|
$COMPOSE exec -T postgres pg_dump -U "$DB_USER" "$DB_NAME" > "$BACKUP_ROOT/kansho-$ENV_NAME-$STAMP.sql"
|
||||||
|
echo "Wrote $BACKUP_ROOT/kansho-$ENV_NAME-$STAMP.sql"
|
||||||
|
echo "Copy the media volume separately; see docs/DEPLOYMENT.md."
|
||||||
29
scripts/sqlite-to-postgres.ps1
Normal file
29
scripts/sqlite-to-postgres.ps1
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
# Wrapper for backend/sqlite_to_postgres.py (personal data, require -Confirm).
|
||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[string]$Sqlite,
|
||||||
|
[string]$MediaSrc,
|
||||||
|
[string]$MediaDest,
|
||||||
|
[switch]$Confirm,
|
||||||
|
[switch]$Replace
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
|
||||||
|
$root = Split-Path -Parent $PSScriptRoot
|
||||||
|
$py = Join-Path $root "backend\.venv\Scripts\python.exe"
|
||||||
|
if (-not (Test-Path $py)) {
|
||||||
|
$py = "python"
|
||||||
|
}
|
||||||
|
|
||||||
|
$env:PYTHONUTF8 = "1"
|
||||||
|
$argsList = @((Join-Path $root "backend\sqlite_to_postgres.py"))
|
||||||
|
if ($Sqlite) { $argsList += @("--sqlite", $Sqlite) }
|
||||||
|
if ($MediaSrc) { $argsList += @("--media-src", $MediaSrc) }
|
||||||
|
if ($MediaDest) { $argsList += @("--media-dest", $MediaDest) }
|
||||||
|
if ($Confirm) { $argsList += "--confirm" }
|
||||||
|
if ($Replace) { $argsList += "--replace" }
|
||||||
|
|
||||||
|
& $py @argsList
|
||||||
|
exit $LASTEXITCODE
|
||||||
|
|
@ -71,24 +71,38 @@ Remove-Item Env:KANSHO_FAKE_DETECT -ErrorAction SilentlyContinue
|
||||||
$env:KANSHO_PROVIDER_KEY = ""
|
$env:KANSHO_PROVIDER_KEY = ""
|
||||||
$env:KANSHO_DETECT_PROVIDER_KEY = ""
|
$env:KANSHO_DETECT_PROVIDER_KEY = ""
|
||||||
$env:KANSHO_MEDIA_ROOT = $iso.Media
|
$env:KANSHO_MEDIA_ROOT = $iso.Media
|
||||||
$env:KANSHO_DB_PATH = $iso.Db
|
|
||||||
$env:KANSHO_DATA_DIR = $iso.Base
|
$env:KANSHO_DATA_DIR = $iso.Base
|
||||||
|
Remove-Item Env:KANSHO_DB_PATH -ErrorAction SilentlyContinue
|
||||||
|
|
||||||
Write-Host "Kansho local MVP tests"
|
function Test-PostgresSuite {
|
||||||
|
$backend = ("$env:KANSHO_DB_BACKEND").ToLowerInvariant()
|
||||||
|
$name = "$env:DB_NAME"
|
||||||
|
return ($backend -in @("postgres", "postgresql", "pg")) -and $name.EndsWith("_test")
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Host "Kansho local tests (pytest wrapper)"
|
||||||
Write-Host "Python: $py"
|
Write-Host "Python: $py"
|
||||||
Write-Host "Isolation: $($iso.Base)"
|
Write-Host "Isolation: $($iso.Base)"
|
||||||
Write-Host "No live provider/detect calls. Production DB/media untouched."
|
Write-Host "No live provider/detect calls. Production DB/media untouched."
|
||||||
Write-Host "Fail-closed stays testable; suites that need a fake provider set it themselves."
|
Write-Host "Canonical backend suite: Gitea pytest on kansho_test after Dev deploy."
|
||||||
|
$runBackend = Test-PostgresSuite
|
||||||
|
if ($runBackend) {
|
||||||
|
Write-Host "Backend pytest: Postgres $($env:DB_NAME)"
|
||||||
|
} else {
|
||||||
|
Write-Host "Backend pytest: unit only (set KANSHO_DB_BACKEND=postgres and DB_NAME=kansho_test for integration)."
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$backendTests = Get-ChildItem -Path (Join-Path $root "backend\tests") -Filter "test_*.py" | Sort-Object Name
|
Invoke-Step -Name "backend pytest" -Action {
|
||||||
if (-not $backendTests) {
|
Push-Location (Join-Path $root "backend")
|
||||||
throw "no backend/tests/test_*.py files found"
|
try {
|
||||||
|
if ($runBackend) {
|
||||||
|
& $py -m pytest tests -m "not slow" -ra --tb=short
|
||||||
|
} else {
|
||||||
|
& $py -m pytest tests -m "unit and not slow" -ra --tb=short
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
foreach ($test in $backendTests) {
|
Pop-Location
|
||||||
Invoke-Step -Name "backend $($test.Name)" -Action {
|
|
||||||
& $py $test.FullName
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -137,5 +151,5 @@ Write-Host ("{0} von {1} Schritten erfolgreich." -f $passed, $ran)
|
||||||
if ($failed) {
|
if ($failed) {
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
Write-Host "MVP local acceptance: OK"
|
Write-Host "local tests: OK"
|
||||||
exit 0
|
exit 0
|
||||||
|
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
---
|
|
||||||
title: "Kanshō – Einmaliger Laptop-Datentransfer über Gitea"
|
|
||||||
status: "Transportartefakt, nach Restore entfernen"
|
|
||||||
date: "2026-09-07"
|
|
||||||
---
|
|
||||||
|
|
||||||
# Einmaliger Datentransfer (Laptop 2026-09-07)
|
|
||||||
|
|
||||||
Dieses Verzeichnis ist **kein** Produkt-Datenspeicher. Es existiert, weil auf dem Urlaubs-Laptop weder USB-Stick noch NAS verbunden werden darf. Der Transport läuft über das selbst gehostete Gitea (`gitea.stommer.de`), das nur in der eigenen Infrastruktur erreichbar ist.
|
|
||||||
|
|
||||||
## Inhalt
|
|
||||||
|
|
||||||
- `kansho-laptop-20260907.zip` – SQLite (Backup-API) + Journalmedien, Manifest, Checksummen
|
|
||||||
- **Nicht enthalten:** `backend/.env`, Provider-Keys, TLS-Zertifikate
|
|
||||||
|
|
||||||
Das Archiv enthält persönliche, **unverschlüsselte** Journalinhalte, Identity-Mappings und ggf. Debug-Spuren. Es ist Klasse A/B. Nicht an Provider, nicht öffentlich klonen, nicht in fremde Remotes spiegeln.
|
|
||||||
|
|
||||||
## Restore auf dem Zielrechner
|
|
||||||
|
|
||||||
Backend aus. Danach:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\scripts\backup-local.ps1 restore -Archive .\transfer\kansho-laptop-20260907.zip -Confirm -Replace
|
|
||||||
```
|
|
||||||
|
|
||||||
`.env` neu anlegen aus `backend/.env.example` und denselben Provider-Keys (Passwortmanager / OpenRouter). Das Admin-Login steckt im SQLite-Hash, nicht in der `.env`.
|
|
||||||
|
|
||||||
## Nach erfolgreichem Restore
|
|
||||||
|
|
||||||
Dieses Verzeichnis aus dem Arbeitsbaum entfernen und in einem eigenen Commit nach Gitea schieben. Die Git-Historie behält die Datei; das ist bewusst der Preis dieses Transportwegs.
|
|
||||||
Binary file not shown.
Loading…
Reference in New Issue
Block a user