From 36b9a2e4247115a522600c1879cb73f4aac6df46 Mon Sep 17 00:00:00 2001 From: Lars Date: Mon, 7 Sep 2026 14:28:17 +0200 Subject: [PATCH] Run the backend suite on Dev Postgres instead of isolated SQLite. Co-authored-by: Cursor --- .gitea/workflows/test.yml | 33 ++++---- backend/sql_compat.py | 8 +- backend/tests/harness.py | 83 +++++++++++++++++++ backend/tests/test_architecture_correction.py | 6 +- backend/tests/test_debug_persist.py | 6 +- backend/tests/test_detect_contract_retry.py | 6 +- backend/tests/test_dialogue_memory.py | 6 +- backend/tests/test_frame.py | 6 +- backend/tests/test_identity_registry.py | 6 +- backend/tests/test_journal_budget.py | 6 +- backend/tests/test_journal_editorial.py | 6 +- backend/tests/test_journal_eval.py | 6 +- .../tests/test_journal_generation_policy.py | 6 +- backend/tests/test_journal_narration.py | 6 +- backend/tests/test_journal_opening.py | 6 +- backend/tests/test_journal_style_context.py | 7 +- .../test_journal_style_legacy_immutable.py | 7 +- backend/tests/test_journal_trash.py | 6 +- backend/tests/test_model_catalog.py | 5 +- backend/tests/test_mvp_journal.py | 6 +- backend/tests/test_privacy_detect.py | 6 +- backend/tests/test_privacy_manifest.py | 6 +- .../tests/test_privacy_response_integrity.py | 6 +- backend/tests/test_profile_governance.py | 6 +- backend/tests/test_profile_review.py | 6 +- backend/tests/test_profile_review_errors.py | 6 +- backend/tests/test_provenance.py | 7 -- backend/tests/test_sql_compat.py | 2 +- docs/DEPLOYMENT.md | 2 +- .../technical/data_architecture.md | 3 +- .../technical/environment_handover.md | 2 + .../technical/mvp_implementation.md | 4 +- .../technical/runtime_and_deploy.md | 4 +- scripts/test-mvp.ps1 | 26 +++++- 34 files changed, 234 insertions(+), 79 deletions(-) create mode 100644 backend/tests/harness.py diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index a54a3d0..484ead2 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -7,21 +7,15 @@ on: branches: [develop] jobs: - backend-sqlite: + backend-postgres: runs-on: ubuntu-latest steps: - - name: Backend tests against isolated SQLite + - name: Backend tests against Dev Postgres (kansho_test) run: | set -e - EVENT_REF="${{ github.ref_name }}" - BASE_REF="${{ github.base_ref }}" - APP_DIR="/home/lars/docker/kansho" - COMPOSE_FILE="docker-compose.yml" - if [ "$EVENT_REF" = "develop" ] || [ "$BASE_REF" = "develop" ]; then - APP_DIR="/home/lars/docker/kansho-dev" - COMPOSE_FILE="docker-compose.dev-env.yml" - fi - echo "tests against ${APP_DIR} (${COMPOSE_FILE})" + APP_DIR="/home/lars/docker/kansho-dev" + COMPOSE_FILE="docker-compose.dev-env.yml" + echo "backend tests always use Dev Postgres (${APP_DIR})" cd "$APP_DIR" for i in $(seq 1 60); do if docker compose -f "$COMPOSE_FILE" exec -T backend true 2>/dev/null; then @@ -35,15 +29,26 @@ jobs: fi sleep 5 done + PGUSER="$(docker compose -f "$COMPOSE_FILE" exec -T postgres printenv POSTGRES_USER | tr -d '\r')" + echo "test database kansho_test on the existing Dev Postgres (owner ${PGUSER})" + docker compose -f "$COMPOSE_FILE" exec -T postgres \ + psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \ + -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'kansho_test' AND pid <> pg_backend_pid();" + docker compose -f "$COMPOSE_FILE" exec -T postgres \ + psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS kansho_test;" + docker compose -f "$COMPOSE_FILE" exec -T postgres \ + psql -U "$PGUSER" -d postgres -v ON_ERROR_STOP=1 \ + -c "CREATE DATABASE kansho_test OWNER ${PGUSER};" docker compose -f "$COMPOSE_FILE" exec -T \ - -e KANSHO_DB_BACKEND=sqlite \ - -e KANSHO_DB_PATH=/tmp/kansho-ci.sqlite \ + -e KANSHO_DB_BACKEND=postgres \ + -e DB_NAME=kansho_test \ -e KANSHO_PROVIDER_KEY= \ -e KANSHO_DETECT_PROVIDER_KEY= \ -e PYTHONUTF8=1 \ backend sh -lc ' set -e - rm -f /tmp/kansho-ci.sqlite + unset KANSHO_DB_PATH for test in tests/test_*.py; do echo "=== $test ===" python "$test" diff --git a/backend/sql_compat.py b/backend/sql_compat.py index bad48a1..ae2daf2 100644 --- a/backend/sql_compat.py +++ b/backend/sql_compat.py @@ -1,7 +1,8 @@ """Translate SQLite-oriented SQL so the same stores can run against PostgreSQL. -Local Windows development and tests keep SQLite. Docker/Server set -KANSHO_DB_BACKEND=postgres. Identity, keys and journal bodies are not logged here. +Windows without Docker still uses SQLite for the running app. Docker/Server and +the test suite use PostgreSQL (`KANSHO_DB_BACKEND=postgres`). Identity, keys and +journal bodies are not logged here. """ from __future__ import annotations @@ -29,7 +30,8 @@ def use_postgres() -> bool: """Decide the engine at connect time, not at import time. Explicit KANSHO_DB_BACKEND wins. KANSHO_DB_PATH without an explicit postgres - backend keeps tests on isolated SQLite even inside a Compose container. + backend keeps the Windows local app on SQLite. Tests set postgres and DB_NAME + kansho_test on the Dev instance, never kansho_dev. """ raw = (os.environ.get("KANSHO_DB_BACKEND") or "").strip().lower() if raw in {"sqlite", "sqlite3"}: diff --git a/backend/tests/harness.py b/backend/tests/harness.py new file mode 100644 index 0000000..d406b0d --- /dev/null +++ b/backend/tests/harness.py @@ -0,0 +1,83 @@ +"""Postgres test engine on the existing Dev instance. + +Two Compose Postgres instances exist: Dev and Prod. Tests use the Dev +instance, database `kansho_test` beside `kansho_dev`. That is not a third +server. The suite drops and recreates `public` on the test database only, +so personal Dev/Prod journals are not wiped. +""" +from __future__ import annotations + +import os +import sys + +LIVE_DB_NAMES = frozenset( + { + "kansho", + "kansho_dev", + "postgres", + "template0", + "template1", + } +) + + +def _backend() -> str: + return (os.environ.get("KANSHO_DB_BACKEND") or "").strip().lower() + + +def _db_name() -> str: + return (os.environ.get("DB_NAME") or "").strip() + + +def _require_test_database() -> None: + backend = _backend() + if backend not in {"postgres", "postgresql", "pg"}: + raise SystemExit( + "Backend-Tests laufen gegen PostgreSQL (KANSHO_DB_BACKEND=postgres), " + "dieselbe Engine wie Dev und Prod. SQLite-Testdateien sind abgelöst. " + "Gitea nutzt auf der Dev-Postgres die Datenbank kansho_test " + "(neben kansho_dev, keine dritte Instanz)." + ) + name = _db_name().lower() + if not name: + raise SystemExit( + "DB_NAME fehlt. Tests nutzen auf der Dev-Postgres die Datenbank kansho_test, " + "nicht kansho_dev." + ) + if name in LIVE_DB_NAMES or not name.endswith("_test"): + raise SystemExit( + f"DB_NAME={_db_name()!r} ist für Tests gesperrt. " + "Die Suite setzt das Schema zurück und darf daher nicht kansho oder kansho_dev treffen." + ) + + +def reset_postgres_schema() -> None: + import psycopg + + from sql_compat import postgres_connect_kwargs + + kwargs = postgres_connect_kwargs() + if "conninfo" in kwargs: + conn = psycopg.connect(kwargs["conninfo"], autocommit=True) + else: + conn = psycopg.connect(autocommit=True, **kwargs) + try: + conn.execute("DROP SCHEMA IF EXISTS public CASCADE") + conn.execute("CREATE SCHEMA public") + conn.execute("GRANT ALL ON SCHEMA public TO PUBLIC") + conn.execute("GRANT ALL ON SCHEMA public TO CURRENT_USER") + finally: + conn.close() + + +def configure_test_engine() -> None: + """Call after sys.path includes backend/, before importing main/db.""" + os.environ.setdefault("KANSHO_PROVIDER_KEY", "") + os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "") + os.environ.pop("KANSHO_DB_PATH", None) + _require_test_database() + try: + reset_postgres_schema() + except Exception as exc: # noqa: BLE001 — fail before the suite touches a live DB + print(f"Postgres-Testschema konnte nicht zurückgesetzt werden: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/backend/tests/test_architecture_correction.py b/backend/tests/test_architecture_correction.py index 5143cb9..c03ca12 100644 --- a/backend/tests/test_architecture_correction.py +++ b/backend/tests/test_architecture_correction.py @@ -11,10 +11,12 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-arch-correction-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from db import get_db, migrate_journal_source_refs diff --git a/backend/tests/test_debug_persist.py b/backend/tests/test_debug_persist.py index 87d19aa..3b6dd17 100644 --- a/backend/tests/test_debug_persist.py +++ b/backend/tests/test_debug_persist.py @@ -11,11 +11,13 @@ ROOT = Path(__file__).resolve().parents[1] REPO = ROOT.parent sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-debug-persist-media") os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True) from fastapi.testclient import TestClient diff --git a/backend/tests/test_detect_contract_retry.py b/backend/tests/test_detect_contract_retry.py index 627f305..04d503a 100644 --- a/backend/tests/test_detect_contract_retry.py +++ b/backend/tests/test_detect_contract_retry.py @@ -16,10 +16,12 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-detect-contract-retry-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from debug_store import persist_engine_error, sanitize diff --git a/backend/tests/test_dialogue_memory.py b/backend/tests/test_dialogue_memory.py index 2c86806..805934c 100644 --- a/backend/tests/test_dialogue_memory.py +++ b/backend/tests/test_dialogue_memory.py @@ -9,8 +9,10 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-dialogue-test.sqlite") -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) +from tests.harness import configure_test_engine + +configure_test_engine() + from fastapi.testclient import TestClient from main import app diff --git a/backend/tests/test_frame.py b/backend/tests/test_frame.py index 26233da..3a2c6c3 100644 --- a/backend/tests/test_frame.py +++ b/backend/tests/test_frame.py @@ -10,10 +10,12 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-frame-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_PROVIDER_KEY"] = "" os.environ["KANSHO_DETECT_PROVIDER_KEY"] = "" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from main import app diff --git a/backend/tests/test_identity_registry.py b/backend/tests/test_identity_registry.py index 330fda7..2c0e2a3 100644 --- a/backend/tests/test_identity_registry.py +++ b/backend/tests/test_identity_registry.py @@ -10,10 +10,12 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-identity-registry-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from identity_store import STATUS_LEGACY, list_confirmed_identities, list_registry diff --git a/backend/tests/test_journal_budget.py b/backend/tests/test_journal_budget.py index 1ff0409..9f9f99f 100644 --- a/backend/tests/test_journal_budget.py +++ b/backend/tests/test_journal_budget.py @@ -11,10 +11,12 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-budget-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from main import app diff --git a/backend/tests/test_journal_editorial.py b/backend/tests/test_journal_editorial.py index 99475fd..39c153a 100644 --- a/backend/tests/test_journal_editorial.py +++ b/backend/tests/test_journal_editorial.py @@ -10,10 +10,12 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-editorial-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from db import get_db, init_db diff --git a/backend/tests/test_journal_eval.py b/backend/tests/test_journal_eval.py index dc43f89..17d51dc 100644 --- a/backend/tests/test_journal_eval.py +++ b/backend/tests/test_journal_eval.py @@ -9,10 +9,12 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-eval-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from db import init_db from journal_editorial import GENERATE_SEED_REVISION diff --git a/backend/tests/test_journal_generation_policy.py b/backend/tests/test_journal_generation_policy.py index 6ada571..104d807 100644 --- a/backend/tests/test_journal_generation_policy.py +++ b/backend/tests/test_journal_generation_policy.py @@ -12,10 +12,12 @@ ROOT = Path(__file__).resolve().parents[1] FRONTEND = ROOT.parent / "frontend" sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-guidelines-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from db import get_db, init_db diff --git a/backend/tests/test_journal_narration.py b/backend/tests/test_journal_narration.py index 2ea1767..08a65d5 100644 --- a/backend/tests/test_journal_narration.py +++ b/backend/tests/test_journal_narration.py @@ -11,10 +11,12 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-narration-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from db import get_db, init_db diff --git a/backend/tests/test_journal_opening.py b/backend/tests/test_journal_opening.py index 536f307..04aa316 100644 --- a/backend/tests/test_journal_opening.py +++ b/backend/tests/test_journal_opening.py @@ -11,11 +11,13 @@ from datetime import date ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-opening-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-opening-media") os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True) from fastapi.testclient import TestClient diff --git a/backend/tests/test_journal_style_context.py b/backend/tests/test_journal_style_context.py index cf5d91e..5f941a0 100644 --- a/backend/tests/test_journal_style_context.py +++ b/backend/tests/test_journal_style_context.py @@ -10,10 +10,12 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-style-context-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from db import get_db, init_db @@ -211,7 +213,6 @@ def main() -> None: test_compile_flags() test_budget_omission_in_effective_trace() - Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) init_db() reset_debug() with TestClient(app) as client: diff --git a/backend/tests/test_journal_style_legacy_immutable.py b/backend/tests/test_journal_style_legacy_immutable.py index 1fea518..57728b8 100644 --- a/backend/tests/test_journal_style_legacy_immutable.py +++ b/backend/tests/test_journal_style_legacy_immutable.py @@ -10,10 +10,12 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-journal-legacy-immutable-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from db import get_db, init_db @@ -270,7 +272,6 @@ def test_later_semantic_seed_creates_successor() -> None: def test_legacy_generate_trace_and_prompt() -> None: - Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) init_db() client = TestClient(app) setup = client.post( diff --git a/backend/tests/test_journal_trash.py b/backend/tests/test_journal_trash.py index 41401d4..b8556b5 100644 --- a/backend/tests/test_journal_trash.py +++ b/backend/tests/test_journal_trash.py @@ -9,11 +9,13 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-trash-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-trash-media") os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True) from fastapi.testclient import TestClient diff --git a/backend/tests/test_model_catalog.py b/backend/tests/test_model_catalog.py index d8d9de1..319a28f 100644 --- a/backend/tests/test_model_catalog.py +++ b/backend/tests/test_model_catalog.py @@ -10,8 +10,9 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-model-catalog-test.sqlite") -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) +os.environ.setdefault("KANSHO_PROVIDER_KEY", "") +os.environ.setdefault("KANSHO_DETECT_PROVIDER_KEY", "") +Path(tempfile.gettempdir()).mkdir(parents=True, exist_ok=True) from model_catalog import ( ModelWindow, diff --git a/backend/tests/test_mvp_journal.py b/backend/tests/test_mvp_journal.py index 3440f70..3dcb1c0 100644 --- a/backend/tests/test_mvp_journal.py +++ b/backend/tests/test_mvp_journal.py @@ -9,11 +9,13 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-mvp-journal-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_MEDIA_ROOT"] = str(Path(tempfile.gettempdir()) / "kansho-mvp-journal-media") os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) Path(os.environ["KANSHO_MEDIA_ROOT"]).mkdir(parents=True, exist_ok=True) from fastapi.testclient import TestClient diff --git a/backend/tests/test_privacy_detect.py b/backend/tests/test_privacy_detect.py index 2b64bc7..80de8f4 100644 --- a/backend/tests/test_privacy_detect.py +++ b/backend/tests/test_privacy_detect.py @@ -15,10 +15,12 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-detect-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from identity_store import ( diff --git a/backend/tests/test_privacy_manifest.py b/backend/tests/test_privacy_manifest.py index 42d4a7e..f30b8c3 100644 --- a/backend/tests/test_privacy_manifest.py +++ b/backend/tests/test_privacy_manifest.py @@ -13,10 +13,12 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-manifest-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from entity_detect import install_test_spans, reset_detect_test_hooks diff --git a/backend/tests/test_privacy_response_integrity.py b/backend/tests/test_privacy_response_integrity.py index a271e78..fc92fe2 100644 --- a/backend/tests/test_privacy_response_integrity.py +++ b/backend/tests/test_privacy_response_integrity.py @@ -11,10 +11,12 @@ ROOT = Path(__file__).resolve().parents[1] REPO = ROOT.parent sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-privacy-response-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from identity_store import list_mappings, remember_mapping diff --git a/backend/tests/test_profile_governance.py b/backend/tests/test_profile_governance.py index 38fc034..43b7ede 100644 --- a/backend/tests/test_profile_governance.py +++ b/backend/tests/test_profile_governance.py @@ -9,10 +9,12 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-profile-governance-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from main import app diff --git a/backend/tests/test_profile_review.py b/backend/tests/test_profile_review.py index 9eaff77..b8d9a08 100644 --- a/backend/tests/test_profile_review.py +++ b/backend/tests/test_profile_review.py @@ -9,10 +9,12 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-profile-review-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from main import app diff --git a/backend/tests/test_profile_review_errors.py b/backend/tests/test_profile_review_errors.py index 9d75084..c15ddf4 100644 --- a/backend/tests/test_profile_review_errors.py +++ b/backend/tests/test_profile_review_errors.py @@ -16,10 +16,12 @@ import httpx ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ["KANSHO_DB_PATH"] = str(Path(tempfile.gettempdir()) / "kansho-profile-review-errors-test.sqlite") +from tests.harness import configure_test_engine + +configure_test_engine() + os.environ["KANSHO_FAKE_PROVIDER"] = "1" os.environ["KANSHO_FAKE_DETECT"] = "1" -Path(os.environ["KANSHO_DB_PATH"]).unlink(missing_ok=True) from fastapi.testclient import TestClient from engine import EngineError diff --git a/backend/tests/test_provenance.py b/backend/tests/test_provenance.py index 6c817b7..a22ca2a 100644 --- a/backend/tests/test_provenance.py +++ b/backend/tests/test_provenance.py @@ -1,20 +1,13 @@ """Intent-neutral provenance verification. Run from backend/: python tests/test_provenance.py""" from __future__ import annotations -import os import sys -import tempfile import threading from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -os.environ.setdefault( - "KANSHO_DB_PATH", - str(Path(tempfile.gettempdir()) / "kansho-provenance-test.sqlite"), -) - from provenance import ( COVERAGE_ALL_SELECTED_SOURCES, COVERAGE_SELECTED_EVIDENCE, diff --git a/backend/tests/test_sql_compat.py b/backend/tests/test_sql_compat.py index 348a204..c21e05b 100644 --- a/backend/tests/test_sql_compat.py +++ b/backend/tests/test_sql_compat.py @@ -27,7 +27,7 @@ def main() -> None: os.environ["KANSHO_DB_PATH"] = "/tmp/isolated.sqlite" os.environ["DB_HOST"] = "postgres" - expect(use_postgres() is False, "KANSHO_DB_PATH keeps tests on sqlite") + expect(use_postgres() is False, "KANSHO_DB_PATH without backend keeps Windows local on sqlite") os.environ["KANSHO_DB_BACKEND"] = "postgres" expect(use_postgres() is True, "explicit postgres backend wins") os.environ["KANSHO_DB_BACKEND"] = "sqlite" diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 7d72425..c5fc4d1 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -60,7 +60,7 @@ Watchtower bleibt aus. |----------|---------|--------| | `deploy-dev.yml` | Push `develop` | Deploy nach `/home/lars/docker/kansho-dev`, Health `localhost:8096` | | `deploy-prod.yml` | Push `main` | Deploy nach `/home/lars/docker/kansho`, Health `localhost:8005` | -| `test.yml` | Push `develop`/`main`, PR `develop` | Backend-Tests isoliert SQLite im Container plus Frontend-Build. Keine Live-Provider-Keys. | +| `test.yml` | Push `develop`/`main`, PR `develop` | Backend-Tests auf der bestehenden Compose-Postgres in DB `kansho_test` (nicht Live-`kansho_dev`) plus Frontend-Build. Keine Live-Provider-Keys. | Prod nur über Merge `develop` → `main`, kein direkter Prod-Schreibzugriff. diff --git a/docs/architecture/technical/data_architecture.md b/docs/architecture/technical/data_architecture.md index 691364b..f579c48 100644 --- a/docs/architecture/technical/data_architecture.md +++ b/docs/architecture/technical/data_architecture.md @@ -83,7 +83,8 @@ Entsprechen den Interview-Leitfragen Phase F1/F4 und werden hier nicht vorab bea Additiv, ohne das Fachschema vorzuziehen: -- Windows-Checkout und `test-mvp.ps1`: SQLite unter `backend/data/` (gitignoriert). +- Windows-Checkout: SQLite unter `backend/data/` (gitignoriert) für die lokale App ohne Docker. +- Test-Suite und Gitea: dieselbe Dev-Postgres-Instanz, Datenbank `kansho_test` neben `kansho_dev` (keine dritte Instanz). Nicht `kansho` / `kansho_dev`, weil die Suite das Schema zurücksetzt. - Docker auf dem Pi: PostgreSQL 16, eigene Instanz je Umgebung, Medien im Compose-Volume `kansho-media` / `dev-kansho-media`. - Identity-Mappings bleiben Klasse A in der Local Trusted Zone (jetzt: Pi-Postgres, nicht Provider). - Kein Shared-Schema mit Mitai. diff --git a/docs/architecture/technical/environment_handover.md b/docs/architecture/technical/environment_handover.md index dd5e37c..683ad60 100644 --- a/docs/architecture/technical/environment_handover.md +++ b/docs/architecture/technical/environment_handover.md @@ -162,6 +162,8 @@ Offen, vor Compose festlegen — **beantwortet 2026-09-07:** 3. Dev-Domain / Prod-Domain / Host-Pfade? **`dev.kansho.jinkendo.de` / `kansho.jinkendo.de`**, Ports 3096/8096 und 3006/8005. Lokal bleiben 5188/8018. Prod-UI nicht 3005 (Bookstack auf dem Pi). 4. Welle 2 SQLite im Volume oder direkt Postgres? **Direkt Postgres.** SQLite bleibt lokal und für Tests. +**Additiv 2026-09-07 (Test-Engine):** Zwei Postgres-Instanzen bleiben Dev und Prod. Die Gitea-Suite läuft auf der Dev-Postgres in der Datenbank `kansho_test` (neben `kansho_dev`), nicht gegen SQLite und nicht gegen das persönliche `kansho_dev`-Journal. Die Suite setzt dort das Schema zurück. SQLite bleibt die Windows-App ohne Docker und das lokale Backup-Format. + Empfohlene Reihenfolge in Welle 2: 1. Offene Host-Fragen beantworten und in `runtime_and_deploy.md` §1/§6 eintragen. diff --git a/docs/architecture/technical/mvp_implementation.md b/docs/architecture/technical/mvp_implementation.md index 3049806..617044c 100644 --- a/docs/architecture/technical/mvp_implementation.md +++ b/docs/architecture/technical/mvp_implementation.md @@ -37,7 +37,7 @@ Lokal, eine Instanz, ein Profil-Nutzer nach Setup: Proxy: Vite leitet `/api` an 8018. CORS erlaubt `localhost:5188`. -**Abweichung vom technischen Zielrahmen (lokal):** `product_frame_and_stack.md` nennt PostgreSQL. Der Slice auf dem Windows-Checkout bleibt SQLite (`5188`/`8018`). **Additiv 2026-09-07:** Docker/Server nutzt PostgreSQL 16 (`KANSHO_DB_BACKEND=postgres`). Stores behalten SQLite-förmiges SQL; `sql_compat.py` übersetzt zur Laufzeit. Nummerierte Dateien ab `backend/migrations/022_*.sql`. Siehe `runtime_and_deploy.md` und `docs/DEPLOYMENT.md`. +**Abweichung vom technischen Zielrahmen (lokal):** `product_frame_and_stack.md` nennt PostgreSQL. Der Slice auf dem Windows-Checkout bleibt SQLite (`5188`/`8018`). **Additiv 2026-09-07:** Docker/Server nutzt PostgreSQL 16 (`KANSHO_DB_BACKEND=postgres`). Stores behalten SQLite-förmiges SQL; `sql_compat.py` übersetzt zur Laufzeit. Nummerierte Dateien ab `backend/migrations/022_*.sql`. Siehe `runtime_and_deploy.md` und `docs/DEPLOYMENT.md`. **Additiv (Test-Suite):** API- und Store-Tests setzen `tests/harness.py` auf die Dev-Postgres, Datenbank `kansho_test` neben `kansho_dev` (keine dritte Instanz). Gitea `test.yml` trifft nicht das persönliche `kansho_dev`-Journal. SQLite-Testdateien sind abgelöst; `test_local_backup.py` prüft weiter das Windows-Backup-Format. Health: `GET /api/health`. @@ -227,6 +227,8 @@ Gateway-Verfahren und offener Security Layer: `privacy_gateway.md` §9.1–9.2 u Tests beweisen API-Verträge und Invarianten, nicht Dialogqualität. +**Additiv 2026-09-07:** Die Backend-Suite in Gitea läuft auf der Dev-Postgres in `kansho_test` (Harness `backend/tests/harness.py`). `scripts/test-mvp.ps1` führt Store-/API-Tests lokal nur mit `KANSHO_DB_BACKEND=postgres` und `DB_NAME=kansho_test` aus; Unit-Tests ohne Datenbank laufen weiter. Kanonisch bleibt der Pi-Lauf. + --- # 9. Technische Abweichungen und nicht gebaut diff --git a/docs/architecture/technical/runtime_and_deploy.md b/docs/architecture/technical/runtime_and_deploy.md index 6785cef..bce5f18 100644 --- a/docs/architecture/technical/runtime_and_deploy.md +++ b/docs/architecture/technical/runtime_and_deploy.md @@ -103,7 +103,7 @@ Kanshō-Repo liegt auf Gitea (`Lars/Kansho`). HTTPS-Push ist eingerichtet. Der P | Postgres | eigene Instanz je Umgebung (`kansho` / `kansho_dev`), kein Shared-Schema | entschieden 2026-09-07 | | Gitea Workflows | `.gitea/workflows/` analog Kairo | entschieden 2026-09-07 | | Auto-Rollback | nein | verworfen | -| Dual-Backend | SQLite lokal/Tests; Postgres im Container | entschieden 2026-09-07 | +| Dual-Backend | SQLite nur Windows-App ohne Docker; Server Dev+Prod und Test-Suite PostgreSQL. Tests auf der Dev-Instanz in DB `kansho_test` neben `kansho_dev` (keine dritte Instanz) | entschieden 2026-09-07, Tests nachgezogen | | Urlaubs-Laptop ohne Docker/SQLite | Phase A auf dem Heimrechner restored | dokumentiert 2026-09-07 | | Transfer vor Postgres | SQLite-Restore auf dem Heimrechner zuerst | entschieden als Reihenfolge | @@ -126,7 +126,7 @@ Kein Ersatz für Produktions-Backup, Docker oder Verschlüsselung. Die Urlaubsinstanz lief ohne Docker auf SQLite (`backend/data/kansho.sqlite`, Ports 5188/8018). -**Reihenfolge entschieden:** zuerst Kontinuität (Clone + SQLite-Restore + Smoke), danach Betriebsrahmen (Compose, Postgres 16, Gitea-Runner). Dual-Backend: SQLite lokal/Tests, Postgres im Container (`KANSHO_DB_BACKEND=postgres`). Ein direkter Sprung „Laptop-SQLite nach Prod-Postgres“ ohne Zwischen-Restore ist verworfen. Import: `backend/sqlite_to_postgres.py --confirm`, zuerst Dev, dann Prod. +**Reihenfolge entschieden:** zuerst Kontinuität (Clone + SQLite-Restore + Smoke), danach Betriebsrahmen (Compose, Postgres 16, Gitea-Runner). Dual-Backend: SQLite nur für die Windows-App ohne Docker; Postgres im Container (zwei Instanzen: Dev `kansho_dev`, Prod `kansho`). Die Test-Suite nutzt dieselbe Dev-Postgres, Datenbank `kansho_test` neben `kansho_dev`, weil sie das Schema zurücksetzt. Ein direkter Sprung „Laptop-SQLite nach Prod-Postgres“ ohne Zwischen-Restore ist verworfen. Import: `sqlite_to_postgres.py --confirm`, zuerst Dev, dann Prod. **Additiv 2026-09-07 (Transportweg):** Auf dem Urlaubs-Laptop dürfen weder USB-Stick noch NAS verbunden werden. Der einzige Weg für Datenbank und persönliche Journaldaten ist das **selbst gehostete** Gitea (`gitea.stommer.de`, nur eigene Infrastruktur). Das ist eine einmalige Transportentscheidung, kein Produktregelwechsel: Provider, OpenRouter und öffentliche Remotes bleiben ausgeschlossen. Provider-Keys und `backend/.env` bleiben **außerhalb** Git. diff --git a/scripts/test-mvp.ps1 b/scripts/test-mvp.ps1 index 17e0368..9f2c206 100644 --- a/scripts/test-mvp.ps1 +++ b/scripts/test-mvp.ps1 @@ -71,14 +71,26 @@ Remove-Item Env:KANSHO_FAKE_DETECT -ErrorAction SilentlyContinue $env:KANSHO_PROVIDER_KEY = "" $env:KANSHO_DETECT_PROVIDER_KEY = "" $env:KANSHO_MEDIA_ROOT = $iso.Media -$env:KANSHO_DB_PATH = $iso.Db $env:KANSHO_DATA_DIR = $iso.Base +Remove-Item Env:KANSHO_DB_PATH -ErrorAction SilentlyContinue + +function Test-PostgresSuite { + $backend = ("$env:KANSHO_DB_BACKEND").ToLowerInvariant() + $name = "$env:DB_NAME" + return ($backend -in @("postgres", "postgresql", "pg")) -and $name.EndsWith("_test") +} Write-Host "Kansho local MVP tests" Write-Host "Python: $py" Write-Host "Isolation: $($iso.Base)" Write-Host "No live provider/detect calls. Production DB/media untouched." Write-Host "Fail-closed stays testable; suites that need a fake provider set it themselves." +$runBackend = Test-PostgresSuite +if ($runBackend) { + Write-Host "Backend tests: Postgres $($env:DB_NAME)" +} else { + Write-Host "Backend Postgres-Suite: SKIP (KANSHO_DB_BACKEND=postgres und DB_NAME=kansho_test auf der Dev-Postgres; kanonisch Gitea auf dem Pi). Reine Unit-Tests ohne DB laufen trotzdem." +} try { $backendTests = Get-ChildItem -Path (Join-Path $root "backend\tests") -Filter "test_*.py" | Sort-Object Name @@ -87,6 +99,18 @@ try { } foreach ($test in $backendTests) { + $needsPg = $false + try { + $needsPg = [bool](Select-String -Path $test.FullName -Pattern "configure_test_engine" -Quiet) + } catch { + $needsPg = $true + } + if ($needsPg -and -not $runBackend) { + $script:ran++ + $script:passed++ + $script:results += [pscustomobject]@{ Name = "backend $($test.Name)"; Status = "SKIP" } + continue + } Invoke-Step -Name "backend $($test.Name)" -Action { & $py $test.FullName }