diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index c0a2997..a27a40c 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -62,7 +62,8 @@ jobs: docker compose -f "$COMPOSE_FILE" exec -T backend sh -lc " pip install -q -r requirements-dev.txt && - python -m pytest tests -ra -vv --tb=short + python -m pytest tests -ra -vv --tb=short && + python run_seeds.py --only seed_001_cleanup_pytest_artifacts --force " echo "✓ pytest OK" diff --git a/README.md b/README.md index 1fe1550..109dc59 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,10 @@ Interaktive API-Doku (Dev): http://localhost:8097/api/docs ### Auth (AP0.2) -Bootstrap (nur wenn noch kein User existiert — `.env` setzen): +**Ersteinrichtung** (nur wenn noch kein User existiert): + +- UI-Registrierung unter http://localhost:3097 (Tab „Registrieren“), oder +- Bootstrap per `.env` (wird als Data-Seed beim Start ausgeführt): ```env KAIRO_BOOTSTRAP_ADMIN_EMAIL=admin@kairo.local @@ -111,6 +114,8 @@ KAIRO_BOOTSTRAP_ADMIN_PASSWORD=… KAIRO_BOOTSTRAP_TENANT_SLUG=default ``` +Migrationen & idempotente Data-Seeds: [docs/MIGRATIONS.md](docs/MIGRATIONS.md) + | Endpoint | Methode | Auth | Beschreibung | |----------|---------|------|--------------| | `/api/auth/setup-status` | GET | — | Ob Erstregistrierung offen ist | diff --git a/backend/main.py b/backend/main.py index 81e8403..3bac90e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -21,9 +21,15 @@ else: print(f"[FAIL] Migrationen fehlgeschlagen (Exit {exit_code})") sys.exit(exit_code) - import bootstrap +if os.getenv("SKIP_SEEDS", "").strip().lower() not in ("1", "true", "yes"): + import run_seeds - bootstrap.bootstrap_admin_if_needed() + exit_code = run_seeds.main() + if exit_code != 0: + print(f"[FAIL] Seeds fehlgeschlagen (Exit {exit_code})") + sys.exit(exit_code) +else: + print("[SKIP_SEEDS] Data-Seeds übersprungen") allowed_origins = [ origin.strip() diff --git a/backend/migrations/003_data_seeds_tracking.sql b/backend/migrations/003_data_seeds_tracking.sql new file mode 100644 index 0000000..51a041f --- /dev/null +++ b/backend/migrations/003_data_seeds_tracking.sql @@ -0,0 +1,8 @@ +-- Tracking für idempotente Data-Seeds (Checksum-basiert, bei Änderung erneut ausführbar) + +CREATE TABLE IF NOT EXISTS data_seeds ( + seed_name VARCHAR(255) PRIMARY KEY, + checksum VARCHAR(64) NOT NULL, + executed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); diff --git a/backend/run_seeds.py b/backend/run_seeds.py new file mode 100644 index 0000000..767c77a --- /dev/null +++ b/backend/run_seeds.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Apply idempotent data seeds with checksum-based tracking and re-run on change.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import os +import re +import sys +import time +from typing import Callable, List, Optional, Tuple + +import psycopg2 +import sqlparse + +from db import db_params, get_connection + +_SEED_PREFIX = re.compile(r"^seed_(\d+)_(.+)$") +_DEV_MARKER = ".dev." + + +def is_production() -> bool: + return os.getenv("ENVIRONMENT", "development").strip().lower() == "production" + + +def seeds_directory() -> str: + docker_path = "/app/seeds" + if os.path.isdir(docker_path): + return docker_path + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "seeds") + + +def init_data_seeds_table(conn) -> None: + with conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE IF NOT EXISTS data_seeds ( + seed_name VARCHAR(255) PRIMARY KEY, + checksum VARCHAR(64) NOT NULL, + executed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + conn.commit() + + +def _seed_sort_key(filename: str) -> Tuple[int, str]: + stem = filename + for suffix in (".sql", ".py"): + if stem.endswith(suffix): + stem = stem[: -len(suffix)] + break + match = _SEED_PREFIX.match(stem) + if match: + return (int(match.group(1)), stem) + return (0, stem) + + +def _seed_stem(filename: str) -> str: + if filename.endswith(".dev.sql"): + return filename[: -len(".dev.sql")] + if filename.endswith(".sql"): + return filename[: -len(".sql")] + if filename.endswith(".py"): + return filename[: -len(".py")] + return filename + + +def seed_files(seeds_dir: str) -> List[Tuple[str, str, str]]: + """Return (seed_name, filepath, kind) sorted by numeric prefix.""" + rows: List[Tuple[str, str, str]] = [] + if not os.path.isdir(seeds_dir): + return rows + + for filename in os.listdir(seeds_dir): + if filename.startswith("_") or not filename.startswith("seed_"): + continue + if filename.endswith(".sql"): + kind = "sql" + elif filename.endswith(".py"): + kind = "py" + else: + continue + stem = _seed_stem(filename) + rows.append((stem, os.path.join(seeds_dir, filename), kind)) + + rows.sort(key=lambda item: _seed_sort_key(item[0])) + return rows + + +def seed_applies_in_environment(filename: str) -> bool: + if _DEV_MARKER in filename and is_production(): + return False + return True + + +def file_checksum(filepath: str) -> str: + digest = hashlib.sha256() + with open(filepath, "rb") as handle: + for chunk in iter(lambda: handle.read(65536), b""): + digest.update(chunk) + return digest.hexdigest() + + +def applied_seed_checksum(conn, seed_name: str) -> Optional[str]: + with conn.cursor() as cur: + cur.execute("SELECT checksum FROM data_seeds WHERE seed_name = %s", (seed_name,)) + row = cur.fetchone() + return row[0] if row else None + + +def record_seed(conn, seed_name: str, checksum: str) -> None: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO data_seeds (seed_name, checksum, executed_at, last_run_at) + VALUES (%s, %s, NOW(), NOW()) + ON CONFLICT (seed_name) DO UPDATE SET + checksum = EXCLUDED.checksum, + last_run_at = NOW() + """, + (seed_name, checksum), + ) + + +def _split_statements(sql_text: str) -> List[str]: + parts = sqlparse.split(sql_text.strip()) + return [part.strip() for part in parts if part and part.strip()] + + +def run_sql_seed(conn, filepath: str) -> None: + with open(filepath, "r", encoding="utf-8") as handle: + body = handle.read() + statements = _split_statements(body) + with conn.cursor() as cur: + for stmt in statements: + cur.execute(stmt) + + +def run_python_seed(filepath: str) -> None: + module_name = f"kairo_seed_{hashlib.md5(filepath.encode()).hexdigest()[:12]}" + spec = importlib.util.spec_from_file_location(module_name, filepath) + if spec is None or spec.loader is None: + raise RuntimeError(f"Seed-Modul nicht ladbar: {filepath}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + run_fn: Callable[[], None] | None = getattr(module, "run", None) + if run_fn is None: + raise RuntimeError(f"Seed {filepath} hat keine run()-Funktion") + run_fn() + + +def connect_with_retry(max_retries: int = 30): + p = db_params() + for attempt in range(max_retries): + try: + conn = get_connection() + conn.autocommit = False + print(f"[OK] Connected to database: {p['dbname']}") + return conn + except psycopg2.OperationalError: + if attempt >= max_retries - 1: + raise + print(f"Waiting for database... ({attempt + 1}/{max_retries})") + time.sleep(2) + + +def run_seed(conn, seed_name: str, filepath: str, kind: str) -> tuple[bool, object]: + print(f"Running seed: {seed_name}") + try: + if kind == "sql": + run_sql_seed(conn, filepath) + else: + conn.commit() + conn.close() + run_python_seed(filepath) + conn = connect_with_retry(max_retries=5) + + checksum = file_checksum(filepath) + record_seed(conn, seed_name, checksum) + conn.commit() + print(f" [OK] {seed_name}") + return True, conn + except Exception as exc: + try: + conn.rollback() + except Exception: + pass + print(f" [FAIL] {seed_name}: {exc}") + return False, conn + + +def pending_seeds( + conn, + seeds_dir: str, + *, + only: Optional[List[str]] = None, + force: bool = False, +) -> List[Tuple[str, str, str]]: + selected: List[Tuple[str, str, str]] = [] + for seed_name, filepath, kind in seed_files(seeds_dir): + filename = os.path.basename(filepath) + if not seed_applies_in_environment(filename): + print(f" [SKIP] {seed_name} (nur Nicht-Prod)") + continue + if only and seed_name not in only: + continue + checksum = file_checksum(filepath) + if force or applied_seed_checksum(conn, seed_name) != checksum: + selected.append((seed_name, filepath, kind)) + return selected + + +def main(argv: Optional[List[str]] = None) -> int: + argv = argv if argv is not None else sys.argv[1:] + only: Optional[List[str]] = None + force = False + + idx = 0 + while idx < len(argv): + arg = argv[idx] + if arg == "--only" and idx + 1 < len(argv): + only = [part.strip() for part in argv[idx + 1].split(",") if part.strip()] + idx += 2 + continue + if arg == "--force": + force = True + idx += 1 + continue + print(f"[FAIL] Unbekanntes Argument: {arg}") + return 1 + + print("=" * 60) + print("Jinkendo Kairo — Data Seeds") + print("=" * 60) + + seeds_dir = seeds_directory() + if not os.path.isdir(seeds_dir): + print(f"[OK] Kein seeds-Verzeichnis ({seeds_dir}) — nichts zu tun.") + return 0 + + try: + conn = connect_with_retry() + init_data_seeds_table(conn) + to_run = pending_seeds(conn, seeds_dir, only=only, force=force) + + if not to_run: + print("[OK] Alle Seeds aktuell — nichts auszuführen.") + conn.close() + return 0 + + print(f"{len(to_run)} Seed(s) ausstehend:") + for seed_name, _, _ in to_run: + print(f" - {seed_name}") + + for seed_name, filepath, kind in to_run: + ok, conn = run_seed(conn, seed_name, filepath, kind) + if not ok: + conn.close() + return 1 + + conn.close() + print("[OK] Seeds abgeschlossen.") + return 0 + except Exception as exc: + print(f"[FAIL] {exc}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/seeds/seed_001_cleanup_pytest_artifacts.dev.sql b/backend/seeds/seed_001_cleanup_pytest_artifacts.dev.sql new file mode 100644 index 0000000..25d422c --- /dev/null +++ b/backend/seeds/seed_001_cleanup_pytest_artifacts.dev.sql @@ -0,0 +1,32 @@ +-- Dev/CI: pytest-Artefakte entfernen (idempotent, sicher mehrfach ausführbar). +-- Nur in Nicht-Prod-Umgebungen (Dateiname *.dev.sql). + +-- Abhängigkeiten vor User-Löschung (human actors: user_id NOT NULL constraint) +DELETE FROM actors +WHERE user_id IN (SELECT id FROM users WHERE email ~* '@example\.com$'); + +DELETE FROM tenant_memberships +WHERE user_id IN (SELECT id FROM users WHERE email ~* '@example\.com$'); + +DELETE FROM sessions +WHERE user_id IN (SELECT id FROM users WHERE email ~* '@example\.com$'); + +DELETE FROM audit_log +WHERE user_id IN (SELECT id FROM users WHERE email ~* '@example\.com$'); + +DELETE FROM users WHERE email ~* '@example\.com$'; + +-- Verwaiste pytest-Tenants (Factory-Slug t-{10 hex}), ohne Memberships +DELETE FROM actors a +USING tenants t +WHERE a.tenant_id = t.id + AND t.slug ~ '^t-[a-f0-9]{10}$' + AND NOT EXISTS ( + SELECT 1 FROM tenant_memberships tm WHERE tm.tenant_id = t.id + ); + +DELETE FROM tenants t +WHERE t.slug ~ '^t-[a-f0-9]{10}$' + AND NOT EXISTS ( + SELECT 1 FROM tenant_memberships tm WHERE tm.tenant_id = t.id + ); diff --git a/backend/seeds/seed_002_bootstrap_admin.py b/backend/seeds/seed_002_bootstrap_admin.py new file mode 100644 index 0000000..9f62c09 --- /dev/null +++ b/backend/seeds/seed_002_bootstrap_admin.py @@ -0,0 +1,9 @@ +"""Bootstrap portal admin from KAIRO_BOOTSTRAP_* when no users exist.""" + +from __future__ import annotations + + +def run() -> None: + from bootstrap import bootstrap_admin_if_needed + + bootstrap_admin_if_needed() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index f62facc..e765d80 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -7,6 +7,7 @@ import os import pytest os.environ.setdefault("SKIP_DB_MIGRATE", "1") +os.environ.setdefault("SKIP_SEEDS", "1") os.environ.setdefault("SKIP_BOOTSTRAP", "1") @@ -37,6 +38,16 @@ def _run_migrations(): assert run_migrations.main() == 0 +@pytest.fixture(scope="session", autouse=True) +def _seed_hygiene(): + """Vor/nach Tests pytest-Artefakte aus der geteilten Dev-DB entfernen.""" + import run_seeds + + run_seeds.main(["--only", "seed_001_cleanup_pytest_artifacts"]) + yield + run_seeds.main(["--only", "seed_001_cleanup_pytest_artifacts"]) + + @pytest.fixture() def client(): import importlib diff --git a/backend/tests/test_migrations.py b/backend/tests/test_migrations.py index 7e9b5f0..9b011c0 100644 --- a/backend/tests/test_migrations.py +++ b/backend/tests/test_migrations.py @@ -33,6 +33,7 @@ def test_migration_runner_finds_migrations(): names = [name for name, _ in files] assert "001_init_core" in names assert "002_auth_identity_tenant_actor" in names + assert "003_data_seeds_tracking" in names def test_migration_runner_is_idempotent(): @@ -44,6 +45,7 @@ def test_migration_runner_is_idempotent(): conn.close() assert "001_init_core" in executed assert "002_auth_identity_tenant_actor" in executed + assert "003_data_seeds_tracking" in executed def test_core_table_exists(): diff --git a/backend/tests/test_seeds.py b/backend/tests/test_seeds.py new file mode 100644 index 0000000..cb46f93 --- /dev/null +++ b/backend/tests/test_seeds.py @@ -0,0 +1,37 @@ +"""Data seed runner tests (require PostgreSQL).""" + +from __future__ import annotations + +import uuid + +import run_seeds +from tests.factories import create_user + + +def test_seed_runner_finds_seeds(): + seeds_dir = run_seeds.seeds_directory() + names = [name for name, _, _ in run_seeds.seed_files(seeds_dir)] + assert "seed_001_cleanup_pytest_artifacts" in names + assert "seed_002_bootstrap_admin" in names + + +def test_cleanup_seed_removes_example_com_users(): + suffix = uuid.uuid4().hex[:8] + user = create_user(email=f"seed-test-{suffix}@example.com") + + assert run_seeds.main(["--only", "seed_001_cleanup_pytest_artifacts", "--force"]) == 0 + + from db import get_connection + + conn = get_connection() + try: + with conn.cursor() as cur: + cur.execute("SELECT COUNT(*) FROM users WHERE id = %s", (user["id"],)) + assert cur.fetchone()[0] == 0 + finally: + conn.close() + + +def test_cleanup_seed_is_idempotent(): + assert run_seeds.main(["--only", "seed_001_cleanup_pytest_artifacts"]) == 0 + assert run_seeds.main(["--only", "seed_001_cleanup_pytest_artifacts"]) == 0 diff --git a/backend/version.py b/backend/version.py index 0220179..721587c 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1,3 +1,3 @@ APP_VERSION = "0.2.0-ap0.2" -DB_SCHEMA_VERSION = "002" +DB_SCHEMA_VERSION = "003" APP_NAME = "jinkendo-kairo" diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md new file mode 100644 index 0000000..2bcbf72 --- /dev/null +++ b/docs/MIGRATIONS.md @@ -0,0 +1,51 @@ +# Migrationen & Data-Seeds + +Kairo trennt **Schema-Migrationen** (einmalig) von **Data-Seeds** (idempotent, bei Änderung erneut ausführbar). + +## Schema-Migrationen + +| | | +|---|---| +| **Pfad** | `backend/migrations/NNN_beschreibung.sql` | +| **Tracking** | Tabelle `schema_migrations` | +| **Ausführung** | Beim Backend-Start (`run_migrations.py`), überspringbar via `SKIP_DB_MIGRATE=1` | +| **Regel** | Jede Datei wird **genau einmal** angewendet. Änderungen → neue nummerierte Datei. | + +## Data-Seeds + +| | | +|---|---| +| **Pfad** | `backend/seeds/seed_NNN_beschreibung.sql` oder `.py` | +| **Tracking** | Tabelle `data_seeds` (Name + SHA256-Checksum) | +| **Ausführung** | Nach Schema-Migrationen (`run_seeds.py`), überspringbar via `SKIP_SEEDS=1` | +| **Regel** | Seed läuft erneut, wenn die Datei **geändert** wurde (Checksum abweicht) oder `--force` gesetzt ist. | + +### Umgebungsfilter + +Dateien mit **`.dev.`** im Namen (z. B. `seed_001_…​.dev.sql`) laufen **nicht in Production** (`ENVIRONMENT=production`). + +### Seeds manuell ausführen + +```bash +docker compose -f docker-compose.dev-env.yml exec backend python run_seeds.py +docker compose -f docker-compose.dev-env.yml exec backend python run_seeds.py --only seed_001_cleanup_pytest_artifacts --force +``` + +## Aktuelle Seeds + +| Seed | Typ | Zweck | +|------|-----|--------| +| `seed_001_cleanup_pytest_artifacts` | SQL (dev) | Entfernt pytest/CI-User (`*@example.com`) und verwaiste Test-Tenants | +| `seed_002_bootstrap_admin` | Python | Legt Systemadmin aus `KAIRO_BOOTSTRAP_*` an, wenn noch kein User existiert | + +## Neuen Seed anlegen + +1. Datei `backend/seeds/seed_NNN_kurzname.sql` oder `.py` anlegen (Python: `def run() -> None:`). +2. SQL idempotent halten (`DELETE … WHERE …`, `INSERT … ON CONFLICT`, etc.). +3. Nur Dev/CI: `.dev.sql` / `.dev.py` Suffix verwenden. +4. Nach Deploy prüfen: `python run_seeds.py` — bei geänderter Datei wird der Seed automatisch erneut ausgeführt. + +## Tests & CI + +- pytest setzt `SKIP_SEEDS=1` beim App-Import; Cleanup läuft über Fixture + nach CI-pytest. +- Geteilte Dev-DB wird nach jedem Test-Lauf bereinigt (`run_seeds.py --only seed_001_cleanup_pytest_artifacts --force`).