diff --git a/backend/main.py b/backend/main.py index 2a9d6ae..9ae6852 100644 --- a/backend/main.py +++ b/backend/main.py @@ -4,26 +4,38 @@ Shinkan Jinkendo - Main Application Entry Point Trainer- und Vereinsplattform für Kampfsport-Trainingsplanung """ from pathlib import Path +from typing import Optional from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from fastapi.staticfiles import StaticFiles import os +import sys from slowapi import _rate_limit_exceeded_handler from slowapi.errors import RateLimitExceeded from version import APP_VERSION, BUILD_DATE, DB_SCHEMA_VERSION, MODULE_VERSIONS -# Run database migrations on startup -try: - import run_migrations - run_migrations.main() - print("✓ Database migrations completed") -except Exception as e: - print(f"⚠ Warning: Migration error: {e}") - print(" Continuing startup - migrations may need manual intervention") +# Run database migrations before API start — halbes Schema ist schlimmer als kein Start +# Lokal ohne DB / nur Tests: SKIP_DB_MIGRATE=1 +if os.getenv("SKIP_DB_MIGRATE", "").strip().lower() in ("1", "true", "yes"): + print("⚠ SKIP_DB_MIGRATE=1 — Migrationen wurden übersprungen (nur für Entwicklung ohne DB)") +else: + try: + import run_migrations + + rc = run_migrations.main() + if rc != 0: + print(f"✗ Datenbank-Migration fehlgeschlagen (Exit-Code {rc}). Start abgebrochen.") + sys.exit(1) + print("✓ Database migrations completed") + except SystemExit: + raise + except Exception as e: + print(f"✗ Migration-Laufzeitfehler: {e}") + sys.exit(1) from routers.auth import limiter as auth_rate_limiter @@ -71,6 +83,63 @@ def health_check(): """Health check endpoint""" return {"status": "healthy", "version": APP_VERSION} + +@app.get("/api/health/ready") +def health_ready(): + """ + Verbindung + Kern-Tabellen prüfen (ohne Login). + Nutzen bei Prod-Debugging: wenn schema_complete=false, Migrationen nicht vollständig. + """ + from db import get_db, get_cursor + + REQUIRED = ( + "schema_migrations", + "skills", + "skill_main_categories", + "skill_categories", + "maturity_models", + "sessions", + ) + tables: dict = {} + err: Optional[str] = None + try: + with get_db() as conn: + cur = get_cursor(conn) + cur.execute("SELECT 1") + for tbl in REQUIRED: + cur.execute( + """ + SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = %s + ) + """, + (tbl,), + ) + row = cur.fetchone() + tables[tbl] = bool(next(iter(row.values()))) if row else False + migration_count = 0 + if tables.get("schema_migrations"): + cur.execute("SELECT COUNT(*)::int FROM schema_migrations") + mig_row = cur.fetchone() + migration_count = ( + int(list(mig_row.values())[0]) if mig_row is not None else 0 + ) + except Exception as e: + err = str(e) + migration_count = 0 + + complete = bool(err is None and all(tables.get(t) for t in REQUIRED)) + return { + "status": "ready" if complete else "degraded", + "database": err is None, + "detail": err, + "schema_complete": complete, + "tables": tables, + "schema_migrations_count": migration_count, + } + + # Root Endpoint @app.get("/") def read_root(): diff --git a/backend/requirements.txt b/backend/requirements.txt index 445b62d..e26b469 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -10,3 +10,4 @@ slowapi==0.1.9 psycopg2-binary==2.9.9 python-dateutil==2.9.0 tzdata>=2024.1 # ZoneInfo (Europe/Berlin) auch unter Windows +sqlparse>=0.5.0 # Migrationen: Statements splitten (Fallback ohne psql) diff --git a/backend/routers/skills.py b/backend/routers/skills.py index a60c062..acc4c66 100644 --- a/backend/routers/skills.py +++ b/backend/routers/skills.py @@ -9,6 +9,7 @@ import json from typing import Any, Optional from fastapi import APIRouter, HTTPException, Depends, Query +from psycopg2.extras import Json from db import get_db, get_cursor, r2d from auth import require_auth @@ -16,6 +17,15 @@ from auth import require_auth router = APIRouter(prefix="/api", tags=["skills"]) +def _jsonb_param(val: Any) -> Any: + """psycopg2 benötigt Json() oder String für JSONB — rohe dict/list sonst ProgrammingError.""" + if val is None: + return None + if isinstance(val, (dict, list)): + return Json(val) + return val + + def _main_id_for_category(cur, category_id: Optional[int]) -> Optional[int]: if not category_id: return None @@ -181,7 +191,7 @@ def create_skill(data: dict, session=Depends(require_auth)): data.get("category"), data.get("description"), data.get("importance"), - data.get("keywords"), + _jsonb_param(data.get("keywords")), data.get("status", "active"), cat_id, main_id, @@ -218,7 +228,6 @@ def update_skill(skill_id: int, data: dict, session=Depends(require_auth)): "category", "description", "importance", - "keywords", "status", "category_id", "main_category_id", @@ -227,6 +236,9 @@ def update_skill(skill_id: int, data: dict, session=Depends(require_auth)): if key in data: sets.append(f"{key} = %s") vals.append(data[key]) + if "keywords" in data: + sets.append("keywords = %s") + vals.append(_jsonb_param(data["keywords"])) if "focus_areas" in data: fa = data["focus_areas"] if isinstance(fa, (list, dict)): @@ -271,7 +283,6 @@ def delete_skill(skill_id: int, session=Depends(require_auth)): # Delete cur.execute("DELETE FROM skills WHERE id = %s", (skill_id,)) - conn.commit() return {"ok": True} diff --git a/backend/run_migrations.py b/backend/run_migrations.py index 12ff6d9..bd5f5d8 100644 --- a/backend/run_migrations.py +++ b/backend/run_migrations.py @@ -1,31 +1,57 @@ #!/usr/bin/env python3 """ -Shinkan Jinkendo - Database Migrations Runner +Shinkan Jinkendo — Datenbank-Migrationen -Runs all SQL migrations in backend/migrations/ directory -Tracks executed migrations in schema_migrations table +**Idempotent** über `schema_migrations`: jede numerische Datei `migrations/*.sql` höchstens einmal +als „erfolgreich“ eingetragen; bei erneutem Start werden nur noch fehlende Dateien abgearbeitet. + +**Reihenfolge:** Alle `NNN_*.sql` nach führender Zahl (001 vor 009 vor 010 …), bei gleicher Zahl +alphabetisch nach Dateinamen — nicht bloß String-Sortierung (vermeidet z. B. `10_` vor `9_`). + +**Pro Datei eine Transaktion:** Entweder `psql -1 -f` (wie bei psql/pg_dump üblich) oder Fallback +über `sqlparse.split` und einzelne `cursor.execute` in einer DB-Transaktion. + +Nach erfolgreicher Ausführung: `INSERT INTO schema_migrations (migration)` (mit `ON CONFLICT DO NOTHING`). """ import os +import re +import shutil +import subprocess import sys -import psycopg2 -from psycopg2 import sql import time +from typing import List, Tuple + +import psycopg2 +import sqlparse + + +def _db_params(): + return { + "host": os.getenv("DB_HOST", "localhost"), + "port": os.getenv("DB_PORT", "5432"), + "dbname": os.getenv("DB_NAME", "shinkan_dev"), + "user": os.getenv("DB_USER", "shinkan_dev"), + "password": os.getenv("DB_PASSWORD", "dev_password"), + } + def get_db_connection(): - """Get database connection with retries""" + """PostgreSQL-Verbindung mit Retry (Container-Start).""" + p = _db_params() max_retries = 30 for i in range(max_retries): try: conn = psycopg2.connect( - host=os.getenv("DB_HOST", "localhost"), - port=os.getenv("DB_PORT", "5432"), - database=os.getenv("DB_NAME", "shinkan_dev"), - user=os.getenv("DB_USER", "shinkan_dev"), - password=os.getenv("DB_PASSWORD", "dev_password") + host=p["host"], + port=p["port"], + database=p["dbname"], + user=p["user"], + password=p["password"], ) - print(f"✓ Connected to database: {os.getenv('DB_NAME')}") + conn.autocommit = False + print(f"✓ Connected to database: {p['dbname']}") return conn - except psycopg2.OperationalError as e: + except psycopg2.OperationalError: if i < max_retries - 1: print(f"Waiting for database... ({i+1}/{max_retries})") time.sleep(2) @@ -33,116 +59,222 @@ def get_db_connection(): print(f"✗ Failed to connect to database after {max_retries} attempts") raise + def init_migrations_table(conn): - """Create schema_migrations table if not exists""" with conn.cursor() as cur: - cur.execute(""" + cur.execute( + """ CREATE TABLE IF NOT EXISTS schema_migrations ( id SERIAL PRIMARY KEY, migration VARCHAR(255) UNIQUE NOT NULL, executed_at TIMESTAMP DEFAULT NOW() ) - """) - conn.commit() - print("✓ Migrations table initialized") + """ + ) + conn.commit() + print("✓ schema_migrations initialisiert") -def get_executed_migrations(conn): - """Get list of already executed migrations""" + +_LEADING_DIGITS = re.compile(r"^(\d+)") + + +def _migration_sort_key_from_stem(stem: str) -> Tuple[int, str]: + """Sortierung: numerisches Präfix am Anfang des Stems, dann voller Stem (stabil).""" + m = _LEADING_DIGITS.match(stem) + n = int(m.group(1)) if m else 0 + return (n, stem) + + +def get_migration_files_ordered(migrations_dir: str) -> List[Tuple[str, str]]: + rows = [] + for filename in os.listdir(migrations_dir): + if not filename.endswith(".sql"): + continue + if not filename[0].isdigit(): + continue + stem = filename[:-4] + rows.append((stem, os.path.join(migrations_dir, filename))) + rows.sort(key=lambda item: _migration_sort_key_from_stem(item[0])) + return rows + + +def get_executed(conn) -> set: with conn.cursor() as cur: - cur.execute("SELECT migration FROM schema_migrations ORDER BY migration") - return set(row[0] for row in cur.fetchall()) + cur.execute("SELECT migration FROM schema_migrations") + return set(r[0] for r in cur.fetchall()) -def get_pending_migrations(executed): - """Get list of pending migrations from migrations directory""" - migrations_dir = "/app/migrations" - if not os.path.exists(migrations_dir): - migrations_dir = "backend/migrations" # Local development - all_migrations = [] - for filename in sorted(os.listdir(migrations_dir)): - if filename.endswith('.sql') and filename[0].isdigit(): - migration_name = filename.replace('.sql', '') - if migration_name not in executed: - all_migrations.append((migration_name, os.path.join(migrations_dir, filename))) +def get_pending(conn, migrations_dir: str): + executed = get_executed(conn) + pending = [] + for name, path in get_migration_files_ordered(migrations_dir): + if name not in executed: + pending.append((name, path)) + return pending - return all_migrations -def run_migration(conn, migration_name, filepath): - """Run a single migration file""" +def _split_statements(sql_text: str) -> List[str]: + """Fallback ohne psql — einzelne Statements (sqlparse.split).""" + stripped = sql_text.strip() + if not stripped: + return [] + parts = sqlparse.split(stripped) + return [p.strip() for p in parts if p and p.strip()] + + +def _run_file_with_psql(filepath: str) -> Tuple[bool, str]: + """ + Ganze Datei wie von psql dokumentiert (-1 eine Transaktion, ON_ERROR_STOP). + Unter Windows oft nicht verfügbar → zurück False ohne Fehlertext wenn kein Binary. + """ + psql = shutil.which("psql") + if not psql: + return False, "" + + p = _db_params() + env = os.environ.copy() + env["PGPASSWORD"] = str(p["password"]) + + cmd = [ + psql, + "-h", + p["host"], + "-p", + str(p["port"]), + "-U", + p["user"], + "-d", + p["dbname"], + "-v", + "ON_ERROR_STOP=1", + "-1", + "-f", + filepath, + ] + + proc = subprocess.run( + cmd, + env=env, + capture_output=True, + text=True, + timeout=7200, + ) + if proc.returncode != 0: + tail = ( + (proc.stderr or "").strip() + + "\n" + + (proc.stdout or "").strip() + ).strip() + return False, tail[:8000] or f"exit {proc.returncode}" + out = (proc.stdout or "").strip() + return True, out + + +def _record_migration(conn, migration_name: str) -> None: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO schema_migrations (migration) + VALUES (%s) + ON CONFLICT (migration) DO NOTHING + """, + (migration_name,), + ) + + +def run_migration(conn, migration_name: str, filepath: str) -> bool: print(f"Running migration: {migration_name}") - with open(filepath, 'r', encoding='utf-8') as f: - sql_content = f.read() - + detail_suffix = "" try: - with conn.cursor() as cur: - # Execute migration - cur.execute(sql_content) + if shutil.which("psql"): + ok, diag = _run_file_with_psql(filepath) + if not ok: + print(f" ✗ psql fehlgeschlagen:\n{diag or '(kein Output)'}") + conn.rollback() + return False + detail_suffix = "(psql -1)" + else: + try: + with open(filepath, "r", encoding="utf-8") as fh: + body = fh.read() + except OSError as e: + print(f" ✗ kann Datei nicht lesen: {e}") + conn.rollback() + return False - # Record migration - cur.execute( - "INSERT INTO schema_migrations (migration) VALUES (%s)", - (migration_name,) - ) + statements = _split_statements(body) + with conn.cursor() as cur: + if not statements: + print( + f" ⚠ keine ausführbaren Statements (leer?) — " + f"Eintrag trotzdem: {migration_name}" + ) + else: + for stmt in statements: + cur.execute(stmt) + detail_suffix = f"(psycopg2 + sqlparse, {len(statements)} Statements)" + _record_migration(conn, migration_name) conn.commit() - print(f" ✓ {migration_name} executed successfully") + print(f" ✓ {migration_name} erfolgreich {detail_suffix}") return True except Exception as e: conn.rollback() - print(f" ✗ {migration_name} failed: {e}") + print(f" ✗ {migration_name}: {e}") return False + def main(): - """Main migrations runner""" print("=" * 60) - print("Shinkan Jinkendo - Database Migrations") + print("Shinkan Jinkendo — Database Migrations") + print("(Warteschlange: nur fehlende *.sql — idempotent)") print("=" * 60) + migrations_dir = "/app/migrations" + if not os.path.isdir(migrations_dir): + migrations_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "migrations") + try: - # Connect to database conn = get_db_connection() - - # Initialize migrations tracking init_migrations_table(conn) - # Get executed and pending migrations - executed = get_executed_migrations(conn) - pending = get_pending_migrations(executed) + pending = get_pending(conn, migrations_dir) if not pending: - print("✓ No pending migrations - database is up to date") + print("✓ Keine ausstehenden Migrationen — Schema aktuell.") + conn.close() return 0 - print(f"\nFound {len(pending)} pending migration(s):") - for name, _ in pending: - print(f" - {name}") + print(f"\n{len(pending)} ausstehende Migration(en):") + for n, _ in pending: + print(f" - {n}") print() - # Run pending migrations - failed = [] + failed = None for migration_name, filepath in pending: if not run_migration(conn, migration_name, filepath): - failed.append(migration_name) - break # Stop on first failure + failed = migration_name + break conn.close() - # Summary print("\n" + "=" * 60) if failed: - print(f"✗ Migration failed: {failed[0]}") + print(f"✗ Abbruch nach: {failed}") + print(" (Bereits erfolgreiche Dateien dieser Session sind committed.)") print("=" * 60) return 1 - else: - print(f"✓ All migrations executed successfully ({len(pending)} total)") - print("=" * 60) - return 0 + + print(f"✓ {len(pending)} Migration(s) angewendet — Schema aktuell.") + print("=" * 60) + return 0 except Exception as e: - print(f"\n✗ Error: {e}") + print(f"\n✗ Fehler: {e}") return 1 + if __name__ == "__main__": sys.exit(main()) diff --git a/backend/version.py b/backend/version.py index 2c1f481..750b412 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1,7 +1,7 @@ # Shinkan Jinkendo Version Information -APP_VERSION = "0.8.0" -BUILD_DATE = "2026-04-28" +APP_VERSION = "0.8.4" +BUILD_DATE = "2026-04-27" DB_SCHEMA_VERSION = "20260428031" MODULE_VERSIONS = { @@ -23,6 +23,38 @@ MODULE_VERSIONS = { } CHANGELOG = [ + { + "version": "0.8.4", + "date": "2026-04-27", + "changes": [ + "run_migrations: Warteschlange nach numerischem Präfix der Dateien (stabile Reihenfolge auch bei ungleich langen Zahlen wie 9 vs 10)", + ], + }, + { + "version": "0.8.3", + "date": "2026-04-28", + "changes": [ + "Migrationen: Warteschlange aller fehlenden *.sql; idempotent über schema_migrations", + "Ausführung vorzugsweise psql -1 -f (eine Transaktion pro Datei); Fallback psycopg2 + sqlparse.split", + "requirements: sqlparse wenn kein psql im PATH", + ], + }, + { + "version": "0.8.2", + "date": "2026-04-28", + "changes": [ + "main: run_migrations — Exit-Code auswerten; bei Fehler kein API-Start (verhindert Prod ohne Skill-Tabellen wie 022)", + "SKIP_DB_MIGRATE=1 optional für lokale Runs ohne Datenbank", + ], + }, + { + "version": "0.8.1", + "date": "2026-04-28", + "changes": [ + "skills: JSONB keywords per psycopg2.extras.Json (verhindert 500 wenn keywords als Objekt/Array gesendet wird)", + "GET /api/health/ready: DB-Verbindung + Kern-Tabellen + schema_migrations_count (ohne Auth, für Prod-Debugging)", + ], + }, { "version": "0.8.0", "date": "2026-04-28", diff --git a/docs/HANDOVER.md b/docs/HANDOVER.md index a6f9fdf..388f537 100644 --- a/docs/HANDOVER.md +++ b/docs/HANDOVER.md @@ -1,11 +1,19 @@ # Shinkan Jinkendo – Entwicklungsstand & Handover -**Stand:** 2026-04-27 -**App-Version:** 0.7.6 (`backend/version.py`) -**DB-Schema-Version:** `20260427027` (Migration 027) +**Stand:** 2026-04-28 +**App-Version / DB-Schema:** siehe `backend/version.py` Diese Datei ist die **Einstiegs-Doku für neue Chat-Sessions**: Anforderungen im Detail stehen in `.claude/docs/` (siehe unten); hier der **implementierte Stand** und **nächste Baustellen**. +### Produktion: `relation … does not exist` (z. B. `skill_main_categories`) + +Das Schema ist gegenüber dem Code zurück: Migration **`022_skills_schema_complete`** (und ggf. Folgende) wurden auf dieser Datenbank noch nicht erfolgreich ausgeführt. + +- **Automatisch:** Die Dateien `migrations/*.sql` (numerisch sortiert) bilden eine **Warteschlange**. Bereits erfolgreiche Läufe stehen in `schema_migrations` — diese Dateien werden **übersprungen** (idempotent). Pro Datei läuft **eine Transaktion** (im Container vorzugsweise `psql -1 -f`, sonst Fallback `sqlparse` + psycopg2). + +- **Fix / manuell:** `docker exec shinkan-api python /app/run_migrations.py` — Exit-Code **0**. +- Aktuelle Builds: Bei fehlgeschlagenem Migrate startet **`main.py`** die API nicht. Lokal ohne DB: **`SKIP_DB_MIGRATE=1`**. + --- ## 1. Pflichtlektüre (Kontext & Anforderungen)