From feb469d47ba27cbc26b30c54cfb5bf8e0bc116d4 Mon Sep 17 00:00:00 2001 From: Lars Date: Thu, 29 Oct 2026 18:32:17 +0100 Subject: [PATCH] feat: add universal-db-restore, app migration contract, targets for kairo/tandoor/bookstack, and uptime kuma guide --- README.md | 22 +- docs/APP_MIGRATION_CONTRACT.md | 127 +++++++ docs/UPTIME_KUMA_INTEGRATION.md | 107 ++++++ hosts/llm-node/targets.json | 63 ++++ hosts/raspberrypi5/targets.json | 30 ++ install.sh | 6 +- orchestrator.py | 11 +- scripts/common/universal-db-restore | 535 ++++++++++++++++++++++++++++ targets.json | 63 ++++ tests/test_universal_restore.py | 127 +++++++ 10 files changed, 1079 insertions(+), 12 deletions(-) create mode 100644 docs/APP_MIGRATION_CONTRACT.md create mode 100644 docs/UPTIME_KUMA_INTEGRATION.md create mode 100644 scripts/common/universal-db-restore create mode 100644 tests/test_universal_restore.py diff --git a/README.md b/README.md index 1b75702..e0e4591 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,12 @@ backup-orchestrator/ ├── install.sh # Server-spezifischer Installer (--host raspberrypi5 | llm-node) ├── targets.json # Globale Target-Registry ├── scripts/ -│ ├── common/ # Gemeinsame, serverunabhängige Backup-Engines -│ │ ├── db-postgres-docker.sh -│ │ ├── db-mariadb-docker.sh -│ │ └── publish-backup-set.sh -│ └── shinkan/ # Anwendungs-Adapter für Shinkan (Recovery Contract) +│ ├── common/ # Gemeinsame, serverunabhängige Backup- & Restore-Engines +│ │ ├── db-postgres-docker.sh # Universeller PostgreSQL-Dump +│ │ ├── db-mariadb-docker.sh # Universeller MariaDB-Dump +│ │ ├── publish-backup-set.sh # Erzeugt Manifest, Checksumme & relativen latest-Link +│ │ └── universal-db-restore # Universeller Staging-Restore für Postgres & MariaDB +│ └── shinkan/ # Spezifische Adapter für Shinkan (Pilot-Applikation) │ ├── shinkan-backup-manifest │ ├── shinkan-restore │ ├── shinkan-restore-test @@ -27,22 +28,25 @@ backup-orchestrator/ │ ├── raspberrypi5/ # Produktiver Backup-Host (Debian 12 ARM64, 192.168.2.49) │ │ ├── host.json # Mounts (/backup), Pfade (/media/lars/paperless/db-backup) │ │ ├── jobs.csv # 17 DB-Jobs (Phase 10), nas-data (Phase 20), system-image (Phase 30) -│ │ ├── targets.json # Host-Targets (pi-prod = source_only) +│ │ ├── targets.json # Host-Targets (pi-prod, kairo-pi-prod, tandoor-pi-prod, ...) │ │ ├── notifications.json.example │ │ ├── scripts/ # Alle 17 DB-Wrapper + NAS-Skripte für raspberrypi5 │ │ └── systemd/ # backup-orchestrator-daily + .service/.timer (05:30 Uhr) │ └── llm-node/ # Neuer Anwendungs- & Recovery-Host (Ubuntu 24.04 x86_64, 192.168.2.144) │ ├── host.json # Mounts (/mnt/nas_backup, /srv/shinkan/prod/media) │ ├── jobs.csv # Backup-Jobs auf llm-node (shinkan-prod-db für Post-Cutover vorbereitet) -│ ├── targets.json # Targets (llm-prod, llm-restoretest, pi-prod = source_only) +│ ├── targets.json # Targets (shinkan, kairo, tandoor, bookstack...) │ ├── notifications.json.example │ ├── scripts/ # Host-spezifische Wrapper für llm-node │ └── systemd/ # backup-orchestrator-daily + .service/.timer (05:00 Uhr) ├── docs/ -│ └── ARCHITECTURE_AND_RECOVERY_PLAN_v0.4.md +│ ├── ARCHITECTURE_AND_RECOVERY_PLAN_v0.4.md +│ ├── APP_MIGRATION_CONTRACT.md # Entwickler-Leitfaden & Agenten-Prompt für App-Migrationen +│ └── UPTIME_KUMA_INTEGRATION.md # Push-Monitoring & Alerting-Konfiguration └── tests/ ├── test_orchestrator.py - └── test_recovery.py + ├── test_recovery.py + └── test_universal_restore.py ``` --- diff --git a/docs/APP_MIGRATION_CONTRACT.md b/docs/APP_MIGRATION_CONTRACT.md new file mode 100644 index 0000000..c3c155f --- /dev/null +++ b/docs/APP_MIGRATION_CONTRACT.md @@ -0,0 +1,127 @@ +# App Migration Contract & Entwickler-Leitfaden für Coding-Agenten + +**Gültig für:** Alle Docker-basierten Anwendungen, die von `raspberrypi5` auf `llm-node` umgezogen oder neu an den `backup-orchestrator` angebunden werden (z. B. Kairo, Tandoor Recipes, BookStack, Paperless-ngx, Mitai, Netbox, Tsumugu, Kansho). + +--- + +## 1. Zielsetzung & Architektur + +Der `backup-orchestrator` führt automatische Backups (Phase 10 Dumps, Phase 20 NAS-Sync) und risikolose Staging-Restores mit atomarem Swap und Rollback-Sicherheit durch. + +Damit eine Anwendung ohne manuelle Eingriffe und ohne Datenverlust migriert und im Regelbetrieb gesichert werden kann, muss jeder Applikations-Entwickler bzw. Coding-Agent die folgenden **5 Kernanforderungen** einhalten. + +```mermaid +flowchart TD + A["Anwendung (Docker Compose)"] --> B["1. DB & App Trennung"] + A --> C["2. Health/Readiness Endpoint (/api/health/ready)"] + A --> D["3. Deterministische Migrationen"] + A --> E["4. Stateless Container & NAS-Storage"] + A --> F["5. Invalidate Sessions bei Restore"] +``` + +--- + +## 2. Die 5 Kernanforderungen an jede Anwendung + +### Anforderung 1: Docker Compose Architektur +1. **Container-Benennung**: + - DB-Container: `-db-prod` (z. B. `kairo-db-prod`, `tandoor-db-prod`, `bookstack-db-prod`). + - App-/Backend-Container: `-backend`, `-frontend` oder sprechende Namen. +2. **Umgebungsvariablen (`.env` / `/etc//prod.env`)**: + - DB-Credentials dürfen niemals im Compose-File fest eincodiert sein. + - Für PostgreSQL: `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD`. + - Für MariaDB: `MARIADB_DATABASE`, `MARIADB_USER`, `MARIADB_PASSWORD`. +3. **Multi-Architektur**: + - Docker-Images müssen Multi-Arch (`linux/amd64` für `llm-node`, `linux/arm64` für Pi) unterstützen oder direkt auf dem Ziel-Host baubar sein. + +### Anforderung 2: Readiness / Health Endpoint (`ready_url`) +Jede Anwendung **muss** einen HTTP-Healthcheck bereitstellen (z. B. `http://127.0.0.1:/api/health/ready` oder `.../healthz`): +- **HTTP 200 OK** darf **erst dann** zurückgegeben werden, wenn: + 1. Die Datenbankverbindung steht und Abfragen erfolgreich sind. + 2. Alle anstehenden Datenbankmigrationen vollständig ausgeführt wurden. +- Wenn die DB noch migriert, blockiert ist oder nicht antwortet, muss der Endpoint HTTP `503 Service Unavailable` zurückgeben. +- Bei JSON-Ausgabe sollte `{"status": "ready"}` oder `{"status": "ok"}` enthalten sein. + +### Anforderung 3: Deterministische Datenbank-Migrationen +1. Schema-Änderungen müssen über Migrationstabellen nachvollziehbar sein: + - Django: `django_migrations` + - Laravel / BookStack: `migrations` + - Shinkan / Custom: `schema_migrations` + - Alembic / FastAPI: `alembic_version` +2. **Migrations-Kompatibilität**: + - Vor einem Restore prüft der Orchestrator, ob der Dump Migrationen enthält, die im deployten Code noch fehlen. + - Ein Rollback auf älteren Code bei neuerem DB-Dump wird verweigert (`EXIT_SCHEMA_INCOMPATIBLE`). + +### Anforderung 4: Invalidation von Sessions bei Umgebungsumzug +- Bei einer Migration zwischen Servern (z. B. von Pi auf llm-node) dürfen alte Session-Tokens nicht unkontrolliert weiterleben. +- Die Session-Tabelle (z. B. `sessions` oder `django_session`) wird beim Restore geleert (`TRUNCATE TABLE`), sofern `--sessions clear` aktiv ist. + +### Anforderung 5: Trennung von Code, Datenbank und Medien (Stateless Container) +- Uploads, Dokumente, PDF-Dateien und Bilder gehören **nicht** in die Datenbank und **nicht** in flüchtige Container-Layer! +- Medienverzeichnisse müssen auf persistenten Mounts (Synology NAS oder lokale Mountpoints wie `/srv//prod/media`) liegen. +- Der DB-Dump bleibt dadurch klein (< 50 MB), transaktionssicher und blitzschnell wiederherstellbar. + +--- + +## 3. Konfiguration in `targets.json` + +Sobald die Anwendung die Kriterien erfüllt, wird sie in `/etc/backup-orchestrator/targets.json` auf `llm-node` eingetragen: + +```json +{ + "kairo-llm-prod": { + "application": "kairo", + "environment": "production", + "host": "llm-node", + "arch": "x86_64", + "role": "production_target", + "allow_restore": true, + "require_confirm": "kairo@llm-node", + "app_dir": "/home/llmadmin/docker/kairo-prod", + "compose_project": "kairo-prod", + "compose_files": ["docker-compose.yml"], + "env_file": "/etc/kairo/prod.env", + "db_engine": "postgresql", + "db_container": "kairo-db-prod", + "db_name": "kairo", + "db_user": "kairo_user", + "ready_url": "http://127.0.0.1:8001/api/health/ready", + "services_to_stop": ["backend"], + "verify_tables_not_empty": ["django_migrations"], + "truncate_tables": ["django_session"] + } +} +``` + +--- + +## 4. Prompt-Vorlage für Coding-Agenten der Ziel-Anwendung + +Wenn ein Coding-Agent (z. B. in Roo-Code, Claude Code, Cursor oder Gemini) beauftragt wird, eine Anwendung für den Umzug nach `llm-node` vorzubereiten, verwende folgenden Prompt: + +````markdown +Du bereitest die Anwendung für den produktiven Betrieb auf unserem neuen Server `llm-node` (Ubuntu 24.04 LTS x86_64) und die Anbindung an den Backup-Orchestrator vor. + +Bitte setze folgende Anforderungen um und verändere keine Produktionsdaten auf dem Raspberry Pi: + +1. **Docker Compose & Multi-Arch Build:** + - Stelle sicher, dass `docker-compose.yml` (oder ein Override `docker-compose.llm-node.yml`) auf `llm-node` fehlerfrei baut (`linux/amd64`). + - Der Datenbank-Container muss den festen Namen `-db-prod` tragen. + - Alle DB-Credentials müssen aus einer externen Environment-Datei (`/etc//prod.env` oder `.env`) geladen werden. + - Nicht-persistente App-Container müssen zustandslos (stateless) sein; persistente Medien/Uploads müssen auf ein Host-Verzeichnis gemountet werden. + +2. **Health / Readiness Endpoint:** + - Implementiere oder verifiziere einen HTTP-Endpoint unter `/api/health/ready` (oder `/healthz`). + - Dieser Endpoint muss HTTP 200 liefern, sobald die Datenbank erreichbar ist UND alle Migrationen erfolgreich durchgelaufen sind. + - Solange Migrationen laufen oder die DB nicht bereit ist, muss HTTP 503 zurückgegeben werden. + +3. **Migrations-Transparenz:** + - Dokumentiere, in welcher Tabelle die DB-Migrationen erfasst werden (z. B. `schema_migrations`, `django_migrations`, `migrations`). + - Dokumentiere, welche 1–2 Kern-Tabellen nach einem Restore mindestens befüllt sein müssen (z. B. `users`, `profiles`, `posts`). + - Dokumentiere den Namen der Session-Tabelle (z. B. `sessions`, `django_session`), damit alte Tokens beim Umzug bereinigt werden können. + +4. **Verifikation auf llm-node:** + - Starte den Stack auf `llm-node` mit einer leeren Datenbank. + - Prüfe mit `curl -s -i http://127.0.0.1:/api/health/ready`, dass der Readiness-Check grün meldet. + - Melde zurück: DB-Engine, Containername, DB-Name, DB-User, Health-URL, Session-Tabelle und nicht-leere Kern-Tabellen. +```` diff --git a/docs/UPTIME_KUMA_INTEGRATION.md b/docs/UPTIME_KUMA_INTEGRATION.md new file mode 100644 index 0000000..8476b97 --- /dev/null +++ b/docs/UPTIME_KUMA_INTEGRATION.md @@ -0,0 +1,107 @@ +# Uptime Kuma Monitoring & Alerting Guide + +Der `backup-orchestrator` verfügt über eine native, out-of-the-box Integration mit **Uptime Kuma**. +Dieses Dokument beschreibt das Monitoring-Konzept, die Schritt-für-Schritt-Einrichtung und die Konfiguration. + +--- + +## 1. Duales Überwachungskonzept + +Das Gesamtsystem wird auf zwei komplementären Ebenen überwacht: + +```mermaid +flowchart TD + subgraph "1. Passive Push-Überwachung (Backup-Läufe & Restores)" + BO["backup-orchestrator
(systemd daily 05:30)"] + BO -->|"HTTP Push: JOB_SUCCESS (up) / JOB_FAILED (down)"| K1["Uptime Kuma
Push Monitor: Einzeljobs"] + BO -->|"HTTP Push: RUN_FINISHED (overall ok/failed)"| K2["Uptime Kuma
Push Monitor: Täglicher Gesamtlauf"] + end + + subgraph "2. Aktive Pull-Überwachung (Live-Dienste auf llm-node)" + K3["Uptime Kuma
HTTP(s) Monitor: 60s Intervall"] + K3 -->|"GET /api/health/ready (Port 8003, 8001...)"| APP1["Shinkan / Kairo Backend"] + K3 -->|"GET / (Port 3003, 8080...)"| APP2["Frontend & Web-UIs"] + end +``` + +### A. Passive Push-Überwachung (Heartbeat / Dead Man's Snitch) +- **Problem bei Crons/Timern**: Wenn ein Cronjob gar nicht erst startet (Server abgestürzt, Timer deaktiviert, NAS-Mount fehlt), gibt es keine Fehlermeldung. +- **Lösung via Kuma Push**: Der Orchestrator sendet am Ende eines Laufs aktiv ein Signal (`/api/push/?status=up&msg=OK`). Bleibt dieses Signal nach 24 Stunden aus, schlägt Uptime Kuma automatisch Alarm ("Monitor DOWN"). +- Einzelne kritische Jobs (z. B. `paperless-db`, `shinkan-prod-db`, `nas-data`) können zudem dedizierte Push-Monitore bedienen. + +### B. Aktive Pull-Überwachung (Service Health) +- Uptime Kuma pollt im Regelbetrieb alle 60 Sekunden die Health-Endpoints (`ready_url`) der Container. +- Nach einem Staging-Restore durch den Orchestrator wird so sofort verifiziert, ob die Anwendung auch von außen im Netzwerk wieder uneingeschränkt antwortet. + +--- + +## 2. Einrichtung in Uptime Kuma (Schritt für Schritt) + +### Schritt 1: Push-Monitor für den täglichen Backup-Lauf anlegen +1. In Uptime Kuma auf **"Neuen Monitor hinzufügen"** klicken. +2. Folgende Werte eintragen: + - **Monitortyp**: `Push` + - **Name**: `Backup: Täglicher Gesamtlauf (Pi 5)` (oder `Backup: Gesamtlauf llm-node`) + - **Intervall (Heartbeat)**: `86400` Sekunden (24 Stunden) + - **Wiederholungsintervall**: `300` Sekunden (5 Minuten) + - **Max. Wiederholungen**: `2` +3. Speichern. Uptime Kuma zeigt nun eine Push-URL an: + ```text + http://:3001/api/push/abc123token?status=up&msg=OK&ping= + ``` +4. Den Basis-Teil der URL kopieren: `http://:3001/api/push/abc123token`. + +### Schritt 2 (Optional): Push-Monitore für Einzeljobs anlegen +Wiederhole den Vorgang für kritische Einzel-Jobs: +- `Backup: Shinkan DB (Phase 10)` +- `Backup: Paperless DB (Phase 10)` +- `Backup: NAS Data Sync (Phase 20)` + +--- + +## 3. Konfiguration im Backup-Orchestrator + +Auf dem jeweiligen Server (`raspberrypi5` oder `llm-node`) wird die Datei `/etc/backup-orchestrator/notifications.json` gepflegt: + +```json +{ + "timeout_seconds": 3, + "webhooks": [], + "kuma": { + "overall_push_url": "http://192.168.2.xxx:3001/api/push/YOUR_OVERALL_TOKEN", + "job_push_urls": { + "shinkan-prod-db": "http://192.168.2.xxx:3001/api/push/YOUR_SHINKAN_TOKEN", + "paperless-db": "http://192.168.2.xxx:3001/api/push/YOUR_PAPERLESS_TOKEN", + "nas-data": "http://192.168.2.xxx:3001/api/push/YOUR_NAS_DATA_TOKEN" + } + } +} +``` + +> [!IMPORTANT] +> - Dateirechte: Die Datei muss mit `chmod 0600 /etc/backup-orchestrator/notifications.json` geschützt werden. +> - Der Orchestrator akzeptiert sowohl die reine Basis-URL (`.../api/push/`) als auch die von Kuma kopierte Beispiel-URL (`.../api/push/?status=up&msg=OK`); Abfrageparameter werden automatisch bereinigt. + +--- + +## 4. Testen der Uptime Kuma Push-Benachrichtigung + +Um die Benachrichtigung zu testen, ohne ein vollständiges Backup auszulösen: + +```bash +# Trockenlauf / Status-Check (zeigt an, wie viele Kuma-URLs konfiguriert sind): +backup-orchestrator --check + +# Testlauf für einen einzelnen Job: +BACKUP_ORCHESTRATOR_CUTOVER_APPROVED=YES backup-orchestrator --run --job shinkan-prod-db +``` + +In Uptime Kuma wechselt der Monitor sofort auf **"UP"** (grün) und zeigt im Ereignisprotokoll die Job-ID, den Laufzeit-Timestamp und die Run-ID an. + +--- + +## 5. Fehlertoleranz & Logging + +- **Entkopplung**: Push-Benachrichtigungen werden asynchron über einen Thread-Pool versendet. +- **Sicherheitsgarantie**: Netzwerk-Timeouts oder Fehler von Uptime Kuma verändern **niemals** den Exit-Code des Backup-Laufs. Ein erfolgreiches Backup bleibt immer `Exit 0`. +- Fehlerhafte Push-Versuche werden in `/var/log/backup-orchestrator//notifications.log` protokolliert. diff --git a/hosts/llm-node/targets.json b/hosts/llm-node/targets.json index 474b41a..2613b56 100644 --- a/hosts/llm-node/targets.json +++ b/hosts/llm-node/targets.json @@ -85,6 +85,69 @@ "frontend_ready_url": "http://127.0.0.1:13003/api/health/ready", "adapter_dir": "/etc/backup-orchestrator/scripts/shinkan", "services_to_stop": ["backend", "frontend"] + }, + "kairo-llm-prod": { + "application": "kairo", + "environment": "production", + "host": "llm-node", + "arch": "x86_64", + "role": "production_target", + "allow_restore": true, + "require_confirm": "kairo@llm-node", + "app_dir": "/home/llmadmin/docker/kairo-prod", + "compose_project": "kairo-prod", + "compose_files": ["docker-compose.yml"], + "env_file": "/etc/kairo/prod.env", + "db_engine": "postgresql", + "db_container": "kairo-db-prod", + "db_name": "kairo", + "db_user": "kairo_user", + "ready_url": "http://127.0.0.1:8001/api/health/ready", + "services_to_stop": ["backend"], + "verify_tables_not_empty": ["django_migrations"], + "truncate_tables": ["django_session"] + }, + "tandoor-llm-prod": { + "application": "tandoor", + "environment": "production", + "host": "llm-node", + "arch": "x86_64", + "role": "production_target", + "allow_restore": true, + "require_confirm": "tandoor@llm-node", + "app_dir": "/home/llmadmin/docker/tandoor-prod", + "compose_project": "tandoor-prod", + "compose_files": ["docker-compose.yml"], + "env_file": "/etc/tandoor/prod.env", + "db_engine": "postgresql", + "db_container": "tandoor-db-prod", + "db_name": "tandoordb", + "db_user": "tandoor_user", + "ready_url": "http://127.0.0.1:8080/", + "services_to_stop": ["web_app"], + "verify_tables_not_empty": ["django_migrations"], + "truncate_tables": ["django_session"] + }, + "bookstack-llm-prod": { + "application": "bookstack", + "environment": "production", + "host": "llm-node", + "arch": "x86_64", + "role": "production_target", + "allow_restore": true, + "require_confirm": "bookstack@llm-node", + "app_dir": "/home/llmadmin/docker/bookstack-prod", + "compose_project": "bookstack-prod", + "compose_files": ["docker-compose.yml"], + "env_file": "/etc/bookstack/prod.env", + "db_engine": "mariadb", + "db_container": "bookstack-db-prod", + "db_name": "bookstack", + "db_user": "bookstack_user", + "ready_url": "http://127.0.0.1:6875/", + "services_to_stop": ["bookstack"], + "verify_tables_not_empty": ["migrations"], + "truncate_tables": ["sessions"] } } } diff --git a/hosts/raspberrypi5/targets.json b/hosts/raspberrypi5/targets.json index d4ef957..0a6e157 100644 --- a/hosts/raspberrypi5/targets.json +++ b/hosts/raspberrypi5/targets.json @@ -27,6 +27,36 @@ "db_name": "shinkan", "db_user": "shinkan", "health_url": "http://localhost:8003/api/health/ready" + }, + "kairo-pi-prod": { + "application": "kairo", + "environment": "prod", + "host": "raspberrypi5", + "role": "source_only", + "allow_restore": false, + "db_engine": "postgresql", + "db_container": "kairo-db-prod", + "db_name": "kairo" + }, + "tandoor-pi-prod": { + "application": "tandoor", + "environment": "prod", + "host": "raspberrypi5", + "role": "source_only", + "allow_restore": false, + "db_engine": "postgresql", + "db_container": "tandoor-db-prod", + "db_name": "tandoordb" + }, + "bookstack-pi-prod": { + "application": "bookstack", + "environment": "prod", + "host": "raspberrypi5", + "role": "source_only", + "allow_restore": false, + "db_engine": "mariadb", + "db_container": "bookstack-db-prod", + "db_name": "bookstack" } } } diff --git a/install.sh b/install.sh index f710b9a..d8de469 100644 --- a/install.sh +++ b/install.sh @@ -73,8 +73,10 @@ if [[ ! -f /etc/backup-orchestrator/notifications.json ]]; then fi # 4. Install shared common scripts + host-specific scripts + Shinkan adapters -for s in "${REPO_ROOT}/scripts/common/"*.sh; do - [[ -f "$s" ]] && install -m 0750 "$s" "/etc/backup-orchestrator/scripts/$(basename "$s")" +for s in "${REPO_ROOT}/scripts/common/"*; do + if [[ -f "$s" ]]; then + install -m 0755 "$s" "/etc/backup-orchestrator/scripts/$(basename "$s")" + fi done for s in "${HOST_DIR}/scripts/"*.sh; do diff --git a/orchestrator.py b/orchestrator.py index 842466e..d673131 100644 --- a/orchestrator.py +++ b/orchestrator.py @@ -1371,7 +1371,16 @@ def execute_recovery_command(mode, args, targets_cfg): cmd = _build_script_cmd(script, extra) else: custom_script = tcfg.get('restore_script') - script = Path(custom_script) if custom_script and Path(custom_script).is_file() else (adapter_dir / f'{app_id}-restore') + if custom_script and Path(custom_script).is_file(): + script = Path(custom_script) + elif (adapter_dir / f'{app_id}-restore').is_file(): + script = adapter_dir / f'{app_id}-restore' + else: + candidates = [ + Path('/etc/backup-orchestrator/scripts/universal-db-restore'), + Path(__file__).resolve().parent / 'scripts' / 'common' / 'universal-db-restore', + ] + script = next((c for c in candidates if c.is_file()), adapter_dir / f'{app_id}-restore') extra = [ '--target', target_name, '--dump', dump_path, '--result', str(result_file), '--sessions', args.sessions, diff --git a/scripts/common/universal-db-restore b/scripts/common/universal-db-restore new file mode 100644 index 0000000..46d8dc1 --- /dev/null +++ b/scripts/common/universal-db-restore @@ -0,0 +1,535 @@ +#!/usr/bin/env python3 +"""Universal transactional database restore adapter for PostgreSQL and MariaDB/MySQL. + +Supports Docker-based deployments with: +- Staging database isolation and integrity verification prior to live cutover. +- Atomic database/table swap. +- Reversible rollback if post-restore health checks fail. +- Plausibility checks (non-empty tables, migration checks). +- Session truncation on cutover. +- Fully parameter-driven via targets.json. + +Exit codes: +- 0: Restore and post-restore checks succeeded +- 1: Execution error +- 2: Precondition / confirmation / host guard error +- 3: Schema incompatibility +- 4: Checksum mismatch +- 5: Post-restore healthcheck failed (automatic rollback executed) +""" + +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import socket +import subprocess +import sys +import time +from urllib.request import urlopen + +EXIT_OK = 0 +EXIT_EXEC_ERROR = 1 +EXIT_PRECONDITION = 2 +EXIT_SCHEMA_INCOMPATIBLE = 3 +EXIT_CHECKSUM_MISMATCH = 4 +EXIT_POST_RESTORE_FAILED = 5 + + +def compute_sha256(path): + h = hashlib.sha256() + with Path(path).open('rb') as fp: + while True: + chunk = fp.read(65536) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def detect_dump_format(path): + with Path(path).open('rb') as fp: + magic = fp.read(5) + if magic == b'PGDMP': + return 'pg_custom' + return 'plain_sql' + + +def write_result(path, data): + if path: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + + +def load_targets(custom_file=None): + candidates = [] + if custom_file: + candidates.append(Path(custom_file)) + candidates.extend([ + Path('/etc/backup-orchestrator/targets.json'), + Path(__file__).resolve().parents[2] / 'targets.json', + Path(__file__).resolve().parents[2] / f'hosts/{socket.gethostname()}/targets.json', + ]) + for c in candidates: + if c.is_file(): + try: + data = json.loads(c.read_text(encoding='utf-8')) + if 'targets' in data: + return data + except (OSError, json.JSONDecodeError): + continue + return {'targets': {}} + + +# ----------------------------------------------------------------------------- +# PostgreSQL Database Helpers +# ----------------------------------------------------------------------------- + +def psql_exec(tcfg, dbname, sql): + cmd = [ + 'docker', 'exec', '-i', tcfg['db_container'], + 'sh', '-ec', + f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -v ON_ERROR_STOP=1 -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}" -At', + ] + return subprocess.run(cmd, input=sql, capture_output=True, text=True, timeout=120) + + +def postgres_drop_db_if_exists(tcfg, dbname): + term_sql = f""" + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = '{dbname}' AND pid <> pg_backend_pid(); + """ + psql_exec(tcfg, 'postgres', term_sql) + psql_exec(tcfg, 'postgres', f'DROP DATABASE IF EXISTS "{dbname}";') + + +def postgres_create_db(tcfg, dbname): + postgres_drop_db_if_exists(tcfg, dbname) + res = psql_exec(tcfg, 'postgres', f'CREATE DATABASE "{dbname}";') + return res.returncode == 0, res.stderr.strip() + + +def postgres_import_dump(tcfg, dbname, dump_path): + fmt = detect_dump_format(dump_path) + if fmt == 'pg_custom': + cmd = [ + 'docker', 'exec', '-i', tcfg['db_container'], + 'sh', '-ec', + f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec pg_restore -v --single-transaction --exit-on-error -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}"', + ] + else: + cmd = [ + 'docker', 'exec', '-i', tcfg['db_container'], + 'sh', '-ec', + f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -v ON_ERROR_STOP=1 --single-transaction -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}"', + ] + with Path(dump_path).open('rb') as fp: + res = subprocess.run(cmd, stdin=fp, capture_output=True, text=True, timeout=1800) + return res.returncode == 0, res.stderr.strip() or res.stdout.strip() + + +def postgres_swap_databases(tcfg, live_db, staging_db, backup_db): + term_sql = f""" + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname IN ('{live_db}', '{staging_db}', '{backup_db}') AND pid <> pg_backend_pid(); + """ + psql_exec(tcfg, 'postgres', term_sql) + # 1. Rename live to backup + r1 = psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{live_db}" RENAME TO "{backup_db}";') + if r1.returncode != 0: + return False, f"Rename {live_db} -> {backup_db} failed: {r1.stderr.strip()}" + # 2. Rename staging to live + r2 = psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{staging_db}" RENAME TO "{live_db}";') + if r2.returncode != 0: + # Emergency rollback + psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{backup_db}" RENAME TO "{live_db}";') + return False, f"Rename {staging_db} -> {live_db} failed: {r2.stderr.strip()}" + return True, "ok" + + +def postgres_rollback_swap(tcfg, live_db, staging_db, backup_db): + term_sql = f""" + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname IN ('{live_db}', '{staging_db}', '{backup_db}') AND pid <> pg_backend_pid(); + """ + psql_exec(tcfg, 'postgres', term_sql) + psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{live_db}" RENAME TO "{staging_db}";') + psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{backup_db}" RENAME TO "{live_db}";') + + +# ----------------------------------------------------------------------------- +# MariaDB / MySQL Database Helpers +# ----------------------------------------------------------------------------- + +def mariadb_exec(tcfg, dbname, sql): + db_flag = f'"{dbname}"' if dbname else '' + cmd = [ + 'docker', 'exec', '-i', tcfg['db_container'], + 'sh', '-ec', + f'export MYSQL_PWD="${{MARIADB_PASSWORD:-${{MYSQL_PASSWORD:-}}}}"; ' + f'exec mariadb --default-character-set=utf8mb4 -u "${{MARIADB_USER:-${{MYSQL_USER:-root}}}}" {db_flag} -sN', + ] + return subprocess.run(cmd, input=sql, capture_output=True, text=True, timeout=120) + + +def mariadb_drop_db_if_exists(tcfg, dbname): + mariadb_exec(tcfg, '', f'DROP DATABASE IF EXISTS `{dbname}`;') + + +def mariadb_create_db(tcfg, dbname): + mariadb_drop_db_if_exists(tcfg, dbname) + res = mariadb_exec(tcfg, '', f'CREATE DATABASE `{dbname}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;') + return res.returncode == 0, res.stderr.strip() + + +def mariadb_import_dump(tcfg, dbname, dump_path, original_dbname): + """Import MariaDB dump into staging DB, rewriting CREATE/USE database references if needed.""" + cmd = [ + 'docker', 'exec', '-i', tcfg['db_container'], + 'sh', '-ec', + f'export MYSQL_PWD="${{MARIADB_PASSWORD:-${{MYSQL_PASSWORD:-}}}}"; ' + f'exec mariadb --default-character-set=utf8mb4 -u "${{MARIADB_USER:-${{MYSQL_USER:-root}}}}" `{dbname}`', + ] + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + # Stream and rewrite USE `original_dbname` / CREATE DATABASE statements to staging db + re_use = re.compile(rf'^\s*USE\s+[`"\']?{re.escape(original_dbname)}[`"\']?\s*;', re.IGNORECASE) + re_create = re.compile(rf'^\s*CREATE\s+DATABASE.*?[`"\']?{re.escape(original_dbname)}[`"\']?.*?;', re.IGNORECASE) + + try: + with Path(dump_path).open('r', encoding='utf-8', errors='replace') as fp: + for line in fp: + if re_use.match(line): + proc.stdin.write(f'USE `{dbname}`;\n') + elif re_create.match(line): + proc.stdin.write(f'-- {line.strip()}\n') + else: + proc.stdin.write(line) + stdout, stderr = proc.communicate(timeout=1800) + return proc.returncode == 0, stderr.strip() or stdout.strip() + except Exception as exc: + proc.kill() + return False, str(exc) + + +def mariadb_get_tables(tcfg, dbname): + sql = f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{dbname}' AND table_type = 'BASE TABLE';" + res = mariadb_exec(tcfg, '', sql) + if res.returncode != 0: + return [] + return [line.strip() for line in res.stdout.splitlines() if line.strip()] + + +def mariadb_swap_databases(tcfg, live_db, staging_db, backup_db): + """Atomically swap tables between schemas using MySQL/MariaDB RENAME TABLE.""" + mariadb_create_db(tcfg, backup_db) + live_tables = set(mariadb_get_tables(tcfg, live_db)) + staging_tables = set(mariadb_get_tables(tcfg, staging_db)) + + rename_clauses = [] + # Move current live tables to backup_db + for tbl in live_tables: + rename_clauses.append(f"`{live_db}`.`{tbl}` TO `{backup_db}`.`{tbl}`") + # Move staging tables to live_db + for tbl in staging_tables: + rename_clauses.append(f"`{staging_db}`.`{tbl}` TO `{live_db}`.`{tbl}`") + + if not rename_clauses: + return False, "No tables to rename in swap operation" + + rename_sql = "RENAME TABLE " + ", ".join(rename_clauses) + ";" + res = mariadb_exec(tcfg, '', rename_sql) + if res.returncode != 0: + return False, f"Atomic table rename failed: {res.stderr.strip()}" + return True, "ok" + + +def mariadb_rollback_swap(tcfg, live_db, staging_db, backup_db): + live_tables = set(mariadb_get_tables(tcfg, live_db)) + backup_tables = set(mariadb_get_tables(tcfg, backup_db)) + rename_clauses = [] + for tbl in live_tables: + rename_clauses.append(f"`{live_db}`.`{tbl}` TO `{staging_db}`.`{tbl}`") + for tbl in backup_tables: + rename_clauses.append(f"`{backup_db}`.`{tbl}` TO `{live_db}`.`{tbl}`") + if rename_clauses: + mariadb_exec(tcfg, '', "RENAME TABLE " + ", ".join(rename_clauses) + ";") + + +# ----------------------------------------------------------------------------- +# Common Compose & Healthcheck Functions +# ----------------------------------------------------------------------------- + +def compose_cmd(tcfg, *args): + cmd = ['docker', 'compose'] + if tcfg.get('compose_project'): + cmd.extend(['-p', tcfg['compose_project']]) + for cf in tcfg.get('compose_files', ['docker-compose.yml']): + cf_path = Path(cf) if Path(cf).is_absolute() else (Path(tcfg.get('app_dir', '.')) / cf) + if cf_path.is_file(): + cmd.extend(['-f', str(cf_path)]) + if tcfg.get('env_file') and Path(tcfg['env_file']).is_file(): + cmd.extend(['--env-file', tcfg['env_file']]) + cmd.extend(args) + cwd = tcfg.get('app_dir') if tcfg.get('app_dir') and Path(tcfg['app_dir']).is_dir() else None + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=300) + + +def wait_ready(url, timeout_sec=180): + deadline = time.monotonic() + timeout_sec + last_body = None + while time.monotonic() < deadline: + try: + with urlopen(url, timeout=5) as resp: + raw = resp.read(8192).decode('utf-8', errors='replace') + if resp.status == 200: + try: + parsed = json.loads(raw) + # If JSON contains 'status', verify it is 'ok' or 'ready' + if isinstance(parsed, dict) and 'status' in parsed: + if parsed['status'] in ('ready', 'ok'): + return True, parsed + else: + return True, {'raw': raw[:200]} + except (ValueError, json.JSONDecodeError): + return True, {'status': 'http_200'} + except Exception as exc: + last_body = {'error': str(exc)} + time.sleep(3) + return False, last_body + + +# ----------------------------------------------------------------------------- +# Main Restore Engine +# ----------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description='Universal transactional database restore adapter') + parser.add_argument('--target', required=True, help='Target name from targets.json') + parser.add_argument('--dump', type=Path, help='Path to SQL dump') + parser.add_argument('--result', required=True, type=Path, help='Output path for result JSON') + parser.add_argument('--confirm', type=str, help='Required confirmation token') + parser.add_argument('--allow-overwrite', action='store_true', help='Allow overwrite of non-empty live database') + parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear') + parser.add_argument('--drop-previous', type=str, help='Explicitly drop a retained _before_ database') + parser.add_argument('--targets-file', type=Path, help='Optional explicit path to targets.json') + args = parser.parse_args() + + targets_registry = load_targets(args.targets_file) + if args.target not in targets_registry.get('targets', {}): + msg = f"Target {args.target!r} not found in targets.json" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f"ERROR: {msg}", file=sys.stderr) + return EXIT_PRECONDITION + + tcfg = targets_registry['targets'][args.target] + engine = tcfg.get('db_engine', 'postgresql').lower() + if engine not in ('postgresql', 'postgres', 'mariadb', 'mysql'): + msg = f"Unsupported db_engine: {engine}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f"ERROR: {msg}", file=sys.stderr) + return EXIT_PRECONDITION + is_postgres = engine in ('postgresql', 'postgres') + + # Host & Arch guard + cur_host = socket.gethostname() + cur_arch = platform.machine() + if os.environ.get('SKIP_HOST_GUARD') != '1': + req_host = tcfg.get('host') + req_arch = tcfg.get('arch') + if req_host and req_host != cur_host: + msg = f"Host mismatch: running on {cur_host}, target requires {req_host}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f"ERROR: {msg}", file=sys.stderr) + return EXIT_PRECONDITION + if req_arch and req_arch != cur_arch: + msg = f"Arch mismatch: running on {cur_arch}, target requires {req_arch}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f"ERROR: {msg}", file=sys.stderr) + return EXIT_PRECONDITION + + # Confirmation token guard + req_confirm = tcfg.get('require_confirm') + if req_confirm and args.confirm != req_confirm: + msg = f"Target {args.target} requires confirmation: --confirm {req_confirm}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f"ERROR: {msg}", file=sys.stderr) + return EXIT_PRECONDITION + + # Drop previous retained DB if requested + live_db = tcfg['db_name'] + if args.drop_previous: + expected_pattern = rf'^{re.escape(live_db)}_before_\d{{8}}_\d{{6}}$' + if not re.match(expected_pattern, args.drop_previous): + msg = f"Refused to drop database with unexpected name: {args.drop_previous}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + return EXIT_PRECONDITION + if is_postgres: + postgres_drop_db_if_exists(tcfg, args.drop_previous) + else: + mariadb_drop_db_if_exists(tcfg, args.drop_previous) + write_result(args.result, {'status': 'ok', 'exit_code': EXIT_OK, 'dropped_database': args.drop_previous}) + print(f"OK: Dropped previous database {args.drop_previous}") + return EXIT_OK + + if not args.dump: + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': '--dump is required'}) + return EXIT_PRECONDITION + + dump_path = args.dump.resolve() + sha_path = Path(str(dump_path) + '.sha256') + if not dump_path.is_file(): + msg = f"Dump file missing: {dump_path}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + return EXIT_PRECONDITION + + # Checksum validation if .sha256 file is present + if sha_path.is_file(): + expected_sha = sha_path.read_text(encoding='utf-8').strip().split()[0].lower() + actual_sha = compute_sha256(dump_path) + if expected_sha != actual_sha: + msg = f"SHA256 mismatch: expected {expected_sha}, got {actual_sha}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_CHECKSUM_MISMATCH, 'detail': msg}) + print(f"ERROR: {msg}", file=sys.stderr) + return EXIT_CHECKSUM_MISMATCH + + # Check if live DB has non-empty tables and --allow-overwrite is needed + check_tables = tcfg.get('verify_tables_not_empty', []) + if check_tables and not args.allow_overwrite: + tbl = check_tables[0] + cnt = 0 + if is_postgres: + r = psql_exec(tcfg, live_db, f'SELECT COUNT(*)::int FROM "{tbl}";') + cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0 + else: + r = mariadb_exec(tcfg, live_db, f'SELECT COUNT(*) FROM `{tbl}`;') + cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0 + if cnt > 0: + msg = f"Target database {live_db} contains {cnt} rows in table {tbl}; pass --allow-overwrite to replace it." + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f"ERROR: {msg}", file=sys.stderr) + return EXIT_PRECONDITION + + ts_label = dt.datetime.now().strftime('%Y%m%d_%H%M%S') + staging_db = f"{live_db}_restore_{ts_label}" + backup_db = f"{live_db}_before_{ts_label}" + + print(f"1. Creating staging database {staging_db} ({engine})...") + if is_postgres: + ok, err = postgres_create_db(tcfg, staging_db) + else: + ok, err = mariadb_create_db(tcfg, staging_db) + if not ok: + msg = f"Failed to create staging DB: {err}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + return EXIT_EXEC_ERROR + + print(f"2. Importing dump into {staging_db}...") + if is_postgres: + ok, err = postgres_import_dump(tcfg, staging_db, dump_path) + else: + ok, err = mariadb_import_dump(tcfg, staging_db, dump_path, live_db) + if not ok: + msg = f"Failed to import dump into staging DB: {err}" + if is_postgres: + postgres_drop_db_if_exists(tcfg, staging_db) + else: + mariadb_drop_db_if_exists(tcfg, staging_db) + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + return EXIT_EXEC_ERROR + + print("3. Validating staging database integrity...") + table_counts = {} + for tbl in tcfg.get('verify_tables_not_empty', []): + if is_postgres: + r = psql_exec(tcfg, staging_db, f'SELECT COUNT(*)::int FROM "{tbl}";') + else: + r = mariadb_exec(tcfg, staging_db, f'SELECT COUNT(*) FROM `{tbl}`;') + cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0 + table_counts[tbl] = cnt + if cnt == 0: + msg = f"Verification failed: required table {tbl} is empty in staging DB" + if is_postgres: + postgres_drop_db_if_exists(tcfg, staging_db) + else: + mariadb_drop_db_if_exists(tcfg, staging_db) + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + return EXIT_EXEC_ERROR + + # Invalidate / truncate session tables if requested + if args.sessions == 'clear': + for tbl in tcfg.get('truncate_tables', ['sessions']): + print(f"Truncating session table: {tbl}") + if is_postgres: + psql_exec(tcfg, staging_db, f'TRUNCATE TABLE "{tbl}" CASCADE;') + else: + mariadb_exec(tcfg, staging_db, f'TRUNCATE TABLE `{tbl}`;') + + services_to_stop = tcfg.get('services_to_stop', []) + if services_to_stop: + print(f"4. Stopping dependent services: {services_to_stop}...") + compose_cmd(tcfg, 'stop', *services_to_stop) + + print(f"5. Performing atomic database swap ({live_db} -> {backup_db}, {staging_db} -> {live_db})...") + if is_postgres: + swap_ok, swap_err = postgres_swap_databases(tcfg, live_db, staging_db, backup_db) + else: + swap_ok, swap_err = mariadb_swap_databases(tcfg, live_db, staging_db, backup_db) + + if not swap_ok: + msg = f"Database swap failed: {swap_err}" + if services_to_stop: + compose_cmd(tcfg, 'up', '-d', *services_to_stop) + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + return EXIT_EXEC_ERROR + + if services_to_stop: + print(f"6. Starting dependent services: {services_to_stop}...") + compose_cmd(tcfg, 'up', '-d', *services_to_stop) + + # Healthcheck / Readiness verification + health_url = tcfg.get('ready_url') or tcfg.get('health_url') + if health_url: + print(f"7. Verifying service readiness at {health_url}...") + timeout_sec = tcfg.get('health_timeout_seconds', 180) + ready_ok, ready_data = wait_ready(health_url, timeout_sec=timeout_sec) + if not ready_ok: + print(f"ERROR: Health check failed after {timeout_sec}s: {ready_data}. Initiating automatic rollback!", file=sys.stderr) + if services_to_stop: + compose_cmd(tcfg, 'stop', *services_to_stop) + if is_postgres: + postgres_rollback_swap(tcfg, live_db, staging_db, backup_db) + else: + mariadb_rollback_swap(tcfg, live_db, staging_db, backup_db) + if services_to_stop: + compose_cmd(tcfg, 'up', '-d', *services_to_stop) + msg = f"Post-restore healthcheck failed: {ready_data}. Automatic rollback executed." + write_result(args.result, {'status': 'error', 'exit_code': EXIT_POST_RESTORE_FAILED, 'detail': msg, 'health_data': ready_data}) + return EXIT_POST_RESTORE_FAILED + + summary = { + 'status': 'ok', + 'exit_code': EXIT_OK, + 'target': args.target, + 'db_engine': engine, + 'live_db': live_db, + 'previous_db': backup_db, + 'verified_tables': table_counts, + 'sessions': args.sessions, + 'completed_at': dt.datetime.now().isoformat(), + } + write_result(args.result, summary) + print(f"OK: Restore completed successfully for target {args.target} (previous DB: {backup_db})") + return EXIT_OK + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/targets.json b/targets.json index 0ac79af..b74212d 100644 --- a/targets.json +++ b/targets.json @@ -93,6 +93,69 @@ "frontend_ready_url": "http://127.0.0.1:13003/api/health/ready", "adapter_dir": "/etc/backup-orchestrator/scripts/shinkan", "services_to_stop": ["backend", "frontend"] + }, + "kairo-llm-prod": { + "application": "kairo", + "environment": "production", + "host": "llm-node", + "arch": "x86_64", + "role": "production_target", + "allow_restore": true, + "require_confirm": "kairo@llm-node", + "app_dir": "/home/llmadmin/docker/kairo-prod", + "compose_project": "kairo-prod", + "compose_files": ["docker-compose.yml"], + "env_file": "/etc/kairo/prod.env", + "db_engine": "postgresql", + "db_container": "kairo-db-prod", + "db_name": "kairo", + "db_user": "kairo_user", + "ready_url": "http://127.0.0.1:8001/api/health/ready", + "services_to_stop": ["backend"], + "verify_tables_not_empty": ["django_migrations"], + "truncate_tables": ["django_session"] + }, + "tandoor-llm-prod": { + "application": "tandoor", + "environment": "production", + "host": "llm-node", + "arch": "x86_64", + "role": "production_target", + "allow_restore": true, + "require_confirm": "tandoor@llm-node", + "app_dir": "/home/llmadmin/docker/tandoor-prod", + "compose_project": "tandoor-prod", + "compose_files": ["docker-compose.yml"], + "env_file": "/etc/tandoor/prod.env", + "db_engine": "postgresql", + "db_container": "tandoor-db-prod", + "db_name": "tandoordb", + "db_user": "tandoor_user", + "ready_url": "http://127.0.0.1:8080/", + "services_to_stop": ["web_app"], + "verify_tables_not_empty": ["django_migrations"], + "truncate_tables": ["django_session"] + }, + "bookstack-llm-prod": { + "application": "bookstack", + "environment": "production", + "host": "llm-node", + "arch": "x86_64", + "role": "production_target", + "allow_restore": true, + "require_confirm": "bookstack@llm-node", + "app_dir": "/home/llmadmin/docker/bookstack-prod", + "compose_project": "bookstack-prod", + "compose_files": ["docker-compose.yml"], + "env_file": "/etc/bookstack/prod.env", + "db_engine": "mariadb", + "db_container": "bookstack-db-prod", + "db_name": "bookstack", + "db_user": "bookstack_user", + "ready_url": "http://127.0.0.1:6875/", + "services_to_stop": ["bookstack"], + "verify_tables_not_empty": ["migrations"], + "truncate_tables": ["sessions"] } } } diff --git a/tests/test_universal_restore.py b/tests/test_universal_restore.py new file mode 100644 index 0000000..5bb9279 --- /dev/null +++ b/tests/test_universal_restore.py @@ -0,0 +1,127 @@ +"""Unit tests for scripts/common/universal-db-restore and generic restore engine.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +ROOT_DIR = Path(__file__).resolve().parents[1] +SCRIPT_PATH = ROOT_DIR / 'scripts' / 'common' / 'universal-db-restore' + + +class UniversalDbRestoreTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + self.targets_file = self.root / 'targets.json' + self.targets_data = { + 'schemaVersion': 1, + 'targets': { + 'test-pg-prod': { + 'application': 'testpg', + 'environment': 'production', + 'host': 'test-host', + 'arch': 'x86_64', + 'role': 'production_target', + 'require_confirm': 'testpg@test-host', + 'db_engine': 'postgresql', + 'db_container': 'test-pg-container', + 'db_name': 'testpgdb', + 'db_user': 'testpguser', + 'services_to_stop': ['backend'], + 'verify_tables_not_empty': ['users'], + 'truncate_tables': ['sessions'], + 'ready_url': 'http://127.0.0.1:8999/healthz', + }, + 'test-maria-prod': { + 'application': 'testmaria', + 'environment': 'production', + 'host': 'test-host', + 'arch': 'x86_64', + 'role': 'production_target', + 'require_confirm': 'testmaria@test-host', + 'db_engine': 'mariadb', + 'db_container': 'test-maria-container', + 'db_name': 'testmariadb', + 'db_user': 'testmariauser', + 'services_to_stop': ['web'], + 'verify_tables_not_empty': ['posts'], + 'truncate_tables': ['sessions'], + }, + }, + } + self.targets_file.write_text(json.dumps(self.targets_data, indent=2), encoding='utf-8') + + def test_missing_target_fails_precondition(self): + result_file = self.root / 'result.json' + cmd = [ + sys.executable, str(SCRIPT_PATH), + '--target', 'unknown-target', + '--result', str(result_file), + '--targets-file', str(self.targets_file), + ] + res = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(res.returncode, 2) # EXIT_PRECONDITION + data = json.loads(result_file.read_text(encoding='utf-8')) + self.assertEqual(data['status'], 'error') + self.assertIn('not found', data['detail']) + + def test_confirmation_guard(self): + dump_path = self.root / 'test.sql' + dump_path.write_text('SELECT 1;', encoding='utf-8') + result_file = self.root / 'result.json' + + env = os.environ.copy() + env['SKIP_HOST_GUARD'] = '1' + + # Without confirmation + cmd = [ + sys.executable, str(SCRIPT_PATH), + '--target', 'test-pg-prod', + '--dump', str(dump_path), + '--result', str(result_file), + '--targets-file', str(self.targets_file), + ] + res = subprocess.run(cmd, env=env, capture_output=True, text=True) + self.assertEqual(res.returncode, 2) + data = json.loads(result_file.read_text(encoding='utf-8')) + self.assertIn('requires confirmation', data['detail']) + + # With wrong confirmation + cmd_wrong = cmd + ['--confirm', 'wrong-token'] + res_wrong = subprocess.run(cmd_wrong, env=env, capture_output=True, text=True) + self.assertEqual(res_wrong.returncode, 2) + + def test_sha256_verification(self): + dump_path = self.root / 'test.sql' + dump_path.write_text('SELECT 1;', encoding='utf-8') + sha_path = self.root / 'test.sql.sha256' + sha_path.write_text('0' * 64 + ' test.sql\n', encoding='utf-8') + result_file = self.root / 'result.json' + + env = os.environ.copy() + env['SKIP_HOST_GUARD'] = '1' + + cmd = [ + sys.executable, str(SCRIPT_PATH), + '--target', 'test-pg-prod', + '--dump', str(dump_path), + '--confirm', 'testpg@test-host', + '--result', str(result_file), + '--targets-file', str(self.targets_file), + ] + res = subprocess.run(cmd, env=env, capture_output=True, text=True) + self.assertEqual(res.returncode, 4) # EXIT_CHECKSUM_MISMATCH + data = json.loads(result_file.read_text(encoding='utf-8')) + self.assertEqual(data['status'], 'error') + self.assertIn('SHA256 mismatch', data['detail']) + + +if __name__ == '__main__': + unittest.main()