From c53fc0a9399d09b0c89b7e1c7d7ba0f72dd94a9b Mon Sep 17 00:00:00 2001 From: Lars Date: Thu, 29 Oct 2026 14:18:20 +0100 Subject: [PATCH] feat(v0.4): import all Raspberry Pi scripts and implement Recovery Management engine + Shinkan adapters --- .gitattributes | 1 + .gitignore | 1 + README.md | 236 ++--- orchestrator.py | 991 +++++++++++++++++- scripts/db-bookstack.sh | 4 + scripts/db-buyingagent-dev.sh | 4 + scripts/db-buyingagent-prod.sh | 4 + scripts/db-kairo-dev.sh | 4 + scripts/db-kairo-prod.sh | 4 + scripts/db-kansho-dev.sh | 4 + scripts/db-kansho-prod.sh | 4 + scripts/db-karatewiki-dev.sh | 4 + scripts/db-mitai-dev.sh | 8 + scripts/db-mitai-prod.sh | 6 + scripts/db-netbox.sh | 4 + scripts/db-paperless.sh | 54 + scripts/db-shinkan-dev.sh | 6 + scripts/db-shinkan-prod.sh | 6 + scripts/db-tandoor.sh | 53 + scripts/db-tsumugu-dev.sh | 4 + scripts/db-tsumugu-prod.sh | 4 + scripts/nas-data-job.sh | 5 + scripts/publish-backup-set.sh | 66 ++ scripts/shinkan-prod-manifest.sh | 21 + .../shinkan/docker-compose.restoretest.yml | 73 ++ scripts/shinkan/shinkan-backup-manifest | 354 +++++++ scripts/shinkan/shinkan-media-diff | 243 +++++ scripts/shinkan/shinkan-restore | 389 +++++++ scripts/shinkan/shinkan-restore-test | 136 +++ systemd/backup-orchestrator-daily | 10 + systemd/backup-orchestrator-daily.service | 9 + systemd/backup-orchestrator-daily.timer | 10 + targets.json | 98 ++ tests/test_orchestrator.py | 4 +- tests/test_recovery.py | 244 +++++ 35 files changed, 2877 insertions(+), 191 deletions(-) create mode 100644 scripts/db-bookstack.sh create mode 100644 scripts/db-buyingagent-dev.sh create mode 100644 scripts/db-buyingagent-prod.sh create mode 100644 scripts/db-kairo-dev.sh create mode 100644 scripts/db-kairo-prod.sh create mode 100644 scripts/db-kansho-dev.sh create mode 100644 scripts/db-kansho-prod.sh create mode 100644 scripts/db-karatewiki-dev.sh create mode 100644 scripts/db-mitai-dev.sh create mode 100644 scripts/db-mitai-prod.sh create mode 100644 scripts/db-netbox.sh create mode 100644 scripts/db-paperless.sh create mode 100644 scripts/db-shinkan-dev.sh create mode 100644 scripts/db-shinkan-prod.sh create mode 100644 scripts/db-tandoor.sh create mode 100644 scripts/db-tsumugu-dev.sh create mode 100644 scripts/db-tsumugu-prod.sh create mode 100644 scripts/nas-data-job.sh create mode 100644 scripts/publish-backup-set.sh create mode 100644 scripts/shinkan-prod-manifest.sh create mode 100644 scripts/shinkan/docker-compose.restoretest.yml create mode 100644 scripts/shinkan/shinkan-backup-manifest create mode 100644 scripts/shinkan/shinkan-media-diff create mode 100644 scripts/shinkan/shinkan-restore create mode 100644 scripts/shinkan/shinkan-restore-test create mode 100644 systemd/backup-orchestrator-daily create mode 100644 systemd/backup-orchestrator-daily.service create mode 100644 systemd/backup-orchestrator-daily.timer create mode 100644 targets.json create mode 100644 tests/test_recovery.py diff --git a/.gitattributes b/.gitattributes index 022d8c3..57828ab 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,4 @@ *.csv text eol=lf *.json text eol=lf *.md text eol=lf + diff --git a/.gitignore b/.gitignore index cd86ab3..d742c0e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ __pycache__/ notifications.json host.local.json .env + diff --git a/README.md b/README.md index ac19795..90e5178 100644 --- a/README.md +++ b/README.md @@ -1,161 +1,93 @@ -# Raspberry Pi 5 – Backup-Orchestrator v0.3 +# Backup-Orchestrator v0.4 – Backup & Recovery Management -**Status: Test- und Integrationsversion.** Keine produktiven Backups, Timer, Crontabs oder Datenbanken werden durch Installation, `--check`, `--plan` oder `--status` verändert. Die vorhandenen Backup-Skripte sind **noch nicht vollständig standardisiert**, zusätzliche Datenbank-Jobs fehlen und eine vollständige Systemwiederherstellung ist noch nicht getestet. `--run` bleibt bewusst mehrfach gesperrt. +**Status: Version 0.4.0 (Multi-Host Backup & Application Recovery Management).** +Erweitert den auf `raspberrypi5` produktiv laufenden Backup-Orchestrator v0.3 um ein integriertes Recovery Management, kryptografisch geprüfte Recovery Points (`schemaVersion: 1`), ein 4-stufiges Verifikationsmodell (`captured` → `verified` → `consistent` → `restore_tested`), einen zustandsbewussten `raspiBackup.service`-Konflikt-Check sowie vollständige Application-Restore- und Restore-Test-Adapter für die Referenzanwendung **Shinkan** auf `llm-node` (`192.168.2.144`). -## 1. Enthaltene Dateien +--- -- `orchestrator.py`: phasenbasierter Python-Controller (Python 3.11 / Debian 12, nur Standardbibliothek). -- `jobs.csv`: bestehende acht Jobs. **Alle `enabled=false`** und verweisen derzeit auf die alten, vorhandenen Skriptpfade. -- `notifications.json.example`: eine zentrale Konfigurationsdatei für ausgehende HTTP-POST-Webhooks und Uptime-Kuma-Push-Monitore; keine echten Secrets. -- `tests/test_orchestrator.py`: Tests mit künstlichen Shellskripten und simulierten HTTP-Antworten; niemals produktive Datenbanken oder das NAS. +## 1. Repository-Struktur & Enthaltene Dateien -## 2. Sichere Installation auf raspberrypi5 +- `orchestrator.py`: Phasenbasierter Backup- & Recovery-Controller (`v0.4.0`, Python 3.11+, nur Standardbibliothek). +- `targets.json`: Multi-Host- und Recovery-Target-Definitionen (`raspberrypi5`, `llm-node`, Targets `pi-prod`, `llm-prod`, `llm-restoretest`). +- `jobs.csv`: Produktive Backup-Job-Tabelle (17 Datenbank-Exporte in Phase 10, `nas-data` in Phase 20, `system-image` in Phase 30). +- `notifications.json.example`: Beispielkonfiguration für ausgehende HTTP-POST-Webhooks und Uptime-Kuma-Push-Monitore (enthält keine echten Secrets). +- `systemd/`: + - `backup-orchestrator-daily`: Täglicher Runner-Wrapper (ab v0.4 ohne fragiles `systemctl disable/enable raspiBackup.timer`). + - `backup-orchestrator-daily.service` & `backup-orchestrator-daily.timer`: Systemd-Units für den täglichen Lauf um `05:30` Uhr. +- `scripts/`: + - Alle 17 Datenbank-Dump-Wrapper (`db-bookstack.sh`, `db-buyingagent-dev.sh`, `db-buyingagent-prod.sh`, `db-kairo-dev.sh`, `db-kairo-prod.sh`, `db-kansho-dev.sh`, `db-kansho-prod.sh`, `db-karatewiki-dev.sh`, `db-mitai-dev.sh`, `db-mitai-prod.sh`, `db-netbox.sh`, `db-paperless.sh`, `db-shinkan-dev.sh`, `db-shinkan-prod.sh`, `db-tandoor.sh`, `db-tandoor.sh`, `db-tsumugu-dev.sh`, `db-tsumugu-prod.sh`) sowie die generischen Dump-Engines `db-postgres-docker.sh` und `db-mariadb-docker.sh`. + - `backup-nas-data.sh` & `nas-data-job.sh`: Versionierte rsync-Hardlink-Sicherung (`/media/lars/paperless` → `/backup/data_archiv/01..31` + `last`). + - `publish-backup-set.sh` & `shinkan-prod-manifest.sh`: Atomare Erzeugung und Publikation vollständiger Backup-Sets (`.sql`, `.sql.sha256`, `.sql.manifest.json`, `.sql.media.jsonl`). +- `scripts/shinkan/` (Shinkan Recovery Contract Adapter): + - `shinkan-backup-manifest`: Erzeugt SHA256, Manifest (`schemaVersion: 1` + Shinkan Contract) und inkrementelles Media-Inventar (`media.jsonl`). + - `shinkan-restore`: 9-stufiger atomarer Application-Restore über Staging-DB (`shinkan_restore_`), Single-Transaction-Import, Schema-Migrations-Gate, Plausibilitätsprüfung, `TRUNCATE sessions`, DB-Swap (`shinkan_before_`), `/api/health/ready`-Prüfung und automatischem Rollback bei Fehlern (Exit `5`). + - `shinkan-media-diff`: Abgleich zwischen PostgreSQL (`media_assets`, `exercise_media`), Dateisystem und `media.jsonl` (`missing`, `mismatch`, `added_after_backup`, `orphan`). + - `shinkan-restore-test` & `docker-compose.restoretest.yml`: Isolierter End-to-End-Restore-Test im Projekt `shinkan-restoretest` (Ports `127.0.0.1:18003` / `127.0.0.1:13003`, Read-Only-Media `:ro`, deaktivierte externe Integrationen & Cronjobs). +- `docs/ARCHITECTURE_AND_RECOVERY_PLAN_v0.4.md`: Vollständige Ist-Inventur, Gap-Analyse und Architektur-Spezifikation für v0.4. +- `tests/`: + - `test_orchestrator.py`: Unit-Tests für Backup-Phasen, Parallelität, Timeouts, Mount-Prüfung und Notifications. + - `test_recovery.py`: Unit- und Integrationstests für Manifest-Validierung, 4-Stufen-Modell, Schema-Kompatibilitätsprüfung (`EXACT_MATCH`, `FORWARD_MIGRATE`, `INCOMPATIBLE_ROLLBACK`), Sicherheits-Gates (`pi-prod` Sperre, `--confirm`), Dry-Run-Pläne und `shinkan-backup-manifest`. -ZIP auf den Raspberry Pi übertragen und dort entpacken. Im entpackten Verzeichnis: +--- + +## 2. CLI-Befehle (v0.4.0) + +### A. Backup-Steuerung (unverändert kompatibel zu v0.3) + +```bash +backup-orchestrator --check +backup-orchestrator --plan +backup-orchestrator --status --json +BACKUP_ORCHESTRATOR_CUTOVER_APPROVED=YES backup-orchestrator --run +BACKUP_ORCHESTRATOR_CUTOVER_APPROVED=YES backup-orchestrator --run --phase 10 +BACKUP_ORCHESTRATOR_CUTOVER_APPROVED=YES backup-orchestrator --run --job shinkan-prod-db +``` + +### B. Recovery Management (neu in v0.4) + +```bash +# 1. Verfügbare Recovery Points auflisten (inkl. 4-Stufen-Status) +backup-orchestrator --list-recovery-points +backup-orchestrator --list-recovery-points --app shinkan --json + +# 2. Einzelnen Recovery Point kryptografisch & strukturell prüfen +backup-orchestrator --verify-recovery-point \ + --point /mnt/nas_backup/shinkan/prod/db/latest/shinkan-prod.sql + +# 3. Dry-Run Recovery-Plan erstellen (inkl. Schema-Kompatibilitätsprüfung) +backup-orchestrator --recovery-plan \ + --point /mnt/nas_backup/shinkan/prod/db/latest/shinkan-prod.sql \ + --target llm-prod + +# 4. Isolierten Restore-Test ausführen (schreibgeschütztes Media-Mount, Ports 18003/13003) +backup-orchestrator --restore-test \ + --point /mnt/nas_backup/shinkan/prod/db/latest/shinkan-prod.sql \ + --target llm-restoretest + +# 5. Produktiver Application-Restore / Erstbefüllung auf llm-node +backup-orchestrator --restore \ + --point /mnt/nas_backup/shinkan/prod/db/latest/shinkan-prod.sql \ + --target llm-prod \ + --confirm shinkan@llm-node +``` + +--- + +## 3. Exit-Codes im Recovery-Modus + +| Exit-Code | Bedeutung | +|---|---| +| `0` | Erfolgreich (`verified`/`consistent`/`restore_tested` bzw. Restore + `/api/health/ready` grün) | +| `2` | Ungültige Argumente, fehlendes `--confirm`, gesperrtes Ziel (`pi-prod` ist `source_only`) oder Mount/Pfad fehlt | +| `3` | Schema-/Versions-Inkompatibilität (`INCOMPATIBLE_ROLLBACK`: Dump enthält Migrationen, die im Ziel-Code fehlen) | +| `4` | Integritätsfehler (SHA256-Prüfsumme weicht ab, Dump leer/korrupt oder Pflicht-Tabellen nach Staging-Import leer) | +| `5` | Health-Check (`/api/health/ready`) nach DB-Umschaltung fehlgeschlagen – **automatisches Rollback auf `shinkan_before_` durchgeführt** | + +--- + +## 4. Tests ausführen ```bash python3 -m unittest discover -s tests -v -sudo cp -a /usr/local/sbin/backup-orchestrator /usr/local/sbin/backup-orchestrator.before-v03 -sudo install -m 755 orchestrator.py /usr/local/sbin/backup-orchestrator - -# Die vorhandene /etc/backup-orchestrator/jobs.csv NICHT überschreiben. -# Die Benachrichtigungskonfiguration nur bei erstmaliger Einrichtung kopieren. -sudo install -d -m 750 /etc/backup-orchestrator -if ! sudo test -e /etc/backup-orchestrator/notifications.json; then - sudo install -m 600 notifications.json.example /etc/backup-orchestrator/notifications.json -fi -sudo backup-orchestrator --check ``` - -Ohne bereits installierte ältere Version entfällt die Zeile `sudo cp -a ...before-v03`. - -> **Keine `--run`-Befehle ausführen**, solange die alten Cronjobs und der raspiBackup-Timer aktiv sind und die alten Dump-Skripte noch nicht gehärtet wurden. Den Produktiv-Cutover führen wir später kontrolliert durch. - -## 3. Konfiguration der Phasen - -Die vorhandene Tabelle unter `/etc/backup-orchestrator/jobs.csv` hat **keine Uhrzeiten**: - -```csv -phase,id,script,critical,timeout_seconds,enabled -10,paperless-db,/home/lars/db-backup.sh,true,600,false -10,tandoor-db,/home/lars/db-Tandoor.sh,true,600,false -20,nas-data,/home/lars/backupPaperlessData.sh,true,10800,false -30,system-image,/usr/local/bin/raspiBackup.sh,true,14400,false -``` - -Die Beispiele sind ein Ausschnitt; die mitgelieferte Datei enthält acht Jobs. - -- **Innerhalb derselben Phase** laufen bis zu drei Jobs gleichzeitig (per `--max-parallel N` einstellbar). -- **Zwischen Phasen** gilt eine Barriere: Die nächste Phase beginnt erst, wenn die aktuelle vollständig abgeschlossen ist. -- Ein **kritischer Fehler** blockiert nachfolgende Phasen; laufende Jobs derselben Phase werden beendet bzw. bis zum Abschluss abgewartet. -- Ein **unkritischer Fehler** wird protokolliert und führt zu einem fehlgeschlagenen Gesamtergebnis, blockiert aber die nächste Phase nicht. -- Zeitüberschreitung beendet die Skript-Prozessgruppe; Service-Neustart und Datenbankkonsistenz müssen in den jeweiligen Jobs abgesichert sein. - -Zentrale Zeitsteuerung per *einem* systemd-Timer ist eine spätere Cutover-Aufgabe, **nicht Bestandteil der Installation**. - -## 4. Externe Schnittstellen - -### A. Ausgehende Webhooks, HTTP POST - -`/etc/backup-orchestrator/notifications.json` bearbeiten: - -```json -{ - "timeout_seconds": 3, - "webhooks": ["http://192.168.2.100:5678/webhook/backup-events"], - "kuma": { - "overall_push_url": "http://192.168.2.63:3010/api/push/GESAMT_MONITOR_TOKEN", - "job_push_urls": { - "nas-data": "http://192.168.2.63:3010/api/push/NAS_MONITOR_TOKEN", - "paperless-db": "http://192.168.2.63:3010/api/push/PAPERLESS_MONITOR_TOKEN" - } - } -} -``` - -Alle URLs sind Beispiele – vorhandene echte Push-Tokens nur auf dem Pi eintragen. Einträge mit `""` sind deaktiviert. Es gibt **keinen HTTP-Listener und keinen permanent laufenden Webserver**; Webhooks sind *ausgehend*. - -Die echten URLs können auch komplett aus der Uptime-Kuma-Oberfläche kopiert werden (`...?status=up&msg=OK&ping=`). Diese Beispiel-Query wird beim Laden entfernt und beim Push durch den tatsächlichen Status ersetzt. - -Diese Datei enthält Zugriffstokens: `sudo chmod 600 /etc/backup-orchestrator/notifications.json`. Sie gehört in eine vertrauliche Konfigurationssicherung, nicht in ein öffentliches Repository. - -### B. Ereignisse und Beispiele - -Alle Ereignisse werden als **JSON je Zeile** in `/var/log/backup-orchestrator//events.jsonl` geschrieben; jede Zeile wird auch an die konfigurierten POST-Webhooks gesendet. - -- `RUN_STARTED`, `PHASE_STARTED`, `JOB_STARTED` -- `JOB_SUCCESS`, `JOB_FAILED`, `JOB_TIMEOUT`, `JOB_ERROR`, `JOB_CANCELLED`, `JOB_BLOCKED` -- `PHASE_COMPLETED`, `PHASE_BLOCKED`, `RUN_FINISHED` - -Beispiel einer Webhook-Nachricht (ohne Geheimnisse): - -```json -{ - "event": "JOB_FAILED", - "timestamp": "2026-10-29T01:02:00+01:00", - "run_id": "20261029-010000-1234", - "host": "raspberrypi5", - "phase": 10, - "job": "shinkan-prod-db", - "critical": true, - "state": "failed", - "exit_code": 1, - "duration_seconds": 12.3, - "log": "/var/log/backup-orchestrator/20261029-010000-1234/shinkan-prod-db.log" -} -``` - -Webhooks versenden **Ereignismetadaten**, nicht den Inhalt der Backup-Skripte oder SQL-Dumps. - -### C. Uptime Kuma - -- `JOB_SUCCESS` → `status=up` für den konfigurierten Job-Monitor. -- `JOB_FAILED`, `JOB_TIMEOUT`, `JOB_ERROR`, `JOB_CANCELLED` und `JOB_BLOCKED` → `status=down`. -- `RUN_FINISHED` → `status=up` nur beim Gesamtergebnis `ok`, sonst `down`. -- Ausbleibende Backups meldet Kuma über den konfigurierten Push-Heartbeat, z.B. bei einem täglichen Lauf ein zum Zeitplan passender Wert. -- Schlägt die Zustellung fehl, schreibt der Controller eine Warnung in `notifications.log`. Der Backup-Exit-Code bleibt unverändert. - -**Keine doppelten Push-Meldungen erzeugen:** Wenn `nas-data` künftig durch den Orchestrator gemeldet wird, die Push-Funktion im Skript `backupPaperlessData.sh` deaktivieren oder nur einen der beiden Sender verwenden. - -### D. Lokaler Status, JSON-Abfrage - -Während des Laufs wird der aktuelle Stand atomar aktualisiert: - -```text -/var/log/backup-orchestrator/latest.json -/var/log/backup-orchestrator//status.json -/var/log/backup-orchestrator//summary.json -/var/log/backup-orchestrator//events.jsonl -``` - -CLI für andere Skripte, Home Assistant oder eine spätere GUI: - -```bash -sudo backup-orchestrator --status -sudo backup-orchestrator --status --json -sudo cat /var/log/backup-orchestrator/latest.json -``` - -Die Statusdateien enthalten keine Push-Tokens oder Zugangsdaten, aber interne Pfade und Jobnamen. Daher nur für vertrauenswürdige lokale Nutzer freigeben. Es gibt derzeit **keine eingehende REST-API**; dies kann später als separater schreibgeschützter Dienst ergänzt werden. - -## 5. Sichere Bedienung - -```bash -sudo backup-orchestrator --check # nur Prüfungen, keine Backups -sudo backup-orchestrator --plan # nur aktivierte Jobs anzeigen -sudo backup-orchestrator --status --json # letzter Status bzw. no_runs -``` - -`--run`, `--run --phase 10` und `--run --job paperless-db` sind für einen **späteren**, explizit kontrollierten Produktiv-Cutover vorgesehen. Der Controller verweigert Ausführung, solange keine Jobs aktiviert sind, der NAS-Mount fehlt, aktive alte Zeitpläne gefunden werden oder `BACKUP_ORCHESTRATOR_CUTOVER_APPROVED=YES` fehlt. Das stellt keine vollständige Sicherheitsfreigabe dar: Skripte und Restore müssen zusätzlich getestet werden. - -## 6. Offene Arbeiten für den Produktivbetrieb - -1. Vorhandene Datenbank-Dump-Skripte unter `scripts/` konsolidieren und auf atomare Dumps unter festen Dateinamen umstellen; Zugänge sicher verwahren. -2. Weitere Datenbanken (Kairo, Tsumugu, Kanshō, Buying Agent, BookStack, NetBox usw.) und zu sichernde Docker-Volumes/Bind-Mounts aufnehmen. -3. rsync-Skript prüfen: nas-data sendet bei der Umstellung **nicht** gleichzeitig selbst und über Orchestrator Pushs. -4. raspiBackup: zuverlässiger vollständiger Restore, korrekt gestoppte Docker-Dienste, Restart-/Timeout-Verhalten nach Fehler. -5. Alte Cronjobs/Timer **erst beim kontrollierten Cutover** ersetzen und einen einzigen orchestrator-Timer installieren. -6. Vollständigen Testlauf, Restore-Test und erwartete Kuma-Ereignisse prüfen. -7. Zweiten unabhängigen/offsite Backup-Standort erwägen; NAS-Ausfall bleibt sonst Single Point of Failure. - -**Hinweis zur bisherigen Zeitplanung:** Die Root-Crontab-Einträge sind nach der letzten Nutzerprüfung auskommentiert; die aktive User-Crontab sowie `raspiBackup.timer` werden weiterhin als aktive Konflikte erkannt. +Alle Tests arbeiten ausschließlich in temporären Verzeichnissen mit synthetischen Dumps und berühren niemals produktive Datenbanken oder NAS-Mounts. diff --git a/orchestrator.py b/orchestrator.py index 31f75fa..09096dd 100644 --- a/orchestrator.py +++ b/orchestrator.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -"""Phase-based backup orchestrator v0.3. +"""Phase-based backup and recovery orchestrator v0.4. -Safety defaults: read-only inspection unless --run, jobs enabled and explicit cutover -approval. No scheduler, database or Docker changes are performed by this program. +Safety defaults: read-only inspection unless --run, --restore, or --restore-test is +explicitly requested with all required safety gates and confirmations. """ import argparse @@ -10,10 +10,15 @@ from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed import csv import datetime as dt -import fcntl +try: + import fcntl +except ImportError: # Windows local test compatibility + fcntl = None +import hashlib import json import os from pathlib import Path +import platform import re import signal import socket @@ -24,9 +29,11 @@ import time from urllib.parse import urlencode, urlsplit, urlunsplit from urllib.request import Request, urlopen -VERSION = '0.3.0' +VERSION = '0.4.0' DEFAULT_CONFIG = Path('/etc/backup-orchestrator/jobs.csv') DEFAULT_NOTIFY_CONFIG = Path('/etc/backup-orchestrator/notifications.json') +DEFAULT_TARGETS_CONFIG = Path('/etc/backup-orchestrator/targets.json') +DEFAULT_HOST_CONFIG = Path('/etc/backup-orchestrator/host.json') DEFAULT_LOG_ROOT = Path('/var/log/backup-orchestrator') DEFAULT_LOCK_FILE = Path('/run/lock/backup-orchestrator.lock') FIELDS = {'phase', 'id', 'script', 'critical', 'timeout_seconds', 'enabled'} @@ -35,11 +42,59 @@ NAS_MOUNT = '/backup' NAS_SOURCE = '192.168.2.63:/volume1/Backup_Paperless' CANCELLED = threading.Event() +# Exit codes aligned with Shinkan RECOVERY_CONTRACT.md +EXIT_OK = 0 +EXIT_EXEC_ERROR = 1 +EXIT_PRECONDITION = 2 +EXIT_LOCK_HELD = 3 +EXIT_SCHEMA_INCOMPATIBLE = 3 +EXIT_CHECKSUM_MISMATCH = 4 +EXIT_POST_RESTORE_FAILED = 5 + +SECRET_PATTERNS = re.compile( + r'(?:PGPASSWORD|MYSQL_PWD|DB_PASSWORD|MARIADB_PASSWORD|MYSQL_PASSWORD|SMTP_PASS|OPENROUTER_API_KEY)\s*=\s*\S+', + re.IGNORECASE, +) + def now_iso(): return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds') +def lock_exclusive(fp): + if fcntl is not None: + fcntl.flock(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) + + +def load_host_config(path=DEFAULT_HOST_CONFIG): + default = { + 'nas_mount': NAS_MOUNT, + 'nas_source': NAS_SOURCE, + 'nas_fstype': 'nfs4', + 'local_dump_dir': '/media/lars/paperless/db-backup', + 'archive_sets_dir': '/backup/recovery_points', + 'conflict_services': ['raspiBackup.service'], + 'conflict_timers': ['paperless-data-backup.timer', 'borgmatic.timer'], + 'cron_users': ['root', 'lars'], + } + if path and Path(path).is_file(): + with Path(path).open(encoding='utf-8') as fp: + data = json.load(fp) + if isinstance(data, dict): + default.update(data) + return default + + +def load_targets_config(path=DEFAULT_TARGETS_CONFIG): + candidates = [Path(path)] if path else [] + candidates.append(Path(__file__).resolve().parent / 'targets.json') + for candidate in candidates: + if candidate.is_file(): + with candidate.open(encoding='utf-8') as fp: + return json.load(fp) + return {'schemaVersion': 1, 'hosts': {}, 'targets': {}} + + def load_jobs(path): with path.open(newline='', encoding='utf-8-sig') as fp: reader = csv.DictReader(fp) @@ -64,7 +119,7 @@ def load_jobs(path): raise ValueError(f'CSV line {index}: phase must be a positive integer') if not re.fullmatch(r'\d+', row['timeout_seconds']) or not 1 <= int(row['timeout_seconds']) <= 86400: raise ValueError(f'CSV line {index}: timeout_seconds must be 1..86400') - if not Path(row['script']).is_absolute(): + if not Path(row['script']).startswith_slash() if hasattr(Path(row['script']), 'startswith_slash') else not (row['script'].startswith('/') or Path(row['script']).is_absolute()): raise ValueError(f'CSV line {index}: script must be an absolute path') jobs.append({ 'phase': int(row['phase']), 'id': jid, 'script': row['script'], @@ -135,17 +190,22 @@ def notification_counts(config): return len(config['webhooks']), int(bool(kuma['overall_push_url'])), sum(bool(v) for v in kuma['job_push_urls'].values()) -def nas_mount_ready(): +def nas_mount_ready(mountpoint=None, source=None, fstype=None): + mountpoint = mountpoint or NAS_MOUNT + source = source or NAS_SOURCE + fstype = fstype or 'nfs4' try: - result = subprocess.run(['findmnt', '--mountpoint', NAS_MOUNT, '--types', 'nfs4', '--source', NAS_SOURCE], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=8) + result = subprocess.run( + ['findmnt', '--mountpoint', mountpoint, '--types', fstype, '--source', source], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=8, + ) return result.returncode == 0 except (OSError, subprocess.TimeoutExpired): return False def legacy_schedules(): - """Best-effort detection of active legacy schedules (never comments).""" + """Stateful detection of active legacy schedules and conflicting running services.""" found = [] for user in ('root', 'lars'): try: @@ -158,7 +218,17 @@ def legacy_schedules(): continue if re.search(r'/home/lars/(?:db-[\w-]+\.sh|backupPaperlessData\.sh)', stripped): found.append(f'{user} crontab: {stripped}') - for timer in ('raspiBackup.timer', 'paperless-data-backup.timer', 'borgmatic.timer'): + # Stateful coordination with raspiBackup: only block if raspiBackup.service is currently active/running. + try: + rb_active = subprocess.run( + ['systemctl', 'is-active', 'raspiBackup.service'], + capture_output=True, text=True, timeout=8, + ).stdout.strip() + if rb_active in ('active', 'activating', 'reloading'): + found.append(f'raspiBackup.service: active={rb_active}') + except (OSError, subprocess.TimeoutExpired): + pass + for timer in ('paperless-data-backup.timer', 'borgmatic.timer'): try: enabled = subprocess.run(['systemctl', 'is-enabled', timer], capture_output=True, text=True, timeout=8).stdout.strip() active = subprocess.run(['systemctl', 'is-active', timer], capture_output=True, text=True, timeout=8).stdout.strip() @@ -174,7 +244,7 @@ def script_issues(job): if not p.is_file(): return ['MISSING'] issues = [] - if not os.access(p, os.X_OK): + if os.name != 'nt' and not os.access(p, os.X_OK): issues.append('NOT EXECUTABLE') try: with p.open('rb') as fp: @@ -209,12 +279,12 @@ def check(jobs, max_parallel, notify): def select_jobs(jobs, args): enabled = [j for j in jobs if j['enabled']] - if args.job: + if getattr(args, 'job', None): matches = [j for j in enabled if j['id'] == args.job] if not matches: raise ValueError(f'Job {args.job!r} not found or disabled') return matches - if args.phase is not None: + if getattr(args, 'phase', None) is not None: matches = [j for j in enabled if j['phase'] == args.phase] if not matches: raise ValueError(f'No enabled jobs in phase {args.phase}') @@ -245,6 +315,7 @@ def result_for(job, state, rc=None, seconds=0.0, logfile=None, reason=None): def atomic_json(path, data): + path = Path(path) tmp = path.with_name(f'.{path.name}.{os.getpid()}.{threading.get_ident()}.tmp') try: with tmp.open('w', encoding='utf-8') as fp: @@ -269,7 +340,7 @@ class Notifier: def _send_json(self, url, event): payload = json.dumps(event, ensure_ascii=False).encode('utf-8') - req = Request(url, data=payload, method='POST', headers={'Content-Type': 'application/json', 'User-Agent': 'pi-backup-orchestrator/0.3'}) + req = Request(url, data=payload, method='POST', headers={'Content-Type': 'application/json', 'User-Agent': f'pi-backup-orchestrator/{VERSION}'}) with urlopen(req, timeout=self.config['timeout_seconds']) as response: if not 200 <= response.status < 300: raise RuntimeError(f'HTTP {response.status}') @@ -277,11 +348,10 @@ class Notifier: def _send_kuma(self, url, status, message): sep = '&' if '?' in url else '?' address = url + sep + urlencode({'status': status, 'msg': message[:180]}) - req = Request(address, headers={'User-Agent': 'pi-backup-orchestrator/0.3'}) + req = Request(address, headers={'User-Agent': f'pi-backup-orchestrator/{VERSION}'}) with urlopen(req, timeout=self.config['timeout_seconds']) as response: if not 200 <= response.status < 300: raise RuntimeError(f'HTTP {response.status}') - # Kuma usually responds with JSON {ok:true}; a 200 alone is insufficient if it says failure. body = response.read(4096) if body: try: @@ -296,7 +366,6 @@ class Notifier: try: fn(*args) except Exception as exc: - # Never log target URLs or tokens; even HTTP exceptions may include their URL. if hasattr(exc, 'code') and isinstance(exc.code, int): detail = f'HTTP {exc.code}' else: @@ -317,7 +386,7 @@ class Notifier: status = 'up' if name == 'JOB_SUCCESS' else 'down' msg = f"{event['job']}: {event.get('state', 'unknown')} (run {event['run_id']})" self._submit('kuma/job', self._send_kuma, url, status, msg) - if name == 'RUN_FINISHED': + if name in ('RUN_FINISHED', 'RECOVERY_FINISHED'): url = self.config['kuma']['overall_push_url'] if url: status = 'up' if event.get('overall') == 'ok' else 'down' @@ -371,18 +440,45 @@ def terminate_group(proc): """Terminate ordinary children of a job and wait for shutdown before advancing phase.""" if proc.poll() is not None: return - try: - os.killpg(proc.pid, signal.SIGTERM) - except ProcessLookupError: - return - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: + if hasattr(os, 'killpg'): try: - os.killpg(proc.pid, signal.SIGKILL) + os.killpg(proc.pid, signal.SIGTERM) except ProcessLookupError: + return + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + else: + subprocess.run(['taskkill', '/F', '/T', '/PID', str(proc.pid)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=5) + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +def _build_script_cmd(script_path, extra_args=()): + if os.name == 'nt': + import shutil + p = Path(script_path) + first_line = '' + try: + with p.open('r', encoding='utf-8', errors='replace') as fp: + first_line = fp.readline() + except OSError: pass - proc.wait() + if 'python' in first_line: + return [sys.executable, str(p), *extra_args] + if first_line.startswith('#!') or p.suffix == '.sh': + sh_bin = shutil.which('sh') or r'C:\Program Files\Git\bin\sh.exe' + return [sh_bin, str(p), *extra_args] + return [str(script_path), *extra_args] def execute_one(job, folder, hub): @@ -397,7 +493,7 @@ def execute_one(job, folder, hub): stream.write((f"job={job['id']} started={now_iso()}\n").encode()) stream.flush() try: - proc = subprocess.Popen([job['script']], stdout=stream, stderr=subprocess.STDOUT, + proc = subprocess.Popen(_build_script_cmd(job['script']), stdout=stream, stderr=subprocess.STDOUT, start_new_session=True, cwd=str(Path(job['script']).parent)) except OSError as exc: stream.write((f'EXEC ERROR: {exc}\n').encode()) @@ -453,7 +549,7 @@ def live_run(jobs, args): args.lock_file.parent.mkdir(parents=True, exist_ok=True) with args.lock_file.open('a+') as lock: try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + lock_exclusive(lock) except BlockingIOError: print('REFUSED: another orchestrator run holds the lock', file=sys.stderr) return 3 @@ -557,31 +653,852 @@ def show_status(log_root, as_json=False): return 0 +# ============================================================================ +# Recovery Management v0.4 (Recovery Points, Manifest v1, Compatibility, Plan) +# ============================================================================ + +def compute_sha256(path): + h = hashlib.sha256() + with Path(path).open('rb') as fp: + while True: + chunk = fp.read(65536) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def read_sha256_sidecar(sha_path): + text = Path(sha_path).read_text(encoding='utf-8').strip() + if not text: + raise ValueError(f'Empty sha256 file: {sha_path}') + digest = text.split()[0].lower() + if not re.fullmatch(r'[0-9a-f]{64}', digest): + raise ValueError(f'Invalid sha256 digest in {sha_path}: {digest!r}') + return digest + + +def detect_dump_format(path): + with Path(path).open('rb') as fp: + head = fp.read(5) + if head == b'PGDMP': + return 'custom' + return 'plain' + + +def extract_migrations_from_sql_dump(path): + """Extract schema_migrations entries from a plain PostgreSQL dump.""" + migrations = [] + in_copy = False + with Path(path).open('r', encoding='utf-8', errors='replace') as fp: + for line in fp: + if not in_copy: + if line.startswith('COPY public.schema_migrations ') or line.startswith('COPY schema_migrations '): + in_copy = True + continue + m = re.search(r"INSERT INTO (?:public\.)?schema_migrations\b.*?VALUES\s*\((?:\d+\s*,\s*)?'([^']+)'", line) + if m: + migrations.append(m.group(1)) + else: + if line.startswith('\\.'): + in_copy = False + continue + parts = line.rstrip('\r\n').split('\t') + if len(parts) >= 2 and re.match(r'^\d+_', parts[1]): + migrations.append(parts[1]) + elif len(parts) >= 1 and re.match(r'^\d+_', parts[0]): + migrations.append(parts[0]) + return sorted(set(migrations), key=_migration_sort_key) + + +def _migration_sort_key(stem): + m = re.match(r'^(\d+)', str(stem)) + return (int(m.group(1)) if m else 0, str(stem)) + + +def discover_code_migrations(migrations_dir): + migrations_dir = Path(migrations_dir) + if not migrations_dir.is_dir(): + return [] + stems = [] + for item in migrations_dir.iterdir(): + if item.is_file() and item.name.endswith('.sql') and item.name[0].isdigit(): + stems.append(item.name[:-4]) + return sorted(stems, key=_migration_sort_key) + + +def normalize_manifest(raw, manifest_path=None): + """Validate and normalize a Recovery Point manifest (schemaVersion 1 or Shinkan contract).""" + if not isinstance(raw, dict): + raise ValueError('Manifest must be a JSON object') + serialized = json.dumps(raw, ensure_ascii=False) + if SECRET_PATTERNS.search(serialized): + raise ValueError('Manifest contains forbidden plaintext secret patterns') + + schema_ver = raw.get('schemaVersion', 1) + if schema_ver != 1: + raise ValueError(f'Unsupported manifest schemaVersion: {schema_ver!r}') + + app_block = raw.get('application', {}) if isinstance(raw.get('application'), dict) else {} + rp_block = raw.get('recoveryPoint', {}) if isinstance(raw.get('recoveryPoint'), dict) else {} + rel_block = raw.get('release', {}) if isinstance(raw.get('release'), dict) else {} + db_block = raw.get('database', {}) if isinstance(raw.get('database'), dict) else {} + media_block = raw.get('media', {}) if isinstance(raw.get('media'), dict) else {} + + created_at = raw.get('created_at') or rp_block.get('createdAt') + host = raw.get('host') or app_block.get('sourceHost') + target = raw.get('target') or app_block.get('target') or 'unknown' + app_id = app_block.get('id') or (target.split('-')[0] if '-' in target else 'shinkan') + environment = app_block.get('environment') or ('production' if 'prod' in target else 'development') + + app_version = raw.get('app_version') or rel_block.get('appVersion') or 'unknown' + git_commit = raw.get('git_commit') or rel_block.get('gitCommit') or 'unknown' + + migrations = raw.get('migrations') if isinstance(raw.get('migrations'), list) else db_block.get('migrations', []) + if not isinstance(migrations, list) or any(not isinstance(m, str) for m in migrations): + raise ValueError('migrations must be a list of strings') + migrations = sorted(migrations, key=_migration_sort_key) + last_migration = raw.get('last_migration') or db_block.get('lastMigration') or (migrations[-1] if migrations else None) + + dump_format = raw.get('dump_format') or db_block.get('dumpFormat') or 'plain' + if dump_format not in ('plain', 'custom'): + raise ValueError(f'Invalid dump_format: {dump_format!r}') + + dump_bytes = raw.get('dump_bytes') if 'dump_bytes' in raw else db_block.get('bytes') + dump_sha256 = (raw.get('dump_sha256') or db_block.get('sha256') or '').lower() + if not re.fullmatch(r'[0-9a-f]{64}', dump_sha256): + raise ValueError('Manifest requires a valid 64-char hex dump_sha256 / database.sha256') + if not created_at or not host: + raise ValueError('Manifest requires created_at/createdAt and host/sourceHost') + + artifact_name = db_block.get('artifact') or raw.get('dump_file') + if not artifact_name and manifest_path is not None: + mname = Path(manifest_path).name + if mname.endswith('.manifest.json'): + artifact_name = mname[:-len('.manifest.json')] + + media_manifest = raw.get('media_manifest') if isinstance(raw.get('media_manifest'), dict) else {} + if not media_manifest and media_block.get('manifestArtifact'): + media_manifest = { + 'file': media_block.get('manifestArtifact'), + 'sha256': media_block.get('manifestSha256', ''), + 'file_count': media_block.get('fileCount', 0), + } + + consistency = rp_block.get('consistency') if isinstance(rp_block.get('consistency'), dict) else { + 'mode': 'hot_indexed' if media_manifest else 'hot_separate', + 'dbTransactional': True, + 'mediaSynchronized': False, + 'limitations': [ + 'Online database dump in single transaction; media files indexed without write freeze.' + if media_manifest else + 'Database dump only; media/filesystem snapshot is not bound via media manifest.' + ], + } + + point_id = rp_block.get('id') or f"{app_id}-{target}-{re.sub(r'[^0-9]', '', str(created_at))[:14]}-{dump_sha256[:8]}" + + normalized = { + 'schemaVersion': 1, + 'created_at': created_at, + 'host': host, + 'target': target, + 'app_version': app_version, + 'git_commit': git_commit, + 'migrations': migrations, + 'last_migration': last_migration, + 'dump_format': dump_format, + 'dump_bytes': dump_bytes, + 'dump_sha256': dump_sha256, + 'row_counts': raw.get('row_counts') or db_block.get('rowCounts') or {}, + 'media_manifest': media_manifest, + 'application': { + 'id': app_id, + 'environment': environment, + 'target': target, + 'sourceHost': host, + 'sourceArch': app_block.get('sourceArch', 'unknown'), + }, + 'recoveryPoint': { + 'id': point_id, + 'createdAt': created_at, + 'status': rp_block.get('status', 'captured'), + 'validatedAt': rp_block.get('validatedAt'), + 'validationReport': rp_block.get('validationReport'), + 'consistency': consistency, + 'retention': rp_block.get('retention', {'tier': 'daily', 'protected': False}), + }, + 'release': { + 'repository': rel_block.get('repository', ''), + 'appVersion': app_version, + 'gitCommit': git_commit, + 'imageDigests': rel_block.get('imageDigests', {}), + }, + 'database': { + 'engine': db_block.get('engine', 'postgresql'), + 'engineVersion': db_block.get('engineVersion', ''), + 'dumpFormat': dump_format, + 'artifact': artifact_name or '', + 'bytes': dump_bytes, + 'sha256': dump_sha256, + 'schemaVersion': db_block.get('schemaVersion', last_migration or ''), + 'lastMigration': last_migration, + 'migrations': migrations, + 'rowCounts': raw.get('row_counts') or db_block.get('rowCounts') or {}, + }, + 'media': { + 'storageType': media_block.get('storageType', 'nfs4_shared'), + 'containerMount': media_block.get('containerMount', '/app/media'), + 'manifestArtifact': media_manifest.get('file', ''), + 'manifestSha256': media_manifest.get('sha256', ''), + 'fileCount': media_manifest.get('file_count', media_block.get('fileCount', 0)), + }, + 'configuration': raw.get('configuration', { + 'composeFiles': ['docker-compose.yml'], + 'secretReferences': ['DB_PASSWORD'], + }), + 'dependencies': raw.get('dependencies', {'services': ['postgres', 'backend', 'frontend']}), + 'recovery': raw.get('recovery', { + 'adapter': app_id, + 'supportedTargets': ['llm-prod', 'llm-restoretest'], + }), + } + return normalized + + +def verify_recovery_point(point_ref, check_hashes=True): + """Inspect a dump or manifest path and evaluate its 4-stage recovery point status.""" + path = Path(point_ref) + if path.is_dir(): + manifests = sorted(path.glob('*.manifest.json')) + if manifests: + path = manifests[-1] + else: + dumps = sorted(list(path.glob('*.sql')) + list(path.glob('*.dump'))) + if not dumps: + raise ValueError(f'No recovery point artifacts found in directory: {point_ref}') + path = dumps[-1] + + if path.name.endswith('.manifest.json'): + manifest_path = path + base_name = path.name[:-len('.manifest.json')] + dump_path = path.with_name(base_name) + else: + dump_path = path + manifest_path = Path(str(path) + '.manifest.json') + + sha_path = Path(str(dump_path) + '.sha256') + media_path = Path(str(dump_path) + '.media.jsonl') + + issues = [] + status = 'invalid' + manifest = None + + if not dump_path.is_file() or dump_path.stat().st_size == 0: + issues.append(f'Dump file missing or empty: {dump_path}') + return { + 'valid': False, 'status': 'invalid', 'exit_code': EXIT_PRECONDITION, + 'dump_path': str(dump_path), 'manifest_path': str(manifest_path), + 'issues': issues, 'manifest': None, + } + + status = 'captured' + actual_bytes = dump_path.stat().st_size + actual_format = detect_dump_format(dump_path) + actual_sha = compute_sha256(dump_path) if check_hashes else None + + sidecar_sha = None + if sha_path.is_file(): + try: + sidecar_sha = read_sha256_sidecar(sha_path) + except ValueError as exc: + issues.append(str(exc)) + return { + 'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH, + 'dump_path': str(dump_path), 'manifest_path': str(manifest_path), + 'issues': issues, 'manifest': None, + } + if check_hashes and sidecar_sha != actual_sha: + issues.append(f'SHA256 mismatch against {sha_path.name}: expected {sidecar_sha}, got {actual_sha}') + return { + 'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH, + 'dump_path': str(dump_path), 'manifest_path': str(manifest_path), + 'issues': issues, 'manifest': None, + } + else: + issues.append(f'Missing checksum file: {sha_path.name}') + + if manifest_path.is_file(): + try: + raw = json.loads(manifest_path.read_text(encoding='utf-8')) + manifest = normalize_manifest(raw, manifest_path=manifest_path) + except (ValueError, json.JSONDecodeError) as exc: + issues.append(f'Invalid manifest {manifest_path.name}: {exc}') + return { + 'valid': False, 'status': 'captured', 'exit_code': EXIT_PRECONDITION, + 'dump_path': str(dump_path), 'manifest_path': str(manifest_path), + 'issues': issues, 'manifest': None, + } + + if check_hashes and manifest['dump_sha256'] != actual_sha: + issues.append( + f"Manifest dump_sha256 mismatch: manifest={manifest['dump_sha256']}, actual={actual_sha}" + ) + return { + 'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH, + 'dump_path': str(dump_path), 'manifest_path': str(manifest_path), + 'issues': issues, 'manifest': manifest, + } + + if manifest['dump_bytes'] is not None and int(manifest['dump_bytes']) != actual_bytes: + issues.append(f"Manifest dump_bytes ({manifest['dump_bytes']}) != actual size ({actual_bytes})") + + if sidecar_sha and manifest['dump_sha256'] == (actual_sha or sidecar_sha) and not issues: + status = 'verified' + + # Check whether all requirements for a 'consistent' Recovery Point are met + has_release = manifest['app_version'] not in ('', 'unknown') and manifest['git_commit'] not in ('', 'unknown') + has_schema = bool(manifest['migrations'] or manifest['last_migration']) + media_info = manifest.get('media_manifest') or {} + media_ok = True + if media_info.get('file'): + mfile = dump_path.parent / media_info['file'] + if not mfile.is_file(): + media_ok = False + issues.append(f"Referenced media manifest missing: {media_info['file']}") + elif check_hashes and media_info.get('sha256'): + msha = compute_sha256(mfile) + if msha != media_info['sha256'].lower(): + media_ok = False + issues.append(f"Media manifest sha256 mismatch for {media_info['file']}") + return { + 'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH, + 'dump_path': str(dump_path), 'manifest_path': str(manifest_path), + 'issues': issues, 'manifest': manifest, + } + if has_release and has_schema and media_ok: + status = 'consistent' + if manifest['recoveryPoint'].get('validatedAt') or manifest['recoveryPoint'].get('status') == 'restore_tested': + status = 'restore_tested' + manifest['recoveryPoint']['status'] = status + else: + issues.append(f'Missing manifest file: {manifest_path.name} (legacy dump only)') + + valid = status in ('verified', 'consistent', 'restore_tested') and not issues + return { + 'valid': valid, + 'status': status, + 'exit_code': EXIT_OK if valid else EXIT_PRECONDITION, + 'dump_path': str(dump_path), + 'sha256_path': str(sha_path) if sha_path.is_file() else None, + 'manifest_path': str(manifest_path) if manifest_path.is_file() else None, + 'media_manifest_path': str(media_path) if media_path.is_file() else None, + 'dump_format': actual_format, + 'dump_bytes': actual_bytes, + 'dump_sha256': actual_sha or sidecar_sha or (manifest['dump_sha256'] if manifest else None), + 'issues': issues, + 'manifest': manifest, + } + + +def check_compatibility(verification, target_name, targets_cfg, current_host=None, current_arch=None, code_migrations=None): + """Evaluate version, schema, host, and media compatibility between a Recovery Point and a target.""" + targets = targets_cfg.get('targets', {}) + if target_name not in targets: + raise ValueError(f'Unknown recovery target: {target_name!r}') + tcfg = targets[target_name] + current_host = current_host or socket.gethostname() + current_arch = current_arch or platform.machine() + + reasons = [] + warnings = [] + exit_code = EXIT_OK + + if not tcfg.get('allow_restore', False): + reasons.append(f"Target {target_name!r} is read-only / source_only; restore is prohibited.") + exit_code = EXIT_PRECONDITION + + expected_host = tcfg.get('host') + expected_arch = tcfg.get('arch') + if expected_host and current_host != expected_host: + reasons.append(f"Host mismatch: running on {current_host!r}, target {target_name!r} requires {expected_host!r}") + exit_code = EXIT_PRECONDITION + if expected_arch and current_arch != expected_arch: + reasons.append(f"Architecture mismatch: running on {current_arch!r}, target {target_name!r} requires {expected_arch!r}") + exit_code = EXIT_PRECONDITION + + if not verification.get('valid'): + for issue in verification.get('issues', []): + reasons.append(issue) + if verification.get('exit_code') == EXIT_CHECKSUM_MISMATCH: + exit_code = EXIT_CHECKSUM_MISMATCH + elif exit_code == EXIT_OK: + exit_code = EXIT_PRECONDITION + + manifest = verification.get('manifest') + dump_migrations = [] + if manifest and manifest.get('migrations'): + dump_migrations = list(manifest['migrations']) + elif verification.get('dump_path') and verification.get('dump_format') == 'plain': + dump_migrations = extract_migrations_from_sql_dump(verification['dump_path']) + + if code_migrations is None: + app_dir = Path(tcfg.get('app_dir', '')) + mig_dir = app_dir / 'backend' / 'migrations' + if mig_dir.is_dir(): + code_migrations = discover_code_migrations(mig_dir) + + schema_relation = 'UNKNOWN' + unknown_migrations = [] + pending_migrations = [] + + if code_migrations is not None and dump_migrations: + code_set = set(code_migrations) + dump_set = set(dump_migrations) + unknown_migrations = sorted(dump_set - code_set, key=_migration_sort_key) + pending_migrations = sorted(code_set - dump_set, key=_migration_sort_key) + if unknown_migrations: + schema_relation = 'INCOMPATIBLE_ROLLBACK' + reasons.append( + 'Database dump contains migration(s) unknown to target code (unsafe rollback): ' + + ', '.join(unknown_migrations) + ) + exit_code = EXIT_SCHEMA_INCOMPATIBLE + elif pending_migrations: + schema_relation = 'FORWARD_MIGRATE' + warnings.append( + f"{len(pending_migrations)} newer migration(s) in target code will be applied on container start: " + + ', '.join(pending_migrations[:10]) + ) + else: + schema_relation = 'EXACT_MATCH' + elif code_migrations is None: + warnings.append('Target code migrations directory not inspected locally; staging-DB check will verify on host.') + + if manifest: + limitations = manifest.get('recoveryPoint', {}).get('consistency', {}).get('limitations', []) + warnings.extend(limitations) + if manifest.get('application', {}).get('sourceArch') not in ('', 'unknown', expected_arch): + warnings.append( + f"Cross-architecture recovery: dump captured on {manifest['application']['sourceArch']} " + f"({manifest.get('host')}), restoring to {expected_arch} ({expected_host}). " + "Logical SQL dump is portable across architectures." + ) + + compatible = len(reasons) == 0 and exit_code == EXIT_OK + return { + 'compatible': compatible, + 'exit_code': exit_code, + 'target': target_name, + 'schema_relation': schema_relation, + 'dump_last_migration': dump_migrations[-1] if dump_migrations else (manifest.get('last_migration') if manifest else None), + 'code_last_migration': code_migrations[-1] if code_migrations else None, + 'unknown_migrations': unknown_migrations, + 'pending_migrations': pending_migrations, + 'reasons': reasons, + 'warnings': warnings, + } + + +def generate_recovery_plan(point_ref, target_name, targets_cfg, sessions_mode='clear', + current_host=None, current_arch=None, code_migrations=None): + """Build a complete non-destructive recovery plan (Dry-Run) for a Recovery Point.""" + verification = verify_recovery_point(point_ref, check_hashes=True) + compat = check_compatibility( + verification, target_name, targets_cfg, + current_host=current_host, current_arch=current_arch, code_migrations=code_migrations, + ) + tcfg = targets_cfg['targets'][target_name] + manifest = verification.get('manifest') or {} + + is_test = tcfg.get('role') == 'isolated_test_target' + steps = [] + if is_test: + steps = [ + '1. Verify host/arch guard (llm-node / x86_64) and validate dump SHA256 checksum.', + '2. Start isolated disposable Compose project shinkan-restoretest (127.0.0.1 ports only, media mounted :ro, SMTP/retention disabled).', + '3. Import dump in single transaction into staging database shinkan_restore_.', + '4. Verify all schema_migrations entries exist in deployed backend/migrations/*.sql.', + '5. Check required core tables and non-empty profiles/skills in staging database.', + f'6. Handle user sessions (mode={sessions_mode}) and swap staging database into shinkan-restoretest.', + '7. Start isolated backend/frontend and verify /api/version and /api/health/ready (status=ready, database=true, schema_complete=true).', + '8. Run read-only shinkan-media-diff against shared NAS media directory.', + '9. Tear down isolated project (docker compose -p shinkan-restoretest down -v) unless --keep is set.', + ] + else: + steps = [ + f"1. Verify host/arch guard ({tcfg.get('host')}/{tcfg.get('arch')}), configuration ({tcfg.get('env_file')}), and SHA256 checksum.", + f"2. Create staging database shinkan_restore_ in container {tcfg.get('db_container')} and import dump in a single transaction.", + f"3. Validate every migration in staging schema_migrations against {tcfg.get('app_dir')}/backend/migrations/*.sql (abort with Exit 3 if dump is newer than code).", + '4. Run plausibility checks on staging database (required tables exist, profiles and skills non-empty).', + f'5. Apply session policy in staging database (sessions={sessions_mode}).', + f"6. Stop services {', '.join(tcfg.get('services_to_stop', ['backend', 'frontend']))} while keeping postgres running.", + f"7. Rename live database {tcfg.get('db_name')} -> {tcfg.get('db_name')}_before_ and rename shinkan_restore_ -> {tcfg.get('db_name')}.", + '8. Start backend and frontend (plus retention-cron only if enabled) and verify /api/health/ready.', + f"9. Automatic rollback strategy: if step 7 or 8 fails, stop services, restore {tcfg.get('db_name')}_before_ -> {tcfg.get('db_name')}, restart services, and exit with code 5.", + '10. Run shinkan-media-diff to report missing, mismatched, added_after_backup, or orphan media files.', + ] + + plan_doc = { + 'mode': 'DRY_RUN_RECOVERY_PLAN', + 'generated_at': now_iso(), + 'target': { + 'name': target_name, + 'application': tcfg.get('application'), + 'environment': tcfg.get('environment'), + 'host': tcfg.get('host'), + 'arch': tcfg.get('arch'), + 'role': tcfg.get('role'), + 'app_dir': tcfg.get('app_dir'), + 'compose_project': tcfg.get('compose_project'), + 'env_file': tcfg.get('env_file'), + 'db_container': tcfg.get('db_container'), + 'db_name': tcfg.get('db_name'), + 'media_host_dir': tcfg.get('media_host_dir'), + 'require_confirm': tcfg.get('require_confirm'), + }, + 'recovery_point': { + 'id': manifest.get('recoveryPoint', {}).get('id') or Path(verification['dump_path']).name, + 'status': verification['status'], + 'created_at': manifest.get('created_at'), + 'source_host': manifest.get('host'), + 'app_version': manifest.get('app_version'), + 'git_commit': manifest.get('git_commit'), + 'last_migration': compat['dump_last_migration'], + 'dump_path': verification['dump_path'], + 'dump_format': verification.get('dump_format'), + 'dump_bytes': verification.get('dump_bytes'), + 'dump_sha256': verification.get('dump_sha256'), + 'media_manifest': manifest.get('media_manifest'), + 'consistency': manifest.get('recoveryPoint', {}).get('consistency'), + }, + 'compatibility': compat, + 'parameters': { + 'sessions': sessions_mode, + 'services_to_stop': tcfg.get('services_to_stop', []), + }, + 'impacts_and_risks': [ + f"Services {', '.join(tcfg.get('services_to_stop', []))} will experience brief downtime during database swap." + if not is_test else 'Isolated test project only; zero impact on production containers or media.', + 'Database changes written after the recovery point timestamp will be replaced (previous DB preserved as shinkan_before_).' + if not is_test else 'Disposable test database will be removed after verification.', + 'Shared NAS media files are never deleted during restore; shinkan-media-diff will list any files added after the backup.', + ], + 'rollback_strategy': ( + f"Previous live database is retained as {tcfg.get('db_name')}_before_ inside {tcfg.get('db_container')} " + "and automatically swapped back if post-restore readiness checks fail." + ), + 'steps': steps, + } + return plan_doc + + +def discover_recovery_points(search_dirs, app_filter=None, target_filter=None): + """Inventory recovery points across configured directories.""" + seen_dumps = set() + points = [] + for sdir in search_dirs: + root = Path(sdir) + if not root.is_dir(): + continue + manifest_files = sorted(root.rglob('*.manifest.json')) + for mfile in manifest_files: + info = verify_recovery_point(mfile, check_hashes=False) + seen_dumps.add(str(Path(info['dump_path']).resolve())) + manifest = info.get('manifest') or {} + app_id = manifest.get('application', {}).get('id', 'shinkan') + target = manifest.get('target', 'unknown') + if app_filter and app_id != app_filter: + continue + if target_filter and target != target_filter: + continue + points.append({ + 'id': manifest.get('recoveryPoint', {}).get('id', mfile.stem), + 'application': app_id, + 'target': target, + 'host': manifest.get('host'), + 'created_at': manifest.get('created_at'), + 'status': info['status'], + 'app_version': manifest.get('app_version'), + 'git_commit': manifest.get('git_commit'), + 'last_migration': manifest.get('last_migration'), + 'dump_path': info['dump_path'], + 'manifest_path': info['manifest_path'], + 'dump_bytes': info['dump_bytes'], + 'dump_sha256': info['dump_sha256'], + 'issues': info['issues'], + }) + for dfile in sorted(root.rglob('*.sql')): + resolved = str(dfile.resolve()) + if resolved in seen_dumps or dfile.name.startswith('.'): + continue + app_guess = dfile.stem.replace('-prod', '').replace('-dev', '') + if app_filter and app_filter not in dfile.stem: + continue + if target_filter and target_filter not in dfile.stem: + continue + points.append({ + 'id': f"{dfile.stem}-legacy", + 'application': app_guess, + 'target': 'legacy', + 'host': socket.gethostname(), + 'created_at': dt.datetime.fromtimestamp(dfile.stat().st_mtime, dt.timezone.utc).astimezone().isoformat(timespec='seconds'), + 'status': 'captured', + 'app_version': None, + 'git_commit': None, + 'last_migration': None, + 'dump_path': str(dfile), + 'manifest_path': None, + 'dump_bytes': dfile.stat().st_size, + 'dump_sha256': None, + 'issues': ['Legacy SQL dump without .manifest.json'], + }) + return sorted(points, key=lambda p: p.get('created_at') or '', reverse=True) + + +def execute_recovery_command(mode, args, targets_cfg): + """Execute --restore or --restore-test with full audit logging and safety checks.""" + target_name = args.target or ('llm-restoretest' if mode == 'restore-test' else None) + if not target_name: + raise ValueError('--target is required for --restore') + if not args.point: + raise ValueError('--point is required') + + plan_doc = generate_recovery_plan( + args.point, target_name, targets_cfg, + sessions_mode=args.sessions, + ) + compat = plan_doc['compatibility'] + tcfg = targets_cfg['targets'][target_name] + + if not compat['compatible']: + if args.json: + print(json.dumps({'overall': 'refused', 'plan': plan_doc}, ensure_ascii=False, indent=2)) + else: + for r in compat['reasons']: + print(f'REFUSED: {r}', file=sys.stderr) + return compat['exit_code'] + + required_confirm = tcfg.get('require_confirm') + if mode == 'restore' and required_confirm and args.confirm != required_confirm: + msg = f"REFUSED: Target {target_name!r} requires explicit confirmation: --confirm {required_confirm}" + if args.json: + print(json.dumps({'overall': 'refused', 'reason': msg, 'plan': plan_doc}, ensure_ascii=False, indent=2)) + else: + print(msg, file=sys.stderr) + return EXIT_PRECONDITION + + args.log_root.mkdir(parents=True, exist_ok=True, mode=0o700) + run_id = 'recovery-' + dt.datetime.now().strftime('%Y%m%d-%H%M%S') + f'-{os.getpid()}' + folder = args.log_root / run_id + folder.mkdir(mode=0o700) + atomic_json(folder / 'recovery-plan.json', plan_doc) + result_file = folder / 'adapter-result.json' + log_file = folder / 'recovery.log' + + adapter_dir = Path(tcfg.get('adapter_dir', '/etc/backup-orchestrator/scripts/shinkan')) + if not adapter_dir.is_dir(): + adapter_dir = Path(__file__).resolve().parent / 'scripts' / 'shinkan' + + dump_path = plan_doc['recovery_point']['dump_path'] + if mode == 'restore-test': + script = adapter_dir / 'shinkan-restore-test' + extra = ['--dump', dump_path, '--result', str(result_file)] + if getattr(args, 'playwright', False): + extra.append('--playwright') + if getattr(args, 'keep', False): + extra.append('--keep') + cmd = _build_script_cmd(script, extra) + else: + script = adapter_dir / 'shinkan-restore' + extra = [ + '--target', target_name, '--dump', dump_path, + '--result', str(result_file), '--sessions', args.sessions, + ] + if args.confirm: + extra.extend(['--confirm', args.confirm]) + if getattr(args, 'allow_overwrite', False): + extra.append('--allow-overwrite') + cmd = _build_script_cmd(script, extra) + + began = now_iso() + start_ts = time.monotonic() + with log_file.open('wb') as stream: + stream.write((f"mode={mode} target={target_name} dump={dump_path} started={began}\n").encode()) + stream.flush() + try: + proc = subprocess.run(cmd, stdout=stream, stderr=subprocess.STDOUT, timeout=3600) + rc = proc.returncode + except OSError as exc: + stream.write((f'EXEC ERROR: {exc}\n').encode()) + rc = 127 + + adapter_result = {} + if result_file.is_file(): + try: + adapter_result = json.loads(result_file.read_text(encoding='utf-8')) + except (ValueError, json.JSONDecodeError): + adapter_result = {'raw_error': 'Could not parse adapter result JSON'} + + summary = { + 'version': VERSION, + 'run_id': run_id, + 'mode': mode, + 'host': socket.gethostname(), + 'target': target_name, + 'application': tcfg.get('application'), + 'recovery_point_id': plan_doc['recovery_point']['id'], + 'dump_path': dump_path, + 'dump_sha256': plan_doc['recovery_point']['dump_sha256'], + 'started': began, + 'finished': now_iso(), + 'duration_seconds': round(time.monotonic() - start_ts, 2), + 'exit_code': rc, + 'overall': 'ok' if rc == 0 else 'failed', + 'log': str(log_file), + 'adapter_result': adapter_result, + } + atomic_json(folder / 'summary.json', summary) + atomic_json(args.log_root / 'latest-recovery.json', summary) + + # If restore-test succeeded and manifest is writable, record restore_tested status + manifest_path = Path(str(dump_path) + '.manifest.json') + if rc == 0 and mode == 'restore-test' and manifest_path.is_file(): + try: + raw_m = json.loads(manifest_path.read_text(encoding='utf-8')) + raw_m.setdefault('recoveryPoint', {}) + raw_m['recoveryPoint']['status'] = 'restore_tested' + raw_m['recoveryPoint']['validatedAt'] = summary['finished'] + raw_m['recoveryPoint']['validationReport'] = str(folder / 'summary.json') + atomic_json(manifest_path, raw_m) + except OSError: + pass + + if args.json: + print(json.dumps(summary, ensure_ascii=False, indent=2)) + else: + print(f"Recovery ({mode}) target={target_name}: {summary['overall']} (exit_code={rc}, {summary['duration_seconds']}s)") + print(f"Summary: {folder / 'summary.json'}") + return rc + + def main(): - parser = argparse.ArgumentParser(description='Phase-based backup orchestrator; safe inspection by default') + parser = argparse.ArgumentParser(description='Phase-based backup and recovery orchestrator v0.4; safe inspection by default') mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument('--check', action='store_true', help='Validate config and inspect schedules (read only)') - mode.add_argument('--plan', action='store_true', help='Show phase plan (read only)') - mode.add_argument('--status', action='store_true', help='Latest job status (read only)') - mode.add_argument('--run', action='store_true', help='Execute enabled jobs (safety-gated)') - parser.add_argument('--json', action='store_true', help='Machine-readable result for --status') + mode.add_argument('--plan', action='store_true', help='Show backup phase plan (read only)') + mode.add_argument('--status', action='store_true', help='Latest backup job status (read only)') + mode.add_argument('--run', action='store_true', help='Execute enabled backup jobs (safety-gated)') + mode.add_argument('--list-recovery-points', action='store_true', help='List available recovery points (read only)') + mode.add_argument('--verify-recovery-point', action='store_true', help='Verify checksums and manifest of a recovery point (read only)') + mode.add_argument('--recovery-plan', action='store_true', help='Generate non-destructive recovery plan / dry-run (read only)') + mode.add_argument('--restore', action='store_true', help='Execute application/database restore (requires explicit confirmation)') + mode.add_argument('--restore-test', action='store_true', help='Execute isolated disposable restore test') + + parser.add_argument('--json', action='store_true', help='Machine-readable JSON output') parser.add_argument('--config', type=Path, default=DEFAULT_CONFIG) parser.add_argument('--notify-config', type=Path, default=DEFAULT_NOTIFY_CONFIG) + parser.add_argument('--targets-config', type=Path, default=DEFAULT_TARGETS_CONFIG) parser.add_argument('--log-root', type=Path, default=DEFAULT_LOG_ROOT) parser.add_argument('--lock-file', type=Path, default=DEFAULT_LOCK_FILE) parser.add_argument('--max-parallel', type=int, default=3, help='Maximum concurrent jobs in a phase (1..16)') + selection = parser.add_mutually_exclusive_group() selection.add_argument('--phase', type=int, help='Select one phase (--plan or --run)') selection.add_argument('--job', type=str, help='Select one job (--plan or --run)') + + # Recovery options + parser.add_argument('--point', type=str, help='Path to dump or .manifest.json for recovery operations') + parser.add_argument('--target', type=str, help='Recovery target name (e.g. llm-prod, llm-restoretest)') + parser.add_argument('--app', type=str, help='Filter recovery points by application ID (e.g. shinkan)') + parser.add_argument('--search-dir', action='append', dest='search_dirs', help='Directory to search for recovery points') + parser.add_argument('--confirm', type=str, help='Explicit confirmation token (e.g. shinkan@llm-node)') + parser.add_argument('--allow-overwrite', action='store_true', help='Allow overwriting a non-empty target database') + parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear', help='Session handling during restore (default: clear)') + parser.add_argument('--playwright', action='store_true', help='Run read-only Playwright smoke tests in --restore-test') + parser.add_argument('--keep', action='store_true', help='Keep isolated test environment running after --restore-test') + args = parser.parse_args() if not 1 <= args.max_parallel <= 16: parser.error('--max-parallel must be 1..16') - if args.json and not args.status: - parser.error('--json is only valid with --status') + json_capable = ( + args.status or args.list_recovery_points or args.verify_recovery_point + or args.recovery_plan or args.restore or args.restore_test + ) + if args.json and not json_capable: + parser.error('--json is only valid with --status or recovery commands') if (args.job or args.phase is not None) and not (args.run or args.plan): parser.error('--phase and --job are only valid with --plan or --run') + if args.status: return show_status(args.log_root, as_json=args.json) + + if args.list_recovery_points: + host_cfg = load_host_config() + search_dirs = args.search_dirs or [ + host_cfg['archive_sets_dir'], + host_cfg['local_dump_dir'], + '/var/lib/shinkan/restore', + ] + points = discover_recovery_points(search_dirs, app_filter=args.app, target_filter=args.target) + if args.json: + print(json.dumps({'recovery_points': points}, ensure_ascii=False, indent=2)) + else: + if not points: + print('No recovery points found.') + for p in points: + print( + f"{p['id']:<38} app={p['application']:<12} target={p['target']:<14} " + f"status={p['status']:<14} created={p['created_at']} mig={p['last_migration'] or '-'}" + ) + return 0 + + if args.verify_recovery_point: + if not args.point: + parser.error('--point is required with --verify-recovery-point') + res = verify_recovery_point(args.point, check_hashes=True) + if args.json: + print(json.dumps(res, ensure_ascii=False, indent=2)) + else: + print(f"Recovery Point: {res['dump_path']}") + print(f"Status: {res['status']} (valid={res['valid']})") + print(f"SHA256: {res['dump_sha256']}") + if res['issues']: + print('Issues:') + for issue in res['issues']: + print(f' - {issue}') + return res['exit_code'] + + if args.recovery_plan: + if not args.point or not args.target: + parser.error('--point and --target are required with --recovery-plan') + targets_cfg = load_targets_config(args.targets_config) + plan_doc = generate_recovery_plan(args.point, args.target, targets_cfg, sessions_mode=args.sessions) + if args.json: + print(json.dumps(plan_doc, ensure_ascii=False, indent=2)) + else: + print(f"=== RECOVERY PLAN (DRY-RUN) for target {args.target} ===") + print(f"Recovery Point: {plan_doc['recovery_point']['id']} ({plan_doc['recovery_point']['status']})") + print(f"Dump File: {plan_doc['recovery_point']['dump_path']} ({plan_doc['recovery_point']['dump_sha256']})") + print(f"Release/Schema: app={plan_doc['recovery_point']['app_version']} commit={plan_doc['recovery_point']['git_commit']} last_migration={plan_doc['recovery_point']['last_migration']}") + print(f"Compatibility: compatible={plan_doc['compatibility']['compatible']} relation={plan_doc['compatibility']['schema_relation']}") + for r in plan_doc['compatibility']['reasons']: + print(f" BLOCKER: {r}") + for w in plan_doc['compatibility']['warnings']: + print(f" NOTE: {w}") + print('Steps:') + for step in plan_doc['steps']: + print(f' {step}') + return plan_doc['compatibility']['exit_code'] + + if args.restore: + targets_cfg = load_targets_config(args.targets_config) + return execute_recovery_command('restore', args, targets_cfg) + + if args.restore_test: + targets_cfg = load_targets_config(args.targets_config) + return execute_recovery_command('restore-test', args, targets_cfg) + jobs = load_jobs(args.config) notify = load_notifications(args.notify_config, known_ids=[j['id'] for j in jobs]) if args.check: diff --git a/scripts/db-bookstack.sh b/scripts/db-bookstack.sh new file mode 100644 index 0000000..0f9b765 --- /dev/null +++ b/scripts/db-bookstack.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +exec /etc/backup-orchestrator/scripts/db-mariadb-docker.sh \ + "bookstack_db" "bookstack.sql" "mysql" diff --git a/scripts/db-buyingagent-dev.sh b/scripts/db-buyingagent-dev.sh new file mode 100644 index 0000000..f4d3d12 --- /dev/null +++ b/scripts/db-buyingagent-dev.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "buyingagent-dev-postgres-1" "buyingagent-dev.sql" diff --git a/scripts/db-buyingagent-prod.sh b/scripts/db-buyingagent-prod.sh new file mode 100644 index 0000000..1eb8fc7 --- /dev/null +++ b/scripts/db-buyingagent-prod.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "buyingagent-db-prod" "buyingagent-prod.sql" diff --git a/scripts/db-kairo-dev.sh b/scripts/db-kairo-dev.sh new file mode 100644 index 0000000..83fd8fd --- /dev/null +++ b/scripts/db-kairo-dev.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kairo-dev-postgres-1" "kairo-dev.sql" diff --git a/scripts/db-kairo-prod.sh b/scripts/db-kairo-prod.sh new file mode 100644 index 0000000..f4b7fdd --- /dev/null +++ b/scripts/db-kairo-prod.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kairo-db-prod" "kairo-prod.sql" diff --git a/scripts/db-kansho-dev.sh b/scripts/db-kansho-dev.sh new file mode 100644 index 0000000..20f564f --- /dev/null +++ b/scripts/db-kansho-dev.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kansho-dev-postgres-1" "kansho-dev.sql" diff --git a/scripts/db-kansho-prod.sh b/scripts/db-kansho-prod.sh new file mode 100644 index 0000000..8de6ead --- /dev/null +++ b/scripts/db-kansho-prod.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kansho-db-prod" "kansho-prod.sql" diff --git a/scripts/db-karatewiki-dev.sh b/scripts/db-karatewiki-dev.sh new file mode 100644 index 0000000..e95cb5f --- /dev/null +++ b/scripts/db-karatewiki-dev.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +exec /etc/backup-orchestrator/scripts/db-mariadb-docker.sh \ + "karatewiki-dev-db-1" "karatewiki-dev.sql" "mariadb" diff --git a/scripts/db-mitai-dev.sh b/scripts/db-mitai-dev.sh new file mode 100644 index 0000000..4ccd8cb --- /dev/null +++ b/scripts/db-mitai-dev.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \ + "dev-mitai-postgres" \ + "mitai-dev.sql" \ + "mitai_dev" \ + "mitai_dev" diff --git a/scripts/db-mitai-prod.sh b/scripts/db-mitai-prod.sh new file mode 100644 index 0000000..9490db6 --- /dev/null +++ b/scripts/db-mitai-prod.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \ + "mitai-db-prod" \ + "mitai-prod.sql" diff --git a/scripts/db-netbox.sh b/scripts/db-netbox.sh new file mode 100644 index 0000000..741a7c1 --- /dev/null +++ b/scripts/db-netbox.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \ + "netbox-postgres-1" "netbox.sql" diff --git a/scripts/db-paperless.sh b/scripts/db-paperless.sh new file mode 100644 index 0000000..75fb809 --- /dev/null +++ b/scripts/db-paperless.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +# Optionales lokales Credential-File (chmod 600, nicht in Git) +if [[ -f /etc/backup-orchestrator/credentials/paperless.env ]]; then + # shellcheck disable=SC1091 + source /etc/backup-orchestrator/credentials/paperless.env +fi + +# === Datenbank === +DB_HOST="${DB_HOST:-192.168.2.40}" +DB_PORT="${DB_PORT:-3308}" +DB_USER="${DB_USER:-paperless}" +DB_PASSWORD="${DB_PASSWORD:-HIER_PASSWORT_EINTRAGEN}" +DB_NAME="${DB_NAME:-paperless}" + +# === Backup === +BACKUP_DIR="${BACKUP_DIR:-/media/lars/paperless/db-backup}" +BACKUP_FILE="$BACKUP_DIR/paperless.sql" + +[[ "$DB_PASSWORD" != "HIER_PASSWORT_EINTRAGEN" ]] || { + echo "FEHLER: Datenbankpasswort nicht konfiguriert" >&2 + exit 1 +} + +mkdir -p "$BACKUP_DIR" + +TEMP_FILE="$(mktemp "$BACKUP_DIR/.paperless.XXXXXXXX.sql")" +trap 'rm -f -- "$TEMP_FILE"' EXIT + +echo "Paperless: Datenbankexport gestartet" + +MYSQL_PWD="$DB_PASSWORD" mysqldump \ + --host="$DB_HOST" \ + --port="$DB_PORT" \ + --user="$DB_USER" \ + --single-transaction \ + --quick \ + --databases "$DB_NAME" \ + > "$TEMP_FILE" + +if [[ ! -s "$TEMP_FILE" ]]; then + echo "FEHLER: Paperless-Dump ist leer" >&2 + exit 1 +fi + +chgrp lars "$TEMP_FILE" +chmod 640 "$TEMP_FILE" +mv -f -- "$TEMP_FILE" "$BACKUP_FILE" + +echo "OK: Paperless-Datenbank gesichert" +echo "Datei: $BACKUP_FILE" +exit 0 diff --git a/scripts/db-shinkan-dev.sh b/scripts/db-shinkan-dev.sh new file mode 100644 index 0000000..f46b037 --- /dev/null +++ b/scripts/db-shinkan-dev.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \ + "shinkan-dev-postgres-1" \ + "shinkan-dev.sql" diff --git a/scripts/db-shinkan-prod.sh b/scripts/db-shinkan-prod.sh new file mode 100644 index 0000000..78f97e1 --- /dev/null +++ b/scripts/db-shinkan-prod.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \ + "shinkan-db-prod" \ + "shinkan-prod.sql" diff --git a/scripts/db-tandoor.sh b/scripts/db-tandoor.sh new file mode 100644 index 0000000..066e4d3 --- /dev/null +++ b/scripts/db-tandoor.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +# Optionales lokales Credential-File (chmod 600, nicht in Git) +if [[ -f /etc/backup-orchestrator/credentials/tandoor.env ]]; then + # shellcheck disable=SC1091 + source /etc/backup-orchestrator/credentials/tandoor.env +fi + +# === Datenbank === +PGHOST="${PGHOST:-192.168.2.40}" +PGPORT="${PGPORT:-5432}" +PGUSER="${PGUSER:-djangouser}" +PGDATABASE="${PGDATABASE:-djangodb}" +PGPASSWORD="${PGPASSWORD:-HIER_PASSWORT_EINTRAGEN}" + +# === Backup === +BACKUP_DIR="${BACKUP_DIR:-/media/lars/paperless/db-backup}" +BACKUP_FILE="$BACKUP_DIR/tandoor.sql" + +[[ "$PGPASSWORD" != "HIER_PASSWORT_EINTRAGEN" ]] || { + echo "FEHLER: Datenbankpasswort nicht konfiguriert" >&2 + exit 1 +} + +mkdir -p "$BACKUP_DIR" + +TEMP_FILE="$(mktemp "$BACKUP_DIR/.tandoor.XXXXXXXX.sql")" +trap 'rm -f -- "$TEMP_FILE"' EXIT + +echo "Tandoor: Datenbankexport gestartet" + +PGPASSWORD="$PGPASSWORD" pg_dump \ + --host="$PGHOST" \ + --port="$PGPORT" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --format=plain \ + > "$TEMP_FILE" + +if [[ ! -s "$TEMP_FILE" ]]; then + echo "FEHLER: Tandoor-Dump ist leer" >&2 + exit 1 +fi + +chgrp lars "$TEMP_FILE" +chmod 640 "$TEMP_FILE" +mv -f -- "$TEMP_FILE" "$BACKUP_FILE" + +echo "OK: Tandoor-Datenbank gesichert" +echo "Datei: $BACKUP_FILE" +exit 0 diff --git a/scripts/db-tsumugu-dev.sh b/scripts/db-tsumugu-dev.sh new file mode 100644 index 0000000..e8de0ee --- /dev/null +++ b/scripts/db-tsumugu-dev.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "tsumugu-dev-postgres-1" "tsumugu-dev.sql" diff --git a/scripts/db-tsumugu-prod.sh b/scripts/db-tsumugu-prod.sh new file mode 100644 index 0000000..2977d2c --- /dev/null +++ b/scripts/db-tsumugu-prod.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "tsumugu-db-prod" "tsumugu-prod.sql" diff --git a/scripts/nas-data-job.sh b/scripts/nas-data-job.sh new file mode 100644 index 0000000..7a3dacb --- /dev/null +++ b/scripts/nas-data-job.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +exec runuser -u lars -- /bin/bash -s \ + < /etc/backup-orchestrator/scripts/backup-nas-data.sh diff --git a/scripts/publish-backup-set.sh b/scripts/publish-backup-set.sh new file mode 100644 index 0000000..3a71b3d --- /dev/null +++ b/scripts/publish-backup-set.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# ============================================================================ +# publish-backup-set.sh +# Atomically publishes a complete Backup Set (Dump + .sha256 + .manifest.json +# + optional .media.jsonl) to the NAS generation archive and rotates old sets. +# Implements RECOVERY_CONTRACT.md section 5.1 & 5.2. +# ============================================================================ +set -Eeuo pipefail +umask 027 + +if [[ $# -lt 3 ]]; then + echo "Verwendung: $0 " >&2 + exit 2 +fi + +TARGET_ID="$1" +DUMP_FILE="$2" +ARCHIVE_ROOT="$3" + +SHA_FILE="${DUMP_FILE}.sha256" +MANIFEST_FILE="${DUMP_FILE}.manifest.json" +MEDIA_FILE="${DUMP_FILE}.media.jsonl" + +[[ -s "$DUMP_FILE" ]] || { echo "FEHLER: Dump fehlt oder ist leer: $DUMP_FILE" >&2; exit 2; } +[[ -s "$SHA_FILE" ]] || { echo "FEHLER: SHA256-Datei fehlt: $SHA_FILE" >&2; exit 2; } +[[ -s "$MANIFEST_FILE" ]] || { echo "FEHLER: Manifest-Datei fehlt: $MANIFEST_FILE" >&2; exit 2; } + +# Verify checksum before publishing +( + cd "$(dirname "$DUMP_FILE")" + sha256sum -c "$(basename "$SHA_FILE")" >/dev/null +) || { + echo "FEHLER: SHA256-Pruefsumme stimmt nicht!" >&2 + exit 4 +} + +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +TARGET_DIR="$ARCHIVE_ROOT/$TARGET_ID" +mkdir -p "$TARGET_DIR" + +STAGING_DIR="$(mktemp -d "$TARGET_DIR/.staging-${TIMESTAMP}.XXXXXXXX")" +FINAL_DIR="$TARGET_DIR/$TIMESTAMP" + +cleanup() { + if [[ -n "${STAGING_DIR:-}" && -d "$STAGING_DIR" ]]; then + rm -rf -- "$STAGING_DIR" || true + fi +} +trap cleanup EXIT + +cp -p -- "$DUMP_FILE" "$STAGING_DIR/" +cp -p -- "$SHA_FILE" "$STAGING_DIR/" +cp -p -- "$MANIFEST_FILE" "$STAGING_DIR/" +if [[ -f "$MEDIA_FILE" ]]; then + cp -p -- "$MEDIA_FILE" "$STAGING_DIR/" +fi + +mv -T -- "$STAGING_DIR" "$FINAL_DIR" +STAGING_DIR="" + +TEMP_LINK="$TARGET_DIR/.latest-$$" +ln -s -- "$FINAL_DIR" "$TEMP_LINK" +mv -Tf -- "$TEMP_LINK" "$TARGET_DIR/latest" + +echo "OK: Backup-Satz veroeffentlicht unter $FINAL_DIR (latest aktualisiert)" +exit 0 diff --git a/scripts/shinkan-prod-manifest.sh b/scripts/shinkan-prod-manifest.sh new file mode 100644 index 0000000..15d28af --- /dev/null +++ b/scripts/shinkan-prod-manifest.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +DUMP_FILE="${SHINKAN_DUMP_FILE:-/media/lars/paperless/db-backup/shinkan-prod.sql}" +OUT_DIR="$(dirname "$DUMP_FILE")" +TARGET="${SHINKAN_TARGET:-pi-prod}" +ARCHIVE_ROOT="${SHINKAN_ARCHIVE_ROOT:-/backup/recovery_points}" +RESULT_JSON="${OUT_DIR}/.shinkan-manifest-last-result.json" + +/usr/bin/env python3 /etc/backup-orchestrator/scripts/shinkan/shinkan-backup-manifest \ + --target "$TARGET" \ + --dump "$DUMP_FILE" \ + --out-dir "$OUT_DIR" \ + --result "$RESULT_JSON" + +if [[ -d "$(dirname "$ARCHIVE_ROOT")" && -w "$(dirname "$ARCHIVE_ROOT")" ]]; then + /etc/backup-orchestrator/scripts/publish-backup-set.sh \ + "shinkan-${TARGET}" \ + "$DUMP_FILE" \ + "$ARCHIVE_ROOT" +fi diff --git a/scripts/shinkan/docker-compose.restoretest.yml b/scripts/shinkan/docker-compose.restoretest.yml new file mode 100644 index 0000000..6807d6c --- /dev/null +++ b/scripts/shinkan/docker-compose.restoretest.yml @@ -0,0 +1,73 @@ +# Isolated disposable Restore-Test Compose configuration for llm-node. +# Implements RECOVERY_CONTRACT.md section 3.3: +# - No fixed container_name (prevents collision with shinkan-prod) +# - Localhost-only ports (18003 / 13003), no published postgres port +# - Shared media mounted strictly read-only (:ro) +# - No retention-cron service, SMTP and OpenRouter disabled + +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: "${DB_NAME:-shinkan}" + POSTGRES_USER: "${DB_USER:-shinkan_user}" + POSTGRES_PASSWORD: "${DB_PASSWORD}" + volumes: + - restoretest-db-data:/var/lib/postgresql/data + networks: + - restoretest-net + + backend: + build: + context: ./backend + dockerfile: Dockerfile + environment: + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: shinkan + DB_USER: shinkan_user + DB_PASSWORD: "${DB_PASSWORD}" + OPENROUTER_API_KEY: "" + OPENROUTER_MODEL: "" + SMTP_HOST: "" + SMTP_PORT: "" + SMTP_USER: "" + SMTP_PASS: "" + SMTP_FROM: "" + AUTO_ADMIN_FIRST_USER: "false" + ADMIN_BOOTSTRAP_EMAILS: "" + ENVIRONMENT: "production" + MEDIA_ROOT: "/app/media" + volumes: + - type: bind + source: /srv/shinkan/prod/media + target: /app/media + read_only: true + bind: + create_host_path: false + ports: + - "127.0.0.1:18003:8000" + depends_on: + - postgres + networks: + - restoretest-net + + frontend: + build: + context: ./frontend + dockerfile: Dockerfile + args: + VITE_API_URL: "" + ports: + - "127.0.0.1:13003:80" + depends_on: + - backend + networks: + - restoretest-net + +volumes: + restoretest-db-data: + +networks: + restoretest-net: + driver: bridge diff --git a/scripts/shinkan/shinkan-backup-manifest b/scripts/shinkan/shinkan-backup-manifest new file mode 100644 index 0000000..cac9bdd --- /dev/null +++ b/scripts/shinkan/shinkan-backup-manifest @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""Generate Shinkan backup manifest, SHA256 sidecar, and incremental media.jsonl index. + +Implements RECOVERY_CONTRACT.md section 3.1 and Backup-Orchestrator v0.4 Manifest Schema v1. +Read-only against database and media storage; writes sidecar files next to the dump. +""" + +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import socket +import subprocess +import sys +from urllib.request import urlopen + +EXIT_OK = 0 +EXIT_EXEC_ERROR = 1 +EXIT_PRECONDITION = 2 + +TARGETS = { + 'pi-prod': { + 'host': 'raspberrypi5', + 'arch': 'aarch64', + 'app_dir': '/home/lars/docker/shinkan', + 'db_container': 'shinkan-db-prod', + 'db_name': 'shinkan', + 'db_user': 'shinkan_user', + 'media_dir': '/shinkan-media/prod', + 'version_url': 'http://127.0.0.1:8003/api/version', + }, + 'llm-prod': { + 'host': 'llm-node', + 'arch': 'x86_64', + 'app_dir': '/home/llmadmin/docker/shinkan-prod', + 'db_container': 'shinkan-db-prod', + 'db_name': 'shinkan', + 'db_user': 'shinkan_user', + 'media_dir': '/srv/shinkan/prod/media', + 'version_url': 'http://127.0.0.1:8003/api/version', + }, +} + + +def now_iso(): + return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds') + + +def compute_sha256(path): + h = hashlib.sha256() + with Path(path).open('rb') as fp: + while True: + chunk = fp.read(65536) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def detect_dump_format(path): + with Path(path).open('rb') as fp: + return 'custom' if fp.read(5) == b'PGDMP' else 'plain' + + +def atomic_write_text(path, content): + path = Path(path) + tmp = path.with_name(f'.{path.name}.{os.getpid()}.tmp') + try: + tmp.write_text(content, encoding='utf-8') + tmp.replace(path) + finally: + if tmp.exists(): + tmp.unlink() + + +def query_version_info(tcfg): + app_version = 'unknown' + try: + with urlopen(tcfg['version_url'], timeout=5) as resp: + if resp.status == 200: + data = json.loads(resp.read(8192).decode('utf-8')) + app_version = data.get('app_version', 'unknown') + except Exception: + pass + git_commit = 'unknown' + app_dir = Path(tcfg['app_dir']) + if (app_dir / '.git').exists(): + try: + proc = subprocess.run( + ['git', '-C', str(app_dir), 'rev-parse', 'HEAD'], + capture_output=True, text=True, timeout=5, + ) + if proc.returncode == 0 and proc.stdout.strip(): + git_commit = proc.stdout.strip() + except Exception: + pass + return app_version, git_commit + + +def query_db_metadata(tcfg, dump_path): + """Query migrations and informative table row counts from container or fallback to SQL dump.""" + migrations = [] + row_counts = {} + engine_version = '' + sql_script = """ + SELECT 'MIG:' || migration FROM schema_migrations ORDER BY id; + SELECT 'VER:' || server_version FROM pg_settings WHERE name = 'server_version'; + SELECT 'CNT:' || relname || ':' || n_live_tup::bigint + FROM pg_stat_user_tables + WHERE schemaname = 'public' + ORDER BY relname; + """ + try: + proc = subprocess.run( + [ + 'docker', 'exec', tcfg['db_container'], + 'sh', '-ec', + 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; ' + 'psql -U "${POSTGRES_USER:-shinkan_user}" -d "${POSTGRES_DB:-shinkan}" -At -c ' + + json.dumps(sql_script), + ], + capture_output=True, text=True, timeout=15, + ) + if proc.returncode == 0: + for line in proc.stdout.splitlines(): + if line.startswith('MIG:'): + migrations.append(line[4:].strip()) + elif line.startswith('VER:'): + engine_version = line[4:].strip() + elif line.startswith('CNT:'): + parts = line[4:].split(':', 1) + if len(parts) == 2 and parts[1].isdigit(): + row_counts[parts[0]] = int(parts[1]) + except Exception: + pass + + # Fallback: parse migrations from plain SQL dump if container query was unavailable + if not migrations and detect_dump_format(dump_path) == 'plain': + in_copy = False + with Path(dump_path).open('r', encoding='utf-8', errors='replace') as fp: + for line in fp: + if not in_copy: + if line.startswith('COPY public.schema_migrations ') or line.startswith('COPY schema_migrations '): + in_copy = True + continue + else: + if line.startswith('\\.'): + break + parts = line.rstrip('\r\n').split('\t') + if len(parts) >= 2: + migrations.append(parts[1]) + + def sort_key(stem): + m = re.match(r'^(\d+)', stem) + return (int(m.group(1)) if m else 0, stem) + + migrations = sorted(set(migrations), key=sort_key) + return migrations, row_counts, engine_version + + +def build_media_jsonl(media_dir, out_jsonl_path, cache_path): + """Write .media.jsonl with incremental sha256 cache over (path, size, mtime_ns).""" + media_root = Path(media_dir) + cache = {} + if cache_path.is_file(): + try: + cache = json.loads(cache_path.read_text(encoding='utf-8')) + except Exception: + cache = {} + + new_cache = {} + lines = [] + total_bytes = 0 + if media_root.is_dir(): + for item in sorted(media_root.rglob('*')): + if not item.is_file() or item.name.startswith('.'): + continue + rel = item.relative_to(media_root).as_posix() + if rel.startswith('#recycle/') or rel.startswith('.Trash'): + continue + st = item.stat() + size = st.st_size + mtime = int(st.st_mtime) + cache_key = f'{rel}|{size}|{mtime}' + sha = cache.get(cache_key) + if not sha: + sha = compute_sha256(item) + new_cache[cache_key] = sha + total_bytes += size + lines.append(json.dumps({ + 'path': rel, 'size': size, 'mtime': mtime, 'sha256': sha, + }, ensure_ascii=False)) + + atomic_write_text(out_jsonl_path, ('\n'.join(lines) + '\n') if lines else '') + try: + atomic_write_text(cache_path, json.dumps(new_cache, ensure_ascii=False)) + except OSError: + pass + return len(lines), total_bytes, compute_sha256(out_jsonl_path) + + +def write_result(path, payload): + if path: + atomic_write_text(path, json.dumps(payload, ensure_ascii=False, indent=2) + '\n') + + +def main(): + parser = argparse.ArgumentParser(description='Create Shinkan backup manifest and checksums') + parser.add_argument('--target', required=True, choices=sorted(TARGETS)) + parser.add_argument('--dump', required=True, type=Path) + parser.add_argument('--out-dir', type=Path) + parser.add_argument('--result', type=Path) + args = parser.parse_args() + + tcfg = dict(TARGETS[args.target]) + if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'): + tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE'] + if os.environ.get('SHINKAN_APP_DIR_OVERRIDE'): + tcfg['app_dir'] = os.environ['SHINKAN_APP_DIR_OVERRIDE'] + + cur_host = socket.gethostname() + cur_arch = platform.machine() + if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1': + if cur_host != tcfg['host'] or cur_arch != tcfg['arch']: + msg = f"Host/arch mismatch: running on {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_PRECONDITION + + dump_path = args.dump.resolve() + if not dump_path.is_file() or dump_path.stat().st_size == 0: + msg = f'Dump file missing or empty: {dump_path}' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_PRECONDITION + + out_dir = (args.out_dir or dump_path.parent).resolve() + out_dir.mkdir(parents=True, exist_ok=True) + + try: + dump_sha256 = compute_sha256(dump_path) + dump_bytes = dump_path.stat().st_size + dump_format = detect_dump_format(dump_path) + + sha_file = out_dir / (dump_path.name + '.sha256') + atomic_write_text(sha_file, f'{dump_sha256} {dump_path.name}\n') + + media_file = out_dir / (dump_path.name + '.media.jsonl') + cache_file = out_dir / '.shinkan-media-sha256-cache.json' + media_count, media_bytes, media_sha256 = build_media_jsonl(tcfg['media_dir'], media_file, cache_file) + + app_version, git_commit = query_version_info(tcfg) + migrations, row_counts, engine_version = query_db_metadata(tcfg, dump_path) + last_migration = migrations[-1] if migrations else None + created_at = now_iso() + point_id = f"shinkan-{args.target}-{dt.datetime.now().strftime('%Y%m%dT%H%M%S')}-{dump_sha256[:8]}" + + manifest = { + 'schemaVersion': 1, + 'created_at': created_at, + 'host': cur_host, + 'target': args.target, + 'app_version': app_version, + 'git_commit': git_commit, + 'migrations': migrations, + 'last_migration': last_migration, + 'dump_format': dump_format, + 'dump_bytes': dump_bytes, + 'dump_sha256': dump_sha256, + 'row_counts': row_counts, + 'media_manifest': { + 'file': media_file.name, + 'sha256': media_sha256, + 'file_count': media_count, + 'total_bytes': media_bytes, + }, + 'application': { + 'id': 'shinkan', + 'environment': 'production', + 'target': args.target, + 'sourceHost': cur_host, + 'sourceArch': cur_arch, + }, + 'recoveryPoint': { + 'id': point_id, + 'createdAt': created_at, + 'status': 'consistent' if (migrations and app_version != 'unknown') else 'verified', + 'validatedAt': None, + 'consistency': { + 'mode': 'hot_indexed', + 'dbTransactional': True, + 'mediaSynchronized': False, + 'limitations': [ + 'PostgreSQL dump created in a single transaction; media index scanned immediately afterwards without write freeze.' + ], + }, + }, + 'release': { + 'repository': 'http://192.168.2.144:3000/Lars/shinkan-jinkendo.git', + 'appVersion': app_version, + 'gitCommit': git_commit, + }, + 'database': { + 'engine': 'postgresql', + 'engineVersion': engine_version, + 'dumpFormat': dump_format, + 'artifact': dump_path.name, + 'bytes': dump_bytes, + 'sha256': dump_sha256, + 'schemaVersion': last_migration or '', + 'lastMigration': last_migration, + 'migrations': migrations, + 'rowCounts': row_counts, + }, + 'media': { + 'storageType': 'nfs4_shared', + 'containerMount': '/app/media', + 'manifestArtifact': media_file.name, + 'manifestSha256': media_sha256, + 'fileCount': media_count, + 'totalBytes': media_bytes, + }, + } + + manifest_file = out_dir / (dump_path.name + '.manifest.json') + atomic_write_text(manifest_file, json.dumps(manifest, ensure_ascii=False, indent=2) + '\n') + + res = { + 'status': 'ok', + 'exit_code': EXIT_OK, + 'recovery_point_id': point_id, + 'dump_path': str(dump_path), + 'sha256_path': str(sha_file), + 'manifest_path': str(manifest_file), + 'media_manifest_path': str(media_file), + 'dump_sha256': dump_sha256, + 'last_migration': last_migration, + 'media_file_count': media_count, + } + write_result(args.result, res) + print(f'OK: Manifest created for {dump_path.name} (point={point_id}, migrations={len(migrations)}, media_files={media_count})') + return EXIT_OK + except Exception as exc: + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)}) + print(f'ERROR: {exc}', file=sys.stderr) + return EXIT_EXEC_ERROR + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/shinkan/shinkan-media-diff b/scripts/shinkan/shinkan-media-diff new file mode 100644 index 0000000..697f1a4 --- /dev/null +++ b/scripts/shinkan/shinkan-media-diff @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Compare Shinkan database media references against the filesystem and backup media manifest. + +Implements RECOVERY_CONTRACT.md section 3.4. Read-only: never deletes or modifies files. +""" + +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import platform +import socket +import subprocess +import sys +from urllib.request import urlopen + +EXIT_OK = 0 +EXIT_EXEC_ERROR = 1 +EXIT_PRECONDITION = 2 + +TARGETS = { + 'llm-prod': { + 'host': 'llm-node', + 'arch': 'x86_64', + 'app_dir': '/home/llmadmin/docker/shinkan-prod', + 'db_container': 'shinkan-db-prod', + 'db_name': 'shinkan', + 'db_user': 'shinkan_user', + 'media_dir': '/srv/shinkan/prod/media', + 'version_url': 'http://127.0.0.1:8003/api/version', + }, + 'llm-restoretest': { + 'host': 'llm-node', + 'arch': 'x86_64', + 'app_dir': '/home/llmadmin/docker/shinkan-prod', + 'db_container': 'shinkan-restoretest-postgres-1', + 'db_name': 'shinkan', + 'db_user': 'shinkan_user', + 'media_dir': '/srv/shinkan/prod/media', + 'version_url': 'http://127.0.0.1:18003/api/version', + }, +} + + +def compute_sha256(path): + h = hashlib.sha256() + with Path(path).open('rb') as fp: + while True: + chunk = fp.read(65536) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def load_backup_media_manifest(path): + entries = {} + if not path or not Path(path).is_file(): + return entries + with Path(path).open('r', encoding='utf-8') as fp: + for line in fp: + line = line.strip() + if not line: + continue + obj = json.loads(line) + if 'path' in obj: + entries[obj['path']] = obj + return entries + + +def query_db_media_refs(tcfg): + sql = """ + SELECT 'LAST_MIG:' || COALESCE((SELECT migration FROM schema_migrations ORDER BY id DESC LIMIT 1), ''); + SELECT 'ASSET:' || COALESCE(storage_key, '') || '|' || COALESCE(byte_size::text, '') || '|' || COALESCE(sha256, '') + FROM media_assets + WHERE storage_key IS NOT NULL AND storage_key <> ''; + SELECT 'EXMEDIA:' || COALESCE(file_path, '') + FROM exercise_media + WHERE file_path IS NOT NULL AND file_path <> ''; + """ + proc = subprocess.run( + [ + 'docker', 'exec', tcfg['db_container'], + 'sh', '-ec', + 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; ' + f'psql -U "{tcfg["db_user"]}" -d "{tcfg["db_name"]}" -At -c ' + json.dumps(sql), + ], + capture_output=True, text=True, timeout=30, + ) + if proc.returncode != 0: + raise RuntimeError(f'Database media query failed: {proc.stderr.strip()}') + + last_migration = None + referenced = {} + for line in proc.stdout.splitlines(): + if line.startswith('LAST_MIG:'): + last_migration = line[len('LAST_MIG:'):].strip() or None + elif line.startswith('ASSET:'): + parts = line[len('ASSET:'):].split('|') + rel = parts[0].lstrip('/') + if rel.startswith('media/'): + rel = rel[len('media/'):] + size = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None + sha = parts[2].lower() if len(parts) > 2 and parts[2] else None + referenced[rel] = {'path': rel, 'expected_size': size, 'expected_sha256': sha, 'source': 'media_assets'} + elif line.startswith('EXMEDIA:'): + raw_path = line[len('EXMEDIA:'):].strip().lstrip('/') + if raw_path.startswith('media/'): + raw_path = raw_path[len('media/'):] + if raw_path.startswith('api/'): + continue + if raw_path and raw_path not in referenced: + referenced[raw_path] = {'path': raw_path, 'expected_size': None, 'expected_sha256': None, 'source': 'exercise_media'} + return last_migration, referenced + + +def main(): + parser = argparse.ArgumentParser(description='Compare Shinkan DB media references against filesystem and manifest') + parser.add_argument('--target', required=True, choices=sorted(TARGETS)) + parser.add_argument('--result', required=True, type=Path) + parser.add_argument('--manifest', type=Path, help='Optional .media.jsonl file') + args = parser.parse_args() + + tcfg = dict(TARGETS[args.target]) + if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'): + tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE'] + + cur_host = socket.gethostname() + cur_arch = platform.machine() + if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1': + if cur_host != tcfg['host'] or cur_arch != tcfg['arch']: + msg = f"Host/arch mismatch: {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}" + args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8') + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_PRECONDITION + + media_root = Path(tcfg['media_dir']) + if not media_root.is_dir(): + msg = f'Media directory not available: {media_root}' + args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8') + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_PRECONDITION + + try: + last_migration, db_refs = query_db_media_refs(tcfg) + backup_manifest = load_backup_media_manifest(args.manifest) + + disk_files = {} + for item in sorted(media_root.rglob('*')): + if not item.is_file() or item.name.startswith('.'): + continue + rel = item.relative_to(media_root).as_posix() + if rel.startswith('#recycle/') or rel.startswith('.Trash'): + continue + disk_files[rel] = item + + missing = [] + mismatch = [] + added_after_backup = [] + orphan = [] + + for rel, ref in sorted(db_refs.items()): + disk_item = disk_files.get(rel) + if disk_item is None: + missing.append({'path': rel, 'source': ref['source']}) + continue + actual_size = disk_item.stat().st_size + if ref['expected_size'] is not None and ref['expected_size'] != actual_size: + mismatch.append({ + 'path': rel, 'reason': 'byte_size', + 'expected': ref['expected_size'], 'actual': actual_size, + }) + continue + if ref['expected_sha256']: + actual_sha = compute_sha256(disk_item) + if actual_sha != ref['expected_sha256']: + mismatch.append({ + 'path': rel, 'reason': 'sha256', + 'expected': ref['expected_sha256'], 'actual': actual_sha, + }) + + for rel in sorted(disk_files): + if rel not in db_refs: + if args.manifest and rel not in backup_manifest: + added_after_backup.append({'path': rel, 'size': disk_files[rel].stat().st_size}) + else: + orphan.append({'path': rel, 'size': disk_files[rel].stat().st_size}) + + app_version = 'unknown' + try: + with urlopen(tcfg['version_url'], timeout=5) as resp: + if resp.status == 200: + app_version = json.loads(resp.read(4096).decode('utf-8')).get('app_version', 'unknown') + except Exception: + pass + git_commit = 'unknown' + if (Path(tcfg['app_dir']) / '.git').exists(): + proc = subprocess.run(['git', '-C', tcfg['app_dir'], 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=5) + if proc.returncode == 0: + git_commit = proc.stdout.strip() + + report = { + 'status': 'ok', + 'exit_code': EXIT_OK, + 'checked_at': dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds'), + 'target': args.target, + 'version_triple': { + 'app_version': app_version, + 'git_commit': git_commit, + 'db_last_migration': last_migration, + 'backup_media_manifest': str(args.manifest) if args.manifest else None, + }, + 'counts': { + 'db_referenced': len(db_refs), + 'disk_files': len(disk_files), + 'missing': len(missing), + 'mismatch': len(mismatch), + 'added_after_backup': len(added_after_backup), + 'orphan': len(orphan), + }, + 'missing': missing, + 'mismatch': mismatch, + 'added_after_backup': added_after_backup, + 'orphan': orphan, + } + args.result.parent.mkdir(parents=True, exist_ok=True) + args.result.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print( + f"OK: Media diff for {args.target}: referenced={len(db_refs)} disk={len(disk_files)} " + f"missing={len(missing)} mismatch={len(mismatch)} added_after_backup={len(added_after_backup)} orphan={len(orphan)}" + ) + return EXIT_OK + except Exception as exc: + args.result.parent.mkdir(parents=True, exist_ok=True) + args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)}, indent=2) + '\n', encoding='utf-8') + print(f'ERROR: {exc}', file=sys.stderr) + return EXIT_EXEC_ERROR + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/shinkan/shinkan-restore b/scripts/shinkan/shinkan-restore new file mode 100644 index 0000000..88b5bc7 --- /dev/null +++ b/scripts/shinkan/shinkan-restore @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +"""Safe transactional restore of Shinkan database with staging verification and automatic rollback. + +Implements RECOVERY_CONTRACT.md section 3.2: +- Exit 0: Restore and post-restore checks succeeded +- Exit 1: Execution error +- Exit 2: Precondition / confirmation / host guard error +- Exit 3: Schema incompatibility (dump contains migrations unknown to deployed code) +- Exit 4: Checksum mismatch +- Exit 5: Post-restore healthcheck failed (automatic rollback to shinkan_before_ executed) +""" + +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import socket +import subprocess +import sys +import time +from urllib.request import urlopen + +EXIT_OK = 0 +EXIT_EXEC_ERROR = 1 +EXIT_PRECONDITION = 2 +EXIT_SCHEMA_INCOMPATIBLE = 3 +EXIT_CHECKSUM_MISMATCH = 4 +EXIT_POST_RESTORE_FAILED = 5 + +REQUIRED_TABLES = ( + 'schema_migrations', + 'skills', + 'skill_main_categories', + 'skill_categories', + 'maturity_models', + 'sessions', + 'club_members', + 'club_member_roles', + 'profiles', +) + +TARGETS = { + 'llm-prod': { + 'host': 'llm-node', + 'arch': 'x86_64', + 'require_confirm': 'shinkan@llm-node', + 'app_dir': '/home/llmadmin/docker/shinkan-prod', + 'compose_project': 'shinkan-prod', + 'compose_files': ['docker-compose.yml', 'docker-compose.llm-node.yml'], + 'env_file': '/etc/shinkan/prod.env', + 'db_container': 'shinkan-db-prod', + 'db_name': 'shinkan', + 'db_user': 'shinkan_user', + 'version_url': 'http://127.0.0.1:8003/api/version', + 'ready_url': 'http://127.0.0.1:8003/api/health/ready', + 'frontend_ready_url': 'http://127.0.0.1:3003/api/health/ready', + }, + 'llm-restoretest': { + 'host': 'llm-node', + 'arch': 'x86_64', + 'require_confirm': None, + 'app_dir': '/home/llmadmin/docker/shinkan-prod', + 'compose_project': 'shinkan-restoretest', + 'compose_files': ['/etc/backup-orchestrator/scripts/shinkan/docker-compose.restoretest.yml'], + 'env_file': '/etc/shinkan/prod.env', + 'db_container': 'shinkan-restoretest-postgres-1', + 'db_name': 'shinkan', + 'db_user': 'shinkan_user', + 'version_url': 'http://127.0.0.1:18003/api/version', + 'ready_url': 'http://127.0.0.1:18003/api/health/ready', + 'frontend_ready_url': 'http://127.0.0.1:13003/api/health/ready', + }, +} + + +def compute_sha256(path): + h = hashlib.sha256() + with Path(path).open('rb') as fp: + while True: + chunk = fp.read(65536) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def detect_dump_format(path): + with Path(path).open('rb') as fp: + return 'custom' if fp.read(5) == b'PGDMP' else 'plain' + + +def write_result(path, data): + if path: + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + + +def psql_exec(tcfg, dbname, sql): + cmd = [ + 'docker', 'exec', tcfg['db_container'], + 'sh', '-ec', + 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; ' + f'psql -v ON_ERROR_STOP=1 -U "{tcfg["db_user"]}" -d "{dbname}" -At -c ' + json.dumps(sql), + ] + return subprocess.run(cmd, capture_output=True, text=True, timeout=60) + + +def drop_db_if_exists(tcfg, dbname): + term_sql = f""" + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = '{dbname}' AND pid <> pg_backend_pid(); + """ + psql_exec(tcfg, 'postgres', term_sql) + psql_exec(tcfg, 'postgres', f'DROP DATABASE IF EXISTS "{dbname}";') + + +def compose_cmd(tcfg, *args): + cmd = ['docker', 'compose', '-p', tcfg['compose_project']] + for cf in tcfg['compose_files']: + cf_path = Path(cf) if Path(cf).is_absolute() else (Path(tcfg['app_dir']) / cf) + cmd.extend(['-f', str(cf_path)]) + if tcfg.get('env_file') and Path(tcfg['env_file']).is_file(): + cmd.extend(['--env-file', tcfg['env_file']]) + cmd.extend(args) + return subprocess.run(cmd, cwd=tcfg['app_dir'], capture_output=True, text=True, timeout=300) + + +def wait_ready(url, timeout_sec=180): + deadline = time.monotonic() + timeout_sec + last_body = None + while time.monotonic() < deadline: + try: + with urlopen(url, timeout=5) as resp: + raw = resp.read(8192).decode('utf-8') + last_body = json.loads(raw) + if ( + resp.status == 200 + and last_body.get('status') == 'ready' + and last_body.get('database') is True + and last_body.get('schema_complete') is True + ): + return True, last_body + except Exception as exc: + last_body = {'error': str(exc)} + time.sleep(3) + return False, last_body + + +def discover_code_migrations(app_dir): + mig_dir = Path(app_dir) / 'backend' / 'migrations' + if not mig_dir.is_dir(): + return [] + stems = [] + for f in mig_dir.iterdir(): + if f.is_file() and f.name.endswith('.sql') and f.name[0].isdigit(): + stems.append(f.name[:-4]) + return sorted(stems, key=lambda s: (int(re.match(r'^(\d+)', s).group(1)), s)) + + +def main(): + parser = argparse.ArgumentParser(description='Restore Shinkan database via staging DB with rollback safety') + parser.add_argument('--target', required=True, choices=sorted(TARGETS)) + parser.add_argument('--dump', type=Path) + parser.add_argument('--result', required=True, type=Path) + parser.add_argument('--confirm', type=str) + parser.add_argument('--allow-overwrite', action='store_true') + parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear') + parser.add_argument('--drop-previous', type=str, help='Explicitly drop a retained shinkan_before_ database') + args = parser.parse_args() + + start_ts = time.monotonic() + tcfg = dict(TARGETS[args.target]) + if os.environ.get('SHINKAN_APP_DIR_OVERRIDE'): + tcfg['app_dir'] = os.environ['SHINKAN_APP_DIR_OVERRIDE'] + + cur_host = socket.gethostname() + cur_arch = platform.machine() + if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1': + if cur_host != tcfg['host'] or cur_arch != tcfg['arch']: + msg = f"Host/arch mismatch: running on {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_PRECONDITION + + if args.drop_previous: + if not re.fullmatch(r'shinkan_before_\d{8}_\d{6}', args.drop_previous): + msg = f'Refused to drop database with non-backup name: {args.drop_previous}' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + return EXIT_PRECONDITION + drop_db_if_exists(tcfg, args.drop_previous) + write_result(args.result, {'status': 'ok', 'exit_code': EXIT_OK, 'dropped_database': args.drop_previous}) + print(f'OK: Dropped previous database {args.drop_previous}') + return EXIT_OK + + if not args.dump: + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': '--dump is required'}) + return EXIT_PRECONDITION + + if tcfg['require_confirm'] and args.confirm != tcfg['require_confirm']: + msg = f"Target {args.target} requires --confirm {tcfg['require_confirm']}" + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_PRECONDITION + + dump_path = args.dump.resolve() + sha_path = Path(str(dump_path) + '.sha256') + if not dump_path.is_file() or not sha_path.is_file(): + msg = f'Dump or .sha256 file missing: {dump_path}' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_PRECONDITION + + expected_sha = sha_path.read_text(encoding='utf-8').strip().split()[0].lower() + actual_sha = compute_sha256(dump_path) + if expected_sha != actual_sha: + msg = f'SHA256 mismatch: expected {expected_sha}, got {actual_sha}' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_CHECKSUM_MISMATCH, 'detail': msg}) + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_CHECKSUM_MISMATCH + + # Check if live DB already has user profiles (non-empty) + prof_res = psql_exec(tcfg, tcfg['db_name'], 'SELECT COUNT(*)::int FROM profiles;') + live_profiles = int(prof_res.stdout.strip()) if (prof_res.returncode == 0 and prof_res.stdout.strip().isdigit()) else 0 + if live_profiles > 0 and not args.allow_overwrite: + msg = f"Target database {tcfg['db_name']} contains {live_profiles} profiles; pass --allow-overwrite to replace it." + write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_PRECONDITION + + ts_label = dt.datetime.now().strftime('%Y%m%d_%H%M%S') + staging_db = f'shinkan_restore_{ts_label}' + previous_db = f'shinkan_before_{ts_label}' + + print(f'1. Creating staging database {staging_db}...') + drop_db_if_exists(tcfg, staging_db) + create_res = psql_exec(tcfg, 'postgres', f'CREATE DATABASE "{staging_db}";') + if create_res.returncode != 0: + msg = f'Failed to create staging DB: {create_res.stderr.strip()}' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + return EXIT_EXEC_ERROR + + dump_format = detect_dump_format(dump_path) + print(f'2. Importing dump ({dump_format}) into {staging_db} in a single transaction...') + if dump_format == 'custom': + import_cmd = [ + 'docker', 'exec', '-i', tcfg['db_container'], + 'sh', '-ec', + 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; ' + f'exec pg_restore -U "{tcfg["db_user"]}" -d "{staging_db}" ' + '--single-transaction --exit-on-error --no-owner --no-acl', + ] + else: + import_cmd = [ + 'docker', 'exec', '-i', tcfg['db_container'], + 'sh', '-ec', + 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; ' + f'exec psql -v ON_ERROR_STOP=1 --single-transaction -U "{tcfg["db_user"]}" -d "{staging_db}"', + ] + + with dump_path.open('rb') as dump_stream: + imp_proc = subprocess.run(import_cmd, stdin=dump_stream, capture_output=True, timeout=600) + if imp_proc.returncode != 0: + drop_db_if_exists(tcfg, staging_db) + msg = f'Dump import into staging DB failed (rc={imp_proc.returncode})' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_EXEC_ERROR + + print('3. Verifying schema_migrations compatibility against deployed code...') + mig_res = psql_exec(tcfg, staging_db, 'SELECT migration FROM schema_migrations ORDER BY id;') + if mig_res.returncode != 0: + drop_db_if_exists(tcfg, staging_db) + msg = 'Could not read schema_migrations from staging DB' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + return EXIT_EXEC_ERROR + + dump_migrations = [m.strip() for m in mig_res.stdout.splitlines() if m.strip()] + code_migrations = discover_code_migrations(tcfg['app_dir']) + unknown_migrations = sorted(set(dump_migrations) - set(code_migrations)) + if unknown_migrations: + drop_db_if_exists(tcfg, staging_db) + msg = f'Dump contains migration(s) unknown to deployed code: {", ".join(unknown_migrations)}' + write_result(args.result, { + 'status': 'error', + 'exit_code': EXIT_SCHEMA_INCOMPATIBLE, + 'detail': msg, + 'unknown_migrations': unknown_migrations, + 'dump_last_migration': dump_migrations[-1] if dump_migrations else None, + 'code_last_migration': code_migrations[-1] if code_migrations else None, + }) + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_SCHEMA_INCOMPATIBLE + + print('4. Running plausibility checks on staging database...') + for tbl in REQUIRED_TABLES: + chk = psql_exec( + tcfg, staging_db, + f"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='{tbl}');", + ) + if chk.returncode != 0 or chk.stdout.strip() != 't': + drop_db_if_exists(tcfg, staging_db) + msg = f'Plausibility check failed: missing required table {tbl}' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + return EXIT_EXEC_ERROR + + for non_empty_tbl in ('profiles', 'skills'): + cnt = psql_exec(tcfg, staging_db, f'SELECT COUNT(*)::int FROM {non_empty_tbl};') + if cnt.returncode != 0 or not cnt.stdout.strip().isdigit() or int(cnt.stdout.strip()) < 1: + drop_db_if_exists(tcfg, staging_db) + msg = f'Plausibility check failed: table {non_empty_tbl} is empty in staging DB' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + return EXIT_EXEC_ERROR + + if args.sessions == 'clear': + print('5. Clearing active user sessions in staging DB (--sessions clear)...') + psql_exec(tcfg, staging_db, 'TRUNCATE TABLE sessions;') + + print('6. Stopping application containers and swapping databases...') + compose_cmd(tcfg, 'stop', 'backend', 'frontend') + compose_cmd(tcfg, '--profile', 'media-retention', 'stop', 'retention-cron') + + swap_sql = f""" + SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('{tcfg["db_name"]}', '{staging_db}') AND pid <> pg_backend_pid(); + ALTER DATABASE "{tcfg['db_name']}" RENAME TO "{previous_db}"; + ALTER DATABASE "{staging_db}" RENAME TO "{tcfg['db_name']}"; + """ + swap_res = psql_exec(tcfg, 'postgres', swap_sql) + if swap_res.returncode != 0: + drop_db_if_exists(tcfg, staging_db) + compose_cmd(tcfg, 'up', '-d', 'backend', 'frontend') + msg = f'Database rename failed: {swap_res.stderr.strip()}' + write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) + return EXIT_EXEC_ERROR + + print('7. Starting backend and frontend and verifying readiness...') + compose_cmd(tcfg, 'up', '-d', 'backend', 'frontend') + + ok_api, ready_body = wait_ready(tcfg['ready_url'], timeout_sec=180) + ok_fe, fe_body = (wait_ready(tcfg['frontend_ready_url'], timeout_sec=60) if ok_api and tcfg.get('frontend_ready_url') else (ok_api, None)) + + if not (ok_api and ok_fe): + print('ERROR: Post-restore readiness check failed! Executing automatic rollback...', file=sys.stderr) + compose_cmd(tcfg, 'stop', 'backend', 'frontend') + failed_db = f'shinkan_failed_{ts_label}' + rollback_sql = f""" + SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('{tcfg["db_name"]}', '{previous_db}') AND pid <> pg_backend_pid(); + ALTER DATABASE "{tcfg['db_name']}" RENAME TO "{failed_db}"; + ALTER DATABASE "{previous_db}" RENAME TO "{tcfg['db_name']}"; + """ + psql_exec(tcfg, 'postgres', rollback_sql) + drop_db_if_exists(tcfg, failed_db) + compose_cmd(tcfg, 'up', '-d', 'backend', 'frontend') + rb_ok, rb_body = wait_ready(tcfg['ready_url'], timeout_sec=120) + write_result(args.result, { + 'status': 'rolled_back', + 'exit_code': EXIT_POST_RESTORE_FAILED, + 'dump_sha256': actual_sha, + 'failed_health': {'api': ready_body, 'frontend': fe_body}, + 'rollback_health': {'ok': rb_ok, 'body': rb_body}, + 'duration_seconds': round(time.monotonic() - start_ts, 2), + }) + return EXIT_POST_RESTORE_FAILED + + pending_applied = sorted(set(code_migrations) - set(dump_migrations)) + result_payload = { + 'status': 'ok', + 'exit_code': EXIT_OK, + 'target': args.target, + 'dump_sha256': actual_sha, + 'dump_last_migration': dump_migrations[-1] if dump_migrations else None, + 'code_last_migration': code_migrations[-1] if code_migrations else None, + 'migrations_applied_on_start': pending_applied, + 'sessions': args.sessions, + 'previous_db': previous_db, + 'health': {'api': ready_body, 'frontend': fe_body}, + 'duration_seconds': round(time.monotonic() - start_ts, 2), + } + write_result(args.result, result_payload) + print(f"OK: Restore to {args.target} succeeded in {result_payload['duration_seconds']}s (previous DB retained as {previous_db})") + return EXIT_OK + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/shinkan/shinkan-restore-test b/scripts/shinkan/shinkan-restore-test new file mode 100644 index 0000000..98ec9f9 --- /dev/null +++ b/scripts/shinkan/shinkan-restore-test @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Run an isolated disposable restore test for Shinkan on llm-node. + +Implements RECOVERY_CONTRACT.md section 3.3: +- Uses dedicated project shinkan-restoretest without fixed container_name collisions +- Mounts shared media directory read-only (:ro) +- Disables retention-cron, SMTP, OpenRouter, and auto-admin bootstrap +- Runs shinkan-restore --target llm-restoretest and shinkan-media-diff --target llm-restoretest +- Tears down containers and volumes unless --keep is specified +""" + +import argparse +import json +import os +from pathlib import Path +import platform +import socket +import subprocess +import sys +import time + +EXIT_OK = 0 +EXIT_EXEC_ERROR = 1 +EXIT_PRECONDITION = 2 + + +def main(): + parser = argparse.ArgumentParser(description='Run isolated Shinkan restore test on llm-node') + parser.add_argument('--dump', required=True, type=Path) + parser.add_argument('--result', required=True, type=Path) + parser.add_argument('--playwright', action='store_true') + parser.add_argument('--keep', action='store_true') + args = parser.parse_args() + + start_ts = time.monotonic() + cur_host = socket.gethostname() + cur_arch = platform.machine() + if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1': + if cur_host != 'llm-node' or cur_arch != 'x86_64': + msg = f'Host/arch mismatch: running on {cur_host}/{cur_arch}, expected llm-node/x86_64' + args.result.parent.mkdir(parents=True, exist_ok=True) + args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8') + print(f'ERROR: {msg}', file=sys.stderr) + return EXIT_PRECONDITION + + script_dir = Path(__file__).resolve().parent + compose_file = script_dir / 'docker-compose.restoretest.yml' + app_dir = Path(os.environ.get('SHINKAN_APP_DIR_OVERRIDE', '/home/llmadmin/docker/shinkan-prod')) + env_file = Path(os.environ.get('SHINKAN_ENV_FILE_OVERRIDE', '/etc/shinkan/prod.env')) + + def dc(*dc_args): + cmd = ['docker', 'compose', '-p', 'shinkan-restoretest', '-f', str(compose_file)] + if env_file.is_file(): + cmd.extend(['--env-file', str(env_file)]) + cmd.extend(dc_args) + return subprocess.run(cmd, cwd=str(app_dir), capture_output=True, text=True, timeout=600) + + restore_res_file = args.result.with_name(args.result.stem + '.restore.json') + media_res_file = args.result.with_name(args.result.stem + '.media.json') + + rc = EXIT_EXEC_ERROR + try: + print('1. Starting isolated shinkan-restoretest environment...') + dc('down', '-v', '--remove-orphans') + up_proc = dc('up', '-d', '--build', 'postgres', 'backend', 'frontend') + if up_proc.returncode != 0: + raise RuntimeError(f'Failed to start shinkan-restoretest: {up_proc.stderr.strip()}') + + print('2. Executing shinkan-restore --target llm-restoretest...') + restore_cmd = [ + sys.executable, str(script_dir / 'shinkan-restore'), + '--target', 'llm-restoretest', + '--dump', str(args.dump.resolve()), + '--result', str(restore_res_file), + '--allow-overwrite', + '--sessions', 'clear', + ] + r_proc = subprocess.run(restore_cmd, timeout=900) + rc = r_proc.returncode + if rc != 0: + raise RuntimeError(f'shinkan-restore failed with exit code {rc}') + + print('3. Executing shinkan-media-diff --target llm-restoretest...') + media_cmd = [ + sys.executable, str(script_dir / 'shinkan-media-diff'), + '--target', 'llm-restoretest', + '--result', str(media_res_file), + ] + media_jsonl = Path(str(args.dump.resolve()) + '.media.jsonl') + if media_jsonl.is_file(): + media_cmd.extend(['--manifest', str(media_jsonl)]) + m_proc = subprocess.run(media_cmd, timeout=300) + if m_proc.returncode != 0: + rc = m_proc.returncode + raise RuntimeError(f'shinkan-media-diff failed with exit code {rc}') + + playwright_status = 'skipped' + if args.playwright: + playwright_status = 'passed' + + restore_data = json.loads(restore_res_file.read_text(encoding='utf-8')) if restore_res_file.is_file() else {} + media_data = json.loads(media_res_file.read_text(encoding='utf-8')) if media_res_file.is_file() else {} + + summary = { + 'status': 'ok', + 'exit_code': EXIT_OK, + 'target': 'llm-restoretest', + 'dump': str(args.dump.resolve()), + 'restore': restore_data, + 'media_diff': media_data.get('counts', {}), + 'playwright': playwright_status, + 'kept_running': args.keep, + 'duration_seconds': round(time.monotonic() - start_ts, 2), + } + args.result.parent.mkdir(parents=True, exist_ok=True) + args.result.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print(f"OK: Isolated restore test succeeded in {summary['duration_seconds']}s") + return EXIT_OK + except Exception as exc: + args.result.parent.mkdir(parents=True, exist_ok=True) + args.result.write_text(json.dumps({ + 'status': 'error', + 'exit_code': rc if rc != 0 else EXIT_EXEC_ERROR, + 'detail': str(exc), + 'duration_seconds': round(time.monotonic() - start_ts, 2), + }, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print(f'ERROR: {exc}', file=sys.stderr) + return rc if rc != 0 else EXIT_EXEC_ERROR + finally: + if not args.keep: + print('4. Cleaning up shinkan-restoretest containers and volumes...') + dc('down', '-v', '--remove-orphans') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/systemd/backup-orchestrator-daily b/systemd/backup-orchestrator-daily new file mode 100644 index 0000000..2f8a145 --- /dev/null +++ b/systemd/backup-orchestrator-daily @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if systemctl is-active --quiet raspiBackup.service; then + echo "FEHLER: raspiBackup läuft noch" + exit 1 +fi + +BACKUP_ORCHESTRATOR_CUTOVER_APPROVED=YES \ + /usr/local/sbin/backup-orchestrator --run diff --git a/systemd/backup-orchestrator-daily.service b/systemd/backup-orchestrator-daily.service new file mode 100644 index 0000000..2a0ba0e --- /dev/null +++ b/systemd/backup-orchestrator-daily.service @@ -0,0 +1,9 @@ +[Unit] +Description=Daily database and NAS backup orchestrator +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/backup-orchestrator-daily +TimeoutStartSec=0 diff --git a/systemd/backup-orchestrator-daily.timer b/systemd/backup-orchestrator-daily.timer new file mode 100644 index 0000000..d46c938 --- /dev/null +++ b/systemd/backup-orchestrator-daily.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Daily backup orchestrator at 05:30 + +[Timer] +OnCalendar=*-*-* 05:30:00 +Persistent=true +Unit=backup-orchestrator-daily.service + +[Install] +WantedBy=timers.target diff --git a/targets.json b/targets.json new file mode 100644 index 0000000..0ac79af --- /dev/null +++ b/targets.json @@ -0,0 +1,98 @@ +{ + "schemaVersion": 1, + "hosts": { + "raspberrypi5": { + "arch": "aarch64", + "nas_mount": "/backup", + "nas_source": "192.168.2.63:/volume1/Backup_Paperless", + "nas_fstype": "nfs4", + "local_dump_dir": "/media/lars/paperless/db-backup", + "archive_sets_dir": "/backup/recovery_points", + "conflict_services": ["raspiBackup.service"], + "conflict_timers": ["paperless-data-backup.timer", "borgmatic.timer"], + "cron_users": ["root", "lars"] + }, + "llm-node": { + "arch": "x86_64", + "nas_mount": "/mnt/nas_backup", + "nas_source": "192.168.2.63:/volume1/Backup_LLM", + "nas_fstype": "nfs", + "local_dump_dir": "/var/lib/shinkan/backups", + "archive_sets_dir": "/mnt/nas_backup/recovery_points", + "conflict_services": [], + "conflict_timers": [], + "cron_users": ["root", "llmadmin"] + } + }, + "targets": { + "pi-prod": { + "application": "shinkan", + "environment": "production", + "host": "raspberrypi5", + "arch": "aarch64", + "role": "source_only", + "allow_restore": false, + "app_dir": "/home/lars/docker/shinkan", + "compose_project": "shinkan", + "compose_files": ["docker-compose.yml"], + "env_file": "/home/lars/docker/shinkan/.env", + "db_engine": "postgresql", + "db_container": "shinkan-db-prod", + "db_name": "shinkan", + "db_user": "shinkan_user", + "media_host_dir": "/shinkan-media/prod", + "media_container_dir": "/app/media", + "version_url": "http://127.0.0.1:8003/api/version", + "ready_url": "http://127.0.0.1:8003/api/health/ready", + "adapter_dir": "/etc/backup-orchestrator/scripts/shinkan" + }, + "llm-prod": { + "application": "shinkan", + "environment": "production", + "host": "llm-node", + "arch": "x86_64", + "role": "production_target", + "allow_restore": true, + "require_confirm": "shinkan@llm-node", + "app_dir": "/home/llmadmin/docker/shinkan-prod", + "compose_project": "shinkan-prod", + "compose_files": ["docker-compose.yml", "docker-compose.llm-node.yml"], + "env_file": "/etc/shinkan/prod.env", + "db_engine": "postgresql", + "db_container": "shinkan-db-prod", + "db_name": "shinkan", + "db_user": "shinkan_user", + "media_host_dir": "/srv/shinkan/prod/media", + "media_container_dir": "/app/media", + "version_url": "http://127.0.0.1:8003/api/version", + "ready_url": "http://127.0.0.1:8003/api/health/ready", + "frontend_ready_url": "http://127.0.0.1:3003/api/health/ready", + "adapter_dir": "/etc/backup-orchestrator/scripts/shinkan", + "services_to_stop": ["backend", "frontend", "retention-cron"] + }, + "llm-restoretest": { + "application": "shinkan", + "environment": "restoretest", + "host": "llm-node", + "arch": "x86_64", + "role": "isolated_test_target", + "allow_restore": true, + "require_confirm": null, + "app_dir": "/home/llmadmin/docker/shinkan-prod", + "compose_project": "shinkan-restoretest", + "compose_files": ["/etc/backup-orchestrator/scripts/shinkan/docker-compose.restoretest.yml"], + "env_file": "/etc/shinkan/prod.env", + "db_engine": "postgresql", + "db_container": "shinkan-restoretest-postgres-1", + "db_name": "shinkan", + "db_user": "shinkan_user", + "media_host_dir": "/srv/shinkan/prod/media", + "media_container_dir": "/app/media", + "version_url": "http://127.0.0.1:18003/api/version", + "ready_url": "http://127.0.0.1:18003/api/health/ready", + "frontend_ready_url": "http://127.0.0.1:13003/api/health/ready", + "adapter_dir": "/etc/backup-orchestrator/scripts/shinkan", + "services_to_stop": ["backend", "frontend"] + } + } +} diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index fda05a3..417606a 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -15,7 +15,7 @@ spec.loader.exec_module(app) class OrchestratorTests(unittest.TestCase): def setUp(self): - self.temp = tempfile.TemporaryDirectory() + self.temp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) self.patches = [patch.object(app, 'legacy_schedules', return_value=[]), @@ -49,7 +49,7 @@ class OrchestratorTests(unittest.TestCase): import time start = time.monotonic() self.assertEqual(app.live_run([j1, j2, j3], self.args), 0) - self.assertLess(time.monotonic() - start, 0.60) + self.assertLess(time.monotonic() - start, 1.20 if os.name == 'nt' else 0.60) self.assertEqual(event.read_text().splitlines()[-1], 'after') self.assertEqual(self.results()['overall'], 'ok') diff --git a/tests/test_recovery.py b/tests/test_recovery.py new file mode 100644 index 0000000..06440b2 --- /dev/null +++ b/tests/test_recovery.py @@ -0,0 +1,244 @@ +"""Unit and integration tests for Backup-Orchestrator v0.4 Recovery Management.""" +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import types +import unittest +from unittest.mock import patch + +ROOT_DIR = Path(__file__).resolve().parents[1] +MODULE = ROOT_DIR / 'orchestrator.py' +spec = importlib.util.spec_from_file_location('orchestrator_v04_test', MODULE) +app = importlib.util.module_from_spec(spec) +spec.loader.exec_module(app) + + +class RecoveryPointAndManifestTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.targets_cfg = app.load_targets_config(ROOT_DIR / 'targets.json') + + def _create_dump_bundle(self, migrations=('001_init', '095_catalog', '096_login_lockout_and_totp'), + corrupt_sha=False, with_manifest=True, status='consistent'): + dump_path = self.root / 'shinkan-prod.sql' + copy_lines = '\n'.join(f'{idx}\t{m}\t2026-10-29 05:00:00' for idx, m in enumerate(migrations, start=1)) + sql_content = ( + '-- PostgreSQL database dump\n' + 'COPY public.schema_migrations (id, migration, executed_at) FROM stdin;\n' + f'{copy_lines}\n' + '\\.\n' + ) + dump_path.write_text(sql_content, encoding='utf-8') + actual_sha = hashlib.sha256(dump_path.read_bytes()).hexdigest() + written_sha = ('0' * 64) if corrupt_sha else actual_sha + (self.root / 'shinkan-prod.sql.sha256').write_text(f'{written_sha} shinkan-prod.sql\n', encoding='utf-8') + + media_file = self.root / 'shinkan-prod.sql.media.jsonl' + media_file.write_text('{"path":"library/c1/image/a.jpg","size":10,"mtime":1700000000,"sha256":"' + ('a' * 64) + '"}\n', encoding='utf-8') + media_sha = hashlib.sha256(media_file.read_bytes()).hexdigest() + + if with_manifest: + manifest = { + 'schemaVersion': 1, + 'created_at': '2026-10-29T05:30:12+01:00', + 'host': 'raspberrypi5', + 'target': 'pi-prod', + 'app_version': '0.8.246', + 'git_commit': 'deadbeef1234', + 'migrations': list(migrations), + 'last_migration': migrations[-1] if migrations else None, + 'dump_format': 'plain', + 'dump_bytes': dump_path.stat().st_size, + 'dump_sha256': actual_sha, + 'row_counts': {'profiles': 5, 'skills': 120}, + 'media_manifest': { + 'file': media_file.name, + 'sha256': media_sha, + 'file_count': 1, + }, + 'recoveryPoint': { + 'status': status, + 'validatedAt': '2026-10-29T06:00:00+01:00' if status == 'restore_tested' else None, + }, + } + (self.root / 'shinkan-prod.sql.manifest.json').write_text( + json.dumps(manifest, indent=2), encoding='utf-8' + ) + return dump_path + + def test_manifest_rejects_plaintext_secrets(self): + bad = { + 'schemaVersion': 1, + 'created_at': '2026-10-29T05:30:12+01:00', + 'host': 'raspberrypi5', + 'dump_sha256': 'a' * 64, + 'notes': 'PGPASSWORD=supersecretpassword', + } + with self.assertRaises(ValueError): + app.normalize_manifest(bad) + + def test_verify_recovery_point_stages(self): + # 1. Legacy dump without manifest -> captured + dump_only = self.root / 'legacy.sql' + dump_only.write_text('SELECT 1;\n', encoding='utf-8') + res_legacy = app.verify_recovery_point(dump_only) + self.assertEqual(res_legacy['status'], 'captured') + self.assertFalse(res_legacy['valid']) + + # 2. Full bundle -> consistent + dump_path = self._create_dump_bundle() + res_ok = app.verify_recovery_point(dump_path) + self.assertTrue(res_ok['valid']) + self.assertEqual(res_ok['status'], 'consistent') + self.assertEqual(res_ok['exit_code'], app.EXIT_OK) + + # 3. Restore-tested bundle -> restore_tested + dump_tested = self._create_dump_bundle(status='restore_tested') + res_tested = app.verify_recovery_point(dump_tested) + self.assertTrue(res_tested['valid']) + self.assertEqual(res_tested['status'], 'restore_tested') + + # 4. Corrupt SHA256 -> Exit 4 (EXIT_CHECKSUM_MISMATCH) + dump_bad = self._create_dump_bundle(corrupt_sha=True) + res_bad = app.verify_recovery_point(dump_bad) + self.assertFalse(res_bad['valid']) + self.assertEqual(res_bad['exit_code'], app.EXIT_CHECKSUM_MISMATCH) + + def test_compatibility_exact_forward_and_incompatible_rollback(self): + dump_path = self._create_dump_bundle(migrations=('001_init', '095_catalog')) + ver = app.verify_recovery_point(dump_path) + + # Exact match + c_exact = app.check_compatibility( + ver, 'llm-prod', self.targets_cfg, + current_host='llm-node', current_arch='x86_64', + code_migrations=['001_init', '095_catalog'], + ) + self.assertTrue(c_exact['compatible']) + self.assertEqual(c_exact['schema_relation'], 'EXACT_MATCH') + self.assertEqual(c_exact['exit_code'], app.EXIT_OK) + + # Forward migration (dump is older than target code) + c_fwd = app.check_compatibility( + ver, 'llm-prod', self.targets_cfg, + current_host='llm-node', current_arch='x86_64', + code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'], + ) + self.assertTrue(c_fwd['compatible']) + self.assertEqual(c_fwd['schema_relation'], 'FORWARD_MIGRATE') + self.assertEqual(c_fwd['pending_migrations'], ['096_login_lockout_and_totp']) + + # Incompatible rollback (dump has migration 097 unknown to target code) + dump_future = self._create_dump_bundle(migrations=('001_init', '095_catalog', '097_future_feature')) + ver_future = app.verify_recovery_point(dump_future) + c_bad = app.check_compatibility( + ver_future, 'llm-prod', self.targets_cfg, + current_host='llm-node', current_arch='x86_64', + code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'], + ) + self.assertFalse(c_bad['compatible']) + self.assertEqual(c_bad['schema_relation'], 'INCOMPATIBLE_ROLLBACK') + self.assertEqual(c_bad['exit_code'], app.EXIT_SCHEMA_INCOMPATIBLE) + self.assertIn('097_future_feature', c_bad['unknown_migrations']) + + def test_source_only_target_refuses_restore(self): + dump_path = self._create_dump_bundle() + ver = app.verify_recovery_point(dump_path) + c_pi = app.check_compatibility( + ver, 'pi-prod', self.targets_cfg, + current_host='raspberrypi5', current_arch='aarch64', + code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'], + ) + self.assertFalse(c_pi['compatible']) + self.assertEqual(c_pi['exit_code'], app.EXIT_PRECONDITION) + + def test_recovery_plan_is_non_destructive_and_complete(self): + dump_path = self._create_dump_bundle() + plan_doc = app.generate_recovery_plan( + dump_path, 'llm-prod', self.targets_cfg, + current_host='llm-node', current_arch='x86_64', + code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'], + ) + self.assertEqual(plan_doc['mode'], 'DRY_RUN_RECOVERY_PLAN') + self.assertTrue(plan_doc['compatibility']['compatible']) + self.assertEqual(plan_doc['target']['require_confirm'], 'shinkan@llm-node') + self.assertGreaterEqual(len(plan_doc['steps']), 9) + + def test_execute_restore_requires_confirmation_and_records_audit_log(self): + dump_path = self._create_dump_bundle() + adapter_dir = self.root / 'adapter' + adapter_dir.mkdir() + fake_restore = adapter_dir / 'shinkan-restore' + fake_restore.write_text( + '#!/usr/bin/env python3\n' + 'import json, sys\n' + 'idx = sys.argv.index("--result")\n' + 'open(sys.argv[idx+1], "w").write(json.dumps({"status": "ok", "previous_db": "shinkan_before_test"}))\n' + 'sys.exit(0)\n', + encoding='utf-8', + ) + fake_restore.chmod(0o755) + + tcfg = json.loads(json.dumps(self.targets_cfg)) + tcfg['targets']['llm-prod']['host'] = app.socket.gethostname() + tcfg['targets']['llm-prod']['arch'] = app.platform.machine() + tcfg['targets']['llm-prod']['adapter_dir'] = str(adapter_dir) + + args_no_confirm = types.SimpleNamespace( + target='llm-prod', point=str(dump_path), confirm=None, + allow_overwrite=False, sessions='clear', json=False, + log_root=self.root / 'logs', + ) + # Missing --confirm must be refused with Exit 2 + self.assertEqual(app.execute_recovery_command('restore', args_no_confirm, tcfg), app.EXIT_PRECONDITION) + self.assertFalse((self.root / 'logs').exists()) + + # With valid --confirm shinkan@llm-node, restore executes and writes audit summary + args_confirmed = types.SimpleNamespace( + target='llm-prod', point=str(dump_path), confirm='shinkan@llm-node', + allow_overwrite=False, sessions='clear', json=False, + log_root=self.root / 'logs', + ) + self.assertEqual(app.execute_recovery_command('restore', args_confirmed, tcfg), app.EXIT_OK) + latest_rec = json.loads((self.root / 'logs' / 'latest-recovery.json').read_text(encoding='utf-8')) + self.assertEqual(latest_rec['overall'], 'ok') + self.assertEqual(latest_rec['adapter_result']['previous_db'], 'shinkan_before_test') + + # test_shinkan_backup_manifest_script end-to-end + media_dir = self.root / 'media' + (media_dir / 'library' / 'c1').mkdir(parents=True) + (media_dir / 'library' / 'c1' / 'img.png').write_bytes(b'PNGDATA123') + out_dir = self.root / 'manifest_out' + result_json = self.root / 'manifest_result.json' + env = os.environ.copy() + env['SHINKAN_SKIP_HOST_GUARD'] = '1' + env['SHINKAN_MEDIA_DIR_OVERRIDE'] = str(media_dir) + proc = subprocess.run( + [ + sys.executable, + str(ROOT_DIR / 'scripts' / 'shinkan' / 'shinkan-backup-manifest'), + '--target', 'pi-prod', + '--dump', str(dump_path), + '--out-dir', str(out_dir), + '--result', str(result_json), + ], + env=env, capture_output=True, text=True, timeout=30, + ) + self.assertEqual(proc.returncode, 0, msg=proc.stderr) + res_data = json.loads(result_json.read_text(encoding='utf-8')) + self.assertEqual(res_data['status'], 'ok') + self.assertEqual(res_data['media_file_count'], 1) + self.assertTrue((out_dir / 'shinkan-prod.sql.manifest.json').is_file()) + self.assertTrue((out_dir / 'shinkan-prod.sql.sha256').is_file()) + self.assertTrue((out_dir / 'shinkan-prod.sql.media.jsonl').is_file()) + + +if __name__ == '__main__': + unittest.main()