Compare commits
11
Commits
c2740c9542
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6882fe397c | ||
|
|
03e09fd0d0 | ||
|
|
feb469d47b | ||
|
|
a17afb4b13 | ||
|
|
0283fa196a | ||
|
|
15974b7977 | ||
|
|
7d291583d1 | ||
|
|
267aca4fbe | ||
|
|
7116215477 | ||
|
|
0bbc4ce440 | ||
|
|
3c1589bd0c |
No files matched your search
@@ -13,11 +13,12 @@ backup-orchestrator/
|
|||||||
├── install.sh # Server-spezifischer Installer (--host raspberrypi5 | llm-node)
|
├── install.sh # Server-spezifischer Installer (--host raspberrypi5 | llm-node)
|
||||||
├── targets.json # Globale Target-Registry
|
├── targets.json # Globale Target-Registry
|
||||||
├── scripts/
|
├── scripts/
|
||||||
│ ├── common/ # Gemeinsame, serverunabhängige Backup-Engines
|
│ ├── common/ # Gemeinsame, serverunabhängige Backup- & Restore-Engines
|
||||||
│ │ ├── db-postgres-docker.sh
|
│ │ ├── db-postgres-docker.sh # Universeller PostgreSQL-Dump
|
||||||
│ │ ├── db-mariadb-docker.sh
|
│ │ ├── db-mariadb-docker.sh # Universeller MariaDB-Dump
|
||||||
│ │ └── publish-backup-set.sh
|
│ │ ├── publish-backup-set.sh # Erzeugt Manifest, Checksumme & relativen latest-Link
|
||||||
│ └── shinkan/ # Anwendungs-Adapter für Shinkan (Recovery Contract)
|
│ │ └── universal-db-restore # Universeller Staging-Restore für Postgres & MariaDB
|
||||||
|
│ └── shinkan/ # Spezifische Adapter für Shinkan (Pilot-Applikation)
|
||||||
│ ├── shinkan-backup-manifest
|
│ ├── shinkan-backup-manifest
|
||||||
│ ├── shinkan-restore
|
│ ├── shinkan-restore
|
||||||
│ ├── shinkan-restore-test
|
│ ├── shinkan-restore-test
|
||||||
@@ -27,22 +28,25 @@ backup-orchestrator/
|
|||||||
│ ├── raspberrypi5/ # Produktiver Backup-Host (Debian 12 ARM64, 192.168.2.49)
|
│ ├── raspberrypi5/ # Produktiver Backup-Host (Debian 12 ARM64, 192.168.2.49)
|
||||||
│ │ ├── host.json # Mounts (/backup), Pfade (/media/lars/paperless/db-backup)
|
│ │ ├── host.json # Mounts (/backup), Pfade (/media/lars/paperless/db-backup)
|
||||||
│ │ ├── jobs.csv # 17 DB-Jobs (Phase 10), nas-data (Phase 20), system-image (Phase 30)
|
│ │ ├── jobs.csv # 17 DB-Jobs (Phase 10), nas-data (Phase 20), system-image (Phase 30)
|
||||||
│ │ ├── targets.json # Host-Targets (pi-prod = source_only)
|
│ │ ├── targets.json # Host-Targets (pi-prod, kairo-pi-prod, tandoor-pi-prod, ...)
|
||||||
│ │ ├── notifications.json.example
|
│ │ ├── notifications.json.example
|
||||||
│ │ ├── scripts/ # Alle 17 DB-Wrapper + NAS-Skripte für raspberrypi5
|
│ │ ├── scripts/ # Alle 17 DB-Wrapper + NAS-Skripte für raspberrypi5
|
||||||
│ │ └── systemd/ # backup-orchestrator-daily + .service/.timer (05:30 Uhr)
|
│ │ └── systemd/ # backup-orchestrator-daily + .service/.timer (05:30 Uhr)
|
||||||
│ └── llm-node/ # Neuer Anwendungs- & Recovery-Host (Ubuntu 24.04 x86_64, 192.168.2.144)
|
│ └── llm-node/ # Neuer Anwendungs- & Recovery-Host (Ubuntu 24.04 x86_64, 192.168.2.144)
|
||||||
│ ├── host.json # Mounts (/mnt/nas_backup, /srv/shinkan/prod/media)
|
│ ├── host.json # Mounts (/mnt/nas_backup, /srv/shinkan/prod/media)
|
||||||
│ ├── jobs.csv # Backup-Jobs auf llm-node (shinkan-prod-db für Post-Cutover vorbereitet)
|
│ ├── jobs.csv # Backup-Jobs auf llm-node (shinkan-prod-db für Post-Cutover vorbereitet)
|
||||||
│ ├── targets.json # Targets (llm-prod, llm-restoretest, pi-prod = source_only)
|
│ ├── targets.json # Targets (shinkan, kairo, tandoor, bookstack...)
|
||||||
│ ├── notifications.json.example
|
│ ├── notifications.json.example
|
||||||
│ ├── scripts/ # Host-spezifische Wrapper für llm-node
|
│ ├── scripts/ # Host-spezifische Wrapper für llm-node
|
||||||
│ └── systemd/ # backup-orchestrator-daily + .service/.timer (05:00 Uhr)
|
│ └── systemd/ # backup-orchestrator-daily + .service/.timer (05:00 Uhr)
|
||||||
├── docs/
|
├── docs/
|
||||||
│ └── ARCHITECTURE_AND_RECOVERY_PLAN_v0.4.md
|
│ ├── ARCHITECTURE_AND_RECOVERY_PLAN_v0.4.md
|
||||||
|
│ ├── APP_MIGRATION_CONTRACT.md # Entwickler-Leitfaden & Agenten-Prompt für App-Migrationen
|
||||||
|
│ └── UPTIME_KUMA_INTEGRATION.md # Push-Monitoring & Alerting-Konfiguration
|
||||||
└── tests/
|
└── tests/
|
||||||
├── test_orchestrator.py
|
├── test_orchestrator.py
|
||||||
└── test_recovery.py
|
├── test_recovery.py
|
||||||
|
└── test_universal_restore.py
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
# App Migration Contract & Entwickler-Leitfaden für Coding-Agenten
|
||||||
|
|
||||||
|
**Gültig für:** Alle Docker-basierten Anwendungen, die von `raspberrypi5` auf `llm-node` umgezogen oder neu an den `backup-orchestrator` angebunden werden (z. B. Kairo, Tandoor Recipes, BookStack, Paperless-ngx, Mitai, Netbox, Tsumugu, Kansho).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zielsetzung & Architektur
|
||||||
|
|
||||||
|
Der `backup-orchestrator` führt automatische Backups (Phase 10 Dumps, Phase 20 NAS-Sync) und risikolose Staging-Restores mit atomarem Swap und Rollback-Sicherheit durch.
|
||||||
|
|
||||||
|
Damit eine Anwendung ohne manuelle Eingriffe und ohne Datenverlust migriert und im Regelbetrieb gesichert werden kann, muss jeder Applikations-Entwickler bzw. Coding-Agent die folgenden **5 Kernanforderungen** einhalten.
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TD
|
||||||
|
A["Anwendung (Docker Compose)"] --> B["1. DB & App Trennung"]
|
||||||
|
A --> C["2. Health/Readiness Endpoint (/api/health/ready)"]
|
||||||
|
A --> D["3. Deterministische Migrationen"]
|
||||||
|
A --> E["4. Stateless Container & NAS-Storage"]
|
||||||
|
A --> F["5. Invalidate Sessions bei Restore"]
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Die 5 Kernanforderungen an jede Anwendung
|
||||||
|
|
||||||
|
### Anforderung 1: Docker Compose Architektur
|
||||||
|
1. **Container-Benennung**:
|
||||||
|
- DB-Container: `<app>-db-prod` (z. B. `kairo-db-prod`, `tandoor-db-prod`, `bookstack-db-prod`).
|
||||||
|
- App-/Backend-Container: `<app>-backend`, `<app>-frontend` oder sprechende Namen.
|
||||||
|
2. **Umgebungsvariablen (`.env` / `/etc/<app>/prod.env`)**:
|
||||||
|
- DB-Credentials dürfen niemals im Compose-File fest eincodiert sein.
|
||||||
|
- Für PostgreSQL: `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD`.
|
||||||
|
- Für MariaDB: `MARIADB_DATABASE`, `MARIADB_USER`, `MARIADB_PASSWORD`.
|
||||||
|
3. **Multi-Architektur**:
|
||||||
|
- Docker-Images müssen Multi-Arch (`linux/amd64` für `llm-node`, `linux/arm64` für Pi) unterstützen oder direkt auf dem Ziel-Host baubar sein.
|
||||||
|
|
||||||
|
### Anforderung 2: Readiness / Health Endpoint (`ready_url`)
|
||||||
|
Jede Anwendung **muss** einen HTTP-Healthcheck bereitstellen (z. B. `http://127.0.0.1:<port>/api/health/ready` oder `.../healthz`):
|
||||||
|
- **HTTP 200 OK** darf **erst dann** zurückgegeben werden, wenn:
|
||||||
|
1. Die Datenbankverbindung steht und Abfragen erfolgreich sind.
|
||||||
|
2. Alle anstehenden Datenbankmigrationen vollständig ausgeführt wurden.
|
||||||
|
- Wenn die DB noch migriert, blockiert ist oder nicht antwortet, muss der Endpoint HTTP `503 Service Unavailable` zurückgeben.
|
||||||
|
- Bei JSON-Ausgabe sollte `{"status": "ready"}` oder `{"status": "ok"}` enthalten sein.
|
||||||
|
|
||||||
|
### Anforderung 3: Deterministische Datenbank-Migrationen
|
||||||
|
1. Schema-Änderungen müssen über Migrationstabellen nachvollziehbar sein:
|
||||||
|
- Django: `django_migrations`
|
||||||
|
- Laravel / BookStack: `migrations`
|
||||||
|
- Shinkan / Custom: `schema_migrations`
|
||||||
|
- Alembic / FastAPI: `alembic_version`
|
||||||
|
2. **Migrations-Kompatibilität**:
|
||||||
|
- Vor einem Restore prüft der Orchestrator, ob der Dump Migrationen enthält, die im deployten Code noch fehlen.
|
||||||
|
- Ein Rollback auf älteren Code bei neuerem DB-Dump wird verweigert (`EXIT_SCHEMA_INCOMPATIBLE`).
|
||||||
|
|
||||||
|
### Anforderung 4: Invalidation von Sessions bei Umgebungsumzug
|
||||||
|
- Bei einer Migration zwischen Servern (z. B. von Pi auf llm-node) dürfen alte Session-Tokens nicht unkontrolliert weiterleben.
|
||||||
|
- Die Session-Tabelle (z. B. `sessions` oder `django_session`) wird beim Restore geleert (`TRUNCATE TABLE`), sofern `--sessions clear` aktiv ist.
|
||||||
|
|
||||||
|
### Anforderung 5: Trennung von Code, Datenbank und Medien (Stateless Container)
|
||||||
|
- Uploads, Dokumente, PDF-Dateien und Bilder gehören **nicht** in die Datenbank und **nicht** in flüchtige Container-Layer!
|
||||||
|
- Medienverzeichnisse müssen auf persistenten Mounts (Synology NAS oder lokale Mountpoints wie `/srv/<app>/prod/media`) liegen.
|
||||||
|
- Der DB-Dump bleibt dadurch klein (< 50 MB), transaktionssicher und blitzschnell wiederherstellbar.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Konfiguration in `targets.json`
|
||||||
|
|
||||||
|
Sobald die Anwendung die Kriterien erfüllt, wird sie in `/etc/backup-orchestrator/targets.json` auf `llm-node` eingetragen:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"kairo-llm-prod": {
|
||||||
|
"application": "kairo",
|
||||||
|
"environment": "production",
|
||||||
|
"host": "llm-node",
|
||||||
|
"arch": "x86_64",
|
||||||
|
"role": "production_target",
|
||||||
|
"allow_restore": true,
|
||||||
|
"require_confirm": "kairo@llm-node",
|
||||||
|
"app_dir": "/home/llmadmin/docker/kairo-prod",
|
||||||
|
"compose_project": "kairo-prod",
|
||||||
|
"compose_files": ["docker-compose.yml"],
|
||||||
|
"env_file": "/etc/kairo/prod.env",
|
||||||
|
"db_engine": "postgresql",
|
||||||
|
"db_container": "kairo-db-prod",
|
||||||
|
"db_name": "kairo",
|
||||||
|
"db_user": "kairo_user",
|
||||||
|
"ready_url": "http://127.0.0.1:8001/api/health/ready",
|
||||||
|
"services_to_stop": ["backend"],
|
||||||
|
"verify_tables_not_empty": ["django_migrations"],
|
||||||
|
"truncate_tables": ["django_session"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Prompt-Vorlage für Coding-Agenten der Ziel-Anwendung
|
||||||
|
|
||||||
|
Wenn ein Coding-Agent (z. B. in Roo-Code, Claude Code, Cursor oder Gemini) beauftragt wird, eine Anwendung für den Umzug nach `llm-node` vorzubereiten, verwende folgenden Prompt:
|
||||||
|
|
||||||
|
````markdown
|
||||||
|
Du bereitest die Anwendung <APP_NAME> für den produktiven Betrieb auf unserem neuen Server `llm-node` (Ubuntu 24.04 LTS x86_64) und die Anbindung an den Backup-Orchestrator vor.
|
||||||
|
|
||||||
|
Bitte setze folgende Anforderungen um und verändere keine Produktionsdaten auf dem Raspberry Pi:
|
||||||
|
|
||||||
|
1. **Docker Compose & Multi-Arch Build:**
|
||||||
|
- Stelle sicher, dass `docker-compose.yml` (oder ein Override `docker-compose.llm-node.yml`) auf `llm-node` fehlerfrei baut (`linux/amd64`).
|
||||||
|
- Der Datenbank-Container muss den festen Namen `<APP_NAME>-db-prod` tragen.
|
||||||
|
- Alle DB-Credentials müssen aus einer externen Environment-Datei (`/etc/<APP_NAME>/prod.env` oder `.env`) geladen werden.
|
||||||
|
- Nicht-persistente App-Container müssen zustandslos (stateless) sein; persistente Medien/Uploads müssen auf ein Host-Verzeichnis gemountet werden.
|
||||||
|
|
||||||
|
2. **Health / Readiness Endpoint:**
|
||||||
|
- Implementiere oder verifiziere einen HTTP-Endpoint unter `/api/health/ready` (oder `/healthz`).
|
||||||
|
- Dieser Endpoint muss HTTP 200 liefern, sobald die Datenbank erreichbar ist UND alle Migrationen erfolgreich durchgelaufen sind.
|
||||||
|
- Solange Migrationen laufen oder die DB nicht bereit ist, muss HTTP 503 zurückgegeben werden.
|
||||||
|
|
||||||
|
3. **Migrations-Transparenz:**
|
||||||
|
- Dokumentiere, in welcher Tabelle die DB-Migrationen erfasst werden (z. B. `schema_migrations`, `django_migrations`, `migrations`).
|
||||||
|
- Dokumentiere, welche 1–2 Kern-Tabellen nach einem Restore mindestens befüllt sein müssen (z. B. `users`, `profiles`, `posts`).
|
||||||
|
- Dokumentiere den Namen der Session-Tabelle (z. B. `sessions`, `django_session`), damit alte Tokens beim Umzug bereinigt werden können.
|
||||||
|
|
||||||
|
4. **Verifikation auf llm-node:**
|
||||||
|
- Starte den Stack auf `llm-node` mit einer leeren Datenbank.
|
||||||
|
- Prüfe mit `curl -s -i http://127.0.0.1:<PORT>/api/health/ready`, dass der Readiness-Check grün meldet.
|
||||||
|
- Melde zurück: DB-Engine, Containername, DB-Name, DB-User, Health-URL, Session-Tabelle und nicht-leere Kern-Tabellen.
|
||||||
|
````
|
||||||
|
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
# Uptime Kuma Monitoring & Alerting Guide
|
||||||
|
|
||||||
|
Der `backup-orchestrator` verfügt über eine native, out-of-the-box Integration mit **Uptime Kuma**.
|
||||||
|
Dieses Dokument beschreibt das Monitoring-Konzept, die Schritt-für-Schritt-Einrichtung und die Konfiguration.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Duales Überwachungskonzept
|
||||||
|
|
||||||
|
Das Gesamtsystem wird auf zwei komplementären Ebenen überwacht:
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TD
|
||||||
|
subgraph "1. Passive Push-Überwachung (Backup-Läufe & Restores)"
|
||||||
|
BO["backup-orchestrator<br/>(systemd daily 05:30)"]
|
||||||
|
BO -->|"HTTP Push: JOB_SUCCESS (up) / JOB_FAILED (down)"| K1["Uptime Kuma<br/>Push Monitor: Einzeljobs"]
|
||||||
|
BO -->|"HTTP Push: RUN_FINISHED (overall ok/failed)"| K2["Uptime Kuma<br/>Push Monitor: Täglicher Gesamtlauf"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "2. Aktive Pull-Überwachung (Live-Dienste auf llm-node)"
|
||||||
|
K3["Uptime Kuma<br/>HTTP(s) Monitor: 60s Intervall"]
|
||||||
|
K3 -->|"GET /api/health/ready (Port 8003, 8001...)"| APP1["Shinkan / Kairo Backend"]
|
||||||
|
K3 -->|"GET / (Port 3003, 8080...)"| APP2["Frontend & Web-UIs"]
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
### A. Passive Push-Überwachung (Heartbeat / Dead Man's Snitch)
|
||||||
|
- **Problem bei Crons/Timern**: Wenn ein Cronjob gar nicht erst startet (Server abgestürzt, Timer deaktiviert, NAS-Mount fehlt), gibt es keine Fehlermeldung.
|
||||||
|
- **Lösung via Kuma Push**: Der Orchestrator sendet am Ende eines Laufs aktiv ein Signal (`/api/push/<token>?status=up&msg=OK`). Bleibt dieses Signal nach 24 Stunden aus, schlägt Uptime Kuma automatisch Alarm ("Monitor DOWN").
|
||||||
|
- Einzelne kritische Jobs (z. B. `paperless-db`, `shinkan-prod-db`, `nas-data`) können zudem dedizierte Push-Monitore bedienen.
|
||||||
|
|
||||||
|
### B. Aktive Pull-Überwachung (Service Health)
|
||||||
|
- Uptime Kuma pollt im Regelbetrieb alle 60 Sekunden die Health-Endpoints (`ready_url`) der Container.
|
||||||
|
- Nach einem Staging-Restore durch den Orchestrator wird so sofort verifiziert, ob die Anwendung auch von außen im Netzwerk wieder uneingeschränkt antwortet.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Einrichtung in Uptime Kuma (Schritt für Schritt)
|
||||||
|
|
||||||
|
### Schritt 1: Push-Monitor für den täglichen Backup-Lauf anlegen
|
||||||
|
1. In Uptime Kuma auf **"Neuen Monitor hinzufügen"** klicken.
|
||||||
|
2. Folgende Werte eintragen:
|
||||||
|
- **Monitortyp**: `Push`
|
||||||
|
- **Name**: `Backup: Täglicher Gesamtlauf (Pi 5)` (oder `Backup: Gesamtlauf llm-node`)
|
||||||
|
- **Intervall (Heartbeat)**: `86400` Sekunden (24 Stunden)
|
||||||
|
- **Wiederholungsintervall**: `300` Sekunden (5 Minuten)
|
||||||
|
- **Max. Wiederholungen**: `2`
|
||||||
|
3. Speichern. Uptime Kuma zeigt nun eine Push-URL an:
|
||||||
|
```text
|
||||||
|
http://<uptime-kuma-host>:3001/api/push/abc123token?status=up&msg=OK&ping=
|
||||||
|
```
|
||||||
|
4. Den Basis-Teil der URL kopieren: `http://<uptime-kuma-host>:3001/api/push/abc123token`.
|
||||||
|
|
||||||
|
### Schritt 2 (Optional): Push-Monitore für Einzeljobs anlegen
|
||||||
|
Wiederhole den Vorgang für kritische Einzel-Jobs:
|
||||||
|
- `Backup: Shinkan DB (Phase 10)`
|
||||||
|
- `Backup: Paperless DB (Phase 10)`
|
||||||
|
- `Backup: NAS Data Sync (Phase 20)`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Konfiguration im Backup-Orchestrator
|
||||||
|
|
||||||
|
Auf dem jeweiligen Server (`raspberrypi5` oder `llm-node`) wird die Datei `/etc/backup-orchestrator/notifications.json` gepflegt:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"timeout_seconds": 3,
|
||||||
|
"webhooks": [],
|
||||||
|
"kuma": {
|
||||||
|
"overall_push_url": "http://192.168.2.xxx:3001/api/push/YOUR_OVERALL_TOKEN",
|
||||||
|
"job_push_urls": {
|
||||||
|
"shinkan-prod-db": "http://192.168.2.xxx:3001/api/push/YOUR_SHINKAN_TOKEN",
|
||||||
|
"paperless-db": "http://192.168.2.xxx:3001/api/push/YOUR_PAPERLESS_TOKEN",
|
||||||
|
"nas-data": "http://192.168.2.xxx:3001/api/push/YOUR_NAS_DATA_TOKEN"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
> [!IMPORTANT]
|
||||||
|
> - Dateirechte: Die Datei muss mit `chmod 0600 /etc/backup-orchestrator/notifications.json` geschützt werden.
|
||||||
|
> - Der Orchestrator akzeptiert sowohl die reine Basis-URL (`.../api/push/<token>`) als auch die von Kuma kopierte Beispiel-URL (`.../api/push/<token>?status=up&msg=OK`); Abfrageparameter werden automatisch bereinigt.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Testen der Uptime Kuma Push-Benachrichtigung
|
||||||
|
|
||||||
|
Um die Benachrichtigung zu testen, ohne ein vollständiges Backup auszulösen:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Trockenlauf / Status-Check (zeigt an, wie viele Kuma-URLs konfiguriert sind):
|
||||||
|
backup-orchestrator --check
|
||||||
|
|
||||||
|
# Testlauf für einen einzelnen Job:
|
||||||
|
BACKUP_ORCHESTRATOR_CUTOVER_APPROVED=YES backup-orchestrator --run --job shinkan-prod-db
|
||||||
|
```
|
||||||
|
|
||||||
|
In Uptime Kuma wechselt der Monitor sofort auf **"UP"** (grün) und zeigt im Ereignisprotokoll die Job-ID, den Laufzeit-Timestamp und die Run-ID an.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Fehlertoleranz & Logging
|
||||||
|
|
||||||
|
- **Entkopplung**: Push-Benachrichtigungen werden asynchron über einen Thread-Pool versendet.
|
||||||
|
- **Sicherheitsgarantie**: Netzwerk-Timeouts oder Fehler von Uptime Kuma verändern **niemals** den Exit-Code des Backup-Laufs. Ein erfolgreiches Backup bleibt immer `Exit 0`.
|
||||||
|
- Fehlerhafte Push-Versuche werden in `/var/log/backup-orchestrator/<run_id>/notifications.log` protokolliert.
|
||||||
|
|
||||||
@@ -6,8 +6,8 @@
|
|||||||
"nas_mount": "/mnt/nas_backup",
|
"nas_mount": "/mnt/nas_backup",
|
||||||
"nas_source": "192.168.2.63:/volume1/Backup_LLM",
|
"nas_source": "192.168.2.63:/volume1/Backup_LLM",
|
||||||
"nas_fstype": "nfs",
|
"nas_fstype": "nfs",
|
||||||
"local_dump_dir": "/mnt/nas_backup/shinkan/prod/db/latest",
|
"local_dump_dir": "/mnt/nas_backup/recovery_points/shinkan-prod/latest",
|
||||||
"archive_sets_dir": "/mnt/nas_backup/shinkan/prod/db",
|
"archive_sets_dir": "/mnt/nas_backup/recovery_points",
|
||||||
"media_mounts": {
|
"media_mounts": {
|
||||||
"shinkan-prod": "/srv/shinkan/prod/media"
|
"shinkan-prod": "/srv/shinkan/prod/media"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,2 +1,2 @@
|
|||||||
phase,id,script,critical,timeout_seconds,enabled
|
phase,id,script,critical,timeout_seconds,enabled
|
||||||
10,shinkan-prod-db,/etc/backup-orchestrator/scripts/db-shinkan-prod.sh,true,900,false
|
10,shinkan-prod-db,/etc/backup-orchestrator/scripts/db-shinkan-prod.sh,true,900,true
|
||||||
|
@@ -1,33 +1,68 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Backup wrapper for Shinkan Production on llm-node (post-cutover)
|
# ============================================================================
|
||||||
|
# Backup wrapper for Shinkan Production on llm-node (Ubuntu 24.04 x86_64)
|
||||||
|
# Captures PostgreSQL dump, creates schema manifest & sha256, and publishes
|
||||||
|
# atomic backup set to NAS recovery_points.
|
||||||
|
# ============================================================================
|
||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
|
umask 027
|
||||||
|
|
||||||
DEST_ROOT="/mnt/nas_backup/shinkan/prod/db"
|
ARCHIVE_ROOT="/mnt/nas_backup/recovery_points"
|
||||||
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
TARGET_ID="shinkan-prod"
|
||||||
GEN_DIR="${DEST_ROOT}/generations/${TS}"
|
DB_CONTAINER="shinkan-db-prod"
|
||||||
DUMP_FILE="${GEN_DIR}/shinkan-prod.sql"
|
DB_NAME="shinkan"
|
||||||
|
DB_USER="shinkan_user"
|
||||||
|
|
||||||
install -d -m 0750 "${GEN_DIR}"
|
# Verify NAS mount
|
||||||
|
if ! mountpoint -q /mnt/nas_backup; then
|
||||||
|
echo "FEHLER: /mnt/nas_backup ist nicht gemountet!" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
export DB_CONTAINER="shinkan-db-prod"
|
TMP_DIR="$(mktemp -d /tmp/shinkan-backup.XXXXXXXX)"
|
||||||
export DB_USER="shinkan"
|
cleanup() {
|
||||||
export DB_NAME="shinkan"
|
rm -rf -- "$TMP_DIR"
|
||||||
export OUTPUT_PATH="${DUMP_FILE}"
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
/etc/backup-orchestrator/scripts/db-postgres-docker.sh
|
DUMP_FILE="${TMP_DIR}/shinkan-prod.sql"
|
||||||
|
RESULT_FILE="${TMP_DIR}/manifest-result.json"
|
||||||
|
|
||||||
# Generate SHA256, Manifest (schemaVersion: 1), and Media Inventory (.media.jsonl)
|
echo "1. Erstelle PostgreSQL-Dump fuer ${DB_CONTAINER} (${DB_NAME})..."
|
||||||
/etc/backup-orchestrator/scripts/shinkan/shinkan-backup-manifest \
|
docker exec "${DB_CONTAINER}" sh -ec '
|
||||||
--dump "${DUMP_FILE}" \
|
export PGPASSWORD="${POSTGRES_PASSWORD:-}"
|
||||||
--env prod \
|
exec pg_dump -U "'"${DB_USER}"'" "'"${DB_NAME}"'"
|
||||||
--host "llm-node" \
|
' > "${DUMP_FILE}"
|
||||||
--compose-dir "/home/llmadmin/docker/shinkan-prod" \
|
|
||||||
--media-dir "/srv/shinkan/prod/media" \
|
|
||||||
--db-container "shinkan-db-prod" \
|
|
||||||
--db-name "shinkan" \
|
|
||||||
--db-user "shinkan"
|
|
||||||
|
|
||||||
# Atomically update 'latest' symlink after complete backup set verification
|
if [[ ! -s "${DUMP_FILE}" ]]; then
|
||||||
ln -sfn "generations/${TS}" "${DEST_ROOT}/latest.tmp.$$"
|
echo "FEHLER: Dump ist leer!" >&2
|
||||||
mv -Tf "${DEST_ROOT}/latest.tmp.$$" "${DEST_ROOT}/latest"
|
exit 1
|
||||||
echo "Published atomic Shinkan backup set on llm-node: ${GEN_DIR}"
|
fi
|
||||||
|
chmod 0640 "${DUMP_FILE}"
|
||||||
|
|
||||||
|
echo "2. Erstelle Manifest, SHA256 und Medien-Index..."
|
||||||
|
MANIFEST_SCRIPT="/etc/backup-orchestrator/scripts/shinkan/shinkan-backup-manifest"
|
||||||
|
if [[ ! -f "$MANIFEST_SCRIPT" ]]; then
|
||||||
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||||
|
MANIFEST_SCRIPT="${REPO_ROOT}/scripts/shinkan/shinkan-backup-manifest"
|
||||||
|
fi
|
||||||
|
|
||||||
|
python3 "${MANIFEST_SCRIPT}" \
|
||||||
|
--target "llm-prod" \
|
||||||
|
--dump "${DUMP_FILE}" \
|
||||||
|
--out-dir "${TMP_DIR}" \
|
||||||
|
--result "${RESULT_FILE}"
|
||||||
|
|
||||||
|
echo "3. Veroeffentliche Backup-Set im Recovery-Archiv..."
|
||||||
|
PUBLISH_SCRIPT="/etc/backup-orchestrator/scripts/publish-backup-set.sh"
|
||||||
|
if [[ ! -f "$PUBLISH_SCRIPT" ]]; then
|
||||||
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||||
|
PUBLISH_SCRIPT="${REPO_ROOT}/scripts/common/publish-backup-set.sh"
|
||||||
|
fi
|
||||||
|
|
||||||
|
bash "${PUBLISH_SCRIPT}" \
|
||||||
|
"${TARGET_ID}" \
|
||||||
|
"${DUMP_FILE}" \
|
||||||
|
"${ARCHIVE_ROOT}"
|
||||||
|
|
||||||
|
echo "OK: Shinkan Production Backup auf llm-node erfolgreich abgeschlossen."
|
||||||
+105
-26
@@ -8,67 +8,146 @@
|
|||||||
"nas_mount": "/mnt/nas_backup",
|
"nas_mount": "/mnt/nas_backup",
|
||||||
"nas_source": "192.168.2.63:/volume1/Backup_LLM",
|
"nas_source": "192.168.2.63:/volume1/Backup_LLM",
|
||||||
"nas_fstype": "nfs",
|
"nas_fstype": "nfs",
|
||||||
"local_dump_dir": "/mnt/nas_backup/shinkan/prod/db/latest",
|
"local_dump_dir": "/mnt/nas_backup/recovery_points/shinkan-pi-prod/latest",
|
||||||
"archive_sets_dir": "/mnt/nas_backup/shinkan/prod/db"
|
"archive_sets_dir": "/mnt/nas_backup/recovery_points"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"targets": {
|
"targets": {
|
||||||
"pi-prod": {
|
"pi-prod": {
|
||||||
"application": "shinkan",
|
"application": "shinkan",
|
||||||
"environment": "prod",
|
"environment": "production",
|
||||||
"host": "raspberrypi5",
|
"host": "raspberrypi5",
|
||||||
|
"arch": "aarch64",
|
||||||
"role": "source_only",
|
"role": "source_only",
|
||||||
"allow_restore": false,
|
"allow_restore": false,
|
||||||
"compose_dir": "/home/lars/docker/shinkan-prod",
|
"app_dir": "/home/lars/docker/shinkan",
|
||||||
|
"compose_project": "shinkan",
|
||||||
"compose_files": ["docker-compose.yml"],
|
"compose_files": ["docker-compose.yml"],
|
||||||
"env_file": "/home/lars/docker/shinkan-prod/.env",
|
"env_file": "/home/lars/docker/shinkan/.env",
|
||||||
"media_dir": "/shinkan-media/prod",
|
"db_engine": "postgresql",
|
||||||
"db_container": "shinkan-db-prod",
|
"db_container": "shinkan-db-prod",
|
||||||
"db_name": "shinkan",
|
"db_name": "shinkan",
|
||||||
"db_user": "shinkan",
|
"db_user": "shinkan_user",
|
||||||
"health_url": "http://localhost:8003/api/health/ready"
|
"media_host_dir": "/shinkan-media/prod",
|
||||||
|
"media_container_dir": "/app/media",
|
||||||
|
"version_url": "http://127.0.0.1:8003/api/version",
|
||||||
|
"ready_url": "http://127.0.0.1:8003/api/health/ready",
|
||||||
|
"adapter_dir": "/etc/backup-orchestrator/scripts/shinkan"
|
||||||
},
|
},
|
||||||
"llm-prod": {
|
"llm-prod": {
|
||||||
"application": "shinkan",
|
"application": "shinkan",
|
||||||
"environment": "prod",
|
"environment": "production",
|
||||||
"host": "llm-node",
|
"host": "llm-node",
|
||||||
|
"arch": "x86_64",
|
||||||
"role": "production_target",
|
"role": "production_target",
|
||||||
"allow_restore": true,
|
"allow_restore": true,
|
||||||
"require_confirmation": "shinkan@llm-node",
|
"require_confirm": "shinkan@llm-node",
|
||||||
"compose_dir": "/home/llmadmin/docker/shinkan-prod",
|
"app_dir": "/home/llmadmin/docker/shinkan-prod",
|
||||||
|
"compose_project": "shinkan-prod",
|
||||||
"compose_files": ["docker-compose.yml", "docker-compose.llm-node.yml"],
|
"compose_files": ["docker-compose.yml", "docker-compose.llm-node.yml"],
|
||||||
"env_file": "/etc/shinkan/prod.env",
|
"env_file": "/etc/shinkan/prod.env",
|
||||||
"media_dir": "/srv/shinkan/prod/media",
|
"db_engine": "postgresql",
|
||||||
"db_container": "shinkan-db-prod",
|
"db_container": "shinkan-db-prod",
|
||||||
"db_name": "shinkan",
|
"db_name": "shinkan",
|
||||||
"db_user": "shinkan",
|
"db_user": "shinkan_user",
|
||||||
"health_url": "http://localhost:8003/api/health/ready",
|
"media_host_dir": "/srv/shinkan/prod/media",
|
||||||
"restore_script": "/etc/backup-orchestrator/scripts/shinkan/shinkan-restore",
|
"media_container_dir": "/app/media",
|
||||||
"media_diff_script": "/etc/backup-orchestrator/scripts/shinkan/shinkan-media-diff"
|
"version_url": "http://127.0.0.1:8003/api/version",
|
||||||
|
"ready_url": "http://127.0.0.1:8003/api/health/ready",
|
||||||
|
"frontend_ready_url": "http://127.0.0.1:3003/api/health/ready",
|
||||||
|
"adapter_dir": "/etc/backup-orchestrator/scripts/shinkan",
|
||||||
|
"services_to_stop": ["backend", "frontend"]
|
||||||
},
|
},
|
||||||
"llm-restoretest": {
|
"llm-restoretest": {
|
||||||
"application": "shinkan",
|
"application": "shinkan",
|
||||||
"environment": "restoretest",
|
"environment": "restoretest",
|
||||||
"host": "llm-node",
|
"host": "llm-node",
|
||||||
|
"arch": "x86_64",
|
||||||
"role": "isolated_test_target",
|
"role": "isolated_test_target",
|
||||||
"allow_restore": true,
|
"allow_restore": true,
|
||||||
"compose_dir": "/home/llmadmin/docker/shinkan-prod",
|
"app_dir": "/home/llmadmin/docker/shinkan-prod",
|
||||||
|
"compose_project": "shinkan-restoretest",
|
||||||
"compose_files": [
|
"compose_files": [
|
||||||
"docker-compose.yml",
|
"docker-compose.yml",
|
||||||
"docker-compose.llm-node.yml",
|
"docker-compose.llm-node.yml",
|
||||||
"/etc/backup-orchestrator/scripts/shinkan/docker-compose.restoretest.yml"
|
"/etc/backup-orchestrator/scripts/shinkan/docker-compose.restoretest.yml"
|
||||||
],
|
],
|
||||||
"project_name": "shinkan-restoretest",
|
|
||||||
"env_file": "/etc/shinkan/prod.env",
|
"env_file": "/etc/shinkan/prod.env",
|
||||||
"media_dir": "/srv/shinkan/prod/media",
|
"db_engine": "postgresql",
|
||||||
|
"db_container": "shinkan-restoretest-postgres-1",
|
||||||
|
"db_name": "shinkan",
|
||||||
|
"db_user": "shinkan_user",
|
||||||
|
"media_host_dir": "/srv/shinkan/prod/media",
|
||||||
|
"media_container_dir": "/app/media",
|
||||||
"media_readonly": true,
|
"media_readonly": true,
|
||||||
"db_container": "shinkan-db-restoretest",
|
"version_url": "http://127.0.0.1:18003/api/version",
|
||||||
"db_name": "shinkan_restoretest",
|
"ready_url": "http://127.0.0.1:18003/api/health/ready",
|
||||||
"db_user": "shinkan",
|
"frontend_ready_url": "http://127.0.0.1:13003/api/health/ready",
|
||||||
"health_url": "http://127.0.0.1:18003/api/health/ready",
|
"adapter_dir": "/etc/backup-orchestrator/scripts/shinkan",
|
||||||
"frontend_url": "http://127.0.0.1:13003",
|
"services_to_stop": ["backend", "frontend"]
|
||||||
"restore_test_script": "/etc/backup-orchestrator/scripts/shinkan/shinkan-restore-test",
|
},
|
||||||
"media_diff_script": "/etc/backup-orchestrator/scripts/shinkan/shinkan-media-diff"
|
"kairo-llm-prod": {
|
||||||
|
"application": "kairo",
|
||||||
|
"environment": "production",
|
||||||
|
"host": "llm-node",
|
||||||
|
"arch": "x86_64",
|
||||||
|
"role": "production_target",
|
||||||
|
"allow_restore": true,
|
||||||
|
"require_confirm": "kairo@llm-node",
|
||||||
|
"app_dir": "/home/llmadmin/docker/kairo-prod",
|
||||||
|
"compose_project": "kairo-prod",
|
||||||
|
"compose_files": ["docker-compose.yml"],
|
||||||
|
"env_file": "/etc/kairo/prod.env",
|
||||||
|
"db_engine": "postgresql",
|
||||||
|
"db_container": "kairo-db-prod",
|
||||||
|
"db_name": "kairo",
|
||||||
|
"db_user": "kairo_user",
|
||||||
|
"ready_url": "http://127.0.0.1:8001/api/health/ready",
|
||||||
|
"services_to_stop": ["backend"],
|
||||||
|
"verify_tables_not_empty": ["django_migrations"],
|
||||||
|
"truncate_tables": ["django_session"]
|
||||||
|
},
|
||||||
|
"tandoor-llm-prod": {
|
||||||
|
"application": "tandoor",
|
||||||
|
"environment": "production",
|
||||||
|
"host": "llm-node",
|
||||||
|
"arch": "x86_64",
|
||||||
|
"role": "production_target",
|
||||||
|
"allow_restore": true,
|
||||||
|
"require_confirm": "tandoor@llm-node",
|
||||||
|
"app_dir": "/home/llmadmin/docker/tandoor-prod",
|
||||||
|
"compose_project": "tandoor-prod",
|
||||||
|
"compose_files": ["docker-compose.yml"],
|
||||||
|
"env_file": "/etc/tandoor/prod.env",
|
||||||
|
"db_engine": "postgresql",
|
||||||
|
"db_container": "tandoor-db-prod",
|
||||||
|
"db_name": "tandoordb",
|
||||||
|
"db_user": "tandoor_user",
|
||||||
|
"ready_url": "http://127.0.0.1:8080/",
|
||||||
|
"services_to_stop": ["web_app"],
|
||||||
|
"verify_tables_not_empty": ["django_migrations"],
|
||||||
|
"truncate_tables": ["django_session"]
|
||||||
|
},
|
||||||
|
"bookstack-llm-prod": {
|
||||||
|
"application": "bookstack",
|
||||||
|
"environment": "production",
|
||||||
|
"host": "llm-node",
|
||||||
|
"arch": "x86_64",
|
||||||
|
"role": "production_target",
|
||||||
|
"allow_restore": true,
|
||||||
|
"require_confirm": "bookstack@llm-node",
|
||||||
|
"app_dir": "/home/llmadmin/docker/bookstack-prod",
|
||||||
|
"compose_project": "bookstack-prod",
|
||||||
|
"compose_files": ["docker-compose.yml"],
|
||||||
|
"env_file": "/etc/bookstack/prod.env",
|
||||||
|
"db_engine": "mariadb",
|
||||||
|
"db_container": "bookstack-db-prod",
|
||||||
|
"db_name": "bookstack",
|
||||||
|
"db_user": "bookstack_user",
|
||||||
|
"ready_url": "http://127.0.0.1:6875/",
|
||||||
|
"services_to_stop": ["bookstack"],
|
||||||
|
"verify_tables_not_empty": ["migrations"],
|
||||||
|
"truncate_tables": ["sessions"]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,7 +3,7 @@ phase,id,script,critical,timeout_seconds,enabled
|
|||||||
10,tandoor-db,/etc/backup-orchestrator/scripts/db-tandoor.sh,true,600,true
|
10,tandoor-db,/etc/backup-orchestrator/scripts/db-tandoor.sh,true,600,true
|
||||||
10,mitai-prod-db,/etc/backup-orchestrator/scripts/db-mitai-prod.sh,true,900,true
|
10,mitai-prod-db,/etc/backup-orchestrator/scripts/db-mitai-prod.sh,true,900,true
|
||||||
10,mitai-dev-db,/etc/backup-orchestrator/scripts/db-mitai-dev.sh,false,900,true
|
10,mitai-dev-db,/etc/backup-orchestrator/scripts/db-mitai-dev.sh,false,900,true
|
||||||
10,shinkan-prod-db,/etc/backup-orchestrator/scripts/db-shinkan-prod.sh,true,600,true
|
10,shinkan-prod-db,/etc/backup-orchestrator/scripts/db-shinkan-prod.sh,true,600,false
|
||||||
10,shinkan-dev-db,/etc/backup-orchestrator/scripts/db-shinkan-dev.sh,false,600,true
|
10,shinkan-dev-db,/etc/backup-orchestrator/scripts/db-shinkan-dev.sh,false,600,true
|
||||||
20,nas-data,/etc/backup-orchestrator/scripts/nas-data-job.sh,true,10800,true
|
20,nas-data,/etc/backup-orchestrator/scripts/nas-data-job.sh,true,10800,true
|
||||||
30,system-image,/etc/backup-orchestrator/scripts/system-image-job.sh,true,14400,false
|
30,system-image,/etc/backup-orchestrator/scripts/system-image-job.sh,true,14400,false
|
||||||
|
|||||||
|
@@ -27,6 +27,36 @@
|
|||||||
"db_name": "shinkan",
|
"db_name": "shinkan",
|
||||||
"db_user": "shinkan",
|
"db_user": "shinkan",
|
||||||
"health_url": "http://localhost:8003/api/health/ready"
|
"health_url": "http://localhost:8003/api/health/ready"
|
||||||
|
},
|
||||||
|
"kairo-pi-prod": {
|
||||||
|
"application": "kairo",
|
||||||
|
"environment": "prod",
|
||||||
|
"host": "raspberrypi5",
|
||||||
|
"role": "source_only",
|
||||||
|
"allow_restore": false,
|
||||||
|
"db_engine": "postgresql",
|
||||||
|
"db_container": "kairo-db-prod",
|
||||||
|
"db_name": "kairo"
|
||||||
|
},
|
||||||
|
"tandoor-pi-prod": {
|
||||||
|
"application": "tandoor",
|
||||||
|
"environment": "prod",
|
||||||
|
"host": "raspberrypi5",
|
||||||
|
"role": "source_only",
|
||||||
|
"allow_restore": false,
|
||||||
|
"db_engine": "postgresql",
|
||||||
|
"db_container": "tandoor-db-prod",
|
||||||
|
"db_name": "tandoordb"
|
||||||
|
},
|
||||||
|
"bookstack-pi-prod": {
|
||||||
|
"application": "bookstack",
|
||||||
|
"environment": "prod",
|
||||||
|
"host": "raspberrypi5",
|
||||||
|
"role": "source_only",
|
||||||
|
"allow_restore": false,
|
||||||
|
"db_engine": "mariadb",
|
||||||
|
"db_container": "bookstack-db-prod",
|
||||||
|
"db_name": "bookstack"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+4
-2
@@ -73,8 +73,10 @@ if [[ ! -f /etc/backup-orchestrator/notifications.json ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# 4. Install shared common scripts + host-specific scripts + Shinkan adapters
|
# 4. Install shared common scripts + host-specific scripts + Shinkan adapters
|
||||||
for s in "${REPO_ROOT}/scripts/common/"*.sh; do
|
for s in "${REPO_ROOT}/scripts/common/"*; do
|
||||||
[[ -f "$s" ]] && install -m 0750 "$s" "/etc/backup-orchestrator/scripts/$(basename "$s")"
|
if [[ -f "$s" ]]; then
|
||||||
|
install -m 0755 "$s" "/etc/backup-orchestrator/scripts/$(basename "$s")"
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
for s in "${HOST_DIR}/scripts/"*.sh; do
|
for s in "${HOST_DIR}/scripts/"*.sh; do
|
||||||
|
|||||||
+129
-62
@@ -67,9 +67,12 @@ def lock_exclusive(fp):
|
|||||||
|
|
||||||
|
|
||||||
def resolve_repo_host_dir(host_name=None):
|
def resolve_repo_host_dir(host_name=None):
|
||||||
host = host_name or socket.gethostname().split('.')[0]
|
try:
|
||||||
candidate = Path(__file__).resolve().parent / 'hosts' / host
|
host = host_name or socket.gethostname().split('.')[0]
|
||||||
return candidate if candidate.is_dir() else None
|
candidate = Path(__file__).resolve().parent / 'hosts' / host
|
||||||
|
return candidate if candidate.is_dir() else None
|
||||||
|
except (OSError, PermissionError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def load_host_config(path=DEFAULT_HOST_CONFIG, host_name=None):
|
def load_host_config(path=DEFAULT_HOST_CONFIG, host_name=None):
|
||||||
@@ -95,12 +98,15 @@ def load_host_config(path=DEFAULT_HOST_CONFIG, host_name=None):
|
|||||||
if host_dir:
|
if host_dir:
|
||||||
candidates.append(host_dir / 'host.json')
|
candidates.append(host_dir / 'host.json')
|
||||||
for candidate in candidates:
|
for candidate in candidates:
|
||||||
if candidate.is_file():
|
try:
|
||||||
with candidate.open(encoding='utf-8') as fp:
|
if candidate.is_file():
|
||||||
data = json.load(fp)
|
with candidate.open(encoding='utf-8') as fp:
|
||||||
if isinstance(data, dict):
|
data = json.load(fp)
|
||||||
default.update(data)
|
if isinstance(data, dict):
|
||||||
break
|
default.update(data)
|
||||||
|
break
|
||||||
|
except (OSError, PermissionError):
|
||||||
|
continue
|
||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
||||||
@@ -117,9 +123,12 @@ def load_targets_config(path=DEFAULT_TARGETS_CONFIG, host_name=None):
|
|||||||
candidates.append(host_dir / 'targets.json')
|
candidates.append(host_dir / 'targets.json')
|
||||||
candidates.append(Path(__file__).resolve().parent / 'targets.json')
|
candidates.append(Path(__file__).resolve().parent / 'targets.json')
|
||||||
for candidate in candidates:
|
for candidate in candidates:
|
||||||
if candidate.is_file():
|
try:
|
||||||
with candidate.open(encoding='utf-8') as fp:
|
if candidate.is_file():
|
||||||
return json.load(fp)
|
with candidate.open(encoding='utf-8') as fp:
|
||||||
|
return json.load(fp)
|
||||||
|
except (OSError, PermissionError):
|
||||||
|
continue
|
||||||
return {'schemaVersion': 1, 'hosts': {}, 'targets': {}}
|
return {'schemaVersion': 1, 'hosts': {}, 'targets': {}}
|
||||||
|
|
||||||
|
|
||||||
@@ -218,10 +227,16 @@ def notification_counts(config):
|
|||||||
return len(config['webhooks']), int(bool(kuma['overall_push_url'])), sum(bool(v) for v in kuma['job_push_urls'].values())
|
return len(config['webhooks']), int(bool(kuma['overall_push_url'])), sum(bool(v) for v in kuma['job_push_urls'].values())
|
||||||
|
|
||||||
|
|
||||||
def nas_mount_ready(mountpoint=None, source=None, fstype=None):
|
def nas_mount_ready(mountpoint=None, source=None, fstype=None, host_cfg=None):
|
||||||
|
if host_cfg:
|
||||||
|
mountpoint = mountpoint or host_cfg.get('nas_mount')
|
||||||
|
source = source or host_cfg.get('nas_source')
|
||||||
|
fstype = fstype or host_cfg.get('nas_fstype')
|
||||||
mountpoint = mountpoint or NAS_MOUNT
|
mountpoint = mountpoint or NAS_MOUNT
|
||||||
source = source or NAS_SOURCE
|
source = source or NAS_SOURCE
|
||||||
fstype = fstype or 'nfs4'
|
fstype = fstype or 'nfs4'
|
||||||
|
if not mountpoint or not source:
|
||||||
|
return True
|
||||||
try:
|
try:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
['findmnt', '--mountpoint', mountpoint, '--types', fstype, '--source', source],
|
['findmnt', '--mountpoint', mountpoint, '--types', fstype, '--source', source],
|
||||||
@@ -232,31 +247,39 @@ def nas_mount_ready(mountpoint=None, source=None, fstype=None):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def legacy_schedules():
|
def legacy_schedules(host_cfg=None):
|
||||||
"""Stateful detection of active legacy schedules and conflicting running services."""
|
"""Stateful detection of active legacy schedules and conflicting running services configured in host.json."""
|
||||||
|
cfg = host_cfg or load_host_config()
|
||||||
|
cron_users = cfg.get('cron_users', ['root', 'lars'])
|
||||||
|
cron_pattern = cfg.get('conflict_cron_pattern', r'/home/lars/(?:db-[\w-]+\.sh|backupPaperlessData\.sh)')
|
||||||
|
conflict_services = cfg.get('conflict_services', ['raspiBackup.service'])
|
||||||
|
conflict_timers = cfg.get('conflict_timers', ['paperless-data-backup.timer', 'borgmatic.timer'])
|
||||||
|
|
||||||
found = []
|
found = []
|
||||||
for user in ('root', 'lars'):
|
if cron_pattern:
|
||||||
try:
|
for user in cron_users:
|
||||||
p = subprocess.run(['crontab', '-u', user, '-l'], capture_output=True, text=True, timeout=8)
|
try:
|
||||||
except (OSError, subprocess.TimeoutExpired):
|
p = subprocess.run(['crontab', '-u', user, '-l'], capture_output=True, text=True, timeout=8)
|
||||||
continue
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
for line in p.stdout.splitlines():
|
|
||||||
stripped = line.strip()
|
|
||||||
if stripped.startswith('#') or not stripped:
|
|
||||||
continue
|
continue
|
||||||
if re.search(r'/home/lars/(?:db-[\w-]+\.sh|backupPaperlessData\.sh)', stripped):
|
for line in p.stdout.splitlines():
|
||||||
found.append(f'{user} crontab: {stripped}')
|
stripped = line.strip()
|
||||||
# Stateful coordination with raspiBackup: only block if raspiBackup.service is currently active/running.
|
if stripped.startswith('#') or not stripped:
|
||||||
try:
|
continue
|
||||||
rb_active = subprocess.run(
|
if re.search(cron_pattern, stripped):
|
||||||
['systemctl', 'is-active', 'raspiBackup.service'],
|
found.append(f'{user} crontab: {stripped}')
|
||||||
capture_output=True, text=True, timeout=8,
|
# Stateful coordination with conflicting services (e.g. raspiBackup.service): only block if currently active/running.
|
||||||
).stdout.strip()
|
for service in conflict_services:
|
||||||
if rb_active in ('active', 'activating', 'reloading'):
|
try:
|
||||||
found.append(f'raspiBackup.service: active={rb_active}')
|
svc_active = subprocess.run(
|
||||||
except (OSError, subprocess.TimeoutExpired):
|
['systemctl', 'is-active', service],
|
||||||
pass
|
capture_output=True, text=True, timeout=8,
|
||||||
for timer in ('paperless-data-backup.timer', 'borgmatic.timer'):
|
).stdout.strip()
|
||||||
|
if svc_active in ('active', 'activating', 'reloading'):
|
||||||
|
found.append(f'{service}: active={svc_active}')
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
pass
|
||||||
|
for timer in conflict_timers:
|
||||||
try:
|
try:
|
||||||
enabled = subprocess.run(['systemctl', 'is-enabled', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
enabled = subprocess.run(['systemctl', 'is-enabled', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
||||||
active = subprocess.run(['systemctl', 'is-active', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
active = subprocess.run(['systemctl', 'is-active', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
||||||
@@ -267,8 +290,25 @@ def legacy_schedules():
|
|||||||
return found
|
return found
|
||||||
|
|
||||||
|
|
||||||
def script_issues(job):
|
def resolve_job_script(job, host_name=None):
|
||||||
p = Path(job['script'])
|
p = Path(job['script'])
|
||||||
|
if p.is_file():
|
||||||
|
return p
|
||||||
|
repo_root = Path(__file__).resolve().parent
|
||||||
|
script_name = p.name
|
||||||
|
host = host_name or socket.gethostname()
|
||||||
|
for c in (
|
||||||
|
repo_root / 'hosts' / host / 'scripts' / script_name,
|
||||||
|
repo_root / 'scripts' / 'common' / script_name,
|
||||||
|
repo_root / 'scripts' / script_name,
|
||||||
|
):
|
||||||
|
if c.is_file():
|
||||||
|
return c
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def script_issues(job, host_name=None):
|
||||||
|
p = resolve_job_script(job, host_name)
|
||||||
if not p.is_file():
|
if not p.is_file():
|
||||||
return ['MISSING']
|
return ['MISSING']
|
||||||
issues = []
|
issues = []
|
||||||
@@ -283,20 +323,23 @@ def script_issues(job):
|
|||||||
return issues
|
return issues
|
||||||
|
|
||||||
|
|
||||||
def check(jobs, max_parallel, notify):
|
def check(jobs, max_parallel, notify, host_cfg=None):
|
||||||
print(f'Backup-Orchestrator {VERSION}: INSPECTION ONLY (no backups, no schedule changes)')
|
cfg = host_cfg or load_host_config()
|
||||||
|
mountpoint = cfg.get('nas_mount', NAS_MOUNT)
|
||||||
|
source = cfg.get('nas_source', NAS_SOURCE)
|
||||||
|
print(f'Backup-Orchestrator {VERSION} (host={cfg.get("hostname", socket.gethostname())}): INSPECTION ONLY (no backups, no schedule changes)')
|
||||||
print(f'Parallel limit per phase: {max_parallel}')
|
print(f'Parallel limit per phase: {max_parallel}')
|
||||||
for job in jobs:
|
for job in jobs:
|
||||||
issues = script_issues(job)
|
issues = script_issues(job, cfg.get('hostname'))
|
||||||
state = 'enabled' if job['enabled'] else 'disabled'
|
state = 'enabled' if job['enabled'] else 'disabled'
|
||||||
print(f"phase {job['phase']:>2} {job['id']:<24} {state:<8} {job['timeout_seconds']:>5}s {job['script']} [{', '.join(issues) if issues else 'OK'}]")
|
print(f"phase {job['phase']:>2} {job['id']:<24} {state:<8} {job['timeout_seconds']:>5}s {job['script']} [{', '.join(issues) if issues else 'OK'}]")
|
||||||
conflicts = legacy_schedules()
|
conflicts = legacy_schedules(cfg)
|
||||||
print('\nLegacy schedules:')
|
print('\nLegacy schedules:')
|
||||||
for line in conflicts:
|
for line in conflicts:
|
||||||
print(' ACTIVE: ' + line)
|
print(' ACTIVE: ' + line)
|
||||||
if not conflicts:
|
if not conflicts:
|
||||||
print(' None detected (best effort)')
|
print(' None detected (best effort)')
|
||||||
print(f'\nNAS mount {NAS_MOUNT}: {"OK" if nas_mount_ready() else "NOT AVAILABLE"} (expected {NAS_SOURCE})')
|
print(f'\nNAS mount {mountpoint}: {"OK" if nas_mount_ready(host_cfg=cfg) else "NOT AVAILABLE"} (expected {source})')
|
||||||
a, b, c = notification_counts(notify)
|
a, b, c = notification_counts(notify)
|
||||||
print(f'Notifications configured: webhooks={a}, Kuma overall={b}, Kuma jobs={c} (tokens hidden)')
|
print(f'Notifications configured: webhooks={a}, Kuma overall={b}, Kuma jobs={c} (tokens hidden)')
|
||||||
print('Same phase: parallel; next phase starts after all members complete.')
|
print('Same phase: parallel; next phase starts after all members complete.')
|
||||||
@@ -517,12 +560,13 @@ def execute_one(job, folder, hub):
|
|||||||
hub.emit('JOB_CANCELLED', **result)
|
hub.emit('JOB_CANCELLED', **result)
|
||||||
return result
|
return result
|
||||||
hub.emit('JOB_STARTED', job=job['id'], phase=job['phase'], critical=job['critical'], state='running')
|
hub.emit('JOB_STARTED', job=job['id'], phase=job['phase'], critical=job['critical'], state='running')
|
||||||
|
resolved_script = resolve_job_script(job)
|
||||||
with logfile.open('wb') as stream:
|
with logfile.open('wb') as stream:
|
||||||
stream.write((f"job={job['id']} started={now_iso()}\n").encode())
|
stream.write((f"job={job['id']} started={now_iso()}\n").encode())
|
||||||
stream.flush()
|
stream.flush()
|
||||||
try:
|
try:
|
||||||
proc = subprocess.Popen(_build_script_cmd(job['script']), stdout=stream, stderr=subprocess.STDOUT,
|
proc = subprocess.Popen(_build_script_cmd(resolved_script), stdout=stream, stderr=subprocess.STDOUT,
|
||||||
start_new_session=True, cwd=str(Path(job['script']).parent))
|
start_new_session=True, cwd=str(resolved_script.parent))
|
||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
stream.write((f'EXEC ERROR: {exc}\n').encode())
|
stream.write((f'EXEC ERROR: {exc}\n').encode())
|
||||||
result = result_for(job, 'error', 127, time.monotonic()-start, logfile, 'Could not start script')
|
result = result_for(job, 'error', 127, time.monotonic()-start, logfile, 'Could not start script')
|
||||||
@@ -551,17 +595,20 @@ def execute_one(job, folder, hub):
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def live_run(jobs, args):
|
def live_run(jobs, args, host_cfg=None):
|
||||||
if not jobs:
|
if not jobs:
|
||||||
print('Nothing enabled; no backups executed.')
|
print('Nothing enabled; no backups executed.')
|
||||||
return 0
|
return 0
|
||||||
if os.environ.get(CUTOVER_ENV) != 'YES':
|
if os.environ.get(CUTOVER_ENV) != 'YES':
|
||||||
print(f'REFUSED: Set {CUTOVER_ENV}=YES only after testing scripts and coordinating existing schedules.', file=sys.stderr)
|
print(f'REFUSED: Set {CUTOVER_ENV}=YES only after testing scripts and coordinating existing schedules.', file=sys.stderr)
|
||||||
return 2
|
return 2
|
||||||
if not nas_mount_ready():
|
cfg = host_cfg or load_host_config(host_name=getattr(args, 'host', None))
|
||||||
print(f'REFUSED: {NAS_MOUNT} is not mounted from {NAS_SOURCE}', file=sys.stderr)
|
mountpoint = cfg.get('nas_mount', NAS_MOUNT)
|
||||||
|
source = cfg.get('nas_source', NAS_SOURCE)
|
||||||
|
if not nas_mount_ready(host_cfg=cfg):
|
||||||
|
print(f'REFUSED: {mountpoint} is not mounted from {source}', file=sys.stderr)
|
||||||
return 2
|
return 2
|
||||||
conflicts = legacy_schedules()
|
conflicts = legacy_schedules() if host_cfg is None else legacy_schedules(cfg)
|
||||||
if conflicts:
|
if conflicts:
|
||||||
print('REFUSED: existing scheduled backups could overlap this run:', file=sys.stderr)
|
print('REFUSED: existing scheduled backups could overlap this run:', file=sys.stderr)
|
||||||
for issue in conflicts:
|
for issue in conflicts:
|
||||||
@@ -1071,9 +1118,10 @@ def check_compatibility(verification, target_name, targets_cfg, current_host=Non
|
|||||||
dump_migrations = extract_migrations_from_sql_dump(verification['dump_path'])
|
dump_migrations = extract_migrations_from_sql_dump(verification['dump_path'])
|
||||||
|
|
||||||
if code_migrations is None:
|
if code_migrations is None:
|
||||||
app_dir = Path(tcfg.get('app_dir', ''))
|
app_dir = Path(tcfg.get('app_dir', tcfg.get('compose_dir', '')))
|
||||||
mig_dir = app_dir / 'backend' / 'migrations'
|
mig_rel = tcfg.get('migrations_dir', 'backend/migrations')
|
||||||
if mig_dir.is_dir():
|
mig_dir = app_dir / mig_rel if app_dir and str(app_dir) != '.' else Path('')
|
||||||
|
if mig_dir and mig_dir.is_dir():
|
||||||
code_migrations = discover_code_migrations(mig_dir)
|
code_migrations = discover_code_migrations(mig_dir)
|
||||||
|
|
||||||
schema_relation = 'UNKNOWN'
|
schema_relation = 'UNKNOWN'
|
||||||
@@ -1324,13 +1372,15 @@ def execute_recovery_command(mode, args, targets_cfg):
|
|||||||
result_file = folder / 'adapter-result.json'
|
result_file = folder / 'adapter-result.json'
|
||||||
log_file = folder / 'recovery.log'
|
log_file = folder / 'recovery.log'
|
||||||
|
|
||||||
adapter_dir = Path(tcfg.get('adapter_dir', '/etc/backup-orchestrator/scripts/shinkan'))
|
app_id = tcfg.get('application', 'app')
|
||||||
|
adapter_dir = Path(tcfg.get('adapter_dir', f'/etc/backup-orchestrator/scripts/{app_id}'))
|
||||||
if not adapter_dir.is_dir():
|
if not adapter_dir.is_dir():
|
||||||
adapter_dir = Path(__file__).resolve().parent / 'scripts' / 'shinkan'
|
adapter_dir = Path(__file__).resolve().parent / 'scripts' / app_id
|
||||||
|
|
||||||
dump_path = plan_doc['recovery_point']['dump_path']
|
dump_path = plan_doc['recovery_point']['dump_path']
|
||||||
if mode == 'restore-test':
|
if mode == 'restore-test':
|
||||||
script = adapter_dir / 'shinkan-restore-test'
|
custom_script = tcfg.get('restore_test_script')
|
||||||
|
script = Path(custom_script) if custom_script and Path(custom_script).is_file() else (adapter_dir / f'{app_id}-restore-test')
|
||||||
extra = ['--dump', dump_path, '--result', str(result_file)]
|
extra = ['--dump', dump_path, '--result', str(result_file)]
|
||||||
if getattr(args, 'playwright', False):
|
if getattr(args, 'playwright', False):
|
||||||
extra.append('--playwright')
|
extra.append('--playwright')
|
||||||
@@ -1338,7 +1388,17 @@ def execute_recovery_command(mode, args, targets_cfg):
|
|||||||
extra.append('--keep')
|
extra.append('--keep')
|
||||||
cmd = _build_script_cmd(script, extra)
|
cmd = _build_script_cmd(script, extra)
|
||||||
else:
|
else:
|
||||||
script = adapter_dir / 'shinkan-restore'
|
custom_script = tcfg.get('restore_script')
|
||||||
|
if custom_script and Path(custom_script).is_file():
|
||||||
|
script = Path(custom_script)
|
||||||
|
elif (adapter_dir / f'{app_id}-restore').is_file():
|
||||||
|
script = adapter_dir / f'{app_id}-restore'
|
||||||
|
else:
|
||||||
|
candidates = [
|
||||||
|
Path('/etc/backup-orchestrator/scripts/universal-db-restore'),
|
||||||
|
Path(__file__).resolve().parent / 'scripts' / 'common' / 'universal-db-restore',
|
||||||
|
]
|
||||||
|
script = next((c for c in candidates if c.is_file()), adapter_dir / f'{app_id}-restore')
|
||||||
extra = [
|
extra = [
|
||||||
'--target', target_name, '--dump', dump_path,
|
'--target', target_name, '--dump', dump_path,
|
||||||
'--result', str(result_file), '--sessions', args.sessions,
|
'--result', str(result_file), '--sessions', args.sessions,
|
||||||
@@ -1448,17 +1508,24 @@ def main():
|
|||||||
parser.add_argument('--keep', action='store_true', help='Keep isolated test environment running after --restore-test')
|
parser.add_argument('--keep', action='store_true', help='Keep isolated test environment running after --restore-test')
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
if args.host:
|
host_name = args.host or socket.gethostname()
|
||||||
host_dir = resolve_repo_host_dir(args.host)
|
host_dir = resolve_repo_host_dir(host_name)
|
||||||
if not host_dir:
|
if args.host and not host_dir:
|
||||||
parser.error(f'Unknown host profile: {args.host!r}')
|
parser.error(f'Unknown host profile: {args.host!r}')
|
||||||
if args.config == DEFAULT_CONFIG:
|
if host_dir:
|
||||||
|
if args.config == DEFAULT_CONFIG and not DEFAULT_CONFIG.is_file():
|
||||||
args.config = host_dir / 'jobs.csv'
|
args.config = host_dir / 'jobs.csv'
|
||||||
if args.notify_config == DEFAULT_NOTIFY_CONFIG and (host_dir / 'notifications.json.example').is_file():
|
if args.notify_config == DEFAULT_NOTIFY_CONFIG and not DEFAULT_NOTIFY_CONFIG.is_file() and (host_dir / 'notifications.json.example').is_file():
|
||||||
args.notify_config = host_dir / 'notifications.json.example'
|
args.notify_config = host_dir / 'notifications.json.example'
|
||||||
if args.targets_config == DEFAULT_TARGETS_CONFIG and (host_dir / 'targets.json').is_file():
|
if args.targets_config == DEFAULT_TARGETS_CONFIG and not DEFAULT_TARGETS_CONFIG.is_file() and (host_dir / 'targets.json').is_file():
|
||||||
args.targets_config = host_dir / 'targets.json'
|
args.targets_config = host_dir / 'targets.json'
|
||||||
|
|
||||||
|
# Fallback to user directories if /var/log or /run/lock are not writable
|
||||||
|
if args.log_root == DEFAULT_LOG_ROOT and not os.access('/var/log', os.W_OK) and not DEFAULT_LOG_ROOT.is_dir():
|
||||||
|
args.log_root = Path.home() / '.local' / 'share' / 'backup-orchestrator' / 'logs'
|
||||||
|
if args.lock_file == DEFAULT_LOCK_FILE and not os.access('/run/lock', os.W_OK) and not DEFAULT_LOCK_FILE.is_file():
|
||||||
|
args.lock_file = Path.home() / '.local' / 'share' / 'backup-orchestrator' / 'lock'
|
||||||
|
|
||||||
if not 1 <= args.max_parallel <= 16:
|
if not 1 <= args.max_parallel <= 16:
|
||||||
parser.error('--max-parallel must be 1..16')
|
parser.error('--max-parallel must be 1..16')
|
||||||
json_capable = (
|
json_capable = (
|
||||||
|
|||||||
@@ -55,11 +55,16 @@ if [[ -f "$MEDIA_FILE" ]]; then
|
|||||||
cp -p -- "$MEDIA_FILE" "$STAGING_DIR/"
|
cp -p -- "$MEDIA_FILE" "$STAGING_DIR/"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Ensure all backup artifacts are readable across hosts (e.g. llmadmin on llm-node)
|
||||||
|
chmod 0755 "$STAGING_DIR"
|
||||||
|
chmod 0644 "$STAGING_DIR"/*
|
||||||
|
|
||||||
mv -T -- "$STAGING_DIR" "$FINAL_DIR"
|
mv -T -- "$STAGING_DIR" "$FINAL_DIR"
|
||||||
STAGING_DIR=""
|
STAGING_DIR=""
|
||||||
|
|
||||||
|
# Use relative symlink so the latest link works regardless of mount point path on different hosts
|
||||||
TEMP_LINK="$TARGET_DIR/.latest-$$"
|
TEMP_LINK="$TARGET_DIR/.latest-$$"
|
||||||
ln -s -- "$FINAL_DIR" "$TEMP_LINK"
|
ln -s -- "$TIMESTAMP" "$TEMP_LINK"
|
||||||
mv -Tf -- "$TEMP_LINK" "$TARGET_DIR/latest"
|
mv -Tf -- "$TEMP_LINK" "$TARGET_DIR/latest"
|
||||||
|
|
||||||
echo "OK: Backup-Satz veroeffentlicht unter $FINAL_DIR (latest aktualisiert)"
|
echo "OK: Backup-Satz veroeffentlicht unter $FINAL_DIR (latest aktualisiert)"
|
||||||
|
|||||||
@@ -0,0 +1,536 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Universal transactional database restore adapter for PostgreSQL and MariaDB/MySQL.
|
||||||
|
|
||||||
|
Supports Docker-based deployments with:
|
||||||
|
- Staging database isolation and integrity verification prior to live cutover.
|
||||||
|
- Atomic database/table swap.
|
||||||
|
- Reversible rollback if post-restore health checks fail.
|
||||||
|
- Plausibility checks (non-empty tables, migration checks).
|
||||||
|
- Session truncation on cutover.
|
||||||
|
- Fully parameter-driven via targets.json.
|
||||||
|
|
||||||
|
Exit codes:
|
||||||
|
- 0: Restore and post-restore checks succeeded
|
||||||
|
- 1: Execution error
|
||||||
|
- 2: Precondition / confirmation / host guard error
|
||||||
|
- 3: Schema incompatibility
|
||||||
|
- 4: Checksum mismatch
|
||||||
|
- 5: Post-restore healthcheck failed (automatic rollback executed)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import datetime as dt
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import platform
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from urllib.request import urlopen
|
||||||
|
|
||||||
|
EXIT_OK = 0
|
||||||
|
EXIT_EXEC_ERROR = 1
|
||||||
|
EXIT_PRECONDITION = 2
|
||||||
|
EXIT_SCHEMA_INCOMPATIBLE = 3
|
||||||
|
EXIT_CHECKSUM_MISMATCH = 4
|
||||||
|
EXIT_POST_RESTORE_FAILED = 5
|
||||||
|
|
||||||
|
|
||||||
|
def compute_sha256(path):
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with Path(path).open('rb') as fp:
|
||||||
|
while True:
|
||||||
|
chunk = fp.read(65536)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
h.update(chunk)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def detect_dump_format(path):
|
||||||
|
with Path(path).open('rb') as fp:
|
||||||
|
magic = fp.read(5)
|
||||||
|
if magic == b'PGDMP':
|
||||||
|
return 'pg_custom'
|
||||||
|
return 'plain_sql'
|
||||||
|
|
||||||
|
|
||||||
|
def write_result(path, data):
|
||||||
|
if path:
|
||||||
|
p = Path(path)
|
||||||
|
p.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
p.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||||
|
|
||||||
|
|
||||||
|
def load_targets(custom_file=None):
|
||||||
|
candidates = []
|
||||||
|
if custom_file:
|
||||||
|
candidates.append(Path(custom_file))
|
||||||
|
candidates.extend([
|
||||||
|
Path('/etc/backup-orchestrator/targets.json'),
|
||||||
|
Path(__file__).resolve().parents[2] / 'targets.json',
|
||||||
|
Path(__file__).resolve().parents[2] / f'hosts/{socket.gethostname()}/targets.json',
|
||||||
|
])
|
||||||
|
for c in candidates:
|
||||||
|
if c.is_file():
|
||||||
|
try:
|
||||||
|
data = json.loads(c.read_text(encoding='utf-8'))
|
||||||
|
if 'targets' in data:
|
||||||
|
return data
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
continue
|
||||||
|
return {'targets': {}}
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# PostgreSQL Database Helpers
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def psql_exec(tcfg, dbname, sql):
|
||||||
|
cmd = [
|
||||||
|
'docker', 'exec', '-i', tcfg['db_container'],
|
||||||
|
'sh', '-ec',
|
||||||
|
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -v ON_ERROR_STOP=1 -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}" -At',
|
||||||
|
]
|
||||||
|
return subprocess.run(cmd, input=sql, capture_output=True, text=True, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_drop_db_if_exists(tcfg, dbname):
|
||||||
|
term_sql = f"""
|
||||||
|
SELECT pg_terminate_backend(pid)
|
||||||
|
FROM pg_stat_activity
|
||||||
|
WHERE datname = '{dbname}' AND pid <> pg_backend_pid();
|
||||||
|
"""
|
||||||
|
psql_exec(tcfg, 'postgres', term_sql)
|
||||||
|
psql_exec(tcfg, 'postgres', f'DROP DATABASE IF EXISTS "{dbname}";')
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_create_db(tcfg, dbname):
|
||||||
|
postgres_drop_db_if_exists(tcfg, dbname)
|
||||||
|
res = psql_exec(tcfg, 'postgres', f'CREATE DATABASE "{dbname}";')
|
||||||
|
return res.returncode == 0, res.stderr.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_import_dump(tcfg, dbname, dump_path):
|
||||||
|
fmt = detect_dump_format(dump_path)
|
||||||
|
if fmt == 'pg_custom':
|
||||||
|
cmd = [
|
||||||
|
'docker', 'exec', '-i', tcfg['db_container'],
|
||||||
|
'sh', '-ec',
|
||||||
|
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec pg_restore -v --single-transaction --exit-on-error -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}"',
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
cmd = [
|
||||||
|
'docker', 'exec', '-i', tcfg['db_container'],
|
||||||
|
'sh', '-ec',
|
||||||
|
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -v ON_ERROR_STOP=1 --single-transaction -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}"',
|
||||||
|
]
|
||||||
|
with Path(dump_path).open('rb') as fp:
|
||||||
|
res = subprocess.run(cmd, stdin=fp, capture_output=True, text=True, timeout=1800)
|
||||||
|
return res.returncode == 0, res.stderr.strip() or res.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_swap_databases(tcfg, live_db, staging_db, backup_db):
|
||||||
|
term_sql = f"""
|
||||||
|
SELECT pg_terminate_backend(pid)
|
||||||
|
FROM pg_stat_activity
|
||||||
|
WHERE datname IN ('{live_db}', '{staging_db}', '{backup_db}') AND pid <> pg_backend_pid();
|
||||||
|
"""
|
||||||
|
psql_exec(tcfg, 'postgres', term_sql)
|
||||||
|
# 1. Rename live to backup
|
||||||
|
r1 = psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{live_db}" RENAME TO "{backup_db}";')
|
||||||
|
if r1.returncode != 0:
|
||||||
|
return False, f"Rename {live_db} -> {backup_db} failed: {r1.stderr.strip()}"
|
||||||
|
# 2. Rename staging to live
|
||||||
|
r2 = psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{staging_db}" RENAME TO "{live_db}";')
|
||||||
|
if r2.returncode != 0:
|
||||||
|
# Emergency rollback
|
||||||
|
psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{backup_db}" RENAME TO "{live_db}";')
|
||||||
|
return False, f"Rename {staging_db} -> {live_db} failed: {r2.stderr.strip()}"
|
||||||
|
return True, "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def postgres_rollback_swap(tcfg, live_db, staging_db, backup_db):
|
||||||
|
term_sql = f"""
|
||||||
|
SELECT pg_terminate_backend(pid)
|
||||||
|
FROM pg_stat_activity
|
||||||
|
WHERE datname IN ('{live_db}', '{staging_db}', '{backup_db}') AND pid <> pg_backend_pid();
|
||||||
|
"""
|
||||||
|
psql_exec(tcfg, 'postgres', term_sql)
|
||||||
|
psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{live_db}" RENAME TO "{staging_db}";')
|
||||||
|
psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{backup_db}" RENAME TO "{live_db}";')
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# MariaDB / MySQL Database Helpers
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def mariadb_exec(tcfg, dbname, sql):
|
||||||
|
db_flag = f'"{dbname}"' if dbname else ''
|
||||||
|
cmd = [
|
||||||
|
'docker', 'exec', '-i', tcfg['db_container'],
|
||||||
|
'sh', '-ec',
|
||||||
|
f'export MYSQL_PWD="${{MARIADB_PASSWORD:-${{MYSQL_PASSWORD:-}}}}"; '
|
||||||
|
f'exec mariadb --default-character-set=utf8mb4 -u "${{MARIADB_USER:-${{MYSQL_USER:-root}}}}" {db_flag} -sN',
|
||||||
|
]
|
||||||
|
return subprocess.run(cmd, input=sql, capture_output=True, text=True, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
def mariadb_drop_db_if_exists(tcfg, dbname):
|
||||||
|
mariadb_exec(tcfg, '', f'DROP DATABASE IF EXISTS `{dbname}`;')
|
||||||
|
|
||||||
|
|
||||||
|
def mariadb_create_db(tcfg, dbname):
|
||||||
|
mariadb_drop_db_if_exists(tcfg, dbname)
|
||||||
|
res = mariadb_exec(tcfg, '', f'CREATE DATABASE `{dbname}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;')
|
||||||
|
return res.returncode == 0, res.stderr.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def mariadb_import_dump(tcfg, dbname, dump_path, original_dbname):
|
||||||
|
"""Import MariaDB dump into staging DB, rewriting CREATE/USE database references if needed."""
|
||||||
|
cmd = [
|
||||||
|
'docker', 'exec', '-i', tcfg['db_container'],
|
||||||
|
'sh', '-ec',
|
||||||
|
f'export MYSQL_PWD="${{MARIADB_PASSWORD:-${{MYSQL_PASSWORD:-}}}}"; '
|
||||||
|
f'exec mariadb --default-character-set=utf8mb4 -u "${{MARIADB_USER:-${{MYSQL_USER:-root}}}}" `{dbname}`',
|
||||||
|
]
|
||||||
|
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||||
|
|
||||||
|
# Stream and rewrite USE `original_dbname` / CREATE DATABASE statements to staging db
|
||||||
|
re_use = re.compile(rf'^\s*USE\s+[`"\']?{re.escape(original_dbname)}[`"\']?\s*;', re.IGNORECASE)
|
||||||
|
re_create = re.compile(rf'^\s*CREATE\s+DATABASE.*?[`"\']?{re.escape(original_dbname)}[`"\']?.*?;', re.IGNORECASE)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with Path(dump_path).open('r', encoding='utf-8', errors='replace') as fp:
|
||||||
|
for line in fp:
|
||||||
|
if re_use.match(line):
|
||||||
|
proc.stdin.write(f'USE `{dbname}`;\n')
|
||||||
|
elif re_create.match(line):
|
||||||
|
proc.stdin.write(f'-- {line.strip()}\n')
|
||||||
|
else:
|
||||||
|
proc.stdin.write(line)
|
||||||
|
stdout, stderr = proc.communicate(timeout=1800)
|
||||||
|
return proc.returncode == 0, stderr.strip() or stdout.strip()
|
||||||
|
except Exception as exc:
|
||||||
|
proc.kill()
|
||||||
|
return False, str(exc)
|
||||||
|
|
||||||
|
|
||||||
|
def mariadb_get_tables(tcfg, dbname):
|
||||||
|
sql = f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{dbname}' AND table_type = 'BASE TABLE';"
|
||||||
|
res = mariadb_exec(tcfg, '', sql)
|
||||||
|
if res.returncode != 0:
|
||||||
|
return []
|
||||||
|
return [line.strip() for line in res.stdout.splitlines() if line.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def mariadb_swap_databases(tcfg, live_db, staging_db, backup_db):
|
||||||
|
"""Atomically swap tables between schemas using MySQL/MariaDB RENAME TABLE."""
|
||||||
|
mariadb_create_db(tcfg, backup_db)
|
||||||
|
live_tables = set(mariadb_get_tables(tcfg, live_db))
|
||||||
|
staging_tables = set(mariadb_get_tables(tcfg, staging_db))
|
||||||
|
|
||||||
|
rename_clauses = []
|
||||||
|
# Move current live tables to backup_db
|
||||||
|
for tbl in live_tables:
|
||||||
|
rename_clauses.append(f"`{live_db}`.`{tbl}` TO `{backup_db}`.`{tbl}`")
|
||||||
|
# Move staging tables to live_db
|
||||||
|
for tbl in staging_tables:
|
||||||
|
rename_clauses.append(f"`{staging_db}`.`{tbl}` TO `{live_db}`.`{tbl}`")
|
||||||
|
|
||||||
|
if not rename_clauses:
|
||||||
|
return False, "No tables to rename in swap operation"
|
||||||
|
|
||||||
|
rename_sql = "RENAME TABLE " + ", ".join(rename_clauses) + ";"
|
||||||
|
res = mariadb_exec(tcfg, '', rename_sql)
|
||||||
|
if res.returncode != 0:
|
||||||
|
return False, f"Atomic table rename failed: {res.stderr.strip()}"
|
||||||
|
return True, "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def mariadb_rollback_swap(tcfg, live_db, staging_db, backup_db):
|
||||||
|
live_tables = set(mariadb_get_tables(tcfg, live_db))
|
||||||
|
backup_tables = set(mariadb_get_tables(tcfg, backup_db))
|
||||||
|
rename_clauses = []
|
||||||
|
for tbl in live_tables:
|
||||||
|
rename_clauses.append(f"`{live_db}`.`{tbl}` TO `{staging_db}`.`{tbl}`")
|
||||||
|
for tbl in backup_tables:
|
||||||
|
rename_clauses.append(f"`{backup_db}`.`{tbl}` TO `{live_db}`.`{tbl}`")
|
||||||
|
if rename_clauses:
|
||||||
|
mariadb_exec(tcfg, '', "RENAME TABLE " + ", ".join(rename_clauses) + ";")
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# Common Compose & Healthcheck Functions
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def compose_cmd(tcfg, *args):
|
||||||
|
cmd = ['docker', 'compose']
|
||||||
|
if tcfg.get('compose_project'):
|
||||||
|
cmd.extend(['-p', tcfg['compose_project']])
|
||||||
|
for cf in tcfg.get('compose_files', ['docker-compose.yml']):
|
||||||
|
cf_path = Path(cf) if Path(cf).is_absolute() else (Path(tcfg.get('app_dir', '.')) / cf)
|
||||||
|
if cf_path.is_file():
|
||||||
|
cmd.extend(['-f', str(cf_path)])
|
||||||
|
if tcfg.get('env_file') and Path(tcfg['env_file']).is_file():
|
||||||
|
cmd.extend(['--env-file', tcfg['env_file']])
|
||||||
|
cmd.extend(args)
|
||||||
|
cwd = tcfg.get('app_dir') if tcfg.get('app_dir') and Path(tcfg['app_dir']).is_dir() else None
|
||||||
|
return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=300)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_ready(url, timeout_sec=180):
|
||||||
|
deadline = time.monotonic() + timeout_sec
|
||||||
|
last_body = None
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
with urlopen(url, timeout=5) as resp:
|
||||||
|
raw = resp.read(8192).decode('utf-8', errors='replace')
|
||||||
|
if resp.status == 200:
|
||||||
|
try:
|
||||||
|
parsed = json.loads(raw)
|
||||||
|
# If JSON contains 'status', verify it is 'ok' or 'ready'
|
||||||
|
if isinstance(parsed, dict) and 'status' in parsed:
|
||||||
|
if parsed['status'] in ('ready', 'ok'):
|
||||||
|
return True, parsed
|
||||||
|
else:
|
||||||
|
return True, {'raw': raw[:200]}
|
||||||
|
except (ValueError, json.JSONDecodeError):
|
||||||
|
return True, {'status': 'http_200'}
|
||||||
|
except Exception as exc:
|
||||||
|
last_body = {'error': str(exc)}
|
||||||
|
time.sleep(3)
|
||||||
|
return False, last_body
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# Main Restore Engine
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description='Universal transactional database restore adapter')
|
||||||
|
parser.add_argument('--target', required=True, help='Target name from targets.json')
|
||||||
|
parser.add_argument('--dump', type=Path, help='Path to SQL dump')
|
||||||
|
parser.add_argument('--result', required=True, type=Path, help='Output path for result JSON')
|
||||||
|
parser.add_argument('--confirm', type=str, help='Required confirmation token')
|
||||||
|
parser.add_argument('--allow-overwrite', action='store_true', help='Allow overwrite of non-empty live database')
|
||||||
|
parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear')
|
||||||
|
parser.add_argument('--drop-previous', type=str, help='Explicitly drop a retained <app>_before_<ts> database')
|
||||||
|
parser.add_argument('--targets-file', type=Path, help='Optional explicit path to targets.json')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
targets_registry = load_targets(args.targets_file)
|
||||||
|
if args.target not in targets_registry.get('targets', {}):
|
||||||
|
msg = f"Target {args.target!r} not found in targets.json"
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||||
|
print(f"ERROR: {msg}", file=sys.stderr)
|
||||||
|
return EXIT_PRECONDITION
|
||||||
|
|
||||||
|
tcfg = targets_registry['targets'][args.target]
|
||||||
|
engine = tcfg.get('db_engine', 'postgresql').lower()
|
||||||
|
if engine not in ('postgresql', 'postgres', 'mariadb', 'mysql'):
|
||||||
|
msg = f"Unsupported db_engine: {engine}"
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||||
|
print(f"ERROR: {msg}", file=sys.stderr)
|
||||||
|
return EXIT_PRECONDITION
|
||||||
|
is_postgres = engine in ('postgresql', 'postgres')
|
||||||
|
|
||||||
|
# Host & Arch guard
|
||||||
|
cur_host = socket.gethostname()
|
||||||
|
cur_arch = platform.machine()
|
||||||
|
if os.environ.get('SKIP_HOST_GUARD') != '1':
|
||||||
|
req_host = tcfg.get('host')
|
||||||
|
req_arch = tcfg.get('arch')
|
||||||
|
if req_host and req_host != cur_host:
|
||||||
|
msg = f"Host mismatch: running on {cur_host}, target requires {req_host}"
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||||
|
print(f"ERROR: {msg}", file=sys.stderr)
|
||||||
|
return EXIT_PRECONDITION
|
||||||
|
if req_arch and req_arch != cur_arch:
|
||||||
|
msg = f"Arch mismatch: running on {cur_arch}, target requires {req_arch}"
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||||
|
print(f"ERROR: {msg}", file=sys.stderr)
|
||||||
|
return EXIT_PRECONDITION
|
||||||
|
|
||||||
|
# Confirmation token guard
|
||||||
|
req_confirm = tcfg.get('require_confirm')
|
||||||
|
if req_confirm and args.confirm != req_confirm:
|
||||||
|
msg = f"Target {args.target} requires confirmation: --confirm {req_confirm}"
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||||
|
print(f"ERROR: {msg}", file=sys.stderr)
|
||||||
|
return EXIT_PRECONDITION
|
||||||
|
|
||||||
|
# Drop previous retained DB if requested
|
||||||
|
live_db = tcfg['db_name']
|
||||||
|
if args.drop_previous:
|
||||||
|
expected_pattern = rf'^{re.escape(live_db)}_before_\d{{8}}_\d{{6}}$'
|
||||||
|
if not re.match(expected_pattern, args.drop_previous):
|
||||||
|
msg = f"Refused to drop database with unexpected name: {args.drop_previous}"
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||||
|
return EXIT_PRECONDITION
|
||||||
|
if is_postgres:
|
||||||
|
postgres_drop_db_if_exists(tcfg, args.drop_previous)
|
||||||
|
else:
|
||||||
|
mariadb_drop_db_if_exists(tcfg, args.drop_previous)
|
||||||
|
write_result(args.result, {'status': 'ok', 'exit_code': EXIT_OK, 'dropped_database': args.drop_previous})
|
||||||
|
print(f"OK: Dropped previous database {args.drop_previous}")
|
||||||
|
return EXIT_OK
|
||||||
|
|
||||||
|
if not args.dump:
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': '--dump is required'})
|
||||||
|
return EXIT_PRECONDITION
|
||||||
|
|
||||||
|
dump_path = args.dump.resolve()
|
||||||
|
sha_path = Path(str(dump_path) + '.sha256')
|
||||||
|
if not dump_path.is_file():
|
||||||
|
msg = f"Dump file missing: {dump_path}"
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||||
|
return EXIT_PRECONDITION
|
||||||
|
|
||||||
|
# Checksum validation if .sha256 file is present
|
||||||
|
if sha_path.is_file():
|
||||||
|
expected_sha = sha_path.read_text(encoding='utf-8').strip().split()[0].lower()
|
||||||
|
actual_sha = compute_sha256(dump_path)
|
||||||
|
if expected_sha != actual_sha:
|
||||||
|
msg = f"SHA256 mismatch: expected {expected_sha}, got {actual_sha}"
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_CHECKSUM_MISMATCH, 'detail': msg})
|
||||||
|
print(f"ERROR: {msg}", file=sys.stderr)
|
||||||
|
return EXIT_CHECKSUM_MISMATCH
|
||||||
|
|
||||||
|
# Check if live DB has non-empty tables and --allow-overwrite is needed
|
||||||
|
check_tables = tcfg.get('verify_tables_not_empty', [])
|
||||||
|
if check_tables and not args.allow_overwrite:
|
||||||
|
tbl = check_tables[0]
|
||||||
|
cnt = 0
|
||||||
|
if is_postgres:
|
||||||
|
r = psql_exec(tcfg, live_db, f'SELECT COUNT(*)::int FROM "{tbl}";')
|
||||||
|
cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0
|
||||||
|
else:
|
||||||
|
r = mariadb_exec(tcfg, live_db, f'SELECT COUNT(*) FROM `{tbl}`;')
|
||||||
|
cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0
|
||||||
|
if cnt > 0:
|
||||||
|
msg = f"Target database {live_db} contains {cnt} rows in table {tbl}; pass --allow-overwrite to replace it."
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||||
|
print(f"ERROR: {msg}", file=sys.stderr)
|
||||||
|
return EXIT_PRECONDITION
|
||||||
|
|
||||||
|
ts_label = dt.datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||||
|
staging_db = f"{live_db}_restore_{ts_label}"
|
||||||
|
backup_db = f"{live_db}_before_{ts_label}"
|
||||||
|
|
||||||
|
print(f"1. Creating staging database {staging_db} ({engine})...")
|
||||||
|
if is_postgres:
|
||||||
|
ok, err = postgres_create_db(tcfg, staging_db)
|
||||||
|
else:
|
||||||
|
ok, err = mariadb_create_db(tcfg, staging_db)
|
||||||
|
if not ok:
|
||||||
|
msg = f"Failed to create staging DB: {err}"
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||||
|
return EXIT_EXEC_ERROR
|
||||||
|
|
||||||
|
print(f"2. Importing dump into {staging_db}...")
|
||||||
|
if is_postgres:
|
||||||
|
ok, err = postgres_import_dump(tcfg, staging_db, dump_path)
|
||||||
|
else:
|
||||||
|
ok, err = mariadb_import_dump(tcfg, staging_db, dump_path, live_db)
|
||||||
|
if not ok:
|
||||||
|
msg = f"Failed to import dump into staging DB: {err}"
|
||||||
|
if is_postgres:
|
||||||
|
postgres_drop_db_if_exists(tcfg, staging_db)
|
||||||
|
else:
|
||||||
|
mariadb_drop_db_if_exists(tcfg, staging_db)
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||||
|
return EXIT_EXEC_ERROR
|
||||||
|
|
||||||
|
print("3. Validating staging database integrity...")
|
||||||
|
table_counts = {}
|
||||||
|
for tbl in tcfg.get('verify_tables_not_empty', []):
|
||||||
|
if is_postgres:
|
||||||
|
r = psql_exec(tcfg, staging_db, f'SELECT COUNT(*)::int FROM "{tbl}";')
|
||||||
|
else:
|
||||||
|
r = mariadb_exec(tcfg, staging_db, f'SELECT COUNT(*) FROM `{tbl}`;')
|
||||||
|
cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0
|
||||||
|
table_counts[tbl] = cnt
|
||||||
|
if cnt == 0:
|
||||||
|
msg = f"Verification failed: required table {tbl} is empty in staging DB"
|
||||||
|
if is_postgres:
|
||||||
|
postgres_drop_db_if_exists(tcfg, staging_db)
|
||||||
|
else:
|
||||||
|
mariadb_drop_db_if_exists(tcfg, staging_db)
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||||
|
return EXIT_EXEC_ERROR
|
||||||
|
|
||||||
|
# Invalidate / truncate session tables if requested
|
||||||
|
if args.sessions == 'clear':
|
||||||
|
for tbl in tcfg.get('truncate_tables', ['sessions']):
|
||||||
|
print(f"Truncating session table: {tbl}")
|
||||||
|
if is_postgres:
|
||||||
|
psql_exec(tcfg, staging_db, f'TRUNCATE TABLE "{tbl}" CASCADE;')
|
||||||
|
else:
|
||||||
|
mariadb_exec(tcfg, staging_db, f'TRUNCATE TABLE `{tbl}`;')
|
||||||
|
|
||||||
|
services_to_stop = tcfg.get('services_to_stop', [])
|
||||||
|
if services_to_stop:
|
||||||
|
print(f"4. Stopping dependent services: {services_to_stop}...")
|
||||||
|
compose_cmd(tcfg, 'stop', *services_to_stop)
|
||||||
|
|
||||||
|
print(f"5. Performing atomic database swap ({live_db} -> {backup_db}, {staging_db} -> {live_db})...")
|
||||||
|
if is_postgres:
|
||||||
|
swap_ok, swap_err = postgres_swap_databases(tcfg, live_db, staging_db, backup_db)
|
||||||
|
else:
|
||||||
|
swap_ok, swap_err = mariadb_swap_databases(tcfg, live_db, staging_db, backup_db)
|
||||||
|
|
||||||
|
if not swap_ok:
|
||||||
|
msg = f"Database swap failed: {swap_err}"
|
||||||
|
if services_to_stop:
|
||||||
|
compose_cmd(tcfg, 'up', '-d', *services_to_stop)
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||||
|
return EXIT_EXEC_ERROR
|
||||||
|
|
||||||
|
if services_to_stop:
|
||||||
|
print(f"6. Starting dependent services: {services_to_stop}...")
|
||||||
|
compose_cmd(tcfg, 'up', '-d', *services_to_stop)
|
||||||
|
|
||||||
|
# Healthcheck / Readiness verification
|
||||||
|
health_url = tcfg.get('ready_url') or tcfg.get('health_url')
|
||||||
|
if health_url:
|
||||||
|
print(f"7. Verifying service readiness at {health_url}...")
|
||||||
|
timeout_sec = tcfg.get('health_timeout_seconds', 180)
|
||||||
|
ready_ok, ready_data = wait_ready(health_url, timeout_sec=timeout_sec)
|
||||||
|
if not ready_ok:
|
||||||
|
print(f"ERROR: Health check failed after {timeout_sec}s: {ready_data}. Initiating automatic rollback!", file=sys.stderr)
|
||||||
|
if services_to_stop:
|
||||||
|
compose_cmd(tcfg, 'stop', *services_to_stop)
|
||||||
|
if is_postgres:
|
||||||
|
postgres_rollback_swap(tcfg, live_db, staging_db, backup_db)
|
||||||
|
else:
|
||||||
|
mariadb_rollback_swap(tcfg, live_db, staging_db, backup_db)
|
||||||
|
if services_to_stop:
|
||||||
|
compose_cmd(tcfg, 'up', '-d', *services_to_stop)
|
||||||
|
msg = f"Post-restore healthcheck failed: {ready_data}. Automatic rollback executed."
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_POST_RESTORE_FAILED, 'detail': msg, 'health_data': ready_data})
|
||||||
|
return EXIT_POST_RESTORE_FAILED
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
'status': 'ok',
|
||||||
|
'exit_code': EXIT_OK,
|
||||||
|
'target': args.target,
|
||||||
|
'db_engine': engine,
|
||||||
|
'live_db': live_db,
|
||||||
|
'previous_db': backup_db,
|
||||||
|
'verified_tables': table_counts,
|
||||||
|
'sessions': args.sessions,
|
||||||
|
'completed_at': dt.datetime.now().isoformat(),
|
||||||
|
}
|
||||||
|
write_result(args.result, summary)
|
||||||
|
print(f"OK: Restore completed successfully for target {args.target} (previous DB: {backup_db})")
|
||||||
|
return EXIT_OK
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main())
|
||||||
|
|
||||||
@@ -138,21 +138,36 @@ def query_db_metadata(tcfg, dump_path):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Fallback: parse migrations from plain SQL dump if container query was unavailable
|
# Fallback: parse migrations and table row counts from plain SQL dump
|
||||||
if not migrations and detect_dump_format(dump_path) == 'plain':
|
if detect_dump_format(dump_path) == 'plain':
|
||||||
|
parse_migrations = not migrations
|
||||||
in_copy = False
|
in_copy = False
|
||||||
|
curr_table = None
|
||||||
|
tcount = 0
|
||||||
with Path(dump_path).open('r', encoding='utf-8', errors='replace') as fp:
|
with Path(dump_path).open('r', encoding='utf-8', errors='replace') as fp:
|
||||||
for line in fp:
|
for line in fp:
|
||||||
if not in_copy:
|
if not in_copy:
|
||||||
if line.startswith('COPY public.schema_migrations ') or line.startswith('COPY schema_migrations '):
|
if line.startswith('COPY public.') or line.startswith('COPY '):
|
||||||
in_copy = True
|
parts = line.split()
|
||||||
continue
|
if len(parts) >= 2:
|
||||||
|
tbl = parts[1].replace('public.', '').replace('"', '').split('(')[0].strip()
|
||||||
|
curr_table = tbl
|
||||||
|
tcount = 0
|
||||||
|
in_copy = True
|
||||||
else:
|
else:
|
||||||
if line.startswith('\\.'):
|
if line.startswith('\\.'):
|
||||||
break
|
if curr_table:
|
||||||
parts = line.rstrip('\r\n').split('\t')
|
if not row_counts or curr_table not in row_counts:
|
||||||
if len(parts) >= 2:
|
row_counts[curr_table] = tcount
|
||||||
migrations.append(parts[1])
|
in_copy = False
|
||||||
|
curr_table = None
|
||||||
|
tcount = 0
|
||||||
|
else:
|
||||||
|
tcount += 1
|
||||||
|
if parse_migrations and curr_table == 'schema_migrations':
|
||||||
|
parts = line.rstrip('\r\n').split('\t')
|
||||||
|
if len(parts) >= 2:
|
||||||
|
migrations.append(parts[1])
|
||||||
|
|
||||||
def sort_key(stem):
|
def sort_key(stem):
|
||||||
m = re.match(r'^(\d+)', stem)
|
m = re.match(r'^(\d+)', stem)
|
||||||
|
|||||||
@@ -82,12 +82,11 @@ def query_db_media_refs(tcfg):
|
|||||||
"""
|
"""
|
||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
[
|
[
|
||||||
'docker', 'exec', tcfg['db_container'],
|
'docker', 'exec', '-i', tcfg['db_container'],
|
||||||
'sh', '-ec',
|
'sh', '-ec',
|
||||||
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
|
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -U "{tcfg["db_user"]}" -d "{tcfg["db_name"]}" -At',
|
||||||
f'psql -U "{tcfg["db_user"]}" -d "{tcfg["db_name"]}" -At -c ' + json.dumps(sql),
|
|
||||||
],
|
],
|
||||||
capture_output=True, text=True, timeout=30,
|
input=sql, capture_output=True, text=True, timeout=30,
|
||||||
)
|
)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise RuntimeError(f'Database media query failed: {proc.stderr.strip()}')
|
raise RuntimeError(f'Database media query failed: {proc.stderr.strip()}')
|
||||||
|
|||||||
@@ -101,12 +101,11 @@ def write_result(path, data):
|
|||||||
|
|
||||||
def psql_exec(tcfg, dbname, sql):
|
def psql_exec(tcfg, dbname, sql):
|
||||||
cmd = [
|
cmd = [
|
||||||
'docker', 'exec', tcfg['db_container'],
|
'docker', 'exec', '-i', tcfg['db_container'],
|
||||||
'sh', '-ec',
|
'sh', '-ec',
|
||||||
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
|
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -v ON_ERROR_STOP=1 -U "{tcfg["db_user"]}" -d "{dbname}" -At',
|
||||||
f'psql -v ON_ERROR_STOP=1 -U "{tcfg["db_user"]}" -d "{dbname}" -At -c ' + json.dumps(sql),
|
|
||||||
]
|
]
|
||||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=60)
|
return subprocess.run(cmd, input=sql, capture_output=True, text=True, timeout=60)
|
||||||
|
|
||||||
|
|
||||||
def drop_db_if_exists(tcfg, dbname):
|
def drop_db_if_exists(tcfg, dbname):
|
||||||
@@ -316,6 +315,16 @@ def main():
|
|||||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||||
return EXIT_EXEC_ERROR
|
return EXIT_EXEC_ERROR
|
||||||
|
|
||||||
|
# Verify active media_assets count in staging database
|
||||||
|
act_res = psql_exec(tcfg, staging_db, "SELECT COUNT(*)::int FROM media_assets WHERE lifecycle_state = 'active';")
|
||||||
|
if act_res.returncode == 0 and act_res.stdout.strip().isdigit():
|
||||||
|
act_cnt = int(act_res.stdout.strip())
|
||||||
|
if act_cnt < 1:
|
||||||
|
drop_db_if_exists(tcfg, staging_db)
|
||||||
|
msg = f'Plausibility check failed: expected active media_assets, found {act_cnt}'
|
||||||
|
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||||
|
return EXIT_EXEC_ERROR
|
||||||
|
|
||||||
if args.sessions == 'clear':
|
if args.sessions == 'clear':
|
||||||
print('5. Clearing active user sessions in staging DB (--sessions clear)...')
|
print('5. Clearing active user sessions in staging DB (--sessions clear)...')
|
||||||
psql_exec(tcfg, staging_db, 'TRUNCATE TABLE sessions;')
|
psql_exec(tcfg, staging_db, 'TRUNCATE TABLE sessions;')
|
||||||
|
|||||||
@@ -93,6 +93,69 @@
|
|||||||
"frontend_ready_url": "http://127.0.0.1:13003/api/health/ready",
|
"frontend_ready_url": "http://127.0.0.1:13003/api/health/ready",
|
||||||
"adapter_dir": "/etc/backup-orchestrator/scripts/shinkan",
|
"adapter_dir": "/etc/backup-orchestrator/scripts/shinkan",
|
||||||
"services_to_stop": ["backend", "frontend"]
|
"services_to_stop": ["backend", "frontend"]
|
||||||
|
},
|
||||||
|
"kairo-llm-prod": {
|
||||||
|
"application": "kairo",
|
||||||
|
"environment": "production",
|
||||||
|
"host": "llm-node",
|
||||||
|
"arch": "x86_64",
|
||||||
|
"role": "production_target",
|
||||||
|
"allow_restore": true,
|
||||||
|
"require_confirm": "kairo@llm-node",
|
||||||
|
"app_dir": "/home/llmadmin/docker/kairo-prod",
|
||||||
|
"compose_project": "kairo-prod",
|
||||||
|
"compose_files": ["docker-compose.yml"],
|
||||||
|
"env_file": "/etc/kairo/prod.env",
|
||||||
|
"db_engine": "postgresql",
|
||||||
|
"db_container": "kairo-db-prod",
|
||||||
|
"db_name": "kairo",
|
||||||
|
"db_user": "kairo_user",
|
||||||
|
"ready_url": "http://127.0.0.1:8001/api/health/ready",
|
||||||
|
"services_to_stop": ["backend"],
|
||||||
|
"verify_tables_not_empty": ["django_migrations"],
|
||||||
|
"truncate_tables": ["django_session"]
|
||||||
|
},
|
||||||
|
"tandoor-llm-prod": {
|
||||||
|
"application": "tandoor",
|
||||||
|
"environment": "production",
|
||||||
|
"host": "llm-node",
|
||||||
|
"arch": "x86_64",
|
||||||
|
"role": "production_target",
|
||||||
|
"allow_restore": true,
|
||||||
|
"require_confirm": "tandoor@llm-node",
|
||||||
|
"app_dir": "/home/llmadmin/docker/tandoor-prod",
|
||||||
|
"compose_project": "tandoor-prod",
|
||||||
|
"compose_files": ["docker-compose.yml"],
|
||||||
|
"env_file": "/etc/tandoor/prod.env",
|
||||||
|
"db_engine": "postgresql",
|
||||||
|
"db_container": "tandoor-db-prod",
|
||||||
|
"db_name": "tandoordb",
|
||||||
|
"db_user": "tandoor_user",
|
||||||
|
"ready_url": "http://127.0.0.1:8080/",
|
||||||
|
"services_to_stop": ["web_app"],
|
||||||
|
"verify_tables_not_empty": ["django_migrations"],
|
||||||
|
"truncate_tables": ["django_session"]
|
||||||
|
},
|
||||||
|
"bookstack-llm-prod": {
|
||||||
|
"application": "bookstack",
|
||||||
|
"environment": "production",
|
||||||
|
"host": "llm-node",
|
||||||
|
"arch": "x86_64",
|
||||||
|
"role": "production_target",
|
||||||
|
"allow_restore": true,
|
||||||
|
"require_confirm": "bookstack@llm-node",
|
||||||
|
"app_dir": "/home/llmadmin/docker/bookstack-prod",
|
||||||
|
"compose_project": "bookstack-prod",
|
||||||
|
"compose_files": ["docker-compose.yml"],
|
||||||
|
"env_file": "/etc/bookstack/prod.env",
|
||||||
|
"db_engine": "mariadb",
|
||||||
|
"db_container": "bookstack-db-prod",
|
||||||
|
"db_name": "bookstack",
|
||||||
|
"db_user": "bookstack_user",
|
||||||
|
"ready_url": "http://127.0.0.1:6875/",
|
||||||
|
"services_to_stop": ["bookstack"],
|
||||||
|
"verify_tables_not_empty": ["migrations"],
|
||||||
|
"truncate_tables": ["sessions"]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -20,6 +20,7 @@ class OrchestratorTests(unittest.TestCase):
|
|||||||
self.root = Path(self.temp.name)
|
self.root = Path(self.temp.name)
|
||||||
self.patches = [patch.object(app, 'legacy_schedules', return_value=[]),
|
self.patches = [patch.object(app, 'legacy_schedules', return_value=[]),
|
||||||
patch.object(app, 'nas_mount_ready', return_value=True),
|
patch.object(app, 'nas_mount_ready', return_value=True),
|
||||||
|
patch.object(app, 'load_host_config', return_value={'hostname':'test-host', 'nas_mount':'/backup', 'nas_source':'192.168.2.63:/volume1/Backup_Paperless'}),
|
||||||
patch.dict(os.environ, {'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED':'YES'})]
|
patch.dict(os.environ, {'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED':'YES'})]
|
||||||
for p in self.patches:
|
for p in self.patches:
|
||||||
p.start()
|
p.start()
|
||||||
|
|||||||
@@ -190,6 +190,8 @@ class RecoveryPointAndManifestTests(unittest.TestCase):
|
|||||||
tcfg['targets']['llm-prod']['host'] = app.socket.gethostname()
|
tcfg['targets']['llm-prod']['host'] = app.socket.gethostname()
|
||||||
tcfg['targets']['llm-prod']['arch'] = app.platform.machine()
|
tcfg['targets']['llm-prod']['arch'] = app.platform.machine()
|
||||||
tcfg['targets']['llm-prod']['adapter_dir'] = str(adapter_dir)
|
tcfg['targets']['llm-prod']['adapter_dir'] = str(adapter_dir)
|
||||||
|
tcfg['targets']['llm-prod']['app_dir'] = str(self.root)
|
||||||
|
tcfg['targets']['llm-prod']['compose_dir'] = str(self.root)
|
||||||
|
|
||||||
args_no_confirm = types.SimpleNamespace(
|
args_no_confirm = types.SimpleNamespace(
|
||||||
target='llm-prod', point=str(dump_path), confirm=None,
|
target='llm-prod', point=str(dump_path), confirm=None,
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
"""Unit tests for scripts/common/universal-db-restore and generic restore engine."""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
ROOT_DIR = Path(__file__).resolve().parents[1]
|
||||||
|
SCRIPT_PATH = ROOT_DIR / 'scripts' / 'common' / 'universal-db-restore'
|
||||||
|
|
||||||
|
|
||||||
|
class UniversalDbRestoreTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.temp.cleanup)
|
||||||
|
self.root = Path(self.temp.name)
|
||||||
|
|
||||||
|
self.targets_file = self.root / 'targets.json'
|
||||||
|
self.targets_data = {
|
||||||
|
'schemaVersion': 1,
|
||||||
|
'targets': {
|
||||||
|
'test-pg-prod': {
|
||||||
|
'application': 'testpg',
|
||||||
|
'environment': 'production',
|
||||||
|
'host': 'test-host',
|
||||||
|
'arch': 'x86_64',
|
||||||
|
'role': 'production_target',
|
||||||
|
'require_confirm': 'testpg@test-host',
|
||||||
|
'db_engine': 'postgresql',
|
||||||
|
'db_container': 'test-pg-container',
|
||||||
|
'db_name': 'testpgdb',
|
||||||
|
'db_user': 'testpguser',
|
||||||
|
'services_to_stop': ['backend'],
|
||||||
|
'verify_tables_not_empty': ['users'],
|
||||||
|
'truncate_tables': ['sessions'],
|
||||||
|
'ready_url': 'http://127.0.0.1:8999/healthz',
|
||||||
|
},
|
||||||
|
'test-maria-prod': {
|
||||||
|
'application': 'testmaria',
|
||||||
|
'environment': 'production',
|
||||||
|
'host': 'test-host',
|
||||||
|
'arch': 'x86_64',
|
||||||
|
'role': 'production_target',
|
||||||
|
'require_confirm': 'testmaria@test-host',
|
||||||
|
'db_engine': 'mariadb',
|
||||||
|
'db_container': 'test-maria-container',
|
||||||
|
'db_name': 'testmariadb',
|
||||||
|
'db_user': 'testmariauser',
|
||||||
|
'services_to_stop': ['web'],
|
||||||
|
'verify_tables_not_empty': ['posts'],
|
||||||
|
'truncate_tables': ['sessions'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
self.targets_file.write_text(json.dumps(self.targets_data, indent=2), encoding='utf-8')
|
||||||
|
|
||||||
|
def test_missing_target_fails_precondition(self):
|
||||||
|
result_file = self.root / 'result.json'
|
||||||
|
cmd = [
|
||||||
|
sys.executable, str(SCRIPT_PATH),
|
||||||
|
'--target', 'unknown-target',
|
||||||
|
'--result', str(result_file),
|
||||||
|
'--targets-file', str(self.targets_file),
|
||||||
|
]
|
||||||
|
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||||
|
self.assertEqual(res.returncode, 2) # EXIT_PRECONDITION
|
||||||
|
data = json.loads(result_file.read_text(encoding='utf-8'))
|
||||||
|
self.assertEqual(data['status'], 'error')
|
||||||
|
self.assertIn('not found', data['detail'])
|
||||||
|
|
||||||
|
def test_confirmation_guard(self):
|
||||||
|
dump_path = self.root / 'test.sql'
|
||||||
|
dump_path.write_text('SELECT 1;', encoding='utf-8')
|
||||||
|
result_file = self.root / 'result.json'
|
||||||
|
|
||||||
|
env = os.environ.copy()
|
||||||
|
env['SKIP_HOST_GUARD'] = '1'
|
||||||
|
|
||||||
|
# Without confirmation
|
||||||
|
cmd = [
|
||||||
|
sys.executable, str(SCRIPT_PATH),
|
||||||
|
'--target', 'test-pg-prod',
|
||||||
|
'--dump', str(dump_path),
|
||||||
|
'--result', str(result_file),
|
||||||
|
'--targets-file', str(self.targets_file),
|
||||||
|
]
|
||||||
|
res = subprocess.run(cmd, env=env, capture_output=True, text=True)
|
||||||
|
self.assertEqual(res.returncode, 2)
|
||||||
|
data = json.loads(result_file.read_text(encoding='utf-8'))
|
||||||
|
self.assertIn('requires confirmation', data['detail'])
|
||||||
|
|
||||||
|
# With wrong confirmation
|
||||||
|
cmd_wrong = cmd + ['--confirm', 'wrong-token']
|
||||||
|
res_wrong = subprocess.run(cmd_wrong, env=env, capture_output=True, text=True)
|
||||||
|
self.assertEqual(res_wrong.returncode, 2)
|
||||||
|
|
||||||
|
def test_sha256_verification(self):
|
||||||
|
dump_path = self.root / 'test.sql'
|
||||||
|
dump_path.write_text('SELECT 1;', encoding='utf-8')
|
||||||
|
sha_path = self.root / 'test.sql.sha256'
|
||||||
|
sha_path.write_text('0' * 64 + ' test.sql\n', encoding='utf-8')
|
||||||
|
result_file = self.root / 'result.json'
|
||||||
|
|
||||||
|
env = os.environ.copy()
|
||||||
|
env['SKIP_HOST_GUARD'] = '1'
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
sys.executable, str(SCRIPT_PATH),
|
||||||
|
'--target', 'test-pg-prod',
|
||||||
|
'--dump', str(dump_path),
|
||||||
|
'--confirm', 'testpg@test-host',
|
||||||
|
'--result', str(result_file),
|
||||||
|
'--targets-file', str(self.targets_file),
|
||||||
|
]
|
||||||
|
res = subprocess.run(cmd, env=env, capture_output=True, text=True)
|
||||||
|
self.assertEqual(res.returncode, 4) # EXIT_CHECKSUM_MISMATCH
|
||||||
|
data = json.loads(result_file.read_text(encoding='utf-8'))
|
||||||
|
self.assertEqual(data['status'], 'error')
|
||||||
|
self.assertIn('SHA256 mismatch', data['detail'])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
|
|
||||||
Reference in new issue
Block a user