diff --git a/docs/APP_MIGRATION_CONTRACT.md b/docs/APP_MIGRATION_CONTRACT.md index c3c155f..b8bb645 100644 --- a/docs/APP_MIGRATION_CONTRACT.md +++ b/docs/APP_MIGRATION_CONTRACT.md @@ -125,3 +125,4 @@ Bitte setze folgende Anforderungen um und verändere keine Produktionsdaten auf - Prüfe mit `curl -s -i http://127.0.0.1:/api/health/ready`, dass der Readiness-Check grün meldet. - Melde zurück: DB-Engine, Containername, DB-Name, DB-User, Health-URL, Session-Tabelle und nicht-leere Kern-Tabellen. ```` + diff --git a/docs/UPTIME_KUMA_INTEGRATION.md b/docs/UPTIME_KUMA_INTEGRATION.md index 8476b97..0a6ab73 100644 --- a/docs/UPTIME_KUMA_INTEGRATION.md +++ b/docs/UPTIME_KUMA_INTEGRATION.md @@ -105,3 +105,4 @@ In Uptime Kuma wechselt der Monitor sofort auf **"UP"** (grün) und zeigt im Ere - **Entkopplung**: Push-Benachrichtigungen werden asynchron über einen Thread-Pool versendet. - **Sicherheitsgarantie**: Netzwerk-Timeouts oder Fehler von Uptime Kuma verändern **niemals** den Exit-Code des Backup-Laufs. Ein erfolgreiches Backup bleibt immer `Exit 0`. - Fehlerhafte Push-Versuche werden in `/var/log/backup-orchestrator//notifications.log` protokolliert. + diff --git a/hosts/llm-node/host.json b/hosts/llm-node/host.json index 76e0688..457db76 100644 --- a/hosts/llm-node/host.json +++ b/hosts/llm-node/host.json @@ -6,8 +6,8 @@ "nas_mount": "/mnt/nas_backup", "nas_source": "192.168.2.63:/volume1/Backup_LLM", "nas_fstype": "nfs", - "local_dump_dir": "/mnt/nas_backup/shinkan/prod/db/latest", - "archive_sets_dir": "/mnt/nas_backup/shinkan/prod/db", + "local_dump_dir": "/mnt/nas_backup/recovery_points/shinkan-prod/latest", + "archive_sets_dir": "/mnt/nas_backup/recovery_points", "media_mounts": { "shinkan-prod": "/srv/shinkan/prod/media" }, diff --git a/hosts/llm-node/jobs.csv b/hosts/llm-node/jobs.csv index c3bc695..b158aca 100644 --- a/hosts/llm-node/jobs.csv +++ b/hosts/llm-node/jobs.csv @@ -1,2 +1,2 @@ phase,id,script,critical,timeout_seconds,enabled -10,shinkan-prod-db,/etc/backup-orchestrator/scripts/db-shinkan-prod.sh,true,900,false +10,shinkan-prod-db,/etc/backup-orchestrator/scripts/db-shinkan-prod.sh,true,900,true diff --git a/hosts/llm-node/scripts/db-shinkan-prod.sh b/hosts/llm-node/scripts/db-shinkan-prod.sh index 7d684e2..721acbf 100644 --- a/hosts/llm-node/scripts/db-shinkan-prod.sh +++ b/hosts/llm-node/scripts/db-shinkan-prod.sh @@ -1,33 +1,66 @@ #!/usr/bin/env bash -# Backup wrapper for Shinkan Production on llm-node (post-cutover) +# ============================================================================ +# Backup wrapper for Shinkan Production on llm-node (Ubuntu 24.04 x86_64) +# Captures PostgreSQL dump, creates schema manifest & sha256, and publishes +# atomic backup set to NAS recovery_points. +# ============================================================================ set -Eeuo pipefail +umask 027 -DEST_ROOT="/mnt/nas_backup/shinkan/prod/db" -TS="$(date -u +%Y%m%dT%H%M%SZ)" -GEN_DIR="${DEST_ROOT}/generations/${TS}" -DUMP_FILE="${GEN_DIR}/shinkan-prod.sql" +ARCHIVE_ROOT="/mnt/nas_backup/recovery_points" +TARGET_ID="shinkan-prod" +DB_CONTAINER="shinkan-db-prod" +DB_NAME="shinkan" +DB_USER="shinkan_user" -install -d -m 0750 "${GEN_DIR}" +# Verify NAS mount +if ! mountpoint -q /mnt/nas_backup; then + echo "FEHLER: /mnt/nas_backup ist nicht gemountet!" >&2 + exit 1 +fi -export DB_CONTAINER="shinkan-db-prod" -export DB_USER="shinkan" -export DB_NAME="shinkan" -export OUTPUT_PATH="${DUMP_FILE}" +TMP_DIR="$(mktemp -d /tmp/shinkan-backup.XXXXXXXX)" +cleanup() { + rm -rf -- "$TMP_DIR" +} +trap cleanup EXIT -/etc/backup-orchestrator/scripts/db-postgres-docker.sh +DUMP_FILE="${TMP_DIR}/shinkan-prod.sql" +RESULT_FILE="${TMP_DIR}/manifest-result.json" -# Generate SHA256, Manifest (schemaVersion: 1), and Media Inventory (.media.jsonl) -/etc/backup-orchestrator/scripts/shinkan/shinkan-backup-manifest \ - --dump "${DUMP_FILE}" \ - --env prod \ - --host "llm-node" \ - --compose-dir "/home/llmadmin/docker/shinkan-prod" \ - --media-dir "/srv/shinkan/prod/media" \ - --db-container "shinkan-db-prod" \ - --db-name "shinkan" \ - --db-user "shinkan" +echo "1. Erstelle PostgreSQL-Dump fuer ${DB_CONTAINER} (${DB_NAME})..." +docker exec "${DB_CONTAINER}" sh -ec ' + export PGPASSWORD="${POSTGRES_PASSWORD:-}" + exec pg_dump -U "'"${DB_USER}"'" "'"${DB_NAME}"'" +' > "${DUMP_FILE}" -# Atomically update 'latest' symlink after complete backup set verification -ln -sfn "generations/${TS}" "${DEST_ROOT}/latest.tmp.$$" -mv -Tf "${DEST_ROOT}/latest.tmp.$$" "${DEST_ROOT}/latest" -echo "Published atomic Shinkan backup set on llm-node: ${GEN_DIR}" +if [[ ! -s "${DUMP_FILE}" ]]; then + echo "FEHLER: Dump ist leer!" >&2 + exit 1 +fi +chmod 0640 "${DUMP_FILE}" + +echo "2. Erstelle Manifest, SHA256 und Medien-Index..." +MANIFEST_SCRIPT="/etc/backup-orchestrator/scripts/shinkan/shinkan-backup-manifest" +if [[ ! -f "$MANIFEST_SCRIPT" ]]; then + MANIFEST_SCRIPT="$(dirname "${BASH_SOURCE[0]}")/../../scripts/shinkan/shinkan-backup-manifest" +fi + +python3 "${MANIFEST_SCRIPT}" \ + --target "llm-prod" \ + --dump "${DUMP_FILE}" \ + --out-dir "${TMP_DIR}" \ + --result "${RESULT_FILE}" + +echo "3. Veroeffentliche Backup-Set im Recovery-Archiv..." +PUBLISH_SCRIPT="/etc/backup-orchestrator/scripts/publish-backup-set.sh" +if [[ ! -f "$PUBLISH_SCRIPT" ]]; then + PUBLISH_SCRIPT="$(dirname "${BASH_SOURCE[0]}")/../../scripts/common/publish-backup-set.sh" +fi + +bash "${PUBLISH_SCRIPT}" \ + "${TARGET_ID}" \ + "${DUMP_FILE}" \ + "${ARCHIVE_ROOT}" + +echo "OK: Shinkan Production Backup auf llm-node erfolgreich abgeschlossen." diff --git a/hosts/raspberrypi5/jobs.csv b/hosts/raspberrypi5/jobs.csv index 69ea5e7..376736b 100644 --- a/hosts/raspberrypi5/jobs.csv +++ b/hosts/raspberrypi5/jobs.csv @@ -3,7 +3,7 @@ phase,id,script,critical,timeout_seconds,enabled 10,tandoor-db,/etc/backup-orchestrator/scripts/db-tandoor.sh,true,600,true 10,mitai-prod-db,/etc/backup-orchestrator/scripts/db-mitai-prod.sh,true,900,true 10,mitai-dev-db,/etc/backup-orchestrator/scripts/db-mitai-dev.sh,false,900,true -10,shinkan-prod-db,/etc/backup-orchestrator/scripts/db-shinkan-prod.sh,true,600,true +10,shinkan-prod-db,/etc/backup-orchestrator/scripts/db-shinkan-prod.sh,true,600,false 10,shinkan-dev-db,/etc/backup-orchestrator/scripts/db-shinkan-dev.sh,false,600,true 20,nas-data,/etc/backup-orchestrator/scripts/nas-data-job.sh,true,10800,true 30,system-image,/etc/backup-orchestrator/scripts/system-image-job.sh,true,14400,false diff --git a/scripts/common/universal-db-restore b/scripts/common/universal-db-restore index 46d8dc1..404ee97 100644 --- a/scripts/common/universal-db-restore +++ b/scripts/common/universal-db-restore @@ -533,3 +533,4 @@ def main(): if __name__ == '__main__': sys.exit(main()) + diff --git a/tests/test_universal_restore.py b/tests/test_universal_restore.py index 5bb9279..ae5a623 100644 --- a/tests/test_universal_restore.py +++ b/tests/test_universal_restore.py @@ -125,3 +125,4 @@ class UniversalDbRestoreTests(unittest.TestCase): if __name__ == '__main__': unittest.main() +