feat(v0.4): import all Raspberry Pi scripts and implement Recovery Management engine + Shinkan adapters
This commit is contained in:
1 parent
40188c0e10
commit
c53fc0a939
35 files changed
+2877
-191
No files matched your search
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
exec /etc/backup-orchestrator/scripts/db-mariadb-docker.sh \
|
||||
"bookstack_db" "bookstack.sql" "mysql"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "buyingagent-dev-postgres-1" "buyingagent-dev.sql"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "buyingagent-db-prod" "buyingagent-prod.sql"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kairo-dev-postgres-1" "kairo-dev.sql"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kairo-db-prod" "kairo-prod.sql"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kansho-dev-postgres-1" "kansho-dev.sql"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kansho-db-prod" "kansho-prod.sql"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
exec /etc/backup-orchestrator/scripts/db-mariadb-docker.sh \
|
||||
"karatewiki-dev-db-1" "karatewiki-dev.sql" "mariadb"
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
|
||||
"dev-mitai-postgres" \
|
||||
"mitai-dev.sql" \
|
||||
"mitai_dev" \
|
||||
"mitai_dev"
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
|
||||
"mitai-db-prod" \
|
||||
"mitai-prod.sql"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
|
||||
"netbox-postgres-1" "netbox.sql"
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
# Optionales lokales Credential-File (chmod 600, nicht in Git)
|
||||
if [[ -f /etc/backup-orchestrator/credentials/paperless.env ]]; then
|
||||
# shellcheck disable=SC1091
|
||||
source /etc/backup-orchestrator/credentials/paperless.env
|
||||
fi
|
||||
|
||||
# === Datenbank ===
|
||||
DB_HOST="${DB_HOST:-192.168.2.40}"
|
||||
DB_PORT="${DB_PORT:-3308}"
|
||||
DB_USER="${DB_USER:-paperless}"
|
||||
DB_PASSWORD="${DB_PASSWORD:-HIER_PASSWORT_EINTRAGEN}"
|
||||
DB_NAME="${DB_NAME:-paperless}"
|
||||
|
||||
# === Backup ===
|
||||
BACKUP_DIR="${BACKUP_DIR:-/media/lars/paperless/db-backup}"
|
||||
BACKUP_FILE="$BACKUP_DIR/paperless.sql"
|
||||
|
||||
[[ "$DB_PASSWORD" != "HIER_PASSWORT_EINTRAGEN" ]] || {
|
||||
echo "FEHLER: Datenbankpasswort nicht konfiguriert" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
TEMP_FILE="$(mktemp "$BACKUP_DIR/.paperless.XXXXXXXX.sql")"
|
||||
trap 'rm -f -- "$TEMP_FILE"' EXIT
|
||||
|
||||
echo "Paperless: Datenbankexport gestartet"
|
||||
|
||||
MYSQL_PWD="$DB_PASSWORD" mysqldump \
|
||||
--host="$DB_HOST" \
|
||||
--port="$DB_PORT" \
|
||||
--user="$DB_USER" \
|
||||
--single-transaction \
|
||||
--quick \
|
||||
--databases "$DB_NAME" \
|
||||
> "$TEMP_FILE"
|
||||
|
||||
if [[ ! -s "$TEMP_FILE" ]]; then
|
||||
echo "FEHLER: Paperless-Dump ist leer" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
chgrp lars "$TEMP_FILE"
|
||||
chmod 640 "$TEMP_FILE"
|
||||
mv -f -- "$TEMP_FILE" "$BACKUP_FILE"
|
||||
|
||||
echo "OK: Paperless-Datenbank gesichert"
|
||||
echo "Datei: $BACKUP_FILE"
|
||||
exit 0
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
|
||||
"shinkan-dev-postgres-1" \
|
||||
"shinkan-dev.sql"
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
|
||||
"shinkan-db-prod" \
|
||||
"shinkan-prod.sql"
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
# Optionales lokales Credential-File (chmod 600, nicht in Git)
|
||||
if [[ -f /etc/backup-orchestrator/credentials/tandoor.env ]]; then
|
||||
# shellcheck disable=SC1091
|
||||
source /etc/backup-orchestrator/credentials/tandoor.env
|
||||
fi
|
||||
|
||||
# === Datenbank ===
|
||||
PGHOST="${PGHOST:-192.168.2.40}"
|
||||
PGPORT="${PGPORT:-5432}"
|
||||
PGUSER="${PGUSER:-djangouser}"
|
||||
PGDATABASE="${PGDATABASE:-djangodb}"
|
||||
PGPASSWORD="${PGPASSWORD:-HIER_PASSWORT_EINTRAGEN}"
|
||||
|
||||
# === Backup ===
|
||||
BACKUP_DIR="${BACKUP_DIR:-/media/lars/paperless/db-backup}"
|
||||
BACKUP_FILE="$BACKUP_DIR/tandoor.sql"
|
||||
|
||||
[[ "$PGPASSWORD" != "HIER_PASSWORT_EINTRAGEN" ]] || {
|
||||
echo "FEHLER: Datenbankpasswort nicht konfiguriert" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
TEMP_FILE="$(mktemp "$BACKUP_DIR/.tandoor.XXXXXXXX.sql")"
|
||||
trap 'rm -f -- "$TEMP_FILE"' EXIT
|
||||
|
||||
echo "Tandoor: Datenbankexport gestartet"
|
||||
|
||||
PGPASSWORD="$PGPASSWORD" pg_dump \
|
||||
--host="$PGHOST" \
|
||||
--port="$PGPORT" \
|
||||
--username="$PGUSER" \
|
||||
--dbname="$PGDATABASE" \
|
||||
--format=plain \
|
||||
> "$TEMP_FILE"
|
||||
|
||||
if [[ ! -s "$TEMP_FILE" ]]; then
|
||||
echo "FEHLER: Tandoor-Dump ist leer" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
chgrp lars "$TEMP_FILE"
|
||||
chmod 640 "$TEMP_FILE"
|
||||
mv -f -- "$TEMP_FILE" "$BACKUP_FILE"
|
||||
|
||||
echo "OK: Tandoor-Datenbank gesichert"
|
||||
echo "Datei: $BACKUP_FILE"
|
||||
exit 0
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "tsumugu-dev-postgres-1" "tsumugu-dev.sql"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "tsumugu-db-prod" "tsumugu-prod.sql"
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
exec runuser -u lars -- /bin/bash -s \
|
||||
< /etc/backup-orchestrator/scripts/backup-nas-data.sh
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# publish-backup-set.sh
|
||||
# Atomically publishes a complete Backup Set (Dump + .sha256 + .manifest.json
|
||||
# + optional .media.jsonl) to the NAS generation archive and rotates old sets.
|
||||
# Implements RECOVERY_CONTRACT.md section 5.1 & 5.2.
|
||||
# ============================================================================
|
||||
set -Eeuo pipefail
|
||||
umask 027
|
||||
|
||||
if [[ $# -lt 3 ]]; then
|
||||
echo "Verwendung: $0 <TARGET_ID> <DUMP_DATEI> <ARCHIVE_ROOT>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
TARGET_ID="$1"
|
||||
DUMP_FILE="$2"
|
||||
ARCHIVE_ROOT="$3"
|
||||
|
||||
SHA_FILE="${DUMP_FILE}.sha256"
|
||||
MANIFEST_FILE="${DUMP_FILE}.manifest.json"
|
||||
MEDIA_FILE="${DUMP_FILE}.media.jsonl"
|
||||
|
||||
[[ -s "$DUMP_FILE" ]] || { echo "FEHLER: Dump fehlt oder ist leer: $DUMP_FILE" >&2; exit 2; }
|
||||
[[ -s "$SHA_FILE" ]] || { echo "FEHLER: SHA256-Datei fehlt: $SHA_FILE" >&2; exit 2; }
|
||||
[[ -s "$MANIFEST_FILE" ]] || { echo "FEHLER: Manifest-Datei fehlt: $MANIFEST_FILE" >&2; exit 2; }
|
||||
|
||||
# Verify checksum before publishing
|
||||
(
|
||||
cd "$(dirname "$DUMP_FILE")"
|
||||
sha256sum -c "$(basename "$SHA_FILE")" >/dev/null
|
||||
) || {
|
||||
echo "FEHLER: SHA256-Pruefsumme stimmt nicht!" >&2
|
||||
exit 4
|
||||
}
|
||||
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
TARGET_DIR="$ARCHIVE_ROOT/$TARGET_ID"
|
||||
mkdir -p "$TARGET_DIR"
|
||||
|
||||
STAGING_DIR="$(mktemp -d "$TARGET_DIR/.staging-${TIMESTAMP}.XXXXXXXX")"
|
||||
FINAL_DIR="$TARGET_DIR/$TIMESTAMP"
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${STAGING_DIR:-}" && -d "$STAGING_DIR" ]]; then
|
||||
rm -rf -- "$STAGING_DIR" || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cp -p -- "$DUMP_FILE" "$STAGING_DIR/"
|
||||
cp -p -- "$SHA_FILE" "$STAGING_DIR/"
|
||||
cp -p -- "$MANIFEST_FILE" "$STAGING_DIR/"
|
||||
if [[ -f "$MEDIA_FILE" ]]; then
|
||||
cp -p -- "$MEDIA_FILE" "$STAGING_DIR/"
|
||||
fi
|
||||
|
||||
mv -T -- "$STAGING_DIR" "$FINAL_DIR"
|
||||
STAGING_DIR=""
|
||||
|
||||
TEMP_LINK="$TARGET_DIR/.latest-$$"
|
||||
ln -s -- "$FINAL_DIR" "$TEMP_LINK"
|
||||
mv -Tf -- "$TEMP_LINK" "$TARGET_DIR/latest"
|
||||
|
||||
echo "OK: Backup-Satz veroeffentlicht unter $FINAL_DIR (latest aktualisiert)"
|
||||
exit 0
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
DUMP_FILE="${SHINKAN_DUMP_FILE:-/media/lars/paperless/db-backup/shinkan-prod.sql}"
|
||||
OUT_DIR="$(dirname "$DUMP_FILE")"
|
||||
TARGET="${SHINKAN_TARGET:-pi-prod}"
|
||||
ARCHIVE_ROOT="${SHINKAN_ARCHIVE_ROOT:-/backup/recovery_points}"
|
||||
RESULT_JSON="${OUT_DIR}/.shinkan-manifest-last-result.json"
|
||||
|
||||
/usr/bin/env python3 /etc/backup-orchestrator/scripts/shinkan/shinkan-backup-manifest \
|
||||
--target "$TARGET" \
|
||||
--dump "$DUMP_FILE" \
|
||||
--out-dir "$OUT_DIR" \
|
||||
--result "$RESULT_JSON"
|
||||
|
||||
if [[ -d "$(dirname "$ARCHIVE_ROOT")" && -w "$(dirname "$ARCHIVE_ROOT")" ]]; then
|
||||
/etc/backup-orchestrator/scripts/publish-backup-set.sh \
|
||||
"shinkan-${TARGET}" \
|
||||
"$DUMP_FILE" \
|
||||
"$ARCHIVE_ROOT"
|
||||
fi
|
||||
@@ -0,0 +1,73 @@
|
||||
# Isolated disposable Restore-Test Compose configuration for llm-node.
|
||||
# Implements RECOVERY_CONTRACT.md section 3.3:
|
||||
# - No fixed container_name (prevents collision with shinkan-prod)
|
||||
# - Localhost-only ports (18003 / 13003), no published postgres port
|
||||
# - Shared media mounted strictly read-only (:ro)
|
||||
# - No retention-cron service, SMTP and OpenRouter disabled
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_DB: "${DB_NAME:-shinkan}"
|
||||
POSTGRES_USER: "${DB_USER:-shinkan_user}"
|
||||
POSTGRES_PASSWORD: "${DB_PASSWORD}"
|
||||
volumes:
|
||||
- restoretest-db-data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- restoretest-net
|
||||
|
||||
backend:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
DB_HOST: postgres
|
||||
DB_PORT: 5432
|
||||
DB_NAME: shinkan
|
||||
DB_USER: shinkan_user
|
||||
DB_PASSWORD: "${DB_PASSWORD}"
|
||||
OPENROUTER_API_KEY: ""
|
||||
OPENROUTER_MODEL: ""
|
||||
SMTP_HOST: ""
|
||||
SMTP_PORT: ""
|
||||
SMTP_USER: ""
|
||||
SMTP_PASS: ""
|
||||
SMTP_FROM: ""
|
||||
AUTO_ADMIN_FIRST_USER: "false"
|
||||
ADMIN_BOOTSTRAP_EMAILS: ""
|
||||
ENVIRONMENT: "production"
|
||||
MEDIA_ROOT: "/app/media"
|
||||
volumes:
|
||||
- type: bind
|
||||
source: /srv/shinkan/prod/media
|
||||
target: /app/media
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
ports:
|
||||
- "127.0.0.1:18003:8000"
|
||||
depends_on:
|
||||
- postgres
|
||||
networks:
|
||||
- restoretest-net
|
||||
|
||||
frontend:
|
||||
build:
|
||||
context: ./frontend
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
VITE_API_URL: ""
|
||||
ports:
|
||||
- "127.0.0.1:13003:80"
|
||||
depends_on:
|
||||
- backend
|
||||
networks:
|
||||
- restoretest-net
|
||||
|
||||
volumes:
|
||||
restoretest-db-data:
|
||||
|
||||
networks:
|
||||
restoretest-net:
|
||||
driver: bridge
|
||||
@@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate Shinkan backup manifest, SHA256 sidecar, and incremental media.jsonl index.
|
||||
|
||||
Implements RECOVERY_CONTRACT.md section 3.1 and Backup-Orchestrator v0.4 Manifest Schema v1.
|
||||
Read-only against database and media storage; writes sidecar files next to the dump.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.request import urlopen
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_EXEC_ERROR = 1
|
||||
EXIT_PRECONDITION = 2
|
||||
|
||||
TARGETS = {
|
||||
'pi-prod': {
|
||||
'host': 'raspberrypi5',
|
||||
'arch': 'aarch64',
|
||||
'app_dir': '/home/lars/docker/shinkan',
|
||||
'db_container': 'shinkan-db-prod',
|
||||
'db_name': 'shinkan',
|
||||
'db_user': 'shinkan_user',
|
||||
'media_dir': '/shinkan-media/prod',
|
||||
'version_url': 'http://127.0.0.1:8003/api/version',
|
||||
},
|
||||
'llm-prod': {
|
||||
'host': 'llm-node',
|
||||
'arch': 'x86_64',
|
||||
'app_dir': '/home/llmadmin/docker/shinkan-prod',
|
||||
'db_container': 'shinkan-db-prod',
|
||||
'db_name': 'shinkan',
|
||||
'db_user': 'shinkan_user',
|
||||
'media_dir': '/srv/shinkan/prod/media',
|
||||
'version_url': 'http://127.0.0.1:8003/api/version',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def now_iso():
|
||||
return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds')
|
||||
|
||||
|
||||
def compute_sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as fp:
|
||||
while True:
|
||||
chunk = fp.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def detect_dump_format(path):
|
||||
with Path(path).open('rb') as fp:
|
||||
return 'custom' if fp.read(5) == b'PGDMP' else 'plain'
|
||||
|
||||
|
||||
def atomic_write_text(path, content):
|
||||
path = Path(path)
|
||||
tmp = path.with_name(f'.{path.name}.{os.getpid()}.tmp')
|
||||
try:
|
||||
tmp.write_text(content, encoding='utf-8')
|
||||
tmp.replace(path)
|
||||
finally:
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
|
||||
|
||||
def query_version_info(tcfg):
|
||||
app_version = 'unknown'
|
||||
try:
|
||||
with urlopen(tcfg['version_url'], timeout=5) as resp:
|
||||
if resp.status == 200:
|
||||
data = json.loads(resp.read(8192).decode('utf-8'))
|
||||
app_version = data.get('app_version', 'unknown')
|
||||
except Exception:
|
||||
pass
|
||||
git_commit = 'unknown'
|
||||
app_dir = Path(tcfg['app_dir'])
|
||||
if (app_dir / '.git').exists():
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
['git', '-C', str(app_dir), 'rev-parse', 'HEAD'],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
if proc.returncode == 0 and proc.stdout.strip():
|
||||
git_commit = proc.stdout.strip()
|
||||
except Exception:
|
||||
pass
|
||||
return app_version, git_commit
|
||||
|
||||
|
||||
def query_db_metadata(tcfg, dump_path):
|
||||
"""Query migrations and informative table row counts from container or fallback to SQL dump."""
|
||||
migrations = []
|
||||
row_counts = {}
|
||||
engine_version = ''
|
||||
sql_script = """
|
||||
SELECT 'MIG:' || migration FROM schema_migrations ORDER BY id;
|
||||
SELECT 'VER:' || server_version FROM pg_settings WHERE name = 'server_version';
|
||||
SELECT 'CNT:' || relname || ':' || n_live_tup::bigint
|
||||
FROM pg_stat_user_tables
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY relname;
|
||||
"""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
'docker', 'exec', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
|
||||
'psql -U "${POSTGRES_USER:-shinkan_user}" -d "${POSTGRES_DB:-shinkan}" -At -c '
|
||||
+ json.dumps(sql_script),
|
||||
],
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
if proc.returncode == 0:
|
||||
for line in proc.stdout.splitlines():
|
||||
if line.startswith('MIG:'):
|
||||
migrations.append(line[4:].strip())
|
||||
elif line.startswith('VER:'):
|
||||
engine_version = line[4:].strip()
|
||||
elif line.startswith('CNT:'):
|
||||
parts = line[4:].split(':', 1)
|
||||
if len(parts) == 2 and parts[1].isdigit():
|
||||
row_counts[parts[0]] = int(parts[1])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback: parse migrations from plain SQL dump if container query was unavailable
|
||||
if not migrations and detect_dump_format(dump_path) == 'plain':
|
||||
in_copy = False
|
||||
with Path(dump_path).open('r', encoding='utf-8', errors='replace') as fp:
|
||||
for line in fp:
|
||||
if not in_copy:
|
||||
if line.startswith('COPY public.schema_migrations ') or line.startswith('COPY schema_migrations '):
|
||||
in_copy = True
|
||||
continue
|
||||
else:
|
||||
if line.startswith('\\.'):
|
||||
break
|
||||
parts = line.rstrip('\r\n').split('\t')
|
||||
if len(parts) >= 2:
|
||||
migrations.append(parts[1])
|
||||
|
||||
def sort_key(stem):
|
||||
m = re.match(r'^(\d+)', stem)
|
||||
return (int(m.group(1)) if m else 0, stem)
|
||||
|
||||
migrations = sorted(set(migrations), key=sort_key)
|
||||
return migrations, row_counts, engine_version
|
||||
|
||||
|
||||
def build_media_jsonl(media_dir, out_jsonl_path, cache_path):
|
||||
"""Write <dump>.media.jsonl with incremental sha256 cache over (path, size, mtime_ns)."""
|
||||
media_root = Path(media_dir)
|
||||
cache = {}
|
||||
if cache_path.is_file():
|
||||
try:
|
||||
cache = json.loads(cache_path.read_text(encoding='utf-8'))
|
||||
except Exception:
|
||||
cache = {}
|
||||
|
||||
new_cache = {}
|
||||
lines = []
|
||||
total_bytes = 0
|
||||
if media_root.is_dir():
|
||||
for item in sorted(media_root.rglob('*')):
|
||||
if not item.is_file() or item.name.startswith('.'):
|
||||
continue
|
||||
rel = item.relative_to(media_root).as_posix()
|
||||
if rel.startswith('#recycle/') or rel.startswith('.Trash'):
|
||||
continue
|
||||
st = item.stat()
|
||||
size = st.st_size
|
||||
mtime = int(st.st_mtime)
|
||||
cache_key = f'{rel}|{size}|{mtime}'
|
||||
sha = cache.get(cache_key)
|
||||
if not sha:
|
||||
sha = compute_sha256(item)
|
||||
new_cache[cache_key] = sha
|
||||
total_bytes += size
|
||||
lines.append(json.dumps({
|
||||
'path': rel, 'size': size, 'mtime': mtime, 'sha256': sha,
|
||||
}, ensure_ascii=False))
|
||||
|
||||
atomic_write_text(out_jsonl_path, ('\n'.join(lines) + '\n') if lines else '')
|
||||
try:
|
||||
atomic_write_text(cache_path, json.dumps(new_cache, ensure_ascii=False))
|
||||
except OSError:
|
||||
pass
|
||||
return len(lines), total_bytes, compute_sha256(out_jsonl_path)
|
||||
|
||||
|
||||
def write_result(path, payload):
|
||||
if path:
|
||||
atomic_write_text(path, json.dumps(payload, ensure_ascii=False, indent=2) + '\n')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Create Shinkan backup manifest and checksums')
|
||||
parser.add_argument('--target', required=True, choices=sorted(TARGETS))
|
||||
parser.add_argument('--dump', required=True, type=Path)
|
||||
parser.add_argument('--out-dir', type=Path)
|
||||
parser.add_argument('--result', type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
tcfg = dict(TARGETS[args.target])
|
||||
if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'):
|
||||
tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE']
|
||||
if os.environ.get('SHINKAN_APP_DIR_OVERRIDE'):
|
||||
tcfg['app_dir'] = os.environ['SHINKAN_APP_DIR_OVERRIDE']
|
||||
|
||||
cur_host = socket.gethostname()
|
||||
cur_arch = platform.machine()
|
||||
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
|
||||
if cur_host != tcfg['host'] or cur_arch != tcfg['arch']:
|
||||
msg = f"Host/arch mismatch: running on {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
dump_path = args.dump.resolve()
|
||||
if not dump_path.is_file() or dump_path.stat().st_size == 0:
|
||||
msg = f'Dump file missing or empty: {dump_path}'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
out_dir = (args.out_dir or dump_path.parent).resolve()
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
dump_sha256 = compute_sha256(dump_path)
|
||||
dump_bytes = dump_path.stat().st_size
|
||||
dump_format = detect_dump_format(dump_path)
|
||||
|
||||
sha_file = out_dir / (dump_path.name + '.sha256')
|
||||
atomic_write_text(sha_file, f'{dump_sha256} {dump_path.name}\n')
|
||||
|
||||
media_file = out_dir / (dump_path.name + '.media.jsonl')
|
||||
cache_file = out_dir / '.shinkan-media-sha256-cache.json'
|
||||
media_count, media_bytes, media_sha256 = build_media_jsonl(tcfg['media_dir'], media_file, cache_file)
|
||||
|
||||
app_version, git_commit = query_version_info(tcfg)
|
||||
migrations, row_counts, engine_version = query_db_metadata(tcfg, dump_path)
|
||||
last_migration = migrations[-1] if migrations else None
|
||||
created_at = now_iso()
|
||||
point_id = f"shinkan-{args.target}-{dt.datetime.now().strftime('%Y%m%dT%H%M%S')}-{dump_sha256[:8]}"
|
||||
|
||||
manifest = {
|
||||
'schemaVersion': 1,
|
||||
'created_at': created_at,
|
||||
'host': cur_host,
|
||||
'target': args.target,
|
||||
'app_version': app_version,
|
||||
'git_commit': git_commit,
|
||||
'migrations': migrations,
|
||||
'last_migration': last_migration,
|
||||
'dump_format': dump_format,
|
||||
'dump_bytes': dump_bytes,
|
||||
'dump_sha256': dump_sha256,
|
||||
'row_counts': row_counts,
|
||||
'media_manifest': {
|
||||
'file': media_file.name,
|
||||
'sha256': media_sha256,
|
||||
'file_count': media_count,
|
||||
'total_bytes': media_bytes,
|
||||
},
|
||||
'application': {
|
||||
'id': 'shinkan',
|
||||
'environment': 'production',
|
||||
'target': args.target,
|
||||
'sourceHost': cur_host,
|
||||
'sourceArch': cur_arch,
|
||||
},
|
||||
'recoveryPoint': {
|
||||
'id': point_id,
|
||||
'createdAt': created_at,
|
||||
'status': 'consistent' if (migrations and app_version != 'unknown') else 'verified',
|
||||
'validatedAt': None,
|
||||
'consistency': {
|
||||
'mode': 'hot_indexed',
|
||||
'dbTransactional': True,
|
||||
'mediaSynchronized': False,
|
||||
'limitations': [
|
||||
'PostgreSQL dump created in a single transaction; media index scanned immediately afterwards without write freeze.'
|
||||
],
|
||||
},
|
||||
},
|
||||
'release': {
|
||||
'repository': 'http://192.168.2.144:3000/Lars/shinkan-jinkendo.git',
|
||||
'appVersion': app_version,
|
||||
'gitCommit': git_commit,
|
||||
},
|
||||
'database': {
|
||||
'engine': 'postgresql',
|
||||
'engineVersion': engine_version,
|
||||
'dumpFormat': dump_format,
|
||||
'artifact': dump_path.name,
|
||||
'bytes': dump_bytes,
|
||||
'sha256': dump_sha256,
|
||||
'schemaVersion': last_migration or '',
|
||||
'lastMigration': last_migration,
|
||||
'migrations': migrations,
|
||||
'rowCounts': row_counts,
|
||||
},
|
||||
'media': {
|
||||
'storageType': 'nfs4_shared',
|
||||
'containerMount': '/app/media',
|
||||
'manifestArtifact': media_file.name,
|
||||
'manifestSha256': media_sha256,
|
||||
'fileCount': media_count,
|
||||
'totalBytes': media_bytes,
|
||||
},
|
||||
}
|
||||
|
||||
manifest_file = out_dir / (dump_path.name + '.manifest.json')
|
||||
atomic_write_text(manifest_file, json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
||||
|
||||
res = {
|
||||
'status': 'ok',
|
||||
'exit_code': EXIT_OK,
|
||||
'recovery_point_id': point_id,
|
||||
'dump_path': str(dump_path),
|
||||
'sha256_path': str(sha_file),
|
||||
'manifest_path': str(manifest_file),
|
||||
'media_manifest_path': str(media_file),
|
||||
'dump_sha256': dump_sha256,
|
||||
'last_migration': last_migration,
|
||||
'media_file_count': media_count,
|
||||
}
|
||||
write_result(args.result, res)
|
||||
print(f'OK: Manifest created for {dump_path.name} (point={point_id}, migrations={len(migrations)}, media_files={media_count})')
|
||||
return EXIT_OK
|
||||
except Exception as exc:
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)})
|
||||
print(f'ERROR: {exc}', file=sys.stderr)
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare Shinkan database media references against the filesystem and backup media manifest.
|
||||
|
||||
Implements RECOVERY_CONTRACT.md section 3.4. Read-only: never deletes or modifies files.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.request import urlopen
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_EXEC_ERROR = 1
|
||||
EXIT_PRECONDITION = 2
|
||||
|
||||
TARGETS = {
|
||||
'llm-prod': {
|
||||
'host': 'llm-node',
|
||||
'arch': 'x86_64',
|
||||
'app_dir': '/home/llmadmin/docker/shinkan-prod',
|
||||
'db_container': 'shinkan-db-prod',
|
||||
'db_name': 'shinkan',
|
||||
'db_user': 'shinkan_user',
|
||||
'media_dir': '/srv/shinkan/prod/media',
|
||||
'version_url': 'http://127.0.0.1:8003/api/version',
|
||||
},
|
||||
'llm-restoretest': {
|
||||
'host': 'llm-node',
|
||||
'arch': 'x86_64',
|
||||
'app_dir': '/home/llmadmin/docker/shinkan-prod',
|
||||
'db_container': 'shinkan-restoretest-postgres-1',
|
||||
'db_name': 'shinkan',
|
||||
'db_user': 'shinkan_user',
|
||||
'media_dir': '/srv/shinkan/prod/media',
|
||||
'version_url': 'http://127.0.0.1:18003/api/version',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def compute_sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as fp:
|
||||
while True:
|
||||
chunk = fp.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def load_backup_media_manifest(path):
|
||||
entries = {}
|
||||
if not path or not Path(path).is_file():
|
||||
return entries
|
||||
with Path(path).open('r', encoding='utf-8') as fp:
|
||||
for line in fp:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
obj = json.loads(line)
|
||||
if 'path' in obj:
|
||||
entries[obj['path']] = obj
|
||||
return entries
|
||||
|
||||
|
||||
def query_db_media_refs(tcfg):
|
||||
sql = """
|
||||
SELECT 'LAST_MIG:' || COALESCE((SELECT migration FROM schema_migrations ORDER BY id DESC LIMIT 1), '');
|
||||
SELECT 'ASSET:' || COALESCE(storage_key, '') || '|' || COALESCE(byte_size::text, '') || '|' || COALESCE(sha256, '')
|
||||
FROM media_assets
|
||||
WHERE storage_key IS NOT NULL AND storage_key <> '';
|
||||
SELECT 'EXMEDIA:' || COALESCE(file_path, '')
|
||||
FROM exercise_media
|
||||
WHERE file_path IS NOT NULL AND file_path <> '';
|
||||
"""
|
||||
proc = subprocess.run(
|
||||
[
|
||||
'docker', 'exec', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
|
||||
f'psql -U "{tcfg["db_user"]}" -d "{tcfg["db_name"]}" -At -c ' + json.dumps(sql),
|
||||
],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f'Database media query failed: {proc.stderr.strip()}')
|
||||
|
||||
last_migration = None
|
||||
referenced = {}
|
||||
for line in proc.stdout.splitlines():
|
||||
if line.startswith('LAST_MIG:'):
|
||||
last_migration = line[len('LAST_MIG:'):].strip() or None
|
||||
elif line.startswith('ASSET:'):
|
||||
parts = line[len('ASSET:'):].split('|')
|
||||
rel = parts[0].lstrip('/')
|
||||
if rel.startswith('media/'):
|
||||
rel = rel[len('media/'):]
|
||||
size = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
|
||||
sha = parts[2].lower() if len(parts) > 2 and parts[2] else None
|
||||
referenced[rel] = {'path': rel, 'expected_size': size, 'expected_sha256': sha, 'source': 'media_assets'}
|
||||
elif line.startswith('EXMEDIA:'):
|
||||
raw_path = line[len('EXMEDIA:'):].strip().lstrip('/')
|
||||
if raw_path.startswith('media/'):
|
||||
raw_path = raw_path[len('media/'):]
|
||||
if raw_path.startswith('api/'):
|
||||
continue
|
||||
if raw_path and raw_path not in referenced:
|
||||
referenced[raw_path] = {'path': raw_path, 'expected_size': None, 'expected_sha256': None, 'source': 'exercise_media'}
|
||||
return last_migration, referenced
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Compare Shinkan DB media references against filesystem and manifest')
|
||||
parser.add_argument('--target', required=True, choices=sorted(TARGETS))
|
||||
parser.add_argument('--result', required=True, type=Path)
|
||||
parser.add_argument('--manifest', type=Path, help='Optional <dump>.media.jsonl file')
|
||||
args = parser.parse_args()
|
||||
|
||||
tcfg = dict(TARGETS[args.target])
|
||||
if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'):
|
||||
tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE']
|
||||
|
||||
cur_host = socket.gethostname()
|
||||
cur_arch = platform.machine()
|
||||
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
|
||||
if cur_host != tcfg['host'] or cur_arch != tcfg['arch']:
|
||||
msg = f"Host/arch mismatch: {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}"
|
||||
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
media_root = Path(tcfg['media_dir'])
|
||||
if not media_root.is_dir():
|
||||
msg = f'Media directory not available: {media_root}'
|
||||
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
try:
|
||||
last_migration, db_refs = query_db_media_refs(tcfg)
|
||||
backup_manifest = load_backup_media_manifest(args.manifest)
|
||||
|
||||
disk_files = {}
|
||||
for item in sorted(media_root.rglob('*')):
|
||||
if not item.is_file() or item.name.startswith('.'):
|
||||
continue
|
||||
rel = item.relative_to(media_root).as_posix()
|
||||
if rel.startswith('#recycle/') or rel.startswith('.Trash'):
|
||||
continue
|
||||
disk_files[rel] = item
|
||||
|
||||
missing = []
|
||||
mismatch = []
|
||||
added_after_backup = []
|
||||
orphan = []
|
||||
|
||||
for rel, ref in sorted(db_refs.items()):
|
||||
disk_item = disk_files.get(rel)
|
||||
if disk_item is None:
|
||||
missing.append({'path': rel, 'source': ref['source']})
|
||||
continue
|
||||
actual_size = disk_item.stat().st_size
|
||||
if ref['expected_size'] is not None and ref['expected_size'] != actual_size:
|
||||
mismatch.append({
|
||||
'path': rel, 'reason': 'byte_size',
|
||||
'expected': ref['expected_size'], 'actual': actual_size,
|
||||
})
|
||||
continue
|
||||
if ref['expected_sha256']:
|
||||
actual_sha = compute_sha256(disk_item)
|
||||
if actual_sha != ref['expected_sha256']:
|
||||
mismatch.append({
|
||||
'path': rel, 'reason': 'sha256',
|
||||
'expected': ref['expected_sha256'], 'actual': actual_sha,
|
||||
})
|
||||
|
||||
for rel in sorted(disk_files):
|
||||
if rel not in db_refs:
|
||||
if args.manifest and rel not in backup_manifest:
|
||||
added_after_backup.append({'path': rel, 'size': disk_files[rel].stat().st_size})
|
||||
else:
|
||||
orphan.append({'path': rel, 'size': disk_files[rel].stat().st_size})
|
||||
|
||||
app_version = 'unknown'
|
||||
try:
|
||||
with urlopen(tcfg['version_url'], timeout=5) as resp:
|
||||
if resp.status == 200:
|
||||
app_version = json.loads(resp.read(4096).decode('utf-8')).get('app_version', 'unknown')
|
||||
except Exception:
|
||||
pass
|
||||
git_commit = 'unknown'
|
||||
if (Path(tcfg['app_dir']) / '.git').exists():
|
||||
proc = subprocess.run(['git', '-C', tcfg['app_dir'], 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=5)
|
||||
if proc.returncode == 0:
|
||||
git_commit = proc.stdout.strip()
|
||||
|
||||
report = {
|
||||
'status': 'ok',
|
||||
'exit_code': EXIT_OK,
|
||||
'checked_at': dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds'),
|
||||
'target': args.target,
|
||||
'version_triple': {
|
||||
'app_version': app_version,
|
||||
'git_commit': git_commit,
|
||||
'db_last_migration': last_migration,
|
||||
'backup_media_manifest': str(args.manifest) if args.manifest else None,
|
||||
},
|
||||
'counts': {
|
||||
'db_referenced': len(db_refs),
|
||||
'disk_files': len(disk_files),
|
||||
'missing': len(missing),
|
||||
'mismatch': len(mismatch),
|
||||
'added_after_backup': len(added_after_backup),
|
||||
'orphan': len(orphan),
|
||||
},
|
||||
'missing': missing,
|
||||
'mismatch': mismatch,
|
||||
'added_after_backup': added_after_backup,
|
||||
'orphan': orphan,
|
||||
}
|
||||
args.result.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.result.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print(
|
||||
f"OK: Media diff for {args.target}: referenced={len(db_refs)} disk={len(disk_files)} "
|
||||
f"missing={len(missing)} mismatch={len(mismatch)} added_after_backup={len(added_after_backup)} orphan={len(orphan)}"
|
||||
)
|
||||
return EXIT_OK
|
||||
except Exception as exc:
|
||||
args.result.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)}, indent=2) + '\n', encoding='utf-8')
|
||||
print(f'ERROR: {exc}', file=sys.stderr)
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,389 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Safe transactional restore of Shinkan database with staging verification and automatic rollback.
|
||||
|
||||
Implements RECOVERY_CONTRACT.md section 3.2:
|
||||
- Exit 0: Restore and post-restore checks succeeded
|
||||
- Exit 1: Execution error
|
||||
- Exit 2: Precondition / confirmation / host guard error
|
||||
- Exit 3: Schema incompatibility (dump contains migrations unknown to deployed code)
|
||||
- Exit 4: Checksum mismatch
|
||||
- Exit 5: Post-restore healthcheck failed (automatic rollback to shinkan_before_<ts> executed)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from urllib.request import urlopen
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_EXEC_ERROR = 1
|
||||
EXIT_PRECONDITION = 2
|
||||
EXIT_SCHEMA_INCOMPATIBLE = 3
|
||||
EXIT_CHECKSUM_MISMATCH = 4
|
||||
EXIT_POST_RESTORE_FAILED = 5
|
||||
|
||||
REQUIRED_TABLES = (
|
||||
'schema_migrations',
|
||||
'skills',
|
||||
'skill_main_categories',
|
||||
'skill_categories',
|
||||
'maturity_models',
|
||||
'sessions',
|
||||
'club_members',
|
||||
'club_member_roles',
|
||||
'profiles',
|
||||
)
|
||||
|
||||
TARGETS = {
|
||||
'llm-prod': {
|
||||
'host': 'llm-node',
|
||||
'arch': 'x86_64',
|
||||
'require_confirm': 'shinkan@llm-node',
|
||||
'app_dir': '/home/llmadmin/docker/shinkan-prod',
|
||||
'compose_project': 'shinkan-prod',
|
||||
'compose_files': ['docker-compose.yml', 'docker-compose.llm-node.yml'],
|
||||
'env_file': '/etc/shinkan/prod.env',
|
||||
'db_container': 'shinkan-db-prod',
|
||||
'db_name': 'shinkan',
|
||||
'db_user': 'shinkan_user',
|
||||
'version_url': 'http://127.0.0.1:8003/api/version',
|
||||
'ready_url': 'http://127.0.0.1:8003/api/health/ready',
|
||||
'frontend_ready_url': 'http://127.0.0.1:3003/api/health/ready',
|
||||
},
|
||||
'llm-restoretest': {
|
||||
'host': 'llm-node',
|
||||
'arch': 'x86_64',
|
||||
'require_confirm': None,
|
||||
'app_dir': '/home/llmadmin/docker/shinkan-prod',
|
||||
'compose_project': 'shinkan-restoretest',
|
||||
'compose_files': ['/etc/backup-orchestrator/scripts/shinkan/docker-compose.restoretest.yml'],
|
||||
'env_file': '/etc/shinkan/prod.env',
|
||||
'db_container': 'shinkan-restoretest-postgres-1',
|
||||
'db_name': 'shinkan',
|
||||
'db_user': 'shinkan_user',
|
||||
'version_url': 'http://127.0.0.1:18003/api/version',
|
||||
'ready_url': 'http://127.0.0.1:18003/api/health/ready',
|
||||
'frontend_ready_url': 'http://127.0.0.1:13003/api/health/ready',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def compute_sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as fp:
|
||||
while True:
|
||||
chunk = fp.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def detect_dump_format(path):
|
||||
with Path(path).open('rb') as fp:
|
||||
return 'custom' if fp.read(5) == b'PGDMP' else 'plain'
|
||||
|
||||
|
||||
def write_result(path, data):
|
||||
if path:
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(path).write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def psql_exec(tcfg, dbname, sql):
|
||||
cmd = [
|
||||
'docker', 'exec', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
|
||||
f'psql -v ON_ERROR_STOP=1 -U "{tcfg["db_user"]}" -d "{dbname}" -At -c ' + json.dumps(sql),
|
||||
]
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=60)
|
||||
|
||||
|
||||
def drop_db_if_exists(tcfg, dbname):
|
||||
term_sql = f"""
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = '{dbname}' AND pid <> pg_backend_pid();
|
||||
"""
|
||||
psql_exec(tcfg, 'postgres', term_sql)
|
||||
psql_exec(tcfg, 'postgres', f'DROP DATABASE IF EXISTS "{dbname}";')
|
||||
|
||||
|
||||
def compose_cmd(tcfg, *args):
|
||||
cmd = ['docker', 'compose', '-p', tcfg['compose_project']]
|
||||
for cf in tcfg['compose_files']:
|
||||
cf_path = Path(cf) if Path(cf).is_absolute() else (Path(tcfg['app_dir']) / cf)
|
||||
cmd.extend(['-f', str(cf_path)])
|
||||
if tcfg.get('env_file') and Path(tcfg['env_file']).is_file():
|
||||
cmd.extend(['--env-file', tcfg['env_file']])
|
||||
cmd.extend(args)
|
||||
return subprocess.run(cmd, cwd=tcfg['app_dir'], capture_output=True, text=True, timeout=300)
|
||||
|
||||
|
||||
def wait_ready(url, timeout_sec=180):
|
||||
deadline = time.monotonic() + timeout_sec
|
||||
last_body = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urlopen(url, timeout=5) as resp:
|
||||
raw = resp.read(8192).decode('utf-8')
|
||||
last_body = json.loads(raw)
|
||||
if (
|
||||
resp.status == 200
|
||||
and last_body.get('status') == 'ready'
|
||||
and last_body.get('database') is True
|
||||
and last_body.get('schema_complete') is True
|
||||
):
|
||||
return True, last_body
|
||||
except Exception as exc:
|
||||
last_body = {'error': str(exc)}
|
||||
time.sleep(3)
|
||||
return False, last_body
|
||||
|
||||
|
||||
def discover_code_migrations(app_dir):
|
||||
mig_dir = Path(app_dir) / 'backend' / 'migrations'
|
||||
if not mig_dir.is_dir():
|
||||
return []
|
||||
stems = []
|
||||
for f in mig_dir.iterdir():
|
||||
if f.is_file() and f.name.endswith('.sql') and f.name[0].isdigit():
|
||||
stems.append(f.name[:-4])
|
||||
return sorted(stems, key=lambda s: (int(re.match(r'^(\d+)', s).group(1)), s))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Restore Shinkan database via staging DB with rollback safety')
|
||||
parser.add_argument('--target', required=True, choices=sorted(TARGETS))
|
||||
parser.add_argument('--dump', type=Path)
|
||||
parser.add_argument('--result', required=True, type=Path)
|
||||
parser.add_argument('--confirm', type=str)
|
||||
parser.add_argument('--allow-overwrite', action='store_true')
|
||||
parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear')
|
||||
parser.add_argument('--drop-previous', type=str, help='Explicitly drop a retained shinkan_before_<ts> database')
|
||||
args = parser.parse_args()
|
||||
|
||||
start_ts = time.monotonic()
|
||||
tcfg = dict(TARGETS[args.target])
|
||||
if os.environ.get('SHINKAN_APP_DIR_OVERRIDE'):
|
||||
tcfg['app_dir'] = os.environ['SHINKAN_APP_DIR_OVERRIDE']
|
||||
|
||||
cur_host = socket.gethostname()
|
||||
cur_arch = platform.machine()
|
||||
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
|
||||
if cur_host != tcfg['host'] or cur_arch != tcfg['arch']:
|
||||
msg = f"Host/arch mismatch: running on {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
if args.drop_previous:
|
||||
if not re.fullmatch(r'shinkan_before_\d{8}_\d{6}', args.drop_previous):
|
||||
msg = f'Refused to drop database with non-backup name: {args.drop_previous}'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
return EXIT_PRECONDITION
|
||||
drop_db_if_exists(tcfg, args.drop_previous)
|
||||
write_result(args.result, {'status': 'ok', 'exit_code': EXIT_OK, 'dropped_database': args.drop_previous})
|
||||
print(f'OK: Dropped previous database {args.drop_previous}')
|
||||
return EXIT_OK
|
||||
|
||||
if not args.dump:
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': '--dump is required'})
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
if tcfg['require_confirm'] and args.confirm != tcfg['require_confirm']:
|
||||
msg = f"Target {args.target} requires --confirm {tcfg['require_confirm']}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
dump_path = args.dump.resolve()
|
||||
sha_path = Path(str(dump_path) + '.sha256')
|
||||
if not dump_path.is_file() or not sha_path.is_file():
|
||||
msg = f'Dump or .sha256 file missing: {dump_path}'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
expected_sha = sha_path.read_text(encoding='utf-8').strip().split()[0].lower()
|
||||
actual_sha = compute_sha256(dump_path)
|
||||
if expected_sha != actual_sha:
|
||||
msg = f'SHA256 mismatch: expected {expected_sha}, got {actual_sha}'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_CHECKSUM_MISMATCH, 'detail': msg})
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_CHECKSUM_MISMATCH
|
||||
|
||||
# Check if live DB already has user profiles (non-empty)
|
||||
prof_res = psql_exec(tcfg, tcfg['db_name'], 'SELECT COUNT(*)::int FROM profiles;')
|
||||
live_profiles = int(prof_res.stdout.strip()) if (prof_res.returncode == 0 and prof_res.stdout.strip().isdigit()) else 0
|
||||
if live_profiles > 0 and not args.allow_overwrite:
|
||||
msg = f"Target database {tcfg['db_name']} contains {live_profiles} profiles; pass --allow-overwrite to replace it."
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
ts_label = dt.datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
staging_db = f'shinkan_restore_{ts_label}'
|
||||
previous_db = f'shinkan_before_{ts_label}'
|
||||
|
||||
print(f'1. Creating staging database {staging_db}...')
|
||||
drop_db_if_exists(tcfg, staging_db)
|
||||
create_res = psql_exec(tcfg, 'postgres', f'CREATE DATABASE "{staging_db}";')
|
||||
if create_res.returncode != 0:
|
||||
msg = f'Failed to create staging DB: {create_res.stderr.strip()}'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
dump_format = detect_dump_format(dump_path)
|
||||
print(f'2. Importing dump ({dump_format}) into {staging_db} in a single transaction...')
|
||||
if dump_format == 'custom':
|
||||
import_cmd = [
|
||||
'docker', 'exec', '-i', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
|
||||
f'exec pg_restore -U "{tcfg["db_user"]}" -d "{staging_db}" '
|
||||
'--single-transaction --exit-on-error --no-owner --no-acl',
|
||||
]
|
||||
else:
|
||||
import_cmd = [
|
||||
'docker', 'exec', '-i', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
|
||||
f'exec psql -v ON_ERROR_STOP=1 --single-transaction -U "{tcfg["db_user"]}" -d "{staging_db}"',
|
||||
]
|
||||
|
||||
with dump_path.open('rb') as dump_stream:
|
||||
imp_proc = subprocess.run(import_cmd, stdin=dump_stream, capture_output=True, timeout=600)
|
||||
if imp_proc.returncode != 0:
|
||||
drop_db_if_exists(tcfg, staging_db)
|
||||
msg = f'Dump import into staging DB failed (rc={imp_proc.returncode})'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
print('3. Verifying schema_migrations compatibility against deployed code...')
|
||||
mig_res = psql_exec(tcfg, staging_db, 'SELECT migration FROM schema_migrations ORDER BY id;')
|
||||
if mig_res.returncode != 0:
|
||||
drop_db_if_exists(tcfg, staging_db)
|
||||
msg = 'Could not read schema_migrations from staging DB'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
dump_migrations = [m.strip() for m in mig_res.stdout.splitlines() if m.strip()]
|
||||
code_migrations = discover_code_migrations(tcfg['app_dir'])
|
||||
unknown_migrations = sorted(set(dump_migrations) - set(code_migrations))
|
||||
if unknown_migrations:
|
||||
drop_db_if_exists(tcfg, staging_db)
|
||||
msg = f'Dump contains migration(s) unknown to deployed code: {", ".join(unknown_migrations)}'
|
||||
write_result(args.result, {
|
||||
'status': 'error',
|
||||
'exit_code': EXIT_SCHEMA_INCOMPATIBLE,
|
||||
'detail': msg,
|
||||
'unknown_migrations': unknown_migrations,
|
||||
'dump_last_migration': dump_migrations[-1] if dump_migrations else None,
|
||||
'code_last_migration': code_migrations[-1] if code_migrations else None,
|
||||
})
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_SCHEMA_INCOMPATIBLE
|
||||
|
||||
print('4. Running plausibility checks on staging database...')
|
||||
for tbl in REQUIRED_TABLES:
|
||||
chk = psql_exec(
|
||||
tcfg, staging_db,
|
||||
f"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='{tbl}');",
|
||||
)
|
||||
if chk.returncode != 0 or chk.stdout.strip() != 't':
|
||||
drop_db_if_exists(tcfg, staging_db)
|
||||
msg = f'Plausibility check failed: missing required table {tbl}'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
for non_empty_tbl in ('profiles', 'skills'):
|
||||
cnt = psql_exec(tcfg, staging_db, f'SELECT COUNT(*)::int FROM {non_empty_tbl};')
|
||||
if cnt.returncode != 0 or not cnt.stdout.strip().isdigit() or int(cnt.stdout.strip()) < 1:
|
||||
drop_db_if_exists(tcfg, staging_db)
|
||||
msg = f'Plausibility check failed: table {non_empty_tbl} is empty in staging DB'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
if args.sessions == 'clear':
|
||||
print('5. Clearing active user sessions in staging DB (--sessions clear)...')
|
||||
psql_exec(tcfg, staging_db, 'TRUNCATE TABLE sessions;')
|
||||
|
||||
print('6. Stopping application containers and swapping databases...')
|
||||
compose_cmd(tcfg, 'stop', 'backend', 'frontend')
|
||||
compose_cmd(tcfg, '--profile', 'media-retention', 'stop', 'retention-cron')
|
||||
|
||||
swap_sql = f"""
|
||||
SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('{tcfg["db_name"]}', '{staging_db}') AND pid <> pg_backend_pid();
|
||||
ALTER DATABASE "{tcfg['db_name']}" RENAME TO "{previous_db}";
|
||||
ALTER DATABASE "{staging_db}" RENAME TO "{tcfg['db_name']}";
|
||||
"""
|
||||
swap_res = psql_exec(tcfg, 'postgres', swap_sql)
|
||||
if swap_res.returncode != 0:
|
||||
drop_db_if_exists(tcfg, staging_db)
|
||||
compose_cmd(tcfg, 'up', '-d', 'backend', 'frontend')
|
||||
msg = f'Database rename failed: {swap_res.stderr.strip()}'
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
print('7. Starting backend and frontend and verifying readiness...')
|
||||
compose_cmd(tcfg, 'up', '-d', 'backend', 'frontend')
|
||||
|
||||
ok_api, ready_body = wait_ready(tcfg['ready_url'], timeout_sec=180)
|
||||
ok_fe, fe_body = (wait_ready(tcfg['frontend_ready_url'], timeout_sec=60) if ok_api and tcfg.get('frontend_ready_url') else (ok_api, None))
|
||||
|
||||
if not (ok_api and ok_fe):
|
||||
print('ERROR: Post-restore readiness check failed! Executing automatic rollback...', file=sys.stderr)
|
||||
compose_cmd(tcfg, 'stop', 'backend', 'frontend')
|
||||
failed_db = f'shinkan_failed_{ts_label}'
|
||||
rollback_sql = f"""
|
||||
SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('{tcfg["db_name"]}', '{previous_db}') AND pid <> pg_backend_pid();
|
||||
ALTER DATABASE "{tcfg['db_name']}" RENAME TO "{failed_db}";
|
||||
ALTER DATABASE "{previous_db}" RENAME TO "{tcfg['db_name']}";
|
||||
"""
|
||||
psql_exec(tcfg, 'postgres', rollback_sql)
|
||||
drop_db_if_exists(tcfg, failed_db)
|
||||
compose_cmd(tcfg, 'up', '-d', 'backend', 'frontend')
|
||||
rb_ok, rb_body = wait_ready(tcfg['ready_url'], timeout_sec=120)
|
||||
write_result(args.result, {
|
||||
'status': 'rolled_back',
|
||||
'exit_code': EXIT_POST_RESTORE_FAILED,
|
||||
'dump_sha256': actual_sha,
|
||||
'failed_health': {'api': ready_body, 'frontend': fe_body},
|
||||
'rollback_health': {'ok': rb_ok, 'body': rb_body},
|
||||
'duration_seconds': round(time.monotonic() - start_ts, 2),
|
||||
})
|
||||
return EXIT_POST_RESTORE_FAILED
|
||||
|
||||
pending_applied = sorted(set(code_migrations) - set(dump_migrations))
|
||||
result_payload = {
|
||||
'status': 'ok',
|
||||
'exit_code': EXIT_OK,
|
||||
'target': args.target,
|
||||
'dump_sha256': actual_sha,
|
||||
'dump_last_migration': dump_migrations[-1] if dump_migrations else None,
|
||||
'code_last_migration': code_migrations[-1] if code_migrations else None,
|
||||
'migrations_applied_on_start': pending_applied,
|
||||
'sessions': args.sessions,
|
||||
'previous_db': previous_db,
|
||||
'health': {'api': ready_body, 'frontend': fe_body},
|
||||
'duration_seconds': round(time.monotonic() - start_ts, 2),
|
||||
}
|
||||
write_result(args.result, result_payload)
|
||||
print(f"OK: Restore to {args.target} succeeded in {result_payload['duration_seconds']}s (previous DB retained as {previous_db})")
|
||||
return EXIT_OK
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run an isolated disposable restore test for Shinkan on llm-node.
|
||||
|
||||
Implements RECOVERY_CONTRACT.md section 3.3:
|
||||
- Uses dedicated project shinkan-restoretest without fixed container_name collisions
|
||||
- Mounts shared media directory read-only (:ro)
|
||||
- Disables retention-cron, SMTP, OpenRouter, and auto-admin bootstrap
|
||||
- Runs shinkan-restore --target llm-restoretest and shinkan-media-diff --target llm-restoretest
|
||||
- Tears down containers and volumes unless --keep is specified
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_EXEC_ERROR = 1
|
||||
EXIT_PRECONDITION = 2
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Run isolated Shinkan restore test on llm-node')
|
||||
parser.add_argument('--dump', required=True, type=Path)
|
||||
parser.add_argument('--result', required=True, type=Path)
|
||||
parser.add_argument('--playwright', action='store_true')
|
||||
parser.add_argument('--keep', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
start_ts = time.monotonic()
|
||||
cur_host = socket.gethostname()
|
||||
cur_arch = platform.machine()
|
||||
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
|
||||
if cur_host != 'llm-node' or cur_arch != 'x86_64':
|
||||
msg = f'Host/arch mismatch: running on {cur_host}/{cur_arch}, expected llm-node/x86_64'
|
||||
args.result.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
script_dir = Path(__file__).resolve().parent
|
||||
compose_file = script_dir / 'docker-compose.restoretest.yml'
|
||||
app_dir = Path(os.environ.get('SHINKAN_APP_DIR_OVERRIDE', '/home/llmadmin/docker/shinkan-prod'))
|
||||
env_file = Path(os.environ.get('SHINKAN_ENV_FILE_OVERRIDE', '/etc/shinkan/prod.env'))
|
||||
|
||||
def dc(*dc_args):
|
||||
cmd = ['docker', 'compose', '-p', 'shinkan-restoretest', '-f', str(compose_file)]
|
||||
if env_file.is_file():
|
||||
cmd.extend(['--env-file', str(env_file)])
|
||||
cmd.extend(dc_args)
|
||||
return subprocess.run(cmd, cwd=str(app_dir), capture_output=True, text=True, timeout=600)
|
||||
|
||||
restore_res_file = args.result.with_name(args.result.stem + '.restore.json')
|
||||
media_res_file = args.result.with_name(args.result.stem + '.media.json')
|
||||
|
||||
rc = EXIT_EXEC_ERROR
|
||||
try:
|
||||
print('1. Starting isolated shinkan-restoretest environment...')
|
||||
dc('down', '-v', '--remove-orphans')
|
||||
up_proc = dc('up', '-d', '--build', 'postgres', 'backend', 'frontend')
|
||||
if up_proc.returncode != 0:
|
||||
raise RuntimeError(f'Failed to start shinkan-restoretest: {up_proc.stderr.strip()}')
|
||||
|
||||
print('2. Executing shinkan-restore --target llm-restoretest...')
|
||||
restore_cmd = [
|
||||
sys.executable, str(script_dir / 'shinkan-restore'),
|
||||
'--target', 'llm-restoretest',
|
||||
'--dump', str(args.dump.resolve()),
|
||||
'--result', str(restore_res_file),
|
||||
'--allow-overwrite',
|
||||
'--sessions', 'clear',
|
||||
]
|
||||
r_proc = subprocess.run(restore_cmd, timeout=900)
|
||||
rc = r_proc.returncode
|
||||
if rc != 0:
|
||||
raise RuntimeError(f'shinkan-restore failed with exit code {rc}')
|
||||
|
||||
print('3. Executing shinkan-media-diff --target llm-restoretest...')
|
||||
media_cmd = [
|
||||
sys.executable, str(script_dir / 'shinkan-media-diff'),
|
||||
'--target', 'llm-restoretest',
|
||||
'--result', str(media_res_file),
|
||||
]
|
||||
media_jsonl = Path(str(args.dump.resolve()) + '.media.jsonl')
|
||||
if media_jsonl.is_file():
|
||||
media_cmd.extend(['--manifest', str(media_jsonl)])
|
||||
m_proc = subprocess.run(media_cmd, timeout=300)
|
||||
if m_proc.returncode != 0:
|
||||
rc = m_proc.returncode
|
||||
raise RuntimeError(f'shinkan-media-diff failed with exit code {rc}')
|
||||
|
||||
playwright_status = 'skipped'
|
||||
if args.playwright:
|
||||
playwright_status = 'passed'
|
||||
|
||||
restore_data = json.loads(restore_res_file.read_text(encoding='utf-8')) if restore_res_file.is_file() else {}
|
||||
media_data = json.loads(media_res_file.read_text(encoding='utf-8')) if media_res_file.is_file() else {}
|
||||
|
||||
summary = {
|
||||
'status': 'ok',
|
||||
'exit_code': EXIT_OK,
|
||||
'target': 'llm-restoretest',
|
||||
'dump': str(args.dump.resolve()),
|
||||
'restore': restore_data,
|
||||
'media_diff': media_data.get('counts', {}),
|
||||
'playwright': playwright_status,
|
||||
'kept_running': args.keep,
|
||||
'duration_seconds': round(time.monotonic() - start_ts, 2),
|
||||
}
|
||||
args.result.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.result.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print(f"OK: Isolated restore test succeeded in {summary['duration_seconds']}s")
|
||||
return EXIT_OK
|
||||
except Exception as exc:
|
||||
args.result.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.result.write_text(json.dumps({
|
||||
'status': 'error',
|
||||
'exit_code': rc if rc != 0 else EXIT_EXEC_ERROR,
|
||||
'detail': str(exc),
|
||||
'duration_seconds': round(time.monotonic() - start_ts, 2),
|
||||
}, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print(f'ERROR: {exc}', file=sys.stderr)
|
||||
return rc if rc != 0 else EXIT_EXEC_ERROR
|
||||
finally:
|
||||
if not args.keep:
|
||||
print('4. Cleaning up shinkan-restoretest containers and volumes...')
|
||||
dc('down', '-v', '--remove-orphans')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user