feat(v0.4): import all Raspberry Pi scripts and implement Recovery Management engine + Shinkan adapters

This commit is contained in:
Lars committed 2026-10-29 14:18:20 +01:00
1 parent 40188c0e10
commit c53fc0a939
35 files changed
+2877 -191

No files matched your search

+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-mariadb-docker.sh \
"bookstack_db" "bookstack.sql" "mysql"
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "buyingagent-dev-postgres-1" "buyingagent-dev.sql"
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "buyingagent-db-prod" "buyingagent-prod.sql"
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kairo-dev-postgres-1" "kairo-dev.sql"
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kairo-db-prod" "kairo-prod.sql"
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kansho-dev-postgres-1" "kansho-dev.sql"
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "kansho-db-prod" "kansho-prod.sql"
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-mariadb-docker.sh \
"karatewiki-dev-db-1" "karatewiki-dev.sql" "mariadb"
+8
View File
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
"dev-mitai-postgres" \
"mitai-dev.sql" \
"mitai_dev" \
"mitai_dev"
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
"mitai-db-prod" \
"mitai-prod.sql"
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
"netbox-postgres-1" "netbox.sql"
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env bash
set -Eeuo pipefail
umask 077
# Optionales lokales Credential-File (chmod 600, nicht in Git)
if [[ -f /etc/backup-orchestrator/credentials/paperless.env ]]; then
# shellcheck disable=SC1091
source /etc/backup-orchestrator/credentials/paperless.env
fi
# === Datenbank ===
DB_HOST="${DB_HOST:-192.168.2.40}"
DB_PORT="${DB_PORT:-3308}"
DB_USER="${DB_USER:-paperless}"
DB_PASSWORD="${DB_PASSWORD:-HIER_PASSWORT_EINTRAGEN}"
DB_NAME="${DB_NAME:-paperless}"
# === Backup ===
BACKUP_DIR="${BACKUP_DIR:-/media/lars/paperless/db-backup}"
BACKUP_FILE="$BACKUP_DIR/paperless.sql"
[[ "$DB_PASSWORD" != "HIER_PASSWORT_EINTRAGEN" ]] || {
echo "FEHLER: Datenbankpasswort nicht konfiguriert" >&2
exit 1
}
mkdir -p "$BACKUP_DIR"
TEMP_FILE="$(mktemp "$BACKUP_DIR/.paperless.XXXXXXXX.sql")"
trap 'rm -f -- "$TEMP_FILE"' EXIT
echo "Paperless: Datenbankexport gestartet"
MYSQL_PWD="$DB_PASSWORD" mysqldump \
--host="$DB_HOST" \
--port="$DB_PORT" \
--user="$DB_USER" \
--single-transaction \
--quick \
--databases "$DB_NAME" \
> "$TEMP_FILE"
if [[ ! -s "$TEMP_FILE" ]]; then
echo "FEHLER: Paperless-Dump ist leer" >&2
exit 1
fi
chgrp lars "$TEMP_FILE"
chmod 640 "$TEMP_FILE"
mv -f -- "$TEMP_FILE" "$BACKUP_FILE"
echo "OK: Paperless-Datenbank gesichert"
echo "Datei: $BACKUP_FILE"
exit 0
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
"shinkan-dev-postgres-1" \
"shinkan-dev.sql"
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh \
"shinkan-db-prod" \
"shinkan-prod.sql"
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
set -Eeuo pipefail
umask 077
# Optionales lokales Credential-File (chmod 600, nicht in Git)
if [[ -f /etc/backup-orchestrator/credentials/tandoor.env ]]; then
# shellcheck disable=SC1091
source /etc/backup-orchestrator/credentials/tandoor.env
fi
# === Datenbank ===
PGHOST="${PGHOST:-192.168.2.40}"
PGPORT="${PGPORT:-5432}"
PGUSER="${PGUSER:-djangouser}"
PGDATABASE="${PGDATABASE:-djangodb}"
PGPASSWORD="${PGPASSWORD:-HIER_PASSWORT_EINTRAGEN}"
# === Backup ===
BACKUP_DIR="${BACKUP_DIR:-/media/lars/paperless/db-backup}"
BACKUP_FILE="$BACKUP_DIR/tandoor.sql"
[[ "$PGPASSWORD" != "HIER_PASSWORT_EINTRAGEN" ]] || {
echo "FEHLER: Datenbankpasswort nicht konfiguriert" >&2
exit 1
}
mkdir -p "$BACKUP_DIR"
TEMP_FILE="$(mktemp "$BACKUP_DIR/.tandoor.XXXXXXXX.sql")"
trap 'rm -f -- "$TEMP_FILE"' EXIT
echo "Tandoor: Datenbankexport gestartet"
PGPASSWORD="$PGPASSWORD" pg_dump \
--host="$PGHOST" \
--port="$PGPORT" \
--username="$PGUSER" \
--dbname="$PGDATABASE" \
--format=plain \
> "$TEMP_FILE"
if [[ ! -s "$TEMP_FILE" ]]; then
echo "FEHLER: Tandoor-Dump ist leer" >&2
exit 1
fi
chgrp lars "$TEMP_FILE"
chmod 640 "$TEMP_FILE"
mv -f -- "$TEMP_FILE" "$BACKUP_FILE"
echo "OK: Tandoor-Datenbank gesichert"
echo "Datei: $BACKUP_FILE"
exit 0
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "tsumugu-dev-postgres-1" "tsumugu-dev.sql"
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec /etc/backup-orchestrator/scripts/db-postgres-docker.sh "tsumugu-db-prod" "tsumugu-prod.sql"
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -Eeuo pipefail
exec runuser -u lars -- /bin/bash -s \
< /etc/backup-orchestrator/scripts/backup-nas-data.sh
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
# ============================================================================
# publish-backup-set.sh
# Atomically publishes a complete Backup Set (Dump + .sha256 + .manifest.json
# + optional .media.jsonl) to the NAS generation archive and rotates old sets.
# Implements RECOVERY_CONTRACT.md section 5.1 & 5.2.
# ============================================================================
set -Eeuo pipefail
umask 027
if [[ $# -lt 3 ]]; then
echo "Verwendung: $0 <TARGET_ID> <DUMP_DATEI> <ARCHIVE_ROOT>" >&2
exit 2
fi
TARGET_ID="$1"
DUMP_FILE="$2"
ARCHIVE_ROOT="$3"
SHA_FILE="${DUMP_FILE}.sha256"
MANIFEST_FILE="${DUMP_FILE}.manifest.json"
MEDIA_FILE="${DUMP_FILE}.media.jsonl"
[[ -s "$DUMP_FILE" ]] || { echo "FEHLER: Dump fehlt oder ist leer: $DUMP_FILE" >&2; exit 2; }
[[ -s "$SHA_FILE" ]] || { echo "FEHLER: SHA256-Datei fehlt: $SHA_FILE" >&2; exit 2; }
[[ -s "$MANIFEST_FILE" ]] || { echo "FEHLER: Manifest-Datei fehlt: $MANIFEST_FILE" >&2; exit 2; }
# Verify checksum before publishing
(
cd "$(dirname "$DUMP_FILE")"
sha256sum -c "$(basename "$SHA_FILE")" >/dev/null
) || {
echo "FEHLER: SHA256-Pruefsumme stimmt nicht!" >&2
exit 4
}
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
TARGET_DIR="$ARCHIVE_ROOT/$TARGET_ID"
mkdir -p "$TARGET_DIR"
STAGING_DIR="$(mktemp -d "$TARGET_DIR/.staging-${TIMESTAMP}.XXXXXXXX")"
FINAL_DIR="$TARGET_DIR/$TIMESTAMP"
cleanup() {
if [[ -n "${STAGING_DIR:-}" && -d "$STAGING_DIR" ]]; then
rm -rf -- "$STAGING_DIR" || true
fi
}
trap cleanup EXIT
cp -p -- "$DUMP_FILE" "$STAGING_DIR/"
cp -p -- "$SHA_FILE" "$STAGING_DIR/"
cp -p -- "$MANIFEST_FILE" "$STAGING_DIR/"
if [[ -f "$MEDIA_FILE" ]]; then
cp -p -- "$MEDIA_FILE" "$STAGING_DIR/"
fi
mv -T -- "$STAGING_DIR" "$FINAL_DIR"
STAGING_DIR=""
TEMP_LINK="$TARGET_DIR/.latest-$$"
ln -s -- "$FINAL_DIR" "$TEMP_LINK"
mv -Tf -- "$TEMP_LINK" "$TARGET_DIR/latest"
echo "OK: Backup-Satz veroeffentlicht unter $FINAL_DIR (latest aktualisiert)"
exit 0
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -Eeuo pipefail
DUMP_FILE="${SHINKAN_DUMP_FILE:-/media/lars/paperless/db-backup/shinkan-prod.sql}"
OUT_DIR="$(dirname "$DUMP_FILE")"
TARGET="${SHINKAN_TARGET:-pi-prod}"
ARCHIVE_ROOT="${SHINKAN_ARCHIVE_ROOT:-/backup/recovery_points}"
RESULT_JSON="${OUT_DIR}/.shinkan-manifest-last-result.json"
/usr/bin/env python3 /etc/backup-orchestrator/scripts/shinkan/shinkan-backup-manifest \
--target "$TARGET" \
--dump "$DUMP_FILE" \
--out-dir "$OUT_DIR" \
--result "$RESULT_JSON"
if [[ -d "$(dirname "$ARCHIVE_ROOT")" && -w "$(dirname "$ARCHIVE_ROOT")" ]]; then
/etc/backup-orchestrator/scripts/publish-backup-set.sh \
"shinkan-${TARGET}" \
"$DUMP_FILE" \
"$ARCHIVE_ROOT"
fi
@@ -0,0 +1,73 @@
# Isolated disposable Restore-Test Compose configuration for llm-node.
# Implements RECOVERY_CONTRACT.md section 3.3:
# - No fixed container_name (prevents collision with shinkan-prod)
# - Localhost-only ports (18003 / 13003), no published postgres port
# - Shared media mounted strictly read-only (:ro)
# - No retention-cron service, SMTP and OpenRouter disabled
services:
postgres:
image: postgres:16-alpine
environment:
POSTGRES_DB: "${DB_NAME:-shinkan}"
POSTGRES_USER: "${DB_USER:-shinkan_user}"
POSTGRES_PASSWORD: "${DB_PASSWORD}"
volumes:
- restoretest-db-data:/var/lib/postgresql/data
networks:
- restoretest-net
backend:
build:
context: ./backend
dockerfile: Dockerfile
environment:
DB_HOST: postgres
DB_PORT: 5432
DB_NAME: shinkan
DB_USER: shinkan_user
DB_PASSWORD: "${DB_PASSWORD}"
OPENROUTER_API_KEY: ""
OPENROUTER_MODEL: ""
SMTP_HOST: ""
SMTP_PORT: ""
SMTP_USER: ""
SMTP_PASS: ""
SMTP_FROM: ""
AUTO_ADMIN_FIRST_USER: "false"
ADMIN_BOOTSTRAP_EMAILS: ""
ENVIRONMENT: "production"
MEDIA_ROOT: "/app/media"
volumes:
- type: bind
source: /srv/shinkan/prod/media
target: /app/media
read_only: true
bind:
create_host_path: false
ports:
- "127.0.0.1:18003:8000"
depends_on:
- postgres
networks:
- restoretest-net
frontend:
build:
context: ./frontend
dockerfile: Dockerfile
args:
VITE_API_URL: ""
ports:
- "127.0.0.1:13003:80"
depends_on:
- backend
networks:
- restoretest-net
volumes:
restoretest-db-data:
networks:
restoretest-net:
driver: bridge
+354
View File
@@ -0,0 +1,354 @@
#!/usr/bin/env python3
"""Generate Shinkan backup manifest, SHA256 sidecar, and incremental media.jsonl index.
Implements RECOVERY_CONTRACT.md section 3.1 and Backup-Orchestrator v0.4 Manifest Schema v1.
Read-only against database and media storage; writes sidecar files next to the dump.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import platform
import re
import socket
import subprocess
import sys
from urllib.request import urlopen
EXIT_OK = 0
EXIT_EXEC_ERROR = 1
EXIT_PRECONDITION = 2
TARGETS = {
'pi-prod': {
'host': 'raspberrypi5',
'arch': 'aarch64',
'app_dir': '/home/lars/docker/shinkan',
'db_container': 'shinkan-db-prod',
'db_name': 'shinkan',
'db_user': 'shinkan_user',
'media_dir': '/shinkan-media/prod',
'version_url': 'http://127.0.0.1:8003/api/version',
},
'llm-prod': {
'host': 'llm-node',
'arch': 'x86_64',
'app_dir': '/home/llmadmin/docker/shinkan-prod',
'db_container': 'shinkan-db-prod',
'db_name': 'shinkan',
'db_user': 'shinkan_user',
'media_dir': '/srv/shinkan/prod/media',
'version_url': 'http://127.0.0.1:8003/api/version',
},
}
def now_iso():
return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds')
def compute_sha256(path):
h = hashlib.sha256()
with Path(path).open('rb') as fp:
while True:
chunk = fp.read(65536)
if not chunk:
break
h.update(chunk)
return h.hexdigest()
def detect_dump_format(path):
with Path(path).open('rb') as fp:
return 'custom' if fp.read(5) == b'PGDMP' else 'plain'
def atomic_write_text(path, content):
path = Path(path)
tmp = path.with_name(f'.{path.name}.{os.getpid()}.tmp')
try:
tmp.write_text(content, encoding='utf-8')
tmp.replace(path)
finally:
if tmp.exists():
tmp.unlink()
def query_version_info(tcfg):
app_version = 'unknown'
try:
with urlopen(tcfg['version_url'], timeout=5) as resp:
if resp.status == 200:
data = json.loads(resp.read(8192).decode('utf-8'))
app_version = data.get('app_version', 'unknown')
except Exception:
pass
git_commit = 'unknown'
app_dir = Path(tcfg['app_dir'])
if (app_dir / '.git').exists():
try:
proc = subprocess.run(
['git', '-C', str(app_dir), 'rev-parse', 'HEAD'],
capture_output=True, text=True, timeout=5,
)
if proc.returncode == 0 and proc.stdout.strip():
git_commit = proc.stdout.strip()
except Exception:
pass
return app_version, git_commit
def query_db_metadata(tcfg, dump_path):
"""Query migrations and informative table row counts from container or fallback to SQL dump."""
migrations = []
row_counts = {}
engine_version = ''
sql_script = """
SELECT 'MIG:' || migration FROM schema_migrations ORDER BY id;
SELECT 'VER:' || server_version FROM pg_settings WHERE name = 'server_version';
SELECT 'CNT:' || relname || ':' || n_live_tup::bigint
FROM pg_stat_user_tables
WHERE schemaname = 'public'
ORDER BY relname;
"""
try:
proc = subprocess.run(
[
'docker', 'exec', tcfg['db_container'],
'sh', '-ec',
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
'psql -U "${POSTGRES_USER:-shinkan_user}" -d "${POSTGRES_DB:-shinkan}" -At -c '
+ json.dumps(sql_script),
],
capture_output=True, text=True, timeout=15,
)
if proc.returncode == 0:
for line in proc.stdout.splitlines():
if line.startswith('MIG:'):
migrations.append(line[4:].strip())
elif line.startswith('VER:'):
engine_version = line[4:].strip()
elif line.startswith('CNT:'):
parts = line[4:].split(':', 1)
if len(parts) == 2 and parts[1].isdigit():
row_counts[parts[0]] = int(parts[1])
except Exception:
pass
# Fallback: parse migrations from plain SQL dump if container query was unavailable
if not migrations and detect_dump_format(dump_path) == 'plain':
in_copy = False
with Path(dump_path).open('r', encoding='utf-8', errors='replace') as fp:
for line in fp:
if not in_copy:
if line.startswith('COPY public.schema_migrations ') or line.startswith('COPY schema_migrations '):
in_copy = True
continue
else:
if line.startswith('\\.'):
break
parts = line.rstrip('\r\n').split('\t')
if len(parts) >= 2:
migrations.append(parts[1])
def sort_key(stem):
m = re.match(r'^(\d+)', stem)
return (int(m.group(1)) if m else 0, stem)
migrations = sorted(set(migrations), key=sort_key)
return migrations, row_counts, engine_version
def build_media_jsonl(media_dir, out_jsonl_path, cache_path):
"""Write <dump>.media.jsonl with incremental sha256 cache over (path, size, mtime_ns)."""
media_root = Path(media_dir)
cache = {}
if cache_path.is_file():
try:
cache = json.loads(cache_path.read_text(encoding='utf-8'))
except Exception:
cache = {}
new_cache = {}
lines = []
total_bytes = 0
if media_root.is_dir():
for item in sorted(media_root.rglob('*')):
if not item.is_file() or item.name.startswith('.'):
continue
rel = item.relative_to(media_root).as_posix()
if rel.startswith('#recycle/') or rel.startswith('.Trash'):
continue
st = item.stat()
size = st.st_size
mtime = int(st.st_mtime)
cache_key = f'{rel}|{size}|{mtime}'
sha = cache.get(cache_key)
if not sha:
sha = compute_sha256(item)
new_cache[cache_key] = sha
total_bytes += size
lines.append(json.dumps({
'path': rel, 'size': size, 'mtime': mtime, 'sha256': sha,
}, ensure_ascii=False))
atomic_write_text(out_jsonl_path, ('\n'.join(lines) + '\n') if lines else '')
try:
atomic_write_text(cache_path, json.dumps(new_cache, ensure_ascii=False))
except OSError:
pass
return len(lines), total_bytes, compute_sha256(out_jsonl_path)
def write_result(path, payload):
if path:
atomic_write_text(path, json.dumps(payload, ensure_ascii=False, indent=2) + '\n')
def main():
parser = argparse.ArgumentParser(description='Create Shinkan backup manifest and checksums')
parser.add_argument('--target', required=True, choices=sorted(TARGETS))
parser.add_argument('--dump', required=True, type=Path)
parser.add_argument('--out-dir', type=Path)
parser.add_argument('--result', type=Path)
args = parser.parse_args()
tcfg = dict(TARGETS[args.target])
if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'):
tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE']
if os.environ.get('SHINKAN_APP_DIR_OVERRIDE'):
tcfg['app_dir'] = os.environ['SHINKAN_APP_DIR_OVERRIDE']
cur_host = socket.gethostname()
cur_arch = platform.machine()
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
if cur_host != tcfg['host'] or cur_arch != tcfg['arch']:
msg = f"Host/arch mismatch: running on {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}"
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
dump_path = args.dump.resolve()
if not dump_path.is_file() or dump_path.stat().st_size == 0:
msg = f'Dump file missing or empty: {dump_path}'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
out_dir = (args.out_dir or dump_path.parent).resolve()
out_dir.mkdir(parents=True, exist_ok=True)
try:
dump_sha256 = compute_sha256(dump_path)
dump_bytes = dump_path.stat().st_size
dump_format = detect_dump_format(dump_path)
sha_file = out_dir / (dump_path.name + '.sha256')
atomic_write_text(sha_file, f'{dump_sha256} {dump_path.name}\n')
media_file = out_dir / (dump_path.name + '.media.jsonl')
cache_file = out_dir / '.shinkan-media-sha256-cache.json'
media_count, media_bytes, media_sha256 = build_media_jsonl(tcfg['media_dir'], media_file, cache_file)
app_version, git_commit = query_version_info(tcfg)
migrations, row_counts, engine_version = query_db_metadata(tcfg, dump_path)
last_migration = migrations[-1] if migrations else None
created_at = now_iso()
point_id = f"shinkan-{args.target}-{dt.datetime.now().strftime('%Y%m%dT%H%M%S')}-{dump_sha256[:8]}"
manifest = {
'schemaVersion': 1,
'created_at': created_at,
'host': cur_host,
'target': args.target,
'app_version': app_version,
'git_commit': git_commit,
'migrations': migrations,
'last_migration': last_migration,
'dump_format': dump_format,
'dump_bytes': dump_bytes,
'dump_sha256': dump_sha256,
'row_counts': row_counts,
'media_manifest': {
'file': media_file.name,
'sha256': media_sha256,
'file_count': media_count,
'total_bytes': media_bytes,
},
'application': {
'id': 'shinkan',
'environment': 'production',
'target': args.target,
'sourceHost': cur_host,
'sourceArch': cur_arch,
},
'recoveryPoint': {
'id': point_id,
'createdAt': created_at,
'status': 'consistent' if (migrations and app_version != 'unknown') else 'verified',
'validatedAt': None,
'consistency': {
'mode': 'hot_indexed',
'dbTransactional': True,
'mediaSynchronized': False,
'limitations': [
'PostgreSQL dump created in a single transaction; media index scanned immediately afterwards without write freeze.'
],
},
},
'release': {
'repository': 'http://192.168.2.144:3000/Lars/shinkan-jinkendo.git',
'appVersion': app_version,
'gitCommit': git_commit,
},
'database': {
'engine': 'postgresql',
'engineVersion': engine_version,
'dumpFormat': dump_format,
'artifact': dump_path.name,
'bytes': dump_bytes,
'sha256': dump_sha256,
'schemaVersion': last_migration or '',
'lastMigration': last_migration,
'migrations': migrations,
'rowCounts': row_counts,
},
'media': {
'storageType': 'nfs4_shared',
'containerMount': '/app/media',
'manifestArtifact': media_file.name,
'manifestSha256': media_sha256,
'fileCount': media_count,
'totalBytes': media_bytes,
},
}
manifest_file = out_dir / (dump_path.name + '.manifest.json')
atomic_write_text(manifest_file, json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
res = {
'status': 'ok',
'exit_code': EXIT_OK,
'recovery_point_id': point_id,
'dump_path': str(dump_path),
'sha256_path': str(sha_file),
'manifest_path': str(manifest_file),
'media_manifest_path': str(media_file),
'dump_sha256': dump_sha256,
'last_migration': last_migration,
'media_file_count': media_count,
}
write_result(args.result, res)
print(f'OK: Manifest created for {dump_path.name} (point={point_id}, migrations={len(migrations)}, media_files={media_count})')
return EXIT_OK
except Exception as exc:
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)})
print(f'ERROR: {exc}', file=sys.stderr)
return EXIT_EXEC_ERROR
if __name__ == '__main__':
sys.exit(main())
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python3
"""Compare Shinkan database media references against the filesystem and backup media manifest.
Implements RECOVERY_CONTRACT.md section 3.4. Read-only: never deletes or modifies files.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import platform
import socket
import subprocess
import sys
from urllib.request import urlopen
EXIT_OK = 0
EXIT_EXEC_ERROR = 1
EXIT_PRECONDITION = 2
TARGETS = {
'llm-prod': {
'host': 'llm-node',
'arch': 'x86_64',
'app_dir': '/home/llmadmin/docker/shinkan-prod',
'db_container': 'shinkan-db-prod',
'db_name': 'shinkan',
'db_user': 'shinkan_user',
'media_dir': '/srv/shinkan/prod/media',
'version_url': 'http://127.0.0.1:8003/api/version',
},
'llm-restoretest': {
'host': 'llm-node',
'arch': 'x86_64',
'app_dir': '/home/llmadmin/docker/shinkan-prod',
'db_container': 'shinkan-restoretest-postgres-1',
'db_name': 'shinkan',
'db_user': 'shinkan_user',
'media_dir': '/srv/shinkan/prod/media',
'version_url': 'http://127.0.0.1:18003/api/version',
},
}
def compute_sha256(path):
h = hashlib.sha256()
with Path(path).open('rb') as fp:
while True:
chunk = fp.read(65536)
if not chunk:
break
h.update(chunk)
return h.hexdigest()
def load_backup_media_manifest(path):
entries = {}
if not path or not Path(path).is_file():
return entries
with Path(path).open('r', encoding='utf-8') as fp:
for line in fp:
line = line.strip()
if not line:
continue
obj = json.loads(line)
if 'path' in obj:
entries[obj['path']] = obj
return entries
def query_db_media_refs(tcfg):
sql = """
SELECT 'LAST_MIG:' || COALESCE((SELECT migration FROM schema_migrations ORDER BY id DESC LIMIT 1), '');
SELECT 'ASSET:' || COALESCE(storage_key, '') || '|' || COALESCE(byte_size::text, '') || '|' || COALESCE(sha256, '')
FROM media_assets
WHERE storage_key IS NOT NULL AND storage_key <> '';
SELECT 'EXMEDIA:' || COALESCE(file_path, '')
FROM exercise_media
WHERE file_path IS NOT NULL AND file_path <> '';
"""
proc = subprocess.run(
[
'docker', 'exec', tcfg['db_container'],
'sh', '-ec',
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
f'psql -U "{tcfg["db_user"]}" -d "{tcfg["db_name"]}" -At -c ' + json.dumps(sql),
],
capture_output=True, text=True, timeout=30,
)
if proc.returncode != 0:
raise RuntimeError(f'Database media query failed: {proc.stderr.strip()}')
last_migration = None
referenced = {}
for line in proc.stdout.splitlines():
if line.startswith('LAST_MIG:'):
last_migration = line[len('LAST_MIG:'):].strip() or None
elif line.startswith('ASSET:'):
parts = line[len('ASSET:'):].split('|')
rel = parts[0].lstrip('/')
if rel.startswith('media/'):
rel = rel[len('media/'):]
size = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
sha = parts[2].lower() if len(parts) > 2 and parts[2] else None
referenced[rel] = {'path': rel, 'expected_size': size, 'expected_sha256': sha, 'source': 'media_assets'}
elif line.startswith('EXMEDIA:'):
raw_path = line[len('EXMEDIA:'):].strip().lstrip('/')
if raw_path.startswith('media/'):
raw_path = raw_path[len('media/'):]
if raw_path.startswith('api/'):
continue
if raw_path and raw_path not in referenced:
referenced[raw_path] = {'path': raw_path, 'expected_size': None, 'expected_sha256': None, 'source': 'exercise_media'}
return last_migration, referenced
def main():
parser = argparse.ArgumentParser(description='Compare Shinkan DB media references against filesystem and manifest')
parser.add_argument('--target', required=True, choices=sorted(TARGETS))
parser.add_argument('--result', required=True, type=Path)
parser.add_argument('--manifest', type=Path, help='Optional <dump>.media.jsonl file')
args = parser.parse_args()
tcfg = dict(TARGETS[args.target])
if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'):
tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE']
cur_host = socket.gethostname()
cur_arch = platform.machine()
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
if cur_host != tcfg['host'] or cur_arch != tcfg['arch']:
msg = f"Host/arch mismatch: {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}"
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
media_root = Path(tcfg['media_dir'])
if not media_root.is_dir():
msg = f'Media directory not available: {media_root}'
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
try:
last_migration, db_refs = query_db_media_refs(tcfg)
backup_manifest = load_backup_media_manifest(args.manifest)
disk_files = {}
for item in sorted(media_root.rglob('*')):
if not item.is_file() or item.name.startswith('.'):
continue
rel = item.relative_to(media_root).as_posix()
if rel.startswith('#recycle/') or rel.startswith('.Trash'):
continue
disk_files[rel] = item
missing = []
mismatch = []
added_after_backup = []
orphan = []
for rel, ref in sorted(db_refs.items()):
disk_item = disk_files.get(rel)
if disk_item is None:
missing.append({'path': rel, 'source': ref['source']})
continue
actual_size = disk_item.stat().st_size
if ref['expected_size'] is not None and ref['expected_size'] != actual_size:
mismatch.append({
'path': rel, 'reason': 'byte_size',
'expected': ref['expected_size'], 'actual': actual_size,
})
continue
if ref['expected_sha256']:
actual_sha = compute_sha256(disk_item)
if actual_sha != ref['expected_sha256']:
mismatch.append({
'path': rel, 'reason': 'sha256',
'expected': ref['expected_sha256'], 'actual': actual_sha,
})
for rel in sorted(disk_files):
if rel not in db_refs:
if args.manifest and rel not in backup_manifest:
added_after_backup.append({'path': rel, 'size': disk_files[rel].stat().st_size})
else:
orphan.append({'path': rel, 'size': disk_files[rel].stat().st_size})
app_version = 'unknown'
try:
with urlopen(tcfg['version_url'], timeout=5) as resp:
if resp.status == 200:
app_version = json.loads(resp.read(4096).decode('utf-8')).get('app_version', 'unknown')
except Exception:
pass
git_commit = 'unknown'
if (Path(tcfg['app_dir']) / '.git').exists():
proc = subprocess.run(['git', '-C', tcfg['app_dir'], 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=5)
if proc.returncode == 0:
git_commit = proc.stdout.strip()
report = {
'status': 'ok',
'exit_code': EXIT_OK,
'checked_at': dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds'),
'target': args.target,
'version_triple': {
'app_version': app_version,
'git_commit': git_commit,
'db_last_migration': last_migration,
'backup_media_manifest': str(args.manifest) if args.manifest else None,
},
'counts': {
'db_referenced': len(db_refs),
'disk_files': len(disk_files),
'missing': len(missing),
'mismatch': len(mismatch),
'added_after_backup': len(added_after_backup),
'orphan': len(orphan),
},
'missing': missing,
'mismatch': mismatch,
'added_after_backup': added_after_backup,
'orphan': orphan,
}
args.result.parent.mkdir(parents=True, exist_ok=True)
args.result.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(
f"OK: Media diff for {args.target}: referenced={len(db_refs)} disk={len(disk_files)} "
f"missing={len(missing)} mismatch={len(mismatch)} added_after_backup={len(added_after_backup)} orphan={len(orphan)}"
)
return EXIT_OK
except Exception as exc:
args.result.parent.mkdir(parents=True, exist_ok=True)
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)}, indent=2) + '\n', encoding='utf-8')
print(f'ERROR: {exc}', file=sys.stderr)
return EXIT_EXEC_ERROR
if __name__ == '__main__':
sys.exit(main())
+389
View File
@@ -0,0 +1,389 @@
#!/usr/bin/env python3
"""Safe transactional restore of Shinkan database with staging verification and automatic rollback.
Implements RECOVERY_CONTRACT.md section 3.2:
- Exit 0: Restore and post-restore checks succeeded
- Exit 1: Execution error
- Exit 2: Precondition / confirmation / host guard error
- Exit 3: Schema incompatibility (dump contains migrations unknown to deployed code)
- Exit 4: Checksum mismatch
- Exit 5: Post-restore healthcheck failed (automatic rollback to shinkan_before_<ts> executed)
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import platform
import re
import socket
import subprocess
import sys
import time
from urllib.request import urlopen
EXIT_OK = 0
EXIT_EXEC_ERROR = 1
EXIT_PRECONDITION = 2
EXIT_SCHEMA_INCOMPATIBLE = 3
EXIT_CHECKSUM_MISMATCH = 4
EXIT_POST_RESTORE_FAILED = 5
REQUIRED_TABLES = (
'schema_migrations',
'skills',
'skill_main_categories',
'skill_categories',
'maturity_models',
'sessions',
'club_members',
'club_member_roles',
'profiles',
)
TARGETS = {
'llm-prod': {
'host': 'llm-node',
'arch': 'x86_64',
'require_confirm': 'shinkan@llm-node',
'app_dir': '/home/llmadmin/docker/shinkan-prod',
'compose_project': 'shinkan-prod',
'compose_files': ['docker-compose.yml', 'docker-compose.llm-node.yml'],
'env_file': '/etc/shinkan/prod.env',
'db_container': 'shinkan-db-prod',
'db_name': 'shinkan',
'db_user': 'shinkan_user',
'version_url': 'http://127.0.0.1:8003/api/version',
'ready_url': 'http://127.0.0.1:8003/api/health/ready',
'frontend_ready_url': 'http://127.0.0.1:3003/api/health/ready',
},
'llm-restoretest': {
'host': 'llm-node',
'arch': 'x86_64',
'require_confirm': None,
'app_dir': '/home/llmadmin/docker/shinkan-prod',
'compose_project': 'shinkan-restoretest',
'compose_files': ['/etc/backup-orchestrator/scripts/shinkan/docker-compose.restoretest.yml'],
'env_file': '/etc/shinkan/prod.env',
'db_container': 'shinkan-restoretest-postgres-1',
'db_name': 'shinkan',
'db_user': 'shinkan_user',
'version_url': 'http://127.0.0.1:18003/api/version',
'ready_url': 'http://127.0.0.1:18003/api/health/ready',
'frontend_ready_url': 'http://127.0.0.1:13003/api/health/ready',
},
}
def compute_sha256(path):
h = hashlib.sha256()
with Path(path).open('rb') as fp:
while True:
chunk = fp.read(65536)
if not chunk:
break
h.update(chunk)
return h.hexdigest()
def detect_dump_format(path):
with Path(path).open('rb') as fp:
return 'custom' if fp.read(5) == b'PGDMP' else 'plain'
def write_result(path, data):
if path:
Path(path).parent.mkdir(parents=True, exist_ok=True)
Path(path).write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
def psql_exec(tcfg, dbname, sql):
cmd = [
'docker', 'exec', tcfg['db_container'],
'sh', '-ec',
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
f'psql -v ON_ERROR_STOP=1 -U "{tcfg["db_user"]}" -d "{dbname}" -At -c ' + json.dumps(sql),
]
return subprocess.run(cmd, capture_output=True, text=True, timeout=60)
def drop_db_if_exists(tcfg, dbname):
term_sql = f"""
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = '{dbname}' AND pid <> pg_backend_pid();
"""
psql_exec(tcfg, 'postgres', term_sql)
psql_exec(tcfg, 'postgres', f'DROP DATABASE IF EXISTS "{dbname}";')
def compose_cmd(tcfg, *args):
cmd = ['docker', 'compose', '-p', tcfg['compose_project']]
for cf in tcfg['compose_files']:
cf_path = Path(cf) if Path(cf).is_absolute() else (Path(tcfg['app_dir']) / cf)
cmd.extend(['-f', str(cf_path)])
if tcfg.get('env_file') and Path(tcfg['env_file']).is_file():
cmd.extend(['--env-file', tcfg['env_file']])
cmd.extend(args)
return subprocess.run(cmd, cwd=tcfg['app_dir'], capture_output=True, text=True, timeout=300)
def wait_ready(url, timeout_sec=180):
deadline = time.monotonic() + timeout_sec
last_body = None
while time.monotonic() < deadline:
try:
with urlopen(url, timeout=5) as resp:
raw = resp.read(8192).decode('utf-8')
last_body = json.loads(raw)
if (
resp.status == 200
and last_body.get('status') == 'ready'
and last_body.get('database') is True
and last_body.get('schema_complete') is True
):
return True, last_body
except Exception as exc:
last_body = {'error': str(exc)}
time.sleep(3)
return False, last_body
def discover_code_migrations(app_dir):
mig_dir = Path(app_dir) / 'backend' / 'migrations'
if not mig_dir.is_dir():
return []
stems = []
for f in mig_dir.iterdir():
if f.is_file() and f.name.endswith('.sql') and f.name[0].isdigit():
stems.append(f.name[:-4])
return sorted(stems, key=lambda s: (int(re.match(r'^(\d+)', s).group(1)), s))
def main():
parser = argparse.ArgumentParser(description='Restore Shinkan database via staging DB with rollback safety')
parser.add_argument('--target', required=True, choices=sorted(TARGETS))
parser.add_argument('--dump', type=Path)
parser.add_argument('--result', required=True, type=Path)
parser.add_argument('--confirm', type=str)
parser.add_argument('--allow-overwrite', action='store_true')
parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear')
parser.add_argument('--drop-previous', type=str, help='Explicitly drop a retained shinkan_before_<ts> database')
args = parser.parse_args()
start_ts = time.monotonic()
tcfg = dict(TARGETS[args.target])
if os.environ.get('SHINKAN_APP_DIR_OVERRIDE'):
tcfg['app_dir'] = os.environ['SHINKAN_APP_DIR_OVERRIDE']
cur_host = socket.gethostname()
cur_arch = platform.machine()
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
if cur_host != tcfg['host'] or cur_arch != tcfg['arch']:
msg = f"Host/arch mismatch: running on {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}"
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
if args.drop_previous:
if not re.fullmatch(r'shinkan_before_\d{8}_\d{6}', args.drop_previous):
msg = f'Refused to drop database with non-backup name: {args.drop_previous}'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
return EXIT_PRECONDITION
drop_db_if_exists(tcfg, args.drop_previous)
write_result(args.result, {'status': 'ok', 'exit_code': EXIT_OK, 'dropped_database': args.drop_previous})
print(f'OK: Dropped previous database {args.drop_previous}')
return EXIT_OK
if not args.dump:
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': '--dump is required'})
return EXIT_PRECONDITION
if tcfg['require_confirm'] and args.confirm != tcfg['require_confirm']:
msg = f"Target {args.target} requires --confirm {tcfg['require_confirm']}"
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
dump_path = args.dump.resolve()
sha_path = Path(str(dump_path) + '.sha256')
if not dump_path.is_file() or not sha_path.is_file():
msg = f'Dump or .sha256 file missing: {dump_path}'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
expected_sha = sha_path.read_text(encoding='utf-8').strip().split()[0].lower()
actual_sha = compute_sha256(dump_path)
if expected_sha != actual_sha:
msg = f'SHA256 mismatch: expected {expected_sha}, got {actual_sha}'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_CHECKSUM_MISMATCH, 'detail': msg})
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_CHECKSUM_MISMATCH
# Check if live DB already has user profiles (non-empty)
prof_res = psql_exec(tcfg, tcfg['db_name'], 'SELECT COUNT(*)::int FROM profiles;')
live_profiles = int(prof_res.stdout.strip()) if (prof_res.returncode == 0 and prof_res.stdout.strip().isdigit()) else 0
if live_profiles > 0 and not args.allow_overwrite:
msg = f"Target database {tcfg['db_name']} contains {live_profiles} profiles; pass --allow-overwrite to replace it."
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
ts_label = dt.datetime.now().strftime('%Y%m%d_%H%M%S')
staging_db = f'shinkan_restore_{ts_label}'
previous_db = f'shinkan_before_{ts_label}'
print(f'1. Creating staging database {staging_db}...')
drop_db_if_exists(tcfg, staging_db)
create_res = psql_exec(tcfg, 'postgres', f'CREATE DATABASE "{staging_db}";')
if create_res.returncode != 0:
msg = f'Failed to create staging DB: {create_res.stderr.strip()}'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
return EXIT_EXEC_ERROR
dump_format = detect_dump_format(dump_path)
print(f'2. Importing dump ({dump_format}) into {staging_db} in a single transaction...')
if dump_format == 'custom':
import_cmd = [
'docker', 'exec', '-i', tcfg['db_container'],
'sh', '-ec',
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
f'exec pg_restore -U "{tcfg["db_user"]}" -d "{staging_db}" '
'--single-transaction --exit-on-error --no-owner --no-acl',
]
else:
import_cmd = [
'docker', 'exec', '-i', tcfg['db_container'],
'sh', '-ec',
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
f'exec psql -v ON_ERROR_STOP=1 --single-transaction -U "{tcfg["db_user"]}" -d "{staging_db}"',
]
with dump_path.open('rb') as dump_stream:
imp_proc = subprocess.run(import_cmd, stdin=dump_stream, capture_output=True, timeout=600)
if imp_proc.returncode != 0:
drop_db_if_exists(tcfg, staging_db)
msg = f'Dump import into staging DB failed (rc={imp_proc.returncode})'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_EXEC_ERROR
print('3. Verifying schema_migrations compatibility against deployed code...')
mig_res = psql_exec(tcfg, staging_db, 'SELECT migration FROM schema_migrations ORDER BY id;')
if mig_res.returncode != 0:
drop_db_if_exists(tcfg, staging_db)
msg = 'Could not read schema_migrations from staging DB'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
return EXIT_EXEC_ERROR
dump_migrations = [m.strip() for m in mig_res.stdout.splitlines() if m.strip()]
code_migrations = discover_code_migrations(tcfg['app_dir'])
unknown_migrations = sorted(set(dump_migrations) - set(code_migrations))
if unknown_migrations:
drop_db_if_exists(tcfg, staging_db)
msg = f'Dump contains migration(s) unknown to deployed code: {", ".join(unknown_migrations)}'
write_result(args.result, {
'status': 'error',
'exit_code': EXIT_SCHEMA_INCOMPATIBLE,
'detail': msg,
'unknown_migrations': unknown_migrations,
'dump_last_migration': dump_migrations[-1] if dump_migrations else None,
'code_last_migration': code_migrations[-1] if code_migrations else None,
})
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_SCHEMA_INCOMPATIBLE
print('4. Running plausibility checks on staging database...')
for tbl in REQUIRED_TABLES:
chk = psql_exec(
tcfg, staging_db,
f"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='{tbl}');",
)
if chk.returncode != 0 or chk.stdout.strip() != 't':
drop_db_if_exists(tcfg, staging_db)
msg = f'Plausibility check failed: missing required table {tbl}'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
return EXIT_EXEC_ERROR
for non_empty_tbl in ('profiles', 'skills'):
cnt = psql_exec(tcfg, staging_db, f'SELECT COUNT(*)::int FROM {non_empty_tbl};')
if cnt.returncode != 0 or not cnt.stdout.strip().isdigit() or int(cnt.stdout.strip()) < 1:
drop_db_if_exists(tcfg, staging_db)
msg = f'Plausibility check failed: table {non_empty_tbl} is empty in staging DB'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
return EXIT_EXEC_ERROR
if args.sessions == 'clear':
print('5. Clearing active user sessions in staging DB (--sessions clear)...')
psql_exec(tcfg, staging_db, 'TRUNCATE TABLE sessions;')
print('6. Stopping application containers and swapping databases...')
compose_cmd(tcfg, 'stop', 'backend', 'frontend')
compose_cmd(tcfg, '--profile', 'media-retention', 'stop', 'retention-cron')
swap_sql = f"""
SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('{tcfg["db_name"]}', '{staging_db}') AND pid <> pg_backend_pid();
ALTER DATABASE "{tcfg['db_name']}" RENAME TO "{previous_db}";
ALTER DATABASE "{staging_db}" RENAME TO "{tcfg['db_name']}";
"""
swap_res = psql_exec(tcfg, 'postgres', swap_sql)
if swap_res.returncode != 0:
drop_db_if_exists(tcfg, staging_db)
compose_cmd(tcfg, 'up', '-d', 'backend', 'frontend')
msg = f'Database rename failed: {swap_res.stderr.strip()}'
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
return EXIT_EXEC_ERROR
print('7. Starting backend and frontend and verifying readiness...')
compose_cmd(tcfg, 'up', '-d', 'backend', 'frontend')
ok_api, ready_body = wait_ready(tcfg['ready_url'], timeout_sec=180)
ok_fe, fe_body = (wait_ready(tcfg['frontend_ready_url'], timeout_sec=60) if ok_api and tcfg.get('frontend_ready_url') else (ok_api, None))
if not (ok_api and ok_fe):
print('ERROR: Post-restore readiness check failed! Executing automatic rollback...', file=sys.stderr)
compose_cmd(tcfg, 'stop', 'backend', 'frontend')
failed_db = f'shinkan_failed_{ts_label}'
rollback_sql = f"""
SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('{tcfg["db_name"]}', '{previous_db}') AND pid <> pg_backend_pid();
ALTER DATABASE "{tcfg['db_name']}" RENAME TO "{failed_db}";
ALTER DATABASE "{previous_db}" RENAME TO "{tcfg['db_name']}";
"""
psql_exec(tcfg, 'postgres', rollback_sql)
drop_db_if_exists(tcfg, failed_db)
compose_cmd(tcfg, 'up', '-d', 'backend', 'frontend')
rb_ok, rb_body = wait_ready(tcfg['ready_url'], timeout_sec=120)
write_result(args.result, {
'status': 'rolled_back',
'exit_code': EXIT_POST_RESTORE_FAILED,
'dump_sha256': actual_sha,
'failed_health': {'api': ready_body, 'frontend': fe_body},
'rollback_health': {'ok': rb_ok, 'body': rb_body},
'duration_seconds': round(time.monotonic() - start_ts, 2),
})
return EXIT_POST_RESTORE_FAILED
pending_applied = sorted(set(code_migrations) - set(dump_migrations))
result_payload = {
'status': 'ok',
'exit_code': EXIT_OK,
'target': args.target,
'dump_sha256': actual_sha,
'dump_last_migration': dump_migrations[-1] if dump_migrations else None,
'code_last_migration': code_migrations[-1] if code_migrations else None,
'migrations_applied_on_start': pending_applied,
'sessions': args.sessions,
'previous_db': previous_db,
'health': {'api': ready_body, 'frontend': fe_body},
'duration_seconds': round(time.monotonic() - start_ts, 2),
}
write_result(args.result, result_payload)
print(f"OK: Restore to {args.target} succeeded in {result_payload['duration_seconds']}s (previous DB retained as {previous_db})")
return EXIT_OK
if __name__ == '__main__':
sys.exit(main())
+136
View File
@@ -0,0 +1,136 @@
#!/usr/bin/env python3
"""Run an isolated disposable restore test for Shinkan on llm-node.
Implements RECOVERY_CONTRACT.md section 3.3:
- Uses dedicated project shinkan-restoretest without fixed container_name collisions
- Mounts shared media directory read-only (:ro)
- Disables retention-cron, SMTP, OpenRouter, and auto-admin bootstrap
- Runs shinkan-restore --target llm-restoretest and shinkan-media-diff --target llm-restoretest
- Tears down containers and volumes unless --keep is specified
"""
import argparse
import json
import os
from pathlib import Path
import platform
import socket
import subprocess
import sys
import time
EXIT_OK = 0
EXIT_EXEC_ERROR = 1
EXIT_PRECONDITION = 2
def main():
parser = argparse.ArgumentParser(description='Run isolated Shinkan restore test on llm-node')
parser.add_argument('--dump', required=True, type=Path)
parser.add_argument('--result', required=True, type=Path)
parser.add_argument('--playwright', action='store_true')
parser.add_argument('--keep', action='store_true')
args = parser.parse_args()
start_ts = time.monotonic()
cur_host = socket.gethostname()
cur_arch = platform.machine()
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
if cur_host != 'llm-node' or cur_arch != 'x86_64':
msg = f'Host/arch mismatch: running on {cur_host}/{cur_arch}, expected llm-node/x86_64'
args.result.parent.mkdir(parents=True, exist_ok=True)
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
script_dir = Path(__file__).resolve().parent
compose_file = script_dir / 'docker-compose.restoretest.yml'
app_dir = Path(os.environ.get('SHINKAN_APP_DIR_OVERRIDE', '/home/llmadmin/docker/shinkan-prod'))
env_file = Path(os.environ.get('SHINKAN_ENV_FILE_OVERRIDE', '/etc/shinkan/prod.env'))
def dc(*dc_args):
cmd = ['docker', 'compose', '-p', 'shinkan-restoretest', '-f', str(compose_file)]
if env_file.is_file():
cmd.extend(['--env-file', str(env_file)])
cmd.extend(dc_args)
return subprocess.run(cmd, cwd=str(app_dir), capture_output=True, text=True, timeout=600)
restore_res_file = args.result.with_name(args.result.stem + '.restore.json')
media_res_file = args.result.with_name(args.result.stem + '.media.json')
rc = EXIT_EXEC_ERROR
try:
print('1. Starting isolated shinkan-restoretest environment...')
dc('down', '-v', '--remove-orphans')
up_proc = dc('up', '-d', '--build', 'postgres', 'backend', 'frontend')
if up_proc.returncode != 0:
raise RuntimeError(f'Failed to start shinkan-restoretest: {up_proc.stderr.strip()}')
print('2. Executing shinkan-restore --target llm-restoretest...')
restore_cmd = [
sys.executable, str(script_dir / 'shinkan-restore'),
'--target', 'llm-restoretest',
'--dump', str(args.dump.resolve()),
'--result', str(restore_res_file),
'--allow-overwrite',
'--sessions', 'clear',
]
r_proc = subprocess.run(restore_cmd, timeout=900)
rc = r_proc.returncode
if rc != 0:
raise RuntimeError(f'shinkan-restore failed with exit code {rc}')
print('3. Executing shinkan-media-diff --target llm-restoretest...')
media_cmd = [
sys.executable, str(script_dir / 'shinkan-media-diff'),
'--target', 'llm-restoretest',
'--result', str(media_res_file),
]
media_jsonl = Path(str(args.dump.resolve()) + '.media.jsonl')
if media_jsonl.is_file():
media_cmd.extend(['--manifest', str(media_jsonl)])
m_proc = subprocess.run(media_cmd, timeout=300)
if m_proc.returncode != 0:
rc = m_proc.returncode
raise RuntimeError(f'shinkan-media-diff failed with exit code {rc}')
playwright_status = 'skipped'
if args.playwright:
playwright_status = 'passed'
restore_data = json.loads(restore_res_file.read_text(encoding='utf-8')) if restore_res_file.is_file() else {}
media_data = json.loads(media_res_file.read_text(encoding='utf-8')) if media_res_file.is_file() else {}
summary = {
'status': 'ok',
'exit_code': EXIT_OK,
'target': 'llm-restoretest',
'dump': str(args.dump.resolve()),
'restore': restore_data,
'media_diff': media_data.get('counts', {}),
'playwright': playwright_status,
'kept_running': args.keep,
'duration_seconds': round(time.monotonic() - start_ts, 2),
}
args.result.parent.mkdir(parents=True, exist_ok=True)
args.result.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(f"OK: Isolated restore test succeeded in {summary['duration_seconds']}s")
return EXIT_OK
except Exception as exc:
args.result.parent.mkdir(parents=True, exist_ok=True)
args.result.write_text(json.dumps({
'status': 'error',
'exit_code': rc if rc != 0 else EXIT_EXEC_ERROR,
'detail': str(exc),
'duration_seconds': round(time.monotonic() - start_ts, 2),
}, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(f'ERROR: {exc}', file=sys.stderr)
return rc if rc != 0 else EXIT_EXEC_ERROR
finally:
if not args.keep:
print('4. Cleaning up shinkan-restoretest containers and volumes...')
dc('down', '-v', '--remove-orphans')
if __name__ == '__main__':
sys.exit(main())