feat: add universal-db-restore, app migration contract, targets for kairo/tandoor/bookstack, and uptime kuma guide
This commit is contained in:
1 parent
a17afb4b13
commit
feb469d47b
10 files changed
+1079
-12
No files matched your search
@@ -0,0 +1,535 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Universal transactional database restore adapter for PostgreSQL and MariaDB/MySQL.
|
||||
|
||||
Supports Docker-based deployments with:
|
||||
- Staging database isolation and integrity verification prior to live cutover.
|
||||
- Atomic database/table swap.
|
||||
- Reversible rollback if post-restore health checks fail.
|
||||
- Plausibility checks (non-empty tables, migration checks).
|
||||
- Session truncation on cutover.
|
||||
- Fully parameter-driven via targets.json.
|
||||
|
||||
Exit codes:
|
||||
- 0: Restore and post-restore checks succeeded
|
||||
- 1: Execution error
|
||||
- 2: Precondition / confirmation / host guard error
|
||||
- 3: Schema incompatibility
|
||||
- 4: Checksum mismatch
|
||||
- 5: Post-restore healthcheck failed (automatic rollback executed)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from urllib.request import urlopen
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_EXEC_ERROR = 1
|
||||
EXIT_PRECONDITION = 2
|
||||
EXIT_SCHEMA_INCOMPATIBLE = 3
|
||||
EXIT_CHECKSUM_MISMATCH = 4
|
||||
EXIT_POST_RESTORE_FAILED = 5
|
||||
|
||||
|
||||
def compute_sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as fp:
|
||||
while True:
|
||||
chunk = fp.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def detect_dump_format(path):
|
||||
with Path(path).open('rb') as fp:
|
||||
magic = fp.read(5)
|
||||
if magic == b'PGDMP':
|
||||
return 'pg_custom'
|
||||
return 'plain_sql'
|
||||
|
||||
|
||||
def write_result(path, data):
|
||||
if path:
|
||||
p = Path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def load_targets(custom_file=None):
|
||||
candidates = []
|
||||
if custom_file:
|
||||
candidates.append(Path(custom_file))
|
||||
candidates.extend([
|
||||
Path('/etc/backup-orchestrator/targets.json'),
|
||||
Path(__file__).resolve().parents[2] / 'targets.json',
|
||||
Path(__file__).resolve().parents[2] / f'hosts/{socket.gethostname()}/targets.json',
|
||||
])
|
||||
for c in candidates:
|
||||
if c.is_file():
|
||||
try:
|
||||
data = json.loads(c.read_text(encoding='utf-8'))
|
||||
if 'targets' in data:
|
||||
return data
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
return {'targets': {}}
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# PostgreSQL Database Helpers
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def psql_exec(tcfg, dbname, sql):
|
||||
cmd = [
|
||||
'docker', 'exec', '-i', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -v ON_ERROR_STOP=1 -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}" -At',
|
||||
]
|
||||
return subprocess.run(cmd, input=sql, capture_output=True, text=True, timeout=120)
|
||||
|
||||
|
||||
def postgres_drop_db_if_exists(tcfg, dbname):
|
||||
term_sql = f"""
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = '{dbname}' AND pid <> pg_backend_pid();
|
||||
"""
|
||||
psql_exec(tcfg, 'postgres', term_sql)
|
||||
psql_exec(tcfg, 'postgres', f'DROP DATABASE IF EXISTS "{dbname}";')
|
||||
|
||||
|
||||
def postgres_create_db(tcfg, dbname):
|
||||
postgres_drop_db_if_exists(tcfg, dbname)
|
||||
res = psql_exec(tcfg, 'postgres', f'CREATE DATABASE "{dbname}";')
|
||||
return res.returncode == 0, res.stderr.strip()
|
||||
|
||||
|
||||
def postgres_import_dump(tcfg, dbname, dump_path):
|
||||
fmt = detect_dump_format(dump_path)
|
||||
if fmt == 'pg_custom':
|
||||
cmd = [
|
||||
'docker', 'exec', '-i', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec pg_restore -v --single-transaction --exit-on-error -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}"',
|
||||
]
|
||||
else:
|
||||
cmd = [
|
||||
'docker', 'exec', '-i', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -v ON_ERROR_STOP=1 --single-transaction -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}"',
|
||||
]
|
||||
with Path(dump_path).open('rb') as fp:
|
||||
res = subprocess.run(cmd, stdin=fp, capture_output=True, text=True, timeout=1800)
|
||||
return res.returncode == 0, res.stderr.strip() or res.stdout.strip()
|
||||
|
||||
|
||||
def postgres_swap_databases(tcfg, live_db, staging_db, backup_db):
|
||||
term_sql = f"""
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname IN ('{live_db}', '{staging_db}', '{backup_db}') AND pid <> pg_backend_pid();
|
||||
"""
|
||||
psql_exec(tcfg, 'postgres', term_sql)
|
||||
# 1. Rename live to backup
|
||||
r1 = psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{live_db}" RENAME TO "{backup_db}";')
|
||||
if r1.returncode != 0:
|
||||
return False, f"Rename {live_db} -> {backup_db} failed: {r1.stderr.strip()}"
|
||||
# 2. Rename staging to live
|
||||
r2 = psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{staging_db}" RENAME TO "{live_db}";')
|
||||
if r2.returncode != 0:
|
||||
# Emergency rollback
|
||||
psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{backup_db}" RENAME TO "{live_db}";')
|
||||
return False, f"Rename {staging_db} -> {live_db} failed: {r2.stderr.strip()}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def postgres_rollback_swap(tcfg, live_db, staging_db, backup_db):
|
||||
term_sql = f"""
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname IN ('{live_db}', '{staging_db}', '{backup_db}') AND pid <> pg_backend_pid();
|
||||
"""
|
||||
psql_exec(tcfg, 'postgres', term_sql)
|
||||
psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{live_db}" RENAME TO "{staging_db}";')
|
||||
psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{backup_db}" RENAME TO "{live_db}";')
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# MariaDB / MySQL Database Helpers
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def mariadb_exec(tcfg, dbname, sql):
|
||||
db_flag = f'"{dbname}"' if dbname else ''
|
||||
cmd = [
|
||||
'docker', 'exec', '-i', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
f'export MYSQL_PWD="${{MARIADB_PASSWORD:-${{MYSQL_PASSWORD:-}}}}"; '
|
||||
f'exec mariadb --default-character-set=utf8mb4 -u "${{MARIADB_USER:-${{MYSQL_USER:-root}}}}" {db_flag} -sN',
|
||||
]
|
||||
return subprocess.run(cmd, input=sql, capture_output=True, text=True, timeout=120)
|
||||
|
||||
|
||||
def mariadb_drop_db_if_exists(tcfg, dbname):
|
||||
mariadb_exec(tcfg, '', f'DROP DATABASE IF EXISTS `{dbname}`;')
|
||||
|
||||
|
||||
def mariadb_create_db(tcfg, dbname):
|
||||
mariadb_drop_db_if_exists(tcfg, dbname)
|
||||
res = mariadb_exec(tcfg, '', f'CREATE DATABASE `{dbname}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;')
|
||||
return res.returncode == 0, res.stderr.strip()
|
||||
|
||||
|
||||
def mariadb_import_dump(tcfg, dbname, dump_path, original_dbname):
|
||||
"""Import MariaDB dump into staging DB, rewriting CREATE/USE database references if needed."""
|
||||
cmd = [
|
||||
'docker', 'exec', '-i', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
f'export MYSQL_PWD="${{MARIADB_PASSWORD:-${{MYSQL_PASSWORD:-}}}}"; '
|
||||
f'exec mariadb --default-character-set=utf8mb4 -u "${{MARIADB_USER:-${{MYSQL_USER:-root}}}}" `{dbname}`',
|
||||
]
|
||||
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
|
||||
# Stream and rewrite USE `original_dbname` / CREATE DATABASE statements to staging db
|
||||
re_use = re.compile(rf'^\s*USE\s+[`"\']?{re.escape(original_dbname)}[`"\']?\s*;', re.IGNORECASE)
|
||||
re_create = re.compile(rf'^\s*CREATE\s+DATABASE.*?[`"\']?{re.escape(original_dbname)}[`"\']?.*?;', re.IGNORECASE)
|
||||
|
||||
try:
|
||||
with Path(dump_path).open('r', encoding='utf-8', errors='replace') as fp:
|
||||
for line in fp:
|
||||
if re_use.match(line):
|
||||
proc.stdin.write(f'USE `{dbname}`;\n')
|
||||
elif re_create.match(line):
|
||||
proc.stdin.write(f'-- {line.strip()}\n')
|
||||
else:
|
||||
proc.stdin.write(line)
|
||||
stdout, stderr = proc.communicate(timeout=1800)
|
||||
return proc.returncode == 0, stderr.strip() or stdout.strip()
|
||||
except Exception as exc:
|
||||
proc.kill()
|
||||
return False, str(exc)
|
||||
|
||||
|
||||
def mariadb_get_tables(tcfg, dbname):
|
||||
sql = f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{dbname}' AND table_type = 'BASE TABLE';"
|
||||
res = mariadb_exec(tcfg, '', sql)
|
||||
if res.returncode != 0:
|
||||
return []
|
||||
return [line.strip() for line in res.stdout.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def mariadb_swap_databases(tcfg, live_db, staging_db, backup_db):
|
||||
"""Atomically swap tables between schemas using MySQL/MariaDB RENAME TABLE."""
|
||||
mariadb_create_db(tcfg, backup_db)
|
||||
live_tables = set(mariadb_get_tables(tcfg, live_db))
|
||||
staging_tables = set(mariadb_get_tables(tcfg, staging_db))
|
||||
|
||||
rename_clauses = []
|
||||
# Move current live tables to backup_db
|
||||
for tbl in live_tables:
|
||||
rename_clauses.append(f"`{live_db}`.`{tbl}` TO `{backup_db}`.`{tbl}`")
|
||||
# Move staging tables to live_db
|
||||
for tbl in staging_tables:
|
||||
rename_clauses.append(f"`{staging_db}`.`{tbl}` TO `{live_db}`.`{tbl}`")
|
||||
|
||||
if not rename_clauses:
|
||||
return False, "No tables to rename in swap operation"
|
||||
|
||||
rename_sql = "RENAME TABLE " + ", ".join(rename_clauses) + ";"
|
||||
res = mariadb_exec(tcfg, '', rename_sql)
|
||||
if res.returncode != 0:
|
||||
return False, f"Atomic table rename failed: {res.stderr.strip()}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def mariadb_rollback_swap(tcfg, live_db, staging_db, backup_db):
|
||||
live_tables = set(mariadb_get_tables(tcfg, live_db))
|
||||
backup_tables = set(mariadb_get_tables(tcfg, backup_db))
|
||||
rename_clauses = []
|
||||
for tbl in live_tables:
|
||||
rename_clauses.append(f"`{live_db}`.`{tbl}` TO `{staging_db}`.`{tbl}`")
|
||||
for tbl in backup_tables:
|
||||
rename_clauses.append(f"`{backup_db}`.`{tbl}` TO `{live_db}`.`{tbl}`")
|
||||
if rename_clauses:
|
||||
mariadb_exec(tcfg, '', "RENAME TABLE " + ", ".join(rename_clauses) + ";")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Common Compose & Healthcheck Functions
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def compose_cmd(tcfg, *args):
|
||||
cmd = ['docker', 'compose']
|
||||
if tcfg.get('compose_project'):
|
||||
cmd.extend(['-p', tcfg['compose_project']])
|
||||
for cf in tcfg.get('compose_files', ['docker-compose.yml']):
|
||||
cf_path = Path(cf) if Path(cf).is_absolute() else (Path(tcfg.get('app_dir', '.')) / cf)
|
||||
if cf_path.is_file():
|
||||
cmd.extend(['-f', str(cf_path)])
|
||||
if tcfg.get('env_file') and Path(tcfg['env_file']).is_file():
|
||||
cmd.extend(['--env-file', tcfg['env_file']])
|
||||
cmd.extend(args)
|
||||
cwd = tcfg.get('app_dir') if tcfg.get('app_dir') and Path(tcfg['app_dir']).is_dir() else None
|
||||
return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=300)
|
||||
|
||||
|
||||
def wait_ready(url, timeout_sec=180):
|
||||
deadline = time.monotonic() + timeout_sec
|
||||
last_body = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urlopen(url, timeout=5) as resp:
|
||||
raw = resp.read(8192).decode('utf-8', errors='replace')
|
||||
if resp.status == 200:
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
# If JSON contains 'status', verify it is 'ok' or 'ready'
|
||||
if isinstance(parsed, dict) and 'status' in parsed:
|
||||
if parsed['status'] in ('ready', 'ok'):
|
||||
return True, parsed
|
||||
else:
|
||||
return True, {'raw': raw[:200]}
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
return True, {'status': 'http_200'}
|
||||
except Exception as exc:
|
||||
last_body = {'error': str(exc)}
|
||||
time.sleep(3)
|
||||
return False, last_body
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Main Restore Engine
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Universal transactional database restore adapter')
|
||||
parser.add_argument('--target', required=True, help='Target name from targets.json')
|
||||
parser.add_argument('--dump', type=Path, help='Path to SQL dump')
|
||||
parser.add_argument('--result', required=True, type=Path, help='Output path for result JSON')
|
||||
parser.add_argument('--confirm', type=str, help='Required confirmation token')
|
||||
parser.add_argument('--allow-overwrite', action='store_true', help='Allow overwrite of non-empty live database')
|
||||
parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear')
|
||||
parser.add_argument('--drop-previous', type=str, help='Explicitly drop a retained <app>_before_<ts> database')
|
||||
parser.add_argument('--targets-file', type=Path, help='Optional explicit path to targets.json')
|
||||
args = parser.parse_args()
|
||||
|
||||
targets_registry = load_targets(args.targets_file)
|
||||
if args.target not in targets_registry.get('targets', {}):
|
||||
msg = f"Target {args.target!r} not found in targets.json"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
tcfg = targets_registry['targets'][args.target]
|
||||
engine = tcfg.get('db_engine', 'postgresql').lower()
|
||||
if engine not in ('postgresql', 'postgres', 'mariadb', 'mysql'):
|
||||
msg = f"Unsupported db_engine: {engine}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
is_postgres = engine in ('postgresql', 'postgres')
|
||||
|
||||
# Host & Arch guard
|
||||
cur_host = socket.gethostname()
|
||||
cur_arch = platform.machine()
|
||||
if os.environ.get('SKIP_HOST_GUARD') != '1':
|
||||
req_host = tcfg.get('host')
|
||||
req_arch = tcfg.get('arch')
|
||||
if req_host and req_host != cur_host:
|
||||
msg = f"Host mismatch: running on {cur_host}, target requires {req_host}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
if req_arch and req_arch != cur_arch:
|
||||
msg = f"Arch mismatch: running on {cur_arch}, target requires {req_arch}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
# Confirmation token guard
|
||||
req_confirm = tcfg.get('require_confirm')
|
||||
if req_confirm and args.confirm != req_confirm:
|
||||
msg = f"Target {args.target} requires confirmation: --confirm {req_confirm}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
# Drop previous retained DB if requested
|
||||
live_db = tcfg['db_name']
|
||||
if args.drop_previous:
|
||||
expected_pattern = rf'^{re.escape(live_db)}_before_\d{{8}}_\d{{6}}$'
|
||||
if not re.match(expected_pattern, args.drop_previous):
|
||||
msg = f"Refused to drop database with unexpected name: {args.drop_previous}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
return EXIT_PRECONDITION
|
||||
if is_postgres:
|
||||
postgres_drop_db_if_exists(tcfg, args.drop_previous)
|
||||
else:
|
||||
mariadb_drop_db_if_exists(tcfg, args.drop_previous)
|
||||
write_result(args.result, {'status': 'ok', 'exit_code': EXIT_OK, 'dropped_database': args.drop_previous})
|
||||
print(f"OK: Dropped previous database {args.drop_previous}")
|
||||
return EXIT_OK
|
||||
|
||||
if not args.dump:
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': '--dump is required'})
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
dump_path = args.dump.resolve()
|
||||
sha_path = Path(str(dump_path) + '.sha256')
|
||||
if not dump_path.is_file():
|
||||
msg = f"Dump file missing: {dump_path}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
# Checksum validation if .sha256 file is present
|
||||
if sha_path.is_file():
|
||||
expected_sha = sha_path.read_text(encoding='utf-8').strip().split()[0].lower()
|
||||
actual_sha = compute_sha256(dump_path)
|
||||
if expected_sha != actual_sha:
|
||||
msg = f"SHA256 mismatch: expected {expected_sha}, got {actual_sha}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_CHECKSUM_MISMATCH, 'detail': msg})
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
return EXIT_CHECKSUM_MISMATCH
|
||||
|
||||
# Check if live DB has non-empty tables and --allow-overwrite is needed
|
||||
check_tables = tcfg.get('verify_tables_not_empty', [])
|
||||
if check_tables and not args.allow_overwrite:
|
||||
tbl = check_tables[0]
|
||||
cnt = 0
|
||||
if is_postgres:
|
||||
r = psql_exec(tcfg, live_db, f'SELECT COUNT(*)::int FROM "{tbl}";')
|
||||
cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0
|
||||
else:
|
||||
r = mariadb_exec(tcfg, live_db, f'SELECT COUNT(*) FROM `{tbl}`;')
|
||||
cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0
|
||||
if cnt > 0:
|
||||
msg = f"Target database {live_db} contains {cnt} rows in table {tbl}; pass --allow-overwrite to replace it."
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg})
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
ts_label = dt.datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
staging_db = f"{live_db}_restore_{ts_label}"
|
||||
backup_db = f"{live_db}_before_{ts_label}"
|
||||
|
||||
print(f"1. Creating staging database {staging_db} ({engine})...")
|
||||
if is_postgres:
|
||||
ok, err = postgres_create_db(tcfg, staging_db)
|
||||
else:
|
||||
ok, err = mariadb_create_db(tcfg, staging_db)
|
||||
if not ok:
|
||||
msg = f"Failed to create staging DB: {err}"
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
print(f"2. Importing dump into {staging_db}...")
|
||||
if is_postgres:
|
||||
ok, err = postgres_import_dump(tcfg, staging_db, dump_path)
|
||||
else:
|
||||
ok, err = mariadb_import_dump(tcfg, staging_db, dump_path, live_db)
|
||||
if not ok:
|
||||
msg = f"Failed to import dump into staging DB: {err}"
|
||||
if is_postgres:
|
||||
postgres_drop_db_if_exists(tcfg, staging_db)
|
||||
else:
|
||||
mariadb_drop_db_if_exists(tcfg, staging_db)
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
print("3. Validating staging database integrity...")
|
||||
table_counts = {}
|
||||
for tbl in tcfg.get('verify_tables_not_empty', []):
|
||||
if is_postgres:
|
||||
r = psql_exec(tcfg, staging_db, f'SELECT COUNT(*)::int FROM "{tbl}";')
|
||||
else:
|
||||
r = mariadb_exec(tcfg, staging_db, f'SELECT COUNT(*) FROM `{tbl}`;')
|
||||
cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0
|
||||
table_counts[tbl] = cnt
|
||||
if cnt == 0:
|
||||
msg = f"Verification failed: required table {tbl} is empty in staging DB"
|
||||
if is_postgres:
|
||||
postgres_drop_db_if_exists(tcfg, staging_db)
|
||||
else:
|
||||
mariadb_drop_db_if_exists(tcfg, staging_db)
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
# Invalidate / truncate session tables if requested
|
||||
if args.sessions == 'clear':
|
||||
for tbl in tcfg.get('truncate_tables', ['sessions']):
|
||||
print(f"Truncating session table: {tbl}")
|
||||
if is_postgres:
|
||||
psql_exec(tcfg, staging_db, f'TRUNCATE TABLE "{tbl}" CASCADE;')
|
||||
else:
|
||||
mariadb_exec(tcfg, staging_db, f'TRUNCATE TABLE `{tbl}`;')
|
||||
|
||||
services_to_stop = tcfg.get('services_to_stop', [])
|
||||
if services_to_stop:
|
||||
print(f"4. Stopping dependent services: {services_to_stop}...")
|
||||
compose_cmd(tcfg, 'stop', *services_to_stop)
|
||||
|
||||
print(f"5. Performing atomic database swap ({live_db} -> {backup_db}, {staging_db} -> {live_db})...")
|
||||
if is_postgres:
|
||||
swap_ok, swap_err = postgres_swap_databases(tcfg, live_db, staging_db, backup_db)
|
||||
else:
|
||||
swap_ok, swap_err = mariadb_swap_databases(tcfg, live_db, staging_db, backup_db)
|
||||
|
||||
if not swap_ok:
|
||||
msg = f"Database swap failed: {swap_err}"
|
||||
if services_to_stop:
|
||||
compose_cmd(tcfg, 'up', '-d', *services_to_stop)
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg})
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
if services_to_stop:
|
||||
print(f"6. Starting dependent services: {services_to_stop}...")
|
||||
compose_cmd(tcfg, 'up', '-d', *services_to_stop)
|
||||
|
||||
# Healthcheck / Readiness verification
|
||||
health_url = tcfg.get('ready_url') or tcfg.get('health_url')
|
||||
if health_url:
|
||||
print(f"7. Verifying service readiness at {health_url}...")
|
||||
timeout_sec = tcfg.get('health_timeout_seconds', 180)
|
||||
ready_ok, ready_data = wait_ready(health_url, timeout_sec=timeout_sec)
|
||||
if not ready_ok:
|
||||
print(f"ERROR: Health check failed after {timeout_sec}s: {ready_data}. Initiating automatic rollback!", file=sys.stderr)
|
||||
if services_to_stop:
|
||||
compose_cmd(tcfg, 'stop', *services_to_stop)
|
||||
if is_postgres:
|
||||
postgres_rollback_swap(tcfg, live_db, staging_db, backup_db)
|
||||
else:
|
||||
mariadb_rollback_swap(tcfg, live_db, staging_db, backup_db)
|
||||
if services_to_stop:
|
||||
compose_cmd(tcfg, 'up', '-d', *services_to_stop)
|
||||
msg = f"Post-restore healthcheck failed: {ready_data}. Automatic rollback executed."
|
||||
write_result(args.result, {'status': 'error', 'exit_code': EXIT_POST_RESTORE_FAILED, 'detail': msg, 'health_data': ready_data})
|
||||
return EXIT_POST_RESTORE_FAILED
|
||||
|
||||
summary = {
|
||||
'status': 'ok',
|
||||
'exit_code': EXIT_OK,
|
||||
'target': args.target,
|
||||
'db_engine': engine,
|
||||
'live_db': live_db,
|
||||
'previous_db': backup_db,
|
||||
'verified_tables': table_counts,
|
||||
'sessions': args.sessions,
|
||||
'completed_at': dt.datetime.now().isoformat(),
|
||||
}
|
||||
write_result(args.result, summary)
|
||||
print(f"OK: Restore completed successfully for target {args.target} (previous DB: {backup_db})")
|
||||
return EXIT_OK
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user