#!/usr/bin/env python3 """Universal transactional database restore adapter for PostgreSQL and MariaDB/MySQL. Supports Docker-based deployments with: - Staging database isolation and integrity verification prior to live cutover. - Atomic database/table swap. - Reversible rollback if post-restore health checks fail. - Plausibility checks (non-empty tables, migration checks). - Session truncation on cutover. - Fully parameter-driven via targets.json. Exit codes: - 0: Restore and post-restore checks succeeded - 1: Execution error - 2: Precondition / confirmation / host guard error - 3: Schema incompatibility - 4: Checksum mismatch - 5: Post-restore healthcheck failed (automatic rollback executed) """ import argparse import datetime as dt import hashlib import json import os from pathlib import Path import platform import re import socket import subprocess import sys import time from urllib.request import urlopen EXIT_OK = 0 EXIT_EXEC_ERROR = 1 EXIT_PRECONDITION = 2 EXIT_SCHEMA_INCOMPATIBLE = 3 EXIT_CHECKSUM_MISMATCH = 4 EXIT_POST_RESTORE_FAILED = 5 def compute_sha256(path): h = hashlib.sha256() with Path(path).open('rb') as fp: while True: chunk = fp.read(65536) if not chunk: break h.update(chunk) return h.hexdigest() def detect_dump_format(path): with Path(path).open('rb') as fp: magic = fp.read(5) if magic == b'PGDMP': return 'pg_custom' return 'plain_sql' def write_result(path, data): if path: p = Path(path) p.parent.mkdir(parents=True, exist_ok=True) p.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') def load_targets(custom_file=None): candidates = [] if custom_file: candidates.append(Path(custom_file)) candidates.extend([ Path('/etc/backup-orchestrator/targets.json'), Path(__file__).resolve().parents[2] / 'targets.json', Path(__file__).resolve().parents[2] / f'hosts/{socket.gethostname()}/targets.json', ]) for c in candidates: if c.is_file(): try: data = json.loads(c.read_text(encoding='utf-8')) if 'targets' in data: return data except (OSError, json.JSONDecodeError): continue return {'targets': {}} # ----------------------------------------------------------------------------- # PostgreSQL Database Helpers # ----------------------------------------------------------------------------- def psql_exec(tcfg, dbname, sql): cmd = [ 'docker', 'exec', '-i', tcfg['db_container'], 'sh', '-ec', f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -v ON_ERROR_STOP=1 -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}" -At', ] return subprocess.run(cmd, input=sql, capture_output=True, text=True, timeout=120) def postgres_drop_db_if_exists(tcfg, dbname): term_sql = f""" SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{dbname}' AND pid <> pg_backend_pid(); """ psql_exec(tcfg, 'postgres', term_sql) psql_exec(tcfg, 'postgres', f'DROP DATABASE IF EXISTS "{dbname}";') def postgres_create_db(tcfg, dbname): postgres_drop_db_if_exists(tcfg, dbname) res = psql_exec(tcfg, 'postgres', f'CREATE DATABASE "{dbname}";') return res.returncode == 0, res.stderr.strip() def postgres_import_dump(tcfg, dbname, dump_path): fmt = detect_dump_format(dump_path) if fmt == 'pg_custom': cmd = [ 'docker', 'exec', '-i', tcfg['db_container'], 'sh', '-ec', f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec pg_restore -v --single-transaction --exit-on-error -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}"', ] else: cmd = [ 'docker', 'exec', '-i', tcfg['db_container'], 'sh', '-ec', f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -v ON_ERROR_STOP=1 --single-transaction -U "{tcfg.get("db_user", "postgres")}" -d "{dbname}"', ] with Path(dump_path).open('rb') as fp: res = subprocess.run(cmd, stdin=fp, capture_output=True, text=True, timeout=1800) return res.returncode == 0, res.stderr.strip() or res.stdout.strip() def postgres_swap_databases(tcfg, live_db, staging_db, backup_db): term_sql = f""" SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('{live_db}', '{staging_db}', '{backup_db}') AND pid <> pg_backend_pid(); """ psql_exec(tcfg, 'postgres', term_sql) # 1. Rename live to backup r1 = psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{live_db}" RENAME TO "{backup_db}";') if r1.returncode != 0: return False, f"Rename {live_db} -> {backup_db} failed: {r1.stderr.strip()}" # 2. Rename staging to live r2 = psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{staging_db}" RENAME TO "{live_db}";') if r2.returncode != 0: # Emergency rollback psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{backup_db}" RENAME TO "{live_db}";') return False, f"Rename {staging_db} -> {live_db} failed: {r2.stderr.strip()}" return True, "ok" def postgres_rollback_swap(tcfg, live_db, staging_db, backup_db): term_sql = f""" SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname IN ('{live_db}', '{staging_db}', '{backup_db}') AND pid <> pg_backend_pid(); """ psql_exec(tcfg, 'postgres', term_sql) psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{live_db}" RENAME TO "{staging_db}";') psql_exec(tcfg, 'postgres', f'ALTER DATABASE "{backup_db}" RENAME TO "{live_db}";') # ----------------------------------------------------------------------------- # MariaDB / MySQL Database Helpers # ----------------------------------------------------------------------------- def mariadb_exec(tcfg, dbname, sql): db_flag = f'"{dbname}"' if dbname else '' cmd = [ 'docker', 'exec', '-i', tcfg['db_container'], 'sh', '-ec', f'export MYSQL_PWD="${{MARIADB_PASSWORD:-${{MYSQL_PASSWORD:-}}}}"; ' f'exec mariadb --default-character-set=utf8mb4 -u "${{MARIADB_USER:-${{MYSQL_USER:-root}}}}" {db_flag} -sN', ] return subprocess.run(cmd, input=sql, capture_output=True, text=True, timeout=120) def mariadb_drop_db_if_exists(tcfg, dbname): mariadb_exec(tcfg, '', f'DROP DATABASE IF EXISTS `{dbname}`;') def mariadb_create_db(tcfg, dbname): mariadb_drop_db_if_exists(tcfg, dbname) res = mariadb_exec(tcfg, '', f'CREATE DATABASE `{dbname}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;') return res.returncode == 0, res.stderr.strip() def mariadb_import_dump(tcfg, dbname, dump_path, original_dbname): """Import MariaDB dump into staging DB, rewriting CREATE/USE database references if needed.""" cmd = [ 'docker', 'exec', '-i', tcfg['db_container'], 'sh', '-ec', f'export MYSQL_PWD="${{MARIADB_PASSWORD:-${{MYSQL_PASSWORD:-}}}}"; ' f'exec mariadb --default-character-set=utf8mb4 -u "${{MARIADB_USER:-${{MYSQL_USER:-root}}}}" `{dbname}`', ] proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) # Stream and rewrite USE `original_dbname` / CREATE DATABASE statements to staging db re_use = re.compile(rf'^\s*USE\s+[`"\']?{re.escape(original_dbname)}[`"\']?\s*;', re.IGNORECASE) re_create = re.compile(rf'^\s*CREATE\s+DATABASE.*?[`"\']?{re.escape(original_dbname)}[`"\']?.*?;', re.IGNORECASE) try: with Path(dump_path).open('r', encoding='utf-8', errors='replace') as fp: for line in fp: if re_use.match(line): proc.stdin.write(f'USE `{dbname}`;\n') elif re_create.match(line): proc.stdin.write(f'-- {line.strip()}\n') else: proc.stdin.write(line) stdout, stderr = proc.communicate(timeout=1800) return proc.returncode == 0, stderr.strip() or stdout.strip() except Exception as exc: proc.kill() return False, str(exc) def mariadb_get_tables(tcfg, dbname): sql = f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{dbname}' AND table_type = 'BASE TABLE';" res = mariadb_exec(tcfg, '', sql) if res.returncode != 0: return [] return [line.strip() for line in res.stdout.splitlines() if line.strip()] def mariadb_swap_databases(tcfg, live_db, staging_db, backup_db): """Atomically swap tables between schemas using MySQL/MariaDB RENAME TABLE.""" mariadb_create_db(tcfg, backup_db) live_tables = set(mariadb_get_tables(tcfg, live_db)) staging_tables = set(mariadb_get_tables(tcfg, staging_db)) rename_clauses = [] # Move current live tables to backup_db for tbl in live_tables: rename_clauses.append(f"`{live_db}`.`{tbl}` TO `{backup_db}`.`{tbl}`") # Move staging tables to live_db for tbl in staging_tables: rename_clauses.append(f"`{staging_db}`.`{tbl}` TO `{live_db}`.`{tbl}`") if not rename_clauses: return False, "No tables to rename in swap operation" rename_sql = "RENAME TABLE " + ", ".join(rename_clauses) + ";" res = mariadb_exec(tcfg, '', rename_sql) if res.returncode != 0: return False, f"Atomic table rename failed: {res.stderr.strip()}" return True, "ok" def mariadb_rollback_swap(tcfg, live_db, staging_db, backup_db): live_tables = set(mariadb_get_tables(tcfg, live_db)) backup_tables = set(mariadb_get_tables(tcfg, backup_db)) rename_clauses = [] for tbl in live_tables: rename_clauses.append(f"`{live_db}`.`{tbl}` TO `{staging_db}`.`{tbl}`") for tbl in backup_tables: rename_clauses.append(f"`{backup_db}`.`{tbl}` TO `{live_db}`.`{tbl}`") if rename_clauses: mariadb_exec(tcfg, '', "RENAME TABLE " + ", ".join(rename_clauses) + ";") # ----------------------------------------------------------------------------- # Common Compose & Healthcheck Functions # ----------------------------------------------------------------------------- def compose_cmd(tcfg, *args): cmd = ['docker', 'compose'] if tcfg.get('compose_project'): cmd.extend(['-p', tcfg['compose_project']]) for cf in tcfg.get('compose_files', ['docker-compose.yml']): cf_path = Path(cf) if Path(cf).is_absolute() else (Path(tcfg.get('app_dir', '.')) / cf) if cf_path.is_file(): cmd.extend(['-f', str(cf_path)]) if tcfg.get('env_file') and Path(tcfg['env_file']).is_file(): cmd.extend(['--env-file', tcfg['env_file']]) cmd.extend(args) cwd = tcfg.get('app_dir') if tcfg.get('app_dir') and Path(tcfg['app_dir']).is_dir() else None return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=300) def wait_ready(url, timeout_sec=180): deadline = time.monotonic() + timeout_sec last_body = None while time.monotonic() < deadline: try: with urlopen(url, timeout=5) as resp: raw = resp.read(8192).decode('utf-8', errors='replace') if resp.status == 200: try: parsed = json.loads(raw) # If JSON contains 'status', verify it is 'ok' or 'ready' if isinstance(parsed, dict) and 'status' in parsed: if parsed['status'] in ('ready', 'ok'): return True, parsed else: return True, {'raw': raw[:200]} except (ValueError, json.JSONDecodeError): return True, {'status': 'http_200'} except Exception as exc: last_body = {'error': str(exc)} time.sleep(3) return False, last_body # ----------------------------------------------------------------------------- # Main Restore Engine # ----------------------------------------------------------------------------- def main(): parser = argparse.ArgumentParser(description='Universal transactional database restore adapter') parser.add_argument('--target', required=True, help='Target name from targets.json') parser.add_argument('--dump', type=Path, help='Path to SQL dump') parser.add_argument('--result', required=True, type=Path, help='Output path for result JSON') parser.add_argument('--confirm', type=str, help='Required confirmation token') parser.add_argument('--allow-overwrite', action='store_true', help='Allow overwrite of non-empty live database') parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear') parser.add_argument('--drop-previous', type=str, help='Explicitly drop a retained _before_ database') parser.add_argument('--targets-file', type=Path, help='Optional explicit path to targets.json') args = parser.parse_args() targets_registry = load_targets(args.targets_file) if args.target not in targets_registry.get('targets', {}): msg = f"Target {args.target!r} not found in targets.json" write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) print(f"ERROR: {msg}", file=sys.stderr) return EXIT_PRECONDITION tcfg = targets_registry['targets'][args.target] engine = tcfg.get('db_engine', 'postgresql').lower() if engine not in ('postgresql', 'postgres', 'mariadb', 'mysql'): msg = f"Unsupported db_engine: {engine}" write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) print(f"ERROR: {msg}", file=sys.stderr) return EXIT_PRECONDITION is_postgres = engine in ('postgresql', 'postgres') # Host & Arch guard cur_host = socket.gethostname() cur_arch = platform.machine() if os.environ.get('SKIP_HOST_GUARD') != '1': req_host = tcfg.get('host') req_arch = tcfg.get('arch') if req_host and req_host != cur_host: msg = f"Host mismatch: running on {cur_host}, target requires {req_host}" write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) print(f"ERROR: {msg}", file=sys.stderr) return EXIT_PRECONDITION if req_arch and req_arch != cur_arch: msg = f"Arch mismatch: running on {cur_arch}, target requires {req_arch}" write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) print(f"ERROR: {msg}", file=sys.stderr) return EXIT_PRECONDITION # Confirmation token guard req_confirm = tcfg.get('require_confirm') if req_confirm and args.confirm != req_confirm: msg = f"Target {args.target} requires confirmation: --confirm {req_confirm}" write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) print(f"ERROR: {msg}", file=sys.stderr) return EXIT_PRECONDITION # Drop previous retained DB if requested live_db = tcfg['db_name'] if args.drop_previous: expected_pattern = rf'^{re.escape(live_db)}_before_\d{{8}}_\d{{6}}$' if not re.match(expected_pattern, args.drop_previous): msg = f"Refused to drop database with unexpected name: {args.drop_previous}" write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) return EXIT_PRECONDITION if is_postgres: postgres_drop_db_if_exists(tcfg, args.drop_previous) else: mariadb_drop_db_if_exists(tcfg, args.drop_previous) write_result(args.result, {'status': 'ok', 'exit_code': EXIT_OK, 'dropped_database': args.drop_previous}) print(f"OK: Dropped previous database {args.drop_previous}") return EXIT_OK if not args.dump: write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': '--dump is required'}) return EXIT_PRECONDITION dump_path = args.dump.resolve() sha_path = Path(str(dump_path) + '.sha256') if not dump_path.is_file(): msg = f"Dump file missing: {dump_path}" write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) return EXIT_PRECONDITION # Checksum validation if .sha256 file is present if sha_path.is_file(): expected_sha = sha_path.read_text(encoding='utf-8').strip().split()[0].lower() actual_sha = compute_sha256(dump_path) if expected_sha != actual_sha: msg = f"SHA256 mismatch: expected {expected_sha}, got {actual_sha}" write_result(args.result, {'status': 'error', 'exit_code': EXIT_CHECKSUM_MISMATCH, 'detail': msg}) print(f"ERROR: {msg}", file=sys.stderr) return EXIT_CHECKSUM_MISMATCH # Check if live DB has non-empty tables and --allow-overwrite is needed check_tables = tcfg.get('verify_tables_not_empty', []) if check_tables and not args.allow_overwrite: tbl = check_tables[0] cnt = 0 if is_postgres: r = psql_exec(tcfg, live_db, f'SELECT COUNT(*)::int FROM "{tbl}";') cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0 else: r = mariadb_exec(tcfg, live_db, f'SELECT COUNT(*) FROM `{tbl}`;') cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0 if cnt > 0: msg = f"Target database {live_db} contains {cnt} rows in table {tbl}; pass --allow-overwrite to replace it." write_result(args.result, {'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}) print(f"ERROR: {msg}", file=sys.stderr) return EXIT_PRECONDITION ts_label = dt.datetime.now().strftime('%Y%m%d_%H%M%S') staging_db = f"{live_db}_restore_{ts_label}" backup_db = f"{live_db}_before_{ts_label}" print(f"1. Creating staging database {staging_db} ({engine})...") if is_postgres: ok, err = postgres_create_db(tcfg, staging_db) else: ok, err = mariadb_create_db(tcfg, staging_db) if not ok: msg = f"Failed to create staging DB: {err}" write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) return EXIT_EXEC_ERROR print(f"2. Importing dump into {staging_db}...") if is_postgres: ok, err = postgres_import_dump(tcfg, staging_db, dump_path) else: ok, err = mariadb_import_dump(tcfg, staging_db, dump_path, live_db) if not ok: msg = f"Failed to import dump into staging DB: {err}" if is_postgres: postgres_drop_db_if_exists(tcfg, staging_db) else: mariadb_drop_db_if_exists(tcfg, staging_db) write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) return EXIT_EXEC_ERROR print("3. Validating staging database integrity...") table_counts = {} for tbl in tcfg.get('verify_tables_not_empty', []): if is_postgres: r = psql_exec(tcfg, staging_db, f'SELECT COUNT(*)::int FROM "{tbl}";') else: r = mariadb_exec(tcfg, staging_db, f'SELECT COUNT(*) FROM `{tbl}`;') cnt = int(r.stdout.strip()) if (r.returncode == 0 and r.stdout.strip().isdigit()) else 0 table_counts[tbl] = cnt if cnt == 0: msg = f"Verification failed: required table {tbl} is empty in staging DB" if is_postgres: postgres_drop_db_if_exists(tcfg, staging_db) else: mariadb_drop_db_if_exists(tcfg, staging_db) write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) return EXIT_EXEC_ERROR # Invalidate / truncate session tables if requested if args.sessions == 'clear': for tbl in tcfg.get('truncate_tables', ['sessions']): print(f"Truncating session table: {tbl}") if is_postgres: psql_exec(tcfg, staging_db, f'TRUNCATE TABLE "{tbl}" CASCADE;') else: mariadb_exec(tcfg, staging_db, f'TRUNCATE TABLE `{tbl}`;') services_to_stop = tcfg.get('services_to_stop', []) if services_to_stop: print(f"4. Stopping dependent services: {services_to_stop}...") compose_cmd(tcfg, 'stop', *services_to_stop) print(f"5. Performing atomic database swap ({live_db} -> {backup_db}, {staging_db} -> {live_db})...") if is_postgres: swap_ok, swap_err = postgres_swap_databases(tcfg, live_db, staging_db, backup_db) else: swap_ok, swap_err = mariadb_swap_databases(tcfg, live_db, staging_db, backup_db) if not swap_ok: msg = f"Database swap failed: {swap_err}" if services_to_stop: compose_cmd(tcfg, 'up', '-d', *services_to_stop) write_result(args.result, {'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': msg}) return EXIT_EXEC_ERROR if services_to_stop: print(f"6. Starting dependent services: {services_to_stop}...") compose_cmd(tcfg, 'up', '-d', *services_to_stop) # Healthcheck / Readiness verification health_url = tcfg.get('ready_url') or tcfg.get('health_url') if health_url: print(f"7. Verifying service readiness at {health_url}...") timeout_sec = tcfg.get('health_timeout_seconds', 180) ready_ok, ready_data = wait_ready(health_url, timeout_sec=timeout_sec) if not ready_ok: print(f"ERROR: Health check failed after {timeout_sec}s: {ready_data}. Initiating automatic rollback!", file=sys.stderr) if services_to_stop: compose_cmd(tcfg, 'stop', *services_to_stop) if is_postgres: postgres_rollback_swap(tcfg, live_db, staging_db, backup_db) else: mariadb_rollback_swap(tcfg, live_db, staging_db, backup_db) if services_to_stop: compose_cmd(tcfg, 'up', '-d', *services_to_stop) msg = f"Post-restore healthcheck failed: {ready_data}. Automatic rollback executed." write_result(args.result, {'status': 'error', 'exit_code': EXIT_POST_RESTORE_FAILED, 'detail': msg, 'health_data': ready_data}) return EXIT_POST_RESTORE_FAILED summary = { 'status': 'ok', 'exit_code': EXIT_OK, 'target': args.target, 'db_engine': engine, 'live_db': live_db, 'previous_db': backup_db, 'verified_tables': table_counts, 'sessions': args.sessions, 'completed_at': dt.datetime.now().isoformat(), } write_result(args.result, summary) print(f"OK: Restore completed successfully for target {args.target} (previous DB: {backup_db})") return EXIT_OK if __name__ == '__main__': sys.exit(main())