Files
backup-orchestrator/orchestrator.py
T

1600 lines
74 KiB
Python

#!/usr/bin/env python3
"""Phase-based backup and recovery orchestrator v0.4.
Safety defaults: read-only inspection unless --run, --restore, or --restore-test is
explicitly requested with all required safety gates and confirmations.
"""
import argparse
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
import csv
import datetime as dt
try:
import fcntl
except ImportError: # Windows local test compatibility
fcntl = None
import hashlib
import json
import os
from pathlib import Path
import platform
import re
import signal
import socket
import subprocess
import sys
import threading
import time
from urllib.parse import urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
VERSION = '0.4.0'
DEFAULT_CONFIG = Path('/etc/backup-orchestrator/jobs.csv')
DEFAULT_NOTIFY_CONFIG = Path('/etc/backup-orchestrator/notifications.json')
DEFAULT_TARGETS_CONFIG = Path('/etc/backup-orchestrator/targets.json')
DEFAULT_HOST_CONFIG = Path('/etc/backup-orchestrator/host.json')
DEFAULT_LOG_ROOT = Path('/var/log/backup-orchestrator')
DEFAULT_LOCK_FILE = Path('/run/lock/backup-orchestrator.lock')
FIELDS = {'phase', 'id', 'script', 'critical', 'timeout_seconds', 'enabled'}
CUTOVER_ENV = 'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED'
NAS_MOUNT = '/backup'
NAS_SOURCE = '192.168.2.63:/volume1/Backup_Paperless'
CANCELLED = threading.Event()
# Exit codes aligned with Shinkan RECOVERY_CONTRACT.md
EXIT_OK = 0
EXIT_EXEC_ERROR = 1
EXIT_PRECONDITION = 2
EXIT_LOCK_HELD = 3
EXIT_SCHEMA_INCOMPATIBLE = 3
EXIT_CHECKSUM_MISMATCH = 4
EXIT_POST_RESTORE_FAILED = 5
SECRET_PATTERNS = re.compile(
r'(?:PGPASSWORD|MYSQL_PWD|DB_PASSWORD|MARIADB_PASSWORD|MYSQL_PASSWORD|SMTP_PASS|OPENROUTER_API_KEY)\s*=\s*\S+',
re.IGNORECASE,
)
def now_iso():
return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds')
def lock_exclusive(fp):
if fcntl is not None:
fcntl.flock(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
def resolve_repo_host_dir(host_name=None):
try:
host = host_name or socket.gethostname().split('.')[0]
candidate = Path(__file__).resolve().parent / 'hosts' / host
return candidate if candidate.is_dir() else None
except (OSError, PermissionError):
return None
def load_host_config(path=DEFAULT_HOST_CONFIG, host_name=None):
default = {
'hostname': host_name or socket.gethostname(),
'nas_mount': NAS_MOUNT,
'nas_source': NAS_SOURCE,
'nas_fstype': 'nfs4',
'local_dump_dir': '/media/lars/paperless/db-backup',
'archive_sets_dir': '/backup/recovery_points',
'conflict_services': ['raspiBackup.service'],
'conflict_timers': ['paperless-data-backup.timer', 'borgmatic.timer'],
'cron_users': ['root', 'lars'],
}
candidates = []
if host_name:
host_dir = resolve_repo_host_dir(host_name)
if host_dir:
candidates.append(host_dir / 'host.json')
if path:
candidates.append(Path(path))
host_dir = resolve_repo_host_dir()
if host_dir:
candidates.append(host_dir / 'host.json')
for candidate in candidates:
try:
if candidate.is_file():
with candidate.open(encoding='utf-8') as fp:
data = json.load(fp)
if isinstance(data, dict):
default.update(data)
break
except (OSError, PermissionError):
continue
return default
def load_targets_config(path=DEFAULT_TARGETS_CONFIG, host_name=None):
candidates = []
if host_name:
host_dir = resolve_repo_host_dir(host_name)
if host_dir:
candidates.append(host_dir / 'targets.json')
if path:
candidates.append(Path(path))
host_dir = resolve_repo_host_dir()
if host_dir:
candidates.append(host_dir / 'targets.json')
candidates.append(Path(__file__).resolve().parent / 'targets.json')
for candidate in candidates:
try:
if candidate.is_file():
with candidate.open(encoding='utf-8') as fp:
return json.load(fp)
except (OSError, PermissionError):
continue
return {'schemaVersion': 1, 'hosts': {}, 'targets': {}}
def load_jobs(path):
with path.open(newline='', encoding='utf-8-sig') as fp:
reader = csv.DictReader(fp)
if reader.fieldnames is None or set(reader.fieldnames) != FIELDS or len(reader.fieldnames) != len(FIELDS):
raise ValueError('CSV header must contain exactly: ' + ','.join(sorted(FIELDS)))
rows = list(reader)
if not rows:
raise ValueError('CSV contains no jobs')
ids = set()
jobs = []
for index, row in enumerate(rows, start=2):
if None in row or any(v is None for v in row.values()):
raise ValueError(f'CSV line {index}: wrong number of columns')
row = {k: v.strip() for k, v in row.items()}
jid = row['id']
if not re.fullmatch(r'[a-z][a-z0-9-]*', jid) or jid in ids:
raise ValueError(f'CSV line {index}: invalid or duplicate job ID: {jid!r}')
ids.add(jid)
if row['critical'] not in ('true', 'false') or row['enabled'] not in ('true', 'false'):
raise ValueError(f'CSV line {index}: critical and enabled must be true or false')
if not re.fullmatch(r'\d+', row['phase']) or int(row['phase']) < 1:
raise ValueError(f'CSV line {index}: phase must be a positive integer')
if not re.fullmatch(r'\d+', row['timeout_seconds']) or not 1 <= int(row['timeout_seconds']) <= 86400:
raise ValueError(f'CSV line {index}: timeout_seconds must be 1..86400')
if not Path(row['script']).startswith_slash() if hasattr(Path(row['script']), 'startswith_slash') else not (row['script'].startswith('/') or Path(row['script']).is_absolute()):
raise ValueError(f'CSV line {index}: script must be an absolute path')
jobs.append({
'phase': int(row['phase']), 'id': jid, 'script': row['script'],
'critical': row['critical'] == 'true',
'timeout_seconds': int(row['timeout_seconds']),
'enabled': row['enabled'] == 'true',
})
return sorted(jobs, key=lambda x: (x['phase'], x['id']))
def push_base_url(value):
"""Accept Kuma's full copied URL and discard its example ?status=... query."""
parsed = urlsplit(value)
if parsed.scheme not in ('http', 'https') or not parsed.netloc or not parsed.path.startswith('/api/push/'):
raise ValueError('Kuma Push URL must be an http(s) URL with /api/push/<token>')
if parsed.username is not None or parsed.password is not None:
raise ValueError('HTTP basic-auth in Push URL is unsupported')
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, '', ''))
def verify_http_url(value):
parsed = urlsplit(value)
if parsed.scheme not in ('http', 'https') or not parsed.netloc:
raise ValueError('Webhook URL must start with http:// or https://')
if parsed.username is not None or parsed.password is not None:
raise ValueError('Webhook URL must not contain user credentials')
return value
def load_notifications(path, known_ids=None):
"""Only outgoing HTTP. URLs are never printed in status/logs."""
default = {'timeout_seconds': 3, 'webhooks': [], 'kuma': {'overall_push_url': '', 'job_push_urls': {}}}
if not path.is_file():
return default
with path.open(encoding='utf-8') as fp:
obj = json.load(fp)
if not isinstance(obj, dict) or set(obj) - {'timeout_seconds', 'webhooks', 'kuma'}:
raise ValueError('Invalid notifications.json root keys')
timeout = obj.get('timeout_seconds', 3)
if type(timeout) not in (int, float) or not 0.5 <= timeout <= 15:
raise ValueError('timeout_seconds must be 0.5..15')
webhooks = obj.get('webhooks', [])
kuma = obj.get('kuma', {})
if not isinstance(webhooks, list) or any(not isinstance(x, str) for x in webhooks):
raise ValueError('webhooks must be a list of URLs')
if not isinstance(kuma, dict) or set(kuma) - {'overall_push_url', 'job_push_urls'}:
raise ValueError('Invalid kuma settings')
overall = kuma.get('overall_push_url', '')
per_job = kuma.get('job_push_urls', {})
if not isinstance(overall, str) or not isinstance(per_job, dict):
raise ValueError('Kuma URLs must be strings; job_push_urls must be an object')
if any(not isinstance(k, str) or not isinstance(v, str) for k, v in per_job.items()):
raise ValueError('Kuma job IDs and URLs must be strings')
if known_ids is not None and set(per_job) - set(known_ids):
raise ValueError('Unknown Kuma job ID(s): ' + ', '.join(sorted(set(per_job) - set(known_ids))))
return {
'timeout_seconds': timeout,
'webhooks': [verify_http_url(x) for x in webhooks],
'kuma': {
'overall_push_url': push_base_url(overall) if overall else '',
'job_push_urls': {k: push_base_url(v) if v else '' for k, v in per_job.items()},
},
}
def notification_counts(config):
kuma = config['kuma']
return len(config['webhooks']), int(bool(kuma['overall_push_url'])), sum(bool(v) for v in kuma['job_push_urls'].values())
def nas_mount_ready(mountpoint=None, source=None, fstype=None, host_cfg=None):
if host_cfg:
mountpoint = mountpoint or host_cfg.get('nas_mount')
source = source or host_cfg.get('nas_source')
fstype = fstype or host_cfg.get('nas_fstype')
mountpoint = mountpoint or NAS_MOUNT
source = source or NAS_SOURCE
fstype = fstype or 'nfs4'
if not mountpoint or not source:
return True
try:
result = subprocess.run(
['findmnt', '--mountpoint', mountpoint, '--types', fstype, '--source', source],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=8,
)
return result.returncode == 0
except (OSError, subprocess.TimeoutExpired):
return False
def legacy_schedules(host_cfg=None):
"""Stateful detection of active legacy schedules and conflicting running services configured in host.json."""
cfg = host_cfg or load_host_config()
cron_users = cfg.get('cron_users', ['root', 'lars'])
cron_pattern = cfg.get('conflict_cron_pattern', r'/home/lars/(?:db-[\w-]+\.sh|backupPaperlessData\.sh)')
conflict_services = cfg.get('conflict_services', ['raspiBackup.service'])
conflict_timers = cfg.get('conflict_timers', ['paperless-data-backup.timer', 'borgmatic.timer'])
found = []
if cron_pattern:
for user in cron_users:
try:
p = subprocess.run(['crontab', '-u', user, '-l'], capture_output=True, text=True, timeout=8)
except (OSError, subprocess.TimeoutExpired):
continue
for line in p.stdout.splitlines():
stripped = line.strip()
if stripped.startswith('#') or not stripped:
continue
if re.search(cron_pattern, stripped):
found.append(f'{user} crontab: {stripped}')
# Stateful coordination with conflicting services (e.g. raspiBackup.service): only block if currently active/running.
for service in conflict_services:
try:
svc_active = subprocess.run(
['systemctl', 'is-active', service],
capture_output=True, text=True, timeout=8,
).stdout.strip()
if svc_active in ('active', 'activating', 'reloading'):
found.append(f'{service}: active={svc_active}')
except (OSError, subprocess.TimeoutExpired):
pass
for timer in conflict_timers:
try:
enabled = subprocess.run(['systemctl', 'is-enabled', timer], capture_output=True, text=True, timeout=8).stdout.strip()
active = subprocess.run(['systemctl', 'is-active', timer], capture_output=True, text=True, timeout=8).stdout.strip()
except (OSError, subprocess.TimeoutExpired):
continue
if enabled in ('enabled', 'enabled-runtime', 'linked', 'linked-runtime') or active == 'active':
found.append(f'{timer}: enabled={enabled or "unknown"}, active={active or "unknown"}')
return found
def script_issues(job):
p = Path(job['script'])
if not p.is_file():
return ['MISSING']
issues = []
if os.name != 'nt' and not os.access(p, os.X_OK):
issues.append('NOT EXECUTABLE')
try:
with p.open('rb') as fp:
if fp.read(2) != b'#!':
issues.append('NO SHEBANG')
except OSError:
issues.append('NOT READABLE')
return issues
def check(jobs, max_parallel, notify, host_cfg=None):
cfg = host_cfg or load_host_config()
mountpoint = cfg.get('nas_mount', NAS_MOUNT)
source = cfg.get('nas_source', NAS_SOURCE)
print(f'Backup-Orchestrator {VERSION} (host={cfg.get("hostname", socket.gethostname())}): INSPECTION ONLY (no backups, no schedule changes)')
print(f'Parallel limit per phase: {max_parallel}')
for job in jobs:
issues = script_issues(job)
state = 'enabled' if job['enabled'] else 'disabled'
print(f"phase {job['phase']:>2} {job['id']:<24} {state:<8} {job['timeout_seconds']:>5}s {job['script']} [{', '.join(issues) if issues else 'OK'}]")
conflicts = legacy_schedules(cfg)
print('\nLegacy schedules:')
for line in conflicts:
print(' ACTIVE: ' + line)
if not conflicts:
print(' None detected (best effort)')
print(f'\nNAS mount {mountpoint}: {"OK" if nas_mount_ready(host_cfg=cfg) else "NOT AVAILABLE"} (expected {source})')
a, b, c = notification_counts(notify)
print(f'Notifications configured: webhooks={a}, Kuma overall={b}, Kuma jobs={c} (tokens hidden)')
print('Same phase: parallel; next phase starts after all members complete.')
print('Critical job failure: later phases BLOCKED; in-flight phase members finish.')
print('No jobs execute without --run, enabled=true and explicit cutover approval.')
return 0
def select_jobs(jobs, args):
enabled = [j for j in jobs if j['enabled']]
if getattr(args, 'job', None):
matches = [j for j in enabled if j['id'] == args.job]
if not matches:
raise ValueError(f'Job {args.job!r} not found or disabled')
return matches
if getattr(args, 'phase', None) is not None:
matches = [j for j in enabled if j['phase'] == args.phase]
if not matches:
raise ValueError(f'No enabled jobs in phase {args.phase}')
return matches
return enabled
def plan(jobs, max_parallel):
groups = defaultdict(list)
for job in jobs:
groups[job['phase']].append(job)
print(f'MODE: PLAN ONLY — no jobs executed. Max parallel jobs per phase: {max_parallel}')
for phase, members in sorted(groups.items()):
print(f'Phase {phase} ({len(members)} job(s)):')
for job in members:
print(f" - {job['id']} ({'CRITICAL' if job['critical'] else 'optional'}, timeout {job['timeout_seconds']}s)")
print(' BARRIER: wait for all jobs before advancing')
def result_for(job, state, rc=None, seconds=0.0, logfile=None, reason=None):
result = dict(phase=job['phase'], job=job['id'], critical=job['critical'],
state=state, exit_code=rc, duration_seconds=round(seconds, 2))
if logfile is not None:
result['log'] = str(logfile)
if reason:
result['reason'] = reason
return result
def atomic_json(path, data):
path = Path(path)
tmp = path.with_name(f'.{path.name}.{os.getpid()}.{threading.get_ident()}.tmp')
try:
with tmp.open('w', encoding='utf-8') as fp:
json.dump(data, fp, ensure_ascii=False, indent=2)
fp.write('\n')
tmp.replace(path)
finally:
if tmp.exists():
tmp.unlink()
class Notifier:
"""Outbound webhooks + Kuma adapter; notification errors never alter backup exit codes."""
def __init__(self, config, folder):
self.config = config
self.folder = folder
self.pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix='backup-notify')
self.futures = []
self.lock = threading.Lock()
self.errors = 0
self.logfile = folder / 'notifications.log'
def _send_json(self, url, event):
payload = json.dumps(event, ensure_ascii=False).encode('utf-8')
req = Request(url, data=payload, method='POST', headers={'Content-Type': 'application/json', 'User-Agent': f'pi-backup-orchestrator/{VERSION}'})
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
if not 200 <= response.status < 300:
raise RuntimeError(f'HTTP {response.status}')
def _send_kuma(self, url, status, message):
sep = '&' if '?' in url else '?'
address = url + sep + urlencode({'status': status, 'msg': message[:180]})
req = Request(address, headers={'User-Agent': f'pi-backup-orchestrator/{VERSION}'})
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
if not 200 <= response.status < 300:
raise RuntimeError(f'HTTP {response.status}')
body = response.read(4096)
if body:
try:
parsed = json.loads(body)
except (ValueError, UnicodeDecodeError):
parsed = None
if isinstance(parsed, dict) and parsed.get('ok') is False:
raise RuntimeError('Kuma rejected push')
def _submit(self, channel, fn, *args):
def guarded():
try:
fn(*args)
except Exception as exc:
if hasattr(exc, 'code') and isinstance(exc.code, int):
detail = f'HTTP {exc.code}'
else:
detail = type(exc).__name__
with self.lock:
self.errors += 1
with self.logfile.open('a', encoding='utf-8') as out:
out.write(f'{now_iso()} {channel}: delivery failed ({detail})\n')
self.futures.append(self.pool.submit(guarded))
def notify(self, event):
for url in self.config['webhooks']:
self._submit('webhook', self._send_json, url, event)
name = event['event']
if name in ('JOB_SUCCESS', 'JOB_FAILED', 'JOB_TIMEOUT', 'JOB_CANCELLED', 'JOB_ERROR', 'JOB_BLOCKED'):
url = self.config['kuma']['job_push_urls'].get(event.get('job'), '')
if url:
status = 'up' if name == 'JOB_SUCCESS' else 'down'
msg = f"{event['job']}: {event.get('state', 'unknown')} (run {event['run_id']})"
self._submit('kuma/job', self._send_kuma, url, status, msg)
if name in ('RUN_FINISHED', 'RECOVERY_FINISHED'):
url = self.config['kuma']['overall_push_url']
if url:
status = 'up' if event.get('overall') == 'ok' else 'down'
self._submit('kuma/overall', self._send_kuma, url, status, f"Run {event['run_id']}: {event.get('overall')}")
def close(self):
self.pool.shutdown(wait=True)
return self.errors
class EventHub:
"""Write append-only events.jsonl plus atomic current snapshot for consumers."""
def __init__(self, folder, log_root, run_id, jobs, notifier, started):
self.folder = folder
self.log_root = log_root
self.run_id = run_id
self.notifier = notifier
self.lock = threading.RLock()
self.snapshot = {
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
'started': started, 'updated': started, 'finished': None,
'overall': 'running',
'jobs': {j['id']: result_for(j, 'pending') for j in jobs},
}
atomic_json(folder / 'status.json', self.snapshot)
atomic_json(log_root / 'latest.json', self.snapshot)
def emit(self, event_name, **fields):
event = {
'event': event_name, 'timestamp': now_iso(), 'run_id': self.run_id,
'host': self.snapshot['host'], **fields,
}
with self.lock:
if 'job' in event and event['job'] in self.snapshot['jobs'] and 'state' in event:
jid = event['job']
self.snapshot['jobs'][jid].update({k: v for k, v in fields.items() if k in (
'state', 'phase', 'critical', 'exit_code', 'duration_seconds', 'log', 'reason')})
if event_name == 'RUN_FINISHED':
self.snapshot['overall'] = fields['overall']
self.snapshot['finished'] = event['timestamp']
self.snapshot['updated'] = event['timestamp']
with (self.folder / 'events.jsonl').open('a', encoding='utf-8') as fp:
fp.write(json.dumps(event, ensure_ascii=False) + '\n')
atomic_json(self.folder / 'status.json', self.snapshot)
atomic_json(self.log_root / 'latest.json', self.snapshot)
self.notifier.notify(event)
return event
def terminate_group(proc):
"""Terminate ordinary children of a job and wait for shutdown before advancing phase."""
if proc.poll() is not None:
return
if hasattr(os, 'killpg'):
try:
os.killpg(proc.pid, signal.SIGTERM)
except ProcessLookupError:
return
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(proc.pid, signal.SIGKILL)
except ProcessLookupError:
pass
proc.wait()
else:
subprocess.run(['taskkill', '/F', '/T', '/PID', str(proc.pid)],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=5)
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
def _build_script_cmd(script_path, extra_args=()):
if os.name == 'nt':
import shutil
p = Path(script_path)
first_line = ''
try:
with p.open('r', encoding='utf-8', errors='replace') as fp:
first_line = fp.readline()
except OSError:
pass
if 'python' in first_line:
return [sys.executable, str(p), *extra_args]
if first_line.startswith('#!') or p.suffix == '.sh':
sh_bin = shutil.which('sh') or r'C:\Program Files\Git\bin\sh.exe'
return [sh_bin, str(p), *extra_args]
return [str(script_path), *extra_args]
def execute_one(job, folder, hub):
start = time.monotonic()
logfile = folder / (job['id'] + '.log')
if CANCELLED.is_set():
result = result_for(job, 'cancelled', reason='Orchestrator interrupted')
hub.emit('JOB_CANCELLED', **result)
return result
hub.emit('JOB_STARTED', job=job['id'], phase=job['phase'], critical=job['critical'], state='running')
with logfile.open('wb') as stream:
stream.write((f"job={job['id']} started={now_iso()}\n").encode())
stream.flush()
try:
proc = subprocess.Popen(_build_script_cmd(job['script']), stdout=stream, stderr=subprocess.STDOUT,
start_new_session=True, cwd=str(Path(job['script']).parent))
except OSError as exc:
stream.write((f'EXEC ERROR: {exc}\n').encode())
result = result_for(job, 'error', 127, time.monotonic()-start, logfile, 'Could not start script')
hub.emit('JOB_ERROR', **result)
return result
deadline = time.monotonic() + job['timeout_seconds']
while True:
if CANCELLED.is_set():
terminate_group(proc)
state, code = 'cancelled', 130
break
if time.monotonic() >= deadline:
terminate_group(proc)
state, code = 'timeout', 124
break
try:
code = proc.wait(timeout=min(0.5, max(0.01, deadline-time.monotonic())))
state = 'ok' if code == 0 else 'failed'
break
except subprocess.TimeoutExpired:
pass
stream.write((f'\njob={job["id"]} finished={now_iso()} status={state} exit_code={code}\n').encode())
result = result_for(job, state, code, time.monotonic()-start, logfile)
event = {'ok': 'JOB_SUCCESS', 'failed': 'JOB_FAILED', 'timeout': 'JOB_TIMEOUT', 'cancelled': 'JOB_CANCELLED'}[state]
hub.emit(event, **result)
return result
def live_run(jobs, args, host_cfg=None):
if not jobs:
print('Nothing enabled; no backups executed.')
return 0
if os.environ.get(CUTOVER_ENV) != 'YES':
print(f'REFUSED: Set {CUTOVER_ENV}=YES only after testing scripts and coordinating existing schedules.', file=sys.stderr)
return 2
cfg = host_cfg or load_host_config(host_name=getattr(args, 'host', None))
mountpoint = cfg.get('nas_mount', NAS_MOUNT)
source = cfg.get('nas_source', NAS_SOURCE)
if not nas_mount_ready(host_cfg=cfg):
print(f'REFUSED: {mountpoint} is not mounted from {source}', file=sys.stderr)
return 2
conflicts = legacy_schedules() if host_cfg is None else legacy_schedules(cfg)
if conflicts:
print('REFUSED: existing scheduled backups could overlap this run:', file=sys.stderr)
for issue in conflicts:
print(' ' + issue, file=sys.stderr)
return 2
unsafe = [(j['id'], script_issues(j)) for j in jobs if script_issues(j)]
if unsafe:
for jid, issues in unsafe:
print(f'REFUSED: {jid}: {", ".join(issues)}', file=sys.stderr)
return 2
notifications = load_notifications(args.notify_config)
args.log_root.mkdir(parents=True, exist_ok=True, mode=0o700)
args.lock_file.parent.mkdir(parents=True, exist_ok=True)
with args.lock_file.open('a+') as lock:
try:
lock_exclusive(lock)
except BlockingIOError:
print('REFUSED: another orchestrator run holds the lock', file=sys.stderr)
return 3
run_id = dt.datetime.now().strftime('%Y%m%d-%H%M%S') + f'-{os.getpid()}'
folder = args.log_root / run_id
folder.mkdir(mode=0o700)
old_int = signal.getsignal(signal.SIGINT)
old_term = signal.getsignal(signal.SIGTERM)
CANCELLED.clear()
signal.signal(signal.SIGINT, lambda signum, frame: CANCELLED.set())
signal.signal(signal.SIGTERM, lambda signum, frame: CANCELLED.set())
results = []
critical_failed = False
groups = defaultdict(list)
for job in jobs:
groups[job['phase']].append(job)
began = now_iso()
notifier = Notifier(notifications, folder)
hub = EventHub(folder, args.log_root, run_id, jobs, notifier, began)
try:
hub.emit('RUN_STARTED', max_parallel=args.max_parallel, job_count=len(jobs))
for phase, members in sorted(groups.items()):
if critical_failed or CANCELLED.is_set():
reason = 'earlier critical failure' if critical_failed else 'orchestrator interrupted'
hub.emit('PHASE_BLOCKED', phase=phase, reason=reason)
for job in members:
result = result_for(job, 'blocked', reason=reason)
results.append(result)
hub.emit('JOB_BLOCKED', **result)
print(f'Phase {phase} BLOCKED: {reason}', flush=True)
continue
hub.emit('PHASE_STARTED', phase=phase, job_count=len(members))
print(f'Phase {phase}: starting {len(members)} job(s), max parallel {args.max_parallel}', flush=True)
phase_results = []
with ThreadPoolExecutor(max_workers=args.max_parallel) as pool:
future_to_job = {pool.submit(execute_one, job, folder, hub): job for job in members}
for future in as_completed(future_to_job):
job = future_to_job[future]
try:
item = future.result()
except Exception as exc:
item = result_for(job, 'error', 127, reason=f'Internal runner error: {type(exc).__name__}')
hub.emit('JOB_ERROR', **item)
phase_results.append(item)
print(f' {item["job"]}: {item["state"]} ({item["exit_code"]})', flush=True)
results.extend(sorted(phase_results, key=lambda r: r['job']))
if any(r['critical'] and r['state'] != 'ok' for r in phase_results):
critical_failed = True
hub.emit('PHASE_COMPLETED', phase=phase, failures=sum(r['state'] != 'ok' for r in phase_results),
critical_failed=critical_failed)
if critical_failed:
print(f'Phase {phase}: critical failure; subsequent phases will not start.', flush=True)
else:
print(f'Phase {phase}: barrier reached; next phase may start.', flush=True)
finally:
signal.signal(signal.SIGINT, old_int)
signal.signal(signal.SIGTERM, old_term)
overall = 'interrupted' if CANCELLED.is_set() else 'ok' if all(r['state'] == 'ok' for r in results) else 'failed'
summary = {
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
'started': began, 'finished': now_iso(), 'max_parallel': args.max_parallel,
'interrupted': CANCELLED.is_set(), 'results': results, 'overall': overall,
}
atomic_json(folder / 'summary.json', summary)
hub.emit('RUN_FINISHED', overall=overall, success_count=sum(r['state'] == 'ok' for r in results),
failure_count=sum(r['state'] != 'ok' for r in results))
delivery_failures = notifier.close()
if delivery_failures:
print(f'WARNING: {delivery_failures} notification delivery failure(s); see {folder / "notifications.log"}', flush=True)
print('Summary: ' + str(folder / 'summary.json'), flush=True)
return 130 if CANCELLED.is_set() else (1 if any(r['state'] != 'ok' for r in results) else 0)
def show_status(log_root, as_json=False):
latest = log_root / 'latest.json'
if latest.is_file():
with latest.open(encoding='utf-8') as fp:
data = json.load(fp)
if as_json:
print(json.dumps(data, ensure_ascii=False, indent=2))
return 0
print(f"Run {data['run_id']}: {data['overall']} ({data['started']} -> {data['finished'] or 'running'})")
for item in sorted(data['jobs'].values(), key=lambda x: (x['phase'], x['job'])):
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
return 0
summaries = sorted(log_root.glob('*/summary.json')) if log_root.is_dir() else []
if not summaries:
if as_json:
print(json.dumps({'overall': 'no_runs', 'jobs': {}}, indent=2))
else:
print('No runs recorded yet')
return 0
with summaries[-1].open(encoding='utf-8') as fp:
data = json.load(fp)
if as_json:
print(json.dumps(data, ensure_ascii=False, indent=2))
else:
print(f"Last run: {data['started']} -> {data['finished']} ; overall={data['overall']}")
for item in data['results']:
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
return 0
# ============================================================================
# Recovery Management v0.4 (Recovery Points, Manifest v1, Compatibility, Plan)
# ============================================================================
def compute_sha256(path):
h = hashlib.sha256()
with Path(path).open('rb') as fp:
while True:
chunk = fp.read(65536)
if not chunk:
break
h.update(chunk)
return h.hexdigest()
def read_sha256_sidecar(sha_path):
text = Path(sha_path).read_text(encoding='utf-8').strip()
if not text:
raise ValueError(f'Empty sha256 file: {sha_path}')
digest = text.split()[0].lower()
if not re.fullmatch(r'[0-9a-f]{64}', digest):
raise ValueError(f'Invalid sha256 digest in {sha_path}: {digest!r}')
return digest
def detect_dump_format(path):
with Path(path).open('rb') as fp:
head = fp.read(5)
if head == b'PGDMP':
return 'custom'
return 'plain'
def extract_migrations_from_sql_dump(path):
"""Extract schema_migrations entries from a plain PostgreSQL dump."""
migrations = []
in_copy = False
with Path(path).open('r', encoding='utf-8', errors='replace') as fp:
for line in fp:
if not in_copy:
if line.startswith('COPY public.schema_migrations ') or line.startswith('COPY schema_migrations '):
in_copy = True
continue
m = re.search(r"INSERT INTO (?:public\.)?schema_migrations\b.*?VALUES\s*\((?:\d+\s*,\s*)?'([^']+)'", line)
if m:
migrations.append(m.group(1))
else:
if line.startswith('\\.'):
in_copy = False
continue
parts = line.rstrip('\r\n').split('\t')
if len(parts) >= 2 and re.match(r'^\d+_', parts[1]):
migrations.append(parts[1])
elif len(parts) >= 1 and re.match(r'^\d+_', parts[0]):
migrations.append(parts[0])
return sorted(set(migrations), key=_migration_sort_key)
def _migration_sort_key(stem):
m = re.match(r'^(\d+)', str(stem))
return (int(m.group(1)) if m else 0, str(stem))
def discover_code_migrations(migrations_dir):
migrations_dir = Path(migrations_dir)
if not migrations_dir.is_dir():
return []
stems = []
for item in migrations_dir.iterdir():
if item.is_file() and item.name.endswith('.sql') and item.name[0].isdigit():
stems.append(item.name[:-4])
return sorted(stems, key=_migration_sort_key)
def normalize_manifest(raw, manifest_path=None):
"""Validate and normalize a Recovery Point manifest (schemaVersion 1 or Shinkan contract)."""
if not isinstance(raw, dict):
raise ValueError('Manifest must be a JSON object')
serialized = json.dumps(raw, ensure_ascii=False)
if SECRET_PATTERNS.search(serialized):
raise ValueError('Manifest contains forbidden plaintext secret patterns')
schema_ver = raw.get('schemaVersion', 1)
if schema_ver != 1:
raise ValueError(f'Unsupported manifest schemaVersion: {schema_ver!r}')
app_block = raw.get('application', {}) if isinstance(raw.get('application'), dict) else {}
rp_block = raw.get('recoveryPoint', {}) if isinstance(raw.get('recoveryPoint'), dict) else {}
rel_block = raw.get('release', {}) if isinstance(raw.get('release'), dict) else {}
db_block = raw.get('database', {}) if isinstance(raw.get('database'), dict) else {}
media_block = raw.get('media', {}) if isinstance(raw.get('media'), dict) else {}
created_at = raw.get('created_at') or rp_block.get('createdAt')
host = raw.get('host') or app_block.get('sourceHost')
target = raw.get('target') or app_block.get('target') or 'unknown'
app_id = app_block.get('id') or (target.split('-')[0] if '-' in target else 'shinkan')
environment = app_block.get('environment') or ('production' if 'prod' in target else 'development')
app_version = raw.get('app_version') or rel_block.get('appVersion') or 'unknown'
git_commit = raw.get('git_commit') or rel_block.get('gitCommit') or 'unknown'
migrations = raw.get('migrations') if isinstance(raw.get('migrations'), list) else db_block.get('migrations', [])
if not isinstance(migrations, list) or any(not isinstance(m, str) for m in migrations):
raise ValueError('migrations must be a list of strings')
migrations = sorted(migrations, key=_migration_sort_key)
last_migration = raw.get('last_migration') or db_block.get('lastMigration') or (migrations[-1] if migrations else None)
dump_format = raw.get('dump_format') or db_block.get('dumpFormat') or 'plain'
if dump_format not in ('plain', 'custom'):
raise ValueError(f'Invalid dump_format: {dump_format!r}')
dump_bytes = raw.get('dump_bytes') if 'dump_bytes' in raw else db_block.get('bytes')
dump_sha256 = (raw.get('dump_sha256') or db_block.get('sha256') or '').lower()
if not re.fullmatch(r'[0-9a-f]{64}', dump_sha256):
raise ValueError('Manifest requires a valid 64-char hex dump_sha256 / database.sha256')
if not created_at or not host:
raise ValueError('Manifest requires created_at/createdAt and host/sourceHost')
artifact_name = db_block.get('artifact') or raw.get('dump_file')
if not artifact_name and manifest_path is not None:
mname = Path(manifest_path).name
if mname.endswith('.manifest.json'):
artifact_name = mname[:-len('.manifest.json')]
media_manifest = raw.get('media_manifest') if isinstance(raw.get('media_manifest'), dict) else {}
if not media_manifest and media_block.get('manifestArtifact'):
media_manifest = {
'file': media_block.get('manifestArtifact'),
'sha256': media_block.get('manifestSha256', ''),
'file_count': media_block.get('fileCount', 0),
}
consistency = rp_block.get('consistency') if isinstance(rp_block.get('consistency'), dict) else {
'mode': 'hot_indexed' if media_manifest else 'hot_separate',
'dbTransactional': True,
'mediaSynchronized': False,
'limitations': [
'Online database dump in single transaction; media files indexed without write freeze.'
if media_manifest else
'Database dump only; media/filesystem snapshot is not bound via media manifest.'
],
}
point_id = rp_block.get('id') or f"{app_id}-{target}-{re.sub(r'[^0-9]', '', str(created_at))[:14]}-{dump_sha256[:8]}"
normalized = {
'schemaVersion': 1,
'created_at': created_at,
'host': host,
'target': target,
'app_version': app_version,
'git_commit': git_commit,
'migrations': migrations,
'last_migration': last_migration,
'dump_format': dump_format,
'dump_bytes': dump_bytes,
'dump_sha256': dump_sha256,
'row_counts': raw.get('row_counts') or db_block.get('rowCounts') or {},
'media_manifest': media_manifest,
'application': {
'id': app_id,
'environment': environment,
'target': target,
'sourceHost': host,
'sourceArch': app_block.get('sourceArch', 'unknown'),
},
'recoveryPoint': {
'id': point_id,
'createdAt': created_at,
'status': rp_block.get('status', 'captured'),
'validatedAt': rp_block.get('validatedAt'),
'validationReport': rp_block.get('validationReport'),
'consistency': consistency,
'retention': rp_block.get('retention', {'tier': 'daily', 'protected': False}),
},
'release': {
'repository': rel_block.get('repository', ''),
'appVersion': app_version,
'gitCommit': git_commit,
'imageDigests': rel_block.get('imageDigests', {}),
},
'database': {
'engine': db_block.get('engine', 'postgresql'),
'engineVersion': db_block.get('engineVersion', ''),
'dumpFormat': dump_format,
'artifact': artifact_name or '',
'bytes': dump_bytes,
'sha256': dump_sha256,
'schemaVersion': db_block.get('schemaVersion', last_migration or ''),
'lastMigration': last_migration,
'migrations': migrations,
'rowCounts': raw.get('row_counts') or db_block.get('rowCounts') or {},
},
'media': {
'storageType': media_block.get('storageType', 'nfs4_shared'),
'containerMount': media_block.get('containerMount', '/app/media'),
'manifestArtifact': media_manifest.get('file', ''),
'manifestSha256': media_manifest.get('sha256', ''),
'fileCount': media_manifest.get('file_count', media_block.get('fileCount', 0)),
},
'configuration': raw.get('configuration', {
'composeFiles': ['docker-compose.yml'],
'secretReferences': ['DB_PASSWORD'],
}),
'dependencies': raw.get('dependencies', {'services': ['postgres', 'backend', 'frontend']}),
'recovery': raw.get('recovery', {
'adapter': app_id,
'supportedTargets': ['llm-prod', 'llm-restoretest'],
}),
}
return normalized
def verify_recovery_point(point_ref, check_hashes=True):
"""Inspect a dump or manifest path and evaluate its 4-stage recovery point status."""
path = Path(point_ref)
if path.is_dir():
manifests = sorted(path.glob('*.manifest.json'))
if manifests:
path = manifests[-1]
else:
dumps = sorted(list(path.glob('*.sql')) + list(path.glob('*.dump')))
if not dumps:
raise ValueError(f'No recovery point artifacts found in directory: {point_ref}')
path = dumps[-1]
if path.name.endswith('.manifest.json'):
manifest_path = path
base_name = path.name[:-len('.manifest.json')]
dump_path = path.with_name(base_name)
else:
dump_path = path
manifest_path = Path(str(path) + '.manifest.json')
sha_path = Path(str(dump_path) + '.sha256')
media_path = Path(str(dump_path) + '.media.jsonl')
issues = []
status = 'invalid'
manifest = None
if not dump_path.is_file() or dump_path.stat().st_size == 0:
issues.append(f'Dump file missing or empty: {dump_path}')
return {
'valid': False, 'status': 'invalid', 'exit_code': EXIT_PRECONDITION,
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
'issues': issues, 'manifest': None,
}
status = 'captured'
actual_bytes = dump_path.stat().st_size
actual_format = detect_dump_format(dump_path)
actual_sha = compute_sha256(dump_path) if check_hashes else None
sidecar_sha = None
if sha_path.is_file():
try:
sidecar_sha = read_sha256_sidecar(sha_path)
except ValueError as exc:
issues.append(str(exc))
return {
'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH,
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
'issues': issues, 'manifest': None,
}
if check_hashes and sidecar_sha != actual_sha:
issues.append(f'SHA256 mismatch against {sha_path.name}: expected {sidecar_sha}, got {actual_sha}')
return {
'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH,
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
'issues': issues, 'manifest': None,
}
else:
issues.append(f'Missing checksum file: {sha_path.name}')
if manifest_path.is_file():
try:
raw = json.loads(manifest_path.read_text(encoding='utf-8'))
manifest = normalize_manifest(raw, manifest_path=manifest_path)
except (ValueError, json.JSONDecodeError) as exc:
issues.append(f'Invalid manifest {manifest_path.name}: {exc}')
return {
'valid': False, 'status': 'captured', 'exit_code': EXIT_PRECONDITION,
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
'issues': issues, 'manifest': None,
}
if check_hashes and manifest['dump_sha256'] != actual_sha:
issues.append(
f"Manifest dump_sha256 mismatch: manifest={manifest['dump_sha256']}, actual={actual_sha}"
)
return {
'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH,
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
'issues': issues, 'manifest': manifest,
}
if manifest['dump_bytes'] is not None and int(manifest['dump_bytes']) != actual_bytes:
issues.append(f"Manifest dump_bytes ({manifest['dump_bytes']}) != actual size ({actual_bytes})")
if sidecar_sha and manifest['dump_sha256'] == (actual_sha or sidecar_sha) and not issues:
status = 'verified'
# Check whether all requirements for a 'consistent' Recovery Point are met
has_release = manifest['app_version'] not in ('', 'unknown') and manifest['git_commit'] not in ('', 'unknown')
has_schema = bool(manifest['migrations'] or manifest['last_migration'])
media_info = manifest.get('media_manifest') or {}
media_ok = True
if media_info.get('file'):
mfile = dump_path.parent / media_info['file']
if not mfile.is_file():
media_ok = False
issues.append(f"Referenced media manifest missing: {media_info['file']}")
elif check_hashes and media_info.get('sha256'):
msha = compute_sha256(mfile)
if msha != media_info['sha256'].lower():
media_ok = False
issues.append(f"Media manifest sha256 mismatch for {media_info['file']}")
return {
'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH,
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
'issues': issues, 'manifest': manifest,
}
if has_release and has_schema and media_ok:
status = 'consistent'
if manifest['recoveryPoint'].get('validatedAt') or manifest['recoveryPoint'].get('status') == 'restore_tested':
status = 'restore_tested'
manifest['recoveryPoint']['status'] = status
else:
issues.append(f'Missing manifest file: {manifest_path.name} (legacy dump only)')
valid = status in ('verified', 'consistent', 'restore_tested') and not issues
return {
'valid': valid,
'status': status,
'exit_code': EXIT_OK if valid else EXIT_PRECONDITION,
'dump_path': str(dump_path),
'sha256_path': str(sha_path) if sha_path.is_file() else None,
'manifest_path': str(manifest_path) if manifest_path.is_file() else None,
'media_manifest_path': str(media_path) if media_path.is_file() else None,
'dump_format': actual_format,
'dump_bytes': actual_bytes,
'dump_sha256': actual_sha or sidecar_sha or (manifest['dump_sha256'] if manifest else None),
'issues': issues,
'manifest': manifest,
}
def check_compatibility(verification, target_name, targets_cfg, current_host=None, current_arch=None, code_migrations=None):
"""Evaluate version, schema, host, and media compatibility between a Recovery Point and a target."""
targets = targets_cfg.get('targets', {})
if target_name not in targets:
raise ValueError(f'Unknown recovery target: {target_name!r}')
tcfg = targets[target_name]
current_host = current_host or socket.gethostname()
current_arch = current_arch or platform.machine()
reasons = []
warnings = []
exit_code = EXIT_OK
if not tcfg.get('allow_restore', False):
reasons.append(f"Target {target_name!r} is read-only / source_only; restore is prohibited.")
exit_code = EXIT_PRECONDITION
expected_host = tcfg.get('host')
expected_arch = tcfg.get('arch')
if expected_host and current_host != expected_host:
reasons.append(f"Host mismatch: running on {current_host!r}, target {target_name!r} requires {expected_host!r}")
exit_code = EXIT_PRECONDITION
if expected_arch and current_arch != expected_arch:
reasons.append(f"Architecture mismatch: running on {current_arch!r}, target {target_name!r} requires {expected_arch!r}")
exit_code = EXIT_PRECONDITION
if not verification.get('valid'):
for issue in verification.get('issues', []):
reasons.append(issue)
if verification.get('exit_code') == EXIT_CHECKSUM_MISMATCH:
exit_code = EXIT_CHECKSUM_MISMATCH
elif exit_code == EXIT_OK:
exit_code = EXIT_PRECONDITION
manifest = verification.get('manifest')
dump_migrations = []
if manifest and manifest.get('migrations'):
dump_migrations = list(manifest['migrations'])
elif verification.get('dump_path') and verification.get('dump_format') == 'plain':
dump_migrations = extract_migrations_from_sql_dump(verification['dump_path'])
if code_migrations is None:
app_dir = Path(tcfg.get('app_dir', tcfg.get('compose_dir', '')))
mig_rel = tcfg.get('migrations_dir', 'backend/migrations')
mig_dir = app_dir / mig_rel if app_dir and str(app_dir) != '.' else Path('')
if mig_dir and mig_dir.is_dir():
code_migrations = discover_code_migrations(mig_dir)
schema_relation = 'UNKNOWN'
unknown_migrations = []
pending_migrations = []
if code_migrations is not None and dump_migrations:
code_set = set(code_migrations)
dump_set = set(dump_migrations)
unknown_migrations = sorted(dump_set - code_set, key=_migration_sort_key)
pending_migrations = sorted(code_set - dump_set, key=_migration_sort_key)
if unknown_migrations:
schema_relation = 'INCOMPATIBLE_ROLLBACK'
reasons.append(
'Database dump contains migration(s) unknown to target code (unsafe rollback): '
+ ', '.join(unknown_migrations)
)
exit_code = EXIT_SCHEMA_INCOMPATIBLE
elif pending_migrations:
schema_relation = 'FORWARD_MIGRATE'
warnings.append(
f"{len(pending_migrations)} newer migration(s) in target code will be applied on container start: "
+ ', '.join(pending_migrations[:10])
)
else:
schema_relation = 'EXACT_MATCH'
elif code_migrations is None:
warnings.append('Target code migrations directory not inspected locally; staging-DB check will verify on host.')
if manifest:
limitations = manifest.get('recoveryPoint', {}).get('consistency', {}).get('limitations', [])
warnings.extend(limitations)
if manifest.get('application', {}).get('sourceArch') not in ('', 'unknown', expected_arch):
warnings.append(
f"Cross-architecture recovery: dump captured on {manifest['application']['sourceArch']} "
f"({manifest.get('host')}), restoring to {expected_arch} ({expected_host}). "
"Logical SQL dump is portable across architectures."
)
compatible = len(reasons) == 0 and exit_code == EXIT_OK
return {
'compatible': compatible,
'exit_code': exit_code,
'target': target_name,
'schema_relation': schema_relation,
'dump_last_migration': dump_migrations[-1] if dump_migrations else (manifest.get('last_migration') if manifest else None),
'code_last_migration': code_migrations[-1] if code_migrations else None,
'unknown_migrations': unknown_migrations,
'pending_migrations': pending_migrations,
'reasons': reasons,
'warnings': warnings,
}
def generate_recovery_plan(point_ref, target_name, targets_cfg, sessions_mode='clear',
current_host=None, current_arch=None, code_migrations=None):
"""Build a complete non-destructive recovery plan (Dry-Run) for a Recovery Point."""
verification = verify_recovery_point(point_ref, check_hashes=True)
compat = check_compatibility(
verification, target_name, targets_cfg,
current_host=current_host, current_arch=current_arch, code_migrations=code_migrations,
)
tcfg = targets_cfg['targets'][target_name]
manifest = verification.get('manifest') or {}
is_test = tcfg.get('role') == 'isolated_test_target'
steps = []
if is_test:
steps = [
'1. Verify host/arch guard (llm-node / x86_64) and validate dump SHA256 checksum.',
'2. Start isolated disposable Compose project shinkan-restoretest (127.0.0.1 ports only, media mounted :ro, SMTP/retention disabled).',
'3. Import dump in single transaction into staging database shinkan_restore_<ts>.',
'4. Verify all schema_migrations entries exist in deployed backend/migrations/*.sql.',
'5. Check required core tables and non-empty profiles/skills in staging database.',
f'6. Handle user sessions (mode={sessions_mode}) and swap staging database into shinkan-restoretest.',
'7. Start isolated backend/frontend and verify /api/version and /api/health/ready (status=ready, database=true, schema_complete=true).',
'8. Run read-only shinkan-media-diff against shared NAS media directory.',
'9. Tear down isolated project (docker compose -p shinkan-restoretest down -v) unless --keep is set.',
]
else:
steps = [
f"1. Verify host/arch guard ({tcfg.get('host')}/{tcfg.get('arch')}), configuration ({tcfg.get('env_file')}), and SHA256 checksum.",
f"2. Create staging database shinkan_restore_<ts> in container {tcfg.get('db_container')} and import dump in a single transaction.",
f"3. Validate every migration in staging schema_migrations against {tcfg.get('app_dir')}/backend/migrations/*.sql (abort with Exit 3 if dump is newer than code).",
'4. Run plausibility checks on staging database (required tables exist, profiles and skills non-empty).',
f'5. Apply session policy in staging database (sessions={sessions_mode}).',
f"6. Stop services {', '.join(tcfg.get('services_to_stop', ['backend', 'frontend']))} while keeping postgres running.",
f"7. Rename live database {tcfg.get('db_name')} -> {tcfg.get('db_name')}_before_<ts> and rename shinkan_restore_<ts> -> {tcfg.get('db_name')}.",
'8. Start backend and frontend (plus retention-cron only if enabled) and verify /api/health/ready.',
f"9. Automatic rollback strategy: if step 7 or 8 fails, stop services, restore {tcfg.get('db_name')}_before_<ts> -> {tcfg.get('db_name')}, restart services, and exit with code 5.",
'10. Run shinkan-media-diff to report missing, mismatched, added_after_backup, or orphan media files.',
]
plan_doc = {
'mode': 'DRY_RUN_RECOVERY_PLAN',
'generated_at': now_iso(),
'target': {
'name': target_name,
'application': tcfg.get('application'),
'environment': tcfg.get('environment'),
'host': tcfg.get('host'),
'arch': tcfg.get('arch'),
'role': tcfg.get('role'),
'app_dir': tcfg.get('app_dir'),
'compose_project': tcfg.get('compose_project'),
'env_file': tcfg.get('env_file'),
'db_container': tcfg.get('db_container'),
'db_name': tcfg.get('db_name'),
'media_host_dir': tcfg.get('media_host_dir'),
'require_confirm': tcfg.get('require_confirm'),
},
'recovery_point': {
'id': manifest.get('recoveryPoint', {}).get('id') or Path(verification['dump_path']).name,
'status': verification['status'],
'created_at': manifest.get('created_at'),
'source_host': manifest.get('host'),
'app_version': manifest.get('app_version'),
'git_commit': manifest.get('git_commit'),
'last_migration': compat['dump_last_migration'],
'dump_path': verification['dump_path'],
'dump_format': verification.get('dump_format'),
'dump_bytes': verification.get('dump_bytes'),
'dump_sha256': verification.get('dump_sha256'),
'media_manifest': manifest.get('media_manifest'),
'consistency': manifest.get('recoveryPoint', {}).get('consistency'),
},
'compatibility': compat,
'parameters': {
'sessions': sessions_mode,
'services_to_stop': tcfg.get('services_to_stop', []),
},
'impacts_and_risks': [
f"Services {', '.join(tcfg.get('services_to_stop', []))} will experience brief downtime during database swap."
if not is_test else 'Isolated test project only; zero impact on production containers or media.',
'Database changes written after the recovery point timestamp will be replaced (previous DB preserved as shinkan_before_<ts>).'
if not is_test else 'Disposable test database will be removed after verification.',
'Shared NAS media files are never deleted during restore; shinkan-media-diff will list any files added after the backup.',
],
'rollback_strategy': (
f"Previous live database is retained as {tcfg.get('db_name')}_before_<ts> inside {tcfg.get('db_container')} "
"and automatically swapped back if post-restore readiness checks fail."
),
'steps': steps,
}
return plan_doc
def discover_recovery_points(search_dirs, app_filter=None, target_filter=None):
"""Inventory recovery points across configured directories."""
seen_dumps = set()
points = []
for sdir in search_dirs:
root = Path(sdir)
if not root.is_dir():
continue
manifest_files = sorted(root.rglob('*.manifest.json'))
for mfile in manifest_files:
info = verify_recovery_point(mfile, check_hashes=False)
seen_dumps.add(str(Path(info['dump_path']).resolve()))
manifest = info.get('manifest') or {}
app_id = manifest.get('application', {}).get('id', 'shinkan')
target = manifest.get('target', 'unknown')
if app_filter and app_id != app_filter:
continue
if target_filter and target != target_filter:
continue
points.append({
'id': manifest.get('recoveryPoint', {}).get('id', mfile.stem),
'application': app_id,
'target': target,
'host': manifest.get('host'),
'created_at': manifest.get('created_at'),
'status': info['status'],
'app_version': manifest.get('app_version'),
'git_commit': manifest.get('git_commit'),
'last_migration': manifest.get('last_migration'),
'dump_path': info['dump_path'],
'manifest_path': info['manifest_path'],
'dump_bytes': info['dump_bytes'],
'dump_sha256': info['dump_sha256'],
'issues': info['issues'],
})
for dfile in sorted(root.rglob('*.sql')):
resolved = str(dfile.resolve())
if resolved in seen_dumps or dfile.name.startswith('.'):
continue
app_guess = dfile.stem.replace('-prod', '').replace('-dev', '')
if app_filter and app_filter not in dfile.stem:
continue
if target_filter and target_filter not in dfile.stem:
continue
points.append({
'id': f"{dfile.stem}-legacy",
'application': app_guess,
'target': 'legacy',
'host': socket.gethostname(),
'created_at': dt.datetime.fromtimestamp(dfile.stat().st_mtime, dt.timezone.utc).astimezone().isoformat(timespec='seconds'),
'status': 'captured',
'app_version': None,
'git_commit': None,
'last_migration': None,
'dump_path': str(dfile),
'manifest_path': None,
'dump_bytes': dfile.stat().st_size,
'dump_sha256': None,
'issues': ['Legacy SQL dump without .manifest.json'],
})
return sorted(points, key=lambda p: p.get('created_at') or '', reverse=True)
def execute_recovery_command(mode, args, targets_cfg):
"""Execute --restore or --restore-test with full audit logging and safety checks."""
target_name = args.target or ('llm-restoretest' if mode == 'restore-test' else None)
if not target_name:
raise ValueError('--target is required for --restore')
if not args.point:
raise ValueError('--point <dump-or-manifest-path> is required')
plan_doc = generate_recovery_plan(
args.point, target_name, targets_cfg,
sessions_mode=args.sessions,
)
compat = plan_doc['compatibility']
tcfg = targets_cfg['targets'][target_name]
if not compat['compatible']:
if args.json:
print(json.dumps({'overall': 'refused', 'plan': plan_doc}, ensure_ascii=False, indent=2))
else:
for r in compat['reasons']:
print(f'REFUSED: {r}', file=sys.stderr)
return compat['exit_code']
required_confirm = tcfg.get('require_confirm')
if mode == 'restore' and required_confirm and args.confirm != required_confirm:
msg = f"REFUSED: Target {target_name!r} requires explicit confirmation: --confirm {required_confirm}"
if args.json:
print(json.dumps({'overall': 'refused', 'reason': msg, 'plan': plan_doc}, ensure_ascii=False, indent=2))
else:
print(msg, file=sys.stderr)
return EXIT_PRECONDITION
args.log_root.mkdir(parents=True, exist_ok=True, mode=0o700)
run_id = 'recovery-' + dt.datetime.now().strftime('%Y%m%d-%H%M%S') + f'-{os.getpid()}'
folder = args.log_root / run_id
folder.mkdir(mode=0o700)
atomic_json(folder / 'recovery-plan.json', plan_doc)
result_file = folder / 'adapter-result.json'
log_file = folder / 'recovery.log'
app_id = tcfg.get('application', 'app')
adapter_dir = Path(tcfg.get('adapter_dir', f'/etc/backup-orchestrator/scripts/{app_id}'))
if not adapter_dir.is_dir():
adapter_dir = Path(__file__).resolve().parent / 'scripts' / app_id
dump_path = plan_doc['recovery_point']['dump_path']
if mode == 'restore-test':
custom_script = tcfg.get('restore_test_script')
script = Path(custom_script) if custom_script and Path(custom_script).is_file() else (adapter_dir / f'{app_id}-restore-test')
extra = ['--dump', dump_path, '--result', str(result_file)]
if getattr(args, 'playwright', False):
extra.append('--playwright')
if getattr(args, 'keep', False):
extra.append('--keep')
cmd = _build_script_cmd(script, extra)
else:
custom_script = tcfg.get('restore_script')
if custom_script and Path(custom_script).is_file():
script = Path(custom_script)
elif (adapter_dir / f'{app_id}-restore').is_file():
script = adapter_dir / f'{app_id}-restore'
else:
candidates = [
Path('/etc/backup-orchestrator/scripts/universal-db-restore'),
Path(__file__).resolve().parent / 'scripts' / 'common' / 'universal-db-restore',
]
script = next((c for c in candidates if c.is_file()), adapter_dir / f'{app_id}-restore')
extra = [
'--target', target_name, '--dump', dump_path,
'--result', str(result_file), '--sessions', args.sessions,
]
if args.confirm:
extra.extend(['--confirm', args.confirm])
if getattr(args, 'allow_overwrite', False):
extra.append('--allow-overwrite')
cmd = _build_script_cmd(script, extra)
began = now_iso()
start_ts = time.monotonic()
with log_file.open('wb') as stream:
stream.write((f"mode={mode} target={target_name} dump={dump_path} started={began}\n").encode())
stream.flush()
try:
proc = subprocess.run(cmd, stdout=stream, stderr=subprocess.STDOUT, timeout=3600)
rc = proc.returncode
except OSError as exc:
stream.write((f'EXEC ERROR: {exc}\n').encode())
rc = 127
adapter_result = {}
if result_file.is_file():
try:
adapter_result = json.loads(result_file.read_text(encoding='utf-8'))
except (ValueError, json.JSONDecodeError):
adapter_result = {'raw_error': 'Could not parse adapter result JSON'}
summary = {
'version': VERSION,
'run_id': run_id,
'mode': mode,
'host': socket.gethostname(),
'target': target_name,
'application': tcfg.get('application'),
'recovery_point_id': plan_doc['recovery_point']['id'],
'dump_path': dump_path,
'dump_sha256': plan_doc['recovery_point']['dump_sha256'],
'started': began,
'finished': now_iso(),
'duration_seconds': round(time.monotonic() - start_ts, 2),
'exit_code': rc,
'overall': 'ok' if rc == 0 else 'failed',
'log': str(log_file),
'adapter_result': adapter_result,
}
atomic_json(folder / 'summary.json', summary)
atomic_json(args.log_root / 'latest-recovery.json', summary)
# If restore-test succeeded and manifest is writable, record restore_tested status
manifest_path = Path(str(dump_path) + '.manifest.json')
if rc == 0 and mode == 'restore-test' and manifest_path.is_file():
try:
raw_m = json.loads(manifest_path.read_text(encoding='utf-8'))
raw_m.setdefault('recoveryPoint', {})
raw_m['recoveryPoint']['status'] = 'restore_tested'
raw_m['recoveryPoint']['validatedAt'] = summary['finished']
raw_m['recoveryPoint']['validationReport'] = str(folder / 'summary.json')
atomic_json(manifest_path, raw_m)
except OSError:
pass
if args.json:
print(json.dumps(summary, ensure_ascii=False, indent=2))
else:
print(f"Recovery ({mode}) target={target_name}: {summary['overall']} (exit_code={rc}, {summary['duration_seconds']}s)")
print(f"Summary: {folder / 'summary.json'}")
return rc
def main():
parser = argparse.ArgumentParser(description='Phase-based backup and recovery orchestrator v0.4; safe inspection by default')
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--check', action='store_true', help='Validate config and inspect schedules (read only)')
mode.add_argument('--plan', action='store_true', help='Show backup phase plan (read only)')
mode.add_argument('--status', action='store_true', help='Latest backup job status (read only)')
mode.add_argument('--run', action='store_true', help='Execute enabled backup jobs (safety-gated)')
mode.add_argument('--list-recovery-points', action='store_true', help='List available recovery points (read only)')
mode.add_argument('--verify-recovery-point', action='store_true', help='Verify checksums and manifest of a recovery point (read only)')
mode.add_argument('--recovery-plan', action='store_true', help='Generate non-destructive recovery plan / dry-run (read only)')
mode.add_argument('--restore', action='store_true', help='Execute application/database restore (requires explicit confirmation)')
mode.add_argument('--restore-test', action='store_true', help='Execute isolated disposable restore test')
parser.add_argument('--json', action='store_true', help='Machine-readable JSON output')
parser.add_argument('--host', type=str, default=None, help='Select server profile from hosts/<host> (e.g. raspberrypi5, llm-node)')
parser.add_argument('--config', type=Path, default=DEFAULT_CONFIG)
parser.add_argument('--notify-config', type=Path, default=DEFAULT_NOTIFY_CONFIG)
parser.add_argument('--targets-config', type=Path, default=DEFAULT_TARGETS_CONFIG)
parser.add_argument('--log-root', type=Path, default=DEFAULT_LOG_ROOT)
parser.add_argument('--lock-file', type=Path, default=DEFAULT_LOCK_FILE)
parser.add_argument('--max-parallel', type=int, default=3, help='Maximum concurrent jobs in a phase (1..16)')
selection = parser.add_mutually_exclusive_group()
selection.add_argument('--phase', type=int, help='Select one phase (--plan or --run)')
selection.add_argument('--job', type=str, help='Select one job (--plan or --run)')
# Recovery options
parser.add_argument('--point', type=str, help='Path to dump or .manifest.json for recovery operations')
parser.add_argument('--target', type=str, help='Recovery target name (e.g. llm-prod, llm-restoretest)')
parser.add_argument('--app', type=str, help='Filter recovery points by application ID (e.g. shinkan)')
parser.add_argument('--search-dir', action='append', dest='search_dirs', help='Directory to search for recovery points')
parser.add_argument('--confirm', type=str, help='Explicit confirmation token (e.g. shinkan@llm-node)')
parser.add_argument('--allow-overwrite', action='store_true', help='Allow overwriting a non-empty target database')
parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear', help='Session handling during restore (default: clear)')
parser.add_argument('--playwright', action='store_true', help='Run read-only Playwright smoke tests in --restore-test')
parser.add_argument('--keep', action='store_true', help='Keep isolated test environment running after --restore-test')
args = parser.parse_args()
if args.host:
host_dir = resolve_repo_host_dir(args.host)
if not host_dir:
parser.error(f'Unknown host profile: {args.host!r}')
if args.config == DEFAULT_CONFIG:
args.config = host_dir / 'jobs.csv'
if args.notify_config == DEFAULT_NOTIFY_CONFIG and (host_dir / 'notifications.json.example').is_file():
args.notify_config = host_dir / 'notifications.json.example'
if args.targets_config == DEFAULT_TARGETS_CONFIG and (host_dir / 'targets.json').is_file():
args.targets_config = host_dir / 'targets.json'
if not 1 <= args.max_parallel <= 16:
parser.error('--max-parallel must be 1..16')
json_capable = (
args.status or args.list_recovery_points or args.verify_recovery_point
or args.recovery_plan or args.restore or args.restore_test
)
if args.json and not json_capable:
parser.error('--json is only valid with --status or recovery commands')
if (args.job or args.phase is not None) and not (args.run or args.plan):
parser.error('--phase and --job are only valid with --plan or --run')
if args.status:
return show_status(args.log_root, as_json=args.json)
if args.list_recovery_points:
host_cfg = load_host_config(host_name=args.host)
search_dirs = args.search_dirs or [
host_cfg['archive_sets_dir'],
host_cfg['local_dump_dir'],
'/var/lib/shinkan/restore',
]
points = discover_recovery_points(search_dirs, app_filter=args.app, target_filter=args.target)
if args.json:
print(json.dumps({'recovery_points': points}, ensure_ascii=False, indent=2))
else:
if not points:
print('No recovery points found.')
for p in points:
print(
f"{p['id']:<38} app={p['application']:<12} target={p['target']:<14} "
f"status={p['status']:<14} created={p['created_at']} mig={p['last_migration'] or '-'}"
)
return 0
if args.verify_recovery_point:
if not args.point:
parser.error('--point is required with --verify-recovery-point')
res = verify_recovery_point(args.point, check_hashes=True)
if args.json:
print(json.dumps(res, ensure_ascii=False, indent=2))
else:
print(f"Recovery Point: {res['dump_path']}")
print(f"Status: {res['status']} (valid={res['valid']})")
print(f"SHA256: {res['dump_sha256']}")
if res['issues']:
print('Issues:')
for issue in res['issues']:
print(f' - {issue}')
return res['exit_code']
if args.recovery_plan:
if not args.point or not args.target:
parser.error('--point and --target are required with --recovery-plan')
targets_cfg = load_targets_config(args.targets_config, host_name=args.host)
plan_doc = generate_recovery_plan(args.point, args.target, targets_cfg, sessions_mode=args.sessions)
if args.json:
print(json.dumps(plan_doc, ensure_ascii=False, indent=2))
else:
print(f"=== RECOVERY PLAN (DRY-RUN) for target {args.target} ===")
print(f"Recovery Point: {plan_doc['recovery_point']['id']} ({plan_doc['recovery_point']['status']})")
print(f"Dump File: {plan_doc['recovery_point']['dump_path']} ({plan_doc['recovery_point']['dump_sha256']})")
print(f"Release/Schema: app={plan_doc['recovery_point']['app_version']} commit={plan_doc['recovery_point']['git_commit']} last_migration={plan_doc['recovery_point']['last_migration']}")
print(f"Compatibility: compatible={plan_doc['compatibility']['compatible']} relation={plan_doc['compatibility']['schema_relation']}")
for r in plan_doc['compatibility']['reasons']:
print(f" BLOCKER: {r}")
for w in plan_doc['compatibility']['warnings']:
print(f" NOTE: {w}")
print('Steps:')
for step in plan_doc['steps']:
print(f' {step}')
return plan_doc['compatibility']['exit_code']
if args.restore:
targets_cfg = load_targets_config(args.targets_config, host_name=args.host)
return execute_recovery_command('restore', args, targets_cfg)
if args.restore_test:
targets_cfg = load_targets_config(args.targets_config, host_name=args.host)
return execute_recovery_command('restore-test', args, targets_cfg)
jobs = load_jobs(args.config)
notify = load_notifications(args.notify_config, known_ids=[j['id'] for j in jobs])
if args.check:
return check(jobs, args.max_parallel, notify)
jobs = select_jobs(jobs, args)
if args.plan:
return plan(jobs, args.max_parallel) or 0
return live_run(jobs, args)
if __name__ == '__main__':
try:
sys.exit(main())
except (ValueError, OSError, json.JSONDecodeError) as exc:
print(f'ERROR: {exc}', file=sys.stderr)
sys.exit(2)