1625 lines
75 KiB
Python
1625 lines
75 KiB
Python
#!/usr/bin/env python3
|
|
"""Phase-based backup and recovery orchestrator v0.4.
|
|
|
|
Safety defaults: read-only inspection unless --run, --restore, or --restore-test is
|
|
explicitly requested with all required safety gates and confirmations.
|
|
"""
|
|
|
|
import argparse
|
|
from collections import defaultdict
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import csv
|
|
import datetime as dt
|
|
try:
|
|
import fcntl
|
|
except ImportError: # Windows local test compatibility
|
|
fcntl = None
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlencode, urlsplit, urlunsplit
|
|
from urllib.request import Request, urlopen
|
|
|
|
VERSION = '0.4.0'
|
|
DEFAULT_CONFIG = Path('/etc/backup-orchestrator/jobs.csv')
|
|
DEFAULT_NOTIFY_CONFIG = Path('/etc/backup-orchestrator/notifications.json')
|
|
DEFAULT_TARGETS_CONFIG = Path('/etc/backup-orchestrator/targets.json')
|
|
DEFAULT_HOST_CONFIG = Path('/etc/backup-orchestrator/host.json')
|
|
DEFAULT_LOG_ROOT = Path('/var/log/backup-orchestrator')
|
|
DEFAULT_LOCK_FILE = Path('/run/lock/backup-orchestrator.lock')
|
|
FIELDS = {'phase', 'id', 'script', 'critical', 'timeout_seconds', 'enabled'}
|
|
CUTOVER_ENV = 'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED'
|
|
NAS_MOUNT = '/backup'
|
|
NAS_SOURCE = '192.168.2.63:/volume1/Backup_Paperless'
|
|
CANCELLED = threading.Event()
|
|
|
|
# Exit codes aligned with Shinkan RECOVERY_CONTRACT.md
|
|
EXIT_OK = 0
|
|
EXIT_EXEC_ERROR = 1
|
|
EXIT_PRECONDITION = 2
|
|
EXIT_LOCK_HELD = 3
|
|
EXIT_SCHEMA_INCOMPATIBLE = 3
|
|
EXIT_CHECKSUM_MISMATCH = 4
|
|
EXIT_POST_RESTORE_FAILED = 5
|
|
|
|
SECRET_PATTERNS = re.compile(
|
|
r'(?:PGPASSWORD|MYSQL_PWD|DB_PASSWORD|MARIADB_PASSWORD|MYSQL_PASSWORD|SMTP_PASS|OPENROUTER_API_KEY)\s*=\s*\S+',
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def now_iso():
|
|
return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds')
|
|
|
|
|
|
def lock_exclusive(fp):
|
|
if fcntl is not None:
|
|
fcntl.flock(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
|
|
|
|
def resolve_repo_host_dir(host_name=None):
|
|
try:
|
|
host = host_name or socket.gethostname().split('.')[0]
|
|
candidate = Path(__file__).resolve().parent / 'hosts' / host
|
|
return candidate if candidate.is_dir() else None
|
|
except (OSError, PermissionError):
|
|
return None
|
|
|
|
|
|
def load_host_config(path=DEFAULT_HOST_CONFIG, host_name=None):
|
|
default = {
|
|
'hostname': host_name or socket.gethostname(),
|
|
'nas_mount': NAS_MOUNT,
|
|
'nas_source': NAS_SOURCE,
|
|
'nas_fstype': 'nfs4',
|
|
'local_dump_dir': '/media/lars/paperless/db-backup',
|
|
'archive_sets_dir': '/backup/recovery_points',
|
|
'conflict_services': ['raspiBackup.service'],
|
|
'conflict_timers': ['paperless-data-backup.timer', 'borgmatic.timer'],
|
|
'cron_users': ['root', 'lars'],
|
|
}
|
|
candidates = []
|
|
if host_name:
|
|
host_dir = resolve_repo_host_dir(host_name)
|
|
if host_dir:
|
|
candidates.append(host_dir / 'host.json')
|
|
if path:
|
|
candidates.append(Path(path))
|
|
host_dir = resolve_repo_host_dir()
|
|
if host_dir:
|
|
candidates.append(host_dir / 'host.json')
|
|
for candidate in candidates:
|
|
try:
|
|
if candidate.is_file():
|
|
with candidate.open(encoding='utf-8') as fp:
|
|
data = json.load(fp)
|
|
if isinstance(data, dict):
|
|
default.update(data)
|
|
break
|
|
except (OSError, PermissionError):
|
|
continue
|
|
return default
|
|
|
|
|
|
def load_targets_config(path=DEFAULT_TARGETS_CONFIG, host_name=None):
|
|
candidates = []
|
|
if host_name:
|
|
host_dir = resolve_repo_host_dir(host_name)
|
|
if host_dir:
|
|
candidates.append(host_dir / 'targets.json')
|
|
if path:
|
|
candidates.append(Path(path))
|
|
host_dir = resolve_repo_host_dir()
|
|
if host_dir:
|
|
candidates.append(host_dir / 'targets.json')
|
|
candidates.append(Path(__file__).resolve().parent / 'targets.json')
|
|
for candidate in candidates:
|
|
try:
|
|
if candidate.is_file():
|
|
with candidate.open(encoding='utf-8') as fp:
|
|
return json.load(fp)
|
|
except (OSError, PermissionError):
|
|
continue
|
|
return {'schemaVersion': 1, 'hosts': {}, 'targets': {}}
|
|
|
|
|
|
def load_jobs(path):
|
|
with path.open(newline='', encoding='utf-8-sig') as fp:
|
|
reader = csv.DictReader(fp)
|
|
if reader.fieldnames is None or set(reader.fieldnames) != FIELDS or len(reader.fieldnames) != len(FIELDS):
|
|
raise ValueError('CSV header must contain exactly: ' + ','.join(sorted(FIELDS)))
|
|
rows = list(reader)
|
|
if not rows:
|
|
raise ValueError('CSV contains no jobs')
|
|
ids = set()
|
|
jobs = []
|
|
for index, row in enumerate(rows, start=2):
|
|
if None in row or any(v is None for v in row.values()):
|
|
raise ValueError(f'CSV line {index}: wrong number of columns')
|
|
row = {k: v.strip() for k, v in row.items()}
|
|
jid = row['id']
|
|
if not re.fullmatch(r'[a-z][a-z0-9-]*', jid) or jid in ids:
|
|
raise ValueError(f'CSV line {index}: invalid or duplicate job ID: {jid!r}')
|
|
ids.add(jid)
|
|
if row['critical'] not in ('true', 'false') or row['enabled'] not in ('true', 'false'):
|
|
raise ValueError(f'CSV line {index}: critical and enabled must be true or false')
|
|
if not re.fullmatch(r'\d+', row['phase']) or int(row['phase']) < 1:
|
|
raise ValueError(f'CSV line {index}: phase must be a positive integer')
|
|
if not re.fullmatch(r'\d+', row['timeout_seconds']) or not 1 <= int(row['timeout_seconds']) <= 86400:
|
|
raise ValueError(f'CSV line {index}: timeout_seconds must be 1..86400')
|
|
if not Path(row['script']).startswith_slash() if hasattr(Path(row['script']), 'startswith_slash') else not (row['script'].startswith('/') or Path(row['script']).is_absolute()):
|
|
raise ValueError(f'CSV line {index}: script must be an absolute path')
|
|
jobs.append({
|
|
'phase': int(row['phase']), 'id': jid, 'script': row['script'],
|
|
'critical': row['critical'] == 'true',
|
|
'timeout_seconds': int(row['timeout_seconds']),
|
|
'enabled': row['enabled'] == 'true',
|
|
})
|
|
return sorted(jobs, key=lambda x: (x['phase'], x['id']))
|
|
|
|
|
|
def push_base_url(value):
|
|
"""Accept Kuma's full copied URL and discard its example ?status=... query."""
|
|
parsed = urlsplit(value)
|
|
if parsed.scheme not in ('http', 'https') or not parsed.netloc or not parsed.path.startswith('/api/push/'):
|
|
raise ValueError('Kuma Push URL must be an http(s) URL with /api/push/<token>')
|
|
if parsed.username is not None or parsed.password is not None:
|
|
raise ValueError('HTTP basic-auth in Push URL is unsupported')
|
|
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, '', ''))
|
|
|
|
|
|
def verify_http_url(value):
|
|
parsed = urlsplit(value)
|
|
if parsed.scheme not in ('http', 'https') or not parsed.netloc:
|
|
raise ValueError('Webhook URL must start with http:// or https://')
|
|
if parsed.username is not None or parsed.password is not None:
|
|
raise ValueError('Webhook URL must not contain user credentials')
|
|
return value
|
|
|
|
|
|
def load_notifications(path, known_ids=None):
|
|
"""Only outgoing HTTP. URLs are never printed in status/logs."""
|
|
default = {'timeout_seconds': 3, 'webhooks': [], 'kuma': {'overall_push_url': '', 'job_push_urls': {}}}
|
|
if not path.is_file():
|
|
return default
|
|
with path.open(encoding='utf-8') as fp:
|
|
obj = json.load(fp)
|
|
if not isinstance(obj, dict) or set(obj) - {'timeout_seconds', 'webhooks', 'kuma'}:
|
|
raise ValueError('Invalid notifications.json root keys')
|
|
timeout = obj.get('timeout_seconds', 3)
|
|
if type(timeout) not in (int, float) or not 0.5 <= timeout <= 15:
|
|
raise ValueError('timeout_seconds must be 0.5..15')
|
|
webhooks = obj.get('webhooks', [])
|
|
kuma = obj.get('kuma', {})
|
|
if not isinstance(webhooks, list) or any(not isinstance(x, str) for x in webhooks):
|
|
raise ValueError('webhooks must be a list of URLs')
|
|
if not isinstance(kuma, dict) or set(kuma) - {'overall_push_url', 'job_push_urls'}:
|
|
raise ValueError('Invalid kuma settings')
|
|
overall = kuma.get('overall_push_url', '')
|
|
per_job = kuma.get('job_push_urls', {})
|
|
if not isinstance(overall, str) or not isinstance(per_job, dict):
|
|
raise ValueError('Kuma URLs must be strings; job_push_urls must be an object')
|
|
if any(not isinstance(k, str) or not isinstance(v, str) for k, v in per_job.items()):
|
|
raise ValueError('Kuma job IDs and URLs must be strings')
|
|
if known_ids is not None and set(per_job) - set(known_ids):
|
|
raise ValueError('Unknown Kuma job ID(s): ' + ', '.join(sorted(set(per_job) - set(known_ids))))
|
|
return {
|
|
'timeout_seconds': timeout,
|
|
'webhooks': [verify_http_url(x) for x in webhooks],
|
|
'kuma': {
|
|
'overall_push_url': push_base_url(overall) if overall else '',
|
|
'job_push_urls': {k: push_base_url(v) if v else '' for k, v in per_job.items()},
|
|
},
|
|
}
|
|
|
|
|
|
def notification_counts(config):
|
|
kuma = config['kuma']
|
|
return len(config['webhooks']), int(bool(kuma['overall_push_url'])), sum(bool(v) for v in kuma['job_push_urls'].values())
|
|
|
|
|
|
def nas_mount_ready(mountpoint=None, source=None, fstype=None, host_cfg=None):
|
|
if host_cfg:
|
|
mountpoint = mountpoint or host_cfg.get('nas_mount')
|
|
source = source or host_cfg.get('nas_source')
|
|
fstype = fstype or host_cfg.get('nas_fstype')
|
|
mountpoint = mountpoint or NAS_MOUNT
|
|
source = source or NAS_SOURCE
|
|
fstype = fstype or 'nfs4'
|
|
if not mountpoint or not source:
|
|
return True
|
|
try:
|
|
result = subprocess.run(
|
|
['findmnt', '--mountpoint', mountpoint, '--types', fstype, '--source', source],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=8,
|
|
)
|
|
return result.returncode == 0
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return False
|
|
|
|
|
|
def legacy_schedules(host_cfg=None):
|
|
"""Stateful detection of active legacy schedules and conflicting running services configured in host.json."""
|
|
cfg = host_cfg or load_host_config()
|
|
cron_users = cfg.get('cron_users', ['root', 'lars'])
|
|
cron_pattern = cfg.get('conflict_cron_pattern', r'/home/lars/(?:db-[\w-]+\.sh|backupPaperlessData\.sh)')
|
|
conflict_services = cfg.get('conflict_services', ['raspiBackup.service'])
|
|
conflict_timers = cfg.get('conflict_timers', ['paperless-data-backup.timer', 'borgmatic.timer'])
|
|
|
|
found = []
|
|
if cron_pattern:
|
|
for user in cron_users:
|
|
try:
|
|
p = subprocess.run(['crontab', '-u', user, '-l'], capture_output=True, text=True, timeout=8)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
continue
|
|
for line in p.stdout.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith('#') or not stripped:
|
|
continue
|
|
if re.search(cron_pattern, stripped):
|
|
found.append(f'{user} crontab: {stripped}')
|
|
# Stateful coordination with conflicting services (e.g. raspiBackup.service): only block if currently active/running.
|
|
for service in conflict_services:
|
|
try:
|
|
svc_active = subprocess.run(
|
|
['systemctl', 'is-active', service],
|
|
capture_output=True, text=True, timeout=8,
|
|
).stdout.strip()
|
|
if svc_active in ('active', 'activating', 'reloading'):
|
|
found.append(f'{service}: active={svc_active}')
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
pass
|
|
for timer in conflict_timers:
|
|
try:
|
|
enabled = subprocess.run(['systemctl', 'is-enabled', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
|
active = subprocess.run(['systemctl', 'is-active', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
continue
|
|
if enabled in ('enabled', 'enabled-runtime', 'linked', 'linked-runtime') or active == 'active':
|
|
found.append(f'{timer}: enabled={enabled or "unknown"}, active={active or "unknown"}')
|
|
return found
|
|
|
|
|
|
def resolve_job_script(job, host_name=None):
|
|
p = Path(job['script'])
|
|
if p.is_file():
|
|
return p
|
|
repo_root = Path(__file__).resolve().parent
|
|
script_name = p.name
|
|
host = host_name or socket.gethostname()
|
|
for c in (
|
|
repo_root / 'hosts' / host / 'scripts' / script_name,
|
|
repo_root / 'scripts' / 'common' / script_name,
|
|
repo_root / 'scripts' / script_name,
|
|
):
|
|
if c.is_file():
|
|
return c
|
|
return p
|
|
|
|
|
|
def script_issues(job, host_name=None):
|
|
p = resolve_job_script(job, host_name)
|
|
if not p.is_file():
|
|
return ['MISSING']
|
|
issues = []
|
|
if os.name != 'nt' and not os.access(p, os.X_OK):
|
|
issues.append('NOT EXECUTABLE')
|
|
try:
|
|
with p.open('rb') as fp:
|
|
if fp.read(2) != b'#!':
|
|
issues.append('NO SHEBANG')
|
|
except OSError:
|
|
issues.append('NOT READABLE')
|
|
return issues
|
|
|
|
|
|
def check(jobs, max_parallel, notify, host_cfg=None):
|
|
cfg = host_cfg or load_host_config()
|
|
mountpoint = cfg.get('nas_mount', NAS_MOUNT)
|
|
source = cfg.get('nas_source', NAS_SOURCE)
|
|
print(f'Backup-Orchestrator {VERSION} (host={cfg.get("hostname", socket.gethostname())}): INSPECTION ONLY (no backups, no schedule changes)')
|
|
print(f'Parallel limit per phase: {max_parallel}')
|
|
for job in jobs:
|
|
issues = script_issues(job, cfg.get('hostname'))
|
|
state = 'enabled' if job['enabled'] else 'disabled'
|
|
print(f"phase {job['phase']:>2} {job['id']:<24} {state:<8} {job['timeout_seconds']:>5}s {job['script']} [{', '.join(issues) if issues else 'OK'}]")
|
|
conflicts = legacy_schedules(cfg)
|
|
print('\nLegacy schedules:')
|
|
for line in conflicts:
|
|
print(' ACTIVE: ' + line)
|
|
if not conflicts:
|
|
print(' None detected (best effort)')
|
|
print(f'\nNAS mount {mountpoint}: {"OK" if nas_mount_ready(host_cfg=cfg) else "NOT AVAILABLE"} (expected {source})')
|
|
a, b, c = notification_counts(notify)
|
|
print(f'Notifications configured: webhooks={a}, Kuma overall={b}, Kuma jobs={c} (tokens hidden)')
|
|
print('Same phase: parallel; next phase starts after all members complete.')
|
|
print('Critical job failure: later phases BLOCKED; in-flight phase members finish.')
|
|
print('No jobs execute without --run, enabled=true and explicit cutover approval.')
|
|
return 0
|
|
|
|
|
|
def select_jobs(jobs, args):
|
|
enabled = [j for j in jobs if j['enabled']]
|
|
if getattr(args, 'job', None):
|
|
matches = [j for j in enabled if j['id'] == args.job]
|
|
if not matches:
|
|
raise ValueError(f'Job {args.job!r} not found or disabled')
|
|
return matches
|
|
if getattr(args, 'phase', None) is not None:
|
|
matches = [j for j in enabled if j['phase'] == args.phase]
|
|
if not matches:
|
|
raise ValueError(f'No enabled jobs in phase {args.phase}')
|
|
return matches
|
|
return enabled
|
|
|
|
|
|
def plan(jobs, max_parallel):
|
|
groups = defaultdict(list)
|
|
for job in jobs:
|
|
groups[job['phase']].append(job)
|
|
print(f'MODE: PLAN ONLY — no jobs executed. Max parallel jobs per phase: {max_parallel}')
|
|
for phase, members in sorted(groups.items()):
|
|
print(f'Phase {phase} ({len(members)} job(s)):')
|
|
for job in members:
|
|
print(f" - {job['id']} ({'CRITICAL' if job['critical'] else 'optional'}, timeout {job['timeout_seconds']}s)")
|
|
print(' BARRIER: wait for all jobs before advancing')
|
|
|
|
|
|
def result_for(job, state, rc=None, seconds=0.0, logfile=None, reason=None):
|
|
result = dict(phase=job['phase'], job=job['id'], critical=job['critical'],
|
|
state=state, exit_code=rc, duration_seconds=round(seconds, 2))
|
|
if logfile is not None:
|
|
result['log'] = str(logfile)
|
|
if reason:
|
|
result['reason'] = reason
|
|
return result
|
|
|
|
|
|
def atomic_json(path, data):
|
|
path = Path(path)
|
|
tmp = path.with_name(f'.{path.name}.{os.getpid()}.{threading.get_ident()}.tmp')
|
|
try:
|
|
with tmp.open('w', encoding='utf-8') as fp:
|
|
json.dump(data, fp, ensure_ascii=False, indent=2)
|
|
fp.write('\n')
|
|
tmp.replace(path)
|
|
finally:
|
|
if tmp.exists():
|
|
tmp.unlink()
|
|
|
|
|
|
class Notifier:
|
|
"""Outbound webhooks + Kuma adapter; notification errors never alter backup exit codes."""
|
|
def __init__(self, config, folder):
|
|
self.config = config
|
|
self.folder = folder
|
|
self.pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix='backup-notify')
|
|
self.futures = []
|
|
self.lock = threading.Lock()
|
|
self.errors = 0
|
|
self.logfile = folder / 'notifications.log'
|
|
|
|
def _send_json(self, url, event):
|
|
payload = json.dumps(event, ensure_ascii=False).encode('utf-8')
|
|
req = Request(url, data=payload, method='POST', headers={'Content-Type': 'application/json', 'User-Agent': f'pi-backup-orchestrator/{VERSION}'})
|
|
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
|
|
if not 200 <= response.status < 300:
|
|
raise RuntimeError(f'HTTP {response.status}')
|
|
|
|
def _send_kuma(self, url, status, message):
|
|
sep = '&' if '?' in url else '?'
|
|
address = url + sep + urlencode({'status': status, 'msg': message[:180]})
|
|
req = Request(address, headers={'User-Agent': f'pi-backup-orchestrator/{VERSION}'})
|
|
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
|
|
if not 200 <= response.status < 300:
|
|
raise RuntimeError(f'HTTP {response.status}')
|
|
body = response.read(4096)
|
|
if body:
|
|
try:
|
|
parsed = json.loads(body)
|
|
except (ValueError, UnicodeDecodeError):
|
|
parsed = None
|
|
if isinstance(parsed, dict) and parsed.get('ok') is False:
|
|
raise RuntimeError('Kuma rejected push')
|
|
|
|
def _submit(self, channel, fn, *args):
|
|
def guarded():
|
|
try:
|
|
fn(*args)
|
|
except Exception as exc:
|
|
if hasattr(exc, 'code') and isinstance(exc.code, int):
|
|
detail = f'HTTP {exc.code}'
|
|
else:
|
|
detail = type(exc).__name__
|
|
with self.lock:
|
|
self.errors += 1
|
|
with self.logfile.open('a', encoding='utf-8') as out:
|
|
out.write(f'{now_iso()} {channel}: delivery failed ({detail})\n')
|
|
self.futures.append(self.pool.submit(guarded))
|
|
|
|
def notify(self, event):
|
|
for url in self.config['webhooks']:
|
|
self._submit('webhook', self._send_json, url, event)
|
|
name = event['event']
|
|
if name in ('JOB_SUCCESS', 'JOB_FAILED', 'JOB_TIMEOUT', 'JOB_CANCELLED', 'JOB_ERROR', 'JOB_BLOCKED'):
|
|
url = self.config['kuma']['job_push_urls'].get(event.get('job'), '')
|
|
if url:
|
|
status = 'up' if name == 'JOB_SUCCESS' else 'down'
|
|
msg = f"{event['job']}: {event.get('state', 'unknown')} (run {event['run_id']})"
|
|
self._submit('kuma/job', self._send_kuma, url, status, msg)
|
|
if name in ('RUN_FINISHED', 'RECOVERY_FINISHED'):
|
|
url = self.config['kuma']['overall_push_url']
|
|
if url:
|
|
status = 'up' if event.get('overall') == 'ok' else 'down'
|
|
self._submit('kuma/overall', self._send_kuma, url, status, f"Run {event['run_id']}: {event.get('overall')}")
|
|
|
|
def close(self):
|
|
self.pool.shutdown(wait=True)
|
|
return self.errors
|
|
|
|
|
|
class EventHub:
|
|
"""Write append-only events.jsonl plus atomic current snapshot for consumers."""
|
|
def __init__(self, folder, log_root, run_id, jobs, notifier, started):
|
|
self.folder = folder
|
|
self.log_root = log_root
|
|
self.run_id = run_id
|
|
self.notifier = notifier
|
|
self.lock = threading.RLock()
|
|
self.snapshot = {
|
|
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
|
|
'started': started, 'updated': started, 'finished': None,
|
|
'overall': 'running',
|
|
'jobs': {j['id']: result_for(j, 'pending') for j in jobs},
|
|
}
|
|
atomic_json(folder / 'status.json', self.snapshot)
|
|
atomic_json(log_root / 'latest.json', self.snapshot)
|
|
|
|
def emit(self, event_name, **fields):
|
|
event = {
|
|
'event': event_name, 'timestamp': now_iso(), 'run_id': self.run_id,
|
|
'host': self.snapshot['host'], **fields,
|
|
}
|
|
with self.lock:
|
|
if 'job' in event and event['job'] in self.snapshot['jobs'] and 'state' in event:
|
|
jid = event['job']
|
|
self.snapshot['jobs'][jid].update({k: v for k, v in fields.items() if k in (
|
|
'state', 'phase', 'critical', 'exit_code', 'duration_seconds', 'log', 'reason')})
|
|
if event_name == 'RUN_FINISHED':
|
|
self.snapshot['overall'] = fields['overall']
|
|
self.snapshot['finished'] = event['timestamp']
|
|
self.snapshot['updated'] = event['timestamp']
|
|
with (self.folder / 'events.jsonl').open('a', encoding='utf-8') as fp:
|
|
fp.write(json.dumps(event, ensure_ascii=False) + '\n')
|
|
atomic_json(self.folder / 'status.json', self.snapshot)
|
|
atomic_json(self.log_root / 'latest.json', self.snapshot)
|
|
self.notifier.notify(event)
|
|
return event
|
|
|
|
|
|
def terminate_group(proc):
|
|
"""Terminate ordinary children of a job and wait for shutdown before advancing phase."""
|
|
if proc.poll() is not None:
|
|
return
|
|
if hasattr(os, 'killpg'):
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
return
|
|
try:
|
|
proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
proc.wait()
|
|
else:
|
|
subprocess.run(['taskkill', '/F', '/T', '/PID', str(proc.pid)],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=5)
|
|
try:
|
|
proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
proc.wait()
|
|
|
|
|
|
def _build_script_cmd(script_path, extra_args=()):
|
|
if os.name == 'nt':
|
|
import shutil
|
|
p = Path(script_path)
|
|
first_line = ''
|
|
try:
|
|
with p.open('r', encoding='utf-8', errors='replace') as fp:
|
|
first_line = fp.readline()
|
|
except OSError:
|
|
pass
|
|
if 'python' in first_line:
|
|
return [sys.executable, str(p), *extra_args]
|
|
if first_line.startswith('#!') or p.suffix == '.sh':
|
|
sh_bin = shutil.which('sh') or r'C:\Program Files\Git\bin\sh.exe'
|
|
return [sh_bin, str(p), *extra_args]
|
|
return [str(script_path), *extra_args]
|
|
|
|
|
|
def execute_one(job, folder, hub):
|
|
start = time.monotonic()
|
|
logfile = folder / (job['id'] + '.log')
|
|
if CANCELLED.is_set():
|
|
result = result_for(job, 'cancelled', reason='Orchestrator interrupted')
|
|
hub.emit('JOB_CANCELLED', **result)
|
|
return result
|
|
hub.emit('JOB_STARTED', job=job['id'], phase=job['phase'], critical=job['critical'], state='running')
|
|
resolved_script = resolve_job_script(job)
|
|
with logfile.open('wb') as stream:
|
|
stream.write((f"job={job['id']} started={now_iso()}\n").encode())
|
|
stream.flush()
|
|
try:
|
|
proc = subprocess.Popen(_build_script_cmd(resolved_script), stdout=stream, stderr=subprocess.STDOUT,
|
|
start_new_session=True, cwd=str(resolved_script.parent))
|
|
except OSError as exc:
|
|
stream.write((f'EXEC ERROR: {exc}\n').encode())
|
|
result = result_for(job, 'error', 127, time.monotonic()-start, logfile, 'Could not start script')
|
|
hub.emit('JOB_ERROR', **result)
|
|
return result
|
|
deadline = time.monotonic() + job['timeout_seconds']
|
|
while True:
|
|
if CANCELLED.is_set():
|
|
terminate_group(proc)
|
|
state, code = 'cancelled', 130
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
terminate_group(proc)
|
|
state, code = 'timeout', 124
|
|
break
|
|
try:
|
|
code = proc.wait(timeout=min(0.5, max(0.01, deadline-time.monotonic())))
|
|
state = 'ok' if code == 0 else 'failed'
|
|
break
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
stream.write((f'\njob={job["id"]} finished={now_iso()} status={state} exit_code={code}\n').encode())
|
|
result = result_for(job, state, code, time.monotonic()-start, logfile)
|
|
event = {'ok': 'JOB_SUCCESS', 'failed': 'JOB_FAILED', 'timeout': 'JOB_TIMEOUT', 'cancelled': 'JOB_CANCELLED'}[state]
|
|
hub.emit(event, **result)
|
|
return result
|
|
|
|
|
|
def live_run(jobs, args, host_cfg=None):
|
|
if not jobs:
|
|
print('Nothing enabled; no backups executed.')
|
|
return 0
|
|
if os.environ.get(CUTOVER_ENV) != 'YES':
|
|
print(f'REFUSED: Set {CUTOVER_ENV}=YES only after testing scripts and coordinating existing schedules.', file=sys.stderr)
|
|
return 2
|
|
cfg = host_cfg or load_host_config(host_name=getattr(args, 'host', None))
|
|
mountpoint = cfg.get('nas_mount', NAS_MOUNT)
|
|
source = cfg.get('nas_source', NAS_SOURCE)
|
|
if not nas_mount_ready(host_cfg=cfg):
|
|
print(f'REFUSED: {mountpoint} is not mounted from {source}', file=sys.stderr)
|
|
return 2
|
|
conflicts = legacy_schedules() if host_cfg is None else legacy_schedules(cfg)
|
|
if conflicts:
|
|
print('REFUSED: existing scheduled backups could overlap this run:', file=sys.stderr)
|
|
for issue in conflicts:
|
|
print(' ' + issue, file=sys.stderr)
|
|
return 2
|
|
unsafe = [(j['id'], script_issues(j)) for j in jobs if script_issues(j)]
|
|
if unsafe:
|
|
for jid, issues in unsafe:
|
|
print(f'REFUSED: {jid}: {", ".join(issues)}', file=sys.stderr)
|
|
return 2
|
|
notifications = load_notifications(args.notify_config)
|
|
args.log_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
args.lock_file.parent.mkdir(parents=True, exist_ok=True)
|
|
with args.lock_file.open('a+') as lock:
|
|
try:
|
|
lock_exclusive(lock)
|
|
except BlockingIOError:
|
|
print('REFUSED: another orchestrator run holds the lock', file=sys.stderr)
|
|
return 3
|
|
run_id = dt.datetime.now().strftime('%Y%m%d-%H%M%S') + f'-{os.getpid()}'
|
|
folder = args.log_root / run_id
|
|
folder.mkdir(mode=0o700)
|
|
old_int = signal.getsignal(signal.SIGINT)
|
|
old_term = signal.getsignal(signal.SIGTERM)
|
|
CANCELLED.clear()
|
|
signal.signal(signal.SIGINT, lambda signum, frame: CANCELLED.set())
|
|
signal.signal(signal.SIGTERM, lambda signum, frame: CANCELLED.set())
|
|
results = []
|
|
critical_failed = False
|
|
groups = defaultdict(list)
|
|
for job in jobs:
|
|
groups[job['phase']].append(job)
|
|
began = now_iso()
|
|
notifier = Notifier(notifications, folder)
|
|
hub = EventHub(folder, args.log_root, run_id, jobs, notifier, began)
|
|
try:
|
|
hub.emit('RUN_STARTED', max_parallel=args.max_parallel, job_count=len(jobs))
|
|
for phase, members in sorted(groups.items()):
|
|
if critical_failed or CANCELLED.is_set():
|
|
reason = 'earlier critical failure' if critical_failed else 'orchestrator interrupted'
|
|
hub.emit('PHASE_BLOCKED', phase=phase, reason=reason)
|
|
for job in members:
|
|
result = result_for(job, 'blocked', reason=reason)
|
|
results.append(result)
|
|
hub.emit('JOB_BLOCKED', **result)
|
|
print(f'Phase {phase} BLOCKED: {reason}', flush=True)
|
|
continue
|
|
hub.emit('PHASE_STARTED', phase=phase, job_count=len(members))
|
|
print(f'Phase {phase}: starting {len(members)} job(s), max parallel {args.max_parallel}', flush=True)
|
|
phase_results = []
|
|
with ThreadPoolExecutor(max_workers=args.max_parallel) as pool:
|
|
future_to_job = {pool.submit(execute_one, job, folder, hub): job for job in members}
|
|
for future in as_completed(future_to_job):
|
|
job = future_to_job[future]
|
|
try:
|
|
item = future.result()
|
|
except Exception as exc:
|
|
item = result_for(job, 'error', 127, reason=f'Internal runner error: {type(exc).__name__}')
|
|
hub.emit('JOB_ERROR', **item)
|
|
phase_results.append(item)
|
|
print(f' {item["job"]}: {item["state"]} ({item["exit_code"]})', flush=True)
|
|
results.extend(sorted(phase_results, key=lambda r: r['job']))
|
|
if any(r['critical'] and r['state'] != 'ok' for r in phase_results):
|
|
critical_failed = True
|
|
hub.emit('PHASE_COMPLETED', phase=phase, failures=sum(r['state'] != 'ok' for r in phase_results),
|
|
critical_failed=critical_failed)
|
|
if critical_failed:
|
|
print(f'Phase {phase}: critical failure; subsequent phases will not start.', flush=True)
|
|
else:
|
|
print(f'Phase {phase}: barrier reached; next phase may start.', flush=True)
|
|
finally:
|
|
signal.signal(signal.SIGINT, old_int)
|
|
signal.signal(signal.SIGTERM, old_term)
|
|
overall = 'interrupted' if CANCELLED.is_set() else 'ok' if all(r['state'] == 'ok' for r in results) else 'failed'
|
|
summary = {
|
|
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
|
|
'started': began, 'finished': now_iso(), 'max_parallel': args.max_parallel,
|
|
'interrupted': CANCELLED.is_set(), 'results': results, 'overall': overall,
|
|
}
|
|
atomic_json(folder / 'summary.json', summary)
|
|
hub.emit('RUN_FINISHED', overall=overall, success_count=sum(r['state'] == 'ok' for r in results),
|
|
failure_count=sum(r['state'] != 'ok' for r in results))
|
|
delivery_failures = notifier.close()
|
|
if delivery_failures:
|
|
print(f'WARNING: {delivery_failures} notification delivery failure(s); see {folder / "notifications.log"}', flush=True)
|
|
print('Summary: ' + str(folder / 'summary.json'), flush=True)
|
|
return 130 if CANCELLED.is_set() else (1 if any(r['state'] != 'ok' for r in results) else 0)
|
|
|
|
|
|
def show_status(log_root, as_json=False):
|
|
latest = log_root / 'latest.json'
|
|
if latest.is_file():
|
|
with latest.open(encoding='utf-8') as fp:
|
|
data = json.load(fp)
|
|
if as_json:
|
|
print(json.dumps(data, ensure_ascii=False, indent=2))
|
|
return 0
|
|
print(f"Run {data['run_id']}: {data['overall']} ({data['started']} -> {data['finished'] or 'running'})")
|
|
for item in sorted(data['jobs'].values(), key=lambda x: (x['phase'], x['job'])):
|
|
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
|
|
return 0
|
|
summaries = sorted(log_root.glob('*/summary.json')) if log_root.is_dir() else []
|
|
if not summaries:
|
|
if as_json:
|
|
print(json.dumps({'overall': 'no_runs', 'jobs': {}}, indent=2))
|
|
else:
|
|
print('No runs recorded yet')
|
|
return 0
|
|
with summaries[-1].open(encoding='utf-8') as fp:
|
|
data = json.load(fp)
|
|
if as_json:
|
|
print(json.dumps(data, ensure_ascii=False, indent=2))
|
|
else:
|
|
print(f"Last run: {data['started']} -> {data['finished']} ; overall={data['overall']}")
|
|
for item in data['results']:
|
|
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
|
|
return 0
|
|
|
|
|
|
# ============================================================================
|
|
# Recovery Management v0.4 (Recovery Points, Manifest v1, Compatibility, Plan)
|
|
# ============================================================================
|
|
|
|
def compute_sha256(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open('rb') as fp:
|
|
while True:
|
|
chunk = fp.read(65536)
|
|
if not chunk:
|
|
break
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def read_sha256_sidecar(sha_path):
|
|
text = Path(sha_path).read_text(encoding='utf-8').strip()
|
|
if not text:
|
|
raise ValueError(f'Empty sha256 file: {sha_path}')
|
|
digest = text.split()[0].lower()
|
|
if not re.fullmatch(r'[0-9a-f]{64}', digest):
|
|
raise ValueError(f'Invalid sha256 digest in {sha_path}: {digest!r}')
|
|
return digest
|
|
|
|
|
|
def detect_dump_format(path):
|
|
with Path(path).open('rb') as fp:
|
|
head = fp.read(5)
|
|
if head == b'PGDMP':
|
|
return 'custom'
|
|
return 'plain'
|
|
|
|
|
|
def extract_migrations_from_sql_dump(path):
|
|
"""Extract schema_migrations entries from a plain PostgreSQL dump."""
|
|
migrations = []
|
|
in_copy = False
|
|
with Path(path).open('r', encoding='utf-8', errors='replace') as fp:
|
|
for line in fp:
|
|
if not in_copy:
|
|
if line.startswith('COPY public.schema_migrations ') or line.startswith('COPY schema_migrations '):
|
|
in_copy = True
|
|
continue
|
|
m = re.search(r"INSERT INTO (?:public\.)?schema_migrations\b.*?VALUES\s*\((?:\d+\s*,\s*)?'([^']+)'", line)
|
|
if m:
|
|
migrations.append(m.group(1))
|
|
else:
|
|
if line.startswith('\\.'):
|
|
in_copy = False
|
|
continue
|
|
parts = line.rstrip('\r\n').split('\t')
|
|
if len(parts) >= 2 and re.match(r'^\d+_', parts[1]):
|
|
migrations.append(parts[1])
|
|
elif len(parts) >= 1 and re.match(r'^\d+_', parts[0]):
|
|
migrations.append(parts[0])
|
|
return sorted(set(migrations), key=_migration_sort_key)
|
|
|
|
|
|
def _migration_sort_key(stem):
|
|
m = re.match(r'^(\d+)', str(stem))
|
|
return (int(m.group(1)) if m else 0, str(stem))
|
|
|
|
|
|
def discover_code_migrations(migrations_dir):
|
|
migrations_dir = Path(migrations_dir)
|
|
if not migrations_dir.is_dir():
|
|
return []
|
|
stems = []
|
|
for item in migrations_dir.iterdir():
|
|
if item.is_file() and item.name.endswith('.sql') and item.name[0].isdigit():
|
|
stems.append(item.name[:-4])
|
|
return sorted(stems, key=_migration_sort_key)
|
|
|
|
|
|
def normalize_manifest(raw, manifest_path=None):
|
|
"""Validate and normalize a Recovery Point manifest (schemaVersion 1 or Shinkan contract)."""
|
|
if not isinstance(raw, dict):
|
|
raise ValueError('Manifest must be a JSON object')
|
|
serialized = json.dumps(raw, ensure_ascii=False)
|
|
if SECRET_PATTERNS.search(serialized):
|
|
raise ValueError('Manifest contains forbidden plaintext secret patterns')
|
|
|
|
schema_ver = raw.get('schemaVersion', 1)
|
|
if schema_ver != 1:
|
|
raise ValueError(f'Unsupported manifest schemaVersion: {schema_ver!r}')
|
|
|
|
app_block = raw.get('application', {}) if isinstance(raw.get('application'), dict) else {}
|
|
rp_block = raw.get('recoveryPoint', {}) if isinstance(raw.get('recoveryPoint'), dict) else {}
|
|
rel_block = raw.get('release', {}) if isinstance(raw.get('release'), dict) else {}
|
|
db_block = raw.get('database', {}) if isinstance(raw.get('database'), dict) else {}
|
|
media_block = raw.get('media', {}) if isinstance(raw.get('media'), dict) else {}
|
|
|
|
created_at = raw.get('created_at') or rp_block.get('createdAt')
|
|
host = raw.get('host') or app_block.get('sourceHost')
|
|
target = raw.get('target') or app_block.get('target') or 'unknown'
|
|
app_id = app_block.get('id') or (target.split('-')[0] if '-' in target else 'shinkan')
|
|
environment = app_block.get('environment') or ('production' if 'prod' in target else 'development')
|
|
|
|
app_version = raw.get('app_version') or rel_block.get('appVersion') or 'unknown'
|
|
git_commit = raw.get('git_commit') or rel_block.get('gitCommit') or 'unknown'
|
|
|
|
migrations = raw.get('migrations') if isinstance(raw.get('migrations'), list) else db_block.get('migrations', [])
|
|
if not isinstance(migrations, list) or any(not isinstance(m, str) for m in migrations):
|
|
raise ValueError('migrations must be a list of strings')
|
|
migrations = sorted(migrations, key=_migration_sort_key)
|
|
last_migration = raw.get('last_migration') or db_block.get('lastMigration') or (migrations[-1] if migrations else None)
|
|
|
|
dump_format = raw.get('dump_format') or db_block.get('dumpFormat') or 'plain'
|
|
if dump_format not in ('plain', 'custom'):
|
|
raise ValueError(f'Invalid dump_format: {dump_format!r}')
|
|
|
|
dump_bytes = raw.get('dump_bytes') if 'dump_bytes' in raw else db_block.get('bytes')
|
|
dump_sha256 = (raw.get('dump_sha256') or db_block.get('sha256') or '').lower()
|
|
if not re.fullmatch(r'[0-9a-f]{64}', dump_sha256):
|
|
raise ValueError('Manifest requires a valid 64-char hex dump_sha256 / database.sha256')
|
|
if not created_at or not host:
|
|
raise ValueError('Manifest requires created_at/createdAt and host/sourceHost')
|
|
|
|
artifact_name = db_block.get('artifact') or raw.get('dump_file')
|
|
if not artifact_name and manifest_path is not None:
|
|
mname = Path(manifest_path).name
|
|
if mname.endswith('.manifest.json'):
|
|
artifact_name = mname[:-len('.manifest.json')]
|
|
|
|
media_manifest = raw.get('media_manifest') if isinstance(raw.get('media_manifest'), dict) else {}
|
|
if not media_manifest and media_block.get('manifestArtifact'):
|
|
media_manifest = {
|
|
'file': media_block.get('manifestArtifact'),
|
|
'sha256': media_block.get('manifestSha256', ''),
|
|
'file_count': media_block.get('fileCount', 0),
|
|
}
|
|
|
|
consistency = rp_block.get('consistency') if isinstance(rp_block.get('consistency'), dict) else {
|
|
'mode': 'hot_indexed' if media_manifest else 'hot_separate',
|
|
'dbTransactional': True,
|
|
'mediaSynchronized': False,
|
|
'limitations': [
|
|
'Online database dump in single transaction; media files indexed without write freeze.'
|
|
if media_manifest else
|
|
'Database dump only; media/filesystem snapshot is not bound via media manifest.'
|
|
],
|
|
}
|
|
|
|
point_id = rp_block.get('id') or f"{app_id}-{target}-{re.sub(r'[^0-9]', '', str(created_at))[:14]}-{dump_sha256[:8]}"
|
|
|
|
normalized = {
|
|
'schemaVersion': 1,
|
|
'created_at': created_at,
|
|
'host': host,
|
|
'target': target,
|
|
'app_version': app_version,
|
|
'git_commit': git_commit,
|
|
'migrations': migrations,
|
|
'last_migration': last_migration,
|
|
'dump_format': dump_format,
|
|
'dump_bytes': dump_bytes,
|
|
'dump_sha256': dump_sha256,
|
|
'row_counts': raw.get('row_counts') or db_block.get('rowCounts') or {},
|
|
'media_manifest': media_manifest,
|
|
'application': {
|
|
'id': app_id,
|
|
'environment': environment,
|
|
'target': target,
|
|
'sourceHost': host,
|
|
'sourceArch': app_block.get('sourceArch', 'unknown'),
|
|
},
|
|
'recoveryPoint': {
|
|
'id': point_id,
|
|
'createdAt': created_at,
|
|
'status': rp_block.get('status', 'captured'),
|
|
'validatedAt': rp_block.get('validatedAt'),
|
|
'validationReport': rp_block.get('validationReport'),
|
|
'consistency': consistency,
|
|
'retention': rp_block.get('retention', {'tier': 'daily', 'protected': False}),
|
|
},
|
|
'release': {
|
|
'repository': rel_block.get('repository', ''),
|
|
'appVersion': app_version,
|
|
'gitCommit': git_commit,
|
|
'imageDigests': rel_block.get('imageDigests', {}),
|
|
},
|
|
'database': {
|
|
'engine': db_block.get('engine', 'postgresql'),
|
|
'engineVersion': db_block.get('engineVersion', ''),
|
|
'dumpFormat': dump_format,
|
|
'artifact': artifact_name or '',
|
|
'bytes': dump_bytes,
|
|
'sha256': dump_sha256,
|
|
'schemaVersion': db_block.get('schemaVersion', last_migration or ''),
|
|
'lastMigration': last_migration,
|
|
'migrations': migrations,
|
|
'rowCounts': raw.get('row_counts') or db_block.get('rowCounts') or {},
|
|
},
|
|
'media': {
|
|
'storageType': media_block.get('storageType', 'nfs4_shared'),
|
|
'containerMount': media_block.get('containerMount', '/app/media'),
|
|
'manifestArtifact': media_manifest.get('file', ''),
|
|
'manifestSha256': media_manifest.get('sha256', ''),
|
|
'fileCount': media_manifest.get('file_count', media_block.get('fileCount', 0)),
|
|
},
|
|
'configuration': raw.get('configuration', {
|
|
'composeFiles': ['docker-compose.yml'],
|
|
'secretReferences': ['DB_PASSWORD'],
|
|
}),
|
|
'dependencies': raw.get('dependencies', {'services': ['postgres', 'backend', 'frontend']}),
|
|
'recovery': raw.get('recovery', {
|
|
'adapter': app_id,
|
|
'supportedTargets': ['llm-prod', 'llm-restoretest'],
|
|
}),
|
|
}
|
|
return normalized
|
|
|
|
|
|
def verify_recovery_point(point_ref, check_hashes=True):
|
|
"""Inspect a dump or manifest path and evaluate its 4-stage recovery point status."""
|
|
path = Path(point_ref)
|
|
if path.is_dir():
|
|
manifests = sorted(path.glob('*.manifest.json'))
|
|
if manifests:
|
|
path = manifests[-1]
|
|
else:
|
|
dumps = sorted(list(path.glob('*.sql')) + list(path.glob('*.dump')))
|
|
if not dumps:
|
|
raise ValueError(f'No recovery point artifacts found in directory: {point_ref}')
|
|
path = dumps[-1]
|
|
|
|
if path.name.endswith('.manifest.json'):
|
|
manifest_path = path
|
|
base_name = path.name[:-len('.manifest.json')]
|
|
dump_path = path.with_name(base_name)
|
|
else:
|
|
dump_path = path
|
|
manifest_path = Path(str(path) + '.manifest.json')
|
|
|
|
sha_path = Path(str(dump_path) + '.sha256')
|
|
media_path = Path(str(dump_path) + '.media.jsonl')
|
|
|
|
issues = []
|
|
status = 'invalid'
|
|
manifest = None
|
|
|
|
if not dump_path.is_file() or dump_path.stat().st_size == 0:
|
|
issues.append(f'Dump file missing or empty: {dump_path}')
|
|
return {
|
|
'valid': False, 'status': 'invalid', 'exit_code': EXIT_PRECONDITION,
|
|
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
|
|
'issues': issues, 'manifest': None,
|
|
}
|
|
|
|
status = 'captured'
|
|
actual_bytes = dump_path.stat().st_size
|
|
actual_format = detect_dump_format(dump_path)
|
|
actual_sha = compute_sha256(dump_path) if check_hashes else None
|
|
|
|
sidecar_sha = None
|
|
if sha_path.is_file():
|
|
try:
|
|
sidecar_sha = read_sha256_sidecar(sha_path)
|
|
except ValueError as exc:
|
|
issues.append(str(exc))
|
|
return {
|
|
'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH,
|
|
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
|
|
'issues': issues, 'manifest': None,
|
|
}
|
|
if check_hashes and sidecar_sha != actual_sha:
|
|
issues.append(f'SHA256 mismatch against {sha_path.name}: expected {sidecar_sha}, got {actual_sha}')
|
|
return {
|
|
'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH,
|
|
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
|
|
'issues': issues, 'manifest': None,
|
|
}
|
|
else:
|
|
issues.append(f'Missing checksum file: {sha_path.name}')
|
|
|
|
if manifest_path.is_file():
|
|
try:
|
|
raw = json.loads(manifest_path.read_text(encoding='utf-8'))
|
|
manifest = normalize_manifest(raw, manifest_path=manifest_path)
|
|
except (ValueError, json.JSONDecodeError) as exc:
|
|
issues.append(f'Invalid manifest {manifest_path.name}: {exc}')
|
|
return {
|
|
'valid': False, 'status': 'captured', 'exit_code': EXIT_PRECONDITION,
|
|
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
|
|
'issues': issues, 'manifest': None,
|
|
}
|
|
|
|
if check_hashes and manifest['dump_sha256'] != actual_sha:
|
|
issues.append(
|
|
f"Manifest dump_sha256 mismatch: manifest={manifest['dump_sha256']}, actual={actual_sha}"
|
|
)
|
|
return {
|
|
'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH,
|
|
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
|
|
'issues': issues, 'manifest': manifest,
|
|
}
|
|
|
|
if manifest['dump_bytes'] is not None and int(manifest['dump_bytes']) != actual_bytes:
|
|
issues.append(f"Manifest dump_bytes ({manifest['dump_bytes']}) != actual size ({actual_bytes})")
|
|
|
|
if sidecar_sha and manifest['dump_sha256'] == (actual_sha or sidecar_sha) and not issues:
|
|
status = 'verified'
|
|
|
|
# Check whether all requirements for a 'consistent' Recovery Point are met
|
|
has_release = manifest['app_version'] not in ('', 'unknown') and manifest['git_commit'] not in ('', 'unknown')
|
|
has_schema = bool(manifest['migrations'] or manifest['last_migration'])
|
|
media_info = manifest.get('media_manifest') or {}
|
|
media_ok = True
|
|
if media_info.get('file'):
|
|
mfile = dump_path.parent / media_info['file']
|
|
if not mfile.is_file():
|
|
media_ok = False
|
|
issues.append(f"Referenced media manifest missing: {media_info['file']}")
|
|
elif check_hashes and media_info.get('sha256'):
|
|
msha = compute_sha256(mfile)
|
|
if msha != media_info['sha256'].lower():
|
|
media_ok = False
|
|
issues.append(f"Media manifest sha256 mismatch for {media_info['file']}")
|
|
return {
|
|
'valid': False, 'status': 'captured', 'exit_code': EXIT_CHECKSUM_MISMATCH,
|
|
'dump_path': str(dump_path), 'manifest_path': str(manifest_path),
|
|
'issues': issues, 'manifest': manifest,
|
|
}
|
|
if has_release and has_schema and media_ok:
|
|
status = 'consistent'
|
|
if manifest['recoveryPoint'].get('validatedAt') or manifest['recoveryPoint'].get('status') == 'restore_tested':
|
|
status = 'restore_tested'
|
|
manifest['recoveryPoint']['status'] = status
|
|
else:
|
|
issues.append(f'Missing manifest file: {manifest_path.name} (legacy dump only)')
|
|
|
|
valid = status in ('verified', 'consistent', 'restore_tested') and not issues
|
|
return {
|
|
'valid': valid,
|
|
'status': status,
|
|
'exit_code': EXIT_OK if valid else EXIT_PRECONDITION,
|
|
'dump_path': str(dump_path),
|
|
'sha256_path': str(sha_path) if sha_path.is_file() else None,
|
|
'manifest_path': str(manifest_path) if manifest_path.is_file() else None,
|
|
'media_manifest_path': str(media_path) if media_path.is_file() else None,
|
|
'dump_format': actual_format,
|
|
'dump_bytes': actual_bytes,
|
|
'dump_sha256': actual_sha or sidecar_sha or (manifest['dump_sha256'] if manifest else None),
|
|
'issues': issues,
|
|
'manifest': manifest,
|
|
}
|
|
|
|
|
|
def check_compatibility(verification, target_name, targets_cfg, current_host=None, current_arch=None, code_migrations=None):
|
|
"""Evaluate version, schema, host, and media compatibility between a Recovery Point and a target."""
|
|
targets = targets_cfg.get('targets', {})
|
|
if target_name not in targets:
|
|
raise ValueError(f'Unknown recovery target: {target_name!r}')
|
|
tcfg = targets[target_name]
|
|
current_host = current_host or socket.gethostname()
|
|
current_arch = current_arch or platform.machine()
|
|
|
|
reasons = []
|
|
warnings = []
|
|
exit_code = EXIT_OK
|
|
|
|
if not tcfg.get('allow_restore', False):
|
|
reasons.append(f"Target {target_name!r} is read-only / source_only; restore is prohibited.")
|
|
exit_code = EXIT_PRECONDITION
|
|
|
|
expected_host = tcfg.get('host')
|
|
expected_arch = tcfg.get('arch')
|
|
if expected_host and current_host != expected_host:
|
|
reasons.append(f"Host mismatch: running on {current_host!r}, target {target_name!r} requires {expected_host!r}")
|
|
exit_code = EXIT_PRECONDITION
|
|
if expected_arch and current_arch != expected_arch:
|
|
reasons.append(f"Architecture mismatch: running on {current_arch!r}, target {target_name!r} requires {expected_arch!r}")
|
|
exit_code = EXIT_PRECONDITION
|
|
|
|
if not verification.get('valid'):
|
|
for issue in verification.get('issues', []):
|
|
reasons.append(issue)
|
|
if verification.get('exit_code') == EXIT_CHECKSUM_MISMATCH:
|
|
exit_code = EXIT_CHECKSUM_MISMATCH
|
|
elif exit_code == EXIT_OK:
|
|
exit_code = EXIT_PRECONDITION
|
|
|
|
manifest = verification.get('manifest')
|
|
dump_migrations = []
|
|
if manifest and manifest.get('migrations'):
|
|
dump_migrations = list(manifest['migrations'])
|
|
elif verification.get('dump_path') and verification.get('dump_format') == 'plain':
|
|
dump_migrations = extract_migrations_from_sql_dump(verification['dump_path'])
|
|
|
|
if code_migrations is None:
|
|
app_dir = Path(tcfg.get('app_dir', tcfg.get('compose_dir', '')))
|
|
mig_rel = tcfg.get('migrations_dir', 'backend/migrations')
|
|
mig_dir = app_dir / mig_rel if app_dir and str(app_dir) != '.' else Path('')
|
|
if mig_dir and mig_dir.is_dir():
|
|
code_migrations = discover_code_migrations(mig_dir)
|
|
|
|
schema_relation = 'UNKNOWN'
|
|
unknown_migrations = []
|
|
pending_migrations = []
|
|
|
|
if code_migrations is not None and dump_migrations:
|
|
code_set = set(code_migrations)
|
|
dump_set = set(dump_migrations)
|
|
unknown_migrations = sorted(dump_set - code_set, key=_migration_sort_key)
|
|
pending_migrations = sorted(code_set - dump_set, key=_migration_sort_key)
|
|
if unknown_migrations:
|
|
schema_relation = 'INCOMPATIBLE_ROLLBACK'
|
|
reasons.append(
|
|
'Database dump contains migration(s) unknown to target code (unsafe rollback): '
|
|
+ ', '.join(unknown_migrations)
|
|
)
|
|
exit_code = EXIT_SCHEMA_INCOMPATIBLE
|
|
elif pending_migrations:
|
|
schema_relation = 'FORWARD_MIGRATE'
|
|
warnings.append(
|
|
f"{len(pending_migrations)} newer migration(s) in target code will be applied on container start: "
|
|
+ ', '.join(pending_migrations[:10])
|
|
)
|
|
else:
|
|
schema_relation = 'EXACT_MATCH'
|
|
elif code_migrations is None:
|
|
warnings.append('Target code migrations directory not inspected locally; staging-DB check will verify on host.')
|
|
|
|
if manifest:
|
|
limitations = manifest.get('recoveryPoint', {}).get('consistency', {}).get('limitations', [])
|
|
warnings.extend(limitations)
|
|
if manifest.get('application', {}).get('sourceArch') not in ('', 'unknown', expected_arch):
|
|
warnings.append(
|
|
f"Cross-architecture recovery: dump captured on {manifest['application']['sourceArch']} "
|
|
f"({manifest.get('host')}), restoring to {expected_arch} ({expected_host}). "
|
|
"Logical SQL dump is portable across architectures."
|
|
)
|
|
|
|
compatible = len(reasons) == 0 and exit_code == EXIT_OK
|
|
return {
|
|
'compatible': compatible,
|
|
'exit_code': exit_code,
|
|
'target': target_name,
|
|
'schema_relation': schema_relation,
|
|
'dump_last_migration': dump_migrations[-1] if dump_migrations else (manifest.get('last_migration') if manifest else None),
|
|
'code_last_migration': code_migrations[-1] if code_migrations else None,
|
|
'unknown_migrations': unknown_migrations,
|
|
'pending_migrations': pending_migrations,
|
|
'reasons': reasons,
|
|
'warnings': warnings,
|
|
}
|
|
|
|
|
|
def generate_recovery_plan(point_ref, target_name, targets_cfg, sessions_mode='clear',
|
|
current_host=None, current_arch=None, code_migrations=None):
|
|
"""Build a complete non-destructive recovery plan (Dry-Run) for a Recovery Point."""
|
|
verification = verify_recovery_point(point_ref, check_hashes=True)
|
|
compat = check_compatibility(
|
|
verification, target_name, targets_cfg,
|
|
current_host=current_host, current_arch=current_arch, code_migrations=code_migrations,
|
|
)
|
|
tcfg = targets_cfg['targets'][target_name]
|
|
manifest = verification.get('manifest') or {}
|
|
|
|
is_test = tcfg.get('role') == 'isolated_test_target'
|
|
steps = []
|
|
if is_test:
|
|
steps = [
|
|
'1. Verify host/arch guard (llm-node / x86_64) and validate dump SHA256 checksum.',
|
|
'2. Start isolated disposable Compose project shinkan-restoretest (127.0.0.1 ports only, media mounted :ro, SMTP/retention disabled).',
|
|
'3. Import dump in single transaction into staging database shinkan_restore_<ts>.',
|
|
'4. Verify all schema_migrations entries exist in deployed backend/migrations/*.sql.',
|
|
'5. Check required core tables and non-empty profiles/skills in staging database.',
|
|
f'6. Handle user sessions (mode={sessions_mode}) and swap staging database into shinkan-restoretest.',
|
|
'7. Start isolated backend/frontend and verify /api/version and /api/health/ready (status=ready, database=true, schema_complete=true).',
|
|
'8. Run read-only shinkan-media-diff against shared NAS media directory.',
|
|
'9. Tear down isolated project (docker compose -p shinkan-restoretest down -v) unless --keep is set.',
|
|
]
|
|
else:
|
|
steps = [
|
|
f"1. Verify host/arch guard ({tcfg.get('host')}/{tcfg.get('arch')}), configuration ({tcfg.get('env_file')}), and SHA256 checksum.",
|
|
f"2. Create staging database shinkan_restore_<ts> in container {tcfg.get('db_container')} and import dump in a single transaction.",
|
|
f"3. Validate every migration in staging schema_migrations against {tcfg.get('app_dir')}/backend/migrations/*.sql (abort with Exit 3 if dump is newer than code).",
|
|
'4. Run plausibility checks on staging database (required tables exist, profiles and skills non-empty).',
|
|
f'5. Apply session policy in staging database (sessions={sessions_mode}).',
|
|
f"6. Stop services {', '.join(tcfg.get('services_to_stop', ['backend', 'frontend']))} while keeping postgres running.",
|
|
f"7. Rename live database {tcfg.get('db_name')} -> {tcfg.get('db_name')}_before_<ts> and rename shinkan_restore_<ts> -> {tcfg.get('db_name')}.",
|
|
'8. Start backend and frontend (plus retention-cron only if enabled) and verify /api/health/ready.',
|
|
f"9. Automatic rollback strategy: if step 7 or 8 fails, stop services, restore {tcfg.get('db_name')}_before_<ts> -> {tcfg.get('db_name')}, restart services, and exit with code 5.",
|
|
'10. Run shinkan-media-diff to report missing, mismatched, added_after_backup, or orphan media files.',
|
|
]
|
|
|
|
plan_doc = {
|
|
'mode': 'DRY_RUN_RECOVERY_PLAN',
|
|
'generated_at': now_iso(),
|
|
'target': {
|
|
'name': target_name,
|
|
'application': tcfg.get('application'),
|
|
'environment': tcfg.get('environment'),
|
|
'host': tcfg.get('host'),
|
|
'arch': tcfg.get('arch'),
|
|
'role': tcfg.get('role'),
|
|
'app_dir': tcfg.get('app_dir'),
|
|
'compose_project': tcfg.get('compose_project'),
|
|
'env_file': tcfg.get('env_file'),
|
|
'db_container': tcfg.get('db_container'),
|
|
'db_name': tcfg.get('db_name'),
|
|
'media_host_dir': tcfg.get('media_host_dir'),
|
|
'require_confirm': tcfg.get('require_confirm'),
|
|
},
|
|
'recovery_point': {
|
|
'id': manifest.get('recoveryPoint', {}).get('id') or Path(verification['dump_path']).name,
|
|
'status': verification['status'],
|
|
'created_at': manifest.get('created_at'),
|
|
'source_host': manifest.get('host'),
|
|
'app_version': manifest.get('app_version'),
|
|
'git_commit': manifest.get('git_commit'),
|
|
'last_migration': compat['dump_last_migration'],
|
|
'dump_path': verification['dump_path'],
|
|
'dump_format': verification.get('dump_format'),
|
|
'dump_bytes': verification.get('dump_bytes'),
|
|
'dump_sha256': verification.get('dump_sha256'),
|
|
'media_manifest': manifest.get('media_manifest'),
|
|
'consistency': manifest.get('recoveryPoint', {}).get('consistency'),
|
|
},
|
|
'compatibility': compat,
|
|
'parameters': {
|
|
'sessions': sessions_mode,
|
|
'services_to_stop': tcfg.get('services_to_stop', []),
|
|
},
|
|
'impacts_and_risks': [
|
|
f"Services {', '.join(tcfg.get('services_to_stop', []))} will experience brief downtime during database swap."
|
|
if not is_test else 'Isolated test project only; zero impact on production containers or media.',
|
|
'Database changes written after the recovery point timestamp will be replaced (previous DB preserved as shinkan_before_<ts>).'
|
|
if not is_test else 'Disposable test database will be removed after verification.',
|
|
'Shared NAS media files are never deleted during restore; shinkan-media-diff will list any files added after the backup.',
|
|
],
|
|
'rollback_strategy': (
|
|
f"Previous live database is retained as {tcfg.get('db_name')}_before_<ts> inside {tcfg.get('db_container')} "
|
|
"and automatically swapped back if post-restore readiness checks fail."
|
|
),
|
|
'steps': steps,
|
|
}
|
|
return plan_doc
|
|
|
|
|
|
def discover_recovery_points(search_dirs, app_filter=None, target_filter=None):
|
|
"""Inventory recovery points across configured directories."""
|
|
seen_dumps = set()
|
|
points = []
|
|
for sdir in search_dirs:
|
|
root = Path(sdir)
|
|
if not root.is_dir():
|
|
continue
|
|
manifest_files = sorted(root.rglob('*.manifest.json'))
|
|
for mfile in manifest_files:
|
|
info = verify_recovery_point(mfile, check_hashes=False)
|
|
seen_dumps.add(str(Path(info['dump_path']).resolve()))
|
|
manifest = info.get('manifest') or {}
|
|
app_id = manifest.get('application', {}).get('id', 'shinkan')
|
|
target = manifest.get('target', 'unknown')
|
|
if app_filter and app_id != app_filter:
|
|
continue
|
|
if target_filter and target != target_filter:
|
|
continue
|
|
points.append({
|
|
'id': manifest.get('recoveryPoint', {}).get('id', mfile.stem),
|
|
'application': app_id,
|
|
'target': target,
|
|
'host': manifest.get('host'),
|
|
'created_at': manifest.get('created_at'),
|
|
'status': info['status'],
|
|
'app_version': manifest.get('app_version'),
|
|
'git_commit': manifest.get('git_commit'),
|
|
'last_migration': manifest.get('last_migration'),
|
|
'dump_path': info['dump_path'],
|
|
'manifest_path': info['manifest_path'],
|
|
'dump_bytes': info['dump_bytes'],
|
|
'dump_sha256': info['dump_sha256'],
|
|
'issues': info['issues'],
|
|
})
|
|
for dfile in sorted(root.rglob('*.sql')):
|
|
resolved = str(dfile.resolve())
|
|
if resolved in seen_dumps or dfile.name.startswith('.'):
|
|
continue
|
|
app_guess = dfile.stem.replace('-prod', '').replace('-dev', '')
|
|
if app_filter and app_filter not in dfile.stem:
|
|
continue
|
|
if target_filter and target_filter not in dfile.stem:
|
|
continue
|
|
points.append({
|
|
'id': f"{dfile.stem}-legacy",
|
|
'application': app_guess,
|
|
'target': 'legacy',
|
|
'host': socket.gethostname(),
|
|
'created_at': dt.datetime.fromtimestamp(dfile.stat().st_mtime, dt.timezone.utc).astimezone().isoformat(timespec='seconds'),
|
|
'status': 'captured',
|
|
'app_version': None,
|
|
'git_commit': None,
|
|
'last_migration': None,
|
|
'dump_path': str(dfile),
|
|
'manifest_path': None,
|
|
'dump_bytes': dfile.stat().st_size,
|
|
'dump_sha256': None,
|
|
'issues': ['Legacy SQL dump without .manifest.json'],
|
|
})
|
|
return sorted(points, key=lambda p: p.get('created_at') or '', reverse=True)
|
|
|
|
|
|
def execute_recovery_command(mode, args, targets_cfg):
|
|
"""Execute --restore or --restore-test with full audit logging and safety checks."""
|
|
target_name = args.target or ('llm-restoretest' if mode == 'restore-test' else None)
|
|
if not target_name:
|
|
raise ValueError('--target is required for --restore')
|
|
if not args.point:
|
|
raise ValueError('--point <dump-or-manifest-path> is required')
|
|
|
|
plan_doc = generate_recovery_plan(
|
|
args.point, target_name, targets_cfg,
|
|
sessions_mode=args.sessions,
|
|
)
|
|
compat = plan_doc['compatibility']
|
|
tcfg = targets_cfg['targets'][target_name]
|
|
|
|
if not compat['compatible']:
|
|
if args.json:
|
|
print(json.dumps({'overall': 'refused', 'plan': plan_doc}, ensure_ascii=False, indent=2))
|
|
else:
|
|
for r in compat['reasons']:
|
|
print(f'REFUSED: {r}', file=sys.stderr)
|
|
return compat['exit_code']
|
|
|
|
required_confirm = tcfg.get('require_confirm')
|
|
if mode == 'restore' and required_confirm and args.confirm != required_confirm:
|
|
msg = f"REFUSED: Target {target_name!r} requires explicit confirmation: --confirm {required_confirm}"
|
|
if args.json:
|
|
print(json.dumps({'overall': 'refused', 'reason': msg, 'plan': plan_doc}, ensure_ascii=False, indent=2))
|
|
else:
|
|
print(msg, file=sys.stderr)
|
|
return EXIT_PRECONDITION
|
|
|
|
args.log_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
run_id = 'recovery-' + dt.datetime.now().strftime('%Y%m%d-%H%M%S') + f'-{os.getpid()}'
|
|
folder = args.log_root / run_id
|
|
folder.mkdir(mode=0o700)
|
|
atomic_json(folder / 'recovery-plan.json', plan_doc)
|
|
result_file = folder / 'adapter-result.json'
|
|
log_file = folder / 'recovery.log'
|
|
|
|
app_id = tcfg.get('application', 'app')
|
|
adapter_dir = Path(tcfg.get('adapter_dir', f'/etc/backup-orchestrator/scripts/{app_id}'))
|
|
if not adapter_dir.is_dir():
|
|
adapter_dir = Path(__file__).resolve().parent / 'scripts' / app_id
|
|
|
|
dump_path = plan_doc['recovery_point']['dump_path']
|
|
if mode == 'restore-test':
|
|
custom_script = tcfg.get('restore_test_script')
|
|
script = Path(custom_script) if custom_script and Path(custom_script).is_file() else (adapter_dir / f'{app_id}-restore-test')
|
|
extra = ['--dump', dump_path, '--result', str(result_file)]
|
|
if getattr(args, 'playwright', False):
|
|
extra.append('--playwright')
|
|
if getattr(args, 'keep', False):
|
|
extra.append('--keep')
|
|
cmd = _build_script_cmd(script, extra)
|
|
else:
|
|
custom_script = tcfg.get('restore_script')
|
|
if custom_script and Path(custom_script).is_file():
|
|
script = Path(custom_script)
|
|
elif (adapter_dir / f'{app_id}-restore').is_file():
|
|
script = adapter_dir / f'{app_id}-restore'
|
|
else:
|
|
candidates = [
|
|
Path('/etc/backup-orchestrator/scripts/universal-db-restore'),
|
|
Path(__file__).resolve().parent / 'scripts' / 'common' / 'universal-db-restore',
|
|
]
|
|
script = next((c for c in candidates if c.is_file()), adapter_dir / f'{app_id}-restore')
|
|
extra = [
|
|
'--target', target_name, '--dump', dump_path,
|
|
'--result', str(result_file), '--sessions', args.sessions,
|
|
]
|
|
if args.confirm:
|
|
extra.extend(['--confirm', args.confirm])
|
|
if getattr(args, 'allow_overwrite', False):
|
|
extra.append('--allow-overwrite')
|
|
cmd = _build_script_cmd(script, extra)
|
|
|
|
began = now_iso()
|
|
start_ts = time.monotonic()
|
|
with log_file.open('wb') as stream:
|
|
stream.write((f"mode={mode} target={target_name} dump={dump_path} started={began}\n").encode())
|
|
stream.flush()
|
|
try:
|
|
proc = subprocess.run(cmd, stdout=stream, stderr=subprocess.STDOUT, timeout=3600)
|
|
rc = proc.returncode
|
|
except OSError as exc:
|
|
stream.write((f'EXEC ERROR: {exc}\n').encode())
|
|
rc = 127
|
|
|
|
adapter_result = {}
|
|
if result_file.is_file():
|
|
try:
|
|
adapter_result = json.loads(result_file.read_text(encoding='utf-8'))
|
|
except (ValueError, json.JSONDecodeError):
|
|
adapter_result = {'raw_error': 'Could not parse adapter result JSON'}
|
|
|
|
summary = {
|
|
'version': VERSION,
|
|
'run_id': run_id,
|
|
'mode': mode,
|
|
'host': socket.gethostname(),
|
|
'target': target_name,
|
|
'application': tcfg.get('application'),
|
|
'recovery_point_id': plan_doc['recovery_point']['id'],
|
|
'dump_path': dump_path,
|
|
'dump_sha256': plan_doc['recovery_point']['dump_sha256'],
|
|
'started': began,
|
|
'finished': now_iso(),
|
|
'duration_seconds': round(time.monotonic() - start_ts, 2),
|
|
'exit_code': rc,
|
|
'overall': 'ok' if rc == 0 else 'failed',
|
|
'log': str(log_file),
|
|
'adapter_result': adapter_result,
|
|
}
|
|
atomic_json(folder / 'summary.json', summary)
|
|
atomic_json(args.log_root / 'latest-recovery.json', summary)
|
|
|
|
# If restore-test succeeded and manifest is writable, record restore_tested status
|
|
manifest_path = Path(str(dump_path) + '.manifest.json')
|
|
if rc == 0 and mode == 'restore-test' and manifest_path.is_file():
|
|
try:
|
|
raw_m = json.loads(manifest_path.read_text(encoding='utf-8'))
|
|
raw_m.setdefault('recoveryPoint', {})
|
|
raw_m['recoveryPoint']['status'] = 'restore_tested'
|
|
raw_m['recoveryPoint']['validatedAt'] = summary['finished']
|
|
raw_m['recoveryPoint']['validationReport'] = str(folder / 'summary.json')
|
|
atomic_json(manifest_path, raw_m)
|
|
except OSError:
|
|
pass
|
|
|
|
if args.json:
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
else:
|
|
print(f"Recovery ({mode}) target={target_name}: {summary['overall']} (exit_code={rc}, {summary['duration_seconds']}s)")
|
|
print(f"Summary: {folder / 'summary.json'}")
|
|
return rc
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Phase-based backup and recovery orchestrator v0.4; safe inspection by default')
|
|
mode = parser.add_mutually_exclusive_group(required=True)
|
|
mode.add_argument('--check', action='store_true', help='Validate config and inspect schedules (read only)')
|
|
mode.add_argument('--plan', action='store_true', help='Show backup phase plan (read only)')
|
|
mode.add_argument('--status', action='store_true', help='Latest backup job status (read only)')
|
|
mode.add_argument('--run', action='store_true', help='Execute enabled backup jobs (safety-gated)')
|
|
mode.add_argument('--list-recovery-points', action='store_true', help='List available recovery points (read only)')
|
|
mode.add_argument('--verify-recovery-point', action='store_true', help='Verify checksums and manifest of a recovery point (read only)')
|
|
mode.add_argument('--recovery-plan', action='store_true', help='Generate non-destructive recovery plan / dry-run (read only)')
|
|
mode.add_argument('--restore', action='store_true', help='Execute application/database restore (requires explicit confirmation)')
|
|
mode.add_argument('--restore-test', action='store_true', help='Execute isolated disposable restore test')
|
|
|
|
parser.add_argument('--json', action='store_true', help='Machine-readable JSON output')
|
|
parser.add_argument('--host', type=str, default=None, help='Select server profile from hosts/<host> (e.g. raspberrypi5, llm-node)')
|
|
parser.add_argument('--config', type=Path, default=DEFAULT_CONFIG)
|
|
parser.add_argument('--notify-config', type=Path, default=DEFAULT_NOTIFY_CONFIG)
|
|
parser.add_argument('--targets-config', type=Path, default=DEFAULT_TARGETS_CONFIG)
|
|
parser.add_argument('--log-root', type=Path, default=DEFAULT_LOG_ROOT)
|
|
parser.add_argument('--lock-file', type=Path, default=DEFAULT_LOCK_FILE)
|
|
parser.add_argument('--max-parallel', type=int, default=3, help='Maximum concurrent jobs in a phase (1..16)')
|
|
|
|
selection = parser.add_mutually_exclusive_group()
|
|
selection.add_argument('--phase', type=int, help='Select one phase (--plan or --run)')
|
|
selection.add_argument('--job', type=str, help='Select one job (--plan or --run)')
|
|
|
|
# Recovery options
|
|
parser.add_argument('--point', type=str, help='Path to dump or .manifest.json for recovery operations')
|
|
parser.add_argument('--target', type=str, help='Recovery target name (e.g. llm-prod, llm-restoretest)')
|
|
parser.add_argument('--app', type=str, help='Filter recovery points by application ID (e.g. shinkan)')
|
|
parser.add_argument('--search-dir', action='append', dest='search_dirs', help='Directory to search for recovery points')
|
|
parser.add_argument('--confirm', type=str, help='Explicit confirmation token (e.g. shinkan@llm-node)')
|
|
parser.add_argument('--allow-overwrite', action='store_true', help='Allow overwriting a non-empty target database')
|
|
parser.add_argument('--sessions', choices=('clear', 'keep'), default='clear', help='Session handling during restore (default: clear)')
|
|
parser.add_argument('--playwright', action='store_true', help='Run read-only Playwright smoke tests in --restore-test')
|
|
parser.add_argument('--keep', action='store_true', help='Keep isolated test environment running after --restore-test')
|
|
|
|
args = parser.parse_args()
|
|
host_name = args.host or socket.gethostname()
|
|
host_dir = resolve_repo_host_dir(host_name)
|
|
if args.host and not host_dir:
|
|
parser.error(f'Unknown host profile: {args.host!r}')
|
|
if host_dir:
|
|
if args.config == DEFAULT_CONFIG and not DEFAULT_CONFIG.is_file():
|
|
args.config = host_dir / 'jobs.csv'
|
|
if args.notify_config == DEFAULT_NOTIFY_CONFIG and not DEFAULT_NOTIFY_CONFIG.is_file() and (host_dir / 'notifications.json.example').is_file():
|
|
args.notify_config = host_dir / 'notifications.json.example'
|
|
if args.targets_config == DEFAULT_TARGETS_CONFIG and not DEFAULT_TARGETS_CONFIG.is_file() and (host_dir / 'targets.json').is_file():
|
|
args.targets_config = host_dir / 'targets.json'
|
|
|
|
# Fallback to user directories if /var/log or /run/lock are not writable
|
|
if args.log_root == DEFAULT_LOG_ROOT and not os.access('/var/log', os.W_OK) and not DEFAULT_LOG_ROOT.is_dir():
|
|
args.log_root = Path.home() / '.local' / 'share' / 'backup-orchestrator' / 'logs'
|
|
if args.lock_file == DEFAULT_LOCK_FILE and not os.access('/run/lock', os.W_OK) and not DEFAULT_LOCK_FILE.is_file():
|
|
args.lock_file = Path.home() / '.local' / 'share' / 'backup-orchestrator' / 'lock'
|
|
|
|
if not 1 <= args.max_parallel <= 16:
|
|
parser.error('--max-parallel must be 1..16')
|
|
json_capable = (
|
|
args.status or args.list_recovery_points or args.verify_recovery_point
|
|
or args.recovery_plan or args.restore or args.restore_test
|
|
)
|
|
if args.json and not json_capable:
|
|
parser.error('--json is only valid with --status or recovery commands')
|
|
if (args.job or args.phase is not None) and not (args.run or args.plan):
|
|
parser.error('--phase and --job are only valid with --plan or --run')
|
|
|
|
if args.status:
|
|
return show_status(args.log_root, as_json=args.json)
|
|
|
|
if args.list_recovery_points:
|
|
host_cfg = load_host_config(host_name=args.host)
|
|
search_dirs = args.search_dirs or [
|
|
host_cfg['archive_sets_dir'],
|
|
host_cfg['local_dump_dir'],
|
|
'/var/lib/shinkan/restore',
|
|
]
|
|
points = discover_recovery_points(search_dirs, app_filter=args.app, target_filter=args.target)
|
|
if args.json:
|
|
print(json.dumps({'recovery_points': points}, ensure_ascii=False, indent=2))
|
|
else:
|
|
if not points:
|
|
print('No recovery points found.')
|
|
for p in points:
|
|
print(
|
|
f"{p['id']:<38} app={p['application']:<12} target={p['target']:<14} "
|
|
f"status={p['status']:<14} created={p['created_at']} mig={p['last_migration'] or '-'}"
|
|
)
|
|
return 0
|
|
|
|
if args.verify_recovery_point:
|
|
if not args.point:
|
|
parser.error('--point is required with --verify-recovery-point')
|
|
res = verify_recovery_point(args.point, check_hashes=True)
|
|
if args.json:
|
|
print(json.dumps(res, ensure_ascii=False, indent=2))
|
|
else:
|
|
print(f"Recovery Point: {res['dump_path']}")
|
|
print(f"Status: {res['status']} (valid={res['valid']})")
|
|
print(f"SHA256: {res['dump_sha256']}")
|
|
if res['issues']:
|
|
print('Issues:')
|
|
for issue in res['issues']:
|
|
print(f' - {issue}')
|
|
return res['exit_code']
|
|
|
|
if args.recovery_plan:
|
|
if not args.point or not args.target:
|
|
parser.error('--point and --target are required with --recovery-plan')
|
|
targets_cfg = load_targets_config(args.targets_config, host_name=args.host)
|
|
plan_doc = generate_recovery_plan(args.point, args.target, targets_cfg, sessions_mode=args.sessions)
|
|
if args.json:
|
|
print(json.dumps(plan_doc, ensure_ascii=False, indent=2))
|
|
else:
|
|
print(f"=== RECOVERY PLAN (DRY-RUN) for target {args.target} ===")
|
|
print(f"Recovery Point: {plan_doc['recovery_point']['id']} ({plan_doc['recovery_point']['status']})")
|
|
print(f"Dump File: {plan_doc['recovery_point']['dump_path']} ({plan_doc['recovery_point']['dump_sha256']})")
|
|
print(f"Release/Schema: app={plan_doc['recovery_point']['app_version']} commit={plan_doc['recovery_point']['git_commit']} last_migration={plan_doc['recovery_point']['last_migration']}")
|
|
print(f"Compatibility: compatible={plan_doc['compatibility']['compatible']} relation={plan_doc['compatibility']['schema_relation']}")
|
|
for r in plan_doc['compatibility']['reasons']:
|
|
print(f" BLOCKER: {r}")
|
|
for w in plan_doc['compatibility']['warnings']:
|
|
print(f" NOTE: {w}")
|
|
print('Steps:')
|
|
for step in plan_doc['steps']:
|
|
print(f' {step}')
|
|
return plan_doc['compatibility']['exit_code']
|
|
|
|
if args.restore:
|
|
targets_cfg = load_targets_config(args.targets_config, host_name=args.host)
|
|
return execute_recovery_command('restore', args, targets_cfg)
|
|
|
|
if args.restore_test:
|
|
targets_cfg = load_targets_config(args.targets_config, host_name=args.host)
|
|
return execute_recovery_command('restore-test', args, targets_cfg)
|
|
|
|
jobs = load_jobs(args.config)
|
|
notify = load_notifications(args.notify_config, known_ids=[j['id'] for j in jobs])
|
|
if args.check:
|
|
return check(jobs, args.max_parallel, notify)
|
|
jobs = select_jobs(jobs, args)
|
|
if args.plan:
|
|
return plan(jobs, args.max_parallel) or 0
|
|
return live_run(jobs, args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
sys.exit(main())
|
|
except (ValueError, OSError, json.JSONDecodeError) as exc:
|
|
print(f'ERROR: {exc}', file=sys.stderr)
|
|
sys.exit(2)
|