601 lines
28 KiB
Python
601 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""Phase-based backup orchestrator v0.3.
|
|
|
|
Safety defaults: read-only inspection unless --run, jobs enabled and explicit cutover
|
|
approval. No scheduler, database or Docker changes are performed by this program.
|
|
"""
|
|
|
|
import argparse
|
|
from collections import defaultdict
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import csv
|
|
import datetime as dt
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlencode, urlsplit, urlunsplit
|
|
from urllib.request import Request, urlopen
|
|
|
|
VERSION = '0.3.0'
|
|
DEFAULT_CONFIG = Path('/etc/backup-orchestrator/jobs.csv')
|
|
DEFAULT_NOTIFY_CONFIG = Path('/etc/backup-orchestrator/notifications.json')
|
|
DEFAULT_LOG_ROOT = Path('/var/log/backup-orchestrator')
|
|
DEFAULT_LOCK_FILE = Path('/run/lock/backup-orchestrator.lock')
|
|
FIELDS = {'phase', 'id', 'script', 'critical', 'timeout_seconds', 'enabled'}
|
|
CUTOVER_ENV = 'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED'
|
|
NAS_MOUNT = '/backup'
|
|
NAS_SOURCE = '192.168.2.63:/volume1/Backup_Paperless'
|
|
CANCELLED = threading.Event()
|
|
|
|
|
|
def now_iso():
|
|
return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds')
|
|
|
|
|
|
def load_jobs(path):
|
|
with path.open(newline='', encoding='utf-8-sig') as fp:
|
|
reader = csv.DictReader(fp)
|
|
if reader.fieldnames is None or set(reader.fieldnames) != FIELDS or len(reader.fieldnames) != len(FIELDS):
|
|
raise ValueError('CSV header must contain exactly: ' + ','.join(sorted(FIELDS)))
|
|
rows = list(reader)
|
|
if not rows:
|
|
raise ValueError('CSV contains no jobs')
|
|
ids = set()
|
|
jobs = []
|
|
for index, row in enumerate(rows, start=2):
|
|
if None in row or any(v is None for v in row.values()):
|
|
raise ValueError(f'CSV line {index}: wrong number of columns')
|
|
row = {k: v.strip() for k, v in row.items()}
|
|
jid = row['id']
|
|
if not re.fullmatch(r'[a-z][a-z0-9-]*', jid) or jid in ids:
|
|
raise ValueError(f'CSV line {index}: invalid or duplicate job ID: {jid!r}')
|
|
ids.add(jid)
|
|
if row['critical'] not in ('true', 'false') or row['enabled'] not in ('true', 'false'):
|
|
raise ValueError(f'CSV line {index}: critical and enabled must be true or false')
|
|
if not re.fullmatch(r'\d+', row['phase']) or int(row['phase']) < 1:
|
|
raise ValueError(f'CSV line {index}: phase must be a positive integer')
|
|
if not re.fullmatch(r'\d+', row['timeout_seconds']) or not 1 <= int(row['timeout_seconds']) <= 86400:
|
|
raise ValueError(f'CSV line {index}: timeout_seconds must be 1..86400')
|
|
if not Path(row['script']).is_absolute():
|
|
raise ValueError(f'CSV line {index}: script must be an absolute path')
|
|
jobs.append({
|
|
'phase': int(row['phase']), 'id': jid, 'script': row['script'],
|
|
'critical': row['critical'] == 'true',
|
|
'timeout_seconds': int(row['timeout_seconds']),
|
|
'enabled': row['enabled'] == 'true',
|
|
})
|
|
return sorted(jobs, key=lambda x: (x['phase'], x['id']))
|
|
|
|
|
|
def push_base_url(value):
|
|
"""Accept Kuma's full copied URL and discard its example ?status=... query."""
|
|
parsed = urlsplit(value)
|
|
if parsed.scheme not in ('http', 'https') or not parsed.netloc or not parsed.path.startswith('/api/push/'):
|
|
raise ValueError('Kuma Push URL must be an http(s) URL with /api/push/<token>')
|
|
if parsed.username is not None or parsed.password is not None:
|
|
raise ValueError('HTTP basic-auth in Push URL is unsupported')
|
|
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, '', ''))
|
|
|
|
|
|
def verify_http_url(value):
|
|
parsed = urlsplit(value)
|
|
if parsed.scheme not in ('http', 'https') or not parsed.netloc:
|
|
raise ValueError('Webhook URL must start with http:// or https://')
|
|
if parsed.username is not None or parsed.password is not None:
|
|
raise ValueError('Webhook URL must not contain user credentials')
|
|
return value
|
|
|
|
|
|
def load_notifications(path, known_ids=None):
|
|
"""Only outgoing HTTP. URLs are never printed in status/logs."""
|
|
default = {'timeout_seconds': 3, 'webhooks': [], 'kuma': {'overall_push_url': '', 'job_push_urls': {}}}
|
|
if not path.is_file():
|
|
return default
|
|
with path.open(encoding='utf-8') as fp:
|
|
obj = json.load(fp)
|
|
if not isinstance(obj, dict) or set(obj) - {'timeout_seconds', 'webhooks', 'kuma'}:
|
|
raise ValueError('Invalid notifications.json root keys')
|
|
timeout = obj.get('timeout_seconds', 3)
|
|
if type(timeout) not in (int, float) or not 0.5 <= timeout <= 15:
|
|
raise ValueError('timeout_seconds must be 0.5..15')
|
|
webhooks = obj.get('webhooks', [])
|
|
kuma = obj.get('kuma', {})
|
|
if not isinstance(webhooks, list) or any(not isinstance(x, str) for x in webhooks):
|
|
raise ValueError('webhooks must be a list of URLs')
|
|
if not isinstance(kuma, dict) or set(kuma) - {'overall_push_url', 'job_push_urls'}:
|
|
raise ValueError('Invalid kuma settings')
|
|
overall = kuma.get('overall_push_url', '')
|
|
per_job = kuma.get('job_push_urls', {})
|
|
if not isinstance(overall, str) or not isinstance(per_job, dict):
|
|
raise ValueError('Kuma URLs must be strings; job_push_urls must be an object')
|
|
if any(not isinstance(k, str) or not isinstance(v, str) for k, v in per_job.items()):
|
|
raise ValueError('Kuma job IDs and URLs must be strings')
|
|
if known_ids is not None and set(per_job) - set(known_ids):
|
|
raise ValueError('Unknown Kuma job ID(s): ' + ', '.join(sorted(set(per_job) - set(known_ids))))
|
|
return {
|
|
'timeout_seconds': timeout,
|
|
'webhooks': [verify_http_url(x) for x in webhooks],
|
|
'kuma': {
|
|
'overall_push_url': push_base_url(overall) if overall else '',
|
|
'job_push_urls': {k: push_base_url(v) if v else '' for k, v in per_job.items()},
|
|
},
|
|
}
|
|
|
|
|
|
def notification_counts(config):
|
|
kuma = config['kuma']
|
|
return len(config['webhooks']), int(bool(kuma['overall_push_url'])), sum(bool(v) for v in kuma['job_push_urls'].values())
|
|
|
|
|
|
def nas_mount_ready():
|
|
try:
|
|
result = subprocess.run(['findmnt', '--mountpoint', NAS_MOUNT, '--types', 'nfs4', '--source', NAS_SOURCE],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=8)
|
|
return result.returncode == 0
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return False
|
|
|
|
|
|
def legacy_schedules():
|
|
"""Best-effort detection of active legacy schedules (never comments)."""
|
|
found = []
|
|
for user in ('root', 'lars'):
|
|
try:
|
|
p = subprocess.run(['crontab', '-u', user, '-l'], capture_output=True, text=True, timeout=8)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
continue
|
|
for line in p.stdout.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith('#') or not stripped:
|
|
continue
|
|
if re.search(r'/home/lars/(?:db-[\w-]+\.sh|backupPaperlessData\.sh)', stripped):
|
|
found.append(f'{user} crontab: {stripped}')
|
|
for timer in ('raspiBackup.timer', 'paperless-data-backup.timer', 'borgmatic.timer'):
|
|
try:
|
|
enabled = subprocess.run(['systemctl', 'is-enabled', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
|
active = subprocess.run(['systemctl', 'is-active', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
continue
|
|
if enabled in ('enabled', 'enabled-runtime', 'linked', 'linked-runtime') or active == 'active':
|
|
found.append(f'{timer}: enabled={enabled or "unknown"}, active={active or "unknown"}')
|
|
return found
|
|
|
|
|
|
def script_issues(job):
|
|
p = Path(job['script'])
|
|
if not p.is_file():
|
|
return ['MISSING']
|
|
issues = []
|
|
if not os.access(p, os.X_OK):
|
|
issues.append('NOT EXECUTABLE')
|
|
try:
|
|
with p.open('rb') as fp:
|
|
if fp.read(2) != b'#!':
|
|
issues.append('NO SHEBANG')
|
|
except OSError:
|
|
issues.append('NOT READABLE')
|
|
return issues
|
|
|
|
|
|
def check(jobs, max_parallel, notify):
|
|
print(f'Backup-Orchestrator {VERSION}: INSPECTION ONLY (no backups, no schedule changes)')
|
|
print(f'Parallel limit per phase: {max_parallel}')
|
|
for job in jobs:
|
|
issues = script_issues(job)
|
|
state = 'enabled' if job['enabled'] else 'disabled'
|
|
print(f"phase {job['phase']:>2} {job['id']:<24} {state:<8} {job['timeout_seconds']:>5}s {job['script']} [{', '.join(issues) if issues else 'OK'}]")
|
|
conflicts = legacy_schedules()
|
|
print('\nLegacy schedules:')
|
|
for line in conflicts:
|
|
print(' ACTIVE: ' + line)
|
|
if not conflicts:
|
|
print(' None detected (best effort)')
|
|
print(f'\nNAS mount {NAS_MOUNT}: {"OK" if nas_mount_ready() else "NOT AVAILABLE"} (expected {NAS_SOURCE})')
|
|
a, b, c = notification_counts(notify)
|
|
print(f'Notifications configured: webhooks={a}, Kuma overall={b}, Kuma jobs={c} (tokens hidden)')
|
|
print('Same phase: parallel; next phase starts after all members complete.')
|
|
print('Critical job failure: later phases BLOCKED; in-flight phase members finish.')
|
|
print('No jobs execute without --run, enabled=true and explicit cutover approval.')
|
|
return 0
|
|
|
|
|
|
def select_jobs(jobs, args):
|
|
enabled = [j for j in jobs if j['enabled']]
|
|
if args.job:
|
|
matches = [j for j in enabled if j['id'] == args.job]
|
|
if not matches:
|
|
raise ValueError(f'Job {args.job!r} not found or disabled')
|
|
return matches
|
|
if args.phase is not None:
|
|
matches = [j for j in enabled if j['phase'] == args.phase]
|
|
if not matches:
|
|
raise ValueError(f'No enabled jobs in phase {args.phase}')
|
|
return matches
|
|
return enabled
|
|
|
|
|
|
def plan(jobs, max_parallel):
|
|
groups = defaultdict(list)
|
|
for job in jobs:
|
|
groups[job['phase']].append(job)
|
|
print(f'MODE: PLAN ONLY — no jobs executed. Max parallel jobs per phase: {max_parallel}')
|
|
for phase, members in sorted(groups.items()):
|
|
print(f'Phase {phase} ({len(members)} job(s)):')
|
|
for job in members:
|
|
print(f" - {job['id']} ({'CRITICAL' if job['critical'] else 'optional'}, timeout {job['timeout_seconds']}s)")
|
|
print(' BARRIER: wait for all jobs before advancing')
|
|
|
|
|
|
def result_for(job, state, rc=None, seconds=0.0, logfile=None, reason=None):
|
|
result = dict(phase=job['phase'], job=job['id'], critical=job['critical'],
|
|
state=state, exit_code=rc, duration_seconds=round(seconds, 2))
|
|
if logfile is not None:
|
|
result['log'] = str(logfile)
|
|
if reason:
|
|
result['reason'] = reason
|
|
return result
|
|
|
|
|
|
def atomic_json(path, data):
|
|
tmp = path.with_name(f'.{path.name}.{os.getpid()}.{threading.get_ident()}.tmp')
|
|
try:
|
|
with tmp.open('w', encoding='utf-8') as fp:
|
|
json.dump(data, fp, ensure_ascii=False, indent=2)
|
|
fp.write('\n')
|
|
tmp.replace(path)
|
|
finally:
|
|
if tmp.exists():
|
|
tmp.unlink()
|
|
|
|
|
|
class Notifier:
|
|
"""Outbound webhooks + Kuma adapter; notification errors never alter backup exit codes."""
|
|
def __init__(self, config, folder):
|
|
self.config = config
|
|
self.folder = folder
|
|
self.pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix='backup-notify')
|
|
self.futures = []
|
|
self.lock = threading.Lock()
|
|
self.errors = 0
|
|
self.logfile = folder / 'notifications.log'
|
|
|
|
def _send_json(self, url, event):
|
|
payload = json.dumps(event, ensure_ascii=False).encode('utf-8')
|
|
req = Request(url, data=payload, method='POST', headers={'Content-Type': 'application/json', 'User-Agent': 'pi-backup-orchestrator/0.3'})
|
|
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
|
|
if not 200 <= response.status < 300:
|
|
raise RuntimeError(f'HTTP {response.status}')
|
|
|
|
def _send_kuma(self, url, status, message):
|
|
sep = '&' if '?' in url else '?'
|
|
address = url + sep + urlencode({'status': status, 'msg': message[:180]})
|
|
req = Request(address, headers={'User-Agent': 'pi-backup-orchestrator/0.3'})
|
|
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
|
|
if not 200 <= response.status < 300:
|
|
raise RuntimeError(f'HTTP {response.status}')
|
|
# Kuma usually responds with JSON {ok:true}; a 200 alone is insufficient if it says failure.
|
|
body = response.read(4096)
|
|
if body:
|
|
try:
|
|
parsed = json.loads(body)
|
|
except (ValueError, UnicodeDecodeError):
|
|
parsed = None
|
|
if isinstance(parsed, dict) and parsed.get('ok') is False:
|
|
raise RuntimeError('Kuma rejected push')
|
|
|
|
def _submit(self, channel, fn, *args):
|
|
def guarded():
|
|
try:
|
|
fn(*args)
|
|
except Exception as exc:
|
|
# Never log target URLs or tokens; even HTTP exceptions may include their URL.
|
|
if hasattr(exc, 'code') and isinstance(exc.code, int):
|
|
detail = f'HTTP {exc.code}'
|
|
else:
|
|
detail = type(exc).__name__
|
|
with self.lock:
|
|
self.errors += 1
|
|
with self.logfile.open('a', encoding='utf-8') as out:
|
|
out.write(f'{now_iso()} {channel}: delivery failed ({detail})\n')
|
|
self.futures.append(self.pool.submit(guarded))
|
|
|
|
def notify(self, event):
|
|
for url in self.config['webhooks']:
|
|
self._submit('webhook', self._send_json, url, event)
|
|
name = event['event']
|
|
if name in ('JOB_SUCCESS', 'JOB_FAILED', 'JOB_TIMEOUT', 'JOB_CANCELLED', 'JOB_ERROR', 'JOB_BLOCKED'):
|
|
url = self.config['kuma']['job_push_urls'].get(event.get('job'), '')
|
|
if url:
|
|
status = 'up' if name == 'JOB_SUCCESS' else 'down'
|
|
msg = f"{event['job']}: {event.get('state', 'unknown')} (run {event['run_id']})"
|
|
self._submit('kuma/job', self._send_kuma, url, status, msg)
|
|
if name == 'RUN_FINISHED':
|
|
url = self.config['kuma']['overall_push_url']
|
|
if url:
|
|
status = 'up' if event.get('overall') == 'ok' else 'down'
|
|
self._submit('kuma/overall', self._send_kuma, url, status, f"Run {event['run_id']}: {event.get('overall')}")
|
|
|
|
def close(self):
|
|
self.pool.shutdown(wait=True)
|
|
return self.errors
|
|
|
|
|
|
class EventHub:
|
|
"""Write append-only events.jsonl plus atomic current snapshot for consumers."""
|
|
def __init__(self, folder, log_root, run_id, jobs, notifier, started):
|
|
self.folder = folder
|
|
self.log_root = log_root
|
|
self.run_id = run_id
|
|
self.notifier = notifier
|
|
self.lock = threading.RLock()
|
|
self.snapshot = {
|
|
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
|
|
'started': started, 'updated': started, 'finished': None,
|
|
'overall': 'running',
|
|
'jobs': {j['id']: result_for(j, 'pending') for j in jobs},
|
|
}
|
|
atomic_json(folder / 'status.json', self.snapshot)
|
|
atomic_json(log_root / 'latest.json', self.snapshot)
|
|
|
|
def emit(self, event_name, **fields):
|
|
event = {
|
|
'event': event_name, 'timestamp': now_iso(), 'run_id': self.run_id,
|
|
'host': self.snapshot['host'], **fields,
|
|
}
|
|
with self.lock:
|
|
if 'job' in event and event['job'] in self.snapshot['jobs'] and 'state' in event:
|
|
jid = event['job']
|
|
self.snapshot['jobs'][jid].update({k: v for k, v in fields.items() if k in (
|
|
'state', 'phase', 'critical', 'exit_code', 'duration_seconds', 'log', 'reason')})
|
|
if event_name == 'RUN_FINISHED':
|
|
self.snapshot['overall'] = fields['overall']
|
|
self.snapshot['finished'] = event['timestamp']
|
|
self.snapshot['updated'] = event['timestamp']
|
|
with (self.folder / 'events.jsonl').open('a', encoding='utf-8') as fp:
|
|
fp.write(json.dumps(event, ensure_ascii=False) + '\n')
|
|
atomic_json(self.folder / 'status.json', self.snapshot)
|
|
atomic_json(self.log_root / 'latest.json', self.snapshot)
|
|
self.notifier.notify(event)
|
|
return event
|
|
|
|
|
|
def terminate_group(proc):
|
|
"""Terminate ordinary children of a job and wait for shutdown before advancing phase."""
|
|
if proc.poll() is not None:
|
|
return
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
return
|
|
try:
|
|
proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
proc.wait()
|
|
|
|
|
|
def execute_one(job, folder, hub):
|
|
start = time.monotonic()
|
|
logfile = folder / (job['id'] + '.log')
|
|
if CANCELLED.is_set():
|
|
result = result_for(job, 'cancelled', reason='Orchestrator interrupted')
|
|
hub.emit('JOB_CANCELLED', **result)
|
|
return result
|
|
hub.emit('JOB_STARTED', job=job['id'], phase=job['phase'], critical=job['critical'], state='running')
|
|
with logfile.open('wb') as stream:
|
|
stream.write((f"job={job['id']} started={now_iso()}\n").encode())
|
|
stream.flush()
|
|
try:
|
|
proc = subprocess.Popen([job['script']], stdout=stream, stderr=subprocess.STDOUT,
|
|
start_new_session=True, cwd=str(Path(job['script']).parent))
|
|
except OSError as exc:
|
|
stream.write((f'EXEC ERROR: {exc}\n').encode())
|
|
result = result_for(job, 'error', 127, time.monotonic()-start, logfile, 'Could not start script')
|
|
hub.emit('JOB_ERROR', **result)
|
|
return result
|
|
deadline = time.monotonic() + job['timeout_seconds']
|
|
while True:
|
|
if CANCELLED.is_set():
|
|
terminate_group(proc)
|
|
state, code = 'cancelled', 130
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
terminate_group(proc)
|
|
state, code = 'timeout', 124
|
|
break
|
|
try:
|
|
code = proc.wait(timeout=min(0.5, max(0.01, deadline-time.monotonic())))
|
|
state = 'ok' if code == 0 else 'failed'
|
|
break
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
stream.write((f'\njob={job["id"]} finished={now_iso()} status={state} exit_code={code}\n').encode())
|
|
result = result_for(job, state, code, time.monotonic()-start, logfile)
|
|
event = {'ok': 'JOB_SUCCESS', 'failed': 'JOB_FAILED', 'timeout': 'JOB_TIMEOUT', 'cancelled': 'JOB_CANCELLED'}[state]
|
|
hub.emit(event, **result)
|
|
return result
|
|
|
|
|
|
def live_run(jobs, args):
|
|
if not jobs:
|
|
print('Nothing enabled; no backups executed.')
|
|
return 0
|
|
if os.environ.get(CUTOVER_ENV) != 'YES':
|
|
print(f'REFUSED: Set {CUTOVER_ENV}=YES only after testing scripts and coordinating existing schedules.', file=sys.stderr)
|
|
return 2
|
|
if not nas_mount_ready():
|
|
print(f'REFUSED: {NAS_MOUNT} is not mounted from {NAS_SOURCE}', file=sys.stderr)
|
|
return 2
|
|
conflicts = legacy_schedules()
|
|
if conflicts:
|
|
print('REFUSED: existing scheduled backups could overlap this run:', file=sys.stderr)
|
|
for issue in conflicts:
|
|
print(' ' + issue, file=sys.stderr)
|
|
return 2
|
|
unsafe = [(j['id'], script_issues(j)) for j in jobs if script_issues(j)]
|
|
if unsafe:
|
|
for jid, issues in unsafe:
|
|
print(f'REFUSED: {jid}: {", ".join(issues)}', file=sys.stderr)
|
|
return 2
|
|
notifications = load_notifications(args.notify_config)
|
|
args.log_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
args.lock_file.parent.mkdir(parents=True, exist_ok=True)
|
|
with args.lock_file.open('a+') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
print('REFUSED: another orchestrator run holds the lock', file=sys.stderr)
|
|
return 3
|
|
run_id = dt.datetime.now().strftime('%Y%m%d-%H%M%S') + f'-{os.getpid()}'
|
|
folder = args.log_root / run_id
|
|
folder.mkdir(mode=0o700)
|
|
old_int = signal.getsignal(signal.SIGINT)
|
|
old_term = signal.getsignal(signal.SIGTERM)
|
|
CANCELLED.clear()
|
|
signal.signal(signal.SIGINT, lambda signum, frame: CANCELLED.set())
|
|
signal.signal(signal.SIGTERM, lambda signum, frame: CANCELLED.set())
|
|
results = []
|
|
critical_failed = False
|
|
groups = defaultdict(list)
|
|
for job in jobs:
|
|
groups[job['phase']].append(job)
|
|
began = now_iso()
|
|
notifier = Notifier(notifications, folder)
|
|
hub = EventHub(folder, args.log_root, run_id, jobs, notifier, began)
|
|
try:
|
|
hub.emit('RUN_STARTED', max_parallel=args.max_parallel, job_count=len(jobs))
|
|
for phase, members in sorted(groups.items()):
|
|
if critical_failed or CANCELLED.is_set():
|
|
reason = 'earlier critical failure' if critical_failed else 'orchestrator interrupted'
|
|
hub.emit('PHASE_BLOCKED', phase=phase, reason=reason)
|
|
for job in members:
|
|
result = result_for(job, 'blocked', reason=reason)
|
|
results.append(result)
|
|
hub.emit('JOB_BLOCKED', **result)
|
|
print(f'Phase {phase} BLOCKED: {reason}', flush=True)
|
|
continue
|
|
hub.emit('PHASE_STARTED', phase=phase, job_count=len(members))
|
|
print(f'Phase {phase}: starting {len(members)} job(s), max parallel {args.max_parallel}', flush=True)
|
|
phase_results = []
|
|
with ThreadPoolExecutor(max_workers=args.max_parallel) as pool:
|
|
future_to_job = {pool.submit(execute_one, job, folder, hub): job for job in members}
|
|
for future in as_completed(future_to_job):
|
|
job = future_to_job[future]
|
|
try:
|
|
item = future.result()
|
|
except Exception as exc:
|
|
item = result_for(job, 'error', 127, reason=f'Internal runner error: {type(exc).__name__}')
|
|
hub.emit('JOB_ERROR', **item)
|
|
phase_results.append(item)
|
|
print(f' {item["job"]}: {item["state"]} ({item["exit_code"]})', flush=True)
|
|
results.extend(sorted(phase_results, key=lambda r: r['job']))
|
|
if any(r['critical'] and r['state'] != 'ok' for r in phase_results):
|
|
critical_failed = True
|
|
hub.emit('PHASE_COMPLETED', phase=phase, failures=sum(r['state'] != 'ok' for r in phase_results),
|
|
critical_failed=critical_failed)
|
|
if critical_failed:
|
|
print(f'Phase {phase}: critical failure; subsequent phases will not start.', flush=True)
|
|
else:
|
|
print(f'Phase {phase}: barrier reached; next phase may start.', flush=True)
|
|
finally:
|
|
signal.signal(signal.SIGINT, old_int)
|
|
signal.signal(signal.SIGTERM, old_term)
|
|
overall = 'interrupted' if CANCELLED.is_set() else 'ok' if all(r['state'] == 'ok' for r in results) else 'failed'
|
|
summary = {
|
|
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
|
|
'started': began, 'finished': now_iso(), 'max_parallel': args.max_parallel,
|
|
'interrupted': CANCELLED.is_set(), 'results': results, 'overall': overall,
|
|
}
|
|
atomic_json(folder / 'summary.json', summary)
|
|
hub.emit('RUN_FINISHED', overall=overall, success_count=sum(r['state'] == 'ok' for r in results),
|
|
failure_count=sum(r['state'] != 'ok' for r in results))
|
|
delivery_failures = notifier.close()
|
|
if delivery_failures:
|
|
print(f'WARNING: {delivery_failures} notification delivery failure(s); see {folder / "notifications.log"}', flush=True)
|
|
print('Summary: ' + str(folder / 'summary.json'), flush=True)
|
|
return 130 if CANCELLED.is_set() else (1 if any(r['state'] != 'ok' for r in results) else 0)
|
|
|
|
|
|
def show_status(log_root, as_json=False):
|
|
latest = log_root / 'latest.json'
|
|
if latest.is_file():
|
|
with latest.open(encoding='utf-8') as fp:
|
|
data = json.load(fp)
|
|
if as_json:
|
|
print(json.dumps(data, ensure_ascii=False, indent=2))
|
|
return 0
|
|
print(f"Run {data['run_id']}: {data['overall']} ({data['started']} -> {data['finished'] or 'running'})")
|
|
for item in sorted(data['jobs'].values(), key=lambda x: (x['phase'], x['job'])):
|
|
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
|
|
return 0
|
|
summaries = sorted(log_root.glob('*/summary.json')) if log_root.is_dir() else []
|
|
if not summaries:
|
|
if as_json:
|
|
print(json.dumps({'overall': 'no_runs', 'jobs': {}}, indent=2))
|
|
else:
|
|
print('No runs recorded yet')
|
|
return 0
|
|
with summaries[-1].open(encoding='utf-8') as fp:
|
|
data = json.load(fp)
|
|
if as_json:
|
|
print(json.dumps(data, ensure_ascii=False, indent=2))
|
|
else:
|
|
print(f"Last run: {data['started']} -> {data['finished']} ; overall={data['overall']}")
|
|
for item in data['results']:
|
|
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
|
|
return 0
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Phase-based backup orchestrator; safe inspection by default')
|
|
mode = parser.add_mutually_exclusive_group(required=True)
|
|
mode.add_argument('--check', action='store_true', help='Validate config and inspect schedules (read only)')
|
|
mode.add_argument('--plan', action='store_true', help='Show phase plan (read only)')
|
|
mode.add_argument('--status', action='store_true', help='Latest job status (read only)')
|
|
mode.add_argument('--run', action='store_true', help='Execute enabled jobs (safety-gated)')
|
|
parser.add_argument('--json', action='store_true', help='Machine-readable result for --status')
|
|
parser.add_argument('--config', type=Path, default=DEFAULT_CONFIG)
|
|
parser.add_argument('--notify-config', type=Path, default=DEFAULT_NOTIFY_CONFIG)
|
|
parser.add_argument('--log-root', type=Path, default=DEFAULT_LOG_ROOT)
|
|
parser.add_argument('--lock-file', type=Path, default=DEFAULT_LOCK_FILE)
|
|
parser.add_argument('--max-parallel', type=int, default=3, help='Maximum concurrent jobs in a phase (1..16)')
|
|
selection = parser.add_mutually_exclusive_group()
|
|
selection.add_argument('--phase', type=int, help='Select one phase (--plan or --run)')
|
|
selection.add_argument('--job', type=str, help='Select one job (--plan or --run)')
|
|
args = parser.parse_args()
|
|
if not 1 <= args.max_parallel <= 16:
|
|
parser.error('--max-parallel must be 1..16')
|
|
if args.json and not args.status:
|
|
parser.error('--json is only valid with --status')
|
|
if (args.job or args.phase is not None) and not (args.run or args.plan):
|
|
parser.error('--phase and --job are only valid with --plan or --run')
|
|
if args.status:
|
|
return show_status(args.log_root, as_json=args.json)
|
|
jobs = load_jobs(args.config)
|
|
notify = load_notifications(args.notify_config, known_ids=[j['id'] for j in jobs])
|
|
if args.check:
|
|
return check(jobs, args.max_parallel, notify)
|
|
jobs = select_jobs(jobs, args)
|
|
if args.plan:
|
|
return plan(jobs, args.max_parallel) or 0
|
|
return live_run(jobs, args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
sys.exit(main())
|
|
except (ValueError, OSError, json.JSONDecodeError) as exc:
|
|
print(f'ERROR: {exc}', file=sys.stderr)
|
|
sys.exit(2)
|