chore: import v0.3.0 baseline and add v0.4 recovery architecture plan
This commit is contained in:
commit
40188c0e10
12 files changed
+1506
No files matched your search
+600
@@ -0,0 +1,600 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase-based backup orchestrator v0.3.
|
||||
|
||||
Safety defaults: read-only inspection unless --run, jobs enabled and explicit cutover
|
||||
approval. No scheduler, database or Docker changes are performed by this program.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import csv
|
||||
import datetime as dt
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlencode, urlsplit, urlunsplit
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
VERSION = '0.3.0'
|
||||
DEFAULT_CONFIG = Path('/etc/backup-orchestrator/jobs.csv')
|
||||
DEFAULT_NOTIFY_CONFIG = Path('/etc/backup-orchestrator/notifications.json')
|
||||
DEFAULT_LOG_ROOT = Path('/var/log/backup-orchestrator')
|
||||
DEFAULT_LOCK_FILE = Path('/run/lock/backup-orchestrator.lock')
|
||||
FIELDS = {'phase', 'id', 'script', 'critical', 'timeout_seconds', 'enabled'}
|
||||
CUTOVER_ENV = 'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED'
|
||||
NAS_MOUNT = '/backup'
|
||||
NAS_SOURCE = '192.168.2.63:/volume1/Backup_Paperless'
|
||||
CANCELLED = threading.Event()
|
||||
|
||||
|
||||
def now_iso():
|
||||
return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds')
|
||||
|
||||
|
||||
def load_jobs(path):
|
||||
with path.open(newline='', encoding='utf-8-sig') as fp:
|
||||
reader = csv.DictReader(fp)
|
||||
if reader.fieldnames is None or set(reader.fieldnames) != FIELDS or len(reader.fieldnames) != len(FIELDS):
|
||||
raise ValueError('CSV header must contain exactly: ' + ','.join(sorted(FIELDS)))
|
||||
rows = list(reader)
|
||||
if not rows:
|
||||
raise ValueError('CSV contains no jobs')
|
||||
ids = set()
|
||||
jobs = []
|
||||
for index, row in enumerate(rows, start=2):
|
||||
if None in row or any(v is None for v in row.values()):
|
||||
raise ValueError(f'CSV line {index}: wrong number of columns')
|
||||
row = {k: v.strip() for k, v in row.items()}
|
||||
jid = row['id']
|
||||
if not re.fullmatch(r'[a-z][a-z0-9-]*', jid) or jid in ids:
|
||||
raise ValueError(f'CSV line {index}: invalid or duplicate job ID: {jid!r}')
|
||||
ids.add(jid)
|
||||
if row['critical'] not in ('true', 'false') or row['enabled'] not in ('true', 'false'):
|
||||
raise ValueError(f'CSV line {index}: critical and enabled must be true or false')
|
||||
if not re.fullmatch(r'\d+', row['phase']) or int(row['phase']) < 1:
|
||||
raise ValueError(f'CSV line {index}: phase must be a positive integer')
|
||||
if not re.fullmatch(r'\d+', row['timeout_seconds']) or not 1 <= int(row['timeout_seconds']) <= 86400:
|
||||
raise ValueError(f'CSV line {index}: timeout_seconds must be 1..86400')
|
||||
if not Path(row['script']).is_absolute():
|
||||
raise ValueError(f'CSV line {index}: script must be an absolute path')
|
||||
jobs.append({
|
||||
'phase': int(row['phase']), 'id': jid, 'script': row['script'],
|
||||
'critical': row['critical'] == 'true',
|
||||
'timeout_seconds': int(row['timeout_seconds']),
|
||||
'enabled': row['enabled'] == 'true',
|
||||
})
|
||||
return sorted(jobs, key=lambda x: (x['phase'], x['id']))
|
||||
|
||||
|
||||
def push_base_url(value):
|
||||
"""Accept Kuma's full copied URL and discard its example ?status=... query."""
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in ('http', 'https') or not parsed.netloc or not parsed.path.startswith('/api/push/'):
|
||||
raise ValueError('Kuma Push URL must be an http(s) URL with /api/push/<token>')
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValueError('HTTP basic-auth in Push URL is unsupported')
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, '', ''))
|
||||
|
||||
|
||||
def verify_http_url(value):
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in ('http', 'https') or not parsed.netloc:
|
||||
raise ValueError('Webhook URL must start with http:// or https://')
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValueError('Webhook URL must not contain user credentials')
|
||||
return value
|
||||
|
||||
|
||||
def load_notifications(path, known_ids=None):
|
||||
"""Only outgoing HTTP. URLs are never printed in status/logs."""
|
||||
default = {'timeout_seconds': 3, 'webhooks': [], 'kuma': {'overall_push_url': '', 'job_push_urls': {}}}
|
||||
if not path.is_file():
|
||||
return default
|
||||
with path.open(encoding='utf-8') as fp:
|
||||
obj = json.load(fp)
|
||||
if not isinstance(obj, dict) or set(obj) - {'timeout_seconds', 'webhooks', 'kuma'}:
|
||||
raise ValueError('Invalid notifications.json root keys')
|
||||
timeout = obj.get('timeout_seconds', 3)
|
||||
if type(timeout) not in (int, float) or not 0.5 <= timeout <= 15:
|
||||
raise ValueError('timeout_seconds must be 0.5..15')
|
||||
webhooks = obj.get('webhooks', [])
|
||||
kuma = obj.get('kuma', {})
|
||||
if not isinstance(webhooks, list) or any(not isinstance(x, str) for x in webhooks):
|
||||
raise ValueError('webhooks must be a list of URLs')
|
||||
if not isinstance(kuma, dict) or set(kuma) - {'overall_push_url', 'job_push_urls'}:
|
||||
raise ValueError('Invalid kuma settings')
|
||||
overall = kuma.get('overall_push_url', '')
|
||||
per_job = kuma.get('job_push_urls', {})
|
||||
if not isinstance(overall, str) or not isinstance(per_job, dict):
|
||||
raise ValueError('Kuma URLs must be strings; job_push_urls must be an object')
|
||||
if any(not isinstance(k, str) or not isinstance(v, str) for k, v in per_job.items()):
|
||||
raise ValueError('Kuma job IDs and URLs must be strings')
|
||||
if known_ids is not None and set(per_job) - set(known_ids):
|
||||
raise ValueError('Unknown Kuma job ID(s): ' + ', '.join(sorted(set(per_job) - set(known_ids))))
|
||||
return {
|
||||
'timeout_seconds': timeout,
|
||||
'webhooks': [verify_http_url(x) for x in webhooks],
|
||||
'kuma': {
|
||||
'overall_push_url': push_base_url(overall) if overall else '',
|
||||
'job_push_urls': {k: push_base_url(v) if v else '' for k, v in per_job.items()},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def notification_counts(config):
|
||||
kuma = config['kuma']
|
||||
return len(config['webhooks']), int(bool(kuma['overall_push_url'])), sum(bool(v) for v in kuma['job_push_urls'].values())
|
||||
|
||||
|
||||
def nas_mount_ready():
|
||||
try:
|
||||
result = subprocess.run(['findmnt', '--mountpoint', NAS_MOUNT, '--types', 'nfs4', '--source', NAS_SOURCE],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=8)
|
||||
return result.returncode == 0
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
|
||||
|
||||
def legacy_schedules():
|
||||
"""Best-effort detection of active legacy schedules (never comments)."""
|
||||
found = []
|
||||
for user in ('root', 'lars'):
|
||||
try:
|
||||
p = subprocess.run(['crontab', '-u', user, '-l'], capture_output=True, text=True, timeout=8)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
continue
|
||||
for line in p.stdout.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith('#') or not stripped:
|
||||
continue
|
||||
if re.search(r'/home/lars/(?:db-[\w-]+\.sh|backupPaperlessData\.sh)', stripped):
|
||||
found.append(f'{user} crontab: {stripped}')
|
||||
for timer in ('raspiBackup.timer', 'paperless-data-backup.timer', 'borgmatic.timer'):
|
||||
try:
|
||||
enabled = subprocess.run(['systemctl', 'is-enabled', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
||||
active = subprocess.run(['systemctl', 'is-active', timer], capture_output=True, text=True, timeout=8).stdout.strip()
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
continue
|
||||
if enabled in ('enabled', 'enabled-runtime', 'linked', 'linked-runtime') or active == 'active':
|
||||
found.append(f'{timer}: enabled={enabled or "unknown"}, active={active or "unknown"}')
|
||||
return found
|
||||
|
||||
|
||||
def script_issues(job):
|
||||
p = Path(job['script'])
|
||||
if not p.is_file():
|
||||
return ['MISSING']
|
||||
issues = []
|
||||
if not os.access(p, os.X_OK):
|
||||
issues.append('NOT EXECUTABLE')
|
||||
try:
|
||||
with p.open('rb') as fp:
|
||||
if fp.read(2) != b'#!':
|
||||
issues.append('NO SHEBANG')
|
||||
except OSError:
|
||||
issues.append('NOT READABLE')
|
||||
return issues
|
||||
|
||||
|
||||
def check(jobs, max_parallel, notify):
|
||||
print(f'Backup-Orchestrator {VERSION}: INSPECTION ONLY (no backups, no schedule changes)')
|
||||
print(f'Parallel limit per phase: {max_parallel}')
|
||||
for job in jobs:
|
||||
issues = script_issues(job)
|
||||
state = 'enabled' if job['enabled'] else 'disabled'
|
||||
print(f"phase {job['phase']:>2} {job['id']:<24} {state:<8} {job['timeout_seconds']:>5}s {job['script']} [{', '.join(issues) if issues else 'OK'}]")
|
||||
conflicts = legacy_schedules()
|
||||
print('\nLegacy schedules:')
|
||||
for line in conflicts:
|
||||
print(' ACTIVE: ' + line)
|
||||
if not conflicts:
|
||||
print(' None detected (best effort)')
|
||||
print(f'\nNAS mount {NAS_MOUNT}: {"OK" if nas_mount_ready() else "NOT AVAILABLE"} (expected {NAS_SOURCE})')
|
||||
a, b, c = notification_counts(notify)
|
||||
print(f'Notifications configured: webhooks={a}, Kuma overall={b}, Kuma jobs={c} (tokens hidden)')
|
||||
print('Same phase: parallel; next phase starts after all members complete.')
|
||||
print('Critical job failure: later phases BLOCKED; in-flight phase members finish.')
|
||||
print('No jobs execute without --run, enabled=true and explicit cutover approval.')
|
||||
return 0
|
||||
|
||||
|
||||
def select_jobs(jobs, args):
|
||||
enabled = [j for j in jobs if j['enabled']]
|
||||
if args.job:
|
||||
matches = [j for j in enabled if j['id'] == args.job]
|
||||
if not matches:
|
||||
raise ValueError(f'Job {args.job!r} not found or disabled')
|
||||
return matches
|
||||
if args.phase is not None:
|
||||
matches = [j for j in enabled if j['phase'] == args.phase]
|
||||
if not matches:
|
||||
raise ValueError(f'No enabled jobs in phase {args.phase}')
|
||||
return matches
|
||||
return enabled
|
||||
|
||||
|
||||
def plan(jobs, max_parallel):
|
||||
groups = defaultdict(list)
|
||||
for job in jobs:
|
||||
groups[job['phase']].append(job)
|
||||
print(f'MODE: PLAN ONLY — no jobs executed. Max parallel jobs per phase: {max_parallel}')
|
||||
for phase, members in sorted(groups.items()):
|
||||
print(f'Phase {phase} ({len(members)} job(s)):')
|
||||
for job in members:
|
||||
print(f" - {job['id']} ({'CRITICAL' if job['critical'] else 'optional'}, timeout {job['timeout_seconds']}s)")
|
||||
print(' BARRIER: wait for all jobs before advancing')
|
||||
|
||||
|
||||
def result_for(job, state, rc=None, seconds=0.0, logfile=None, reason=None):
|
||||
result = dict(phase=job['phase'], job=job['id'], critical=job['critical'],
|
||||
state=state, exit_code=rc, duration_seconds=round(seconds, 2))
|
||||
if logfile is not None:
|
||||
result['log'] = str(logfile)
|
||||
if reason:
|
||||
result['reason'] = reason
|
||||
return result
|
||||
|
||||
|
||||
def atomic_json(path, data):
|
||||
tmp = path.with_name(f'.{path.name}.{os.getpid()}.{threading.get_ident()}.tmp')
|
||||
try:
|
||||
with tmp.open('w', encoding='utf-8') as fp:
|
||||
json.dump(data, fp, ensure_ascii=False, indent=2)
|
||||
fp.write('\n')
|
||||
tmp.replace(path)
|
||||
finally:
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
|
||||
|
||||
class Notifier:
|
||||
"""Outbound webhooks + Kuma adapter; notification errors never alter backup exit codes."""
|
||||
def __init__(self, config, folder):
|
||||
self.config = config
|
||||
self.folder = folder
|
||||
self.pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix='backup-notify')
|
||||
self.futures = []
|
||||
self.lock = threading.Lock()
|
||||
self.errors = 0
|
||||
self.logfile = folder / 'notifications.log'
|
||||
|
||||
def _send_json(self, url, event):
|
||||
payload = json.dumps(event, ensure_ascii=False).encode('utf-8')
|
||||
req = Request(url, data=payload, method='POST', headers={'Content-Type': 'application/json', 'User-Agent': 'pi-backup-orchestrator/0.3'})
|
||||
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
|
||||
if not 200 <= response.status < 300:
|
||||
raise RuntimeError(f'HTTP {response.status}')
|
||||
|
||||
def _send_kuma(self, url, status, message):
|
||||
sep = '&' if '?' in url else '?'
|
||||
address = url + sep + urlencode({'status': status, 'msg': message[:180]})
|
||||
req = Request(address, headers={'User-Agent': 'pi-backup-orchestrator/0.3'})
|
||||
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
|
||||
if not 200 <= response.status < 300:
|
||||
raise RuntimeError(f'HTTP {response.status}')
|
||||
# Kuma usually responds with JSON {ok:true}; a 200 alone is insufficient if it says failure.
|
||||
body = response.read(4096)
|
||||
if body:
|
||||
try:
|
||||
parsed = json.loads(body)
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
parsed = None
|
||||
if isinstance(parsed, dict) and parsed.get('ok') is False:
|
||||
raise RuntimeError('Kuma rejected push')
|
||||
|
||||
def _submit(self, channel, fn, *args):
|
||||
def guarded():
|
||||
try:
|
||||
fn(*args)
|
||||
except Exception as exc:
|
||||
# Never log target URLs or tokens; even HTTP exceptions may include their URL.
|
||||
if hasattr(exc, 'code') and isinstance(exc.code, int):
|
||||
detail = f'HTTP {exc.code}'
|
||||
else:
|
||||
detail = type(exc).__name__
|
||||
with self.lock:
|
||||
self.errors += 1
|
||||
with self.logfile.open('a', encoding='utf-8') as out:
|
||||
out.write(f'{now_iso()} {channel}: delivery failed ({detail})\n')
|
||||
self.futures.append(self.pool.submit(guarded))
|
||||
|
||||
def notify(self, event):
|
||||
for url in self.config['webhooks']:
|
||||
self._submit('webhook', self._send_json, url, event)
|
||||
name = event['event']
|
||||
if name in ('JOB_SUCCESS', 'JOB_FAILED', 'JOB_TIMEOUT', 'JOB_CANCELLED', 'JOB_ERROR', 'JOB_BLOCKED'):
|
||||
url = self.config['kuma']['job_push_urls'].get(event.get('job'), '')
|
||||
if url:
|
||||
status = 'up' if name == 'JOB_SUCCESS' else 'down'
|
||||
msg = f"{event['job']}: {event.get('state', 'unknown')} (run {event['run_id']})"
|
||||
self._submit('kuma/job', self._send_kuma, url, status, msg)
|
||||
if name == 'RUN_FINISHED':
|
||||
url = self.config['kuma']['overall_push_url']
|
||||
if url:
|
||||
status = 'up' if event.get('overall') == 'ok' else 'down'
|
||||
self._submit('kuma/overall', self._send_kuma, url, status, f"Run {event['run_id']}: {event.get('overall')}")
|
||||
|
||||
def close(self):
|
||||
self.pool.shutdown(wait=True)
|
||||
return self.errors
|
||||
|
||||
|
||||
class EventHub:
|
||||
"""Write append-only events.jsonl plus atomic current snapshot for consumers."""
|
||||
def __init__(self, folder, log_root, run_id, jobs, notifier, started):
|
||||
self.folder = folder
|
||||
self.log_root = log_root
|
||||
self.run_id = run_id
|
||||
self.notifier = notifier
|
||||
self.lock = threading.RLock()
|
||||
self.snapshot = {
|
||||
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
|
||||
'started': started, 'updated': started, 'finished': None,
|
||||
'overall': 'running',
|
||||
'jobs': {j['id']: result_for(j, 'pending') for j in jobs},
|
||||
}
|
||||
atomic_json(folder / 'status.json', self.snapshot)
|
||||
atomic_json(log_root / 'latest.json', self.snapshot)
|
||||
|
||||
def emit(self, event_name, **fields):
|
||||
event = {
|
||||
'event': event_name, 'timestamp': now_iso(), 'run_id': self.run_id,
|
||||
'host': self.snapshot['host'], **fields,
|
||||
}
|
||||
with self.lock:
|
||||
if 'job' in event and event['job'] in self.snapshot['jobs'] and 'state' in event:
|
||||
jid = event['job']
|
||||
self.snapshot['jobs'][jid].update({k: v for k, v in fields.items() if k in (
|
||||
'state', 'phase', 'critical', 'exit_code', 'duration_seconds', 'log', 'reason')})
|
||||
if event_name == 'RUN_FINISHED':
|
||||
self.snapshot['overall'] = fields['overall']
|
||||
self.snapshot['finished'] = event['timestamp']
|
||||
self.snapshot['updated'] = event['timestamp']
|
||||
with (self.folder / 'events.jsonl').open('a', encoding='utf-8') as fp:
|
||||
fp.write(json.dumps(event, ensure_ascii=False) + '\n')
|
||||
atomic_json(self.folder / 'status.json', self.snapshot)
|
||||
atomic_json(self.log_root / 'latest.json', self.snapshot)
|
||||
self.notifier.notify(event)
|
||||
return event
|
||||
|
||||
|
||||
def terminate_group(proc):
|
||||
"""Terminate ordinary children of a job and wait for shutdown before advancing phase."""
|
||||
if proc.poll() is not None:
|
||||
return
|
||||
try:
|
||||
os.killpg(proc.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
proc.wait()
|
||||
|
||||
|
||||
def execute_one(job, folder, hub):
|
||||
start = time.monotonic()
|
||||
logfile = folder / (job['id'] + '.log')
|
||||
if CANCELLED.is_set():
|
||||
result = result_for(job, 'cancelled', reason='Orchestrator interrupted')
|
||||
hub.emit('JOB_CANCELLED', **result)
|
||||
return result
|
||||
hub.emit('JOB_STARTED', job=job['id'], phase=job['phase'], critical=job['critical'], state='running')
|
||||
with logfile.open('wb') as stream:
|
||||
stream.write((f"job={job['id']} started={now_iso()}\n").encode())
|
||||
stream.flush()
|
||||
try:
|
||||
proc = subprocess.Popen([job['script']], stdout=stream, stderr=subprocess.STDOUT,
|
||||
start_new_session=True, cwd=str(Path(job['script']).parent))
|
||||
except OSError as exc:
|
||||
stream.write((f'EXEC ERROR: {exc}\n').encode())
|
||||
result = result_for(job, 'error', 127, time.monotonic()-start, logfile, 'Could not start script')
|
||||
hub.emit('JOB_ERROR', **result)
|
||||
return result
|
||||
deadline = time.monotonic() + job['timeout_seconds']
|
||||
while True:
|
||||
if CANCELLED.is_set():
|
||||
terminate_group(proc)
|
||||
state, code = 'cancelled', 130
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
terminate_group(proc)
|
||||
state, code = 'timeout', 124
|
||||
break
|
||||
try:
|
||||
code = proc.wait(timeout=min(0.5, max(0.01, deadline-time.monotonic())))
|
||||
state = 'ok' if code == 0 else 'failed'
|
||||
break
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
stream.write((f'\njob={job["id"]} finished={now_iso()} status={state} exit_code={code}\n').encode())
|
||||
result = result_for(job, state, code, time.monotonic()-start, logfile)
|
||||
event = {'ok': 'JOB_SUCCESS', 'failed': 'JOB_FAILED', 'timeout': 'JOB_TIMEOUT', 'cancelled': 'JOB_CANCELLED'}[state]
|
||||
hub.emit(event, **result)
|
||||
return result
|
||||
|
||||
|
||||
def live_run(jobs, args):
|
||||
if not jobs:
|
||||
print('Nothing enabled; no backups executed.')
|
||||
return 0
|
||||
if os.environ.get(CUTOVER_ENV) != 'YES':
|
||||
print(f'REFUSED: Set {CUTOVER_ENV}=YES only after testing scripts and coordinating existing schedules.', file=sys.stderr)
|
||||
return 2
|
||||
if not nas_mount_ready():
|
||||
print(f'REFUSED: {NAS_MOUNT} is not mounted from {NAS_SOURCE}', file=sys.stderr)
|
||||
return 2
|
||||
conflicts = legacy_schedules()
|
||||
if conflicts:
|
||||
print('REFUSED: existing scheduled backups could overlap this run:', file=sys.stderr)
|
||||
for issue in conflicts:
|
||||
print(' ' + issue, file=sys.stderr)
|
||||
return 2
|
||||
unsafe = [(j['id'], script_issues(j)) for j in jobs if script_issues(j)]
|
||||
if unsafe:
|
||||
for jid, issues in unsafe:
|
||||
print(f'REFUSED: {jid}: {", ".join(issues)}', file=sys.stderr)
|
||||
return 2
|
||||
notifications = load_notifications(args.notify_config)
|
||||
args.log_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
args.lock_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.lock_file.open('a+') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
print('REFUSED: another orchestrator run holds the lock', file=sys.stderr)
|
||||
return 3
|
||||
run_id = dt.datetime.now().strftime('%Y%m%d-%H%M%S') + f'-{os.getpid()}'
|
||||
folder = args.log_root / run_id
|
||||
folder.mkdir(mode=0o700)
|
||||
old_int = signal.getsignal(signal.SIGINT)
|
||||
old_term = signal.getsignal(signal.SIGTERM)
|
||||
CANCELLED.clear()
|
||||
signal.signal(signal.SIGINT, lambda signum, frame: CANCELLED.set())
|
||||
signal.signal(signal.SIGTERM, lambda signum, frame: CANCELLED.set())
|
||||
results = []
|
||||
critical_failed = False
|
||||
groups = defaultdict(list)
|
||||
for job in jobs:
|
||||
groups[job['phase']].append(job)
|
||||
began = now_iso()
|
||||
notifier = Notifier(notifications, folder)
|
||||
hub = EventHub(folder, args.log_root, run_id, jobs, notifier, began)
|
||||
try:
|
||||
hub.emit('RUN_STARTED', max_parallel=args.max_parallel, job_count=len(jobs))
|
||||
for phase, members in sorted(groups.items()):
|
||||
if critical_failed or CANCELLED.is_set():
|
||||
reason = 'earlier critical failure' if critical_failed else 'orchestrator interrupted'
|
||||
hub.emit('PHASE_BLOCKED', phase=phase, reason=reason)
|
||||
for job in members:
|
||||
result = result_for(job, 'blocked', reason=reason)
|
||||
results.append(result)
|
||||
hub.emit('JOB_BLOCKED', **result)
|
||||
print(f'Phase {phase} BLOCKED: {reason}', flush=True)
|
||||
continue
|
||||
hub.emit('PHASE_STARTED', phase=phase, job_count=len(members))
|
||||
print(f'Phase {phase}: starting {len(members)} job(s), max parallel {args.max_parallel}', flush=True)
|
||||
phase_results = []
|
||||
with ThreadPoolExecutor(max_workers=args.max_parallel) as pool:
|
||||
future_to_job = {pool.submit(execute_one, job, folder, hub): job for job in members}
|
||||
for future in as_completed(future_to_job):
|
||||
job = future_to_job[future]
|
||||
try:
|
||||
item = future.result()
|
||||
except Exception as exc:
|
||||
item = result_for(job, 'error', 127, reason=f'Internal runner error: {type(exc).__name__}')
|
||||
hub.emit('JOB_ERROR', **item)
|
||||
phase_results.append(item)
|
||||
print(f' {item["job"]}: {item["state"]} ({item["exit_code"]})', flush=True)
|
||||
results.extend(sorted(phase_results, key=lambda r: r['job']))
|
||||
if any(r['critical'] and r['state'] != 'ok' for r in phase_results):
|
||||
critical_failed = True
|
||||
hub.emit('PHASE_COMPLETED', phase=phase, failures=sum(r['state'] != 'ok' for r in phase_results),
|
||||
critical_failed=critical_failed)
|
||||
if critical_failed:
|
||||
print(f'Phase {phase}: critical failure; subsequent phases will not start.', flush=True)
|
||||
else:
|
||||
print(f'Phase {phase}: barrier reached; next phase may start.', flush=True)
|
||||
finally:
|
||||
signal.signal(signal.SIGINT, old_int)
|
||||
signal.signal(signal.SIGTERM, old_term)
|
||||
overall = 'interrupted' if CANCELLED.is_set() else 'ok' if all(r['state'] == 'ok' for r in results) else 'failed'
|
||||
summary = {
|
||||
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
|
||||
'started': began, 'finished': now_iso(), 'max_parallel': args.max_parallel,
|
||||
'interrupted': CANCELLED.is_set(), 'results': results, 'overall': overall,
|
||||
}
|
||||
atomic_json(folder / 'summary.json', summary)
|
||||
hub.emit('RUN_FINISHED', overall=overall, success_count=sum(r['state'] == 'ok' for r in results),
|
||||
failure_count=sum(r['state'] != 'ok' for r in results))
|
||||
delivery_failures = notifier.close()
|
||||
if delivery_failures:
|
||||
print(f'WARNING: {delivery_failures} notification delivery failure(s); see {folder / "notifications.log"}', flush=True)
|
||||
print('Summary: ' + str(folder / 'summary.json'), flush=True)
|
||||
return 130 if CANCELLED.is_set() else (1 if any(r['state'] != 'ok' for r in results) else 0)
|
||||
|
||||
|
||||
def show_status(log_root, as_json=False):
|
||||
latest = log_root / 'latest.json'
|
||||
if latest.is_file():
|
||||
with latest.open(encoding='utf-8') as fp:
|
||||
data = json.load(fp)
|
||||
if as_json:
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
print(f"Run {data['run_id']}: {data['overall']} ({data['started']} -> {data['finished'] or 'running'})")
|
||||
for item in sorted(data['jobs'].values(), key=lambda x: (x['phase'], x['job'])):
|
||||
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
|
||||
return 0
|
||||
summaries = sorted(log_root.glob('*/summary.json')) if log_root.is_dir() else []
|
||||
if not summaries:
|
||||
if as_json:
|
||||
print(json.dumps({'overall': 'no_runs', 'jobs': {}}, indent=2))
|
||||
else:
|
||||
print('No runs recorded yet')
|
||||
return 0
|
||||
with summaries[-1].open(encoding='utf-8') as fp:
|
||||
data = json.load(fp)
|
||||
if as_json:
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"Last run: {data['started']} -> {data['finished']} ; overall={data['overall']}")
|
||||
for item in data['results']:
|
||||
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Phase-based backup orchestrator; safe inspection by default')
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--check', action='store_true', help='Validate config and inspect schedules (read only)')
|
||||
mode.add_argument('--plan', action='store_true', help='Show phase plan (read only)')
|
||||
mode.add_argument('--status', action='store_true', help='Latest job status (read only)')
|
||||
mode.add_argument('--run', action='store_true', help='Execute enabled jobs (safety-gated)')
|
||||
parser.add_argument('--json', action='store_true', help='Machine-readable result for --status')
|
||||
parser.add_argument('--config', type=Path, default=DEFAULT_CONFIG)
|
||||
parser.add_argument('--notify-config', type=Path, default=DEFAULT_NOTIFY_CONFIG)
|
||||
parser.add_argument('--log-root', type=Path, default=DEFAULT_LOG_ROOT)
|
||||
parser.add_argument('--lock-file', type=Path, default=DEFAULT_LOCK_FILE)
|
||||
parser.add_argument('--max-parallel', type=int, default=3, help='Maximum concurrent jobs in a phase (1..16)')
|
||||
selection = parser.add_mutually_exclusive_group()
|
||||
selection.add_argument('--phase', type=int, help='Select one phase (--plan or --run)')
|
||||
selection.add_argument('--job', type=str, help='Select one job (--plan or --run)')
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.max_parallel <= 16:
|
||||
parser.error('--max-parallel must be 1..16')
|
||||
if args.json and not args.status:
|
||||
parser.error('--json is only valid with --status')
|
||||
if (args.job or args.phase is not None) and not (args.run or args.plan):
|
||||
parser.error('--phase and --job are only valid with --plan or --run')
|
||||
if args.status:
|
||||
return show_status(args.log_root, as_json=args.json)
|
||||
jobs = load_jobs(args.config)
|
||||
notify = load_notifications(args.notify_config, known_ids=[j['id'] for j in jobs])
|
||||
if args.check:
|
||||
return check(jobs, args.max_parallel, notify)
|
||||
jobs = select_jobs(jobs, args)
|
||||
if args.plan:
|
||||
return plan(jobs, args.max_parallel) or 0
|
||||
return live_run(jobs, args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
sys.exit(main())
|
||||
except (ValueError, OSError, json.JSONDecodeError) as exc:
|
||||
print(f'ERROR: {exc}', file=sys.stderr)
|
||||
sys.exit(2)
|
||||
Reference in new issue
Block a user