Files
backup-orchestrator/orchestrator.py
T

601 lines
28 KiB
Python

#!/usr/bin/env python3
"""Phase-based backup orchestrator v0.3.
Safety defaults: read-only inspection unless --run, jobs enabled and explicit cutover
approval. No scheduler, database or Docker changes are performed by this program.
"""
import argparse
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
import csv
import datetime as dt
import fcntl
import json
import os
from pathlib import Path
import re
import signal
import socket
import subprocess
import sys
import threading
import time
from urllib.parse import urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
VERSION = '0.3.0'
DEFAULT_CONFIG = Path('/etc/backup-orchestrator/jobs.csv')
DEFAULT_NOTIFY_CONFIG = Path('/etc/backup-orchestrator/notifications.json')
DEFAULT_LOG_ROOT = Path('/var/log/backup-orchestrator')
DEFAULT_LOCK_FILE = Path('/run/lock/backup-orchestrator.lock')
FIELDS = {'phase', 'id', 'script', 'critical', 'timeout_seconds', 'enabled'}
CUTOVER_ENV = 'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED'
NAS_MOUNT = '/backup'
NAS_SOURCE = '192.168.2.63:/volume1/Backup_Paperless'
CANCELLED = threading.Event()
def now_iso():
return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds')
def load_jobs(path):
with path.open(newline='', encoding='utf-8-sig') as fp:
reader = csv.DictReader(fp)
if reader.fieldnames is None or set(reader.fieldnames) != FIELDS or len(reader.fieldnames) != len(FIELDS):
raise ValueError('CSV header must contain exactly: ' + ','.join(sorted(FIELDS)))
rows = list(reader)
if not rows:
raise ValueError('CSV contains no jobs')
ids = set()
jobs = []
for index, row in enumerate(rows, start=2):
if None in row or any(v is None for v in row.values()):
raise ValueError(f'CSV line {index}: wrong number of columns')
row = {k: v.strip() for k, v in row.items()}
jid = row['id']
if not re.fullmatch(r'[a-z][a-z0-9-]*', jid) or jid in ids:
raise ValueError(f'CSV line {index}: invalid or duplicate job ID: {jid!r}')
ids.add(jid)
if row['critical'] not in ('true', 'false') or row['enabled'] not in ('true', 'false'):
raise ValueError(f'CSV line {index}: critical and enabled must be true or false')
if not re.fullmatch(r'\d+', row['phase']) or int(row['phase']) < 1:
raise ValueError(f'CSV line {index}: phase must be a positive integer')
if not re.fullmatch(r'\d+', row['timeout_seconds']) or not 1 <= int(row['timeout_seconds']) <= 86400:
raise ValueError(f'CSV line {index}: timeout_seconds must be 1..86400')
if not Path(row['script']).is_absolute():
raise ValueError(f'CSV line {index}: script must be an absolute path')
jobs.append({
'phase': int(row['phase']), 'id': jid, 'script': row['script'],
'critical': row['critical'] == 'true',
'timeout_seconds': int(row['timeout_seconds']),
'enabled': row['enabled'] == 'true',
})
return sorted(jobs, key=lambda x: (x['phase'], x['id']))
def push_base_url(value):
"""Accept Kuma's full copied URL and discard its example ?status=... query."""
parsed = urlsplit(value)
if parsed.scheme not in ('http', 'https') or not parsed.netloc or not parsed.path.startswith('/api/push/'):
raise ValueError('Kuma Push URL must be an http(s) URL with /api/push/<token>')
if parsed.username is not None or parsed.password is not None:
raise ValueError('HTTP basic-auth in Push URL is unsupported')
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, '', ''))
def verify_http_url(value):
parsed = urlsplit(value)
if parsed.scheme not in ('http', 'https') or not parsed.netloc:
raise ValueError('Webhook URL must start with http:// or https://')
if parsed.username is not None or parsed.password is not None:
raise ValueError('Webhook URL must not contain user credentials')
return value
def load_notifications(path, known_ids=None):
"""Only outgoing HTTP. URLs are never printed in status/logs."""
default = {'timeout_seconds': 3, 'webhooks': [], 'kuma': {'overall_push_url': '', 'job_push_urls': {}}}
if not path.is_file():
return default
with path.open(encoding='utf-8') as fp:
obj = json.load(fp)
if not isinstance(obj, dict) or set(obj) - {'timeout_seconds', 'webhooks', 'kuma'}:
raise ValueError('Invalid notifications.json root keys')
timeout = obj.get('timeout_seconds', 3)
if type(timeout) not in (int, float) or not 0.5 <= timeout <= 15:
raise ValueError('timeout_seconds must be 0.5..15')
webhooks = obj.get('webhooks', [])
kuma = obj.get('kuma', {})
if not isinstance(webhooks, list) or any(not isinstance(x, str) for x in webhooks):
raise ValueError('webhooks must be a list of URLs')
if not isinstance(kuma, dict) or set(kuma) - {'overall_push_url', 'job_push_urls'}:
raise ValueError('Invalid kuma settings')
overall = kuma.get('overall_push_url', '')
per_job = kuma.get('job_push_urls', {})
if not isinstance(overall, str) or not isinstance(per_job, dict):
raise ValueError('Kuma URLs must be strings; job_push_urls must be an object')
if any(not isinstance(k, str) or not isinstance(v, str) for k, v in per_job.items()):
raise ValueError('Kuma job IDs and URLs must be strings')
if known_ids is not None and set(per_job) - set(known_ids):
raise ValueError('Unknown Kuma job ID(s): ' + ', '.join(sorted(set(per_job) - set(known_ids))))
return {
'timeout_seconds': timeout,
'webhooks': [verify_http_url(x) for x in webhooks],
'kuma': {
'overall_push_url': push_base_url(overall) if overall else '',
'job_push_urls': {k: push_base_url(v) if v else '' for k, v in per_job.items()},
},
}
def notification_counts(config):
kuma = config['kuma']
return len(config['webhooks']), int(bool(kuma['overall_push_url'])), sum(bool(v) for v in kuma['job_push_urls'].values())
def nas_mount_ready():
try:
result = subprocess.run(['findmnt', '--mountpoint', NAS_MOUNT, '--types', 'nfs4', '--source', NAS_SOURCE],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=8)
return result.returncode == 0
except (OSError, subprocess.TimeoutExpired):
return False
def legacy_schedules():
"""Best-effort detection of active legacy schedules (never comments)."""
found = []
for user in ('root', 'lars'):
try:
p = subprocess.run(['crontab', '-u', user, '-l'], capture_output=True, text=True, timeout=8)
except (OSError, subprocess.TimeoutExpired):
continue
for line in p.stdout.splitlines():
stripped = line.strip()
if stripped.startswith('#') or not stripped:
continue
if re.search(r'/home/lars/(?:db-[\w-]+\.sh|backupPaperlessData\.sh)', stripped):
found.append(f'{user} crontab: {stripped}')
for timer in ('raspiBackup.timer', 'paperless-data-backup.timer', 'borgmatic.timer'):
try:
enabled = subprocess.run(['systemctl', 'is-enabled', timer], capture_output=True, text=True, timeout=8).stdout.strip()
active = subprocess.run(['systemctl', 'is-active', timer], capture_output=True, text=True, timeout=8).stdout.strip()
except (OSError, subprocess.TimeoutExpired):
continue
if enabled in ('enabled', 'enabled-runtime', 'linked', 'linked-runtime') or active == 'active':
found.append(f'{timer}: enabled={enabled or "unknown"}, active={active or "unknown"}')
return found
def script_issues(job):
p = Path(job['script'])
if not p.is_file():
return ['MISSING']
issues = []
if not os.access(p, os.X_OK):
issues.append('NOT EXECUTABLE')
try:
with p.open('rb') as fp:
if fp.read(2) != b'#!':
issues.append('NO SHEBANG')
except OSError:
issues.append('NOT READABLE')
return issues
def check(jobs, max_parallel, notify):
print(f'Backup-Orchestrator {VERSION}: INSPECTION ONLY (no backups, no schedule changes)')
print(f'Parallel limit per phase: {max_parallel}')
for job in jobs:
issues = script_issues(job)
state = 'enabled' if job['enabled'] else 'disabled'
print(f"phase {job['phase']:>2} {job['id']:<24} {state:<8} {job['timeout_seconds']:>5}s {job['script']} [{', '.join(issues) if issues else 'OK'}]")
conflicts = legacy_schedules()
print('\nLegacy schedules:')
for line in conflicts:
print(' ACTIVE: ' + line)
if not conflicts:
print(' None detected (best effort)')
print(f'\nNAS mount {NAS_MOUNT}: {"OK" if nas_mount_ready() else "NOT AVAILABLE"} (expected {NAS_SOURCE})')
a, b, c = notification_counts(notify)
print(f'Notifications configured: webhooks={a}, Kuma overall={b}, Kuma jobs={c} (tokens hidden)')
print('Same phase: parallel; next phase starts after all members complete.')
print('Critical job failure: later phases BLOCKED; in-flight phase members finish.')
print('No jobs execute without --run, enabled=true and explicit cutover approval.')
return 0
def select_jobs(jobs, args):
enabled = [j for j in jobs if j['enabled']]
if args.job:
matches = [j for j in enabled if j['id'] == args.job]
if not matches:
raise ValueError(f'Job {args.job!r} not found or disabled')
return matches
if args.phase is not None:
matches = [j for j in enabled if j['phase'] == args.phase]
if not matches:
raise ValueError(f'No enabled jobs in phase {args.phase}')
return matches
return enabled
def plan(jobs, max_parallel):
groups = defaultdict(list)
for job in jobs:
groups[job['phase']].append(job)
print(f'MODE: PLAN ONLY — no jobs executed. Max parallel jobs per phase: {max_parallel}')
for phase, members in sorted(groups.items()):
print(f'Phase {phase} ({len(members)} job(s)):')
for job in members:
print(f" - {job['id']} ({'CRITICAL' if job['critical'] else 'optional'}, timeout {job['timeout_seconds']}s)")
print(' BARRIER: wait for all jobs before advancing')
def result_for(job, state, rc=None, seconds=0.0, logfile=None, reason=None):
result = dict(phase=job['phase'], job=job['id'], critical=job['critical'],
state=state, exit_code=rc, duration_seconds=round(seconds, 2))
if logfile is not None:
result['log'] = str(logfile)
if reason:
result['reason'] = reason
return result
def atomic_json(path, data):
tmp = path.with_name(f'.{path.name}.{os.getpid()}.{threading.get_ident()}.tmp')
try:
with tmp.open('w', encoding='utf-8') as fp:
json.dump(data, fp, ensure_ascii=False, indent=2)
fp.write('\n')
tmp.replace(path)
finally:
if tmp.exists():
tmp.unlink()
class Notifier:
"""Outbound webhooks + Kuma adapter; notification errors never alter backup exit codes."""
def __init__(self, config, folder):
self.config = config
self.folder = folder
self.pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix='backup-notify')
self.futures = []
self.lock = threading.Lock()
self.errors = 0
self.logfile = folder / 'notifications.log'
def _send_json(self, url, event):
payload = json.dumps(event, ensure_ascii=False).encode('utf-8')
req = Request(url, data=payload, method='POST', headers={'Content-Type': 'application/json', 'User-Agent': 'pi-backup-orchestrator/0.3'})
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
if not 200 <= response.status < 300:
raise RuntimeError(f'HTTP {response.status}')
def _send_kuma(self, url, status, message):
sep = '&' if '?' in url else '?'
address = url + sep + urlencode({'status': status, 'msg': message[:180]})
req = Request(address, headers={'User-Agent': 'pi-backup-orchestrator/0.3'})
with urlopen(req, timeout=self.config['timeout_seconds']) as response:
if not 200 <= response.status < 300:
raise RuntimeError(f'HTTP {response.status}')
# Kuma usually responds with JSON {ok:true}; a 200 alone is insufficient if it says failure.
body = response.read(4096)
if body:
try:
parsed = json.loads(body)
except (ValueError, UnicodeDecodeError):
parsed = None
if isinstance(parsed, dict) and parsed.get('ok') is False:
raise RuntimeError('Kuma rejected push')
def _submit(self, channel, fn, *args):
def guarded():
try:
fn(*args)
except Exception as exc:
# Never log target URLs or tokens; even HTTP exceptions may include their URL.
if hasattr(exc, 'code') and isinstance(exc.code, int):
detail = f'HTTP {exc.code}'
else:
detail = type(exc).__name__
with self.lock:
self.errors += 1
with self.logfile.open('a', encoding='utf-8') as out:
out.write(f'{now_iso()} {channel}: delivery failed ({detail})\n')
self.futures.append(self.pool.submit(guarded))
def notify(self, event):
for url in self.config['webhooks']:
self._submit('webhook', self._send_json, url, event)
name = event['event']
if name in ('JOB_SUCCESS', 'JOB_FAILED', 'JOB_TIMEOUT', 'JOB_CANCELLED', 'JOB_ERROR', 'JOB_BLOCKED'):
url = self.config['kuma']['job_push_urls'].get(event.get('job'), '')
if url:
status = 'up' if name == 'JOB_SUCCESS' else 'down'
msg = f"{event['job']}: {event.get('state', 'unknown')} (run {event['run_id']})"
self._submit('kuma/job', self._send_kuma, url, status, msg)
if name == 'RUN_FINISHED':
url = self.config['kuma']['overall_push_url']
if url:
status = 'up' if event.get('overall') == 'ok' else 'down'
self._submit('kuma/overall', self._send_kuma, url, status, f"Run {event['run_id']}: {event.get('overall')}")
def close(self):
self.pool.shutdown(wait=True)
return self.errors
class EventHub:
"""Write append-only events.jsonl plus atomic current snapshot for consumers."""
def __init__(self, folder, log_root, run_id, jobs, notifier, started):
self.folder = folder
self.log_root = log_root
self.run_id = run_id
self.notifier = notifier
self.lock = threading.RLock()
self.snapshot = {
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
'started': started, 'updated': started, 'finished': None,
'overall': 'running',
'jobs': {j['id']: result_for(j, 'pending') for j in jobs},
}
atomic_json(folder / 'status.json', self.snapshot)
atomic_json(log_root / 'latest.json', self.snapshot)
def emit(self, event_name, **fields):
event = {
'event': event_name, 'timestamp': now_iso(), 'run_id': self.run_id,
'host': self.snapshot['host'], **fields,
}
with self.lock:
if 'job' in event and event['job'] in self.snapshot['jobs'] and 'state' in event:
jid = event['job']
self.snapshot['jobs'][jid].update({k: v for k, v in fields.items() if k in (
'state', 'phase', 'critical', 'exit_code', 'duration_seconds', 'log', 'reason')})
if event_name == 'RUN_FINISHED':
self.snapshot['overall'] = fields['overall']
self.snapshot['finished'] = event['timestamp']
self.snapshot['updated'] = event['timestamp']
with (self.folder / 'events.jsonl').open('a', encoding='utf-8') as fp:
fp.write(json.dumps(event, ensure_ascii=False) + '\n')
atomic_json(self.folder / 'status.json', self.snapshot)
atomic_json(self.log_root / 'latest.json', self.snapshot)
self.notifier.notify(event)
return event
def terminate_group(proc):
"""Terminate ordinary children of a job and wait for shutdown before advancing phase."""
if proc.poll() is not None:
return
try:
os.killpg(proc.pid, signal.SIGTERM)
except ProcessLookupError:
return
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(proc.pid, signal.SIGKILL)
except ProcessLookupError:
pass
proc.wait()
def execute_one(job, folder, hub):
start = time.monotonic()
logfile = folder / (job['id'] + '.log')
if CANCELLED.is_set():
result = result_for(job, 'cancelled', reason='Orchestrator interrupted')
hub.emit('JOB_CANCELLED', **result)
return result
hub.emit('JOB_STARTED', job=job['id'], phase=job['phase'], critical=job['critical'], state='running')
with logfile.open('wb') as stream:
stream.write((f"job={job['id']} started={now_iso()}\n").encode())
stream.flush()
try:
proc = subprocess.Popen([job['script']], stdout=stream, stderr=subprocess.STDOUT,
start_new_session=True, cwd=str(Path(job['script']).parent))
except OSError as exc:
stream.write((f'EXEC ERROR: {exc}\n').encode())
result = result_for(job, 'error', 127, time.monotonic()-start, logfile, 'Could not start script')
hub.emit('JOB_ERROR', **result)
return result
deadline = time.monotonic() + job['timeout_seconds']
while True:
if CANCELLED.is_set():
terminate_group(proc)
state, code = 'cancelled', 130
break
if time.monotonic() >= deadline:
terminate_group(proc)
state, code = 'timeout', 124
break
try:
code = proc.wait(timeout=min(0.5, max(0.01, deadline-time.monotonic())))
state = 'ok' if code == 0 else 'failed'
break
except subprocess.TimeoutExpired:
pass
stream.write((f'\njob={job["id"]} finished={now_iso()} status={state} exit_code={code}\n').encode())
result = result_for(job, state, code, time.monotonic()-start, logfile)
event = {'ok': 'JOB_SUCCESS', 'failed': 'JOB_FAILED', 'timeout': 'JOB_TIMEOUT', 'cancelled': 'JOB_CANCELLED'}[state]
hub.emit(event, **result)
return result
def live_run(jobs, args):
if not jobs:
print('Nothing enabled; no backups executed.')
return 0
if os.environ.get(CUTOVER_ENV) != 'YES':
print(f'REFUSED: Set {CUTOVER_ENV}=YES only after testing scripts and coordinating existing schedules.', file=sys.stderr)
return 2
if not nas_mount_ready():
print(f'REFUSED: {NAS_MOUNT} is not mounted from {NAS_SOURCE}', file=sys.stderr)
return 2
conflicts = legacy_schedules()
if conflicts:
print('REFUSED: existing scheduled backups could overlap this run:', file=sys.stderr)
for issue in conflicts:
print(' ' + issue, file=sys.stderr)
return 2
unsafe = [(j['id'], script_issues(j)) for j in jobs if script_issues(j)]
if unsafe:
for jid, issues in unsafe:
print(f'REFUSED: {jid}: {", ".join(issues)}', file=sys.stderr)
return 2
notifications = load_notifications(args.notify_config)
args.log_root.mkdir(parents=True, exist_ok=True, mode=0o700)
args.lock_file.parent.mkdir(parents=True, exist_ok=True)
with args.lock_file.open('a+') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
print('REFUSED: another orchestrator run holds the lock', file=sys.stderr)
return 3
run_id = dt.datetime.now().strftime('%Y%m%d-%H%M%S') + f'-{os.getpid()}'
folder = args.log_root / run_id
folder.mkdir(mode=0o700)
old_int = signal.getsignal(signal.SIGINT)
old_term = signal.getsignal(signal.SIGTERM)
CANCELLED.clear()
signal.signal(signal.SIGINT, lambda signum, frame: CANCELLED.set())
signal.signal(signal.SIGTERM, lambda signum, frame: CANCELLED.set())
results = []
critical_failed = False
groups = defaultdict(list)
for job in jobs:
groups[job['phase']].append(job)
began = now_iso()
notifier = Notifier(notifications, folder)
hub = EventHub(folder, args.log_root, run_id, jobs, notifier, began)
try:
hub.emit('RUN_STARTED', max_parallel=args.max_parallel, job_count=len(jobs))
for phase, members in sorted(groups.items()):
if critical_failed or CANCELLED.is_set():
reason = 'earlier critical failure' if critical_failed else 'orchestrator interrupted'
hub.emit('PHASE_BLOCKED', phase=phase, reason=reason)
for job in members:
result = result_for(job, 'blocked', reason=reason)
results.append(result)
hub.emit('JOB_BLOCKED', **result)
print(f'Phase {phase} BLOCKED: {reason}', flush=True)
continue
hub.emit('PHASE_STARTED', phase=phase, job_count=len(members))
print(f'Phase {phase}: starting {len(members)} job(s), max parallel {args.max_parallel}', flush=True)
phase_results = []
with ThreadPoolExecutor(max_workers=args.max_parallel) as pool:
future_to_job = {pool.submit(execute_one, job, folder, hub): job for job in members}
for future in as_completed(future_to_job):
job = future_to_job[future]
try:
item = future.result()
except Exception as exc:
item = result_for(job, 'error', 127, reason=f'Internal runner error: {type(exc).__name__}')
hub.emit('JOB_ERROR', **item)
phase_results.append(item)
print(f' {item["job"]}: {item["state"]} ({item["exit_code"]})', flush=True)
results.extend(sorted(phase_results, key=lambda r: r['job']))
if any(r['critical'] and r['state'] != 'ok' for r in phase_results):
critical_failed = True
hub.emit('PHASE_COMPLETED', phase=phase, failures=sum(r['state'] != 'ok' for r in phase_results),
critical_failed=critical_failed)
if critical_failed:
print(f'Phase {phase}: critical failure; subsequent phases will not start.', flush=True)
else:
print(f'Phase {phase}: barrier reached; next phase may start.', flush=True)
finally:
signal.signal(signal.SIGINT, old_int)
signal.signal(signal.SIGTERM, old_term)
overall = 'interrupted' if CANCELLED.is_set() else 'ok' if all(r['state'] == 'ok' for r in results) else 'failed'
summary = {
'version': VERSION, 'run_id': run_id, 'host': socket.gethostname(),
'started': began, 'finished': now_iso(), 'max_parallel': args.max_parallel,
'interrupted': CANCELLED.is_set(), 'results': results, 'overall': overall,
}
atomic_json(folder / 'summary.json', summary)
hub.emit('RUN_FINISHED', overall=overall, success_count=sum(r['state'] == 'ok' for r in results),
failure_count=sum(r['state'] != 'ok' for r in results))
delivery_failures = notifier.close()
if delivery_failures:
print(f'WARNING: {delivery_failures} notification delivery failure(s); see {folder / "notifications.log"}', flush=True)
print('Summary: ' + str(folder / 'summary.json'), flush=True)
return 130 if CANCELLED.is_set() else (1 if any(r['state'] != 'ok' for r in results) else 0)
def show_status(log_root, as_json=False):
latest = log_root / 'latest.json'
if latest.is_file():
with latest.open(encoding='utf-8') as fp:
data = json.load(fp)
if as_json:
print(json.dumps(data, ensure_ascii=False, indent=2))
return 0
print(f"Run {data['run_id']}: {data['overall']} ({data['started']} -> {data['finished'] or 'running'})")
for item in sorted(data['jobs'].values(), key=lambda x: (x['phase'], x['job'])):
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
return 0
summaries = sorted(log_root.glob('*/summary.json')) if log_root.is_dir() else []
if not summaries:
if as_json:
print(json.dumps({'overall': 'no_runs', 'jobs': {}}, indent=2))
else:
print('No runs recorded yet')
return 0
with summaries[-1].open(encoding='utf-8') as fp:
data = json.load(fp)
if as_json:
print(json.dumps(data, ensure_ascii=False, indent=2))
else:
print(f"Last run: {data['started']} -> {data['finished']} ; overall={data['overall']}")
for item in data['results']:
print(f" Phase {item['phase']:<3} {item['job']:<24} {item['state']:<10} {item['duration_seconds']}s")
return 0
def main():
parser = argparse.ArgumentParser(description='Phase-based backup orchestrator; safe inspection by default')
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--check', action='store_true', help='Validate config and inspect schedules (read only)')
mode.add_argument('--plan', action='store_true', help='Show phase plan (read only)')
mode.add_argument('--status', action='store_true', help='Latest job status (read only)')
mode.add_argument('--run', action='store_true', help='Execute enabled jobs (safety-gated)')
parser.add_argument('--json', action='store_true', help='Machine-readable result for --status')
parser.add_argument('--config', type=Path, default=DEFAULT_CONFIG)
parser.add_argument('--notify-config', type=Path, default=DEFAULT_NOTIFY_CONFIG)
parser.add_argument('--log-root', type=Path, default=DEFAULT_LOG_ROOT)
parser.add_argument('--lock-file', type=Path, default=DEFAULT_LOCK_FILE)
parser.add_argument('--max-parallel', type=int, default=3, help='Maximum concurrent jobs in a phase (1..16)')
selection = parser.add_mutually_exclusive_group()
selection.add_argument('--phase', type=int, help='Select one phase (--plan or --run)')
selection.add_argument('--job', type=str, help='Select one job (--plan or --run)')
args = parser.parse_args()
if not 1 <= args.max_parallel <= 16:
parser.error('--max-parallel must be 1..16')
if args.json and not args.status:
parser.error('--json is only valid with --status')
if (args.job or args.phase is not None) and not (args.run or args.plan):
parser.error('--phase and --job are only valid with --plan or --run')
if args.status:
return show_status(args.log_root, as_json=args.json)
jobs = load_jobs(args.config)
notify = load_notifications(args.notify_config, known_ids=[j['id'] for j in jobs])
if args.check:
return check(jobs, args.max_parallel, notify)
jobs = select_jobs(jobs, args)
if args.plan:
return plan(jobs, args.max_parallel) or 0
return live_run(jobs, args)
if __name__ == '__main__':
try:
sys.exit(main())
except (ValueError, OSError, json.JSONDecodeError) as exc:
print(f'ERROR: {exc}', file=sys.stderr)
sys.exit(2)