"""Tests use temporary dummy scripts only. They never touch Raspberry Pi backups.""" import importlib.util import os from pathlib import Path import tempfile import types import unittest from unittest.mock import patch MODULE = Path(__file__).resolve().parents[1] / 'orchestrator.py' spec = importlib.util.spec_from_file_location('orchestrator_test', MODULE) app = importlib.util.module_from_spec(spec) spec.loader.exec_module(app) class OrchestratorTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) self.patches = [patch.object(app, 'legacy_schedules', return_value=[]), patch.object(app, 'nas_mount_ready', return_value=True), patch.dict(os.environ, {'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED':'YES'})] for p in self.patches: p.start() self.addCleanup(p.stop) self.args = types.SimpleNamespace(max_parallel=3, log_root=self.root/'logs', lock_file=self.root/'runner.lock', notify_config=self.root/'notifications.json') def script(self, name, body): p = self.root / f'{name}.sh' p.write_text('#!/bin/sh\n' + body + '\n') p.chmod(0o700) return str(p) def job(self, phase, name, body, critical=True, timeout=10): return {'phase': phase, 'id': name, 'script': self.script(name, body), 'critical': critical, 'timeout_seconds': timeout, 'enabled': True} def results(self): import json path = list((self.root / 'logs').glob('*/summary.json'))[0] return json.loads(path.read_text()) def test_parallel_phase_then_barrier(self): event = self.root / 'events.txt' j1 = self.job(10, 'a', f'sleep 0.3; echo a >> "{event}"') j2 = self.job(10, 'b', f'sleep 0.3; echo b >> "{event}"') j3 = self.job(20, 'after', f'count=$(wc -l < "{event}"); [ "$count" -eq 2 ] && echo after >> "{event}"') import time start = time.monotonic() self.assertEqual(app.live_run([j1, j2, j3], self.args), 0) self.assertLess(time.monotonic() - start, 1.20 if os.name == 'nt' else 0.60) self.assertEqual(event.read_text().splitlines()[-1], 'after') self.assertEqual(self.results()['overall'], 'ok') def test_critical_failure_blocks_next_phase(self): event = self.root / 'events.txt' j1 = self.job(10, 'fail', 'exit 7', critical=True) j2 = self.job(10, 'optional', 'exit 0', critical=False) j3 = self.job(20, 'after', f'echo wrong > "{event}"') self.assertEqual(app.live_run([j1, j2, j3], self.args), 1) self.assertFalse(event.exists()) by_name = {r['job']:r['state'] for r in self.results()['results']} self.assertEqual(by_name, {'fail':'failed', 'optional':'ok', 'after':'blocked'}) def test_noncritical_failure_allows_next_phase(self): event = self.root / 'events.txt' j1 = self.job(10, 'optional-fail', 'exit 8', critical=False) j2 = self.job(20, 'after', f'echo ran > "{event}"') self.assertEqual(app.live_run([j1, j2], self.args), 1) self.assertTrue(event.exists()) self.assertEqual([r['state'] for r in self.results()['results']], ['failed', 'ok']) def test_timeout(self): j = self.job(10, 'hang', 'sleep 8', timeout=1) self.assertEqual(app.live_run([j], self.args), 1) self.assertEqual(self.results()['results'][0]['state'], 'timeout') def test_conflicts_block_runs(self): j = self.job(10, 'no-run', 'exit 0') with patch.object(app, 'legacy_schedules', return_value=['raspiBackup.timer: enabled']): self.assertEqual(app.live_run([j], self.args), 2) self.assertFalse((self.root/'logs').exists()) def test_config_validation(self): path = self.root/'jobs.csv' path.write_text('phase,id,script,critical,timeout_seconds,enabled\n10,one,/bin/true,true,30,false\n') self.assertEqual(len(app.load_jobs(path)), 1) path.write_text('phase,id,script,critical,timeout_seconds,enabled\n10,one,/bin/true,maybe,30,false\n') with self.assertRaises(ValueError): app.load_jobs(path) # New outbound interface tests are intentionally declared outside the legacy test class. class NotificationTests(OrchestratorTests): class FakeResponse: status = 200 def __enter__(self): return self def __exit__(self, *args): return False def read(self, *_): return b'{"ok":true}' def test_outbound_webhooks_kuma_and_status_json(self): import json from urllib.parse import parse_qs, urlsplit observed = [] def fake_request(req, timeout): observed.append((req.get_method(), req.full_url, req.data)) return self.FakeResponse() self.args.notify_config.write_text(json.dumps({ 'timeout_seconds': 2, 'webhooks': ['http://127.0.0.1:99/events'], 'kuma': {'overall_push_url': 'http://localhost:3010/api/push/SUPERSECRET?status=up&msg=OK&ping=', 'job_push_urls': {'one': 'http://localhost:3010/api/push/ANOTHERSECRET'}} })) job = self.job(10, 'one', 'echo test') with patch.object(app, 'urlopen', side_effect=fake_request): self.assertEqual(app.live_run([job], self.args), 0) methods = [a[0] for a in observed] self.assertIn('POST', methods) sent_events = [json.loads(a[2])['event'] for a in observed if a[0] == 'POST'] self.assertIn('JOB_STARTED', sent_events) self.assertIn('JOB_SUCCESS', sent_events) self.assertIn('RUN_FINISHED', sent_events) gets = [a for a in observed if a[0] == 'GET'] self.assertEqual(len(gets), 2) for _, url, _ in gets: self.assertEqual(parse_qs(urlsplit(url).query)['status'], ['up']) latest = json.loads((self.root/'logs'/'latest.json').read_text()) self.assertEqual(latest['overall'], 'ok') self.assertEqual(latest['jobs']['one']['state'], 'ok') self.assertNotIn('SUPERSECRET', json.dumps(latest)) events = list((self.root/'logs').glob('*/events.jsonl'))[0].read_text() self.assertNotIn('ANOTHERSECRET', events) self.assertIn('RUN_FINISHED', events) def test_job_failure_sends_kuma_down_and_later_phase_blocked(self): import json from urllib.parse import parse_qs, urlsplit observed = [] def fake_request(req, timeout): observed.append(req.full_url) return self.FakeResponse() self.args.notify_config.write_text(json.dumps({ 'kuma': {'overall_push_url': 'http://localhost/api/push/overall', 'job_push_urls': {'fail': 'http://localhost/api/push/fail', 'after': 'http://localhost/api/push/after'}} })) jobs = [self.job(10, 'fail', 'exit 9'), self.job(20, 'after', 'exit 0')] with patch.object(app, 'urlopen', side_effect=fake_request): self.assertEqual(app.live_run(jobs, self.args), 1) self.assertEqual(len(observed), 3) for url in observed: self.assertEqual(parse_qs(urlsplit(url).query)['status'], ['down']) latest = json.loads((self.root/'logs'/'latest.json').read_text()) self.assertEqual(latest['jobs']['fail']['state'], 'failed') self.assertEqual(latest['jobs']['after']['state'], 'blocked') self.assertEqual(latest['overall'], 'failed') def test_notification_failure_keeps_backup_result_and_sanitizes_logs(self): job = self.job(10, 'one', 'exit 0') self.args.notify_config.write_text('{"kuma":{"overall_push_url":"http://localhost/api/push/SECRET"}}') with patch.object(app, 'urlopen', side_effect=OSError('http://localhost/api/push/SECRET')): self.assertEqual(app.live_run([job], self.args), 0) log = list((self.root/'logs').glob('*/notifications.log'))[0].read_text() self.assertIn('delivery failed', log) self.assertNotIn('SECRET', log) def test_validation_rejects_wrong_kuma_url(self): import json self.args.notify_config.write_text(json.dumps({'kuma': {'overall_push_url': 'file:///secrets'}})) with self.assertRaises(ValueError): app.load_notifications(self.args.notify_config) if __name__ == '__main__': unittest.main()