179 lines
8.3 KiB
Python
179 lines
8.3 KiB
Python
"""Tests use temporary dummy scripts only. They never touch Raspberry Pi backups."""
|
|
import importlib.util
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import types
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
MODULE = Path(__file__).resolve().parents[1] / 'orchestrator.py'
|
|
spec = importlib.util.spec_from_file_location('orchestrator_test', MODULE)
|
|
app = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(app)
|
|
|
|
|
|
class OrchestratorTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.patches = [patch.object(app, 'legacy_schedules', return_value=[]),
|
|
patch.object(app, 'nas_mount_ready', return_value=True),
|
|
patch.dict(os.environ, {'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED':'YES'})]
|
|
for p in self.patches:
|
|
p.start()
|
|
self.addCleanup(p.stop)
|
|
self.args = types.SimpleNamespace(max_parallel=3, log_root=self.root/'logs', lock_file=self.root/'runner.lock', notify_config=self.root/'notifications.json')
|
|
|
|
def script(self, name, body):
|
|
p = self.root / f'{name}.sh'
|
|
p.write_text('#!/bin/sh\n' + body + '\n')
|
|
p.chmod(0o700)
|
|
return str(p)
|
|
|
|
def job(self, phase, name, body, critical=True, timeout=10):
|
|
return {'phase': phase, 'id': name, 'script': self.script(name, body),
|
|
'critical': critical, 'timeout_seconds': timeout, 'enabled': True}
|
|
|
|
def results(self):
|
|
import json
|
|
path = list((self.root / 'logs').glob('*/summary.json'))[0]
|
|
return json.loads(path.read_text())
|
|
|
|
def test_parallel_phase_then_barrier(self):
|
|
event = self.root / 'events.txt'
|
|
j1 = self.job(10, 'a', f'sleep 0.3; echo a >> "{event}"')
|
|
j2 = self.job(10, 'b', f'sleep 0.3; echo b >> "{event}"')
|
|
j3 = self.job(20, 'after', f'count=$(wc -l < "{event}"); [ "$count" -eq 2 ] && echo after >> "{event}"')
|
|
import time
|
|
start = time.monotonic()
|
|
self.assertEqual(app.live_run([j1, j2, j3], self.args), 0)
|
|
self.assertLess(time.monotonic() - start, 0.60)
|
|
self.assertEqual(event.read_text().splitlines()[-1], 'after')
|
|
self.assertEqual(self.results()['overall'], 'ok')
|
|
|
|
def test_critical_failure_blocks_next_phase(self):
|
|
event = self.root / 'events.txt'
|
|
j1 = self.job(10, 'fail', 'exit 7', critical=True)
|
|
j2 = self.job(10, 'optional', 'exit 0', critical=False)
|
|
j3 = self.job(20, 'after', f'echo wrong > "{event}"')
|
|
self.assertEqual(app.live_run([j1, j2, j3], self.args), 1)
|
|
self.assertFalse(event.exists())
|
|
by_name = {r['job']:r['state'] for r in self.results()['results']}
|
|
self.assertEqual(by_name, {'fail':'failed', 'optional':'ok', 'after':'blocked'})
|
|
|
|
def test_noncritical_failure_allows_next_phase(self):
|
|
event = self.root / 'events.txt'
|
|
j1 = self.job(10, 'optional-fail', 'exit 8', critical=False)
|
|
j2 = self.job(20, 'after', f'echo ran > "{event}"')
|
|
self.assertEqual(app.live_run([j1, j2], self.args), 1)
|
|
self.assertTrue(event.exists())
|
|
self.assertEqual([r['state'] for r in self.results()['results']], ['failed', 'ok'])
|
|
|
|
def test_timeout(self):
|
|
j = self.job(10, 'hang', 'sleep 8', timeout=1)
|
|
self.assertEqual(app.live_run([j], self.args), 1)
|
|
self.assertEqual(self.results()['results'][0]['state'], 'timeout')
|
|
|
|
def test_conflicts_block_runs(self):
|
|
j = self.job(10, 'no-run', 'exit 0')
|
|
with patch.object(app, 'legacy_schedules', return_value=['raspiBackup.timer: enabled']):
|
|
self.assertEqual(app.live_run([j], self.args), 2)
|
|
self.assertFalse((self.root/'logs').exists())
|
|
|
|
def test_config_validation(self):
|
|
path = self.root/'jobs.csv'
|
|
path.write_text('phase,id,script,critical,timeout_seconds,enabled\n10,one,/bin/true,true,30,false\n')
|
|
self.assertEqual(len(app.load_jobs(path)), 1)
|
|
path.write_text('phase,id,script,critical,timeout_seconds,enabled\n10,one,/bin/true,maybe,30,false\n')
|
|
with self.assertRaises(ValueError):
|
|
app.load_jobs(path)
|
|
|
|
|
|
# New outbound interface tests are intentionally declared outside the legacy test class.
|
|
class NotificationTests(OrchestratorTests):
|
|
class FakeResponse:
|
|
status = 200
|
|
def __enter__(self):
|
|
return self
|
|
def __exit__(self, *args):
|
|
return False
|
|
def read(self, *_):
|
|
return b'{"ok":true}'
|
|
|
|
def test_outbound_webhooks_kuma_and_status_json(self):
|
|
import json
|
|
from urllib.parse import parse_qs, urlsplit
|
|
observed = []
|
|
def fake_request(req, timeout):
|
|
observed.append((req.get_method(), req.full_url, req.data))
|
|
return self.FakeResponse()
|
|
self.args.notify_config.write_text(json.dumps({
|
|
'timeout_seconds': 2,
|
|
'webhooks': ['http://127.0.0.1:99/events'],
|
|
'kuma': {'overall_push_url': 'http://localhost:3010/api/push/SUPERSECRET?status=up&msg=OK&ping=',
|
|
'job_push_urls': {'one': 'http://localhost:3010/api/push/ANOTHERSECRET'}}
|
|
}))
|
|
job = self.job(10, 'one', 'echo test')
|
|
with patch.object(app, 'urlopen', side_effect=fake_request):
|
|
self.assertEqual(app.live_run([job], self.args), 0)
|
|
methods = [a[0] for a in observed]
|
|
self.assertIn('POST', methods)
|
|
sent_events = [json.loads(a[2])['event'] for a in observed if a[0] == 'POST']
|
|
self.assertIn('JOB_STARTED', sent_events)
|
|
self.assertIn('JOB_SUCCESS', sent_events)
|
|
self.assertIn('RUN_FINISHED', sent_events)
|
|
gets = [a for a in observed if a[0] == 'GET']
|
|
self.assertEqual(len(gets), 2)
|
|
for _, url, _ in gets:
|
|
self.assertEqual(parse_qs(urlsplit(url).query)['status'], ['up'])
|
|
latest = json.loads((self.root/'logs'/'latest.json').read_text())
|
|
self.assertEqual(latest['overall'], 'ok')
|
|
self.assertEqual(latest['jobs']['one']['state'], 'ok')
|
|
self.assertNotIn('SUPERSECRET', json.dumps(latest))
|
|
events = list((self.root/'logs').glob('*/events.jsonl'))[0].read_text()
|
|
self.assertNotIn('ANOTHERSECRET', events)
|
|
self.assertIn('RUN_FINISHED', events)
|
|
|
|
def test_job_failure_sends_kuma_down_and_later_phase_blocked(self):
|
|
import json
|
|
from urllib.parse import parse_qs, urlsplit
|
|
observed = []
|
|
def fake_request(req, timeout):
|
|
observed.append(req.full_url)
|
|
return self.FakeResponse()
|
|
self.args.notify_config.write_text(json.dumps({
|
|
'kuma': {'overall_push_url': 'http://localhost/api/push/overall',
|
|
'job_push_urls': {'fail': 'http://localhost/api/push/fail',
|
|
'after': 'http://localhost/api/push/after'}}
|
|
}))
|
|
jobs = [self.job(10, 'fail', 'exit 9'), self.job(20, 'after', 'exit 0')]
|
|
with patch.object(app, 'urlopen', side_effect=fake_request):
|
|
self.assertEqual(app.live_run(jobs, self.args), 1)
|
|
self.assertEqual(len(observed), 3)
|
|
for url in observed:
|
|
self.assertEqual(parse_qs(urlsplit(url).query)['status'], ['down'])
|
|
latest = json.loads((self.root/'logs'/'latest.json').read_text())
|
|
self.assertEqual(latest['jobs']['fail']['state'], 'failed')
|
|
self.assertEqual(latest['jobs']['after']['state'], 'blocked')
|
|
self.assertEqual(latest['overall'], 'failed')
|
|
|
|
def test_notification_failure_keeps_backup_result_and_sanitizes_logs(self):
|
|
job = self.job(10, 'one', 'exit 0')
|
|
self.args.notify_config.write_text('{"kuma":{"overall_push_url":"http://localhost/api/push/SECRET"}}')
|
|
with patch.object(app, 'urlopen', side_effect=OSError('http://localhost/api/push/SECRET')):
|
|
self.assertEqual(app.live_run([job], self.args), 0)
|
|
log = list((self.root/'logs').glob('*/notifications.log'))[0].read_text()
|
|
self.assertIn('delivery failed', log)
|
|
self.assertNotIn('SECRET', log)
|
|
|
|
def test_validation_rejects_wrong_kuma_url(self):
|
|
import json
|
|
self.args.notify_config.write_text(json.dumps({'kuma': {'overall_push_url': 'file:///secrets'}}))
|
|
with self.assertRaises(ValueError):
|
|
app.load_notifications(self.args.notify_config)
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|