Files

180 lines
8.6 KiB
Python

"""Tests use temporary dummy scripts only. They never touch Raspberry Pi backups."""
import importlib.util
import os
from pathlib import Path
import tempfile
import types
import unittest
from unittest.mock import patch
MODULE = Path(__file__).resolve().parents[1] / 'orchestrator.py'
spec = importlib.util.spec_from_file_location('orchestrator_test', MODULE)
app = importlib.util.module_from_spec(spec)
spec.loader.exec_module(app)
class OrchestratorTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.patches = [patch.object(app, 'legacy_schedules', return_value=[]),
patch.object(app, 'nas_mount_ready', return_value=True),
patch.object(app, 'load_host_config', return_value={'hostname':'test-host', 'nas_mount':'/backup', 'nas_source':'192.168.2.63:/volume1/Backup_Paperless'}),
patch.dict(os.environ, {'BACKUP_ORCHESTRATOR_CUTOVER_APPROVED':'YES'})]
for p in self.patches:
p.start()
self.addCleanup(p.stop)
self.args = types.SimpleNamespace(max_parallel=3, log_root=self.root/'logs', lock_file=self.root/'runner.lock', notify_config=self.root/'notifications.json')
def script(self, name, body):
p = self.root / f'{name}.sh'
p.write_text('#!/bin/sh\n' + body + '\n')
p.chmod(0o700)
return str(p)
def job(self, phase, name, body, critical=True, timeout=10):
return {'phase': phase, 'id': name, 'script': self.script(name, body),
'critical': critical, 'timeout_seconds': timeout, 'enabled': True}
def results(self):
import json
path = list((self.root / 'logs').glob('*/summary.json'))[0]
return json.loads(path.read_text())
def test_parallel_phase_then_barrier(self):
event = self.root / 'events.txt'
j1 = self.job(10, 'a', f'sleep 0.3; echo a >> "{event}"')
j2 = self.job(10, 'b', f'sleep 0.3; echo b >> "{event}"')
j3 = self.job(20, 'after', f'count=$(wc -l < "{event}"); [ "$count" -eq 2 ] && echo after >> "{event}"')
import time
start = time.monotonic()
self.assertEqual(app.live_run([j1, j2, j3], self.args), 0)
self.assertLess(time.monotonic() - start, 1.20 if os.name == 'nt' else 0.60)
self.assertEqual(event.read_text().splitlines()[-1], 'after')
self.assertEqual(self.results()['overall'], 'ok')
def test_critical_failure_blocks_next_phase(self):
event = self.root / 'events.txt'
j1 = self.job(10, 'fail', 'exit 7', critical=True)
j2 = self.job(10, 'optional', 'exit 0', critical=False)
j3 = self.job(20, 'after', f'echo wrong > "{event}"')
self.assertEqual(app.live_run([j1, j2, j3], self.args), 1)
self.assertFalse(event.exists())
by_name = {r['job']:r['state'] for r in self.results()['results']}
self.assertEqual(by_name, {'fail':'failed', 'optional':'ok', 'after':'blocked'})
def test_noncritical_failure_allows_next_phase(self):
event = self.root / 'events.txt'
j1 = self.job(10, 'optional-fail', 'exit 8', critical=False)
j2 = self.job(20, 'after', f'echo ran > "{event}"')
self.assertEqual(app.live_run([j1, j2], self.args), 1)
self.assertTrue(event.exists())
self.assertEqual([r['state'] for r in self.results()['results']], ['failed', 'ok'])
def test_timeout(self):
j = self.job(10, 'hang', 'sleep 8', timeout=1)
self.assertEqual(app.live_run([j], self.args), 1)
self.assertEqual(self.results()['results'][0]['state'], 'timeout')
def test_conflicts_block_runs(self):
j = self.job(10, 'no-run', 'exit 0')
with patch.object(app, 'legacy_schedules', return_value=['raspiBackup.timer: enabled']):
self.assertEqual(app.live_run([j], self.args), 2)
self.assertFalse((self.root/'logs').exists())
def test_config_validation(self):
path = self.root/'jobs.csv'
path.write_text('phase,id,script,critical,timeout_seconds,enabled\n10,one,/bin/true,true,30,false\n')
self.assertEqual(len(app.load_jobs(path)), 1)
path.write_text('phase,id,script,critical,timeout_seconds,enabled\n10,one,/bin/true,maybe,30,false\n')
with self.assertRaises(ValueError):
app.load_jobs(path)
# New outbound interface tests are intentionally declared outside the legacy test class.
class NotificationTests(OrchestratorTests):
class FakeResponse:
status = 200
def __enter__(self):
return self
def __exit__(self, *args):
return False
def read(self, *_):
return b'{"ok":true}'
def test_outbound_webhooks_kuma_and_status_json(self):
import json
from urllib.parse import parse_qs, urlsplit
observed = []
def fake_request(req, timeout):
observed.append((req.get_method(), req.full_url, req.data))
return self.FakeResponse()
self.args.notify_config.write_text(json.dumps({
'timeout_seconds': 2,
'webhooks': ['http://127.0.0.1:99/events'],
'kuma': {'overall_push_url': 'http://localhost:3010/api/push/SUPERSECRET?status=up&msg=OK&ping=',
'job_push_urls': {'one': 'http://localhost:3010/api/push/ANOTHERSECRET'}}
}))
job = self.job(10, 'one', 'echo test')
with patch.object(app, 'urlopen', side_effect=fake_request):
self.assertEqual(app.live_run([job], self.args), 0)
methods = [a[0] for a in observed]
self.assertIn('POST', methods)
sent_events = [json.loads(a[2])['event'] for a in observed if a[0] == 'POST']
self.assertIn('JOB_STARTED', sent_events)
self.assertIn('JOB_SUCCESS', sent_events)
self.assertIn('RUN_FINISHED', sent_events)
gets = [a for a in observed if a[0] == 'GET']
self.assertEqual(len(gets), 2)
for _, url, _ in gets:
self.assertEqual(parse_qs(urlsplit(url).query)['status'], ['up'])
latest = json.loads((self.root/'logs'/'latest.json').read_text())
self.assertEqual(latest['overall'], 'ok')
self.assertEqual(latest['jobs']['one']['state'], 'ok')
self.assertNotIn('SUPERSECRET', json.dumps(latest))
events = list((self.root/'logs').glob('*/events.jsonl'))[0].read_text()
self.assertNotIn('ANOTHERSECRET', events)
self.assertIn('RUN_FINISHED', events)
def test_job_failure_sends_kuma_down_and_later_phase_blocked(self):
import json
from urllib.parse import parse_qs, urlsplit
observed = []
def fake_request(req, timeout):
observed.append(req.full_url)
return self.FakeResponse()
self.args.notify_config.write_text(json.dumps({
'kuma': {'overall_push_url': 'http://localhost/api/push/overall',
'job_push_urls': {'fail': 'http://localhost/api/push/fail',
'after': 'http://localhost/api/push/after'}}
}))
jobs = [self.job(10, 'fail', 'exit 9'), self.job(20, 'after', 'exit 0')]
with patch.object(app, 'urlopen', side_effect=fake_request):
self.assertEqual(app.live_run(jobs, self.args), 1)
self.assertEqual(len(observed), 3)
for url in observed:
self.assertEqual(parse_qs(urlsplit(url).query)['status'], ['down'])
latest = json.loads((self.root/'logs'/'latest.json').read_text())
self.assertEqual(latest['jobs']['fail']['state'], 'failed')
self.assertEqual(latest['jobs']['after']['state'], 'blocked')
self.assertEqual(latest['overall'], 'failed')
def test_notification_failure_keeps_backup_result_and_sanitizes_logs(self):
job = self.job(10, 'one', 'exit 0')
self.args.notify_config.write_text('{"kuma":{"overall_push_url":"http://localhost/api/push/SECRET"}}')
with patch.object(app, 'urlopen', side_effect=OSError('http://localhost/api/push/SECRET')):
self.assertEqual(app.live_run([job], self.args), 0)
log = list((self.root/'logs').glob('*/notifications.log'))[0].read_text()
self.assertIn('delivery failed', log)
self.assertNotIn('SECRET', log)
def test_validation_rejects_wrong_kuma_url(self):
import json
self.args.notify_config.write_text(json.dumps({'kuma': {'overall_push_url': 'file:///secrets'}}))
with self.assertRaises(ValueError):
app.load_notifications(self.args.notify_config)
if __name__ == '__main__':
unittest.main()