Files
backup-orchestrator/tests/test_recovery.py
T

276 lines
13 KiB
Python

"""Unit and integration tests for Backup-Orchestrator v0.4 Recovery Management."""
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import types
import unittest
from unittest.mock import patch
ROOT_DIR = Path(__file__).resolve().parents[1]
MODULE = ROOT_DIR / 'orchestrator.py'
spec = importlib.util.spec_from_file_location('orchestrator_v04_test', MODULE)
app = importlib.util.module_from_spec(spec)
spec.loader.exec_module(app)
class RecoveryPointAndManifestTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.targets_cfg = app.load_targets_config(ROOT_DIR / 'targets.json')
def _create_dump_bundle(self, migrations=('001_init', '095_catalog', '096_login_lockout_and_totp'),
corrupt_sha=False, with_manifest=True, status='consistent'):
dump_path = self.root / 'shinkan-prod.sql'
copy_lines = '\n'.join(f'{idx}\t{m}\t2026-10-29 05:00:00' for idx, m in enumerate(migrations, start=1))
sql_content = (
'-- PostgreSQL database dump\n'
'COPY public.schema_migrations (id, migration, executed_at) FROM stdin;\n'
f'{copy_lines}\n'
'\\.\n'
)
dump_path.write_text(sql_content, encoding='utf-8')
actual_sha = hashlib.sha256(dump_path.read_bytes()).hexdigest()
written_sha = ('0' * 64) if corrupt_sha else actual_sha
(self.root / 'shinkan-prod.sql.sha256').write_text(f'{written_sha} shinkan-prod.sql\n', encoding='utf-8')
media_file = self.root / 'shinkan-prod.sql.media.jsonl'
media_file.write_text('{"path":"library/c1/image/a.jpg","size":10,"mtime":1700000000,"sha256":"' + ('a' * 64) + '"}\n', encoding='utf-8')
media_sha = hashlib.sha256(media_file.read_bytes()).hexdigest()
if with_manifest:
manifest = {
'schemaVersion': 1,
'created_at': '2026-10-29T05:30:12+01:00',
'host': 'raspberrypi5',
'target': 'pi-prod',
'app_version': '0.8.246',
'git_commit': 'deadbeef1234',
'migrations': list(migrations),
'last_migration': migrations[-1] if migrations else None,
'dump_format': 'plain',
'dump_bytes': dump_path.stat().st_size,
'dump_sha256': actual_sha,
'row_counts': {'profiles': 5, 'skills': 120},
'media_manifest': {
'file': media_file.name,
'sha256': media_sha,
'file_count': 1,
},
'recoveryPoint': {
'status': status,
'validatedAt': '2026-10-29T06:00:00+01:00' if status == 'restore_tested' else None,
},
}
(self.root / 'shinkan-prod.sql.manifest.json').write_text(
json.dumps(manifest, indent=2), encoding='utf-8'
)
return dump_path
def test_manifest_rejects_plaintext_secrets(self):
bad = {
'schemaVersion': 1,
'created_at': '2026-10-29T05:30:12+01:00',
'host': 'raspberrypi5',
'dump_sha256': 'a' * 64,
'notes': 'PGPASSWORD=supersecretpassword',
}
with self.assertRaises(ValueError):
app.normalize_manifest(bad)
def test_verify_recovery_point_stages(self):
# 1. Legacy dump without manifest -> captured
dump_only = self.root / 'legacy.sql'
dump_only.write_text('SELECT 1;\n', encoding='utf-8')
res_legacy = app.verify_recovery_point(dump_only)
self.assertEqual(res_legacy['status'], 'captured')
self.assertFalse(res_legacy['valid'])
# 2. Full bundle -> consistent
dump_path = self._create_dump_bundle()
res_ok = app.verify_recovery_point(dump_path)
self.assertTrue(res_ok['valid'])
self.assertEqual(res_ok['status'], 'consistent')
self.assertEqual(res_ok['exit_code'], app.EXIT_OK)
# 3. Restore-tested bundle -> restore_tested
dump_tested = self._create_dump_bundle(status='restore_tested')
res_tested = app.verify_recovery_point(dump_tested)
self.assertTrue(res_tested['valid'])
self.assertEqual(res_tested['status'], 'restore_tested')
# 4. Corrupt SHA256 -> Exit 4 (EXIT_CHECKSUM_MISMATCH)
dump_bad = self._create_dump_bundle(corrupt_sha=True)
res_bad = app.verify_recovery_point(dump_bad)
self.assertFalse(res_bad['valid'])
self.assertEqual(res_bad['exit_code'], app.EXIT_CHECKSUM_MISMATCH)
def test_compatibility_exact_forward_and_incompatible_rollback(self):
dump_path = self._create_dump_bundle(migrations=('001_init', '095_catalog'))
ver = app.verify_recovery_point(dump_path)
# Exact match
c_exact = app.check_compatibility(
ver, 'llm-prod', self.targets_cfg,
current_host='llm-node', current_arch='x86_64',
code_migrations=['001_init', '095_catalog'],
)
self.assertTrue(c_exact['compatible'])
self.assertEqual(c_exact['schema_relation'], 'EXACT_MATCH')
self.assertEqual(c_exact['exit_code'], app.EXIT_OK)
# Forward migration (dump is older than target code)
c_fwd = app.check_compatibility(
ver, 'llm-prod', self.targets_cfg,
current_host='llm-node', current_arch='x86_64',
code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'],
)
self.assertTrue(c_fwd['compatible'])
self.assertEqual(c_fwd['schema_relation'], 'FORWARD_MIGRATE')
self.assertEqual(c_fwd['pending_migrations'], ['096_login_lockout_and_totp'])
# Incompatible rollback (dump has migration 097 unknown to target code)
dump_future = self._create_dump_bundle(migrations=('001_init', '095_catalog', '097_future_feature'))
ver_future = app.verify_recovery_point(dump_future)
c_bad = app.check_compatibility(
ver_future, 'llm-prod', self.targets_cfg,
current_host='llm-node', current_arch='x86_64',
code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'],
)
self.assertFalse(c_bad['compatible'])
self.assertEqual(c_bad['schema_relation'], 'INCOMPATIBLE_ROLLBACK')
self.assertEqual(c_bad['exit_code'], app.EXIT_SCHEMA_INCOMPATIBLE)
self.assertIn('097_future_feature', c_bad['unknown_migrations'])
def test_source_only_target_refuses_restore(self):
dump_path = self._create_dump_bundle()
ver = app.verify_recovery_point(dump_path)
c_pi = app.check_compatibility(
ver, 'pi-prod', self.targets_cfg,
current_host='raspberrypi5', current_arch='aarch64',
code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'],
)
self.assertFalse(c_pi['compatible'])
self.assertEqual(c_pi['exit_code'], app.EXIT_PRECONDITION)
def test_recovery_plan_is_non_destructive_and_complete(self):
dump_path = self._create_dump_bundle()
plan_doc = app.generate_recovery_plan(
dump_path, 'llm-prod', self.targets_cfg,
current_host='llm-node', current_arch='x86_64',
code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'],
)
self.assertEqual(plan_doc['mode'], 'DRY_RUN_RECOVERY_PLAN')
self.assertTrue(plan_doc['compatibility']['compatible'])
self.assertEqual(plan_doc['target']['require_confirm'], 'shinkan@llm-node')
self.assertGreaterEqual(len(plan_doc['steps']), 9)
def test_execute_restore_requires_confirmation_and_records_audit_log(self):
dump_path = self._create_dump_bundle()
adapter_dir = self.root / 'adapter'
adapter_dir.mkdir()
fake_restore = adapter_dir / 'shinkan-restore'
fake_restore.write_text(
'#!/usr/bin/env python3\n'
'import json, sys\n'
'idx = sys.argv.index("--result")\n'
'open(sys.argv[idx+1], "w").write(json.dumps({"status": "ok", "previous_db": "shinkan_before_test"}))\n'
'sys.exit(0)\n',
encoding='utf-8',
)
fake_restore.chmod(0o755)
tcfg = json.loads(json.dumps(self.targets_cfg))
tcfg['targets']['llm-prod']['host'] = app.socket.gethostname()
tcfg['targets']['llm-prod']['arch'] = app.platform.machine()
tcfg['targets']['llm-prod']['adapter_dir'] = str(adapter_dir)
tcfg['targets']['llm-prod']['app_dir'] = str(self.root)
tcfg['targets']['llm-prod']['compose_dir'] = str(self.root)
args_no_confirm = types.SimpleNamespace(
target='llm-prod', point=str(dump_path), confirm=None,
allow_overwrite=False, sessions='clear', json=False,
log_root=self.root / 'logs',
)
# Missing --confirm must be refused with Exit 2
self.assertEqual(app.execute_recovery_command('restore', args_no_confirm, tcfg), app.EXIT_PRECONDITION)
self.assertFalse((self.root / 'logs').exists())
# With valid --confirm shinkan@llm-node, restore executes and writes audit summary
args_confirmed = types.SimpleNamespace(
target='llm-prod', point=str(dump_path), confirm='shinkan@llm-node',
allow_overwrite=False, sessions='clear', json=False,
log_root=self.root / 'logs',
)
self.assertEqual(app.execute_recovery_command('restore', args_confirmed, tcfg), app.EXIT_OK)
latest_rec = json.loads((self.root / 'logs' / 'latest-recovery.json').read_text(encoding='utf-8'))
self.assertEqual(latest_rec['overall'], 'ok')
self.assertEqual(latest_rec['adapter_result']['previous_db'], 'shinkan_before_test')
# test_shinkan_backup_manifest_script end-to-end
media_dir = self.root / 'media'
(media_dir / 'library' / 'c1').mkdir(parents=True)
(media_dir / 'library' / 'c1' / 'img.png').write_bytes(b'PNGDATA123')
out_dir = self.root / 'manifest_out'
result_json = self.root / 'manifest_result.json'
env = os.environ.copy()
env['SHINKAN_SKIP_HOST_GUARD'] = '1'
env['SHINKAN_MEDIA_DIR_OVERRIDE'] = str(media_dir)
proc = subprocess.run(
[
sys.executable,
str(ROOT_DIR / 'scripts' / 'shinkan' / 'shinkan-backup-manifest'),
'--target', 'pi-prod',
'--dump', str(dump_path),
'--out-dir', str(out_dir),
'--result', str(result_json),
],
env=env, capture_output=True, text=True, timeout=30,
)
self.assertEqual(proc.returncode, 0, msg=proc.stderr)
res_data = json.loads(result_json.read_text(encoding='utf-8'))
self.assertEqual(res_data['status'], 'ok')
self.assertEqual(res_data['media_file_count'], 1)
self.assertTrue((out_dir / 'shinkan-prod.sql.manifest.json').is_file())
self.assertTrue((out_dir / 'shinkan-prod.sql.sha256').is_file())
self.assertTrue((out_dir / 'shinkan-prod.sql.media.jsonl').is_file())
def test_host_profiles_raspberrypi5_and_llm_node(self):
repo_root = Path(__file__).resolve().parents[1]
for host_name, expected_jobs in (('raspberrypi5', 19), ('llm-node', 1)):
host_dir = app.resolve_repo_host_dir(host_name)
self.assertIsNotNone(host_dir, f'Missing host dir for {host_name}')
host_cfg = app.load_host_config(host_name=host_name)
self.assertEqual(host_cfg['hostname'], host_name)
jobs = app.load_jobs(host_dir / 'jobs.csv')
self.assertEqual(len(jobs), expected_jobs)
notify = app.load_notifications(
host_dir / 'notifications.json.example',
known_ids=[j['id'] for j in jobs],
)
self.assertIn('kuma', notify)
targets = app.load_targets_config(host_name=host_name)
self.assertEqual(targets['host'], host_name)
self.assertIn('pi-prod', targets['targets'])
# Verify every job script exists in either hosts/<host>/scripts/ or scripts/common/
for job in jobs:
script_name = Path(job['script']).name
if job['id'] == 'system-image':
continue
exists_in_host = (host_dir / 'scripts' / script_name).is_file()
exists_in_common = (repo_root / 'scripts' / 'common' / script_name).is_file()
self.assertTrue(
exists_in_host or exists_in_common,
f"Missing script {script_name} for job {job['id']} on host {host_name}",
)
if __name__ == '__main__':
unittest.main()