"""Unit and integration tests for Backup-Orchestrator v0.4 Recovery Management.""" import hashlib import importlib.util import json import os from pathlib import Path import subprocess import sys import tempfile import types import unittest from unittest.mock import patch ROOT_DIR = Path(__file__).resolve().parents[1] MODULE = ROOT_DIR / 'orchestrator.py' spec = importlib.util.spec_from_file_location('orchestrator_v04_test', MODULE) app = importlib.util.module_from_spec(spec) spec.loader.exec_module(app) class RecoveryPointAndManifestTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) self.targets_cfg = app.load_targets_config(ROOT_DIR / 'targets.json') def _create_dump_bundle(self, migrations=('001_init', '095_catalog', '096_login_lockout_and_totp'), corrupt_sha=False, with_manifest=True, status='consistent'): dump_path = self.root / 'shinkan-prod.sql' copy_lines = '\n'.join(f'{idx}\t{m}\t2026-10-29 05:00:00' for idx, m in enumerate(migrations, start=1)) sql_content = ( '-- PostgreSQL database dump\n' 'COPY public.schema_migrations (id, migration, executed_at) FROM stdin;\n' f'{copy_lines}\n' '\\.\n' ) dump_path.write_text(sql_content, encoding='utf-8') actual_sha = hashlib.sha256(dump_path.read_bytes()).hexdigest() written_sha = ('0' * 64) if corrupt_sha else actual_sha (self.root / 'shinkan-prod.sql.sha256').write_text(f'{written_sha} shinkan-prod.sql\n', encoding='utf-8') media_file = self.root / 'shinkan-prod.sql.media.jsonl' media_file.write_text('{"path":"library/c1/image/a.jpg","size":10,"mtime":1700000000,"sha256":"' + ('a' * 64) + '"}\n', encoding='utf-8') media_sha = hashlib.sha256(media_file.read_bytes()).hexdigest() if with_manifest: manifest = { 'schemaVersion': 1, 'created_at': '2026-10-29T05:30:12+01:00', 'host': 'raspberrypi5', 'target': 'pi-prod', 'app_version': '0.8.246', 'git_commit': 'deadbeef1234', 'migrations': list(migrations), 'last_migration': migrations[-1] if migrations else None, 'dump_format': 'plain', 'dump_bytes': dump_path.stat().st_size, 'dump_sha256': actual_sha, 'row_counts': {'profiles': 5, 'skills': 120}, 'media_manifest': { 'file': media_file.name, 'sha256': media_sha, 'file_count': 1, }, 'recoveryPoint': { 'status': status, 'validatedAt': '2026-10-29T06:00:00+01:00' if status == 'restore_tested' else None, }, } (self.root / 'shinkan-prod.sql.manifest.json').write_text( json.dumps(manifest, indent=2), encoding='utf-8' ) return dump_path def test_manifest_rejects_plaintext_secrets(self): bad = { 'schemaVersion': 1, 'created_at': '2026-10-29T05:30:12+01:00', 'host': 'raspberrypi5', 'dump_sha256': 'a' * 64, 'notes': 'PGPASSWORD=supersecretpassword', } with self.assertRaises(ValueError): app.normalize_manifest(bad) def test_verify_recovery_point_stages(self): # 1. Legacy dump without manifest -> captured dump_only = self.root / 'legacy.sql' dump_only.write_text('SELECT 1;\n', encoding='utf-8') res_legacy = app.verify_recovery_point(dump_only) self.assertEqual(res_legacy['status'], 'captured') self.assertFalse(res_legacy['valid']) # 2. Full bundle -> consistent dump_path = self._create_dump_bundle() res_ok = app.verify_recovery_point(dump_path) self.assertTrue(res_ok['valid']) self.assertEqual(res_ok['status'], 'consistent') self.assertEqual(res_ok['exit_code'], app.EXIT_OK) # 3. Restore-tested bundle -> restore_tested dump_tested = self._create_dump_bundle(status='restore_tested') res_tested = app.verify_recovery_point(dump_tested) self.assertTrue(res_tested['valid']) self.assertEqual(res_tested['status'], 'restore_tested') # 4. Corrupt SHA256 -> Exit 4 (EXIT_CHECKSUM_MISMATCH) dump_bad = self._create_dump_bundle(corrupt_sha=True) res_bad = app.verify_recovery_point(dump_bad) self.assertFalse(res_bad['valid']) self.assertEqual(res_bad['exit_code'], app.EXIT_CHECKSUM_MISMATCH) def test_compatibility_exact_forward_and_incompatible_rollback(self): dump_path = self._create_dump_bundle(migrations=('001_init', '095_catalog')) ver = app.verify_recovery_point(dump_path) # Exact match c_exact = app.check_compatibility( ver, 'llm-prod', self.targets_cfg, current_host='llm-node', current_arch='x86_64', code_migrations=['001_init', '095_catalog'], ) self.assertTrue(c_exact['compatible']) self.assertEqual(c_exact['schema_relation'], 'EXACT_MATCH') self.assertEqual(c_exact['exit_code'], app.EXIT_OK) # Forward migration (dump is older than target code) c_fwd = app.check_compatibility( ver, 'llm-prod', self.targets_cfg, current_host='llm-node', current_arch='x86_64', code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'], ) self.assertTrue(c_fwd['compatible']) self.assertEqual(c_fwd['schema_relation'], 'FORWARD_MIGRATE') self.assertEqual(c_fwd['pending_migrations'], ['096_login_lockout_and_totp']) # Incompatible rollback (dump has migration 097 unknown to target code) dump_future = self._create_dump_bundle(migrations=('001_init', '095_catalog', '097_future_feature')) ver_future = app.verify_recovery_point(dump_future) c_bad = app.check_compatibility( ver_future, 'llm-prod', self.targets_cfg, current_host='llm-node', current_arch='x86_64', code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'], ) self.assertFalse(c_bad['compatible']) self.assertEqual(c_bad['schema_relation'], 'INCOMPATIBLE_ROLLBACK') self.assertEqual(c_bad['exit_code'], app.EXIT_SCHEMA_INCOMPATIBLE) self.assertIn('097_future_feature', c_bad['unknown_migrations']) def test_source_only_target_refuses_restore(self): dump_path = self._create_dump_bundle() ver = app.verify_recovery_point(dump_path) c_pi = app.check_compatibility( ver, 'pi-prod', self.targets_cfg, current_host='raspberrypi5', current_arch='aarch64', code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'], ) self.assertFalse(c_pi['compatible']) self.assertEqual(c_pi['exit_code'], app.EXIT_PRECONDITION) def test_recovery_plan_is_non_destructive_and_complete(self): dump_path = self._create_dump_bundle() plan_doc = app.generate_recovery_plan( dump_path, 'llm-prod', self.targets_cfg, current_host='llm-node', current_arch='x86_64', code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'], ) self.assertEqual(plan_doc['mode'], 'DRY_RUN_RECOVERY_PLAN') self.assertTrue(plan_doc['compatibility']['compatible']) self.assertEqual(plan_doc['target']['require_confirm'], 'shinkan@llm-node') self.assertGreaterEqual(len(plan_doc['steps']), 9) def test_execute_restore_requires_confirmation_and_records_audit_log(self): dump_path = self._create_dump_bundle() adapter_dir = self.root / 'adapter' adapter_dir.mkdir() fake_restore = adapter_dir / 'shinkan-restore' fake_restore.write_text( '#!/usr/bin/env python3\n' 'import json, sys\n' 'idx = sys.argv.index("--result")\n' 'open(sys.argv[idx+1], "w").write(json.dumps({"status": "ok", "previous_db": "shinkan_before_test"}))\n' 'sys.exit(0)\n', encoding='utf-8', ) fake_restore.chmod(0o755) tcfg = json.loads(json.dumps(self.targets_cfg)) tcfg['targets']['llm-prod']['host'] = app.socket.gethostname() tcfg['targets']['llm-prod']['arch'] = app.platform.machine() tcfg['targets']['llm-prod']['adapter_dir'] = str(adapter_dir) tcfg['targets']['llm-prod']['app_dir'] = str(self.root) tcfg['targets']['llm-prod']['compose_dir'] = str(self.root) args_no_confirm = types.SimpleNamespace( target='llm-prod', point=str(dump_path), confirm=None, allow_overwrite=False, sessions='clear', json=False, log_root=self.root / 'logs', ) # Missing --confirm must be refused with Exit 2 self.assertEqual(app.execute_recovery_command('restore', args_no_confirm, tcfg), app.EXIT_PRECONDITION) self.assertFalse((self.root / 'logs').exists()) # With valid --confirm shinkan@llm-node, restore executes and writes audit summary args_confirmed = types.SimpleNamespace( target='llm-prod', point=str(dump_path), confirm='shinkan@llm-node', allow_overwrite=False, sessions='clear', json=False, log_root=self.root / 'logs', ) self.assertEqual(app.execute_recovery_command('restore', args_confirmed, tcfg), app.EXIT_OK) latest_rec = json.loads((self.root / 'logs' / 'latest-recovery.json').read_text(encoding='utf-8')) self.assertEqual(latest_rec['overall'], 'ok') self.assertEqual(latest_rec['adapter_result']['previous_db'], 'shinkan_before_test') # test_shinkan_backup_manifest_script end-to-end media_dir = self.root / 'media' (media_dir / 'library' / 'c1').mkdir(parents=True) (media_dir / 'library' / 'c1' / 'img.png').write_bytes(b'PNGDATA123') out_dir = self.root / 'manifest_out' result_json = self.root / 'manifest_result.json' env = os.environ.copy() env['SHINKAN_SKIP_HOST_GUARD'] = '1' env['SHINKAN_MEDIA_DIR_OVERRIDE'] = str(media_dir) proc = subprocess.run( [ sys.executable, str(ROOT_DIR / 'scripts' / 'shinkan' / 'shinkan-backup-manifest'), '--target', 'pi-prod', '--dump', str(dump_path), '--out-dir', str(out_dir), '--result', str(result_json), ], env=env, capture_output=True, text=True, timeout=30, ) self.assertEqual(proc.returncode, 0, msg=proc.stderr) res_data = json.loads(result_json.read_text(encoding='utf-8')) self.assertEqual(res_data['status'], 'ok') self.assertEqual(res_data['media_file_count'], 1) self.assertTrue((out_dir / 'shinkan-prod.sql.manifest.json').is_file()) self.assertTrue((out_dir / 'shinkan-prod.sql.sha256').is_file()) self.assertTrue((out_dir / 'shinkan-prod.sql.media.jsonl').is_file()) def test_host_profiles_raspberrypi5_and_llm_node(self): repo_root = Path(__file__).resolve().parents[1] for host_name, expected_jobs in (('raspberrypi5', 19), ('llm-node', 1)): host_dir = app.resolve_repo_host_dir(host_name) self.assertIsNotNone(host_dir, f'Missing host dir for {host_name}') host_cfg = app.load_host_config(host_name=host_name) self.assertEqual(host_cfg['hostname'], host_name) jobs = app.load_jobs(host_dir / 'jobs.csv') self.assertEqual(len(jobs), expected_jobs) notify = app.load_notifications( host_dir / 'notifications.json.example', known_ids=[j['id'] for j in jobs], ) self.assertIn('kuma', notify) targets = app.load_targets_config(host_name=host_name) self.assertEqual(targets['host'], host_name) self.assertIn('pi-prod', targets['targets']) # Verify every job script exists in either hosts//scripts/ or scripts/common/ for job in jobs: script_name = Path(job['script']).name if job['id'] == 'system-image': continue exists_in_host = (host_dir / 'scripts' / script_name).is_file() exists_in_common = (repo_root / 'scripts' / 'common' / script_name).is_file() self.assertTrue( exists_in_host or exists_in_common, f"Missing script {script_name} for job {job['id']} on host {host_name}", ) if __name__ == '__main__': unittest.main()