245 lines
11 KiB
Python
245 lines
11 KiB
Python
"""Unit and integration tests for Backup-Orchestrator v0.4 Recovery Management."""
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import types
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
ROOT_DIR = Path(__file__).resolve().parents[1]
|
|
MODULE = ROOT_DIR / 'orchestrator.py'
|
|
spec = importlib.util.spec_from_file_location('orchestrator_v04_test', MODULE)
|
|
app = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(app)
|
|
|
|
|
|
class RecoveryPointAndManifestTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.targets_cfg = app.load_targets_config(ROOT_DIR / 'targets.json')
|
|
|
|
def _create_dump_bundle(self, migrations=('001_init', '095_catalog', '096_login_lockout_and_totp'),
|
|
corrupt_sha=False, with_manifest=True, status='consistent'):
|
|
dump_path = self.root / 'shinkan-prod.sql'
|
|
copy_lines = '\n'.join(f'{idx}\t{m}\t2026-10-29 05:00:00' for idx, m in enumerate(migrations, start=1))
|
|
sql_content = (
|
|
'-- PostgreSQL database dump\n'
|
|
'COPY public.schema_migrations (id, migration, executed_at) FROM stdin;\n'
|
|
f'{copy_lines}\n'
|
|
'\\.\n'
|
|
)
|
|
dump_path.write_text(sql_content, encoding='utf-8')
|
|
actual_sha = hashlib.sha256(dump_path.read_bytes()).hexdigest()
|
|
written_sha = ('0' * 64) if corrupt_sha else actual_sha
|
|
(self.root / 'shinkan-prod.sql.sha256').write_text(f'{written_sha} shinkan-prod.sql\n', encoding='utf-8')
|
|
|
|
media_file = self.root / 'shinkan-prod.sql.media.jsonl'
|
|
media_file.write_text('{"path":"library/c1/image/a.jpg","size":10,"mtime":1700000000,"sha256":"' + ('a' * 64) + '"}\n', encoding='utf-8')
|
|
media_sha = hashlib.sha256(media_file.read_bytes()).hexdigest()
|
|
|
|
if with_manifest:
|
|
manifest = {
|
|
'schemaVersion': 1,
|
|
'created_at': '2026-10-29T05:30:12+01:00',
|
|
'host': 'raspberrypi5',
|
|
'target': 'pi-prod',
|
|
'app_version': '0.8.246',
|
|
'git_commit': 'deadbeef1234',
|
|
'migrations': list(migrations),
|
|
'last_migration': migrations[-1] if migrations else None,
|
|
'dump_format': 'plain',
|
|
'dump_bytes': dump_path.stat().st_size,
|
|
'dump_sha256': actual_sha,
|
|
'row_counts': {'profiles': 5, 'skills': 120},
|
|
'media_manifest': {
|
|
'file': media_file.name,
|
|
'sha256': media_sha,
|
|
'file_count': 1,
|
|
},
|
|
'recoveryPoint': {
|
|
'status': status,
|
|
'validatedAt': '2026-10-29T06:00:00+01:00' if status == 'restore_tested' else None,
|
|
},
|
|
}
|
|
(self.root / 'shinkan-prod.sql.manifest.json').write_text(
|
|
json.dumps(manifest, indent=2), encoding='utf-8'
|
|
)
|
|
return dump_path
|
|
|
|
def test_manifest_rejects_plaintext_secrets(self):
|
|
bad = {
|
|
'schemaVersion': 1,
|
|
'created_at': '2026-10-29T05:30:12+01:00',
|
|
'host': 'raspberrypi5',
|
|
'dump_sha256': 'a' * 64,
|
|
'notes': 'PGPASSWORD=supersecretpassword',
|
|
}
|
|
with self.assertRaises(ValueError):
|
|
app.normalize_manifest(bad)
|
|
|
|
def test_verify_recovery_point_stages(self):
|
|
# 1. Legacy dump without manifest -> captured
|
|
dump_only = self.root / 'legacy.sql'
|
|
dump_only.write_text('SELECT 1;\n', encoding='utf-8')
|
|
res_legacy = app.verify_recovery_point(dump_only)
|
|
self.assertEqual(res_legacy['status'], 'captured')
|
|
self.assertFalse(res_legacy['valid'])
|
|
|
|
# 2. Full bundle -> consistent
|
|
dump_path = self._create_dump_bundle()
|
|
res_ok = app.verify_recovery_point(dump_path)
|
|
self.assertTrue(res_ok['valid'])
|
|
self.assertEqual(res_ok['status'], 'consistent')
|
|
self.assertEqual(res_ok['exit_code'], app.EXIT_OK)
|
|
|
|
# 3. Restore-tested bundle -> restore_tested
|
|
dump_tested = self._create_dump_bundle(status='restore_tested')
|
|
res_tested = app.verify_recovery_point(dump_tested)
|
|
self.assertTrue(res_tested['valid'])
|
|
self.assertEqual(res_tested['status'], 'restore_tested')
|
|
|
|
# 4. Corrupt SHA256 -> Exit 4 (EXIT_CHECKSUM_MISMATCH)
|
|
dump_bad = self._create_dump_bundle(corrupt_sha=True)
|
|
res_bad = app.verify_recovery_point(dump_bad)
|
|
self.assertFalse(res_bad['valid'])
|
|
self.assertEqual(res_bad['exit_code'], app.EXIT_CHECKSUM_MISMATCH)
|
|
|
|
def test_compatibility_exact_forward_and_incompatible_rollback(self):
|
|
dump_path = self._create_dump_bundle(migrations=('001_init', '095_catalog'))
|
|
ver = app.verify_recovery_point(dump_path)
|
|
|
|
# Exact match
|
|
c_exact = app.check_compatibility(
|
|
ver, 'llm-prod', self.targets_cfg,
|
|
current_host='llm-node', current_arch='x86_64',
|
|
code_migrations=['001_init', '095_catalog'],
|
|
)
|
|
self.assertTrue(c_exact['compatible'])
|
|
self.assertEqual(c_exact['schema_relation'], 'EXACT_MATCH')
|
|
self.assertEqual(c_exact['exit_code'], app.EXIT_OK)
|
|
|
|
# Forward migration (dump is older than target code)
|
|
c_fwd = app.check_compatibility(
|
|
ver, 'llm-prod', self.targets_cfg,
|
|
current_host='llm-node', current_arch='x86_64',
|
|
code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'],
|
|
)
|
|
self.assertTrue(c_fwd['compatible'])
|
|
self.assertEqual(c_fwd['schema_relation'], 'FORWARD_MIGRATE')
|
|
self.assertEqual(c_fwd['pending_migrations'], ['096_login_lockout_and_totp'])
|
|
|
|
# Incompatible rollback (dump has migration 097 unknown to target code)
|
|
dump_future = self._create_dump_bundle(migrations=('001_init', '095_catalog', '097_future_feature'))
|
|
ver_future = app.verify_recovery_point(dump_future)
|
|
c_bad = app.check_compatibility(
|
|
ver_future, 'llm-prod', self.targets_cfg,
|
|
current_host='llm-node', current_arch='x86_64',
|
|
code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'],
|
|
)
|
|
self.assertFalse(c_bad['compatible'])
|
|
self.assertEqual(c_bad['schema_relation'], 'INCOMPATIBLE_ROLLBACK')
|
|
self.assertEqual(c_bad['exit_code'], app.EXIT_SCHEMA_INCOMPATIBLE)
|
|
self.assertIn('097_future_feature', c_bad['unknown_migrations'])
|
|
|
|
def test_source_only_target_refuses_restore(self):
|
|
dump_path = self._create_dump_bundle()
|
|
ver = app.verify_recovery_point(dump_path)
|
|
c_pi = app.check_compatibility(
|
|
ver, 'pi-prod', self.targets_cfg,
|
|
current_host='raspberrypi5', current_arch='aarch64',
|
|
code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'],
|
|
)
|
|
self.assertFalse(c_pi['compatible'])
|
|
self.assertEqual(c_pi['exit_code'], app.EXIT_PRECONDITION)
|
|
|
|
def test_recovery_plan_is_non_destructive_and_complete(self):
|
|
dump_path = self._create_dump_bundle()
|
|
plan_doc = app.generate_recovery_plan(
|
|
dump_path, 'llm-prod', self.targets_cfg,
|
|
current_host='llm-node', current_arch='x86_64',
|
|
code_migrations=['001_init', '095_catalog', '096_login_lockout_and_totp'],
|
|
)
|
|
self.assertEqual(plan_doc['mode'], 'DRY_RUN_RECOVERY_PLAN')
|
|
self.assertTrue(plan_doc['compatibility']['compatible'])
|
|
self.assertEqual(plan_doc['target']['require_confirm'], 'shinkan@llm-node')
|
|
self.assertGreaterEqual(len(plan_doc['steps']), 9)
|
|
|
|
def test_execute_restore_requires_confirmation_and_records_audit_log(self):
|
|
dump_path = self._create_dump_bundle()
|
|
adapter_dir = self.root / 'adapter'
|
|
adapter_dir.mkdir()
|
|
fake_restore = adapter_dir / 'shinkan-restore'
|
|
fake_restore.write_text(
|
|
'#!/usr/bin/env python3\n'
|
|
'import json, sys\n'
|
|
'idx = sys.argv.index("--result")\n'
|
|
'open(sys.argv[idx+1], "w").write(json.dumps({"status": "ok", "previous_db": "shinkan_before_test"}))\n'
|
|
'sys.exit(0)\n',
|
|
encoding='utf-8',
|
|
)
|
|
fake_restore.chmod(0o755)
|
|
|
|
tcfg = json.loads(json.dumps(self.targets_cfg))
|
|
tcfg['targets']['llm-prod']['host'] = app.socket.gethostname()
|
|
tcfg['targets']['llm-prod']['arch'] = app.platform.machine()
|
|
tcfg['targets']['llm-prod']['adapter_dir'] = str(adapter_dir)
|
|
|
|
args_no_confirm = types.SimpleNamespace(
|
|
target='llm-prod', point=str(dump_path), confirm=None,
|
|
allow_overwrite=False, sessions='clear', json=False,
|
|
log_root=self.root / 'logs',
|
|
)
|
|
# Missing --confirm must be refused with Exit 2
|
|
self.assertEqual(app.execute_recovery_command('restore', args_no_confirm, tcfg), app.EXIT_PRECONDITION)
|
|
self.assertFalse((self.root / 'logs').exists())
|
|
|
|
# With valid --confirm shinkan@llm-node, restore executes and writes audit summary
|
|
args_confirmed = types.SimpleNamespace(
|
|
target='llm-prod', point=str(dump_path), confirm='shinkan@llm-node',
|
|
allow_overwrite=False, sessions='clear', json=False,
|
|
log_root=self.root / 'logs',
|
|
)
|
|
self.assertEqual(app.execute_recovery_command('restore', args_confirmed, tcfg), app.EXIT_OK)
|
|
latest_rec = json.loads((self.root / 'logs' / 'latest-recovery.json').read_text(encoding='utf-8'))
|
|
self.assertEqual(latest_rec['overall'], 'ok')
|
|
self.assertEqual(latest_rec['adapter_result']['previous_db'], 'shinkan_before_test')
|
|
|
|
# test_shinkan_backup_manifest_script end-to-end
|
|
media_dir = self.root / 'media'
|
|
(media_dir / 'library' / 'c1').mkdir(parents=True)
|
|
(media_dir / 'library' / 'c1' / 'img.png').write_bytes(b'PNGDATA123')
|
|
out_dir = self.root / 'manifest_out'
|
|
result_json = self.root / 'manifest_result.json'
|
|
env = os.environ.copy()
|
|
env['SHINKAN_SKIP_HOST_GUARD'] = '1'
|
|
env['SHINKAN_MEDIA_DIR_OVERRIDE'] = str(media_dir)
|
|
proc = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(ROOT_DIR / 'scripts' / 'shinkan' / 'shinkan-backup-manifest'),
|
|
'--target', 'pi-prod',
|
|
'--dump', str(dump_path),
|
|
'--out-dir', str(out_dir),
|
|
'--result', str(result_json),
|
|
],
|
|
env=env, capture_output=True, text=True, timeout=30,
|
|
)
|
|
self.assertEqual(proc.returncode, 0, msg=proc.stderr)
|
|
res_data = json.loads(result_json.read_text(encoding='utf-8'))
|
|
self.assertEqual(res_data['status'], 'ok')
|
|
self.assertEqual(res_data['media_file_count'], 1)
|
|
self.assertTrue((out_dir / 'shinkan-prod.sql.manifest.json').is_file())
|
|
self.assertTrue((out_dir / 'shinkan-prod.sql.sha256').is_file())
|
|
self.assertTrue((out_dir / 'shinkan-prod.sql.media.jsonl').is_file())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|