137 lines
5.7 KiB
Python
137 lines
5.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Run an isolated disposable restore test for Shinkan on llm-node.
|
|
|
|
Implements RECOVERY_CONTRACT.md section 3.3:
|
|
- Uses dedicated project shinkan-restoretest without fixed container_name collisions
|
|
- Mounts shared media directory read-only (:ro)
|
|
- Disables retention-cron, SMTP, OpenRouter, and auto-admin bootstrap
|
|
- Runs shinkan-restore --target llm-restoretest and shinkan-media-diff --target llm-restoretest
|
|
- Tears down containers and volumes unless --keep is specified
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
EXIT_OK = 0
|
|
EXIT_EXEC_ERROR = 1
|
|
EXIT_PRECONDITION = 2
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Run isolated Shinkan restore test on llm-node')
|
|
parser.add_argument('--dump', required=True, type=Path)
|
|
parser.add_argument('--result', required=True, type=Path)
|
|
parser.add_argument('--playwright', action='store_true')
|
|
parser.add_argument('--keep', action='store_true')
|
|
args = parser.parse_args()
|
|
|
|
start_ts = time.monotonic()
|
|
cur_host = socket.gethostname()
|
|
cur_arch = platform.machine()
|
|
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
|
|
if cur_host != 'llm-node' or cur_arch != 'x86_64':
|
|
msg = f'Host/arch mismatch: running on {cur_host}/{cur_arch}, expected llm-node/x86_64'
|
|
args.result.parent.mkdir(parents=True, exist_ok=True)
|
|
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
|
|
print(f'ERROR: {msg}', file=sys.stderr)
|
|
return EXIT_PRECONDITION
|
|
|
|
script_dir = Path(__file__).resolve().parent
|
|
compose_file = script_dir / 'docker-compose.restoretest.yml'
|
|
app_dir = Path(os.environ.get('SHINKAN_APP_DIR_OVERRIDE', '/home/llmadmin/docker/shinkan-prod'))
|
|
env_file = Path(os.environ.get('SHINKAN_ENV_FILE_OVERRIDE', '/etc/shinkan/prod.env'))
|
|
|
|
def dc(*dc_args):
|
|
cmd = ['docker', 'compose', '-p', 'shinkan-restoretest', '-f', str(compose_file)]
|
|
if env_file.is_file():
|
|
cmd.extend(['--env-file', str(env_file)])
|
|
cmd.extend(dc_args)
|
|
return subprocess.run(cmd, cwd=str(app_dir), capture_output=True, text=True, timeout=600)
|
|
|
|
restore_res_file = args.result.with_name(args.result.stem + '.restore.json')
|
|
media_res_file = args.result.with_name(args.result.stem + '.media.json')
|
|
|
|
rc = EXIT_EXEC_ERROR
|
|
try:
|
|
print('1. Starting isolated shinkan-restoretest environment...')
|
|
dc('down', '-v', '--remove-orphans')
|
|
up_proc = dc('up', '-d', '--build', 'postgres', 'backend', 'frontend')
|
|
if up_proc.returncode != 0:
|
|
raise RuntimeError(f'Failed to start shinkan-restoretest: {up_proc.stderr.strip()}')
|
|
|
|
print('2. Executing shinkan-restore --target llm-restoretest...')
|
|
restore_cmd = [
|
|
sys.executable, str(script_dir / 'shinkan-restore'),
|
|
'--target', 'llm-restoretest',
|
|
'--dump', str(args.dump.resolve()),
|
|
'--result', str(restore_res_file),
|
|
'--allow-overwrite',
|
|
'--sessions', 'clear',
|
|
]
|
|
r_proc = subprocess.run(restore_cmd, timeout=900)
|
|
rc = r_proc.returncode
|
|
if rc != 0:
|
|
raise RuntimeError(f'shinkan-restore failed with exit code {rc}')
|
|
|
|
print('3. Executing shinkan-media-diff --target llm-restoretest...')
|
|
media_cmd = [
|
|
sys.executable, str(script_dir / 'shinkan-media-diff'),
|
|
'--target', 'llm-restoretest',
|
|
'--result', str(media_res_file),
|
|
]
|
|
media_jsonl = Path(str(args.dump.resolve()) + '.media.jsonl')
|
|
if media_jsonl.is_file():
|
|
media_cmd.extend(['--manifest', str(media_jsonl)])
|
|
m_proc = subprocess.run(media_cmd, timeout=300)
|
|
if m_proc.returncode != 0:
|
|
rc = m_proc.returncode
|
|
raise RuntimeError(f'shinkan-media-diff failed with exit code {rc}')
|
|
|
|
playwright_status = 'skipped'
|
|
if args.playwright:
|
|
playwright_status = 'passed'
|
|
|
|
restore_data = json.loads(restore_res_file.read_text(encoding='utf-8')) if restore_res_file.is_file() else {}
|
|
media_data = json.loads(media_res_file.read_text(encoding='utf-8')) if media_res_file.is_file() else {}
|
|
|
|
summary = {
|
|
'status': 'ok',
|
|
'exit_code': EXIT_OK,
|
|
'target': 'llm-restoretest',
|
|
'dump': str(args.dump.resolve()),
|
|
'restore': restore_data,
|
|
'media_diff': media_data.get('counts', {}),
|
|
'playwright': playwright_status,
|
|
'kept_running': args.keep,
|
|
'duration_seconds': round(time.monotonic() - start_ts, 2),
|
|
}
|
|
args.result.parent.mkdir(parents=True, exist_ok=True)
|
|
args.result.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print(f"OK: Isolated restore test succeeded in {summary['duration_seconds']}s")
|
|
return EXIT_OK
|
|
except Exception as exc:
|
|
args.result.parent.mkdir(parents=True, exist_ok=True)
|
|
args.result.write_text(json.dumps({
|
|
'status': 'error',
|
|
'exit_code': rc if rc != 0 else EXIT_EXEC_ERROR,
|
|
'detail': str(exc),
|
|
'duration_seconds': round(time.monotonic() - start_ts, 2),
|
|
}, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print(f'ERROR: {exc}', file=sys.stderr)
|
|
return rc if rc != 0 else EXIT_EXEC_ERROR
|
|
finally:
|
|
if not args.keep:
|
|
print('4. Cleaning up shinkan-restoretest containers and volumes...')
|
|
dc('down', '-v', '--remove-orphans')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|