#!/usr/bin/env python3 """Compare Shinkan database media references against the filesystem and backup media manifest. Implements RECOVERY_CONTRACT.md section 3.4. Read-only: never deletes or modifies files. """ import argparse import datetime as dt import hashlib import json import os from pathlib import Path import platform import socket import subprocess import sys from urllib.request import urlopen EXIT_OK = 0 EXIT_EXEC_ERROR = 1 EXIT_PRECONDITION = 2 TARGETS = { 'llm-prod': { 'host': 'llm-node', 'arch': 'x86_64', 'app_dir': '/home/llmadmin/docker/shinkan-prod', 'db_container': 'shinkan-db-prod', 'db_name': 'shinkan', 'db_user': 'shinkan_user', 'media_dir': '/srv/shinkan/prod/media', 'version_url': 'http://127.0.0.1:8003/api/version', }, 'llm-restoretest': { 'host': 'llm-node', 'arch': 'x86_64', 'app_dir': '/home/llmadmin/docker/shinkan-prod', 'db_container': 'shinkan-restoretest-postgres-1', 'db_name': 'shinkan', 'db_user': 'shinkan_user', 'media_dir': '/srv/shinkan/prod/media', 'version_url': 'http://127.0.0.1:18003/api/version', }, } def compute_sha256(path): h = hashlib.sha256() with Path(path).open('rb') as fp: while True: chunk = fp.read(65536) if not chunk: break h.update(chunk) return h.hexdigest() def load_backup_media_manifest(path): entries = {} if not path or not Path(path).is_file(): return entries with Path(path).open('r', encoding='utf-8') as fp: for line in fp: line = line.strip() if not line: continue obj = json.loads(line) if 'path' in obj: entries[obj['path']] = obj return entries def query_db_media_refs(tcfg): sql = """ SELECT 'LAST_MIG:' || COALESCE((SELECT migration FROM schema_migrations ORDER BY id DESC LIMIT 1), ''); SELECT 'ASSET:' || COALESCE(storage_key, '') || '|' || COALESCE(byte_size::text, '') || '|' || COALESCE(sha256, '') FROM media_assets WHERE storage_key IS NOT NULL AND storage_key <> ''; SELECT 'EXMEDIA:' || COALESCE(file_path, '') FROM exercise_media WHERE file_path IS NOT NULL AND file_path <> ''; """ proc = subprocess.run( [ 'docker', 'exec', tcfg['db_container'], 'sh', '-ec', 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; ' f'psql -U "{tcfg["db_user"]}" -d "{tcfg["db_name"]}" -At -c ' + json.dumps(sql), ], capture_output=True, text=True, timeout=30, ) if proc.returncode != 0: raise RuntimeError(f'Database media query failed: {proc.stderr.strip()}') last_migration = None referenced = {} for line in proc.stdout.splitlines(): if line.startswith('LAST_MIG:'): last_migration = line[len('LAST_MIG:'):].strip() or None elif line.startswith('ASSET:'): parts = line[len('ASSET:'):].split('|') rel = parts[0].lstrip('/') if rel.startswith('media/'): rel = rel[len('media/'):] size = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None sha = parts[2].lower() if len(parts) > 2 and parts[2] else None referenced[rel] = {'path': rel, 'expected_size': size, 'expected_sha256': sha, 'source': 'media_assets'} elif line.startswith('EXMEDIA:'): raw_path = line[len('EXMEDIA:'):].strip().lstrip('/') if raw_path.startswith('media/'): raw_path = raw_path[len('media/'):] if raw_path.startswith('api/'): continue if raw_path and raw_path not in referenced: referenced[raw_path] = {'path': raw_path, 'expected_size': None, 'expected_sha256': None, 'source': 'exercise_media'} return last_migration, referenced def main(): parser = argparse.ArgumentParser(description='Compare Shinkan DB media references against filesystem and manifest') parser.add_argument('--target', required=True, choices=sorted(TARGETS)) parser.add_argument('--result', required=True, type=Path) parser.add_argument('--manifest', type=Path, help='Optional .media.jsonl file') args = parser.parse_args() tcfg = dict(TARGETS[args.target]) if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'): tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE'] cur_host = socket.gethostname() cur_arch = platform.machine() if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1': if cur_host != tcfg['host'] or cur_arch != tcfg['arch']: msg = f"Host/arch mismatch: {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}" args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8') print(f'ERROR: {msg}', file=sys.stderr) return EXIT_PRECONDITION media_root = Path(tcfg['media_dir']) if not media_root.is_dir(): msg = f'Media directory not available: {media_root}' args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8') print(f'ERROR: {msg}', file=sys.stderr) return EXIT_PRECONDITION try: last_migration, db_refs = query_db_media_refs(tcfg) backup_manifest = load_backup_media_manifest(args.manifest) disk_files = {} for item in sorted(media_root.rglob('*')): if not item.is_file() or item.name.startswith('.'): continue rel = item.relative_to(media_root).as_posix() if rel.startswith('#recycle/') or rel.startswith('.Trash'): continue disk_files[rel] = item missing = [] mismatch = [] added_after_backup = [] orphan = [] for rel, ref in sorted(db_refs.items()): disk_item = disk_files.get(rel) if disk_item is None: missing.append({'path': rel, 'source': ref['source']}) continue actual_size = disk_item.stat().st_size if ref['expected_size'] is not None and ref['expected_size'] != actual_size: mismatch.append({ 'path': rel, 'reason': 'byte_size', 'expected': ref['expected_size'], 'actual': actual_size, }) continue if ref['expected_sha256']: actual_sha = compute_sha256(disk_item) if actual_sha != ref['expected_sha256']: mismatch.append({ 'path': rel, 'reason': 'sha256', 'expected': ref['expected_sha256'], 'actual': actual_sha, }) for rel in sorted(disk_files): if rel not in db_refs: if args.manifest and rel not in backup_manifest: added_after_backup.append({'path': rel, 'size': disk_files[rel].stat().st_size}) else: orphan.append({'path': rel, 'size': disk_files[rel].stat().st_size}) app_version = 'unknown' try: with urlopen(tcfg['version_url'], timeout=5) as resp: if resp.status == 200: app_version = json.loads(resp.read(4096).decode('utf-8')).get('app_version', 'unknown') except Exception: pass git_commit = 'unknown' if (Path(tcfg['app_dir']) / '.git').exists(): proc = subprocess.run(['git', '-C', tcfg['app_dir'], 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=5) if proc.returncode == 0: git_commit = proc.stdout.strip() report = { 'status': 'ok', 'exit_code': EXIT_OK, 'checked_at': dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds'), 'target': args.target, 'version_triple': { 'app_version': app_version, 'git_commit': git_commit, 'db_last_migration': last_migration, 'backup_media_manifest': str(args.manifest) if args.manifest else None, }, 'counts': { 'db_referenced': len(db_refs), 'disk_files': len(disk_files), 'missing': len(missing), 'mismatch': len(mismatch), 'added_after_backup': len(added_after_backup), 'orphan': len(orphan), }, 'missing': missing, 'mismatch': mismatch, 'added_after_backup': added_after_backup, 'orphan': orphan, } args.result.parent.mkdir(parents=True, exist_ok=True) args.result.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') print( f"OK: Media diff for {args.target}: referenced={len(db_refs)} disk={len(disk_files)} " f"missing={len(missing)} mismatch={len(mismatch)} added_after_backup={len(added_after_backup)} orphan={len(orphan)}" ) return EXIT_OK except Exception as exc: args.result.parent.mkdir(parents=True, exist_ok=True) args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)}, indent=2) + '\n', encoding='utf-8') print(f'ERROR: {exc}', file=sys.stderr) return EXIT_EXEC_ERROR if __name__ == '__main__': sys.exit(main())