Files

243 lines
9.5 KiB
Python

#!/usr/bin/env python3
"""Compare Shinkan database media references against the filesystem and backup media manifest.
Implements RECOVERY_CONTRACT.md section 3.4. Read-only: never deletes or modifies files.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import platform
import socket
import subprocess
import sys
from urllib.request import urlopen
EXIT_OK = 0
EXIT_EXEC_ERROR = 1
EXIT_PRECONDITION = 2
TARGETS = {
'llm-prod': {
'host': 'llm-node',
'arch': 'x86_64',
'app_dir': '/home/llmadmin/docker/shinkan-prod',
'db_container': 'shinkan-db-prod',
'db_name': 'shinkan',
'db_user': 'shinkan_user',
'media_dir': '/srv/shinkan/prod/media',
'version_url': 'http://127.0.0.1:8003/api/version',
},
'llm-restoretest': {
'host': 'llm-node',
'arch': 'x86_64',
'app_dir': '/home/llmadmin/docker/shinkan-prod',
'db_container': 'shinkan-restoretest-postgres-1',
'db_name': 'shinkan',
'db_user': 'shinkan_user',
'media_dir': '/srv/shinkan/prod/media',
'version_url': 'http://127.0.0.1:18003/api/version',
},
}
def compute_sha256(path):
h = hashlib.sha256()
with Path(path).open('rb') as fp:
while True:
chunk = fp.read(65536)
if not chunk:
break
h.update(chunk)
return h.hexdigest()
def load_backup_media_manifest(path):
entries = {}
if not path or not Path(path).is_file():
return entries
with Path(path).open('r', encoding='utf-8') as fp:
for line in fp:
line = line.strip()
if not line:
continue
obj = json.loads(line)
if 'path' in obj:
entries[obj['path']] = obj
return entries
def query_db_media_refs(tcfg):
sql = """
SELECT 'LAST_MIG:' || COALESCE((SELECT migration FROM schema_migrations ORDER BY id DESC LIMIT 1), '');
SELECT 'ASSET:' || COALESCE(storage_key, '') || '|' || COALESCE(byte_size::text, '') || '|' || COALESCE(sha256, '')
FROM media_assets
WHERE storage_key IS NOT NULL AND storage_key <> '';
SELECT 'EXMEDIA:' || COALESCE(file_path, '')
FROM exercise_media
WHERE file_path IS NOT NULL AND file_path <> '';
"""
proc = subprocess.run(
[
'docker', 'exec', '-i', tcfg['db_container'],
'sh', '-ec',
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -U "{tcfg["db_user"]}" -d "{tcfg["db_name"]}" -At',
],
input=sql, capture_output=True, text=True, timeout=30,
)
if proc.returncode != 0:
raise RuntimeError(f'Database media query failed: {proc.stderr.strip()}')
last_migration = None
referenced = {}
for line in proc.stdout.splitlines():
if line.startswith('LAST_MIG:'):
last_migration = line[len('LAST_MIG:'):].strip() or None
elif line.startswith('ASSET:'):
parts = line[len('ASSET:'):].split('|')
rel = parts[0].lstrip('/')
if rel.startswith('media/'):
rel = rel[len('media/'):]
size = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
sha = parts[2].lower() if len(parts) > 2 and parts[2] else None
referenced[rel] = {'path': rel, 'expected_size': size, 'expected_sha256': sha, 'source': 'media_assets'}
elif line.startswith('EXMEDIA:'):
raw_path = line[len('EXMEDIA:'):].strip().lstrip('/')
if raw_path.startswith('media/'):
raw_path = raw_path[len('media/'):]
if raw_path.startswith('api/'):
continue
if raw_path and raw_path not in referenced:
referenced[raw_path] = {'path': raw_path, 'expected_size': None, 'expected_sha256': None, 'source': 'exercise_media'}
return last_migration, referenced
def main():
parser = argparse.ArgumentParser(description='Compare Shinkan DB media references against filesystem and manifest')
parser.add_argument('--target', required=True, choices=sorted(TARGETS))
parser.add_argument('--result', required=True, type=Path)
parser.add_argument('--manifest', type=Path, help='Optional <dump>.media.jsonl file')
args = parser.parse_args()
tcfg = dict(TARGETS[args.target])
if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'):
tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE']
cur_host = socket.gethostname()
cur_arch = platform.machine()
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
if cur_host != tcfg['host'] or cur_arch != tcfg['arch']:
msg = f"Host/arch mismatch: {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}"
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
media_root = Path(tcfg['media_dir'])
if not media_root.is_dir():
msg = f'Media directory not available: {media_root}'
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
print(f'ERROR: {msg}', file=sys.stderr)
return EXIT_PRECONDITION
try:
last_migration, db_refs = query_db_media_refs(tcfg)
backup_manifest = load_backup_media_manifest(args.manifest)
disk_files = {}
for item in sorted(media_root.rglob('*')):
if not item.is_file() or item.name.startswith('.'):
continue
rel = item.relative_to(media_root).as_posix()
if rel.startswith('#recycle/') or rel.startswith('.Trash'):
continue
disk_files[rel] = item
missing = []
mismatch = []
added_after_backup = []
orphan = []
for rel, ref in sorted(db_refs.items()):
disk_item = disk_files.get(rel)
if disk_item is None:
missing.append({'path': rel, 'source': ref['source']})
continue
actual_size = disk_item.stat().st_size
if ref['expected_size'] is not None and ref['expected_size'] != actual_size:
mismatch.append({
'path': rel, 'reason': 'byte_size',
'expected': ref['expected_size'], 'actual': actual_size,
})
continue
if ref['expected_sha256']:
actual_sha = compute_sha256(disk_item)
if actual_sha != ref['expected_sha256']:
mismatch.append({
'path': rel, 'reason': 'sha256',
'expected': ref['expected_sha256'], 'actual': actual_sha,
})
for rel in sorted(disk_files):
if rel not in db_refs:
if args.manifest and rel not in backup_manifest:
added_after_backup.append({'path': rel, 'size': disk_files[rel].stat().st_size})
else:
orphan.append({'path': rel, 'size': disk_files[rel].stat().st_size})
app_version = 'unknown'
try:
with urlopen(tcfg['version_url'], timeout=5) as resp:
if resp.status == 200:
app_version = json.loads(resp.read(4096).decode('utf-8')).get('app_version', 'unknown')
except Exception:
pass
git_commit = 'unknown'
if (Path(tcfg['app_dir']) / '.git').exists():
proc = subprocess.run(['git', '-C', tcfg['app_dir'], 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=5)
if proc.returncode == 0:
git_commit = proc.stdout.strip()
report = {
'status': 'ok',
'exit_code': EXIT_OK,
'checked_at': dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds'),
'target': args.target,
'version_triple': {
'app_version': app_version,
'git_commit': git_commit,
'db_last_migration': last_migration,
'backup_media_manifest': str(args.manifest) if args.manifest else None,
},
'counts': {
'db_referenced': len(db_refs),
'disk_files': len(disk_files),
'missing': len(missing),
'mismatch': len(mismatch),
'added_after_backup': len(added_after_backup),
'orphan': len(orphan),
},
'missing': missing,
'mismatch': mismatch,
'added_after_backup': added_after_backup,
'orphan': orphan,
}
args.result.parent.mkdir(parents=True, exist_ok=True)
args.result.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(
f"OK: Media diff for {args.target}: referenced={len(db_refs)} disk={len(disk_files)} "
f"missing={len(missing)} mismatch={len(mismatch)} added_after_backup={len(added_after_backup)} orphan={len(orphan)}"
)
return EXIT_OK
except Exception as exc:
args.result.parent.mkdir(parents=True, exist_ok=True)
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)}, indent=2) + '\n', encoding='utf-8')
print(f'ERROR: {exc}', file=sys.stderr)
return EXIT_EXEC_ERROR
if __name__ == '__main__':
sys.exit(main())