243 lines
9.5 KiB
Python
243 lines
9.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare Shinkan database media references against the filesystem and backup media manifest.
|
|
|
|
Implements RECOVERY_CONTRACT.md section 3.4. Read-only: never deletes or modifies files.
|
|
"""
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
from urllib.request import urlopen
|
|
|
|
EXIT_OK = 0
|
|
EXIT_EXEC_ERROR = 1
|
|
EXIT_PRECONDITION = 2
|
|
|
|
TARGETS = {
|
|
'llm-prod': {
|
|
'host': 'llm-node',
|
|
'arch': 'x86_64',
|
|
'app_dir': '/home/llmadmin/docker/shinkan-prod',
|
|
'db_container': 'shinkan-db-prod',
|
|
'db_name': 'shinkan',
|
|
'db_user': 'shinkan_user',
|
|
'media_dir': '/srv/shinkan/prod/media',
|
|
'version_url': 'http://127.0.0.1:8003/api/version',
|
|
},
|
|
'llm-restoretest': {
|
|
'host': 'llm-node',
|
|
'arch': 'x86_64',
|
|
'app_dir': '/home/llmadmin/docker/shinkan-prod',
|
|
'db_container': 'shinkan-restoretest-postgres-1',
|
|
'db_name': 'shinkan',
|
|
'db_user': 'shinkan_user',
|
|
'media_dir': '/srv/shinkan/prod/media',
|
|
'version_url': 'http://127.0.0.1:18003/api/version',
|
|
},
|
|
}
|
|
|
|
|
|
def compute_sha256(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open('rb') as fp:
|
|
while True:
|
|
chunk = fp.read(65536)
|
|
if not chunk:
|
|
break
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def load_backup_media_manifest(path):
|
|
entries = {}
|
|
if not path or not Path(path).is_file():
|
|
return entries
|
|
with Path(path).open('r', encoding='utf-8') as fp:
|
|
for line in fp:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
obj = json.loads(line)
|
|
if 'path' in obj:
|
|
entries[obj['path']] = obj
|
|
return entries
|
|
|
|
|
|
def query_db_media_refs(tcfg):
|
|
sql = """
|
|
SELECT 'LAST_MIG:' || COALESCE((SELECT migration FROM schema_migrations ORDER BY id DESC LIMIT 1), '');
|
|
SELECT 'ASSET:' || COALESCE(storage_key, '') || '|' || COALESCE(byte_size::text, '') || '|' || COALESCE(sha256, '')
|
|
FROM media_assets
|
|
WHERE storage_key IS NOT NULL AND storage_key <> '';
|
|
SELECT 'EXMEDIA:' || COALESCE(file_path, '')
|
|
FROM exercise_media
|
|
WHERE file_path IS NOT NULL AND file_path <> '';
|
|
"""
|
|
proc = subprocess.run(
|
|
[
|
|
'docker', 'exec', '-i', tcfg['db_container'],
|
|
'sh', '-ec',
|
|
f'export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; exec psql -U "{tcfg["db_user"]}" -d "{tcfg["db_name"]}" -At',
|
|
],
|
|
input=sql, capture_output=True, text=True, timeout=30,
|
|
)
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f'Database media query failed: {proc.stderr.strip()}')
|
|
|
|
last_migration = None
|
|
referenced = {}
|
|
for line in proc.stdout.splitlines():
|
|
if line.startswith('LAST_MIG:'):
|
|
last_migration = line[len('LAST_MIG:'):].strip() or None
|
|
elif line.startswith('ASSET:'):
|
|
parts = line[len('ASSET:'):].split('|')
|
|
rel = parts[0].lstrip('/')
|
|
if rel.startswith('media/'):
|
|
rel = rel[len('media/'):]
|
|
size = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
|
|
sha = parts[2].lower() if len(parts) > 2 and parts[2] else None
|
|
referenced[rel] = {'path': rel, 'expected_size': size, 'expected_sha256': sha, 'source': 'media_assets'}
|
|
elif line.startswith('EXMEDIA:'):
|
|
raw_path = line[len('EXMEDIA:'):].strip().lstrip('/')
|
|
if raw_path.startswith('media/'):
|
|
raw_path = raw_path[len('media/'):]
|
|
if raw_path.startswith('api/'):
|
|
continue
|
|
if raw_path and raw_path not in referenced:
|
|
referenced[raw_path] = {'path': raw_path, 'expected_size': None, 'expected_sha256': None, 'source': 'exercise_media'}
|
|
return last_migration, referenced
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Compare Shinkan DB media references against filesystem and manifest')
|
|
parser.add_argument('--target', required=True, choices=sorted(TARGETS))
|
|
parser.add_argument('--result', required=True, type=Path)
|
|
parser.add_argument('--manifest', type=Path, help='Optional <dump>.media.jsonl file')
|
|
args = parser.parse_args()
|
|
|
|
tcfg = dict(TARGETS[args.target])
|
|
if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'):
|
|
tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE']
|
|
|
|
cur_host = socket.gethostname()
|
|
cur_arch = platform.machine()
|
|
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
|
|
if cur_host != tcfg['host'] or cur_arch != tcfg['arch']:
|
|
msg = f"Host/arch mismatch: {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}"
|
|
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
|
|
print(f'ERROR: {msg}', file=sys.stderr)
|
|
return EXIT_PRECONDITION
|
|
|
|
media_root = Path(tcfg['media_dir'])
|
|
if not media_root.is_dir():
|
|
msg = f'Media directory not available: {media_root}'
|
|
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
|
|
print(f'ERROR: {msg}', file=sys.stderr)
|
|
return EXIT_PRECONDITION
|
|
|
|
try:
|
|
last_migration, db_refs = query_db_media_refs(tcfg)
|
|
backup_manifest = load_backup_media_manifest(args.manifest)
|
|
|
|
disk_files = {}
|
|
for item in sorted(media_root.rglob('*')):
|
|
if not item.is_file() or item.name.startswith('.'):
|
|
continue
|
|
rel = item.relative_to(media_root).as_posix()
|
|
if rel.startswith('#recycle/') or rel.startswith('.Trash'):
|
|
continue
|
|
disk_files[rel] = item
|
|
|
|
missing = []
|
|
mismatch = []
|
|
added_after_backup = []
|
|
orphan = []
|
|
|
|
for rel, ref in sorted(db_refs.items()):
|
|
disk_item = disk_files.get(rel)
|
|
if disk_item is None:
|
|
missing.append({'path': rel, 'source': ref['source']})
|
|
continue
|
|
actual_size = disk_item.stat().st_size
|
|
if ref['expected_size'] is not None and ref['expected_size'] != actual_size:
|
|
mismatch.append({
|
|
'path': rel, 'reason': 'byte_size',
|
|
'expected': ref['expected_size'], 'actual': actual_size,
|
|
})
|
|
continue
|
|
if ref['expected_sha256']:
|
|
actual_sha = compute_sha256(disk_item)
|
|
if actual_sha != ref['expected_sha256']:
|
|
mismatch.append({
|
|
'path': rel, 'reason': 'sha256',
|
|
'expected': ref['expected_sha256'], 'actual': actual_sha,
|
|
})
|
|
|
|
for rel in sorted(disk_files):
|
|
if rel not in db_refs:
|
|
if args.manifest and rel not in backup_manifest:
|
|
added_after_backup.append({'path': rel, 'size': disk_files[rel].stat().st_size})
|
|
else:
|
|
orphan.append({'path': rel, 'size': disk_files[rel].stat().st_size})
|
|
|
|
app_version = 'unknown'
|
|
try:
|
|
with urlopen(tcfg['version_url'], timeout=5) as resp:
|
|
if resp.status == 200:
|
|
app_version = json.loads(resp.read(4096).decode('utf-8')).get('app_version', 'unknown')
|
|
except Exception:
|
|
pass
|
|
git_commit = 'unknown'
|
|
if (Path(tcfg['app_dir']) / '.git').exists():
|
|
proc = subprocess.run(['git', '-C', tcfg['app_dir'], 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=5)
|
|
if proc.returncode == 0:
|
|
git_commit = proc.stdout.strip()
|
|
|
|
report = {
|
|
'status': 'ok',
|
|
'exit_code': EXIT_OK,
|
|
'checked_at': dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds'),
|
|
'target': args.target,
|
|
'version_triple': {
|
|
'app_version': app_version,
|
|
'git_commit': git_commit,
|
|
'db_last_migration': last_migration,
|
|
'backup_media_manifest': str(args.manifest) if args.manifest else None,
|
|
},
|
|
'counts': {
|
|
'db_referenced': len(db_refs),
|
|
'disk_files': len(disk_files),
|
|
'missing': len(missing),
|
|
'mismatch': len(mismatch),
|
|
'added_after_backup': len(added_after_backup),
|
|
'orphan': len(orphan),
|
|
},
|
|
'missing': missing,
|
|
'mismatch': mismatch,
|
|
'added_after_backup': added_after_backup,
|
|
'orphan': orphan,
|
|
}
|
|
args.result.parent.mkdir(parents=True, exist_ok=True)
|
|
args.result.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print(
|
|
f"OK: Media diff for {args.target}: referenced={len(db_refs)} disk={len(disk_files)} "
|
|
f"missing={len(missing)} mismatch={len(mismatch)} added_after_backup={len(added_after_backup)} orphan={len(orphan)}"
|
|
)
|
|
return EXIT_OK
|
|
except Exception as exc:
|
|
args.result.parent.mkdir(parents=True, exist_ok=True)
|
|
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)}, indent=2) + '\n', encoding='utf-8')
|
|
print(f'ERROR: {exc}', file=sys.stderr)
|
|
return EXIT_EXEC_ERROR
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|