feat(v0.4): import all Raspberry Pi scripts and implement Recovery Management engine + Shinkan adapters
This commit is contained in:
1 parent
40188c0e10
commit
c53fc0a939
35 files changed
+2877
-191
No files matched your search
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare Shinkan database media references against the filesystem and backup media manifest.
|
||||
|
||||
Implements RECOVERY_CONTRACT.md section 3.4. Read-only: never deletes or modifies files.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.request import urlopen
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_EXEC_ERROR = 1
|
||||
EXIT_PRECONDITION = 2
|
||||
|
||||
TARGETS = {
|
||||
'llm-prod': {
|
||||
'host': 'llm-node',
|
||||
'arch': 'x86_64',
|
||||
'app_dir': '/home/llmadmin/docker/shinkan-prod',
|
||||
'db_container': 'shinkan-db-prod',
|
||||
'db_name': 'shinkan',
|
||||
'db_user': 'shinkan_user',
|
||||
'media_dir': '/srv/shinkan/prod/media',
|
||||
'version_url': 'http://127.0.0.1:8003/api/version',
|
||||
},
|
||||
'llm-restoretest': {
|
||||
'host': 'llm-node',
|
||||
'arch': 'x86_64',
|
||||
'app_dir': '/home/llmadmin/docker/shinkan-prod',
|
||||
'db_container': 'shinkan-restoretest-postgres-1',
|
||||
'db_name': 'shinkan',
|
||||
'db_user': 'shinkan_user',
|
||||
'media_dir': '/srv/shinkan/prod/media',
|
||||
'version_url': 'http://127.0.0.1:18003/api/version',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def compute_sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as fp:
|
||||
while True:
|
||||
chunk = fp.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def load_backup_media_manifest(path):
|
||||
entries = {}
|
||||
if not path or not Path(path).is_file():
|
||||
return entries
|
||||
with Path(path).open('r', encoding='utf-8') as fp:
|
||||
for line in fp:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
obj = json.loads(line)
|
||||
if 'path' in obj:
|
||||
entries[obj['path']] = obj
|
||||
return entries
|
||||
|
||||
|
||||
def query_db_media_refs(tcfg):
|
||||
sql = """
|
||||
SELECT 'LAST_MIG:' || COALESCE((SELECT migration FROM schema_migrations ORDER BY id DESC LIMIT 1), '');
|
||||
SELECT 'ASSET:' || COALESCE(storage_key, '') || '|' || COALESCE(byte_size::text, '') || '|' || COALESCE(sha256, '')
|
||||
FROM media_assets
|
||||
WHERE storage_key IS NOT NULL AND storage_key <> '';
|
||||
SELECT 'EXMEDIA:' || COALESCE(file_path, '')
|
||||
FROM exercise_media
|
||||
WHERE file_path IS NOT NULL AND file_path <> '';
|
||||
"""
|
||||
proc = subprocess.run(
|
||||
[
|
||||
'docker', 'exec', tcfg['db_container'],
|
||||
'sh', '-ec',
|
||||
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; '
|
||||
f'psql -U "{tcfg["db_user"]}" -d "{tcfg["db_name"]}" -At -c ' + json.dumps(sql),
|
||||
],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f'Database media query failed: {proc.stderr.strip()}')
|
||||
|
||||
last_migration = None
|
||||
referenced = {}
|
||||
for line in proc.stdout.splitlines():
|
||||
if line.startswith('LAST_MIG:'):
|
||||
last_migration = line[len('LAST_MIG:'):].strip() or None
|
||||
elif line.startswith('ASSET:'):
|
||||
parts = line[len('ASSET:'):].split('|')
|
||||
rel = parts[0].lstrip('/')
|
||||
if rel.startswith('media/'):
|
||||
rel = rel[len('media/'):]
|
||||
size = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else None
|
||||
sha = parts[2].lower() if len(parts) > 2 and parts[2] else None
|
||||
referenced[rel] = {'path': rel, 'expected_size': size, 'expected_sha256': sha, 'source': 'media_assets'}
|
||||
elif line.startswith('EXMEDIA:'):
|
||||
raw_path = line[len('EXMEDIA:'):].strip().lstrip('/')
|
||||
if raw_path.startswith('media/'):
|
||||
raw_path = raw_path[len('media/'):]
|
||||
if raw_path.startswith('api/'):
|
||||
continue
|
||||
if raw_path and raw_path not in referenced:
|
||||
referenced[raw_path] = {'path': raw_path, 'expected_size': None, 'expected_sha256': None, 'source': 'exercise_media'}
|
||||
return last_migration, referenced
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Compare Shinkan DB media references against filesystem and manifest')
|
||||
parser.add_argument('--target', required=True, choices=sorted(TARGETS))
|
||||
parser.add_argument('--result', required=True, type=Path)
|
||||
parser.add_argument('--manifest', type=Path, help='Optional <dump>.media.jsonl file')
|
||||
args = parser.parse_args()
|
||||
|
||||
tcfg = dict(TARGETS[args.target])
|
||||
if os.environ.get('SHINKAN_MEDIA_DIR_OVERRIDE'):
|
||||
tcfg['media_dir'] = os.environ['SHINKAN_MEDIA_DIR_OVERRIDE']
|
||||
|
||||
cur_host = socket.gethostname()
|
||||
cur_arch = platform.machine()
|
||||
if os.environ.get('SHINKAN_SKIP_HOST_GUARD') != '1':
|
||||
if cur_host != tcfg['host'] or cur_arch != tcfg['arch']:
|
||||
msg = f"Host/arch mismatch: {cur_host}/{cur_arch}, expected {tcfg['host']}/{tcfg['arch']}"
|
||||
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
media_root = Path(tcfg['media_dir'])
|
||||
if not media_root.is_dir():
|
||||
msg = f'Media directory not available: {media_root}'
|
||||
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_PRECONDITION, 'detail': msg}, indent=2) + '\n', encoding='utf-8')
|
||||
print(f'ERROR: {msg}', file=sys.stderr)
|
||||
return EXIT_PRECONDITION
|
||||
|
||||
try:
|
||||
last_migration, db_refs = query_db_media_refs(tcfg)
|
||||
backup_manifest = load_backup_media_manifest(args.manifest)
|
||||
|
||||
disk_files = {}
|
||||
for item in sorted(media_root.rglob('*')):
|
||||
if not item.is_file() or item.name.startswith('.'):
|
||||
continue
|
||||
rel = item.relative_to(media_root).as_posix()
|
||||
if rel.startswith('#recycle/') or rel.startswith('.Trash'):
|
||||
continue
|
||||
disk_files[rel] = item
|
||||
|
||||
missing = []
|
||||
mismatch = []
|
||||
added_after_backup = []
|
||||
orphan = []
|
||||
|
||||
for rel, ref in sorted(db_refs.items()):
|
||||
disk_item = disk_files.get(rel)
|
||||
if disk_item is None:
|
||||
missing.append({'path': rel, 'source': ref['source']})
|
||||
continue
|
||||
actual_size = disk_item.stat().st_size
|
||||
if ref['expected_size'] is not None and ref['expected_size'] != actual_size:
|
||||
mismatch.append({
|
||||
'path': rel, 'reason': 'byte_size',
|
||||
'expected': ref['expected_size'], 'actual': actual_size,
|
||||
})
|
||||
continue
|
||||
if ref['expected_sha256']:
|
||||
actual_sha = compute_sha256(disk_item)
|
||||
if actual_sha != ref['expected_sha256']:
|
||||
mismatch.append({
|
||||
'path': rel, 'reason': 'sha256',
|
||||
'expected': ref['expected_sha256'], 'actual': actual_sha,
|
||||
})
|
||||
|
||||
for rel in sorted(disk_files):
|
||||
if rel not in db_refs:
|
||||
if args.manifest and rel not in backup_manifest:
|
||||
added_after_backup.append({'path': rel, 'size': disk_files[rel].stat().st_size})
|
||||
else:
|
||||
orphan.append({'path': rel, 'size': disk_files[rel].stat().st_size})
|
||||
|
||||
app_version = 'unknown'
|
||||
try:
|
||||
with urlopen(tcfg['version_url'], timeout=5) as resp:
|
||||
if resp.status == 200:
|
||||
app_version = json.loads(resp.read(4096).decode('utf-8')).get('app_version', 'unknown')
|
||||
except Exception:
|
||||
pass
|
||||
git_commit = 'unknown'
|
||||
if (Path(tcfg['app_dir']) / '.git').exists():
|
||||
proc = subprocess.run(['git', '-C', tcfg['app_dir'], 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=5)
|
||||
if proc.returncode == 0:
|
||||
git_commit = proc.stdout.strip()
|
||||
|
||||
report = {
|
||||
'status': 'ok',
|
||||
'exit_code': EXIT_OK,
|
||||
'checked_at': dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec='seconds'),
|
||||
'target': args.target,
|
||||
'version_triple': {
|
||||
'app_version': app_version,
|
||||
'git_commit': git_commit,
|
||||
'db_last_migration': last_migration,
|
||||
'backup_media_manifest': str(args.manifest) if args.manifest else None,
|
||||
},
|
||||
'counts': {
|
||||
'db_referenced': len(db_refs),
|
||||
'disk_files': len(disk_files),
|
||||
'missing': len(missing),
|
||||
'mismatch': len(mismatch),
|
||||
'added_after_backup': len(added_after_backup),
|
||||
'orphan': len(orphan),
|
||||
},
|
||||
'missing': missing,
|
||||
'mismatch': mismatch,
|
||||
'added_after_backup': added_after_backup,
|
||||
'orphan': orphan,
|
||||
}
|
||||
args.result.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.result.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print(
|
||||
f"OK: Media diff for {args.target}: referenced={len(db_refs)} disk={len(disk_files)} "
|
||||
f"missing={len(missing)} mismatch={len(mismatch)} added_after_backup={len(added_after_backup)} orphan={len(orphan)}"
|
||||
)
|
||||
return EXIT_OK
|
||||
except Exception as exc:
|
||||
args.result.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.result.write_text(json.dumps({'status': 'error', 'exit_code': EXIT_EXEC_ERROR, 'detail': str(exc)}, indent=2) + '\n', encoding='utf-8')
|
||||
print(f'ERROR: {exc}', file=sys.stderr)
|
||||
return EXIT_EXEC_ERROR
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user