Import Script und Logging für WP15b
This commit is contained in:
1 parent
f6b2375d65
commit
c676c8263f
2 files changed
+82
-51
No files matched your search
+28
-14
@@ -2,7 +2,9 @@
|
||||
"""
|
||||
scripts/import_markdown.py
|
||||
CLI-Tool zum Importieren von Markdown-Dateien in Qdrant.
|
||||
Updated for Mindnet v2.3.6 (Async Ingestion Support).
|
||||
WP-15b: Implementiert den Two-Pass Workflow (Pre-Scan + Processing).
|
||||
Sorgt dafür, dass der LocalBatchCache vor der Verarbeitung gefüllt wird.
|
||||
VERSION: 2.4.0
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
@@ -11,21 +13,16 @@ import logging
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
import logging
|
||||
# Setzt das Level global auf INFO, damit Sie den Fortschritt sehen
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
|
||||
# Wenn Sie TIEFE Einblicke wollen, setzen Sie den SemanticAnalyzer spezifisch auf DEBUG:
|
||||
logging.getLogger("app.services.semantic_analyzer").setLevel(logging.DEBUG)
|
||||
|
||||
# Importiere den neuen Async Service
|
||||
# Stellen wir sicher, dass der Pfad stimmt (Pythonpath)
|
||||
import sys
|
||||
sys.path.append(os.getcwd())
|
||||
|
||||
from app.core.ingestion import IngestionService
|
||||
from app.core.parser import pre_scan_markdown
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("importer")
|
||||
|
||||
async def main_async(args):
|
||||
@@ -34,7 +31,7 @@ async def main_async(args):
|
||||
logger.error(f"Vault path does not exist: {vault_path}")
|
||||
return
|
||||
|
||||
# Service initialisieren (startet Async Clients)
|
||||
# 1. Service initialisieren
|
||||
logger.info(f"Initializing IngestionService (Prefix: {args.prefix})")
|
||||
service = IngestionService(collection_prefix=args.prefix)
|
||||
|
||||
@@ -46,14 +43,31 @@ async def main_async(args):
|
||||
|
||||
logger.info(f"Found {len(files)} markdown files.")
|
||||
|
||||
stats = {"processed": 0, "skipped": 0, "errors": 0}
|
||||
# =========================================================================
|
||||
# PASS 1: Global Pre-Scan (WP-15b)
|
||||
# Füllt den LocalBatchCache für die semantische Kanten-Validierung.
|
||||
# =========================================================================
|
||||
logger.info(f"🔍 [Pass 1] Pre-scanning {len(files)} files for global context cache...")
|
||||
for f_path in files:
|
||||
try:
|
||||
ctx = pre_scan_markdown(str(f_path))
|
||||
if ctx:
|
||||
service.batch_cache[ctx.note_id] = ctx
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Could not pre-scan {f_path}: {e}")
|
||||
|
||||
# Wir nutzen eine Semaphore, um nicht zu viele Files gleichzeitig zu öffnen/embedden
|
||||
sem = asyncio.Semaphore(5) # Max 5 concurrent files to avoid OOM or Rate Limit
|
||||
logger.info(f"✅ Cache populated with {len(service.batch_cache)} note contexts.")
|
||||
|
||||
# =========================================================================
|
||||
# PASS 2: Processing (Batch-Verarbeitung)
|
||||
# =========================================================================
|
||||
stats = {"processed": 0, "skipped": 0, "errors": 0}
|
||||
sem = asyncio.Semaphore(5) # Max 5 parallele Dateien für Stabilität
|
||||
|
||||
async def process_with_limit(f_path):
|
||||
async with sem:
|
||||
try:
|
||||
# Nutzt den nun gefüllten Batch-Cache für die Validierung
|
||||
res = await service.process_file(
|
||||
file_path=str(f_path),
|
||||
vault_root=str(vault_path),
|
||||
@@ -65,8 +79,8 @@ async def main_async(args):
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": str(e), "path": str(f_path)}
|
||||
|
||||
# Batch Processing
|
||||
# Wir verarbeiten in Chunks, um den Progress zu sehen
|
||||
logger.info(f"🚀 [Pass 2] Starting semantic processing in batches...")
|
||||
|
||||
batch_size = 20
|
||||
for i in range(0, len(files), batch_size):
|
||||
batch = files[i:i+batch_size]
|
||||
@@ -92,7 +106,7 @@ def main():
|
||||
load_dotenv()
|
||||
default_prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
|
||||
|
||||
parser = argparse.ArgumentParser(description="Import Vault to Qdrant (Async)")
|
||||
parser = argparse.ArgumentParser(description="Import Vault to Qdrant (Two-Pass Ingestion)")
|
||||
parser.add_argument("--vault", default="./vault", help="Path to vault root")
|
||||
parser.add_argument("--prefix", default=default_prefix, help="Collection prefix")
|
||||
parser.add_argument("--force", action="store_true", help="Force re-index all files")
|
||||
|
||||
Reference in new issue
Block a user