Update qdrant_points.py, graph_utils.py, ingestion_processor.py, and import_markdown.py: Enhance ID generation and error handling, centralize identity logic to prevent ID drift, and improve documentation clarity. Update versioning to reflect changes in functionality and maintain compatibility across modules.
This commit is contained in:
1 parent
c33b1c644a
commit
b0f4309a29
4 files changed
+235
-155
No files matched your search
@@ -4,10 +4,10 @@ DESCRIPTION: Der zentrale IngestionService (Orchestrator).
|
||||
WP-25a: Integration der Mixture of Experts (MoE) Architektur.
|
||||
WP-15b: Two-Pass Workflow mit globalem Kontext-Cache.
|
||||
WP-20/22: Cloud-Resilienz und Content-Lifecycle integriert.
|
||||
AUDIT v3.4.1: Strikte 2-Phasen-Strategie (Authority-First).
|
||||
Lösung des Ghost-ID Problems via Cache-Resolution.
|
||||
Fix für Pydantic 'None'-ID Crash.
|
||||
VERSION: 3.4.1 (WP-24c: Robust Global Orchestration)
|
||||
AUDIT v3.4.2: Strikte 2-Phasen-Strategie (Authority-First).
|
||||
Lösung des Ghost-ID Problems & Pydantic-Crash Fix.
|
||||
Zentralisierte ID-Generierung zur Vermeidung von Duplikaten.
|
||||
VERSION: 3.4.2 (WP-24c: Unified ID Orchestration)
|
||||
STATUS: Active
|
||||
"""
|
||||
import logging
|
||||
@@ -22,8 +22,8 @@ from app.core.parser import (
|
||||
validate_required_frontmatter, NoteContext
|
||||
)
|
||||
from app.core.chunking import assemble_chunks
|
||||
# WP-24c: Import für die deterministische ID-Vorabberechnung aus graph_utils
|
||||
from app.core.graph.graph_utils import _mk_edge_id
|
||||
# WP-24c: Import der zentralen Identitäts-Logik und Pfad-Getter
|
||||
from app.core.graph.graph_utils import _mk_edge_id, get_vocab_path, get_schema_path
|
||||
|
||||
# Datenbank-Ebene (Modularisierte database-Infrastruktur)
|
||||
from app.core.database.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
||||
@@ -56,10 +56,16 @@ class IngestionService:
|
||||
from app.config import get_settings
|
||||
self.settings = get_settings()
|
||||
|
||||
# --- LOGGING CLEANUP (Header-Noise unterdrücken, Business erhalten) ---
|
||||
# --- LOGGING CLEANUP ---
|
||||
for lib in ["httpx", "httpcore", "qdrant_client", "urllib3", "openai"]:
|
||||
logging.getLogger(lib).setLevel(logging.WARNING)
|
||||
|
||||
# WP-24c: Explizite Initialisierung der Registry mit .env Pfaden
|
||||
edge_registry.initialize(
|
||||
vocab_path=get_vocab_path(),
|
||||
schema_path=get_schema_path()
|
||||
)
|
||||
|
||||
self.prefix = collection_prefix or self.settings.COLLECTION_PREFIX
|
||||
self.cfg = QdrantConfig.from_env()
|
||||
self.cfg.prefix = self.prefix
|
||||
@@ -73,7 +79,6 @@ class IngestionService:
|
||||
embed_cfg = self.llm.profiles.get("embedding_expert", {})
|
||||
self.dim = embed_cfg.get("dimensions") or self.settings.VECTOR_SIZE
|
||||
|
||||
# Festlegen des Change-Detection Modus
|
||||
self.active_hash_mode = self.settings.CHANGE_DETECTION_MODE
|
||||
|
||||
# WP-15b: Kontext-Gedächtnis für ID-Auflösung (Globaler Cache)
|
||||
@@ -83,7 +88,6 @@ class IngestionService:
|
||||
self.symmetry_buffer: List[Dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
# Aufruf der modularisierten Schema-Logik
|
||||
ensure_collections(self.client, self.prefix, self.dim)
|
||||
ensure_payload_indexes(self.client, self.prefix)
|
||||
except Exception as e:
|
||||
@@ -113,7 +117,6 @@ class IngestionService:
|
||||
if ctx:
|
||||
self.batch_cache[ctx.note_id] = ctx
|
||||
self.batch_cache[ctx.title] = ctx
|
||||
# Auch Dateinamen ohne Endung auflösbar machen
|
||||
self.batch_cache[os.path.splitext(os.path.basename(path))[0]] = ctx
|
||||
except Exception as e:
|
||||
logger.warning(f" ⚠️ Pre-scan fehlgeschlagen für {path}: {e}")
|
||||
@@ -142,7 +145,6 @@ class IngestionService:
|
||||
Sorgt dafür, dass virtuelle Kanten niemals Nutzer-Autorität überschreiben.
|
||||
"""
|
||||
if not self.symmetry_buffer:
|
||||
logger.info("⏭️ Symmetrie-Puffer leer. Keine Aktion erforderlich.")
|
||||
return {"status": "skipped", "reason": "buffer_empty"}
|
||||
|
||||
logger.info(f"🔄 PHASE 2: Validiere {len(self.symmetry_buffer)} Symmetrien gegen Live-DB...")
|
||||
@@ -151,7 +153,7 @@ class IngestionService:
|
||||
src, tgt, kind = v_edge.get("note_id"), v_edge.get("target_id"), v_edge.get("kind")
|
||||
if not src or not tgt: continue
|
||||
|
||||
# Deterministische ID berechnen (WP-24c Standard)
|
||||
# WP-Fix v3.4.2: NUTZUNG DER ZENTRALEN FUNKTION STATT MANUELLEM STRING
|
||||
try:
|
||||
v_id = _mk_edge_id(kind, src, tgt, "note")
|
||||
except ValueError:
|
||||
@@ -162,16 +164,14 @@ class IngestionService:
|
||||
final_virtuals.append(v_edge)
|
||||
logger.info(f" 🔄 [SYMMETRY] Add inverse: {src} --({kind})--> {tgt}")
|
||||
else:
|
||||
logger.debug(f" 🛡️ Schutz: Manuelle Kante verhindert Symmetrie {v_id}")
|
||||
logger.info(f" 🛡️ [PROTECTED] Manuelle Kante gefunden. Symmetrie für {kind} unterdrückt.")
|
||||
|
||||
if final_virtuals:
|
||||
logger.info(f"📤 Schreibe {len(final_virtuals)} geschützte Symmetrie-Kanten in Qdrant.")
|
||||
col, pts = points_for_edges(self.prefix, final_virtuals)
|
||||
# Nutzt upsert_batch mit wait=True für atomare Konsistenz
|
||||
upsert_batch(self.client, col, pts, wait=True)
|
||||
|
||||
count = len(final_virtuals)
|
||||
self.symmetry_buffer.clear() # Puffer nach Commit leeren
|
||||
self.symmetry_buffer.clear()
|
||||
return {"status": "success", "added": count}
|
||||
|
||||
async def process_file(self, file_path: str, vault_root: str, **kwargs) -> Dict[str, Any]:
|
||||
@@ -201,7 +201,6 @@ class IngestionService:
|
||||
note_id = note_pl.get("note_id")
|
||||
|
||||
if not note_id:
|
||||
logger.warning(f" ⚠️ Keine ID für {file_path}. Überspringe.")
|
||||
return {**result, "status": "error", "error": "missing_id"}
|
||||
|
||||
logger.info(f"📄 Bearbeite: '{note_id}'")
|
||||
@@ -229,10 +228,7 @@ class IngestionService:
|
||||
if not self._is_valid_id(t_id): continue
|
||||
|
||||
if cand.get("provenance") == "global_pool" and enable_smart:
|
||||
# LLM Logging
|
||||
logger.info(f" ⚖️ [VALIDATING] Relation to '{t_id}' via Experts...")
|
||||
is_valid = await validate_edge_candidate(ch.text, cand, self.batch_cache, self.llm)
|
||||
logger.info(f" 🧠 [SMART EDGE] {t_id} -> {'✅ OK' if is_valid else '❌ SKIP'}")
|
||||
if is_valid: new_pool.append(cand)
|
||||
else:
|
||||
new_pool.append(cand)
|
||||
@@ -283,11 +279,10 @@ class IngestionService:
|
||||
|
||||
if explicit_edges:
|
||||
col_e, pts_e = points_for_edges(self.prefix, explicit_edges)
|
||||
# WICHTIG: wait=True garantiert, dass die Kanten indiziert sind, bevor Phase 2 prüft
|
||||
upsert_batch(self.client, col_e, pts_e, wait=True)
|
||||
|
||||
logger.info(f" ✨ Phase 1 fertig: {len(explicit_edges)} explizite Kanten für '{note_id}'.")
|
||||
return {"status": "success", "note_id": note_id, "edges_count": len(explicit_edges)}
|
||||
return {"status": "success", "note_id": note_id}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Fehler bei {file_path}: {e}", exc_info=True)
|
||||
|
||||
Reference in new issue
Block a user