Update qdrant_points.py, graph_utils.py, ingestion_processor.py, and import_markdown.py: Enhance ID generation and error handling, centralize identity logic to prevent ID drift, and improve documentation clarity. Update versioning to reflect changes in functionality and maintain compatibility across modules.

This commit is contained in:
Lars committed 2026-01-10 14:00:12 +01:00
1 parent c33b1c644a
commit b0f4309a29
4 files changed
+235 -155

No files matched your search

+17 -22
View File
@@ -4,10 +4,10 @@ DESCRIPTION: Der zentrale IngestionService (Orchestrator).
WP-25a: Integration der Mixture of Experts (MoE) Architektur.
WP-15b: Two-Pass Workflow mit globalem Kontext-Cache.
WP-20/22: Cloud-Resilienz und Content-Lifecycle integriert.
AUDIT v3.4.1: Strikte 2-Phasen-Strategie (Authority-First).
Lösung des Ghost-ID Problems via Cache-Resolution.
Fix für Pydantic 'None'-ID Crash.
VERSION: 3.4.1 (WP-24c: Robust Global Orchestration)
AUDIT v3.4.2: Strikte 2-Phasen-Strategie (Authority-First).
Lösung des Ghost-ID Problems & Pydantic-Crash Fix.
Zentralisierte ID-Generierung zur Vermeidung von Duplikaten.
VERSION: 3.4.2 (WP-24c: Unified ID Orchestration)
STATUS: Active
"""
import logging
@@ -22,8 +22,8 @@ from app.core.parser import (
validate_required_frontmatter, NoteContext
)
from app.core.chunking import assemble_chunks
# WP-24c: Import für die deterministische ID-Vorabberechnung aus graph_utils
from app.core.graph.graph_utils import _mk_edge_id
# WP-24c: Import der zentralen Identitäts-Logik und Pfad-Getter
from app.core.graph.graph_utils import _mk_edge_id, get_vocab_path, get_schema_path
# Datenbank-Ebene (Modularisierte database-Infrastruktur)
from app.core.database.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
@@ -56,10 +56,16 @@ class IngestionService:
from app.config import get_settings
self.settings = get_settings()
# --- LOGGING CLEANUP (Header-Noise unterdrücken, Business erhalten) ---
# --- LOGGING CLEANUP ---
for lib in ["httpx", "httpcore", "qdrant_client", "urllib3", "openai"]:
logging.getLogger(lib).setLevel(logging.WARNING)
# WP-24c: Explizite Initialisierung der Registry mit .env Pfaden
edge_registry.initialize(
vocab_path=get_vocab_path(),
schema_path=get_schema_path()
)
self.prefix = collection_prefix or self.settings.COLLECTION_PREFIX
self.cfg = QdrantConfig.from_env()
self.cfg.prefix = self.prefix
@@ -73,7 +79,6 @@ class IngestionService:
embed_cfg = self.llm.profiles.get("embedding_expert", {})
self.dim = embed_cfg.get("dimensions") or self.settings.VECTOR_SIZE
# Festlegen des Change-Detection Modus
self.active_hash_mode = self.settings.CHANGE_DETECTION_MODE
# WP-15b: Kontext-Gedächtnis für ID-Auflösung (Globaler Cache)
@@ -83,7 +88,6 @@ class IngestionService:
self.symmetry_buffer: List[Dict[str, Any]] = []
try:
# Aufruf der modularisierten Schema-Logik
ensure_collections(self.client, self.prefix, self.dim)
ensure_payload_indexes(self.client, self.prefix)
except Exception as e:
@@ -113,7 +117,6 @@ class IngestionService:
if ctx:
self.batch_cache[ctx.note_id] = ctx
self.batch_cache[ctx.title] = ctx
# Auch Dateinamen ohne Endung auflösbar machen
self.batch_cache[os.path.splitext(os.path.basename(path))[0]] = ctx
except Exception as e:
logger.warning(f" ⚠️ Pre-scan fehlgeschlagen für {path}: {e}")
@@ -142,7 +145,6 @@ class IngestionService:
Sorgt dafür, dass virtuelle Kanten niemals Nutzer-Autorität überschreiben.
"""
if not self.symmetry_buffer:
logger.info("⏭️ Symmetrie-Puffer leer. Keine Aktion erforderlich.")
return {"status": "skipped", "reason": "buffer_empty"}
logger.info(f"🔄 PHASE 2: Validiere {len(self.symmetry_buffer)} Symmetrien gegen Live-DB...")
@@ -151,7 +153,7 @@ class IngestionService:
src, tgt, kind = v_edge.get("note_id"), v_edge.get("target_id"), v_edge.get("kind")
if not src or not tgt: continue
# Deterministische ID berechnen (WP-24c Standard)
# WP-Fix v3.4.2: NUTZUNG DER ZENTRALEN FUNKTION STATT MANUELLEM STRING
try:
v_id = _mk_edge_id(kind, src, tgt, "note")
except ValueError:
@@ -162,16 +164,14 @@ class IngestionService:
final_virtuals.append(v_edge)
logger.info(f" 🔄 [SYMMETRY] Add inverse: {src} --({kind})--> {tgt}")
else:
logger.debug(f" 🛡️ Schutz: Manuelle Kante verhindert Symmetrie {v_id}")
logger.info(f" 🛡️ [PROTECTED] Manuelle Kante gefunden. Symmetrie für {kind} unterdrückt.")
if final_virtuals:
logger.info(f"📤 Schreibe {len(final_virtuals)} geschützte Symmetrie-Kanten in Qdrant.")
col, pts = points_for_edges(self.prefix, final_virtuals)
# Nutzt upsert_batch mit wait=True für atomare Konsistenz
upsert_batch(self.client, col, pts, wait=True)
count = len(final_virtuals)
self.symmetry_buffer.clear() # Puffer nach Commit leeren
self.symmetry_buffer.clear()
return {"status": "success", "added": count}
async def process_file(self, file_path: str, vault_root: str, **kwargs) -> Dict[str, Any]:
@@ -201,7 +201,6 @@ class IngestionService:
note_id = note_pl.get("note_id")
if not note_id:
logger.warning(f" ⚠️ Keine ID für {file_path}. Überspringe.")
return {**result, "status": "error", "error": "missing_id"}
logger.info(f"📄 Bearbeite: '{note_id}'")
@@ -229,10 +228,7 @@ class IngestionService:
if not self._is_valid_id(t_id): continue
if cand.get("provenance") == "global_pool" and enable_smart:
# LLM Logging
logger.info(f" ⚖️ [VALIDATING] Relation to '{t_id}' via Experts...")
is_valid = await validate_edge_candidate(ch.text, cand, self.batch_cache, self.llm)
logger.info(f" 🧠 [SMART EDGE] {t_id} -> {'✅ OK' if is_valid else '❌ SKIP'}")
if is_valid: new_pool.append(cand)
else:
new_pool.append(cand)
@@ -283,11 +279,10 @@ class IngestionService:
if explicit_edges:
col_e, pts_e = points_for_edges(self.prefix, explicit_edges)
# WICHTIG: wait=True garantiert, dass die Kanten indiziert sind, bevor Phase 2 prüft
upsert_batch(self.client, col_e, pts_e, wait=True)
logger.info(f" ✨ Phase 1 fertig: {len(explicit_edges)} explizite Kanten für '{note_id}'.")
return {"status": "success", "note_id": note_id, "edges_count": len(explicit_edges)}
return {"status": "success", "note_id": note_id}
except Exception as e:
logger.error(f"❌ Fehler bei {file_path}: {e}", exc_info=True)