Update qdrant_points.py, graph_utils.py, ingestion_processor.py, and import_markdown.py: Enhance ID generation and error handling, centralize identity logic to prevent ID drift, and improve documentation clarity. Update versioning to reflect changes in functionality and maintain compatibility across modules.

This commit is contained in:
Lars committed 2026-01-10 14:00:12 +01:00
1 parent c33b1c644a
commit b0f4309a29
4 files changed
+235 -155

No files matched your search

+87 -33
View File
@@ -1,9 +1,10 @@
"""
FILE: app/core/database/qdrant_points.py
DESCRIPTION: Object-Mapper für Qdrant. Konvertiert JSON-Payloads (Notes, Chunks, Edges) in PointStructs und generiert deterministische UUIDs.
VERSION: 1.5.2 (WP-Fix: Atomic Consistency & Canonical Edge IDs)
DESCRIPTION: Object-Mapper für Qdrant. Konvertiert JSON-Payloads (Notes, Chunks, Edges)
in PointStructs und generiert deterministische UUIDs.
VERSION: 1.5.3 (WP-Fix: Centralized Identity Enforcement)
STATUS: Active
DEPENDENCIES: qdrant_client, uuid, os
DEPENDENCIES: qdrant_client, uuid, os, app.core.graph.graph_utils
LAST_ANALYSIS: 2026-01-10
"""
from __future__ import annotations
@@ -14,6 +15,9 @@ from typing import List, Tuple, Iterable, Optional, Dict, Any
from qdrant_client.http import models as rest
from qdrant_client import QdrantClient
# WP-24c: Import der zentralen Identitäts-Logik zur Vermeidung von ID-Drift
from app.core.graph.graph_utils import _mk_edge_id
# --------------------- ID helpers ---------------------
def _to_uuid(stable_key: str) -> str:
@@ -26,19 +30,29 @@ def _to_uuid(stable_key: str) -> str:
return str(uuid.uuid5(uuid.NAMESPACE_URL, str(stable_key)))
def _names(prefix: str) -> Tuple[str, str, str]:
"""Interne Auflösung der Collection-Namen basierend auf dem Präfix."""
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
# --------------------- Points builders ---------------------
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
"""Konvertiert Note-Metadaten in Qdrant Points."""
notes_col, _, _ = _names(prefix)
# Nutzt Null-Vektor als Fallback, falls kein Embedding vorhanden ist
vector = note_vec if note_vec is not None else [0.0] * int(dim)
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
point_id = _to_uuid(raw_note_id)
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
pt = rest.PointStruct(
id=point_id,
vector=vector,
payload=note_payload
)
return notes_col, [pt]
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
"""Konvertiert Chunks und deren Vektoren in Qdrant Points."""
_, chunks_col, _ = _names(prefix)
points: List[rest.PointStruct] = []
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
@@ -47,8 +61,13 @@ def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[Lis
note_id = pl.get("note_id") or pl.get("parent_note_id") or "missing-note"
chunk_id = f"{note_id}#{i}"
pl["chunk_id"] = chunk_id
point_id = _to_uuid(chunk_id)
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
points.append(rest.PointStruct(
id=point_id,
vector=vec,
payload=pl
))
return chunks_col, points
def _normalize_edge_payload(pl: dict) -> dict:
@@ -76,30 +95,54 @@ def _normalize_edge_payload(pl: dict) -> dict:
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
"""
Konvertiert Kanten-Payloads in PointStructs.
WP-24c: Nutzt strikte ID-Kanonisierung für die Symmetrie-Integrität.
WP-24c Audit v1.5.3: Nutzt die zentrale _mk_edge_id Funktion aus graph_utils.
Dies eliminiert den ID-Drift zwischen manuellen und virtuellen Kanten.
"""
_, _, edges_col = _names(prefix)
points: List[rest.PointStruct] = []
for raw in edge_payloads:
pl = _normalize_edge_payload(raw)
# WP-24c: Deterministische ID-Generierung zur Kollisionsvermeidung
# Extraktion der Identitäts-Parameter
kind = pl.get("kind", "edge")
s = pl.get("source_id", "unknown-src")
t = pl.get("target_id", "unknown-tgt")
scope = pl.get("scope", "note")
# Stabiler Schlüssel für UUIDv5
edge_id = f"edge:{kind}:{s}:{t}:{scope}"
pl["edge_id"] = edge_id
# Optionale Differenzierung (falls von graph_derive_edges gesetzt)
rule_id = pl.get("rule_id")
variant = pl.get("variant")
point_id = _to_uuid(edge_id)
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
try:
# Aufruf der Single-Source-of-Truth für IDs
point_id = _mk_edge_id(
kind=kind,
s=s,
t=t,
scope=scope,
rule_id=rule_id,
variant=variant
)
# Synchronisierung des Payloads mit der berechneten ID
pl["edge_id"] = point_id
points.append(rest.PointStruct(
id=point_id,
vector=[0.0],
payload=pl
))
except ValueError as e:
# Fehlerhaft definierte Kanten werden übersprungen, um Pydantic-Crashes zu vermeiden
continue
return edges_col, points
# --------------------- Vector schema & overrides ---------------------
def _preferred_name(candidates: List[str]) -> str:
"""Ermittelt den primären Vektor-Namen aus einer Liste von Kandidaten."""
for k in ("text", "default", "embedding", "content"):
if k in candidates:
return k
@@ -107,10 +150,11 @@ def _preferred_name(candidates: List[str]) -> str:
def _env_override_for_collection(collection: str) -> Optional[str]:
"""
Prüft auf Umgebungsvariablen-Overrides für Vektor-Namen.
Returns:
- "__single__" to force single-vector
- concrete name (str) to force named-vector with that name
- None to auto-detect
- "__single__" für erzwungenen Single-Vector Modus
- Name (str) für spezifischen Named-Vector
- None für automatische Erkennung
"""
base = os.getenv("MINDNET_VECTOR_NAME")
if collection.endswith("_notes"):
@@ -125,19 +169,17 @@ def _env_override_for_collection(collection: str) -> Optional[str]:
val = base.strip()
if val.lower() in ("__single__", "single"):
return "__single__"
return val # concrete name
return val
def _get_vector_schema(client: QdrantClient, collection_name: str) -> dict:
"""
Return {"kind": "single", "size": int} or {"kind": "named", "names": [...], "primary": str}.
"""
"""Ermittelt das Vektor-Schema einer existierenden Collection via API."""
try:
info = client.get_collection(collection_name=collection_name)
vecs = getattr(info, "vectors", None)
# Single-vector config
# Prüfung auf Single-Vector Konfiguration
if hasattr(vecs, "size") and isinstance(vecs.size, int):
return {"kind": "single", "size": vecs.size}
# Named-vectors config (dict-like in .config)
# Prüfung auf Named-Vectors Konfiguration
cfg = getattr(vecs, "config", None)
if isinstance(cfg, dict) and cfg:
names = list(cfg.keys())
@@ -148,6 +190,7 @@ def _get_vector_schema(client: QdrantClient, collection_name: str) -> dict:
return {"kind": "single", "size": None}
def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruct]:
"""Transformiert PointStructs in das Named-Vector Format."""
out: List[rest.PointStruct] = []
for pt in points:
vec = getattr(pt, "vector", None)
@@ -155,7 +198,6 @@ def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruc
if name in vec:
out.append(pt)
else:
# take any existing entry; if empty dict fallback to [0.0]
fallback_vec = None
try:
fallback_vec = list(next(iter(vec.values())))
@@ -172,13 +214,14 @@ def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruc
def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointStruct], wait: bool = True) -> None:
"""
Schreibt Points in eine Collection.
WP-Fix: Unterstützt den 'wait' Parameter (Default True für Kompatibilität zu v1.5.1).
Schreibt Points hocheffizient in eine Collection.
Unterstützt automatische Schema-Erkennung und Named-Vector Transformation.
WP-Fix: 'wait=True' ist Default für Datenkonsistenz zwischen den Ingest-Phasen.
"""
if not points:
return
# 1) ENV overrides come first
# 1) ENV overrides prüfen
override = _env_override_for_collection(collection)
if override == "__single__":
client.upsert(collection_name=collection, points=points, wait=wait)
@@ -187,22 +230,24 @@ def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointS
client.upsert(collection_name=collection, points=_as_named(points, override), wait=wait)
return
# 2) Auto-detect schema
# 2) Automatische Schema-Erkennung (Live-Check)
schema = _get_vector_schema(client, collection)
if schema.get("kind") == "named":
name = schema.get("primary") or _preferred_name(schema.get("names") or [])
client.upsert(collection_name=collection, points=_as_named(points, name), wait=wait)
return
# 3) Fallback single-vector
# 3) Fallback: Single-Vector Upsert
client.upsert(collection_name=collection, points=points, wait=wait)
# --- Optional search helpers ---
def _filter_any(field: str, values: Iterable[str]) -> rest.Filter:
"""Hilfsfunktion für händische Filter-Konstruktion (Logical OR)."""
return rest.Filter(should=[rest.FieldCondition(key=field, match=rest.MatchValue(value=v)) for v in values])
def _merge_filters(*filters: Optional[rest.Filter]) -> Optional[rest.Filter]:
"""Führt mehrere Filter-Objekte zu einem konsolidierten Filter zusammen."""
fs = [f for f in filters if f is not None]
if not fs:
return None
@@ -217,6 +262,7 @@ def _merge_filters(*filters: Optional[rest.Filter]) -> Optional[rest.Filter]:
return rest.Filter(must=must)
def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter]:
"""Konvertiert ein Python-Dict in ein Qdrant-Filter Objekt."""
if not filters:
return None
parts = []
@@ -228,9 +274,17 @@ def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter
return _merge_filters(*parts)
def search_chunks_by_vector(client: QdrantClient, prefix: str, vector: List[float], top: int = 10, filters: Optional[Dict[str, Any]] = None) -> List[Tuple[str, float, dict]]:
"""Sucht semantisch ähnliche Chunks in der Vektordatenbank."""
_, chunks_col, _ = _names(prefix)
flt = _filter_from_dict(filters)
res = client.search(collection_name=chunks_col, query_vector=vector, limit=top, with_payload=True, with_vectors=False, query_filter=flt)
res = client.search(
collection_name=chunks_col,
query_vector=vector,
limit=top,
with_payload=True,
with_vectors=False,
query_filter=flt
)
out: List[Tuple[str, float, dict]] = []
for r in res:
out.append((str(r.id), float(r.score), dict(r.payload or {})))
@@ -246,18 +300,18 @@ def get_edges_for_sources(
edge_types: Optional[Iterable[str]] = None,
limit: int = 2048,
) -> List[Dict[str, Any]]:
"""Retrieve edge payloads from the <prefix>_edges collection."""
"""Ruft alle Kanten ab, die von einer Menge von Quell-Notizen ausgehen."""
source_ids = list(source_ids)
if not source_ids or limit <= 0:
return []
# Resolve collection name
# Namen der Edges-Collection auflösen
_, _, edges_col = _names(prefix)
# Build filter: source_id IN source_ids
# Filter-Bau: source_id IN source_ids
src_filter = _filter_any("source_id", [str(s) for s in source_ids])
# Optional: kind IN edge_types
# Optionaler Filter auf den Kanten-Typ
kind_filter = None
if edge_types:
kind_filter = _filter_any("kind", [str(k) for k in edge_types])
@@ -268,7 +322,7 @@ def get_edges_for_sources(
next_page = None
remaining = int(limit)
# Use paginated scroll API
# Paginated Scroll API (NUR Payload, keine Vektoren)
while remaining > 0:
batch_limit = min(256, remaining)
res, next_page = client.scroll(