Update qdrant_points.py, graph_utils.py, ingestion_processor.py, and import_markdown.py: Enhance ID generation and error handling, centralize identity logic to prevent ID drift, and improve documentation clarity. Update versioning to reflect changes in functionality and maintain compatibility across modules.
This commit is contained in:
1 parent
c33b1c644a
commit
b0f4309a29
4 files changed
+235
-155
No files matched your search
@@ -1,9 +1,10 @@
|
||||
"""
|
||||
FILE: app/core/database/qdrant_points.py
|
||||
DESCRIPTION: Object-Mapper für Qdrant. Konvertiert JSON-Payloads (Notes, Chunks, Edges) in PointStructs und generiert deterministische UUIDs.
|
||||
VERSION: 1.5.2 (WP-Fix: Atomic Consistency & Canonical Edge IDs)
|
||||
DESCRIPTION: Object-Mapper für Qdrant. Konvertiert JSON-Payloads (Notes, Chunks, Edges)
|
||||
in PointStructs und generiert deterministische UUIDs.
|
||||
VERSION: 1.5.3 (WP-Fix: Centralized Identity Enforcement)
|
||||
STATUS: Active
|
||||
DEPENDENCIES: qdrant_client, uuid, os
|
||||
DEPENDENCIES: qdrant_client, uuid, os, app.core.graph.graph_utils
|
||||
LAST_ANALYSIS: 2026-01-10
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -14,6 +15,9 @@ from typing import List, Tuple, Iterable, Optional, Dict, Any
|
||||
from qdrant_client.http import models as rest
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
# WP-24c: Import der zentralen Identitäts-Logik zur Vermeidung von ID-Drift
|
||||
from app.core.graph.graph_utils import _mk_edge_id
|
||||
|
||||
# --------------------- ID helpers ---------------------
|
||||
|
||||
def _to_uuid(stable_key: str) -> str:
|
||||
@@ -26,19 +30,29 @@ def _to_uuid(stable_key: str) -> str:
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, str(stable_key)))
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
"""Interne Auflösung der Collection-Namen basierend auf dem Präfix."""
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
# --------------------- Points builders ---------------------
|
||||
|
||||
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""Konvertiert Note-Metadaten in Qdrant Points."""
|
||||
notes_col, _, _ = _names(prefix)
|
||||
# Nutzt Null-Vektor als Fallback, falls kein Embedding vorhanden ist
|
||||
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
||||
|
||||
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
||||
point_id = _to_uuid(raw_note_id)
|
||||
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
||||
|
||||
pt = rest.PointStruct(
|
||||
id=point_id,
|
||||
vector=vector,
|
||||
payload=note_payload
|
||||
)
|
||||
return notes_col, [pt]
|
||||
|
||||
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""Konvertiert Chunks und deren Vektoren in Qdrant Points."""
|
||||
_, chunks_col, _ = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
||||
@@ -47,8 +61,13 @@ def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[Lis
|
||||
note_id = pl.get("note_id") or pl.get("parent_note_id") or "missing-note"
|
||||
chunk_id = f"{note_id}#{i}"
|
||||
pl["chunk_id"] = chunk_id
|
||||
|
||||
point_id = _to_uuid(chunk_id)
|
||||
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
||||
points.append(rest.PointStruct(
|
||||
id=point_id,
|
||||
vector=vec,
|
||||
payload=pl
|
||||
))
|
||||
return chunks_col, points
|
||||
|
||||
def _normalize_edge_payload(pl: dict) -> dict:
|
||||
@@ -76,30 +95,54 @@ def _normalize_edge_payload(pl: dict) -> dict:
|
||||
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Konvertiert Kanten-Payloads in PointStructs.
|
||||
WP-24c: Nutzt strikte ID-Kanonisierung für die Symmetrie-Integrität.
|
||||
WP-24c Audit v1.5.3: Nutzt die zentrale _mk_edge_id Funktion aus graph_utils.
|
||||
Dies eliminiert den ID-Drift zwischen manuellen und virtuellen Kanten.
|
||||
"""
|
||||
_, _, edges_col = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
|
||||
for raw in edge_payloads:
|
||||
pl = _normalize_edge_payload(raw)
|
||||
|
||||
# WP-24c: Deterministische ID-Generierung zur Kollisionsvermeidung
|
||||
# Extraktion der Identitäts-Parameter
|
||||
kind = pl.get("kind", "edge")
|
||||
s = pl.get("source_id", "unknown-src")
|
||||
t = pl.get("target_id", "unknown-tgt")
|
||||
scope = pl.get("scope", "note")
|
||||
|
||||
# Stabiler Schlüssel für UUIDv5
|
||||
edge_id = f"edge:{kind}:{s}:{t}:{scope}"
|
||||
pl["edge_id"] = edge_id
|
||||
# Optionale Differenzierung (falls von graph_derive_edges gesetzt)
|
||||
rule_id = pl.get("rule_id")
|
||||
variant = pl.get("variant")
|
||||
|
||||
point_id = _to_uuid(edge_id)
|
||||
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
||||
try:
|
||||
# Aufruf der Single-Source-of-Truth für IDs
|
||||
point_id = _mk_edge_id(
|
||||
kind=kind,
|
||||
s=s,
|
||||
t=t,
|
||||
scope=scope,
|
||||
rule_id=rule_id,
|
||||
variant=variant
|
||||
)
|
||||
|
||||
# Synchronisierung des Payloads mit der berechneten ID
|
||||
pl["edge_id"] = point_id
|
||||
|
||||
points.append(rest.PointStruct(
|
||||
id=point_id,
|
||||
vector=[0.0],
|
||||
payload=pl
|
||||
))
|
||||
except ValueError as e:
|
||||
# Fehlerhaft definierte Kanten werden übersprungen, um Pydantic-Crashes zu vermeiden
|
||||
continue
|
||||
|
||||
return edges_col, points
|
||||
|
||||
# --------------------- Vector schema & overrides ---------------------
|
||||
|
||||
def _preferred_name(candidates: List[str]) -> str:
|
||||
"""Ermittelt den primären Vektor-Namen aus einer Liste von Kandidaten."""
|
||||
for k in ("text", "default", "embedding", "content"):
|
||||
if k in candidates:
|
||||
return k
|
||||
@@ -107,10 +150,11 @@ def _preferred_name(candidates: List[str]) -> str:
|
||||
|
||||
def _env_override_for_collection(collection: str) -> Optional[str]:
|
||||
"""
|
||||
Prüft auf Umgebungsvariablen-Overrides für Vektor-Namen.
|
||||
Returns:
|
||||
- "__single__" to force single-vector
|
||||
- concrete name (str) to force named-vector with that name
|
||||
- None to auto-detect
|
||||
- "__single__" für erzwungenen Single-Vector Modus
|
||||
- Name (str) für spezifischen Named-Vector
|
||||
- None für automatische Erkennung
|
||||
"""
|
||||
base = os.getenv("MINDNET_VECTOR_NAME")
|
||||
if collection.endswith("_notes"):
|
||||
@@ -125,19 +169,17 @@ def _env_override_for_collection(collection: str) -> Optional[str]:
|
||||
val = base.strip()
|
||||
if val.lower() in ("__single__", "single"):
|
||||
return "__single__"
|
||||
return val # concrete name
|
||||
return val
|
||||
|
||||
def _get_vector_schema(client: QdrantClient, collection_name: str) -> dict:
|
||||
"""
|
||||
Return {"kind": "single", "size": int} or {"kind": "named", "names": [...], "primary": str}.
|
||||
"""
|
||||
"""Ermittelt das Vektor-Schema einer existierenden Collection via API."""
|
||||
try:
|
||||
info = client.get_collection(collection_name=collection_name)
|
||||
vecs = getattr(info, "vectors", None)
|
||||
# Single-vector config
|
||||
# Prüfung auf Single-Vector Konfiguration
|
||||
if hasattr(vecs, "size") and isinstance(vecs.size, int):
|
||||
return {"kind": "single", "size": vecs.size}
|
||||
# Named-vectors config (dict-like in .config)
|
||||
# Prüfung auf Named-Vectors Konfiguration
|
||||
cfg = getattr(vecs, "config", None)
|
||||
if isinstance(cfg, dict) and cfg:
|
||||
names = list(cfg.keys())
|
||||
@@ -148,6 +190,7 @@ def _get_vector_schema(client: QdrantClient, collection_name: str) -> dict:
|
||||
return {"kind": "single", "size": None}
|
||||
|
||||
def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruct]:
|
||||
"""Transformiert PointStructs in das Named-Vector Format."""
|
||||
out: List[rest.PointStruct] = []
|
||||
for pt in points:
|
||||
vec = getattr(pt, "vector", None)
|
||||
@@ -155,7 +198,6 @@ def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruc
|
||||
if name in vec:
|
||||
out.append(pt)
|
||||
else:
|
||||
# take any existing entry; if empty dict fallback to [0.0]
|
||||
fallback_vec = None
|
||||
try:
|
||||
fallback_vec = list(next(iter(vec.values())))
|
||||
@@ -172,13 +214,14 @@ def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruc
|
||||
|
||||
def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointStruct], wait: bool = True) -> None:
|
||||
"""
|
||||
Schreibt Points in eine Collection.
|
||||
WP-Fix: Unterstützt den 'wait' Parameter (Default True für Kompatibilität zu v1.5.1).
|
||||
Schreibt Points hocheffizient in eine Collection.
|
||||
Unterstützt automatische Schema-Erkennung und Named-Vector Transformation.
|
||||
WP-Fix: 'wait=True' ist Default für Datenkonsistenz zwischen den Ingest-Phasen.
|
||||
"""
|
||||
if not points:
|
||||
return
|
||||
|
||||
# 1) ENV overrides come first
|
||||
# 1) ENV overrides prüfen
|
||||
override = _env_override_for_collection(collection)
|
||||
if override == "__single__":
|
||||
client.upsert(collection_name=collection, points=points, wait=wait)
|
||||
@@ -187,22 +230,24 @@ def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointS
|
||||
client.upsert(collection_name=collection, points=_as_named(points, override), wait=wait)
|
||||
return
|
||||
|
||||
# 2) Auto-detect schema
|
||||
# 2) Automatische Schema-Erkennung (Live-Check)
|
||||
schema = _get_vector_schema(client, collection)
|
||||
if schema.get("kind") == "named":
|
||||
name = schema.get("primary") or _preferred_name(schema.get("names") or [])
|
||||
client.upsert(collection_name=collection, points=_as_named(points, name), wait=wait)
|
||||
return
|
||||
|
||||
# 3) Fallback single-vector
|
||||
# 3) Fallback: Single-Vector Upsert
|
||||
client.upsert(collection_name=collection, points=points, wait=wait)
|
||||
|
||||
# --- Optional search helpers ---
|
||||
|
||||
def _filter_any(field: str, values: Iterable[str]) -> rest.Filter:
|
||||
"""Hilfsfunktion für händische Filter-Konstruktion (Logical OR)."""
|
||||
return rest.Filter(should=[rest.FieldCondition(key=field, match=rest.MatchValue(value=v)) for v in values])
|
||||
|
||||
def _merge_filters(*filters: Optional[rest.Filter]) -> Optional[rest.Filter]:
|
||||
"""Führt mehrere Filter-Objekte zu einem konsolidierten Filter zusammen."""
|
||||
fs = [f for f in filters if f is not None]
|
||||
if not fs:
|
||||
return None
|
||||
@@ -217,6 +262,7 @@ def _merge_filters(*filters: Optional[rest.Filter]) -> Optional[rest.Filter]:
|
||||
return rest.Filter(must=must)
|
||||
|
||||
def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter]:
|
||||
"""Konvertiert ein Python-Dict in ein Qdrant-Filter Objekt."""
|
||||
if not filters:
|
||||
return None
|
||||
parts = []
|
||||
@@ -228,9 +274,17 @@ def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter
|
||||
return _merge_filters(*parts)
|
||||
|
||||
def search_chunks_by_vector(client: QdrantClient, prefix: str, vector: List[float], top: int = 10, filters: Optional[Dict[str, Any]] = None) -> List[Tuple[str, float, dict]]:
|
||||
"""Sucht semantisch ähnliche Chunks in der Vektordatenbank."""
|
||||
_, chunks_col, _ = _names(prefix)
|
||||
flt = _filter_from_dict(filters)
|
||||
res = client.search(collection_name=chunks_col, query_vector=vector, limit=top, with_payload=True, with_vectors=False, query_filter=flt)
|
||||
res = client.search(
|
||||
collection_name=chunks_col,
|
||||
query_vector=vector,
|
||||
limit=top,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
query_filter=flt
|
||||
)
|
||||
out: List[Tuple[str, float, dict]] = []
|
||||
for r in res:
|
||||
out.append((str(r.id), float(r.score), dict(r.payload or {})))
|
||||
@@ -246,18 +300,18 @@ def get_edges_for_sources(
|
||||
edge_types: Optional[Iterable[str]] = None,
|
||||
limit: int = 2048,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Retrieve edge payloads from the <prefix>_edges collection."""
|
||||
"""Ruft alle Kanten ab, die von einer Menge von Quell-Notizen ausgehen."""
|
||||
source_ids = list(source_ids)
|
||||
if not source_ids or limit <= 0:
|
||||
return []
|
||||
|
||||
# Resolve collection name
|
||||
# Namen der Edges-Collection auflösen
|
||||
_, _, edges_col = _names(prefix)
|
||||
|
||||
# Build filter: source_id IN source_ids
|
||||
# Filter-Bau: source_id IN source_ids
|
||||
src_filter = _filter_any("source_id", [str(s) for s in source_ids])
|
||||
|
||||
# Optional: kind IN edge_types
|
||||
# Optionaler Filter auf den Kanten-Typ
|
||||
kind_filter = None
|
||||
if edge_types:
|
||||
kind_filter = _filter_any("kind", [str(k) for k in edge_types])
|
||||
@@ -268,7 +322,7 @@ def get_edges_for_sources(
|
||||
next_page = None
|
||||
remaining = int(limit)
|
||||
|
||||
# Use paginated scroll API
|
||||
# Paginated Scroll API (NUR Payload, keine Vektoren)
|
||||
while remaining > 0:
|
||||
batch_limit = min(256, remaining)
|
||||
res, next_page = client.scroll(
|
||||
|
||||
Reference in new issue
Block a user