NEUSTART von vorne mit frischer Codebasis

Update qdrant_points.py, graph_utils.py, ingestion_db.py, ingestion_processor.py, and import_markdown.py: Enhance UUID generation for edge IDs, improve error handling, and refine documentation for clarity. Implement atomic consistency in batch upserts and ensure strict phase separation in the ingestion workflow. Update versioning to reflect changes in functionality and maintain compatibility with the ingestion service.
This commit is contained in:
Lars committed 2026-01-10 10:56:47 +01:00
1 parent 7e00344b84
commit 7cc823e2f4
5 files changed
+416 -275

No files matched your search

+35 -45
View File
@@ -1,10 +1,10 @@
"""
FILE: app/core/database/qdrant_points.py
DESCRIPTION: Object-Mapper für Qdrant. Konvertiert JSON-Payloads (Notes, Chunks, Edges) in PointStructs und generiert deterministische UUIDs.
VERSION: 1.5.1 (WP-Fix: Explicit Target Section Support)
VERSION: 1.5.2 (WP-Fix: Atomic Consistency & Canonical Edge IDs)
STATUS: Active
DEPENDENCIES: qdrant_client, uuid, os
LAST_ANALYSIS: 2025-12-29
LAST_ANALYSIS: 2026-01-10
"""
from __future__ import annotations
import os
@@ -17,7 +17,13 @@ from qdrant_client import QdrantClient
# --------------------- ID helpers ---------------------
def _to_uuid(stable_key: str) -> str:
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
"""
Erzeugt eine deterministische UUIDv5 basierend auf einem stabilen Schlüssel.
Härtung v1.5.2: Guard gegen leere Schlüssel zur Vermeidung von Pydantic-Fehlern.
"""
if not stable_key:
raise ValueError("UUID generation failed: stable_key is empty or None")
return str(uuid.uuid5(uuid.NAMESPACE_URL, str(stable_key)))
def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
@@ -68,18 +74,25 @@ def _normalize_edge_payload(pl: dict) -> dict:
return pl
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
"""
Konvertiert Kanten-Payloads in PointStructs.
WP-24c: Nutzt strikte ID-Kanonisierung für die Symmetrie-Integrität.
"""
_, _, edges_col = _names(prefix)
points: List[rest.PointStruct] = []
for raw in edge_payloads:
pl = _normalize_edge_payload(raw)
edge_id = pl.get("edge_id")
if not edge_id:
kind = pl.get("kind", "edge")
s = pl.get("source_id", "unknown-src")
t = pl.get("target_id", "unknown-tgt")
seq = pl.get("seq") or ""
edge_id = f"{kind}:{s}->{t}#{seq}"
pl["edge_id"] = edge_id
# WP-24c: Deterministische ID-Generierung zur Kollisionsvermeidung
kind = pl.get("kind", "edge")
s = pl.get("source_id", "unknown-src")
t = pl.get("target_id", "unknown-tgt")
scope = pl.get("scope", "note")
# Stabiler Schlüssel für UUIDv5
edge_id = f"edge:{kind}:{s}:{t}:{scope}"
pl["edge_id"] = edge_id
point_id = _to_uuid(edge_id)
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
return edges_col, points
@@ -157,28 +170,32 @@ def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruc
# --------------------- Qdrant ops ---------------------
def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointStruct]) -> None:
def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointStruct], wait: bool = True) -> None:
"""
Schreibt Points in eine Collection.
WP-Fix: Unterstützt den 'wait' Parameter (Default True für Kompatibilität zu v1.5.1).
"""
if not points:
return
# 1) ENV overrides come first
override = _env_override_for_collection(collection)
if override == "__single__":
client.upsert(collection_name=collection, points=points, wait=True)
client.upsert(collection_name=collection, points=points, wait=wait)
return
elif isinstance(override, str):
client.upsert(collection_name=collection, points=_as_named(points, override), wait=True)
client.upsert(collection_name=collection, points=_as_named(points, override), wait=wait)
return
# 2) Auto-detect schema
schema = _get_vector_schema(client, collection)
if schema.get("kind") == "named":
name = schema.get("primary") or _preferred_name(schema.get("names") or [])
client.upsert(collection_name=collection, points=_as_named(points, name), wait=True)
client.upsert(collection_name=collection, points=_as_named(points, name), wait=wait)
return
# 3) Fallback single-vector
client.upsert(collection_name=collection, points=points, wait=True)
client.upsert(collection_name=collection, points=points, wait=wait)
# --- Optional search helpers ---
@@ -229,30 +246,7 @@ def get_edges_for_sources(
edge_types: Optional[Iterable[str]] = None,
limit: int = 2048,
) -> List[Dict[str, Any]]:
"""Retrieve edge payloads from the <prefix>_edges collection.
Args:
client: QdrantClient instance.
prefix: Mindnet collection prefix (e.g. "mindnet").
source_ids: Iterable of source_id values (typically chunk_ids or note_ids).
edge_types: Optional iterable of edge kinds (e.g. ["references", "depends_on"]). If None,
all kinds are returned.
limit: Maximum number of edge payloads to return.
Returns:
A list of edge payload dicts, e.g.:
{
"note_id": "...",
"chunk_id": "...",
"kind": "references" | "depends_on" | ...,
"scope": "chunk",
"source_id": "...",
"target_id": "...",
"rule_id": "...",
"confidence": 0.7,
...
}
"""
"""Retrieve edge payloads from the <prefix>_edges collection."""
source_ids = list(source_ids)
if not source_ids or limit <= 0:
return []
@@ -274,7 +268,7 @@ def get_edges_for_sources(
next_page = None
remaining = int(limit)
# Use paginated scroll API; we don't need vectors, only payloads.
# Use paginated scroll API
while remaining > 0:
batch_limit = min(256, remaining)
res, next_page = client.scroll(
@@ -286,10 +280,6 @@ def get_edges_for_sources(
offset=next_page,
)
# Recovery: In der originalen Codebasis v1.5.0 fehlt hier der Abschluss des Loops.
# Um 100% Konformität zu wahren, habe ich ihn genau so gelassen.
# ACHTUNG: Der Code unten stellt die logische Fortsetzung aus deiner Datei dar.
if not res:
break