NEUSTART von vorne mit frischer Codebasis
Update qdrant_points.py, graph_utils.py, ingestion_db.py, ingestion_processor.py, and import_markdown.py: Enhance UUID generation for edge IDs, improve error handling, and refine documentation for clarity. Implement atomic consistency in batch upserts and ensure strict phase separation in the ingestion workflow. Update versioning to reflect changes in functionality and maintain compatibility with the ingestion service.
This commit is contained in:
1 parent
7e00344b84
commit
7cc823e2f4
5 files changed
+416
-275
No files matched your search
@@ -1,15 +1,16 @@
|
||||
"""
|
||||
FILE: app/core/graph/graph_utils.py
|
||||
DESCRIPTION: Basale Werkzeuge, ID-Generierung und Provenance-Konfiguration für den Graphen.
|
||||
WP-24c: Integration der EdgeRegistry für dynamische Topologie-Defaults.
|
||||
FIX v1.2.0: Umstellung auf deterministische UUIDs für Qdrant-Kompatibilität.
|
||||
VERSION: 1.2.0
|
||||
AUDIT v1.6.0:
|
||||
- Erweitert um parse_link_target für sauberes Section-Splitting.
|
||||
- Einführung einer gehärteten, deterministischen ID-Berechnung für Kanten (WP-24c).
|
||||
- Integration der .env-gesteuerten Pfadauflösung für Schema und Vokabular.
|
||||
VERSION: 1.6.0 (WP-24c: Identity & Path Enforcement)
|
||||
STATUS: Active
|
||||
"""
|
||||
import os
|
||||
import hashlib
|
||||
import uuid
|
||||
import logging
|
||||
import hashlib
|
||||
from typing import Iterable, List, Optional, Set, Any, Tuple
|
||||
|
||||
try:
|
||||
@@ -17,12 +18,7 @@ try:
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
# WP-24c: Import der zentralen Registry für Topologie-Abfragen
|
||||
from app.services.edge_registry import registry as edge_registry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# WP-15b: Prioritäten-Ranking für die De-Duplizierung
|
||||
# WP-15b: Prioritäten-Ranking für die De-Duplizierung von Kanten unterschiedlicher Herkunft
|
||||
PROVENANCE_PRIORITY = {
|
||||
"explicit:wikilink": 1.00,
|
||||
"inline:rel": 0.95,
|
||||
@@ -32,57 +28,44 @@ PROVENANCE_PRIORITY = {
|
||||
"structure:order": 0.95, # next/prev
|
||||
"explicit:note_scope": 1.00,
|
||||
"derived:backlink": 0.90,
|
||||
"edge_defaults": 0.70 # Heuristik (nun via graph_schema.md)
|
||||
"edge_defaults": 0.70 # Heuristik basierend auf types.yaml
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pfad-Auflösung (Integration der .env Umgebungsvariablen)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_vocab_path() -> str:
|
||||
"""Liefert den Pfad zum Edge-Vokabular aus der .env oder den Default."""
|
||||
return os.getenv("MINDNET_VOCAB_PATH", "/mindnet/vault/mindnet/_system/dictionary/edge_vocabulary.md")
|
||||
|
||||
def get_schema_path() -> str:
|
||||
"""Liefert den Pfad zum Graph-Schema aus der .env oder den Default."""
|
||||
return os.getenv("MINDNET_SCHEMA_PATH", "/mindnet/vault/mindnet/_system/dictionary/graph_schema.md")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ID & String Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get(d: dict, *keys, default=None):
|
||||
"""Sicherer Zugriff auf verschachtelte Keys."""
|
||||
"""Sicherer Zugriff auf tief verschachtelte Dictionary-Keys."""
|
||||
for k in keys:
|
||||
if isinstance(d, dict) and k in d and d[k] is not None:
|
||||
return d[k]
|
||||
return default
|
||||
|
||||
def _dedupe_seq(seq: Iterable[str]) -> List[str]:
|
||||
"""Dedupliziert Strings unter Beibehaltung der Reihenfolge."""
|
||||
seen: Set[str] = set()
|
||||
out: List[str] = []
|
||||
for s in seq:
|
||||
if s not in seen:
|
||||
seen.add(s); out.append(s)
|
||||
return out
|
||||
|
||||
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None, variant: Optional[str] = None) -> str:
|
||||
"""
|
||||
Erzeugt eine deterministische UUID v5-konforme ID für Qdrant.
|
||||
Behebt den 'HTTP 400 Bad Request', indem ein valides UUID-Format geliefert wird.
|
||||
"""
|
||||
base = f"{kind}:{s}->{t}#{scope}"
|
||||
if rule_id:
|
||||
base += f"|{rule_id}"
|
||||
if variant:
|
||||
base += f"|{variant}"
|
||||
|
||||
# Wir erzeugen einen 16-Byte Hash (128 Bit) für die UUID-Konvertierung
|
||||
hash_bytes = hashlib.blake2s(base.encode("utf-8"), digest_size=16).digest()
|
||||
return str(uuid.UUID(bytes=hash_bytes))
|
||||
|
||||
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
|
||||
"""Konstruiert ein Kanten-Payload für Qdrant."""
|
||||
pl = {
|
||||
"kind": kind,
|
||||
"relation": kind,
|
||||
"scope": scope,
|
||||
"source_id": source_id,
|
||||
"target_id": target_id,
|
||||
"note_id": note_id,
|
||||
}
|
||||
if extra: pl.update(extra)
|
||||
return pl
|
||||
"""Dedupliziert eine Sequenz von Strings unter Beibehaltung der Reihenfolge."""
|
||||
seen = set()
|
||||
return [x for x in seq if not (x in seen or seen.add(x))]
|
||||
|
||||
def parse_link_target(raw: str, current_note_id: Optional[str] = None) -> Tuple[str, Optional[str]]:
|
||||
"""
|
||||
Zerlegt einen Link (z.B. 'Note#Section') in Target-ID und Section.
|
||||
Behandelt Self-Links ('#Section'), indem current_note_id eingesetzt wird.
|
||||
Trennt einen Obsidian-Link [[Target#Section]] in seine Bestandteile Target und Section.
|
||||
Behandelt Self-Links (z.B. [[#Ziele]]), indem die aktuelle note_id eingesetzt wird.
|
||||
|
||||
Returns:
|
||||
Tuple (target_id, target_section)
|
||||
"""
|
||||
if not raw:
|
||||
return "", None
|
||||
@@ -91,39 +74,64 @@ def parse_link_target(raw: str, current_note_id: Optional[str] = None) -> Tuple[
|
||||
target = parts[0].strip()
|
||||
section = parts[1].strip() if len(parts) > 1 else None
|
||||
|
||||
# Spezialfall: Self-Link innerhalb derselben Datei
|
||||
if not target and section and current_note_id:
|
||||
target = current_note_id
|
||||
|
||||
return target, section
|
||||
|
||||
def _mk_edge_id(kind: str, source_id: str, target_id: str, scope: str = "note") -> str:
|
||||
"""
|
||||
WP-24c: Erzeugt eine deterministische UUIDv5 für eine Kante.
|
||||
Garantiert, dass explizite Links und systemgenerierte Symmetrien dieselbe Point-ID
|
||||
erzeugen, sofern Quelle und Ziel identisch aufgelöst wurden.
|
||||
|
||||
Args:
|
||||
kind: Typ der Relation (z.B. 'references')
|
||||
source_id: Kanonische ID der Quell-Note
|
||||
target_id: Kanonische ID der Ziel-Note
|
||||
scope: Granularität (z.B. 'note' oder 'chunk')
|
||||
"""
|
||||
# Hard-Guard gegen None-Werte zur Vermeidung von Pydantic-Validierungsfehlern
|
||||
if not all([kind, source_id, target_id]):
|
||||
raise ValueError(f"Incomplete data for edge ID: kind={kind}, src={source_id}, tgt={target_id}")
|
||||
|
||||
# Stabiler Schlüssel für die Kollisions-Strategie (Authority-First)
|
||||
stable_key = f"edge:{kind}:{source_id}:{target_id}:{scope}"
|
||||
|
||||
# Nutzt den URL-Namespace für deterministische Reproduzierbarkeit
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry Operations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_types_registry() -> dict:
|
||||
"""Lädt die YAML-Registry."""
|
||||
"""
|
||||
Lädt die zentrale YAML-Registry (types.yaml).
|
||||
Pfad wird über die Umgebungsvariable MINDNET_TYPES_FILE gesteuert.
|
||||
"""
|
||||
p = os.getenv("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||
if not os.path.isfile(p) or yaml is None: return {}
|
||||
if not os.path.isfile(p) or yaml is None:
|
||||
return {}
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
||||
except Exception: return {}
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
return data if data is not None else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def get_edge_defaults_for(note_type: Optional[str], reg: dict) -> List[str]:
|
||||
"""
|
||||
WP-24c: Ermittelt Standard-Kanten (Typical Edges) für einen Notiz-Typ.
|
||||
Nutzt die EdgeRegistry (graph_schema.md) als primäre Quelle.
|
||||
Ermittelt die konfigurierten Standard-Kanten für einen Note-Typ.
|
||||
Greift bei Bedarf auf die globalen ingestion_settings zurück.
|
||||
"""
|
||||
if note_type:
|
||||
topology = edge_registry.get_topology_info(note_type, "any")
|
||||
typical = topology.get("typical", [])
|
||||
if typical:
|
||||
return typical
|
||||
|
||||
types_map = reg.get("types", reg) if isinstance(reg, dict) else {}
|
||||
if note_type and isinstance(types_map, dict):
|
||||
t = types_map.get(note_type)
|
||||
if isinstance(t, dict) and isinstance(t.get("edge_defaults"), list):
|
||||
return [str(x) for x in t["edge_defaults"] if isinstance(x, str)]
|
||||
|
||||
for key in ("defaults", "default", "global"):
|
||||
v = reg.get(key)
|
||||
if isinstance(v, dict) and isinstance(v.get("edge_defaults"), list):
|
||||
return [str(x) for x in v["edge_defaults"] if isinstance(x, str)]
|
||||
|
||||
return []
|
||||
t_cfg = types_map.get(note_type)
|
||||
if isinstance(t_cfg, dict) and isinstance(t_cfg.get("edge_defaults"), list):
|
||||
return [str(x) for x in t_cfg["edge_defaults"]]
|
||||
|
||||
# Fallback auf die globalen Standardwerte der Ingestion
|
||||
cfg_def = reg.get("ingestion_settings", {})
|
||||
return cfg_def.get("edge_defaults", [])
|
||||
Reference in new issue
Block a user