Refactor provenance handling in EdgeDTO and graph utilities

- Updated provenance priorities and introduced a mapping from internal provenance values to EdgeDTO-compliant literals.
- Added a new function `normalize_provenance` to standardize internal provenance strings.
- Enhanced the `_edge` function to include an `is_internal` flag and provenance normalization.
- Modified the `EdgeDTO` model to include a new `source_hint` field for detailed provenance information and an `is_internal` flag for intra-note edges.
- Reduced the provenance options in `EdgeDTO` to valid literals, improving data integrity.
This commit is contained in:
Lars committed 2026-01-25 16:27:09 +01:00
1 parent 0d61a9e191
commit cc258008dc
9 files changed
+2337 -52

No files matched your search

+90 -9
View File
@@ -12,28 +12,85 @@ STATUS: Active
import os
import uuid
import hashlib
from typing import Iterable, List, Optional, Set, Any, Tuple
from typing import Dict, Iterable, List, Optional, Set, Any, Tuple
try:
import yaml
except ImportError:
yaml = None
# WP-15b: Prioritäten-Ranking für die De-Duplizierung von Kanten unterschiedlicher Herkunft
# WP-26 v1.0: Provenance-Literale auf valide EdgeDTO-Werte reduziert
# Legacy-Prioritäten für interne Verarbeitung (werden zu source_hint gemappt)
PROVENANCE_PRIORITY = {
# Explizite Kanten (provenance: "explicit")
"explicit:wikilink": 1.00,
"inline:rel": 0.95,
"callout:edge": 0.90,
"explicit:callout": 0.90, # WP-24c v4.2.7: Callout-Kanten aus candidate_pool
"semantic_ai": 0.90, # Validierte KI-Kanten
"structure:belongs_to": 1.00,
"structure:order": 0.95, # next/prev
"explicit:callout": 0.90,
"explicit:note_scope": 1.00,
"explicit:note_zone": 1.00, # WP-24c v4.2.0: Note-Scope Zonen (höchste Priorität)
"explicit:note_zone": 1.00,
# Regel-basierte Kanten (provenance: "rule")
"derived:backlink": 0.90,
"edge_defaults": 0.70 # Heuristik basierend auf types.yaml
"edge_defaults": 0.70,
"schema_default": 0.85,
# Struktur-Kanten (provenance: "structure")
"structure:belongs_to": 1.00,
"structure:order": 0.95,
# KI-generierte Kanten (provenance: "smart")
"semantic_ai": 0.90,
"global_pool": 0.80,
}
# WP-26 v1.0: Mapping von internen Provenance-Werten zu EdgeDTO-konformen Literalen
PROVENANCE_TO_DTO = {
# explicit
"explicit:wikilink": ("explicit", "wikilink"),
"explicit:callout": ("explicit", "callout"),
"explicit:note_scope": ("explicit", "note_scope"),
"explicit:note_zone": ("explicit", "note_zone"),
"inline:rel": ("explicit", "inline_rel"),
"callout:edge": ("explicit", "callout"),
"explicit": ("explicit", None),
# rule
"derived:backlink": ("rule", "backlink"),
"edge_defaults": ("rule", "edge_defaults"),
"schema_default": ("rule", "schema_default"),
"inferred:schema": ("rule", "schema_default"),
"rule": ("rule", None),
# structure
"structure:belongs_to": ("structure", "belongs_to"),
"structure:order": ("structure", "order"),
"structure": ("structure", None),
# smart
"semantic_ai": ("smart", None),
"global_pool": ("smart", "global_pool"),
"smart": ("smart", None),
}
def normalize_provenance(internal_provenance: str) -> Tuple[str, Optional[str]]:
"""
WP-26 v1.0: Normalisiert interne Provenance-Werte zu EdgeDTO-konformen Literalen.
Args:
internal_provenance: Interner Provenance-String (z.B. "explicit:callout")
Returns:
Tuple (provenance, source_hint) mit validen EdgeDTO-Werten
"""
if internal_provenance in PROVENANCE_TO_DTO:
return PROVENANCE_TO_DTO[internal_provenance]
# Fallback: Versuche Präfix-Matching
if internal_provenance.startswith("explicit"):
return ("explicit", internal_provenance.split(":")[-1] if ":" in internal_provenance else None)
if internal_provenance.startswith("structure"):
return ("structure", internal_provenance.split(":")[-1] if ":" in internal_provenance else None)
if internal_provenance.startswith("rule") or internal_provenance.startswith("derived"):
return ("rule", internal_provenance.split(":")[-1] if ":" in internal_provenance else None)
# Default: explicit ohne source_hint
return ("explicit", None)
# ---------------------------------------------------------------------------
# Pfad-Auflösung (Integration der .env Umgebungsvariablen)
# ---------------------------------------------------------------------------
@@ -123,7 +180,15 @@ def _mk_edge_id(kind: str, s: str, t: str, scope: str, target_section: Optional[
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
"""
Konstruiert ein standardisiertes Kanten-Payload für Qdrant.
Wird von graph_derive_edges.py benötigt.
WP-26 v1.0: Erweitert um is_internal Flag und Provenance-Normalisierung.
Args:
kind: Kantentyp (z.B. "derives", "caused_by")
scope: Granularität ("chunk" oder "note")
source_id: ID der Quelle (Chunk oder Note)
target_id: ID des Ziels (Chunk oder Note)
note_id: ID der Note (für Kontext)
extra: Zusätzliche Payload-Felder
"""
pl = {
"kind": kind,
@@ -134,8 +199,24 @@ def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, e
"note_id": note_id,
"virtual": False # Standardmäßig explizit, solange nicht anders in Phase 2 gesetzt
}
# WP-26 v1.0: is_internal Flag berechnen
# Intra-Note-Edge: Source und Target gehören zur gleichen Note
source_note = source_id.split("#")[0] if "#" in source_id else source_id
target_note = target_id.split("#")[0] if "#" in target_id else target_id
pl["is_internal"] = (source_note == target_note) or (source_note == note_id and target_note == note_id)
if extra:
pl.update(extra)
# WP-26 v1.0: Provenance normalisieren, falls vorhanden
if "provenance" in extra:
internal_prov = extra["provenance"]
dto_prov, source_hint = normalize_provenance(internal_prov)
pl["provenance"] = dto_prov
if source_hint:
pl["source_hint"] = source_hint
return pl
# ---------------------------------------------------------------------------