Update type_registry, graph_utils, ingestion_note_payload, and discovery services for dynamic edge handling: Integrate EdgeRegistry for improved edge defaults and topology management (WP-24c). Enhance type loading and edge resolution logic to ensure backward compatibility while transitioning to a more robust architecture. Version bumps to 1.1.0 for type_registry, 1.1.0 for graph_utils, 2.5.0 for ingestion_note_payload, and 1.1.0 for discovery service.
This commit is contained in:
1 parent
5e2a074019
commit
a392dc2786
6 files changed
+416
-207
No files matched your search
@@ -1,10 +1,14 @@
|
||||
"""
|
||||
FILE: app/core/graph/graph_utils.py
|
||||
DESCRIPTION: Basale Werkzeuge, ID-Generierung und Provenance-Konfiguration für den Graphen.
|
||||
AUDIT: Erweitert um parse_link_target für sauberes Section-Splitting (WP-Fix).
|
||||
WP-24c: Integration der EdgeRegistry für dynamische Topologie-Defaults.
|
||||
AUDIT: Erweitert um parse_link_target für sauberes Section-Splitting.
|
||||
VERSION: 1.1.0 (WP-24c: Dynamic Topology Implementation)
|
||||
STATUS: Active
|
||||
"""
|
||||
import os
|
||||
import hashlib
|
||||
import logging
|
||||
from typing import Iterable, List, Optional, Set, Any, Tuple
|
||||
|
||||
try:
|
||||
@@ -12,6 +16,11 @@ try:
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
# WP-24c: Import der zentralen Registry für Topologie-Abfragen
|
||||
from app.services.edge_registry import registry as edge_registry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# WP-15b: Prioritäten-Ranking für die De-Duplizierung
|
||||
PROVENANCE_PRIORITY = {
|
||||
"explicit:wikilink": 1.00,
|
||||
@@ -22,7 +31,7 @@ PROVENANCE_PRIORITY = {
|
||||
"structure:order": 0.95, # next/prev
|
||||
"explicit:note_scope": 1.00,
|
||||
"derived:backlink": 0.90,
|
||||
"edge_defaults": 0.70 # Heuristik (types.yaml)
|
||||
"edge_defaults": 0.70 # Heuristik (nun via graph_schema.md)
|
||||
}
|
||||
|
||||
def _get(d: dict, *keys, default=None):
|
||||
@@ -52,7 +61,7 @@ def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] =
|
||||
if rule_id:
|
||||
base += f"|{rule_id}"
|
||||
if variant:
|
||||
base += f"|{variant}" # <--- Hier entsteht die Eindeutigkeit für verschiedene Sections
|
||||
base += f"|{variant}"
|
||||
|
||||
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||
|
||||
@@ -73,9 +82,6 @@ def parse_link_target(raw: str, current_note_id: Optional[str] = None) -> Tuple[
|
||||
"""
|
||||
Zerlegt einen Link (z.B. 'Note#Section') in Target-ID und Section.
|
||||
Behandelt Self-Links ('#Section'), indem current_note_id eingesetzt wird.
|
||||
|
||||
Returns:
|
||||
(target_id, target_section)
|
||||
"""
|
||||
if not raw:
|
||||
return "", None
|
||||
@@ -84,7 +90,6 @@ def parse_link_target(raw: str, current_note_id: Optional[str] = None) -> Tuple[
|
||||
target = parts[0].strip()
|
||||
section = parts[1].strip() if len(parts) > 1 else None
|
||||
|
||||
# Handle Self-Link [[#Section]] -> target wird zu current_note_id
|
||||
if not target and section and current_note_id:
|
||||
target = current_note_id
|
||||
|
||||
@@ -99,14 +104,30 @@ def load_types_registry() -> dict:
|
||||
except Exception: return {}
|
||||
|
||||
def get_edge_defaults_for(note_type: Optional[str], reg: dict) -> List[str]:
|
||||
"""Ermittelt Standard-Kanten für einen Typ."""
|
||||
"""
|
||||
WP-24c: Ermittelt Standard-Kanten (Typical Edges) für einen Notiz-Typ.
|
||||
Nutzt die EdgeRegistry (graph_schema.md) als primäre Quelle.
|
||||
"""
|
||||
# 1. Dynamische Abfrage über die neue Topologie-Engine (WP-24c)
|
||||
# Behebt das Audit-Problem 1a/1b: Suche in graph_schema.md statt types.yaml
|
||||
if note_type:
|
||||
topology = edge_registry.get_topology_info(note_type, "any")
|
||||
typical = topology.get("typical", [])
|
||||
if typical:
|
||||
return typical
|
||||
|
||||
# 2. Legacy-Fallback: Suche in der geladenen Registry (types.yaml)
|
||||
# Sichert 100% Rückwärtskompatibilität, falls Reste in types.yaml verblieben sind.
|
||||
types_map = reg.get("types", reg) if isinstance(reg, dict) else {}
|
||||
if note_type and isinstance(types_map, dict):
|
||||
t = types_map.get(note_type)
|
||||
if isinstance(t, dict) and isinstance(t.get("edge_defaults"), list):
|
||||
return [str(x) for x in t["edge_defaults"] if isinstance(x, str)]
|
||||
|
||||
# 3. Globaler Default-Fallback aus der Registry
|
||||
for key in ("defaults", "default", "global"):
|
||||
v = reg.get(key)
|
||||
if isinstance(v, dict) and isinstance(v.get("edge_defaults"), list):
|
||||
return [str(x) for x in v["edge_defaults"] if isinstance(x, str)]
|
||||
|
||||
return []
|
||||
@@ -1,10 +1,10 @@
|
||||
"""
|
||||
FILE: app/core/ingestion/ingestion_note_payload.py
|
||||
DESCRIPTION: Baut das JSON-Objekt für mindnet_notes.
|
||||
FEATURES:
|
||||
- Multi-Hash (body/full) für flexible Change Detection.
|
||||
- Fix v2.4.5: Präzise Hash-Logik für Profil-Änderungen.
|
||||
- Integration der zentralen Registry (WP-14).
|
||||
WP-14: Integration der zentralen Registry.
|
||||
WP-24c: Dynamische Ermittlung von edge_defaults aus dem Graph-Schema.
|
||||
VERSION: 2.5.0 (WP-24c: Dynamic Topology Integration)
|
||||
STATUS: Active
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, Tuple, Optional
|
||||
@@ -15,6 +15,8 @@ import hashlib
|
||||
|
||||
# Import der zentralen Registry-Logik
|
||||
from app.core.registry import load_type_registry
|
||||
# WP-24c: Zugriff auf das dynamische Graph-Schema
|
||||
from app.services.edge_registry import registry as edge_registry
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
@@ -46,15 +48,14 @@ def _compute_hash(content: str) -> str:
|
||||
def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str:
|
||||
"""
|
||||
Generiert den Hash-Input-String basierend auf Body oder Metadaten.
|
||||
Fix: Inkludiert nun alle entscheidungsrelevanten Profil-Parameter.
|
||||
Inkludiert alle entscheidungsrelevanten Profil-Parameter.
|
||||
"""
|
||||
body = str(n.get("body") or "").strip()
|
||||
if mode == "body": return body
|
||||
if mode == "full":
|
||||
fm = n.get("frontmatter") or {}
|
||||
meta_parts = []
|
||||
# Wir inkludieren alle Felder, die das Chunking oder Retrieval beeinflussen
|
||||
# Jede Änderung hier führt nun zwingend zu einem neuen Full-Hash
|
||||
# Alle Felder, die das Chunking oder Retrieval beeinflussen
|
||||
keys = [
|
||||
"title", "type", "status", "tags",
|
||||
"chunking_profile", "chunk_profile",
|
||||
@@ -87,7 +88,7 @@ def _cfg_defaults(reg: dict) -> dict:
|
||||
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
"""
|
||||
Baut das Note-Payload inklusive Multi-Hash und Audit-Validierung.
|
||||
WP-14: Nutzt die zentrale Registry für alle Fallbacks.
|
||||
WP-24c: Nutzt die EdgeRegistry zur dynamischen Auflösung von Typical Edges.
|
||||
"""
|
||||
n = _as_dict(note)
|
||||
|
||||
@@ -120,10 +121,16 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
if chunk_profile is None:
|
||||
chunk_profile = ingest_cfg.get("default_chunk_profile", cfg_def.get("chunking_profile", "sliding_standard"))
|
||||
|
||||
# --- edge_defaults Audit ---
|
||||
# --- WP-24c: edge_defaults Dynamisierung ---
|
||||
# 1. Priorität: Manuelle Definition im Frontmatter
|
||||
edge_defaults = fm.get("edge_defaults")
|
||||
|
||||
# 2. Priorität: Dynamische Abfrage der 'Typical Edges' aus dem Graph-Schema
|
||||
if edge_defaults is None:
|
||||
edge_defaults = cfg_type.get("edge_defaults", cfg_def.get("edge_defaults", []))
|
||||
topology = edge_registry.get_topology_info(note_type, "any")
|
||||
edge_defaults = topology.get("typical", [])
|
||||
|
||||
# 3. Fallback: Leere Liste, falls kein Schema-Eintrag existiert
|
||||
edge_defaults = _ensure_list(edge_defaults)
|
||||
|
||||
# --- Basis-Metadaten ---
|
||||
|
||||
+16
-11
@@ -1,11 +1,12 @@
|
||||
"""
|
||||
FILE: app/core/type_registry.py
|
||||
DESCRIPTION: Loader für types.yaml. Achtung: Wird in der aktuellen Pipeline meist durch lokale Loader in 'ingestion.py' oder 'note_payload.py' umgangen.
|
||||
VERSION: 1.0.0
|
||||
STATUS: Deprecated (Redundant)
|
||||
DESCRIPTION: Loader für types.yaml.
|
||||
WP-24c: Robustheits-Fix für chunking_profile vs chunk_profile.
|
||||
WP-14: Support für zentrale Registry-Strukturen.
|
||||
VERSION: 1.1.0 (Audit-Fix: Profile Key Consistency)
|
||||
STATUS: Active (Support für Legacy-Loader)
|
||||
DEPENDENCIES: yaml, os, functools
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -18,12 +19,12 @@ try:
|
||||
except Exception:
|
||||
yaml = None # wird erst benötigt, wenn eine Datei gelesen werden soll
|
||||
|
||||
# Konservativer Default – bewusst minimal
|
||||
# Konservativer Default – WP-24c: Nutzt nun konsistent 'chunking_profile'
|
||||
_DEFAULT_REGISTRY: Dict[str, Any] = {
|
||||
"version": "1.0",
|
||||
"types": {
|
||||
"concept": {
|
||||
"chunk_profile": "medium",
|
||||
"chunking_profile": "medium",
|
||||
"edge_defaults": ["references", "related_to"],
|
||||
"retriever_weight": 1.0,
|
||||
}
|
||||
@@ -33,7 +34,6 @@ _DEFAULT_REGISTRY: Dict[str, Any] = {
|
||||
}
|
||||
|
||||
# Chunk-Profile → Overlap-Empfehlungen (nur für synthetische Fensterbildung)
|
||||
# Die absoluten Chunk-Längen bleiben Aufgabe des Chunkers (assemble_chunks).
|
||||
_PROFILE_TO_OVERLAP: Dict[str, Tuple[int, int]] = {
|
||||
"short": (20, 30),
|
||||
"medium": (40, 60),
|
||||
@@ -45,7 +45,7 @@ _PROFILE_TO_OVERLAP: Dict[str, Tuple[int, int]] = {
|
||||
def load_type_registry(path: str = "config/types.yaml") -> Dict[str, Any]:
|
||||
"""
|
||||
Lädt die Registry aus 'path'. Bei Fehlern wird ein konserviver Default geliefert.
|
||||
Die Rückgabe ist *prozessweit* gecached.
|
||||
Die Rückgabe ist prozessweit gecached.
|
||||
"""
|
||||
if not path:
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
@@ -54,7 +54,6 @@ def load_type_registry(path: str = "config/types.yaml") -> Dict[str, Any]:
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
if yaml is None:
|
||||
# PyYAML fehlt → auf Default zurückfallen
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
try:
|
||||
@@ -71,6 +70,7 @@ def load_type_registry(path: str = "config/types.yaml") -> Dict[str, Any]:
|
||||
|
||||
|
||||
def get_type_config(note_type: Optional[str], reg: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Extrahiert die Konfiguration für einen spezifischen Typ."""
|
||||
t = (note_type or "concept").strip().lower()
|
||||
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
|
||||
return types.get(t) or types.get("concept") or _DEFAULT_REGISTRY["types"]["concept"]
|
||||
@@ -84,8 +84,13 @@ def resolve_note_type(fm_type: Optional[str], reg: Dict[str, Any]) -> str:
|
||||
|
||||
|
||||
def effective_chunk_profile(note_type: Optional[str], reg: Dict[str, Any]) -> Optional[str]:
|
||||
"""
|
||||
Ermittelt das aktive Chunking-Profil für einen Notiz-Typ.
|
||||
Fix (Audit-Problem 2): Prüft beide Key-Varianten für 100% Kompatibilität.
|
||||
"""
|
||||
cfg = get_type_config(note_type, reg)
|
||||
prof = cfg.get("chunk_profile")
|
||||
# Check 'chunking_profile' (Standard) OR 'chunk_profile' (Legacy/Fallback)
|
||||
prof = cfg.get("chunking_profile") or cfg.get("chunk_profile")
|
||||
if isinstance(prof, str) and prof.strip():
|
||||
return prof.strip().lower()
|
||||
return None
|
||||
@@ -95,4 +100,4 @@ def profile_overlap(profile: Optional[str]) -> Tuple[int, int]:
|
||||
"""Gibt eine Overlap-Empfehlung (low, high) für das Profil zurück."""
|
||||
if not profile:
|
||||
return _PROFILE_TO_OVERLAP["medium"]
|
||||
return _PROFILE_TO_OVERLAP.get(profile.strip().lower(), _PROFILE_TO_OVERLAP["medium"])
|
||||
return _PROFILE_TO_OVERLAP.get(profile.strip().lower(), _PROFILE_TO_OVERLAP["medium"])
|
||||
Reference in new issue
Block a user