Implement WP-15c enhancements across graph and retrieval modules, including full metadata support for Super-Edge aggregation and Note-Level Diversity Pooling. Update scoring logic to reflect new edge handling and improve retrieval accuracy. Version updates to reflect these changes.
This commit is contained in:
1 parent
cd5056d4c9
commit
d35bdc64b9
5 files changed
+202
-91
No files matched your search
@@ -1,13 +1,15 @@
|
||||
"""
|
||||
FILE: app/core/graph/graph_db_adapter.py
|
||||
DESCRIPTION: Datenbeschaffung aus Qdrant für den Graphen.
|
||||
AUDIT v1.1.0: Nutzt nun die zentrale database-Infrastruktur für Namen.
|
||||
AUDIT v1.1.1: Volle Unterstützung für WP-15c Metadaten.
|
||||
Stellt sicher, dass 'target_section' und 'provenance' für die
|
||||
Super-Edge-Aggregation im Retriever geladen werden.
|
||||
"""
|
||||
from typing import List, Dict, Optional
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
# ENTSCHEIDENDER FIX: Nutzt die neue Infrastruktur für konsistente Collection-Namen
|
||||
# Nutzt die zentrale Infrastruktur für konsistente Collection-Namen (WP-14)
|
||||
from app.core.database import collection_names
|
||||
|
||||
def fetch_edges_from_qdrant(
|
||||
@@ -19,14 +21,16 @@ def fetch_edges_from_qdrant(
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Holt Edges aus der Datenbank basierend auf Seed-IDs.
|
||||
Filtert auf source_id, target_id oder note_id.
|
||||
WP-15c: Erhält alle Metadaten für das Note-Level Diversity Pooling.
|
||||
"""
|
||||
if not seeds or limit <= 0:
|
||||
return []
|
||||
|
||||
# Konsistente Namensauflösung via database-Paket
|
||||
# Rückgabe: (notes_col, chunks_col, edges_col)
|
||||
_, _, edges_col = collection_names(prefix)
|
||||
|
||||
# Wir suchen Kanten, bei denen die Seed-IDs entweder Quelle, Ziel oder Kontext-Note sind.
|
||||
seed_conditions = []
|
||||
for field in ("source_id", "target_id", "note_id"):
|
||||
for s in seeds:
|
||||
@@ -35,6 +39,7 @@ def fetch_edges_from_qdrant(
|
||||
)
|
||||
seeds_filter = rest.Filter(should=seed_conditions) if seed_conditions else None
|
||||
|
||||
# Optionaler Filter auf spezifische Kanten-Typen (z.B. für Intent-Routing)
|
||||
type_filter = None
|
||||
if edge_types:
|
||||
type_conds = [
|
||||
@@ -52,6 +57,7 @@ def fetch_edges_from_qdrant(
|
||||
flt = rest.Filter(must=must) if must else None
|
||||
|
||||
# Abfrage via Qdrant Scroll API
|
||||
# WICHTIG: with_payload=True lädt alle Metadaten (target_section, provenance etc.)
|
||||
pts, _ = client.scroll(
|
||||
collection_name=edges_col,
|
||||
scroll_filter=flt,
|
||||
@@ -60,4 +66,6 @@ def fetch_edges_from_qdrant(
|
||||
with_vectors=False,
|
||||
)
|
||||
|
||||
# Wir geben das vollständige Payload zurück, damit der Retriever
|
||||
# alle Signale für die Super-Edge-Aggregation und das Scoring hat.
|
||||
return [dict(p.payload) for p in pts if p.payload]
|
||||
@@ -1,10 +1,10 @@
|
||||
"""
|
||||
FILE: app/core/graph/graph_derive_edges.py
|
||||
DESCRIPTION: Hauptlogik zur Kanten-Aggregation und De-Duplizierung.
|
||||
AUDIT:
|
||||
- Nutzt parse_link_target
|
||||
- Übergibt Section als 'variant' an ID-Gen
|
||||
- Dedup basiert jetzt auf Edge-ID (erlaubt Multigraph für Sections)
|
||||
WP-15b/c Audit:
|
||||
- Präzises Sektions-Splitting via parse_link_target.
|
||||
- Eindeutige ID-Generierung pro Sektions-Variante (Multigraph).
|
||||
- Ermöglicht dem Retriever die Super-Edge-Aggregation.
|
||||
"""
|
||||
from typing import List, Optional, Dict, Tuple
|
||||
from .graph_utils import (
|
||||
@@ -21,31 +21,45 @@ def build_edges_for_note(
|
||||
note_level_references: Optional[List[str]] = None,
|
||||
include_note_scope_refs: bool = False,
|
||||
) -> List[dict]:
|
||||
"""Erzeugt und aggregiert alle Kanten für eine Note (WP-15b)."""
|
||||
"""
|
||||
Erzeugt und aggregiert alle Kanten für eine Note.
|
||||
Sorgt für die physische Trennung von Sektions-Links via Edge-ID.
|
||||
"""
|
||||
edges: List[dict] = []
|
||||
# note_type für die Ermittlung der edge_defaults (types.yaml)
|
||||
note_type = _get(chunks[0], "type") if chunks else "concept"
|
||||
|
||||
# 1) Struktur-Kanten (belongs_to, next/prev)
|
||||
# 1) Struktur-Kanten (Internal: belongs_to, next/prev)
|
||||
# Diese erhalten die Provenienz 'structure' und sind in der Registry geschützt.
|
||||
for idx, ch in enumerate(chunks):
|
||||
cid = _get(ch, "chunk_id", "id")
|
||||
if not cid: continue
|
||||
|
||||
# Verbindung Chunk -> Note
|
||||
edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {
|
||||
"chunk_id": cid, "edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to"),
|
||||
"provenance": "structure", "rule_id": "structure:belongs_to", "confidence": PROVENANCE_PRIORITY["structure:belongs_to"]
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to"),
|
||||
"provenance": "structure",
|
||||
"rule_id": "structure:belongs_to",
|
||||
"confidence": PROVENANCE_PRIORITY["structure:belongs_to"]
|
||||
}))
|
||||
|
||||
# Horizontale Verkettung (Ordnung)
|
||||
if idx < len(chunks) - 1:
|
||||
next_id = _get(chunks[idx+1], "chunk_id", "id")
|
||||
if next_id:
|
||||
edges.append(_edge("next", "chunk", cid, next_id, note_id, {
|
||||
"chunk_id": cid, "edge_id": _mk_edge_id("next", cid, next_id, "chunk", "structure:order"),
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id("next", cid, next_id, "chunk", "structure:order"),
|
||||
"provenance": "structure", "rule_id": "structure:order", "confidence": PROVENANCE_PRIORITY["structure:order"]
|
||||
}))
|
||||
edges.append(_edge("prev", "chunk", next_id, cid, note_id, {
|
||||
"chunk_id": next_id, "edge_id": _mk_edge_id("prev", next_id, cid, "chunk", "structure:order"),
|
||||
"chunk_id": next_id,
|
||||
"edge_id": _mk_edge_id("prev", next_id, cid, "chunk", "structure:order"),
|
||||
"provenance": "structure", "rule_id": "structure:order", "confidence": PROVENANCE_PRIORITY["structure:order"]
|
||||
}))
|
||||
|
||||
# 2) Inhaltliche Kanten
|
||||
# 2) Inhaltliche Kanten (Explicit & Candidate Pool)
|
||||
reg = load_types_registry()
|
||||
defaults = get_edge_defaults_for(note_type, reg)
|
||||
refs_all: List[str] = []
|
||||
@@ -55,7 +69,7 @@ def build_edges_for_note(
|
||||
if not cid: continue
|
||||
raw = _get(ch, "window") or _get(ch, "text") or ""
|
||||
|
||||
# Typed & Candidate Pool (WP-15b Integration)
|
||||
# A. Typed Relations (Inline [[rel:kind|target]])
|
||||
typed, rem = extract_typed_relations(raw)
|
||||
for k, raw_t in typed:
|
||||
t, sec = parse_link_target(raw_t, note_id)
|
||||
@@ -63,14 +77,14 @@ def build_edges_for_note(
|
||||
|
||||
payload = {
|
||||
"chunk_id": cid,
|
||||
# Variant=sec sorgt für eindeutige ID pro Abschnitt
|
||||
# WP-Fix: Variant=sec sorgt für eindeutige ID pro Sektion
|
||||
"edge_id": _mk_edge_id(k, cid, t, "chunk", "inline:rel", variant=sec),
|
||||
"provenance": "explicit", "rule_id": "inline:rel", "confidence": PROVENANCE_PRIORITY["inline:rel"]
|
||||
}
|
||||
if sec: payload["target_section"] = sec
|
||||
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
|
||||
|
||||
# B. Candidate Pool (WP-15b Validierte KI-Kanten)
|
||||
pool = ch.get("candidate_pool") or ch.get("candidate_edges") or []
|
||||
for cand in pool:
|
||||
raw_t, k, p = cand.get("to"), cand.get("kind", "related_to"), cand.get("provenance", "semantic_ai")
|
||||
@@ -82,10 +96,9 @@ def build_edges_for_note(
|
||||
"provenance": p, "rule_id": f"candidate:{p}", "confidence": PROVENANCE_PRIORITY.get(p, 0.90)
|
||||
}
|
||||
if sec: payload["target_section"] = sec
|
||||
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
|
||||
|
||||
# Callouts & Wikilinks
|
||||
# C. Callouts (> [!edge])
|
||||
call_pairs, rem2 = extract_callout_relations(rem)
|
||||
for k, raw_t in call_pairs:
|
||||
t, sec = parse_link_target(raw_t, note_id)
|
||||
@@ -97,9 +110,9 @@ def build_edges_for_note(
|
||||
"provenance": "explicit", "rule_id": "callout:edge", "confidence": PROVENANCE_PRIORITY["callout:edge"]
|
||||
}
|
||||
if sec: payload["target_section"] = sec
|
||||
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
|
||||
|
||||
# D. Standard Wikilinks & Typ-Defaults
|
||||
refs = extract_wikilinks(rem2)
|
||||
for raw_r in refs:
|
||||
r, sec = parse_link_target(raw_r, note_id)
|
||||
@@ -111,9 +124,9 @@ def build_edges_for_note(
|
||||
"provenance": "explicit", "rule_id": "explicit:wikilink", "confidence": PROVENANCE_PRIORITY["explicit:wikilink"]
|
||||
}
|
||||
if sec: payload["target_section"] = sec
|
||||
|
||||
edges.append(_edge("references", "chunk", cid, r, note_id, payload))
|
||||
|
||||
# Automatische Kanten-Vererbung aus types.yaml
|
||||
for rel in defaults:
|
||||
if rel != "references":
|
||||
def_payload = {
|
||||
@@ -124,13 +137,10 @@ def build_edges_for_note(
|
||||
if sec: def_payload["target_section"] = sec
|
||||
edges.append(_edge(rel, "chunk", cid, r, note_id, def_payload))
|
||||
|
||||
# Für Note-Scope Sammlung nutzen wir den Original-String zur Dedup, aber gesäubert
|
||||
refs_all.extend([parse_link_target(r, note_id)[0] for r in refs])
|
||||
|
||||
# 3) Note-Scope & De-Duplizierung
|
||||
# 3) Note-Scope (Grobe Struktur-Verbindungen)
|
||||
if include_note_scope_refs:
|
||||
# refs_all ist jetzt schon gesäubert (nur Targets)
|
||||
# note_level_references müssen auch gesäubert werden
|
||||
cleaned_note_refs = [parse_link_target(r, note_id)[0] for r in (note_level_references or [])]
|
||||
refs_note = _dedupe_seq((refs_all or []) + cleaned_note_refs)
|
||||
|
||||
@@ -140,17 +150,19 @@ def build_edges_for_note(
|
||||
"edge_id": _mk_edge_id("references", note_id, r, "note", "explicit:note_scope"),
|
||||
"provenance": "explicit", "confidence": PROVENANCE_PRIORITY["explicit:note_scope"]
|
||||
}))
|
||||
# Backlinks zur Stärkung der Bidirektionalität
|
||||
edges.append(_edge("backlink", "note", r, note_id, note_id, {
|
||||
"edge_id": _mk_edge_id("backlink", r, note_id, "note", "derived:backlink"),
|
||||
"provenance": "rule", "confidence": PROVENANCE_PRIORITY["derived:backlink"]
|
||||
}))
|
||||
|
||||
# Deduplizierung: Wir nutzen jetzt die EDGE-ID als Schlüssel.
|
||||
# Da die Edge-ID nun 'variant' (Section) enthält, bleiben unterschiedliche Sections erhalten.
|
||||
# 4) De-Duplizierung (In-Place)
|
||||
# Da die EDGE-ID nun die Sektion (variant) enthält, bleiben Links auf
|
||||
# unterschiedliche Abschnitte derselben Note erhalten.
|
||||
unique_map: Dict[str, dict] = {}
|
||||
for e in edges:
|
||||
eid = e["edge_id"]
|
||||
# Bei Konflikt (gleiche ID = exakt gleiche Kante und Section) gewinnt die höhere Confidence
|
||||
# Höhere Confidence gewinnt bei identischer ID
|
||||
if eid not in unique_map or e.get("confidence", 0) > unique_map[eid].get("confidence", 0):
|
||||
unique_map[eid] = e
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
FILE: app/core/graph/graph_subgraph.py
|
||||
DESCRIPTION: In-Memory Repräsentation eines Graphen für Scoring und Analyse.
|
||||
Zentrale Komponente für die Graph-Expansion (BFS) und Bonus-Berechnung.
|
||||
MODULARISIERUNG: Teil des graph-Pakets (WP-14).
|
||||
VERSION: 1.1.0
|
||||
WP-15c Update: Erhalt von Metadaten (target_section, provenance)
|
||||
für präzises Retrieval-Reasoning.
|
||||
VERSION: 1.2.0
|
||||
STATUS: Active
|
||||
"""
|
||||
import math
|
||||
@@ -22,6 +23,7 @@ class Subgraph:
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# adj speichert nun vollständige Payloads statt nur Tripel
|
||||
self.adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||
self.reverse_adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||
self.in_degree: DefaultDict[str, int] = defaultdict(int)
|
||||
@@ -30,31 +32,49 @@ class Subgraph:
|
||||
def add_edge(self, e: Dict) -> None:
|
||||
"""
|
||||
Fügt eine Kante hinzu und aktualisiert Indizes.
|
||||
Unterstützt Kontext-Notes für verbesserte Graph-Konnektivität.
|
||||
WP-15c: Speichert das vollständige Payload für den Explanation Layer.
|
||||
"""
|
||||
src = e.get("source")
|
||||
tgt = e.get("target")
|
||||
kind = e.get("kind")
|
||||
weight = e.get("weight", EDGE_BASE_WEIGHTS.get(kind, 0.0))
|
||||
|
||||
# Das gesamte Payload wird als Kanten-Objekt behalten
|
||||
# Wir stellen sicher, dass alle relevanten Metadaten vorhanden sind
|
||||
edge_data = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"kind": kind,
|
||||
"weight": e.get("weight", EDGE_BASE_WEIGHTS.get(kind, 0.0)),
|
||||
"provenance": e.get("provenance", "rule"),
|
||||
"confidence": e.get("confidence", 1.0),
|
||||
"target_section": e.get("target_section"), # Essentiell für Präzision
|
||||
"is_super_edge": e.get("is_super_edge", False)
|
||||
}
|
||||
|
||||
owner = e.get("note_id")
|
||||
|
||||
if not src or not tgt:
|
||||
return
|
||||
|
||||
# 1. Forward-Kante
|
||||
self.adj[src].append({"target": tgt, "kind": kind, "weight": weight})
|
||||
self.adj[src].append(edge_data)
|
||||
self.out_degree[src] += 1
|
||||
self.in_degree[tgt] += 1
|
||||
|
||||
# 2. Reverse-Kante (für WP-04b Explanation Layer)
|
||||
self.reverse_adj[tgt].append({"source": src, "kind": kind, "weight": weight})
|
||||
# 2. Reverse-Kante (für Explanation Layer & Backlinks)
|
||||
self.reverse_adj[tgt].append(edge_data)
|
||||
|
||||
# 3. Kontext-Note Handling (erhöht die Zentralität der Parent-Note)
|
||||
if owner and owner != src:
|
||||
self.adj[owner].append({"target": tgt, "kind": kind, "weight": weight})
|
||||
# Wir erstellen eine virtuelle Kontext-Kante
|
||||
ctx_edge = edge_data.copy()
|
||||
ctx_edge["source"] = owner
|
||||
ctx_edge["via_context"] = True
|
||||
|
||||
self.adj[owner].append(ctx_edge)
|
||||
self.out_degree[owner] += 1
|
||||
if owner != tgt:
|
||||
self.reverse_adj[tgt].append({"source": owner, "kind": kind, "weight": weight, "via_context": True})
|
||||
self.reverse_adj[tgt].append(ctx_edge)
|
||||
self.in_degree[owner] += 1
|
||||
|
||||
def aggregate_edge_bonus(self, node_id: str) -> float:
|
||||
@@ -73,14 +93,15 @@ class Subgraph:
|
||||
indeg = self.in_degree.get(node_id, 0)
|
||||
if indeg <= 0:
|
||||
return 0.0
|
||||
# math.log1p(x) entspricht log(1+x)
|
||||
return min(math.log1p(indeg) / 10.0, 0.15)
|
||||
|
||||
def get_outgoing_edges(self, node_id: str) -> List[Dict[str, Any]]:
|
||||
"""Gibt alle ausgehenden Kanten einer Node zurück."""
|
||||
"""Gibt alle ausgehenden Kanten einer Node inkl. Metadaten zurück."""
|
||||
return self.adj.get(node_id, [])
|
||||
|
||||
def get_incoming_edges(self, node_id: str) -> List[Dict[str, Any]]:
|
||||
"""Gibt alle eingehenden Kanten einer Node zurück."""
|
||||
"""Gibt alle eingehenden Kanten einer Node inkl. Metadaten zurück."""
|
||||
return self.reverse_adj.get(node_id, [])
|
||||
|
||||
|
||||
@@ -111,13 +132,19 @@ def expand(
|
||||
src, tgt = pl.get("source_id"), pl.get("target_id")
|
||||
if not src or not tgt: continue
|
||||
|
||||
sg.add_edge({
|
||||
# WP-15c: Wir übergeben das vollständige Payload an add_edge
|
||||
edge_payload = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"kind": pl.get("kind", "edge"),
|
||||
"weight": calculate_edge_weight(pl),
|
||||
"note_id": pl.get("note_id"),
|
||||
})
|
||||
"provenance": pl.get("provenance", "rule"),
|
||||
"confidence": pl.get("confidence", 1.0),
|
||||
"target_section": pl.get("target_section")
|
||||
}
|
||||
|
||||
sg.add_edge(edge_payload)
|
||||
|
||||
# BFS Logik: Neue Ziele in die nächste Frontier aufnehmen
|
||||
if tgt not in visited:
|
||||
|
||||
Reference in new issue
Block a user