Implement WP-15c enhancements across graph and retrieval modules, including full metadata support for Super-Edge aggregation and Note-Level Diversity Pooling. Update scoring logic to reflect new edge handling and improve retrieval accuracy. Version updates to reflect these changes.

This commit is contained in:
Lars committed 2025-12-30 21:47:18 +01:00
1 parent cd5056d4c9
commit d35bdc64b9
5 files changed
+202 -91

No files matched your search

+37 -25
View File
@@ -1,10 +1,10 @@
"""
FILE: app/core/graph/graph_derive_edges.py
DESCRIPTION: Hauptlogik zur Kanten-Aggregation und De-Duplizierung.
AUDIT:
- Nutzt parse_link_target
- Übergibt Section als 'variant' an ID-Gen
- Dedup basiert jetzt auf Edge-ID (erlaubt Multigraph für Sections)
WP-15b/c Audit:
- Präzises Sektions-Splitting via parse_link_target.
- Eindeutige ID-Generierung pro Sektions-Variante (Multigraph).
- Ermöglicht dem Retriever die Super-Edge-Aggregation.
"""
from typing import List, Optional, Dict, Tuple
from .graph_utils import (
@@ -21,31 +21,45 @@ def build_edges_for_note(
note_level_references: Optional[List[str]] = None,
include_note_scope_refs: bool = False,
) -> List[dict]:
"""Erzeugt und aggregiert alle Kanten für eine Note (WP-15b)."""
"""
Erzeugt und aggregiert alle Kanten für eine Note.
Sorgt für die physische Trennung von Sektions-Links via Edge-ID.
"""
edges: List[dict] = []
# note_type für die Ermittlung der edge_defaults (types.yaml)
note_type = _get(chunks[0], "type") if chunks else "concept"
# 1) Struktur-Kanten (belongs_to, next/prev)
# 1) Struktur-Kanten (Internal: belongs_to, next/prev)
# Diese erhalten die Provenienz 'structure' und sind in der Registry geschützt.
for idx, ch in enumerate(chunks):
cid = _get(ch, "chunk_id", "id")
if not cid: continue
# Verbindung Chunk -> Note
edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {
"chunk_id": cid, "edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to"),
"provenance": "structure", "rule_id": "structure:belongs_to", "confidence": PROVENANCE_PRIORITY["structure:belongs_to"]
"chunk_id": cid,
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to"),
"provenance": "structure",
"rule_id": "structure:belongs_to",
"confidence": PROVENANCE_PRIORITY["structure:belongs_to"]
}))
# Horizontale Verkettung (Ordnung)
if idx < len(chunks) - 1:
next_id = _get(chunks[idx+1], "chunk_id", "id")
if next_id:
edges.append(_edge("next", "chunk", cid, next_id, note_id, {
"chunk_id": cid, "edge_id": _mk_edge_id("next", cid, next_id, "chunk", "structure:order"),
"chunk_id": cid,
"edge_id": _mk_edge_id("next", cid, next_id, "chunk", "structure:order"),
"provenance": "structure", "rule_id": "structure:order", "confidence": PROVENANCE_PRIORITY["structure:order"]
}))
edges.append(_edge("prev", "chunk", next_id, cid, note_id, {
"chunk_id": next_id, "edge_id": _mk_edge_id("prev", next_id, cid, "chunk", "structure:order"),
"chunk_id": next_id,
"edge_id": _mk_edge_id("prev", next_id, cid, "chunk", "structure:order"),
"provenance": "structure", "rule_id": "structure:order", "confidence": PROVENANCE_PRIORITY["structure:order"]
}))
# 2) Inhaltliche Kanten
# 2) Inhaltliche Kanten (Explicit & Candidate Pool)
reg = load_types_registry()
defaults = get_edge_defaults_for(note_type, reg)
refs_all: List[str] = []
@@ -55,7 +69,7 @@ def build_edges_for_note(
if not cid: continue
raw = _get(ch, "window") or _get(ch, "text") or ""
# Typed & Candidate Pool (WP-15b Integration)
# A. Typed Relations (Inline [[rel:kind|target]])
typed, rem = extract_typed_relations(raw)
for k, raw_t in typed:
t, sec = parse_link_target(raw_t, note_id)
@@ -63,14 +77,14 @@ def build_edges_for_note(
payload = {
"chunk_id": cid,
# Variant=sec sorgt für eindeutige ID pro Abschnitt
# WP-Fix: Variant=sec sorgt für eindeutige ID pro Sektion
"edge_id": _mk_edge_id(k, cid, t, "chunk", "inline:rel", variant=sec),
"provenance": "explicit", "rule_id": "inline:rel", "confidence": PROVENANCE_PRIORITY["inline:rel"]
}
if sec: payload["target_section"] = sec
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
# B. Candidate Pool (WP-15b Validierte KI-Kanten)
pool = ch.get("candidate_pool") or ch.get("candidate_edges") or []
for cand in pool:
raw_t, k, p = cand.get("to"), cand.get("kind", "related_to"), cand.get("provenance", "semantic_ai")
@@ -82,10 +96,9 @@ def build_edges_for_note(
"provenance": p, "rule_id": f"candidate:{p}", "confidence": PROVENANCE_PRIORITY.get(p, 0.90)
}
if sec: payload["target_section"] = sec
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
# Callouts & Wikilinks
# C. Callouts (> [!edge])
call_pairs, rem2 = extract_callout_relations(rem)
for k, raw_t in call_pairs:
t, sec = parse_link_target(raw_t, note_id)
@@ -97,9 +110,9 @@ def build_edges_for_note(
"provenance": "explicit", "rule_id": "callout:edge", "confidence": PROVENANCE_PRIORITY["callout:edge"]
}
if sec: payload["target_section"] = sec
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
# D. Standard Wikilinks & Typ-Defaults
refs = extract_wikilinks(rem2)
for raw_r in refs:
r, sec = parse_link_target(raw_r, note_id)
@@ -111,9 +124,9 @@ def build_edges_for_note(
"provenance": "explicit", "rule_id": "explicit:wikilink", "confidence": PROVENANCE_PRIORITY["explicit:wikilink"]
}
if sec: payload["target_section"] = sec
edges.append(_edge("references", "chunk", cid, r, note_id, payload))
# Automatische Kanten-Vererbung aus types.yaml
for rel in defaults:
if rel != "references":
def_payload = {
@@ -124,13 +137,10 @@ def build_edges_for_note(
if sec: def_payload["target_section"] = sec
edges.append(_edge(rel, "chunk", cid, r, note_id, def_payload))
# Für Note-Scope Sammlung nutzen wir den Original-String zur Dedup, aber gesäubert
refs_all.extend([parse_link_target(r, note_id)[0] for r in refs])
# 3) Note-Scope & De-Duplizierung
# 3) Note-Scope (Grobe Struktur-Verbindungen)
if include_note_scope_refs:
# refs_all ist jetzt schon gesäubert (nur Targets)
# note_level_references müssen auch gesäubert werden
cleaned_note_refs = [parse_link_target(r, note_id)[0] for r in (note_level_references or [])]
refs_note = _dedupe_seq((refs_all or []) + cleaned_note_refs)
@@ -140,17 +150,19 @@ def build_edges_for_note(
"edge_id": _mk_edge_id("references", note_id, r, "note", "explicit:note_scope"),
"provenance": "explicit", "confidence": PROVENANCE_PRIORITY["explicit:note_scope"]
}))
# Backlinks zur Stärkung der Bidirektionalität
edges.append(_edge("backlink", "note", r, note_id, note_id, {
"edge_id": _mk_edge_id("backlink", r, note_id, "note", "derived:backlink"),
"provenance": "rule", "confidence": PROVENANCE_PRIORITY["derived:backlink"]
}))
# Deduplizierung: Wir nutzen jetzt die EDGE-ID als Schlüssel.
# Da die Edge-ID nun 'variant' (Section) enthält, bleiben unterschiedliche Sections erhalten.
# 4) De-Duplizierung (In-Place)
# Da die EDGE-ID nun die Sektion (variant) enthält, bleiben Links auf
# unterschiedliche Abschnitte derselben Note erhalten.
unique_map: Dict[str, dict] = {}
for e in edges:
eid = e["edge_id"]
# Bei Konflikt (gleiche ID = exakt gleiche Kante und Section) gewinnt die höhere Confidence
# Höhere Confidence gewinnt bei identischer ID
if eid not in unique_map or e.get("confidence", 0) > unique_map[eid].get("confidence", 0):
unique_map[eid] = e