Update graph_db_adapter.py, graph_derive_edges.py, graph_subgraph.py, graph_utils.py, ingestion_processor.py, and retriever.py to version 4.1.0: Introduce Scope-Awareness and Section-Filtering features, enhancing edge retrieval and processing. Implement Note-Scope Zones extraction from Markdown, improve edge ID generation with target_section, and prioritize Note-Scope Links during de-duplication. Update documentation for clarity and consistency across modules.
This commit is contained in:
1 parent
be2bed9927
commit
39fd15b565
14 files changed
+1004
-37
No files matched your search
@@ -1,9 +1,11 @@
|
||||
"""
|
||||
FILE: app/core/graph/graph_db_adapter.py
|
||||
DESCRIPTION: Datenbeschaffung aus Qdrant für den Graphen.
|
||||
AUDIT v1.1.1: Volle Unterstützung für WP-15c Metadaten.
|
||||
Stellt sicher, dass 'target_section' und 'provenance' für die
|
||||
Super-Edge-Aggregation im Retriever geladen werden.
|
||||
AUDIT v1.2.0: Gold-Standard v4.1.0 - Scope-Awareness & Section-Filtering.
|
||||
- Erweiterte Suche nach chunk_id-Edges für Scope-Awareness
|
||||
- Optionales target_section-Filtering für präzise Section-Links
|
||||
- Vollständige Metadaten-Unterstützung (provenance, confidence, virtual)
|
||||
VERSION: 1.2.0 (WP-24c: Gold-Standard v4.1.0)
|
||||
"""
|
||||
from typing import List, Dict, Optional
|
||||
from qdrant_client import QdrantClient
|
||||
@@ -17,11 +19,22 @@ def fetch_edges_from_qdrant(
|
||||
prefix: str,
|
||||
seeds: List[str],
|
||||
edge_types: Optional[List[str]] = None,
|
||||
target_section: Optional[str] = None,
|
||||
chunk_ids: Optional[List[str]] = None,
|
||||
limit: int = 2048,
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Holt Edges aus der Datenbank basierend auf Seed-IDs.
|
||||
WP-15c: Erhält alle Metadaten für das Note-Level Diversity Pooling.
|
||||
WP-24c v4.1.0: Scope-Aware Edge Retrieval mit Section-Filtering.
|
||||
|
||||
Args:
|
||||
client: Qdrant Client
|
||||
prefix: Collection-Präfix
|
||||
seeds: Liste von Note-IDs für die Suche
|
||||
edge_types: Optionale Filterung nach Kanten-Typen
|
||||
target_section: Optionales Section-Filtering (für präzise Section-Links)
|
||||
chunk_ids: Optionale Liste von Chunk-IDs für Scope-Awareness (Chunk-Level Edges)
|
||||
limit: Maximale Anzahl zurückgegebener Edges
|
||||
"""
|
||||
if not seeds or limit <= 0:
|
||||
return []
|
||||
@@ -30,13 +43,21 @@ def fetch_edges_from_qdrant(
|
||||
# Rückgabe: (notes_col, chunks_col, edges_col)
|
||||
_, _, edges_col = collection_names(prefix)
|
||||
|
||||
# Wir suchen Kanten, bei denen die Seed-IDs entweder Quelle, Ziel oder Kontext-Note sind.
|
||||
# WP-24c v4.1.0: Scope-Awareness - Suche nach Note- UND Chunk-Level Edges
|
||||
seed_conditions = []
|
||||
for field in ("source_id", "target_id", "note_id"):
|
||||
for s in seeds:
|
||||
seed_conditions.append(
|
||||
rest.FieldCondition(key=field, match=rest.MatchValue(value=str(s)))
|
||||
)
|
||||
|
||||
# Chunk-Level Edges: Wenn chunk_ids angegeben, suche auch nach chunk_id als source_id
|
||||
if chunk_ids:
|
||||
for cid in chunk_ids:
|
||||
seed_conditions.append(
|
||||
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=str(cid)))
|
||||
)
|
||||
|
||||
seeds_filter = rest.Filter(should=seed_conditions) if seed_conditions else None
|
||||
|
||||
# Optionaler Filter auf spezifische Kanten-Typen (z.B. für Intent-Routing)
|
||||
@@ -48,11 +69,20 @@ def fetch_edges_from_qdrant(
|
||||
]
|
||||
type_filter = rest.Filter(should=type_conds)
|
||||
|
||||
# WP-24c v4.1.0: Section-Filtering für präzise Section-Links
|
||||
section_filter = None
|
||||
if target_section:
|
||||
section_filter = rest.Filter(must=[
|
||||
rest.FieldCondition(key="target_section", match=rest.MatchValue(value=str(target_section)))
|
||||
])
|
||||
|
||||
must = []
|
||||
if seeds_filter:
|
||||
must.append(seeds_filter)
|
||||
if type_filter:
|
||||
must.append(type_filter)
|
||||
if section_filter:
|
||||
must.append(section_filter)
|
||||
|
||||
flt = rest.Filter(must=must) if must else None
|
||||
|
||||
|
||||
@@ -5,7 +5,14 @@ DESCRIPTION: Hauptlogik zur Kanten-Aggregation und De-Duplizierung.
|
||||
- Präzises Sektions-Splitting via parse_link_target.
|
||||
- v4.1.0: Eindeutige ID-Generierung pro Sektions-Variante (Multigraph).
|
||||
- Ermöglicht dem Retriever die Super-Edge-Aggregation.
|
||||
WP-24c v4.2.0: Note-Scope Extraktions-Zonen für globale Referenzen.
|
||||
- Header-basierte Identifikation von Note-Scope Zonen
|
||||
- Automatische Scope-Umschaltung (chunk -> note)
|
||||
- Priorisierung: Note-Scope Links haben Vorrang bei Duplikaten
|
||||
VERSION: 4.2.0 (WP-24c: Note-Scope Zones)
|
||||
STATUS: Active
|
||||
"""
|
||||
import re
|
||||
from typing import List, Optional, Dict, Tuple
|
||||
from .graph_utils import (
|
||||
_get, _edge, _mk_edge_id, _dedupe_seq, parse_link_target,
|
||||
@@ -15,19 +22,138 @@ from .graph_extractors import (
|
||||
extract_typed_relations, extract_callout_relations, extract_wikilinks
|
||||
)
|
||||
|
||||
# WP-24c v4.2.0: Header-basierte Identifikation von Note-Scope Zonen
|
||||
NOTE_SCOPE_ZONE_HEADERS = [
|
||||
"Smart Edges",
|
||||
"Relationen",
|
||||
"Global Links",
|
||||
"Note-Level Relations",
|
||||
"Globale Verbindungen"
|
||||
]
|
||||
|
||||
def extract_note_scope_zones(markdown_body: str) -> List[Tuple[str, str]]:
|
||||
"""
|
||||
WP-24c v4.2.0: Extrahiert Note-Scope Zonen aus Markdown.
|
||||
|
||||
Identifiziert Sektionen mit spezifischen Headern (z.B. "## Smart Edges")
|
||||
und extrahiert alle darin enthaltenen Links.
|
||||
|
||||
Returns:
|
||||
List[Tuple[str, str]]: Liste von (kind, target) Tupeln
|
||||
"""
|
||||
if not markdown_body:
|
||||
return []
|
||||
|
||||
edges: List[Tuple[str, str]] = []
|
||||
|
||||
# Regex für Header-Erkennung (## oder ###)
|
||||
header_pattern = r'^#{2,3}\s+(.+?)$'
|
||||
|
||||
lines = markdown_body.split('\n')
|
||||
in_zone = False
|
||||
zone_content = []
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
# Prüfe auf Header
|
||||
header_match = re.match(header_pattern, line.strip())
|
||||
if header_match:
|
||||
header_text = header_match.group(1).strip()
|
||||
|
||||
# Prüfe, ob dieser Header eine Note-Scope Zone ist
|
||||
is_zone_header = any(
|
||||
header_text.lower() == zone_header.lower()
|
||||
for zone_header in NOTE_SCOPE_ZONE_HEADERS
|
||||
)
|
||||
|
||||
if is_zone_header:
|
||||
in_zone = True
|
||||
zone_content = []
|
||||
continue
|
||||
else:
|
||||
# Neuer Header gefunden, der keine Zone ist -> Zone beendet
|
||||
if in_zone:
|
||||
# Verarbeite gesammelten Inhalt
|
||||
zone_text = '\n'.join(zone_content)
|
||||
# Extrahiere Typed Relations
|
||||
typed, _ = extract_typed_relations(zone_text)
|
||||
edges.extend(typed)
|
||||
# Extrahiere Wikilinks (als related_to)
|
||||
wikilinks = extract_wikilinks(zone_text)
|
||||
for wl in wikilinks:
|
||||
edges.append(("related_to", wl))
|
||||
# Extrahiere Callouts
|
||||
callouts, _ = extract_callout_relations(zone_text)
|
||||
edges.extend(callouts)
|
||||
in_zone = False
|
||||
zone_content = []
|
||||
|
||||
# Sammle Inhalt, wenn wir in einer Zone sind
|
||||
if in_zone:
|
||||
zone_content.append(line)
|
||||
|
||||
# Verarbeite letzte Zone (falls am Ende des Dokuments)
|
||||
if in_zone and zone_content:
|
||||
zone_text = '\n'.join(zone_content)
|
||||
typed, _ = extract_typed_relations(zone_text)
|
||||
edges.extend(typed)
|
||||
wikilinks = extract_wikilinks(zone_text)
|
||||
for wl in wikilinks:
|
||||
edges.append(("related_to", wl))
|
||||
callouts, _ = extract_callout_relations(zone_text)
|
||||
edges.extend(callouts)
|
||||
|
||||
return edges
|
||||
|
||||
def build_edges_for_note(
|
||||
note_id: str,
|
||||
chunks: List[dict],
|
||||
note_level_references: Optional[List[str]] = None,
|
||||
include_note_scope_refs: bool = False,
|
||||
markdown_body: Optional[str] = None,
|
||||
) -> List[dict]:
|
||||
"""
|
||||
Erzeugt und aggregiert alle Kanten für eine Note.
|
||||
Sorgt für die physische Trennung von Sektions-Links via Edge-ID.
|
||||
WP-24c v4.2.0: Unterstützt Note-Scope Extraktions-Zonen.
|
||||
|
||||
Args:
|
||||
note_id: ID der Note
|
||||
chunks: Liste von Chunk-Payloads
|
||||
note_level_references: Optionale Liste von Note-Level Referenzen
|
||||
include_note_scope_refs: Ob Note-Scope Referenzen eingeschlossen werden sollen
|
||||
markdown_body: Optionaler Original-Markdown-Text für Note-Scope Zonen-Extraktion
|
||||
"""
|
||||
edges: List[dict] = []
|
||||
# note_type für die Ermittlung der edge_defaults (types.yaml)
|
||||
note_type = _get(chunks[0], "type") if chunks else "concept"
|
||||
|
||||
# WP-24c v4.2.0: Note-Scope Zonen Extraktion (VOR Chunk-Verarbeitung)
|
||||
note_scope_edges: List[dict] = []
|
||||
if markdown_body:
|
||||
zone_links = extract_note_scope_zones(markdown_body)
|
||||
for kind, raw_target in zone_links:
|
||||
target, sec = parse_link_target(raw_target, note_id)
|
||||
if not target:
|
||||
continue
|
||||
|
||||
# WP-24c v4.2.0: Note-Scope Links mit scope: "note" und source_id: note_id
|
||||
# ID-Konsistenz: Exakt wie in Phase 2 (Symmetrie-Prüfung)
|
||||
payload = {
|
||||
"edge_id": _mk_edge_id(kind, note_id, target, "note", target_section=sec),
|
||||
"provenance": "explicit:note_zone",
|
||||
"rule_id": "explicit:note_zone",
|
||||
"confidence": PROVENANCE_PRIORITY.get("explicit:note_zone", 1.0)
|
||||
}
|
||||
if sec:
|
||||
payload["target_section"] = sec
|
||||
|
||||
note_scope_edges.append(_edge(
|
||||
kind=kind,
|
||||
scope="note",
|
||||
source_id=note_id, # WP-24c v4.2.0: source_id = note_id (nicht chunk_id)
|
||||
target_id=target,
|
||||
note_id=note_id,
|
||||
extra=payload
|
||||
))
|
||||
|
||||
# 1) Struktur-Kanten (Internal: belongs_to, next/prev)
|
||||
# Diese erhalten die Provenienz 'structure' und sind in der Registry geschützt.
|
||||
@@ -162,15 +288,45 @@ def build_edges_for_note(
|
||||
"provenance": "rule", "rule_id": "derived:backlink", "confidence": PROVENANCE_PRIORITY["derived:backlink"]
|
||||
}))
|
||||
|
||||
# 4) De-Duplizierung (In-Place)
|
||||
# 4) WP-24c v4.2.0: Note-Scope Edges hinzufügen (VOR De-Duplizierung)
|
||||
# Diese werden mit höherer Priorität behandelt, da sie explizite Note-Level Verbindungen sind
|
||||
edges.extend(note_scope_edges)
|
||||
|
||||
# 5) De-Duplizierung (In-Place) mit Priorisierung
|
||||
# WP-24c v4.2.0: Note-Scope Links haben Vorrang bei Duplikaten
|
||||
# WP-24c v4.1.0: Da die EDGE-ID nun auf 5 Parametern basiert (inkl. target_section),
|
||||
# bleiben Links auf unterschiedliche Abschnitte derselben Note als eigenständige
|
||||
# Kanten erhalten. Nur identische Sektions-Links werden nach Confidence konsolidiert.
|
||||
# Kanten erhalten. Nur identische Sektions-Links werden nach Confidence und Provenance konsolidiert.
|
||||
unique_map: Dict[str, dict] = {}
|
||||
for e in edges:
|
||||
eid = e["edge_id"]
|
||||
# Höhere Confidence gewinnt bei identischer ID
|
||||
if eid not in unique_map or e.get("confidence", 0) > unique_map[eid].get("confidence", 0):
|
||||
|
||||
# WP-24c v4.2.0: Priorisierung bei Duplikaten
|
||||
# 1. Note-Scope Links (explicit:note_zone) haben höchste Priorität
|
||||
# 2. Dann Confidence
|
||||
# 3. Dann Provenance-Priority
|
||||
if eid not in unique_map:
|
||||
unique_map[eid] = e
|
||||
else:
|
||||
existing = unique_map[eid]
|
||||
existing_prov = existing.get("provenance", "")
|
||||
new_prov = e.get("provenance", "")
|
||||
|
||||
# Note-Scope Zone Links haben Vorrang
|
||||
is_existing_note_zone = existing_prov == "explicit:note_zone"
|
||||
is_new_note_zone = new_prov == "explicit:note_zone"
|
||||
|
||||
if is_new_note_zone and not is_existing_note_zone:
|
||||
# Neuer Link ist Note-Scope Zone -> ersetze
|
||||
unique_map[eid] = e
|
||||
elif is_existing_note_zone and not is_new_note_zone:
|
||||
# Bestehender Link ist Note-Scope Zone -> behalte
|
||||
pass
|
||||
else:
|
||||
# Beide sind Note-Scope oder beide nicht -> vergleiche Confidence
|
||||
existing_conf = existing.get("confidence", 0)
|
||||
new_conf = e.get("confidence", 0)
|
||||
if new_conf > existing_conf:
|
||||
unique_map[eid] = e
|
||||
|
||||
return list(unique_map.values())
|
||||
@@ -4,7 +4,8 @@ DESCRIPTION: In-Memory Repräsentation eines Graphen für Scoring und Analyse.
|
||||
Zentrale Komponente für die Graph-Expansion (BFS) und Bonus-Berechnung.
|
||||
WP-15c Update: Erhalt von Metadaten (target_section, provenance)
|
||||
für präzises Retrieval-Reasoning.
|
||||
VERSION: 1.2.0
|
||||
WP-24c v4.1.0: Scope-Awareness und Section-Filtering Support.
|
||||
VERSION: 1.3.0 (WP-24c: Gold-Standard v4.1.0)
|
||||
STATUS: Active
|
||||
"""
|
||||
import math
|
||||
@@ -28,6 +29,8 @@ class Subgraph:
|
||||
self.reverse_adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||
self.in_degree: DefaultDict[str, int] = defaultdict(int)
|
||||
self.out_degree: DefaultDict[str, int] = defaultdict(int)
|
||||
# WP-24c v4.1.0: Chunk-Level In-Degree für präzise Scoring-Aggregation
|
||||
self.chunk_level_in_degree: DefaultDict[str, int] = defaultdict(int)
|
||||
|
||||
def add_edge(self, e: Dict) -> None:
|
||||
"""
|
||||
@@ -48,7 +51,9 @@ class Subgraph:
|
||||
"provenance": e.get("provenance", "rule"),
|
||||
"confidence": e.get("confidence", 1.0),
|
||||
"target_section": e.get("target_section"), # Essentiell für Präzision
|
||||
"is_super_edge": e.get("is_super_edge", False)
|
||||
"is_super_edge": e.get("is_super_edge", False),
|
||||
"virtual": e.get("virtual", False), # WP-24c v4.1.0: Für Authority-Priorisierung
|
||||
"chunk_id": e.get("chunk_id") # WP-24c v4.1.0: Für RAG-Kontext
|
||||
}
|
||||
|
||||
owner = e.get("note_id")
|
||||
@@ -111,10 +116,21 @@ def expand(
|
||||
seeds: List[str],
|
||||
depth: int = 1,
|
||||
edge_types: Optional[List[str]] = None,
|
||||
chunk_ids: Optional[List[str]] = None,
|
||||
target_section: Optional[str] = None,
|
||||
) -> Subgraph:
|
||||
"""
|
||||
Expandiert ab Seeds entlang von Edges bis zu einer bestimmten Tiefe.
|
||||
Nutzt fetch_edges_from_qdrant für den Datenbankzugriff.
|
||||
WP-24c v4.1.0: Unterstützt Scope-Awareness (chunk_ids) und Section-Filtering.
|
||||
|
||||
Args:
|
||||
client: Qdrant Client
|
||||
prefix: Collection-Präfix
|
||||
seeds: Liste von Note-IDs für die Expansion
|
||||
depth: Maximale Tiefe der Expansion
|
||||
edge_types: Optionale Filterung nach Kanten-Typen
|
||||
chunk_ids: Optionale Liste von Chunk-IDs für Scope-Awareness
|
||||
target_section: Optionales Section-Filtering
|
||||
"""
|
||||
sg = Subgraph()
|
||||
frontier = set(seeds)
|
||||
@@ -124,8 +140,13 @@ def expand(
|
||||
if not frontier:
|
||||
break
|
||||
|
||||
# Batch-Abfrage der Kanten für die aktuelle Ebene
|
||||
payloads = fetch_edges_from_qdrant(client, prefix, list(frontier), edge_types)
|
||||
# WP-24c v4.1.0: Erweiterte Edge-Retrieval mit Scope-Awareness und Section-Filtering
|
||||
payloads = fetch_edges_from_qdrant(
|
||||
client, prefix, list(frontier),
|
||||
edge_types=edge_types,
|
||||
chunk_ids=chunk_ids,
|
||||
target_section=target_section
|
||||
)
|
||||
next_frontier: Set[str] = set()
|
||||
|
||||
for pl in payloads:
|
||||
@@ -133,6 +154,7 @@ def expand(
|
||||
if not src or not tgt: continue
|
||||
|
||||
# WP-15c: Wir übergeben das vollständige Payload an add_edge
|
||||
# WP-24c v4.1.0: virtual Flag wird für Authority-Priorisierung benötigt
|
||||
edge_payload = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
@@ -141,7 +163,9 @@ def expand(
|
||||
"note_id": pl.get("note_id"),
|
||||
"provenance": pl.get("provenance", "rule"),
|
||||
"confidence": pl.get("confidence", 1.0),
|
||||
"target_section": pl.get("target_section")
|
||||
"target_section": pl.get("target_section"),
|
||||
"virtual": pl.get("virtual", False), # WP-24c v4.1.0: Für Authority-Priorisierung
|
||||
"chunk_id": pl.get("chunk_id") # WP-24c v4.1.0: Für RAG-Kontext
|
||||
}
|
||||
|
||||
sg.add_edge(edge_payload)
|
||||
|
||||
@@ -28,6 +28,7 @@ PROVENANCE_PRIORITY = {
|
||||
"structure:belongs_to": 1.00,
|
||||
"structure:order": 0.95, # next/prev
|
||||
"explicit:note_scope": 1.00,
|
||||
"explicit:note_zone": 1.00, # WP-24c v4.2.0: Note-Scope Zonen (höchste Priorität)
|
||||
"derived:backlink": 0.90,
|
||||
"edge_defaults": 0.70 # Heuristik basierend auf types.yaml
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user