Update graph_db_adapter.py, graph_derive_edges.py, graph_subgraph.py, graph_utils.py, ingestion_processor.py, and retriever.py to version 4.1.0: Introduce Scope-Awareness and Section-Filtering features, enhancing edge retrieval and processing. Implement Note-Scope Zones extraction from Markdown, improve edge ID generation with target_section, and prioritize Note-Scope Links during de-duplication. Update documentation for clarity and consistency across modules.

This commit is contained in:
Lars committed 2026-01-10 19:55:51 +01:00
1 parent be2bed9927
commit 39fd15b565
14 files changed
+1004 -37

No files matched your search

+35 -5
View File
@@ -1,9 +1,11 @@
"""
FILE: app/core/graph/graph_db_adapter.py
DESCRIPTION: Datenbeschaffung aus Qdrant für den Graphen.
AUDIT v1.1.1: Volle Unterstützung für WP-15c Metadaten.
Stellt sicher, dass 'target_section' und 'provenance' für die
Super-Edge-Aggregation im Retriever geladen werden.
AUDIT v1.2.0: Gold-Standard v4.1.0 - Scope-Awareness & Section-Filtering.
- Erweiterte Suche nach chunk_id-Edges für Scope-Awareness
- Optionales target_section-Filtering für präzise Section-Links
- Vollständige Metadaten-Unterstützung (provenance, confidence, virtual)
VERSION: 1.2.0 (WP-24c: Gold-Standard v4.1.0)
"""
from typing import List, Dict, Optional
from qdrant_client import QdrantClient
@@ -17,11 +19,22 @@ def fetch_edges_from_qdrant(
prefix: str,
seeds: List[str],
edge_types: Optional[List[str]] = None,
target_section: Optional[str] = None,
chunk_ids: Optional[List[str]] = None,
limit: int = 2048,
) -> List[Dict]:
"""
Holt Edges aus der Datenbank basierend auf Seed-IDs.
WP-15c: Erhält alle Metadaten für das Note-Level Diversity Pooling.
WP-24c v4.1.0: Scope-Aware Edge Retrieval mit Section-Filtering.
Args:
client: Qdrant Client
prefix: Collection-Präfix
seeds: Liste von Note-IDs für die Suche
edge_types: Optionale Filterung nach Kanten-Typen
target_section: Optionales Section-Filtering (für präzise Section-Links)
chunk_ids: Optionale Liste von Chunk-IDs für Scope-Awareness (Chunk-Level Edges)
limit: Maximale Anzahl zurückgegebener Edges
"""
if not seeds or limit <= 0:
return []
@@ -30,13 +43,21 @@ def fetch_edges_from_qdrant(
# Rückgabe: (notes_col, chunks_col, edges_col)
_, _, edges_col = collection_names(prefix)
# Wir suchen Kanten, bei denen die Seed-IDs entweder Quelle, Ziel oder Kontext-Note sind.
# WP-24c v4.1.0: Scope-Awareness - Suche nach Note- UND Chunk-Level Edges
seed_conditions = []
for field in ("source_id", "target_id", "note_id"):
for s in seeds:
seed_conditions.append(
rest.FieldCondition(key=field, match=rest.MatchValue(value=str(s)))
)
# Chunk-Level Edges: Wenn chunk_ids angegeben, suche auch nach chunk_id als source_id
if chunk_ids:
for cid in chunk_ids:
seed_conditions.append(
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=str(cid)))
)
seeds_filter = rest.Filter(should=seed_conditions) if seed_conditions else None
# Optionaler Filter auf spezifische Kanten-Typen (z.B. für Intent-Routing)
@@ -48,11 +69,20 @@ def fetch_edges_from_qdrant(
]
type_filter = rest.Filter(should=type_conds)
# WP-24c v4.1.0: Section-Filtering für präzise Section-Links
section_filter = None
if target_section:
section_filter = rest.Filter(must=[
rest.FieldCondition(key="target_section", match=rest.MatchValue(value=str(target_section)))
])
must = []
if seeds_filter:
must.append(seeds_filter)
if type_filter:
must.append(type_filter)
if section_filter:
must.append(section_filter)
flt = rest.Filter(must=must) if must else None
+161 -5
View File
@@ -5,7 +5,14 @@ DESCRIPTION: Hauptlogik zur Kanten-Aggregation und De-Duplizierung.
- Präzises Sektions-Splitting via parse_link_target.
- v4.1.0: Eindeutige ID-Generierung pro Sektions-Variante (Multigraph).
- Ermöglicht dem Retriever die Super-Edge-Aggregation.
WP-24c v4.2.0: Note-Scope Extraktions-Zonen für globale Referenzen.
- Header-basierte Identifikation von Note-Scope Zonen
- Automatische Scope-Umschaltung (chunk -> note)
- Priorisierung: Note-Scope Links haben Vorrang bei Duplikaten
VERSION: 4.2.0 (WP-24c: Note-Scope Zones)
STATUS: Active
"""
import re
from typing import List, Optional, Dict, Tuple
from .graph_utils import (
_get, _edge, _mk_edge_id, _dedupe_seq, parse_link_target,
@@ -15,19 +22,138 @@ from .graph_extractors import (
extract_typed_relations, extract_callout_relations, extract_wikilinks
)
# WP-24c v4.2.0: Header-basierte Identifikation von Note-Scope Zonen
NOTE_SCOPE_ZONE_HEADERS = [
"Smart Edges",
"Relationen",
"Global Links",
"Note-Level Relations",
"Globale Verbindungen"
]
def extract_note_scope_zones(markdown_body: str) -> List[Tuple[str, str]]:
"""
WP-24c v4.2.0: Extrahiert Note-Scope Zonen aus Markdown.
Identifiziert Sektionen mit spezifischen Headern (z.B. "## Smart Edges")
und extrahiert alle darin enthaltenen Links.
Returns:
List[Tuple[str, str]]: Liste von (kind, target) Tupeln
"""
if not markdown_body:
return []
edges: List[Tuple[str, str]] = []
# Regex für Header-Erkennung (## oder ###)
header_pattern = r'^#{2,3}\s+(.+?)$'
lines = markdown_body.split('\n')
in_zone = False
zone_content = []
for i, line in enumerate(lines):
# Prüfe auf Header
header_match = re.match(header_pattern, line.strip())
if header_match:
header_text = header_match.group(1).strip()
# Prüfe, ob dieser Header eine Note-Scope Zone ist
is_zone_header = any(
header_text.lower() == zone_header.lower()
for zone_header in NOTE_SCOPE_ZONE_HEADERS
)
if is_zone_header:
in_zone = True
zone_content = []
continue
else:
# Neuer Header gefunden, der keine Zone ist -> Zone beendet
if in_zone:
# Verarbeite gesammelten Inhalt
zone_text = '\n'.join(zone_content)
# Extrahiere Typed Relations
typed, _ = extract_typed_relations(zone_text)
edges.extend(typed)
# Extrahiere Wikilinks (als related_to)
wikilinks = extract_wikilinks(zone_text)
for wl in wikilinks:
edges.append(("related_to", wl))
# Extrahiere Callouts
callouts, _ = extract_callout_relations(zone_text)
edges.extend(callouts)
in_zone = False
zone_content = []
# Sammle Inhalt, wenn wir in einer Zone sind
if in_zone:
zone_content.append(line)
# Verarbeite letzte Zone (falls am Ende des Dokuments)
if in_zone and zone_content:
zone_text = '\n'.join(zone_content)
typed, _ = extract_typed_relations(zone_text)
edges.extend(typed)
wikilinks = extract_wikilinks(zone_text)
for wl in wikilinks:
edges.append(("related_to", wl))
callouts, _ = extract_callout_relations(zone_text)
edges.extend(callouts)
return edges
def build_edges_for_note(
note_id: str,
chunks: List[dict],
note_level_references: Optional[List[str]] = None,
include_note_scope_refs: bool = False,
markdown_body: Optional[str] = None,
) -> List[dict]:
"""
Erzeugt und aggregiert alle Kanten für eine Note.
Sorgt für die physische Trennung von Sektions-Links via Edge-ID.
WP-24c v4.2.0: Unterstützt Note-Scope Extraktions-Zonen.
Args:
note_id: ID der Note
chunks: Liste von Chunk-Payloads
note_level_references: Optionale Liste von Note-Level Referenzen
include_note_scope_refs: Ob Note-Scope Referenzen eingeschlossen werden sollen
markdown_body: Optionaler Original-Markdown-Text für Note-Scope Zonen-Extraktion
"""
edges: List[dict] = []
# note_type für die Ermittlung der edge_defaults (types.yaml)
note_type = _get(chunks[0], "type") if chunks else "concept"
# WP-24c v4.2.0: Note-Scope Zonen Extraktion (VOR Chunk-Verarbeitung)
note_scope_edges: List[dict] = []
if markdown_body:
zone_links = extract_note_scope_zones(markdown_body)
for kind, raw_target in zone_links:
target, sec = parse_link_target(raw_target, note_id)
if not target:
continue
# WP-24c v4.2.0: Note-Scope Links mit scope: "note" und source_id: note_id
# ID-Konsistenz: Exakt wie in Phase 2 (Symmetrie-Prüfung)
payload = {
"edge_id": _mk_edge_id(kind, note_id, target, "note", target_section=sec),
"provenance": "explicit:note_zone",
"rule_id": "explicit:note_zone",
"confidence": PROVENANCE_PRIORITY.get("explicit:note_zone", 1.0)
}
if sec:
payload["target_section"] = sec
note_scope_edges.append(_edge(
kind=kind,
scope="note",
source_id=note_id, # WP-24c v4.2.0: source_id = note_id (nicht chunk_id)
target_id=target,
note_id=note_id,
extra=payload
))
# 1) Struktur-Kanten (Internal: belongs_to, next/prev)
# Diese erhalten die Provenienz 'structure' und sind in der Registry geschützt.
@@ -162,15 +288,45 @@ def build_edges_for_note(
"provenance": "rule", "rule_id": "derived:backlink", "confidence": PROVENANCE_PRIORITY["derived:backlink"]
}))
# 4) De-Duplizierung (In-Place)
# 4) WP-24c v4.2.0: Note-Scope Edges hinzufügen (VOR De-Duplizierung)
# Diese werden mit höherer Priorität behandelt, da sie explizite Note-Level Verbindungen sind
edges.extend(note_scope_edges)
# 5) De-Duplizierung (In-Place) mit Priorisierung
# WP-24c v4.2.0: Note-Scope Links haben Vorrang bei Duplikaten
# WP-24c v4.1.0: Da die EDGE-ID nun auf 5 Parametern basiert (inkl. target_section),
# bleiben Links auf unterschiedliche Abschnitte derselben Note als eigenständige
# Kanten erhalten. Nur identische Sektions-Links werden nach Confidence konsolidiert.
# Kanten erhalten. Nur identische Sektions-Links werden nach Confidence und Provenance konsolidiert.
unique_map: Dict[str, dict] = {}
for e in edges:
eid = e["edge_id"]
# Höhere Confidence gewinnt bei identischer ID
if eid not in unique_map or e.get("confidence", 0) > unique_map[eid].get("confidence", 0):
# WP-24c v4.2.0: Priorisierung bei Duplikaten
# 1. Note-Scope Links (explicit:note_zone) haben höchste Priorität
# 2. Dann Confidence
# 3. Dann Provenance-Priority
if eid not in unique_map:
unique_map[eid] = e
else:
existing = unique_map[eid]
existing_prov = existing.get("provenance", "")
new_prov = e.get("provenance", "")
# Note-Scope Zone Links haben Vorrang
is_existing_note_zone = existing_prov == "explicit:note_zone"
is_new_note_zone = new_prov == "explicit:note_zone"
if is_new_note_zone and not is_existing_note_zone:
# Neuer Link ist Note-Scope Zone -> ersetze
unique_map[eid] = e
elif is_existing_note_zone and not is_new_note_zone:
# Bestehender Link ist Note-Scope Zone -> behalte
pass
else:
# Beide sind Note-Scope oder beide nicht -> vergleiche Confidence
existing_conf = existing.get("confidence", 0)
new_conf = e.get("confidence", 0)
if new_conf > existing_conf:
unique_map[eid] = e
return list(unique_map.values())
+30 -6
View File
@@ -4,7 +4,8 @@ DESCRIPTION: In-Memory Repräsentation eines Graphen für Scoring und Analyse.
Zentrale Komponente für die Graph-Expansion (BFS) und Bonus-Berechnung.
WP-15c Update: Erhalt von Metadaten (target_section, provenance)
für präzises Retrieval-Reasoning.
VERSION: 1.2.0
WP-24c v4.1.0: Scope-Awareness und Section-Filtering Support.
VERSION: 1.3.0 (WP-24c: Gold-Standard v4.1.0)
STATUS: Active
"""
import math
@@ -28,6 +29,8 @@ class Subgraph:
self.reverse_adj: DefaultDict[str, List[Dict]] = defaultdict(list)
self.in_degree: DefaultDict[str, int] = defaultdict(int)
self.out_degree: DefaultDict[str, int] = defaultdict(int)
# WP-24c v4.1.0: Chunk-Level In-Degree für präzise Scoring-Aggregation
self.chunk_level_in_degree: DefaultDict[str, int] = defaultdict(int)
def add_edge(self, e: Dict) -> None:
"""
@@ -48,7 +51,9 @@ class Subgraph:
"provenance": e.get("provenance", "rule"),
"confidence": e.get("confidence", 1.0),
"target_section": e.get("target_section"), # Essentiell für Präzision
"is_super_edge": e.get("is_super_edge", False)
"is_super_edge": e.get("is_super_edge", False),
"virtual": e.get("virtual", False), # WP-24c v4.1.0: Für Authority-Priorisierung
"chunk_id": e.get("chunk_id") # WP-24c v4.1.0: Für RAG-Kontext
}
owner = e.get("note_id")
@@ -111,10 +116,21 @@ def expand(
seeds: List[str],
depth: int = 1,
edge_types: Optional[List[str]] = None,
chunk_ids: Optional[List[str]] = None,
target_section: Optional[str] = None,
) -> Subgraph:
"""
Expandiert ab Seeds entlang von Edges bis zu einer bestimmten Tiefe.
Nutzt fetch_edges_from_qdrant für den Datenbankzugriff.
WP-24c v4.1.0: Unterstützt Scope-Awareness (chunk_ids) und Section-Filtering.
Args:
client: Qdrant Client
prefix: Collection-Präfix
seeds: Liste von Note-IDs für die Expansion
depth: Maximale Tiefe der Expansion
edge_types: Optionale Filterung nach Kanten-Typen
chunk_ids: Optionale Liste von Chunk-IDs für Scope-Awareness
target_section: Optionales Section-Filtering
"""
sg = Subgraph()
frontier = set(seeds)
@@ -124,8 +140,13 @@ def expand(
if not frontier:
break
# Batch-Abfrage der Kanten für die aktuelle Ebene
payloads = fetch_edges_from_qdrant(client, prefix, list(frontier), edge_types)
# WP-24c v4.1.0: Erweiterte Edge-Retrieval mit Scope-Awareness und Section-Filtering
payloads = fetch_edges_from_qdrant(
client, prefix, list(frontier),
edge_types=edge_types,
chunk_ids=chunk_ids,
target_section=target_section
)
next_frontier: Set[str] = set()
for pl in payloads:
@@ -133,6 +154,7 @@ def expand(
if not src or not tgt: continue
# WP-15c: Wir übergeben das vollständige Payload an add_edge
# WP-24c v4.1.0: virtual Flag wird für Authority-Priorisierung benötigt
edge_payload = {
"source": src,
"target": tgt,
@@ -141,7 +163,9 @@ def expand(
"note_id": pl.get("note_id"),
"provenance": pl.get("provenance", "rule"),
"confidence": pl.get("confidence", 1.0),
"target_section": pl.get("target_section")
"target_section": pl.get("target_section"),
"virtual": pl.get("virtual", False), # WP-24c v4.1.0: Für Authority-Priorisierung
"chunk_id": pl.get("chunk_id") # WP-24c v4.1.0: Für RAG-Kontext
}
sg.add_edge(edge_payload)
+1
View File
@@ -28,6 +28,7 @@ PROVENANCE_PRIORITY = {
"structure:belongs_to": 1.00,
"structure:order": 0.95, # next/prev
"explicit:note_scope": 1.00,
"explicit:note_zone": 1.00, # WP-24c v4.2.0: Note-Scope Zonen (höchste Priorität)
"derived:backlink": 0.90,
"edge_defaults": 0.70 # Heuristik basierend auf types.yaml
}
+9 -2
View File
@@ -244,8 +244,15 @@ class IngestionService:
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, file_path=file_path, types_cfg=self.registry)
vecs = await self.embedder.embed_documents([c.get("window") or "" for c in chunk_pls]) if chunk_pls else []
# Kanten-Extraktion
raw_edges = build_edges_for_note(note_id, chunk_pls, note_level_references=note_pl.get("references", []))
# WP-24c v4.2.0: Kanten-Extraktion mit Note-Scope Zonen Support
# Übergabe des Original-Markdown-Texts für Note-Scope Zonen-Extraktion
markdown_body = getattr(parsed, "body", "")
raw_edges = build_edges_for_note(
note_id,
chunk_pls,
note_level_references=note_pl.get("references", []),
markdown_body=markdown_body
)
explicit_edges = []
for e in raw_edges:
+129 -16
View File
@@ -2,7 +2,8 @@
FILE: app/core/retrieval/retriever.py
DESCRIPTION: Haupt-Schnittstelle für die Suche. Orchestriert Vektorsuche und Graph-Expansion.
WP-15c Update: Note-Level Diversity Pooling & Super-Edge Aggregation.
VERSION: 0.7.0
WP-24c v4.1.0: Gold-Standard - Scope-Awareness, Section-Filtering, Authority-Priorisierung.
VERSION: 0.8.0 (WP-24c: Gold-Standard v4.1.0)
STATUS: Active
DEPENDENCIES: app.config, app.models.dto, app.core.database*, app.core.graph_adapter
"""
@@ -26,6 +27,9 @@ import app.core.database.qdrant_points as qp
import app.services.embeddings_client as ec
import app.core.graph.graph_subgraph as ga
import app.core.graph.graph_db_adapter as gdb
from app.core.graph.graph_utils import PROVENANCE_PRIORITY
from qdrant_client.http import models as rest
# Mathematische Engine importieren
from app.core.retrieval.retriever_scoring import get_weights, compute_wp22_score
@@ -63,14 +67,64 @@ def _get_query_vector(req: QueryRequest) -> List[float]:
return ec.embed_text(req.query)
def _get_chunk_ids_for_notes(
client: Any,
prefix: str,
note_ids: List[str]
) -> List[str]:
"""
WP-24c v4.1.0: Lädt alle Chunk-IDs für gegebene Note-IDs.
Wird für Scope-Aware Edge Retrieval benötigt.
"""
if not note_ids:
return []
_, chunks_col, _ = qp._names(prefix)
chunk_ids = []
try:
# Filter: note_id IN note_ids
note_filter = rest.Filter(should=[
rest.FieldCondition(key="note_id", match=rest.MatchValue(value=str(nid)))
for nid in note_ids
])
pts, _ = client.scroll(
collection_name=chunks_col,
scroll_filter=note_filter,
limit=2048,
with_payload=True,
with_vectors=False
)
for pt in pts:
pl = pt.payload or {}
cid = pl.get("chunk_id")
if cid:
chunk_ids.append(str(cid))
except Exception as e:
logger.warning(f"Failed to load chunk IDs for notes: {e}")
return chunk_ids
def _semantic_hits(
client: Any,
prefix: str,
vector: List[float],
top_k: int,
filters: Optional[Dict] = None
filters: Optional[Dict] = None,
target_section: Optional[str] = None
) -> List[Tuple[str, float, Dict[str, Any]]]:
"""Führt die Vektorsuche via database-Points-Modul durch."""
"""
Führt die Vektorsuche via database-Points-Modul durch.
WP-24c v4.1.0: Unterstützt optionales Section-Filtering.
"""
# WP-24c v4.1.0: Section-Filtering für präzise Section-Links
if target_section and filters:
filters = {**filters, "section": target_section}
elif target_section:
filters = {"section": target_section}
raw_hits = qp.search_chunks_by_vector(client, prefix, vector, top=top_k, filters=filters)
# Strikte Typkonvertierung für Stabilität
return [(str(hit[0]), float(hit[1]), dict(hit[2] or {})) for hit in raw_hits]
@@ -254,6 +308,16 @@ def _build_hits_from_semantic(
text_content = pl.get("page_content") or pl.get("text") or pl.get("content", "[Kein Text]")
# WP-24c v4.1.0: RAG-Kontext - source_chunk_id aus Edge-Payload extrahieren
source_chunk_id = None
if explanation_obj and explanation_obj.related_edges:
# Finde die erste Edge mit chunk_id als source
for edge in explanation_obj.related_edges:
# Prüfe, ob source eine Chunk-ID ist (enthält # oder ist chunk_id)
if edge.source and ("#" in edge.source or edge.source.startswith("chunk:")):
source_chunk_id = edge.source
break
results.append(QueryHit(
node_id=str(pid),
note_id=str(pl.get("note_id", "unknown")),
@@ -267,7 +331,8 @@ def _build_hits_from_semantic(
"text": text_content
},
payload=pl,
explanation=explanation_obj
explanation=explanation_obj,
source_chunk_id=source_chunk_id # WP-24c v4.1.0: RAG-Kontext
))
return QueryResponse(results=results, used_mode=used_mode, latency_ms=int((time.time() - t0) * 1000))
@@ -283,7 +348,9 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
top_k = req.top_k or 10
# 1. Semantische Seed-Suche (Wir laden etwas mehr für das Pooling)
hits = _semantic_hits(client, prefix, vector, top_k=top_k * 3, filters=req.filters)
# WP-24c v4.1.0: Section-Filtering unterstützen
target_section = getattr(req, "target_section", None)
hits = _semantic_hits(client, prefix, vector, top_k=top_k * 3, filters=req.filters, target_section=target_section)
# 2. Graph Expansion Konfiguration
expand_cfg = req.expand if isinstance(req.expand, dict) else {}
@@ -296,36 +363,71 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
if seed_ids:
try:
subgraph = ga.expand(client, prefix, seed_ids, depth=depth, edge_types=expand_cfg.get("edge_types"))
# WP-24c v4.1.0: Scope-Awareness - Lade Chunk-IDs für Note-IDs
chunk_ids = _get_chunk_ids_for_notes(client, prefix, seed_ids)
# --- WP-15c: Edge-Aggregation & Deduplizierung (Super-Kanten) ---
# Erweiterte Edge-Retrieval mit Chunk-Scope und Section-Filtering
subgraph = ga.expand(
client, prefix, seed_ids,
depth=depth,
edge_types=expand_cfg.get("edge_types"),
chunk_ids=chunk_ids,
target_section=target_section
)
# --- WP-24c v4.1.0: Chunk-Level Edge-Aggregation & Deduplizierung ---
# Verhindert Score-Explosion durch multiple Links auf versch. Abschnitte.
# Logik: 1. Kante zählt voll, weitere dämpfen auf Faktor 0.1.
# Erweitert um Chunk-Level Tracking für präzise In-Degree-Berechnung.
if subgraph and hasattr(subgraph, "adj"):
# WP-24c v4.1.0: Chunk-Level In-Degree Tracking
chunk_level_in_degree = defaultdict(int) # target -> count of chunk sources
for src, edge_list in subgraph.adj.items():
# Gruppiere Kanten nach Ziel-Note (Deduplizierung ID_A -> ID_B)
by_target = defaultdict(list)
for e in edge_list:
by_target[e["target"]].append(e)
# WP-24c v4.1.0: Chunk-Level In-Degree Tracking
# Wenn source eine Chunk-ID ist, zähle für Chunk-Level In-Degree
if e.get("chunk_id") or (src and ("#" in src or src.startswith("chunk:"))):
chunk_level_in_degree[e["target"]] += 1
aggregated_list = []
for tgt, edges in by_target.items():
if len(edges) > 1:
# Sortiere: Stärkste Kante zuerst
sorted_edges = sorted(edges, key=lambda x: x.get("weight", 0.0), reverse=True)
# Sortiere: Stärkste Kante zuerst (Authority-Priorisierung)
sorted_edges = sorted(
edges,
key=lambda x: (
x.get("weight", 0.0) *
(1.0 if not x.get("virtual", False) else 0.5) * # Virtual-Penalty
float(x.get("confidence", 1.0)) # Confidence-Boost
),
reverse=True
)
primary = sorted_edges[0]
# Aggregiertes Gewicht berechnen (Sättigungs-Logik)
total_w = primary.get("weight", 0.0)
chunk_count = 0
for secondary in sorted_edges[1:]:
total_w += secondary.get("weight", 0.0) * 0.1
if secondary.get("chunk_id") or (secondary.get("source") and ("#" in secondary.get("source", "") or secondary.get("source", "").startswith("chunk:"))):
chunk_count += 1
primary["weight"] = total_w
primary["is_super_edge"] = True # Flag für Explanation Layer
primary["edge_count"] = len(edges)
primary["chunk_source_count"] = chunk_count + (1 if (primary.get("chunk_id") or (primary.get("source") and ("#" in primary.get("source", "") or primary.get("source", "").startswith("chunk:")))) else 0)
aggregated_list.append(primary)
else:
aggregated_list.append(edges[0])
edge = edges[0]
# WP-24c v4.1.0: Chunk-Count auch für einzelne Edges
if edge.get("chunk_id") or (edge.get("source") and ("#" in edge.get("source", "") or edge.get("source", "").startswith("chunk:"))):
edge["chunk_source_count"] = 1
aggregated_list.append(edge)
# In-Place Update der Adjazenzliste des Graphen
subgraph.adj[src] = aggregated_list
@@ -335,21 +437,32 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
for src, edges in subgraph.adj.items():
for e in edges:
subgraph.in_degree[e["target"]] += 1
# WP-24c v4.1.0: Chunk-Level In-Degree als Attribut speichern
subgraph.chunk_level_in_degree = chunk_level_in_degree
# --- WP-22: Kanten-Gewichtung (Provenance & Intent Boost) ---
# --- WP-24c v4.1.0: Authority-Priorisierung (Provenance & Confidence) ---
if subgraph and hasattr(subgraph, "adj"):
for src, edges in subgraph.adj.items():
for e in edges:
# A. Provenance Weighting
# A. Provenance Weighting (nutzt PROVENANCE_PRIORITY aus graph_utils)
prov = e.get("provenance", "rule")
prov_w = 1.0 if prov == "explicit" else (0.9 if prov == "smart" else 0.7)
prov_key = f"{prov}:{e.get('kind', 'related_to')}" if ":" not in prov else prov
prov_w = PROVENANCE_PRIORITY.get(prov_key, PROVENANCE_PRIORITY.get(prov, 0.7))
# B. Intent Boost Multiplikator
# B. Confidence-Weighting (aus Edge-Payload)
confidence = float(e.get("confidence", 1.0))
# C. Virtual-Flag De-Priorisierung
is_virtual = e.get("virtual", False)
virtual_penalty = 0.5 if is_virtual else 1.0
# D. Intent Boost Multiplikator
kind = e.get("kind")
intent_multiplier = boost_edges.get(kind, 1.0)
# Gewichtung anpassen
e["weight"] = e.get("weight", 1.0) * prov_w * intent_multiplier
# Gewichtung anpassen (Authority-Priorisierung)
e["weight"] = e.get("weight", 1.0) * prov_w * confidence * virtual_penalty * intent_multiplier
except Exception as e:
logger.error(f"Graph Expansion failed: {e}")
+7 -1
View File
@@ -56,6 +56,7 @@ class EdgeDTO(BaseModel):
class QueryRequest(BaseModel):
"""
Request für /query. Unterstützt Multi-Stream Isolation via filters.
WP-24c v4.1.0: Erweitert um Section-Filtering und Scope-Awareness.
"""
mode: Literal["semantic", "edge", "hybrid"] = "hybrid"
query: Optional[str] = None
@@ -67,7 +68,10 @@ class QueryRequest(BaseModel):
explain: bool = False
# WP-22/25: Dynamische Gewichtung der Graphen-Highways
boost_edges: Optional[Dict[str, float]] = None
boost_edges: Optional[Dict[str, float]] = None
# WP-24c v4.1.0: Section-Filtering für präzise Section-Links
target_section: Optional[str] = None
class FeedbackRequest(BaseModel):
@@ -125,6 +129,7 @@ class QueryHit(BaseModel):
"""
Einzelnes Trefferobjekt.
WP-25: stream_origin hinzugefügt für Tracing und Feedback-Optimierung.
WP-24c v4.1.0: source_chunk_id für RAG-Kontext hinzugefügt.
"""
node_id: str
note_id: str
@@ -137,6 +142,7 @@ class QueryHit(BaseModel):
payload: Optional[Dict] = None
explanation: Optional[Explanation] = None
stream_origin: Optional[str] = Field(None, description="Name des Ursprungs-Streams")
source_chunk_id: Optional[str] = Field(None, description="Chunk-ID der Quelle (für RAG-Kontext)")
class QueryResponse(BaseModel):