Update graph_db_adapter.py, graph_derive_edges.py, graph_subgraph.py, graph_utils.py, ingestion_processor.py, and retriever.py to version 4.1.0: Introduce Scope-Awareness and Section-Filtering features, enhancing edge retrieval and processing. Implement Note-Scope Zones extraction from Markdown, improve edge ID generation with target_section, and prioritize Note-Scope Links during de-duplication. Update documentation for clarity and consistency across modules.
This commit is contained in:
1 parent
be2bed9927
commit
39fd15b565
14 files changed
+1004
-37
No files matched your search
+129
-16
@@ -2,7 +2,8 @@
|
||||
FILE: app/core/retrieval/retriever.py
|
||||
DESCRIPTION: Haupt-Schnittstelle für die Suche. Orchestriert Vektorsuche und Graph-Expansion.
|
||||
WP-15c Update: Note-Level Diversity Pooling & Super-Edge Aggregation.
|
||||
VERSION: 0.7.0
|
||||
WP-24c v4.1.0: Gold-Standard - Scope-Awareness, Section-Filtering, Authority-Priorisierung.
|
||||
VERSION: 0.8.0 (WP-24c: Gold-Standard v4.1.0)
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.config, app.models.dto, app.core.database*, app.core.graph_adapter
|
||||
"""
|
||||
@@ -26,6 +27,9 @@ import app.core.database.qdrant_points as qp
|
||||
|
||||
import app.services.embeddings_client as ec
|
||||
import app.core.graph.graph_subgraph as ga
|
||||
import app.core.graph.graph_db_adapter as gdb
|
||||
from app.core.graph.graph_utils import PROVENANCE_PRIORITY
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
# Mathematische Engine importieren
|
||||
from app.core.retrieval.retriever_scoring import get_weights, compute_wp22_score
|
||||
@@ -63,14 +67,64 @@ def _get_query_vector(req: QueryRequest) -> List[float]:
|
||||
return ec.embed_text(req.query)
|
||||
|
||||
|
||||
def _get_chunk_ids_for_notes(
|
||||
client: Any,
|
||||
prefix: str,
|
||||
note_ids: List[str]
|
||||
) -> List[str]:
|
||||
"""
|
||||
WP-24c v4.1.0: Lädt alle Chunk-IDs für gegebene Note-IDs.
|
||||
Wird für Scope-Aware Edge Retrieval benötigt.
|
||||
"""
|
||||
if not note_ids:
|
||||
return []
|
||||
|
||||
_, chunks_col, _ = qp._names(prefix)
|
||||
chunk_ids = []
|
||||
|
||||
try:
|
||||
# Filter: note_id IN note_ids
|
||||
note_filter = rest.Filter(should=[
|
||||
rest.FieldCondition(key="note_id", match=rest.MatchValue(value=str(nid)))
|
||||
for nid in note_ids
|
||||
])
|
||||
|
||||
pts, _ = client.scroll(
|
||||
collection_name=chunks_col,
|
||||
scroll_filter=note_filter,
|
||||
limit=2048,
|
||||
with_payload=True,
|
||||
with_vectors=False
|
||||
)
|
||||
|
||||
for pt in pts:
|
||||
pl = pt.payload or {}
|
||||
cid = pl.get("chunk_id")
|
||||
if cid:
|
||||
chunk_ids.append(str(cid))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load chunk IDs for notes: {e}")
|
||||
|
||||
return chunk_ids
|
||||
|
||||
def _semantic_hits(
|
||||
client: Any,
|
||||
prefix: str,
|
||||
vector: List[float],
|
||||
top_k: int,
|
||||
filters: Optional[Dict] = None
|
||||
filters: Optional[Dict] = None,
|
||||
target_section: Optional[str] = None
|
||||
) -> List[Tuple[str, float, Dict[str, Any]]]:
|
||||
"""Führt die Vektorsuche via database-Points-Modul durch."""
|
||||
"""
|
||||
Führt die Vektorsuche via database-Points-Modul durch.
|
||||
WP-24c v4.1.0: Unterstützt optionales Section-Filtering.
|
||||
"""
|
||||
# WP-24c v4.1.0: Section-Filtering für präzise Section-Links
|
||||
if target_section and filters:
|
||||
filters = {**filters, "section": target_section}
|
||||
elif target_section:
|
||||
filters = {"section": target_section}
|
||||
|
||||
raw_hits = qp.search_chunks_by_vector(client, prefix, vector, top=top_k, filters=filters)
|
||||
# Strikte Typkonvertierung für Stabilität
|
||||
return [(str(hit[0]), float(hit[1]), dict(hit[2] or {})) for hit in raw_hits]
|
||||
@@ -254,6 +308,16 @@ def _build_hits_from_semantic(
|
||||
|
||||
text_content = pl.get("page_content") or pl.get("text") or pl.get("content", "[Kein Text]")
|
||||
|
||||
# WP-24c v4.1.0: RAG-Kontext - source_chunk_id aus Edge-Payload extrahieren
|
||||
source_chunk_id = None
|
||||
if explanation_obj and explanation_obj.related_edges:
|
||||
# Finde die erste Edge mit chunk_id als source
|
||||
for edge in explanation_obj.related_edges:
|
||||
# Prüfe, ob source eine Chunk-ID ist (enthält # oder ist chunk_id)
|
||||
if edge.source and ("#" in edge.source or edge.source.startswith("chunk:")):
|
||||
source_chunk_id = edge.source
|
||||
break
|
||||
|
||||
results.append(QueryHit(
|
||||
node_id=str(pid),
|
||||
note_id=str(pl.get("note_id", "unknown")),
|
||||
@@ -267,7 +331,8 @@ def _build_hits_from_semantic(
|
||||
"text": text_content
|
||||
},
|
||||
payload=pl,
|
||||
explanation=explanation_obj
|
||||
explanation=explanation_obj,
|
||||
source_chunk_id=source_chunk_id # WP-24c v4.1.0: RAG-Kontext
|
||||
))
|
||||
|
||||
return QueryResponse(results=results, used_mode=used_mode, latency_ms=int((time.time() - t0) * 1000))
|
||||
@@ -283,7 +348,9 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
||||
top_k = req.top_k or 10
|
||||
|
||||
# 1. Semantische Seed-Suche (Wir laden etwas mehr für das Pooling)
|
||||
hits = _semantic_hits(client, prefix, vector, top_k=top_k * 3, filters=req.filters)
|
||||
# WP-24c v4.1.0: Section-Filtering unterstützen
|
||||
target_section = getattr(req, "target_section", None)
|
||||
hits = _semantic_hits(client, prefix, vector, top_k=top_k * 3, filters=req.filters, target_section=target_section)
|
||||
|
||||
# 2. Graph Expansion Konfiguration
|
||||
expand_cfg = req.expand if isinstance(req.expand, dict) else {}
|
||||
@@ -296,36 +363,71 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
||||
|
||||
if seed_ids:
|
||||
try:
|
||||
subgraph = ga.expand(client, prefix, seed_ids, depth=depth, edge_types=expand_cfg.get("edge_types"))
|
||||
# WP-24c v4.1.0: Scope-Awareness - Lade Chunk-IDs für Note-IDs
|
||||
chunk_ids = _get_chunk_ids_for_notes(client, prefix, seed_ids)
|
||||
|
||||
# --- WP-15c: Edge-Aggregation & Deduplizierung (Super-Kanten) ---
|
||||
# Erweiterte Edge-Retrieval mit Chunk-Scope und Section-Filtering
|
||||
subgraph = ga.expand(
|
||||
client, prefix, seed_ids,
|
||||
depth=depth,
|
||||
edge_types=expand_cfg.get("edge_types"),
|
||||
chunk_ids=chunk_ids,
|
||||
target_section=target_section
|
||||
)
|
||||
|
||||
# --- WP-24c v4.1.0: Chunk-Level Edge-Aggregation & Deduplizierung ---
|
||||
# Verhindert Score-Explosion durch multiple Links auf versch. Abschnitte.
|
||||
# Logik: 1. Kante zählt voll, weitere dämpfen auf Faktor 0.1.
|
||||
# Erweitert um Chunk-Level Tracking für präzise In-Degree-Berechnung.
|
||||
if subgraph and hasattr(subgraph, "adj"):
|
||||
# WP-24c v4.1.0: Chunk-Level In-Degree Tracking
|
||||
chunk_level_in_degree = defaultdict(int) # target -> count of chunk sources
|
||||
|
||||
for src, edge_list in subgraph.adj.items():
|
||||
# Gruppiere Kanten nach Ziel-Note (Deduplizierung ID_A -> ID_B)
|
||||
by_target = defaultdict(list)
|
||||
for e in edge_list:
|
||||
by_target[e["target"]].append(e)
|
||||
|
||||
# WP-24c v4.1.0: Chunk-Level In-Degree Tracking
|
||||
# Wenn source eine Chunk-ID ist, zähle für Chunk-Level In-Degree
|
||||
if e.get("chunk_id") or (src and ("#" in src or src.startswith("chunk:"))):
|
||||
chunk_level_in_degree[e["target"]] += 1
|
||||
|
||||
aggregated_list = []
|
||||
for tgt, edges in by_target.items():
|
||||
if len(edges) > 1:
|
||||
# Sortiere: Stärkste Kante zuerst
|
||||
sorted_edges = sorted(edges, key=lambda x: x.get("weight", 0.0), reverse=True)
|
||||
# Sortiere: Stärkste Kante zuerst (Authority-Priorisierung)
|
||||
sorted_edges = sorted(
|
||||
edges,
|
||||
key=lambda x: (
|
||||
x.get("weight", 0.0) *
|
||||
(1.0 if not x.get("virtual", False) else 0.5) * # Virtual-Penalty
|
||||
float(x.get("confidence", 1.0)) # Confidence-Boost
|
||||
),
|
||||
reverse=True
|
||||
)
|
||||
primary = sorted_edges[0]
|
||||
|
||||
# Aggregiertes Gewicht berechnen (Sättigungs-Logik)
|
||||
total_w = primary.get("weight", 0.0)
|
||||
chunk_count = 0
|
||||
for secondary in sorted_edges[1:]:
|
||||
total_w += secondary.get("weight", 0.0) * 0.1
|
||||
if secondary.get("chunk_id") or (secondary.get("source") and ("#" in secondary.get("source", "") or secondary.get("source", "").startswith("chunk:"))):
|
||||
chunk_count += 1
|
||||
|
||||
primary["weight"] = total_w
|
||||
primary["is_super_edge"] = True # Flag für Explanation Layer
|
||||
primary["edge_count"] = len(edges)
|
||||
primary["chunk_source_count"] = chunk_count + (1 if (primary.get("chunk_id") or (primary.get("source") and ("#" in primary.get("source", "") or primary.get("source", "").startswith("chunk:")))) else 0)
|
||||
aggregated_list.append(primary)
|
||||
else:
|
||||
aggregated_list.append(edges[0])
|
||||
edge = edges[0]
|
||||
# WP-24c v4.1.0: Chunk-Count auch für einzelne Edges
|
||||
if edge.get("chunk_id") or (edge.get("source") and ("#" in edge.get("source", "") or edge.get("source", "").startswith("chunk:"))):
|
||||
edge["chunk_source_count"] = 1
|
||||
aggregated_list.append(edge)
|
||||
|
||||
# In-Place Update der Adjazenzliste des Graphen
|
||||
subgraph.adj[src] = aggregated_list
|
||||
@@ -335,21 +437,32 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
||||
for src, edges in subgraph.adj.items():
|
||||
for e in edges:
|
||||
subgraph.in_degree[e["target"]] += 1
|
||||
|
||||
# WP-24c v4.1.0: Chunk-Level In-Degree als Attribut speichern
|
||||
subgraph.chunk_level_in_degree = chunk_level_in_degree
|
||||
|
||||
# --- WP-22: Kanten-Gewichtung (Provenance & Intent Boost) ---
|
||||
# --- WP-24c v4.1.0: Authority-Priorisierung (Provenance & Confidence) ---
|
||||
if subgraph and hasattr(subgraph, "adj"):
|
||||
for src, edges in subgraph.adj.items():
|
||||
for e in edges:
|
||||
# A. Provenance Weighting
|
||||
# A. Provenance Weighting (nutzt PROVENANCE_PRIORITY aus graph_utils)
|
||||
prov = e.get("provenance", "rule")
|
||||
prov_w = 1.0 if prov == "explicit" else (0.9 if prov == "smart" else 0.7)
|
||||
prov_key = f"{prov}:{e.get('kind', 'related_to')}" if ":" not in prov else prov
|
||||
prov_w = PROVENANCE_PRIORITY.get(prov_key, PROVENANCE_PRIORITY.get(prov, 0.7))
|
||||
|
||||
# B. Intent Boost Multiplikator
|
||||
# B. Confidence-Weighting (aus Edge-Payload)
|
||||
confidence = float(e.get("confidence", 1.0))
|
||||
|
||||
# C. Virtual-Flag De-Priorisierung
|
||||
is_virtual = e.get("virtual", False)
|
||||
virtual_penalty = 0.5 if is_virtual else 1.0
|
||||
|
||||
# D. Intent Boost Multiplikator
|
||||
kind = e.get("kind")
|
||||
intent_multiplier = boost_edges.get(kind, 1.0)
|
||||
|
||||
# Gewichtung anpassen
|
||||
e["weight"] = e.get("weight", 1.0) * prov_w * intent_multiplier
|
||||
# Gewichtung anpassen (Authority-Priorisierung)
|
||||
e["weight"] = e.get("weight", 1.0) * prov_w * confidence * virtual_penalty * intent_multiplier
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Graph Expansion failed: {e}")
|
||||
|
||||
Reference in new issue
Block a user