Implement WP-26 v1.0 - Phase 2: Enhance edge scoring and aggregation configuration
- Introduced configurable edge scoring with internal and external boosts for intra-note edges. - Added aggregation configuration to support note-level and chunk-level retrieval strategies. - Updated retriever and graph subgraph modules to utilize new scoring and aggregation logic. - Enhanced YAML configuration to include new parameters for edge scoring and aggregation levels. - Added boolean indexing for filtering based on edge properties in the setup script.
This commit is contained in:
1 parent
e86e9f2313
commit
c5215e22e7
5 files changed
+384
-19
No files matched your search
@@ -5,18 +5,56 @@ DESCRIPTION: In-Memory Repräsentation eines Graphen für Scoring und Analyse.
|
||||
WP-15c Update: Erhalt von Metadaten (target_section, provenance)
|
||||
für präzises Retrieval-Reasoning.
|
||||
WP-24c v4.1.0: Scope-Awareness und Section-Filtering Support.
|
||||
VERSION: 1.3.0 (WP-24c: Gold-Standard v4.1.0)
|
||||
WP-26 v1.0: is_internal-Boost für Intra-Note-Edges.
|
||||
VERSION: 1.4.0 (WP-26: Intra-Note-Edge-Boost)
|
||||
STATUS: Active
|
||||
"""
|
||||
import os
|
||||
import math
|
||||
from functools import lru_cache
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Optional, DefaultDict, Any, Set
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
# Lokale Paket-Imports
|
||||
from .graph_weights import EDGE_BASE_WEIGHTS, calculate_edge_weight
|
||||
from .graph_db_adapter import fetch_edges_from_qdrant
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_edge_scoring_config() -> Dict[str, float]:
|
||||
"""
|
||||
WP-26 v1.0: Lädt Edge-Scoring-Konfiguration aus retriever.yaml.
|
||||
|
||||
Returns:
|
||||
Dict mit internal_edge_boost und external_edge_boost
|
||||
"""
|
||||
defaults = {
|
||||
"internal_edge_boost": 1.2, # +20% Boost für Intra-Note-Edges
|
||||
"external_edge_boost": 1.0 # Standard für Inter-Note-Edges
|
||||
}
|
||||
|
||||
config_path = os.getenv("MINDNET_RETRIEVER_CONFIG", "config/retriever.yaml")
|
||||
if yaml and os.path.exists(config_path):
|
||||
try:
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
edge_scoring = data.get("edge_scoring", {})
|
||||
defaults["internal_edge_boost"] = float(edge_scoring.get("internal_edge_boost", defaults["internal_edge_boost"]))
|
||||
defaults["external_edge_boost"] = float(edge_scoring.get("external_edge_boost", defaults["external_edge_boost"]))
|
||||
except Exception as e:
|
||||
logger.warning(f"Edge-Scoring-Konfiguration konnte nicht geladen werden: {e}")
|
||||
|
||||
return defaults
|
||||
|
||||
class Subgraph:
|
||||
"""
|
||||
Leichtgewichtiger Subgraph mit Adjazenzlisten & Kennzahlen.
|
||||
@@ -36,24 +74,39 @@ class Subgraph:
|
||||
"""
|
||||
Fügt eine Kante hinzu und aktualisiert Indizes.
|
||||
WP-15c: Speichert das vollständige Payload für den Explanation Layer.
|
||||
WP-26 v1.0: Wendet is_internal-Boost auf Intra-Note-Edges an.
|
||||
"""
|
||||
src = e.get("source")
|
||||
tgt = e.get("target")
|
||||
kind = e.get("kind")
|
||||
|
||||
# Basis-Gewicht aus Payload oder Edge-Weights
|
||||
base_weight = e.get("weight", EDGE_BASE_WEIGHTS.get(kind, 0.0))
|
||||
|
||||
# WP-26 v1.0: is_internal-Boost anwenden
|
||||
is_internal = e.get("is_internal", False)
|
||||
edge_scoring = get_edge_scoring_config()
|
||||
if is_internal:
|
||||
weight_multiplier = edge_scoring["internal_edge_boost"]
|
||||
else:
|
||||
weight_multiplier = edge_scoring["external_edge_boost"]
|
||||
|
||||
final_weight = base_weight * weight_multiplier
|
||||
|
||||
# Das gesamte Payload wird als Kanten-Objekt behalten
|
||||
# Wir stellen sicher, dass alle relevanten Metadaten vorhanden sind
|
||||
edge_data = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"kind": kind,
|
||||
"weight": e.get("weight", EDGE_BASE_WEIGHTS.get(kind, 0.0)),
|
||||
"weight": final_weight,
|
||||
"provenance": e.get("provenance", "rule"),
|
||||
"confidence": e.get("confidence", 1.0),
|
||||
"target_section": e.get("target_section"), # Essentiell für Präzision
|
||||
"is_super_edge": e.get("is_super_edge", False),
|
||||
"virtual": e.get("virtual", False), # WP-24c v4.1.0: Für Authority-Priorisierung
|
||||
"chunk_id": e.get("chunk_id") # WP-24c v4.1.0: Für RAG-Kontext
|
||||
"chunk_id": e.get("chunk_id"), # WP-24c v4.1.0: Für RAG-Kontext
|
||||
"is_internal": is_internal # WP-26 v1.0: Flag für Debugging
|
||||
}
|
||||
|
||||
owner = e.get("note_id")
|
||||
|
||||
@@ -3,7 +3,8 @@ FILE: app/core/retrieval/retriever.py
|
||||
DESCRIPTION: Haupt-Schnittstelle für die Suche. Orchestriert Vektorsuche und Graph-Expansion.
|
||||
WP-15c Update: Note-Level Diversity Pooling & Super-Edge Aggregation.
|
||||
WP-24c v4.1.0: Gold-Standard - Scope-Awareness, Section-Filtering, Authority-Priorisierung.
|
||||
VERSION: 0.8.0 (WP-24c: Gold-Standard v4.1.0)
|
||||
WP-26 v1.0: Konfigurierbare Aggregation (note/chunk Level).
|
||||
VERSION: 0.9.0 (WP-26: Aggregation-Level)
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.config, app.models.dto, app.core.database*, app.core.graph_adapter
|
||||
"""
|
||||
@@ -34,8 +35,39 @@ from qdrant_client.http import models as rest
|
||||
# Mathematische Engine importieren
|
||||
from app.core.retrieval.retriever_scoring import get_weights, compute_wp22_score
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_aggregation_config() -> Dict[str, Any]:
|
||||
"""
|
||||
WP-26 v1.0: Lädt Aggregation-Konfiguration aus retriever.yaml.
|
||||
|
||||
Returns:
|
||||
Dict mit level ("note" oder "chunk") und max_chunks_per_note
|
||||
"""
|
||||
defaults = {
|
||||
"level": "note", # "note" (Default) oder "chunk"
|
||||
"max_chunks_per_note": 3 # Limit bei "note"-Level
|
||||
}
|
||||
|
||||
config_path = os.getenv("MINDNET_RETRIEVER_CONFIG", "config/retriever.yaml")
|
||||
if yaml and os.path.exists(config_path):
|
||||
try:
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
aggregation = data.get("aggregation", {})
|
||||
defaults["level"] = str(aggregation.get("level", defaults["level"])).lower()
|
||||
defaults["max_chunks_per_note"] = int(aggregation.get("max_chunks_per_note", defaults["max_chunks_per_note"]))
|
||||
except Exception as e:
|
||||
logger.warning(f"Aggregation-Konfiguration konnte nicht geladen werden: {e}")
|
||||
|
||||
return defaults
|
||||
|
||||
# ==============================================================================
|
||||
# 1. CORE HELPERS & CONFIG LOADERS
|
||||
# ==============================================================================
|
||||
@@ -316,22 +348,33 @@ def _build_hits_from_semantic(
|
||||
# 1. Sortierung nach finalem mathematischen Score
|
||||
enriched_sorted = sorted(enriched, key=lambda h: h[3]["total"], reverse=True)
|
||||
|
||||
# 2. WP-15c: Note-Level Diversity Pooling
|
||||
# Wir behalten pro note_id nur den Hit mit dem höchsten total_score.
|
||||
# Dies verhindert, dass 10 Chunks derselben Note andere KeyNotes verdrängen.
|
||||
unique_note_hits = []
|
||||
seen_notes = set()
|
||||
# 2. WP-26 v1.0: Konfigurierbare Aggregation (Note-Level oder Chunk-Level)
|
||||
aggregation_config = _get_aggregation_config()
|
||||
aggregation_level = aggregation_config["level"]
|
||||
max_chunks_per_note = aggregation_config["max_chunks_per_note"]
|
||||
|
||||
for item in enriched_sorted:
|
||||
_, _, payload, _ = item
|
||||
note_id = str(payload.get("note_id", "unknown"))
|
||||
if aggregation_level == "chunk":
|
||||
# WP-26 v1.0: Chunk-Level - alle Chunks individuell ranken (keine Deduplizierung)
|
||||
logger.debug(f"📊 [AGGREGATION] Chunk-Level: Alle {len(enriched_sorted)} Chunks individuell")
|
||||
pooled_hits = enriched_sorted
|
||||
else:
|
||||
# WP-15c: Note-Level Diversity Pooling (Default)
|
||||
# Behalten pro note_id bis zu max_chunks_per_note Hits
|
||||
pooled_hits = []
|
||||
note_chunk_count: Dict[str, int] = defaultdict(int)
|
||||
|
||||
if note_id not in seen_notes:
|
||||
unique_note_hits.append(item)
|
||||
seen_notes.add(note_id)
|
||||
for item in enriched_sorted:
|
||||
_, _, payload, _ = item
|
||||
note_id = str(payload.get("note_id", "unknown"))
|
||||
|
||||
if note_chunk_count[note_id] < max_chunks_per_note:
|
||||
pooled_hits.append(item)
|
||||
note_chunk_count[note_id] += 1
|
||||
|
||||
logger.debug(f"📊 [AGGREGATION] Note-Level: {len(pooled_hits)} Chunks (max {max_chunks_per_note}/Note)")
|
||||
|
||||
# 3. Begrenzung auf top_k nach dem Diversity-Pooling
|
||||
limited_hits = unique_note_hits[: max(1, top_k)]
|
||||
limited_hits = pooled_hits[: max(1, top_k)]
|
||||
|
||||
results: List[QueryHit] = []
|
||||
for pid, s_score, pl, dbg in limited_hits:
|
||||
|
||||
Reference in new issue
Block a user