Implement WP-26 v1.0 - Phase 2: Enhance edge scoring and aggregation configuration

- Introduced configurable edge scoring with internal and external boosts for intra-note edges.
- Added aggregation configuration to support note-level and chunk-level retrieval strategies.
- Updated retriever and graph subgraph modules to utilize new scoring and aggregation logic.
- Enhanced YAML configuration to include new parameters for edge scoring and aggregation levels.
- Added boolean indexing for filtering based on edge properties in the setup script.
This commit is contained in:
Lars committed 2026-01-25 21:06:13 +01:00
1 parent e86e9f2313
commit c5215e22e7
5 files changed
+384 -19

No files matched your search

+56 -13
View File
@@ -3,7 +3,8 @@ FILE: app/core/retrieval/retriever.py
DESCRIPTION: Haupt-Schnittstelle für die Suche. Orchestriert Vektorsuche und Graph-Expansion.
WP-15c Update: Note-Level Diversity Pooling & Super-Edge Aggregation.
WP-24c v4.1.0: Gold-Standard - Scope-Awareness, Section-Filtering, Authority-Priorisierung.
VERSION: 0.8.0 (WP-24c: Gold-Standard v4.1.0)
WP-26 v1.0: Konfigurierbare Aggregation (note/chunk Level).
VERSION: 0.9.0 (WP-26: Aggregation-Level)
STATUS: Active
DEPENDENCIES: app.config, app.models.dto, app.core.database*, app.core.graph_adapter
"""
@@ -34,8 +35,39 @@ from qdrant_client.http import models as rest
# Mathematische Engine importieren
from app.core.retrieval.retriever_scoring import get_weights, compute_wp22_score
try:
import yaml
except ImportError:
yaml = None
logger = logging.getLogger(__name__)
def _get_aggregation_config() -> Dict[str, Any]:
"""
WP-26 v1.0: Lädt Aggregation-Konfiguration aus retriever.yaml.
Returns:
Dict mit level ("note" oder "chunk") und max_chunks_per_note
"""
defaults = {
"level": "note", # "note" (Default) oder "chunk"
"max_chunks_per_note": 3 # Limit bei "note"-Level
}
config_path = os.getenv("MINDNET_RETRIEVER_CONFIG", "config/retriever.yaml")
if yaml and os.path.exists(config_path):
try:
with open(config_path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
aggregation = data.get("aggregation", {})
defaults["level"] = str(aggregation.get("level", defaults["level"])).lower()
defaults["max_chunks_per_note"] = int(aggregation.get("max_chunks_per_note", defaults["max_chunks_per_note"]))
except Exception as e:
logger.warning(f"Aggregation-Konfiguration konnte nicht geladen werden: {e}")
return defaults
# ==============================================================================
# 1. CORE HELPERS & CONFIG LOADERS
# ==============================================================================
@@ -316,22 +348,33 @@ def _build_hits_from_semantic(
# 1. Sortierung nach finalem mathematischen Score
enriched_sorted = sorted(enriched, key=lambda h: h[3]["total"], reverse=True)
# 2. WP-15c: Note-Level Diversity Pooling
# Wir behalten pro note_id nur den Hit mit dem höchsten total_score.
# Dies verhindert, dass 10 Chunks derselben Note andere KeyNotes verdrängen.
unique_note_hits = []
seen_notes = set()
# 2. WP-26 v1.0: Konfigurierbare Aggregation (Note-Level oder Chunk-Level)
aggregation_config = _get_aggregation_config()
aggregation_level = aggregation_config["level"]
max_chunks_per_note = aggregation_config["max_chunks_per_note"]
for item in enriched_sorted:
_, _, payload, _ = item
note_id = str(payload.get("note_id", "unknown"))
if aggregation_level == "chunk":
# WP-26 v1.0: Chunk-Level - alle Chunks individuell ranken (keine Deduplizierung)
logger.debug(f"📊 [AGGREGATION] Chunk-Level: Alle {len(enriched_sorted)} Chunks individuell")
pooled_hits = enriched_sorted
else:
# WP-15c: Note-Level Diversity Pooling (Default)
# Behalten pro note_id bis zu max_chunks_per_note Hits
pooled_hits = []
note_chunk_count: Dict[str, int] = defaultdict(int)
if note_id not in seen_notes:
unique_note_hits.append(item)
seen_notes.add(note_id)
for item in enriched_sorted:
_, _, payload, _ = item
note_id = str(payload.get("note_id", "unknown"))
if note_chunk_count[note_id] < max_chunks_per_note:
pooled_hits.append(item)
note_chunk_count[note_id] += 1
logger.debug(f"📊 [AGGREGATION] Note-Level: {len(pooled_hits)} Chunks (max {max_chunks_per_note}/Note)")
# 3. Begrenzung auf top_k nach dem Diversity-Pooling
limited_hits = unique_note_hits[: max(1, top_k)]
limited_hits = pooled_hits[: max(1, top_k)]
results: List[QueryHit] = []
for pid, s_score, pl, dbg in limited_hits: