diff --git a/app/core/retriever.py b/app/core/retriever.py index 7db9472..c96a892 100644 --- a/app/core/retriever.py +++ b/app/core/retriever.py @@ -3,7 +3,7 @@ FILE: app/core/retriever.py DESCRIPTION: Implementiert die Hybrid-Suche (Vektor + Graph-Expansion) und das Scoring-Modell (Explainability). WP-22 Update: Dynamic Edge Boosting, Lifecycle Scoring & Provenance Awareness. Enthält detaillierte Debug-Informationen für die mathematische Verifizierung. -VERSION: 0.6.11 (WP-22 Full, Debug & Stable) +VERSION: 0.6.12 (WP-22 Full, Debug & Stable) STATUS: Active DEPENDENCIES: app.config, app.models.dto, app.core.qdrant*, app.services.embeddings_client, app.core.graph_adapter LAST_ANALYSIS: 2025-12-18 @@ -47,7 +47,6 @@ def _get_scoring_weights() -> Tuple[float, float, float]: """ Liefert die Basis-Gewichtung (semantic_weight, edge_weight, centrality_weight). Priorität: 1. retriever.yaml -> 2. Environment/Settings -> 3. Hardcoded Defaults - """ settings = get_settings() sem = float(getattr(settings, "RETRIEVER_W_SEM", 1.0)) @@ -119,7 +118,6 @@ def _get_status_multiplier(payload: Dict[str, Any]) -> float: WP-22 A: Lifecycle-Scoring. - stable: 1.2 (Validiertes Wissen fördern) - draft: 0.5 (Entwürfe de-priorisieren) - """ status = str(payload.get("status", "active")).lower().strip() if status == "stable": @@ -139,7 +137,6 @@ def _compute_total_score( """ Die zentrale mathematische Scoring-Formel von WP-22. Score = (Similarity * StatusMult) * (1 + (Weight-1) + DynamicBoost) - """ _sem_w, edge_w_cfg, cent_w_cfg = _get_scoring_weights() status_mult = _get_status_multiplier(payload) @@ -149,33 +146,37 @@ def _compute_total_score( base_val = float(semantic_score) * status_mult # 2. Graph Boost Factor (WP-22 C) + # Globaler Verstärker für Graph-Signale bei spezifischen Intents graph_boost_factor = 1.5 if dynamic_edge_boosts and (edge_bonus_raw > 0 or cent_bonus_raw > 0) else 1.0 # 3. Graph Contributions - edge_impact = (edge_w_cfg * edge_bonus_raw) * graph_boost_factor - cent_impact = (cent_w_cfg * cent_bonus_raw) * graph_boost_factor - dynamic_graph_impact = edge_impact + cent_impact + edge_impact_raw = edge_w_cfg * edge_bonus_raw + cent_impact_raw = cent_w_cfg * cent_bonus_raw + + # Finaler Impact unter Einbeziehung des Intent-Boosters + edge_impact_final = edge_impact_raw * graph_boost_factor + cent_impact_final = cent_impact_raw * graph_boost_factor + + dynamic_graph_impact = edge_impact_final + cent_impact_final # 4. Final Merge total = base_val * (1.0 + (node_weight - 1.0) + dynamic_graph_impact) - # Floor-Schutz - final_score = max(0.001, float(total)) - + # Debug Logging für Berechnungs-Validierung + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Scoring Node {payload.get('note_id')}: Base={base_val:.3f}, GraphI={dynamic_graph_impact:.3f} -> Total={total:.3f}") + return { - "total": final_score, + "total": max(0.001, float(total)), "edge_bonus": float(edge_bonus_raw), "cent_bonus": float(cent_bonus_raw), "status_multiplier": status_mult, "graph_boost_factor": graph_boost_factor, "type_impact": node_weight - 1.0, "base_val": base_val, - "edge_impact_final": edge_impact, - "cent_impact_final": cent_impact + "edge_impact_final": edge_impact_final, + "cent_impact_final": cent_impact_final } - - - # ============================================================================== # 3. EXPLANATION LAYER (DEBUG & VERIFIABILITY) # ============================================================================== @@ -189,7 +190,7 @@ def _build_explanation( applied_boosts: Optional[Dict[str, float]] = None ) -> Explanation: """Erstellt ein detailliertes Explanation-Objekt inkl. WP-22 Metriken.""" - _, edge_w_cfg, cent_w_cfg = _get_scoring_weights() + _, edge_w_cfg, _ = _get_scoring_weights() type_weight = float(payload.get("retriever_weight", 1.0)) status_mult = scoring_debug["status_multiplier"] @@ -214,8 +215,10 @@ def _build_explanation( edges_dto: List[EdgeDTO] = [] # 2. Gründe generieren - if semantic_score > 0.70: - reasons.append(Reason(kind="semantic", message="Textuelle Übereinstimmung.", score_impact=base_val)) + if semantic_score > 0.85: + reasons.append(Reason(kind="semantic", message="Herausragende inhaltliche Übereinstimmung.", score_impact=base_val)) + elif semantic_score > 0.70: + reasons.append(Reason(kind="semantic", message="Gute inhaltliche Übereinstimmung.", score_impact=base_val)) if type_weight != 1.0: msg = "Bevorzugt" if type_weight > 1.0 else "Abgewertet" @@ -225,7 +228,7 @@ def _build_explanation( txt = "Bonus" if status_mult > 1.0 else "Malus" reasons.append(Reason(kind="lifecycle", message=f"Status-{txt} ({payload.get('status')}).", score_impact=0.0)) - # 3. Kanten-Details (WP-22 B) - Beachtet eingehende UND ausgehende Kanten + # 3. Kanten-Details (WP-22 B) if subgraph and target_note_id and scoring_debug["edge_bonus"] > 0: raw_edges = [] if hasattr(subgraph, "get_incoming_edges"): @@ -243,8 +246,8 @@ def _build_explanation( is_incoming = (tgt == target_note_id) direction = "in" if is_incoming else "out" - # neighbor_id FIX: Variable sicher innerhalb der Schleife definieren - neighbor_id = src if is_incoming else tgt + # Peer-ID bestimmen (für die Anzeige) + neighbor_name = src if is_incoming else tgt edge_obj = EdgeDTO( id=f"{src}->{tgt}:{k}", kind=k, source=src, target=tgt, @@ -253,19 +256,18 @@ def _build_explanation( ) edges_dto.append(edge_obj) - # Die 3 stärksten Signale als Gründe formulieren top_edges = sorted(edges_dto, key=lambda e: e.confidence, reverse=True) for e in top_edges[:3]: prov_txt = "Explizite" if e.provenance == "explicit" else "Heuristische" boost_txt = f" [Boost x{applied_boosts.get(e.kind)}]" if applied_boosts and e.kind in applied_boosts else "" - # e.source/e.target sind durch e.direction eindeutig zugeordnet - peer_name = e.source if e.direction == "in" else e.target - msg = f"{prov_txt} Kante '{e.kind}'{boost_txt} von/zu '{peer_name}'." + # Richtigen Nachbarn für die Reason-Message finden + target_name = e.source if e.direction == "in" else e.target + msg = f"{prov_txt} Kante '{e.kind}'{boost_txt} von/zu '{target_name}'." reasons.append(Reason(kind="edge", message=msg, score_impact=edge_w_cfg * e.confidence)) if scoring_debug["cent_bonus"] > 0.01: - reasons.append(Reason(kind="centrality", message="Knoten liegt zentral im aktuellen Kontext.", score_impact=breakdown.centrality_contribution)) + reasons.append(Reason(kind="centrality", message="Knoten liegt zentral im Kontext.", score_impact=breakdown.centrality_contribution)) return Explanation( breakdown=breakdown, @@ -276,17 +278,13 @@ def _build_explanation( def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]: - """Extrahiert Expansion-Tiefe und Kanten-Filter aus dem Request.""" + """Extrahiert Expansion-Tiefe und Kanten-Filter.""" expand = getattr(req, "expand", None) - if not expand: - return 0, None - + if not expand: return 0, None if isinstance(expand, dict): return int(expand.get("depth", 1)), expand.get("edge_types") - if hasattr(expand, "depth"): return int(getattr(expand, "depth", 1)), getattr(expand, "edge_types", None) - return 1, None @@ -298,29 +296,23 @@ def _build_hits_from_semantic( explain: bool = False, dynamic_edge_boosts: Dict[str, float] = None ) -> QueryResponse: - """Wandelt semantische Roh-Treffer in strukturierte QueryHits um und berechnet WP-22 Scores.""" + """Wandelt semantische Roh-Treffer in strukturierte QueryHits um.""" t0 = time.time() enriched = [] for pid, semantic_score, payload in hits: - edge_bonus = 0.0 - cent_bonus = 0.0 + edge_bonus, cent_bonus = 0.0, 0.0 target_note_id = payload.get("note_id") if subgraph is not None and target_note_id: try: edge_bonus = float(subgraph.edge_bonus(target_note_id)) cent_bonus = float(subgraph.centrality_bonus(target_note_id)) - except Exception: - pass + except Exception: pass - # Messbare Scoring-Daten berechnen debug_data = _compute_total_score( - semantic_score, - payload, - edge_bonus_raw=edge_bonus, - cent_bonus_raw=cent_bonus, - dynamic_edge_boosts=dynamic_edge_boosts + semantic_score, payload, edge_bonus_raw=edge_bonus, + cent_bonus_raw=cent_bonus, dynamic_edge_boosts=dynamic_edge_boosts ) enriched.append((pid, float(semantic_score), payload, debug_data)) @@ -334,21 +326,19 @@ def _build_hits_from_semantic( if explain: explanation_obj = _build_explanation( semantic_score=float(semantic_score), - payload=payload, - scoring_debug=debug, - subgraph=subgraph, - target_note_id=payload.get("note_id"), + payload=payload, scoring_debug=debug, + subgraph=subgraph, target_note_id=payload.get("note_id"), applied_boosts=dynamic_edge_boosts ) text_content = payload.get("page_content") or payload.get("text") or payload.get("content") results.append(QueryHit( - node_id=str(pid), + node_id=str(pid), note_id=payload.get("note_id", "unknown"), - semantic_score=float(semantic_score), + semantic_score=float(semantic_score), edge_bonus=debug["edge_bonus"], - centrality_bonus=debug["cent_bonus"], + centrality_bonus=debug["cent_bonus"], total_score=debug["total"], source={ "path": payload.get("path"), @@ -359,64 +349,57 @@ def _build_hits_from_semantic( explanation=explanation_obj )) - dt_ms = int((time.time() - t0) * 1000) - return QueryResponse(results=results, used_mode=used_mode, latency_ms=dt_ms) + return QueryResponse(results=results, used_mode=used_mode, latency_ms=int((time.time() - t0) * 1000)) # ============================================================================== # 4. PUBLIC INTERFACE # ============================================================================== +def semantic_retrieve(req: QueryRequest) -> QueryResponse: + """Standard-Vektorsuche ohne Graph-Einfluss (WP-02).""" + client, prefix = _get_client_and_prefix() + vector = _get_query_vector(req) + top_k = req.top_k or 10 + hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters) + return _build_hits_from_semantic(hits, top_k=top_k, used_mode="semantic", subgraph=None, explain=req.explain) + + def hybrid_retrieve(req: QueryRequest) -> QueryResponse: """Hybrid-Suche: Semantik + WP-22 Graph Intelligence.""" client, prefix = _get_client_and_prefix() vector = list(req.query_vector) if req.query_vector else _get_query_vector(req) top_k = req.top_k or 10 - - # 1. Semantische Suche hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters) - # 2. Graph Expansion & Custom Weighting expand_depth, edge_types = _extract_expand_options(req) boost_edges = getattr(req, "boost_edges", {}) or {} subgraph: ga.Subgraph | None = None if expand_depth > 0 and hits: seed_ids = list({h[2].get("note_id") for h in hits if h[2].get("note_id")}) - if seed_ids: try: subgraph = ga.expand(client, prefix, seed_ids, depth=expand_depth, edge_types=edge_types) - - # WP-22: Transformation der Gewichte im RAM-Graphen vor Bonus-Berechnung if subgraph and hasattr(subgraph, "graph"): for u, v, data in subgraph.graph.edges(data=True): # Provenance Weighting (Concept 2.6) prov = data.get("provenance", "rule") prov_w = 1.0 if prov == "explicit" else (0.9 if prov == "smart" else 0.7) - # Intent Boost Multiplikator + # Intent Boost Mapping k = data.get("kind") intent_multiplier = boost_edges.get(k, 1.0) - - # Finales Gewicht setzen + + # Finales Kanten-Gewicht im Graphen setzen data["weight"] = data.get("weight", 1.0) * prov_w * intent_multiplier - - except Exception as e: + except Exception as e: logger.error(f"Graph expansion failed: {e}") subgraph = None - # 3. Scoring & Result Generation - return _build_hits_from_semantic( - hits, - top_k, - "hybrid", - subgraph, - req.explain, - boost_edges - ) + return _build_hits_from_semantic(hits, top_k, "hybrid", subgraph, req.explain, boost_edges) class Retriever: - """Wrapper-Klasse für FastAPI-Integration.""" + """Asynchroner Wrapper für FastAPI-Integration.""" async def search(self, request: QueryRequest) -> QueryResponse: - return hybrid_retrieve(request) \ No newline at end of file + return await ga.run_in_threadpool(hybrid_retrieve, request) if hasattr(ga, "run_in_threadpool") else hybrid_retrieve(request) \ No newline at end of file