Lars/mindnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 21 Wiki Activity

retriever in zwei Teilen

Browse Source
This commit is contained in:
Lars 2025-12-18 17:05:14 +01:00
parent cc12dcf993
commit 5dd58f49f0
1 changed files with 58 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
app/core/retriever.py
View File

@ -3,7 +3,7 @@ FILE: app/core/retriever.py
DESCRIPTION: Implementiert die Hybrid-Suche (Vektor + Graph-Expansion) und das Scoring-Modell (Explainability). DESCRIPTION: Implementiert die Hybrid-Suche (Vektor + Graph-Expansion) und das Scoring-Modell (Explainability).
WP-22 Update: Dynamic Edge Boosting, Lifecycle Scoring & Provenance Awareness. WP-22 Update: Dynamic Edge Boosting, Lifecycle Scoring & Provenance Awareness.
Enthält detaillierte Debug-Informationen für die mathematische Verifizierung. Enthält detaillierte Debug-Informationen für die mathematische Verifizierung.
VERSION: 0.6.11 (WP-22 Full, Debug & Stable) VERSION: 0.6.12 (WP-22 Full, Debug & Stable)
STATUS: Active STATUS: Active
DEPENDENCIES: app.config, app.models.dto, app.core.qdrant*, app.services.embeddings_client, app.core.graph_adapter DEPENDENCIES: app.config, app.models.dto, app.core.qdrant*, app.services.embeddings_client, app.core.graph_adapter
LAST_ANALYSIS: 2025-12-18 LAST_ANALYSIS: 2025-12-18
@ -47,7 +47,6 @@ def _get_scoring_weights() -> Tuple[float, float, float]:
""" """
Liefert die Basis-Gewichtung (semantic_weight, edge_weight, centrality_weight). Liefert die Basis-Gewichtung (semantic_weight, edge_weight, centrality_weight).
Priorität: 1. retriever.yaml -> 2. Environment/Settings -> 3. Hardcoded Defaults Priorität: 1. retriever.yaml -> 2. Environment/Settings -> 3. Hardcoded Defaults
""" """
settings = get_settings() settings = get_settings()
sem = float(getattr(settings, "RETRIEVER_W_SEM", 1.0)) sem = float(getattr(settings, "RETRIEVER_W_SEM", 1.0))
@ -119,7 +118,6 @@ def _get_status_multiplier(payload: Dict[str, Any]) -> float:
WP-22 A: Lifecycle-Scoring. WP-22 A: Lifecycle-Scoring.
- stable: 1.2 (Validiertes Wissen fördern) - stable: 1.2 (Validiertes Wissen fördern)
- draft: 0.5 (Entwürfe de-priorisieren) - draft: 0.5 (Entwürfe de-priorisieren)
""" """
status = str(payload.get("status", "active")).lower().strip() status = str(payload.get("status", "active")).lower().strip()
if status == "stable": if status == "stable":
@ -139,7 +137,6 @@ def _compute_total_score(
""" """
Die zentrale mathematische Scoring-Formel von WP-22. Die zentrale mathematische Scoring-Formel von WP-22.
Score = (Similarity * StatusMult) * (1 + (Weight-1) + DynamicBoost) Score = (Similarity * StatusMult) * (1 + (Weight-1) + DynamicBoost)
""" """
_sem_w, edge_w_cfg, cent_w_cfg = _get_scoring_weights() _sem_w, edge_w_cfg, cent_w_cfg = _get_scoring_weights()
status_mult = _get_status_multiplier(payload) status_mult = _get_status_multiplier(payload)
@ -149,33 +146,37 @@ def _compute_total_score(
base_val = float(semantic_score) * status_mult base_val = float(semantic_score) * status_mult
# 2. Graph Boost Factor (WP-22 C) # 2. Graph Boost Factor (WP-22 C)
# Globaler Verstärker für Graph-Signale bei spezifischen Intents
graph_boost_factor = 1.5 if dynamic_edge_boosts and (edge_bonus_raw > 0 or cent_bonus_raw > 0) else 1.0 graph_boost_factor = 1.5 if dynamic_edge_boosts and (edge_bonus_raw > 0 or cent_bonus_raw > 0) else 1.0
# 3. Graph Contributions # 3. Graph Contributions
edge_impact = (edge_w_cfg * edge_bonus_raw) * graph_boost_factor edge_impact_raw = edge_w_cfg * edge_bonus_raw
cent_impact = (cent_w_cfg * cent_bonus_raw) * graph_boost_factor cent_impact_raw = cent_w_cfg * cent_bonus_raw
dynamic_graph_impact = edge_impact + cent_impact
# Finaler Impact unter Einbeziehung des Intent-Boosters
edge_impact_final = edge_impact_raw * graph_boost_factor
cent_impact_final = cent_impact_raw * graph_boost_factor
dynamic_graph_impact = edge_impact_final + cent_impact_final
# 4. Final Merge # 4. Final Merge
total = base_val * (1.0 + (node_weight - 1.0) + dynamic_graph_impact) total = base_val * (1.0 + (node_weight - 1.0) + dynamic_graph_impact)
# Floor-Schutz # Debug Logging für Berechnungs-Validierung
final_score = max(0.001, float(total)) if logger.isEnabledFor(logging.DEBUG):
logger.debug(f"Scoring Node {payload.get('note_id')}: Base={base_val:.3f}, GraphI={dynamic_graph_impact:.3f} -> Total={total:.3f}")
return { return {
"total": final_score, "total": max(0.001, float(total)),
"edge_bonus": float(edge_bonus_raw), "edge_bonus": float(edge_bonus_raw),
"cent_bonus": float(cent_bonus_raw), "cent_bonus": float(cent_bonus_raw),
"status_multiplier": status_mult, "status_multiplier": status_mult,
"graph_boost_factor": graph_boost_factor, "graph_boost_factor": graph_boost_factor,
"type_impact": node_weight - 1.0, "type_impact": node_weight - 1.0,
"base_val": base_val, "base_val": base_val,
"edge_impact_final": edge_impact, "edge_impact_final": edge_impact_final,
"cent_impact_final": cent_impact "cent_impact_final": cent_impact_final
} }
# ============================================================================== # ==============================================================================
# 3. EXPLANATION LAYER (DEBUG & VERIFIABILITY) # 3. EXPLANATION LAYER (DEBUG & VERIFIABILITY)
# ============================================================================== # ==============================================================================
@ -189,7 +190,7 @@ def _build_explanation(
applied_boosts: Optional[Dict[str, float]] = None applied_boosts: Optional[Dict[str, float]] = None
) -> Explanation: ) -> Explanation:
"""Erstellt ein detailliertes Explanation-Objekt inkl. WP-22 Metriken.""" """Erstellt ein detailliertes Explanation-Objekt inkl. WP-22 Metriken."""
_, edge_w_cfg, cent_w_cfg = _get_scoring_weights() _, edge_w_cfg, _ = _get_scoring_weights()
type_weight = float(payload.get("retriever_weight", 1.0)) type_weight = float(payload.get("retriever_weight", 1.0))
status_mult = scoring_debug["status_multiplier"] status_mult = scoring_debug["status_multiplier"]
@ -214,8 +215,10 @@ def _build_explanation(
edges_dto: List[EdgeDTO] = [] edges_dto: List[EdgeDTO] = []
# 2. Gründe generieren # 2. Gründe generieren
if semantic_score > 0.70: if semantic_score > 0.85:
reasons.append(Reason(kind="semantic", message="Textuelle Übereinstimmung.", score_impact=base_val)) reasons.append(Reason(kind="semantic", message="Herausragende inhaltliche Übereinstimmung.", score_impact=base_val))
elif semantic_score > 0.70:
reasons.append(Reason(kind="semantic", message="Gute inhaltliche Übereinstimmung.", score_impact=base_val))
if type_weight != 1.0: if type_weight != 1.0:
msg = "Bevorzugt" if type_weight > 1.0 else "Abgewertet" msg = "Bevorzugt" if type_weight > 1.0 else "Abgewertet"
@ -225,7 +228,7 @@ def _build_explanation(
txt = "Bonus" if status_mult > 1.0 else "Malus" txt = "Bonus" if status_mult > 1.0 else "Malus"
reasons.append(Reason(kind="lifecycle", message=f"Status-{txt} ({payload.get('status')}).", score_impact=0.0)) reasons.append(Reason(kind="lifecycle", message=f"Status-{txt} ({payload.get('status')}).", score_impact=0.0))
# 3. Kanten-Details (WP-22 B) - Beachtet eingehende UND ausgehende Kanten # 3. Kanten-Details (WP-22 B)
if subgraph and target_note_id and scoring_debug["edge_bonus"] > 0: if subgraph and target_note_id and scoring_debug["edge_bonus"] > 0:
raw_edges = [] raw_edges = []
if hasattr(subgraph, "get_incoming_edges"): if hasattr(subgraph, "get_incoming_edges"):
@ -243,8 +246,8 @@ def _build_explanation(
is_incoming = (tgt == target_note_id) is_incoming = (tgt == target_note_id)
direction = "in" if is_incoming else "out" direction = "in" if is_incoming else "out"
# neighbor_id FIX: Variable sicher innerhalb der Schleife definieren # Peer-ID bestimmen (für die Anzeige)
neighbor_id = src if is_incoming else tgt neighbor_name = src if is_incoming else tgt
edge_obj = EdgeDTO( edge_obj = EdgeDTO(
id=f"{src}->{tgt}:{k}", kind=k, source=src, target=tgt, id=f"{src}->{tgt}:{k}", kind=k, source=src, target=tgt,
@ -253,19 +256,18 @@ def _build_explanation(
) )
edges_dto.append(edge_obj) edges_dto.append(edge_obj)
# Die 3 stärksten Signale als Gründe formulieren
top_edges = sorted(edges_dto, key=lambda e: e.confidence, reverse=True) top_edges = sorted(edges_dto, key=lambda e: e.confidence, reverse=True)
for e in top_edges[:3]: for e in top_edges[:3]:
prov_txt = "Explizite" if e.provenance == "explicit" else "Heuristische" prov_txt = "Explizite" if e.provenance == "explicit" else "Heuristische"
boost_txt = f" [Boost x{applied_boosts.get(e.kind)}]" if applied_boosts and e.kind in applied_boosts else "" boost_txt = f" [Boost x{applied_boosts.get(e.kind)}]" if applied_boosts and e.kind in applied_boosts else ""
# e.source/e.target sind durch e.direction eindeutig zugeordnet # Richtigen Nachbarn für die Reason-Message finden
peer_name = e.source if e.direction == "in" else e.target target_name = e.source if e.direction == "in" else e.target
msg = f"{prov_txt} Kante '{e.kind}'{boost_txt} von/zu '{peer_name}'." msg = f"{prov_txt} Kante '{e.kind}'{boost_txt} von/zu '{target_name}'."
reasons.append(Reason(kind="edge", message=msg, score_impact=edge_w_cfg * e.confidence)) reasons.append(Reason(kind="edge", message=msg, score_impact=edge_w_cfg * e.confidence))
if scoring_debug["cent_bonus"] > 0.01: if scoring_debug["cent_bonus"] > 0.01:
reasons.append(Reason(kind="centrality", message="Knoten liegt zentral im aktuellen Kontext.", score_impact=breakdown.centrality_contribution)) reasons.append(Reason(kind="centrality", message="Knoten liegt zentral im Kontext.", score_impact=breakdown.centrality_contribution))
return Explanation( return Explanation(
breakdown=breakdown, breakdown=breakdown,
@ -276,17 +278,13 @@ def _build_explanation(
def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]: def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
"""Extrahiert Expansion-Tiefe und Kanten-Filter aus dem Request.""" """Extrahiert Expansion-Tiefe und Kanten-Filter."""
expand = getattr(req, "expand", None) expand = getattr(req, "expand", None)
if not expand: if not expand: return 0, None
return 0, None
if isinstance(expand, dict): if isinstance(expand, dict):
return int(expand.get("depth", 1)), expand.get("edge_types") return int(expand.get("depth", 1)), expand.get("edge_types")
if hasattr(expand, "depth"): if hasattr(expand, "depth"):
return int(getattr(expand, "depth", 1)), getattr(expand, "edge_types", None) return int(getattr(expand, "depth", 1)), getattr(expand, "edge_types", None)
return 1, None return 1, None
@ -298,29 +296,23 @@ def _build_hits_from_semantic(
explain: bool = False, explain: bool = False,
dynamic_edge_boosts: Dict[str, float] = None dynamic_edge_boosts: Dict[str, float] = None
) -> QueryResponse: ) -> QueryResponse:
"""Wandelt semantische Roh-Treffer in strukturierte QueryHits um und berechnet WP-22 Scores.""" """Wandelt semantische Roh-Treffer in strukturierte QueryHits um."""
t0 = time.time() t0 = time.time()
enriched = [] enriched = []
for pid, semantic_score, payload in hits: for pid, semantic_score, payload in hits:
edge_bonus = 0.0 edge_bonus, cent_bonus = 0.0, 0.0
cent_bonus = 0.0
target_note_id = payload.get("note_id") target_note_id = payload.get("note_id")
if subgraph is not None and target_note_id: if subgraph is not None and target_note_id:
try: try:
edge_bonus = float(subgraph.edge_bonus(target_note_id)) edge_bonus = float(subgraph.edge_bonus(target_note_id))
cent_bonus = float(subgraph.centrality_bonus(target_note_id)) cent_bonus = float(subgraph.centrality_bonus(target_note_id))
except Exception: except Exception: pass
pass
# Messbare Scoring-Daten berechnen
debug_data = _compute_total_score( debug_data = _compute_total_score(
semantic_score, semantic_score, payload, edge_bonus_raw=edge_bonus,
payload, cent_bonus_raw=cent_bonus, dynamic_edge_boosts=dynamic_edge_boosts
edge_bonus_raw=edge_bonus,
cent_bonus_raw=cent_bonus,
dynamic_edge_boosts=dynamic_edge_boosts
) )
enriched.append((pid, float(semantic_score), payload, debug_data)) enriched.append((pid, float(semantic_score), payload, debug_data))
@ -334,10 +326,8 @@ def _build_hits_from_semantic(
if explain: if explain:
explanation_obj = _build_explanation( explanation_obj = _build_explanation(
semantic_score=float(semantic_score), semantic_score=float(semantic_score),
payload=payload, payload=payload, scoring_debug=debug,
scoring_debug=debug, subgraph=subgraph, target_note_id=payload.get("note_id"),
subgraph=subgraph,
target_note_id=payload.get("note_id"),
applied_boosts=dynamic_edge_boosts applied_boosts=dynamic_edge_boosts
) )
@ -359,64 +349,57 @@ def _build_hits_from_semantic(
explanation=explanation_obj explanation=explanation_obj
)) ))
dt_ms = int((time.time() - t0) * 1000) return QueryResponse(results=results, used_mode=used_mode, latency_ms=int((time.time() - t0) * 1000))
return QueryResponse(results=results, used_mode=used_mode, latency_ms=dt_ms)
# ============================================================================== # ==============================================================================
# 4. PUBLIC INTERFACE # 4. PUBLIC INTERFACE
# ============================================================================== # ==============================================================================
def semantic_retrieve(req: QueryRequest) -> QueryResponse:
"""Standard-Vektorsuche ohne Graph-Einfluss (WP-02)."""
client, prefix = _get_client_and_prefix()
vector = _get_query_vector(req)
top_k = req.top_k or 10
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="semantic", subgraph=None, explain=req.explain)
def hybrid_retrieve(req: QueryRequest) -> QueryResponse: def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
"""Hybrid-Suche: Semantik + WP-22 Graph Intelligence.""" """Hybrid-Suche: Semantik + WP-22 Graph Intelligence."""
client, prefix = _get_client_and_prefix() client, prefix = _get_client_and_prefix()
vector = list(req.query_vector) if req.query_vector else _get_query_vector(req) vector = list(req.query_vector) if req.query_vector else _get_query_vector(req)
top_k = req.top_k or 10 top_k = req.top_k or 10
# 1. Semantische Suche
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters) hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
# 2. Graph Expansion & Custom Weighting
expand_depth, edge_types = _extract_expand_options(req) expand_depth, edge_types = _extract_expand_options(req)
boost_edges = getattr(req, "boost_edges", {}) or {} boost_edges = getattr(req, "boost_edges", {}) or {}
subgraph: ga.Subgraph | None = None subgraph: ga.Subgraph | None = None
if expand_depth > 0 and hits: if expand_depth > 0 and hits:
seed_ids = list({h[2].get("note_id") for h in hits if h[2].get("note_id")}) seed_ids = list({h[2].get("note_id") for h in hits if h[2].get("note_id")})
if seed_ids: if seed_ids:
try: try:
subgraph = ga.expand(client, prefix, seed_ids, depth=expand_depth, edge_types=edge_types) subgraph = ga.expand(client, prefix, seed_ids, depth=expand_depth, edge_types=edge_types)
# WP-22: Transformation der Gewichte im RAM-Graphen vor Bonus-Berechnung
if subgraph and hasattr(subgraph, "graph"): if subgraph and hasattr(subgraph, "graph"):
for u, v, data in subgraph.graph.edges(data=True): for u, v, data in subgraph.graph.edges(data=True):
# Provenance Weighting (Concept 2.6) # Provenance Weighting (Concept 2.6)
prov = data.get("provenance", "rule") prov = data.get("provenance", "rule")
prov_w = 1.0 if prov == "explicit" else (0.9 if prov == "smart" else 0.7) prov_w = 1.0 if prov == "explicit" else (0.9 if prov == "smart" else 0.7)
# Intent Boost Multiplikator # Intent Boost Mapping
k = data.get("kind") k = data.get("kind")
intent_multiplier = boost_edges.get(k, 1.0) intent_multiplier = boost_edges.get(k, 1.0)
# Finales Gewicht setzen # Finales Kanten-Gewicht im Graphen setzen
data["weight"] = data.get("weight", 1.0) * prov_w * intent_multiplier data["weight"] = data.get("weight", 1.0) * prov_w * intent_multiplier
except Exception as e: except Exception as e:
logger.error(f"Graph expansion failed: {e}") logger.error(f"Graph expansion failed: {e}")
subgraph = None subgraph = None
# 3. Scoring & Result Generation return _build_hits_from_semantic(hits, top_k, "hybrid", subgraph, req.explain, boost_edges)
return _build_hits_from_semantic(
hits,
top_k,
"hybrid",
subgraph,
req.explain,
boost_edges
)
class Retriever: class Retriever:
"""Wrapper-Klasse für FastAPI-Integration.""" """Asynchroner Wrapper für FastAPI-Integration."""
async def search(self, request: QueryRequest) -> QueryResponse: async def search(self, request: QueryRequest) -> QueryResponse:
return hybrid_retrieve(request) return await ga.run_in_threadpool(hybrid_retrieve, request) if hasattr(ga, "run_in_threadpool") else hybrid_retrieve(request)