neue debug Version im retriever
This commit is contained in:
parent
c61d9c8236
commit
babab3167b
|
|
@ -2,7 +2,8 @@
|
||||||
FILE: app/core/retriever.py
|
FILE: app/core/retriever.py
|
||||||
DESCRIPTION: Implementiert die Hybrid-Suche (Vektor + Graph-Expansion) und das Scoring-Modell (Explainability).
|
DESCRIPTION: Implementiert die Hybrid-Suche (Vektor + Graph-Expansion) und das Scoring-Modell (Explainability).
|
||||||
WP-22 Update: Dynamic Edge Boosting, Lifecycle Scoring & Provenance Awareness.
|
WP-22 Update: Dynamic Edge Boosting, Lifecycle Scoring & Provenance Awareness.
|
||||||
VERSION: 0.6.7 (WP-22 Scoring & Provenance Fix)
|
Enthält detaillierte Debug-Informationen für die mathematische Verifizierung.
|
||||||
|
VERSION: 0.6.8 (WP-22 Debug & Verifiability)
|
||||||
STATUS: Active
|
STATUS: Active
|
||||||
DEPENDENCIES: app.config, app.models.dto, app.core.qdrant*, app.services.embeddings_client, app.core.graph_adapter
|
DEPENDENCIES: app.config, app.models.dto, app.core.qdrant*, app.services.embeddings_client, app.core.graph_adapter
|
||||||
LAST_ANALYSIS: 2025-12-18
|
LAST_ANALYSIS: 2025-12-18
|
||||||
|
|
@ -39,7 +40,10 @@ logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@lru_cache
|
@lru_cache
|
||||||
def _get_scoring_weights() -> Tuple[float, float, float]:
|
def _get_scoring_weights() -> Tuple[float, float, float]:
|
||||||
"""Liefert (semantic_weight, edge_weight, centrality_weight) für den Retriever."""
|
"""
|
||||||
|
Liefert die Basis-Gewichtung (semantic_weight, edge_weight, centrality_weight) aus der Config.
|
||||||
|
Priorität: 1. retriever.yaml -> 2. Environment/Settings -> 3. Hardcoded Defaults
|
||||||
|
"""
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
sem = float(getattr(settings, "RETRIEVER_W_SEM", 1.0))
|
sem = float(getattr(settings, "RETRIEVER_W_SEM", 1.0))
|
||||||
edge = float(getattr(settings, "RETRIEVER_W_EDGE", 0.0))
|
edge = float(getattr(settings, "RETRIEVER_W_EDGE", 0.0))
|
||||||
|
|
@ -56,32 +60,38 @@ def _get_scoring_weights() -> Tuple[float, float, float]:
|
||||||
sem = float(scoring.get("semantic_weight", sem))
|
sem = float(scoring.get("semantic_weight", sem))
|
||||||
edge = float(scoring.get("edge_weight", edge))
|
edge = float(scoring.get("edge_weight", edge))
|
||||||
cent = float(scoring.get("centrality_weight", cent))
|
cent = float(scoring.get("centrality_weight", cent))
|
||||||
except Exception:
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to load retriever weights from {config_path}: {e}")
|
||||||
return sem, edge, cent
|
return sem, edge, cent
|
||||||
return sem, edge, cent
|
return sem, edge, cent
|
||||||
|
|
||||||
|
|
||||||
def _get_client_and_prefix() -> Tuple[Any, str]:
|
def _get_client_and_prefix() -> Tuple[Any, str]:
|
||||||
"""Liefert (QdrantClient, prefix)."""
|
"""Liefert das initialisierte Qdrant-Client-Objekt und das aktuelle Collection-Präfix."""
|
||||||
cfg = qdr.QdrantConfig.from_env()
|
cfg = qdr.QdrantConfig.from_env()
|
||||||
client = qdr.get_client(cfg)
|
client = qdr.get_client(cfg)
|
||||||
return client, cfg.prefix
|
return client, cfg.prefix
|
||||||
|
|
||||||
|
|
||||||
def _get_query_vector(req: QueryRequest) -> List[float]:
|
def _get_query_vector(req: QueryRequest) -> List[float]:
|
||||||
"""Liefert den Query-Vektor aus dem Request."""
|
"""
|
||||||
|
Stellt sicher, dass ein Query-Vektor vorhanden ist.
|
||||||
|
Wandelt Text-Queries via EmbeddingsClient um, falls kein Vektor im Request liegt.
|
||||||
|
"""
|
||||||
if req.query_vector:
|
if req.query_vector:
|
||||||
return list(req.query_vector)
|
return list(req.query_vector)
|
||||||
|
|
||||||
if not req.query:
|
if not req.query:
|
||||||
raise ValueError("QueryRequest benötigt entweder query oder query_vector")
|
raise ValueError("QueryRequest benötigt entweder 'query' oder 'query_vector'")
|
||||||
|
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
model_name = settings.MODEL_NAME
|
model_name = settings.MODEL_NAME
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
# Versuch mit modernem Interface (WP-03 kompatibel)
|
||||||
return ec.embed_text(req.query, model_name=model_name)
|
return ec.embed_text(req.query, model_name=model_name)
|
||||||
except TypeError:
|
except TypeError:
|
||||||
|
# Fallback für ältere EmbeddingsClient-Signaturen
|
||||||
return ec.embed_text(req.query)
|
return ec.embed_text(req.query)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -92,7 +102,7 @@ def _semantic_hits(
|
||||||
top_k: int,
|
top_k: int,
|
||||||
filters: Dict[str, Any] | None = None,
|
filters: Dict[str, Any] | None = None,
|
||||||
) -> List[Tuple[str, float, Dict[str, Any]]]:
|
) -> List[Tuple[str, float, Dict[str, Any]]]:
|
||||||
"""Führt eine semantische Suche aus."""
|
"""Führt eine reine Vektorsuche in Qdrant aus und gibt die Roh-Treffer zurück."""
|
||||||
flt = filters or None
|
flt = filters or None
|
||||||
raw_hits = qp.search_chunks_by_vector(client, prefix, vector, top=top_k, filters=flt)
|
raw_hits = qp.search_chunks_by_vector(client, prefix, vector, top=top_k, filters=flt)
|
||||||
results: List[Tuple[str, float, Dict[str, Any]]] = []
|
results: List[Tuple[str, float, Dict[str, Any]]] = []
|
||||||
|
|
@ -101,59 +111,76 @@ def _semantic_hits(
|
||||||
return results
|
return results
|
||||||
|
|
||||||
# --- WP-22 Helper: Lifecycle Multipliers (Teil A) ---
|
# --- WP-22 Helper: Lifecycle Multipliers (Teil A) ---
|
||||||
|
|
||||||
def _get_status_multiplier(payload: Dict[str, Any]) -> float:
|
def _get_status_multiplier(payload: Dict[str, Any]) -> float:
|
||||||
"""
|
"""
|
||||||
WP-22: stable (1.2), active/default (1.0), draft (0.5).
|
Ermittelt den Multiplikator basierend auf dem Content-Status.
|
||||||
|
- stable: 1.2 (Belohnung für validiertes Wissen)
|
||||||
|
- active/default: 1.0
|
||||||
|
- draft: 0.5 (Bestrafung für Unfertiges)
|
||||||
"""
|
"""
|
||||||
status = str(payload.get("status", "active")).lower()
|
status = str(payload.get("status", "active")).lower().strip()
|
||||||
if status == "stable": return 1.2
|
if status == "stable":
|
||||||
if status == "draft": return 0.5
|
return 1.2
|
||||||
|
if status == "draft":
|
||||||
|
return 0.5
|
||||||
return 1.0
|
return 1.0
|
||||||
|
|
||||||
# --- WP-22: Dynamic Scoring Formula (Teil C) ---
|
# --- WP-22: Dynamic Scoring Formula (Teil C) ---
|
||||||
|
|
||||||
def _compute_total_score(
|
def _compute_total_score(
|
||||||
semantic_score: float,
|
semantic_score: float,
|
||||||
payload: Dict[str, Any],
|
payload: Dict[str, Any],
|
||||||
edge_bonus_raw: float = 0.0,
|
edge_bonus_raw: float = 0.0,
|
||||||
cent_bonus_raw: float = 0.0,
|
cent_bonus_raw: float = 0.0,
|
||||||
dynamic_edge_boosts: Dict[str, float] = None
|
dynamic_edge_boosts: Dict[str, float] = None
|
||||||
) -> Tuple[float, float, float]:
|
) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
WP-22 Mathematische Logik:
|
Die zentrale mathematische Scoring-Formel von WP-22.
|
||||||
Score = BaseScore * (1 + ConfigWeight + DynamicBoost)
|
|
||||||
|
FORMEL:
|
||||||
|
Score = (SemanticScore * StatusMultiplier) * (1 + (Weight-1) + DynamicGraphBoost)
|
||||||
|
|
||||||
Hierbei gilt:
|
Hierbei gilt:
|
||||||
- BaseScore: semantic_similarity * status_multiplier
|
- BaseScore: semantic_similarity * status_multiplier
|
||||||
- ConfigWeight: retriever_weight (Type Boost) - 1.0
|
- TypeImpact: retriever_weight (z.B. 1.1 für Decisions)
|
||||||
- DynamicBoost: (edge_weight * edge_bonus) + (centrality_weight * centrality_bonus)
|
- DynamicBoost: (EdgeW * EdgeBonus) + (CentW * CentBonus)
|
||||||
"""
|
"""
|
||||||
|
# 1. Basis-Parameter laden
|
||||||
# 1. Base Score (Semantik * Lifecycle)
|
|
||||||
status_mult = _get_status_multiplier(payload)
|
|
||||||
base_score = float(semantic_score) * status_mult
|
|
||||||
|
|
||||||
# 2. Config Weight (Static Type Boost)
|
|
||||||
# Ein neutrales retriever_weight von 1.0 ergibt 0.0 Einfluss.
|
|
||||||
config_weight = float(payload.get("retriever_weight", 1.0)) - 1.0
|
|
||||||
|
|
||||||
# 3. Dynamic Boost (Graph-Signale)
|
|
||||||
_sem_w, edge_w_cfg, cent_w_cfg = _get_scoring_weights()
|
_sem_w, edge_w_cfg, cent_w_cfg = _get_scoring_weights()
|
||||||
|
status_mult = _get_status_multiplier(payload)
|
||||||
|
node_weight = float(payload.get("retriever_weight", 1.0))
|
||||||
|
|
||||||
# Multiplikator für Intent-Boosting (Teil C)
|
# 2. Base Score (Semantik gewichtet durch Lifecycle)
|
||||||
|
base_val = float(semantic_score) * status_mult
|
||||||
|
|
||||||
|
# 3. Graph-Intelligence Boost (WP-22 C)
|
||||||
|
# Globaler Verstärker für Graph-Signale bei spezifischen Intents (z.B. WHY/EMPATHY)
|
||||||
graph_boost_factor = 1.5 if dynamic_edge_boosts and (edge_bonus_raw > 0 or cent_bonus_raw > 0) else 1.0
|
graph_boost_factor = 1.5 if dynamic_edge_boosts and (edge_bonus_raw > 0 or cent_bonus_raw > 0) else 1.0
|
||||||
|
|
||||||
edge_impact = (edge_w_cfg * edge_bonus_raw) * graph_boost_factor
|
edge_contribution_raw = edge_w_cfg * edge_bonus_raw
|
||||||
cent_impact = (cent_w_cfg * cent_bonus_raw) * graph_boost_factor
|
cent_contribution_raw = cent_w_cfg * cent_bonus_raw
|
||||||
|
|
||||||
dynamic_boost = edge_impact + cent_impact
|
dynamic_graph_impact = (edge_contribution_raw + cent_contribution_raw) * graph_boost_factor
|
||||||
|
|
||||||
total = base_score * (1.0 + config_weight + dynamic_boost)
|
# 4. Zusammenführung (Die "Dicke" des Knotens und die Verknüpfung)
|
||||||
|
# (node_weight - 1.0) ermöglicht negative oder positive Type-Impacts relativ zu 1.0
|
||||||
|
total = base_val * (1.0 + (node_weight - 1.0) + dynamic_graph_impact)
|
||||||
|
|
||||||
# Debug Logging für Berechnungs-Validierung
|
# Schutz vor negativen Scores (Floor)
|
||||||
if logger.isEnabledFor(logging.DEBUG):
|
final_score = max(0.001, float(total))
|
||||||
logger.debug(f"Scoring Node {payload.get('note_id')}: Base={base_score:.3f}, ConfigW={config_weight:.3f}, GraphB={dynamic_boost:.3f} -> Total={total:.3f}")
|
|
||||||
|
# Debug-Daten für den Explanation-Layer sammeln
|
||||||
|
return {
|
||||||
|
"total": final_score,
|
||||||
|
"edge_bonus": float(edge_bonus_raw),
|
||||||
|
"cent_bonus": float(cent_bonus_raw),
|
||||||
|
"status_multiplier": status_mult,
|
||||||
|
"graph_boost_factor": graph_boost_factor,
|
||||||
|
"type_impact": node_weight - 1.0,
|
||||||
|
"base_val": base_val
|
||||||
|
}
|
||||||
|
|
||||||
return float(total), float(edge_bonus_raw), float(cent_bonus_raw)
|
|
||||||
|
|
||||||
|
|
||||||
# --- WP-04b Explanation Logic ---
|
# --- WP-04b Explanation Logic ---
|
||||||
|
|
@ -161,119 +188,119 @@ def _compute_total_score(
|
||||||
def _build_explanation(
|
def _build_explanation(
|
||||||
semantic_score: float,
|
semantic_score: float,
|
||||||
payload: Dict[str, Any],
|
payload: Dict[str, Any],
|
||||||
edge_bonus: float,
|
scoring_debug: Dict[str, Any],
|
||||||
cent_bonus: float,
|
|
||||||
subgraph: Optional[ga.Subgraph],
|
subgraph: Optional[ga.Subgraph],
|
||||||
target_note_id: Optional[str]
|
target_note_id: Optional[str],
|
||||||
|
applied_boosts: Optional[Dict[str, float]] = None
|
||||||
) -> Explanation:
|
) -> Explanation:
|
||||||
"""Erstellt ein Explanation-Objekt mit Provenance-Details."""
|
"""
|
||||||
|
Erstellt ein detailliertes Explanation-Objekt für maximale Transparenz (WP-04b).
|
||||||
|
Enthält nun WP-22 Debug-Metriken wie StatusMultiplier und GraphBoostFactor.
|
||||||
|
"""
|
||||||
_, edge_w_cfg, cent_w_cfg = _get_scoring_weights()
|
_, edge_w_cfg, cent_w_cfg = _get_scoring_weights()
|
||||||
|
|
||||||
type_weight = float(payload.get("retriever_weight", 1.0))
|
type_weight = float(payload.get("retriever_weight", 1.0))
|
||||||
status_mult = _get_status_multiplier(payload)
|
status_mult = scoring_debug["status_multiplier"]
|
||||||
|
graph_bf = scoring_debug["graph_boost_factor"]
|
||||||
note_type = payload.get("type", "unknown")
|
note_type = payload.get("type", "unknown")
|
||||||
|
base_val = scoring_debug["base_val"]
|
||||||
|
|
||||||
# Breakdown für Explanation (Reflektiert die WP-22 Formel exakt)
|
# 1. Score Breakdown Objekt
|
||||||
base_val = semantic_score * status_mult
|
|
||||||
config_w_impact = type_weight - 1.0
|
|
||||||
|
|
||||||
# Zentrale Berechnung der Kontributionen für den Breakdown
|
|
||||||
breakdown = ScoreBreakdown(
|
breakdown = ScoreBreakdown(
|
||||||
semantic_contribution=base_val,
|
semantic_contribution=base_val,
|
||||||
edge_contribution=base_val * (edge_w_cfg * edge_bonus),
|
edge_contribution=base_val * (edge_w_cfg * scoring_debug["edge_bonus"] * graph_bf),
|
||||||
centrality_contribution=base_val * (cent_w_cfg * cent_bonus),
|
centrality_contribution=base_val * (cent_w_cfg * scoring_debug["cent_bonus"] * graph_bf),
|
||||||
raw_semantic=semantic_score,
|
raw_semantic=semantic_score,
|
||||||
raw_edge_bonus=edge_bonus,
|
raw_edge_bonus=scoring_debug["edge_bonus"],
|
||||||
raw_centrality=cent_bonus,
|
raw_centrality=scoring_debug["cent_bonus"],
|
||||||
node_weight=type_weight
|
node_weight=type_weight,
|
||||||
|
status_multiplier=status_mult,
|
||||||
|
graph_boost_factor=graph_bf
|
||||||
)
|
)
|
||||||
|
|
||||||
reasons: List[Reason] = []
|
reasons: List[Reason] = []
|
||||||
edges_dto: List[EdgeDTO] = []
|
edges_dto: List[EdgeDTO] = []
|
||||||
|
|
||||||
# 1. Semantische Gründe
|
# 2. Gründe generieren
|
||||||
if semantic_score > 0.85:
|
if semantic_score > 0.85:
|
||||||
reasons.append(Reason(kind="semantic", message="Sehr hohe textuelle Übereinstimmung.", score_impact=breakdown.semantic_contribution))
|
reasons.append(Reason(kind="semantic", message="Herausragende inhaltliche Übereinstimmung.", score_impact=base_val))
|
||||||
elif semantic_score > 0.70:
|
elif semantic_score > 0.70:
|
||||||
reasons.append(Reason(kind="semantic", message="Gute textuelle Übereinstimmung.", score_impact=breakdown.semantic_contribution))
|
reasons.append(Reason(kind="semantic", message="Gute inhaltliche Übereinstimmung.", score_impact=base_val))
|
||||||
|
|
||||||
# 2. Typ-Gründe
|
|
||||||
if type_weight != 1.0:
|
if type_weight != 1.0:
|
||||||
msg = "Bevorzugt" if type_weight > 1.0 else "Leicht abgewertet"
|
direction = "Bevorzugt" if type_weight > 1.0 else "Abgewertet"
|
||||||
reasons.append(Reason(kind="type", message=f"{msg} aufgrund des Typs '{note_type}'.", score_impact=base_val * config_w_impact))
|
reasons.append(Reason(kind="type", message=f"{direction} durch Typ-Profil '{note_type}'.", score_impact=base_val * (type_weight - 1.0)))
|
||||||
|
|
||||||
# 3. Lifecycle-Gründe
|
|
||||||
if status_mult != 1.0:
|
if status_mult != 1.0:
|
||||||
msg = "Status-Bonus" if status_mult > 1.0 else "Status-Malus"
|
impact_txt = "Belohnt" if status_mult > 1.0 else "Zurückgestellt"
|
||||||
reasons.append(Reason(kind="lifecycle", message=f"{msg} (Notiz ist '{payload.get('status', 'unknown')}').", score_impact=0.0))
|
reasons.append(Reason(kind="lifecycle", message=f"{impact_txt} (Status: {payload.get('status', 'draft')}).", score_impact=0.0))
|
||||||
|
|
||||||
# 4. Graph-Gründe (Edges) - FIX: Beachtet eingehende UND ausgehende Kanten
|
# 3. Kanten-Details extrahieren (Incoming + Outgoing für volle Sichtbarkeit)
|
||||||
if subgraph and target_note_id and edge_bonus > 0:
|
if subgraph and target_note_id and scoring_debug["edge_bonus"] > 0:
|
||||||
# Sammle alle relevanten Kanten (Incoming + Outgoing)
|
raw_edges = []
|
||||||
edges_raw = []
|
|
||||||
if hasattr(subgraph, "get_incoming_edges"):
|
if hasattr(subgraph, "get_incoming_edges"):
|
||||||
edges_raw.extend(subgraph.get_incoming_edges(target_note_id) or [])
|
raw_edges.extend(subgraph.get_incoming_edges(target_note_id) or [])
|
||||||
if hasattr(subgraph, "get_outgoing_edges"):
|
if hasattr(subgraph, "get_outgoing_edges"):
|
||||||
edges_raw.extend(subgraph.get_outgoing_edges(target_note_id) or [])
|
raw_edges.extend(subgraph.get_outgoing_edges(target_note_id) or [])
|
||||||
|
|
||||||
for edge in edges_raw:
|
for edge in raw_edges:
|
||||||
src = edge.get("source", target_note_id)
|
src, tgt = edge.get("source"), edge.get("target")
|
||||||
tgt = edge.get("target", target_note_id)
|
|
||||||
k = edge.get("kind", "edge")
|
k = edge.get("kind", "edge")
|
||||||
prov = edge.get("provenance", "rule")
|
prov = edge.get("provenance", "rule")
|
||||||
conf = float(edge.get("confidence", 1.0))
|
conf = float(edge.get("confidence", 1.0))
|
||||||
|
|
||||||
# Richtung bestimmen
|
# Richtung und Nachbar bestimmen
|
||||||
direction = "in" if tgt == target_note_id else "out"
|
is_incoming = (tgt == target_note_id)
|
||||||
peer_id = src if direction == "in" else tgt
|
neighbor = src if is_incoming else tgt
|
||||||
|
|
||||||
edges_dto.append(EdgeDTO(
|
edge_obj = EdgeDTO(
|
||||||
id=f"{src}->{tgt}:{k}", kind=k, source=src, target=tgt,
|
id=f"{src}->{tgt}:{k}", kind=k, source=src, target=tgt,
|
||||||
weight=conf, direction=direction, provenance=prov, confidence=conf
|
weight=conf, direction="in" if is_incoming else "out",
|
||||||
))
|
provenance=prov, confidence=conf
|
||||||
|
)
|
||||||
|
edges_dto.append(edge_obj)
|
||||||
|
|
||||||
# Die 3 stärksten Kanten als Begründung auflisten
|
# Die 3 stärksten Signale als Gründe formulieren
|
||||||
all_edges = sorted(edges_dto, key=lambda e: e.confidence, reverse=True)
|
top_edges = sorted(edges_dto, key=lambda e: e.confidence, reverse=True)
|
||||||
for top_e in all_edges[:3]:
|
for e in top_edges[:3]:
|
||||||
prov_txt = "Bestätigte" if top_e.provenance == "explicit" else "Vermutete (KI)"
|
prov_label = "Explizite" if e.provenance == "explicit" else "Heuristische"
|
||||||
dir_txt = "Referenz von" if top_e.direction == "in" else "Verweis auf"
|
boost_label = f" [Boost x{applied_boosts.get(e.kind)}]" if applied_boosts and e.kind in applied_boosts else ""
|
||||||
reasons.append(Reason(
|
|
||||||
kind="edge",
|
|
||||||
message=f"{prov_txt} Kante '{top_e.kind}': {dir_txt} '{top_e.peer_id if hasattr(top_e, 'peer_id') else (top_e.source if top_e.direction=='in' else top_e.target)}'.",
|
|
||||||
score_impact=edge_w_cfg * top_e.confidence,
|
|
||||||
details={"provenance": top_e.provenance, "kind": top_e.kind}
|
|
||||||
))
|
|
||||||
|
|
||||||
# 5. Zentralitäts-Gründe
|
msg = f"{prov_label} Verbindung ({e.kind}){boost_label} zu '{neighbor}'."
|
||||||
if cent_bonus > 0.01:
|
reasons.append(Reason(kind="edge", message=msg, score_impact=edge_w_cfg * e.confidence))
|
||||||
reasons.append(Reason(kind="centrality", message="Knoten liegt zentral im aktuellen Kontext-Graphen.", score_impact=breakdown.centrality_contribution))
|
|
||||||
|
|
||||||
return Explanation(breakdown=breakdown, reasons=reasons, related_edges=edges_dto if edges_dto else None)
|
if scoring_debug["cent_bonus"] > 0.01:
|
||||||
|
reasons.append(Reason(kind="centrality", message="Knoten ist ein zentraler Hub im Kontext.", score_impact=breakdown.centrality_contribution))
|
||||||
|
|
||||||
|
return Explanation(
|
||||||
|
breakdown=breakdown,
|
||||||
|
reasons=reasons,
|
||||||
|
related_edges=edges_dto if edges_dto else None,
|
||||||
|
applied_intent=getattr(ga, "_LAST_INTENT", "UNKNOWN"), # Debugging-Zweck
|
||||||
|
applied_boosts=applied_boosts
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
|
def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
|
||||||
"""Extrahiert depth und edge_types für die Expansion."""
|
"""Extrahiert Expansion-Tiefe und Kanten-Filter aus dem Request."""
|
||||||
expand = getattr(req, "expand", None)
|
expand = getattr(req, "expand", None)
|
||||||
if not expand:
|
if not expand:
|
||||||
return 0, None
|
return 0, None
|
||||||
|
|
||||||
depth = 1
|
depth = 1
|
||||||
edge_types: List[str] | None = None
|
edge_types = None
|
||||||
|
|
||||||
if hasattr(expand, "depth") or hasattr(expand, "edge_types"):
|
|
||||||
depth = int(getattr(expand, "depth", 1) or 1)
|
|
||||||
types_val = getattr(expand, "edge_types", None)
|
|
||||||
if types_val:
|
|
||||||
edge_types = list(types_val)
|
|
||||||
return depth, edge_types
|
|
||||||
|
|
||||||
if isinstance(expand, dict):
|
if isinstance(expand, dict):
|
||||||
if "depth" in expand:
|
depth = int(expand.get("depth", 1))
|
||||||
depth = int(expand.get("depth") or 1)
|
edge_types = expand.get("edge_types")
|
||||||
if "edge_types" in expand and expand["edge_types"] is not None:
|
if edge_types:
|
||||||
edge_types = list(expand["edge_types"])
|
edge_types = list(edge_types)
|
||||||
return depth, edge_types
|
return depth, edge_types
|
||||||
|
|
||||||
|
# Fallback für Pydantic Objekte
|
||||||
|
if hasattr(expand, "depth"):
|
||||||
|
return int(getattr(expand, "depth", 1)), getattr(expand, "edge_types", None)
|
||||||
|
|
||||||
return 0, None
|
return 0, None
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -285,51 +312,51 @@ def _build_hits_from_semantic(
|
||||||
explain: bool = False,
|
explain: bool = False,
|
||||||
dynamic_edge_boosts: Dict[str, float] = None
|
dynamic_edge_boosts: Dict[str, float] = None
|
||||||
) -> QueryResponse:
|
) -> QueryResponse:
|
||||||
"""Baut strukturierte QueryHits basierend auf Hybrid-Scoring."""
|
"""
|
||||||
|
Wandelt semantische Roh-Treffer in strukturierte QueryHits um.
|
||||||
|
Berechnet den finalen Score pro Hit unter Einbeziehung des Subgraphen.
|
||||||
|
"""
|
||||||
t0 = time.time()
|
t0 = time.time()
|
||||||
enriched: List[Tuple[str, float, Dict[str, Any], float, float, float]] = []
|
enriched = []
|
||||||
|
|
||||||
for pid, semantic_score, payload in hits:
|
for pid, semantic_score, payload in hits:
|
||||||
edge_bonus = 0.0
|
edge_bonus = 0.0
|
||||||
cent_bonus = 0.0
|
cent_bonus = 0.0
|
||||||
# WICHTIG für WP-22: Graph-Abfragen IMMER über die Note-ID, nicht Chunk-ID
|
# Graph-Abfrage erfolgt IMMER über die Note-ID
|
||||||
target_note_id = payload.get("note_id")
|
target_note_id = payload.get("note_id")
|
||||||
|
|
||||||
if subgraph is not None and target_note_id:
|
if subgraph is not None and target_note_id:
|
||||||
try:
|
try:
|
||||||
# edge_bonus nutzt intern bereits die confidence-gewichteten Pfade
|
|
||||||
edge_bonus = float(subgraph.edge_bonus(target_note_id))
|
edge_bonus = float(subgraph.edge_bonus(target_note_id))
|
||||||
except Exception:
|
|
||||||
edge_bonus = 0.0
|
|
||||||
try:
|
|
||||||
cent_bonus = float(subgraph.centrality_bonus(target_note_id))
|
cent_bonus = float(subgraph.centrality_bonus(target_note_id))
|
||||||
except Exception:
|
except Exception as e:
|
||||||
cent_bonus = 0.0
|
logger.debug(f"Graph signal failed for {target_note_id}: {e}")
|
||||||
|
|
||||||
total, eb, cb = _compute_total_score(
|
# Messbare Scoring-Daten via WP-22 Formel
|
||||||
|
debug_data = _compute_total_score(
|
||||||
semantic_score,
|
semantic_score,
|
||||||
payload,
|
payload,
|
||||||
edge_bonus_raw=edge_bonus,
|
edge_bonus_raw=edge_bonus,
|
||||||
cent_bonus_raw=cent_bonus,
|
cent_bonus_raw=cent_bonus,
|
||||||
dynamic_edge_boosts=dynamic_edge_boosts
|
dynamic_edge_boosts=dynamic_edge_boosts
|
||||||
)
|
)
|
||||||
enriched.append((pid, float(semantic_score), payload, total, eb, cb))
|
enriched.append((pid, float(semantic_score), payload, debug_data))
|
||||||
|
|
||||||
# Sortierung nach finalem Score
|
# Sortierung nach berechnetem Total Score
|
||||||
enriched_sorted = sorted(enriched, key=lambda h: h[3], reverse=True)
|
enriched_sorted = sorted(enriched, key=lambda h: h[3]["total"], reverse=True)
|
||||||
limited = enriched_sorted[: max(1, top_k)]
|
limited_hits = enriched_sorted[: max(1, top_k)]
|
||||||
|
|
||||||
results: List[QueryHit] = []
|
results: List[QueryHit] = []
|
||||||
for pid, semantic_score, payload, total, eb, cb in limited:
|
for pid, semantic_score, payload, debug in limited_hits:
|
||||||
explanation_obj = None
|
explanation_obj = None
|
||||||
if explain:
|
if explain:
|
||||||
explanation_obj = _build_explanation(
|
explanation_obj = _build_explanation(
|
||||||
semantic_score=float(semantic_score),
|
semantic_score=float(semantic_score),
|
||||||
payload=payload,
|
payload=payload,
|
||||||
edge_bonus=eb,
|
scoring_debug=debug,
|
||||||
cent_bonus=cb,
|
|
||||||
subgraph=subgraph,
|
subgraph=subgraph,
|
||||||
target_note_id=payload.get("note_id")
|
target_note_id=payload.get("note_id"),
|
||||||
|
applied_boosts=dynamic_edge_boosts
|
||||||
)
|
)
|
||||||
|
|
||||||
text_content = payload.get("page_content") or payload.get("text") or payload.get("content")
|
text_content = payload.get("page_content") or payload.get("text") or payload.get("content")
|
||||||
|
|
@ -338,10 +365,9 @@ def _build_hits_from_semantic(
|
||||||
node_id=str(pid),
|
node_id=str(pid),
|
||||||
note_id=payload.get("note_id", "unknown"),
|
note_id=payload.get("note_id", "unknown"),
|
||||||
semantic_score=float(semantic_score),
|
semantic_score=float(semantic_score),
|
||||||
edge_bonus=eb,
|
edge_bonus=debug["edge_bonus"],
|
||||||
centrality_bonus=cb,
|
centrality_bonus=debug["cent_bonus"],
|
||||||
total_score=total,
|
total_score=debug["total"],
|
||||||
paths=None,
|
|
||||||
source={
|
source={
|
||||||
"path": payload.get("path"),
|
"path": payload.get("path"),
|
||||||
"section": payload.get("section") or payload.get("section_title"),
|
"section": payload.get("section") or payload.get("section_title"),
|
||||||
|
|
@ -351,85 +377,78 @@ def _build_hits_from_semantic(
|
||||||
explanation=explanation_obj
|
explanation=explanation_obj
|
||||||
))
|
))
|
||||||
|
|
||||||
dt = int((time.time() - t0) * 1000)
|
dt_ms = int((time.time() - t0) * 1000)
|
||||||
return QueryResponse(results=results, used_mode=used_mode, latency_ms=dt)
|
return QueryResponse(results=results, used_mode=used_mode, latency_ms=dt_ms)
|
||||||
|
|
||||||
|
|
||||||
def semantic_retrieve(req: QueryRequest) -> QueryResponse:
|
def semantic_retrieve(req: QueryRequest) -> QueryResponse:
|
||||||
"""Reiner semantischer Retriever (WP-02)."""
|
"""Standard-Vektorsuche ohne Graph-Einfluss (WP-02)."""
|
||||||
client, prefix = _get_client_and_prefix()
|
client, prefix = _get_client_and_prefix()
|
||||||
vector = _get_query_vector(req)
|
vector = _get_query_vector(req)
|
||||||
top_k = req.top_k or get_settings().RETRIEVER_TOP_K
|
top_k = req.top_k or 10
|
||||||
|
|
||||||
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
||||||
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="semantic", subgraph=None, explain=req.explain)
|
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="semantic", subgraph=None, explain=req.explain)
|
||||||
|
|
||||||
|
|
||||||
def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
||||||
"""Hybrid-Retriever: semantische Suche + optionale Edge-Expansion (WP-04a)."""
|
"""
|
||||||
|
Hybrid-Suche: Kombiniert Semantik mit WP-22 Graph Intelligence.
|
||||||
|
Führt Expansion durch, gewichtet nach Provenance und appliziert Intent-Boosts.
|
||||||
|
"""
|
||||||
client, prefix = _get_client_and_prefix()
|
client, prefix = _get_client_and_prefix()
|
||||||
if req.query_vector:
|
vector = list(req.query_vector) if req.query_vector else _get_query_vector(req)
|
||||||
vector = list(req.query_vector)
|
top_k = req.top_k or 10
|
||||||
else:
|
|
||||||
vector = _get_query_vector(req)
|
|
||||||
|
|
||||||
top_k = req.top_k or get_settings().RETRIEVER_TOP_K
|
# 1. Semantische Seed-Suche
|
||||||
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
||||||
|
|
||||||
depth, edge_types = _extract_expand_options(req)
|
# 2. Graph Expansion & Custom Weighting
|
||||||
|
expand_depth, edge_types = _extract_expand_options(req)
|
||||||
# WP-22: Dynamic Boosts aus dem Request (vom Router)
|
boost_edges = getattr(req, "boost_edges", {}) or {}
|
||||||
boost_edges = getattr(req, "boost_edges", {})
|
|
||||||
|
|
||||||
subgraph: ga.Subgraph | None = None
|
subgraph: ga.Subgraph | None = None
|
||||||
if depth and depth > 0:
|
if expand_depth > 0 and hits:
|
||||||
seed_ids: List[str] = []
|
# Extrahiere Note-IDs der Treffer als Startpunkte für den Graphen
|
||||||
for _pid, _score, payload in hits:
|
seed_ids = list({h[2].get("note_id") for h in hits if h[2].get("note_id")})
|
||||||
key = payload.get("note_id")
|
|
||||||
if key and key not in seed_ids:
|
|
||||||
seed_ids.append(key)
|
|
||||||
if seed_ids:
|
if seed_ids:
|
||||||
try:
|
try:
|
||||||
# Subgraph laden
|
# Subgraph aus Qdrant laden
|
||||||
subgraph = ga.expand(client, prefix, seed_ids, depth=depth, edge_types=edge_types)
|
subgraph = ga.expand(client, prefix, seed_ids, depth=expand_depth, edge_types=edge_types)
|
||||||
|
|
||||||
# --- WP-22: Kanten-Boosts & Provenance-Weighting im RAM-Graphen ---
|
# WP-22: Transformation der Gewichte im RAM-Graphen vor Bonus-Berechnung
|
||||||
if subgraph and hasattr(subgraph, "graph"):
|
if subgraph and hasattr(subgraph, "graph"):
|
||||||
for u, v, data in subgraph.graph.edges(data=True):
|
for u, v, data in subgraph.graph.edges(data=True):
|
||||||
# 1. Herkunfts-Basisgewichtung (Concept 2.6)
|
# A. Provenance Weighting (WP-22 Herkunfts-Bonus)
|
||||||
prov = data.get("provenance", "rule")
|
prov = data.get("provenance", "rule")
|
||||||
prov_weight = 1.0
|
# Explicit=1.0, Smart=0.9, Rule=0.7
|
||||||
if prov == "smart": prov_weight = 0.9
|
prov_w = 1.0 if prov == "explicit" else (0.9 if prov == "smart" else 0.7)
|
||||||
elif prov == "rule": prov_weight = 0.7
|
|
||||||
|
|
||||||
# 2. Intent-basierter Multiplikator (Teil C)
|
# B. Intent Boost Multiplikator (Vom Router geladen)
|
||||||
k = data.get("kind")
|
k = data.get("kind")
|
||||||
intent_boost = 1.0
|
intent_multiplier = boost_edges.get(k, 1.0)
|
||||||
if boost_edges and k in boost_edges:
|
|
||||||
intent_boost = boost_edges[k]
|
|
||||||
|
|
||||||
# Finales Gewicht im Graphen setzen
|
# Finales Kanten-Gewicht im Graphen setzen
|
||||||
data["weight"] = data.get("weight", 1.0) * prov_weight * intent_boost
|
data["weight"] = data.get("weight", 1.0) * prov_w * intent_multiplier
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
|
logger.error(f"Graph expansion failed: {e}")
|
||||||
subgraph = None
|
subgraph = None
|
||||||
|
|
||||||
|
# 3. Scoring & Explanation Generierung
|
||||||
return _build_hits_from_semantic(
|
return _build_hits_from_semantic(
|
||||||
hits,
|
hits,
|
||||||
top_k=top_k,
|
top_k,
|
||||||
used_mode="hybrid",
|
"hybrid",
|
||||||
subgraph=subgraph,
|
subgraph,
|
||||||
explain=req.explain,
|
req.explain,
|
||||||
dynamic_edge_boosts=boost_edges
|
boost_edges
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class Retriever:
|
class Retriever:
|
||||||
"""
|
"""Wrapper-Klasse für die konsolidierte Retrieval-Logik."""
|
||||||
Wrapper-Klasse für Suchoperationen.
|
|
||||||
"""
|
|
||||||
def __init__(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
async def search(self, request: QueryRequest) -> QueryResponse:
|
async def search(self, request: QueryRequest) -> QueryResponse:
|
||||||
|
"""Führt eine hybride Suche aus. Asynchron für FastAPI-Integration."""
|
||||||
return hybrid_retrieve(request)
|
return hybrid_retrieve(request)
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
"""
|
"""
|
||||||
FILE: app/models/dto.py
|
FILE: app/models/dto.py
|
||||||
DESCRIPTION: Pydantic-Modelle (DTOs) für Request/Response Bodies. Definiert das API-Schema.
|
DESCRIPTION: Pydantic-Modelle (DTOs) für Request/Response Bodies. Definiert das API-Schema.
|
||||||
VERSION: 0.6.4 (WP-22 Semantic Graph Routing, Lifecycle & Provenance)
|
VERSION: 0.6.5 (WP-22 Debug & Verifiability Update)
|
||||||
STATUS: Active
|
STATUS: Active
|
||||||
DEPENDENCIES: pydantic, typing, uuid
|
DEPENDENCIES: pydantic, typing, uuid
|
||||||
LAST_ANALYSIS: 2025-12-18
|
LAST_ANALYSIS: 2025-12-18
|
||||||
|
|
@ -12,7 +12,6 @@ from pydantic import BaseModel, Field
|
||||||
from typing import List, Literal, Optional, Dict, Any
|
from typing import List, Literal, Optional, Dict, Any
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
# WP-22: Definition der gültigen Kanten-Typen gemäß Manual
|
|
||||||
EdgeKind = Literal["references", "references_at", "backlink", "next", "prev", "belongs_to", "depends_on", "related_to", "similar_to", "caused_by", "derived_from", "based_on", "solves", "blocks", "uses", "guides"]
|
EdgeKind = Literal["references", "references_at", "backlink", "next", "prev", "belongs_to", "depends_on", "related_to", "similar_to", "caused_by", "derived_from", "based_on", "solves", "blocks", "uses", "guides"]
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -41,7 +40,6 @@ class EdgeDTO(BaseModel):
|
||||||
target: str
|
target: str
|
||||||
weight: float
|
weight: float
|
||||||
direction: Literal["out", "in", "undirected"] = "out"
|
direction: Literal["out", "in", "undirected"] = "out"
|
||||||
# WP-22: Provenance Tracking (Herkunft und Vertrauen)
|
|
||||||
provenance: Optional[Literal["explicit", "rule", "smart", "structure"]] = "explicit"
|
provenance: Optional[Literal["explicit", "rule", "smart", "structure"]] = "explicit"
|
||||||
confidence: float = 1.0
|
confidence: float = 1.0
|
||||||
|
|
||||||
|
|
@ -60,22 +58,16 @@ class QueryRequest(BaseModel):
|
||||||
filters: Optional[Dict] = None
|
filters: Optional[Dict] = None
|
||||||
ret: Dict = {"with_paths": True, "with_notes": True, "with_chunks": True}
|
ret: Dict = {"with_paths": True, "with_notes": True, "with_chunks": True}
|
||||||
explain: bool = False
|
explain: bool = False
|
||||||
|
|
||||||
# WP-22: Semantic Graph Routing
|
|
||||||
# Erlaubt dem Router, Kantengewichte dynamisch zu überschreiben.
|
|
||||||
# Format: {"caused_by": 3.0, "related_to": 0.5}
|
|
||||||
boost_edges: Optional[Dict[str, float]] = None
|
boost_edges: Optional[Dict[str, float]] = None
|
||||||
|
|
||||||
|
|
||||||
class FeedbackRequest(BaseModel):
|
class FeedbackRequest(BaseModel):
|
||||||
"""
|
"""
|
||||||
User-Feedback zu einem spezifischen Treffer oder der Gesamtantwort (Basis für WP-08).
|
User-Feedback zu einem spezifischen Treffer oder der Gesamtantwort.
|
||||||
|
Basis für WP-08 (Self-Tuning).
|
||||||
"""
|
"""
|
||||||
query_id: str = Field(..., description="ID der ursprünglichen Suche")
|
query_id: str = Field(..., description="ID der ursprünglichen Suche")
|
||||||
# node_id ist optional: Wenn leer oder "generated_answer", gilt es für die Antwort.
|
|
||||||
# Wenn eine echte Chunk-ID, gilt es für die Quelle.
|
|
||||||
node_id: str = Field(..., description="ID des bewerteten Treffers oder 'generated_answer'")
|
node_id: str = Field(..., description="ID des bewerteten Treffers oder 'generated_answer'")
|
||||||
# Update: Range auf 1-5 erweitert für differenziertes Tuning
|
|
||||||
score: int = Field(..., ge=1, le=5, description="1 (Irrelevant/Falsch) bis 5 (Perfekt)")
|
score: int = Field(..., ge=1, le=5, description="1 (Irrelevant/Falsch) bis 5 (Perfekt)")
|
||||||
comment: Optional[str] = None
|
comment: Optional[str] = None
|
||||||
|
|
||||||
|
|
@ -86,7 +78,6 @@ class ChatRequest(BaseModel):
|
||||||
"""
|
"""
|
||||||
message: str = Field(..., description="Die Nachricht des Users")
|
message: str = Field(..., description="Die Nachricht des Users")
|
||||||
conversation_id: Optional[str] = Field(None, description="Optional: ID für Chat-Verlauf (noch nicht implementiert)")
|
conversation_id: Optional[str] = Field(None, description="Optional: ID für Chat-Verlauf (noch nicht implementiert)")
|
||||||
# RAG Parameter (Override defaults)
|
|
||||||
top_k: int = 5
|
top_k: int = 5
|
||||||
explain: bool = False
|
explain: bool = False
|
||||||
|
|
||||||
|
|
@ -102,11 +93,13 @@ class ScoreBreakdown(BaseModel):
|
||||||
raw_edge_bonus: float
|
raw_edge_bonus: float
|
||||||
raw_centrality: float
|
raw_centrality: float
|
||||||
node_weight: float
|
node_weight: float
|
||||||
|
# WP-22 Debug Fields
|
||||||
|
status_multiplier: float = 1.0
|
||||||
|
graph_boost_factor: float = 1.0
|
||||||
|
|
||||||
|
|
||||||
class Reason(BaseModel):
|
class Reason(BaseModel):
|
||||||
"""Ein semantischer Grund für das Ranking."""
|
"""Ein semantischer Grund für das Ranking."""
|
||||||
# WP-22: 'lifecycle' hinzugefügt für Status-Begründungen (Draft vs Stable)
|
|
||||||
kind: Literal["semantic", "edge", "type", "centrality", "lifecycle"]
|
kind: Literal["semantic", "edge", "type", "centrality", "lifecycle"]
|
||||||
message: str
|
message: str
|
||||||
score_impact: Optional[float] = None
|
score_impact: Optional[float] = None
|
||||||
|
|
@ -118,6 +111,9 @@ class Explanation(BaseModel):
|
||||||
breakdown: ScoreBreakdown
|
breakdown: ScoreBreakdown
|
||||||
reasons: List[Reason]
|
reasons: List[Reason]
|
||||||
related_edges: Optional[List[EdgeDTO]] = None
|
related_edges: Optional[List[EdgeDTO]] = None
|
||||||
|
# WP-22 Debug: Verifizierung des Routings
|
||||||
|
applied_intent: Optional[str] = None
|
||||||
|
applied_boosts: Optional[Dict[str, float]] = None
|
||||||
|
|
||||||
|
|
||||||
# --- Response Models ---
|
# --- Response Models ---
|
||||||
|
|
@ -132,7 +128,7 @@ class QueryHit(BaseModel):
|
||||||
total_score: float
|
total_score: float
|
||||||
paths: Optional[List[List[Dict]]] = None
|
paths: Optional[List[List[Dict]]] = None
|
||||||
source: Optional[Dict] = None
|
source: Optional[Dict] = None
|
||||||
payload: Optional[Dict] = None # Added for flexibility & WP-06 meta-data
|
payload: Optional[Dict] = None
|
||||||
explanation: Optional[Explanation] = None
|
explanation: Optional[Explanation] = None
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user