Erste Version des WP04b
This commit is contained in:
parent
9e7104e362
commit
34cbc0c0eb
|
|
@ -1,28 +1,22 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
app/core/graph_adapter.py — Adjazenzaufbau & Subgraph-Expansion (WP-04)
|
||||
app/core/graph_adapter.py — Adjazenzaufbau & Subgraph-Expansion
|
||||
|
||||
Zweck:
|
||||
Baut aus Qdrant-Edges (Collection: *_edges) einen leichten In-Memory-Graph
|
||||
und liefert Edge-basierte Kennzahlen (In-Degree, Out-Degree, edge_bonus).
|
||||
Baut aus Qdrant-Edges (Collection: *_edges) einen leichten In-Memory-Graph.
|
||||
|
||||
Kompatibilität:
|
||||
- Python 3.12+, qdrant-client 1.x
|
||||
- Wird von app/core/retriever.py im Hybrid-Modus genutzt.
|
||||
- Signaturen bleiben kompatibel zu den bestehenden Tests
|
||||
(tests/test_retriever_edges.py patcht expand()).
|
||||
- WP-04a: Liefert Scores (edge_bonus, centrality).
|
||||
- WP-04b: Liefert jetzt auch Struktur-Daten für Erklärungen (Reverse-Lookup).
|
||||
|
||||
Version:
|
||||
0.3.1 (2025-12-04 – note_id-bewusste Expansion:
|
||||
* Seeds = Note-IDs
|
||||
* Subgraph pflegt zusätzlich Adjazenz auf note_id-Basis,
|
||||
damit der Retriever mit note_id arbeiten kann)
|
||||
0.4.0 (Update für WP-04b: Reverse Adjacency für Explainability)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Optional, DefaultDict
|
||||
from typing import Dict, List, Optional, DefaultDict, Any
|
||||
from collections import defaultdict
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
|
|
@ -30,36 +24,27 @@ from qdrant_client.http import models as rest
|
|||
|
||||
from app.core.qdrant import collection_names
|
||||
|
||||
# Legacy-Import (wird aktuell nicht mehr verwendet, bleibt aber erhalten,
|
||||
# damit bestehende Importe/Mocks nicht brechen).
|
||||
# Legacy-Import Fallback
|
||||
try: # pragma: no cover
|
||||
from app.core.qdrant_points import get_edges_for_sources # type: ignore
|
||||
except Exception: # pragma: no cover
|
||||
get_edges_for_sources = None # type: ignore
|
||||
|
||||
|
||||
# Basisgewichte je Edge-Typ.
|
||||
# Diese Werte werden mit der in der Edge-Payload hinterlegten "confidence"
|
||||
# multipliziert, falls vorhanden.
|
||||
#
|
||||
# Hinweis:
|
||||
# - Strukturkanten (belongs_to, next/prev) sind schwächer gewichtet.
|
||||
# - Wissenskanten (depends_on, related_to, similar_to, references) erhalten
|
||||
# höhere Basisgewichte, damit sie sich im Retriever bemerkbar machen.
|
||||
# Basisgewichte je Edge-Typ (WP-04a Config)
|
||||
EDGE_BASE_WEIGHTS: Dict[str, float] = {
|
||||
# Struktur / Navigationskanten
|
||||
# Struktur
|
||||
"belongs_to": 0.10,
|
||||
"next": 0.06,
|
||||
"prev": 0.06,
|
||||
"backlink": 0.04,
|
||||
"references_at": 0.08,
|
||||
|
||||
# Wissenskanten
|
||||
# Wissen
|
||||
"references": 0.20,
|
||||
"depends_on": 0.18,
|
||||
"related_to": 0.15,
|
||||
"similar_to": 0.12,
|
||||
# weitere Typen erhalten per Default 0.0 und wirken nur über centrality
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -77,11 +62,8 @@ def _edge_weight(pl: Dict) -> float:
|
|||
if conf is None:
|
||||
return base
|
||||
|
||||
# Confidence vorsichtig in [0.0, 1.0] clampen
|
||||
if conf < 0.0:
|
||||
conf = 0.0
|
||||
if conf > 1.0:
|
||||
conf = 1.0
|
||||
if conf < 0.0: conf = 0.0
|
||||
if conf > 1.0: conf = 1.0
|
||||
|
||||
return base * conf
|
||||
|
||||
|
|
@ -95,48 +77,34 @@ def _fetch_edges(
|
|||
) -> List[Dict]:
|
||||
"""
|
||||
Holt Edges direkt aus der *_edges Collection.
|
||||
|
||||
Filter:
|
||||
- source_id IN seeds ODER target_id IN seeds ODER note_id IN seeds
|
||||
- optional: kind IN edge_types
|
||||
Filter: source_id IN seeds OR target_id IN seeds OR note_id IN seeds
|
||||
"""
|
||||
if not seeds or limit <= 0:
|
||||
return []
|
||||
|
||||
_, _, edges_col = collection_names(prefix)
|
||||
|
||||
# OR über source_id / target_id / note_id für alle Seeds
|
||||
seed_conditions = []
|
||||
for field in ("source_id", "target_id", "note_id"):
|
||||
for s in seeds:
|
||||
seed_conditions.append(
|
||||
rest.FieldCondition(
|
||||
key=field,
|
||||
match=rest.MatchValue(value=str(s)),
|
||||
)
|
||||
rest.FieldCondition(key=field, match=rest.MatchValue(value=str(s)))
|
||||
)
|
||||
seeds_filter = rest.Filter(should=seed_conditions) if seed_conditions else None
|
||||
|
||||
# Optional: Filter auf bestimmte Edge-Typen (kind)
|
||||
type_filter = None
|
||||
if edge_types:
|
||||
type_conds = [
|
||||
rest.FieldCondition(
|
||||
key="kind",
|
||||
match=rest.MatchValue(value=str(k)),
|
||||
)
|
||||
rest.FieldCondition(key="kind", match=rest.MatchValue(value=str(k)))
|
||||
for k in edge_types
|
||||
]
|
||||
type_filter = rest.Filter(should=type_conds)
|
||||
|
||||
flt = None
|
||||
must = []
|
||||
if seeds_filter:
|
||||
must.append(seeds_filter)
|
||||
if type_filter:
|
||||
must.append(type_filter)
|
||||
if must:
|
||||
flt = rest.Filter(must=must)
|
||||
if seeds_filter: must.append(seeds_filter)
|
||||
if type_filter: must.append(type_filter)
|
||||
|
||||
flt = rest.Filter(must=must) if must else None
|
||||
|
||||
pts, _ = client.scroll(
|
||||
collection_name=edges_col,
|
||||
|
|
@ -155,28 +123,21 @@ def _fetch_edges(
|
|||
|
||||
|
||||
class Subgraph:
|
||||
"""Leichtgewichtiger Subgraph mit Adjazenzlisten & einfachen Kennzahlen."""
|
||||
"""Leichtgewichtiger Subgraph mit Adjazenzlisten & Kennzahlen."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Forward: source -> [targets]
|
||||
self.adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||
# Reverse: target -> [sources] (Neu für WP-04b Explanation)
|
||||
self.reverse_adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||
|
||||
self.in_degree: DefaultDict[str, int] = defaultdict(int)
|
||||
self.out_degree: DefaultDict[str, int] = defaultdict(int)
|
||||
|
||||
def add_edge(self, e: Dict) -> None:
|
||||
"""
|
||||
Fügt eine Kante zum Subgraph hinzu.
|
||||
|
||||
e enthält:
|
||||
- source: Knoten-ID (z. B. source_id)
|
||||
- target: Knoten-ID (z. B. target_id)
|
||||
- kind: Kantentyp
|
||||
- weight: bereits berechnetes Gewicht
|
||||
- note_id: (optional) Kontext-Note dieser Kante
|
||||
|
||||
Wichtig für den Retriever:
|
||||
- Wir pflegen Adjazenz primär unter `source`.
|
||||
- Zusätzlich pflegen wir, falls vorhanden, eine Adjazenz
|
||||
unter `note_id`, damit der Retriever mit note_id arbeiten kann.
|
||||
Fügt eine Kante hinzu und aktualisiert Forward/Reverse Indizes.
|
||||
e muss enthalten: source, target, kind, weight.
|
||||
"""
|
||||
src = e.get("source")
|
||||
tgt = e.get("target")
|
||||
|
|
@ -187,65 +148,57 @@ class Subgraph:
|
|||
if not src or not tgt:
|
||||
return
|
||||
|
||||
# Primäre Adjazenz (z. B. source_id → target_id)
|
||||
self.adj[src].append(
|
||||
{
|
||||
"target": tgt,
|
||||
"kind": kind,
|
||||
"weight": weight,
|
||||
}
|
||||
)
|
||||
# 1. Primäre Adjazenz (Forward)
|
||||
edge_data = {"target": tgt, "kind": kind, "weight": weight}
|
||||
self.adj[src].append(edge_data)
|
||||
self.out_degree[src] += 1
|
||||
self.in_degree[tgt] += 1
|
||||
|
||||
# Sekundäre Adjazenz auf Note-Ebene:
|
||||
# Falls eine Kontext-Note existiert, bekommt diese ebenfalls
|
||||
# "Ausgangskanten" angerechnet. So kann der Retriever über note_id
|
||||
# edge_bonus(...) und centrality_bonus(...) abfragen.
|
||||
if owner:
|
||||
# same edge unter owner registrieren (falls nicht identisch mit src),
|
||||
# damit edge_bonus(owner) > 0 werden kann.
|
||||
if owner != src:
|
||||
self.adj[owner].append(
|
||||
{
|
||||
"target": tgt,
|
||||
"kind": kind,
|
||||
"weight": weight,
|
||||
}
|
||||
)
|
||||
# 2. Reverse Adjazenz (Neu für Explanation)
|
||||
# Wir speichern, woher die Kante kam.
|
||||
rev_data = {"source": src, "kind": kind, "weight": weight}
|
||||
self.reverse_adj[tgt].append(rev_data)
|
||||
|
||||
# 3. Kontext-Note Handling (Forward & Reverse)
|
||||
# Wenn eine Kante "im Kontext einer Note" (owner) definiert ist,
|
||||
# schreiben wir sie der Note gut, damit der Retriever Scores auf Note-Ebene findet.
|
||||
if owner and owner != src:
|
||||
# Forward: Owner -> Target
|
||||
self.adj[owner].append(edge_data)
|
||||
self.out_degree[owner] += 1
|
||||
|
||||
# Zentralität der Kontext-Note leicht erhöhen
|
||||
# Reverse: Target wird vom Owner referenziert (indirekt)
|
||||
if owner != tgt:
|
||||
self.in_degree[owner] += 1
|
||||
rev_owner_data = {"source": owner, "kind": kind, "weight": weight, "via_context": True}
|
||||
self.reverse_adj[tgt].append(rev_owner_data)
|
||||
self.in_degree[owner] += 1 # Leichter Centrality Boost für den Owner
|
||||
|
||||
def aggregate_edge_bonus(self, node_id: str) -> float:
|
||||
"""
|
||||
Summe der ausgehenden Kantengewichte für einen Knoten.
|
||||
"""
|
||||
"""Summe der ausgehenden Kantengewichte (Hub-Score)."""
|
||||
return sum(edge["weight"] for edge in self.adj.get(node_id, []))
|
||||
|
||||
def edge_bonus(self, node_id: str) -> float:
|
||||
"""
|
||||
Kompatibilitäts-Methode für den Retriever.
|
||||
|
||||
Der Retriever ruft subgraph.edge_bonus(node_id) auf. Intern verwenden
|
||||
wir aggregate_edge_bonus(...), um bestehende Tests nicht zu brechen.
|
||||
"""
|
||||
"""API für Retriever (WP-04a Kompatibilität)."""
|
||||
return self.aggregate_edge_bonus(node_id)
|
||||
|
||||
def centrality_bonus(self, node_id: str) -> float:
|
||||
"""
|
||||
Einfache log-gedämpfte Zentralität auf Basis des In-Degree.
|
||||
Obergrenze: 0.15
|
||||
"""
|
||||
"""Log-gedämpfte Zentralität (In-Degree)."""
|
||||
import math
|
||||
|
||||
indeg = self.in_degree.get(node_id, 0)
|
||||
if indeg <= 0:
|
||||
return 0.0
|
||||
return min(math.log1p(indeg) / 10.0, 0.15)
|
||||
|
||||
# --- WP-04b Explanation Helpers ---
|
||||
|
||||
def get_outgoing_edges(self, node_id: str) -> List[Dict[str, Any]]:
|
||||
"""Liefert Liste aller Ziele, auf die dieser Knoten zeigt."""
|
||||
return self.adj.get(node_id, [])
|
||||
|
||||
def get_incoming_edges(self, node_id: str) -> List[Dict[str, Any]]:
|
||||
"""Liefert Liste aller Quellen, die auf diesen Knoten zeigen."""
|
||||
return self.reverse_adj.get(node_id, [])
|
||||
|
||||
|
||||
def expand(
|
||||
client: QdrantClient,
|
||||
|
|
@ -255,12 +208,7 @@ def expand(
|
|||
edge_types: Optional[List[str]] = None,
|
||||
) -> Subgraph:
|
||||
"""
|
||||
Expandiert ab Seeds entlang von Edges (bis `depth`), optional gefiltert
|
||||
nach Edge-Typen.
|
||||
|
||||
Seeds sind stabile payload-IDs (in unserem Fall vor allem note_id).
|
||||
Es werden Edges berücksichtigt, bei denen source_id ODER target_id ODER
|
||||
note_id einem der Seeds entspricht.
|
||||
Expandiert ab Seeds entlang von Edges (bis `depth`).
|
||||
"""
|
||||
sg = Subgraph()
|
||||
frontier = set(seeds)
|
||||
|
|
@ -284,19 +232,22 @@ def expand(
|
|||
for pl in edges_payloads:
|
||||
src = pl.get("source_id")
|
||||
tgt = pl.get("target_id")
|
||||
kind = pl.get("kind", "edge")
|
||||
owner = pl.get("note_id")
|
||||
|
||||
# Skip invalid edges
|
||||
if not src or not tgt:
|
||||
continue
|
||||
|
||||
e = {
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"kind": kind,
|
||||
"kind": pl.get("kind", "edge"),
|
||||
"weight": _edge_weight(pl),
|
||||
"note_id": owner,
|
||||
"note_id": pl.get("note_id"),
|
||||
}
|
||||
sg.add_edge(e)
|
||||
|
||||
if tgt:
|
||||
# Nur weitersuchen, wenn Target noch nicht besucht
|
||||
if tgt and tgt not in visited:
|
||||
next_frontier.add(tgt)
|
||||
|
||||
visited |= frontier
|
||||
|
|
|
|||
|
|
@ -3,10 +3,18 @@ from __future__ import annotations
|
|||
import os
|
||||
import time
|
||||
from functools import lru_cache
|
||||
from typing import Any, Dict, List, Tuple, Iterable
|
||||
from typing import Any, Dict, List, Tuple, Iterable, Optional
|
||||
|
||||
from app.config import get_settings
|
||||
from app.models.dto import QueryRequest, QueryResponse, QueryHit
|
||||
from app.models.dto import (
|
||||
QueryRequest,
|
||||
QueryResponse,
|
||||
QueryHit,
|
||||
Explanation,
|
||||
ScoreBreakdown,
|
||||
Reason,
|
||||
EdgeDTO
|
||||
)
|
||||
import app.core.qdrant as qdr
|
||||
import app.core.qdrant_points as qp
|
||||
import app.services.embeddings_client as ec
|
||||
|
|
@ -23,12 +31,7 @@ def _get_scoring_weights() -> Tuple[float, float, float]:
|
|||
"""Liefert (semantic_weight, edge_weight, centrality_weight) für den Retriever.
|
||||
|
||||
Priorität:
|
||||
1. Werte aus config/retriever.yaml (falls vorhanden und gültig),
|
||||
Abschnitt:
|
||||
scoring:
|
||||
semantic_weight: 1.0
|
||||
edge_weight: 0.5
|
||||
centrality_weight: 0.5
|
||||
1. Werte aus config/retriever.yaml (falls vorhanden und gültig).
|
||||
2. Fallback auf Settings.RETRIEVER_W_* (ENV-basiert).
|
||||
"""
|
||||
settings = get_settings()
|
||||
|
|
@ -64,10 +67,6 @@ def _get_client_and_prefix() -> Tuple[Any, str]:
|
|||
def _get_query_vector(req: QueryRequest) -> List[float]:
|
||||
"""
|
||||
Liefert den Query-Vektor aus dem Request.
|
||||
|
||||
- Falls req.query_vector gesetzt ist, wird dieser unverändert genutzt.
|
||||
- Andernfalls wird req.query über den Embedding-Service in einen Vektor
|
||||
transformiert.
|
||||
"""
|
||||
if req.query_vector:
|
||||
return list(req.query_vector)
|
||||
|
|
@ -78,11 +77,9 @@ def _get_query_vector(req: QueryRequest) -> List[float]:
|
|||
settings = get_settings()
|
||||
model_name = settings.MODEL_NAME
|
||||
|
||||
# Kompatibel mit Fakes in Unit-Tests (ohne model_name-Parameter)
|
||||
try:
|
||||
return ec.embed_text(req.query, model_name=model_name) # type: ignore[call-arg]
|
||||
except TypeError:
|
||||
# Fallback: einfache Signatur embed_text(text)
|
||||
return ec.embed_text(req.query) # type: ignore[call-arg]
|
||||
|
||||
|
||||
|
|
@ -93,13 +90,7 @@ def _semantic_hits(
|
|||
top_k: int,
|
||||
filters: Dict[str, Any] | None = None,
|
||||
) -> List[Tuple[str, float, Dict[str, Any]]]:
|
||||
"""Führt eine semantische Suche über mindnet_chunks aus und liefert Roh-Treffer.
|
||||
|
||||
Rückgabeformat: Liste von (point_id, score, payload)
|
||||
|
||||
Erwartetes Format von qp.search_chunks_by_vector:
|
||||
List[Tuple[str, float, dict]]
|
||||
"""
|
||||
"""Führt eine semantische Suche über mindnet_chunks aus und liefert Roh-Treffer."""
|
||||
flt = filters or None
|
||||
raw_hits = qp.search_chunks_by_vector(client, prefix, vector, top=top_k, filters=flt)
|
||||
results: List[Tuple[str, float, Dict[str, Any]]] = []
|
||||
|
|
@ -108,30 +99,13 @@ def _semantic_hits(
|
|||
return results
|
||||
|
||||
|
||||
def _get_semantic_score_and_payload(
|
||||
hit: Tuple[str, float, Dict[str, Any]]
|
||||
) -> Tuple[float, Dict[str, Any]]:
|
||||
"""Extrahiert semantic_score und Payload aus einem Raw-Hit."""
|
||||
_, score, payload = hit
|
||||
return float(score), payload or {}
|
||||
|
||||
|
||||
def _compute_total_score(
|
||||
semantic_score: float,
|
||||
payload: Dict[str, Any],
|
||||
edge_bonus: float = 0.0,
|
||||
cent_bonus: float = 0.0,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""Berechnet total_score aus semantic_score, retriever_weight und Graph-Boni.
|
||||
|
||||
Formel (WP-04, konfigurierbar über config/retriever.yaml bzw. ENV):
|
||||
total_score = W_sem * semantic_score * max(retriever_weight, 0.0)
|
||||
+ W_edge * edge_bonus
|
||||
+ W_cent * cent_bonus
|
||||
|
||||
- retriever_weight stammt aus dem Chunk-Payload (types.yaml).
|
||||
- W_sem / W_edge / W_cent kommen aus _get_scoring_weights().
|
||||
"""
|
||||
"""Berechnet total_score aus semantic_score, retriever_weight und Graph-Boni."""
|
||||
raw_weight = payload.get("retriever_weight", 1.0)
|
||||
try:
|
||||
weight = float(raw_weight)
|
||||
|
|
@ -145,12 +119,128 @@ def _compute_total_score(
|
|||
return float(total), float(edge_bonus), float(cent_bonus)
|
||||
|
||||
|
||||
def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
|
||||
"""Extrahiert depth und edge_types aus req.expand, falls vorhanden.
|
||||
# --- WP-04b Explanation Logic ---
|
||||
|
||||
- Falls expand nicht gesetzt ist: depth=0, edge_types=None (keine Expansion).
|
||||
- Unterstützt sowohl Pydantic-Modelle als auch plain dicts.
|
||||
def _build_explanation(
|
||||
semantic_score: float,
|
||||
payload: Dict[str, Any],
|
||||
edge_bonus: float,
|
||||
cent_bonus: float,
|
||||
subgraph: Optional[ga.Subgraph],
|
||||
node_key: Optional[str]
|
||||
) -> Explanation:
|
||||
"""
|
||||
Erstellt ein detailliertes Explanation-Objekt für einen Treffer.
|
||||
Analysiert Scores, Typen und eingehende Kanten.
|
||||
"""
|
||||
# 1. Weights & Config laden
|
||||
sem_w, edge_w, cent_w = _get_scoring_weights()
|
||||
|
||||
try:
|
||||
type_weight = float(payload.get("retriever_weight", 1.0))
|
||||
except (TypeError, ValueError):
|
||||
type_weight = 1.0
|
||||
|
||||
note_type = payload.get("type", "unknown")
|
||||
|
||||
# 2. Score Breakdown berechnen
|
||||
breakdown = ScoreBreakdown(
|
||||
semantic_contribution=(sem_w * semantic_score * type_weight),
|
||||
edge_contribution=(edge_w * edge_bonus),
|
||||
centrality_contribution=(cent_w * cent_bonus),
|
||||
raw_semantic=semantic_score,
|
||||
raw_edge_bonus=edge_bonus,
|
||||
raw_centrality=cent_bonus,
|
||||
node_weight=type_weight
|
||||
)
|
||||
|
||||
reasons: List[Reason] = []
|
||||
edges_dto: List[EdgeDTO] = []
|
||||
|
||||
# 3. Semantische Gründe
|
||||
if semantic_score > 0.85:
|
||||
reasons.append(Reason(
|
||||
kind="semantic",
|
||||
message="Sehr hohe textuelle Übereinstimmung mit der Anfrage.",
|
||||
score_impact=breakdown.semantic_contribution
|
||||
))
|
||||
elif semantic_score > 0.75:
|
||||
reasons.append(Reason(
|
||||
kind="semantic",
|
||||
message="Gute textuelle Übereinstimmung.",
|
||||
score_impact=breakdown.semantic_contribution
|
||||
))
|
||||
|
||||
# 4. Typ-Gründe
|
||||
if type_weight > 1.0:
|
||||
reasons.append(Reason(
|
||||
kind="type",
|
||||
message=f"Bevorzugt aufgrund des Typs '{note_type}' (Gewicht: {type_weight}).",
|
||||
score_impact=(sem_w * semantic_score * (type_weight - 1.0)) # Delta
|
||||
))
|
||||
elif type_weight < 1.0:
|
||||
reasons.append(Reason(
|
||||
kind="type",
|
||||
message=f"Abgewertet aufgrund des Typs '{note_type}' (Gewicht: {type_weight}).",
|
||||
score_impact=None
|
||||
))
|
||||
|
||||
# 5. Graph-Gründe (Incoming Edges)
|
||||
if subgraph and node_key and edge_bonus > 0:
|
||||
# Wir suchen nach eingehenden Kanten, die diesen Bonus verursacht haben.
|
||||
# graph_adapter.py (v0.4.0) muss get_incoming_edges bereitstellen.
|
||||
if hasattr(subgraph, "get_incoming_edges"):
|
||||
incoming = subgraph.get_incoming_edges(node_key)
|
||||
|
||||
# Sortieren nach Gewicht (stärkste zuerst)
|
||||
incoming_sorted = sorted(incoming, key=lambda e: e.get("weight", 0.0), reverse=True)
|
||||
|
||||
# Top-3 Gründe extrahieren
|
||||
for idx, edge in enumerate(incoming_sorted[:3]):
|
||||
src = edge.get("source", "Unknown")
|
||||
kind = edge.get("kind", "edge")
|
||||
weight = edge.get("weight", 0.0)
|
||||
|
||||
msg = f"Verbunden mit '{src}' via '{kind}'"
|
||||
reasons.append(Reason(
|
||||
kind="edge",
|
||||
message=msg,
|
||||
score_impact=(edge_w * weight),
|
||||
details={"source": src, "kind": kind, "weight": weight}
|
||||
))
|
||||
|
||||
# EdgeDTO für die API
|
||||
edges_dto.append(EdgeDTO(
|
||||
id=f"{src}->{node_key}:{kind}", # Synthetische ID für Anzeige
|
||||
kind=kind,
|
||||
source=src,
|
||||
target=node_key,
|
||||
weight=weight,
|
||||
direction="in"
|
||||
))
|
||||
else:
|
||||
# Fallback, falls GraphAdapter noch alt
|
||||
reasons.append(Reason(kind="edge", message="Knoten ist im Kontext-Graphen vernetzt."))
|
||||
|
||||
# 6. Centrality Gründe
|
||||
if cent_bonus > 0.05:
|
||||
reasons.append(Reason(
|
||||
kind="centrality",
|
||||
message="Knoten ist ein zentraler Hub im Kontext der Anfrage.",
|
||||
score_impact=breakdown.centrality_contribution
|
||||
))
|
||||
|
||||
return Explanation(
|
||||
breakdown=breakdown,
|
||||
reasons=reasons,
|
||||
related_edges=edges_dto if edges_dto else None
|
||||
)
|
||||
|
||||
# --- End Explanation Logic ---
|
||||
|
||||
|
||||
def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
|
||||
"""Extrahiert depth und edge_types aus req.expand."""
|
||||
expand = getattr(req, "expand", None)
|
||||
if not expand:
|
||||
return 0, None
|
||||
|
|
@ -158,7 +248,6 @@ def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
|
|||
depth = 1
|
||||
edge_types: List[str] | None = None
|
||||
|
||||
# Pydantic-Modell oder Objekt mit Attributen
|
||||
if hasattr(expand, "depth") or hasattr(expand, "edge_types"):
|
||||
depth = int(getattr(expand, "depth", 1) or 1)
|
||||
types_val = getattr(expand, "edge_types", None)
|
||||
|
|
@ -166,7 +255,6 @@ def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
|
|||
edge_types = list(types_val)
|
||||
return depth, edge_types
|
||||
|
||||
# Plain dict
|
||||
if isinstance(expand, dict):
|
||||
if "depth" in expand:
|
||||
depth = int(expand.get("depth") or 1)
|
||||
|
|
@ -182,24 +270,22 @@ def _build_hits_from_semantic(
|
|||
top_k: int,
|
||||
used_mode: str,
|
||||
subgraph: ga.Subgraph | None = None,
|
||||
explain: bool = False, # WP-04b
|
||||
) -> QueryResponse:
|
||||
"""Baut aus Raw-Hits und optionalem Subgraph strukturierte QueryHits.
|
||||
|
||||
- Aggregation auf Note-Ebene (note_id),
|
||||
- Berechnung von total_score unter Nutzung von retriever_weight + Graph-Boni,
|
||||
- Rückgabe als QueryResponse mit Latenz.
|
||||
WP-04b: Wenn explain=True, wird _build_explanation aufgerufen.
|
||||
"""
|
||||
t0 = time.time()
|
||||
|
||||
enriched: List[Tuple[str, float, Dict[str, Any], float, float, float]] = []
|
||||
|
||||
for pid, semantic_score, payload in hits:
|
||||
# Graph-Scores, falls Subgraph und stabiler Key vorhanden
|
||||
edge_bonus = 0.0
|
||||
cent_bonus = 0.0
|
||||
if subgraph is not None:
|
||||
node_key = payload.get("chunk_id") or payload.get("note_id")
|
||||
if node_key:
|
||||
|
||||
if subgraph is not None and node_key:
|
||||
try:
|
||||
edge_bonus = float(subgraph.edge_bonus(node_key))
|
||||
except Exception:
|
||||
|
|
@ -225,8 +311,20 @@ def _build_hits_from_semantic(
|
|||
for pid, semantic_score, payload, total, edge_bonus, cent_bonus in limited:
|
||||
note_id = payload.get("note_id")
|
||||
path = payload.get("path")
|
||||
# mindnet_chunks: section (aktuell genutzt); ältere Stände nutzen ggf. section_title
|
||||
section = payload.get("section") or payload.get("section_title")
|
||||
node_key = payload.get("chunk_id") or payload.get("note_id")
|
||||
|
||||
# WP-04b: Explanation bauen?
|
||||
explanation_obj = None
|
||||
if explain:
|
||||
explanation_obj = _build_explanation(
|
||||
semantic_score=float(semantic_score),
|
||||
payload=payload,
|
||||
edge_bonus=edge_bonus,
|
||||
cent_bonus=cent_bonus,
|
||||
subgraph=subgraph,
|
||||
node_key=node_key
|
||||
)
|
||||
|
||||
results.append(
|
||||
QueryHit(
|
||||
|
|
@ -241,6 +339,7 @@ def _build_hits_from_semantic(
|
|||
"path": path,
|
||||
"section": section,
|
||||
},
|
||||
explanation=explanation_obj
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -255,22 +354,21 @@ def semantic_retrieve(req: QueryRequest) -> QueryResponse:
|
|||
top_k = req.top_k or get_settings().RETRIEVER_TOP_K
|
||||
|
||||
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
||||
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="semantic", subgraph=None)
|
||||
|
||||
# explain Flag durchreichen
|
||||
return _build_hits_from_semantic(
|
||||
hits,
|
||||
top_k=top_k,
|
||||
used_mode="semantic",
|
||||
subgraph=None,
|
||||
explain=req.explain
|
||||
)
|
||||
|
||||
|
||||
def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
||||
"""Hybrid-Retriever: semantische Suche + optionale Edge-Expansion.
|
||||
|
||||
Aktueller Stand (Step 3):
|
||||
- Semantische Suche über mindnet_chunks (wie semantic_retrieve),
|
||||
- Bei expand.depth > 0 wird ein Subgraph aus mindnet_edges aufgebaut,
|
||||
und edge_bonus / centrality_bonus in das Scoring einbezogen.
|
||||
"""
|
||||
"""Hybrid-Retriever: semantische Suche + optionale Edge-Expansion."""
|
||||
client, prefix = _get_client_and_prefix()
|
||||
|
||||
# Query-Vektor:
|
||||
# - Falls explizit gesetzt, unverändert nutzen.
|
||||
# - Andernfalls über embed_text erzeugen.
|
||||
if req.query_vector:
|
||||
vector = list(req.query_vector)
|
||||
else:
|
||||
|
|
@ -283,7 +381,6 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
|||
depth, edge_types = _extract_expand_options(req)
|
||||
subgraph: ga.Subgraph | None = None
|
||||
if depth and depth > 0:
|
||||
# Seeds: stabile IDs aus dem Payload (chunk_id bevorzugt, sonst note_id)
|
||||
seed_ids: List[str] = []
|
||||
for _pid, _score, payload in hits:
|
||||
key = payload.get("chunk_id") or payload.get("note_id")
|
||||
|
|
@ -294,7 +391,13 @@ def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
|||
try:
|
||||
subgraph = ga.expand(client, prefix, seed_ids, depth=depth, edge_types=edge_types)
|
||||
except Exception:
|
||||
# Edge-Expansion ist optional: bei Fehlern weiter ohne Graph-Boni
|
||||
subgraph = None
|
||||
|
||||
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="hybrid", subgraph=subgraph)
|
||||
# explain Flag durchreichen
|
||||
return _build_hits_from_semantic(
|
||||
hits,
|
||||
top_k=top_k,
|
||||
used_mode="hybrid",
|
||||
subgraph=subgraph,
|
||||
explain=req.explain
|
||||
)
|
||||
|
|
@ -3,27 +3,27 @@ app/models/dto.py — Pydantic-Modelle (DTOs) für WP-04 Endpunkte
|
|||
|
||||
Zweck:
|
||||
Laufzeit-Modelle für FastAPI (Requests/Responses), getrennt von JSON-Schemas.
|
||||
Deckt die Graph-/Retriever-Endpunkte ab, ohne bestehende Funktionen zu ändern.
|
||||
Deckt die Graph-/Retriever-Endpunkte ab.
|
||||
Enthält Erweiterungen für WP-04b (Explanation Layer).
|
||||
|
||||
Kompatibilität:
|
||||
Python 3.12+, Pydantic 2.x, FastAPI 0.110+
|
||||
Version:
|
||||
0.1.0 (Erstanlage für WP-04)
|
||||
0.2.0 (Update für WP-04b Explanation Layer)
|
||||
Stand:
|
||||
2025-10-07
|
||||
2025-12-07
|
||||
Bezug:
|
||||
- schemas/*.json (Speicherschema für Notes/Chunks/Edges)
|
||||
- WP-04 API-Design (Query- und Graph-Endpunkte)
|
||||
Nutzung:
|
||||
from app.models.dto import QueryRequest, QueryResponse, GraphResponse
|
||||
Änderungsverlauf:
|
||||
0.1.0 (2025-10-07) – Erstanlage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Literal, Optional, Dict
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Literal, Optional, Dict, Any
|
||||
|
||||
EdgeKind = Literal["references", "references_at", "backlink", "next", "prev", "belongs_to"]
|
||||
EdgeKind = Literal["references", "references_at", "backlink", "next", "prev", "belongs_to", "depends_on", "related_to", "similar_to"]
|
||||
|
||||
|
||||
class NodeDTO(BaseModel):
|
||||
|
|
@ -44,7 +44,7 @@ class NodeDTO(BaseModel):
|
|||
class EdgeDTO(BaseModel):
|
||||
"""Darstellung einer Kante im API-Graph."""
|
||||
id: str
|
||||
kind: EdgeKind
|
||||
kind: str # String statt Literal, um flexibel für Custom-Types zu bleiben
|
||||
source: str
|
||||
target: str
|
||||
weight: float
|
||||
|
|
@ -57,27 +57,79 @@ class QueryRequest(BaseModel):
|
|||
- mode: 'semantic' | 'edge' | 'hybrid'
|
||||
- query: (optional) Freitext; Embedding wird später angebunden
|
||||
- query_vector: (optional) direkter Vektor (384d) für Quick-Tests ohne Embedding
|
||||
- explain: (optional) Fordert detaillierte Erklärungen an (WP-04b)
|
||||
"""
|
||||
mode: Literal["semantic", "edge", "hybrid"] = "hybrid"
|
||||
query: Optional[str] = None
|
||||
query_vector: Optional[List[float]] = None
|
||||
top_k: int = 10
|
||||
expand: Dict = {"depth": 1, "edge_types": ["references", "belongs_to", "prev", "next"]}
|
||||
expand: Dict = {"depth": 1, "edge_types": ["references", "belongs_to", "prev", "next", "depends_on", "related_to"]}
|
||||
filters: Optional[Dict] = None
|
||||
# Flags zur Steuerung der Rückgabe
|
||||
ret: Dict = {"with_paths": True, "with_notes": True, "with_chunks": True}
|
||||
# WP-04b: Soll eine Erklärung generiert werden?
|
||||
explain: bool = False
|
||||
|
||||
|
||||
# --- WP-04b Explanation Models ---
|
||||
|
||||
class ScoreBreakdown(BaseModel):
|
||||
"""
|
||||
Aufschlüsselung der Score-Komponenten.
|
||||
Zeigt die gewichteten Beiträge zum Total Score.
|
||||
"""
|
||||
semantic_contribution: float = Field(..., description="W_sem * semantic_score * weight")
|
||||
edge_contribution: float = Field(..., description="W_edge * edge_bonus")
|
||||
centrality_contribution: float = Field(..., description="W_cent * centrality_bonus")
|
||||
|
||||
# Rohwerte für Transparenz
|
||||
raw_semantic: float
|
||||
raw_edge_bonus: float
|
||||
raw_centrality: float
|
||||
node_weight: float = Field(..., description="Typ-Gewicht (retriever_weight)")
|
||||
|
||||
|
||||
class Reason(BaseModel):
|
||||
"""
|
||||
Ein semantischer Grund für das Ranking.
|
||||
z.B. 'Verlinkt von Projekt X', 'Hohe Textähnlichkeit'.
|
||||
"""
|
||||
kind: Literal["semantic", "edge", "type", "centrality"]
|
||||
message: str
|
||||
score_impact: Optional[float] = None
|
||||
details: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class Explanation(BaseModel):
|
||||
"""
|
||||
Container für alle Erklärungsdaten eines Treffers.
|
||||
"""
|
||||
breakdown: ScoreBreakdown
|
||||
reasons: List[Reason]
|
||||
# Optional: Pfade im Graphen, die zu diesem Treffer geführt haben
|
||||
related_edges: Optional[List[EdgeDTO]] = None
|
||||
|
||||
|
||||
# --- End Explanation Models ---
|
||||
|
||||
|
||||
class QueryHit(BaseModel):
|
||||
"""Einzelnes Trefferobjekt für /query."""
|
||||
node_id: str
|
||||
note_id: Optional[str]
|
||||
|
||||
# Flache Scores (Kompatibilität WP-04a)
|
||||
semantic_score: float
|
||||
edge_bonus: float
|
||||
centrality_bonus: float
|
||||
total_score: float
|
||||
|
||||
paths: Optional[List[List[Dict]]] = None
|
||||
source: Optional[Dict] = None
|
||||
|
||||
# WP-04b: Erklärungsobjekt (nur gefüllt, wenn explain=True)
|
||||
explanation: Optional[Explanation] = None
|
||||
|
||||
|
||||
class QueryResponse(BaseModel):
|
||||
"""Antwortstruktur für /query (Liste von Treffern + Telemetrie)."""
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user