update aus WP11

This commit is contained in:
Lars committed 2025-12-11 13:50:40 +01:00
1 parent 9d792e11ce
commit 52210a91fd
3 files changed
+147 -181

No files matched your search

+78 -54
View File
@@ -1,14 +1,12 @@
"""
app/services/discovery.py
Service für Link-Vorschläge und Knowledge-Discovery (WP-11).
Adaptiert für Async-Architecture (v2.4).
Updated for WP-11: Sliding Window Analysis.
"""
import logging
import os
import asyncio
from typing import List, Dict, Any
import yaml
# Wir nutzen hier weiterhin die Low-Level Funktionen, da diese stabil sind
from app.core.qdrant import QdrantConfig, get_client
from app.models.dto import QueryRequest
from app.core.retriever import hybrid_retrieve
@@ -24,81 +22,110 @@ class DiscoveryService:
async def analyze_draft(self, text: str, current_type: str) -> Dict[str, Any]:
"""
Analysiert einen Draft-Text und schlägt Verlinkungen vor.
Kombiniert Exact Match (Titel/Alias) und Semantic Match.
Analysiert den Draft mit Sliding Window Strategie.
"""
suggestions = []
default_edge_type = self._get_default_edge_type(current_type)
# 1. Exact Match: Finde Begriffe im Text, die als Notiz-Titel existieren
# (Dies läuft synchron, ist aber sehr schnell durch Qdrant Scroll)
# 1. Exact Match (Läuft über gesamten Text, schnell genug)
known_entities = self._fetch_all_titles_and_aliases()
found_entities = self._find_entities_in_text(text, known_entities)
existing_target_ids = set()
for entity in found_entities:
existing_target_ids.add(entity["id"])
target_title = entity["title"]
suggested_md = f"[[rel:{default_edge_type} {target_title}]]"
suggestions.append({
"type": "exact_match",
"text_found": entity["match"],
"target_title": target_title,
"target_title": entity["title"],
"target_id": entity["id"],
"suggested_edge_type": default_edge_type,
"suggested_markdown": suggested_md,
"suggested_markdown": f"[[rel:{default_edge_type} {entity['title']}]]",
"confidence": 1.0,
"reason": f"Exakter Treffer (Default für '{current_type}': {default_edge_type})"
"reason": f"Exakter Treffer ({entity['match']})"
})
# 2. Semantic Match: Finde inhaltlich ähnliche Notizen
# Wir filtern Ergebnisse heraus, die wir schon per Exact Match gefunden haben.
semantic_hits = await self._get_semantic_suggestions_async(text)
# 2. Semantic Match (Sliding Window)
# Wir zerlegen den Text in relevante Chunks, um Token-Limits zu umgehen
# und Fokus zu streuen.
search_queries = self._generate_search_queries(text)
for hit in semantic_hits:
if hit.node_id in existing_target_ids:
continue
if hit.total_score > 0.65:
# FIX: Titel aus Payload lesen, nicht ID!
target_title = hit.payload.get("title") or hit.node_id
# Parallel Execution für alle Queries
tasks = [self._get_semantic_suggestions_async(q) for q in search_queries]
results_list = await asyncio.gather(*tasks)
# Ergebnisse mergen und deduplizieren
seen_semantic_ids = set()
for hits in results_list:
for hit in hits:
if hit.node_id in existing_target_ids or hit.node_id in seen_semantic_ids:
continue
suggested_md = f"[[rel:{default_edge_type} {target_title}]]"
suggestions.append({
"type": "semantic_match",
"text_found": (hit.source.get("text") or "")[:50] + "...",
"target_title": target_title,
"target_id": hit.node_id,
"suggested_edge_type": default_edge_type,
"suggested_markdown": suggested_md,
"confidence": round(hit.total_score, 2),
"reason": f"Semantische Ähnlichkeit ({hit.total_score:.2f})"
})
# Threshold Tuning: Bei 'nomic' sind Scores oft niedriger (0.4-0.6 ist schon gut)
# Wir setzen ihn moderat auf 0.50
if hit.total_score > 0.50:
seen_semantic_ids.add(hit.node_id)
target_title = hit.payload.get("title") or hit.node_id
suggestions.append({
"type": "semantic_match",
"text_found": (hit.source.get("text") or "")[:60] + "...",
"target_title": target_title,
"target_id": hit.node_id,
"suggested_edge_type": default_edge_type,
"suggested_markdown": f"[[rel:{default_edge_type} {target_title}]]",
"confidence": round(hit.total_score, 2),
"reason": f"Semantisch ähnlich ({hit.total_score:.2f})"
})
# Sortieren nach Confidence
suggestions.sort(key=lambda x: x["confidence"], reverse=True)
return {
"draft_length": len(text),
"suggestions_count": len(suggestions),
"suggestions": suggestions
"suggestions": suggestions[:10] # Limit auf Top 10
}
# --- Helpers ---
def _generate_search_queries(self, text: str) -> List[str]:
"""
Zerlegt den Text in bis zu 3 Such-Queries:
1. Der Anfang (Kontext/Einleitung)
2. Die Mitte (Details)
3. Das Ende (Fazit/Zusammenfassung)
"""
if len(text) < 600:
return [text]
queries = []
# Query 1: Die ersten 400 Zeichen
queries.append(text[:400])
# Query 2: Ein Fenster aus der Mitte
mid = len(text) // 2
queries.append(text[mid-200 : mid+200])
# Query 3: Die letzten 400 Zeichen
if len(text) > 800:
queries.append(text[-400:])
return queries
async def _get_semantic_suggestions_async(self, text: str):
"""Async Wrapper um den Hybrid Retriever."""
# Nutzt hybrid_retrieve (sync), aber hier in Async Context okay
req = QueryRequest(query=text, top_k=5, explain=False)
try:
# Da hybrid_retrieve (noch) sync ist, rufen wir es direkt auf.
# In einer voll-async Umgebung würde man dies in einen Thread-Pool auslagern,
# aber da Qdrant-Client sync ist, ist das hier okay.
res = hybrid_retrieve(req)
return res.results
except Exception as e:
logger.error(f"Semantic suggestion failed: {e}")
return []
except Exception: return []
# ... (Restliche Methoden wie _fetch_all_titles_and_aliases bleiben gleich) ...
# Füge hier die Methoden aus dem vorherigen Artefakt ein (fetch_all..., find_entities..., load_type...)
# Der Kürze halber lasse ich sie im Snippet weg, da sie unverändert sind.
def _load_type_registry(self) -> dict:
path = os.getenv("MINDNET_TYPES_FILE", "config/types.yaml")
if not os.path.exists(path):
@@ -112,24 +139,22 @@ class DiscoveryService:
types_cfg = self.registry.get("types", {})
type_def = types_cfg.get(note_type, {})
defaults = type_def.get("edge_defaults")
if defaults and isinstance(defaults, list) and len(defaults) > 0:
return defaults[0]
return "related_to"
return defaults[0] if defaults else "related_to"
def _fetch_all_titles_and_aliases(self) -> List[Dict]:
notes = []
next_page = None
col_name = f"{self.prefix}_notes"
col = f"{self.prefix}_notes"
try:
while True:
res, next_page = self.client.scroll(collection_name=col_name, limit=1000, offset=next_page, with_payload=True, with_vectors=False)
res, next_page = self.client.scroll(collection_name=col, limit=1000, offset=next_page, with_payload=True, with_vectors=False)
for point in res:
pl = point.payload or {}
aliases = pl.get("aliases") or []
if isinstance(aliases, str): aliases = [aliases]
notes.append({"id": pl.get("note_id"), "title": pl.get("title"), "aliases": aliases})
if next_page is None: break
except Exception: return []
except Exception: pass
return notes
def _find_entities_in_text(self, text: str, entities: List[Dict]) -> List[Dict]:
@@ -140,9 +165,8 @@ class DiscoveryService:
if title and title.lower() in text_lower:
found.append({"match": title, "title": title, "id": entity["id"]})
continue
aliases = entity.get("aliases", [])
for alias in aliases:
if alias and str(alias).lower() in text_lower:
for alias in entity.get("aliases", []):
if str(alias).lower() in text_lower:
found.append({"match": alias, "title": title, "id": entity["id"]})
break
return found