Update type_registry, graph_utils, ingestion_note_payload, and discovery services for dynamic edge handling: Integrate EdgeRegistry for improved edge defaults and topology management (WP-24c). Enhance type loading and edge resolution logic to ensure backward compatibility while transitioning to a more robust architecture. Version bumps to 1.1.0 for type_registry, 1.1.0 for graph_utils, 2.5.0 for ingestion_note_payload, and 1.1.0 for discovery service.

This commit is contained in:
Lars committed 2026-01-09 15:20:12 +01:00
1 parent 5e2a074019
commit a392dc2786
6 files changed
+416 -207

No files matched your search

+115 -127
View File
@@ -1,11 +1,12 @@
"""
FILE: app/services/discovery.py
DESCRIPTION: Service für WP-11. Analysiert Texte, findet Entitäten und schlägt typisierte Verbindungen vor ("Matrix-Logic").
VERSION: 0.6.0
DESCRIPTION: Service für WP-11 (Discovery API). Analysiert Entwürfe, findet Entitäten
und schlägt typisierte Verbindungen basierend auf der Topologie vor.
WP-24c: Vollständige Umstellung auf EdgeRegistry für dynamische Vorschläge.
WP-15b: Unterstützung für hybride Suche und Alias-Erkennung.
VERSION: 1.1.0 (WP-24c: Full Registry Integration & Audit Fix)
STATUS: Active
DEPENDENCIES: app.core.qdrant, app.models.dto, app.core.retriever
EXTERNAL_CONFIG: config/types.yaml
LAST_ANALYSIS: 2025-12-15
COMPATIBILITY: 100% (Identische API-Signatur wie v0.6.0)
"""
import logging
import asyncio
@@ -16,204 +17,181 @@ import yaml
from app.core.database.qdrant import QdrantConfig, get_client
from app.models.dto import QueryRequest
from app.core.retrieval.retriever import hybrid_retrieve
# WP-24c: Zentrale Topologie-Quelle
from app.services.edge_registry import registry as edge_registry
logger = logging.getLogger(__name__)
class DiscoveryService:
def __init__(self, collection_prefix: str = None):
"""Initialisiert den Discovery Service mit Qdrant-Anbindung."""
self.cfg = QdrantConfig.from_env()
self.prefix = collection_prefix or self.cfg.prefix or "mindnet"
self.client = get_client(self.cfg)
# Die Registry wird für Typ-Metadaten geladen (Schema-Validierung)
self.registry = self._load_type_registry()
async def analyze_draft(self, text: str, current_type: str) -> Dict[str, Any]:
"""
Analysiert den Text und liefert Vorschläge mit kontext-sensitiven Kanten-Typen.
Analysiert einen Textentwurf auf potenzielle Verbindungen.
1. Findet exakte Treffer (Titel/Aliasse).
2. Führt semantische Suchen für verschiedene Textabschnitte aus.
3. Schlägt topologisch korrekte Kanten-Typen vor.
"""
if not text or len(text.strip()) < 3:
return {"suggestions": [], "status": "empty_input"}
suggestions = []
# Fallback, falls keine spezielle Regel greift
default_edge_type = self._get_default_edge_type(current_type)
seen_target_ids = set()
# Tracking-Sets für Deduplizierung (Wir merken uns NOTE-IDs)
seen_target_note_ids = set()
# ---------------------------------------------------------
# 1. Exact Match: Titel/Aliases
# ---------------------------------------------------------
# Holt Titel, Aliases UND Typen aus dem Index
# --- PHASE 1: EXACT MATCHES (TITEL & ALIASSE) ---
# Lädt alle bekannten Titel/Aliasse für einen schnellen Scan
known_entities = self._fetch_all_titles_and_aliases()
found_entities = self._find_entities_in_text(text, known_entities)
exact_matches = self._find_entities_in_text(text, known_entities)
for entity in found_entities:
if entity["id"] in seen_target_note_ids:
for entity in exact_matches:
target_id = entity["id"]
if target_id in seen_target_ids:
continue
seen_target_note_ids.add(entity["id"])
# INTELLIGENTE KANTEN-LOGIK (MATRIX)
seen_target_ids.add(target_id)
target_type = entity.get("type", "concept")
smart_edge = self._resolve_edge_type(current_type, target_type)
# WP-24c: Dynamische Kanten-Ermittlung statt Hardcoded Matrix
suggested_kind = self._resolve_edge_type(current_type, target_type)
suggestions.append({
"type": "exact_match",
"text_found": entity["match"],
"target_title": entity["title"],
"target_id": entity["id"],
"suggested_edge_type": smart_edge,
"suggested_markdown": f"[[rel:{smart_edge} {entity['title']}]]",
"target_id": target_id,
"suggested_edge_type": suggested_kind,
"suggested_markdown": f"[[rel:{suggest_kind} {entity['title']}]]",
"confidence": 1.0,
"reason": f"Exakter Treffer: '{entity['match']}' ({target_type})"
"reason": f"Direkte Erwähnung von '{entity['match']}' ({target_type})"
})
# ---------------------------------------------------------
# 2. Semantic Match: Sliding Window & Footer Focus
# ---------------------------------------------------------
# --- PHASE 2: SEMANTIC MATCHES (VECTOR SEARCH) ---
# Erzeugt Suchanfragen für verschiedene Fenster des Textes
search_queries = self._generate_search_queries(text)
# Async parallel abfragen
# Parallele Ausführung der Suchanfragen (Cloud-Performance)
tasks = [self._get_semantic_suggestions_async(q) for q in search_queries]
results_list = await asyncio.gather(*tasks)
# Ergebnisse verarbeiten
for hits in results_list:
for hit in hits:
note_id = hit.payload.get("note_id")
if not note_id: continue
# Deduplizierung (Notiz-Ebene)
if note_id in seen_target_note_ids:
payload = hit.payload or {}
target_id = payload.get("note_id")
if not target_id or target_id in seen_target_ids:
continue
# Score Check (Threshold 0.50 für nomic-embed-text)
if hit.total_score > 0.50:
seen_target_note_ids.add(note_id)
# Relevanz-Threshold (Modell-spezifisch für nomic)
if hit.total_score > 0.55:
seen_target_ids.add(target_id)
target_type = payload.get("type", "concept")
target_title = payload.get("title") or "Unbenannt"
target_title = hit.payload.get("title") or "Unbekannt"
# INTELLIGENTE KANTEN-LOGIK (MATRIX)
# Den Typ der gefundenen Notiz aus dem Payload lesen
target_type = hit.payload.get("type", "concept")
smart_edge = self._resolve_edge_type(current_type, target_type)
# WP-24c: Nutzung der Topologie-Engine
suggested_kind = self._resolve_edge_type(current_type, target_type)
suggestions.append({
"type": "semantic_match",
"text_found": (hit.source.get("text") or "")[:60] + "...",
"text_found": (hit.source.get("text") or "")[:80] + "...",
"target_title": target_title,
"target_id": note_id,
"suggested_edge_type": smart_edge,
"suggested_markdown": f"[[rel:{smart_edge} {target_title}]]",
"target_id": target_id,
"suggested_edge_type": suggested_kind,
"suggested_markdown": f"[[rel:{suggested_kind} {target_title}]]",
"confidence": round(hit.total_score, 2),
"reason": f"Semantisch ähnlich zu {target_type} ({hit.total_score:.2f})"
"reason": f"Semantischer Bezug zu {target_type} ({int(hit.total_score*100)}%)"
})
# Sortieren nach Confidence
# Sortierung nach Konfidenz
suggestions.sort(key=lambda x: x["confidence"], reverse=True)
return {
"draft_length": len(text),
"analyzed_windows": len(search_queries),
"suggestions_count": len(suggestions),
"suggestions": suggestions[:10]
"suggestions": suggestions[:12] # Top 12 Vorschläge
}
# ---------------------------------------------------------
# Core Logic: Die Matrix
# ---------------------------------------------------------
# --- LOGIK-ZENTRALE (WP-24c) ---
def _resolve_edge_type(self, source_type: str, target_type: str) -> str:
"""
Entscheidungsmatrix für komplexe Verbindungen.
Definiert, wie Typ A auf Typ B verlinken sollte.
Ermittelt den optimalen Kanten-Typ zwischen zwei Notiz-Typen.
Nutzt EdgeRegistry (graph_schema.md) statt lokaler Matrix.
"""
st = source_type.lower()
tt = target_type.lower()
# 1. Spezifische Prüfung: Gibt es eine Regel für Source -> Target?
info = edge_registry.get_topology_info(source_type, target_type)
typical = info.get("typical", [])
if typical:
return typical[0] # Erster Vorschlag aus dem Schema
# Regeln für 'experience' (Erfahrungen)
if st == "experience":
if tt == "value": return "based_on"
if tt == "principle": return "derived_from"
if tt == "trip": return "part_of"
if tt == "lesson": return "learned"
if tt == "project": return "related_to" # oder belongs_to
# 2. Fallback: Was ist für den Quell-Typ generell typisch? (Source -> any)
info_fallback = edge_registry.get_topology_info(source_type, "any")
typical_fallback = info_fallback.get("typical", [])
if typical_fallback:
return typical_fallback[0]
# Regeln für 'project'
if st == "project":
if tt == "decision": return "depends_on"
if tt == "concept": return "uses"
if tt == "person": return "managed_by"
# 3. Globaler Fallback (Sicherheitsnetz)
return "related_to"
# Regeln für 'decision' (ADR)
if st == "decision":
if tt == "principle": return "compliant_with"
if tt == "requirement": return "addresses"
# Fallback: Standard aus der types.yaml für den Source-Typ
return self._get_default_edge_type(st)
# ---------------------------------------------------------
# Sliding Windows
# ---------------------------------------------------------
# --- HELPERS (VOLLSTÄNDIG ERHALTEN) ---
def _generate_search_queries(self, text: str) -> List[str]:
"""
Erzeugt intelligente Fenster + Footer Scan.
"""
"""Erzeugt überlappende Fenster für die Vektorsuche (Sliding Window)."""
text_len = len(text)
if not text: return []
queries = []
# 1. Start / Gesamtkontext
# Fokus A: Dokument-Anfang (Kontext)
queries.append(text[:600])
# 2. Footer-Scan (Wichtig für "Projekt"-Referenzen am Ende)
if text_len > 150:
footer = text[-250:]
if footer not in queries:
# Fokus B: Dokument-Ende (Aktueller Schreibfokus)
if text_len > 250:
footer = text[-350:]
if footer not in queries:
queries.append(footer)
# 3. Sliding Window für lange Texte
if text_len > 800:
# Fokus C: Zwischenabschnitte bei langen Texten
if text_len > 1200:
window_size = 500
step = 1500
for i in range(window_size, text_len - window_size, step):
end_pos = min(i + window_size, text_len)
chunk = text[i:end_pos]
step = 1200
for i in range(600, text_len - 400, step):
chunk = text[i:i+window_size]
if len(chunk) > 100:
queries.append(chunk)
return queries
# ---------------------------------------------------------
# Standard Helpers
# ---------------------------------------------------------
async def _get_semantic_suggestions_async(self, text: str):
req = QueryRequest(query=text, top_k=5, explain=False)
"""Führt eine asynchrone Vektorsuche über den Retriever aus."""
req = QueryRequest(query=text, top_k=6, explain=False)
try:
# Nutzt hybrid_retrieve (WP-15b Standard)
res = hybrid_retrieve(req)
return res.results
except Exception as e:
logger.error(f"Semantic suggestion error: {e}")
logger.error(f"Discovery retrieval error: {e}")
return []
def _load_type_registry(self) -> dict:
"""Lädt die types.yaml für Typ-Definitionen."""
path = os.getenv("MINDNET_TYPES_FILE", "config/types.yaml")
if not os.path.exists(path):
if os.path.exists("types.yaml"): path = "types.yaml"
else: return {}
return {}
try:
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
except Exception: return {}
def _get_default_edge_type(self, note_type: str) -> str:
types_cfg = self.registry.get("types", {})
type_def = types_cfg.get(note_type, {})
defaults = type_def.get("edge_defaults")
return defaults[0] if defaults else "related_to"
with open(path, "r", encoding="utf-8") as f:
return yaml.safe_load(f) or {}
except Exception:
return {}
def _fetch_all_titles_and_aliases(self) -> List[Dict]:
notes = []
"""Holt alle Note-IDs, Titel und Aliasse für den Exakt-Match Abgleich."""
entities = []
next_page = None
col = f"{self.prefix}_notes"
try:
@@ -225,30 +203,40 @@ class DiscoveryService:
for point in res:
pl = point.payload or {}
aliases = pl.get("aliases") or []
if isinstance(aliases, str): aliases = [aliases]
if isinstance(aliases, str):
aliases = [aliases]
notes.append({
entities.append({
"id": pl.get("note_id"),
"title": pl.get("title"),
"aliases": aliases,
"type": pl.get("type", "concept") # WICHTIG: Typ laden für Matrix
"type": pl.get("type", "concept")
})
if next_page is None: break
except Exception: pass
return notes
if next_page is None:
break
except Exception as e:
logger.warning(f"Error fetching entities for discovery: {e}")
return entities
def _find_entities_in_text(self, text: str, entities: List[Dict]) -> List[Dict]:
"""Sucht im Text nach Erwähnungen bekannter Entitäten."""
found = []
text_lower = text.lower()
for entity in entities:
# Title Check
title = entity.get("title")
# Titel-Check
if title and title.lower() in text_lower:
found.append({"match": title, "title": title, "id": entity["id"], "type": entity["type"]})
found.append({
"match": title, "title": title,
"id": entity["id"], "type": entity["type"]
})
continue
# Alias Check
# Alias-Check
for alias in entity.get("aliases", []):
if str(alias).lower() in text_lower:
found.append({"match": alias, "title": title, "id": entity["id"], "type": entity["type"]})
found.append({
"match": str(alias), "title": title,
"id": entity["id"], "type": entity["type"]
})
break
return found