überarbeitet mit Gemini
This commit is contained in:
parent
4ab44e36a2
commit
079cf174d4
|
|
@ -1,10 +1,10 @@
|
|||
"""
|
||||
FILE: app/core/ingestion.py
|
||||
DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen.
|
||||
WP-20: Smart Edge Allocation via Hybrid LLM (OpenRouter/Gemini).
|
||||
WP-22: Content Lifecycle, Edge Registry Validation & Multi-Hash.
|
||||
FIX: Behebung des AttributeError und Härtung des Prompt-Formattings.
|
||||
VERSION: 2.11.7
|
||||
WP-20: Optimiert für OpenRouter (openai/gpt-oss-20b:free) als Primary.
|
||||
WP-22: Fallback-Unterstützung für Google Gemini und Ollama.
|
||||
FIX: Dynamische Provider-Wahl und Modell-Zuweisung für den Turbo-Modus.
|
||||
VERSION: 2.11.9
|
||||
STATUS: Active
|
||||
"""
|
||||
import os
|
||||
|
|
@ -131,9 +131,19 @@ class IngestionService:
|
|||
return get_chunk_config(note_type)
|
||||
|
||||
async def _perform_smart_edge_allocation(self, text: str, note_id: str) -> List[Dict]:
|
||||
"""Nutzt das Hybrid LLM für die semantische Kanten-Extraktion."""
|
||||
provider = "openrouter" if self.settings.OPENROUTER_API_KEY else self.settings.MINDNET_LLM_PROVIDER
|
||||
model = self.settings.GEMMA_MODEL
|
||||
"""
|
||||
WP-20: Nutzt das Hybrid LLM für die semantische Kanten-Extraktion.
|
||||
Bevorzugt den primär eingestellten Provider (z.B. OpenRouter).
|
||||
"""
|
||||
# 1. Provider & Modell Bestimmung (User-Request: OpenRouter Primary)
|
||||
provider = self.settings.MINDNET_LLM_PROVIDER
|
||||
|
||||
if provider == "openrouter":
|
||||
model = self.settings.OPENROUTER_MODEL
|
||||
elif provider == "gemini":
|
||||
model = self.settings.GEMINI_MODEL
|
||||
else:
|
||||
model = self.settings.LLM_MODEL
|
||||
|
||||
logger.info(f"🚀 [Ingestion] Turbo-Mode: Extracting edges for '{note_id}' using {model} on {provider}")
|
||||
|
||||
|
|
@ -238,7 +248,6 @@ class IngestionService:
|
|||
body_text = getattr(parsed, "body", "") or ""
|
||||
if hasattr(edge_registry, "ensure_latest"): edge_registry.ensure_latest()
|
||||
|
||||
# FIX: Behebung des AttributeError durch korrekten Aufruf der Klassenmethode
|
||||
chunk_config = self._get_chunk_config_by_profile(effective_profile, note_type)
|
||||
chunks = await assemble_chunks(fm["id"], body_text, fm["type"], config=chunk_config)
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
"""
|
||||
FILE: app/services/semantic_analyzer.py
|
||||
DESCRIPTION: KI-gestützte Kanten-Validierung. Nutzt LLM (Background-Priority), um Kanten präzise einem Chunk zuzuordnen.
|
||||
WP-20 Fix: Volle Kompatibilität mit der gehärteten LLMService (v3.3.2) Kaskade.
|
||||
WP-20 Fix: Volle Kompatibilität mit der provider-basierten Routing-Logik (OpenRouter Primary).
|
||||
WP-22: Integration von valid_types zur Halluzinations-Vermeidung.
|
||||
VERSION: 2.2.3
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.services.llm_service, app.services.edge_registry, json, logging
|
||||
LAST_ANALYSIS: 2025-12-23
|
||||
LAST_ANALYSIS: 2025-12-24
|
||||
"""
|
||||
|
||||
import json
|
||||
|
|
@ -29,53 +29,44 @@ class SemanticAnalyzer:
|
|||
"""
|
||||
Prüft, ob ein String eine valide Kante im Format 'kind:target' ist.
|
||||
Verhindert, dass LLM-Geschwätz als Kante durchrutscht.
|
||||
|
||||
WP-22 Erweiterung:
|
||||
- kind muss (wenn valid_types verfügbar) im kontrollierten Vokabular enthalten sein.
|
||||
"""
|
||||
if not isinstance(edge_str, str) or ":" not in edge_str:
|
||||
return False
|
||||
|
||||
|
||||
parts = edge_str.split(":", 1)
|
||||
kind = parts[0].strip()
|
||||
target = parts[1].strip()
|
||||
|
||||
|
||||
# Regel 1: Ein 'kind' (Beziehungstyp) darf keine Leerzeichen enthalten.
|
||||
if " " in kind:
|
||||
return False
|
||||
|
||||
|
||||
# Regel 2: Plausible Länge für den Typ
|
||||
if len(kind) > 40 or len(kind) < 2:
|
||||
return False
|
||||
|
||||
|
||||
# Regel 3: Target darf nicht leer sein
|
||||
if not target:
|
||||
return False
|
||||
|
||||
# WP-22: kontrolliertes Vokabular erzwingen (falls vorhanden/geladen)
|
||||
try:
|
||||
if hasattr(edge_registry, "valid_types") and edge_registry.valid_types:
|
||||
if kind not in edge_registry.valid_types:
|
||||
return False
|
||||
except Exception:
|
||||
# Bei Registry-Problemen lieber nicht crashen -> konservativ: ablehnen wäre auch möglich,
|
||||
# aber wir bleiben kompatibel und robust.
|
||||
pass
|
||||
|
||||
|
||||
return True
|
||||
|
||||
async def assign_edges_to_chunk(self, chunk_text: str, all_edges: List[str], note_type: str) -> List[str]:
|
||||
"""
|
||||
Sendet einen Chunk und eine Liste potenzieller Kanten an das LLM.
|
||||
Das LLM filtert heraus, welche Kanten für diesen Chunk relevant sind.
|
||||
WP-20: Nutzt primär den Provider aus MINDNET_LLM_PROVIDER (OpenRouter).
|
||||
"""
|
||||
if not all_edges:
|
||||
return []
|
||||
|
||||
# 1. Prompt laden via get_prompt (handelt die Provider-Kaskade automatisch ab)
|
||||
prompt_template = self.llm.get_prompt("edge_allocation_template")
|
||||
# 1. Bestimmung des Providers und Modells (WP-20)
|
||||
# Wir ziehen die Werte direkt aus dem Service-Kontext
|
||||
provider = self.llm.settings.MINDNET_LLM_PROVIDER
|
||||
model = self.llm.settings.OPENROUTER_MODEL if provider == "openrouter" else None
|
||||
|
||||
# Sicherheits-Check für die Format-Methode
|
||||
# 2. Prompt laden via get_prompt
|
||||
prompt_template = self.llm.get_prompt("edge_allocation_template", provider)
|
||||
|
||||
if not prompt_template or isinstance(prompt_template, dict):
|
||||
logger.warning("⚠️ [SemanticAnalyzer] Prompt 'edge_allocation_template' konnte nicht als String geladen werden. Nutze Not-Fallback.")
|
||||
prompt_template = (
|
||||
|
|
@ -85,45 +76,42 @@ class SemanticAnalyzer:
|
|||
"OUTPUT: JSON Liste von Strings [\"kind:target\"]."
|
||||
)
|
||||
|
||||
# 2. Daten für Template vorbereiten (WP-22 Integration)
|
||||
# Wir laden die validen Typen, um sie dem LLM als Leitplanken zu geben
|
||||
# 3. Daten für Template vorbereiten (WP-22 Integration)
|
||||
edge_registry.ensure_latest()
|
||||
valid_types_str = ", ".join(sorted(list(edge_registry.valid_types)))
|
||||
edges_str = "\n".join([f"- {e}" for e in all_edges])
|
||||
|
||||
# LOG: Request Info
|
||||
|
||||
logger.debug(f"🔍 [SemanticAnalyzer] Request: {len(chunk_text)} chars Text, {len(all_edges)} Candidates.")
|
||||
|
||||
# 3. Prompt füllen (FIX: valid_types hinzugefügt, um FormatError zu beheben)
|
||||
# 4. Prompt füllen (FIX: valid_types hinzugefügt, um Format Error zu beheben)
|
||||
try:
|
||||
final_prompt = prompt_template.format(
|
||||
chunk_text=chunk_text[:3500],
|
||||
chunk_text=chunk_text[:3500],
|
||||
edge_list=edges_str,
|
||||
valid_types=valid_types_str
|
||||
)
|
||||
except Exception as format_err:
|
||||
logger.error(f"❌ [SemanticAnalyzer] Format Error im Prompt-Template (Fehlender Parameter): {format_err}")
|
||||
logger.error(f"❌ [SemanticAnalyzer] Format Error im Prompt-Template: {format_err}")
|
||||
return []
|
||||
|
||||
try:
|
||||
# 4. LLM Call mit Traffic Control (Background Priority)
|
||||
# NOTE: Keine neuen Parameter hier, damit es mit deinem aktuellen llm_service.py kompatibel bleibt.
|
||||
# 5. LLM Call mit Traffic Control (Background Priority)
|
||||
response_json = await self.llm.generate_raw_response(
|
||||
prompt=final_prompt,
|
||||
force_json=True,
|
||||
max_retries=5,
|
||||
max_retries=5,
|
||||
base_delay=5.0,
|
||||
priority="background"
|
||||
priority="background",
|
||||
provider=provider,
|
||||
model_override=model
|
||||
)
|
||||
|
||||
# LOG: Raw Response Preview
|
||||
logger.debug(f"📥 [SemanticAnalyzer] Raw Response (Preview): {response_json[:200]}...")
|
||||
|
||||
# 5. Parsing & Cleaning
|
||||
# 6. Parsing & Cleaning
|
||||
clean_json = response_json.replace("```json", "").replace("```", "").strip()
|
||||
|
||||
if not clean_json:
|
||||
logger.warning("⚠️ [SemanticAnalyzer] Leere Antwort vom LLM erhalten.")
|
||||
|
||||
if not clean_json:
|
||||
return []
|
||||
|
||||
try:
|
||||
|
|
@ -134,17 +122,15 @@ class SemanticAnalyzer:
|
|||
|
||||
valid_edges = []
|
||||
|
||||
# 6. Robuste Validierung (List vs Dict)
|
||||
# 7. Robuste Validierung (List vs Dict)
|
||||
raw_candidates = []
|
||||
|
||||
if isinstance(data, list):
|
||||
raw_candidates = data
|
||||
|
||||
elif isinstance(data, dict):
|
||||
logger.info(f"ℹ️ [SemanticAnalyzer] LLM lieferte Dict statt Liste. Versuche Reparatur.")
|
||||
for key, val in data.items():
|
||||
if key.lower() in ["edges", "results", "kanten", "matches"] and isinstance(val, list):
|
||||
raw_candidates.extend(val)
|
||||
raw_candidates.extend(val)
|
||||
elif isinstance(val, str):
|
||||
raw_candidates.append(f"{key}:{val}")
|
||||
elif isinstance(val, list):
|
||||
|
|
@ -152,7 +138,7 @@ class SemanticAnalyzer:
|
|||
if isinstance(target, str):
|
||||
raw_candidates.append(f"{key}:{target}")
|
||||
|
||||
# 7. Strict Validation Loop
|
||||
# 8. Strict Validation Loop
|
||||
for e in raw_candidates:
|
||||
e_str = str(e)
|
||||
if self._is_valid_edge_string(e_str):
|
||||
|
|
@ -161,12 +147,9 @@ class SemanticAnalyzer:
|
|||
logger.debug(f" [SemanticAnalyzer] Invalid edge format rejected: '{e_str}'")
|
||||
|
||||
final_result = [e for e in valid_edges if ":" in e]
|
||||
|
||||
|
||||
if final_result:
|
||||
logger.info(f"✅ [SemanticAnalyzer] Success. {len(final_result)} Kanten zugewiesen.")
|
||||
else:
|
||||
logger.debug(" [SemanticAnalyzer] Keine spezifischen Kanten erkannt (Empty Result).")
|
||||
|
||||
return final_result
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -183,4 +166,4 @@ def get_semantic_analyzer():
|
|||
global _analyzer_instance
|
||||
if _analyzer_instance is None:
|
||||
_analyzer_instance = SemanticAnalyzer()
|
||||
return _analyzer_instance
|
||||
return _analyzer_instance
|
||||
Loading…
Reference in New Issue
Block a user