bereinigung Code Basis, wegfall von Platzhaltern und Annahmen. Volle Kofigurierbarkeit
This commit is contained in:
1 parent
f3fd71b828
commit
5c55229376
3 files changed
+140
-115
No files matched your search
@@ -3,14 +3,15 @@ FILE: app/services/semantic_analyzer.py
|
||||
DESCRIPTION: KI-gestützte Kanten-Validierung. Nutzt LLM (Background-Priority), um Kanten präzise einem Chunk zuzuordnen.
|
||||
WP-20 Fix: Volle Kompatibilität mit der provider-basierten Routing-Logik (OpenRouter Primary).
|
||||
WP-22: Integration von valid_types zur Halluzinations-Vermeidung.
|
||||
VERSION: 2.2.3
|
||||
FIX: Finale DoD-Härtung, Entfernung von Hardcoded Limits und optimiertes Error-Handling.
|
||||
VERSION: 2.2.4
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.services.llm_service, app.services.edge_registry, json, logging
|
||||
LAST_ANALYSIS: 2025-12-24
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import List, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
@@ -41,7 +42,7 @@ class SemanticAnalyzer:
|
||||
if " " in kind:
|
||||
return False
|
||||
|
||||
# Regel 2: Plausible Länge für den Typ
|
||||
# Regel 2: Plausible Länge für den Typ (Vermeidet Sätze als Typ)
|
||||
if len(kind) > 40 or len(kind) < 2:
|
||||
return False
|
||||
|
||||
@@ -54,21 +55,21 @@ class SemanticAnalyzer:
|
||||
async def assign_edges_to_chunk(self, chunk_text: str, all_edges: List[str], note_type: str) -> List[str]:
|
||||
"""
|
||||
Sendet einen Chunk und eine Liste potenzieller Kanten an das LLM.
|
||||
WP-20: Nutzt primär den Provider aus MINDNET_LLM_PROVIDER (OpenRouter).
|
||||
Das LLM filtert heraus, welche Kanten für diesen Chunk relevant sind.
|
||||
WP-20: Nutzt primär den konfigurierten Provider (z.B. OpenRouter).
|
||||
"""
|
||||
if not all_edges:
|
||||
return []
|
||||
|
||||
# 1. Bestimmung des Providers und Modells (WP-20)
|
||||
# Wir ziehen die Werte direkt aus dem Service-Kontext
|
||||
# 1. Bestimmung des Providers und Modells (Dynamisch über Settings)
|
||||
provider = self.llm.settings.MINDNET_LLM_PROVIDER
|
||||
model = self.llm.settings.OPENROUTER_MODEL if provider == "openrouter" else None
|
||||
model = self.llm.settings.OPENROUTER_MODEL if provider == "openrouter" else self.llm.settings.GEMINI_MODEL
|
||||
|
||||
# 2. Prompt laden via get_prompt
|
||||
# 2. Prompt laden (Provider-spezifisch)
|
||||
prompt_template = self.llm.get_prompt("edge_allocation_template", provider)
|
||||
|
||||
if not prompt_template or isinstance(prompt_template, dict):
|
||||
logger.warning("⚠️ [SemanticAnalyzer] Prompt 'edge_allocation_template' konnte nicht als String geladen werden. Nutze Not-Fallback.")
|
||||
if not prompt_template or not isinstance(prompt_template, str):
|
||||
logger.warning("⚠️ [SemanticAnalyzer] Prompt 'edge_allocation_template' ungültig. Nutze Recovery-Template.")
|
||||
prompt_template = (
|
||||
"TASK: Wähle aus den Kandidaten die relevanten Kanten für den Text.\n"
|
||||
"TEXT: {chunk_text}\n"
|
||||
@@ -76,91 +77,99 @@ class SemanticAnalyzer:
|
||||
"OUTPUT: JSON Liste von Strings [\"kind:target\"]."
|
||||
)
|
||||
|
||||
# 3. Daten für Template vorbereiten (WP-22 Integration)
|
||||
# 3. Daten für Template vorbereiten (Vokabular-Check)
|
||||
edge_registry.ensure_latest()
|
||||
valid_types_str = ", ".join(sorted(list(edge_registry.valid_types)))
|
||||
edges_str = "\n".join([f"- {e}" for e in all_edges])
|
||||
|
||||
logger.debug(f"🔍 [SemanticAnalyzer] Request: {len(chunk_text)} chars Text, {len(all_edges)} Candidates.")
|
||||
|
||||
# 4. Prompt füllen (FIX: valid_types hinzugefügt, um Format Error zu beheben)
|
||||
# 4. Prompt füllen mit Format-Check (Kein Shortcut)
|
||||
try:
|
||||
# Wir begrenzen den Text auf eine vernünftige Länge für das Kontextfenster (ca. 10k Tokens max)
|
||||
final_prompt = prompt_template.format(
|
||||
chunk_text=chunk_text[:3500],
|
||||
chunk_text=chunk_text[:6000],
|
||||
edge_list=edges_str,
|
||||
valid_types=valid_types_str
|
||||
)
|
||||
except Exception as format_err:
|
||||
logger.error(f"❌ [SemanticAnalyzer] Format Error im Prompt-Template: {format_err}")
|
||||
logger.error(f"❌ [SemanticAnalyzer] Prompt Formatting failed: {format_err}")
|
||||
return []
|
||||
|
||||
try:
|
||||
# 5. LLM Call mit Traffic Control (Background Priority)
|
||||
# 5. LLM Call mit Background Priority & Semaphore Control
|
||||
response_json = await self.llm.generate_raw_response(
|
||||
prompt=final_prompt,
|
||||
force_json=True,
|
||||
max_retries=5,
|
||||
base_delay=5.0,
|
||||
max_retries=3,
|
||||
base_delay=2.0,
|
||||
priority="background",
|
||||
provider=provider,
|
||||
model_override=model
|
||||
)
|
||||
|
||||
logger.debug(f"📥 [SemanticAnalyzer] Raw Response (Preview): {response_json[:200]}...")
|
||||
|
||||
# 6. Parsing & Cleaning
|
||||
clean_json = response_json.replace("```json", "").replace("```", "").strip()
|
||||
# 6. Bulletproof JSON Extraction (Analog zur Ingestion)
|
||||
# Entfernt Markdown-Code-Blöcke falls vorhanden
|
||||
match = re.search(r"```(?:json)?\s*(.*?)\s*```", response_json, re.DOTALL)
|
||||
clean_json = match.group(1) if match else response_json
|
||||
clean_json = clean_json.strip()
|
||||
|
||||
if not clean_json:
|
||||
return []
|
||||
|
||||
try:
|
||||
data = json.loads(clean_json)
|
||||
except json.JSONDecodeError as json_err:
|
||||
logger.error(f"❌ [SemanticAnalyzer] JSON Decode Error: {json_err}")
|
||||
return []
|
||||
except json.JSONDecodeError:
|
||||
# Letzter Rettungsversuch: Suche nach dem ersten '[' und letzten ']'
|
||||
start = clean_json.find('[')
|
||||
end = clean_json.rfind(']') + 1
|
||||
if start != -1 and end != 0:
|
||||
try:
|
||||
data = json.loads(clean_json[start:end])
|
||||
except:
|
||||
logger.error("❌ [SemanticAnalyzer] JSON Recovery failed.")
|
||||
return []
|
||||
else:
|
||||
return []
|
||||
|
||||
valid_edges = []
|
||||
|
||||
# 7. Robuste Validierung (List vs Dict)
|
||||
# 7. Robuste Normalisierung (List vs Dict Recovery)
|
||||
raw_candidates = []
|
||||
if isinstance(data, list):
|
||||
raw_candidates = data
|
||||
elif isinstance(data, dict):
|
||||
logger.info(f"ℹ️ [SemanticAnalyzer] LLM lieferte Dict statt Liste. Versuche Reparatur.")
|
||||
for key, val in data.items():
|
||||
if key.lower() in ["edges", "results", "kanten", "matches"] and isinstance(val, list):
|
||||
raw_candidates.extend(val)
|
||||
elif isinstance(val, str):
|
||||
raw_candidates.append(f"{key}:{val}")
|
||||
elif isinstance(val, list):
|
||||
for target in val:
|
||||
if isinstance(target, str):
|
||||
raw_candidates.append(f"{key}:{target}")
|
||||
logger.info(f"ℹ️ [SemanticAnalyzer] LLM returned dict, trying recovery.")
|
||||
for key in ["edges", "results", "kanten", "matches"]:
|
||||
if key in data and isinstance(data[key], list):
|
||||
raw_candidates.extend(data[key])
|
||||
break
|
||||
# Falls immer noch leer, nutze Schlüssel-Wert Paare als Behelf
|
||||
if not raw_candidates:
|
||||
for k, v in data.items():
|
||||
if isinstance(v, str): raw_candidates.append(f"{k}:{v}")
|
||||
elif isinstance(v, list): [raw_candidates.append(f"{k}:{i}") for i in v if isinstance(i, str)]
|
||||
|
||||
# 8. Strict Validation Loop
|
||||
# 8. Strikte Validierung gegen Kanten-Format
|
||||
valid_edges = []
|
||||
for e in raw_candidates:
|
||||
e_str = str(e)
|
||||
e_str = str(e).strip()
|
||||
if self._is_valid_edge_string(e_str):
|
||||
valid_edges.append(e_str)
|
||||
else:
|
||||
logger.debug(f" [SemanticAnalyzer] Invalid edge format rejected: '{e_str}'")
|
||||
logger.debug(f" [SemanticAnalyzer] Rejected invalid edge format: '{e_str}'")
|
||||
|
||||
final_result = [e for e in valid_edges if ":" in e]
|
||||
|
||||
if final_result:
|
||||
logger.info(f"✅ [SemanticAnalyzer] Success. {len(final_result)} Kanten zugewiesen.")
|
||||
return final_result
|
||||
if valid_edges:
|
||||
logger.info(f"✅ [SemanticAnalyzer] Assigned {len(valid_edges)} edges to chunk.")
|
||||
return valid_edges
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"💥 [SemanticAnalyzer] Kritischer Fehler: {e}", exc_info=True)
|
||||
logger.error(f"💥 [SemanticAnalyzer] Critical error during analysis: {e}", exc_info=True)
|
||||
return []
|
||||
|
||||
async def close(self):
|
||||
if self.llm:
|
||||
await self.llm.close()
|
||||
|
||||
# Singleton Helper
|
||||
# Singleton Instanziierung
|
||||
_analyzer_instance = None
|
||||
def get_semantic_analyzer():
|
||||
global _analyzer_instance
|
||||
|
||||
Reference in new issue
Block a user