Mistral sichere Parser implemntierung
This commit is contained in:
1 parent
ecfdc67485
commit
16e128668c
2 files changed
+87
-53
No files matched your search
@@ -3,16 +3,16 @@ FILE: app/services/semantic_analyzer.py
|
||||
DESCRIPTION: KI-gestützte Kanten-Validierung. Nutzt LLM (Background-Priority), um Kanten präzise einem Chunk zuzuordnen.
|
||||
WP-20 Fix: Volle Kompatibilität mit der provider-basierten Routing-Logik (OpenRouter Primary).
|
||||
WP-22: Integration von valid_types zur Halluzinations-Vermeidung.
|
||||
FIX: Finale DoD-Härtung, Entfernung von Hardcoded Limits und optimiertes Error-Handling.
|
||||
VERSION: 2.2.4
|
||||
FIX: Mistral-sicheres JSON-Parsing (<s> & [OUT] Handling) und 100% Logik-Erhalt.
|
||||
VERSION: 2.2.6
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.services.llm_service, app.services.edge_registry, json, logging
|
||||
DEPENDENCIES: app.services.llm_service, app.services.edge_registry, json, logging, re
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Any
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Importe
|
||||
@@ -52,6 +52,43 @@ class SemanticAnalyzer:
|
||||
|
||||
return True
|
||||
|
||||
def _extract_json_safely(self, text: str) -> Any:
|
||||
"""
|
||||
Extrahiert JSON-Daten und bereinigt LLM-Steuerzeichen (Mistral/Llama).
|
||||
Implementiert robuste Recovery-Logik für Cloud-Provider.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# 1. Entferne Mistral/Llama Steuerzeichen und Tags
|
||||
clean = text.replace("<s>", "").replace("</s>", "")
|
||||
clean = clean.replace("[OUT]", "").replace("[/OUT]", "")
|
||||
clean = clean.strip()
|
||||
|
||||
# 2. Suche nach Markdown JSON-Blöcken
|
||||
match = re.search(r"```(?:json)?\s*(.*?)\s*```", clean, re.DOTALL)
|
||||
payload = match.group(1) if match else clean
|
||||
|
||||
try:
|
||||
return json.loads(payload.strip())
|
||||
except json.JSONDecodeError:
|
||||
# 3. Recovery: Suche nach der ersten [ und letzten ]
|
||||
start = payload.find('[')
|
||||
end = payload.rfind(']') + 1
|
||||
if start != -1 and end > start:
|
||||
try:
|
||||
return json.loads(payload[start:end])
|
||||
except: pass
|
||||
|
||||
# 4. Zweite Recovery: Suche nach der ersten { und letzten }
|
||||
start_obj = payload.find('{')
|
||||
end_obj = payload.rfind('}') + 1
|
||||
if start_obj != -1 and end_obj > start_obj:
|
||||
try:
|
||||
return json.loads(payload[start_obj:end_obj])
|
||||
except: pass
|
||||
return []
|
||||
|
||||
async def assign_edges_to_chunk(self, chunk_text: str, all_edges: List[str], note_type: str) -> List[str]:
|
||||
"""
|
||||
Sendet einen Chunk und eine Liste potenzieller Kanten an das LLM.
|
||||
@@ -65,7 +102,7 @@ class SemanticAnalyzer:
|
||||
provider = self.llm.settings.MINDNET_LLM_PROVIDER
|
||||
model = self.llm.settings.OPENROUTER_MODEL if provider == "openrouter" else self.llm.settings.GEMINI_MODEL
|
||||
|
||||
# 2. Prompt laden (Provider-spezifisch)
|
||||
# 2. Prompt laden (Provider-spezifisch via get_prompt)
|
||||
prompt_template = self.llm.get_prompt("edge_allocation_template", provider)
|
||||
|
||||
if not prompt_template or not isinstance(prompt_template, str):
|
||||
@@ -86,7 +123,7 @@ class SemanticAnalyzer:
|
||||
|
||||
# 4. Prompt füllen mit Format-Check (Kein Shortcut)
|
||||
try:
|
||||
# Wir begrenzen den Text auf eine vernünftige Länge für das Kontextfenster (ca. 10k Tokens max)
|
||||
# Wir begrenzen den Text auf eine vernünftige Länge für das Kontextfenster
|
||||
final_prompt = prompt_template.format(
|
||||
chunk_text=chunk_text[:6000],
|
||||
edge_list=edges_str,
|
||||
@@ -108,30 +145,12 @@ class SemanticAnalyzer:
|
||||
model_override=model
|
||||
)
|
||||
|
||||
# 6. Bulletproof JSON Extraction (Analog zur Ingestion)
|
||||
# Entfernt Markdown-Code-Blöcke falls vorhanden
|
||||
match = re.search(r"```(?:json)?\s*(.*?)\s*```", response_json, re.DOTALL)
|
||||
clean_json = match.group(1) if match else response_json
|
||||
clean_json = clean_json.strip()
|
||||
# 6. Mistral-sicheres JSON Parsing via Helper
|
||||
data = self._extract_json_safely(response_json)
|
||||
|
||||
if not clean_json:
|
||||
if not data:
|
||||
return []
|
||||
|
||||
try:
|
||||
data = json.loads(clean_json)
|
||||
except json.JSONDecodeError:
|
||||
# Letzter Rettungsversuch: Suche nach dem ersten '[' und letzten ']'
|
||||
start = clean_json.find('[')
|
||||
end = clean_json.rfind(']') + 1
|
||||
if start != -1 and end != 0:
|
||||
try:
|
||||
data = json.loads(clean_json[start:end])
|
||||
except:
|
||||
logger.error("❌ [SemanticAnalyzer] JSON Recovery failed.")
|
||||
return []
|
||||
else:
|
||||
return []
|
||||
|
||||
# 7. Robuste Normalisierung (List vs Dict Recovery)
|
||||
raw_candidates = []
|
||||
if isinstance(data, list):
|
||||
@@ -146,7 +165,9 @@ class SemanticAnalyzer:
|
||||
if not raw_candidates:
|
||||
for k, v in data.items():
|
||||
if isinstance(v, str): raw_candidates.append(f"{k}:{v}")
|
||||
elif isinstance(v, list): [raw_candidates.append(f"{k}:{i}") for i in v if isinstance(i, str)]
|
||||
elif isinstance(v, list):
|
||||
for target in v:
|
||||
if isinstance(target, str): raw_candidates.append(f"{k}:{target}")
|
||||
|
||||
# 8. Strikte Validierung gegen Kanten-Format
|
||||
valid_edges = []
|
||||
|
||||
Reference in new issue
Block a user