Mistral sichere Parser implemntierung
This commit is contained in:
1 parent
ecfdc67485
commit
16e128668c
2 files changed
+87
-53
No files matched your search
+38
-25
@@ -1,10 +1,10 @@
|
||||
"""
|
||||
FILE: app/core/ingestion.py
|
||||
DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen.
|
||||
WP-20: Optimiert für OpenRouter (openai/gpt-oss-20b:free) als Primary.
|
||||
WP-20: Optimiert für OpenRouter (mistralai/mistral-7b-instruct:free).
|
||||
WP-22: Content Lifecycle, Edge Registry Validation & Multi-Hash.
|
||||
FIX: Finale DoD-Härtung, Entfernung aller Shortcuts und Stabilitätspatch.
|
||||
VERSION: 2.11.10
|
||||
FIX: Finale Mistral-Härtung (<s> & [OUT] Tags), robuste JSON-Recovery & DoD-Sync.
|
||||
VERSION: 2.11.11
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.services.llm_service, app.services.edge_registry
|
||||
"""
|
||||
@@ -49,21 +49,41 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Global Helpers ---
|
||||
def extract_json_from_response(text: str) -> Any:
|
||||
"""Extrahiert JSON-Daten, selbst wenn sie in Markdown-Blöcken stehen."""
|
||||
"""
|
||||
Extrahiert JSON-Daten und bereinigt LLM-Steuerzeichen (Mistral/Llama).
|
||||
Entfernt <s>, [OUT], [/OUT] und Markdown-Blöcke für maximale Robustheit.
|
||||
"""
|
||||
if not text: return []
|
||||
# Suche nach ```json ... ``` oder ``` ... ```
|
||||
match = re.search(r"```(?:json)?\s*(.*?)\s*```", text, re.DOTALL)
|
||||
clean_text = match.group(1) if match else text
|
||||
|
||||
# 1. Entferne Mistral/Llama Steuerzeichen und Tags
|
||||
clean = text.replace("<s>", "").replace("</s>", "")
|
||||
clean = clean.replace("[OUT]", "").replace("[/OUT]", "")
|
||||
clean = clean.strip()
|
||||
|
||||
# 2. Suche nach Markdown JSON-Blöcken (```json ... ```)
|
||||
match = re.search(r"```(?:json)?\s*(.*?)\s*```", clean, re.DOTALL)
|
||||
payload = match.group(1) if match else clean
|
||||
|
||||
try:
|
||||
return json.loads(clean_text.strip())
|
||||
return json.loads(payload.strip())
|
||||
except json.JSONDecodeError:
|
||||
# Versuch: Alles vor der ersten [ und nach der letzten ] entfernen (Recovery)
|
||||
start = clean_text.find('[')
|
||||
end = clean_text.rfind(']') + 1
|
||||
if start != -1 and end != 0:
|
||||
try: return json.loads(clean_text[start:end])
|
||||
# 3. Recovery: Suche nach der ersten [ und letzten ] (Liste)
|
||||
start = payload.find('[')
|
||||
end = payload.rfind(']') + 1
|
||||
if start != -1 and end > start:
|
||||
try:
|
||||
return json.loads(payload[start:end])
|
||||
except: pass
|
||||
raise
|
||||
|
||||
# 4. Zweite Recovery: Suche nach der ersten { und letzten } (Objekt)
|
||||
start_obj = payload.find('{')
|
||||
end_obj = payload.rfind('}') + 1
|
||||
if start_obj != -1 and end_obj > start_obj:
|
||||
try:
|
||||
return json.loads(payload[start_obj:end_obj])
|
||||
except: pass
|
||||
|
||||
return []
|
||||
|
||||
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
||||
"""Lädt die types.yaml zur Steuerung der typ-spezifischen Ingestion."""
|
||||
@@ -121,14 +141,7 @@ class IngestionService:
|
||||
Respektiert die Provider-Einstellung (OpenRouter Primary).
|
||||
"""
|
||||
provider = self.settings.MINDNET_LLM_PROVIDER
|
||||
|
||||
# Modell-Zuordnung basierend auf Provider-Wahl (Keine festen Annahmen)
|
||||
if provider == "openrouter":
|
||||
model = self.settings.OPENROUTER_MODEL
|
||||
elif provider == "gemini":
|
||||
model = self.settings.GEMINI_MODEL
|
||||
else:
|
||||
model = self.settings.LLM_MODEL
|
||||
model = self.settings.OPENROUTER_MODEL if provider == "openrouter" else self.settings.GEMINI_MODEL
|
||||
|
||||
logger.info(f"🚀 [Ingestion] Turbo-Mode: Extracting edges for '{note_id}' using {model} on {provider}")
|
||||
|
||||
@@ -140,14 +153,14 @@ class IngestionService:
|
||||
try:
|
||||
# Sicherheits-Check: Formatierung des Templates gegen KeyError schützen
|
||||
try:
|
||||
# Nutzt die ersten 6000 Zeichen als Kontext-Fenster (DoD: Explizit dokumentiert)
|
||||
# Nutzt die ersten 6000 Zeichen als Kontext-Fenster
|
||||
prompt = template.format(
|
||||
text=text[:6000],
|
||||
note_id=note_id,
|
||||
valid_types=valid_types_str
|
||||
)
|
||||
except KeyError as ke:
|
||||
logger.error(f"❌ [Ingestion] Prompt-Template Fehler (Variable {ke} fehlt). Prüfe prompts.yaml Maskierung.")
|
||||
logger.error(f"❌ [Ingestion] Prompt-Template Fehler (Variable {ke} fehlt).")
|
||||
return []
|
||||
|
||||
response_json = await self.llm.generate_raw_response(
|
||||
@@ -155,7 +168,7 @@ class IngestionService:
|
||||
provider=provider, model_override=model
|
||||
)
|
||||
|
||||
# Robustes JSON-Parsing via Helper
|
||||
# Nutzt den verbesserten Mistral-sicheren JSON-Extraktor
|
||||
raw_data = extract_json_from_response(response_json)
|
||||
|
||||
# Recovery: Suche nach Listen in Dictionaries (z.B. {"edges": [...]})
|
||||
|
||||
Reference in new issue
Block a user