bug fix
This commit is contained in:
1 parent
e045371969
commit
37ec8b614e
3 files changed
+59
-45
No files matched your search
@@ -1,46 +1,25 @@
|
||||
"""
|
||||
FILE: app/core/ingestion/ingestion_utils.py
|
||||
DESCRIPTION: Hilfswerkzeuge für JSON-Recovery, Typ-Registry und Konfigurations-Lookups.
|
||||
AUDIT v2.13.8: Zentralisierung der Text-Bereinigung für LLM-Antworten.
|
||||
AUDIT v2.13.9: Behebung des Circular Imports durch Nutzung der app.core.registry.
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import yaml
|
||||
from typing import Any, Optional, Dict, List
|
||||
from typing import Any, Optional, Dict
|
||||
|
||||
def clean_llm_text(text: str, registry: Optional[dict] = None) -> str:
|
||||
"""
|
||||
Entfernt LLM-Steuerzeichen und Artefakte aus einem Text.
|
||||
Nutzt die cleanup_patterns aus der Registry oder Standardwerte.
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return ""
|
||||
|
||||
# Fallback-Patterns, falls die Registry nicht greift
|
||||
default_patterns = ["<s>", "</s>", "[OUT]", "[/OUT]"]
|
||||
|
||||
# Falls keine Registry übergeben wurde, versuchen wir sie zu laden
|
||||
reg = registry or load_type_registry()
|
||||
|
||||
# Lade Patterns aus llm_settings (WP-14 Erweiterung)
|
||||
patterns: List[str] = reg.get("llm_settings", {}).get("cleanup_patterns", default_patterns)
|
||||
|
||||
clean = text
|
||||
for p in patterns:
|
||||
clean = clean.replace(p, "")
|
||||
|
||||
return clean.strip()
|
||||
# ENTSCHEIDENDER FIX: Import der Basis-Logik aus dem neutralen Registry-Modul.
|
||||
# Dies bricht den Zirkelbezug auf, da dieses Modul keine Services mehr importiert.
|
||||
from app.core.registry import load_type_registry, clean_llm_text
|
||||
|
||||
def extract_json_from_response(text: str, registry: Optional[dict] = None) -> Any:
|
||||
"""
|
||||
Extrahiert JSON-Daten und bereinigt LLM-Steuerzeichen.
|
||||
WP-14: Nutzt nun die zentrale clean_llm_text Funktion.
|
||||
WP-14: Nutzt nun die zentrale clean_llm_text Funktion aus app.core.registry.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# 1. Text zentral bereinigen
|
||||
# 1. Text zentral bereinigen via neutralem Modul
|
||||
clean = clean_llm_text(text, registry)
|
||||
|
||||
# 2. Markdown-Code-Blöcke extrahieren
|
||||
@@ -65,16 +44,6 @@ def extract_json_from_response(text: str, registry: Optional[dict] = None) -> An
|
||||
except: pass
|
||||
return []
|
||||
|
||||
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
||||
"""Lädt die types.yaml zur Steuerung der typ-spezifischen Ingestion."""
|
||||
from app.config import get_settings
|
||||
settings = get_settings()
|
||||
path = custom_path or settings.MINDNET_TYPES_FILE
|
||||
if not os.path.exists(path): return {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
||||
except Exception: return {}
|
||||
|
||||
def resolve_note_type(registry: dict, requested: Optional[str]) -> str:
|
||||
"""
|
||||
Bestimmt den finalen Notiz-Typ.
|
||||
@@ -89,7 +58,9 @@ def resolve_note_type(registry: dict, requested: Optional[str]) -> str:
|
||||
return ingest_cfg.get("default_note_type", "concept")
|
||||
|
||||
def get_chunk_config_by_profile(registry: dict, profile_name: str, note_type: str) -> Dict[str, Any]:
|
||||
"""Holt die Chunker-Parameter für ein spezifisches Profil aus der Registry."""
|
||||
"""
|
||||
Holt die Chunker-Parameter für ein spezifisches Profil aus der Registry.
|
||||
"""
|
||||
from app.core.chunking import get_chunk_config
|
||||
profiles = registry.get("chunking_profiles", {})
|
||||
if profile_name in profiles:
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""
|
||||
FILE: app/core/registry.py
|
||||
DESCRIPTION: Zentraler Base-Layer für Konfigurations-Loading und Text-Bereinigung.
|
||||
Bricht Zirkelbezüge zwischen Ingestion und LLMService auf.
|
||||
VERSION: 1.0.0
|
||||
"""
|
||||
import os
|
||||
import yaml
|
||||
from typing import Optional, List
|
||||
|
||||
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
||||
"""Lädt die types.yaml zur Steuerung der typ-spezifischen Logik."""
|
||||
# Wir nutzen hier einen direkten Import von Settings, um Zyklen zu vermeiden
|
||||
from app.config import get_settings
|
||||
settings = get_settings()
|
||||
path = custom_path or settings.MINDNET_TYPES_FILE
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def clean_llm_text(text: str, registry: Optional[dict] = None) -> str:
|
||||
"""
|
||||
Entfernt LLM-Steuerzeichen (<s>, [OUT] etc.) aus einem Text.
|
||||
Wird sowohl für JSON-Parsing als auch für Chat-Antworten genutzt.
|
||||
"""
|
||||
if not text or not isinstance(text, str):
|
||||
return ""
|
||||
|
||||
default_patterns = ["<s>", "</s>", "[OUT]", "[/OUT]"]
|
||||
reg = registry or load_type_registry()
|
||||
|
||||
# Lade Patterns aus llm_settings (WP-14)
|
||||
patterns: List[str] = reg.get("llm_settings", {}).get("cleanup_patterns", default_patterns)
|
||||
|
||||
clean = text
|
||||
for p in patterns:
|
||||
clean = clean.replace(p, "")
|
||||
|
||||
return clean.strip()
|
||||
Reference in new issue
Block a user