komplett neues WP20 deployment
This commit is contained in:
1 parent
f1bfa40b5b
commit
0157faab89
7 files changed
+203
-137
No files matched your search
@@ -2,7 +2,10 @@
|
||||
FILE: app/services/edge_registry.py
|
||||
DESCRIPTION: Single Source of Truth für Kanten-Typen mit dynamischem Reload.
|
||||
WP-22: Fix für absolute Pfade außerhalb des Vaults (Prod-Dictionary).
|
||||
VERSION: 0.7.4 (Fix: Absolute Path Escaping & Quote Stripping)
|
||||
WP-20: Synchronisation mit zentralen Settings (v0.6.2).
|
||||
VERSION: 0.7.5
|
||||
STATUS: Active
|
||||
DEPENDENCIES: re, os, json, logging, time, app.config
|
||||
"""
|
||||
import re
|
||||
import os
|
||||
@@ -25,69 +28,49 @@ class EdgeRegistry:
|
||||
cls._instance.initialized = False
|
||||
return cls._instance
|
||||
|
||||
def __init__(self, vault_root: Optional[str] = None):
|
||||
def __init__(self):
|
||||
if self.initialized:
|
||||
return
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
# 1. ENV-Werte laden und von Anführungszeichen bereinigen
|
||||
env_vocab_path = os.getenv("MINDNET_VOCAB_PATH")
|
||||
if env_vocab_path:
|
||||
env_vocab_path = env_vocab_path.strip('"').strip("'")
|
||||
|
||||
env_vault_root = os.getenv("MINDNET_VAULT_ROOT") or getattr(settings, "MINDNET_VAULT_ROOT", "./vault")
|
||||
if env_vault_root:
|
||||
env_vault_root = env_vault_root.strip('"').strip("'")
|
||||
# 1. Pfad aus den zentralen Settings laden (WP-20 Synchronisation)
|
||||
# Priorisiert den Pfad aus der .env / config.py (v0.6.2)
|
||||
self.full_vocab_path = os.path.abspath(settings.MINDNET_VOCAB_PATH)
|
||||
|
||||
# 2. Pfad-Priorität: Wenn absolut (/), dann direkt nutzen!
|
||||
if env_vocab_path:
|
||||
if env_vocab_path.startswith("/"):
|
||||
# Absoluter Pfad (z.B. aus Produktion)
|
||||
self.full_vocab_path = env_vocab_path
|
||||
else:
|
||||
# Relativer Pfad zum aktuellen Verzeichnis oder Vault
|
||||
self.full_vocab_path = os.path.abspath(env_vocab_path)
|
||||
else:
|
||||
# Fallback: Suche im Vault-Verzeichnis
|
||||
possible_paths = [
|
||||
os.path.join(env_vault_root, "_system", "dictionary", "edge_vocabulary.md"),
|
||||
os.path.join(env_vault_root, "01_User_Manual", "01_edge_vocabulary.md")
|
||||
]
|
||||
self.full_vocab_path = None
|
||||
for p in possible_paths:
|
||||
if os.path.exists(p):
|
||||
self.full_vocab_path = os.path.abspath(p)
|
||||
break
|
||||
|
||||
if not self.full_vocab_path:
|
||||
self.full_vocab_path = os.path.abspath(possible_paths[0])
|
||||
|
||||
self.unknown_log_path = "data/logs/unknown_edges.jsonl"
|
||||
self.canonical_map: Dict[str, str] = {}
|
||||
self.valid_types: Set[str] = set()
|
||||
self._last_mtime = 0.0
|
||||
|
||||
print(f"\n>>> [EDGE-REGISTRY] Initializing with Path: {self.full_vocab_path}", flush=True)
|
||||
# Initialer Ladevorgang
|
||||
logger.info(f">>> [EDGE-REGISTRY] Initializing with Path: {self.full_vocab_path}")
|
||||
self.ensure_latest()
|
||||
self.initialized = True
|
||||
|
||||
def ensure_latest(self):
|
||||
"""Prüft den Zeitstempel und lädt bei Bedarf neu."""
|
||||
"""
|
||||
Prüft den Zeitstempel der Vokabular-Datei und lädt bei Bedarf neu.
|
||||
Verhindert den AttributeError in der Ingestion-Pipeline.
|
||||
"""
|
||||
if not os.path.exists(self.full_vocab_path):
|
||||
print(f"!!! [EDGE-REGISTRY ERROR] File not found: {self.full_vocab_path} !!!", flush=True)
|
||||
logger.error(f"!!! [EDGE-REGISTRY ERROR] File not found: {self.full_vocab_path} !!!")
|
||||
return
|
||||
|
||||
current_mtime = os.path.getmtime(self.full_vocab_path)
|
||||
if current_mtime > self._last_mtime:
|
||||
self._load_vocabulary()
|
||||
self._last_mtime = current_mtime
|
||||
try:
|
||||
current_mtime = os.path.getmtime(self.full_vocab_path)
|
||||
if current_mtime > self._last_mtime:
|
||||
self._load_vocabulary()
|
||||
self._last_mtime = current_mtime
|
||||
except Exception as e:
|
||||
logger.error(f"!!! [EDGE-REGISTRY] Error checking file time: {e}")
|
||||
|
||||
def _load_vocabulary(self):
|
||||
"""Parst das Wörterbuch."""
|
||||
"""Parst das Markdown-Wörterbuch und baut die Canonical-Map auf."""
|
||||
self.canonical_map.clear()
|
||||
self.valid_types.clear()
|
||||
|
||||
# Regex für Tabellen-Struktur: | **Typ** | Aliase |
|
||||
pattern = re.compile(r"\|\s*\*\*`?([a-zA-Z0-9_-]+)`?\*\*\s*\|\s*([^|]+)\|")
|
||||
|
||||
try:
|
||||
@@ -106,38 +89,48 @@ class EdgeRegistry:
|
||||
if aliases_str and "Kein Alias" not in aliases_str:
|
||||
aliases = [a.strip() for a in aliases_str.split(",") if a.strip()]
|
||||
for alias in aliases:
|
||||
# Normalisierung: Kleinschreibung, Underscores statt Leerzeichen
|
||||
clean_alias = alias.replace("`", "").lower().strip().replace(" ", "_")
|
||||
self.canonical_map[clean_alias] = canonical
|
||||
c_aliases += 1
|
||||
|
||||
print(f"=== [EDGE-REGISTRY SUCCESS] Loaded {c_types} Canonical Types and {c_aliases} Aliases ===", flush=True)
|
||||
logger.info(f"=== [EDGE-REGISTRY SUCCESS] Loaded {c_types} Canonical Types and {c_aliases} Aliases ===")
|
||||
|
||||
except Exception as e:
|
||||
print(f"!!! [EDGE-REGISTRY FATAL] Error reading file: {e} !!!", flush=True)
|
||||
logger.error(f"!!! [EDGE-REGISTRY FATAL] Error reading file: {e} !!!")
|
||||
|
||||
def resolve(self, edge_type: str, provenance: str = "explicit", context: dict = None) -> str:
|
||||
"""Validierung mit Fundort-Logging."""
|
||||
"""
|
||||
Validiert einen Kanten-Typ gegen das Vokabular.
|
||||
Loggt unbekannte Typen für die spätere manuelle Pflege.
|
||||
"""
|
||||
self.ensure_latest()
|
||||
if not edge_type: return "related_to"
|
||||
if not edge_type:
|
||||
return "related_to"
|
||||
|
||||
# Normalisierung des Typs
|
||||
clean_type = edge_type.lower().strip().replace(" ", "_").replace("-", "_")
|
||||
ctx = context or {}
|
||||
|
||||
# System-Kanten dürfen nicht manuell vergeben werden
|
||||
if provenance == "explicit" and clean_type in self.FORBIDDEN_SYSTEM_EDGES:
|
||||
self._log_issue(clean_type, "forbidden_system_usage", ctx)
|
||||
return "related_to"
|
||||
|
||||
# System-Kanten sind nur bei struktureller Provenienz erlaubt
|
||||
if provenance == "structure" and clean_type in self.FORBIDDEN_SYSTEM_EDGES:
|
||||
return clean_type
|
||||
|
||||
# Mapping auf kanonischen Namen
|
||||
if clean_type in self.canonical_map:
|
||||
return self.canonical_map[clean_type]
|
||||
|
||||
# Fallback und Logging
|
||||
self._log_issue(clean_type, "unknown_type", ctx)
|
||||
return clean_type
|
||||
|
||||
def _log_issue(self, edge_type: str, error_kind: str, ctx: dict):
|
||||
"""Detailliertes JSONL-Logging für Debugging."""
|
||||
"""Detailliertes JSONL-Logging für die Vokabular-Optimierung."""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(self.unknown_log_path), exist_ok=True)
|
||||
entry = {
|
||||
@@ -150,6 +143,8 @@ class EdgeRegistry:
|
||||
}
|
||||
with open(self.unknown_log_path, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(entry) + "\n")
|
||||
except Exception: pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Singleton Export
|
||||
registry = EdgeRegistry()
|
||||
@@ -3,7 +3,8 @@ FILE: app/services/llm_service.py
|
||||
DESCRIPTION: Hybrid-Client für Ollama, Google GenAI (Gemini) und OpenRouter.
|
||||
Verwaltet provider-spezifische Prompts und Background-Last.
|
||||
WP-20: Optimiertes Fallback-Management zum Schutz von Cloud-Quoten.
|
||||
VERSION: 3.3.1
|
||||
WP-20 Fix: Bulletproof Prompt-Auflösung für format() Aufrufe.
|
||||
VERSION: 3.3.2
|
||||
STATUS: Active
|
||||
"""
|
||||
import httpx
|
||||
@@ -58,11 +59,14 @@ class LLMService:
|
||||
def _load_prompts(self) -> dict:
|
||||
"""Lädt die Prompt-Konfiguration aus der YAML-Datei."""
|
||||
path = Path(self.settings.PROMPTS_PATH)
|
||||
if not path.exists(): return {}
|
||||
if not path.exists():
|
||||
logger.error(f"❌ Prompts file not found at {path}")
|
||||
return {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load prompts: {e}")
|
||||
logger.error(f"❌ Failed to load prompts: {e}")
|
||||
return {}
|
||||
|
||||
def get_prompt(self, key: str, provider: str = None) -> str:
|
||||
@@ -70,12 +74,22 @@ class LLMService:
|
||||
Hole provider-spezifisches Template mit intelligenter Text-Kaskade.
|
||||
HINWEIS: Dies ist nur ein Text-Lookup und verbraucht kein API-Kontingent.
|
||||
Kaskade: Gewählter Provider -> Gemini (Cloud-Stil) -> Ollama (Basis-Stil).
|
||||
|
||||
WP-20 Fix: Garantiert die Rückgabe eines Strings, um AttributeError zu vermeiden.
|
||||
"""
|
||||
active_provider = provider or self.settings.MINDNET_LLM_PROVIDER
|
||||
data = self.prompts.get(key, "")
|
||||
|
||||
if isinstance(data, dict):
|
||||
# Wir versuchen erst den Provider, dann Gemini (weil ähnlich leistungsfähig), dann Ollama
|
||||
return data.get(active_provider, data.get("gemini", data.get("ollama", "")))
|
||||
val = data.get(active_provider, data.get("gemini", data.get("ollama", "")))
|
||||
|
||||
# Falls val durch YAML-Fehler immer noch ein Dict ist, extrahiere ersten String
|
||||
if isinstance(val, dict):
|
||||
logger.warning(f"⚠️ [LLMService] Nested dictionary detected for key '{key}'. Using first entry.")
|
||||
val = next(iter(val.values()), "") if val else ""
|
||||
return str(val)
|
||||
|
||||
return str(data)
|
||||
|
||||
async def generate_raw_response(
|
||||
@@ -116,6 +130,7 @@ class LLMService:
|
||||
|
||||
async def _execute_google(self, prompt, system, force_json, model_override):
|
||||
"""Native Google SDK Integration (Gemini)."""
|
||||
# Nutzt GEMINI_MODEL aus config.py falls kein override (z.B. für Ingestion) übergeben wurde
|
||||
model = model_override or self.settings.GEMINI_MODEL
|
||||
config = types.GenerateContentConfig(
|
||||
system_instruction=system,
|
||||
@@ -130,7 +145,8 @@ class LLMService:
|
||||
|
||||
async def _execute_openrouter(self, prompt, system, force_json, model_override):
|
||||
"""OpenRouter API Integration (OpenAI-kompatibel)."""
|
||||
model = model_override or getattr(self.settings, "OPENROUTER_MODEL", "google/gemma-2-9b-it:free")
|
||||
# Nutzt OPENROUTER_MODEL aus config.py (v0.6.2)
|
||||
model = model_override or self.settings.OPENROUTER_MODEL
|
||||
messages = []
|
||||
if system:
|
||||
messages.append({"role": "system", "content": system})
|
||||
@@ -166,7 +182,7 @@ class LLMService:
|
||||
except Exception as e:
|
||||
attempt += 1
|
||||
if attempt > max_retries:
|
||||
logger.error(f"Ollama Error after {attempt} retries: {e}")
|
||||
logger.error(f"❌ Ollama Error after {attempt} retries: {e}")
|
||||
raise e
|
||||
wait_time = base_delay * (2 ** (attempt - 1))
|
||||
logger.warning(f"⚠️ Ollama attempt {attempt} failed. Retrying in {wait_time}s...")
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""
|
||||
FILE: app/services/semantic_analyzer.py
|
||||
DESCRIPTION: KI-gestützte Kanten-Validierung. Nutzt LLM (Background-Priority), um Kanten präzise einem Chunk zuzuordnen.
|
||||
WP-20 Fix: Kompatibilität mit Provider-basierten Prompt-Dictionaries (Hybrid-Modus).
|
||||
VERSION: 2.2.0
|
||||
WP-20 Fix: Volle Kompatibilität mit der gehärteten LLMService (v3.3.2) Kaskade.
|
||||
WP-20: Unterstützung für Provider-spezifische Routing-Logik beim Import.
|
||||
VERSION: 2.2.1
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.services.llm_service, json, logging
|
||||
LAST_ANALYSIS: 2025-12-23
|
||||
@@ -53,16 +54,17 @@ class SemanticAnalyzer:
|
||||
Sendet einen Chunk und eine Liste potenzieller Kanten an das LLM.
|
||||
Das LLM filtert heraus, welche Kanten für diesen Chunk relevant sind.
|
||||
|
||||
WP-20 Fix: Nutzt get_prompt(), um den 'AttributeError: dict object' zu vermeiden.
|
||||
WP-20 Fix: Nutzt get_prompt(), um den 'AttributeError: dict object' sicher zu vermeiden.
|
||||
"""
|
||||
if not all_edges:
|
||||
return []
|
||||
|
||||
# 1. Prompt laden via get_prompt (handelt die Provider-Kaskade automatisch ab) [WP-20 Fix]
|
||||
# 1. Prompt laden via get_prompt (handelt die Provider-Kaskade automatisch ab)
|
||||
prompt_template = self.llm.get_prompt("edge_allocation_template")
|
||||
|
||||
# Sicherheits-Check für die Format-Methode
|
||||
if not prompt_template or isinstance(prompt_template, dict):
|
||||
logger.warning("⚠️ [SemanticAnalyzer] Prompt 'edge_allocation_template' konnte nicht als String geladen werden. Nutze Hard-Fallback.")
|
||||
logger.warning("⚠️ [SemanticAnalyzer] Prompt 'edge_allocation_template' konnte nicht als String geladen werden. Nutze Not-Fallback.")
|
||||
prompt_template = (
|
||||
"TASK: Wähle aus den Kandidaten die relevanten Kanten für den Text.\n"
|
||||
"TEXT: {chunk_text}\n"
|
||||
@@ -76,7 +78,7 @@ class SemanticAnalyzer:
|
||||
# LOG: Request Info
|
||||
logger.debug(f"🔍 [SemanticAnalyzer] Request: {len(chunk_text)} chars Text, {len(all_edges)} Candidates.")
|
||||
|
||||
# 3. Prompt füllen (Hier trat der AttributeError auf, wenn prompt_template ein dict war)
|
||||
# 3. Prompt füllen (Nutzt nun sicher einen String dank LLMService v3.3.2)
|
||||
try:
|
||||
final_prompt = prompt_template.format(
|
||||
chunk_text=chunk_text[:3500],
|
||||
@@ -88,6 +90,7 @@ class SemanticAnalyzer:
|
||||
|
||||
try:
|
||||
# 4. LLM Call mit Traffic Control (Background Priority)
|
||||
# Der gewählte Provider wird über die .env gesteuert (MINDNET_LLM_PROVIDER)
|
||||
response_json = await self.llm.generate_raw_response(
|
||||
prompt=final_prompt,
|
||||
force_json=True,
|
||||
@@ -132,7 +135,7 @@ class SemanticAnalyzer:
|
||||
if isinstance(target, str):
|
||||
raw_candidates.append(f"{key}:{target}")
|
||||
|
||||
# 7. Strict Validation Loop
|
||||
# 7. Strict Validation Loop (Übernahme aus deiner V2.2.0)
|
||||
for e in raw_candidates:
|
||||
e_str = str(e)
|
||||
if self._is_valid_edge_string(e_str):
|
||||
|
||||
Reference in new issue
Block a user