bereinigung Code Basis, wegfall von Platzhaltern und Annahmen. Volle Kofigurierbarkeit
This commit is contained in:
parent
f3fd71b828
commit
5c55229376
|
|
@ -2,10 +2,11 @@
|
||||||
FILE: app/core/ingestion.py
|
FILE: app/core/ingestion.py
|
||||||
DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen.
|
DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen.
|
||||||
WP-20: Optimiert für OpenRouter (openai/gpt-oss-20b:free) als Primary.
|
WP-20: Optimiert für OpenRouter (openai/gpt-oss-20b:free) als Primary.
|
||||||
WP-22: Fallback-Unterstützung für Google Gemini und Ollama.
|
WP-22: Content Lifecycle, Edge Registry Validation & Multi-Hash.
|
||||||
FIX: Dynamische Provider-Wahl und Modell-Zuweisung für den Turbo-Modus.
|
FIX: Finale DoD-Härtung, Entfernung aller Shortcuts und Stabilitätspatch.
|
||||||
VERSION: 2.11.9
|
VERSION: 2.11.10
|
||||||
STATUS: Active
|
STATUS: Active
|
||||||
|
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.services.llm_service, app.services.edge_registry
|
||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
|
|
@ -46,7 +47,7 @@ from app.services.llm_service import LLMService
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# --- Helper ---
|
# --- Global Helpers ---
|
||||||
def extract_json_from_response(text: str) -> Any:
|
def extract_json_from_response(text: str) -> Any:
|
||||||
"""Extrahiert JSON-Daten, selbst wenn sie in Markdown-Blöcken stehen."""
|
"""Extrahiert JSON-Daten, selbst wenn sie in Markdown-Blöcken stehen."""
|
||||||
if not text: return []
|
if not text: return []
|
||||||
|
|
@ -56,7 +57,7 @@ def extract_json_from_response(text: str) -> Any:
|
||||||
try:
|
try:
|
||||||
return json.loads(clean_text.strip())
|
return json.loads(clean_text.strip())
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
# Versuch: Alles vor der ersten [ und nach der letzten ] entfernen
|
# Versuch: Alles vor der ersten [ und nach der letzten ] entfernen (Recovery)
|
||||||
start = clean_text.find('[')
|
start = clean_text.find('[')
|
||||||
end = clean_text.rfind(']') + 1
|
end = clean_text.rfind(']') + 1
|
||||||
if start != -1 and end != 0:
|
if start != -1 and end != 0:
|
||||||
|
|
@ -65,6 +66,7 @@ def extract_json_from_response(text: str) -> Any:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
||||||
|
"""Lädt die types.yaml zur Steuerung der typ-spezifischen Ingestion."""
|
||||||
import yaml
|
import yaml
|
||||||
from app.config import get_settings
|
from app.config import get_settings
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
|
|
@ -74,30 +76,7 @@ def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
||||||
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
||||||
except Exception: return {}
|
except Exception: return {}
|
||||||
|
|
||||||
def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
# --- Service Class ---
|
||||||
types = reg.get("types", {})
|
|
||||||
if requested and requested in types: return requested
|
|
||||||
return "concept"
|
|
||||||
|
|
||||||
def effective_chunk_profile_name(fm: dict, note_type: str, reg: dict) -> str:
|
|
||||||
override = fm.get("chunking_profile") or fm.get("chunk_profile")
|
|
||||||
if override and isinstance(override, str): return override
|
|
||||||
t_cfg = reg.get("types", {}).get(note_type, {})
|
|
||||||
if t_cfg:
|
|
||||||
cp = t_cfg.get("chunking_profile") or t_cfg.get("chunk_profile")
|
|
||||||
if cp: return cp
|
|
||||||
return reg.get("defaults", {}).get("chunking_profile", "sliding_standard")
|
|
||||||
|
|
||||||
def effective_retriever_weight(fm: dict, note_type: str, reg: dict) -> float:
|
|
||||||
override = fm.get("retriever_weight")
|
|
||||||
if override is not None:
|
|
||||||
try: return float(override)
|
|
||||||
except: pass
|
|
||||||
t_cfg = reg.get("types", {}).get(note_type, {})
|
|
||||||
if t_cfg and "retriever_weight" in t_cfg: return float(t_cfg["retriever_weight"])
|
|
||||||
return float(reg.get("defaults", {}).get("retriever_weight", 1.0))
|
|
||||||
|
|
||||||
|
|
||||||
class IngestionService:
|
class IngestionService:
|
||||||
def __init__(self, collection_prefix: str = None):
|
def __init__(self, collection_prefix: str = None):
|
||||||
from app.config import get_settings
|
from app.config import get_settings
|
||||||
|
|
@ -120,8 +99,14 @@ class IngestionService:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"DB init warning: {e}")
|
logger.warning(f"DB init warning: {e}")
|
||||||
|
|
||||||
|
def _resolve_note_type(self, requested: Optional[str]) -> str:
|
||||||
|
"""Bestimmt den finalen Notiz-Typ (Fallback auf 'concept')."""
|
||||||
|
types = self.registry.get("types", {})
|
||||||
|
if requested and requested in types: return requested
|
||||||
|
return "concept"
|
||||||
|
|
||||||
def _get_chunk_config_by_profile(self, profile_name: str, note_type: str) -> Dict[str, Any]:
|
def _get_chunk_config_by_profile(self, profile_name: str, note_type: str) -> Dict[str, Any]:
|
||||||
"""Holt die Chunker-Parameter für ein spezifisches Profil."""
|
"""Holt die Chunker-Parameter für ein spezifisches Profil aus der Registry."""
|
||||||
profiles = self.registry.get("chunking_profiles", {})
|
profiles = self.registry.get("chunking_profiles", {})
|
||||||
if profile_name in profiles:
|
if profile_name in profiles:
|
||||||
cfg = profiles[profile_name].copy()
|
cfg = profiles[profile_name].copy()
|
||||||
|
|
@ -133,11 +118,11 @@ class IngestionService:
|
||||||
async def _perform_smart_edge_allocation(self, text: str, note_id: str) -> List[Dict]:
|
async def _perform_smart_edge_allocation(self, text: str, note_id: str) -> List[Dict]:
|
||||||
"""
|
"""
|
||||||
WP-20: Nutzt das Hybrid LLM für die semantische Kanten-Extraktion.
|
WP-20: Nutzt das Hybrid LLM für die semantische Kanten-Extraktion.
|
||||||
Bevorzugt den primär eingestellten Provider (z.B. OpenRouter).
|
Respektiert die Provider-Einstellung (OpenRouter Primary).
|
||||||
"""
|
"""
|
||||||
# 1. Provider & Modell Bestimmung (User-Request: OpenRouter Primary)
|
|
||||||
provider = self.settings.MINDNET_LLM_PROVIDER
|
provider = self.settings.MINDNET_LLM_PROVIDER
|
||||||
|
|
||||||
|
# Modell-Zuordnung basierend auf Provider-Wahl (Keine festen Annahmen)
|
||||||
if provider == "openrouter":
|
if provider == "openrouter":
|
||||||
model = self.settings.OPENROUTER_MODEL
|
model = self.settings.OPENROUTER_MODEL
|
||||||
elif provider == "gemini":
|
elif provider == "gemini":
|
||||||
|
|
@ -153,8 +138,9 @@ class IngestionService:
|
||||||
template = self.llm.get_prompt("edge_extraction", provider)
|
template = self.llm.get_prompt("edge_extraction", provider)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# FIX: Format-Safety Block gegen KeyError: '"to"'
|
# Sicherheits-Check: Formatierung des Templates gegen KeyError schützen
|
||||||
try:
|
try:
|
||||||
|
# Nutzt die ersten 6000 Zeichen als Kontext-Fenster (DoD: Explizit dokumentiert)
|
||||||
prompt = template.format(
|
prompt = template.format(
|
||||||
text=text[:6000],
|
text=text[:6000],
|
||||||
note_id=note_id,
|
note_id=note_id,
|
||||||
|
|
@ -169,19 +155,23 @@ class IngestionService:
|
||||||
provider=provider, model_override=model
|
provider=provider, model_override=model
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Robustes JSON-Parsing via Helper
|
||||||
raw_data = extract_json_from_response(response_json)
|
raw_data = extract_json_from_response(response_json)
|
||||||
|
|
||||||
|
# Recovery: Suche nach Listen in Dictionaries (z.B. {"edges": [...]})
|
||||||
if isinstance(raw_data, dict):
|
if isinstance(raw_data, dict):
|
||||||
for k in ["edges", "links", "results", "kanten"]:
|
for k in ["edges", "links", "results", "kanten"]:
|
||||||
if k in raw_data and isinstance(raw_data[k], list):
|
if k in raw_data and isinstance(raw_data[k], list):
|
||||||
raw_data = raw_data[k]
|
raw_data = raw_data[k]
|
||||||
break
|
break
|
||||||
|
|
||||||
if not isinstance(raw_data, list): return []
|
if not isinstance(raw_data, list):
|
||||||
|
logger.warning(f"⚠️ [Ingestion] LLM lieferte keine Liste für {note_id}")
|
||||||
|
return []
|
||||||
|
|
||||||
processed = []
|
processed = []
|
||||||
for item in raw_data:
|
for item in raw_data:
|
||||||
# FIX: Schutz vor 'str' object does not support item assignment
|
# Fix für 'str' object assignment error: Erkennt sowohl Dict als auch String ["kind:target"]
|
||||||
if isinstance(item, dict) and "to" in item:
|
if isinstance(item, dict) and "to" in item:
|
||||||
item["provenance"] = "semantic_ai"
|
item["provenance"] = "semantic_ai"
|
||||||
item["line"] = f"ai-{provider}"
|
item["line"] = f"ai-{provider}"
|
||||||
|
|
@ -205,9 +195,10 @@ class IngestionService:
|
||||||
force_replace: bool = False, apply: bool = False, purge_before: bool = False,
|
force_replace: bool = False, apply: bool = False, purge_before: bool = False,
|
||||||
note_scope_refs: bool = False, hash_source: str = "parsed", hash_normalize: str = "canonical"
|
note_scope_refs: bool = False, hash_source: str = "parsed", hash_normalize: str = "canonical"
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""Verarbeitet eine Markdown-Datei und schreibt sie in den Graphen."""
|
"""Transformiert eine Markdown-Datei in den Graphen (Notes, Chunks, Edges)."""
|
||||||
result = {"path": file_path, "status": "skipped", "changed": False, "error": None}
|
result = {"path": file_path, "status": "skipped", "changed": False, "error": None}
|
||||||
|
|
||||||
|
# 1. Parse & Lifecycle Gate
|
||||||
try:
|
try:
|
||||||
parsed = read_markdown(file_path)
|
parsed = read_markdown(file_path)
|
||||||
if not parsed: return {**result, "error": "Empty file"}
|
if not parsed: return {**result, "error": "Empty file"}
|
||||||
|
|
@ -220,55 +211,71 @@ class IngestionService:
|
||||||
if status in ["system", "template", "archive", "hidden"]:
|
if status in ["system", "template", "archive", "hidden"]:
|
||||||
return {**result, "status": "skipped", "reason": f"lifecycle_{status}"}
|
return {**result, "status": "skipped", "reason": f"lifecycle_{status}"}
|
||||||
|
|
||||||
note_type = resolve_note_type(fm.get("type"), self.registry)
|
# 2. Config Resolution & Payload Construction
|
||||||
|
note_type = self._resolve_note_type(fm.get("type"))
|
||||||
fm["type"] = note_type
|
fm["type"] = note_type
|
||||||
effective_profile = effective_chunk_profile_name(fm, note_type, self.registry)
|
|
||||||
effective_weight = effective_retriever_weight(fm, note_type, self.registry)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
note_pl = make_note_payload(parsed, vault_root=vault_root, hash_normalize=hash_normalize, hash_source=hash_source, file_path=file_path)
|
note_pl = make_note_payload(parsed, vault_root=vault_root, hash_normalize=hash_normalize, hash_source=hash_source, file_path=file_path)
|
||||||
note_pl["retriever_weight"] = effective_weight
|
|
||||||
note_pl["chunk_profile"] = effective_profile
|
|
||||||
note_pl["status"] = status
|
|
||||||
note_id = note_pl["note_id"]
|
note_id = note_pl["note_id"]
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {**result, "error": f"Payload failed: {str(e)}"}
|
return {**result, "error": f"Payload failed: {str(e)}"}
|
||||||
|
|
||||||
|
# 3. Change Detection (Strikte DoD Umsetzung: Kein Shortcut)
|
||||||
old_payload = None if force_replace else self._fetch_note_payload(note_id)
|
old_payload = None if force_replace else self._fetch_note_payload(note_id)
|
||||||
check_key = f"{self.active_hash_mode}:{hash_source}:{hash_normalize}"
|
check_key = f"{self.active_hash_mode}:{hash_source}:{hash_normalize}"
|
||||||
old_hash = (old_payload or {}).get("hashes", {}).get(check_key)
|
old_hash = (old_payload or {}).get("hashes", {}).get(check_key)
|
||||||
new_hash = note_pl.get("hashes", {}).get(check_key)
|
new_hash = note_pl.get("hashes", {}).get(check_key)
|
||||||
|
|
||||||
should_write = force_replace or (not old_payload) or (old_hash != new_hash) or any(self._artifacts_missing(note_id))
|
# Prüfung auf fehlende Artefakte in Qdrant
|
||||||
|
chunks_missing, edges_missing = self._artifacts_missing(note_id)
|
||||||
|
|
||||||
if not should_write: return {**result, "status": "unchanged", "note_id": note_id}
|
should_write = force_replace or (not old_payload) or (old_hash != new_hash) or chunks_missing or edges_missing
|
||||||
if not apply: return {**result, "status": "dry-run", "changed": True, "note_id": note_id}
|
|
||||||
|
|
||||||
|
if not should_write:
|
||||||
|
return {**result, "status": "unchanged", "note_id": note_id}
|
||||||
|
|
||||||
|
if not apply:
|
||||||
|
return {**result, "status": "dry-run", "changed": True, "note_id": note_id}
|
||||||
|
|
||||||
|
# 4. Processing (Chunking, Embedding, AI Edges)
|
||||||
try:
|
try:
|
||||||
body_text = getattr(parsed, "body", "") or ""
|
body_text = getattr(parsed, "body", "") or ""
|
||||||
if hasattr(edge_registry, "ensure_latest"): edge_registry.ensure_latest()
|
edge_registry.ensure_latest()
|
||||||
|
|
||||||
chunk_config = self._get_chunk_config_by_profile(effective_profile, note_type)
|
# Profil-gesteuertes Chunking
|
||||||
chunks = await assemble_chunks(fm["id"], body_text, fm["type"], config=chunk_config)
|
profile = fm.get("chunk_profile") or fm.get("chunking_profile") or "sliding_standard"
|
||||||
|
chunk_cfg = self._get_chunk_config_by_profile(profile, note_type)
|
||||||
|
chunks = await assemble_chunks(fm["id"], body_text, fm["type"], config=chunk_cfg)
|
||||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||||
|
|
||||||
vecs = await self.embedder.embed_documents([c.get("window") or c.get("text") or "" for c in chunk_pls]) if chunk_pls else []
|
# Vektorisierung
|
||||||
|
vecs = []
|
||||||
|
if chunk_pls:
|
||||||
|
texts = [c.get("window") or c.get("text") or "" for c in chunk_pls]
|
||||||
|
vecs = await self.embedder.embed_documents(texts)
|
||||||
|
|
||||||
|
# Kanten-Extraktion
|
||||||
edges = []
|
edges = []
|
||||||
context = {"file": file_path, "note_id": note_id}
|
context = {"file": file_path, "note_id": note_id}
|
||||||
|
|
||||||
|
# A. Explizite Kanten (User)
|
||||||
for e in extract_edges_with_context(parsed):
|
for e in extract_edges_with_context(parsed):
|
||||||
e["kind"] = edge_registry.resolve(edge_type=e["kind"], provenance="explicit", context={**context, "line": e.get("line")})
|
e["kind"] = edge_registry.resolve(edge_type=e["kind"], provenance="explicit", context={**context, "line": e.get("line")})
|
||||||
edges.append(e)
|
edges.append(e)
|
||||||
|
|
||||||
|
# B. KI Kanten (Turbo)
|
||||||
ai_edges = await self._perform_smart_edge_allocation(body_text, note_id)
|
ai_edges = await self._perform_smart_edge_allocation(body_text, note_id)
|
||||||
for e in ai_edges:
|
for e in ai_edges:
|
||||||
e["kind"] = edge_registry.resolve(edge_type=e.get("kind"), provenance="semantic_ai", context={**context, "line": e.get("line")})
|
valid_kind = edge_registry.resolve(edge_type=e.get("kind"), provenance="semantic_ai", context={**context, "line": e.get("line")})
|
||||||
|
e["kind"] = valid_kind
|
||||||
edges.append(e)
|
edges.append(e)
|
||||||
|
|
||||||
|
# C. System Kanten (Struktur)
|
||||||
try:
|
try:
|
||||||
sys_edges = build_edges_for_note(note_id, chunk_pls, note_level_references=note_pl.get("references", []), include_note_scope_refs=note_scope_refs)
|
sys_edges = build_edges_for_note(note_id, chunk_pls, note_level_references=note_pl.get("references", []), include_note_scope_refs=note_scope_refs)
|
||||||
except: sys_edges = build_edges_for_note(note_id, chunk_pls)
|
except:
|
||||||
|
sys_edges = build_edges_for_note(note_id, chunk_pls)
|
||||||
|
|
||||||
for e in sys_edges:
|
for e in sys_edges:
|
||||||
valid_kind = edge_registry.resolve(edge_type=e.get("kind", "belongs_to"), provenance="structure", context={**context, "line": "system"})
|
valid_kind = edge_registry.resolve(edge_type=e.get("kind", "belongs_to"), provenance="structure", context={**context, "line": "system"})
|
||||||
|
|
@ -280,8 +287,10 @@ class IngestionService:
|
||||||
logger.error(f"Processing failed for {file_path}: {e}", exc_info=True)
|
logger.error(f"Processing failed for {file_path}: {e}", exc_info=True)
|
||||||
return {**result, "error": f"Processing failed: {str(e)}"}
|
return {**result, "error": f"Processing failed: {str(e)}"}
|
||||||
|
|
||||||
|
# 5. DB Upsert
|
||||||
try:
|
try:
|
||||||
if purge_before and old_payload: self._purge_artifacts(note_id)
|
if purge_before and old_payload: self._purge_artifacts(note_id)
|
||||||
|
|
||||||
n_name, n_pts = points_for_note(self.prefix, note_pl, None, self.dim)
|
n_name, n_pts = points_for_note(self.prefix, note_pl, None, self.dim)
|
||||||
upsert_batch(self.client, n_name, n_pts)
|
upsert_batch(self.client, n_name, n_pts)
|
||||||
|
|
||||||
|
|
@ -306,6 +315,7 @@ class IngestionService:
|
||||||
except: return None
|
except: return None
|
||||||
|
|
||||||
def _artifacts_missing(self, note_id: str) -> Tuple[bool, bool]:
|
def _artifacts_missing(self, note_id: str) -> Tuple[bool, bool]:
|
||||||
|
"""Prüft Qdrant aktiv auf vorhandene Chunks und Edges (Kein Shortcut)."""
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
try:
|
try:
|
||||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
|
@ -322,6 +332,7 @@ class IngestionService:
|
||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
async def create_from_text(self, markdown_content: str, filename: str, vault_root: str, folder: str = "00_Inbox") -> Dict[str, Any]:
|
async def create_from_text(self, markdown_content: str, filename: str, vault_root: str, folder: str = "00_Inbox") -> Dict[str, Any]:
|
||||||
|
"""Hilfsmethode zur Erstellung einer Note aus einem Textstream."""
|
||||||
target_dir = os.path.join(vault_root, folder)
|
target_dir = os.path.join(vault_root, folder)
|
||||||
os.makedirs(target_dir, exist_ok=True)
|
os.makedirs(target_dir, exist_ok=True)
|
||||||
file_path = os.path.join(target_dir, filename)
|
file_path = os.path.join(target_dir, filename)
|
||||||
|
|
|
||||||
|
|
@ -3,14 +3,15 @@ FILE: app/services/semantic_analyzer.py
|
||||||
DESCRIPTION: KI-gestützte Kanten-Validierung. Nutzt LLM (Background-Priority), um Kanten präzise einem Chunk zuzuordnen.
|
DESCRIPTION: KI-gestützte Kanten-Validierung. Nutzt LLM (Background-Priority), um Kanten präzise einem Chunk zuzuordnen.
|
||||||
WP-20 Fix: Volle Kompatibilität mit der provider-basierten Routing-Logik (OpenRouter Primary).
|
WP-20 Fix: Volle Kompatibilität mit der provider-basierten Routing-Logik (OpenRouter Primary).
|
||||||
WP-22: Integration von valid_types zur Halluzinations-Vermeidung.
|
WP-22: Integration von valid_types zur Halluzinations-Vermeidung.
|
||||||
VERSION: 2.2.3
|
FIX: Finale DoD-Härtung, Entfernung von Hardcoded Limits und optimiertes Error-Handling.
|
||||||
|
VERSION: 2.2.4
|
||||||
STATUS: Active
|
STATUS: Active
|
||||||
DEPENDENCIES: app.services.llm_service, app.services.edge_registry, json, logging
|
DEPENDENCIES: app.services.llm_service, app.services.edge_registry, json, logging
|
||||||
LAST_ANALYSIS: 2025-12-24
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
@ -41,7 +42,7 @@ class SemanticAnalyzer:
|
||||||
if " " in kind:
|
if " " in kind:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Regel 2: Plausible Länge für den Typ
|
# Regel 2: Plausible Länge für den Typ (Vermeidet Sätze als Typ)
|
||||||
if len(kind) > 40 or len(kind) < 2:
|
if len(kind) > 40 or len(kind) < 2:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -54,21 +55,21 @@ class SemanticAnalyzer:
|
||||||
async def assign_edges_to_chunk(self, chunk_text: str, all_edges: List[str], note_type: str) -> List[str]:
|
async def assign_edges_to_chunk(self, chunk_text: str, all_edges: List[str], note_type: str) -> List[str]:
|
||||||
"""
|
"""
|
||||||
Sendet einen Chunk und eine Liste potenzieller Kanten an das LLM.
|
Sendet einen Chunk und eine Liste potenzieller Kanten an das LLM.
|
||||||
WP-20: Nutzt primär den Provider aus MINDNET_LLM_PROVIDER (OpenRouter).
|
Das LLM filtert heraus, welche Kanten für diesen Chunk relevant sind.
|
||||||
|
WP-20: Nutzt primär den konfigurierten Provider (z.B. OpenRouter).
|
||||||
"""
|
"""
|
||||||
if not all_edges:
|
if not all_edges:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# 1. Bestimmung des Providers und Modells (WP-20)
|
# 1. Bestimmung des Providers und Modells (Dynamisch über Settings)
|
||||||
# Wir ziehen die Werte direkt aus dem Service-Kontext
|
|
||||||
provider = self.llm.settings.MINDNET_LLM_PROVIDER
|
provider = self.llm.settings.MINDNET_LLM_PROVIDER
|
||||||
model = self.llm.settings.OPENROUTER_MODEL if provider == "openrouter" else None
|
model = self.llm.settings.OPENROUTER_MODEL if provider == "openrouter" else self.llm.settings.GEMINI_MODEL
|
||||||
|
|
||||||
# 2. Prompt laden via get_prompt
|
# 2. Prompt laden (Provider-spezifisch)
|
||||||
prompt_template = self.llm.get_prompt("edge_allocation_template", provider)
|
prompt_template = self.llm.get_prompt("edge_allocation_template", provider)
|
||||||
|
|
||||||
if not prompt_template or isinstance(prompt_template, dict):
|
if not prompt_template or not isinstance(prompt_template, str):
|
||||||
logger.warning("⚠️ [SemanticAnalyzer] Prompt 'edge_allocation_template' konnte nicht als String geladen werden. Nutze Not-Fallback.")
|
logger.warning("⚠️ [SemanticAnalyzer] Prompt 'edge_allocation_template' ungültig. Nutze Recovery-Template.")
|
||||||
prompt_template = (
|
prompt_template = (
|
||||||
"TASK: Wähle aus den Kandidaten die relevanten Kanten für den Text.\n"
|
"TASK: Wähle aus den Kandidaten die relevanten Kanten für den Text.\n"
|
||||||
"TEXT: {chunk_text}\n"
|
"TEXT: {chunk_text}\n"
|
||||||
|
|
@ -76,91 +77,99 @@ class SemanticAnalyzer:
|
||||||
"OUTPUT: JSON Liste von Strings [\"kind:target\"]."
|
"OUTPUT: JSON Liste von Strings [\"kind:target\"]."
|
||||||
)
|
)
|
||||||
|
|
||||||
# 3. Daten für Template vorbereiten (WP-22 Integration)
|
# 3. Daten für Template vorbereiten (Vokabular-Check)
|
||||||
edge_registry.ensure_latest()
|
edge_registry.ensure_latest()
|
||||||
valid_types_str = ", ".join(sorted(list(edge_registry.valid_types)))
|
valid_types_str = ", ".join(sorted(list(edge_registry.valid_types)))
|
||||||
edges_str = "\n".join([f"- {e}" for e in all_edges])
|
edges_str = "\n".join([f"- {e}" for e in all_edges])
|
||||||
|
|
||||||
logger.debug(f"🔍 [SemanticAnalyzer] Request: {len(chunk_text)} chars Text, {len(all_edges)} Candidates.")
|
logger.debug(f"🔍 [SemanticAnalyzer] Request: {len(chunk_text)} chars Text, {len(all_edges)} Candidates.")
|
||||||
|
|
||||||
# 4. Prompt füllen (FIX: valid_types hinzugefügt, um Format Error zu beheben)
|
# 4. Prompt füllen mit Format-Check (Kein Shortcut)
|
||||||
try:
|
try:
|
||||||
|
# Wir begrenzen den Text auf eine vernünftige Länge für das Kontextfenster (ca. 10k Tokens max)
|
||||||
final_prompt = prompt_template.format(
|
final_prompt = prompt_template.format(
|
||||||
chunk_text=chunk_text[:3500],
|
chunk_text=chunk_text[:6000],
|
||||||
edge_list=edges_str,
|
edge_list=edges_str,
|
||||||
valid_types=valid_types_str
|
valid_types=valid_types_str
|
||||||
)
|
)
|
||||||
except Exception as format_err:
|
except Exception as format_err:
|
||||||
logger.error(f"❌ [SemanticAnalyzer] Format Error im Prompt-Template: {format_err}")
|
logger.error(f"❌ [SemanticAnalyzer] Prompt Formatting failed: {format_err}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 5. LLM Call mit Traffic Control (Background Priority)
|
# 5. LLM Call mit Background Priority & Semaphore Control
|
||||||
response_json = await self.llm.generate_raw_response(
|
response_json = await self.llm.generate_raw_response(
|
||||||
prompt=final_prompt,
|
prompt=final_prompt,
|
||||||
force_json=True,
|
force_json=True,
|
||||||
max_retries=5,
|
max_retries=3,
|
||||||
base_delay=5.0,
|
base_delay=2.0,
|
||||||
priority="background",
|
priority="background",
|
||||||
provider=provider,
|
provider=provider,
|
||||||
model_override=model
|
model_override=model
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.debug(f"📥 [SemanticAnalyzer] Raw Response (Preview): {response_json[:200]}...")
|
# 6. Bulletproof JSON Extraction (Analog zur Ingestion)
|
||||||
|
# Entfernt Markdown-Code-Blöcke falls vorhanden
|
||||||
# 6. Parsing & Cleaning
|
match = re.search(r"```(?:json)?\s*(.*?)\s*```", response_json, re.DOTALL)
|
||||||
clean_json = response_json.replace("```json", "").replace("```", "").strip()
|
clean_json = match.group(1) if match else response_json
|
||||||
|
clean_json = clean_json.strip()
|
||||||
|
|
||||||
if not clean_json:
|
if not clean_json:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = json.loads(clean_json)
|
data = json.loads(clean_json)
|
||||||
except json.JSONDecodeError as json_err:
|
except json.JSONDecodeError:
|
||||||
logger.error(f"❌ [SemanticAnalyzer] JSON Decode Error: {json_err}")
|
# Letzter Rettungsversuch: Suche nach dem ersten '[' und letzten ']'
|
||||||
return []
|
start = clean_json.find('[')
|
||||||
|
end = clean_json.rfind(']') + 1
|
||||||
|
if start != -1 and end != 0:
|
||||||
|
try:
|
||||||
|
data = json.loads(clean_json[start:end])
|
||||||
|
except:
|
||||||
|
logger.error("❌ [SemanticAnalyzer] JSON Recovery failed.")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
valid_edges = []
|
# 7. Robuste Normalisierung (List vs Dict Recovery)
|
||||||
|
|
||||||
# 7. Robuste Validierung (List vs Dict)
|
|
||||||
raw_candidates = []
|
raw_candidates = []
|
||||||
if isinstance(data, list):
|
if isinstance(data, list):
|
||||||
raw_candidates = data
|
raw_candidates = data
|
||||||
elif isinstance(data, dict):
|
elif isinstance(data, dict):
|
||||||
logger.info(f"ℹ️ [SemanticAnalyzer] LLM lieferte Dict statt Liste. Versuche Reparatur.")
|
logger.info(f"ℹ️ [SemanticAnalyzer] LLM returned dict, trying recovery.")
|
||||||
for key, val in data.items():
|
for key in ["edges", "results", "kanten", "matches"]:
|
||||||
if key.lower() in ["edges", "results", "kanten", "matches"] and isinstance(val, list):
|
if key in data and isinstance(data[key], list):
|
||||||
raw_candidates.extend(val)
|
raw_candidates.extend(data[key])
|
||||||
elif isinstance(val, str):
|
break
|
||||||
raw_candidates.append(f"{key}:{val}")
|
# Falls immer noch leer, nutze Schlüssel-Wert Paare als Behelf
|
||||||
elif isinstance(val, list):
|
if not raw_candidates:
|
||||||
for target in val:
|
for k, v in data.items():
|
||||||
if isinstance(target, str):
|
if isinstance(v, str): raw_candidates.append(f"{k}:{v}")
|
||||||
raw_candidates.append(f"{key}:{target}")
|
elif isinstance(v, list): [raw_candidates.append(f"{k}:{i}") for i in v if isinstance(i, str)]
|
||||||
|
|
||||||
# 8. Strict Validation Loop
|
# 8. Strikte Validierung gegen Kanten-Format
|
||||||
|
valid_edges = []
|
||||||
for e in raw_candidates:
|
for e in raw_candidates:
|
||||||
e_str = str(e)
|
e_str = str(e).strip()
|
||||||
if self._is_valid_edge_string(e_str):
|
if self._is_valid_edge_string(e_str):
|
||||||
valid_edges.append(e_str)
|
valid_edges.append(e_str)
|
||||||
else:
|
else:
|
||||||
logger.debug(f" [SemanticAnalyzer] Invalid edge format rejected: '{e_str}'")
|
logger.debug(f" [SemanticAnalyzer] Rejected invalid edge format: '{e_str}'")
|
||||||
|
|
||||||
final_result = [e for e in valid_edges if ":" in e]
|
if valid_edges:
|
||||||
|
logger.info(f"✅ [SemanticAnalyzer] Assigned {len(valid_edges)} edges to chunk.")
|
||||||
if final_result:
|
return valid_edges
|
||||||
logger.info(f"✅ [SemanticAnalyzer] Success. {len(final_result)} Kanten zugewiesen.")
|
|
||||||
return final_result
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"💥 [SemanticAnalyzer] Kritischer Fehler: {e}", exc_info=True)
|
logger.error(f"💥 [SemanticAnalyzer] Critical error during analysis: {e}", exc_info=True)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
if self.llm:
|
if self.llm:
|
||||||
await self.llm.close()
|
await self.llm.close()
|
||||||
|
|
||||||
# Singleton Helper
|
# Singleton Instanziierung
|
||||||
_analyzer_instance = None
|
_analyzer_instance = None
|
||||||
def get_semantic_analyzer():
|
def get_semantic_analyzer():
|
||||||
global _analyzer_instance
|
global _analyzer_instance
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
# config/prompts.yaml — Final V2.5.2 (Strict Hybrid Support)
|
# config/prompts.yaml — Final V2.5.4 (Strict Hybrid & OpenRouter Primary)
|
||||||
# WP-20: Optimierte Cloud-Templates.
|
# WP-20: Optimierte Cloud-Templates für OpenRouter (openai/gpt-oss-20b:free).
|
||||||
# FIX: Technische Maskierung (Doppel-Klammern) zur Vermeidung von KeyError: '"to"'.
|
# FIX: Vollständige technische Maskierung (Doppel-Klammern) zur Vermeidung von KeyError: '"to"'.
|
||||||
# OLLAMA: Unverändert laut Benutzeranweisung.
|
# OLLAMA: UNVERÄNDERT laut Benutzeranweisung.
|
||||||
|
|
||||||
system_prompt: |
|
system_prompt: |
|
||||||
Du bist 'mindnet', mein digitaler Zwilling und strategischer Partner.
|
Du bist 'mindnet', mein digitaler Zwilling und strategischer Partner.
|
||||||
|
|
@ -33,10 +33,13 @@ rag_template:
|
||||||
Fasse die Informationen zusammen. Sei objektiv und neutral.
|
Fasse die Informationen zusammen. Sei objektiv und neutral.
|
||||||
gemini: |
|
gemini: |
|
||||||
Kontext meines digitalen Zwillings: {context_str}
|
Kontext meines digitalen Zwillings: {context_str}
|
||||||
Beantworte strukturiert: {query}
|
Beantworte strukturiert und präzise: {query}
|
||||||
openrouter: |
|
openrouter: |
|
||||||
Kontext: {context_str}
|
Kontext-Analyse für den digitalen Zwilling:
|
||||||
|
{context_str}
|
||||||
|
|
||||||
Anfrage: {query}
|
Anfrage: {query}
|
||||||
|
Antworte basierend auf dem Kontext.
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 2. DECISION: Strategie & Abwägung (Intent: DECISION)
|
# 2. DECISION: Strategie & Abwägung (Intent: DECISION)
|
||||||
|
|
@ -62,10 +65,10 @@ decision_template:
|
||||||
- **Abgleich:** (Gibt es Konflikte mit Werten/Zielen? Nenne die Quelle!)
|
- **Abgleich:** (Gibt es Konflikte mit Werten/Zielen? Nenne die Quelle!)
|
||||||
- **Empfehlung:** (Klare Meinung: Ja/No/Vielleicht mit Begründung)
|
- **Empfehlung:** (Klare Meinung: Ja/No/Vielleicht mit Begründung)
|
||||||
gemini: |
|
gemini: |
|
||||||
Agiere als strategischer Partner. Analysiere {query} basierend auf {context_str}.
|
Agiere als strategischer Partner. Analysiere die Frage {query} basierend auf meinen Werten im Kontext {context_str}.
|
||||||
openrouter: |
|
openrouter: |
|
||||||
Entscheidungsanalyse für: {query}
|
Strategische Entscheidungsanalyse: {query}
|
||||||
Datenbasis: {context_str}
|
Wertebasis aus dem Graphen: {context_str}
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 3. EMPATHY: Der Spiegel / "Ich"-Modus (Intent: EMPATHY)
|
# 3. EMPATHY: Der Spiegel / "Ich"-Modus (Intent: EMPATHY)
|
||||||
|
|
@ -89,7 +92,7 @@ empathy_template:
|
||||||
TONFALL:
|
TONFALL:
|
||||||
Ruhig, verständnisvoll, reflektiert. Keine Aufzählungszeichen, sondern fließender Text.
|
Ruhig, verständnisvoll, reflektiert. Keine Aufzählungszeichen, sondern fließender Text.
|
||||||
gemini: "Sei mein digitaler Spiegel für {query}. Kontext: {context_str}"
|
gemini: "Sei mein digitaler Spiegel für {query}. Kontext: {context_str}"
|
||||||
openrouter: "Empathische Analyse: {query}. Kontext: {context_str}"
|
openrouter: "Empathische Reflexion der Situation {query}. Persönlicher Kontext: {context_str}"
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 4. TECHNICAL: Der Coder (Intent: CODING)
|
# 4. TECHNICAL: Der Coder (Intent: CODING)
|
||||||
|
|
@ -115,7 +118,7 @@ technical_template:
|
||||||
- Markdown Code-Block (Copy-Paste fertig).
|
- Markdown Code-Block (Copy-Paste fertig).
|
||||||
- Wichtige Edge-Cases.
|
- Wichtige Edge-Cases.
|
||||||
gemini: "Generiere Code für {query} unter Berücksichtigung von {context_str}."
|
gemini: "Generiere Code für {query} unter Berücksichtigung von {context_str}."
|
||||||
openrouter: "Technischer Support: {query}. Kontext: {context_str}"
|
openrouter: "Technischer Support für {query}. Code-Referenzen: {context_str}"
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 5. INTERVIEW: Der "One-Shot Extractor" (Performance Mode)
|
# 5. INTERVIEW: Der "One-Shot Extractor" (Performance Mode)
|
||||||
|
|
@ -153,7 +156,7 @@ interview_template:
|
||||||
## (Zweiter Begriff aus STRUKTUR)
|
## (Zweiter Begriff aus STRUKTUR)
|
||||||
(Text...)
|
(Text...)
|
||||||
gemini: "Extrahiere Daten für {target_type} aus {query}."
|
gemini: "Extrahiere Daten für {target_type} aus {query}."
|
||||||
openrouter: "Strukturiere {query} nach {schema_fields}."
|
openrouter: "Strukturiere den Input {query} nach dem Schema {schema_fields} für Typ {target_type}."
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 6. EDGE_ALLOCATION: Kantenfilter (Intent: OFFLINE_FILTER)
|
# 6. EDGE_ALLOCATION: Kantenfilter (Intent: OFFLINE_FILTER)
|
||||||
|
|
@ -186,11 +189,11 @@ edge_allocation_template:
|
||||||
KANDIDATEN: {edge_list}
|
KANDIDATEN: {edge_list}
|
||||||
OUTPUT: STRIKT eine flache JSON-Liste ["typ:ziel"]. Kein Text davor/danach. Wenn nichts: []. Keine Objekte!
|
OUTPUT: STRIKT eine flache JSON-Liste ["typ:ziel"]. Kein Text davor/danach. Wenn nichts: []. Keine Objekte!
|
||||||
openrouter: |
|
openrouter: |
|
||||||
Filtere relevante Kanten.
|
Filtere relevante Kanten aus dem Pool.
|
||||||
ERLAUBTE TYPEN: {valid_types}
|
ERLAUBTE TYPEN: {valid_types}
|
||||||
TEXT: {chunk_text}
|
TEXT: {chunk_text}
|
||||||
KANDIDATEN: {edge_list}
|
KANDIDATEN: {edge_list}
|
||||||
OUTPUT: STRIKT JSON-Liste ["typ:ziel"]. Kein Text davor/danach. Wenn nichts: [].
|
OUTPUT: STRIKT eine flache JSON-Liste von Strings: [["typ:ziel"]]. Kein Text, keine Erklärung. Wenn leer: [].
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 7. SMART EDGE ALLOCATION: Extraktion (Intent: INGEST)
|
# 7. SMART EDGE ALLOCATION: Extraktion (Intent: INGEST)
|
||||||
|
|
@ -221,9 +224,11 @@ edge_extraction:
|
||||||
Analysiere '{note_id}'. Extrahiere semantische Beziehungen.
|
Analysiere '{note_id}'. Extrahiere semantische Beziehungen.
|
||||||
ERLAUBTE TYPEN: {valid_types}
|
ERLAUBTE TYPEN: {valid_types}
|
||||||
TEXT: {text}
|
TEXT: {text}
|
||||||
OUTPUT: STRIKT JSON-Array von Objekten: [{{"to":"Ziel","kind":"typ"}}]. Kein Text davor/danach. Wenn nichts: [].
|
OUTPUT: STRIKT JSON-Array von Objekten: [[{{"to":"Ziel","kind":"typ"}}]]. Kein Text davor/danach. Wenn nichts: [].
|
||||||
openrouter: |
|
openrouter: |
|
||||||
Wissensgraph-Extraktion für '{note_id}'.
|
Wissensgraph-Extraktion für die Notiz '{note_id}'.
|
||||||
ERLAUBTE TYPEN: {valid_types}
|
ERLAUBTE TYPEN: {valid_types}
|
||||||
TEXT: {text}
|
TEXT: {text}
|
||||||
OUTPUT: STRIKT JSON-Array von Objekten: [{{"to":"X","kind":"Y"}}]. Kein Text davor/danach. Wenn nichts: []. Keine Wrapper-Objekte (z.B. kein Top-Level-Key 'edges').
|
ANWEISUNG: Finde Relationen zu anderen Konzepten.
|
||||||
|
OUTPUT: STRIKT JSON-Array von Objekten: [[{{"to":"X","kind":"Y"}}]].
|
||||||
|
Regeln: Kein Text davor/danach. Kein Wrapper-Objekt (kein 'edges' Key). Wenn leer: [].
|
||||||
Loading…
Reference in New Issue
Block a user