komplett neues WP20 deployment
This commit is contained in:
parent
f1bfa40b5b
commit
0157faab89
|
|
@ -2,8 +2,11 @@
|
||||||
FILE: app/config.py
|
FILE: app/config.py
|
||||||
DESCRIPTION: Zentrale Pydantic-Konfiguration. Enthält Parameter für Qdrant,
|
DESCRIPTION: Zentrale Pydantic-Konfiguration. Enthält Parameter für Qdrant,
|
||||||
Embeddings, Ollama, Google GenAI und OpenRouter.
|
Embeddings, Ollama, Google GenAI und OpenRouter.
|
||||||
VERSION: 0.6.0 (WP-20 Hybrid & OpenRouter Integration)
|
WP-20: Optimiert für Hybrid-Cloud Modus und Vektor-Synchronisation.
|
||||||
|
WP-22: Integration von Change-Detection und Vocab-Paths.
|
||||||
|
VERSION: 0.6.2
|
||||||
STATUS: Active
|
STATUS: Active
|
||||||
|
DEPENDENCIES: os, functools, pathlib
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import os
|
import os
|
||||||
|
|
@ -15,22 +18,27 @@ class Settings:
|
||||||
QDRANT_URL: str = os.getenv("QDRANT_URL", "http://127.0.0.1:6333")
|
QDRANT_URL: str = os.getenv("QDRANT_URL", "http://127.0.0.1:6333")
|
||||||
QDRANT_API_KEY: str | None = os.getenv("QDRANT_API_KEY")
|
QDRANT_API_KEY: str | None = os.getenv("QDRANT_API_KEY")
|
||||||
COLLECTION_PREFIX: str = os.getenv("MINDNET_PREFIX", "mindnet")
|
COLLECTION_PREFIX: str = os.getenv("MINDNET_PREFIX", "mindnet")
|
||||||
VECTOR_SIZE: int = int(os.getenv("MINDNET_VECTOR_SIZE", "384"))
|
# WP-22: Vektor-Dimension muss mit dem Embedding-Modell (nomic) übereinstimmen
|
||||||
|
VECTOR_SIZE: int = int(os.getenv("VECTOR_DIM", "768"))
|
||||||
DISTANCE: str = os.getenv("MINDNET_DISTANCE", "Cosine")
|
DISTANCE: str = os.getenv("MINDNET_DISTANCE", "Cosine")
|
||||||
|
|
||||||
# --- Lokale Embeddings ---
|
# --- Lokale Embeddings (Ollama & Sentence-Transformers) ---
|
||||||
|
# Modell für die Vektorisierung (z.B. nomic-embed-text)
|
||||||
|
EMBEDDING_MODEL: str = os.getenv("MINDNET_EMBEDDING_MODEL", "nomic-embed-text")
|
||||||
|
# Legacy Fallback Modellname
|
||||||
MODEL_NAME: str = os.getenv("MINDNET_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
|
MODEL_NAME: str = os.getenv("MINDNET_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
|
||||||
|
|
||||||
# --- WP-20 Hybrid LLM Provider ---
|
# --- WP-20 Hybrid LLM Provider ---
|
||||||
# Optionen: "ollama" | "gemini" | "openrouter"
|
# Erlaubt: "ollama" | "gemini" | "openrouter"
|
||||||
MINDNET_LLM_PROVIDER: str = os.getenv("MINDNET_LLM_PROVIDER", "ollama").lower()
|
MINDNET_LLM_PROVIDER: str = os.getenv("MINDNET_LLM_PROVIDER", "ollama").lower()
|
||||||
|
|
||||||
# Google AI Studio
|
# Google AI Studio (Direkt)
|
||||||
GOOGLE_API_KEY: str | None = os.getenv("GOOGLE_API_KEY")
|
GOOGLE_API_KEY: str | None = os.getenv("GOOGLE_API_KEY")
|
||||||
GEMINI_MODEL: str = os.getenv("MINDNET_GEMINI_MODEL", "gemini-1.5-flash")
|
GEMINI_MODEL: str = os.getenv("MINDNET_GEMINI_MODEL", "gemini-1.5-flash")
|
||||||
GEMMA_MODEL: str = os.getenv("MINDNET_GEMMA_MODEL", "gemma2-9b-it")
|
# Gemma-Modell für hohen Durchsatz bei der Ingestion
|
||||||
|
GEMMA_MODEL: str = os.getenv("MINDNET_GEMMA_MODEL", "google/gemma-2-9b-it:free")
|
||||||
|
|
||||||
# OpenRouter
|
# OpenRouter Integration
|
||||||
OPENROUTER_API_KEY: str | None = os.getenv("OPENROUTER_API_KEY")
|
OPENROUTER_API_KEY: str | None = os.getenv("OPENROUTER_API_KEY")
|
||||||
OPENROUTER_MODEL: str = os.getenv("OPENROUTER_MODEL", "google/gemma-2-9b-it:free")
|
OPENROUTER_MODEL: str = os.getenv("OPENROUTER_MODEL", "google/gemma-2-9b-it:free")
|
||||||
|
|
||||||
|
|
@ -42,16 +50,20 @@ class Settings:
|
||||||
PROMPTS_PATH: str = os.getenv("MINDNET_PROMPTS_PATH", "config/prompts.yaml")
|
PROMPTS_PATH: str = os.getenv("MINDNET_PROMPTS_PATH", "config/prompts.yaml")
|
||||||
|
|
||||||
# --- WP-06 / WP-14 Performance & Last-Steuerung ---
|
# --- WP-06 / WP-14 Performance & Last-Steuerung ---
|
||||||
LLM_TIMEOUT: float = float(os.getenv("MINDNET_LLM_TIMEOUT", "120.0"))
|
LLM_TIMEOUT: float = float(os.getenv("MINDNET_LLM_TIMEOUT", "300.0"))
|
||||||
DECISION_CONFIG_PATH: str = os.getenv("MINDNET_DECISION_CONFIG", "config/decision_engine.yaml")
|
DECISION_CONFIG_PATH: str = os.getenv("MINDNET_DECISION_CONFIG", "config/decision_engine.yaml")
|
||||||
BACKGROUND_LIMIT: int = int(os.getenv("MINDNET_LLM_BACKGROUND_LIMIT", "2"))
|
BACKGROUND_LIMIT: int = int(os.getenv("MINDNET_LLM_BACKGROUND_LIMIT", "2"))
|
||||||
|
|
||||||
# --- System-Pfade ---
|
# --- System-Pfade & Ingestion-Logik ---
|
||||||
DEBUG: bool = os.getenv("DEBUG", "false").lower() == "true"
|
DEBUG: bool = os.getenv("DEBUG", "false").lower() == "true"
|
||||||
MINDNET_VAULT_ROOT: str = os.getenv("MINDNET_VAULT_ROOT", "./vault")
|
MINDNET_VAULT_ROOT: str = os.getenv("MINDNET_VAULT_ROOT", "./vault")
|
||||||
MINDNET_TYPES_FILE: str = os.getenv("MINDNET_TYPES_FILE", "config/types.yaml")
|
MINDNET_TYPES_FILE: str = os.getenv("MINDNET_TYPES_FILE", "config/types.yaml")
|
||||||
|
# WP-22: Dictionary für die Edge-Validierung
|
||||||
|
MINDNET_VOCAB_PATH: str = os.getenv("MINDNET_VOCAB_PATH", "vault/_system/dictionary/edge_vocabulary.md")
|
||||||
|
# WP-22: Change Detection (Modus: "full" oder "body")
|
||||||
|
CHANGE_DETECTION_MODE: str = os.getenv("MINDNET_CHANGE_DETECTION_MODE", "full")
|
||||||
|
|
||||||
# --- WP-04 Retriever Gewichte ---
|
# --- WP-04 Retriever Gewichte (Semantik vs. Graph) ---
|
||||||
RETRIEVER_W_SEM: float = float(os.getenv("MINDNET_WP04_W_SEM", "0.70"))
|
RETRIEVER_W_SEM: float = float(os.getenv("MINDNET_WP04_W_SEM", "0.70"))
|
||||||
RETRIEVER_W_EDGE: float = float(os.getenv("MINDNET_WP04_W_EDGE", "0.25"))
|
RETRIEVER_W_EDGE: float = float(os.getenv("MINDNET_WP04_W_EDGE", "0.25"))
|
||||||
RETRIEVER_W_CENT: float = float(os.getenv("MINDNET_WP04_W_CENT", "0.05"))
|
RETRIEVER_W_CENT: float = float(os.getenv("MINDNET_WP04_W_CENT", "0.05"))
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen (Notes
|
||||||
WP-22: Integration von Content Lifecycle (Status Gate) und Edge Registry Validation.
|
WP-22: Integration von Content Lifecycle (Status Gate) und Edge Registry Validation.
|
||||||
WP-22: Kontextsensitive Kanten-Validierung mit Fundort-Reporting (Zeilennummern).
|
WP-22: Kontextsensitive Kanten-Validierung mit Fundort-Reporting (Zeilennummern).
|
||||||
WP-22: Multi-Hash Refresh für konsistente Change Detection.
|
WP-22: Multi-Hash Refresh für konsistente Change Detection.
|
||||||
VERSION: 2.11.3 (WP-20 Quota Protection & Stability Patch)
|
VERSION: 2.11.4
|
||||||
STATUS: Active
|
STATUS: Active
|
||||||
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.services.llm_service, app.services.edge_registry
|
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.services.llm_service, app.services.edge_registry
|
||||||
EXTERNAL_CONFIG: config/types.yaml, config/prompts.yaml
|
EXTERNAL_CONFIG: config/types.yaml, config/prompts.yaml
|
||||||
|
|
@ -22,7 +22,7 @@ from app.core.parser import (
|
||||||
read_markdown,
|
read_markdown,
|
||||||
normalize_frontmatter,
|
normalize_frontmatter,
|
||||||
validate_required_frontmatter,
|
validate_required_frontmatter,
|
||||||
extract_edges_with_context, # WP-22: Neue Funktion für Zeilennummern
|
extract_edges_with_context,
|
||||||
)
|
)
|
||||||
from app.core.note_payload import make_note_payload
|
from app.core.note_payload import make_note_payload
|
||||||
from app.core.chunker import assemble_chunks, get_chunk_config
|
from app.core.chunker import assemble_chunks, get_chunk_config
|
||||||
|
|
@ -44,7 +44,7 @@ from app.core.qdrant_points import (
|
||||||
|
|
||||||
from app.services.embeddings_client import EmbeddingsClient
|
from app.services.embeddings_client import EmbeddingsClient
|
||||||
from app.services.edge_registry import registry as edge_registry
|
from app.services.edge_registry import registry as edge_registry
|
||||||
from app.services.llm_service import LLMService # WP-20 Integration
|
from app.services.llm_service import LLMService
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -52,7 +52,9 @@ logger = logging.getLogger(__name__)
|
||||||
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
||||||
"""Lädt die types.yaml zur Steuerung der typ-spezifischen Ingestion."""
|
"""Lädt die types.yaml zur Steuerung der typ-spezifischen Ingestion."""
|
||||||
import yaml
|
import yaml
|
||||||
path = custom_path or os.getenv("MINDNET_TYPES_FILE", "config/types.yaml")
|
from app.config import get_settings
|
||||||
|
settings = get_settings()
|
||||||
|
path = custom_path or settings.MINDNET_TYPES_FILE
|
||||||
if not os.path.exists(path): return {}
|
if not os.path.exists(path): return {}
|
||||||
try:
|
try:
|
||||||
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
||||||
|
|
@ -65,35 +67,23 @@ def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
||||||
return "concept"
|
return "concept"
|
||||||
|
|
||||||
def effective_chunk_profile_name(fm: dict, note_type: str, reg: dict) -> str:
|
def effective_chunk_profile_name(fm: dict, note_type: str, reg: dict) -> str:
|
||||||
"""
|
"""Ermittelt den Namen des zu nutzenden Chunk-Profils."""
|
||||||
Ermittelt den Namen des zu nutzenden Chunk-Profils.
|
|
||||||
Priorität: 1. Frontmatter Override -> 2. Type Config -> 3. Global Default
|
|
||||||
"""
|
|
||||||
override = fm.get("chunking_profile") or fm.get("chunk_profile")
|
override = fm.get("chunking_profile") or fm.get("chunk_profile")
|
||||||
if override and isinstance(override, str):
|
if override and isinstance(override, str): return override
|
||||||
return override
|
|
||||||
|
|
||||||
t_cfg = reg.get("types", {}).get(note_type, {})
|
t_cfg = reg.get("types", {}).get(note_type, {})
|
||||||
if t_cfg:
|
if t_cfg:
|
||||||
cp = t_cfg.get("chunking_profile") or t_cfg.get("chunk_profile")
|
cp = t_cfg.get("chunking_profile") or t_cfg.get("chunk_profile")
|
||||||
if cp: return cp
|
if cp: return cp
|
||||||
|
|
||||||
return reg.get("defaults", {}).get("chunking_profile", "sliding_standard")
|
return reg.get("defaults", {}).get("chunking_profile", "sliding_standard")
|
||||||
|
|
||||||
def effective_retriever_weight(fm: dict, note_type: str, reg: dict) -> float:
|
def effective_retriever_weight(fm: dict, note_type: str, reg: dict) -> float:
|
||||||
"""
|
"""Ermittelt das effektive retriever_weight für das Scoring."""
|
||||||
Ermittelt das effektive retriever_weight für das Scoring.
|
|
||||||
Priorität: 1. Frontmatter Override -> 2. Type Config -> 3. Global Default
|
|
||||||
"""
|
|
||||||
override = fm.get("retriever_weight")
|
override = fm.get("retriever_weight")
|
||||||
if override is not None:
|
if override is not None:
|
||||||
try: return float(override)
|
try: return float(override)
|
||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
t_cfg = reg.get("types", {}).get(note_type, {})
|
t_cfg = reg.get("types", {}).get(note_type, {})
|
||||||
if t_cfg and "retriever_weight" in t_cfg:
|
if t_cfg and "retriever_weight" in t_cfg: return float(t_cfg["retriever_weight"])
|
||||||
return float(t_cfg["retriever_weight"])
|
|
||||||
|
|
||||||
return float(reg.get("defaults", {}).get("retriever_weight", 1.0))
|
return float(reg.get("defaults", {}).get("retriever_weight", 1.0))
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -106,13 +96,13 @@ class IngestionService:
|
||||||
self.cfg = QdrantConfig.from_env()
|
self.cfg = QdrantConfig.from_env()
|
||||||
self.cfg.prefix = self.prefix
|
self.cfg.prefix = self.prefix
|
||||||
self.client = get_client(self.cfg)
|
self.client = get_client(self.cfg)
|
||||||
self.dim = self.cfg.dim if hasattr(self.cfg, 'dim') else self.settings.VECTOR_SIZE
|
self.dim = self.settings.VECTOR_SIZE # Synchronisiert mit Settings v0.6.2
|
||||||
self.registry = load_type_registry()
|
self.registry = load_type_registry()
|
||||||
self.embedder = EmbeddingsClient()
|
self.embedder = EmbeddingsClient()
|
||||||
self.llm = LLMService() # WP-20
|
self.llm = LLMService()
|
||||||
|
|
||||||
# Change Detection Modus (full oder body)
|
# WP-22: Change Detection Modus aus Settings
|
||||||
self.active_hash_mode = os.getenv("MINDNET_CHANGE_DETECTION_MODE", "full")
|
self.active_hash_mode = self.settings.CHANGE_DETECTION_MODE
|
||||||
|
|
||||||
try:
|
try:
|
||||||
ensure_collections(self.client, self.prefix, self.dim)
|
ensure_collections(self.client, self.prefix, self.dim)
|
||||||
|
|
@ -133,20 +123,20 @@ class IngestionService:
|
||||||
async def _perform_smart_edge_allocation(self, text: str, note_id: str) -> List[Dict]:
|
async def _perform_smart_edge_allocation(self, text: str, note_id: str) -> List[Dict]:
|
||||||
"""
|
"""
|
||||||
WP-20: Nutzt den Hybrid LLM Service für die semantische Kanten-Extraktion.
|
WP-20: Nutzt den Hybrid LLM Service für die semantische Kanten-Extraktion.
|
||||||
QUOTEN-SCHUTZ: Bevorzugt OpenRouter (Gemma), um Gemini-Tageslimits zu schützen.
|
QUOTEN-SCHUTZ: Bevorzugt OpenRouter (Gemma 2), um Gemini-Tageslimits zu schonen.
|
||||||
"""
|
"""
|
||||||
# Bestimme Provider: Nutze OpenRouter falls Key vorhanden, sonst Global Default
|
# Bestimme Provider: Nutze OpenRouter falls Key vorhanden
|
||||||
provider = "openrouter" if getattr(self.settings, "OPENROUTER_API_KEY", None) else self.settings.MINDNET_LLM_PROVIDER
|
provider = "openrouter" if self.settings.OPENROUTER_API_KEY else self.settings.MINDNET_LLM_PROVIDER
|
||||||
|
model = self.settings.GEMMA_MODEL # Hochdurchsatz-Modell aus config.py
|
||||||
|
|
||||||
# Nutze Gemma-Modell für hohen Durchsatz via OpenRouter/Google
|
logger.info(f"🚀 [Ingestion] Turbo-Mode: Extracting edges for '{note_id}' using {model} on {provider}")
|
||||||
model = getattr(self.settings, "GEMMA_MODEL", None)
|
|
||||||
|
|
||||||
# Hole das optimierte Prompt-Template (Key: edge_extraction)
|
# Hole das optimierte Prompt-Template (Kaskade: Provider -> gemini -> ollama)
|
||||||
template = self.llm.get_prompt("edge_extraction", provider)
|
template = self.llm.get_prompt("edge_extraction", provider)
|
||||||
prompt = template.format(text=text[:6000], note_id=note_id)
|
prompt = template.format(text=text[:6000], note_id=note_id)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Hintergrund-Task mit Semaphore via LLMService
|
# Hintergrund-Task mit Semaphore via LLMService (WP-06)
|
||||||
response_json = await self.llm.generate_raw_response(
|
response_json = await self.llm.generate_raw_response(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
priority="background",
|
priority="background",
|
||||||
|
|
@ -156,13 +146,12 @@ class IngestionService:
|
||||||
)
|
)
|
||||||
data = json.loads(response_json)
|
data = json.loads(response_json)
|
||||||
|
|
||||||
# Metadaten für die Edge-Herkunft setzen
|
|
||||||
for item in data:
|
for item in data:
|
||||||
item["provenance"] = "semantic_ai"
|
item["provenance"] = "semantic_ai"
|
||||||
item["line"] = f"ai-{provider}"
|
item["line"] = f"ai-{provider}"
|
||||||
return data
|
return data
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Smart Edge Allocation failed for {note_id}: {e}")
|
logger.warning(f"⚠️ [Ingestion] Smart Edge Allocation failed for {note_id} on {provider}: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
async def process_file(
|
async def process_file(
|
||||||
|
|
@ -214,7 +203,7 @@ class IngestionService:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {**result, "error": f"Payload build failed: {str(e)}"}
|
return {**result, "error": f"Payload build failed: {str(e)}"}
|
||||||
|
|
||||||
# 4. Change Detection (Multi-Hash)
|
# 4. Change Detection (WP-22 Multi-Hash)
|
||||||
old_payload = None
|
old_payload = None
|
||||||
if not force_replace:
|
if not force_replace:
|
||||||
old_payload = self._fetch_note_payload(note_id)
|
old_payload = self._fetch_note_payload(note_id)
|
||||||
|
|
@ -241,7 +230,7 @@ class IngestionService:
|
||||||
try:
|
try:
|
||||||
body_text = getattr(parsed, "body", "") or ""
|
body_text = getattr(parsed, "body", "") or ""
|
||||||
|
|
||||||
# STABILITY PATCH: Prüfen, ob ensure_latest existiert (verhindert AttributeError)
|
# WP-22 STABILITY PATCH: Prüfen, ob ensure_latest existiert
|
||||||
if hasattr(edge_registry, "ensure_latest"):
|
if hasattr(edge_registry, "ensure_latest"):
|
||||||
edge_registry.ensure_latest()
|
edge_registry.ensure_latest()
|
||||||
|
|
||||||
|
|
@ -264,13 +253,13 @@ class IngestionService:
|
||||||
e["kind"] = edge_registry.resolve(edge_type=e["kind"], provenance="explicit", context={**context, "line": e.get("line")})
|
e["kind"] = edge_registry.resolve(edge_type=e["kind"], provenance="explicit", context={**context, "line": e.get("line")})
|
||||||
edges.append(e)
|
edges.append(e)
|
||||||
|
|
||||||
# B. WP-20: Smart AI Edges (Hybrid Acceleration)
|
# B. WP-20: Smart AI Edges (Hybrid Turbo Acceleration)
|
||||||
ai_edges = await self._perform_smart_edge_allocation(body_text, note_id)
|
ai_edges = await self._perform_smart_edge_allocation(body_text, note_id)
|
||||||
for e in ai_edges:
|
for e in ai_edges:
|
||||||
e["kind"] = edge_registry.resolve(edge_type=e.get("kind"), provenance="semantic_ai", context={**context, "line": e.get("line")})
|
e["kind"] = edge_registry.resolve(edge_type=e.get("kind"), provenance="semantic_ai", context={**context, "line": e.get("line")})
|
||||||
edges.append(e)
|
edges.append(e)
|
||||||
|
|
||||||
# C. System-Kanten
|
# C. System-Kanten (Struktur)
|
||||||
try:
|
try:
|
||||||
raw_system_edges = build_edges_for_note(note_id, chunk_pls, note_level_references=note_pl.get("references", []), include_note_scope_refs=note_scope_refs)
|
raw_system_edges = build_edges_for_note(note_id, chunk_pls, note_level_references=note_pl.get("references", []), include_note_scope_refs=note_scope_refs)
|
||||||
except TypeError:
|
except TypeError:
|
||||||
|
|
@ -283,7 +272,7 @@ class IngestionService:
|
||||||
edges.append(e)
|
edges.append(e)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Processing failed: {e}", exc_info=True)
|
logger.error(f"Processing failed for {file_path}: {e}", exc_info=True)
|
||||||
return {**result, "error": f"Processing failed: {str(e)}"}
|
return {**result, "error": f"Processing failed: {str(e)}"}
|
||||||
|
|
||||||
# 6. Upsert
|
# 6. Upsert
|
||||||
|
|
@ -303,7 +292,7 @@ class IngestionService:
|
||||||
|
|
||||||
return {"path": file_path, "status": "success", "changed": True, "note_id": note_id, "chunks_count": len(chunk_pls), "edges_count": len(edges)}
|
return {"path": file_path, "status": "success", "changed": True, "note_id": note_id, "chunks_count": len(chunk_pls), "edges_count": len(edges)}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Upsert failed: {e}", exc_info=True)
|
logger.error(f"Upsert failed for {note_id}: {e}", exc_info=True)
|
||||||
return {**result, "error": f"DB Upsert failed: {e}"}
|
return {**result, "error": f"DB Upsert failed: {e}"}
|
||||||
|
|
||||||
def _fetch_note_payload(self, note_id: str) -> Optional[dict]:
|
def _fetch_note_payload(self, note_id: str) -> Optional[dict]:
|
||||||
|
|
@ -330,4 +319,20 @@ class IngestionService:
|
||||||
selector = rest.FilterSelector(filter=f)
|
selector = rest.FilterSelector(filter=f)
|
||||||
for suffix in ["chunks", "edges"]:
|
for suffix in ["chunks", "edges"]:
|
||||||
try: self.client.delete(collection_name=f"{self.prefix}_{suffix}", points_selector=selector)
|
try: self.client.delete(collection_name=f"{self.prefix}_{suffix}", points_selector=selector)
|
||||||
except Exception: pass
|
except Exception: pass
|
||||||
|
|
||||||
|
async def create_from_text(self, markdown_content: str, filename: str, vault_root: str, folder: str = "00_Inbox") -> Dict[str, Any]:
|
||||||
|
"""Hilfsmethode zur Erstellung einer Note aus einem Textstream."""
|
||||||
|
target_dir = os.path.join(vault_root, folder)
|
||||||
|
os.makedirs(target_dir, exist_ok=True)
|
||||||
|
file_path = os.path.join(target_dir, filename)
|
||||||
|
try:
|
||||||
|
with open(file_path, "w", encoding="utf-8") as f:
|
||||||
|
f.write(markdown_content)
|
||||||
|
f.flush()
|
||||||
|
os.fsync(f.fileno())
|
||||||
|
await asyncio.sleep(0.1)
|
||||||
|
logger.info(f"Written file to {file_path}")
|
||||||
|
except Exception as e:
|
||||||
|
return {"status": "error", "error": f"Disk write failed: {str(e)}"}
|
||||||
|
return await self.process_file(file_path=file_path, vault_root=vault_root, apply=True, force_replace=True, purge_before=True)
|
||||||
|
|
@ -2,7 +2,10 @@
|
||||||
FILE: app/services/edge_registry.py
|
FILE: app/services/edge_registry.py
|
||||||
DESCRIPTION: Single Source of Truth für Kanten-Typen mit dynamischem Reload.
|
DESCRIPTION: Single Source of Truth für Kanten-Typen mit dynamischem Reload.
|
||||||
WP-22: Fix für absolute Pfade außerhalb des Vaults (Prod-Dictionary).
|
WP-22: Fix für absolute Pfade außerhalb des Vaults (Prod-Dictionary).
|
||||||
VERSION: 0.7.4 (Fix: Absolute Path Escaping & Quote Stripping)
|
WP-20: Synchronisation mit zentralen Settings (v0.6.2).
|
||||||
|
VERSION: 0.7.5
|
||||||
|
STATUS: Active
|
||||||
|
DEPENDENCIES: re, os, json, logging, time, app.config
|
||||||
"""
|
"""
|
||||||
import re
|
import re
|
||||||
import os
|
import os
|
||||||
|
|
@ -25,69 +28,49 @@ class EdgeRegistry:
|
||||||
cls._instance.initialized = False
|
cls._instance.initialized = False
|
||||||
return cls._instance
|
return cls._instance
|
||||||
|
|
||||||
def __init__(self, vault_root: Optional[str] = None):
|
def __init__(self):
|
||||||
if self.initialized:
|
if self.initialized:
|
||||||
return
|
return
|
||||||
|
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
|
|
||||||
# 1. ENV-Werte laden und von Anführungszeichen bereinigen
|
# 1. Pfad aus den zentralen Settings laden (WP-20 Synchronisation)
|
||||||
env_vocab_path = os.getenv("MINDNET_VOCAB_PATH")
|
# Priorisiert den Pfad aus der .env / config.py (v0.6.2)
|
||||||
if env_vocab_path:
|
self.full_vocab_path = os.path.abspath(settings.MINDNET_VOCAB_PATH)
|
||||||
env_vocab_path = env_vocab_path.strip('"').strip("'")
|
|
||||||
|
|
||||||
env_vault_root = os.getenv("MINDNET_VAULT_ROOT") or getattr(settings, "MINDNET_VAULT_ROOT", "./vault")
|
|
||||||
if env_vault_root:
|
|
||||||
env_vault_root = env_vault_root.strip('"').strip("'")
|
|
||||||
|
|
||||||
# 2. Pfad-Priorität: Wenn absolut (/), dann direkt nutzen!
|
|
||||||
if env_vocab_path:
|
|
||||||
if env_vocab_path.startswith("/"):
|
|
||||||
# Absoluter Pfad (z.B. aus Produktion)
|
|
||||||
self.full_vocab_path = env_vocab_path
|
|
||||||
else:
|
|
||||||
# Relativer Pfad zum aktuellen Verzeichnis oder Vault
|
|
||||||
self.full_vocab_path = os.path.abspath(env_vocab_path)
|
|
||||||
else:
|
|
||||||
# Fallback: Suche im Vault-Verzeichnis
|
|
||||||
possible_paths = [
|
|
||||||
os.path.join(env_vault_root, "_system", "dictionary", "edge_vocabulary.md"),
|
|
||||||
os.path.join(env_vault_root, "01_User_Manual", "01_edge_vocabulary.md")
|
|
||||||
]
|
|
||||||
self.full_vocab_path = None
|
|
||||||
for p in possible_paths:
|
|
||||||
if os.path.exists(p):
|
|
||||||
self.full_vocab_path = os.path.abspath(p)
|
|
||||||
break
|
|
||||||
|
|
||||||
if not self.full_vocab_path:
|
|
||||||
self.full_vocab_path = os.path.abspath(possible_paths[0])
|
|
||||||
|
|
||||||
self.unknown_log_path = "data/logs/unknown_edges.jsonl"
|
self.unknown_log_path = "data/logs/unknown_edges.jsonl"
|
||||||
self.canonical_map: Dict[str, str] = {}
|
self.canonical_map: Dict[str, str] = {}
|
||||||
self.valid_types: Set[str] = set()
|
self.valid_types: Set[str] = set()
|
||||||
self._last_mtime = 0.0
|
self._last_mtime = 0.0
|
||||||
|
|
||||||
print(f"\n>>> [EDGE-REGISTRY] Initializing with Path: {self.full_vocab_path}", flush=True)
|
# Initialer Ladevorgang
|
||||||
|
logger.info(f">>> [EDGE-REGISTRY] Initializing with Path: {self.full_vocab_path}")
|
||||||
self.ensure_latest()
|
self.ensure_latest()
|
||||||
self.initialized = True
|
self.initialized = True
|
||||||
|
|
||||||
def ensure_latest(self):
|
def ensure_latest(self):
|
||||||
"""Prüft den Zeitstempel und lädt bei Bedarf neu."""
|
"""
|
||||||
|
Prüft den Zeitstempel der Vokabular-Datei und lädt bei Bedarf neu.
|
||||||
|
Verhindert den AttributeError in der Ingestion-Pipeline.
|
||||||
|
"""
|
||||||
if not os.path.exists(self.full_vocab_path):
|
if not os.path.exists(self.full_vocab_path):
|
||||||
print(f"!!! [EDGE-REGISTRY ERROR] File not found: {self.full_vocab_path} !!!", flush=True)
|
logger.error(f"!!! [EDGE-REGISTRY ERROR] File not found: {self.full_vocab_path} !!!")
|
||||||
return
|
return
|
||||||
|
|
||||||
current_mtime = os.path.getmtime(self.full_vocab_path)
|
try:
|
||||||
if current_mtime > self._last_mtime:
|
current_mtime = os.path.getmtime(self.full_vocab_path)
|
||||||
self._load_vocabulary()
|
if current_mtime > self._last_mtime:
|
||||||
self._last_mtime = current_mtime
|
self._load_vocabulary()
|
||||||
|
self._last_mtime = current_mtime
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"!!! [EDGE-REGISTRY] Error checking file time: {e}")
|
||||||
|
|
||||||
def _load_vocabulary(self):
|
def _load_vocabulary(self):
|
||||||
"""Parst das Wörterbuch."""
|
"""Parst das Markdown-Wörterbuch und baut die Canonical-Map auf."""
|
||||||
self.canonical_map.clear()
|
self.canonical_map.clear()
|
||||||
self.valid_types.clear()
|
self.valid_types.clear()
|
||||||
|
|
||||||
|
# Regex für Tabellen-Struktur: | **Typ** | Aliase |
|
||||||
pattern = re.compile(r"\|\s*\*\*`?([a-zA-Z0-9_-]+)`?\*\*\s*\|\s*([^|]+)\|")
|
pattern = re.compile(r"\|\s*\*\*`?([a-zA-Z0-9_-]+)`?\*\*\s*\|\s*([^|]+)\|")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
@ -106,38 +89,48 @@ class EdgeRegistry:
|
||||||
if aliases_str and "Kein Alias" not in aliases_str:
|
if aliases_str and "Kein Alias" not in aliases_str:
|
||||||
aliases = [a.strip() for a in aliases_str.split(",") if a.strip()]
|
aliases = [a.strip() for a in aliases_str.split(",") if a.strip()]
|
||||||
for alias in aliases:
|
for alias in aliases:
|
||||||
|
# Normalisierung: Kleinschreibung, Underscores statt Leerzeichen
|
||||||
clean_alias = alias.replace("`", "").lower().strip().replace(" ", "_")
|
clean_alias = alias.replace("`", "").lower().strip().replace(" ", "_")
|
||||||
self.canonical_map[clean_alias] = canonical
|
self.canonical_map[clean_alias] = canonical
|
||||||
c_aliases += 1
|
c_aliases += 1
|
||||||
|
|
||||||
print(f"=== [EDGE-REGISTRY SUCCESS] Loaded {c_types} Canonical Types and {c_aliases} Aliases ===", flush=True)
|
logger.info(f"=== [EDGE-REGISTRY SUCCESS] Loaded {c_types} Canonical Types and {c_aliases} Aliases ===")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"!!! [EDGE-REGISTRY FATAL] Error reading file: {e} !!!", flush=True)
|
logger.error(f"!!! [EDGE-REGISTRY FATAL] Error reading file: {e} !!!")
|
||||||
|
|
||||||
def resolve(self, edge_type: str, provenance: str = "explicit", context: dict = None) -> str:
|
def resolve(self, edge_type: str, provenance: str = "explicit", context: dict = None) -> str:
|
||||||
"""Validierung mit Fundort-Logging."""
|
"""
|
||||||
|
Validiert einen Kanten-Typ gegen das Vokabular.
|
||||||
|
Loggt unbekannte Typen für die spätere manuelle Pflege.
|
||||||
|
"""
|
||||||
self.ensure_latest()
|
self.ensure_latest()
|
||||||
if not edge_type: return "related_to"
|
if not edge_type:
|
||||||
|
return "related_to"
|
||||||
|
|
||||||
|
# Normalisierung des Typs
|
||||||
clean_type = edge_type.lower().strip().replace(" ", "_").replace("-", "_")
|
clean_type = edge_type.lower().strip().replace(" ", "_").replace("-", "_")
|
||||||
ctx = context or {}
|
ctx = context or {}
|
||||||
|
|
||||||
|
# System-Kanten dürfen nicht manuell vergeben werden
|
||||||
if provenance == "explicit" and clean_type in self.FORBIDDEN_SYSTEM_EDGES:
|
if provenance == "explicit" and clean_type in self.FORBIDDEN_SYSTEM_EDGES:
|
||||||
self._log_issue(clean_type, "forbidden_system_usage", ctx)
|
self._log_issue(clean_type, "forbidden_system_usage", ctx)
|
||||||
return "related_to"
|
return "related_to"
|
||||||
|
|
||||||
|
# System-Kanten sind nur bei struktureller Provenienz erlaubt
|
||||||
if provenance == "structure" and clean_type in self.FORBIDDEN_SYSTEM_EDGES:
|
if provenance == "structure" and clean_type in self.FORBIDDEN_SYSTEM_EDGES:
|
||||||
return clean_type
|
return clean_type
|
||||||
|
|
||||||
|
# Mapping auf kanonischen Namen
|
||||||
if clean_type in self.canonical_map:
|
if clean_type in self.canonical_map:
|
||||||
return self.canonical_map[clean_type]
|
return self.canonical_map[clean_type]
|
||||||
|
|
||||||
|
# Fallback und Logging
|
||||||
self._log_issue(clean_type, "unknown_type", ctx)
|
self._log_issue(clean_type, "unknown_type", ctx)
|
||||||
return clean_type
|
return clean_type
|
||||||
|
|
||||||
def _log_issue(self, edge_type: str, error_kind: str, ctx: dict):
|
def _log_issue(self, edge_type: str, error_kind: str, ctx: dict):
|
||||||
"""Detailliertes JSONL-Logging für Debugging."""
|
"""Detailliertes JSONL-Logging für die Vokabular-Optimierung."""
|
||||||
try:
|
try:
|
||||||
os.makedirs(os.path.dirname(self.unknown_log_path), exist_ok=True)
|
os.makedirs(os.path.dirname(self.unknown_log_path), exist_ok=True)
|
||||||
entry = {
|
entry = {
|
||||||
|
|
@ -150,6 +143,8 @@ class EdgeRegistry:
|
||||||
}
|
}
|
||||||
with open(self.unknown_log_path, "a", encoding="utf-8") as f:
|
with open(self.unknown_log_path, "a", encoding="utf-8") as f:
|
||||||
f.write(json.dumps(entry) + "\n")
|
f.write(json.dumps(entry) + "\n")
|
||||||
except Exception: pass
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Singleton Export
|
||||||
registry = EdgeRegistry()
|
registry = EdgeRegistry()
|
||||||
|
|
@ -3,7 +3,8 @@ FILE: app/services/llm_service.py
|
||||||
DESCRIPTION: Hybrid-Client für Ollama, Google GenAI (Gemini) und OpenRouter.
|
DESCRIPTION: Hybrid-Client für Ollama, Google GenAI (Gemini) und OpenRouter.
|
||||||
Verwaltet provider-spezifische Prompts und Background-Last.
|
Verwaltet provider-spezifische Prompts und Background-Last.
|
||||||
WP-20: Optimiertes Fallback-Management zum Schutz von Cloud-Quoten.
|
WP-20: Optimiertes Fallback-Management zum Schutz von Cloud-Quoten.
|
||||||
VERSION: 3.3.1
|
WP-20 Fix: Bulletproof Prompt-Auflösung für format() Aufrufe.
|
||||||
|
VERSION: 3.3.2
|
||||||
STATUS: Active
|
STATUS: Active
|
||||||
"""
|
"""
|
||||||
import httpx
|
import httpx
|
||||||
|
|
@ -58,11 +59,14 @@ class LLMService:
|
||||||
def _load_prompts(self) -> dict:
|
def _load_prompts(self) -> dict:
|
||||||
"""Lädt die Prompt-Konfiguration aus der YAML-Datei."""
|
"""Lädt die Prompt-Konfiguration aus der YAML-Datei."""
|
||||||
path = Path(self.settings.PROMPTS_PATH)
|
path = Path(self.settings.PROMPTS_PATH)
|
||||||
if not path.exists(): return {}
|
if not path.exists():
|
||||||
|
logger.error(f"❌ Prompts file not found at {path}")
|
||||||
|
return {}
|
||||||
try:
|
try:
|
||||||
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
return yaml.safe_load(f) or {}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Failed to load prompts: {e}")
|
logger.error(f"❌ Failed to load prompts: {e}")
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
def get_prompt(self, key: str, provider: str = None) -> str:
|
def get_prompt(self, key: str, provider: str = None) -> str:
|
||||||
|
|
@ -70,12 +74,22 @@ class LLMService:
|
||||||
Hole provider-spezifisches Template mit intelligenter Text-Kaskade.
|
Hole provider-spezifisches Template mit intelligenter Text-Kaskade.
|
||||||
HINWEIS: Dies ist nur ein Text-Lookup und verbraucht kein API-Kontingent.
|
HINWEIS: Dies ist nur ein Text-Lookup und verbraucht kein API-Kontingent.
|
||||||
Kaskade: Gewählter Provider -> Gemini (Cloud-Stil) -> Ollama (Basis-Stil).
|
Kaskade: Gewählter Provider -> Gemini (Cloud-Stil) -> Ollama (Basis-Stil).
|
||||||
|
|
||||||
|
WP-20 Fix: Garantiert die Rückgabe eines Strings, um AttributeError zu vermeiden.
|
||||||
"""
|
"""
|
||||||
active_provider = provider or self.settings.MINDNET_LLM_PROVIDER
|
active_provider = provider or self.settings.MINDNET_LLM_PROVIDER
|
||||||
data = self.prompts.get(key, "")
|
data = self.prompts.get(key, "")
|
||||||
|
|
||||||
if isinstance(data, dict):
|
if isinstance(data, dict):
|
||||||
# Wir versuchen erst den Provider, dann Gemini (weil ähnlich leistungsfähig), dann Ollama
|
# Wir versuchen erst den Provider, dann Gemini (weil ähnlich leistungsfähig), dann Ollama
|
||||||
return data.get(active_provider, data.get("gemini", data.get("ollama", "")))
|
val = data.get(active_provider, data.get("gemini", data.get("ollama", "")))
|
||||||
|
|
||||||
|
# Falls val durch YAML-Fehler immer noch ein Dict ist, extrahiere ersten String
|
||||||
|
if isinstance(val, dict):
|
||||||
|
logger.warning(f"⚠️ [LLMService] Nested dictionary detected for key '{key}'. Using first entry.")
|
||||||
|
val = next(iter(val.values()), "") if val else ""
|
||||||
|
return str(val)
|
||||||
|
|
||||||
return str(data)
|
return str(data)
|
||||||
|
|
||||||
async def generate_raw_response(
|
async def generate_raw_response(
|
||||||
|
|
@ -116,6 +130,7 @@ class LLMService:
|
||||||
|
|
||||||
async def _execute_google(self, prompt, system, force_json, model_override):
|
async def _execute_google(self, prompt, system, force_json, model_override):
|
||||||
"""Native Google SDK Integration (Gemini)."""
|
"""Native Google SDK Integration (Gemini)."""
|
||||||
|
# Nutzt GEMINI_MODEL aus config.py falls kein override (z.B. für Ingestion) übergeben wurde
|
||||||
model = model_override or self.settings.GEMINI_MODEL
|
model = model_override or self.settings.GEMINI_MODEL
|
||||||
config = types.GenerateContentConfig(
|
config = types.GenerateContentConfig(
|
||||||
system_instruction=system,
|
system_instruction=system,
|
||||||
|
|
@ -130,7 +145,8 @@ class LLMService:
|
||||||
|
|
||||||
async def _execute_openrouter(self, prompt, system, force_json, model_override):
|
async def _execute_openrouter(self, prompt, system, force_json, model_override):
|
||||||
"""OpenRouter API Integration (OpenAI-kompatibel)."""
|
"""OpenRouter API Integration (OpenAI-kompatibel)."""
|
||||||
model = model_override or getattr(self.settings, "OPENROUTER_MODEL", "google/gemma-2-9b-it:free")
|
# Nutzt OPENROUTER_MODEL aus config.py (v0.6.2)
|
||||||
|
model = model_override or self.settings.OPENROUTER_MODEL
|
||||||
messages = []
|
messages = []
|
||||||
if system:
|
if system:
|
||||||
messages.append({"role": "system", "content": system})
|
messages.append({"role": "system", "content": system})
|
||||||
|
|
@ -166,7 +182,7 @@ class LLMService:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
attempt += 1
|
attempt += 1
|
||||||
if attempt > max_retries:
|
if attempt > max_retries:
|
||||||
logger.error(f"Ollama Error after {attempt} retries: {e}")
|
logger.error(f"❌ Ollama Error after {attempt} retries: {e}")
|
||||||
raise e
|
raise e
|
||||||
wait_time = base_delay * (2 ** (attempt - 1))
|
wait_time = base_delay * (2 ** (attempt - 1))
|
||||||
logger.warning(f"⚠️ Ollama attempt {attempt} failed. Retrying in {wait_time}s...")
|
logger.warning(f"⚠️ Ollama attempt {attempt} failed. Retrying in {wait_time}s...")
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
"""
|
"""
|
||||||
FILE: app/services/semantic_analyzer.py
|
FILE: app/services/semantic_analyzer.py
|
||||||
DESCRIPTION: KI-gestützte Kanten-Validierung. Nutzt LLM (Background-Priority), um Kanten präzise einem Chunk zuzuordnen.
|
DESCRIPTION: KI-gestützte Kanten-Validierung. Nutzt LLM (Background-Priority), um Kanten präzise einem Chunk zuzuordnen.
|
||||||
WP-20 Fix: Kompatibilität mit Provider-basierten Prompt-Dictionaries (Hybrid-Modus).
|
WP-20 Fix: Volle Kompatibilität mit der gehärteten LLMService (v3.3.2) Kaskade.
|
||||||
VERSION: 2.2.0
|
WP-20: Unterstützung für Provider-spezifische Routing-Logik beim Import.
|
||||||
|
VERSION: 2.2.1
|
||||||
STATUS: Active
|
STATUS: Active
|
||||||
DEPENDENCIES: app.services.llm_service, json, logging
|
DEPENDENCIES: app.services.llm_service, json, logging
|
||||||
LAST_ANALYSIS: 2025-12-23
|
LAST_ANALYSIS: 2025-12-23
|
||||||
|
|
@ -53,16 +54,17 @@ class SemanticAnalyzer:
|
||||||
Sendet einen Chunk und eine Liste potenzieller Kanten an das LLM.
|
Sendet einen Chunk und eine Liste potenzieller Kanten an das LLM.
|
||||||
Das LLM filtert heraus, welche Kanten für diesen Chunk relevant sind.
|
Das LLM filtert heraus, welche Kanten für diesen Chunk relevant sind.
|
||||||
|
|
||||||
WP-20 Fix: Nutzt get_prompt(), um den 'AttributeError: dict object' zu vermeiden.
|
WP-20 Fix: Nutzt get_prompt(), um den 'AttributeError: dict object' sicher zu vermeiden.
|
||||||
"""
|
"""
|
||||||
if not all_edges:
|
if not all_edges:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# 1. Prompt laden via get_prompt (handelt die Provider-Kaskade automatisch ab) [WP-20 Fix]
|
# 1. Prompt laden via get_prompt (handelt die Provider-Kaskade automatisch ab)
|
||||||
prompt_template = self.llm.get_prompt("edge_allocation_template")
|
prompt_template = self.llm.get_prompt("edge_allocation_template")
|
||||||
|
|
||||||
|
# Sicherheits-Check für die Format-Methode
|
||||||
if not prompt_template or isinstance(prompt_template, dict):
|
if not prompt_template or isinstance(prompt_template, dict):
|
||||||
logger.warning("⚠️ [SemanticAnalyzer] Prompt 'edge_allocation_template' konnte nicht als String geladen werden. Nutze Hard-Fallback.")
|
logger.warning("⚠️ [SemanticAnalyzer] Prompt 'edge_allocation_template' konnte nicht als String geladen werden. Nutze Not-Fallback.")
|
||||||
prompt_template = (
|
prompt_template = (
|
||||||
"TASK: Wähle aus den Kandidaten die relevanten Kanten für den Text.\n"
|
"TASK: Wähle aus den Kandidaten die relevanten Kanten für den Text.\n"
|
||||||
"TEXT: {chunk_text}\n"
|
"TEXT: {chunk_text}\n"
|
||||||
|
|
@ -76,7 +78,7 @@ class SemanticAnalyzer:
|
||||||
# LOG: Request Info
|
# LOG: Request Info
|
||||||
logger.debug(f"🔍 [SemanticAnalyzer] Request: {len(chunk_text)} chars Text, {len(all_edges)} Candidates.")
|
logger.debug(f"🔍 [SemanticAnalyzer] Request: {len(chunk_text)} chars Text, {len(all_edges)} Candidates.")
|
||||||
|
|
||||||
# 3. Prompt füllen (Hier trat der AttributeError auf, wenn prompt_template ein dict war)
|
# 3. Prompt füllen (Nutzt nun sicher einen String dank LLMService v3.3.2)
|
||||||
try:
|
try:
|
||||||
final_prompt = prompt_template.format(
|
final_prompt = prompt_template.format(
|
||||||
chunk_text=chunk_text[:3500],
|
chunk_text=chunk_text[:3500],
|
||||||
|
|
@ -88,6 +90,7 @@ class SemanticAnalyzer:
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 4. LLM Call mit Traffic Control (Background Priority)
|
# 4. LLM Call mit Traffic Control (Background Priority)
|
||||||
|
# Der gewählte Provider wird über die .env gesteuert (MINDNET_LLM_PROVIDER)
|
||||||
response_json = await self.llm.generate_raw_response(
|
response_json = await self.llm.generate_raw_response(
|
||||||
prompt=final_prompt,
|
prompt=final_prompt,
|
||||||
force_json=True,
|
force_json=True,
|
||||||
|
|
@ -132,7 +135,7 @@ class SemanticAnalyzer:
|
||||||
if isinstance(target, str):
|
if isinstance(target, str):
|
||||||
raw_candidates.append(f"{key}:{target}")
|
raw_candidates.append(f"{key}:{target}")
|
||||||
|
|
||||||
# 7. Strict Validation Loop
|
# 7. Strict Validation Loop (Übernahme aus deiner V2.2.0)
|
||||||
for e in raw_candidates:
|
for e in raw_candidates:
|
||||||
e_str = str(e)
|
e_str = str(e)
|
||||||
if self._is_valid_edge_string(e_str):
|
if self._is_valid_edge_string(e_str):
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ strategies:
|
||||||
# 1. Fakten-Abfrage (Fallback & Default)
|
# 1. Fakten-Abfrage (Fallback & Default)
|
||||||
FACT:
|
FACT:
|
||||||
description: "Reine Wissensabfrage."
|
description: "Reine Wissensabfrage."
|
||||||
preferred_provider: "ollama" # Schnell und lokal ausreichend
|
preferred_provider: "openrouter" # Schnell und lokal ausreichend
|
||||||
trigger_keywords: []
|
trigger_keywords: []
|
||||||
inject_types: []
|
inject_types: []
|
||||||
# WP-22: Definitionen & Hierarchien bevorzugen
|
# WP-22: Definitionen & Hierarchien bevorzugen
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
# config/prompts.yaml — Final V2.3.1 (Multi-Personality Support)
|
# config/prompts.yaml — Final V2.4.0 (Hybrid & Multi-Provider Support)
|
||||||
|
# WP-20: Optimierte Cloud-Templates bei unveränderten Ollama-Prompts.
|
||||||
|
|
||||||
system_prompt: |
|
system_prompt: |
|
||||||
Du bist 'mindnet', mein digitaler Zwilling und strategischer Partner.
|
Du bist 'mindnet', mein digitaler Zwilling und strategischer Partner.
|
||||||
|
|
@ -29,11 +30,13 @@ rag_template:
|
||||||
Beantworte die Frage präzise basierend auf den Quellen.
|
Beantworte die Frage präzise basierend auf den Quellen.
|
||||||
Fasse die Informationen zusammen. Sei objektiv und neutral.
|
Fasse die Informationen zusammen. Sei objektiv und neutral.
|
||||||
gemini: |
|
gemini: |
|
||||||
Analysiere diesen Kontext meines digitalen Zwillings:
|
Nutze das Wissen meines digitalen Zwillings aus folgendem Kontext: {context_str}
|
||||||
|
Beantworte die Anfrage präzise, detailliert und strukturiert: {query}
|
||||||
|
openrouter: |
|
||||||
|
Kontext-Analyse für Gemma/Llama:
|
||||||
{context_str}
|
{context_str}
|
||||||
Beantworte die Anfrage detailliert und prüfe auf Widersprüche: {query}
|
|
||||||
openrouter: "Kontext-Analyse für Gemma/Llama: {context_str}\n\nAnfrage: {query}"
|
Anfrage: {query}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 2. DECISION: Strategie & Abwägung (Intent: DECISION)
|
# 2. DECISION: Strategie & Abwägung (Intent: DECISION)
|
||||||
|
|
@ -57,11 +60,16 @@ decision_template:
|
||||||
FORMAT:
|
FORMAT:
|
||||||
- **Analyse:** (Kurze Zusammenfassung der Fakten)
|
- **Analyse:** (Kurze Zusammenfassung der Fakten)
|
||||||
- **Abgleich:** (Gibt es Konflikte mit Werten/Zielen? Nenne die Quelle!)
|
- **Abgleich:** (Gibt es Konflikte mit Werten/Zielen? Nenne die Quelle!)
|
||||||
- **Empfehlung:** (Klare Meinung: Ja/Nein/Vielleicht mit Begründung)
|
- **Empfehlung:** (Klare Meinung: Ja/No/Vielleicht mit Begründung)
|
||||||
gemini: |
|
gemini: |
|
||||||
Agierte als Senior Strategy Consultant. Nutze den Kontext {context_str}, um die Frage {query}
|
Agiere als Senior Strategy Consultant für meinen digitalen Zwilling.
|
||||||
tiefgreifend gegen meine langfristigen Ziele abzuwägen.
|
Wäge die Frage {query} multiperspektivisch gegen meine Werte und langfristigen Ziele ab.
|
||||||
openrouter: "Strategischer Check (OpenRouter): {query}\n\nReferenzdaten: {context_str}"
|
Kontext: {context_str}
|
||||||
|
openrouter: |
|
||||||
|
Strategischer Check via OpenRouter/Gemma:
|
||||||
|
Analyse der Entscheidungsfrage: {query}
|
||||||
|
Referenzdaten aus dem Graph: {context_str}
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 3. EMPATHY: Der Spiegel / "Ich"-Modus (Intent: EMPATHY)
|
# 3. EMPATHY: Der Spiegel / "Ich"-Modus (Intent: EMPATHY)
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
|
|
@ -83,6 +91,13 @@ empathy_template:
|
||||||
|
|
||||||
TONFALL:
|
TONFALL:
|
||||||
Ruhig, verständnisvoll, reflektiert. Keine Aufzählungszeichen, sondern fließender Text.
|
Ruhig, verständnisvoll, reflektiert. Keine Aufzählungszeichen, sondern fließender Text.
|
||||||
|
gemini: |
|
||||||
|
Reflektiere meine aktuelle Situation {query} basierend auf meinen Werten {context_str}.
|
||||||
|
Sei mein empathischer digitaler Zwilling. Antworte als 'Ich'.
|
||||||
|
openrouter: |
|
||||||
|
Empathische Reflexion (OpenRouter):
|
||||||
|
Situation: {query}
|
||||||
|
Persönlicher Kontext: {context_str}
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 4. TECHNICAL: Der Coder (Intent: CODING)
|
# 4. TECHNICAL: Der Coder (Intent: CODING)
|
||||||
|
|
@ -107,6 +122,14 @@ technical_template:
|
||||||
- Kurze Erklärung des Ansatzes.
|
- Kurze Erklärung des Ansatzes.
|
||||||
- Markdown Code-Block (Copy-Paste fertig).
|
- Markdown Code-Block (Copy-Paste fertig).
|
||||||
- Wichtige Edge-Cases.
|
- Wichtige Edge-Cases.
|
||||||
|
gemini: |
|
||||||
|
Du bist Senior Software Engineer. Löse die technische Aufgabe {query}
|
||||||
|
unter Berücksichtigung meiner Dokumentation: {context_str}.
|
||||||
|
openrouter: |
|
||||||
|
Technischer Support via OpenRouter:
|
||||||
|
Task: {query}
|
||||||
|
Kontext-Snippets: {context_str}
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 5. INTERVIEW: Der "One-Shot Extractor" (Performance Mode)
|
# 5. INTERVIEW: Der "One-Shot Extractor" (Performance Mode)
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
|
|
@ -142,9 +165,8 @@ interview_template:
|
||||||
|
|
||||||
## (Zweiter Begriff aus STRUKTUR)
|
## (Zweiter Begriff aus STRUKTUR)
|
||||||
(Text...)
|
(Text...)
|
||||||
|
gemini: "Transformiere den Input {query} in das Schema {schema_fields} für Typ {target_type}."
|
||||||
(usw.)
|
openrouter: "Extrahiere Daten für Typ {target_type} aus {query}. Schema: {schema_fields}."
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# 6. EDGE_ALLOCATION: Kantenfilter (Intent: OFFLINE_FILTER)
|
# 6. EDGE_ALLOCATION: Kantenfilter (Intent: OFFLINE_FILTER)
|
||||||
|
|
@ -164,24 +186,37 @@ edge_allocation_template:
|
||||||
{edge_list}
|
{edge_list}
|
||||||
|
|
||||||
REGELN:
|
REGELN:
|
||||||
1. Die Kanten haben das Format "typ:ziel". Der "typ" ist variabel und kann ALLES sein (z.B. uses, blocks, inspired_by, loves, etc.).
|
1. Die Kanten haben das Format "typ:ziel". Der "typ" ist variabel und kann ALLES sein.
|
||||||
2. Gib NUR die Strings aus der Kandidaten-Liste zurück, die zum Text passen.
|
2. Gib NUR die Strings aus der Kandidaten-Liste zurück, die zum Text passen.
|
||||||
3. Erfinde KEINE neuen Kanten. Nutze exakt die Schreibweise aus der Liste.
|
3. Erfinde KEINE neuen Kanten.
|
||||||
4. Antworte als flache JSON-Liste.
|
4. Antworte als flache JSON-Liste.
|
||||||
|
|
||||||
BEISPIEL (Zur Demonstration der Logik):
|
|
||||||
Input Text: "Das Projekt Alpha scheitert, weil Budget fehlt."
|
|
||||||
Input Kandidaten: ["blocks:Projekt Alpha", "inspired_by:Buch der Weisen", "needs:Budget"]
|
|
||||||
Output: ["blocks:Projekt Alpha", "needs:Budget"]
|
|
||||||
|
|
||||||
DEIN OUTPUT (JSON):
|
DEIN OUTPUT (JSON):
|
||||||
|
|
||||||
gemini: |
|
gemini: |
|
||||||
Extrahiere semantische Kanten für den Graphen ({note_id}).
|
Analysiere den Textabschnitt: {chunk_text}
|
||||||
Finde auch implizite Verbindungen.
|
Wähle aus folgender Liste alle relevanten Kanten aus: {edge_list}
|
||||||
JSON: [{"to": "X", "kind": "Y", "reason": "Z"}].
|
Antworte STRIKT als JSON-Liste von Strings im Format ["typ:ziel"].
|
||||||
TEXT: {text}
|
Kein Text davor oder danach!
|
||||||
|
openrouter: |
|
||||||
|
Filtere die relevanten Kanten für den Graphen.
|
||||||
|
Kandidaten: {edge_list}
|
||||||
|
Text: {chunk_text}
|
||||||
|
Output: JSON-Liste ["typ:ziel"].
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# 7. SMART EDGE ALLOCATION: Extraktion (Intent: INGEST)
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
edge_extraction:
|
||||||
|
ollama: |
|
||||||
|
Extrahiere Kanten als JSON: [{"to": "X", "kind": "Y"}].
|
||||||
|
Text: {text}
|
||||||
|
gemini: |
|
||||||
|
Führe eine semantische Analyse der Notiz '{note_id}' durch.
|
||||||
|
Finde explizite und implizite Relationen.
|
||||||
|
Antworte als JSON: [{"to": "Ziel", "kind": "typ", "reason": "begründung"}]
|
||||||
|
Keine Erklärungen, nur JSON.
|
||||||
|
Text: {text}
|
||||||
openrouter: |
|
openrouter: |
|
||||||
Analysiere den Text für den Graphen. Identifiziere semantische Verbindungen.
|
Analysiere den Text für den Graphen. Identifiziere semantische Verbindungen.
|
||||||
Output JSON: [{"to": "X", "kind": "Y"}].
|
Output STRIKT als JSON-Liste: [{"to": "X", "kind": "Y"}].
|
||||||
Text: {text}
|
Text: {text}
|
||||||
Loading…
Reference in New Issue
Block a user