komplett neues WP20 deployment
This commit is contained in:
1 parent
f1bfa40b5b
commit
0157faab89
7 files changed
+203
-137
No files matched your search
+45
-40
@@ -5,7 +5,7 @@ DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen (Notes
|
||||
WP-22: Integration von Content Lifecycle (Status Gate) und Edge Registry Validation.
|
||||
WP-22: Kontextsensitive Kanten-Validierung mit Fundort-Reporting (Zeilennummern).
|
||||
WP-22: Multi-Hash Refresh für konsistente Change Detection.
|
||||
VERSION: 2.11.3 (WP-20 Quota Protection & Stability Patch)
|
||||
VERSION: 2.11.4
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.services.llm_service, app.services.edge_registry
|
||||
EXTERNAL_CONFIG: config/types.yaml, config/prompts.yaml
|
||||
@@ -22,7 +22,7 @@ from app.core.parser import (
|
||||
read_markdown,
|
||||
normalize_frontmatter,
|
||||
validate_required_frontmatter,
|
||||
extract_edges_with_context, # WP-22: Neue Funktion für Zeilennummern
|
||||
extract_edges_with_context,
|
||||
)
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.chunker import assemble_chunks, get_chunk_config
|
||||
@@ -44,7 +44,7 @@ from app.core.qdrant_points import (
|
||||
|
||||
from app.services.embeddings_client import EmbeddingsClient
|
||||
from app.services.edge_registry import registry as edge_registry
|
||||
from app.services.llm_service import LLMService # WP-20 Integration
|
||||
from app.services.llm_service import LLMService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -52,7 +52,9 @@ logger = logging.getLogger(__name__)
|
||||
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
||||
"""Lädt die types.yaml zur Steuerung der typ-spezifischen Ingestion."""
|
||||
import yaml
|
||||
path = custom_path or os.getenv("MINDNET_TYPES_FILE", "config/types.yaml")
|
||||
from app.config import get_settings
|
||||
settings = get_settings()
|
||||
path = custom_path or settings.MINDNET_TYPES_FILE
|
||||
if not os.path.exists(path): return {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
||||
@@ -65,35 +67,23 @@ def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
||||
return "concept"
|
||||
|
||||
def effective_chunk_profile_name(fm: dict, note_type: str, reg: dict) -> str:
|
||||
"""
|
||||
Ermittelt den Namen des zu nutzenden Chunk-Profils.
|
||||
Priorität: 1. Frontmatter Override -> 2. Type Config -> 3. Global Default
|
||||
"""
|
||||
"""Ermittelt den Namen des zu nutzenden Chunk-Profils."""
|
||||
override = fm.get("chunking_profile") or fm.get("chunk_profile")
|
||||
if override and isinstance(override, str):
|
||||
return override
|
||||
|
||||
if override and isinstance(override, str): return override
|
||||
t_cfg = reg.get("types", {}).get(note_type, {})
|
||||
if t_cfg:
|
||||
cp = t_cfg.get("chunking_profile") or t_cfg.get("chunk_profile")
|
||||
if cp: return cp
|
||||
|
||||
return reg.get("defaults", {}).get("chunking_profile", "sliding_standard")
|
||||
|
||||
def effective_retriever_weight(fm: dict, note_type: str, reg: dict) -> float:
|
||||
"""
|
||||
Ermittelt das effektive retriever_weight für das Scoring.
|
||||
Priorität: 1. Frontmatter Override -> 2. Type Config -> 3. Global Default
|
||||
"""
|
||||
"""Ermittelt das effektive retriever_weight für das Scoring."""
|
||||
override = fm.get("retriever_weight")
|
||||
if override is not None:
|
||||
try: return float(override)
|
||||
except: pass
|
||||
|
||||
t_cfg = reg.get("types", {}).get(note_type, {})
|
||||
if t_cfg and "retriever_weight" in t_cfg:
|
||||
return float(t_cfg["retriever_weight"])
|
||||
|
||||
if t_cfg and "retriever_weight" in t_cfg: return float(t_cfg["retriever_weight"])
|
||||
return float(reg.get("defaults", {}).get("retriever_weight", 1.0))
|
||||
|
||||
|
||||
@@ -106,13 +96,13 @@ class IngestionService:
|
||||
self.cfg = QdrantConfig.from_env()
|
||||
self.cfg.prefix = self.prefix
|
||||
self.client = get_client(self.cfg)
|
||||
self.dim = self.cfg.dim if hasattr(self.cfg, 'dim') else self.settings.VECTOR_SIZE
|
||||
self.dim = self.settings.VECTOR_SIZE # Synchronisiert mit Settings v0.6.2
|
||||
self.registry = load_type_registry()
|
||||
self.embedder = EmbeddingsClient()
|
||||
self.llm = LLMService() # WP-20
|
||||
self.llm = LLMService()
|
||||
|
||||
# Change Detection Modus (full oder body)
|
||||
self.active_hash_mode = os.getenv("MINDNET_CHANGE_DETECTION_MODE", "full")
|
||||
# WP-22: Change Detection Modus aus Settings
|
||||
self.active_hash_mode = self.settings.CHANGE_DETECTION_MODE
|
||||
|
||||
try:
|
||||
ensure_collections(self.client, self.prefix, self.dim)
|
||||
@@ -133,20 +123,20 @@ class IngestionService:
|
||||
async def _perform_smart_edge_allocation(self, text: str, note_id: str) -> List[Dict]:
|
||||
"""
|
||||
WP-20: Nutzt den Hybrid LLM Service für die semantische Kanten-Extraktion.
|
||||
QUOTEN-SCHUTZ: Bevorzugt OpenRouter (Gemma), um Gemini-Tageslimits zu schützen.
|
||||
QUOTEN-SCHUTZ: Bevorzugt OpenRouter (Gemma 2), um Gemini-Tageslimits zu schonen.
|
||||
"""
|
||||
# Bestimme Provider: Nutze OpenRouter falls Key vorhanden, sonst Global Default
|
||||
provider = "openrouter" if getattr(self.settings, "OPENROUTER_API_KEY", None) else self.settings.MINDNET_LLM_PROVIDER
|
||||
# Bestimme Provider: Nutze OpenRouter falls Key vorhanden
|
||||
provider = "openrouter" if self.settings.OPENROUTER_API_KEY else self.settings.MINDNET_LLM_PROVIDER
|
||||
model = self.settings.GEMMA_MODEL # Hochdurchsatz-Modell aus config.py
|
||||
|
||||
# Nutze Gemma-Modell für hohen Durchsatz via OpenRouter/Google
|
||||
model = getattr(self.settings, "GEMMA_MODEL", None)
|
||||
logger.info(f"🚀 [Ingestion] Turbo-Mode: Extracting edges for '{note_id}' using {model} on {provider}")
|
||||
|
||||
# Hole das optimierte Prompt-Template (Key: edge_extraction)
|
||||
# Hole das optimierte Prompt-Template (Kaskade: Provider -> gemini -> ollama)
|
||||
template = self.llm.get_prompt("edge_extraction", provider)
|
||||
prompt = template.format(text=text[:6000], note_id=note_id)
|
||||
|
||||
try:
|
||||
# Hintergrund-Task mit Semaphore via LLMService
|
||||
# Hintergrund-Task mit Semaphore via LLMService (WP-06)
|
||||
response_json = await self.llm.generate_raw_response(
|
||||
prompt=prompt,
|
||||
priority="background",
|
||||
@@ -156,13 +146,12 @@ class IngestionService:
|
||||
)
|
||||
data = json.loads(response_json)
|
||||
|
||||
# Metadaten für die Edge-Herkunft setzen
|
||||
for item in data:
|
||||
item["provenance"] = "semantic_ai"
|
||||
item["line"] = f"ai-{provider}"
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning(f"Smart Edge Allocation failed for {note_id}: {e}")
|
||||
logger.warning(f"⚠️ [Ingestion] Smart Edge Allocation failed for {note_id} on {provider}: {e}")
|
||||
return []
|
||||
|
||||
async def process_file(
|
||||
@@ -214,7 +203,7 @@ class IngestionService:
|
||||
except Exception as e:
|
||||
return {**result, "error": f"Payload build failed: {str(e)}"}
|
||||
|
||||
# 4. Change Detection (Multi-Hash)
|
||||
# 4. Change Detection (WP-22 Multi-Hash)
|
||||
old_payload = None
|
||||
if not force_replace:
|
||||
old_payload = self._fetch_note_payload(note_id)
|
||||
@@ -241,7 +230,7 @@ class IngestionService:
|
||||
try:
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
|
||||
# STABILITY PATCH: Prüfen, ob ensure_latest existiert (verhindert AttributeError)
|
||||
# WP-22 STABILITY PATCH: Prüfen, ob ensure_latest existiert
|
||||
if hasattr(edge_registry, "ensure_latest"):
|
||||
edge_registry.ensure_latest()
|
||||
|
||||
@@ -264,13 +253,13 @@ class IngestionService:
|
||||
e["kind"] = edge_registry.resolve(edge_type=e["kind"], provenance="explicit", context={**context, "line": e.get("line")})
|
||||
edges.append(e)
|
||||
|
||||
# B. WP-20: Smart AI Edges (Hybrid Acceleration)
|
||||
# B. WP-20: Smart AI Edges (Hybrid Turbo Acceleration)
|
||||
ai_edges = await self._perform_smart_edge_allocation(body_text, note_id)
|
||||
for e in ai_edges:
|
||||
e["kind"] = edge_registry.resolve(edge_type=e.get("kind"), provenance="semantic_ai", context={**context, "line": e.get("line")})
|
||||
edges.append(e)
|
||||
|
||||
# C. System-Kanten
|
||||
# C. System-Kanten (Struktur)
|
||||
try:
|
||||
raw_system_edges = build_edges_for_note(note_id, chunk_pls, note_level_references=note_pl.get("references", []), include_note_scope_refs=note_scope_refs)
|
||||
except TypeError:
|
||||
@@ -283,7 +272,7 @@ class IngestionService:
|
||||
edges.append(e)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Processing failed: {e}", exc_info=True)
|
||||
logger.error(f"Processing failed for {file_path}: {e}", exc_info=True)
|
||||
return {**result, "error": f"Processing failed: {str(e)}"}
|
||||
|
||||
# 6. Upsert
|
||||
@@ -303,7 +292,7 @@ class IngestionService:
|
||||
|
||||
return {"path": file_path, "status": "success", "changed": True, "note_id": note_id, "chunks_count": len(chunk_pls), "edges_count": len(edges)}
|
||||
except Exception as e:
|
||||
logger.error(f"Upsert failed: {e}", exc_info=True)
|
||||
logger.error(f"Upsert failed for {note_id}: {e}", exc_info=True)
|
||||
return {**result, "error": f"DB Upsert failed: {e}"}
|
||||
|
||||
def _fetch_note_payload(self, note_id: str) -> Optional[dict]:
|
||||
@@ -330,4 +319,20 @@ class IngestionService:
|
||||
selector = rest.FilterSelector(filter=f)
|
||||
for suffix in ["chunks", "edges"]:
|
||||
try: self.client.delete(collection_name=f"{self.prefix}_{suffix}", points_selector=selector)
|
||||
except Exception: pass
|
||||
except Exception: pass
|
||||
|
||||
async def create_from_text(self, markdown_content: str, filename: str, vault_root: str, folder: str = "00_Inbox") -> Dict[str, Any]:
|
||||
"""Hilfsmethode zur Erstellung einer Note aus einem Textstream."""
|
||||
target_dir = os.path.join(vault_root, folder)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
file_path = os.path.join(target_dir, filename)
|
||||
try:
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(markdown_content)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
await asyncio.sleep(0.1)
|
||||
logger.info(f"Written file to {file_path}")
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": f"Disk write failed: {str(e)}"}
|
||||
return await self.process_file(file_path=file_path, vault_root=vault_root, apply=True, force_replace=True, purge_before=True)
|
||||
Reference in new issue
Block a user