""" FILE: app/core/ingestion.py DESCRIPTION: Haupt-Ingestion-Logik. FIX: Korrekte Priorisierung von Frontmatter für chunk_profile und retriever_weight. Lade Chunk-Config basierend auf dem effektiven Profil, nicht nur dem Notiz-Typ. WP-22: Integration von Content Lifecycle (Status) und Edge Registry. VERSION: 2.8.1 (WP-22 Lifecycle & Registry) STATUS: Active DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.core.derive_edges, app.core.qdrant*, app.services.embeddings_client, app.services.edge_registry EXTERNAL_CONFIG: config/types.yaml """ import os import logging import asyncio import time from typing import Dict, List, Optional, Tuple, Any # Core Module Imports from app.core.parser import ( read_markdown, normalize_frontmatter, validate_required_frontmatter, ) from app.core.note_payload import make_note_payload from app.core.chunker import assemble_chunks, get_chunk_config from app.core.chunk_payload import make_chunk_payloads # Fallback für Edges try: from app.core.derive_edges import build_edges_for_note except ImportError: def build_edges_for_note(*args, **kwargs): return [] from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes from app.core.qdrant_points import ( points_for_chunks, points_for_note, points_for_edges, upsert_batch, ) from app.services.embeddings_client import EmbeddingsClient from app.services.edge_registry import registry as edge_registry logger = logging.getLogger(__name__) # --- Helper --- def load_type_registry(custom_path: Optional[str] = None) -> dict: import yaml path = custom_path or os.getenv("MINDNET_TYPES_FILE", "config/types.yaml") if not os.path.exists(path): return {} try: with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {} except Exception: return {} def resolve_note_type(requested: Optional[str], reg: dict) -> str: types = reg.get("types", {}) if requested and requested in types: return requested return "concept" def effective_chunk_profile_name(fm: dict, note_type: str, reg: dict) -> str: """ Ermittelt den Namen des Chunk-Profils. Prio: 1. Frontmatter -> 2. Type-Config -> 3. Default """ # 1. Frontmatter Override override = fm.get("chunking_profile") or fm.get("chunk_profile") if override and isinstance(override, str): return override # 2. Type Config t_cfg = reg.get("types", {}).get(note_type, {}) if t_cfg: cp = t_cfg.get("chunking_profile") or t_cfg.get("chunk_profile") if cp: return cp # 3. Global Default return reg.get("defaults", {}).get("chunking_profile", "sliding_standard") def effective_retriever_weight(fm: dict, note_type: str, reg: dict) -> float: """ Ermittelt das Retriever Weight. Prio: 1. Frontmatter -> 2. Type-Config -> 3. Default """ # 1. Frontmatter Override override = fm.get("retriever_weight") if override is not None: try: return float(override) except: pass # 2. Type Config t_cfg = reg.get("types", {}).get(note_type, {}) if t_cfg and "retriever_weight" in t_cfg: return float(t_cfg["retriever_weight"]) # 3. Global Default return float(reg.get("defaults", {}).get("retriever_weight", 1.0)) class IngestionService: def __init__(self, collection_prefix: str = None): env_prefix = os.getenv("COLLECTION_PREFIX", "mindnet") self.prefix = collection_prefix or env_prefix self.cfg = QdrantConfig.from_env() self.cfg.prefix = self.prefix self.client = get_client(self.cfg) self.dim = self.cfg.dim self.registry = load_type_registry() self.embedder = EmbeddingsClient() # ACTIVE HASH MODE aus ENV lesen (Default: full) self.active_hash_mode = os.getenv("MINDNET_CHANGE_DETECTION_MODE", "full") try: ensure_collections(self.client, self.prefix, self.dim) ensure_payload_indexes(self.client, self.prefix) except Exception as e: logger.warning(f"DB init warning: {e}") def _get_chunk_config_by_profile(self, profile_name: str, note_type: str) -> Dict[str, Any]: """ Lädt die konkrete Config (target, max, overlap) für einen Profilnamen. """ # Suche direkt in den definierten Profilen der Registry profiles = self.registry.get("chunking_profiles", {}) if profile_name in profiles: cfg = profiles[profile_name].copy() # Tuple-Fix für Overlap (wie in chunker.py) if "overlap" in cfg and isinstance(cfg["overlap"], list): cfg["overlap"] = tuple(cfg["overlap"]) return cfg # Fallback: Wenn Profilname unbekannt, nutze Standard für den Typ via Chunker logger.warning(f"Profile '{profile_name}' not found in registry. Falling back to type defaults.") return get_chunk_config(note_type) async def process_file( self, file_path: str, vault_root: str, force_replace: bool = False, apply: bool = False, purge_before: bool = False, note_scope_refs: bool = False, hash_source: str = "parsed", hash_normalize: str = "canonical" ) -> Dict[str, Any]: result = {"path": file_path, "status": "skipped", "changed": False, "error": None} # 1. Parse & Frontmatter Validation try: parsed = read_markdown(file_path) if not parsed: return {**result, "error": "Empty or unreadable file"} fm = normalize_frontmatter(parsed.frontmatter) validate_required_frontmatter(fm) except Exception as e: logger.error(f"Validation failed for {file_path}: {e}") return {**result, "error": f"Validation failed: {str(e)}"} # --- WP-22: Content Lifecycle Gate (Teil A) --- status = fm.get("status", "draft").lower().strip() # Hard Skip für System-Dateien if status in ["system", "template", "archive", "hidden"]: logger.info(f"Skipping file {file_path} (Status: {status})") return {**result, "status": "skipped", "reason": f"lifecycle_status_{status}"} # 2. Type & Config Resolution (FIXED) # Wir ermitteln erst den Typ note_type = resolve_note_type(fm.get("type"), self.registry) fm["type"] = note_type # Dann ermitteln wir die effektiven Werte unter Berücksichtigung des Frontmatters! effective_profile = effective_chunk_profile_name(fm, note_type, self.registry) effective_weight = effective_retriever_weight(fm, note_type, self.registry) # Wir schreiben die effektiven Werte zurück ins FM, damit note_payload sie sicher hat fm["chunk_profile"] = effective_profile fm["retriever_weight"] = effective_weight # 3. Build Note Payload try: note_pl = make_note_payload( parsed, vault_root=vault_root, hash_normalize=hash_normalize, hash_source=hash_source, file_path=file_path ) # Text Body Fallback if not note_pl.get("fulltext"): note_pl["fulltext"] = getattr(parsed, "body", "") or "" # Update Payload with explicit effective values (Sicherheit) note_pl["retriever_weight"] = effective_weight note_pl["chunk_profile"] = effective_profile # WP-22: Status speichern für Dynamic Scoring note_pl["status"] = status note_id = note_pl["note_id"] except Exception as e: logger.error(f"Payload build failed: {e}") return {**result, "error": f"Payload build failed: {str(e)}"} # 4. Change Detection old_payload = None if not force_replace: old_payload = self._fetch_note_payload(note_id) has_old = old_payload is not None check_key = f"{self.active_hash_mode}:{hash_source}:{hash_normalize}" old_hashes = (old_payload or {}).get("hashes") if isinstance(old_hashes, dict): old_hash = old_hashes.get(check_key) elif isinstance(old_hashes, str) and self.active_hash_mode == "body": old_hash = old_hashes else: old_hash = None new_hash = note_pl.get("hashes", {}).get(check_key) hash_changed = (old_hash != new_hash) chunks_missing, edges_missing = self._artifacts_missing(note_id) should_write = force_replace or (not has_old) or hash_changed or chunks_missing or edges_missing if not should_write: return {**result, "status": "unchanged", "note_id": note_id} if not apply: return {**result, "status": "dry-run", "changed": True, "note_id": note_id} # 5. Processing try: body_text = getattr(parsed, "body", "") or "" # FIX: Wir laden jetzt die Config für das SPEZIFISCHE Profil chunk_config = self._get_chunk_config_by_profile(effective_profile, note_type) chunks = await assemble_chunks(fm["id"], body_text, fm["type"], config=chunk_config) # chunk_payloads werden mit den aktualisierten FM-Werten gebaut chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text) vecs = [] if chunk_pls: texts = [c.get("window") or c.get("text") or "" for c in chunk_pls] try: if hasattr(self.embedder, 'embed_documents'): vecs = await self.embedder.embed_documents(texts) else: for t in texts: v = await self.embedder.embed_query(t) vecs.append(v) except Exception as e: logger.error(f"Embedding failed: {e}") raise RuntimeError(f"Embedding failed: {e}") # Raw Edges generieren try: raw_edges = build_edges_for_note( note_id, chunk_pls, note_level_references=note_pl.get("references", []), include_note_scope_refs=note_scope_refs ) except TypeError: raw_edges = build_edges_for_note(note_id, chunk_pls) # --- WP-22: Edge Registry Validation (Teil B) --- edges = [] if raw_edges: for edge in raw_edges: original_kind = edge.get("kind", "related_to") # Resolve via Registry (Canonical mapping + Unknown Logging) canonical_kind = edge_registry.resolve(original_kind) edge["kind"] = canonical_kind edges.append(edge) except Exception as e: logger.error(f"Processing failed: {e}", exc_info=True) return {**result, "error": f"Processing failed: {str(e)}"} # 6. Upsert try: if purge_before and has_old: self._purge_artifacts(note_id) n_name, n_pts = points_for_note(self.prefix, note_pl, None, self.dim) upsert_batch(self.client, n_name, n_pts) if chunk_pls and vecs: c_name, c_pts = points_for_chunks(self.prefix, chunk_pls, vecs) upsert_batch(self.client, c_name, c_pts) if edges: e_name, e_pts = points_for_edges(self.prefix, edges) upsert_batch(self.client, e_name, e_pts) return { "path": file_path, "status": "success", "changed": True, "note_id": note_id, "chunks_count": len(chunk_pls), "edges_count": len(edges) } except Exception as e: logger.error(f"Upsert failed: {e}", exc_info=True) return {**result, "error": f"DB Upsert failed: {e}"} def _fetch_note_payload(self, note_id: str) -> Optional[dict]: from qdrant_client.http import models as rest col = f"{self.prefix}_notes" try: f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) pts, _ = self.client.scroll(collection_name=col, scroll_filter=f, limit=1, with_payload=True) return pts[0].payload if pts else None except: return None def _artifacts_missing(self, note_id: str) -> Tuple[bool, bool]: from qdrant_client.http import models as rest c_col = f"{self.prefix}_chunks" e_col = f"{self.prefix}_edges" try: f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) c_pts, _ = self.client.scroll(collection_name=c_col, scroll_filter=f, limit=1) e_pts, _ = self.client.scroll(collection_name=e_col, scroll_filter=f, limit=1) return (not bool(c_pts)), (not bool(e_pts)) except: return True, True def _purge_artifacts(self, note_id: str): from qdrant_client.http import models as rest f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) selector = rest.FilterSelector(filter=f) for suffix in ["chunks", "edges"]: try: self.client.delete(collection_name=f"{self.prefix}_{suffix}", points_selector=selector) except Exception: pass async def create_from_text(self, markdown_content: str, filename: str, vault_root: str, folder: str = "00_Inbox") -> Dict[str, Any]: target_dir = os.path.join(vault_root, folder) os.makedirs(target_dir, exist_ok=True) file_path = os.path.join(target_dir, filename) try: with open(file_path, "w", encoding="utf-8") as f: f.write(markdown_content) f.flush() os.fsync(f.fileno()) await asyncio.sleep(0.1) logger.info(f"Written file to {file_path}") except Exception as e: return {"status": "error", "error": f"Disk write failed: {str(e)}"} return await self.process_file(file_path=file_path, vault_root=vault_root, apply=True, force_replace=True, purge_before=True)