""" FILE: app/core/ingestion/ingestion_chunk_payload.py DESCRIPTION: Baut das JSON-Objekt für 'mindnet_chunks'. Fix v2.4.3: Integration der zentralen Registry (WP-14) für konsistente Defaults. WP-24c v4.3.0: candidate_pool wird explizit übernommen für Chunk-Attribution. WP-26 v1.0: Erweiterung um effective_type (section_type || note_type) und note_type-Feld. VERSION: 2.5.0 (WP-26 v1.0) STATUS: Active """ from __future__ import annotations from typing import Any, Dict, List, Optional import logging # ENTSCHEIDENDER FIX: Import der neutralen Registry-Logik zur Vermeidung von Circular Imports from app.core.registry import load_type_registry logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Resolution Helpers (Audited) # --------------------------------------------------------------------------- def _as_list(x): """Sichert die Listen-Integrität für Metadaten wie Tags.""" if x is None: return [] return x if isinstance(x, list) else [x] def _resolve_val(note_type: str, reg: dict, key: str, default: Any) -> Any: """ Hierarchische Suche in der Registry: Type-Spezifisch > Globaler Default. WP-14: Erlaubt dynamische Konfiguration via types.yaml. """ types = reg.get("types", {}) if isinstance(types, dict): t_cfg = types.get(note_type, {}) if isinstance(t_cfg, dict): # Fallback für Key-Varianten (z.B. chunking_profile vs chunk_profile) val = t_cfg.get(key) or t_cfg.get(key.replace("ing", "")) if val is not None: return val defs = reg.get("defaults", {}) or reg.get("global", {}) if isinstance(defs, dict): val = defs.get(key) or defs.get(key.replace("ing", "")) if val is not None: return val return default # --------------------------------------------------------------------------- # Haupt-API # --------------------------------------------------------------------------- def make_chunk_payloads(note: Dict[str, Any], note_path: str, chunks_from_chunker: List[Any], **kwargs) -> List[Dict[str, Any]]: """ Erstellt die Payloads für die Chunks inklusive Audit-Resolution. Nutzt nun die zentrale Registry für alle Fallbacks. """ if isinstance(note, dict) and "frontmatter" in note: fm = note["frontmatter"] else: fm = note or {} # WP-14 Fix: Nutzt übergebene Registry oder lädt sie global reg = kwargs.get("types_cfg") or load_type_registry() note_type = fm.get("type") or "concept" title = fm.get("title") or fm.get("id") or "Untitled" tags = _as_list(fm.get("tags") or []) # Audit: Resolution Hierarchie (Frontmatter > Registry) cp = fm.get("chunking_profile") or fm.get("chunk_profile") if not cp: cp = _resolve_val(note_type, reg, "chunking_profile", "sliding_standard") rw = fm.get("retriever_weight") if rw is None: rw = _resolve_val(note_type, reg, "retriever_weight", 1.0) try: rw = float(rw) except: rw = 1.0 out: List[Dict[str, Any]] = [] for idx, ch in enumerate(chunks_from_chunker): is_dict = isinstance(ch, dict) cid = getattr(ch, "id", None) if not is_dict else ch.get("id") nid = getattr(ch, "note_id", None) if not is_dict else ch.get("note_id") index = getattr(ch, "index", idx) if not is_dict else ch.get("index", idx) text = getattr(ch, "text", "") if not is_dict else ch.get("text", "") window = getattr(ch, "window", text) if not is_dict else ch.get("window", text) prev_id = getattr(ch, "neighbors_prev", None) if not is_dict else ch.get("neighbors_prev") next_id = getattr(ch, "neighbors_next", None) if not is_dict else ch.get("neighbors_next") section = getattr(ch, "section_title", "") if not is_dict else ch.get("section", "") # WP-24c v4.3.0: candidate_pool muss erhalten bleiben für Chunk-Attribution candidate_pool = getattr(ch, "candidate_pool", []) if not is_dict else ch.get("candidate_pool", []) # WP-26 v1.0: Section-Type für typ-spezifische Sektionen section_type = getattr(ch, "section_type", None) if not is_dict else ch.get("section_type") # WP-26 v1.0: Block-ID für Intra-Note-Links block_id = getattr(ch, "block_id", None) if not is_dict else ch.get("block_id") # WP-26 v1.0: Effektiver Typ = section_type || note_type (FA-03) effective_type = section_type if section_type else note_type # WP-26 v1.0: retriever_weight basiert auf effektivem Typ (FA-09b) # Wenn section_type vorhanden, nutze dessen retriever_weight effective_rw = rw if section_type: effective_rw = _resolve_val(section_type, reg, "retriever_weight", rw) try: effective_rw = float(effective_rw) except: effective_rw = rw pl: Dict[str, Any] = { "note_id": nid or fm.get("id"), "chunk_id": cid, "title": title, "index": int(index), "ord": int(index) + 1, # WP-26 v1.0: type enthält den effektiven Typ (section_type || note_type) "type": effective_type, # WP-26 v1.0: note_type ist immer der ursprüngliche Note-Typ (für Filterung) "note_type": note_type, "tags": tags, "text": text, "window": window, "neighbors_prev": _as_list(prev_id), "neighbors_next": _as_list(next_id), "section": section, "path": note_path, "source_path": kwargs.get("file_path") or note_path, # WP-26 v1.0: retriever_weight basiert auf effektivem Typ "retriever_weight": effective_rw, "chunk_profile": cp, "candidate_pool": candidate_pool, # WP-24c v4.3.0: Kritisch für Chunk-Attribution # WP-26 v1.0: Optionale Felder für Section-Type-Tracking "section_type": section_type, # Expliziter Section-Type (oder None) "block_id": block_id, # Block-ID für Intra-Note-Links (oder None) } # Audit: Cleanup Pop (Vermeidung von redundanten Alias-Feldern) for alias in ("chunk_num", "Chunk_Number"): pl.pop(alias, None) # WP-24c v4.4.0-DEBUG: Schnittstelle 2 - Transfer # Log-Output unmittelbar bevor das Dictionary zurückgegeben wird pool_size = len(candidate_pool) if candidate_pool else 0 pool_content = candidate_pool if candidate_pool else [] explicit_callout_in_pool = [c for c in pool_content if isinstance(c, dict) and c.get("provenance") == "explicit:callout"] logger.debug(f"DEBUG-TRACER [Payload]: Chunk ID: {cid}, Index: {index}, Pool-Size: {pool_size}, Pool-Inhalt: {pool_content}, Explicit-Callout-Count: {len(explicit_callout_in_pool)}, Has_Candidate_Pool_Key: {'candidate_pool' in pl}") if explicit_callout_in_pool: for ec in explicit_callout_in_pool: logger.debug(f"DEBUG-TRACER [Payload]: Explicit-Callout Detail - Kind: {ec.get('kind')}, To: {ec.get('to')}, Provenance: {ec.get('provenance')}") out.append(pl) return out