131 lines
5.7 KiB
Python
131 lines
5.7 KiB
Python
"""
|
|
FILE: app/core/ingestion/ingestion_chunk_payload.py
|
|
DESCRIPTION: Baut das JSON-Objekt für 'mindnet_chunks'.
|
|
Fix v2.4.3: Integration der zentralen Registry (WP-14) für konsistente Defaults.
|
|
WP-24c v4.3.0: candidate_pool wird explizit übernommen für Chunk-Attribution.
|
|
VERSION: 2.4.4 (WP-24c v4.3.0)
|
|
STATUS: Active
|
|
"""
|
|
from __future__ import annotations
|
|
from typing import Any, Dict, List, Optional
|
|
import logging
|
|
|
|
# ENTSCHEIDENDER FIX: Import der neutralen Registry-Logik zur Vermeidung von Circular Imports
|
|
from app.core.registry import load_type_registry
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Resolution Helpers (Audited)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _as_list(x):
|
|
"""Sichert die Listen-Integrität für Metadaten wie Tags."""
|
|
if x is None: return []
|
|
return x if isinstance(x, list) else [x]
|
|
|
|
def _resolve_val(note_type: str, reg: dict, key: str, default: Any) -> Any:
|
|
"""
|
|
Hierarchische Suche in der Registry: Type-Spezifisch > Globaler Default.
|
|
WP-14: Erlaubt dynamische Konfiguration via types.yaml.
|
|
"""
|
|
types = reg.get("types", {})
|
|
if isinstance(types, dict):
|
|
t_cfg = types.get(note_type, {})
|
|
if isinstance(t_cfg, dict):
|
|
# Fallback für Key-Varianten (z.B. chunking_profile vs chunk_profile)
|
|
val = t_cfg.get(key) or t_cfg.get(key.replace("ing", ""))
|
|
if val is not None: return val
|
|
|
|
defs = reg.get("defaults", {}) or reg.get("global", {})
|
|
if isinstance(defs, dict):
|
|
val = defs.get(key) or defs.get(key.replace("ing", ""))
|
|
if val is not None: return val
|
|
|
|
return default
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Haupt-API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def make_chunk_payloads(note: Dict[str, Any], note_path: str, chunks_from_chunker: List[Any], **kwargs) -> List[Dict[str, Any]]:
|
|
"""
|
|
Erstellt die Payloads für die Chunks inklusive Audit-Resolution.
|
|
Nutzt nun die zentrale Registry für alle Fallbacks.
|
|
"""
|
|
if isinstance(note, dict) and "frontmatter" in note:
|
|
fm = note["frontmatter"]
|
|
else:
|
|
fm = note or {}
|
|
|
|
# WP-14 Fix: Nutzt übergebene Registry oder lädt sie global
|
|
reg = kwargs.get("types_cfg") or load_type_registry()
|
|
|
|
note_type = fm.get("type") or "concept"
|
|
title = fm.get("title") or fm.get("id") or "Untitled"
|
|
tags = _as_list(fm.get("tags") or [])
|
|
|
|
# Audit: Resolution Hierarchie (Frontmatter > Registry)
|
|
cp = fm.get("chunking_profile") or fm.get("chunk_profile")
|
|
if not cp:
|
|
cp = _resolve_val(note_type, reg, "chunking_profile", "sliding_standard")
|
|
|
|
rw = fm.get("retriever_weight")
|
|
if rw is None:
|
|
rw = _resolve_val(note_type, reg, "retriever_weight", 1.0)
|
|
try:
|
|
rw = float(rw)
|
|
except:
|
|
rw = 1.0
|
|
|
|
out: List[Dict[str, Any]] = []
|
|
for idx, ch in enumerate(chunks_from_chunker):
|
|
is_dict = isinstance(ch, dict)
|
|
cid = getattr(ch, "id", None) if not is_dict else ch.get("id")
|
|
nid = getattr(ch, "note_id", None) if not is_dict else ch.get("note_id")
|
|
index = getattr(ch, "index", idx) if not is_dict else ch.get("index", idx)
|
|
text = getattr(ch, "text", "") if not is_dict else ch.get("text", "")
|
|
window = getattr(ch, "window", text) if not is_dict else ch.get("window", text)
|
|
prev_id = getattr(ch, "neighbors_prev", None) if not is_dict else ch.get("neighbors_prev")
|
|
next_id = getattr(ch, "neighbors_next", None) if not is_dict else ch.get("neighbors_next")
|
|
section = getattr(ch, "section_title", "") if not is_dict else ch.get("section", "")
|
|
# WP-24c v4.3.0: candidate_pool muss erhalten bleiben für Chunk-Attribution
|
|
candidate_pool = getattr(ch, "candidate_pool", []) if not is_dict else ch.get("candidate_pool", [])
|
|
|
|
pl: Dict[str, Any] = {
|
|
"note_id": nid or fm.get("id"),
|
|
"chunk_id": cid,
|
|
"title": title,
|
|
"index": int(index),
|
|
"ord": int(index) + 1,
|
|
"type": note_type,
|
|
"tags": tags,
|
|
"text": text,
|
|
"window": window,
|
|
"neighbors_prev": _as_list(prev_id),
|
|
"neighbors_next": _as_list(next_id),
|
|
"section": section,
|
|
"path": note_path,
|
|
"source_path": kwargs.get("file_path") or note_path,
|
|
"retriever_weight": rw,
|
|
"chunk_profile": cp,
|
|
"candidate_pool": candidate_pool # WP-24c v4.3.0: Kritisch für Chunk-Attribution
|
|
}
|
|
|
|
# Audit: Cleanup Pop (Vermeidung von redundanten Alias-Feldern)
|
|
for alias in ("chunk_num", "Chunk_Number"):
|
|
pl.pop(alias, None)
|
|
|
|
# WP-24c v4.4.0-DEBUG: Schnittstelle 2 - Transfer
|
|
# Log-Output unmittelbar bevor das Dictionary zurückgegeben wird
|
|
pool_size = len(candidate_pool) if candidate_pool else 0
|
|
pool_content = candidate_pool if candidate_pool else []
|
|
explicit_callout_in_pool = [c for c in pool_content if isinstance(c, dict) and c.get("provenance") == "explicit:callout"]
|
|
logger.debug(f"DEBUG-TRACER [Payload]: Chunk ID: {cid}, Index: {index}, Pool-Size: {pool_size}, Pool-Inhalt: {pool_content}, Explicit-Callout-Count: {len(explicit_callout_in_pool)}, Has_Candidate_Pool_Key: {'candidate_pool' in pl}")
|
|
if explicit_callout_in_pool:
|
|
for ec in explicit_callout_in_pool:
|
|
logger.debug(f"DEBUG-TRACER [Payload]: Explicit-Callout Detail - Kind: {ec.get('kind')}, To: {ec.get('to')}, Provenance: {ec.get('provenance')}")
|
|
|
|
out.append(pl)
|
|
|
|
return out |