chunk_payload nimmt auch das richtige chunking_profile

This commit is contained in:
Lars 2025-12-16 15:50:24 +01:00
parent a272c39613
commit b3833f2051

View File

@ -1,11 +1,13 @@
""" """
FILE: app/core/chunk_payload.py FILE: app/core/chunk_payload.py
DESCRIPTION: Baut das JSON-Objekt für 'mindnet_chunks'. Inkludiert Nachbarschafts-IDs (prev/next) und Titel. DESCRIPTION: Baut das JSON-Objekt für 'mindnet_chunks'.
VERSION: 2.0.0 FEATURES:
- Inkludiert Nachbarschafts-IDs (prev/next) und Titel.
- FIX: Korrektes Auslesen von 'chunking_profile' (Frontmatter > Type > Default).
VERSION: 2.1.0
STATUS: Active STATUS: Active
DEPENDENCIES: yaml, os DEPENDENCIES: yaml, os
EXTERNAL_CONFIG: config/types.yaml EXTERNAL_CONFIG: config/types.yaml
LAST_ANALYSIS: 2025-12-15
""" """
from __future__ import annotations from __future__ import annotations
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@ -36,21 +38,32 @@ def _get_defaults(reg: dict) -> dict:
return {} return {}
def _as_float(x: Any): def _as_float(x: Any):
try: try: return float(x)
return float(x) except Exception: return None
except Exception:
return None
def _resolve_chunk_profile(note_type: str, reg: dict) -> str: def _resolve_chunk_profile_from_config(note_type: str, reg: dict) -> Optional[str]:
"""
Liest das Profil aus der Config (Type > Default).
Prüft 'chunking_profile' UND 'chunk_profile'.
"""
# 1. Type Level
types = _get_types_map(reg) types = _get_types_map(reg)
if isinstance(types, dict): if isinstance(types, dict):
t = types.get(note_type, {}) t = types.get(note_type, {})
if isinstance(t, dict) and isinstance(t.get("chunk_profile"), str): if isinstance(t, dict):
return t["chunk_profile"] # Prüfe beide Schreibweisen
cp = t.get("chunking_profile") or t.get("chunk_profile")
if isinstance(cp, str) and cp:
return cp
# 2. Defaults Level
defs = _get_defaults(reg) defs = _get_defaults(reg)
if isinstance(defs, dict) and isinstance(defs.get("chunk_profile"), str): if isinstance(defs, dict):
return defs["chunk_profile"] cp = defs.get("chunking_profile") or defs.get("chunk_profile")
return "default" if isinstance(cp, str) and cp:
return cp
return None
def _resolve_retriever_weight(note_type: str, reg: dict) -> float: def _resolve_retriever_weight(note_type: str, reg: dict) -> float:
types = _get_types_map(reg) types = _get_types_map(reg)
@ -58,20 +71,16 @@ def _resolve_retriever_weight(note_type: str, reg: dict) -> float:
t = types.get(note_type, {}) t = types.get(note_type, {})
if isinstance(t, dict) and (t.get("retriever_weight") is not None): if isinstance(t, dict) and (t.get("retriever_weight") is not None):
v = _as_float(t.get("retriever_weight")) v = _as_float(t.get("retriever_weight"))
if v is not None: if v is not None: return float(v)
return float(v)
defs = _get_defaults(reg) defs = _get_defaults(reg)
if isinstance(defs, dict) and (defs.get("retriever_weight") is not None): if isinstance(defs, dict) and (defs.get("retriever_weight") is not None):
v = _as_float(defs.get("retriever_weight")) v = _as_float(defs.get("retriever_weight"))
if v is not None: if v is not None: return float(v)
return float(v)
return 1.0 return 1.0
def _as_list(x): def _as_list(x):
if x is None: if x is None: return []
return [] if isinstance(x, list): return x
if isinstance(x, list):
return x
return [x] return [x]
def make_chunk_payloads(note: Dict[str, Any], def make_chunk_payloads(note: Dict[str, Any],
@ -81,17 +90,29 @@ def make_chunk_payloads(note: Dict[str, Any],
note_text: str = "", note_text: str = "",
types_cfg: Optional[dict] = None, types_cfg: Optional[dict] = None,
file_path: Optional[str] = None) -> List[Dict[str, Any]]: file_path: Optional[str] = None) -> List[Dict[str, Any]]:
"""
Erstellt die Payloads für die Chunks.
"""
fm = (note or {}).get("frontmatter", {}) or {} fm = (note or {}).get("frontmatter", {}) or {}
note_type = fm.get("type") or note.get("type") or "concept" note_type = fm.get("type") or note.get("type") or "concept"
# WP-11 FIX: Title Extraction für Discovery Service # Title Extraction
# Wir holen den Titel aus Frontmatter oder Fallback ID/Untitled
title = fm.get("title") or note.get("title") or fm.get("id") or "Untitled" title = fm.get("title") or note.get("title") or fm.get("id") or "Untitled"
reg = types_cfg if isinstance(types_cfg, dict) else _load_types() reg = types_cfg if isinstance(types_cfg, dict) else _load_types()
# types.yaml authoritative # --- FIX: Profil-Ermittlung ---
cp = _resolve_chunk_profile(note_type, reg) # 1. Frontmatter (Override)
cp = fm.get("chunking_profile") or fm.get("chunk_profile")
# 2. Config (Type / Default)
if not cp:
cp = _resolve_chunk_profile_from_config(note_type, reg)
# 3. Hard Fallback
if not cp:
cp = "sliding_standard" # Statt "default"
rw = _resolve_retriever_weight(note_type, reg) rw = _resolve_retriever_weight(note_type, reg)
tags = fm.get("tags") or [] tags = fm.get("tags") or []
@ -100,7 +121,7 @@ def make_chunk_payloads(note: Dict[str, Any],
out: List[Dict[str, Any]] = [] out: List[Dict[str, Any]] = []
for idx, ch in enumerate(chunks_from_chunker): for idx, ch in enumerate(chunks_from_chunker):
# Attribute oder Keys (Chunk-Objekt oder Dict) # Attribute sicher extrahieren
cid = getattr(ch, "id", None) or (ch.get("id") if isinstance(ch, dict) else None) cid = getattr(ch, "id", None) or (ch.get("id") if isinstance(ch, dict) else None)
nid = getattr(ch, "note_id", None) or (ch.get("note_id") if isinstance(ch, dict) else fm.get("id")) nid = getattr(ch, "note_id", None) or (ch.get("note_id") if isinstance(ch, dict) else fm.get("id"))
index = getattr(ch, "index", None) or (ch.get("index") if isinstance(ch, dict) else idx) index = getattr(ch, "index", None) or (ch.get("index") if isinstance(ch, dict) else idx)
@ -112,7 +133,7 @@ def make_chunk_payloads(note: Dict[str, Any],
pl: Dict[str, Any] = { pl: Dict[str, Any] = {
"note_id": nid, "note_id": nid,
"chunk_id": cid, "chunk_id": cid,
"title": title, # <--- HIER: Titel in Payload einfügen "title": title,
"index": int(index), "index": int(index),
"ord": int(index) + 1, "ord": int(index) + 1,
"type": note_type, "type": note_type,
@ -125,9 +146,10 @@ def make_chunk_payloads(note: Dict[str, Any],
"path": note_path, "path": note_path,
"source_path": file_path or note_path, "source_path": file_path or note_path,
"retriever_weight": float(rw), "retriever_weight": float(rw),
"chunk_profile": cp, "chunk_profile": cp, # Jetzt korrekt
} }
# Aufräumen von Alt-Feldern
# Cleanup
for alias in ("chunk_num", "Chunk_Number"): for alias in ("chunk_num", "Chunk_Number"):
pl.pop(alias, None) pl.pop(alias, None)