46 lines
1.9 KiB
Python
46 lines
1.9 KiB
Python
"""
|
|
FILE: app/core/ingestion/ingestion_chunk_payload.py
|
|
DESCRIPTION: Baut das JSON-Objekt für mindnet_chunks.
|
|
VERSION: 2.4.0
|
|
"""
|
|
from __future__ import annotations
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
def _as_list(x):
|
|
if x is None: return []
|
|
return x if isinstance(x, list) else [x]
|
|
|
|
def make_chunk_payloads(note: Dict[str, Any], note_path: str, chunks_from_chunker: List[Any], **kwargs) -> List[Dict[str, Any]]:
|
|
"""Erstellt die Payloads für die Chunks eines Dokuments."""
|
|
if isinstance(note, dict) and "frontmatter" in note: fm = note["frontmatter"]
|
|
else: fm = note or {}
|
|
|
|
note_type = fm.get("type") or "concept"
|
|
title = fm.get("title") or fm.get("id") or "Untitled"
|
|
tags = _as_list(fm.get("tags") or [])
|
|
cp = fm.get("chunking_profile") or fm.get("chunk_profile") or "sliding_standard"
|
|
rw = float(fm.get("retriever_weight", 1.0))
|
|
|
|
out: List[Dict[str, Any]] = []
|
|
for idx, ch in enumerate(chunks_from_chunker):
|
|
text = getattr(ch, "text", "") or ch.get("text", "")
|
|
pl: Dict[str, Any] = {
|
|
"note_id": getattr(ch, "note_id", None) or fm.get("id"),
|
|
"chunk_id": getattr(ch, "id", None),
|
|
"title": title,
|
|
"index": int(getattr(ch, "index", idx)),
|
|
"ord": int(getattr(ch, "index", idx)) + 1,
|
|
"type": note_type,
|
|
"tags": tags,
|
|
"text": text,
|
|
"window": getattr(ch, "window", text),
|
|
"neighbors_prev": _as_list(getattr(ch, "neighbors_prev", None)),
|
|
"neighbors_next": _as_list(getattr(ch, "neighbors_next", None)),
|
|
"section": getattr(ch, "section_title", "") or ch.get("section", ""),
|
|
"path": note_path,
|
|
"source_path": kwargs.get("file_path") or note_path,
|
|
"retriever_weight": rw,
|
|
"chunk_profile": cp
|
|
}
|
|
out.append(pl)
|
|
return out |