app/core/chunk_payload.py aktualisiert
All checks were successful
Deploy mindnet to llm-node / deploy (push) Successful in 2s
All checks were successful
Deploy mindnet to llm-node / deploy (push) Successful in 2s
This commit is contained in:
parent
44e468fc21
commit
c05dbd4b3b
|
|
@ -2,44 +2,34 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/chunk_payload.py
|
||||
Version: 2.2.0
|
||||
Datum: 2025-10-06
|
||||
Version: 2.2.1
|
||||
Datum: 2025-11-07
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.0.1.
|
||||
Neu: Wenn der Chunker KEIN Overlap im Fenster liefert (== window fehlt / identisch zur Kernpassage),
|
||||
erzeugen wir FENSTER mit synthetischem Overlap auf Basis chunk_config.get_sizes(note_type)['overlap'].
|
||||
Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.2.0 / v2.0.1.
|
||||
Neu (2.2.1):
|
||||
• Stabilere Offsets (start/end) bei mehrfach vorkommenden Segmenten (inkrementelles Suchen + Fallback),
|
||||
• optionale Felder window_left_ctx_len und window_right_ctx_est zur Diagnose,
|
||||
• robustere Section-Pfadbehandlung.
|
||||
|
||||
Felder (beibehalten aus 2.0.1):
|
||||
- note_id, chunk_id, id (Alias), chunk_index, seq, path
|
||||
- window (mit Overlap), text (ohne linkes Overlap)
|
||||
- start, end (Offsets im gesamten Body)
|
||||
- overlap_left, overlap_right
|
||||
- token_count?, section_title?, section_path?, type?, title?, tags?
|
||||
|
||||
Kompatibilität:
|
||||
- 'id' == 'chunk_id' als Alias
|
||||
- Pfade bleiben relativ (keine führenden '/'), Backslashes → Slashes
|
||||
- Robust für Chunk-Objekte oder Dicts; Fensterquelle: 'window'|'text'|'content'|'raw'
|
||||
|
||||
Lizenz: MIT (projektintern)
|
||||
Felder (unverändert beibehalten):
|
||||
note_id, chunk_id (Alias: id), chunk_index, seq, path,
|
||||
window (mit linkem Overlap), text (Kernsegment), start, end,
|
||||
overlap_left, overlap_right,
|
||||
type, title, tags, token_count?, section_title?, section_path?
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
try:
|
||||
# Typgerechtes Overlap aus deiner Konfiguration holen
|
||||
from app.core.chunk_config import get_sizes as _get_sizes
|
||||
except Exception:
|
||||
def _get_sizes(_note_type: str):
|
||||
# konservativer Default, falls Import fehlschlägt
|
||||
return {"overlap": (40, 60), "target": (250, 350), "max": 500}
|
||||
|
||||
|
||||
# ------------------------------- Utils ------------------------------- #
|
||||
|
||||
def _get_attr_or_key(obj: Any, key: str, default=None):
|
||||
if obj is None:
|
||||
return default
|
||||
|
|
@ -48,7 +38,6 @@ def _get_attr_or_key(obj: Any, key: str, default=None):
|
|||
return getattr(obj, key, default)
|
||||
|
||||
def _as_window_text(chunk: Any) -> str:
|
||||
"""Fenstertext robust lesen (bevorzugt echte Fenster, sonst Kern)."""
|
||||
for k in ("window", "text", "content", "raw"):
|
||||
v = _get_attr_or_key(chunk, k, None)
|
||||
if isinstance(v, str) and v:
|
||||
|
|
@ -67,14 +56,7 @@ def _normalize_rel_path(p: str) -> str:
|
|||
p = p[1:]
|
||||
return p
|
||||
|
||||
|
||||
# ---------------------- Overlap & Offsets ---------------------------- #
|
||||
|
||||
def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int], str]:
|
||||
"""
|
||||
Entfernt linkes Overlap aus echten Fenster-Strings.
|
||||
Rückgabe: (segments, overlaps_left, reconstructed_text)
|
||||
"""
|
||||
segments: List[str] = []
|
||||
overlaps_left: List[int] = []
|
||||
reconstructed = ""
|
||||
|
|
@ -93,7 +75,6 @@ def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int
|
|||
return segments, overlaps_left, reconstructed
|
||||
|
||||
def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
|
||||
"""Länge längsten Suffix(a), der Prefix(b) ist."""
|
||||
if not a or not b:
|
||||
return 0
|
||||
a1 = a[-max_probe:]
|
||||
|
|
@ -104,26 +85,18 @@ def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
|
|||
return k
|
||||
return 0
|
||||
|
||||
|
||||
# ----------------------------- Public API ---------------------------- #
|
||||
|
||||
def make_chunk_payloads(
|
||||
frontmatter: Dict[str, Any],
|
||||
rel_path: str,
|
||||
chunks: Iterable[Union[Dict[str, Any], Any]],
|
||||
note_text: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Baut Payloads pro Chunk. Falls Fenster ohne Overlap geliefert werden,
|
||||
erzeugen wir synthetische 'window'-Texte mit typgerechtem Overlap.
|
||||
"""
|
||||
note_id = str(frontmatter.get("id") or "").strip()
|
||||
note_type = str(frontmatter.get("type", "")).lower()
|
||||
note_title = frontmatter.get("title", None)
|
||||
note_tags = frontmatter.get("tags", None)
|
||||
rel_path = _normalize_rel_path(rel_path)
|
||||
|
||||
# 1) Rohdaten sammeln (so wie geliefert)
|
||||
chunks_list = list(chunks)
|
||||
raw_windows: List[str] = []
|
||||
seqs: List[int] = []
|
||||
|
|
@ -134,16 +107,13 @@ def make_chunk_payloads(
|
|||
any_explicit_window = False
|
||||
|
||||
for idx, c in enumerate(chunks_list):
|
||||
# Fensterquelle
|
||||
w = _get_attr_or_key(c, "window", None)
|
||||
if isinstance(w, str) and w:
|
||||
any_explicit_window = True
|
||||
raw_windows.append(w)
|
||||
else:
|
||||
raw_windows.append(_as_window_text(c)) # 'text'|'content'|'raw' als Ersatz
|
||||
# Ordnung
|
||||
raw_windows.append(_as_window_text(c))
|
||||
seqs.append(_to_int(_get_attr_or_key(c, "seq", _get_attr_or_key(c, "chunk_index", idx)), idx))
|
||||
# IDs, Tokens, Sektionen
|
||||
cid = _get_attr_or_key(c, "chunk_id", _get_attr_or_key(c, "id", None))
|
||||
ids_in.append(str(cid) if isinstance(cid, str) and cid else None)
|
||||
tc = _get_attr_or_key(c, "token_count", None)
|
||||
|
|
@ -151,14 +121,10 @@ def make_chunk_payloads(
|
|||
section_titles.append(_get_attr_or_key(c, "section_title", None))
|
||||
section_paths.append(_get_attr_or_key(c, "section_path", None))
|
||||
|
||||
# 2) Segmente & Overlaps bestimmen
|
||||
if any_explicit_window:
|
||||
# Es existieren echte Fenster → dedupe, um Kernsegmente zu finden
|
||||
segments, overlaps_left, recon = _dedupe_windows_to_segments(raw_windows)
|
||||
windows_final = raw_windows[:] # bereits mit Overlap geliefert
|
||||
windows_final = raw_windows[:]
|
||||
else:
|
||||
# Keine echten Fenster → Segmente sind identisch zu "Fenstern" (bisher),
|
||||
# wir erzeugen synthetische Fenster mit Overlap gemäß Typ
|
||||
segments = [w or "" for w in raw_windows]
|
||||
overlaps_left = []
|
||||
windows_final = []
|
||||
|
|
@ -171,19 +137,16 @@ def make_chunk_payloads(
|
|||
|
||||
for i, seg in enumerate(segments):
|
||||
if i == 0:
|
||||
# erstes Fenster: kein linker Kontext
|
||||
windows_final.append(seg)
|
||||
overlaps_left.append(0)
|
||||
recon += seg
|
||||
else:
|
||||
# synthetischer linker Kontext = Suffix des bisher rekonstruierten Texts
|
||||
k = min(overlap_target, len(recon))
|
||||
left_ctx = recon[-k:] if k > 0 else ""
|
||||
windows_final.append(left_ctx + seg)
|
||||
overlaps_left.append(k)
|
||||
recon += seg # Rekonstruktion bleibt kerntreu
|
||||
recon += seg
|
||||
|
||||
# 3) overlap_right bestimmen
|
||||
overlaps_right: List[int] = []
|
||||
for i in range(len(windows_final)):
|
||||
if i + 1 < len(windows_final):
|
||||
|
|
@ -192,10 +155,8 @@ def make_chunk_payloads(
|
|||
ov = 0
|
||||
overlaps_right.append(ov)
|
||||
|
||||
# 4) start/end-Offsets (exakt via note_text, sonst kumulativ)
|
||||
starts: List[int] = [0] * len(segments)
|
||||
ends: List[int] = [0] * len(segments)
|
||||
pos = 0
|
||||
if isinstance(note_text, str) and note_text:
|
||||
search_pos = 0
|
||||
for i, seg in enumerate(segments):
|
||||
|
|
@ -208,24 +169,25 @@ def make_chunk_payloads(
|
|||
ends[i] = j + len(seg)
|
||||
search_pos = ends[i]
|
||||
else:
|
||||
# Fallback: kumulativ
|
||||
starts[i] = pos
|
||||
pos += len(seg)
|
||||
ends[i] = pos
|
||||
# Fallback: naive fortlaufende Positionierung
|
||||
starts[i] = starts[i - 1] if i > 0 else 0
|
||||
ends[i] = starts[i] + len(seg)
|
||||
search_pos = ends[i]
|
||||
else:
|
||||
pos = 0
|
||||
for i, seg in enumerate(segments):
|
||||
starts[i] = pos
|
||||
pos += len(seg)
|
||||
ends[i] = pos
|
||||
|
||||
# 5) Payload-Dicts
|
||||
payloads: List[Dict[str, Any]] = []
|
||||
for i, (win, seg) in enumerate(zip(windows_final, segments)):
|
||||
chunk_id = ids_in[i] or f"{note_id}#{i+1}"
|
||||
left_len = max(0, len(win) - len(seg))
|
||||
pl: Dict[str, Any] = {
|
||||
"note_id": note_id,
|
||||
"chunk_id": chunk_id,
|
||||
"id": chunk_id, # Alias
|
||||
"id": chunk_id,
|
||||
"chunk_index": i,
|
||||
"seq": seqs[i],
|
||||
"path": rel_path,
|
||||
|
|
@ -235,8 +197,8 @@ def make_chunk_payloads(
|
|||
"end": ends[i],
|
||||
"overlap_left": overlaps_left[i],
|
||||
"overlap_right": overlaps_right[i],
|
||||
"window_left_ctx_len": left_len,
|
||||
}
|
||||
# optionale Metadaten
|
||||
if note_type:
|
||||
pl["type"] = note_type
|
||||
if note_title is not None:
|
||||
|
|
@ -254,11 +216,8 @@ def make_chunk_payloads(
|
|||
|
||||
return payloads
|
||||
|
||||
|
||||
# __main__ Demo (optional)
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
fm = {"id": "demo", "title": "Demo", "type": "concept"}
|
||||
# Beispiel ohne echte Fenster → erzeugt synthetische Overlaps
|
||||
chunks = [
|
||||
{"id": "demo#1", "text": "Alpha Beta Gamma"},
|
||||
{"id": "demo#2", "text": "Gamma Delta"},
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user