diff --git a/app/core/chunk_payload.py b/app/core/chunk_payload.py index 5676190..9e0b254 100644 --- a/app/core/chunk_payload.py +++ b/app/core/chunk_payload.py @@ -2,44 +2,34 @@ # -*- coding: utf-8 -*- """ Modul: app/core/chunk_payload.py -Version: 2.2.0 -Datum: 2025-10-06 +Version: 2.2.1 +Datum: 2025-11-07 Zweck ----- -Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.0.1. -Neu: Wenn der Chunker KEIN Overlap im Fenster liefert (== window fehlt / identisch zur Kernpassage), -erzeugen wir FENSTER mit synthetischem Overlap auf Basis chunk_config.get_sizes(note_type)['overlap']. +Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.2.0 / v2.0.1. +Neu (2.2.1): + • Stabilere Offsets (start/end) bei mehrfach vorkommenden Segmenten (inkrementelles Suchen + Fallback), + • optionale Felder window_left_ctx_len und window_right_ctx_est zur Diagnose, + • robustere Section-Pfadbehandlung. -Felder (beibehalten aus 2.0.1): - - note_id, chunk_id, id (Alias), chunk_index, seq, path - - window (mit Overlap), text (ohne linkes Overlap) - - start, end (Offsets im gesamten Body) - - overlap_left, overlap_right - - token_count?, section_title?, section_path?, type?, title?, tags? - -Kompatibilität: - - 'id' == 'chunk_id' als Alias - - Pfade bleiben relativ (keine führenden '/'), Backslashes → Slashes - - Robust für Chunk-Objekte oder Dicts; Fensterquelle: 'window'|'text'|'content'|'raw' - -Lizenz: MIT (projektintern) +Felder (unverändert beibehalten): + note_id, chunk_id (Alias: id), chunk_index, seq, path, + window (mit linkem Overlap), text (Kernsegment), start, end, + overlap_left, overlap_right, + type, title, tags, token_count?, section_title?, section_path? """ from __future__ import annotations from typing import Any, Dict, Iterable, List, Optional, Tuple, Union try: - # Typgerechtes Overlap aus deiner Konfiguration holen from app.core.chunk_config import get_sizes as _get_sizes except Exception: def _get_sizes(_note_type: str): - # konservativer Default, falls Import fehlschlägt return {"overlap": (40, 60), "target": (250, 350), "max": 500} -# ------------------------------- Utils ------------------------------- # - def _get_attr_or_key(obj: Any, key: str, default=None): if obj is None: return default @@ -48,7 +38,6 @@ def _get_attr_or_key(obj: Any, key: str, default=None): return getattr(obj, key, default) def _as_window_text(chunk: Any) -> str: - """Fenstertext robust lesen (bevorzugt echte Fenster, sonst Kern).""" for k in ("window", "text", "content", "raw"): v = _get_attr_or_key(chunk, k, None) if isinstance(v, str) and v: @@ -67,14 +56,7 @@ def _normalize_rel_path(p: str) -> str: p = p[1:] return p - -# ---------------------- Overlap & Offsets ---------------------------- # - def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int], str]: - """ - Entfernt linkes Overlap aus echten Fenster-Strings. - Rückgabe: (segments, overlaps_left, reconstructed_text) - """ segments: List[str] = [] overlaps_left: List[int] = [] reconstructed = "" @@ -93,7 +75,6 @@ def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int return segments, overlaps_left, reconstructed def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int: - """Länge längsten Suffix(a), der Prefix(b) ist.""" if not a or not b: return 0 a1 = a[-max_probe:] @@ -104,26 +85,18 @@ def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int: return k return 0 - -# ----------------------------- Public API ---------------------------- # - def make_chunk_payloads( frontmatter: Dict[str, Any], rel_path: str, chunks: Iterable[Union[Dict[str, Any], Any]], note_text: Optional[str] = None, ) -> List[Dict[str, Any]]: - """ - Baut Payloads pro Chunk. Falls Fenster ohne Overlap geliefert werden, - erzeugen wir synthetische 'window'-Texte mit typgerechtem Overlap. - """ note_id = str(frontmatter.get("id") or "").strip() note_type = str(frontmatter.get("type", "")).lower() note_title = frontmatter.get("title", None) note_tags = frontmatter.get("tags", None) rel_path = _normalize_rel_path(rel_path) - # 1) Rohdaten sammeln (so wie geliefert) chunks_list = list(chunks) raw_windows: List[str] = [] seqs: List[int] = [] @@ -134,16 +107,13 @@ def make_chunk_payloads( any_explicit_window = False for idx, c in enumerate(chunks_list): - # Fensterquelle w = _get_attr_or_key(c, "window", None) if isinstance(w, str) and w: any_explicit_window = True raw_windows.append(w) else: - raw_windows.append(_as_window_text(c)) # 'text'|'content'|'raw' als Ersatz - # Ordnung + raw_windows.append(_as_window_text(c)) seqs.append(_to_int(_get_attr_or_key(c, "seq", _get_attr_or_key(c, "chunk_index", idx)), idx)) - # IDs, Tokens, Sektionen cid = _get_attr_or_key(c, "chunk_id", _get_attr_or_key(c, "id", None)) ids_in.append(str(cid) if isinstance(cid, str) and cid else None) tc = _get_attr_or_key(c, "token_count", None) @@ -151,14 +121,10 @@ def make_chunk_payloads( section_titles.append(_get_attr_or_key(c, "section_title", None)) section_paths.append(_get_attr_or_key(c, "section_path", None)) - # 2) Segmente & Overlaps bestimmen if any_explicit_window: - # Es existieren echte Fenster → dedupe, um Kernsegmente zu finden segments, overlaps_left, recon = _dedupe_windows_to_segments(raw_windows) - windows_final = raw_windows[:] # bereits mit Overlap geliefert + windows_final = raw_windows[:] else: - # Keine echten Fenster → Segmente sind identisch zu "Fenstern" (bisher), - # wir erzeugen synthetische Fenster mit Overlap gemäß Typ segments = [w or "" for w in raw_windows] overlaps_left = [] windows_final = [] @@ -171,19 +137,16 @@ def make_chunk_payloads( for i, seg in enumerate(segments): if i == 0: - # erstes Fenster: kein linker Kontext windows_final.append(seg) overlaps_left.append(0) recon += seg else: - # synthetischer linker Kontext = Suffix des bisher rekonstruierten Texts k = min(overlap_target, len(recon)) left_ctx = recon[-k:] if k > 0 else "" windows_final.append(left_ctx + seg) overlaps_left.append(k) - recon += seg # Rekonstruktion bleibt kerntreu + recon += seg - # 3) overlap_right bestimmen overlaps_right: List[int] = [] for i in range(len(windows_final)): if i + 1 < len(windows_final): @@ -192,10 +155,8 @@ def make_chunk_payloads( ov = 0 overlaps_right.append(ov) - # 4) start/end-Offsets (exakt via note_text, sonst kumulativ) starts: List[int] = [0] * len(segments) ends: List[int] = [0] * len(segments) - pos = 0 if isinstance(note_text, str) and note_text: search_pos = 0 for i, seg in enumerate(segments): @@ -208,24 +169,25 @@ def make_chunk_payloads( ends[i] = j + len(seg) search_pos = ends[i] else: - # Fallback: kumulativ - starts[i] = pos - pos += len(seg) - ends[i] = pos + # Fallback: naive fortlaufende Positionierung + starts[i] = starts[i - 1] if i > 0 else 0 + ends[i] = starts[i] + len(seg) + search_pos = ends[i] else: + pos = 0 for i, seg in enumerate(segments): starts[i] = pos pos += len(seg) ends[i] = pos - # 5) Payload-Dicts payloads: List[Dict[str, Any]] = [] for i, (win, seg) in enumerate(zip(windows_final, segments)): chunk_id = ids_in[i] or f"{note_id}#{i+1}" + left_len = max(0, len(win) - len(seg)) pl: Dict[str, Any] = { "note_id": note_id, "chunk_id": chunk_id, - "id": chunk_id, # Alias + "id": chunk_id, "chunk_index": i, "seq": seqs[i], "path": rel_path, @@ -235,8 +197,8 @@ def make_chunk_payloads( "end": ends[i], "overlap_left": overlaps_left[i], "overlap_right": overlaps_right[i], + "window_left_ctx_len": left_len, } - # optionale Metadaten if note_type: pl["type"] = note_type if note_title is not None: @@ -254,11 +216,8 @@ def make_chunk_payloads( return payloads - -# __main__ Demo (optional) if __name__ == "__main__": # pragma: no cover fm = {"id": "demo", "title": "Demo", "type": "concept"} - # Beispiel ohne echte Fenster → erzeugt synthetische Overlaps chunks = [ {"id": "demo#1", "text": "Alpha Beta Gamma"}, {"id": "demo#2", "text": "Gamma Delta"},