app/core/chunk_payload.py aktualisiert

app/core/chunk_payload.py

@@ -2,44 +2,34 @@
 # -*- coding: utf-8 -*-
 """
 Modul: app/core/chunk_payload.py
-Version: 2.2.0
+Version: 2.2.1
-Datum: 2025-10-06
+Datum: 2025-11-07
 
 Zweck
 -----
-Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.0.1.
+Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.2.0 / v2.0.1.
-Neu: Wenn der Chunker KEIN Overlap im Fenster liefert (== window fehlt / identisch zur Kernpassage),
+Neu (2.2.1):
-erzeugen wir FENSTER mit synthetischem Overlap auf Basis chunk_config.get_sizes(note_type)['overlap'].
+  • Stabilere Offsets (start/end) bei mehrfach vorkommenden Segmenten (inkrementelles Suchen + Fallback),
+  • optionale Felder window_left_ctx_len und window_right_ctx_est zur Diagnose,
+  • robustere Section-Pfadbehandlung.
 
-Felder (beibehalten aus 2.0.1):
+Felder (unverändert beibehalten):
-  - note_id, chunk_id, id (Alias), chunk_index, seq, path
+  note_id, chunk_id (Alias: id), chunk_index, seq, path,
-  - window (mit Overlap), text (ohne linkes Overlap)
+  window (mit linkem Overlap), text (Kernsegment), start, end,
-  - start, end (Offsets im gesamten Body)
+  overlap_left, overlap_right,
-  - overlap_left, overlap_right
+  type, title, tags, token_count?, section_title?, section_path?
-  - token_count?, section_title?, section_path?, type?, title?, tags?
-
-Kompatibilität:
-  - 'id' == 'chunk_id' als Alias
-  - Pfade bleiben relativ (keine führenden '/'), Backslashes → Slashes
-  - Robust für Chunk-Objekte oder Dicts; Fensterquelle: 'window'|'text'|'content'|'raw'
-
-Lizenz: MIT (projektintern)
 """
 from __future__ import annotations
 
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 
 try:
-    # Typgerechtes Overlap aus deiner Konfiguration holen
     from app.core.chunk_config import get_sizes as _get_sizes
 except Exception:
     def _get_sizes(_note_type: str):
-        # konservativer Default, falls Import fehlschlägt
         return {"overlap": (40, 60), "target": (250, 350), "max": 500}
 
 
-# ------------------------------- Utils ------------------------------- #
-
 def _get_attr_or_key(obj: Any, key: str, default=None):
     if obj is None:
         return default
@@ -48,7 +38,6 @@ def _get_attr_or_key(obj: Any, key: str, default=None):
     return getattr(obj, key, default)
 
 def _as_window_text(chunk: Any) -> str:
-    """Fenstertext robust lesen (bevorzugt echte Fenster, sonst Kern)."""
     for k in ("window", "text", "content", "raw"):
         v = _get_attr_or_key(chunk, k, None)
         if isinstance(v, str) and v:
@@ -67,14 +56,7 @@ def _normalize_rel_path(p: str) -> str:
         p = p[1:]
     return p
 
-
-# ---------------------- Overlap & Offsets ---------------------------- #
-
 def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int], str]:
-    """
-    Entfernt linkes Overlap aus echten Fenster-Strings.
-    Rückgabe: (segments, overlaps_left, reconstructed_text)
-    """
     segments: List[str] = []
     overlaps_left: List[int] = []
     reconstructed = ""
@@ -93,7 +75,6 @@ def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int
     return segments, overlaps_left, reconstructed
 
 def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
-    """Länge längsten Suffix(a), der Prefix(b) ist."""
     if not a or not b:
         return 0
     a1 = a[-max_probe:]
@@ -104,26 +85,18 @@ def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
             return k
     return 0
 
-
-# ----------------------------- Public API ---------------------------- #
-
 def make_chunk_payloads(
     frontmatter: Dict[str, Any],
     rel_path: str,
     chunks: Iterable[Union[Dict[str, Any], Any]],
     note_text: Optional[str] = None,
 ) -> List[Dict[str, Any]]:
-    """
-    Baut Payloads pro Chunk. Falls Fenster ohne Overlap geliefert werden,
-    erzeugen wir synthetische 'window'-Texte mit typgerechtem Overlap.
-    """
     note_id = str(frontmatter.get("id") or "").strip()
     note_type = str(frontmatter.get("type", "")).lower()
     note_title = frontmatter.get("title", None)
     note_tags = frontmatter.get("tags", None)
     rel_path = _normalize_rel_path(rel_path)
 
-    # 1) Rohdaten sammeln (so wie geliefert)
     chunks_list = list(chunks)
     raw_windows: List[str] = []
     seqs: List[int] = []
@@ -134,16 +107,13 @@ def make_chunk_payloads(
     any_explicit_window = False
 
     for idx, c in enumerate(chunks_list):
-        # Fensterquelle
         w = _get_attr_or_key(c, "window", None)
         if isinstance(w, str) and w:
             any_explicit_window = True
             raw_windows.append(w)
         else:
-            raw_windows.append(_as_window_text(c))  # 'text'|'content'|'raw' als Ersatz
+            raw_windows.append(_as_window_text(c))
-        # Ordnung
         seqs.append(_to_int(_get_attr_or_key(c, "seq", _get_attr_or_key(c, "chunk_index", idx)), idx))
-        # IDs, Tokens, Sektionen
         cid = _get_attr_or_key(c, "chunk_id", _get_attr_or_key(c, "id", None))
         ids_in.append(str(cid) if isinstance(cid, str) and cid else None)
         tc = _get_attr_or_key(c, "token_count", None)
@@ -151,14 +121,10 @@ def make_chunk_payloads(
         section_titles.append(_get_attr_or_key(c, "section_title", None))
         section_paths.append(_get_attr_or_key(c, "section_path", None))
 
-    # 2) Segmente & Overlaps bestimmen
     if any_explicit_window:
-        # Es existieren echte Fenster → dedupe, um Kernsegmente zu finden
         segments, overlaps_left, recon = _dedupe_windows_to_segments(raw_windows)
-        windows_final = raw_windows[:]  # bereits mit Overlap geliefert
+        windows_final = raw_windows[:]
     else:
-        # Keine echten Fenster → Segmente sind identisch zu "Fenstern" (bisher),
-        # wir erzeugen synthetische Fenster mit Overlap gemäß Typ
         segments = [w or "" for w in raw_windows]
         overlaps_left = []
         windows_final = []
@@ -171,19 +137,16 @@ def make_chunk_payloads(
 
         for i, seg in enumerate(segments):
             if i == 0:
-                # erstes Fenster: kein linker Kontext
                 windows_final.append(seg)
                 overlaps_left.append(0)
                 recon += seg
             else:
-                # synthetischer linker Kontext = Suffix des bisher rekonstruierten Texts
                 k = min(overlap_target, len(recon))
                 left_ctx = recon[-k:] if k > 0 else ""
                 windows_final.append(left_ctx + seg)
                 overlaps_left.append(k)
-                recon += seg  # Rekonstruktion bleibt kerntreu
+                recon += seg
 
-    # 3) overlap_right bestimmen
     overlaps_right: List[int] = []
     for i in range(len(windows_final)):
         if i + 1 < len(windows_final):
@@ -192,10 +155,8 @@ def make_chunk_payloads(
             ov = 0
         overlaps_right.append(ov)
 
-    # 4) start/end-Offsets (exakt via note_text, sonst kumulativ)
     starts: List[int] = [0] * len(segments)
     ends: List[int] = [0] * len(segments)
-    pos = 0
     if isinstance(note_text, str) and note_text:
         search_pos = 0
         for i, seg in enumerate(segments):
@@ -208,24 +169,25 @@ def make_chunk_payloads(
                 ends[i] = j + len(seg)
                 search_pos = ends[i]
             else:
-                # Fallback: kumulativ
+                # Fallback: naive fortlaufende Positionierung
-                starts[i] = pos
+                starts[i] = starts[i - 1] if i > 0 else 0
-                pos += len(seg)
+                ends[i] = starts[i] + len(seg)
-                ends[i] = pos
+                search_pos = ends[i]
     else:
+        pos = 0
         for i, seg in enumerate(segments):
             starts[i] = pos
             pos += len(seg)
             ends[i] = pos
 
-    # 5) Payload-Dicts
     payloads: List[Dict[str, Any]] = []
     for i, (win, seg) in enumerate(zip(windows_final, segments)):
         chunk_id = ids_in[i] or f"{note_id}#{i+1}"
+        left_len = max(0, len(win) - len(seg))
         pl: Dict[str, Any] = {
             "note_id": note_id,
             "chunk_id": chunk_id,
-            "id": chunk_id,  # Alias
+            "id": chunk_id,
             "chunk_index": i,
             "seq": seqs[i],
             "path": rel_path,
@@ -235,8 +197,8 @@ def make_chunk_payloads(
             "end": ends[i],
             "overlap_left": overlaps_left[i],
             "overlap_right": overlaps_right[i],
+            "window_left_ctx_len": left_len,
         }
-        # optionale Metadaten
         if note_type:
             pl["type"] = note_type
         if note_title is not None:
@@ -254,11 +216,8 @@ def make_chunk_payloads(
 
     return payloads
 
-
-# __main__ Demo (optional)
 if __name__ == "__main__":  # pragma: no cover
     fm = {"id": "demo", "title": "Demo", "type": "concept"}
-    # Beispiel ohne echte Fenster → erzeugt synthetische Overlaps
     chunks = [
         {"id": "demo#1", "text": "Alpha Beta Gamma"},
         {"id": "demo#2", "text": "Gamma Delta"},