app/core/chunk_payload.py aktualisiert

app/core/chunk_payload.py

@@ -1,158 +1,220 @@
+# app/core/chunk_payload.py
+# Line count: 214
+
 from __future__ import annotations
-from typing import Any, Dict, Iterable, List, Optional, Union
 
-# ---- Helpers ----
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
-def _coerce_float(val: Any) -> Optional[float]:
+
+
+def _get(obj: Any, key: str, default: Any = None) -> Any:
+    if obj is None:
+        return default
+    if hasattr(obj, key):
+        try:
+            val = getattr(obj, key)
+            return val if val is not None else default
+        except Exception:
+            pass
+    if isinstance(obj, dict):
+        if key in obj:
+            val = obj.get(key, default)
+            return val if val is not None else default
+    return default
+
+
+def _get_frontmatter(note: Any) -> Dict[str, Any]:
+    fm = _get(note, "frontmatter", None)
+    if isinstance(fm, dict):
+        return fm
+    meta = _get(note, "meta", None)
+    if isinstance(meta, dict) and isinstance(meta.get("frontmatter"), dict):
+        return meta["frontmatter"]
+    return {}
+
+
+def _get_from_frontmatter(fm: Dict[str, Any], key: str, default: Any = None) -> Any:
+    if not isinstance(fm, dict):
+        return default
+    if key in fm:
+        val = fm.get(key, default)
+        return val if val is not None else default
+    return default
+
+
+def _coerce_tags(val: Any) -> List[str]:
     if val is None:
-        return None
-    try:
-        if isinstance(val, (int, float)):
-            return float(val)
-        if isinstance(val, str):
-            v = val.strip()
-            if not v:
-                return None
-            return float(v.replace(",", "."))
-    except Exception:
-        return None
-    return None
-
-def _extract_weight(frontmatter: Dict[str, Any], explicit: Optional[float]) -> Optional[float]:
-    if explicit is not None:
-        return _coerce_float(explicit)
-    if frontmatter is None:
-        return None
-    if "retriever_weight" in frontmatter:
-        return _coerce_float(frontmatter.get("retriever_weight"))
-    # also accept nested style: retriever: { weight: 0.8 }
-    retriever = frontmatter.get("retriever")
-    if isinstance(retriever, dict) and "weight" in retriever:
-        return _coerce_float(retriever.get("weight"))
-    return None
-
-def _ensure_list(x: Any) -> List[Any]:
-    if x is None:
         return []
-    if isinstance(x, list):
+    if isinstance(val, list):
-        return x
+        return [str(x) for x in val]
-    return [x]
+    if isinstance(val, str):
+        parts = [t.strip() for t in val.split(",")]
+        return [p for p in parts if p]
+    return []
 
-def _resolve_note_id(frontmatter: Dict[str, Any], kw_note_id: Optional[str]) -> Optional[str]:
-    if kw_note_id:
-        return kw_note_id
-    if not isinstance(frontmatter, dict):
-        return None
-    return frontmatter.get("id") or frontmatter.get("note_id")
 
-def _base_fields(frontmatter: Dict[str, Any], note_id: Optional[str], path: str) -> Dict[str, Any]:
+def _resolve_retriever_weight(
-    title = None
+    fm: Dict[str, Any],
-    typ = None
+    explicit: Optional[float],
-    tags = None
+) -> Optional[float]:
-    if isinstance(frontmatter, dict):
+    if explicit is not None:
-        title = frontmatter.get("title")
+        return explicit
-        typ = frontmatter.get("type") or frontmatter.get("note_type")
+    val = _get_from_frontmatter(fm, "retriever_weight", None)
-        # tags can be list[str] or comma separated string
+    if isinstance(val, (int, float)):
-        tags = frontmatter.get("tags")
+        return float(val)
-        if isinstance(tags, str):
+    retr = fm.get("retriever")
-            tags = [t.strip() for t in tags.split(",") if t.strip()]
+    if isinstance(retr, dict):
+        v = retr.get("weight")
+        if isinstance(v, (int, float)):
+            return float(v)
+    return None
+
+
+def _resolve_note_fields(note: Any) -> Dict[str, Any]:
+    fm = _get_frontmatter(note)
+
+    note_id = _get_from_frontmatter(fm, "id", None)
+    if note_id is None:
+        note_id = _get(note, "note_id", None)
+    if note_id is None:
+        note_id = _get(note, "id", None)
+
+    title = _get_from_frontmatter(fm, "title", None)
+    if title is None:
+        title = _get(note, "title", None)
+
+    ntype = _get_from_frontmatter(fm, "type", None)
+    if ntype is None:
+        ntype = _get(note, "type", None)
+
+    tags = _get_from_frontmatter(fm, "tags", None)
+    if tags is None:
+        tags = _get(note, "tags", None)
+    tags = _coerce_tags(tags)
+
+    path = _get_from_frontmatter(fm, "path", None)
+    if path is None:
+        path = _get(note, "path", None)
+    if path is None:
+        path = _get(note, "source", None)
+    if path is None:
+        path = _get(note, "filepath", None)
+
     return {
         "note_id": note_id,
         "title": title,
-        "type": typ,
+        "type": ntype,
         "tags": tags,
-        "path": path or None,
+        "path": path,
+        "frontmatter": fm,
     }
 
-# ---- Public API ----
+
+def _extract_chunk_text_and_index(
+    chunk: Any,
+    fallback_index: int,
+) -> Tuple[str, int]:
+    """
+    Akzeptiert verschiedene Chunk-Formate:
+    - str (reiner Text)
+    - dict mit keys: text | window | body | content
+    - Objekt mit Attributen: text | window | body | content
+    - (text, idx) Tuple
+    """
+    # Tuple (text, idx)
+    if isinstance(chunk, tuple) and len(chunk) == 2 and isinstance(chunk[0], str):
+        txt, idx = chunk
+        try:
+            idx_int = int(idx)
+        except Exception:
+            idx_int = fallback_index
+        return txt, idx_int
+
+    # String
+    if isinstance(chunk, str):
+        return chunk, fallback_index
+
+    # Dict
+    if isinstance(chunk, dict):
+        txt = (
+            chunk.get("text")
+            or chunk.get("window")
+            or chunk.get("body")
+            or chunk.get("content")
+        )
+        if isinstance(txt, str):
+            idx = chunk.get("index")
+            try:
+                idx_int = int(idx) if idx is not None else fallback_index
+            except Exception:
+                idx_int = fallback_index
+            return txt, idx_int
+
+    # Objekt mit Attributen
+    for attr in ("text", "window", "body", "content"):
+        if hasattr(chunk, attr):
+            try:
+                txt = getattr(chunk, attr)
+            except Exception:
+                txt = None
+            if isinstance(txt, str):
+                # Optionale "index"-Quelle
+                idx = None
+                if hasattr(chunk, "index"):
+                    try:
+                        idx = getattr(chunk, "index")
+                    except Exception:
+                        idx = None
+                try:
+                    idx_int = int(idx) if idx is not None else fallback_index
+                except Exception:
+                    idx_int = fallback_index
+                return txt, idx_int
+
+    # Wenn nichts passt -> klarer Fehler
+    raise ValueError("Unsupported chunk format: cannot extract text/index")
+
+
 def make_chunk_payloads(
-    frontmatter: Dict[str, Any],
+    note: Any,
-    *args,
+    chunks: Iterable[Any],
-    note_id: Optional[str] = None,
+    *,
-    chunks: Optional[Iterable[Any]] = None,
-    path: str = "",
-    chunk_profile: Optional[str] = None,
     retriever_weight: Optional[float] = None,
-    **kwargs,
+    base_payload: Optional[Dict[str, Any]] = None,
 ) -> List[Dict[str, Any]]:
     """
-    Build chunk payload dictionaries for Qdrant.
+    Erzeugt Qdrant-Payloads für Chunk-Punkte.
-
+    - Kopiert Note-Metadaten (note_id/title/type/tags/path)
-    This function is intentionally permissive to stay compatible with older callers:
+    - Schreibt text + chunk_index je Chunk
-    - If `chunks` is a list of dictionaries that already contain payload-like fields,
+    - Setzt retriever_weight, wenn vorhanden/angegeben
-      those are augmented.
-    - If `chunks` is a list of strings, minimal payloads are created.
-    - If `chunks` is a list of dicts with keys like `text`, `window`, or `index`, they are normalized.
-
-    Always injects `retriever_weight` into each payload when available (from explicit arg or frontmatter).
     """
-    # Backward-compat for callers that might pass via kwargs
-    if chunks is None:
-        chunks = kwargs.get("payloads") or kwargs.get("pls") or kwargs.get("items") or kwargs.get("chunk_items")
-
-    note_id_resolved = _resolve_note_id(frontmatter, note_id)
-    weight = _extract_weight(frontmatter, retriever_weight)
-    base = _base_fields(frontmatter, note_id_resolved, path)
-
     out: List[Dict[str, Any]] = []
-    for idx, item in enumerate(_ensure_list(chunks)):
+    note_fields = _resolve_note_fields(note)
-        # Case A: already a full payload dict (heuristic: has 'text' or 'window' or 'note_id' keys)
+    fm = note_fields["frontmatter"]
-        if isinstance(item, dict) and ("text" in item or "window" in item or "note_id" in item):
+    rw = _resolve_retriever_weight(fm, retriever_weight)
-            pl = dict(item)  # shallow copy
-            # ensure base fields exist if missing
-            for k, v in base.items():
-                pl.setdefault(k, v)
-            # ensure chunk_index if not present
-            pl.setdefault("chunk_index", item.get("index", idx))
-            # inject retriever_weight
-            if weight is not None:
-                pl["retriever_weight"] = weight
-            out.append(pl)
-            continue
 
-        # Case B: item is a dict with nested 'payload'
+    # Basisfelder, die jeder Chunk tragen soll
-        if isinstance(item, dict) and "payload" in item and isinstance(item["payload"], dict):
+    common: Dict[str, Any] = {}
-            pl = dict(item["payload"])
+    if base_payload:
-            for k, v in base.items():
+        common.update({k: v for k, v in base_payload.items() if v is not None})
-                pl.setdefault(k, v)
-            pl.setdefault("chunk_index", pl.get("index", idx))
-            if weight is not None:
-                pl["retriever_weight"] = weight
-            out.append(pl)
-            continue
 
-        # Case C: item is a plain string -> treat as text (no window context)
+    if note_fields.get("note_id") is not None:
-        if isinstance(item, str):
+        common["note_id"] = note_fields["note_id"]
-            text_val = item
+    if note_fields.get("title") is not None:
-            pl = {
+        common["title"] = note_fields["title"]
-                **base,
+    if note_fields.get("type") is not None:
-                "chunk_index": idx,
+        common["type"] = note_fields["type"]
-                "text": text_val,
+    if note_fields.get("tags"):
-                "window": text_val,
+        common["tags"] = note_fields["tags"]
-            }
+    if note_fields.get("path") is not None:
-            if weight is not None:
+        common["path"] = note_fields["path"]
-                pl["retriever_weight"] = weight
+    if rw is not None:
-            out.append(pl)
+        common["retriever_weight"] = rw
-            continue
 
-        # Case D: item has 'text'/'window' under different names
+    for i, ch in enumerate(chunks):
-        if isinstance(item, dict):
+        text, idx = _extract_chunk_text_and_index(ch, i)
-            text_val = item.get("text") or item.get("body") or item.get("content") or ""
+        payload = dict(common)  # copy
-            window_val = item.get("window") or text_val
+        payload["chunk_index"] = idx
-            pl = {
+        payload["text"] = text
-                **base,
+        out.append(payload)
-                "chunk_index": item.get("chunk_index", item.get("index", idx)),
-                "text": text_val,
-                "window": window_val,
-            }
-            if weight is not None:
-                pl["retriever_weight"] = weight
-            out.append(pl)
-            continue
-
-        # Fallback: minimal payload
-        pl = {**base, "chunk_index": idx}
-        if weight is not None:
-            pl["retriever_weight"] = weight
-        out.append(pl)
 
     return out