From f8eebd91cbb44f6bc8eb8258b3ac3fd086341e94 Mon Sep 17 00:00:00 2001 From: Lars Date: Thu, 4 Dec 2025 12:37:31 +0100 Subject: [PATCH] app/core/note_payload.py aktualisiert --- app/core/note_payload.py | 138 +++++++++++++++++++++++++++------------ 1 file changed, 97 insertions(+), 41 deletions(-) diff --git a/app/core/note_payload.py b/app/core/note_payload.py index 2688672..1c5e6bc 100644 --- a/app/core/note_payload.py +++ b/app/core/note_payload.py @@ -13,57 +13,95 @@ und `edge_defaults` werden in folgender Priorität bestimmt: 3) Registry-Defaults (config/types.yaml: defaults.*) 4) ENV-Defaults (MINDNET_DEFAULT_RETRIEVER_WEIGHT / MINDNET_DEFAULT_CHUNK_PROFILE) """ + from __future__ import annotations from typing import Any, Dict, Tuple, Optional -import os, json, pathlib +import os +import json +import pathlib try: - import yaml + import yaml # type: ignore except Exception: yaml = None + +# --------------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------------- + def _as_dict(x) -> Dict[str, Any]: + """Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen.""" if isinstance(x, dict): return dict(x) - # Versuche, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen + out: Dict[str, Any] = {} - # bekannte Attribute übernehmen - for attr in ("frontmatter", "body", "id", "note_id", "title", "path", "tags", "type", "created", "modified", "date"): + # bekannte Attribute übernehmen, sofern vorhanden + for attr in ( + "frontmatter", + "body", + "id", + "note_id", + "title", + "path", + "tags", + "type", + "created", + "modified", + "date", + ): if hasattr(x, attr): val = getattr(x, attr) if val is not None: out[attr] = val + # Fallback: wenn immer noch leer, raw speichern if not out: out["raw"] = str(x) + return out + def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]: + """Extrahiert optionale Zusatzargumente wie path und types_cfg.""" path = kwargs.get("path") or (args[0] if args else None) types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None return path, types_cfg + def _env_float(name: str, default: float) -> float: + """Liest einen Float-Wert aus der Umgebung, mit robustem Fallback.""" try: return float(os.environ.get(name, default)) except Exception: return default + def _ensure_list(x) -> list: - if x is None: return [] - if isinstance(x, list): return [str(i) for i in x] - if isinstance(x, (set, tuple)): return [str(i) for i in x] + """Garantiert eine String-Liste.""" + if x is None: + return [] + if isinstance(x, list): + return [str(i) for i in x] + if isinstance(x, (set, tuple)): + return [str(i) for i in x] return [str(x)] -# ---- Registry laden -------------------------------------------------------- + +# --------------------------------------------------------------------------- +# Type-Registry laden +# --------------------------------------------------------------------------- def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict: + """Lädt die Type-Registry aus YAML/JSON oder nutzt ein explizit übergebenes Dict.""" if explicit_cfg and isinstance(explicit_cfg, dict): return explicit_cfg + path = os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml" if not os.path.isfile(path) or yaml is None: return {} + try: with open(path, "r", encoding="utf-8") as f: data = yaml.safe_load(f) or {} @@ -71,13 +109,17 @@ def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict: except Exception: return {} + def _cfg_for_type(note_type: str, reg: dict) -> dict: + """Liefert die Konfiguration für einen konkreten Notiztyp aus der Registry.""" if not isinstance(reg, dict): return {} types = reg.get("types") if isinstance(reg.get("types"), dict) else reg return types.get(note_type, {}) if isinstance(types, dict) else {} + def _cfg_defaults(reg: dict) -> dict: + """Liefert den Default-Block aus der Registry (defaults/global).""" if not isinstance(reg, dict): return {} for key in ("defaults", "default", "global"): @@ -86,9 +128,24 @@ def _cfg_defaults(reg: dict) -> dict: return v return {} -# ---- Haupt-API ------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# Haupt-API +# --------------------------------------------------------------------------- def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]: + """ + Baut das Note-Payload für mindnet_notes auf. + + Erwartete Felder im Payload: + - note_id: stabile ID aus Frontmatter (id) oder Note-Objekt + - title: Titel der Notiz + - type: Notiztyp (z. B. concept, project, journal, ...) + - path: Pfad im Vault + - retriever_weight: effektives Gewicht für den Retriever + - chunk_profile: Profil für Chunking (short|medium|long|default|...) + - edge_defaults: Liste von Kanten-Typen, die als Defaults gelten + """ n = _as_dict(note) path_arg, types_cfg_explicit = _pick_args(*args, **kwargs) reg = _load_types_config(types_cfg_explicit) @@ -98,35 +155,51 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]: note_type = str(fm_type) cfg_type = _cfg_for_type(note_type, reg) - cfg_def = _cfg_defaults(reg) + cfg_def = _cfg_defaults(reg) + # --- retriever_weight: Frontmatter > Typ-Config > Registry-Defaults > ENV --- default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0) retriever_weight = fm.get("retriever_weight") if retriever_weight is None: - retriever_weight = cfg_type.get("retriever_weight", cfg_def.get("retriever_weight", default_rw)) + retriever_weight = cfg_type.get( + "retriever_weight", + cfg_def.get("retriever_weight", default_rw), + ) try: retriever_weight = float(retriever_weight) except Exception: retriever_weight = default_rw + # --- chunk_profile: Frontmatter > Typ-Config > Registry-Defaults > ENV --- chunk_profile = fm.get("chunk_profile") if chunk_profile is None: - chunk_profile = cfg_type.get("chunk_profile", cfg_def.get("chunk_profile", os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE", "medium"))) + chunk_profile = cfg_type.get( + "chunk_profile", + cfg_def.get( + "chunk_profile", + os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE", "medium"), + ), + ) if not isinstance(chunk_profile, str): chunk_profile = "medium" + # --- edge_defaults: Frontmatter > Typ-Config > Registry-Defaults --- edge_defaults = fm.get("edge_defaults") if edge_defaults is None: - edge_defaults = cfg_type.get("edge_defaults", cfg_def.get("edge_defaults", [])) + edge_defaults = cfg_type.get( + "edge_defaults", + cfg_def.get("edge_defaults", []), + ) edge_defaults = _ensure_list(edge_defaults) + # --- Basis-Metadaten (IDs, Titel, Pfad) --- note_id = n.get("note_id") or n.get("id") or fm.get("id") title = n.get("title") or fm.get("title") or "" path = n.get("path") or path_arg if isinstance(path, pathlib.Path): path = str(path) - payload = { + payload: Dict[str, Any] = { "note_id": note_id, "title": title, "type": note_type, @@ -136,35 +209,18 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]: "edge_defaults": edge_defaults, } + # Tags / Keywords übernehmen tags = fm.get("tags") or fm.get("keywords") or n.get("tags") - if tags: payload["tags"] = _ensure_list(tags) - for k in ("created","modified","date"): + if tags: + payload["tags"] = _ensure_list(tags) + + # Zeitliche Metadaten (sofern vorhanden) + for k in ("created", "modified", "date"): v = fm.get(k) or n.get(k) - if v: payload[k] = str(v) + if v: + payload[k] = str(v) # JSON-Roundtrip zur harten Validierung (ASCII beibehalten) json.loads(json.dumps(payload, ensure_ascii=False)) + return payload -``` :contentReference[oaicite:0]{index=0} - -Kurz zur Einordnung: - -- `retriever_weight`, `chunk_profile`, `edge_defaults` kommen jetzt **immer** aus dieser Priorität: - - 1. Frontmatter der Note (`retriever_weight`, `chunk_profile`, `edge_defaults`) - 2. `config/types.yaml` → `types..retriever_weight` / `chunk_profile` / `edge_defaults` - 3. `config/types.yaml` → `defaults.*` - 4. ENV (`MINDNET_DEFAULT_RETRIEVER_WEIGHT`, `MINDNET_DEFAULT_CHUNK_PROFILE`) - -- Für **Notes** (Collection `mindnet_notes`) ist damit alles korrekt verdrahtet. - -Dass in deinem `payload_dryrun` die Chunks noch überall `retriever_weight: 1.0` und `chunk_profile: "default"` haben, liegt nicht an dieser Datei, sondern an `chunk_payload.py`. Die nächste saubere Maßnahme ist: - -1. In `app/core/chunk_payload.py` dieselbe Resolve-Logik (Types-Registry + Defaults) ergänzen. -2. Danach: - - `scripts.reset_qdrant` laufen lassen, - - Test-Vault neu importieren, - - `scripts.payload_dryrun` erneut prüfen, ob die Chunk-Werte jetzt typabhängig sind. - -Wenn du magst, können wir im nächsten Schritt direkt `chunk_payload.py` analog umbauen, so dass Notes und Chunks konsistent aus `types.yaml` lesen. -::contentReference[oaicite:1]{index=1}