app/core/note_payload.py aktualisiert
All checks were successful
Deploy mindnet to llm-node / deploy (push) Successful in 4s

This commit is contained in:
Lars 2025-12-04 12:37:31 +01:00
parent 00297fc354
commit f8eebd91cb

View File

@ -13,57 +13,95 @@ und `edge_defaults` werden in folgender Priorität bestimmt:
3) Registry-Defaults (config/types.yaml: defaults.*) 3) Registry-Defaults (config/types.yaml: defaults.*)
4) ENV-Defaults (MINDNET_DEFAULT_RETRIEVER_WEIGHT / MINDNET_DEFAULT_CHUNK_PROFILE) 4) ENV-Defaults (MINDNET_DEFAULT_RETRIEVER_WEIGHT / MINDNET_DEFAULT_CHUNK_PROFILE)
""" """
from __future__ import annotations from __future__ import annotations
from typing import Any, Dict, Tuple, Optional from typing import Any, Dict, Tuple, Optional
import os, json, pathlib import os
import json
import pathlib
try: try:
import yaml import yaml # type: ignore
except Exception: except Exception:
yaml = None yaml = None
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
def _as_dict(x) -> Dict[str, Any]: def _as_dict(x) -> Dict[str, Any]:
"""Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen."""
if isinstance(x, dict): if isinstance(x, dict):
return dict(x) return dict(x)
# Versuche, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen
out: Dict[str, Any] = {} out: Dict[str, Any] = {}
# bekannte Attribute übernehmen # bekannte Attribute übernehmen, sofern vorhanden
for attr in ("frontmatter", "body", "id", "note_id", "title", "path", "tags", "type", "created", "modified", "date"): for attr in (
"frontmatter",
"body",
"id",
"note_id",
"title",
"path",
"tags",
"type",
"created",
"modified",
"date",
):
if hasattr(x, attr): if hasattr(x, attr):
val = getattr(x, attr) val = getattr(x, attr)
if val is not None: if val is not None:
out[attr] = val out[attr] = val
# Fallback: wenn immer noch leer, raw speichern # Fallback: wenn immer noch leer, raw speichern
if not out: if not out:
out["raw"] = str(x) out["raw"] = str(x)
return out return out
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]: def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]:
"""Extrahiert optionale Zusatzargumente wie path und types_cfg."""
path = kwargs.get("path") or (args[0] if args else None) path = kwargs.get("path") or (args[0] if args else None)
types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None
return path, types_cfg return path, types_cfg
def _env_float(name: str, default: float) -> float: def _env_float(name: str, default: float) -> float:
"""Liest einen Float-Wert aus der Umgebung, mit robustem Fallback."""
try: try:
return float(os.environ.get(name, default)) return float(os.environ.get(name, default))
except Exception: except Exception:
return default return default
def _ensure_list(x) -> list: def _ensure_list(x) -> list:
if x is None: return [] """Garantiert eine String-Liste."""
if isinstance(x, list): return [str(i) for i in x] if x is None:
if isinstance(x, (set, tuple)): return [str(i) for i in x] return []
if isinstance(x, list):
return [str(i) for i in x]
if isinstance(x, (set, tuple)):
return [str(i) for i in x]
return [str(x)] return [str(x)]
# ---- Registry laden --------------------------------------------------------
# ---------------------------------------------------------------------------
# Type-Registry laden
# ---------------------------------------------------------------------------
def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict: def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict:
"""Lädt die Type-Registry aus YAML/JSON oder nutzt ein explizit übergebenes Dict."""
if explicit_cfg and isinstance(explicit_cfg, dict): if explicit_cfg and isinstance(explicit_cfg, dict):
return explicit_cfg return explicit_cfg
path = os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml" path = os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
if not os.path.isfile(path) or yaml is None: if not os.path.isfile(path) or yaml is None:
return {} return {}
try: try:
with open(path, "r", encoding="utf-8") as f: with open(path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {} data = yaml.safe_load(f) or {}
@ -71,13 +109,17 @@ def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict:
except Exception: except Exception:
return {} return {}
def _cfg_for_type(note_type: str, reg: dict) -> dict: def _cfg_for_type(note_type: str, reg: dict) -> dict:
"""Liefert die Konfiguration für einen konkreten Notiztyp aus der Registry."""
if not isinstance(reg, dict): if not isinstance(reg, dict):
return {} return {}
types = reg.get("types") if isinstance(reg.get("types"), dict) else reg types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
return types.get(note_type, {}) if isinstance(types, dict) else {} return types.get(note_type, {}) if isinstance(types, dict) else {}
def _cfg_defaults(reg: dict) -> dict: def _cfg_defaults(reg: dict) -> dict:
"""Liefert den Default-Block aus der Registry (defaults/global)."""
if not isinstance(reg, dict): if not isinstance(reg, dict):
return {} return {}
for key in ("defaults", "default", "global"): for key in ("defaults", "default", "global"):
@ -86,9 +128,24 @@ def _cfg_defaults(reg: dict) -> dict:
return v return v
return {} return {}
# ---- Haupt-API -------------------------------------------------------------
# ---------------------------------------------------------------------------
# Haupt-API
# ---------------------------------------------------------------------------
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]: def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
"""
Baut das Note-Payload für mindnet_notes auf.
Erwartete Felder im Payload:
- note_id: stabile ID aus Frontmatter (id) oder Note-Objekt
- title: Titel der Notiz
- type: Notiztyp (z. B. concept, project, journal, ...)
- path: Pfad im Vault
- retriever_weight: effektives Gewicht für den Retriever
- chunk_profile: Profil für Chunking (short|medium|long|default|...)
- edge_defaults: Liste von Kanten-Typen, die als Defaults gelten
"""
n = _as_dict(note) n = _as_dict(note)
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs) path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
reg = _load_types_config(types_cfg_explicit) reg = _load_types_config(types_cfg_explicit)
@ -100,33 +157,49 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
cfg_type = _cfg_for_type(note_type, reg) cfg_type = _cfg_for_type(note_type, reg)
cfg_def = _cfg_defaults(reg) cfg_def = _cfg_defaults(reg)
# --- retriever_weight: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0) default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
retriever_weight = fm.get("retriever_weight") retriever_weight = fm.get("retriever_weight")
if retriever_weight is None: if retriever_weight is None:
retriever_weight = cfg_type.get("retriever_weight", cfg_def.get("retriever_weight", default_rw)) retriever_weight = cfg_type.get(
"retriever_weight",
cfg_def.get("retriever_weight", default_rw),
)
try: try:
retriever_weight = float(retriever_weight) retriever_weight = float(retriever_weight)
except Exception: except Exception:
retriever_weight = default_rw retriever_weight = default_rw
# --- chunk_profile: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
chunk_profile = fm.get("chunk_profile") chunk_profile = fm.get("chunk_profile")
if chunk_profile is None: if chunk_profile is None:
chunk_profile = cfg_type.get("chunk_profile", cfg_def.get("chunk_profile", os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE", "medium"))) chunk_profile = cfg_type.get(
"chunk_profile",
cfg_def.get(
"chunk_profile",
os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE", "medium"),
),
)
if not isinstance(chunk_profile, str): if not isinstance(chunk_profile, str):
chunk_profile = "medium" chunk_profile = "medium"
# --- edge_defaults: Frontmatter > Typ-Config > Registry-Defaults ---
edge_defaults = fm.get("edge_defaults") edge_defaults = fm.get("edge_defaults")
if edge_defaults is None: if edge_defaults is None:
edge_defaults = cfg_type.get("edge_defaults", cfg_def.get("edge_defaults", [])) edge_defaults = cfg_type.get(
"edge_defaults",
cfg_def.get("edge_defaults", []),
)
edge_defaults = _ensure_list(edge_defaults) edge_defaults = _ensure_list(edge_defaults)
# --- Basis-Metadaten (IDs, Titel, Pfad) ---
note_id = n.get("note_id") or n.get("id") or fm.get("id") note_id = n.get("note_id") or n.get("id") or fm.get("id")
title = n.get("title") or fm.get("title") or "" title = n.get("title") or fm.get("title") or ""
path = n.get("path") or path_arg path = n.get("path") or path_arg
if isinstance(path, pathlib.Path): if isinstance(path, pathlib.Path):
path = str(path) path = str(path)
payload = { payload: Dict[str, Any] = {
"note_id": note_id, "note_id": note_id,
"title": title, "title": title,
"type": note_type, "type": note_type,
@ -136,35 +209,18 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
"edge_defaults": edge_defaults, "edge_defaults": edge_defaults,
} }
# Tags / Keywords übernehmen
tags = fm.get("tags") or fm.get("keywords") or n.get("tags") tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
if tags: payload["tags"] = _ensure_list(tags) if tags:
payload["tags"] = _ensure_list(tags)
# Zeitliche Metadaten (sofern vorhanden)
for k in ("created", "modified", "date"): for k in ("created", "modified", "date"):
v = fm.get(k) or n.get(k) v = fm.get(k) or n.get(k)
if v: payload[k] = str(v) if v:
payload[k] = str(v)
# JSON-Roundtrip zur harten Validierung (ASCII beibehalten) # JSON-Roundtrip zur harten Validierung (ASCII beibehalten)
json.loads(json.dumps(payload, ensure_ascii=False)) json.loads(json.dumps(payload, ensure_ascii=False))
return payload return payload
``` :contentReference[oaicite:0]{index=0}
Kurz zur Einordnung:
- `retriever_weight`, `chunk_profile`, `edge_defaults` kommen jetzt **immer** aus dieser Priorität:
1. Frontmatter der Note (`retriever_weight`, `chunk_profile`, `edge_defaults`)
2. `config/types.yaml` `types.<type>.retriever_weight` / `chunk_profile` / `edge_defaults`
3. `config/types.yaml` `defaults.*`
4. ENV (`MINDNET_DEFAULT_RETRIEVER_WEIGHT`, `MINDNET_DEFAULT_CHUNK_PROFILE`)
- Für **Notes** (Collection `mindnet_notes`) ist damit alles korrekt verdrahtet.
Dass in deinem `payload_dryrun` die Chunks noch überall `retriever_weight: 1.0` und `chunk_profile: "default"` haben, liegt nicht an dieser Datei, sondern an `chunk_payload.py`. Die nächste saubere Maßnahme ist:
1. In `app/core/chunk_payload.py` dieselbe Resolve-Logik (Types-Registry + Defaults) ergänzen.
2. Danach:
- `scripts.reset_qdrant` laufen lassen,
- Test-Vault neu importieren,
- `scripts.payload_dryrun` erneut prüfen, ob die Chunk-Werte jetzt typabhängig sind.
Wenn du magst, können wir im nächsten Schritt direkt `chunk_payload.py` analog umbauen, so dass Notes und Chunks konsistent aus `types.yaml` lesen.
::contentReference[oaicite:1]{index=1}