""" FILE: app/core/note_payload.py DESCRIPTION: Baut das JSON-Objekt. FEATURES: 1. Multi-Hash: Berechnet immer 'body' AND 'full' Hashes für flexible Change Detection. 2. Config-Fix: Liest korrekt 'chunking_profile' aus types.yaml (statt Legacy 'chunk_profile'). VERSION: 2.3.0 STATUS: Active DEPENDENCIES: yaml, os, json, pathlib, hashlib EXTERNAL_CONFIG: config/types.yaml """ from __future__ import annotations from typing import Any, Dict, Tuple, Optional import os import json import pathlib import hashlib try: import yaml # type: ignore except Exception: yaml = None # --------------------------------------------------------------------------- # Helper # --------------------------------------------------------------------------- def _as_dict(x) -> Dict[str, Any]: """Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen.""" if isinstance(x, dict): return dict(x) out: Dict[str, Any] = {} for attr in ( "frontmatter", "body", "id", "note_id", "title", "path", "tags", "type", "created", "modified", "date", ): if hasattr(x, attr): val = getattr(x, attr) if val is not None: out[attr] = val if not out: out["raw"] = str(x) return out def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]: path = kwargs.get("path") or (args[0] if args else None) types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None return path, types_cfg def _env_float(name: str, default: float) -> float: try: return float(os.environ.get(name, default)) except Exception: return default def _ensure_list(x) -> list: if x is None: return [] if isinstance(x, list): return [str(i) for i in x] if isinstance(x, (set, tuple)): return [str(i) for i in x] return [str(x)] # --- Hash Logic --- def _compute_hash(content: str) -> str: """Berechnet einen SHA-256 Hash für den gegebenen String.""" if not content: return "" return hashlib.sha256(content.encode("utf-8")).hexdigest() def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str: """ Stellt den String zusammen, der gehasht werden soll. """ body = str(n.get("body") or "") if mode == "body": return body if mode == "full": fm = n.get("frontmatter") or {} # Wichtig: Sortierte Keys für deterministisches Verhalten! # Wir nehmen alle steuernden Metadaten auf meta_parts = [] # Hier checken wir keys, die eine Neu-Indizierung rechtfertigen würden for k in sorted(["title", "type", "status", "tags", "chunking_profile", "chunk_profile", "retriever_weight"]): val = fm.get(k) if val is not None: meta_parts.append(f"{k}:{val}") meta_str = "|".join(meta_parts) return f"{meta_str}||{body}" return body # --------------------------------------------------------------------------- # Type-Registry laden # --------------------------------------------------------------------------- def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict: if explicit_cfg and isinstance(explicit_cfg, dict): return explicit_cfg path = os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml" if not os.path.isfile(path) or yaml is None: return {} try: with open(path, "r", encoding="utf-8") as f: data = yaml.safe_load(f) or {} return data if isinstance(data, dict) else {} except Exception: return {} def _cfg_for_type(note_type: str, reg: dict) -> dict: if not isinstance(reg, dict): return {} types = reg.get("types") if isinstance(reg.get("types"), dict) else reg return types.get(note_type, {}) if isinstance(types, dict) else {} def _cfg_defaults(reg: dict) -> dict: if not isinstance(reg, dict): return {} for key in ("defaults", "default", "global"): v = reg.get(key) if isinstance(v, dict): return v return {} # --------------------------------------------------------------------------- # Haupt-API # --------------------------------------------------------------------------- def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]: """ Baut das Note-Payload für mindnet_notes auf. Inkludiert Hash-Berechnung (Body & Full) und korrigierte Config-Lookups. """ n = _as_dict(note) path_arg, types_cfg_explicit = _pick_args(*args, **kwargs) reg = _load_types_config(types_cfg_explicit) # Hash Config (Parameter für Source/Normalize, Mode ist hardcoded auf 'beide') hash_source = kwargs.get("hash_source", "parsed") hash_normalize = kwargs.get("hash_normalize", "canonical") fm = n.get("frontmatter") or {} fm_type = fm.get("type") or n.get("type") or "concept" note_type = str(fm_type) cfg_type = _cfg_for_type(note_type, reg) cfg_def = _cfg_defaults(reg) # --- retriever_weight --- default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0) retriever_weight = fm.get("retriever_weight") if retriever_weight is None: retriever_weight = cfg_type.get( "retriever_weight", cfg_def.get("retriever_weight", default_rw), ) try: retriever_weight = float(retriever_weight) except Exception: retriever_weight = default_rw # --- chunk_profile (FIXED LOGIC) --- # 1. Frontmatter Override (beide Schreibweisen erlaubt) chunk_profile = fm.get("chunking_profile") or fm.get("chunk_profile") # 2. Type Config (Korrekter Key 'chunking_profile' aus types.yaml) if chunk_profile is None: chunk_profile = cfg_type.get("chunking_profile") # 3. Default Config (Fallback auf sliding_standard statt medium) if chunk_profile is None: chunk_profile = cfg_def.get("chunking_profile", "sliding_standard") # 4. Safety Fallback if not isinstance(chunk_profile, str) or not chunk_profile: chunk_profile = "sliding_standard" # --- edge_defaults --- edge_defaults = fm.get("edge_defaults") if edge_defaults is None: edge_defaults = cfg_type.get( "edge_defaults", cfg_def.get("edge_defaults", []), ) edge_defaults = _ensure_list(edge_defaults) # --- Basis-Metadaten --- note_id = n.get("note_id") or n.get("id") or fm.get("id") title = n.get("title") or fm.get("title") or "" path = n.get("path") or path_arg if isinstance(path, pathlib.Path): path = str(path) payload: Dict[str, Any] = { "note_id": note_id, "title": title, "type": note_type, "path": path or "", "retriever_weight": retriever_weight, "chunk_profile": chunk_profile, "edge_defaults": edge_defaults, "hashes": {} # Init Hash Dict } # --- MULTI-HASH CALCULATION (Strategy Decoupling) --- # Wir berechnen immer BEIDE Strategien und speichern sie. # ingestion.py entscheidet dann anhand der ENV-Variable, welcher verglichen wird. modes_to_calc = ["body", "full"] for mode in modes_to_calc: content_to_hash = _get_hash_source_content(n, mode) computed_hash = _compute_hash(content_to_hash) # Key Schema: mode:source:normalize (z.B. "full:parsed:canonical") key = f"{mode}:{hash_source}:{hash_normalize}" payload["hashes"][key] = computed_hash # Tags / Keywords tags = fm.get("tags") or fm.get("keywords") or n.get("tags") if tags: payload["tags"] = _ensure_list(tags) # Aliases aliases = fm.get("aliases") if aliases: payload["aliases"] = _ensure_list(aliases) # Zeit for k in ("created", "modified", "date"): v = fm.get(k) or n.get(k) if v: payload[k] = str(v) # Fulltext if "body" in n and n["body"]: payload["fulltext"] = str(n["body"]) # JSON Validation json.loads(json.dumps(payload, ensure_ascii=False)) return payload