268 lines
8.1 KiB
Python
268 lines
8.1 KiB
Python
"""
|
|
FILE: app/core/note_payload.py
|
|
DESCRIPTION: Baut das JSON-Objekt.
|
|
FEATURES:
|
|
1. Multi-Hash: Berechnet immer 'body' AND 'full' Hashes für flexible Change Detection.
|
|
2. Config-Fix: Liest korrekt 'chunking_profile' aus types.yaml (statt Legacy 'chunk_profile').
|
|
VERSION: 2.3.0
|
|
STATUS: Active
|
|
DEPENDENCIES: yaml, os, json, pathlib, hashlib
|
|
EXTERNAL_CONFIG: config/types.yaml
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Dict, Tuple, Optional
|
|
import os
|
|
import json
|
|
import pathlib
|
|
import hashlib
|
|
|
|
try:
|
|
import yaml # type: ignore
|
|
except Exception:
|
|
yaml = None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _as_dict(x) -> Dict[str, Any]:
|
|
"""Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen."""
|
|
if isinstance(x, dict):
|
|
return dict(x)
|
|
|
|
out: Dict[str, Any] = {}
|
|
for attr in (
|
|
"frontmatter",
|
|
"body",
|
|
"id",
|
|
"note_id",
|
|
"title",
|
|
"path",
|
|
"tags",
|
|
"type",
|
|
"created",
|
|
"modified",
|
|
"date",
|
|
):
|
|
if hasattr(x, attr):
|
|
val = getattr(x, attr)
|
|
if val is not None:
|
|
out[attr] = val
|
|
|
|
if not out:
|
|
out["raw"] = str(x)
|
|
|
|
return out
|
|
|
|
|
|
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]:
|
|
path = kwargs.get("path") or (args[0] if args else None)
|
|
types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None
|
|
return path, types_cfg
|
|
|
|
|
|
def _env_float(name: str, default: float) -> float:
|
|
try:
|
|
return float(os.environ.get(name, default))
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def _ensure_list(x) -> list:
|
|
if x is None:
|
|
return []
|
|
if isinstance(x, list):
|
|
return [str(i) for i in x]
|
|
if isinstance(x, (set, tuple)):
|
|
return [str(i) for i in x]
|
|
return [str(x)]
|
|
|
|
# --- Hash Logic ---
|
|
def _compute_hash(content: str) -> str:
|
|
"""Berechnet einen SHA-256 Hash für den gegebenen String."""
|
|
if not content:
|
|
return ""
|
|
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
|
|
def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str:
|
|
"""
|
|
Stellt den String zusammen, der gehasht werden soll.
|
|
"""
|
|
body = str(n.get("body") or "")
|
|
|
|
if mode == "body":
|
|
return body
|
|
|
|
if mode == "full":
|
|
fm = n.get("frontmatter") or {}
|
|
# Wichtig: Sortierte Keys für deterministisches Verhalten!
|
|
# Wir nehmen alle steuernden Metadaten auf
|
|
meta_parts = []
|
|
# Hier checken wir keys, die eine Neu-Indizierung rechtfertigen würden
|
|
for k in sorted(["title", "type", "status", "tags", "chunking_profile", "chunk_profile", "retriever_weight"]):
|
|
val = fm.get(k)
|
|
if val is not None:
|
|
meta_parts.append(f"{k}:{val}")
|
|
|
|
meta_str = "|".join(meta_parts)
|
|
return f"{meta_str}||{body}"
|
|
|
|
return body
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Type-Registry laden
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict:
|
|
if explicit_cfg and isinstance(explicit_cfg, dict):
|
|
return explicit_cfg
|
|
|
|
path = os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
|
|
if not os.path.isfile(path) or yaml is None:
|
|
return {}
|
|
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
data = yaml.safe_load(f) or {}
|
|
return data if isinstance(data, dict) else {}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _cfg_for_type(note_type: str, reg: dict) -> dict:
|
|
if not isinstance(reg, dict):
|
|
return {}
|
|
types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
|
|
return types.get(note_type, {}) if isinstance(types, dict) else {}
|
|
|
|
|
|
def _cfg_defaults(reg: dict) -> dict:
|
|
if not isinstance(reg, dict):
|
|
return {}
|
|
for key in ("defaults", "default", "global"):
|
|
v = reg.get(key)
|
|
if isinstance(v, dict):
|
|
return v
|
|
return {}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Haupt-API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
|
"""
|
|
Baut das Note-Payload für mindnet_notes auf.
|
|
Inkludiert Hash-Berechnung (Body & Full) und korrigierte Config-Lookups.
|
|
"""
|
|
n = _as_dict(note)
|
|
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
|
|
reg = _load_types_config(types_cfg_explicit)
|
|
|
|
# Hash Config (Parameter für Source/Normalize, Mode ist hardcoded auf 'beide')
|
|
hash_source = kwargs.get("hash_source", "parsed")
|
|
hash_normalize = kwargs.get("hash_normalize", "canonical")
|
|
|
|
fm = n.get("frontmatter") or {}
|
|
fm_type = fm.get("type") or n.get("type") or "concept"
|
|
note_type = str(fm_type)
|
|
|
|
cfg_type = _cfg_for_type(note_type, reg)
|
|
cfg_def = _cfg_defaults(reg)
|
|
|
|
# --- retriever_weight ---
|
|
default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
|
|
retriever_weight = fm.get("retriever_weight")
|
|
if retriever_weight is None:
|
|
retriever_weight = cfg_type.get(
|
|
"retriever_weight",
|
|
cfg_def.get("retriever_weight", default_rw),
|
|
)
|
|
try:
|
|
retriever_weight = float(retriever_weight)
|
|
except Exception:
|
|
retriever_weight = default_rw
|
|
|
|
# --- chunk_profile (FIXED LOGIC) ---
|
|
# 1. Frontmatter Override (beide Schreibweisen erlaubt)
|
|
chunk_profile = fm.get("chunking_profile") or fm.get("chunk_profile")
|
|
|
|
# 2. Type Config (Korrekter Key 'chunking_profile' aus types.yaml)
|
|
if chunk_profile is None:
|
|
chunk_profile = cfg_type.get("chunking_profile")
|
|
|
|
# 3. Default Config (Fallback auf sliding_standard statt medium)
|
|
if chunk_profile is None:
|
|
chunk_profile = cfg_def.get("chunking_profile", "sliding_standard")
|
|
|
|
# 4. Safety Fallback
|
|
if not isinstance(chunk_profile, str) or not chunk_profile:
|
|
chunk_profile = "sliding_standard"
|
|
|
|
# --- edge_defaults ---
|
|
edge_defaults = fm.get("edge_defaults")
|
|
if edge_defaults is None:
|
|
edge_defaults = cfg_type.get(
|
|
"edge_defaults",
|
|
cfg_def.get("edge_defaults", []),
|
|
)
|
|
edge_defaults = _ensure_list(edge_defaults)
|
|
|
|
# --- Basis-Metadaten ---
|
|
note_id = n.get("note_id") or n.get("id") or fm.get("id")
|
|
title = n.get("title") or fm.get("title") or ""
|
|
path = n.get("path") or path_arg
|
|
if isinstance(path, pathlib.Path):
|
|
path = str(path)
|
|
|
|
payload: Dict[str, Any] = {
|
|
"note_id": note_id,
|
|
"title": title,
|
|
"type": note_type,
|
|
"path": path or "",
|
|
"retriever_weight": retriever_weight,
|
|
"chunk_profile": chunk_profile,
|
|
"edge_defaults": edge_defaults,
|
|
"hashes": {} # Init Hash Dict
|
|
}
|
|
|
|
# --- MULTI-HASH CALCULATION (Strategy Decoupling) ---
|
|
# Wir berechnen immer BEIDE Strategien und speichern sie.
|
|
# ingestion.py entscheidet dann anhand der ENV-Variable, welcher verglichen wird.
|
|
modes_to_calc = ["body", "full"]
|
|
|
|
for mode in modes_to_calc:
|
|
content_to_hash = _get_hash_source_content(n, mode)
|
|
computed_hash = _compute_hash(content_to_hash)
|
|
# Key Schema: mode:source:normalize (z.B. "full:parsed:canonical")
|
|
key = f"{mode}:{hash_source}:{hash_normalize}"
|
|
payload["hashes"][key] = computed_hash
|
|
|
|
# Tags / Keywords
|
|
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
|
|
if tags:
|
|
payload["tags"] = _ensure_list(tags)
|
|
|
|
# Aliases
|
|
aliases = fm.get("aliases")
|
|
if aliases:
|
|
payload["aliases"] = _ensure_list(aliases)
|
|
|
|
# Zeit
|
|
for k in ("created", "modified", "date"):
|
|
v = fm.get(k) or n.get(k)
|
|
if v:
|
|
payload[k] = str(v)
|
|
|
|
# Fulltext
|
|
if "body" in n and n["body"]:
|
|
payload["fulltext"] = str(n["body"])
|
|
|
|
# JSON Validation
|
|
json.loads(json.dumps(payload, ensure_ascii=False))
|
|
|
|
return payload |