Dateien nach "app/core" hochladen

app/core/chunk_payload.py

@@ -1,12 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-app/core/chunk_payload.py  (Mindnet V2 — robust v2)
-
-Änderungen ggü. v1:
-- neighbors_prev / neighbors_next werden als **Array** persistiert ([], [id]).
-- retriever_weight / chunk_profile werden je Chunk aufgelöst (Frontmatter > types.yaml > Defaults).
-- Lädt config/types.yaml selbst, wenn types_cfg nicht übergeben wurde.
+app/core/chunk_payload.py  (Mindnet V2 — types.yaml authoritative)
+- neighbors_prev / neighbors_next sind Listen ([], [id]).
+- retriever_weight / chunk_profile kommen aus types.yaml (Frontmatter wird ignoriert).
+- Fallbacks: defaults.* in types.yaml; sonst 1.0 / "default".
 """
 from __future__ import annotations
 from typing import Any, Dict, List, Optional
@@ -16,21 +14,7 @@ def _env(n: str, d: Optional[str]=None) -> str:
     v = os.getenv(n)
     return v if v is not None else (d or "")
 
-def _deep_get(root: Any, path: str) -> Any:
-    cur = root
-    for key in path.split("."):
-        if not isinstance(cur, dict) or key not in cur:
-            return None
-        cur = cur[key]
-    return cur
-
-def _as_float(x: Any):
-    try:
-        return float(x)
-    except Exception:
-        return None
-
-def _load_types_local() -> dict:
+def _load_types() -> dict:
     p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
     try:
         with open(p, "r", encoding="utf-8") as f:
@@ -38,38 +22,46 @@ def _load_types_local() -> dict:
     except Exception:
         return {}
 
-def _effective_chunk_profile(note_type: str, fm: Dict[str, Any], reg: dict) -> Optional[str]:
-    if isinstance(fm.get("chunk_profile"), str):
-        return fm.get("chunk_profile")
-    types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
-    if isinstance(types, dict):
-        v = types.get(note_type, {})
-        if isinstance(v, dict):
-            cp = v.get("chunk_profile")
-            if isinstance(cp, str):
-                return cp
+def _get_types_map(reg: dict) -> dict:
+    if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
+        return reg["types"]
+    return reg if isinstance(reg, dict) else {}
+
+def _get_defaults(reg: dict) -> dict:
+    if isinstance(reg, dict) and isinstance(reg.get("defaults"), dict):
+        return reg["defaults"]
+    if isinstance(reg, dict) and isinstance(reg.get("global"), dict):
+        return reg["global"]
+    return {}
+
+def _as_float(x: Any):
+    try:
+        return float(x)
+    except Exception:
         return None
 
-def _effective_retriever_weight(note_type: str, fm: Dict[str, Any], reg: dict) -> float:
-    if fm.get("retriever_weight") is not None:
-        v = _as_float(fm.get("retriever_weight"))
+def _resolve_chunk_profile(note_type: str, reg: dict) -> str:
+    types = _get_types_map(reg)
+    if isinstance(types, dict):
+        t = types.get(note_type, {})
+        if isinstance(t, dict) and isinstance(t.get("chunk_profile"), str):
+            return t["chunk_profile"]
+    defs = _get_defaults(reg)
+    if isinstance(defs, dict) and isinstance(defs.get("chunk_profile"), str):
+        return defs["chunk_profile"]
+    return "default"
+
+def _resolve_retriever_weight(note_type: str, reg: dict) -> float:
+    types = _get_types_map(reg)
+    if isinstance(types, dict):
+        t = types.get(note_type, {})
+        if isinstance(t, dict) and (t.get("retriever_weight") is not None):
+            v = _as_float(t.get("retriever_weight"))
             if v is not None:
                 return float(v)
-    types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
-    candidates = [
-        f"{note_type}.retriever_weight",
-        f"{note_type}.retriever.weight",
-        f"{note_type}.retrieval.weight",
-        "defaults.retriever_weight",
-        "defaults.retriever.weight",
-        "global.retriever_weight",
-        "global.retriever.weight",
-    ]
-    for path in candidates:
-        val = _deep_get(types, path) if "." in path else (types.get(path) if isinstance(types, dict) else None)
-        if val is None and isinstance(reg, dict):
-            val = _deep_get(reg, f"types.{path}")
-        v = _as_float(val)
+    defs = _get_defaults(reg)
+    if isinstance(defs, dict) and (defs.get("retriever_weight") is not None):
+        v = _as_float(defs.get("retriever_weight"))
         if v is not None:
             return float(v)
     return 1.0
@@ -90,10 +82,11 @@ def make_chunk_payloads(note: Dict[str, Any],
                         file_path: Optional[str] = None) -> List[Dict[str, Any]]:
     fm = (note or {}).get("frontmatter", {}) or {}
     note_type = fm.get("type") or note.get("type") or "concept"
-    reg = types_cfg if isinstance(types_cfg, dict) else _load_types_local()
+    reg = types_cfg if isinstance(types_cfg, dict) else _load_types()
 
-    cp = _effective_chunk_profile(note_type, fm, reg)
-    rw = _effective_retriever_weight(note_type, fm, reg)
+    # types.yaml authoritative
+    cp = _resolve_chunk_profile(note_type, reg)
+    rw = _resolve_retriever_weight(note_type, reg)
 
     tags = fm.get("tags") or []
     if isinstance(tags, str):
@@ -125,11 +118,9 @@ def make_chunk_payloads(note: Dict[str, Any],
             "path": note_path,
             "source_path": file_path or note_path,
             "retriever_weight": float(rw),
+            "chunk_profile": cp,
         }
-        if cp is not None:
-            pl["chunk_profile"] = cp
-
-        # Aufräumen
+        # Aufräumen von Alt-Feldern
         for alias in ("chunk_num", "Chunk_Number"):
             pl.pop(alias, None)
 

app/core/note_payload.py

@@ -1,104 +1,159 @@
-# note_payload.py
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+app/core/note_payload.py  (Mindnet V2 — types.yaml authoritative)
+- retriever_weight und chunk_profile werden aus config/types.yaml gelesen.
+- Reihenfolge: types.<type>  >  defaults.*  >  Fallbacks (1.0 / "default").
+- Frontmatter-Overrides für diese beiden Felder werden bewusst IGNORIERT.
+- edge_defaults (falls in types.yaml definiert) werden in die Note-Payload gespiegelt.
+- MINDNET_TYPES_FILE kann absoluten Pfad liefern, sonst ./config/types.yaml.
+"""
+
 from __future__ import annotations
-from typing import Any, Dict, Optional, Tuple
-import os, json, pathlib, yaml
+from typing import Any, Dict, Optional, List
+import os, yaml, hashlib, datetime
 
-def _as_dict(note: Any) -> Dict[str, Any]:
-    if isinstance(note, dict):
-        return note
-    d: Dict[str, Any] = {}
-    for attr in ("id","note_id","title","path","frontmatter","meta","metadata","type","created","modified","date","tags"):
-        if hasattr(note, attr):
-            d[attr] = getattr(note, attr)
-    # Normalisiere Frontmatter
-    fm = d.get("frontmatter") or d.get("meta") or d.get("metadata") or {}
-    d["frontmatter"] = fm if isinstance(fm, dict) else {}
-    return d
+# ----------------------- helpers -----------------------
 
-def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[Dict[str,Any]]]:
-    path = kwargs.get("path")
-    types_cfg = kwargs.get("types_config")
-    # legacy positional: (path, types_config) oder (types_config, ...)
-    for a in args:
-        if path is None and isinstance(a, (str, pathlib.Path)):
-            path = str(a)
-        if types_cfg is None and isinstance(a, dict):
-            types_cfg = a
-    return path, types_cfg
+def _env(n: str, d: Optional[str]=None) -> str:
+    v = os.getenv(n)
+    return v if v is not None else (d or "")
 
-def _load_types_config(explicit: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
-    if isinstance(explicit, dict):
-        return explicit
-    for rel in ("config/config.yaml", "config/types.yaml"):
-        p = pathlib.Path(rel)
-        if p.exists():
-            with p.open("r", encoding="utf-8") as f:
-                data = yaml.safe_load(f) or {}
-            if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
-                return data["types"]
-            return data if isinstance(data, dict) else {}
+def _load_types() -> dict:
+    p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
+    try:
+        with open(p, "r", encoding="utf-8") as f:
+            return yaml.safe_load(f) or {}
+    except Exception:
         return {}
 
-def _coalesce(*vals):
-    for v in vals:
-        if v is not None:
-            return v
+def _get_types_map(reg: dict) -> dict:
+    if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
+        return reg["types"]
+    return reg if isinstance(reg, dict) else {}
+
+def _get_defaults(reg: dict) -> dict:
+    if isinstance(reg, dict) and isinstance(reg.get("defaults"), dict):
+        return reg["defaults"]
+    # alias "global" erlaubt
+    if isinstance(reg, dict) and isinstance(reg.get("global"), dict):
+        return reg["global"]
+    return {}
+
+def _resolve_chunk_profile(note_type: str, reg: dict) -> str:
+    types = _get_types_map(reg)
+    if isinstance(types, dict):
+        t = types.get(note_type, {})
+        if isinstance(t, dict) and isinstance(t.get("chunk_profile"), str):
+            return t["chunk_profile"]
+    defs = _get_defaults(reg)
+    if isinstance(defs, dict) and isinstance(defs.get("chunk_profile"), str):
+        return defs["chunk_profile"]
+    return "default"
+
+def _as_float(x: Any) -> Optional[float]:
+    try:
+        return float(x)
+    except Exception:
         return None
 
-def _env_float(name: str, default: float) -> float:
-    try:
-        return float(os.environ.get(name, default))
-    except Exception:
-        return default
+def _resolve_retriever_weight(note_type: str, reg: dict) -> float:
+    types = _get_types_map(reg)
+    if isinstance(types, dict):
+        t = types.get(note_type, {})
+        if isinstance(t, dict) and (t.get("retriever_weight") is not None):
+            v = _as_float(t.get("retriever_weight"))
+            if v is not None:
+                return float(v)
+    defs = _get_defaults(reg)
+    if isinstance(defs, dict) and (defs.get("retriever_weight") is not None):
+        v = _as_float(defs.get("retriever_weight"))
+        if v is not None:
+            return float(v)
+    return 1.0
 
-def _ensure_list(x) -> list:
-    if x is None: return []
-    if isinstance(x, list): return [str(i) for i in x]
-    if isinstance(x, (set, tuple)): return [str(i) for i in x]
-    return [str(x)]
+# ------------------- public API -------------------
 
-def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
-    n = _as_dict(note)
-    path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
-    types_cfg = _load_types_config(types_cfg_explicit)
+def make_note_payload(parsed_note: Any,
+                      *,
+                      vault_root: str,
+                      hash_mode: str = "body",
+                      hash_normalize: str = "canonical",
+                      hash_source: str = "parsed",
+                      file_path: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Erwartet ein Parsed-Objekt mit Attributen:
+      - frontmatter (dict)
+      - body (str)
+      - path (optional)
+    Liefert Note-Payload mit deterministischer note_id und Typ-Werten aus types.yaml.
+    """
+    fm = (getattr(parsed_note, "frontmatter", None) or {}) if parsed_note else {}
+    title = fm.get("title") or ""
+    note_type = fm.get("type") or "concept"
+    note_id = fm.get("id") or _stable_id_from_path_or_title(file_path or "", title)
+    tags = fm.get("tags") or []
+    if isinstance(tags, str):
+        tags = [tags]
 
-    fm = n.get("frontmatter") or {}
-    note_type = str(fm.get("type") or n.get("type") or "note")
-    cfg_for_type = types_cfg.get(note_type, {}) if isinstance(types_cfg, dict) else {}
+    # types.yaml authoritative:
+    reg = _load_types()
+    chunk_profile = _resolve_chunk_profile(note_type, reg)
+    retriever_weight = _resolve_retriever_weight(note_type, reg)
+    # edge_defaults (falls vorhanden)
+    edge_defaults = None
+    types = _get_types_map(reg)
+    if isinstance(types, dict):
+        t = types.get(note_type, {})
+        if isinstance(t, dict) and isinstance(t.get("edge_defaults"), list):
+            edge_defaults = t["edge_defaults"]
 
-    default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
-    retriever_weight = _coalesce(fm.get("retriever_weight"), cfg_for_type.get("retriever_weight"), default_rw)
-    try:
-        retriever_weight = float(retriever_weight)
-    except Exception:
-        retriever_weight = default_rw
+    updated = _ts_to_int(fm.get("updated"))
 
-    chunk_profile = _coalesce(fm.get("chunk_profile"), cfg_for_type.get("chunk_profile"), os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE","medium"))
-    chunk_profile = chunk_profile if isinstance(chunk_profile, str) else "medium"
-
-    edge_defaults = _ensure_list(_coalesce(fm.get("edge_defaults"), cfg_for_type.get("edge_defaults"), []))
-
-    note_id = n.get("note_id") or n.get("id") or fm.get("id")
-    title = n.get("title") or fm.get("title") or ""
-    path = n.get("path") or path_arg
-    if isinstance(path, pathlib.Path):
-        path = str(path)
-
-    payload = {
+    payload: Dict[str, Any] = {
         "note_id": note_id,
-        "title": title,
         "type": note_type,
-        "path": path or "",                 # immer vorhanden
-        "retriever_weight": retriever_weight,
+        "title": title,
+        "tags": tags,
+        "updated": updated if updated is not None else 0,
+        "path": file_path or "",
         "chunk_profile": chunk_profile,
-        "edge_defaults": edge_defaults,
+        "retriever_weight": float(retriever_weight),
     }
+    if edge_defaults is not None:
+        payload["edge_defaults"] = edge_defaults
 
-    tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
-    if tags: payload["tags"] = _ensure_list(tags)
-    for k in ("created","modified","date"):
-        v = fm.get(k) or n.get(k)
-        if v: payload[k] = str(v)
-
-    json.loads(json.dumps(payload, ensure_ascii=False))
     return payload
+
+# ------------------- internal utilities -------------------
+
+def _stable_id_from_path_or_title(path: str, title: str) -> str:
+    base = path or title or ""
+    if not base:
+        base = "note"
+    h = hashlib.sha1(base.encode("utf-8")).hexdigest()[:6]
+    # title-sourced; in V2 typischerweise durch Frontmatter id ersetzt
+    return f"auto-{h}"
+
+def _ts_to_int(val: Any) -> Optional[int]:
+    # akzeptiert YYYY-MM-DD oder epoch int; None bei Fehler
+    if val is None:
+        return None
+    if isinstance(val, int):
+        return val
+    if isinstance(val, float):
+        return int(val)
+    if isinstance(val, str):
+        val = val.strip()
+        # YYYY-MM-DD
+        try:
+            dt = datetime.date.fromisoformat(val)
+            return int(datetime.datetime(dt.year, dt.month, dt.day).timestamp())
+        except Exception:
+            pass
+        # epoch string
+        try:
+            return int(val)
+        except Exception:
+            return None
+    return None