WP15b #15

Merged
Lars merged 23 commits from WP15b into main 2025-12-27 22:15:27 +01:00
Showing only changes of commit cfcaa926cd - Show all commits

View File

@ -1,8 +1,11 @@
""" """
FILE: app/core/ingestion/ingestion_note_payload.py FILE: app/core/ingestion/ingestion_note_payload.py
DESCRIPTION: Baut das JSON-Objekt für mindnet_notes. DESCRIPTION: Baut das JSON-Objekt für mindnet_notes.
FEATURES: Multi-Hash (body/full), Config-Fix für chunking_profile. FEATURES:
VERSION: 2.4.0 - Multi-Hash (body/full) für flexible Change Detection.
- Fix v2.4.2: edge_defaults Logik wiederhergestellt (DoD-Korrektur).
VERSION: 2.4.2
STATUS: Active
""" """
from __future__ import annotations from __future__ import annotations
from typing import Any, Dict, Tuple, Optional from typing import Any, Dict, Tuple, Optional
@ -12,7 +15,12 @@ import pathlib
import hashlib import hashlib
import yaml import yaml
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
def _as_dict(x) -> Dict[str, Any]: def _as_dict(x) -> Dict[str, Any]:
"""Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen."""
if isinstance(x, dict): return dict(x) if isinstance(x, dict): return dict(x)
out: Dict[str, Any] = {} out: Dict[str, Any] = {}
for attr in ("frontmatter", "body", "id", "note_id", "title", "path", "tags", "type", "created", "modified", "date"): for attr in ("frontmatter", "body", "id", "note_id", "title", "path", "tags", "type", "created", "modified", "date"):
@ -23,29 +31,53 @@ def _as_dict(x) -> Dict[str, Any]:
return out return out
def _ensure_list(x) -> list: def _ensure_list(x) -> list:
"""Sichert, dass das Ergebnis eine Liste von Strings ist."""
if x is None: return [] if x is None: return []
if isinstance(x, list): return [str(i) for i in x] if isinstance(x, list): return [str(i) for i in x]
if isinstance(x, (set, tuple)): return [str(i) for i in x] if isinstance(x, (set, tuple)): return [str(i) for i in x]
return [str(x)] return [str(x)]
def _compute_hash(content: str) -> str: def _compute_hash(content: str) -> str:
"""Berechnet einen SHA-256 Hash."""
if not content: return "" if not content: return ""
return hashlib.sha256(content.encode("utf-8")).hexdigest() return hashlib.sha256(content.encode("utf-8")).hexdigest()
def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str: def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str:
"""Stellt den zu hashenden Content deterministisch zusammen."""
body = str(n.get("body") or "") body = str(n.get("body") or "")
if mode == "body": return body if mode == "body": return body
if mode == "full": if mode == "full":
fm = n.get("frontmatter") or {} fm = n.get("frontmatter") or {}
meta_parts = [] meta_parts = []
# Steuernde Metadaten für Change Detection
for k in sorted(["title", "type", "status", "tags", "chunking_profile", "chunk_profile", "retriever_weight"]): for k in sorted(["title", "type", "status", "tags", "chunking_profile", "chunk_profile", "retriever_weight"]):
val = fm.get(k) val = fm.get(k)
if val is not None: meta_parts.append(f"{k}:{val}") if val is not None: meta_parts.append(f"{k}:{val}")
return f" {'|'.join(meta_parts)}||{body}" return f"{'|'.join(meta_parts)}||{body}"
return body return body
def _cfg_for_type(note_type: str, reg: dict) -> dict:
"""Holt die typ-spezifische Konfiguration."""
if not isinstance(reg, dict): return {}
types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
return types.get(note_type, {}) if isinstance(types, dict) else {}
def _cfg_defaults(reg: dict) -> dict:
"""Holt die globalen Default-Werte aus der Registry."""
if not isinstance(reg, dict): return {}
for key in ("defaults", "default", "global"):
v = reg.get(key)
if isinstance(v, dict): return v
return {}
# ---------------------------------------------------------------------------
# Haupt-API
# ---------------------------------------------------------------------------
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]: def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
"""Baut das Note-Payload inklusive Multi-Hash.""" """
Baut das Note-Payload inklusive Multi-Hash und edge_defaults.
"""
n = _as_dict(note) n = _as_dict(note)
reg = kwargs.get("types_cfg") or {} reg = kwargs.get("types_cfg") or {}
hash_source = kwargs.get("hash_source", "parsed") hash_source = kwargs.get("hash_source", "parsed")
@ -54,24 +86,48 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
fm = n.get("frontmatter") or {} fm = n.get("frontmatter") or {}
note_type = str(fm.get("type") or n.get("type") or "concept") note_type = str(fm.get("type") or n.get("type") or "concept")
# Weights & Profiles cfg_type = _cfg_for_type(note_type, reg)
retriever_weight = fm.get("retriever_weight", 1.0) cfg_def = _cfg_defaults(reg)
chunk_profile = fm.get("chunking_profile") or fm.get("chunk_profile") or "sliding_standard"
# --- retriever_weight ---
retriever_weight = fm.get("retriever_weight")
if retriever_weight is None:
retriever_weight = cfg_type.get("retriever_weight", cfg_def.get("retriever_weight", 1.0))
try: retriever_weight = float(retriever_weight)
except: retriever_weight = 1.0
# --- chunk_profile ---
chunk_profile = fm.get("chunking_profile") or fm.get("chunk_profile")
if chunk_profile is None:
chunk_profile = cfg_type.get("chunking_profile", cfg_def.get("chunking_profile", "sliding_standard"))
# --- edge_defaults (WIEDERHERGESTELLT) ---
edge_defaults = fm.get("edge_defaults")
if edge_defaults is None:
edge_defaults = cfg_type.get("edge_defaults", cfg_def.get("edge_defaults", []))
edge_defaults = _ensure_list(edge_defaults)
# --- Basis-Metadaten ---
note_id = n.get("note_id") or n.get("id") or fm.get("id")
title = n.get("title") or fm.get("title") or ""
payload: Dict[str, Any] = { payload: Dict[str, Any] = {
"note_id": n.get("note_id") or n.get("id") or fm.get("id"), "note_id": note_id,
"title": n.get("title") or fm.get("title") or "", "title": title,
"type": note_type, "type": note_type,
"path": str(n.get("path") or kwargs.get("path") or ""), "path": str(n.get("path") or kwargs.get("file_path") or ""),
"retriever_weight": float(retriever_weight), "retriever_weight": retriever_weight,
"chunk_profile": chunk_profile, "chunk_profile": chunk_profile,
"edge_defaults": edge_defaults, # Feld jetzt wieder enthalten
"hashes": {} "hashes": {}
} }
# --- MULTI-HASH ---
for mode in ["body", "full"]: for mode in ["body", "full"]:
key = f"{mode}:{hash_source}:{hash_normalize}" key = f"{mode}:{hash_source}:{hash_normalize}"
payload["hashes"][key] = _compute_hash(_get_hash_source_content(n, mode)) payload["hashes"][key] = _compute_hash(_get_hash_source_content(n, mode))
# Metadaten-Felder
if fm.get("tags") or n.get("tags"): payload["tags"] = _ensure_list(fm.get("tags") or n.get("tags")) if fm.get("tags") or n.get("tags"): payload["tags"] = _ensure_list(fm.get("tags") or n.get("tags"))
if fm.get("aliases"): payload["aliases"] = _ensure_list(fm.get("aliases")) if fm.get("aliases"): payload["aliases"] = _ensure_list(fm.get("aliases"))
for k in ("created", "modified", "date"): for k in ("created", "modified", "date"):