herstellung vollständiger Kompaitibilität

This commit is contained in:
Lars committed 2025-12-27 13:20:37 +01:00
1 parent cfcaa926cd
commit f08a331bc6
3 files changed
+99 -60

No files matched your search

+31 -26
View File
@@ -3,8 +3,8 @@ FILE: app/core/ingestion/ingestion_note_payload.py
DESCRIPTION: Baut das JSON-Objekt für mindnet_notes.
FEATURES:
- Multi-Hash (body/full) für flexible Change Detection.
- Fix v2.4.2: edge_defaults Logik wiederhergestellt (DoD-Korrektur).
VERSION: 2.4.2
- Fix v2.4.3: Vollständiger Audit-Check (Env-Vars, JSON-Validation, Edge-Defaults).
VERSION: 2.4.3
STATUS: Active
"""
from __future__ import annotations
@@ -13,14 +13,13 @@ import os
import json
import pathlib
import hashlib
import yaml
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
def _as_dict(x) -> Dict[str, Any]:
"""Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen."""
"""Versucht, ein Objekt in ein Dict zu überführen."""
if isinstance(x, dict): return dict(x)
out: Dict[str, Any] = {}
for attr in ("frontmatter", "body", "id", "note_id", "title", "path", "tags", "type", "created", "modified", "date"):
@@ -31,25 +30,24 @@ def _as_dict(x) -> Dict[str, Any]:
return out
def _ensure_list(x) -> list:
"""Sichert, dass das Ergebnis eine Liste von Strings ist."""
"""Sichert String-Listen Integrität."""
if x is None: return []
if isinstance(x, list): return [str(i) for i in x]
if isinstance(x, (set, tuple)): return [str(i) for i in x]
return [str(x)]
def _compute_hash(content: str) -> str:
"""Berechnet einen SHA-256 Hash."""
"""SHA-256 Hash-Berechnung."""
if not content: return ""
return hashlib.sha256(content.encode("utf-8")).hexdigest()
def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str:
"""Stellt den zu hashenden Content deterministisch zusammen."""
"""Generiert den Hash-Input-String."""
body = str(n.get("body") or "")
if mode == "body": return body
if mode == "full":
fm = n.get("frontmatter") or {}
meta_parts = []
# Steuernde Metadaten für Change Detection
for k in sorted(["title", "type", "status", "tags", "chunking_profile", "chunk_profile", "retriever_weight"]):
val = fm.get(k)
if val is not None: meta_parts.append(f"{k}:{val}")
@@ -57,13 +55,13 @@ def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str:
return body
def _cfg_for_type(note_type: str, reg: dict) -> dict:
"""Holt die typ-spezifische Konfiguration."""
"""Extrahiert Typ-spezifische Config."""
if not isinstance(reg, dict): return {}
types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
return types.get(note_type, {}) if isinstance(types, dict) else {}
def _cfg_defaults(reg: dict) -> dict:
"""Holt die globalen Default-Werte aus der Registry."""
"""Extrahiert globale Default-Werte."""
if not isinstance(reg, dict): return {}
for key in ("defaults", "default", "global"):
v = reg.get(key)
@@ -75,9 +73,7 @@ def _cfg_defaults(reg: dict) -> dict:
# ---------------------------------------------------------------------------
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
"""
Baut das Note-Payload inklusive Multi-Hash und edge_defaults.
"""
"""Baut das Note-Payload inklusive Multi-Hash und Audit-Validierung."""
n = _as_dict(note)
reg = kwargs.get("types_cfg") or {}
hash_source = kwargs.get("hash_source", "parsed")
@@ -89,19 +85,22 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
cfg_type = _cfg_for_type(note_type, reg)
cfg_def = _cfg_defaults(reg)
# --- retriever_weight ---
# --- retriever_weight Audit ---
default_rw = float(os.environ.get("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0))
retriever_weight = fm.get("retriever_weight")
if retriever_weight is None:
retriever_weight = cfg_type.get("retriever_weight", cfg_def.get("retriever_weight", 1.0))
retriever_weight = cfg_type.get("retriever_weight", cfg_def.get("retriever_weight", default_rw))
try: retriever_weight = float(retriever_weight)
except: retriever_weight = 1.0
except: retriever_weight = default_rw
# --- chunk_profile ---
# --- chunk_profile Audit ---
chunk_profile = fm.get("chunking_profile") or fm.get("chunk_profile")
if chunk_profile is None:
chunk_profile = cfg_type.get("chunking_profile", cfg_def.get("chunking_profile", "sliding_standard"))
chunk_profile = cfg_type.get("chunking_profile")
if chunk_profile is None:
chunk_profile = cfg_def.get("chunking_profile", "sliding_standard")
# --- edge_defaults (WIEDERHERGESTELLT) ---
# --- edge_defaults ---
edge_defaults = fm.get("edge_defaults")
if edge_defaults is None:
edge_defaults = cfg_type.get("edge_defaults", cfg_def.get("edge_defaults", []))
@@ -110,29 +109,35 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
# --- Basis-Metadaten ---
note_id = n.get("note_id") or n.get("id") or fm.get("id")
title = n.get("title") or fm.get("title") or ""
path = n.get("path") or kwargs.get("file_path") or ""
if isinstance(path, pathlib.Path): path = str(path)
payload: Dict[str, Any] = {
"note_id": note_id,
"title": title,
"type": note_type,
"path": str(n.get("path") or kwargs.get("file_path") or ""),
"path": path,
"retriever_weight": retriever_weight,
"chunk_profile": chunk_profile,
"edge_defaults": edge_defaults, # Feld jetzt wieder enthalten
"edge_defaults": edge_defaults,
"hashes": {}
}
# --- MULTI-HASH ---
for mode in ["body", "full"]:
key = f"{mode}:{hash_source}:{hash_normalize}"
payload["hashes"][key] = _compute_hash(_get_hash_source_content(n, mode))
content = _get_hash_source_content(n, mode)
payload["hashes"][f"{mode}:{hash_source}:{hash_normalize}"] = _compute_hash(content)
# Metadaten-Felder
if fm.get("tags") or n.get("tags"): payload["tags"] = _ensure_list(fm.get("tags") or n.get("tags"))
# Metadaten
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
if tags: payload["tags"] = _ensure_list(tags)
if fm.get("aliases"): payload["aliases"] = _ensure_list(fm.get("aliases"))
for k in ("created", "modified", "date"):
v = fm.get(k) or n.get(k)
if v: payload[k] = str(v)
if n.get("body"): payload["fulltext"] = str(n["body"])
# Final JSON Validation Audit
json.loads(json.dumps(payload, ensure_ascii=False))
return payload