scriptAudit #11
|
|
@ -1,11 +1,10 @@
|
|||
"""
|
||||
FILE: app/core/note_payload.py
|
||||
DESCRIPTION: Baut das JSON-Objekt für 'mindnet_notes'. Wendet Vererbung für Configs an (Frontmatter > Type > Default).
|
||||
VERSION: 2.1.0
|
||||
DESCRIPTION: Baut das JSON-Objekt für 'mindnet_notes'. Inkludiert Hash-Berechnung für Change Detection.
|
||||
VERSION: 2.2.0 (Fix: Missing Hash Calculation)
|
||||
STATUS: Active
|
||||
DEPENDENCIES: yaml, os, json, pathlib
|
||||
DEPENDENCIES: yaml, os, json, pathlib, hashlib
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -14,6 +13,7 @@ from typing import Any, Dict, Tuple, Optional
|
|||
import os
|
||||
import json
|
||||
import pathlib
|
||||
import hashlib
|
||||
|
||||
try:
|
||||
import yaml # type: ignore
|
||||
|
|
@ -31,7 +31,6 @@ def _as_dict(x) -> Dict[str, Any]:
|
|||
return dict(x)
|
||||
|
||||
out: Dict[str, Any] = {}
|
||||
# bekannte Attribute übernehmen, sofern vorhanden
|
||||
for attr in (
|
||||
"frontmatter",
|
||||
"body",
|
||||
|
|
@ -50,7 +49,6 @@ def _as_dict(x) -> Dict[str, Any]:
|
|||
if val is not None:
|
||||
out[attr] = val
|
||||
|
||||
# Fallback: wenn immer noch leer, raw speichern
|
||||
if not out:
|
||||
out["raw"] = str(x)
|
||||
|
||||
|
|
@ -58,14 +56,12 @@ def _as_dict(x) -> Dict[str, Any]:
|
|||
|
||||
|
||||
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]:
|
||||
"""Extrahiert optionale Zusatzargumente wie path und types_cfg."""
|
||||
path = kwargs.get("path") or (args[0] if args else None)
|
||||
types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None
|
||||
return path, types_cfg
|
||||
|
||||
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
"""Liest einen Float-Wert aus der Umgebung, mit robustem Fallback."""
|
||||
try:
|
||||
return float(os.environ.get(name, default))
|
||||
except Exception:
|
||||
|
|
@ -73,7 +69,6 @@ def _env_float(name: str, default: float) -> float:
|
|||
|
||||
|
||||
def _ensure_list(x) -> list:
|
||||
"""Garantiert eine String-Liste."""
|
||||
if x is None:
|
||||
return []
|
||||
if isinstance(x, list):
|
||||
|
|
@ -82,13 +77,47 @@ def _ensure_list(x) -> list:
|
|||
return [str(i) for i in x]
|
||||
return [str(x)]
|
||||
|
||||
# --- NEW: Hash Logic ---
|
||||
def _compute_hash(content: str) -> str:
|
||||
"""Berechnet einen SHA-256 Hash für den gegebenen String."""
|
||||
if not content:
|
||||
return ""
|
||||
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str:
|
||||
"""
|
||||
Stellt den String zusammen, der gehasht werden soll.
|
||||
mode="body": Nur der Textinhalt.
|
||||
mode="full": Textinhalt + relevante Metadaten (Titel, Typ, Tags, Status).
|
||||
"""
|
||||
body = str(n.get("body") or "")
|
||||
|
||||
if mode == "body":
|
||||
return body
|
||||
|
||||
if mode == "full":
|
||||
# Wir nehmen die wichtigsten strukturellen Metadaten dazu.
|
||||
# Wichtig: Sortierte Keys für deterministisches Verhalten!
|
||||
fm = n.get("frontmatter") or {}
|
||||
|
||||
# Extrahieren relevanter Meta-Felder, die Änderungen auslösen sollen
|
||||
meta_parts = []
|
||||
for k in sorted(["title", "type", "status", "tags", "chunk_profile"]):
|
||||
val = fm.get(k)
|
||||
if val is not None:
|
||||
meta_parts.append(f"{k}:{val}")
|
||||
|
||||
meta_str = "|".join(meta_parts)
|
||||
return f"{meta_str}||{body}"
|
||||
|
||||
return body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Type-Registry laden
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict:
|
||||
"""Lädt die Type-Registry aus YAML/JSON oder nutzt ein explizit übergebenes Dict."""
|
||||
if explicit_cfg and isinstance(explicit_cfg, dict):
|
||||
return explicit_cfg
|
||||
|
||||
|
|
@ -105,7 +134,6 @@ def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict:
|
|||
|
||||
|
||||
def _cfg_for_type(note_type: str, reg: dict) -> dict:
|
||||
"""Liefert die Konfiguration für einen konkreten Notiztyp aus der Registry."""
|
||||
if not isinstance(reg, dict):
|
||||
return {}
|
||||
types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
|
||||
|
|
@ -113,7 +141,6 @@ def _cfg_for_type(note_type: str, reg: dict) -> dict:
|
|||
|
||||
|
||||
def _cfg_defaults(reg: dict) -> dict:
|
||||
"""Liefert den Default-Block aus der Registry (defaults/global)."""
|
||||
if not isinstance(reg, dict):
|
||||
return {}
|
||||
for key in ("defaults", "default", "global"):
|
||||
|
|
@ -130,20 +157,16 @@ def _cfg_defaults(reg: dict) -> dict:
|
|||
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
"""
|
||||
Baut das Note-Payload für mindnet_notes auf.
|
||||
|
||||
Erwartete Felder im Payload:
|
||||
- note_id: stabile ID aus Frontmatter (id) oder Note-Objekt
|
||||
- title: Titel der Notiz
|
||||
- type: Notiztyp (z. B. concept, project, journal, ...)
|
||||
- path: Pfad im Vault
|
||||
- retriever_weight: effektives Gewicht für den Retriever
|
||||
- chunk_profile: Profil für Chunking (short|medium|long|default|...)
|
||||
- edge_defaults: Liste von Kanten-Typen, die als Defaults gelten
|
||||
- aliases: Liste von Synonymen (WP-11)
|
||||
Inkludiert Hash-Berechnung.
|
||||
"""
|
||||
n = _as_dict(note)
|
||||
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
|
||||
reg = _load_types_config(types_cfg_explicit)
|
||||
|
||||
# Hash Config extrahieren (Defaults wie in ingestion.py)
|
||||
hash_mode = kwargs.get("hash_mode", "full")
|
||||
hash_source = kwargs.get("hash_source", "parsed")
|
||||
hash_normalize = kwargs.get("hash_normalize", "canonical")
|
||||
|
||||
fm = n.get("frontmatter") or {}
|
||||
fm_type = fm.get("type") or n.get("type") or "concept"
|
||||
|
|
@ -152,7 +175,7 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
|||
cfg_type = _cfg_for_type(note_type, reg)
|
||||
cfg_def = _cfg_defaults(reg)
|
||||
|
||||
# --- retriever_weight: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
|
||||
# --- retriever_weight ---
|
||||
default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
|
||||
retriever_weight = fm.get("retriever_weight")
|
||||
if retriever_weight is None:
|
||||
|
|
@ -165,7 +188,7 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
|||
except Exception:
|
||||
retriever_weight = default_rw
|
||||
|
||||
# --- chunk_profile: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
|
||||
# --- chunk_profile ---
|
||||
chunk_profile = fm.get("chunk_profile")
|
||||
if chunk_profile is None:
|
||||
chunk_profile = cfg_type.get(
|
||||
|
|
@ -178,7 +201,7 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
|||
if not isinstance(chunk_profile, str):
|
||||
chunk_profile = "medium"
|
||||
|
||||
# --- edge_defaults: Frontmatter > Typ-Config > Registry-Defaults ---
|
||||
# --- edge_defaults ---
|
||||
edge_defaults = fm.get("edge_defaults")
|
||||
if edge_defaults is None:
|
||||
edge_defaults = cfg_type.get(
|
||||
|
|
@ -187,7 +210,7 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
|||
)
|
||||
edge_defaults = _ensure_list(edge_defaults)
|
||||
|
||||
# --- Basis-Metadaten (IDs, Titel, Pfad) ---
|
||||
# --- Basis-Metadaten ---
|
||||
note_id = n.get("note_id") or n.get("id") or fm.get("id")
|
||||
title = n.get("title") or fm.get("title") or ""
|
||||
path = n.get("path") or path_arg
|
||||
|
|
@ -203,28 +226,41 @@ def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
|||
"chunk_profile": chunk_profile,
|
||||
"edge_defaults": edge_defaults,
|
||||
}
|
||||
|
||||
# --- FIX: Hash Calculation ---
|
||||
# Wir berechnen den Hash basierend auf dem Modus
|
||||
content_to_hash = _get_hash_source_content(n, hash_mode)
|
||||
computed_hash = _compute_hash(content_to_hash)
|
||||
|
||||
# Der Key muss exakt dem entsprechen, was ingestion.py erwartet:
|
||||
# key_current = f"{hash_mode}:{hash_source}:{hash_normalize}"
|
||||
hash_key = f"{hash_mode}:{hash_source}:{hash_normalize}"
|
||||
|
||||
payload["hashes"] = {
|
||||
hash_key: computed_hash
|
||||
}
|
||||
|
||||
# Tags / Keywords übernehmen
|
||||
# Tags / Keywords
|
||||
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
|
||||
if tags:
|
||||
payload["tags"] = _ensure_list(tags)
|
||||
|
||||
# WP-11: Aliases übernehmen (für Discovery Service)
|
||||
# Aliases
|
||||
aliases = fm.get("aliases")
|
||||
if aliases:
|
||||
payload["aliases"] = _ensure_list(aliases)
|
||||
|
||||
# Zeitliche Metadaten (sofern vorhanden)
|
||||
# Zeit
|
||||
for k in ("created", "modified", "date"):
|
||||
v = fm.get(k) or n.get(k)
|
||||
if v:
|
||||
payload[k] = str(v)
|
||||
|
||||
# Fulltext (Fallback, falls body im Input)
|
||||
# Fulltext
|
||||
if "body" in n and n["body"]:
|
||||
payload["fulltext"] = str(n["body"])
|
||||
|
||||
# JSON-Roundtrip zur harten Validierung (ASCII beibehalten)
|
||||
# JSON Validation
|
||||
json.loads(json.dumps(payload, ensure_ascii=False))
|
||||
|
||||
return payload
|
||||
Loading…
Reference in New Issue
Block a user