aufräumen und löschen von Alt-Scripten WP19b
This commit is contained in:
1 parent
f08a331bc6
commit
e3858e8bc3
14 files changed
+15
-680
No files matched your search
@@ -1,176 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/chunk_payload.py
|
||||
DESCRIPTION: Baut das JSON-Objekt für 'mindnet_chunks'.
|
||||
FEATURES:
|
||||
- Inkludiert Nachbarschafts-IDs (prev/next) und Titel.
|
||||
- FIX 3: Robuste Erkennung des Inputs (Frontmatter-Dict vs. Note-Objekt), damit Overrides ankommen.
|
||||
VERSION: 2.3.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: yaml, os
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, List, Optional
|
||||
import os, yaml
|
||||
|
||||
def _env(n: str, d: Optional[str]=None) -> str:
|
||||
v = os.getenv(n)
|
||||
return v if v is not None else (d or "")
|
||||
|
||||
def _load_types() -> dict:
|
||||
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def _get_types_map(reg: dict) -> dict:
|
||||
if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
|
||||
return reg["types"]
|
||||
return reg if isinstance(reg, dict) else {}
|
||||
|
||||
def _get_defaults(reg: dict) -> dict:
|
||||
if isinstance(reg, dict) and isinstance(reg.get("defaults"), dict):
|
||||
return reg["defaults"]
|
||||
if isinstance(reg, dict) and isinstance(reg.get("global"), dict):
|
||||
return reg["global"]
|
||||
return {}
|
||||
|
||||
def _as_float(x: Any):
|
||||
try: return float(x)
|
||||
except Exception: return None
|
||||
|
||||
def _resolve_chunk_profile_from_config(note_type: str, reg: dict) -> Optional[str]:
|
||||
# 1. Type Level
|
||||
types = _get_types_map(reg)
|
||||
if isinstance(types, dict):
|
||||
t = types.get(note_type, {})
|
||||
if isinstance(t, dict):
|
||||
cp = t.get("chunking_profile") or t.get("chunk_profile")
|
||||
if isinstance(cp, str) and cp: return cp
|
||||
# 2. Defaults Level
|
||||
defs = _get_defaults(reg)
|
||||
if isinstance(defs, dict):
|
||||
cp = defs.get("chunking_profile") or defs.get("chunk_profile")
|
||||
if isinstance(cp, str) and cp: return cp
|
||||
return None
|
||||
|
||||
def _resolve_retriever_weight_from_config(note_type: str, reg: dict) -> float:
|
||||
"""
|
||||
Liest Weight nur aus Config (Type > Default).
|
||||
Wird aufgerufen, wenn im Frontmatter nichts steht.
|
||||
"""
|
||||
# 1. Type Level
|
||||
types = _get_types_map(reg)
|
||||
if isinstance(types, dict):
|
||||
t = types.get(note_type, {})
|
||||
if isinstance(t, dict) and (t.get("retriever_weight") is not None):
|
||||
v = _as_float(t.get("retriever_weight"))
|
||||
if v is not None: return float(v)
|
||||
|
||||
# 2. Defaults Level
|
||||
defs = _get_defaults(reg)
|
||||
if isinstance(defs, dict) and (defs.get("retriever_weight") is not None):
|
||||
v = _as_float(defs.get("retriever_weight"))
|
||||
if v is not None: return float(v)
|
||||
|
||||
return 1.0
|
||||
|
||||
def _as_list(x):
|
||||
if x is None: return []
|
||||
if isinstance(x, list): return x
|
||||
return [x]
|
||||
|
||||
def make_chunk_payloads(note: Dict[str, Any],
|
||||
note_path: str,
|
||||
chunks_from_chunker: List[Any],
|
||||
*,
|
||||
note_text: str = "",
|
||||
types_cfg: Optional[dict] = None,
|
||||
file_path: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Erstellt die Payloads für die Chunks.
|
||||
|
||||
Argument 'note' kann sein:
|
||||
A) Ein komplexes Objekt/Dict mit Key "frontmatter" (Legacy / Tests)
|
||||
B) Direkt das Frontmatter-Dictionary (Call aus ingestion.py)
|
||||
"""
|
||||
|
||||
# --- FIX 3: Intelligente Erkennung der Input-Daten ---
|
||||
# Wir prüfen: Ist 'note' ein Container MIT 'frontmatter', oder IST es das 'frontmatter'?
|
||||
if isinstance(note, dict) and "frontmatter" in note and isinstance(note["frontmatter"], dict):
|
||||
# Fall A: Container (wir müssen auspacken)
|
||||
fm = note["frontmatter"]
|
||||
else:
|
||||
# Fall B: Direktes Dict (so ruft ingestion.py es auf!)
|
||||
fm = note or {}
|
||||
|
||||
note_type = fm.get("type") or note.get("type") or "concept"
|
||||
|
||||
# Title Extraction (Fallback Chain)
|
||||
title = fm.get("title") or note.get("title") or fm.get("id") or "Untitled"
|
||||
|
||||
reg = types_cfg if isinstance(types_cfg, dict) else _load_types()
|
||||
|
||||
# --- Profil-Ermittlung ---
|
||||
# Da wir 'fm' jetzt korrekt haben, funktionieren diese lookups:
|
||||
cp = fm.get("chunking_profile") or fm.get("chunk_profile")
|
||||
|
||||
if not cp:
|
||||
cp = _resolve_chunk_profile_from_config(note_type, reg)
|
||||
if not cp:
|
||||
cp = "sliding_standard"
|
||||
|
||||
# --- Retriever Weight Ermittlung ---
|
||||
rw = fm.get("retriever_weight")
|
||||
|
||||
if rw is None:
|
||||
rw = _resolve_retriever_weight_from_config(note_type, reg)
|
||||
|
||||
try:
|
||||
rw = float(rw)
|
||||
except Exception:
|
||||
rw = 1.0
|
||||
|
||||
tags = fm.get("tags") or []
|
||||
if isinstance(tags, str):
|
||||
tags = [tags]
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
for idx, ch in enumerate(chunks_from_chunker):
|
||||
# Attribute extrahieren
|
||||
cid = getattr(ch, "id", None) or (ch.get("id") if isinstance(ch, dict) else None)
|
||||
nid = getattr(ch, "note_id", None) or (ch.get("note_id") if isinstance(ch, dict) else fm.get("id"))
|
||||
index = getattr(ch, "index", None) or (ch.get("index") if isinstance(ch, dict) else idx)
|
||||
text = getattr(ch, "text", None) or (ch.get("text") if isinstance(ch, dict) else "")
|
||||
window = getattr(ch, "window", None) or (ch.get("window") if isinstance(ch, dict) else text)
|
||||
prev_id = getattr(ch, "neighbors_prev", None) or (ch.get("neighbors_prev") if isinstance(ch, dict) else None)
|
||||
next_id = getattr(ch, "neighbors_next", None) or (ch.get("neighbors_next") if isinstance(ch, dict) else None)
|
||||
|
||||
pl: Dict[str, Any] = {
|
||||
"note_id": nid,
|
||||
"chunk_id": cid,
|
||||
"title": title,
|
||||
"index": int(index),
|
||||
"ord": int(index) + 1,
|
||||
"type": note_type,
|
||||
"tags": tags,
|
||||
"text": text,
|
||||
"window": window,
|
||||
"neighbors_prev": _as_list(prev_id),
|
||||
"neighbors_next": _as_list(next_id),
|
||||
"section": getattr(ch, "section", None) or (ch.get("section") if isinstance(ch, dict) else ""),
|
||||
"path": note_path,
|
||||
"source_path": file_path or note_path,
|
||||
"retriever_weight": float(rw),
|
||||
"chunk_profile": cp, # Jetzt endlich mit dem Override-Wert!
|
||||
}
|
||||
|
||||
# Cleanup
|
||||
for alias in ("chunk_num", "Chunk_Number"):
|
||||
pl.pop(alias, None)
|
||||
|
||||
out.append(pl)
|
||||
|
||||
return out
|
||||
@@ -1,10 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/chunker.py
|
||||
DESCRIPTION: Facade für das Chunking-Package. Stellt 100% Abwärtskompatibilität sicher.
|
||||
VERSION: 3.3.0
|
||||
"""
|
||||
from .chunking.chunking_processor import assemble_chunks
|
||||
from .chunking.chunking_utils import get_chunk_config, extract_frontmatter_from_text
|
||||
from .chunking.chunking_models import Chunk
|
||||
|
||||
__all__ = ["assemble_chunks", "get_chunk_config", "extract_frontmatter_from_text", "Chunk"]
|
||||
@@ -1,15 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/ingestion.py
|
||||
DESCRIPTION: Facade für das Ingestion-Package. Stellt 100% Abwärtskompatibilität sicher.
|
||||
WP-14: Modularisierung der Ingestion-Pipeline abgeschlossen.
|
||||
Nutzt interne Module mit 'ingestion_' Präfix für maximale Wartbarkeit.
|
||||
VERSION: 2.13.0
|
||||
STATUS: Active
|
||||
"""
|
||||
# Export der Hauptklasse für externe Module (z.B. scripts/import_markdown.py)
|
||||
from .ingestion.ingestion_processor import IngestionService
|
||||
|
||||
# Export der Hilfsfunktionen für Abwärtskompatibilität
|
||||
from .ingestion.ingestion_utils import extract_json_from_response, load_type_registry
|
||||
|
||||
__all__ = ["IngestionService", "extract_json_from_response", "load_type_registry"]
|
||||
@@ -18,7 +18,7 @@ from app.core.parser import (
|
||||
read_markdown, pre_scan_markdown, normalize_frontmatter,
|
||||
validate_required_frontmatter, NoteContext
|
||||
)
|
||||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunking import assemble_chunks
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
||||
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ def resolve_note_type(registry: dict, requested: Optional[str]) -> str:
|
||||
|
||||
def get_chunk_config_by_profile(registry: dict, profile_name: str, note_type: str) -> Dict[str, Any]:
|
||||
"""Holt die Chunker-Parameter für ein spezifisches Profil aus der Registry."""
|
||||
from app.core.chunker import get_chunk_config
|
||||
from app.core.chunking import get_chunk_config
|
||||
profiles = registry.get("chunking_profiles", {})
|
||||
if profile_name in profiles:
|
||||
cfg = profiles[profile_name].copy()
|
||||
|
||||
@@ -1,268 +0,0 @@
|
||||
"""
|
||||
FILE: app/core/note_payload.py
|
||||
DESCRIPTION: Baut das JSON-Objekt.
|
||||
FEATURES:
|
||||
1. Multi-Hash: Berechnet immer 'body' AND 'full' Hashes für flexible Change Detection.
|
||||
2. Config-Fix: Liest korrekt 'chunking_profile' aus types.yaml (statt Legacy 'chunk_profile').
|
||||
VERSION: 2.3.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: yaml, os, json, pathlib, hashlib
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Tuple, Optional
|
||||
import os
|
||||
import json
|
||||
import pathlib
|
||||
import hashlib
|
||||
|
||||
try:
|
||||
import yaml # type: ignore
|
||||
except Exception:
|
||||
yaml = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _as_dict(x) -> Dict[str, Any]:
|
||||
"""Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen."""
|
||||
if isinstance(x, dict):
|
||||
return dict(x)
|
||||
|
||||
out: Dict[str, Any] = {}
|
||||
for attr in (
|
||||
"frontmatter",
|
||||
"body",
|
||||
"id",
|
||||
"note_id",
|
||||
"title",
|
||||
"path",
|
||||
"tags",
|
||||
"type",
|
||||
"created",
|
||||
"modified",
|
||||
"date",
|
||||
):
|
||||
if hasattr(x, attr):
|
||||
val = getattr(x, attr)
|
||||
if val is not None:
|
||||
out[attr] = val
|
||||
|
||||
if not out:
|
||||
out["raw"] = str(x)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]:
|
||||
path = kwargs.get("path") or (args[0] if args else None)
|
||||
types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None
|
||||
return path, types_cfg
|
||||
|
||||
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
try:
|
||||
return float(os.environ.get(name, default))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _ensure_list(x) -> list:
|
||||
if x is None:
|
||||
return []
|
||||
if isinstance(x, list):
|
||||
return [str(i) for i in x]
|
||||
if isinstance(x, (set, tuple)):
|
||||
return [str(i) for i in x]
|
||||
return [str(x)]
|
||||
|
||||
# --- Hash Logic ---
|
||||
def _compute_hash(content: str) -> str:
|
||||
"""Berechnet einen SHA-256 Hash für den gegebenen String."""
|
||||
if not content:
|
||||
return ""
|
||||
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
def _get_hash_source_content(n: Dict[str, Any], mode: str) -> str:
|
||||
"""
|
||||
Stellt den String zusammen, der gehasht werden soll.
|
||||
"""
|
||||
body = str(n.get("body") or "")
|
||||
|
||||
if mode == "body":
|
||||
return body
|
||||
|
||||
if mode == "full":
|
||||
fm = n.get("frontmatter") or {}
|
||||
# Wichtig: Sortierte Keys für deterministisches Verhalten!
|
||||
# Wir nehmen alle steuernden Metadaten auf
|
||||
meta_parts = []
|
||||
# Hier checken wir keys, die eine Neu-Indizierung rechtfertigen würden
|
||||
for k in sorted(["title", "type", "status", "tags", "chunking_profile", "chunk_profile", "retriever_weight"]):
|
||||
val = fm.get(k)
|
||||
if val is not None:
|
||||
meta_parts.append(f"{k}:{val}")
|
||||
|
||||
meta_str = "|".join(meta_parts)
|
||||
return f"{meta_str}||{body}"
|
||||
|
||||
return body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Type-Registry laden
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict:
|
||||
if explicit_cfg and isinstance(explicit_cfg, dict):
|
||||
return explicit_cfg
|
||||
|
||||
path = os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
|
||||
if not os.path.isfile(path) or yaml is None:
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _cfg_for_type(note_type: str, reg: dict) -> dict:
|
||||
if not isinstance(reg, dict):
|
||||
return {}
|
||||
types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
|
||||
return types.get(note_type, {}) if isinstance(types, dict) else {}
|
||||
|
||||
|
||||
def _cfg_defaults(reg: dict) -> dict:
|
||||
if not isinstance(reg, dict):
|
||||
return {}
|
||||
for key in ("defaults", "default", "global"):
|
||||
v = reg.get(key)
|
||||
if isinstance(v, dict):
|
||||
return v
|
||||
return {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Haupt-API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
"""
|
||||
Baut das Note-Payload für mindnet_notes auf.
|
||||
Inkludiert Hash-Berechnung (Body & Full) und korrigierte Config-Lookups.
|
||||
"""
|
||||
n = _as_dict(note)
|
||||
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
|
||||
reg = _load_types_config(types_cfg_explicit)
|
||||
|
||||
# Hash Config (Parameter für Source/Normalize, Mode ist hardcoded auf 'beide')
|
||||
hash_source = kwargs.get("hash_source", "parsed")
|
||||
hash_normalize = kwargs.get("hash_normalize", "canonical")
|
||||
|
||||
fm = n.get("frontmatter") or {}
|
||||
fm_type = fm.get("type") or n.get("type") or "concept"
|
||||
note_type = str(fm_type)
|
||||
|
||||
cfg_type = _cfg_for_type(note_type, reg)
|
||||
cfg_def = _cfg_defaults(reg)
|
||||
|
||||
# --- retriever_weight ---
|
||||
default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
|
||||
retriever_weight = fm.get("retriever_weight")
|
||||
if retriever_weight is None:
|
||||
retriever_weight = cfg_type.get(
|
||||
"retriever_weight",
|
||||
cfg_def.get("retriever_weight", default_rw),
|
||||
)
|
||||
try:
|
||||
retriever_weight = float(retriever_weight)
|
||||
except Exception:
|
||||
retriever_weight = default_rw
|
||||
|
||||
# --- chunk_profile (FIXED LOGIC) ---
|
||||
# 1. Frontmatter Override (beide Schreibweisen erlaubt)
|
||||
chunk_profile = fm.get("chunking_profile") or fm.get("chunk_profile")
|
||||
|
||||
# 2. Type Config (Korrekter Key 'chunking_profile' aus types.yaml)
|
||||
if chunk_profile is None:
|
||||
chunk_profile = cfg_type.get("chunking_profile")
|
||||
|
||||
# 3. Default Config (Fallback auf sliding_standard statt medium)
|
||||
if chunk_profile is None:
|
||||
chunk_profile = cfg_def.get("chunking_profile", "sliding_standard")
|
||||
|
||||
# 4. Safety Fallback
|
||||
if not isinstance(chunk_profile, str) or not chunk_profile:
|
||||
chunk_profile = "sliding_standard"
|
||||
|
||||
# --- edge_defaults ---
|
||||
edge_defaults = fm.get("edge_defaults")
|
||||
if edge_defaults is None:
|
||||
edge_defaults = cfg_type.get(
|
||||
"edge_defaults",
|
||||
cfg_def.get("edge_defaults", []),
|
||||
)
|
||||
edge_defaults = _ensure_list(edge_defaults)
|
||||
|
||||
# --- Basis-Metadaten ---
|
||||
note_id = n.get("note_id") or n.get("id") or fm.get("id")
|
||||
title = n.get("title") or fm.get("title") or ""
|
||||
path = n.get("path") or path_arg
|
||||
if isinstance(path, pathlib.Path):
|
||||
path = str(path)
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"note_id": note_id,
|
||||
"title": title,
|
||||
"type": note_type,
|
||||
"path": path or "",
|
||||
"retriever_weight": retriever_weight,
|
||||
"chunk_profile": chunk_profile,
|
||||
"edge_defaults": edge_defaults,
|
||||
"hashes": {} # Init Hash Dict
|
||||
}
|
||||
|
||||
# --- MULTI-HASH CALCULATION (Strategy Decoupling) ---
|
||||
# Wir berechnen immer BEIDE Strategien und speichern sie.
|
||||
# ingestion.py entscheidet dann anhand der ENV-Variable, welcher verglichen wird.
|
||||
modes_to_calc = ["body", "full"]
|
||||
|
||||
for mode in modes_to_calc:
|
||||
content_to_hash = _get_hash_source_content(n, mode)
|
||||
computed_hash = _compute_hash(content_to_hash)
|
||||
# Key Schema: mode:source:normalize (z.B. "full:parsed:canonical")
|
||||
key = f"{mode}:{hash_source}:{hash_normalize}"
|
||||
payload["hashes"][key] = computed_hash
|
||||
|
||||
# Tags / Keywords
|
||||
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
|
||||
if tags:
|
||||
payload["tags"] = _ensure_list(tags)
|
||||
|
||||
# Aliases
|
||||
aliases = fm.get("aliases")
|
||||
if aliases:
|
||||
payload["aliases"] = _ensure_list(aliases)
|
||||
|
||||
# Zeit
|
||||
for k in ("created", "modified", "date"):
|
||||
v = fm.get(k) or n.get(k)
|
||||
if v:
|
||||
payload[k] = str(v)
|
||||
|
||||
# Fulltext
|
||||
if "body" in n and n["body"]:
|
||||
payload["fulltext"] = str(n["body"])
|
||||
|
||||
# JSON Validation
|
||||
json.loads(json.dumps(payload, ensure_ascii=False))
|
||||
|
||||
return payload
|
||||
Reference in new issue
Block a user