Dateien nach "scripts" hochladen

scripts/import_markdown.py

@@ -1,33 +1,28 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-scripts/import_markdown.py  (V2.3.1)
+scripts/import_markdown.py  (Mindnet V2 — Importer, v2.5.0)
 
 Zweck
 -----
 - Liest Markdown-Notizen aus einem Vault ein
 - Erzeugt Note-Payload, Chunk-Payloads (+ optionale Embeddings) und Edges
 - Schreibt alles idempotent in Qdrant (Notes, Chunks, Edges)
-- Integriert eine optionale Type-Registry (types.yaml), um z. B. chunk_profile
-  und retriever_weight pro Notiz-Typ zu steuern.
+- Nutzt optional eine Type-Registry (`./config/types.yaml`) für:
+  - `chunk_profile` pro type
+  - `retriever_weight` pro type (Frontmatter > Registry > Default 1.0)
 
-Fix in V2.3.1
--------------
-- `retriever_weight` wird nun **immer deterministisch** gesetzt:
-  Frontmatter-Override > types.yaml > Default=1.0 (falls nichts konfiguriert).
-  Das Feld wird **zwingend** in mindnet_notes **und** mindnet_chunks geschrieben.
+Kernfeatures
+------------
+- Deterministische Hashes via `make_note_payload(...)` (hash_mode/source/normalize)
+- Idempotenter Upsert; Baseline-Hash-Merge je Modus
+- Optionales `--sync-deletes` (Vault → Qdrant)
+- Edges aus realen Referenzen + Nachbarschaft (next/prev/belongs_to) via `derive_edges.py`/`edges.py`
+- `retriever_weight` wird **immer** in Notes & Chunks geschrieben
+- Keine Duplikate wie `chunk_num`/`Chunk_Number` → wir halten uns an `index` (0‑basiert) und `ord` (= index+1)
 
-Kompatibilität & Fixes
-----------------------
-- Unterstützt sowohl app.core.derive_edges (bevorzugt) als auch app.core.edges als Fallback
-  → Aufruf erfolgt mit POSITIONSARGUMENTEN, damit alte Signaturen (note_level_refs vs. note_level_references)
-    nicht zu TypeError führen.
-- `scroll_filter` wird für alle Scrolls verwendet (Qdrant >= 1.7.x).
-- `--purge-before-upsert` entfernt alte Chunks/Edges einer Note, wenn sich die Note geändert hat.
-- Baseline-Hash-Strategie: hash_mode (body|frontmatter|full), hash_source (parsed|raw), hash_normalize (canonical|none).
-
-Aufrufbeispiele
----------------
+Aufruf
+------
     # Import (Apply + Purge-Update)
     python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
 
@@ -44,52 +39,53 @@ from typing import Dict, List, Optional, Tuple, Any, Set
 
 from dotenv import load_dotenv
 from qdrant_client.http import models as rest
+import yaml
 
 # --- Projekt-Imports ---
-from app.core.parser import (
-    read_markdown,
-    normalize_frontmatter,
-    validate_required_frontmatter,
-)
+from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
 from app.core.note_payload import make_note_payload
 from app.core.chunker import assemble_chunks
 from app.core.chunk_payload import make_chunk_payloads
 try:
-    # bevorzugt der robuste Builder
+    # bevorzugt: derive_edges (regelbasiert + real)
     from app.core.derive_edges import build_edges_for_note
 except Exception:  # pragma: no cover
-    # Fallback auf die einfache Variante
+    # Fallback: einfache Kanten in edges.py
     from app.core.edges import build_edges_for_note  # type: ignore
-from app.core.qdrant import (
-    QdrantConfig,
-    get_client,
-    ensure_collections,
-    ensure_payload_indexes,
-)
-from app.core.qdrant_points import (
-    points_for_chunks,
-    points_for_note,
-    points_for_edges,
-    upsert_batch,
-)
+from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
+from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
 
-# embeddings sind optional (z. B. beim Payload-Backfill)
+# embeddings optional
 try:
     from app.core.embed import embed_texts  # optional
 except Exception:  # pragma: no cover
-    embed_texts = None
+    embed_texts = None  # type: ignore
 
-# ------------------------------------------------------------
-# Type-Registry (types.yaml)
-# ------------------------------------------------------------
-import yaml
+
+# ============================================================
+# Type-Registry (config/types.yaml)
+# ============================================================
 
 def _env(name: str, default: Optional[str] = None) -> str:
     v = os.getenv(name)
     return v if v is not None else (default or "")
 
+def _deep_get(root: Any, path: str) -> Any:
+    cur = root
+    for key in path.split("."):
+        if not isinstance(cur, dict) or key not in cur:
+            return None
+        cur = cur[key]
+    return cur
+
+def _as_float(x: Any) -> Optional[float]:
+    try:
+        return float(x)
+    except Exception:
+        return None
+
 def load_type_registry() -> dict:
-    # ENV kann Pfad überschreiben; Standard: ./config/types.yaml
+    # ENV überschreibt Pfad
     path = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
     if not os.path.isfile(path):
         return {}
@@ -118,48 +114,51 @@ def effective_chunk_profile(note_type: str, reg: dict) -> Optional[str]:
         return prof
     return None
 
-def effective_retriever_weight_from_registry(note_type: str, reg: dict) -> Optional[float]:
-    cfg = get_type_config(note_type, reg)
-    w = cfg.get("retriever_weight")
-    try:
-        return float(w) if w is not None else None
-    except Exception:
-        return None
-
-def compute_effective_retriever_weight(fm: Dict[str, Any], note_type: str, reg: dict) -> float:
-    """Ermittelt den finalen retriever_weight:
-       1) Frontmatter-Override
-       2) types.yaml (für den type)
-       3) Default 1.0
+def effective_retriever_weight_from_registry(note_type: str, reg: dict) -> Tuple[Optional[float], Optional[str]]:
     """
-    # 1) Frontmatter-Override
-    if fm.get("retriever_weight") is not None:
-        try:
-            return float(fm.get("retriever_weight"))
-        except Exception:
-            pass
-    # 2) Registry
-    r = effective_retriever_weight_from_registry(note_type, reg)
-    if r is not None:
-        return float(r)
-    # 3) Default
-    return 1.0
+    Liefert (Wert, Pfad) aus Registry, wenn vorhanden.
+    Unterstützte Pfade (in dieser Reihenfolge):
+      - types.<type>.retriever_weight
+      - types.<type>.retriever.weight
+      - types.<type>.retrieval.weight
+      - defaults.retriever_weight
+      - defaults.retriever.weight
+      - global.retriever_weight
+      - global.retriever.weight
+    """
+    candidates = [
+        f"types.{note_type}.retriever_weight",
+        f"types.{note_type}.retriever.weight",
+        f"types.{note_type}.retrieval.weight",
+        "defaults.retriever_weight",
+        "defaults.retriever.weight",
+        "global.retriever_weight",
+        "global.retriever.weight",
+    ]
+    for path in candidates:
+        val = _deep_get(reg, path)
+        v = _as_float(val)
+        if v is not None:
+            return v, path
+    return None, None
 
-# ------------------------------------------------------------
-# Sonstige Helper
-# ------------------------------------------------------------
-def iter_md(root: str) -> List[str]:
-    out: List[str] = []
-    for dirpath, _, filenames in os.walk(root):
-        for fn in filenames:
-            if not fn.lower().endswith(".md"):
-                continue
-            p = os.path.join(dirpath, fn)
-            pn = p.replace("\\", "/")
-            if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
-                continue
-            out.append(p)
-    return sorted(out)
+def compute_effective_retriever_weight(fm: Dict[str, Any], note_type: str, reg: dict) -> Tuple[float, str]:
+    """
+    Priorität: Frontmatter > Registry > Default (1.0). Gibt (Wert, Quelle) zurück.
+    """
+    if fm.get("retriever_weight") is not None:
+        v = _as_float(fm.get("retriever_weight"))
+        if v is not None:
+            return v, "frontmatter.retriever_weight"
+    r, rpath = effective_retriever_weight_from_registry(note_type, reg)
+    if r is not None:
+        return float(r), f"types.yaml:{rpath}"
+    return 1.0, "default:1.0"
+
+
+# ============================================================
+# Qdrant Helpers
+# ============================================================
 
 def collections(prefix: str) -> Tuple[str, str, str]:
     return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
@@ -169,7 +168,7 @@ def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[D
     f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
     points, _ = client.scroll(
         collection_name=notes_col,
-        scroll_filter=f,  # wichtig: scroll_filter (nicht: filter)
+        scroll_filter=f,
         with_payload=True,
         with_vectors=False,
         limit=1,
@@ -244,9 +243,11 @@ def artifacts_missing(client, prefix: str, note_id: str) -> Tuple[bool, bool]:
     edges_missing = not _has_any_point(client, edges_col, note_id)
     return chunks_missing, edges_missing
 
-# ------------------------------------------------------------
+
+# ============================================================
 # Main
-# ------------------------------------------------------------
+# ============================================================
+
 def _resolve_mode(m: Optional[str]) -> str:
     m = (m or "body").strip().lower()
     return m if m in {"body", "frontmatter", "full"} else "body"
@@ -270,7 +271,7 @@ def main() -> None:
     ap.add_argument("--hash-normalize", help="canonical|none (Default canonical)")
     ap.add_argument("--compare-text", action="store_true", help="Parsed fulltext zusätzlich direkt vergleichen")
     ap.add_argument("--baseline-modes", action="store_true", help="Fehlende Hash-Varianten still nachtragen (Notes)")
-    ap.add_argument("--sync-deletes", action="store_true", help="Qdrant->Vault Lösch-Sync (Dry-Run; mit --apply ausführen)")
+    ap.add_argument("--sync-deletes", action="store_true", help="Qdrant->Vault Lösch-Sync (mit --apply ausführen)")
     ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
     args = ap.parse_args()
 
@@ -289,7 +290,7 @@ def main() -> None:
     ensure_collections(client, cfg.prefix, cfg.dim)
     ensure_payload_indexes(client, cfg.prefix)
 
-    # Type-Registry laden (optional)
+    # Type-Registry laden
     reg = load_type_registry()
 
     root = os.path.abspath(args.vault)
@@ -299,7 +300,17 @@ def main() -> None:
         only = os.path.abspath(args.only_path)
         files = [only]
     else:
-        files = iter_md(root)
+        files: List[str] = []
+        for dirpath, _, filenames in os.walk(root):
+            for fn in filenames:
+                if not fn.lower().endswith(".md"):
+                    continue
+                p = os.path.join(dirpath, fn)
+                pn = p.replace("\\", "/")
+                if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
+                    continue
+                files.append(p)
+        files.sort()
     if not files:
         print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
         sys.exit(2)
@@ -358,7 +369,7 @@ def main() -> None:
 
         processed += 1
 
-        # --- Type-Registry anwenden (chunk_profile / retriever_weight) ---
+        # --- Typ & Profile aus Registry übernehmen ---
         try:
             note_type = resolve_note_type(fm.get("type"), reg)
         except Exception:
@@ -369,9 +380,9 @@ def main() -> None:
         if prof:
             fm["chunk_profile"] = prof
 
-        # NEU: finalen retriever_weight deterministisch bestimmen
-        rw = compute_effective_retriever_weight(fm, note_type, reg)
-        fm["retriever_weight"] = rw  # Frontmatter spiegeln, damit nachfolgende Builder konsistent sind
+        # --- retriever_weight auflösen (FM > Registry > 1.0) ---
+        rw, rw_source = compute_effective_retriever_weight(fm, note_type, reg)
+        fm["retriever_weight"] = rw  # Spiegeln ins FM für Builder
 
         # --- Payload aufbauen (inkl. Hashes) ---
         try:
@@ -390,7 +401,7 @@ def main() -> None:
         if not note_pl.get("fulltext"):
             note_pl["fulltext"] = getattr(parsed, "body", "") or ""
 
-        # NEU: retriever_weight **immer** in Note-Payload setzen
+        # retriever_weight **immer** in Note-Payload schreiben
         try:
             note_pl["retriever_weight"] = float(rw)
         except Exception:
@@ -430,11 +441,18 @@ def main() -> None:
             print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
             continue
 
-        # NEU: retriever_weight **immer** auf Chunk-Payload spiegeln
-        try:
-            rwf = float(rw)
-        except Exception:
-            rwf = 1.0
+        # **Felder-Policy**: `index` (0-based) & `ord` (1-based) sicherstellen; keine doppelten Alias-Felder
+        for i, pl in enumerate(chunk_pls):
+            if "index" not in pl:
+                pl["index"] = i
+            pl["ord"] = int(pl.get("index", i)) + 1
+            # Entferne ggf. Alt-Aliase, um Duplikate zu vermeiden
+            for alias in ("chunk_num", "Chunk_Number"):
+                if alias in pl:
+                    pl.pop(alias, None)
+
+        # retriever_weight **immer** auf Chunk-Payload spiegeln
+        rwf = float(rw) if isinstance(rw, (int, float)) else 1.0
         for pl in chunk_pls:
             pl["retriever_weight"] = rwf
 
@@ -456,8 +474,8 @@ def main() -> None:
         should_build_edges = (changed and (not do_baseline_only)) or edges_missing
         if should_build_edges:
             try:
-                # ACHTUNG: POSITIONSARGUMENTE benutzen → kompatibel zu edges.py UND derive_edges.py
                 note_refs = note_pl.get("references") or []
+                # Positionsargumente → kompatibel mit derive_edges.py UND edges.py
                 edges = build_edges_for_note(note_id, chunk_pls, note_refs, include_note_scope_refs=note_scope_refs)
             except Exception as e:
                 edges_failed = True
@@ -470,6 +488,7 @@ def main() -> None:
             "title": fm.get("title"),
             "type": fm.get("type"),
             "rw": rw,
+            "rw_source": rw_source,
             "chunks": len(chunk_pls),
             "edges": len(edges),
             "edges_failed": edges_failed,