Dateien nach "scripts" hochladen

scripts/import_markdown.py

@@ -1,58 +1,57 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
+Script: scripts/import_markdown.py - Markdown -> Qdrant (Notes, Chunks, Edges)
-Version: 3.7.2
+Version: 3.7.3
-Datum: 2025-09-30
+Date: 2025-11-08
 
-Kurzbeschreibung
+Summary
-----------------
+-------
-- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
+- Reads Markdown files and creates Notes/Chunks/Edges idempotently.
-- Änderungserkennung „Option C“: mehrere Hash-Varianten werden parallel in der Note
+- Change detection "Option C": multiple hash variants are stored in note payload
-  gespeichert (Feld `hashes` mit Schlüsseln `<mode>:<source>:<normalize>`). Der Vergleich
+  (field `hashes` with keys `<mode>:<source>:<normalize>`). Comparison uses ONLY
-  nutzt NUR den aktuellen Modus-Key — ein Moduswechsel triggert keine Massenänderungen mehr.
+  the current-mode key. Switching modes no longer triggers bulk "changed".
-- „Erstimport-Fix“: Bei leerem Qdrant gilt Create-Fall automatisch als geändert.
+- "First import" fix: if Qdrant is empty for the note, we treat it as changed.
-- `--baseline-modes`: fehlende Hash-Varianten still nachtragen (nur Notes upserten).
+- `--baseline-modes`: silently add any missing hash variants (notes only).
-- `--sync-deletes`: gezielte Lösch-Synchronisation (Dry-Run + Apply).
+- `--sync-deletes`: selective deletes (Dry-Run + Apply).
-- `--only-path`: exakt **eine** Datei (Pfad) importieren — nützlich für Diagnosefälle.
+- `--only-path`: process exactly one file (useful for diagnostics).
+- NEW in 3.7.3: Type registry (if present) is loaded and the derived
+  `retriever_weight` is written to both note and chunk payload. If the registry
+  defines a `chunk_profile`, it is injected into frontmatter for payloads
+  (does NOT change the chunking behavior here).
 
-Neu in 3.7.1/3.7.2
+Hash/Compare configuration
-------------------
-- Chunk-Payloads: `window` (für Embeddings), `text` (überlappungsfrei, verlustfrei rekonstruierbar),
-  `start/end/overlap_*`. Embeddings nutzen `window`.
-- **3.7.2:** Edges-Fehler führen nicht mehr zum Abbruch der gesamten Note; Note/Chunks werden trotzdem geschrieben.
-
-Hash/Compare Konfiguration
 --------------------------
-- Vergleichsmodus:
+- Compare mode:
     --hash-mode body|frontmatter|full
-    oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE
+    or ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE
-- Quelle:
+- Source:
     --hash-source parsed|raw          (ENV: MINDNET_HASH_SOURCE, Default parsed)
-- Normalisierung:
+- Normalization:
     --hash-normalize canonical|none   (ENV: MINDNET_HASH_NORMALIZE, Default canonical)
-- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich
+- Optional: --compare-text (or ENV MINDNET_COMPARE_TEXT=true) compares
-  den parsed Body-Text direkt.
+  parsed body text, in addition to the hash.
 
 Qdrant / ENV
 ------------
 - QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
-- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
+- COLLECTION_PREFIX (Default: mindnet), can be overridden with --prefix
 - VECTOR_DIM (Default: 384)
 - MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
+- MINDNET_TYPES_PATH: optional path to config/types.yaml
 
-Beispiele
+Examples
----------
+--------
     # Standard (Body, parsed, canonical)
     python3 -m scripts.import_markdown --vault ./vault
 
-    # Erstimport nach truncate (Create-Fall)
+    # First import after truncate (create case)
     python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
 
-    # Nur eine Datei (Diagnose)
+    # Single file (diagnostics)
     python3 -m scripts.import_markdown --vault ./vault --only-path ./vault/30_projects/project-demo.md --apply
 
-    # Sync-Deletes (Dry-Run → Apply)
+    # Sync-Deletes (Dry-Run -> Apply)
     python3 -m scripts.import_markdown --vault ./vault --sync-deletes
     python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply
 """
@@ -92,11 +91,14 @@ from app.core.qdrant_points import (
     upsert_batch,
 )
 
+# ---- Type-Registry (optional) ------------------------------------------------
 try:
-    from app.core.embed import embed_texts  # optional
+    # Expected API
-except Exception:
+    from app.core.type_registry import load_type_registry  # type: ignore
-    embed_texts = None
+except Exception:  # pragma: no cover
-
+    def load_type_registry(path: str) -> dict:
+        """Fallback loader if module is absent. Returns empty dict."""
+        return {}
 
 # ---------------------------------------------------------------------
 # Helper
@@ -200,6 +202,33 @@ def _env(key: str, default: str) -> str:
     return (os.environ.get(key) or default).strip().lower()
 
 
+# --- Type-Registry helpers (pure) --------------------------------------------
+
+def _effective_chunk_profile(note_type: str, registry: dict) -> Optional[str]:
+    try:
+        types = (registry or {}).get("types", {}) if isinstance(registry, dict) else {}
+        cfg = types.get(note_type) or types.get("concept") or {}
+        prof = cfg.get("chunk_profile")
+        if isinstance(prof, str) and prof in {"short", "medium", "long"}:
+            return prof
+    except Exception:
+        pass
+    return None
+
+
+def _effective_retriever_weight(note_type: str, registry: dict) -> Optional[float]:
+    try:
+        types = (registry or {}).get("types", {}) if isinstance(registry, dict) else {}
+        cfg = types.get(note_type) or types.get("concept") or {}
+        w = cfg.get("retriever_weight")
+        if w is None:
+            return None
+        return float(w)
+    except Exception:
+        # be tolerant
+        return None
+
+
 # ---------------------------------------------------------------------
 # Main
 # ---------------------------------------------------------------------
@@ -207,31 +236,31 @@ def _env(key: str, default: str) -> str:
 def main() -> None:
     load_dotenv()
     ap = argparse.ArgumentParser()
-    ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
+    ap.add_argument("--vault", required=True, help="Path to the Obsidian vault (root folder)")
-    ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
+    ap.add_argument("--apply", action="store_true", help="Write to Qdrant; otherwise Dry-Run")
     ap.add_argument("--purge-before-upsert", action="store_true",
-                    help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
+                    help="Before upsert, delete Chunks & Edges of the CHANGED note")
-    ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
+    ap.add_argument("--note-id", help="Process only a specific note-id")
-    ap.add_argument("--only-path", help="Exakt diesen Markdown-Pfad verarbeiten (ignoriert --note-id)")
+    ap.add_argument("--only-path", help="Process exactly this Markdown path (ignores --note-id)")
-    ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
+    ap.add_argument("--embed-note", action="store_true", help="Optionally embed the full note text")
     ap.add_argument("--force-replace", action="store_true",
-                    help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
+                    help="Ignore change detection and always upsert (+ optional purge)")
     ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None,
-                    help="Vergleichsmodus (Body | Frontmatter | Full)")
+                    help="Compare mode (Body | Frontmatter | Full)")
     ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
     ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
-                    help="Quelle für die Hash-Berechnung (Default: parsed)")
+                    help="Source for hash calculation (Default: parsed)")
     ap.add_argument("--note-scope-refs", action="store_true",
-                    help="(Optional) erzeugt zusätzlich references:note/backlink:note (Default: aus)")
+                    help="(Optional) also create references:note/backlink:note (Default: off)")
     ap.add_argument("--debug-hash-diff", action="store_true",
-                    help="(reserviert) optionaler Body-Diff")
+                    help="(reserved) optional body diff")
     ap.add_argument("--compare-text", action="store_true",
-                    help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
+                    help="Additionally compare parsed fulltext (beyond the hash)")
     ap.add_argument("--baseline-modes", action="store_true",
-                    help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
+                    help="Add missing hash variants in 'hashes' silently (Upsert notes only)")
     ap.add_argument("--sync-deletes", action="store_true",
-                    help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)")
+                    help="Delete notes/chunks/edges that exist in Qdrant but are missing in the vault (Dry-Run; use --apply to execute)")
-    ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
+    ap.add_argument("--prefix", help="Collection prefix (overrides ENV COLLECTION_PREFIX)")
     args = ap.parse_args()
 
     mode = _resolve_mode(args.hash_mode)  # body|frontmatter|full
@@ -241,6 +270,7 @@ def main() -> None:
     note_scope_refs = args.note_scope_refs or note_scope_refs_env
     compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
 
+    # Prepare Qdrant client
     cfg = QdrantConfig.from_env()
     if args.prefix:
         cfg.prefix = args.prefix.strip()
@@ -248,19 +278,27 @@ def main() -> None:
     ensure_collections(client, cfg.prefix, cfg.dim)
     ensure_payload_indexes(client, cfg.prefix)
 
+    # Load type registry (optional)
+    types_path = os.environ.get("MINDNET_TYPES_PATH") or os.path.join(os.getcwd(), "config", "types.yaml")
+    try:
+        type_registry = load_type_registry(types_path) or {}
+    except Exception as e:  # tolerant
+        print(json.dumps({"warn": f"type-registry load failed ({types_path}): {type(e).__name__}: {e}"}))
+        type_registry = {}
+
     root = os.path.abspath(args.vault)
 
-    # Dateiliste bestimmen
+    # Build file list
     if args.only_path:
         only = os.path.abspath(args.only_path)
         files = [only]
     else:
         files = iter_md(root)
     if not files:
-        print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
+        print("No Markdown files found.", file=sys.stderr)
         sys.exit(2)
 
-    # Optional: Sync-Deletes vorab
+    # Optional: Sync-Deletes upfront
     if args.sync_deletes:
         vault_note_ids: Set[str] = set()
         for path in files:
@@ -315,6 +353,15 @@ def main() -> None:
 
         processed += 1
 
+        # -------- Type-Registry derivation (tolerant) --------
+        note_type = (fm.get("type") or "concept").strip().lower()
+        prof = _effective_chunk_profile(note_type, type_registry)
+        if prof and not fm.get("chunk_profile"):
+            fm["chunk_profile"] = prof
+        weight = _effective_retriever_weight(note_type, type_registry)
+        if weight is not None:
+            fm["retriever_weight"] = weight
+
         # -------- Build new payload (includes 'hashes') --------
         note_pl = make_note_payload(
             parsed,
@@ -326,6 +373,8 @@ def main() -> None:
         )
         if not note_pl.get("fulltext"):
             note_pl["fulltext"] = getattr(parsed, "body", "") or ""
+        if weight is not None:
+            note_pl["retriever_weight"] = weight
 
         note_id = note_pl.get("note_id") or fm.get("id")
         if not note_id:
@@ -337,6 +386,7 @@ def main() -> None:
         has_old = old_payload is not None
 
         old_hashes = (old_payload or {}).get("hashes") or {}
+        key_current = f"{mode}:{src}:{norm}"
         old_hash_exact = old_hashes.get(key_current)
         new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
         needs_baseline = (old_hash_exact is None)
@@ -356,8 +406,12 @@ def main() -> None:
         chunk_pls: List[Dict[str, Any]] = []
         try:
             body_text = getattr(parsed, "body", "") or ""
-            chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
+            chunks = assemble_chunks(fm["id"], body_text, note_type)
             chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
+            if weight is not None:
+                for pl in chunk_pls:
+                    if pl.get("retriever_weight") is None:
+                        pl["retriever_weight"] = weight
         except Exception as e:
             print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
             continue
@@ -385,13 +439,15 @@ def main() -> None:
             except Exception as e:
                 edges_failed = True
                 edges = []
-                # WICHTIG: Wir brechen NICHT mehr ab — Note & Chunks werden geschrieben.
                 print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
 
         # -------- Summary --------
         summary = {
             "note_id": note_id,
             "title": fm.get("title"),
+            "type": note_type,
+            "chunk_profile": fm.get("chunk_profile"),
+            "retriever_weight": weight,
             "chunks": len(chunk_pls),
             "edges": len(edges),
             "edges_failed": edges_failed,
@@ -448,31 +504,3 @@ def main() -> None:
 
 if __name__ == "__main__":
     main()
-
-
-# --- Type-Registry helper shims (safe if unused) ---
-
-def effective_chunk_profile(note_type: str, registry: dict) -> str | None:
-    try:
-        reg = registry or {}
-        types = reg.get("types", {}) if isinstance(reg, dict) else {}
-        # take exact type or fallback to concept
-        cfg = types.get(note_type) or types.get("concept") or {}
-        prof = cfg.get("chunk_profile")
-        if isinstance(prof, str) and prof in {"short", "medium", "long"}:
-            return prof
-    except Exception:
-        pass
-    return None
-
-def effective_retriever_weight(note_type: str, registry: dict) -> float | None:
-    try:
-        reg = registry or {}
-        types = reg.get("types", {}) if isinstance(reg, dict) else {}
-        cfg = types.get(note_type) or types.get("concept") or {}
-        w = cfg.get("retriever_weight")
-        if w is None:
-            return None
-        return float(w)
-    except Exception:
-        return None