scripts/import_markdown.py aktualisiert

scripts/import_markdown.py

@@ -1,69 +1,490 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-tests/inspect_one_note.py
-Diagnose einer einzelnen Markdown-Datei (kein Upsert).
-Nutzung:
-  python3 tests/inspect_one_note.py --path ./test_vault/30_projects/project-demo.md
+Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
+Version: 3.7.1
+Datum: 2025-09-30
+
+Kurzbeschreibung
+----------------
+- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
+- Robuste Änderungserkennung (Option C): Mehrfach-Hashes werden parallel in der Note
+  gespeichert (Feld `hashes` mit Schlüsseln `<mode>:<source>:<normalize>`). Der Vergleich
+  erfolgt **modusgenau** anhand dieses Keys, sodass ein Wechsel des Vergleichsmodus **keine
+  Massenänderungen** mehr triggert.
+- **Erstimport-Fix:** Bei leerem Qdrant gilt Create-Fall automatisch als geändert.
+- **--baseline-modes:** fehlende Hash-Varianten „still“ nachtragen (nur Notes upserten).
+- **--sync-deletes:** Punkte in Qdrant, die im Vault fehlen, sicher löschen (Dry-Run + Apply).
+- **--prefix**: CLI-Override für COLLECTION_PREFIX.
+
+Neu in 3.7.1
+-------------
+- Chunk-Payloads enthalten nun
+    * `window` (Fenster inkl. Overlap, für Embeddings)
+    * `text`   (überlappungsfreies Segment, für verlustfreie Rekonstruktion)
+    * `start`, `end`, `overlap_left`, `overlap_right`
+- Embeddings werden aus `window` gebildet (mehr Kontext, bessere Retrieval-Qualität).
+- Die Offsets werden beim Build der Chunk-Payloads mitgegeben (`note_text`).
+
+Hash/Compare Konfiguration
+--------------------------
+- Vergleichsmodus:
+    --hash-mode body|frontmatter|full
+    oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE
+- Quelle:
+    --hash-source parsed|raw          (ENV: MINDNET_HASH_SOURCE, Default parsed)
+- Normalisierung:
+    --hash-normalize canonical|none   (ENV: MINDNET_HASH_NORMALIZE, Default canonical)
+- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich
+  den parsed Body-Text direkt.
+
+Qdrant / ENV
+------------
+- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
+- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
+- VECTOR_DIM (Default: 384)
+- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
+
+Beispiele
+---------
+    # Standard (Body, parsed, canonical)
+    python3 -m scripts.import_markdown --vault ./vault
+
+    # Erstimport nach truncate (Create-Fall)
+    python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
+
+    # Baseline für parsed:canonical „still“ auffüllen
+    MINDNET_HASH_SOURCE=parsed MINDNET_HASH_NORMALIZE=canonical \
+    python3 -m scripts.import_markdown --vault ./vault --apply --baseline-modes
+
+    # Frontmatter-Vergleich ohne Massenänderungen (nach Baseline)
+    MINDNET_HASH_COMPARE=Frontmatter \
+    python3 -m scripts.import_markdown --vault ./vault
+
+    # Sync-Deletes (Dry-Run → Apply)
+    python3 -m scripts.import_markdown --vault ./vault --sync-deletes
+    python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply
+
+    # Prefix explizit setzen
+    python3 -m scripts.import_markdown --vault ./vault --prefix mindnet
 """
 from __future__ import annotations
-import os, sys, json, argparse
 
-PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
-if PROJECT_ROOT not in sys.path:
-    sys.path.insert(0, PROJECT_ROOT)
+import argparse
+import json
+import os
+import sys
+from typing import Dict, List, Optional, Tuple, Any, Set
 
-from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
+from dotenv import load_dotenv
+from qdrant_client.http import models as rest
+
+from app.core.parser import (
+    read_markdown,
+    normalize_frontmatter,
+    validate_required_frontmatter,
+)
+from app.core.note_payload import make_note_payload
 from app.core.chunker import assemble_chunks
 from app.core.chunk_payload import make_chunk_payloads
+# Kompatibel zu beiden Modulnamen:
 try:
     from app.core.derive_edges import build_edges_for_note
-except Exception:
+except Exception:  # pragma: no cover
     from app.core.edges import build_edges_for_note  # type: ignore
 
-def main():
+from app.core.qdrant import (
+    QdrantConfig,
+    get_client,
+    ensure_collections,
+    ensure_payload_indexes,
+)
+from app.core.qdrant_points import (
+    points_for_chunks,
+    points_for_note,
+    points_for_edges,
+    upsert_batch,
+)
+
+try:
+    from app.core.embed import embed_texts  # optional
+except Exception:
+    embed_texts = None
+
+
+# ---------------------------------------------------------------------
+# Helper
+# ---------------------------------------------------------------------
+
+def iter_md(root: str) -> List[str]:
+    out: List[str] = []
+    for dirpath, _, filenames in os.walk(root):
+        for fn in filenames:
+            if not fn.lower().endswith(".md"):
+                continue
+            p = os.path.join(dirpath, fn)
+            pn = p.replace("\\", "/")
+            if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
+                continue
+            out.append(p)
+    return sorted(out)
+
+
+def collections(prefix: str) -> Tuple[str, str, str]:
+    return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
+
+
+def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
+    notes_col, _, _ = collections(prefix)
+    f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
+    points, _ = client.scroll(
+        collection_name=notes_col,
+        scroll_filter=f,
+        with_payload=True,
+        with_vectors=False,
+        limit=1,
+    )
+    if not points:
+        return None
+    return points[0].payload or {}
+
+
+def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
+    """Liest alle note_ids aus der Notes-Collection (per Scroll)."""
+    notes_col, _, _ = collections(prefix)
+    out: Set[str] = set()
+    next_page = None
+    while True:
+        pts, next_page = client.scroll(
+            collection_name=notes_col,
+            with_payload=True,
+            with_vectors=False,
+            limit=256,
+            offset=next_page,
+        )
+        if not pts:
+            break
+        for p in pts:
+            pl = p.payload or {}
+            nid = pl.get("note_id")
+            if isinstance(nid, str):
+                out.add(nid)
+        if next_page is None:
+            break
+    return out
+
+
+def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
+    """
+    Löscht alle Chunks & Edges zu einer Note mittels Filter-Selector (kompatibel mit aktuellen Qdrant-Clients).
+    """
+    _, chunks_col, edges_col = collections(prefix)
+    filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
+
+    try:
+        client.delete(
+            collection_name=chunks_col,
+            points_selector=rest.FilterSelector(filter=filt),
+            wait=True
+        )
+    except Exception as e:
+        print(json.dumps({"note_id": note_id, "warn": f"delete chunks via filter failed: {e}"}))
+
+    try:
+        client.delete(
+            collection_name=edges_col,
+            points_selector=rest.FilterSelector(filter=filt),
+            wait=True
+        )
+    except Exception as e:
+        print(json.dumps({"note_id": note_id, "warn": f"delete edges via filter failed: {e}"}))
+
+
+def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
+    """Löscht Note + zugehörige Chunks + Edges per Filter."""
+    notes_col, chunks_col, edges_col = collections(prefix)
+    filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
+    # Reihenfolge: erst edges/chunks, dann note
+    for col in (edges_col, chunks_col, notes_col):
+        try:
+            client.delete(
+                collection_name=col,
+                points_selector=rest.FilterSelector(filter=filt),
+                wait=True
+            )
+        except Exception as e:
+            print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
+
+
+def _resolve_mode(val: Optional[str]) -> str:
+    v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
+    if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
+        return "full"
+    if v in ("frontmatter", "fm"):
+        return "frontmatter"
+    return "body"
+
+
+def _env(key: str, default: str) -> str:
+    return (os.environ.get(key) or default).strip().lower()
+
+
+# ---------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------
+
+def main() -> None:
+    load_dotenv()
     ap = argparse.ArgumentParser()
-    ap.add_argument("--path", required=True)
+    ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
+    ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
+    ap.add_argument("--purge-before-upsert", action="store_true",
+                    help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
+    ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
+    ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
+    ap.add_argument("--force-replace", action="store_true",
+                    help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
+    ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None,
+                    help="Vergleichsmodus (Body | Frontmatter | Full)")
+    ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
+    ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
+                    help="Quelle für die Hash-Berechnung (Default: parsed)")
+    ap.add_argument("--note-scope-refs", action="store_true",
+                    help="(Optional) erzeugt zusätzlich references:note (Default: aus)")
+    ap.add_argument("--debug-hash-diff", action="store_true",
+                    help="(reserviert) optionaler Body-Diff (nicht nötig bei Option C)")
+    ap.add_argument("--compare-text", action="store_true",
+                    help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
+    ap.add_argument("--baseline-modes", action="store_true",
+                    help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
+    ap.add_argument("--sync-deletes", action="store_true",
+                    help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)")
+    ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
     args = ap.parse_args()
-    p = os.path.abspath(args.path)
 
-    try:
-        parsed = read_markdown(p)
+    mode = _resolve_mode(args.hash_mode)  # body|frontmatter|full
+    src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed")  # parsed|raw
+    norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical")  # canonical|none
+    note_scope_refs_env = (_env("MINDNET_NOTE_SCOPE_REFS", "false") == "true")
+    note_scope_refs = args.note_scoop_refs if hasattr(args, "note_scoop_refs") else (args.note_scope_refs or note_scope_refs_env)  # typo-safe
+    compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
+
+    cfg = QdrantConfig.from_env()
+    if args.prefix:
+        cfg.prefix = args.prefix.strip()
+    client = get_client(cfg)
+    ensure_collections(client, cfg.prefix, cfg.dim)
+    ensure_payload_indexes(client, cfg.prefix)
+
+    root = os.path.abspath(args.vault)
+    files = iter_md(root)
+    if not files:
+        print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
+        sys.exit(2)
+
+    # Optional: Sync-Deletes vorab
+    if args.sync_deletes:
+        # Vault-Note-IDs sammeln
+        vault_note_ids: Set[str] = set()
+        for path in files:
+            try:
+                parsed = read_markdown(path)
+                if not parsed:
+                    continue
+                fm = normalize_frontmatter(parsed.frontmatter)
+                nid = fm.get("id")
+                if isinstance(nid, str):
+                    vault_note_ids.add(nid)
+            except Exception:
+                continue
+        # Qdrant-Note-IDs sammeln
+        qdrant_note_ids = list_qdrant_note_ids(client, cfg.prefix)
+        to_delete = sorted(qdrant_note_ids - vault_note_ids)
+        print(json.dumps({
+            "action": "sync-deletes",
+            "prefix": cfg.prefix,
+            "qdrant_total": len(qdrant_note_ids),
+            "vault_total": len(vault_note_ids),
+            "to_delete_count": len(to_delete),
+            "to_delete": to_delete[:50] + (["…"] if len(to_delete) > 50 else [])
+        }, ensure_ascii=False))
+        if args.apply and to_delete:
+            for nid in to_delete:
+                print(json.dumps({"action": "delete", "note_id": nid, "decision": "apply"}))
+                delete_note_everywhere(client, cfg.prefix, nid)
+        # Danach normal mit Import fortfahren
+
+    key_current = f"{mode}:{src}:{norm}"
+
+    processed = 0
+    for path in files:
+        # -------- Parse & Validate --------
+        try:
+            parsed = read_markdown(path)
+        except Exception as e:
+            print(json.dumps({"path": path, "error": f"read_markdown failed: {e}"}))
+            continue
         if parsed is None:
-            print(json.dumps({"path": p, "error": "read_markdown returned None"}))
-            return
-    except Exception as e:
-        print(json.dumps({"path": p, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
-        return
+            print(json.dumps({"path": path, "error": "read_markdown returned None"}))
+            continue
 
-    try:
-        fm = normalize_frontmatter(parsed.frontmatter)
-        validate_required_frontmatter(fm)
-    except Exception as e:
-        print(json.dumps({"path": p, "error": f"frontmatter invalid: {type(e).__name__}: {e}"}))
-        return
+        try:
+            fm = normalize_frontmatter(parsed.frontmatter)
+            validate_required_frontmatter(fm)
+        except Exception as e:
+            print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"}))
+            continue
 
-    body = getattr(parsed, "body", "") or ""
-    chunks = assemble_chunks(fm["id"], body, fm.get("type", "concept"))
-    chunk_pls = make_chunk_payloads(fm, os.path.relpath(p, os.path.join(p, "..", "..")), chunks, note_text=body)
+        if args.note_id and fm.get("id") != args.note_id:
+            continue
 
-    edge_error = None
-    edges_count = 0
-    try:
-        edges = build_edges_for_note(fm["id"], chunk_pls, include_note_scope_refs=False)
-        edges_count = len(edges)
-    except Exception as e:
-        edge_error = f"{type(e).__name__}: {e}"
+        processed += 1
+
+        # -------- Build new payload (includes 'hashes') --------
+        note_pl = make_note_payload(
+            parsed,
+            vault_root=root,
+            hash_mode=mode,
+            hash_normalize=norm,
+            hash_source=src,
+            file_path=path,
+        )
+        if not note_pl.get("fulltext"):
+            note_pl["fulltext"] = getattr(parsed, "body", "") or ""
+
+        note_id = note_pl.get("note_id") or fm.get("id")
+        if not note_id:
+            print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
+            continue
+
+        # -------- Fetch old payload --------
+        old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
+        has_old = old_payload is not None
+
+        old_hashes = (old_payload or {}).get("hashes") or {}
+        old_hash_exact = old_hashes.get(key_current)
+
+        # Neu-Hash (aktueller Modus) aus neuem Payload
+        new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
+
+        needs_baseline = (old_hash_exact is None)
+
+        # Change-Detection:
+        # - CREATE: wenn es KEIN Alt-Payload gibt -> changed=True
+        # - UPDATE: baseline existiert UND Hash differiert
+        # - force/text_changed wie gehabt
+        hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact)
+
+        text_changed = False
+        if compare_text:
+            old_text = (old_payload or {}).get("fulltext") or ""
+            new_text = note_pl.get("fulltext") or ""
+            text_changed = (old_text != new_text)
+
+        changed = args.force_replace or (not has_old) or hash_changed or text_changed
+
+        # Baseline-only nur, wenn Alt-Payload existiert UND Key fehlt UND keine sonstige Änderung
+        do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
+
+        # -------- Optional: Chunks / Embeddings / Edges vorbereiten --------
+        chunk_pls: List[Dict[str, Any]] = []
+        edges: List[Dict[str, Any]] = []
+        vecs: List[List[float]] = []
+
+        if changed and (not do_baseline_only):
+            try:
+                # Chunks assemblen
+                body_text = getattr(parsed, "body", "") or ""
+                chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
+                # Chunk-Payloads inkl. Offsets/Overlap (neu in 3.7.1)
+                chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
+            except Exception as e:
+                print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {e}"}))
+                continue
+
+            # Embeddings: aus window (Fallback text)
+            if embed_texts and chunk_pls:
+                try:
+                    texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls]
+                    vecs = embed_texts(texts_for_embed)
+                except Exception as e:
+                    print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
+                    vecs = [[0.0] * cfg.dim for _ in chunk_pls]
+            else:
+                vecs = [[0.0] * cfg.dim for _ in chunk_pls]
+
+            # Edges
+            try:
+                note_refs = note_pl.get("references") or []
+                edges = build_edges_for_note(
+                    note_id,
+                    chunk_pls,
+                    note_refs,
+                    include_note_scope_refs=note_scope_refs,
+                )
+            except Exception as e:
+                print(json.dumps({"path": path, "note_id": note_id, "error": f"build_edges_for_note failed: {e}"}))
+                continue
+
+        # -------- Summary --------
+        summary = {
+            "note_id": note_id,
+            "title": fm.get("title"),
+            "chunks": len(chunk_pls),
+            "edges": len(edges),
+            "changed": changed,
+            "needs_baseline_for_mode": needs_baseline,
+            "decision": ("baseline-only" if args.apply and do_baseline_only else
+                         "apply" if args.apply and changed else
+                         "apply-skip-unchanged" if args.apply and not changed else
+                         "dry-run"),
+            "path": note_pl["path"],
+            "hash_mode": mode,
+            "hash_normalize": norm,
+            "hash_source": src,
+            "prefix": cfg.prefix,
+        }
+        print(json.dumps(summary, ensure_ascii=False))
+
+        # -------- Writes --------
+        if not args.apply:
+            continue
+
+        # BASELINE-ONLY: fehlenden Key nachtragen, ohne legacy Felder anzutasten
+        if do_baseline_only:
+            merged_hashes = {}
+            merged_hashes.update(old_hashes)
+            merged_hashes.update(note_pl.get("hashes") or {})
+            # Legacy-Hashfelder unverändert lassen, falls vorhanden
+            if old_payload:
+                note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext"))
+                note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature"))
+            note_pl["hashes"] = merged_hashes
+            notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
+            upsert_batch(client, notes_name, note_pts)
+            continue
+
+        # Normale CREATE/UPDATE
+        if not changed:
+            continue
+
+        if args.purge_before_upsert and has_old:
+            try:
+                purge_note_artifacts(client, cfg.prefix, note_id)
+            except Exception as e:
+                print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
+
+        notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
+        upsert_batch(client, notes_name, note_pts)
+        if chunk_pls:
+            chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
+            upsert_batch(client, chunks_name, chunk_pts)
+        if edges:
+            edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
+            upsert_batch(client, edges_name, edge_pts)
+
+    print(f"Done. Processed notes: {processed}")
 
-    print(json.dumps({
-        "path": p,
-        "id": fm.get("id"),
-        "title": fm.get("title"),
-        "body_len": len(body),
-        "chunks": len(chunk_pls),
-        "edges": edges_count,
-        "edge_error": edge_error
-    }, ensure_ascii=False, indent=2))
 
 if __name__ == "__main__":
     main()