scripts/import_markdown.py aktualisiert

scripts/import_markdown.py

@@ -2,52 +2,66 @@
 # -*- coding: utf-8 -*-
 """
 Script:      scripts/import_markdown.py
-Version:     v2.3.1 (2025-09-05)
+Version:     v2.4.0 (2025-09-05)
 
 Beschreibung
-  Importiert Markdown-Notizen in Qdrant (Notes, Chunks, Edges).
+  Importiert Markdown-Notizen in Qdrant (Notes, Chunks, Edges) mit Delta-Detection.
   - Chunking + Embedding (MiniLM 384d, externer Embed-Server)
   - Edges direkt beim Import aus Wikilinks ([[…]]) ableiten
-  - Idempotente UUIDv5-IDs; Collections werden bereitgestellt
+  - Idempotenz via stabile UUIDv5-IDs und hash_fulltext
+  - Create/Update/Skip pro Note:
+      * Unverändert (hash_fulltext gleich) ⇒ Skip
+      * Geändert ⇒ Chunks & Edges der Note purge + Replace (Upsert)
+  - Dry-Run löscht/ändert nichts; zeigt die Entscheidung je Note
 
 Aufruf
-  python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id ID] [--embed-note] [--force-replace]
+  python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id ID]
+                                     [--embed-note] [--force-replace]
 
 Parameter
-  --vault         Pfad zum Obsidian-Vault (erforderlich)
+  --vault           Pfad zum Obsidian-Vault (erforderlich)
-  --apply         Ohne Flag: Dry-Run (nur JSON-Zeilen). Mit Flag: schreibt in Qdrant.
+  --apply           Ohne Flag: Dry-Run (nur JSON-Zeilen). Mit Flag: schreibt in Qdrant.
-  --note-id       Nur eine spezifische Note-ID verarbeiten
+  --note-id         Nur eine spezifische Note-ID verarbeiten (Filter)
-  --embed-note    Optional: Note-Volltext zusätzlich einbetten
+  --embed-note      Optional: Note-Volltext zusätzlich einbetten
-  --force-replace Vor Upsert: zugehörige Edges der Quell-Note in Qdrant löschen (harte Ersetzung)
+  --force-replace   Erzwingt Neuaufbau von Chunks/Edges der Note (auch wenn Hash unverändert)
 
 Hinweise
-  - Im venv laufen: `source .venv/bin/activate`
+  - Im venv arbeiten: `source .venv/bin/activate`
-  - Erwartet laufenden Embed-Server (http://127.0.0.1:8990)
+  - Embed-Server muss laufen (http://127.0.0.1:8990)
-  - Qdrant via env: QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM
+  - Qdrant via ENV: QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM
 
 Changelog
-  v2.3.1: FIX – Für derive_wikilink_edges werden jetzt echte Chunk-Texte übergeben
+  v2.4.0: NEU – Delta-Detection über hash_fulltext; Skip/Replace-Entscheidung.
-          (chunks_for_links mit {"chunk_id","text"}), damit `references_at` erzeugt werden.
+          Purge bei Updates: löscht Chunks & Edges der Quelle, dann Upsert.
-  v2.3.0: Umstellung auf app.core.derive_edges; Edge-IDs unterstützen Occurrence.
+          Dry-Run garantiert ohne Mutationen.
-  v2.2.x: Fix Filter-API (pydantic v2) bei Purge; Kleinkosmetik.
+  v2.3.1: FIX – Für derive_wikilink_edges werden echte Chunk-Texte übergeben
+          ({"chunk_id","text"}) → erzeugt `references_at`. :contentReference[oaicite:5]{index=5}
+  v2.3.0: Umstellung auf app.core.derive_edges; Edge-IDs inkl. Occurrence.
 """
 
 from __future__ import annotations
-import argparse, os, glob, json, sys
+import argparse, os, glob, json, sys, hashlib
+from typing import Optional, Tuple, List
 from dotenv import load_dotenv
+from qdrant_client.http import models as rest
+from qdrant_client import QdrantClient
 
-from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
+from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter  # FM-Pflichtfelder usw. :contentReference[oaicite:6]{index=6}
 from app.core.note_payload import make_note_payload
 from app.core.validate_note import validate_note_payload
 from app.core.chunker import assemble_chunks
 from app.core.chunk_payload import make_chunk_payloads
 from app.core.embed import embed_texts, embed_one
 from app.core.qdrant import QdrantConfig, ensure_collections, get_client
-from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
+from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch  # stabile IDs, 1D-Dummy-Vektor für Edges :contentReference[oaicite:7]{index=7}
-from app.core.derive_edges import build_note_index, derive_wikilink_edges   # nutzt 'text' je Chunk  :contentReference[oaicite:1]{index=1}
+from app.core.derive_edges import build_note_index, derive_wikilink_edges  # benötigt Chunk-texte für references_at :contentReference[oaicite:8]{index=8}
-from qdrant_client.http import models as rest
 
-def iter_md(root: str, exclude_dirs=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")):
+
+# -------------------------------
+# Utility
+# -------------------------------
+
+def iter_md(root: str, exclude_dirs=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")) -> List[str]:
     files = [p for p in glob.glob(os.path.join(root, "**", "*.md"), recursive=True)]
     out = []
     for p in files:
@@ -57,22 +71,45 @@ def iter_md(root: str, exclude_dirs=("/.obsidian/", "/_backup_frontmatter/", "/_
         out.append(p)
     return out
 
-def purge_note_edges(client, prefix: str, source_note_id: str):
+def sha256_hex(text: str) -> str:
-    # Löscht Edges, deren source_id == source_note_id ODER (kind==backlink && target_id==source_note_id)
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
-    edges_col = f"{prefix}_edges"
+
-    f = rest.Filter(
+def collection_names(prefix: str) -> Tuple[str, str, str]:
-        should=[
+    return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
-            rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_note_id)),
+
-            rest.Filter(
+def fetch_existing_note_hash(client: QdrantClient, prefix: str, note_id: str) -> Optional[str]:
-                must=[
+    """Liest hash_fulltext der Note aus Qdrant (falls vorhanden)."""
-                    rest.FieldCondition(key="kind", match=rest.MatchValue(value="backlink")),
+    notes_col, _, _ = collection_names(prefix)
-                    rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_note_id)),
+    f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
-                ]
+    pts, _, _ = client.scroll(collection_name=notes_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=1)
-            ),
+    if not pts:
-        ]
+        return None
-    )
+    pl = (pts[0].payload or {})
-    # pydantic v2: client.delete(...) statt delete_points(...)
+    return pl.get("hash_fulltext") or None  # kann bei Altbeständen fehlen
-    client.delete(collection_name=edges_col, points_selector=f, wait=True)
+
+def purge_note_edges(client: QdrantClient, prefix: str, source_note_id: str) -> None:
+    """
+    Löscht Edges der Quelle:
+      - alle mit source_id == note
+      - alle Backlinks, die auf die Quelle zeigen (kind=backlink & target_id=note)
+    """
+    _, _, edges_col = collection_names(prefix)
+    cond_source = rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_note_id))
+    cond_kind = rest.FieldCondition(key="kind", match=rest.MatchValue(value="backlink"))
+    cond_target = rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_note_id))
+    filt = rest.Filter(should=[cond_source, rest.Filter(must=[cond_kind, cond_target])])
+    client.delete(collection_name=edges_col, points_selector=filt, wait=True)
+
+def purge_note_chunks(client: QdrantClient, prefix: str, note_id: str) -> None:
+    """Löscht alle Chunks einer Note (payload.note_id == note_id)."""
+    _, chunks_col, _ = collection_names(prefix)
+    f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
+    client.delete(collection_name=chunks_col, points_selector=f, wait=True)
+
+
+# -------------------------------
+# Hauptlogik
+# -------------------------------
 
 def main():
     load_dotenv()
@@ -81,7 +118,7 @@ def main():
     ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant (sonst Dry-Run)")
     ap.add_argument("--note-id", help="Nur eine Note-ID verarbeiten")
     ap.add_argument("--embed-note", action="store_true", help="Auch Note-Volltext einbetten (optional)")
-    ap.add_argument("--force-replace", action="store_true", help="Vor Upsert alte Edges der Note löschen")
+    ap.add_argument("--force-replace", action="store_true", help="Erzwingt Purge+Replace der Note (auch wenn Hash gleich)")
     args = ap.parse_args()
 
     # Qdrant
@@ -99,7 +136,7 @@ def main():
     if not files:
         print("Keine Markdown-Dateien gefunden.", file=sys.stderr); sys.exit(2)
 
-    # --- Note-Index (für robuste Zielauflösung) ---
+    # --- Vorab: Note-Index für Linkauflösung (by id/slug/path) ---
     note_stubs = []
     for path in files:
         parsed = read_markdown(path)
@@ -112,60 +149,80 @@ def main():
             continue
         rel = os.path.relpath(parsed.path, root).replace("\\","/")
         note_stubs.append({"note_id": fm["id"], "title": fm.get("title",""), "path": rel})
-    note_index = build_note_index(note_stubs)
+    note_index = build_note_index(note_stubs)  # benötigt für Wikilink-Auflösung :contentReference[oaicite:9]{index=9}
 
     total_notes = 0
     for path in files:
         parsed = read_markdown(path)
         fm = normalize_frontmatter(parsed.frontmatter)
         try:
-            validate_required_frontmatter(fm)
+            validate_required_frontmatter(fm)  # Pflichtfelder lt. Schema/Design 
         except Exception:
             continue
         if args.note_id and fm.get("id") != args.note_id:
             continue
 
         total_notes += 1
+        note_id = fm["id"]
+
+        # --- Delta-Detection: Hash berechnen + bestehenden Hash aus Qdrant lesen ---
+        fulltext = parsed.body
+        new_hash = sha256_hex(fulltext)
+        old_hash = fetch_existing_note_hash(client, cfg.prefix, note_id)
+        changed = (old_hash != new_hash) or (old_hash is None) or args.force_replace
 
         # Note-Payload
         note_pl = make_note_payload(parsed, vault_root=root)
+        note_pl["fulltext"] = fulltext                 # für derive_edges (references) :contentReference[oaicite:11]{index=11}
+        note_pl["hash_fulltext"] = new_hash            # Schema-Feld vorhanden :contentReference[oaicite:12]{index=12}
         validate_note_payload(note_pl)
-        note_pl["fulltext"] = parsed.body  # für derive_edges
 
-        # Chunks
+        # Früher Exit (Dry-Run/Skip): wenn unverändert und kein force-replace, nur melden
-        chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
+        if not changed:
+            print(json.dumps({
+                "note_id": note_id,
+                "title": fm["title"],
+                "changed": False,
+                "decision": "skip",
+                "path": note_pl["path"]
+            }, ensure_ascii=False))
+            continue
+
+        # Chunks (inkl. Texte für references_at)
+        chunks = assemble_chunks(note_id, fulltext, fm.get("type", "concept"))
         chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
+        chunks_for_links = [
+            {"chunk_id": (pl.get("chunk_id") or pl.get("id") or f"{note_id}#{i+1}"),
+             "text": chunks[i].text}
+            for i, pl in enumerate(chunk_pls)
+            if i < len(chunks)
+        ]
 
         # Embeddings (Chunks)
         texts = [ch.text for ch in chunks]
         vectors = embed_texts(texts)
 
         # Optional: Note-Vektor
-        note_vec = embed_one(parsed.body) if args.embed_note else None
+        note_vec = embed_one(fulltext) if args.embed_note else None
-
-        # --- WICHTIG: Chunk-Texte für derive_wikilink_edges ---
-        chunks_for_links = [
-            {"chunk_id": (pl.get("chunk_id") or pl.get("id") or f"{fm['id']}#{i+1}"),
-             "text": chunks[i].text}
-            for i, pl in enumerate(chunk_pls)
-            if i < len(chunks)
-        ]
 
         # Kanten (Note- & Chunk-Ebene)
-        edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index)
+        edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index)  # erzeugt references, backlink, references_at :contentReference[oaicite:13]{index=13}
 
         # Dry-Run-Ausgabe
         print(json.dumps({
-            "note_id": fm["id"],
+            "note_id": note_id,
             "title": fm["title"],
             "chunks": len(chunk_pls),
             "edges": len(edges),
+            "changed": True,
+            "decision": "replace" if args.apply else "dry-run",
             "path": note_pl["path"]
         }, ensure_ascii=False))
 
         if args.apply:
-            if args.force_replace:
+            # Purge alte Daten der Note (Chunks + Edges), dann Upsert
-                purge_note_edges(client, cfg.prefix, fm["id"])
+            purge_note_chunks(client, cfg.prefix, note_id)
+            purge_note_edges(client, cfg.prefix, note_id)
 
             # Notes upsert
             notes_col, note_pts = points_for_note(cfg.prefix, note_pl, note_vec, cfg.dim)