diff --git a/scripts/import_markdown.py b/scripts/import_markdown.py index dfdfbe0..f542115 100644 --- a/scripts/import_markdown.py +++ b/scripts/import_markdown.py @@ -1,69 +1,490 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -tests/inspect_one_note.py -Diagnose einer einzelnen Markdown-Datei (kein Upsert). -Nutzung: - python3 tests/inspect_one_note.py --path ./test_vault/30_projects/project-demo.md +Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges) +Version: 3.7.1 +Datum: 2025-09-30 + +Kurzbeschreibung +---------------- +- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**. +- Robuste Änderungserkennung (Option C): Mehrfach-Hashes werden parallel in der Note + gespeichert (Feld `hashes` mit Schlüsseln `::`). Der Vergleich + erfolgt **modusgenau** anhand dieses Keys, sodass ein Wechsel des Vergleichsmodus **keine + Massenänderungen** mehr triggert. +- **Erstimport-Fix:** Bei leerem Qdrant gilt Create-Fall automatisch als geändert. +- **--baseline-modes:** fehlende Hash-Varianten „still“ nachtragen (nur Notes upserten). +- **--sync-deletes:** Punkte in Qdrant, die im Vault fehlen, sicher löschen (Dry-Run + Apply). +- **--prefix**: CLI-Override für COLLECTION_PREFIX. + +Neu in 3.7.1 +------------- +- Chunk-Payloads enthalten nun + * `window` (Fenster inkl. Overlap, für Embeddings) + * `text` (überlappungsfreies Segment, für verlustfreie Rekonstruktion) + * `start`, `end`, `overlap_left`, `overlap_right` +- Embeddings werden aus `window` gebildet (mehr Kontext, bessere Retrieval-Qualität). +- Die Offsets werden beim Build der Chunk-Payloads mitgegeben (`note_text`). + +Hash/Compare Konfiguration +-------------------------- +- Vergleichsmodus: + --hash-mode body|frontmatter|full + oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE +- Quelle: + --hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE, Default parsed) +- Normalisierung: + --hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE, Default canonical) +- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich + den parsed Body-Text direkt. + +Qdrant / ENV +------------ +- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY +- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar +- VECTOR_DIM (Default: 384) +- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false) + +Beispiele +--------- + # Standard (Body, parsed, canonical) + python3 -m scripts.import_markdown --vault ./vault + + # Erstimport nach truncate (Create-Fall) + python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert + + # Baseline für parsed:canonical „still“ auffüllen + MINDNET_HASH_SOURCE=parsed MINDNET_HASH_NORMALIZE=canonical \ + python3 -m scripts.import_markdown --vault ./vault --apply --baseline-modes + + # Frontmatter-Vergleich ohne Massenänderungen (nach Baseline) + MINDNET_HASH_COMPARE=Frontmatter \ + python3 -m scripts.import_markdown --vault ./vault + + # Sync-Deletes (Dry-Run → Apply) + python3 -m scripts.import_markdown --vault ./vault --sync-deletes + python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply + + # Prefix explizit setzen + python3 -m scripts.import_markdown --vault ./vault --prefix mindnet """ from __future__ import annotations -import os, sys, json, argparse -PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -if PROJECT_ROOT not in sys.path: - sys.path.insert(0, PROJECT_ROOT) +import argparse +import json +import os +import sys +from typing import Dict, List, Optional, Tuple, Any, Set -from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter +from dotenv import load_dotenv +from qdrant_client.http import models as rest + +from app.core.parser import ( + read_markdown, + normalize_frontmatter, + validate_required_frontmatter, +) +from app.core.note_payload import make_note_payload from app.core.chunker import assemble_chunks from app.core.chunk_payload import make_chunk_payloads +# Kompatibel zu beiden Modulnamen: try: from app.core.derive_edges import build_edges_for_note -except Exception: +except Exception: # pragma: no cover from app.core.edges import build_edges_for_note # type: ignore -def main(): +from app.core.qdrant import ( + QdrantConfig, + get_client, + ensure_collections, + ensure_payload_indexes, +) +from app.core.qdrant_points import ( + points_for_chunks, + points_for_note, + points_for_edges, + upsert_batch, +) + +try: + from app.core.embed import embed_texts # optional +except Exception: + embed_texts = None + + +# --------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------- + +def iter_md(root: str) -> List[str]: + out: List[str] = [] + for dirpath, _, filenames in os.walk(root): + for fn in filenames: + if not fn.lower().endswith(".md"): + continue + p = os.path.join(dirpath, fn) + pn = p.replace("\\", "/") + if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]): + continue + out.append(p) + return sorted(out) + + +def collections(prefix: str) -> Tuple[str, str, str]: + return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges" + + +def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]: + notes_col, _, _ = collections(prefix) + f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) + points, _ = client.scroll( + collection_name=notes_col, + scroll_filter=f, + with_payload=True, + with_vectors=False, + limit=1, + ) + if not points: + return None + return points[0].payload or {} + + +def list_qdrant_note_ids(client, prefix: str) -> Set[str]: + """Liest alle note_ids aus der Notes-Collection (per Scroll).""" + notes_col, _, _ = collections(prefix) + out: Set[str] = set() + next_page = None + while True: + pts, next_page = client.scroll( + collection_name=notes_col, + with_payload=True, + with_vectors=False, + limit=256, + offset=next_page, + ) + if not pts: + break + for p in pts: + pl = p.payload or {} + nid = pl.get("note_id") + if isinstance(nid, str): + out.add(nid) + if next_page is None: + break + return out + + +def purge_note_artifacts(client, prefix: str, note_id: str) -> None: + """ + Löscht alle Chunks & Edges zu einer Note mittels Filter-Selector (kompatibel mit aktuellen Qdrant-Clients). + """ + _, chunks_col, edges_col = collections(prefix) + filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) + + try: + client.delete( + collection_name=chunks_col, + points_selector=rest.FilterSelector(filter=filt), + wait=True + ) + except Exception as e: + print(json.dumps({"note_id": note_id, "warn": f"delete chunks via filter failed: {e}"})) + + try: + client.delete( + collection_name=edges_col, + points_selector=rest.FilterSelector(filter=filt), + wait=True + ) + except Exception as e: + print(json.dumps({"note_id": note_id, "warn": f"delete edges via filter failed: {e}"})) + + +def delete_note_everywhere(client, prefix: str, note_id: str) -> None: + """Löscht Note + zugehörige Chunks + Edges per Filter.""" + notes_col, chunks_col, edges_col = collections(prefix) + filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) + # Reihenfolge: erst edges/chunks, dann note + for col in (edges_col, chunks_col, notes_col): + try: + client.delete( + collection_name=col, + points_selector=rest.FilterSelector(filter=filt), + wait=True + ) + except Exception as e: + print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"})) + + +def _resolve_mode(val: Optional[str]) -> str: + v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower() + if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"): + return "full" + if v in ("frontmatter", "fm"): + return "frontmatter" + return "body" + + +def _env(key: str, default: str) -> str: + return (os.environ.get(key) or default).strip().lower() + + +# --------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------- + +def main() -> None: + load_dotenv() ap = argparse.ArgumentParser() - ap.add_argument("--path", required=True) + ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)") + ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run") + ap.add_argument("--purge-before-upsert", action="store_true", + help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen") + ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten") + ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten") + ap.add_argument("--force-replace", action="store_true", + help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)") + ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None, + help="Vergleichsmodus (Body | Frontmatter | Full)") + ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None) + ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None, + help="Quelle für die Hash-Berechnung (Default: parsed)") + ap.add_argument("--note-scope-refs", action="store_true", + help="(Optional) erzeugt zusätzlich references:note (Default: aus)") + ap.add_argument("--debug-hash-diff", action="store_true", + help="(reserviert) optionaler Body-Diff (nicht nötig bei Option C)") + ap.add_argument("--compare-text", action="store_true", + help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)") + ap.add_argument("--baseline-modes", action="store_true", + help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)") + ap.add_argument("--sync-deletes", action="store_true", + help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)") + ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)") args = ap.parse_args() - p = os.path.abspath(args.path) - try: - parsed = read_markdown(p) + mode = _resolve_mode(args.hash_mode) # body|frontmatter|full + src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed") # parsed|raw + norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical") # canonical|none + note_scope_refs_env = (_env("MINDNET_NOTE_SCOPE_REFS", "false") == "true") + note_scope_refs = args.note_scoop_refs if hasattr(args, "note_scoop_refs") else (args.note_scope_refs or note_scope_refs_env) # typo-safe + compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true") + + cfg = QdrantConfig.from_env() + if args.prefix: + cfg.prefix = args.prefix.strip() + client = get_client(cfg) + ensure_collections(client, cfg.prefix, cfg.dim) + ensure_payload_indexes(client, cfg.prefix) + + root = os.path.abspath(args.vault) + files = iter_md(root) + if not files: + print("Keine Markdown-Dateien gefunden.", file=sys.stderr) + sys.exit(2) + + # Optional: Sync-Deletes vorab + if args.sync_deletes: + # Vault-Note-IDs sammeln + vault_note_ids: Set[str] = set() + for path in files: + try: + parsed = read_markdown(path) + if not parsed: + continue + fm = normalize_frontmatter(parsed.frontmatter) + nid = fm.get("id") + if isinstance(nid, str): + vault_note_ids.add(nid) + except Exception: + continue + # Qdrant-Note-IDs sammeln + qdrant_note_ids = list_qdrant_note_ids(client, cfg.prefix) + to_delete = sorted(qdrant_note_ids - vault_note_ids) + print(json.dumps({ + "action": "sync-deletes", + "prefix": cfg.prefix, + "qdrant_total": len(qdrant_note_ids), + "vault_total": len(vault_note_ids), + "to_delete_count": len(to_delete), + "to_delete": to_delete[:50] + (["…"] if len(to_delete) > 50 else []) + }, ensure_ascii=False)) + if args.apply and to_delete: + for nid in to_delete: + print(json.dumps({"action": "delete", "note_id": nid, "decision": "apply"})) + delete_note_everywhere(client, cfg.prefix, nid) + # Danach normal mit Import fortfahren + + key_current = f"{mode}:{src}:{norm}" + + processed = 0 + for path in files: + # -------- Parse & Validate -------- + try: + parsed = read_markdown(path) + except Exception as e: + print(json.dumps({"path": path, "error": f"read_markdown failed: {e}"})) + continue if parsed is None: - print(json.dumps({"path": p, "error": "read_markdown returned None"})) - return - except Exception as e: - print(json.dumps({"path": p, "error": f"read_markdown failed: {type(e).__name__}: {e}"})) - return + print(json.dumps({"path": path, "error": "read_markdown returned None"})) + continue - try: - fm = normalize_frontmatter(parsed.frontmatter) - validate_required_frontmatter(fm) - except Exception as e: - print(json.dumps({"path": p, "error": f"frontmatter invalid: {type(e).__name__}: {e}"})) - return + try: + fm = normalize_frontmatter(parsed.frontmatter) + validate_required_frontmatter(fm) + except Exception as e: + print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"})) + continue - body = getattr(parsed, "body", "") or "" - chunks = assemble_chunks(fm["id"], body, fm.get("type", "concept")) - chunk_pls = make_chunk_payloads(fm, os.path.relpath(p, os.path.join(p, "..", "..")), chunks, note_text=body) + if args.note_id and fm.get("id") != args.note_id: + continue - edge_error = None - edges_count = 0 - try: - edges = build_edges_for_note(fm["id"], chunk_pls, include_note_scope_refs=False) - edges_count = len(edges) - except Exception as e: - edge_error = f"{type(e).__name__}: {e}" + processed += 1 + + # -------- Build new payload (includes 'hashes') -------- + note_pl = make_note_payload( + parsed, + vault_root=root, + hash_mode=mode, + hash_normalize=norm, + hash_source=src, + file_path=path, + ) + if not note_pl.get("fulltext"): + note_pl["fulltext"] = getattr(parsed, "body", "") or "" + + note_id = note_pl.get("note_id") or fm.get("id") + if not note_id: + print(json.dumps({"path": path, "error": "Missing note_id after payload build"})) + continue + + # -------- Fetch old payload -------- + old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id) + has_old = old_payload is not None + + old_hashes = (old_payload or {}).get("hashes") or {} + old_hash_exact = old_hashes.get(key_current) + + # Neu-Hash (aktueller Modus) aus neuem Payload + new_hash_exact = (note_pl.get("hashes") or {}).get(key_current) + + needs_baseline = (old_hash_exact is None) + + # Change-Detection: + # - CREATE: wenn es KEIN Alt-Payload gibt -> changed=True + # - UPDATE: baseline existiert UND Hash differiert + # - force/text_changed wie gehabt + hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact) + + text_changed = False + if compare_text: + old_text = (old_payload or {}).get("fulltext") or "" + new_text = note_pl.get("fulltext") or "" + text_changed = (old_text != new_text) + + changed = args.force_replace or (not has_old) or hash_changed or text_changed + + # Baseline-only nur, wenn Alt-Payload existiert UND Key fehlt UND keine sonstige Änderung + do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed) + + # -------- Optional: Chunks / Embeddings / Edges vorbereiten -------- + chunk_pls: List[Dict[str, Any]] = [] + edges: List[Dict[str, Any]] = [] + vecs: List[List[float]] = [] + + if changed and (not do_baseline_only): + try: + # Chunks assemblen + body_text = getattr(parsed, "body", "") or "" + chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept")) + # Chunk-Payloads inkl. Offsets/Overlap (neu in 3.7.1) + chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text) + except Exception as e: + print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {e}"})) + continue + + # Embeddings: aus window (Fallback text) + if embed_texts and chunk_pls: + try: + texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls] + vecs = embed_texts(texts_for_embed) + except Exception as e: + print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"})) + vecs = [[0.0] * cfg.dim for _ in chunk_pls] + else: + vecs = [[0.0] * cfg.dim for _ in chunk_pls] + + # Edges + try: + note_refs = note_pl.get("references") or [] + edges = build_edges_for_note( + note_id, + chunk_pls, + note_refs, + include_note_scope_refs=note_scope_refs, + ) + except Exception as e: + print(json.dumps({"path": path, "note_id": note_id, "error": f"build_edges_for_note failed: {e}"})) + continue + + # -------- Summary -------- + summary = { + "note_id": note_id, + "title": fm.get("title"), + "chunks": len(chunk_pls), + "edges": len(edges), + "changed": changed, + "needs_baseline_for_mode": needs_baseline, + "decision": ("baseline-only" if args.apply and do_baseline_only else + "apply" if args.apply and changed else + "apply-skip-unchanged" if args.apply and not changed else + "dry-run"), + "path": note_pl["path"], + "hash_mode": mode, + "hash_normalize": norm, + "hash_source": src, + "prefix": cfg.prefix, + } + print(json.dumps(summary, ensure_ascii=False)) + + # -------- Writes -------- + if not args.apply: + continue + + # BASELINE-ONLY: fehlenden Key nachtragen, ohne legacy Felder anzutasten + if do_baseline_only: + merged_hashes = {} + merged_hashes.update(old_hashes) + merged_hashes.update(note_pl.get("hashes") or {}) + # Legacy-Hashfelder unverändert lassen, falls vorhanden + if old_payload: + note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext")) + note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature")) + note_pl["hashes"] = merged_hashes + notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim) + upsert_batch(client, notes_name, note_pts) + continue + + # Normale CREATE/UPDATE + if not changed: + continue + + if args.purge_before_upsert and has_old: + try: + purge_note_artifacts(client, cfg.prefix, note_id) + except Exception as e: + print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"})) + + notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim) + upsert_batch(client, notes_name, note_pts) + if chunk_pls: + chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs) + upsert_batch(client, chunks_name, chunk_pts) + if edges: + edges_name, edge_pts = points_for_edges(cfg.prefix, edges) + upsert_batch(client, edges_name, edge_pts) + + print(f"Done. Processed notes: {processed}") - print(json.dumps({ - "path": p, - "id": fm.get("id"), - "title": fm.get("title"), - "body_len": len(body), - "chunks": len(chunk_pls), - "edges": edges_count, - "edge_error": edge_error - }, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()