diff --git a/scripts/import_markdown.py b/scripts/import_markdown.py index f542115..dfdfbe0 100644 --- a/scripts/import_markdown.py +++ b/scripts/import_markdown.py @@ -1,490 +1,69 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges) -Version: 3.7.1 -Datum: 2025-09-30 - -Kurzbeschreibung ----------------- -- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**. -- Robuste Änderungserkennung (Option C): Mehrfach-Hashes werden parallel in der Note - gespeichert (Feld `hashes` mit Schlüsseln `::`). Der Vergleich - erfolgt **modusgenau** anhand dieses Keys, sodass ein Wechsel des Vergleichsmodus **keine - Massenänderungen** mehr triggert. -- **Erstimport-Fix:** Bei leerem Qdrant gilt Create-Fall automatisch als geändert. -- **--baseline-modes:** fehlende Hash-Varianten „still“ nachtragen (nur Notes upserten). -- **--sync-deletes:** Punkte in Qdrant, die im Vault fehlen, sicher löschen (Dry-Run + Apply). -- **--prefix**: CLI-Override für COLLECTION_PREFIX. - -Neu in 3.7.1 -------------- -- Chunk-Payloads enthalten nun - * `window` (Fenster inkl. Overlap, für Embeddings) - * `text` (überlappungsfreies Segment, für verlustfreie Rekonstruktion) - * `start`, `end`, `overlap_left`, `overlap_right` -- Embeddings werden aus `window` gebildet (mehr Kontext, bessere Retrieval-Qualität). -- Die Offsets werden beim Build der Chunk-Payloads mitgegeben (`note_text`). - -Hash/Compare Konfiguration --------------------------- -- Vergleichsmodus: - --hash-mode body|frontmatter|full - oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE -- Quelle: - --hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE, Default parsed) -- Normalisierung: - --hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE, Default canonical) -- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich - den parsed Body-Text direkt. - -Qdrant / ENV ------------- -- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY -- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar -- VECTOR_DIM (Default: 384) -- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false) - -Beispiele ---------- - # Standard (Body, parsed, canonical) - python3 -m scripts.import_markdown --vault ./vault - - # Erstimport nach truncate (Create-Fall) - python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert - - # Baseline für parsed:canonical „still“ auffüllen - MINDNET_HASH_SOURCE=parsed MINDNET_HASH_NORMALIZE=canonical \ - python3 -m scripts.import_markdown --vault ./vault --apply --baseline-modes - - # Frontmatter-Vergleich ohne Massenänderungen (nach Baseline) - MINDNET_HASH_COMPARE=Frontmatter \ - python3 -m scripts.import_markdown --vault ./vault - - # Sync-Deletes (Dry-Run → Apply) - python3 -m scripts.import_markdown --vault ./vault --sync-deletes - python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply - - # Prefix explizit setzen - python3 -m scripts.import_markdown --vault ./vault --prefix mindnet +tests/inspect_one_note.py +Diagnose einer einzelnen Markdown-Datei (kein Upsert). +Nutzung: + python3 tests/inspect_one_note.py --path ./test_vault/30_projects/project-demo.md """ from __future__ import annotations +import os, sys, json, argparse -import argparse -import json -import os -import sys -from typing import Dict, List, Optional, Tuple, Any, Set +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) -from dotenv import load_dotenv -from qdrant_client.http import models as rest - -from app.core.parser import ( - read_markdown, - normalize_frontmatter, - validate_required_frontmatter, -) -from app.core.note_payload import make_note_payload +from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter from app.core.chunker import assemble_chunks from app.core.chunk_payload import make_chunk_payloads -# Kompatibel zu beiden Modulnamen: try: from app.core.derive_edges import build_edges_for_note -except Exception: # pragma: no cover +except Exception: from app.core.edges import build_edges_for_note # type: ignore -from app.core.qdrant import ( - QdrantConfig, - get_client, - ensure_collections, - ensure_payload_indexes, -) -from app.core.qdrant_points import ( - points_for_chunks, - points_for_note, - points_for_edges, - upsert_batch, -) - -try: - from app.core.embed import embed_texts # optional -except Exception: - embed_texts = None - - -# --------------------------------------------------------------------- -# Helper -# --------------------------------------------------------------------- - -def iter_md(root: str) -> List[str]: - out: List[str] = [] - for dirpath, _, filenames in os.walk(root): - for fn in filenames: - if not fn.lower().endswith(".md"): - continue - p = os.path.join(dirpath, fn) - pn = p.replace("\\", "/") - if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]): - continue - out.append(p) - return sorted(out) - - -def collections(prefix: str) -> Tuple[str, str, str]: - return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges" - - -def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]: - notes_col, _, _ = collections(prefix) - f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) - points, _ = client.scroll( - collection_name=notes_col, - scroll_filter=f, - with_payload=True, - with_vectors=False, - limit=1, - ) - if not points: - return None - return points[0].payload or {} - - -def list_qdrant_note_ids(client, prefix: str) -> Set[str]: - """Liest alle note_ids aus der Notes-Collection (per Scroll).""" - notes_col, _, _ = collections(prefix) - out: Set[str] = set() - next_page = None - while True: - pts, next_page = client.scroll( - collection_name=notes_col, - with_payload=True, - with_vectors=False, - limit=256, - offset=next_page, - ) - if not pts: - break - for p in pts: - pl = p.payload or {} - nid = pl.get("note_id") - if isinstance(nid, str): - out.add(nid) - if next_page is None: - break - return out - - -def purge_note_artifacts(client, prefix: str, note_id: str) -> None: - """ - Löscht alle Chunks & Edges zu einer Note mittels Filter-Selector (kompatibel mit aktuellen Qdrant-Clients). - """ - _, chunks_col, edges_col = collections(prefix) - filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) - - try: - client.delete( - collection_name=chunks_col, - points_selector=rest.FilterSelector(filter=filt), - wait=True - ) - except Exception as e: - print(json.dumps({"note_id": note_id, "warn": f"delete chunks via filter failed: {e}"})) - - try: - client.delete( - collection_name=edges_col, - points_selector=rest.FilterSelector(filter=filt), - wait=True - ) - except Exception as e: - print(json.dumps({"note_id": note_id, "warn": f"delete edges via filter failed: {e}"})) - - -def delete_note_everywhere(client, prefix: str, note_id: str) -> None: - """Löscht Note + zugehörige Chunks + Edges per Filter.""" - notes_col, chunks_col, edges_col = collections(prefix) - filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) - # Reihenfolge: erst edges/chunks, dann note - for col in (edges_col, chunks_col, notes_col): - try: - client.delete( - collection_name=col, - points_selector=rest.FilterSelector(filter=filt), - wait=True - ) - except Exception as e: - print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"})) - - -def _resolve_mode(val: Optional[str]) -> str: - v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower() - if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"): - return "full" - if v in ("frontmatter", "fm"): - return "frontmatter" - return "body" - - -def _env(key: str, default: str) -> str: - return (os.environ.get(key) or default).strip().lower() - - -# --------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------- - -def main() -> None: - load_dotenv() +def main(): ap = argparse.ArgumentParser() - ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)") - ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run") - ap.add_argument("--purge-before-upsert", action="store_true", - help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen") - ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten") - ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten") - ap.add_argument("--force-replace", action="store_true", - help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)") - ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None, - help="Vergleichsmodus (Body | Frontmatter | Full)") - ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None) - ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None, - help="Quelle für die Hash-Berechnung (Default: parsed)") - ap.add_argument("--note-scope-refs", action="store_true", - help="(Optional) erzeugt zusätzlich references:note (Default: aus)") - ap.add_argument("--debug-hash-diff", action="store_true", - help="(reserviert) optionaler Body-Diff (nicht nötig bei Option C)") - ap.add_argument("--compare-text", action="store_true", - help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)") - ap.add_argument("--baseline-modes", action="store_true", - help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)") - ap.add_argument("--sync-deletes", action="store_true", - help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)") - ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)") + ap.add_argument("--path", required=True) args = ap.parse_args() + p = os.path.abspath(args.path) - mode = _resolve_mode(args.hash_mode) # body|frontmatter|full - src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed") # parsed|raw - norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical") # canonical|none - note_scope_refs_env = (_env("MINDNET_NOTE_SCOPE_REFS", "false") == "true") - note_scope_refs = args.note_scoop_refs if hasattr(args, "note_scoop_refs") else (args.note_scope_refs or note_scope_refs_env) # typo-safe - compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true") - - cfg = QdrantConfig.from_env() - if args.prefix: - cfg.prefix = args.prefix.strip() - client = get_client(cfg) - ensure_collections(client, cfg.prefix, cfg.dim) - ensure_payload_indexes(client, cfg.prefix) - - root = os.path.abspath(args.vault) - files = iter_md(root) - if not files: - print("Keine Markdown-Dateien gefunden.", file=sys.stderr) - sys.exit(2) - - # Optional: Sync-Deletes vorab - if args.sync_deletes: - # Vault-Note-IDs sammeln - vault_note_ids: Set[str] = set() - for path in files: - try: - parsed = read_markdown(path) - if not parsed: - continue - fm = normalize_frontmatter(parsed.frontmatter) - nid = fm.get("id") - if isinstance(nid, str): - vault_note_ids.add(nid) - except Exception: - continue - # Qdrant-Note-IDs sammeln - qdrant_note_ids = list_qdrant_note_ids(client, cfg.prefix) - to_delete = sorted(qdrant_note_ids - vault_note_ids) - print(json.dumps({ - "action": "sync-deletes", - "prefix": cfg.prefix, - "qdrant_total": len(qdrant_note_ids), - "vault_total": len(vault_note_ids), - "to_delete_count": len(to_delete), - "to_delete": to_delete[:50] + (["…"] if len(to_delete) > 50 else []) - }, ensure_ascii=False)) - if args.apply and to_delete: - for nid in to_delete: - print(json.dumps({"action": "delete", "note_id": nid, "decision": "apply"})) - delete_note_everywhere(client, cfg.prefix, nid) - # Danach normal mit Import fortfahren - - key_current = f"{mode}:{src}:{norm}" - - processed = 0 - for path in files: - # -------- Parse & Validate -------- - try: - parsed = read_markdown(path) - except Exception as e: - print(json.dumps({"path": path, "error": f"read_markdown failed: {e}"})) - continue + try: + parsed = read_markdown(p) if parsed is None: - print(json.dumps({"path": path, "error": "read_markdown returned None"})) - continue + print(json.dumps({"path": p, "error": "read_markdown returned None"})) + return + except Exception as e: + print(json.dumps({"path": p, "error": f"read_markdown failed: {type(e).__name__}: {e}"})) + return - try: - fm = normalize_frontmatter(parsed.frontmatter) - validate_required_frontmatter(fm) - except Exception as e: - print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"})) - continue + try: + fm = normalize_frontmatter(parsed.frontmatter) + validate_required_frontmatter(fm) + except Exception as e: + print(json.dumps({"path": p, "error": f"frontmatter invalid: {type(e).__name__}: {e}"})) + return - if args.note_id and fm.get("id") != args.note_id: - continue + body = getattr(parsed, "body", "") or "" + chunks = assemble_chunks(fm["id"], body, fm.get("type", "concept")) + chunk_pls = make_chunk_payloads(fm, os.path.relpath(p, os.path.join(p, "..", "..")), chunks, note_text=body) - processed += 1 - - # -------- Build new payload (includes 'hashes') -------- - note_pl = make_note_payload( - parsed, - vault_root=root, - hash_mode=mode, - hash_normalize=norm, - hash_source=src, - file_path=path, - ) - if not note_pl.get("fulltext"): - note_pl["fulltext"] = getattr(parsed, "body", "") or "" - - note_id = note_pl.get("note_id") or fm.get("id") - if not note_id: - print(json.dumps({"path": path, "error": "Missing note_id after payload build"})) - continue - - # -------- Fetch old payload -------- - old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id) - has_old = old_payload is not None - - old_hashes = (old_payload or {}).get("hashes") or {} - old_hash_exact = old_hashes.get(key_current) - - # Neu-Hash (aktueller Modus) aus neuem Payload - new_hash_exact = (note_pl.get("hashes") or {}).get(key_current) - - needs_baseline = (old_hash_exact is None) - - # Change-Detection: - # - CREATE: wenn es KEIN Alt-Payload gibt -> changed=True - # - UPDATE: baseline existiert UND Hash differiert - # - force/text_changed wie gehabt - hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact) - - text_changed = False - if compare_text: - old_text = (old_payload or {}).get("fulltext") or "" - new_text = note_pl.get("fulltext") or "" - text_changed = (old_text != new_text) - - changed = args.force_replace or (not has_old) or hash_changed or text_changed - - # Baseline-only nur, wenn Alt-Payload existiert UND Key fehlt UND keine sonstige Änderung - do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed) - - # -------- Optional: Chunks / Embeddings / Edges vorbereiten -------- - chunk_pls: List[Dict[str, Any]] = [] - edges: List[Dict[str, Any]] = [] - vecs: List[List[float]] = [] - - if changed and (not do_baseline_only): - try: - # Chunks assemblen - body_text = getattr(parsed, "body", "") or "" - chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept")) - # Chunk-Payloads inkl. Offsets/Overlap (neu in 3.7.1) - chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text) - except Exception as e: - print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {e}"})) - continue - - # Embeddings: aus window (Fallback text) - if embed_texts and chunk_pls: - try: - texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls] - vecs = embed_texts(texts_for_embed) - except Exception as e: - print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"})) - vecs = [[0.0] * cfg.dim for _ in chunk_pls] - else: - vecs = [[0.0] * cfg.dim for _ in chunk_pls] - - # Edges - try: - note_refs = note_pl.get("references") or [] - edges = build_edges_for_note( - note_id, - chunk_pls, - note_refs, - include_note_scope_refs=note_scope_refs, - ) - except Exception as e: - print(json.dumps({"path": path, "note_id": note_id, "error": f"build_edges_for_note failed: {e}"})) - continue - - # -------- Summary -------- - summary = { - "note_id": note_id, - "title": fm.get("title"), - "chunks": len(chunk_pls), - "edges": len(edges), - "changed": changed, - "needs_baseline_for_mode": needs_baseline, - "decision": ("baseline-only" if args.apply and do_baseline_only else - "apply" if args.apply and changed else - "apply-skip-unchanged" if args.apply and not changed else - "dry-run"), - "path": note_pl["path"], - "hash_mode": mode, - "hash_normalize": norm, - "hash_source": src, - "prefix": cfg.prefix, - } - print(json.dumps(summary, ensure_ascii=False)) - - # -------- Writes -------- - if not args.apply: - continue - - # BASELINE-ONLY: fehlenden Key nachtragen, ohne legacy Felder anzutasten - if do_baseline_only: - merged_hashes = {} - merged_hashes.update(old_hashes) - merged_hashes.update(note_pl.get("hashes") or {}) - # Legacy-Hashfelder unverändert lassen, falls vorhanden - if old_payload: - note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext")) - note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature")) - note_pl["hashes"] = merged_hashes - notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim) - upsert_batch(client, notes_name, note_pts) - continue - - # Normale CREATE/UPDATE - if not changed: - continue - - if args.purge_before_upsert and has_old: - try: - purge_note_artifacts(client, cfg.prefix, note_id) - except Exception as e: - print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"})) - - notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim) - upsert_batch(client, notes_name, note_pts) - if chunk_pls: - chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs) - upsert_batch(client, chunks_name, chunk_pts) - if edges: - edges_name, edge_pts = points_for_edges(cfg.prefix, edges) - upsert_batch(client, edges_name, edge_pts) - - print(f"Done. Processed notes: {processed}") + edge_error = None + edges_count = 0 + try: + edges = build_edges_for_note(fm["id"], chunk_pls, include_note_scope_refs=False) + edges_count = len(edges) + except Exception as e: + edge_error = f"{type(e).__name__}: {e}" + print(json.dumps({ + "path": p, + "id": fm.get("id"), + "title": fm.get("title"), + "body_len": len(body), + "chunks": len(chunk_pls), + "edges": edges_count, + "edge_error": edge_error + }, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()