From da70f0e00c22a4886668a5055a8b82276d8231e3 Mon Sep 17 00:00:00 2001 From: Lars Date: Sat, 8 Nov 2025 18:13:07 +0100 Subject: [PATCH] Dateien nach "scripts" hochladen --- scripts/import_markdown.py | 196 +++++++++++++++++++++---------------- 1 file changed, 112 insertions(+), 84 deletions(-) diff --git a/scripts/import_markdown.py b/scripts/import_markdown.py index efb14a3..3968f43 100644 --- a/scripts/import_markdown.py +++ b/scripts/import_markdown.py @@ -1,58 +1,57 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges) -Version: 3.7.2 -Datum: 2025-09-30 +Script: scripts/import_markdown.py - Markdown -> Qdrant (Notes, Chunks, Edges) +Version: 3.7.3 +Date: 2025-11-08 -Kurzbeschreibung ----------------- -- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**. -- Änderungserkennung „Option C“: mehrere Hash-Varianten werden parallel in der Note - gespeichert (Feld `hashes` mit Schlüsseln `::`). Der Vergleich - nutzt NUR den aktuellen Modus-Key — ein Moduswechsel triggert keine Massenänderungen mehr. -- „Erstimport-Fix“: Bei leerem Qdrant gilt Create-Fall automatisch als geändert. -- `--baseline-modes`: fehlende Hash-Varianten still nachtragen (nur Notes upserten). -- `--sync-deletes`: gezielte Lösch-Synchronisation (Dry-Run + Apply). -- `--only-path`: exakt **eine** Datei (Pfad) importieren — nützlich für Diagnosefälle. +Summary +------- +- Reads Markdown files and creates Notes/Chunks/Edges idempotently. +- Change detection "Option C": multiple hash variants are stored in note payload + (field `hashes` with keys `::`). Comparison uses ONLY + the current-mode key. Switching modes no longer triggers bulk "changed". +- "First import" fix: if Qdrant is empty for the note, we treat it as changed. +- `--baseline-modes`: silently add any missing hash variants (notes only). +- `--sync-deletes`: selective deletes (Dry-Run + Apply). +- `--only-path`: process exactly one file (useful for diagnostics). +- NEW in 3.7.3: Type registry (if present) is loaded and the derived + `retriever_weight` is written to both note and chunk payload. If the registry + defines a `chunk_profile`, it is injected into frontmatter for payloads + (does NOT change the chunking behavior here). -Neu in 3.7.1/3.7.2 ------------------- -- Chunk-Payloads: `window` (für Embeddings), `text` (überlappungsfrei, verlustfrei rekonstruierbar), - `start/end/overlap_*`. Embeddings nutzen `window`. -- **3.7.2:** Edges-Fehler führen nicht mehr zum Abbruch der gesamten Note; Note/Chunks werden trotzdem geschrieben. - -Hash/Compare Konfiguration +Hash/Compare configuration -------------------------- -- Vergleichsmodus: +- Compare mode: --hash-mode body|frontmatter|full - oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE -- Quelle: + or ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE +- Source: --hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE, Default parsed) -- Normalisierung: +- Normalization: --hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE, Default canonical) -- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich - den parsed Body-Text direkt. +- Optional: --compare-text (or ENV MINDNET_COMPARE_TEXT=true) compares + parsed body text, in addition to the hash. Qdrant / ENV ------------ - QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY -- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar +- COLLECTION_PREFIX (Default: mindnet), can be overridden with --prefix - VECTOR_DIM (Default: 384) - MINDNET_NOTE_SCOPE_REFS: true|false (Default: false) +- MINDNET_TYPES_PATH: optional path to config/types.yaml -Beispiele ---------- +Examples +-------- # Standard (Body, parsed, canonical) python3 -m scripts.import_markdown --vault ./vault - # Erstimport nach truncate (Create-Fall) + # First import after truncate (create case) python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert - # Nur eine Datei (Diagnose) + # Single file (diagnostics) python3 -m scripts.import_markdown --vault ./vault --only-path ./vault/30_projects/project-demo.md --apply - # Sync-Deletes (Dry-Run → Apply) + # Sync-Deletes (Dry-Run -> Apply) python3 -m scripts.import_markdown --vault ./vault --sync-deletes python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply """ @@ -92,11 +91,14 @@ from app.core.qdrant_points import ( upsert_batch, ) +# ---- Type-Registry (optional) ------------------------------------------------ try: - from app.core.embed import embed_texts # optional -except Exception: - embed_texts = None - + # Expected API + from app.core.type_registry import load_type_registry # type: ignore +except Exception: # pragma: no cover + def load_type_registry(path: str) -> dict: + """Fallback loader if module is absent. Returns empty dict.""" + return {} # --------------------------------------------------------------------- # Helper @@ -200,6 +202,33 @@ def _env(key: str, default: str) -> str: return (os.environ.get(key) or default).strip().lower() +# --- Type-Registry helpers (pure) -------------------------------------------- + +def _effective_chunk_profile(note_type: str, registry: dict) -> Optional[str]: + try: + types = (registry or {}).get("types", {}) if isinstance(registry, dict) else {} + cfg = types.get(note_type) or types.get("concept") or {} + prof = cfg.get("chunk_profile") + if isinstance(prof, str) and prof in {"short", "medium", "long"}: + return prof + except Exception: + pass + return None + + +def _effective_retriever_weight(note_type: str, registry: dict) -> Optional[float]: + try: + types = (registry or {}).get("types", {}) if isinstance(registry, dict) else {} + cfg = types.get(note_type) or types.get("concept") or {} + w = cfg.get("retriever_weight") + if w is None: + return None + return float(w) + except Exception: + # be tolerant + return None + + # --------------------------------------------------------------------- # Main # --------------------------------------------------------------------- @@ -207,31 +236,31 @@ def _env(key: str, default: str) -> str: def main() -> None: load_dotenv() ap = argparse.ArgumentParser() - ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)") - ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run") + ap.add_argument("--vault", required=True, help="Path to the Obsidian vault (root folder)") + ap.add_argument("--apply", action="store_true", help="Write to Qdrant; otherwise Dry-Run") ap.add_argument("--purge-before-upsert", action="store_true", - help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen") - ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten") - ap.add_argument("--only-path", help="Exakt diesen Markdown-Pfad verarbeiten (ignoriert --note-id)") - ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten") + help="Before upsert, delete Chunks & Edges of the CHANGED note") + ap.add_argument("--note-id", help="Process only a specific note-id") + ap.add_argument("--only-path", help="Process exactly this Markdown path (ignores --note-id)") + ap.add_argument("--embed-note", action="store_true", help="Optionally embed the full note text") ap.add_argument("--force-replace", action="store_true", - help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)") + help="Ignore change detection and always upsert (+ optional purge)") ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None, - help="Vergleichsmodus (Body | Frontmatter | Full)") + help="Compare mode (Body | Frontmatter | Full)") ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None) ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None, - help="Quelle für die Hash-Berechnung (Default: parsed)") + help="Source for hash calculation (Default: parsed)") ap.add_argument("--note-scope-refs", action="store_true", - help="(Optional) erzeugt zusätzlich references:note/backlink:note (Default: aus)") + help="(Optional) also create references:note/backlink:note (Default: off)") ap.add_argument("--debug-hash-diff", action="store_true", - help="(reserviert) optionaler Body-Diff") + help="(reserved) optional body diff") ap.add_argument("--compare-text", action="store_true", - help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)") + help="Additionally compare parsed fulltext (beyond the hash)") ap.add_argument("--baseline-modes", action="store_true", - help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)") + help="Add missing hash variants in 'hashes' silently (Upsert notes only)") ap.add_argument("--sync-deletes", action="store_true", - help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)") - ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)") + help="Delete notes/chunks/edges that exist in Qdrant but are missing in the vault (Dry-Run; use --apply to execute)") + ap.add_argument("--prefix", help="Collection prefix (overrides ENV COLLECTION_PREFIX)") args = ap.parse_args() mode = _resolve_mode(args.hash_mode) # body|frontmatter|full @@ -241,6 +270,7 @@ def main() -> None: note_scope_refs = args.note_scope_refs or note_scope_refs_env compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true") + # Prepare Qdrant client cfg = QdrantConfig.from_env() if args.prefix: cfg.prefix = args.prefix.strip() @@ -248,19 +278,27 @@ def main() -> None: ensure_collections(client, cfg.prefix, cfg.dim) ensure_payload_indexes(client, cfg.prefix) + # Load type registry (optional) + types_path = os.environ.get("MINDNET_TYPES_PATH") or os.path.join(os.getcwd(), "config", "types.yaml") + try: + type_registry = load_type_registry(types_path) or {} + except Exception as e: # tolerant + print(json.dumps({"warn": f"type-registry load failed ({types_path}): {type(e).__name__}: {e}"})) + type_registry = {} + root = os.path.abspath(args.vault) - # Dateiliste bestimmen + # Build file list if args.only_path: only = os.path.abspath(args.only_path) files = [only] else: files = iter_md(root) if not files: - print("Keine Markdown-Dateien gefunden.", file=sys.stderr) + print("No Markdown files found.", file=sys.stderr) sys.exit(2) - # Optional: Sync-Deletes vorab + # Optional: Sync-Deletes upfront if args.sync_deletes: vault_note_ids: Set[str] = set() for path in files: @@ -315,6 +353,15 @@ def main() -> None: processed += 1 + # -------- Type-Registry derivation (tolerant) -------- + note_type = (fm.get("type") or "concept").strip().lower() + prof = _effective_chunk_profile(note_type, type_registry) + if prof and not fm.get("chunk_profile"): + fm["chunk_profile"] = prof + weight = _effective_retriever_weight(note_type, type_registry) + if weight is not None: + fm["retriever_weight"] = weight + # -------- Build new payload (includes 'hashes') -------- note_pl = make_note_payload( parsed, @@ -326,6 +373,8 @@ def main() -> None: ) if not note_pl.get("fulltext"): note_pl["fulltext"] = getattr(parsed, "body", "") or "" + if weight is not None: + note_pl["retriever_weight"] = weight note_id = note_pl.get("note_id") or fm.get("id") if not note_id: @@ -337,6 +386,7 @@ def main() -> None: has_old = old_payload is not None old_hashes = (old_payload or {}).get("hashes") or {} + key_current = f"{mode}:{src}:{norm}" old_hash_exact = old_hashes.get(key_current) new_hash_exact = (note_pl.get("hashes") or {}).get(key_current) needs_baseline = (old_hash_exact is None) @@ -356,8 +406,12 @@ def main() -> None: chunk_pls: List[Dict[str, Any]] = [] try: body_text = getattr(parsed, "body", "") or "" - chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept")) + chunks = assemble_chunks(fm["id"], body_text, note_type) chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text) + if weight is not None: + for pl in chunk_pls: + if pl.get("retriever_weight") is None: + pl["retriever_weight"] = weight except Exception as e: print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"})) continue @@ -385,13 +439,15 @@ def main() -> None: except Exception as e: edges_failed = True edges = [] - # WICHTIG: Wir brechen NICHT mehr ab — Note & Chunks werden geschrieben. print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"})) # -------- Summary -------- summary = { "note_id": note_id, "title": fm.get("title"), + "type": note_type, + "chunk_profile": fm.get("chunk_profile"), + "retriever_weight": weight, "chunks": len(chunk_pls), "edges": len(edges), "edges_failed": edges_failed, @@ -448,31 +504,3 @@ def main() -> None: if __name__ == "__main__": main() - - -# --- Type-Registry helper shims (safe if unused) --- - -def effective_chunk_profile(note_type: str, registry: dict) -> str | None: - try: - reg = registry or {} - types = reg.get("types", {}) if isinstance(reg, dict) else {} - # take exact type or fallback to concept - cfg = types.get(note_type) or types.get("concept") or {} - prof = cfg.get("chunk_profile") - if isinstance(prof, str) and prof in {"short", "medium", "long"}: - return prof - except Exception: - pass - return None - -def effective_retriever_weight(note_type: str, registry: dict) -> float | None: - try: - reg = registry or {} - types = reg.get("types", {}) if isinstance(reg, dict) else {} - cfg = types.get(note_type) or types.get("concept") or {} - w = cfg.get("retriever_weight") - if w is None: - return None - return float(w) - except Exception: - return None