mindnet/scripts/import_markdown.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name:        scripts/import_markdown.py
Version:     v2.1.1 (2025-09-05)
Kurzbeschreibung:
  Importiert Obsidian-Markdown-Notes in Qdrant (Notes/Chunks/Edges).
  Leitet Wikilink-Edges (references/backlink/references_at) direkt beim Import
  aus Volltext + echten Chunk-Texten ab. Keine Abhängigkeit zu edges.py (Legacy).

Aufruf (aus Projekt-Root, im venv):
  python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id NOTE_ID] [--embed-note]

Parameter:
  --vault       Pfad zum Vault (z. B. ./vault)
  --apply       Führt Upserts in Qdrant aus (ohne Flag = Dry-Run mit JSON-Summaries)
  --note-id     Bearbeite nur eine konkrete Note-ID
  --embed-note  Optional: Note-Vektor (Volltext) zusätzlich einbetten

Umgebungsvariablen (optional):
  QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)

Exitcodes:
  0 = OK, 2 = keine Markdown-Dateien gefunden

Wichtige Hinweise:
  - Verwendet ausschließlich app.core.derive_edges für Wikilink-Kanten.
  - Chunk-Texte werden an derive_wikilink_edges übergeben; nur so entstehen references_at.
  - Edges-IDs sind deterministisch (UUIDv5 in qdrant_points), idempotent.
  - Edges-Collection bleibt 1D-Dummy-Vektor (DOT), reine Payload-Nutzung.

Changelog:
  v2.1.1: Entfernt Abhängigkeit auf app.core.edges; direkte Nutzung von derive_wikilink_edges;
          sichert Übergabe der Chunk-Texte → references_at werden erzeugt.
  v2.1.0: Note-Index über gesamten Vault; erste Fassung mit direkter Edge-Ableitung.
  v2.0.x: Import Notes/Chunks/Embeddings, Edges via Backfill (separat).
"""
from __future__ import annotations
import argparse
import os
import glob
import json
import sys
from typing import List, Dict

from dotenv import load_dotenv

# Core-Module (gemäß Projektstruktur)
from app.core.parser import (
    read_markdown,
    normalize_frontmatter,
    validate_required_frontmatter,
)
from app.core.note_payload import make_note_payload
from app.core.validate_note import validate_note_payload
from app.core.chunker import assemble_chunks
from app.core.chunk_payload import make_chunk_payloads
from app.core.embed import embed_texts, embed_one
from app.core.qdrant import QdrantConfig, ensure_collections, get_client
from app.core.qdrant_points import (
    points_for_chunks,
    points_for_note,
    points_for_edges,
    upsert_batch,
)
# Nur Wikilink-Ableitung (keine Kollision mit edges.py):
from app.core.derive_edges import build_note_index, derive_wikilink_edges


def iter_md(root: str, exclude=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")) -> List[str]:
    files = [p for p in glob.glob(os.path.join(root, "**", "*.md"), recursive=True)]
    out: List[str] = []
    for p in files:
        pn = p.replace("\\", "/")
        if any(ex in pn for ex in exclude):
            continue
        out.append(p)
    return out


def minimal_note_index_payload(abs_path: str, vault_root: str) -> Dict:
    """Nur Felder, die der Resolver braucht (id/title/path)."""
    parsed = read_markdown(abs_path)
    fm = normalize_frontmatter(parsed.frontmatter)
    validate_required_frontmatter(fm)
    relpath = os.path.relpath(abs_path, vault_root).replace("\\", "/")
    return {
        "note_id": fm.get("id") or fm.get("note_id"),
        "title": fm.get("title"),
        "path": relpath,
    }


def main():
    load_dotenv()

    ap = argparse.ArgumentParser()
    ap.add_argument("--vault", required=True, help="Obsidian Vault Pfad (z.B. mindnet/vault)")
    ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant (sonst Dry-Run)")
    ap.add_argument("--note-id", help="Nur eine Note-ID verarbeiten")
    ap.add_argument("--embed-note", action="store_true", help="Auch Note-Volltext einbetten (optional)")
    args = ap.parse_args()

    # Qdrant-Konfiguration (+ Collections sicherstellen)
    cfg = QdrantConfig(
        url=os.getenv("QDRANT_URL", "http://127.0.0.1:6333"),
        api_key=os.getenv("QDRANT_API_KEY") or None,
        prefix=os.getenv("COLLECTION_PREFIX", "mindnet"),
        dim=int(os.getenv("VECTOR_DIM", "384")),
    )
    client = get_client(cfg)
    ensure_collections(client, cfg.prefix, cfg.dim)

    root = os.path.abspath(args.vault)
    files = iter_md(root)
    if not files:
        print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
        sys.exit(2)

    # (1) Vorab-Lauf: Note-Index (robuste Auflösung via ID/Titel-Slug/File-Slug)
    index_payloads: List[Dict] = []
    for path in files:
        try:
            pl = minimal_note_index_payload(path, root)
            if not pl.get("note_id"):
                continue
            if args.note_id and pl["note_id"] != args.note_id:
                continue
            index_payloads.append(pl)
        except Exception:
            # Einzeldefekte nicht fatal
            continue

    note_index = build_note_index(index_payloads)

    # (2) Hauptlauf: Import + direkte Wikilink-Edge-Ableitung
    total_notes = 0
    for path in files:
        parsed = read_markdown(path)
        fm = normalize_frontmatter(parsed.frontmatter)
        try:
            validate_required_frontmatter(fm)
        except Exception:
            continue
        if args.note_id and fm.get("id") != args.note_id:
            continue

        total_notes += 1

        # Note-Payload (vollständig für notes-Collection)
        note_pl = make_note_payload(parsed, vault_root=root)
        validate_note_payload(note_pl)

        # Chunks aus Body gemäß Chunking-Strategie
        chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
        chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)

        # Embeddings (Chunks)
        texts = [ch.text for ch in chunks]
        vectors = embed_texts(texts)

        # Optional: Note-Vektor (z. B. für Doc-Suche/Clustering)
        note_vec = None
        if args.embed_note:
            note_vec = embed_one(parsed.body)

        # Edges direkt aus realen Chunks + Note-Index
        # ACHTUNG: derive_wikilink_edges erwartet Chunks mit Text → wir bauen eine
        # minimalistische Liste {"chunk_id","text"} in gleicher Reihenfolge.
        note_pl_for_edges = {
            "note_id": fm["id"],
            "title": fm.get("title"),
            "path": note_pl["path"],
            "fulltext": parsed.body,
        }
        chunks_for_links = []
        for i, pl in enumerate(chunk_pls):
            cid = pl.get("chunk_id") or pl.get("id")
            txt = chunks[i].text if i < len(chunks) else ""
            chunks_for_links.append({"chunk_id": cid, "text": txt})

        edges = derive_wikilink_edges(note_pl_for_edges, chunks_for_links, note_index)

        # Dry-Run-Summary
        print(json.dumps({
            "note_id": fm["id"],
            "title": fm["title"],
            "chunks": len(chunk_pls),
            "edges": len(edges),
            "path": note_pl["path"],
        }, ensure_ascii=False))

        if args.apply:
            # Notes upsert (idempotent; UUIDv5)
            notes_col, note_pts = points_for_note(cfg.prefix, note_pl, note_vec, cfg.dim)
            upsert_batch(client, notes_col, note_pts)

            # Chunks upsert (idempotent)
            chunks_col, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vectors)
            upsert_batch(client, chunks_col, chunk_pts)

            # Edges upsert (idempotent; deterministisches edge_id)
            edges_col, edge_pts = points_for_edges(cfg.prefix, edges)
            upsert_batch(client, edges_col, edge_pts)

    print(f"Done. Processed notes: {total_notes}")


if __name__ == "__main__":
    main()