mindnet/scripts/export_markdown.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: export_markdown.py — Qdrant → Markdown (Obsidian-kompatibel)
Version: 1.4.0
Datum: 2025-09-09

Kurzbeschreibung
----------------
Exportiert Markdown-Notizen aus Qdrant in einen Zielordner (Vault).
Rekonstruiert YAML-Frontmatter und Body.

Body-Rekonstruktions-Priorität (abwärtskompatibel):
  1) notes.payload.fulltext (verlustfrei, wenn beim Import gespeichert)
  2) ansonsten aus allen zugehörigen Chunks: payload.text → payload.content → payload.raw
     in stabiler, sequentieller Reihenfolge (seq/chunk_index/ID-Nummer)

Wichtige Fixes
--------------
- **Pfad-Normalisierung**: erzwingt relative Pfade (führe führende '/' ab, backslashes → slashes),
  damit ``--out`` nicht ignoriert wird.
- **`--prefix` (optional)**: Überschreibt COLLECTION_PREFIX; ENV bleibt Default (rückwärtskompatibel).

ENV / Qdrant
------------
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
- QDRANT_API_KEY (optional)
- COLLECTION_PREFIX (Default: mindnet)

Aufruf
------
    python3 -m scripts.export_markdown --out ./_exportVault
    python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-foo
    python3 -m scripts.export_markdown --out ./_exportVault --overwrite
    python3 -m scripts.export_markdown --out ./_exportVault --prefix mindnet_dev   # optional

Beispiele
---------
COLLECTION_PREFIX=mindnet QDRANT_URL=http://127.0.0.1:6333 \\
  python3 -m scripts.export_markdown --out ./_exportVault --overwrite
"""

from __future__ import annotations

import argparse
import json
import os
import re
from typing import Dict, Iterable, List, Optional, Tuple

import yaml
from qdrant_client.http import models as rest
from qdrant_client import QdrantClient

from app.core.qdrant import QdrantConfig, get_client, ensure_collections


# -----------------------------------------------------------------------------
# Utilities
# -----------------------------------------------------------------------------

def _names(prefix: str) -> Tuple[str, str, str]:
    return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"


def _ensure_dir(path: str) -> None:
    d = os.path.dirname(path)
    if d and not os.path.isdir(d):
        os.makedirs(d, exist_ok=True)


def _normalize_rel_path(p: str) -> str:
    """Pfad relativ halten & normalisieren (slashes, führende / entfernen)."""
    p = (p or "").replace("\\", "/")
    return p.lstrip("/")


def _to_md(frontmatter: dict, body: str) -> str:
    fm = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=True).strip()
    return f"---\n{fm}\n---\n{(body or '').rstrip()}\n"


def _scroll_all(
    client: QdrantClient,
    col: str,
    flt: Optional[rest.Filter] = None,
    with_payload: bool = True,
    with_vectors: bool = False,
    limit: int = 256,
):
    """Scrollt durch alle Punkte einer Collection und liefert eine Liste mit Points."""
    out = []
    next_page = None
    while True:
        pts, next_page = client.scroll(
            collection_name=col,
            scroll_filter=flt,
            with_payload=with_payload,
            with_vectors=with_vectors,
            limit=limit,
            offset=next_page,
        )
        if not pts:
            break
        out.extend(pts)
        if not next_page:
            break
    return out


def _load_chunks_for_note(client: QdrantClient, chunks_col: str, note_id: str) -> List[dict]:
    flt = rest.Filter(must=[rest.FieldCondition(
        key="note_id",
        match=rest.MatchValue(value=note_id),
    )])
    pts = _scroll_all(client, chunks_col, flt, with_payload=True, with_vectors=False)
    # Sortierung: bevorzugt seq → chunk_index → Nummer in id
    def _seq(pl: dict) -> Tuple[int, int, int]:
        s1 = pl.get("seq", pl.get("chunk_index", -1))
        s2 = pl.get("chunk_index", -1)
        # Nummer-Anteil aus "noteid#<n>"
        s3 = 0
        try:
            m = re.search(r"#(\\d+)$", pl.get("id") or "")
            if m:
                s3 = int(m.group(1))
        except Exception:
            pass
        return (int(s1) if isinstance(s1, int) else -1, int(s2) if isinstance(s2, int) else -1, s3)

    pts_sorted = sorted(pts, key=lambda p: _seq(p.payload or {}))
    return [p.payload or {} for p in pts_sorted]


def _reconstruct_body(note_pl: dict, chunk_payloads: List[dict]) -> str:
    # 1) Volltext vorhanden?
    fulltext = note_pl.get("fulltext")
    if isinstance(fulltext, str) and fulltext.strip():
        return fulltext

    # 2) Aus Chunks zusammensetzen: text → content → raw
    parts: List[str] = []
    for ch in chunk_payloads:
        text = ch.get("text") or ch.get("content") or ch.get("raw")
        if isinstance(text, str) and text.strip():
            parts.append(text.rstrip())
    return ("\n\n".join(parts)).rstrip() + ("\n" if parts else "")


def _export_one_note(
    client: QdrantClient,
    prefix: str,
    note_pl: dict,
    out_root: str,
    overwrite: bool,
) -> dict:
    notes_col, chunks_col, _ = _names(prefix)
    note_id = note_pl.get("note_id") or note_pl.get("id")

    # Pfad robust bestimmen und relativ halten
    path = note_pl.get("path") or f"{note_id}.md"
    path = _normalize_rel_path(path)
    out_path = os.path.join(out_root, path).replace("\\", "/")

    # Frontmatter aus Payload zurückführen (nur bekannte Felder)
    fm: Dict[str, object] = {}
    for k in [
        "title", "id", "type", "status", "created", "updated", "tags",
        "priority", "effort_min", "due", "people", "aliases",
        "depends_on", "assigned_to", "lang",
    ]:
        v = note_pl.get(k) if k in note_pl else note_pl.get(f"note_{k}")
        if v not in (None, [], ""):
            fm[k] = v

    # Mindestfelder
    if "id" not in fm and note_id:
        fm["id"] = note_id
    if "title" not in fm and note_pl.get("title"):
        fm["title"] = note_pl["title"]

    # Body beschaffen
    chunks = _load_chunks_for_note(client, chunks_col, note_id)
    body = _reconstruct_body(note_pl, chunks)

    # Schreiben?
    if os.path.exists(out_path) and not overwrite:
        return {"note_id": note_id, "path": path, "status": "skip_exists"}

    _ensure_dir(out_path)
    with open(out_path, "w", encoding="utf-8") as f:
        f.write(_to_md(fm, body))

    return {"note_id": note_id, "path": path, "status": "written"}


# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------

def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", required=True, help="Zielordner für den Export-Vault")
    ap.add_argument("--note-id", help="Nur eine Note exportieren (Note-ID)")
    ap.add_argument("--overwrite", action="store_true", help="Bestehende Dateien überschreiben")
    ap.add_argument("--prefix", help="(Optional) überschreibt COLLECTION_PREFIX aus ENV")
    args = ap.parse_args()

    # Qdrant-Konfiguration
    cfg = QdrantConfig.from_env()
    if args.prefix:
        cfg.prefix = args.prefix  # abwärtskompatibel: ENV bleibt Default

    client = get_client(cfg)
    ensure_collections(client, cfg.prefix, cfg.dim)

    notes_col, _, _ = _names(cfg.prefix)

    # Notes holen (optional gefiltert)
    flt = None
    if args.note_id:
        flt = rest.Filter(must=[rest.FieldCondition(
            key="note_id",
            match=rest.MatchValue(value=args.note_id),
        )])

    note_pts = _scroll_all(client, notes_col, flt, with_payload=True, with_vectors=False)
    if not note_pts:
        print(json.dumps({"exported": 0, "out": args.out, "message": "Keine Notes gefunden."}, ensure_ascii=False))
        return

    results = []
    for p in note_pts:
        pl = p.payload or {}
        try:
            res = _export_one_note(client, cfg.prefix, pl, args.out, args.overwrite)
        except Exception as e:
            res = {"note_id": pl.get("note_id") or pl.get("id"), "error": str(e)}
        results.append(res)

    print(json.dumps({
        "exported": len([r for r in results if r.get("status") == "written"]),
        "skipped": len([r for r in results if r.get("status") == "skip_exists"]),
        "out": args.out,
        "details": results,
    }, ensure_ascii=False))


if __name__ == "__main__":
    main()