Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s

This commit is contained in:
Lars committed 2025-11-08 17:00:27 +01:00
1 parent 2794d26181
commit 3b192e2eed
2 files changed
+263 -389

No files matched your search

+185 -138
View File
@@ -1,180 +1,227 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Modul: scripts/export_markdown.py
Version: 1.6.1
Datum: 2025-11-07
Script: scripts/export_markdown.py — Qdrant → Markdown (Vault)
Version: 1.4.1
Datum: 2025-09-10
Zweck
-----
Exportiert Notes aus Qdrant zurück in Markdown-Dateien (verlustarm):
• Pfade relativieren, Backslashes → Slashes
• Body aus 'fulltext' (falls vorhanden) oder Rekonstruktion via Chunks (seq/chunk_index)
• Optional: vorhandene Edges pro Note mit exportieren (--include-edges yaml|footer)
Funktion
--------
Exportiert Notes (Frontmatter + Body) aus Qdrant in einen Zielordner. Der Body wird
bevorzugt aus dem Feld `fulltext` rekonstruiert; falls leer/nicht vorhanden, aus Chunks
(Sortierung: seq → chunk_index → Nummer in chunk_id). Pfade werden **relativ** geschrieben.
CLI
Optionen
--------
--out PATH Zielordner (erforderlich)
--prefix TEXT Collection-Prefix (CLI überschreibt ENV COLLECTION_PREFIX)
--note-id ID Nur eine Note exportieren
--overwrite Existierende Dateien überschreiben
--include-edges MODE none|yaml|footer (Default: none)
--flatten-paths Alle Dateien flach schreiben; Originalpfad in FM: orig_path
ENV
---
COLLECTION_PREFIX, QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
Beispiele
---------
export COLLECTION_PREFIX="mindnet"
python3 -m scripts.export_markdown --out ./_exportVault
python3 -m scripts.export_markdown --out ./_exportVault --note-id <ID>
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
python3 -m scripts.export_markdown --out ./_exportVault --include-edges yaml
python3 -m scripts.export_markdown --out ./_exportVault --include-edges footer
Parameter
---------
--out Zielwurzel (Ordner wird angelegt)
--prefix überschreibt ENV COLLECTION_PREFIX (Default: mindnet)
--note-id nur eine bestimmte Note exportieren
--overwrite vorhandene Dateien überschreiben
--include-edges none|yaml|footer (Default: none)
# Nur eine Note, Edges als YAML-Feld 'references'
python3 -m scripts.export_markdown --out ./_exportVault --note-id concept-alpha --include-edges yaml
# Flach schreiben mit Überschreiben
python3 -m scripts.export_markdown --out ./_exportVault --flatten-paths --overwrite
"""
from __future__ import annotations
import argparse
import os
import json
from pathlib import Path
from typing import Dict, List, Tuple, Optional
from typing import Dict, List, Optional, Tuple, Any
from app.core.qdrant import (
QdrantConfig,
get_client,
fetch_all_notes,
fetch_chunks_for_note,
fetch_edges_for_note, # <— jetzt angebunden
ensure_collections,
)
import yaml
from qdrant_client.http import models as rest
def _normalize_rel_path(p: str) -> str:
p = (p or "").replace("\\", "/")
while p.startswith("/"):
p = p[1:]
return p
from app.core.qdrant import QdrantConfig, get_client
from app.core.qdrant import ensure_collections # safety
def _ensure_parent(p: Path):
p.parent.mkdir(parents=True, exist_ok=True)
# ---------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------
def _yaml_frontmatter(d: Dict) -> str:
import io
def _ser(obj):
if isinstance(obj, str):
if any(ch in obj for ch in [":", "-", "{", "}", "[", "]", ",", "#", "&", "*", "!", "|", ">", "'", "\"", "%", "@", "`"]):
return '"' + obj.replace('"', '\\"') + '"'
return obj
if isinstance(obj, bool):
return "true" if obj else "false"
if obj is None:
return "null"
if isinstance(obj, (int, float)):
return str(obj)
if isinstance(obj, list):
return "[" + ", ".join(_ser(x) for x in obj) + "]"
if isinstance(obj, dict):
inner = []
for k in sorted(obj.keys()):
inner.append(f"{k}: {_ser(obj[k])}")
return "{ " + ", ".join(inner) + " }"
return '"' + str(obj).replace('"', '\\"') + '"'
def collections(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
buf = io.StringIO()
buf.write("---\n")
for k in sorted(d.keys()):
buf.write(f"{k}: {_ser(d[k])}\n")
buf.write("---\n")
return buf.getvalue()
def _norm_rel_path(path: str) -> str:
p = (path or "").replace("\\", "/").lstrip("/")
return p if p else ""
def _reconstruct_body_from_chunks(chunks: List[Dict]) -> str:
if not chunks:
return ""
def _num_from_chunk_id(cid: str) -> int:
try:
if "#" in cid:
return int(cid.split("#", 1)[1])
return 0
except Exception:
return 0
chunks_sorted = sorted(
chunks,
key=lambda c: (
int(c.get("seq", c.get("chunk_index", 0))),
int(c.get("chunk_index", 0)),
_num_from_chunk_id(str(c.get("chunk_id", ""))),
def _ensure_dir(path: str) -> None:
d = os.path.dirname(path)
if d and not os.path.exists(d):
os.makedirs(d, exist_ok=True)
def _yaml_dump(data: Dict[str, Any]) -> str:
return yaml.safe_dump(data, allow_unicode=True, sort_keys=False).strip()
def _frontmatter_block(fm: Dict[str, Any]) -> str:
y = _yaml_dump(fm)
return f"---\n{y}\n---\n"
def _scroll_all(client, collection: str, flt: Optional[rest.Filter] = None) -> List[Any]:
out = []
nextp = None
while True:
pts, nextp = client.scroll(
collection_name=collection,
with_payload=True,
with_vectors=False,
limit=256,
scroll_filter=flt,
offset=nextp,
)
)
body = "".join(c.get("text") or "" for c in chunks_sorted)
return body
if not pts:
break
out.extend(pts)
if nextp is None:
break
return out
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(prog="export_markdown.py", description="Exportiert Notes aus Qdrant in Markdown.")
p.add_argument("--out", required=True, help="Zielordner")
p.add_argument("--prefix", default="", help="Collections-Prefix; überschreibt ENV COLLECTION_PREFIX")
p.add_argument("--note-id", default="", help="nur eine Note exportieren")
p.add_argument("--overwrite", action="store_true", help="vorhandene Dateien überschreiben")
p.add_argument("--include-edges", default="none", choices=["none", "yaml", "footer"], help="Edges im Export anzeigen")
return p.parse_args()
def _reconstruct_body_from_chunks(chunks: List[Any]) -> str:
def seq_key(pl: Dict[str, Any]) -> Tuple[int, int, int]:
s = pl.get("seq")
ci = pl.get("chunk_index")
cid = pl.get("chunk_id") or ""
n = 0
if isinstance(cid, str) and "#" in cid:
try:
n = int(cid.rsplit("#", 1)[-1])
except Exception:
n = 0
return (int(s) if isinstance(s, int) else 0,
int(ci) if isinstance(ci, int) else 0,
n)
chunks_sorted = sorted(chunks, key=lambda p: seq_key(p.payload or {}))
texts: List[str] = []
for p in chunks_sorted:
pl = p.payload or {}
t = pl.get("text") or pl.get("content") or pl.get("raw") or ""
if isinstance(t, str) and t:
texts.append(t)
return "\n".join(texts).strip()
def main():
args = parse_args()
out_root = Path(args.out).resolve()
out_root.mkdir(parents=True, exist_ok=True)
def _collect_forward_refs_from_edges(edges: List[Any]) -> List[str]:
refs = []
for p in edges:
pl = p.payload or {}
if pl.get("kind") == "references" and isinstance(pl.get("target_id"), str):
refs.append(pl["target_id"])
# de-dupe, preserve order
seen = set()
out = []
for r in refs:
if r not in seen:
seen.add(r)
out.append(r)
return out
prefix = args.prefix.strip() or os.environ.get("COLLECTION_PREFIX", "").strip() or "mindnet"
cfg = QdrantConfig.from_env(prefix=prefix)
# ---------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--out", required=True, help="Zielordner für exportierte Markdown-Dateien")
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID exportieren")
ap.add_argument("--overwrite", action="store_true", help="Existierende Dateien überschreiben")
ap.add_argument("--include-edges", choices=["none", "yaml", "footer"], default="none",
help="Forward-Links mit exportieren (aus Edges oder Note-Payload)")
ap.add_argument("--flatten-paths", action="store_true", help="Alle Dateien flach schreiben (orig_path in Frontmatter)")
args = ap.parse_args()
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix.strip()
client = get_client(cfg)
ensure_collections(client, cfg)
ensure_collections(client, cfg.prefix, cfg.dim)
out_root = os.path.abspath(args.out)
os.makedirs(out_root, exist_ok=True)
notes_col, chunks_col, edges_col = collections(cfg.prefix)
# Filter nach note-id (optional)
flt = None
if args.note_id:
notes = fetch_all_notes(client, cfg, only_note_id=args.note_id)
else:
notes = fetch_all_notes(client, cfg)
flt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=args.note_id))])
exported = 0
notes = _scroll_all(client, notes_col, flt)
total = 0
for n in notes:
note_id = n.get("note_id") or n.get("id")
if not note_id:
continue
pl = n.payload or {}
nid = pl.get("note_id")
rel_path = _norm_rel_path(pl.get("path") or "")
if args.flatten_paths or not rel_path:
fname = f"{(nid or 'note')}.md"
out_path = os.path.join(out_root, fname)
else:
out_path = os.path.join(out_root, rel_path)
out_path = out_path.replace("\\", "/")
_ensure_dir(out_path)
rel = _normalize_rel_path(str(n.get("path") or f"{note_id}.md"))
dst = out_root.joinpath(rel)
# Frontmatter aufbauen (nur sinnvolle Felder)
fm_fields = ["id","title","type","status","created","updated","tags","area","project","source","lang","slug","aliases"]
fm: Dict[str, Any] = {}
fm["id"] = nid
for k in fm_fields:
if k == "id":
continue
if k in pl and pl[k] is not None:
fm[k] = pl[k]
if args.flatten_paths and rel_path:
fm["orig_path"] = rel_path
body = str(n.get("fulltext") or "")
# Body ermitteln (fulltext oder Chunks)
body = (pl.get("fulltext") or "").strip()
if not body:
chunks = fetch_chunks_for_note(client, cfg, note_id)
flt_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
chunks = _scroll_all(client, chunks_col, flt_chunks)
body = _reconstruct_body_from_chunks(chunks)
fm = {}
for k in ("id", "title", "type", "status", "created", "tags", "priority", "due", "effort_min", "values", "goals", "embedding_exclude"):
if k in n:
fm[k] = n[k]
for k in ("hash_signature", "hash_fulltext", "hash_body", "hash_frontmatter"):
if k in n:
fm[k] = n[k]
# Edges (optional)
refs: List[str] = []
if args.include_edges != "none":
# aus Note-Payload, falls vorhanden
if isinstance(pl.get("references"), list) and pl["references"]:
refs = [r for r in pl["references"] if isinstance(r, str)]
else:
flt_edges = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
edges = _scroll_all(client, edges_col, flt_edges)
refs = _collect_forward_refs_from_edges(edges)
if args.include_edges == "yaml" and refs:
fm["references"] = refs
edges_block = ""
if args.include_edges in ("yaml", "footer"):
try:
edges = fetch_edges_for_note(client, cfg, note_id) or []
if args.include_edges == "yaml":
fm["_edges"] = edges
else:
edges_block = "\n\n---\n_edges_:\n" + json.dumps(edges, ensure_ascii=False, indent=2) + "\n"
except Exception:
pass
# Datei schreiben
if (not args.overwrite) and os.path.exists(out_path):
print(json.dumps({"note_id": nid, "path": out_path, "decision": "skip-exists"}))
continue
if dst.exists() and not args.overwrite:
decision = "skip"
else:
_ensure_parent(dst)
content = _yaml_frontmatter(fm) + (body or "") + edges_block
dst.write_text(content, encoding="utf-8")
decision = "write"
content = _frontmatter_block(fm) + (body + "\n" if body else "")
if args.include_edges == "footer" and refs:
content += "\n---\nLinks:\n" + "\n".join(f"- [[{r}]]" for r in refs) + "\n"
print(json.dumps({"note_id": note_id, "path": str(dst), "decision": decision}, ensure_ascii=False))
if decision == "write":
exported += 1
with open(out_path, "w", encoding="utf-8") as f:
f.write(content)
print(json.dumps({"note_id": nid, "path": out_path, "decision": "write"}))
total += 1
print(f"Done. Exported notes: {total}")
print(f"Done. Exported notes: {exported}")
if __name__ == "__main__":
main()