# scripts/backfill_edges.py #!/usr/bin/env python3 from __future__ import annotations import argparse, glob, json, os from typing import List, Tuple from app.core.parser import read_markdown # liefert (frontmatter_dict, body_str) from app.core.qdrant import QdrantConfig, get_client, ensure_collections from app.core.qdrant_points import points_for_edges, upsert_batch from app.core.derive_edges import build_note_index, derive_wikilink_edges def make_note_stub(path: str, fm: dict, body: str) -> dict: """ Minimaler Note-Payload nur für die Link-Auflösung und Kanten: - note_id (aus Frontmatter), title, path - fulltext = body (für Links im Gesamtdokument) """ note_id = fm.get("id") or fm.get("note_id") title = fm.get("title") or os.path.basename(path).rsplit(".", 1)[0] return { "note_id": note_id, "title": title, "path": path.replace("\\", "/"), "fulltext": body, } def iter_notes(vault: str, excludes: List[str]) -> List[Tuple[dict, List[dict]]]: """ Liefert eine Liste von (note_stub, chunks_for_link_scan). Für den Backfill reicht 1 Chunk (= gesamter Body), damit wir [[...]] finden. """ files = [p for p in glob.glob(os.path.join(vault, "**/*.md"), recursive=True)] out: List[Tuple[dict, List[dict]]] = [] for path in files: if any(ex in path for ex in excludes): continue try: fm, body = read_markdown(path) stub = make_note_stub(path=os.path.relpath(path, vault), fm=fm, body=body) if not stub.get("note_id"): # ohne stabile ID können wir keine Edges sinnvoll referenzieren continue chunk = { "chunk_id": f"{stub['note_id']}#1", "note_id": stub["note_id"], "text": body, } out.append((stub, [chunk])) except Exception as e: print(f"skip {path}: {e}") return out def main(): ap = argparse.ArgumentParser() ap.add_argument("--vault", required=True) ap.add_argument("--exclude", nargs="*", default=["/.obsidian/", "/_backup_frontmatter/"]) args = ap.parse_args() cfg = QdrantConfig.from_env() client = get_client(cfg) ensure_collections(client, cfg.prefix, cfg.dim) # 1) Notizen sammeln (stubs) + 1-Chunk pro Note für den Scan note_tuples = iter_notes(args.vault, args.exclude) note_payloads = [n for n, _ in note_tuples] # 2) Index für Zielauflösung idx = build_note_index(note_payloads) # 3) Edges ableiten all_edges = [] for note_stub, chunks in note_tuples: edges = derive_wikilink_edges(note_stub, chunks, idx) all_edges.extend(edges) # 4) Upsert edges_col, edge_pts = points_for_edges(cfg.prefix, all_edges) upsert_batch(client, edges_col, edge_pts) print(json.dumps({"edges_upserted": len(edge_pts)}, ensure_ascii=False)) if __name__ == "__main__": main()