Compare commits
34
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e040fcc0dd | ||
|
|
305089fcf6 | ||
|
|
f9c0b5df19 | ||
|
|
77fa57a2a6 | ||
|
|
dcb02dd194 | ||
|
|
35338dea64 | ||
|
|
c34df96839 | ||
|
|
0617163c42 | ||
|
|
47797ecd29 | ||
|
|
897d0c9e6d | ||
|
|
6c25d76135 | ||
|
|
22885a4857 | ||
|
|
a807cc8bd1 | ||
|
|
c2954b5663 | ||
|
|
8f97f42618 | ||
|
|
0a116ea3f4 | ||
|
|
aa7d0190c8 | ||
|
|
af5d0a0c91 | ||
|
|
fb016c6c6a | ||
|
|
77732445ea | ||
|
|
1b833f76ce | ||
|
|
b394cadf73 | ||
|
|
f3b6166daa | ||
|
|
ce720090ec | ||
|
|
bf2149bbeb | ||
|
|
80a7db0e54 | ||
|
|
47e6d56b21 | ||
|
|
df33293621 | ||
|
|
a202c53594 | ||
|
|
4f54e750b2 | ||
|
|
5478be60c1 | ||
|
|
3deb99f053 | ||
|
|
8df2069c6f | ||
|
|
d790c0ef75 |
No files matched your search
+149
-27
@@ -1,37 +1,159 @@
|
||||
from __future__ import annotations
|
||||
from typing import Dict, List
|
||||
from .chunker import Chunk
|
||||
from .parser import extract_wikilinks
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/chunk_payload.py
|
||||
Version: 1.1.3
|
||||
Datum: 2025-09-09
|
||||
|
||||
def make_chunk_payloads(note_meta: Dict, path: str, chunks: List[Chunk]) -> List[Dict]:
|
||||
res = []
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
Erzeugt Qdrant-Payloads für Text-Chunks einer Note. Jeder Chunk enthält
|
||||
den tatsächlichen Text unter dem Schlüssel ``text``. Dadurch kann der
|
||||
Exporter den vollständigen Body verlässlich aus Chunks rekonstruieren,
|
||||
falls ``notes.payload.fulltext`` fehlt.
|
||||
|
||||
Wesentliche Features
|
||||
--------------------
|
||||
- Stabile, idempotente Payload-Struktur für Chunks
|
||||
- Persistenter Chunk-Text (``text``)
|
||||
- Extraktion von Wikilinks pro Chunk (``wikilinks`` & ``references``)
|
||||
- Pfadübernahme (relativ zum Vault, wird vom Aufrufer geliefert)
|
||||
- Bereinigung leerer Felder (keine ``None``/leere Collections im Payload)
|
||||
|
||||
Abhängigkeiten
|
||||
--------------
|
||||
- ``app.core.chunker.Chunk`` und ``assemble_chunks`` (nur für CLI-Demo)
|
||||
- ``app.core.parser.extract_wikilinks`` und ``read_markdown`` (nur CLI-Demo)
|
||||
|
||||
Beispiele (CLI – Sichtprüfung)
|
||||
------------------------------
|
||||
python3 -m app.core.chunk_payload --from-file ./vault/demo.md --print
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Dict, List, Any
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
|
||||
# WICHTIG: Keine direkten Selbst- oder Kreuz-Imports, die Zyklen erzeugen!
|
||||
# Für die Kernfunktion reicht extract_wikilinks als Typ-unabhängige Hilfsfunktion.
|
||||
try:
|
||||
from app.core.parser import extract_wikilinks # nur Funktion, erzeugt keine Zyklen
|
||||
except Exception: # pragma: no cover
|
||||
from .parser import extract_wikilinks # type: ignore
|
||||
|
||||
# Typ-Hinweis für Chunk (zur Laufzeit nicht notwendig)
|
||||
try:
|
||||
from app.core.chunker import Chunk # nur Typ-Annotation / Nutzung im CLI
|
||||
except Exception: # pragma: no cover
|
||||
class Chunk: # type: ignore
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utils
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _drop_empty(d: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Entfernt leere/None-Felder aus einem Dict (für saubere Payloads)."""
|
||||
return {k: v for k, v in d.items() if v not in (None, [], {}, "")}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kernfunktion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_chunk_payloads(note_meta: Dict[str, Any], path: str, chunks: List[Chunk]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Baut Payloads für alle ``chunks`` der gegebenen Note.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
note_meta : Dict[str, Any]
|
||||
Minimale Metadaten der Note (mind. ``id``, ``title``; optional ``type``,
|
||||
``area``, ``project``, ``tags``, ``lang``).
|
||||
path : str
|
||||
Relativer Pfad der Note innerhalb des Vaults (z. B. "area/topic/file.md").
|
||||
chunks : List[Chunk]
|
||||
Liste vorsegmentierter Chunks (vgl. app.core.chunker.Chunk).
|
||||
|
||||
Returns
|
||||
-------
|
||||
List[Dict[str, Any]]
|
||||
Payload-Objekte, bereit für Qdrant-Upserts.
|
||||
"""
|
||||
res: List[Dict[str, Any]] = []
|
||||
for ch in chunks:
|
||||
wikilinks = extract_wikilinks(ch.text)
|
||||
# 'text' kann bei leerem Abschnitt None/'' sein → robust behandeln
|
||||
text_val = getattr(ch, "text", "") or ""
|
||||
wikilinks = extract_wikilinks(text_val)
|
||||
payload = {
|
||||
"id": ch.id,
|
||||
"note_id": note_meta["id"],
|
||||
"note_title": note_meta["title"],
|
||||
"chunk_index": ch.index,
|
||||
"char_start": ch.char_start,
|
||||
"char_end": ch.char_end,
|
||||
"token_count": ch.token_count,
|
||||
"type": note_meta.get("type"),
|
||||
"area": note_meta.get("area"),
|
||||
"project": note_meta.get("project"),
|
||||
"tags": note_meta.get("tags", []),
|
||||
"section_title": ch.section_title,
|
||||
"section_path": ch.section_path,
|
||||
"id": getattr(ch, "id", None),
|
||||
"note_id": note_meta.get("id"),
|
||||
"note_title": note_meta.get("title"),
|
||||
"chunk_index": getattr(ch, "index", None),
|
||||
"char_start": getattr(ch, "char_start", None),
|
||||
"char_end": getattr(ch, "char_end", None),
|
||||
"token_count": getattr(ch, "token_count", None),
|
||||
"section_title": getattr(ch, "section_title", None),
|
||||
"section_path": getattr(ch, "section_path", None),
|
||||
"lang": note_meta.get("lang"),
|
||||
"wikilinks": wikilinks,
|
||||
"external_links": [], # (optional später ergänzen)
|
||||
"external_links": [],
|
||||
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
|
||||
"neighbors": {
|
||||
"prev": ch.neighbors_prev,
|
||||
"next": ch.neighbors_next
|
||||
"prev": getattr(ch, "neighbors_prev", None),
|
||||
"next": getattr(ch, "neighbors_next", None),
|
||||
},
|
||||
"path": path,
|
||||
"path": path, # vom Aufrufer relativ geliefert
|
||||
"text": text_val, # WICHTIG für Export/Rekonstruktion
|
||||
}
|
||||
# None/Leere bereinigen
|
||||
payload = {k: v for k, v in payload.items() if v not in (None, [], {})}
|
||||
res.append(payload)
|
||||
res.append(_drop_empty(payload))
|
||||
return res
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI – nur zur Sichtprüfung / Debug
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _cli() -> None:
|
||||
# CLI lädt hier *lazy* die schweren Abhängigkeiten,
|
||||
# damit die Modul-Initialisierung zirkularfrei bleibt.
|
||||
try:
|
||||
from app.core.parser import read_markdown
|
||||
from app.core.chunker import assemble_chunks
|
||||
except Exception: # pragma: no cover
|
||||
from .parser import read_markdown # type: ignore
|
||||
from .chunker import assemble_chunks # type: ignore
|
||||
|
||||
ap = argparse.ArgumentParser(description="Vorschau: Chunk-Payloads erzeugen und anzeigen")
|
||||
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zu einer Markdown-Datei")
|
||||
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
|
||||
args = ap.parse_args()
|
||||
|
||||
note = read_markdown(args.src)
|
||||
note_meta = {
|
||||
"id": note.frontmatter.get("id") or note.frontmatter.get("note_id"),
|
||||
"title": note.frontmatter.get("title"),
|
||||
"type": note.frontmatter.get("type"),
|
||||
"area": note.frontmatter.get("area"),
|
||||
"project": note.frontmatter.get("project"),
|
||||
"tags": note.frontmatter.get("tags", []),
|
||||
"lang": note.frontmatter.get("lang"),
|
||||
}
|
||||
|
||||
chunks = assemble_chunks(note.frontmatter.get("id"), note.body or "", note.frontmatter.get("type", "concept"))
|
||||
|
||||
# Vault-Root heuristisch relativieren (nur für Demo)
|
||||
vault_root = os.path.dirname(os.path.dirname(args.src)) # heuristisch
|
||||
rel_path = os.path.relpath(args.src, vault_root).replace("\\", "/").lstrip("/")
|
||||
|
||||
payloads = make_chunk_payloads(note_meta, rel_path, chunks)
|
||||
|
||||
if args.do_print:
|
||||
print(json.dumps(payloads, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
_cli()
|
||||
+187
-28
@@ -1,39 +1,198 @@
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import hashlib
|
||||
from typing import Dict
|
||||
from .parser import ParsedNote
|
||||
from .parser import extract_wikilinks
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/note_payload.py
|
||||
Version: 1.4.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
Erzeugt den Qdrant-Payload für **Notes**. Neben stabiler Hash-Bildung zur
|
||||
Idempotenz wird der vollständige Body unter ``fulltext`` persistiert und
|
||||
der Dateipfad relativ zum Vault gespeichert. Das erlaubt eine verlustfreie
|
||||
Rekonstruktion im Export (erst ``fulltext``, sonst Chunks).
|
||||
|
||||
Hash-Steuerung
|
||||
--------------
|
||||
- Modus: ``body`` (Default) | ``frontmatter`` | ``body+frontmatter``
|
||||
* per Funktionsparameter ``hash_mode`` (prio 1)
|
||||
* oder ENV ``MINDNET_HASH_MODE`` (prio 2)
|
||||
- Normalisierung: ``canonical`` (Default) | ``none``
|
||||
* per Funktionsparameter ``hash_normalize`` (prio 1)
|
||||
* oder ENV ``MINDNET_HASH_NORMALIZE`` (prio 2)
|
||||
|
||||
Beispiele (CLI – Sichtprüfung)
|
||||
------------------------------
|
||||
python3 -m app.core.note_payload --from-file ./vault/demo.md --vault-root ./vault --print
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
try:
|
||||
from app.core.parser import read_markdown, extract_wikilinks
|
||||
except Exception: # pragma: no cover
|
||||
from .parser import read_markdown, extract_wikilinks # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hashing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
|
||||
"""Kanonische, stabile JSON-Serialisierung der Frontmatter für Hashbildung."""
|
||||
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
def _normalize_body(body: str, mode: str) -> str:
|
||||
"""Normalisiert den Body für reproduzierbare Hashes (oder nicht)."""
|
||||
if mode == "none":
|
||||
return body if body is not None else ""
|
||||
# canonical: \r\n→\n, trailing spaces, mehrfach-blankzeilen trimmen
|
||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
|
||||
# trailing whitespace am Zeilenende entfernen
|
||||
text = "\n".join(line.rstrip() for line in text.split("\n"))
|
||||
# keine aggressivere Logik (keine inhaltliche Änderung)
|
||||
return text
|
||||
|
||||
def compute_hash(
|
||||
*,
|
||||
body: Optional[str],
|
||||
frontmatter: Optional[Dict[str, Any]],
|
||||
mode: Optional[str] = None,
|
||||
normalize: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Berechnet einen Hex-Hash gemäß ``mode`` und ``normalize``.
|
||||
|
||||
mode:
|
||||
- "body" (Default)
|
||||
- "frontmatter"
|
||||
- "body+frontmatter"
|
||||
normalize:
|
||||
- "canonical" (Default)
|
||||
- "none"
|
||||
"""
|
||||
mode = (mode or os.environ.get("MINDNET_HASH_MODE", "body")).strip().lower()
|
||||
normalize = (normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
|
||||
|
||||
body = _normalize_body(body or "", normalize)
|
||||
fm_s = _canon_frontmatter(frontmatter or {})
|
||||
|
||||
def sha256_text(text: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
h.update(text.encode("utf-8"))
|
||||
if mode == "frontmatter":
|
||||
h.update(fm_s.encode("utf-8"))
|
||||
elif mode == "body+frontmatter":
|
||||
h.update(body.encode("utf-8"))
|
||||
h.update(b"\n--FM--\n")
|
||||
h.update(fm_s.encode("utf-8"))
|
||||
else: # body
|
||||
h.update(body.encode("utf-8"))
|
||||
return h.hexdigest()
|
||||
|
||||
def make_note_payload(parsed: ParsedNote, vault_root: str) -> Dict:
|
||||
fm = parsed.frontmatter
|
||||
body = parsed.body
|
||||
rel_path = os.path.relpath(parsed.path, vault_root) if vault_root else parsed.path
|
||||
|
||||
payload = {
|
||||
"note_id": fm.get("id"),
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kernfunktion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_note_payload(
|
||||
parsed: Any,
|
||||
vault_root: Optional[str] = None,
|
||||
*,
|
||||
hash_mode: Optional[str] = None,
|
||||
hash_normalize: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Erzeugt den Payload für eine geparste Note.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
parsed : Any
|
||||
Objekt mit Attributen/Keys ``frontmatter``, ``body``, ``path``.
|
||||
vault_root : Optional[str]
|
||||
Vault-Wurzel (für Pfad-Relativierung). Wenn ``None``, wird ``path`` unverändert übernommen.
|
||||
hash_mode : Optional[str]
|
||||
"body" | "frontmatter" | "body+frontmatter" (überschreibt ENV).
|
||||
hash_normalize : Optional[str]
|
||||
"canonical" | "none" (überschreibt ENV).
|
||||
|
||||
Returns
|
||||
-------
|
||||
Dict[str, Any]
|
||||
Qdrant-Payload für die Notes-Collection.
|
||||
"""
|
||||
# "Duck typing": dict oder Objekt akzeptieren
|
||||
fm = (
|
||||
getattr(parsed, "frontmatter", None)
|
||||
or getattr(parsed, "fm", None)
|
||||
or getattr(parsed, "front_matter", None)
|
||||
or (parsed.get("frontmatter") if isinstance(parsed, dict) else {})
|
||||
) or {}
|
||||
body = getattr(parsed, "body", None) or (parsed.get("body") if isinstance(parsed, dict) else "") or ""
|
||||
path = getattr(parsed, "path", None) or (parsed.get("path") if isinstance(parsed, dict) else "") or ""
|
||||
|
||||
# Hash gem. Modus bilden
|
||||
hash_fulltext = compute_hash(body=body, frontmatter=fm, mode=hash_mode, normalize=hash_normalize)
|
||||
|
||||
# Pfad relativieren
|
||||
rel_path = path
|
||||
try:
|
||||
if vault_root:
|
||||
rel = os.path.relpath(path, vault_root)
|
||||
rel = rel.replace("\\", "/").lstrip("/") # normalisieren
|
||||
rel_path = rel
|
||||
except Exception:
|
||||
pass # Pfad nicht kritisch für Hash/ID
|
||||
|
||||
# Optionale Note-Level-Wikilinks (Fallback, wenn Chunks nicht geliefert werden)
|
||||
note_level_refs = list(dict.fromkeys(extract_wikilinks(body))) if body else []
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"note_id": fm.get("id") or fm.get("note_id"),
|
||||
"title": fm.get("title"),
|
||||
"type": fm.get("type"),
|
||||
"status": fm.get("status"),
|
||||
"created": fm.get("created"),
|
||||
"updated": fm.get("updated"),
|
||||
"tags": fm.get("tags") or [],
|
||||
"priority": fm.get("priority"),
|
||||
"effort_min": fm.get("effort_min"),
|
||||
"due": fm.get("due"),
|
||||
"people": fm.get("people") or [],
|
||||
"aliases": fm.get("aliases") or [],
|
||||
"depends_on": fm.get("depends_on") or [],
|
||||
"assigned_to": fm.get("assigned_to") or [],
|
||||
"references": list(sorted(set(extract_wikilinks(body)))),
|
||||
"lang": fm.get("lang") or None,
|
||||
"path": rel_path.replace("\\", "/"),
|
||||
"hash_fulltext": sha256_text(body.strip()),
|
||||
"path": rel_path or fm.get("path"),
|
||||
"tags": fm.get("tags"),
|
||||
"hash_fulltext": hash_fulltext,
|
||||
# Volltext persistieren (verlustfreie Rekonstruktion)
|
||||
"fulltext": body,
|
||||
# Fallback-Refs auf Note-Ebene
|
||||
"references": note_level_refs,
|
||||
}
|
||||
# Entferne None-Werte für saubere Payloads
|
||||
return {k: v for k, v in payload.items() if v not in (None, [])}
|
||||
|
||||
for k in ("area", "project", "source", "lang", "slug"):
|
||||
if k in fm:
|
||||
payload[k] = fm[k]
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI – Sichtprüfung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _cli() -> None:
|
||||
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
|
||||
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zur Markdown-Datei")
|
||||
ap.add_argument("--vault-root", dest="vault_root", default=None, help="Vault-Wurzel zur Pfad-Relativierung")
|
||||
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter"], default=None)
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
parsed = read_markdown(args.src)
|
||||
payload = make_note_payload(parsed, vault_root=args.vault_root,
|
||||
hash_mode=args.hash_mode, hash_normalize=args.hash_normalize)
|
||||
|
||||
if args.do_print:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
_cli()
|
||||
+79
-21
@@ -1,19 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: app/core/qdrant_points.py
|
||||
Version: v1.2.0 (2025-09-05)
|
||||
app/core/qdrant_points.py
|
||||
|
||||
Kurzbeschreibung
|
||||
Qdrant-Points für Notes/Chunks/Edges.
|
||||
NEU: Edge-ID berücksichtigt optional 'occ' (Occurrence je Match):
|
||||
"{kind}:{src}->{tgt}#{seq}@{occ}"
|
||||
Zweck
|
||||
- Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
|
||||
- Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
|
||||
- alt: {'edge_type','src_id','dst_id', ...}
|
||||
- neu: {'kind','source_id','target_id', ...}
|
||||
|
||||
Aufruf
|
||||
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
|
||||
Version
|
||||
- 1.3 (2025-09-08)
|
||||
|
||||
Changelog
|
||||
v1.2.0: Edge-ID mit '@occ' falls vorhanden.
|
||||
Änderungen (ggü. 1.2)
|
||||
- points_for_edges() akzeptiert jetzt beide Edge-Schemata.
|
||||
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
|
||||
stabile 'edge_id' zurück in die Payload.
|
||||
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
|
||||
|
||||
Aufruf / Verwendung
|
||||
- Wird von Import-/Backfill-Skripten via:
|
||||
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
|
||||
eingebunden. Keine CLI.
|
||||
|
||||
Hinweise
|
||||
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
|
||||
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,13 +33,23 @@ import uuid
|
||||
from typing import List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
|
||||
def _to_uuid(stable_key: str) -> str:
|
||||
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||
|
||||
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
|
||||
|
||||
def points_for_note(
|
||||
prefix: str,
|
||||
note_payload: dict,
|
||||
note_vec: List[float] | None,
|
||||
dim: int,
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
|
||||
notes_col, _, _ = _names(prefix)
|
||||
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
||||
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
||||
@@ -35,7 +57,18 @@ def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | Non
|
||||
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
||||
return notes_col, [pt]
|
||||
|
||||
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
|
||||
def points_for_chunks(
|
||||
prefix: str,
|
||||
chunk_payloads: List[dict],
|
||||
vectors: List[List[float]],
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Chunks-Collection: erwartet pro Chunk einen Vektor.
|
||||
Robustheit:
|
||||
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
|
||||
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
|
||||
"""
|
||||
_, chunks_col, _ = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
||||
@@ -48,30 +81,55 @@ def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[Lis
|
||||
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
||||
return chunks_col, points
|
||||
|
||||
|
||||
def _normalize_edge_payload(pl: dict) -> dict:
|
||||
"""
|
||||
Sorgt für kompatible Feldnamen.
|
||||
akzeptiert:
|
||||
- neu: kind, source_id, target_id, seq?
|
||||
- alt: edge_type, src_id, dst_id, order?/index?
|
||||
schreibt zurück: kind, source_id, target_id, seq?
|
||||
"""
|
||||
# bereits neu?
|
||||
kind = pl.get("kind") or pl.get("edge_type") or "edge"
|
||||
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
|
||||
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
|
||||
seq = pl.get("seq") or pl.get("order") or pl.get("index")
|
||||
|
||||
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
|
||||
pl.setdefault("kind", kind)
|
||||
pl.setdefault("source_id", source_id)
|
||||
pl.setdefault("target_id", target_id)
|
||||
if seq is not None and "seq" not in pl:
|
||||
pl["seq"] = seq
|
||||
return pl
|
||||
|
||||
|
||||
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Edges-Collection mit 1D-Dummy-Vektor (keine Vektorsuche auf Edges).
|
||||
Stable-ID:
|
||||
Basis: "{kind}:{src}->{tgt}#{seq}"
|
||||
Neu: Falls 'occ' vorhanden -> "{…}@{occ}"
|
||||
Edges-Collection mit 1D-Dummy-Vektor.
|
||||
- Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
|
||||
- Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
|
||||
"""
|
||||
_, _, edges_col = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for pl in edge_payloads:
|
||||
for raw in edge_payloads:
|
||||
pl = _normalize_edge_payload(raw)
|
||||
|
||||
edge_id = pl.get("edge_id")
|
||||
if not edge_id:
|
||||
kind = pl.get("kind", "edge")
|
||||
s = pl.get("source_id", "unknown-src")
|
||||
t = pl.get("target_id", "unknown-tgt")
|
||||
seq = pl.get("seq") or pl.get("order") or ""
|
||||
occ = pl.get("occ")
|
||||
base = f"{kind}:{s}->{t}#{seq}"
|
||||
edge_id = f"{base}@{occ}" if occ is not None else base
|
||||
seq = pl.get("seq") or ""
|
||||
edge_id = f"{kind}:{s}->{t}#{seq}"
|
||||
pl["edge_id"] = edge_id
|
||||
|
||||
point_id = _to_uuid(edge_id)
|
||||
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
||||
return edges_col, points
|
||||
|
||||
|
||||
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
|
||||
if not points:
|
||||
return
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
# 📘 Mindnet Skript-Handbuch
|
||||
|
||||
Dieses Handbuch dokumentiert die implementierten Skripte für das Projekt **mindnet Wissensnetzwerk**.
|
||||
Es beschreibt Zweck, Parameter und typische Aufrufe. Alle Beispiele sind geprüft und lauffähig.
|
||||
|
||||
Alle Skripte liegen in `~/mindnet/scripts/`.
|
||||
Ausführung erfolgt **im aktivierten venv**:
|
||||
|
||||
cd ~/mindnet
|
||||
source .venv/bin/activate
|
||||
|
||||
---
|
||||
|
||||
## 1. Vault anlegen (für Tests)
|
||||
|
||||
### `make_test_vault.py`
|
||||
|
||||
**Zweck:**
|
||||
Erstellt einen kleinen, nachvollziehbaren Test-Vault mit Markdown-Dateien.
|
||||
Szenarien: offene Links, nachträgliche Anlage fehlender Notes, geänderte Chunk-Grenzen.
|
||||
|
||||
**Parameter:**
|
||||
- `--out PATH` : Zielverzeichnis (Default: `./test_vault`)
|
||||
- `--force` : Existierenden Ordner löschen und neu anlegen
|
||||
- `--with-missing` : Optional – auch die `missing-note` gleich anlegen
|
||||
|
||||
**Hinweise für Tests**
|
||||
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
|
||||
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
|
||||
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
|
||||
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
|
||||
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
|
||||
|
||||
Kompatibel mit:
|
||||
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
|
||||
|
||||
**Beispiele:**
|
||||
|
||||
python3 -m scripts.make_test_vault --out ./test_vault --force
|
||||
python3 -m scripts.make_test_vault --out ./test_vault --force --with-missing
|
||||
|
||||
---
|
||||
|
||||
## 2. Import in Qdrant
|
||||
|
||||
### `import_markdown.py`
|
||||
|
||||
**Zweck:**
|
||||
Importiert Notes/Chunks/Edges aus einem Vault nach Qdrant.
|
||||
- prüft Frontmatter
|
||||
- chunked Inhalte + Embeddings
|
||||
- erzeugt Edges (`references`, `references_at`, `backlink`)
|
||||
- erkennt geänderte Dateien automatisch
|
||||
- optional: Purge der alten Chunks/Edges pro Note
|
||||
|
||||
**Parameter:**
|
||||
- `--vault PATH` : (Pflicht) Vault-Verzeichnis
|
||||
- `--apply` : Änderungen wirklich durchführen (sonst Dry-Run)
|
||||
- `--purge-before-upsert` : Vor Upsert alte Chunks/Edges **nur der geänderten Notes** löschen
|
||||
- `--note-id ID` : Nur eine bestimmte Note importieren
|
||||
|
||||
**Beispiele:**
|
||||
|
||||
python3 -m scripts.import_markdown --vault ./vault
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
||||
|
||||
---
|
||||
|
||||
## 3. Konsistenzprüfungen
|
||||
|
||||
### `audit_vault_vs_qdrant.py`
|
||||
|
||||
**Zweck:**
|
||||
Vergleicht Vault-Inhalte mit Qdrant.
|
||||
- Zählt Notes/Chunks/Edges
|
||||
- Verifiziert Anzahl Wikilinks vs. Edges
|
||||
- Meldet Mismatches pro Note
|
||||
|
||||
**Parameter:**
|
||||
- `--vault PATH` : Vault-Verzeichnis
|
||||
- `--prefix STR` : Prefix für Qdrant-Collections (Default: `mindnet`)
|
||||
|
||||
**Beispiel:**
|
||||
|
||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
||||
|
||||
---
|
||||
|
||||
### `validate_edges.py`
|
||||
|
||||
**Zweck:**
|
||||
Prüft die Edges-Collection in Qdrant.
|
||||
- Zählt Edges pro Typ
|
||||
- Kontrolliert Invarianten (z. B. jede Reference hat Backlink)
|
||||
- Listet unresolved Edges auf
|
||||
|
||||
**Parameter:**
|
||||
- `--prefix STR` : Prefix für Qdrant-Collections
|
||||
- `--details` : Detailausgabe mit Beispielen
|
||||
|
||||
**Beispiel:**
|
||||
|
||||
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||
|
||||
---
|
||||
|
||||
## 4. Qdrant zurücksetzen
|
||||
|
||||
### `reset_qdrant.py`
|
||||
|
||||
**Zweck:**
|
||||
Setzt die Qdrant-Collections zurück.
|
||||
- „wipe“: löscht Collections komplett (inkl. Schema)
|
||||
- „truncate“: löscht nur Inhalte, behält Collections
|
||||
|
||||
**Parameter:**
|
||||
- `--mode {wipe,truncate}` : Aktion
|
||||
- `--prefix STR` : Prefix der Collections (Default: `mindnet`)
|
||||
- `--yes` : Automatische Bestätigung
|
||||
- `--dry-run` : Nur anzeigen, nicht ausführen
|
||||
|
||||
**Beispiele:**
|
||||
|
||||
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
||||
python3 -m scripts.reset_qdrant --mode wipe --prefix mindnet --yes
|
||||
|
||||
---
|
||||
|
||||
## 5. Diagnose
|
||||
|
||||
### `diagnose_changed.py`
|
||||
|
||||
**Zweck:**
|
||||
Prüft, ob die Datei-Hashes (`sha256`) mit den gespeicherten `hash_fulltext` in Qdrant übereinstimmen.
|
||||
Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt.
|
||||
|
||||
**Parameter:**
|
||||
- `--vault PATH` : Vault-Verzeichnis
|
||||
- `--prefix STR` : Prefix (Default: `mindnet`)
|
||||
- `--note-id ID` : Nur bestimmte Note prüfen
|
||||
- `--all` : Alle Notes prüfen
|
||||
|
||||
**Beispiel:**
|
||||
|
||||
python3 -m scripts.diagnose_changed --vault ./vault --prefix mindnet --all
|
||||
|
||||
---
|
||||
|
||||
## 6. Unresolved Links nachträglich auflösen
|
||||
|
||||
### `resolve_unresolved_references.py`
|
||||
|
||||
**Zweck:**
|
||||
Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwischen passende Notizen existieren.
|
||||
|
||||
**Parameter:**
|
||||
- `--prefix STR` : Prefix für Qdrant-Collections
|
||||
- `--apply` : Änderungen wirklich ausführen (sonst nur Vorschau)
|
||||
|
||||
**Beispiele:**
|
||||
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
||||
|
||||
---
|
||||
|
||||
# ✅ Typische Workflows
|
||||
|
||||
### A) Vault neu importieren
|
||||
|
||||
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
||||
|
||||
### B) Änderungen einspielen (mit Purge für geänderte Notes)
|
||||
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
||||
|
||||
### C) Unresolved Links nachziehen
|
||||
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
||||
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||
|
||||
### D) Test-Vault Szenario durchspielen
|
||||
|
||||
python3 -m scripts.make_test_vault --out ./test_vault --force
|
||||
python3 -m scripts.import_markdown --vault ./test_vault --apply
|
||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
|
||||
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: audit_edges_vs_expectations.py — Prüfe Kanten in Qdrant gegen Vault-Erwartungen
|
||||
Version: 1.0.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
Zweck
|
||||
-----
|
||||
- Liest Edges/Chunks/Notes aus Qdrant.
|
||||
- Ermittelt erwartete Kanten-Anzahlen aus dem Vault:
|
||||
* belongs_to : sollte == #Chunks
|
||||
* next / prev : je Note (#Chunks_in_Note - 1)
|
||||
* references : Summe aller Chunk-Wikilinks
|
||||
* backlink : Summe einzigartiger Wikilinks pro Note (Note-Level)
|
||||
- Vergleicht IST vs. SOLL und meldet Abweichungen.
|
||||
|
||||
ENV/Qdrant
|
||||
----------
|
||||
QDRANT_URL, QDRANT_API_KEY (optional), COLLECTION_PREFIX (Default: mindnet)
|
||||
|
||||
Aufrufe
|
||||
-------
|
||||
# Gesamtaudit
|
||||
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault
|
||||
|
||||
# Mit anderem Prefix
|
||||
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --prefix mindnet_dev
|
||||
|
||||
# Details anzeigen
|
||||
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --details
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict, Counter
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Projektmodule – nur leichtgewichtige Funktionen
|
||||
try:
|
||||
from app.core.parser import read_markdown
|
||||
except Exception:
|
||||
# sehr einfacher Fallback für Wikilinks
|
||||
read_markdown = None
|
||||
|
||||
WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||
|
||||
# ------------------------------
|
||||
# Qdrant Helpers
|
||||
# ------------------------------
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def _scroll_all(client: QdrantClient, col: str, flt=None, with_payload=True, with_vectors=False, limit=256):
|
||||
out, next_page = [], None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=col,
|
||||
scroll_filter=flt,
|
||||
with_payload=with_payload,
|
||||
with_vectors=with_vectors,
|
||||
limit=limit,
|
||||
offset=next_page,
|
||||
)
|
||||
if not pts:
|
||||
break
|
||||
out.extend(pts)
|
||||
if not next_page:
|
||||
break
|
||||
return out
|
||||
|
||||
# ------------------------------
|
||||
# Vault scan
|
||||
# ------------------------------
|
||||
|
||||
def _iter_md(root: str) -> List[str]:
|
||||
out: List[str] = []
|
||||
for dp, _, fns in os.walk(root):
|
||||
for fn in fns:
|
||||
if fn.lower().endswith(".md"):
|
||||
p = os.path.join(dp, fn)
|
||||
if "/.obsidian/" in p.replace("\\", "/"):
|
||||
continue
|
||||
out.append(p)
|
||||
return sorted(out)
|
||||
|
||||
def _wikilinks_in_text(text: str) -> List[str]:
|
||||
return WIKILINK_RE.findall(text or "")
|
||||
|
||||
def _wikilinks_per_note(vault_root: str) -> Dict[str, List[str]]:
|
||||
res: Dict[str, List[str]] = {}
|
||||
for p in _iter_md(vault_root):
|
||||
body = ""
|
||||
try:
|
||||
if read_markdown:
|
||||
parsed = read_markdown(p)
|
||||
body = parsed.body or ""
|
||||
fm = parsed.frontmatter or {}
|
||||
nid = fm.get("id") or fm.get("note_id") or os.path.splitext(os.path.basename(p))[0]
|
||||
else:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
txt = f.read()
|
||||
# sehr einfacher Split: YAML-Frontmatter rausnehmen
|
||||
if txt.lstrip().startswith("---"):
|
||||
parts = txt.split("\n---", 1)
|
||||
body = parts[1] if len(parts) > 1 else txt
|
||||
else:
|
||||
body = txt
|
||||
nid = os.path.splitext(os.path.basename(p))[0]
|
||||
res[nid] = _wikilinks_in_text(body)
|
||||
except Exception:
|
||||
continue
|
||||
return res
|
||||
|
||||
# ------------------------------
|
||||
# Main Audit
|
||||
# ------------------------------
|
||||
|
||||
def main():
|
||||
load_dotenv()
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Vault (für Erwartungswerte)")
|
||||
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX", "mindnet"), help="Collection-Prefix")
|
||||
ap.add_argument("--details", action="store_true", help="Detail-Listen ausgeben")
|
||||
args = ap.parse_args()
|
||||
|
||||
client = QdrantClient(url=os.environ.get("QDRANT_URL", "http://127.0.0.1:6333"),
|
||||
api_key=os.environ.get("QDRANT_API_KEY") or None)
|
||||
notes_col, chunks_col, edges_col = _names(args.prefix)
|
||||
|
||||
# Qdrant laden
|
||||
notes = _scroll_all(client, notes_col, with_payload=True, with_vectors=False)
|
||||
chunks = _scroll_all(client, chunks_col, with_payload=True, with_vectors=False)
|
||||
edges = _scroll_all(client, edges_col, with_payload=True, with_vectors=False)
|
||||
|
||||
# --- Ist-Zähler
|
||||
cnt_kind = Counter()
|
||||
cnt_scope = Counter()
|
||||
by_note_chunks: Dict[str, int] = defaultdict(int)
|
||||
chunk_wikilinks_total = 0
|
||||
|
||||
for p in chunks:
|
||||
pl = p.payload or {}
|
||||
by_note_chunks[pl.get("note_id")] += 1
|
||||
wl = pl.get("wikilinks") or []
|
||||
if isinstance(wl, list):
|
||||
chunk_wikilinks_total += len(wl)
|
||||
|
||||
for p in edges:
|
||||
pl = p.payload or {}
|
||||
kind = pl.get("kind") or pl.get("edge_type") or "?"
|
||||
scope = pl.get("scope") or "?"
|
||||
cnt_kind[kind] += 1
|
||||
cnt_scope[f"{kind}:{scope}"] += 1
|
||||
|
||||
total_chunks = sum(by_note_chunks.values())
|
||||
|
||||
# --- Soll-Zähler aus Vault
|
||||
wl_per_note = _wikilinks_per_note(args.vault)
|
||||
backlink_expected = sum(len(set(v)) for v in wl_per_note.values())
|
||||
|
||||
next_expected = sum(max(c - 1, 0) for c in by_note_chunks.values())
|
||||
prev_expected = next_expected # symmetrische Kanten
|
||||
|
||||
belongs_to_expected = total_chunks
|
||||
references_expected = chunk_wikilinks_total # aus Chunk-Payloads
|
||||
|
||||
# --- Ergebnis
|
||||
result = {
|
||||
"qdrant_counts": dict(cnt_kind),
|
||||
"qdrant_counts_by_scope": dict(cnt_scope),
|
||||
"chunks_total": total_chunks,
|
||||
"by_note_chunks": dict(by_note_chunks),
|
||||
"vault_expected": {
|
||||
"belongs_to": belongs_to_expected,
|
||||
"next": next_expected,
|
||||
"prev": prev_expected,
|
||||
"references": references_expected,
|
||||
"backlink": backlink_expected,
|
||||
},
|
||||
"deltas": {
|
||||
"belongs_to": cnt_kind.get("belongs_to", 0) - belongs_to_expected,
|
||||
"next": cnt_kind.get("next", 0) - next_expected,
|
||||
"prev": cnt_kind.get("prev", 0) - prev_expected,
|
||||
"references": cnt_kind.get("references", 0) - references_expected,
|
||||
"backlink": cnt_kind.get("backlink", 0) - backlink_expected,
|
||||
},
|
||||
"collections": {
|
||||
"notes": notes_col, "chunks": chunks_col, "edges": edges_col
|
||||
}
|
||||
}
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
if args.details:
|
||||
# optionale Stichproben (z. B. fehlerhafte Kantenarten)
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: export_markdown.py — Qdrant → Markdown (Obsidian-kompatibel)
|
||||
Version: 1.4.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
Exportiert Markdown-Notizen aus Qdrant in einen Zielordner (Vault).
|
||||
Rekonstruiert YAML-Frontmatter und Body.
|
||||
|
||||
Body-Rekonstruktions-Priorität (abwärtskompatibel):
|
||||
1) notes.payload.fulltext (verlustfrei, wenn beim Import gespeichert)
|
||||
2) ansonsten aus allen zugehörigen Chunks: payload.text → payload.content → payload.raw
|
||||
in stabiler, sequentieller Reihenfolge (seq/chunk_index/ID-Nummer)
|
||||
|
||||
Wichtige Fixes
|
||||
--------------
|
||||
- **Pfad-Normalisierung**: erzwingt relative Pfade (führe führende '/' ab, backslashes → slashes),
|
||||
damit ``--out`` nicht ignoriert wird.
|
||||
- **`--prefix` (optional)**: Überschreibt COLLECTION_PREFIX; ENV bleibt Default (rückwärtskompatibel).
|
||||
|
||||
ENV / Qdrant
|
||||
------------
|
||||
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
|
||||
- QDRANT_API_KEY (optional)
|
||||
- COLLECTION_PREFIX (Default: mindnet)
|
||||
|
||||
Aufruf
|
||||
------
|
||||
python3 -m scripts.export_markdown --out ./_exportVault
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-foo
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --prefix mindnet_dev # optional
|
||||
|
||||
Beispiele
|
||||
---------
|
||||
COLLECTION_PREFIX=mindnet QDRANT_URL=http://127.0.0.1:6333 \\
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
import yaml
|
||||
from qdrant_client.http import models as rest
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Utilities
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
|
||||
def _ensure_dir(path: str) -> None:
|
||||
d = os.path.dirname(path)
|
||||
if d and not os.path.isdir(d):
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
|
||||
def _normalize_rel_path(p: str) -> str:
|
||||
"""Pfad relativ halten & normalisieren (slashes, führende / entfernen)."""
|
||||
p = (p or "").replace("\\", "/")
|
||||
return p.lstrip("/")
|
||||
|
||||
|
||||
def _to_md(frontmatter: dict, body: str) -> str:
|
||||
fm = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=True).strip()
|
||||
return f"---\n{fm}\n---\n{(body or '').rstrip()}\n"
|
||||
|
||||
|
||||
def _scroll_all(
|
||||
client: QdrantClient,
|
||||
col: str,
|
||||
flt: Optional[rest.Filter] = None,
|
||||
with_payload: bool = True,
|
||||
with_vectors: bool = False,
|
||||
limit: int = 256,
|
||||
):
|
||||
"""Scrollt durch alle Punkte einer Collection und liefert eine Liste mit Points."""
|
||||
out = []
|
||||
next_page = None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=col,
|
||||
scroll_filter=flt,
|
||||
with_payload=with_payload,
|
||||
with_vectors=with_vectors,
|
||||
limit=limit,
|
||||
offset=next_page,
|
||||
)
|
||||
if not pts:
|
||||
break
|
||||
out.extend(pts)
|
||||
if not next_page:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _load_chunks_for_note(client: QdrantClient, chunks_col: str, note_id: str) -> List[dict]:
|
||||
flt = rest.Filter(must=[rest.FieldCondition(
|
||||
key="note_id",
|
||||
match=rest.MatchValue(value=note_id),
|
||||
)])
|
||||
pts = _scroll_all(client, chunks_col, flt, with_payload=True, with_vectors=False)
|
||||
# Sortierung: bevorzugt seq → chunk_index → Nummer in id
|
||||
def _seq(pl: dict) -> Tuple[int, int, int]:
|
||||
s1 = pl.get("seq", pl.get("chunk_index", -1))
|
||||
s2 = pl.get("chunk_index", -1)
|
||||
# Nummer-Anteil aus "noteid#<n>"
|
||||
s3 = 0
|
||||
try:
|
||||
m = re.search(r"#(\\d+)$", pl.get("id") or "")
|
||||
if m:
|
||||
s3 = int(m.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
return (int(s1) if isinstance(s1, int) else -1, int(s2) if isinstance(s2, int) else -1, s3)
|
||||
|
||||
pts_sorted = sorted(pts, key=lambda p: _seq(p.payload or {}))
|
||||
return [p.payload or {} for p in pts_sorted]
|
||||
|
||||
|
||||
def _reconstruct_body(note_pl: dict, chunk_payloads: List[dict]) -> str:
|
||||
# 1) Volltext vorhanden?
|
||||
fulltext = note_pl.get("fulltext")
|
||||
if isinstance(fulltext, str) and fulltext.strip():
|
||||
return fulltext
|
||||
|
||||
# 2) Aus Chunks zusammensetzen: text → content → raw
|
||||
parts: List[str] = []
|
||||
for ch in chunk_payloads:
|
||||
text = ch.get("text") or ch.get("content") or ch.get("raw")
|
||||
if isinstance(text, str) and text.strip():
|
||||
parts.append(text.rstrip())
|
||||
return ("\n\n".join(parts)).rstrip() + ("\n" if parts else "")
|
||||
|
||||
|
||||
def _export_one_note(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
note_pl: dict,
|
||||
out_root: str,
|
||||
overwrite: bool,
|
||||
) -> dict:
|
||||
notes_col, chunks_col, _ = _names(prefix)
|
||||
note_id = note_pl.get("note_id") or note_pl.get("id")
|
||||
|
||||
# Pfad robust bestimmen und relativ halten
|
||||
path = note_pl.get("path") or f"{note_id}.md"
|
||||
path = _normalize_rel_path(path)
|
||||
out_path = os.path.join(out_root, path).replace("\\", "/")
|
||||
|
||||
# Frontmatter aus Payload zurückführen (nur bekannte Felder)
|
||||
fm: Dict[str, object] = {}
|
||||
for k in [
|
||||
"title", "id", "type", "status", "created", "updated", "tags",
|
||||
"priority", "effort_min", "due", "people", "aliases",
|
||||
"depends_on", "assigned_to", "lang",
|
||||
]:
|
||||
v = note_pl.get(k) if k in note_pl else note_pl.get(f"note_{k}")
|
||||
if v not in (None, [], ""):
|
||||
fm[k] = v
|
||||
|
||||
# Mindestfelder
|
||||
if "id" not in fm and note_id:
|
||||
fm["id"] = note_id
|
||||
if "title" not in fm and note_pl.get("title"):
|
||||
fm["title"] = note_pl["title"]
|
||||
|
||||
# Body beschaffen
|
||||
chunks = _load_chunks_for_note(client, chunks_col, note_id)
|
||||
body = _reconstruct_body(note_pl, chunks)
|
||||
|
||||
# Schreiben?
|
||||
if os.path.exists(out_path) and not overwrite:
|
||||
return {"note_id": note_id, "path": path, "status": "skip_exists"}
|
||||
|
||||
_ensure_dir(out_path)
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
f.write(_to_md(fm, body))
|
||||
|
||||
return {"note_id": note_id, "path": path, "status": "written"}
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Main
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", required=True, help="Zielordner für den Export-Vault")
|
||||
ap.add_argument("--note-id", help="Nur eine Note exportieren (Note-ID)")
|
||||
ap.add_argument("--overwrite", action="store_true", help="Bestehende Dateien überschreiben")
|
||||
ap.add_argument("--prefix", help="(Optional) überschreibt COLLECTION_PREFIX aus ENV")
|
||||
args = ap.parse_args()
|
||||
|
||||
# Qdrant-Konfiguration
|
||||
cfg = QdrantConfig.from_env()
|
||||
if args.prefix:
|
||||
cfg.prefix = args.prefix # abwärtskompatibel: ENV bleibt Default
|
||||
|
||||
client = get_client(cfg)
|
||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
|
||||
notes_col, _, _ = _names(cfg.prefix)
|
||||
|
||||
# Notes holen (optional gefiltert)
|
||||
flt = None
|
||||
if args.note_id:
|
||||
flt = rest.Filter(must=[rest.FieldCondition(
|
||||
key="note_id",
|
||||
match=rest.MatchValue(value=args.note_id),
|
||||
)])
|
||||
|
||||
note_pts = _scroll_all(client, notes_col, flt, with_payload=True, with_vectors=False)
|
||||
if not note_pts:
|
||||
print(json.dumps({"exported": 0, "out": args.out, "message": "Keine Notes gefunden."}, ensure_ascii=False))
|
||||
return
|
||||
|
||||
results = []
|
||||
for p in note_pts:
|
||||
pl = p.payload or {}
|
||||
try:
|
||||
res = _export_one_note(client, cfg.prefix, pl, args.out, args.overwrite)
|
||||
except Exception as e:
|
||||
res = {"note_id": pl.get("note_id") or pl.get("id"), "error": str(e)}
|
||||
results.append(res)
|
||||
|
||||
print(json.dumps({
|
||||
"exported": len([r for r in results if r.get("status") == "written"]),
|
||||
"skipped": len([r for r in results if r.get("status") == "skip_exists"]),
|
||||
"out": args.out,
|
||||
"details": results,
|
||||
}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
# scripts/gc_qdrant_after_vault_scan.py
|
||||
# Version: 1.0.0 (2025-09-05)
|
||||
#
|
||||
# Zweck
|
||||
# -----
|
||||
# Garbage-Collector für Qdrant: löscht Einträge (Edges, Chunks, Notes), deren Note-ID
|
||||
# nicht mehr im Obsidian-Vault vorhanden ist. So bleiben Vektorindex/Graph mit dem Vault konsistent.
|
||||
#
|
||||
# Aufrufparameter
|
||||
# ---------------
|
||||
# --vault PATH Pfad zum Obsidian-Vault (z. B. ./vault) [erforderlich]
|
||||
# --mode {edges,content,all}
|
||||
# edges -> nur Kanten mit source_id/target_id ∉ Vault-Notes löschen
|
||||
# content -> Chunks & Notes löschen, deren note_id ∉ Vault-Notes (keine Edges)
|
||||
# all -> zuerst edges, dann content
|
||||
# --prefix PREFIX Collection-Präfix (Default: aus ENV COLLECTION_PREFIX oder "mindnet")
|
||||
# --apply Ohne diesen Schalter: Dry-Run (nur Vorschau). Mit Schalter: ausführen.
|
||||
# --yes Sicherheitsabfrage überspringen (non-interaktiv)
|
||||
# --batch-size N Größe für In-Filter (Default: 1000)
|
||||
#
|
||||
# Hinweise
|
||||
# --------
|
||||
# - Benötigt funktionierendes Python-venv mit allen Abhängigkeiten von mindnet (qdrant-client usw.).
|
||||
# - Läuft gegen die drei Collections {prefix}_notes, {prefix}_chunks, {prefix}_edges (keine anderen Projekte).
|
||||
# - Nutzt OR-Filter (Filter.should) korrekt, keine minimum_should (Pydantic v2 lässt die Option nicht zu).
|
||||
#
|
||||
# Änderungen ggü. vorher
|
||||
# ----------------------
|
||||
# - Neu: eigenes GC-Tool; berührt ausschließlich das übergebene Projekt-Präfix.
|
||||
# - Sichere Dry-Run-Vorschau mit Zählung pro Collection; interaktive Bestätigung.
|
||||
# - Löscht Edges mittels Filter auf (source_id ∈ MISSING) ODER (target_id ∈ MISSING).
|
||||
# - Löscht Chunks/Notes mittels Filter auf note_id ∈ MISSING.
|
||||
#
|
||||
# Beispiele
|
||||
# ---------
|
||||
# DRY RUN: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all
|
||||
# APPLY: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all --apply
|
||||
# CI-Modus: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode edges --apply --yes
|
||||
#
|
||||
# Kompatibilität / Kontext
|
||||
# ------------------------
|
||||
# - Frontmatter-Validierung & Parsing wie im Projekt (parser.read_markdown, validate_required_frontmatter). :contentReference[oaicite:0]{index=0}
|
||||
# - Note-Payload-Schema (u. a. note_id, hash_fulltext) wie spezifiziert. :contentReference[oaicite:1]{index=1}
|
||||
# - Collection-Namen & Anlage-Logik über app.core.qdrant (notes/chunks = dim, edges = 1D). :contentReference[oaicite:2]{index=2}
|
||||
# - Edges nutzen Felder source_id/target_id; Chunks/Notes tragen note_id in payload. :contentReference[oaicite:3]{index=3} :contentReference[oaicite:4]{index=4}
|
||||
#
|
||||
from __future__ import annotations
|
||||
import argparse, os, glob, json, sys
|
||||
from typing import Iterable, Set, Tuple, List
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter # :contentReference[oaicite:5]{index=5}
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names # :contentReference[oaicite:6]{index=6}
|
||||
|
||||
def iter_note_ids_from_vault(vault_root: str) -> Set[str]:
|
||||
"""Liest alle Markdown-Dateien im Vault und sammelt valide Frontmatter-IDs."""
|
||||
paths = [p for p in glob.glob(os.path.join(vault_root, "**", "*.md"), recursive=True)]
|
||||
out: Set[str] = set()
|
||||
for p in paths:
|
||||
try:
|
||||
parsed = read_markdown(p)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
nid = fm.get("id") or fm.get("note_id")
|
||||
if nid:
|
||||
out.add(nid)
|
||||
except Exception:
|
||||
# ungültige oder Template-Dateien ohne Pflichtfelder ignorieren
|
||||
continue
|
||||
return out
|
||||
|
||||
def scroll_payload_values(client: QdrantClient, collection: str, field: str, limit: int = 2000) -> Set[str]:
|
||||
"""Scrollt eine Collection und sammelt alle Payload-Werte eines Feldes (falls vorhanden)."""
|
||||
out: Set[str] = set()
|
||||
offset = None
|
||||
while True:
|
||||
res = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=None,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=limit,
|
||||
offset=offset
|
||||
)
|
||||
# qdrant-client (neuere Versionen) geben (points, next_offset)
|
||||
if isinstance(res, tuple) and len(res) == 2:
|
||||
points, offset = res
|
||||
else:
|
||||
# Fallback, falls sich API signifikant ändert
|
||||
points, offset = res[0], res[1] if len(res) > 1 else None
|
||||
|
||||
for pt in points or []:
|
||||
pl = getattr(pt, "payload", None) or {}
|
||||
v = pl.get(field)
|
||||
if isinstance(v, str):
|
||||
out.add(v)
|
||||
if not offset:
|
||||
break
|
||||
return out
|
||||
|
||||
def chunked(iterable: Iterable[str], n: int) -> Iterable[List[str]]:
|
||||
buf: List[str] = []
|
||||
for x in iterable:
|
||||
buf.append(x)
|
||||
if len(buf) >= n:
|
||||
yield buf
|
||||
buf = []
|
||||
if buf:
|
||||
yield buf
|
||||
|
||||
def count_matches(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
|
||||
"""Grober Zähler über scroll (Qdrant Count-API ist optional je Version)."""
|
||||
total = 0
|
||||
offset = None
|
||||
while True:
|
||||
points, offset = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=filter_,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
limit=2000,
|
||||
offset=offset
|
||||
)
|
||||
total += len(points or [])
|
||||
if not offset:
|
||||
break
|
||||
return total
|
||||
|
||||
def build_filters_for_missing(prefix: str, missing_note_ids: Set[str]) -> Tuple[rest.Filter, rest.Filter, rest.Filter]:
|
||||
"""
|
||||
Erzeugt je Collection passende Filter:
|
||||
- Notes/Chunks: note_id ∈ missing
|
||||
- Edges: (source_id ∈ missing) ODER (target_id ∈ missing)
|
||||
"""
|
||||
notes_col, chunks_col, edges_col = collection_names(prefix) # :contentReference[oaicite:7]{index=7}
|
||||
# MatchAny statt MatchValue für große Mengen
|
||||
any_missing = rest.MatchAny(any=list(missing_note_ids))
|
||||
|
||||
f_notes = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
|
||||
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
|
||||
f_edges = rest.Filter(should=[
|
||||
rest.FieldCondition(key="source_id", match=any_missing),
|
||||
rest.FieldCondition(key="target_id", match=any_missing),
|
||||
])
|
||||
return f_notes, f_chunks, f_edges
|
||||
|
||||
def preview(client: QdrantClient, prefix: str, missing: Set[str]) -> dict:
|
||||
"""Erstellt eine Dry-Run-Vorschau (Anzahl Treffer pro Collection)."""
|
||||
notes_col, chunks_col, edges_col = collection_names(prefix)
|
||||
f_notes, f_chunks, f_edges = build_filters_for_missing(prefix, missing)
|
||||
|
||||
return {
|
||||
"prefix": prefix,
|
||||
"collections": {"notes": notes_col, "chunks": chunks_col, "edges": edges_col},
|
||||
"missing_note_ids_count": len(missing),
|
||||
"to_delete_counts": {
|
||||
"edges": count_matches(client, edges_col, f_edges),
|
||||
"chunks": count_matches(client, chunks_col, f_chunks),
|
||||
"notes": count_matches(client, notes_col, f_notes),
|
||||
}
|
||||
}
|
||||
|
||||
def delete_with_filter(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
|
||||
"""Löscht alle Punkte, die den Filter matchen. Gibt grobe Trefferzahl zurück."""
|
||||
# erst zählen (für Feedback)
|
||||
to_del = count_matches(client, collection, filter_)
|
||||
if to_del == 0:
|
||||
return 0
|
||||
client.delete(
|
||||
collection_name=collection,
|
||||
points_selector=filter_,
|
||||
wait=True
|
||||
)
|
||||
return to_del
|
||||
|
||||
def main():
|
||||
load_dotenv()
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (z. B. ./vault)")
|
||||
ap.add_argument("--mode", choices=["edges", "content", "all"], default="all",
|
||||
help="Löschmodus (edges=nur Kanten, content=Chunks+Notes, all=beides)")
|
||||
ap.add_argument("--prefix", help="Collection-Präfix (Default aus ENV COLLECTION_PREFIX)")
|
||||
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter: Dry-Run")
|
||||
ap.add_argument("--yes", action="store_true", help="Rückfrage unterdrücken")
|
||||
ap.add_argument("--batch-size", type=int, default=1000, help="Batch-Größe für MatchAny")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
if args.prefix:
|
||||
cfg.prefix = args.prefix
|
||||
client = get_client(cfg)
|
||||
|
||||
notes_col, chunks_col, edges_col = collection_names(cfg.prefix)
|
||||
|
||||
# 1) Vault lesen → Menge gültiger Note-IDs
|
||||
vault_root = os.path.abspath(args.vault)
|
||||
vault_ids = iter_note_ids_from_vault(vault_root)
|
||||
|
||||
# 2) Qdrant vorhandene Note-IDs einsammeln
|
||||
existing_note_ids = scroll_payload_values(client, notes_col, "note_id")
|
||||
|
||||
# 3) Differenz ermitteln
|
||||
missing = existing_note_ids - vault_ids
|
||||
|
||||
# 4) Vorschau
|
||||
summary = preview(client, cfg.prefix, missing)
|
||||
print(json.dumps({"mode": args.mode, "apply": args.apply, "summary": summary}, ensure_ascii=False, indent=2))
|
||||
|
||||
if not missing:
|
||||
# nichts zu tun
|
||||
return
|
||||
|
||||
if not args.apply:
|
||||
# Dry-Run Ende
|
||||
return
|
||||
|
||||
if not args.yes:
|
||||
resp = input("Fortfahren und die oben gezeigten Objekte löschen? (yes/no): ").strip().lower()
|
||||
if resp not in ("y", "yes"):
|
||||
print("Abgebrochen.")
|
||||
return
|
||||
|
||||
# 5) Löschen je Modus
|
||||
f_notes, f_chunks, f_edges = build_filters_for_missing(cfg.prefix, missing)
|
||||
report = {"deleted": {"edges": 0, "chunks": 0, "notes": 0}}
|
||||
|
||||
try:
|
||||
if args.mode in ("edges", "all"):
|
||||
report["deleted"]["edges"] = delete_with_filter(client, edges_col, f_edges)
|
||||
if args.mode in ("content", "all"):
|
||||
# Reihenfolge: erst chunks, dann notes
|
||||
report["deleted"]["chunks"] = delete_with_filter(client, chunks_col, f_chunks)
|
||||
report["deleted"]["notes"] = delete_with_filter(client, notes_col, f_notes)
|
||||
finally:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+218
-133
@@ -1,185 +1,270 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: scripts/import_markdown.py
|
||||
Version: v2.3.1 (2025-09-05)
|
||||
Script: import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
|
||||
Version: 3.2.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
Beschreibung
|
||||
Importiert Markdown-Notizen in Qdrant (Notes, Chunks, Edges).
|
||||
- Chunking + Embedding (MiniLM 384d, externer Embed-Server)
|
||||
- Edges direkt beim Import aus Wikilinks ([[…]]) ableiten
|
||||
- Idempotente UUIDv5-IDs; Collections werden bereitgestellt
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
Liest Markdown-Dateien aus einem Vault ein und schreibt Notes, Chunks und Edges
|
||||
idempotent nach Qdrant. Change-Detection basiert standardmäßig auf dem **Body-Hash**.
|
||||
Neu: Hash-Modus und Normalisierung sind auch per **CLI** steuerbar.
|
||||
|
||||
ENV / Qdrant
|
||||
------------
|
||||
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
|
||||
- QDRANT_API_KEY (optional)
|
||||
- COLLECTION_PREFIX (Default: mindnet)
|
||||
- VECTOR_DIM (Default: 384)
|
||||
- MINDNET_HASH_MODE: "body" (Default) | "frontmatter" | "body+frontmatter"
|
||||
- MINDNET_HASH_NORMALIZE: "canonical" (Default) | "none"
|
||||
|
||||
CLI (übersteuert ENV)
|
||||
---------------------
|
||||
--hash-mode body|frontmatter|body+frontmatter
|
||||
--hash-normalize canonical|none
|
||||
|
||||
Aufruf
|
||||
python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id ID] [--embed-note] [--force-replace]
|
||||
|
||||
Parameter
|
||||
--vault Pfad zum Obsidian-Vault (erforderlich)
|
||||
--apply Ohne Flag: Dry-Run (nur JSON-Zeilen). Mit Flag: schreibt in Qdrant.
|
||||
--note-id Nur eine spezifische Note-ID verarbeiten
|
||||
--embed-note Optional: Note-Volltext zusätzlich einbetten
|
||||
--force-replace Vor Upsert: zugehörige Edges der Quell-Note in Qdrant löschen (harte Ersetzung)
|
||||
|
||||
Hinweise
|
||||
- Im venv laufen: `source .venv/bin/activate`
|
||||
- Erwartet laufenden Embed-Server (http://127.0.0.1:8990)
|
||||
- Qdrant via env: QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM
|
||||
|
||||
Changelog
|
||||
v2.3.1: FIX – Für derive_wikilink_edges werden jetzt echte Chunk-Texte übergeben
|
||||
(chunks_for_links mit {"chunk_id","text"}), damit `references_at` erzeugt werden.
|
||||
v2.3.0: Umstellung auf app.core.derive_edges; Edge-IDs unterstützen Occurrence.
|
||||
v2.2.x: Fix Filter-API (pydantic v2) bei Purge; Kleinkosmetik.
|
||||
------
|
||||
python3 -m scripts.import_markdown --vault ./vault
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
||||
python3 -m scripts.import_markdown --vault ./vault --note-id 20250821-foo --apply
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --embed-note
|
||||
# Feingranulare Erkennung (jede Kleinigkeit im Body zählt):
|
||||
python3 -m scripts.import_markdown --vault ./vault --hash-normalize none
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse, os, glob, json, sys
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.validate_note import validate_note_payload
|
||||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunk_payload import make_chunk_payloads
|
||||
from app.core.embed import embed_texts, embed_one
|
||||
from app.core.qdrant import QdrantConfig, ensure_collections, get_client
|
||||
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
|
||||
from app.core.derive_edges import build_note_index, derive_wikilink_edges # nutzt 'text' je Chunk :contentReference[oaicite:1]{index=1}
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
def iter_md(root: str, exclude_dirs=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")):
|
||||
files = [p for p in glob.glob(os.path.join(root, "**", "*.md"), recursive=True)]
|
||||
out = []
|
||||
for p in files:
|
||||
pn = p.replace("\\","/")
|
||||
if any(ex in pn for ex in exclude_dirs):
|
||||
continue
|
||||
out.append(p)
|
||||
return out
|
||||
from app.core.parser import (
|
||||
read_markdown,
|
||||
normalize_frontmatter,
|
||||
validate_required_frontmatter,
|
||||
extract_wikilinks,
|
||||
)
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunk_payload import make_chunk_payloads
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
|
||||
from app.core.qdrant_points import (
|
||||
points_for_chunks,
|
||||
points_for_note,
|
||||
points_for_edges,
|
||||
upsert_batch,
|
||||
)
|
||||
|
||||
def purge_note_edges(client, prefix: str, source_note_id: str):
|
||||
# Löscht Edges, deren source_id == source_note_id ODER (kind==backlink && target_id==source_note_id)
|
||||
edges_col = f"{prefix}_edges"
|
||||
f = rest.Filter(
|
||||
should=[
|
||||
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_note_id)),
|
||||
rest.Filter(
|
||||
must=[
|
||||
rest.FieldCondition(key="kind", match=rest.MatchValue(value="backlink")),
|
||||
rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_note_id)),
|
||||
]
|
||||
),
|
||||
]
|
||||
try:
|
||||
from app.core.embed import embed_texts, embed_one # optional
|
||||
except Exception:
|
||||
embed_texts = None
|
||||
embed_one = None
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Hilfsfunktionen
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def iter_md(root: str) -> List[str]:
|
||||
out: List[str] = []
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
for fn in filenames:
|
||||
if not fn.lower().endswith(".md"):
|
||||
continue
|
||||
p = os.path.join(dirpath, fn)
|
||||
pn = p.replace("\\", "/")
|
||||
if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
|
||||
continue
|
||||
out.append(p)
|
||||
return sorted(out)
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def fetch_existing_note_hash(client, prefix: str, note_id: str) -> Optional[str]:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
f = rest.Filter(must=[rest.FieldCondition(
|
||||
key="note_id",
|
||||
match=rest.MatchValue(value=note_id),
|
||||
)])
|
||||
points, _ = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=f,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1,
|
||||
)
|
||||
# pydantic v2: client.delete(...) statt delete_points(...)
|
||||
client.delete(collection_name=edges_col, points_selector=f, wait=True)
|
||||
if not points:
|
||||
return None
|
||||
pl = points[0].payload or {}
|
||||
return pl.get("hash_fulltext")
|
||||
|
||||
def main():
|
||||
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||
_, chunks_col, edges_col = collections(prefix)
|
||||
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
client.delete(collection_name=chunks_col, points_selector=f_chunks, wait=True)
|
||||
should = [
|
||||
rest.FieldCondition(key="source_id", match=rest.MatchText(text=f"{note_id}#")),
|
||||
rest.FieldCondition(key="target_id", match=rest.MatchText(text=f"{note_id}#")),
|
||||
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=note_id)),
|
||||
rest.FieldCondition(key="target_id", match=rest.MatchValue(value=note_id)),
|
||||
]
|
||||
f_edges = rest.Filter(should=should)
|
||||
client.delete(collection_name=edges_col, points_selector=f_edges, wait=True)
|
||||
|
||||
def _normalize_rel_path(abs_path: str, vault_root: str) -> str:
|
||||
try:
|
||||
rel = os.path.relpath(abs_path, vault_root)
|
||||
except Exception:
|
||||
rel = abs_path
|
||||
return rel.replace("\\", "/").lstrip("/")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Main
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
load_dotenv()
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian Vault (z.B. ./vault)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant (sonst Dry-Run)")
|
||||
ap.add_argument("--note-id", help="Nur eine Note-ID verarbeiten")
|
||||
ap.add_argument("--embed-note", action="store_true", help="Auch Note-Volltext einbetten (optional)")
|
||||
ap.add_argument("--force-replace", action="store_true", help="Vor Upsert alte Edges der Note löschen")
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
|
||||
ap.add_argument("--purge-before-upsert", action="store_true",
|
||||
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
|
||||
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
|
||||
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
|
||||
ap.add_argument("--force-replace", action="store_true",
|
||||
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
|
||||
# NEU: Hash-Steuerung per CLI
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter"], default=None)
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
# Qdrant
|
||||
cfg = QdrantConfig(
|
||||
url=os.getenv("QDRANT_URL", "http://127.0.0.1:6333"),
|
||||
api_key=os.getenv("QDRANT_API_KEY") or None,
|
||||
prefix=os.getenv("COLLECTION_PREFIX", "mindnet"),
|
||||
dim=int(os.getenv("VECTOR_DIM","384")),
|
||||
)
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
notes_col, chunks_col, edges_col = collections(cfg.prefix)
|
||||
|
||||
root = os.path.abspath(args.vault)
|
||||
files = iter_md(root)
|
||||
if not files:
|
||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr); sys.exit(2)
|
||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# --- Note-Index (für robuste Zielauflösung) ---
|
||||
note_stubs = []
|
||||
processed = 0
|
||||
for path in files:
|
||||
parsed = read_markdown(path)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
try:
|
||||
validate_required_frontmatter(fm)
|
||||
except Exception:
|
||||
continue
|
||||
if args.note_id and fm.get("id") != args.note_id:
|
||||
continue
|
||||
rel = os.path.relpath(parsed.path, root).replace("\\","/")
|
||||
note_stubs.append({"note_id": fm["id"], "title": fm.get("title",""), "path": rel})
|
||||
note_index = build_note_index(note_stubs)
|
||||
|
||||
total_notes = 0
|
||||
for path in files:
|
||||
parsed = read_markdown(path)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
try:
|
||||
validate_required_frontmatter(fm)
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"}))
|
||||
continue
|
||||
|
||||
if args.note_id and fm.get("id") != args.note_id:
|
||||
continue
|
||||
|
||||
total_notes += 1
|
||||
processed += 1
|
||||
|
||||
# Note-Payload
|
||||
note_pl = make_note_payload(parsed, vault_root=root)
|
||||
validate_note_payload(note_pl)
|
||||
note_pl["fulltext"] = parsed.body # für derive_edges
|
||||
# Note-Payload (mit expliziten Hash-Parametern)
|
||||
note_pl = make_note_payload(parsed, vault_root=root,
|
||||
hash_mode=args.hash_mode, hash_normalize=args.hash_normalize)
|
||||
if "fulltext" not in (note_pl or {}):
|
||||
note_pl["fulltext"] = parsed.body or ""
|
||||
if note_pl.get("path"):
|
||||
note_pl["path"] = _normalize_rel_path(
|
||||
os.path.join(root, note_pl["path"]) if not os.path.isabs(note_pl["path"]) else note_pl["path"], root
|
||||
)
|
||||
else:
|
||||
note_pl["path"] = _normalize_rel_path(parsed.path, root)
|
||||
|
||||
# Chunks
|
||||
note_id = note_pl["note_id"]
|
||||
|
||||
# Change-Detection
|
||||
new_hash = note_pl.get("hash_fulltext")
|
||||
old_hash = None if args.force_replace else fetch_existing_note_hash(client, cfg.prefix, note_id)
|
||||
changed = args.force_replace or (old_hash != new_hash)
|
||||
|
||||
# Chunks + Embeddings (Nullvektor-Fallback)
|
||||
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
|
||||
if embed_texts:
|
||||
vecs = embed_texts([getattr(c, "text", "") for c in chunks])
|
||||
else:
|
||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
||||
|
||||
# Embeddings (Chunks)
|
||||
texts = [ch.text for ch in chunks]
|
||||
vectors = embed_texts(texts)
|
||||
# Edges (leichtgewichtig direkt hier ableiten)
|
||||
# belongs_to / prev/next aus Chunk-Nachbarschaften + Wikilinks als references
|
||||
edges: List[Dict] = []
|
||||
for ch in chunk_pls:
|
||||
cid = ch["id"]
|
||||
edges.append({"src_id": cid, "dst_id": note_id, "edge_type": "belongs_to", "scope": "chunk"})
|
||||
nb = ch.get("neighbors") or {}
|
||||
if nb.get("prev"):
|
||||
edges.append({"src_id": nb["prev"], "dst_id": cid, "edge_type": "next", "scope": "chunk"})
|
||||
edges.append({"src_id": cid, "dst_id": nb["prev"], "edge_type": "prev", "scope": "chunk"})
|
||||
if nb.get("next"):
|
||||
edges.append({"src_id": cid, "dst_id": nb["next"], "edge_type": "next", "scope": "chunk"})
|
||||
edges.append({"src_id": nb["next"], "dst_id": cid, "edge_type": "prev", "scope": "chunk"})
|
||||
for ref in (ch.get("references") or []):
|
||||
tid = ref.get("target_id")
|
||||
if tid:
|
||||
edges.append({"src_id": cid, "dst_id": tid, "edge_type": "references", "scope": "chunk"})
|
||||
for tid in (note_pl.get("references") or []):
|
||||
edges.append({"src_id": note_id, "dst_id": tid, "edge_type": "references", "scope": "note"})
|
||||
edges.append({"src_id": tid, "dst_id": note_id, "edge_type": "backlink", "scope": "note"})
|
||||
# Dedupe
|
||||
_uniq = {}
|
||||
for e in edges:
|
||||
_uniq[(e["src_id"], e["dst_id"], e["edge_type"], e.get("scope", ""))] = e
|
||||
edges = list(_uniq.values())
|
||||
|
||||
# Optional: Note-Vektor
|
||||
note_vec = embed_one(parsed.body) if args.embed_note else None
|
||||
|
||||
# --- WICHTIG: Chunk-Texte für derive_wikilink_edges ---
|
||||
chunks_for_links = [
|
||||
{"chunk_id": (pl.get("chunk_id") or pl.get("id") or f"{fm['id']}#{i+1}"),
|
||||
"text": chunks[i].text}
|
||||
for i, pl in enumerate(chunk_pls)
|
||||
if i < len(chunks)
|
||||
]
|
||||
|
||||
# Kanten (Note- & Chunk-Ebene)
|
||||
edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index)
|
||||
|
||||
# Dry-Run-Ausgabe
|
||||
print(json.dumps({
|
||||
"note_id": fm["id"],
|
||||
"title": fm["title"],
|
||||
# Zusammenfassung
|
||||
summary = {
|
||||
"note_id": note_id,
|
||||
"title": fm.get("title"),
|
||||
"chunks": len(chunk_pls),
|
||||
"edges": len(edges),
|
||||
"path": note_pl["path"]
|
||||
}, ensure_ascii=False))
|
||||
"changed": changed,
|
||||
"decision": ("apply" if args.apply and changed else
|
||||
"apply-skip-unchanged" if args.apply and not changed else
|
||||
"dry-run"),
|
||||
"path": note_pl["path"],
|
||||
"hash_mode": args.hash_mode or os.environ.get("MINDNET_HASH_MODE", "body"),
|
||||
"hash_normalize": args.hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical"),
|
||||
"hash_old": old_hash,
|
||||
"hash_new": new_hash,
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
|
||||
if args.apply:
|
||||
if args.force_replace:
|
||||
purge_note_edges(client, cfg.prefix, fm["id"])
|
||||
if not args.apply:
|
||||
continue
|
||||
|
||||
# Notes upsert
|
||||
notes_col, note_pts = points_for_note(cfg.prefix, note_pl, note_vec, cfg.dim)
|
||||
upsert_batch(client, notes_col, note_pts)
|
||||
if changed and args.purge_before_upsert:
|
||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||
|
||||
# Chunks upsert
|
||||
chunks_col, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vectors)
|
||||
upsert_batch(client, chunks_col, chunk_pts)
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
|
||||
# Edges upsert
|
||||
edges_col, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||
upsert_batch(client, edges_col, edge_pts)
|
||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||
upsert_batch(client, chunks_name, chunk_pts)
|
||||
|
||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||
upsert_batch(client, edges_name, edge_pts)
|
||||
|
||||
print(f"Done. Processed notes: {processed}")
|
||||
|
||||
print(f"Done. Processed notes: {total_notes}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
make_test_vault.py — erzeugt einen minimalen, nachvollziehbaren Test-Vault
|
||||
|
||||
Version: 1.1 (2025-09-05)
|
||||
Änderungen ggü. 1.0:
|
||||
- created/updated werden als QUOTED ISO-Strings geschrieben, damit der YAML-Parser
|
||||
sie nicht in datetime-Objekte konvertiert (Schema verlangt strings).
|
||||
- Kommentare/Output gestrafft.
|
||||
|
||||
Zweck
|
||||
- Kleiner Obsidian-ähnlicher Vault zum Durchspielen des Importers (Chunks/Edges).
|
||||
- Szenarien: leere Links, spätere Anlage fehlender Noten, Chunk-Neuaufteilung.
|
||||
|
||||
Struktur (Default: ./test_vault)
|
||||
- 40_concepts/concept-alpha.md
|
||||
- 20_experiences/exp-one.md (verlinkt auf [[concept-alpha]] und [[missing-note]])
|
||||
- 20_experiences/exp-two.md (verlinkt auf [[concept-alpha]])
|
||||
- 30_projects/project-demo.md (verlinkt auf [[concept-alpha]] und [[exp-one]])
|
||||
|
||||
Voraussetzungen
|
||||
- Keine. Rein Dateigenerierung.
|
||||
|
||||
Aufruf
|
||||
python3 -m scripts.make_test_vault [--out ./test_vault] [--force]
|
||||
|
||||
Parameter
|
||||
--out Zielverzeichnis (Standard: ./test_vault)
|
||||
--force Bestehenden Ordner löschen und neu anlegen.
|
||||
|
||||
Hinweise für Tests
|
||||
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
|
||||
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
|
||||
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
|
||||
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
|
||||
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
|
||||
|
||||
Kompatibel mit:
|
||||
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
|
||||
def iso_now() -> str:
|
||||
# ISO 8601 mit Offset; als String zurückgeben
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
def q(s: str) -> str:
|
||||
# Sicheres Quoten für YAML-Frontmatter
|
||||
# (einfaches doppelt-Quoten reicht hier, da wir keine eingebetteten """ verwenden)
|
||||
return f"\"{s}\""
|
||||
|
||||
TEMPLATE_CONCEPT = """---
|
||||
title: "Concept Alpha"
|
||||
id: "concept-alpha"
|
||||
type: "concept"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/concept","topic/test"]
|
||||
---
|
||||
|
||||
## Beschreibung
|
||||
Dies ist ein einfaches Konzept für Testzwecke.
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[exp-one]]
|
||||
- [[project-demo]]
|
||||
"""
|
||||
|
||||
TEMPLATE_EXP_ONE = """---
|
||||
title: "Experience One"
|
||||
id: "exp-one"
|
||||
type: "experience"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/experience","topic/test"]
|
||||
---
|
||||
|
||||
## Kontext
|
||||
Diese Notiz verlinkt auf ein existierendes Konzept und auf eine noch **fehlende** Note.
|
||||
|
||||
## Beobachtung
|
||||
- Bezug zu [[concept-alpha]]
|
||||
- Offener Verweis auf [[missing-note]]
|
||||
|
||||
## Interpretation
|
||||
Wenn später eine Datei für 'missing-note' angelegt wird, sollte der Importer die Kanten nachziehen.
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-alpha]]
|
||||
- [[missing-note]]
|
||||
"""
|
||||
|
||||
TEMPLATE_EXP_TWO = """---
|
||||
title: "Experience Two"
|
||||
id: "exp-two"
|
||||
type: "experience"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/experience","topic/test"]
|
||||
---
|
||||
|
||||
## Kontext
|
||||
Zweite Experience, die auf dasselbe Konzept verweist.
|
||||
|
||||
## Beobachtung
|
||||
Verweis auf [[concept-alpha]].
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-alpha]]
|
||||
"""
|
||||
|
||||
TEMPLATE_PROJECT = """---
|
||||
title: "Project Demo"
|
||||
id: "project-demo"
|
||||
type: "project"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/project","topic/test"]
|
||||
---
|
||||
|
||||
## Scope
|
||||
Ein kleines Demo-Projekt, das andere Notizen referenziert.
|
||||
|
||||
## Arbeitspakete
|
||||
- AP-1: Zusammenhang erklären (siehe [[concept-alpha]])
|
||||
- AP-2: Ereignis berücksichtigen (siehe [[exp-one]])
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-alpha]]
|
||||
- [[exp-one]]
|
||||
"""
|
||||
|
||||
TEMPLATE_MISSING = """---
|
||||
title: "Missing Note (jetzt angelegt)"
|
||||
id: "missing-note"
|
||||
type: "concept"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/concept","topic/test"]
|
||||
---
|
||||
|
||||
## Kontext
|
||||
Diese Notiz wurde **nachträglich** erstellt, um einen zuvor offenen Link zu schließen.
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[exp-one]]
|
||||
"""
|
||||
|
||||
def write_file(path: str, content: str) -> None:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
def build_vault(base: str, include_missing_note: bool) -> None:
|
||||
created = q(iso_now())
|
||||
updated = q(iso_now())
|
||||
|
||||
# 40_concepts
|
||||
write_file(
|
||||
os.path.join(base, "40_concepts", "concept-alpha.md"),
|
||||
TEMPLATE_CONCEPT.format(created=created, updated=updated),
|
||||
)
|
||||
|
||||
# 20_experiences
|
||||
write_file(
|
||||
os.path.join(base, "20_experiences", "exp-one.md"),
|
||||
TEMPLATE_EXP_ONE.format(created=created, updated=updated),
|
||||
)
|
||||
write_file(
|
||||
os.path.join(base, "20_experiences", "exp-two.md"),
|
||||
TEMPLATE_EXP_TWO.format(created=created, updated=updated),
|
||||
)
|
||||
|
||||
# 30_projects
|
||||
write_file(
|
||||
os.path.join(base, "30_projects", "project-demo.md"),
|
||||
TEMPLATE_PROJECT.format(created=created, updated=updated),
|
||||
)
|
||||
|
||||
# Optional: zunächst NICHT anlegen (Szenario „leerer Link“),
|
||||
# später für den 2. Testlauf anlegen.
|
||||
if include_missing_note:
|
||||
write_file(
|
||||
os.path.join(base, "40_concepts", "missing-note.md"),
|
||||
TEMPLATE_MISSING.format(created=created, updated=updated),
|
||||
)
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="./test_vault", help="Zielverzeichnis für den Test-Vault")
|
||||
ap.add_argument("--force", action="store_true", help="Falls vorhanden: Verzeichnis löschen und neu anlegen")
|
||||
ap.add_argument("--with-missing", action="store_true",
|
||||
help="Auch die 'missing-note' direkt anlegen (Standard: erst im 2. Lauf)")
|
||||
args = ap.parse_args()
|
||||
|
||||
base = os.path.abspath(args.out)
|
||||
if os.path.exists(base):
|
||||
if args.force:
|
||||
shutil.rmtree(base)
|
||||
else:
|
||||
print(f"Ziel existiert bereits: {base}\nNutze --force zum Überschreiben.")
|
||||
return
|
||||
|
||||
build_vault(base, include_missing_note=args.with_missing)
|
||||
print(f"Test-Vault erstellt unter: {base}")
|
||||
print("Hinweis:")
|
||||
print(" 1) Zuerst ohne missing-note anlegen (Standard), importieren, Edges prüfen.")
|
||||
print(" 2) Dann erneut ausführen mit --force --with-missing (missing-note wird erzeugt),")
|
||||
print(" und Import wiederholen → Edges/Backlinks sollten nachgezogen werden.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,232 @@
|
||||
# scripts/prune_qdrant_vs_vault.py
|
||||
# -----------------------------------------------------------------------------
|
||||
# Name: prune_qdrant_vs_vault.py
|
||||
# Version: 1.0.0 (2025-09-08)
|
||||
# Zweck: Entfernt verwaiste Qdrant-Einträge (notes/chunks/edges), wenn
|
||||
# die zugehörigen Markdown-Dateien im Vault nicht mehr existieren.
|
||||
#
|
||||
# Was es macht:
|
||||
# - Liest alle note_id aus dem Vault (Frontmatter: id / note_id).
|
||||
# - Liest alle note_id aus Qdrant (mindnet_notes).
|
||||
# - Bildet die Differenz (nur in Qdrant vorhandene, im Vault fehlende).
|
||||
# - Löscht für jede verwaiste note_id:
|
||||
# * Notes-Point(s)
|
||||
# * Alle Chunks der Note
|
||||
# * Alle Edges, die auf diese Note referenzieren
|
||||
# (source_id == note_id ODER target_id == note_id ODER
|
||||
# source_note_id == note_id ODER target_note_id == note_id)
|
||||
#
|
||||
# Hinweise:
|
||||
# - Kein globaler Delete. Nur betroffene note_id.
|
||||
# - Dry-Run standardmäßig; tatsächliches Löschen erst mit --apply.
|
||||
# - Interaktive Bestätigung (abschaltbar mit --yes).
|
||||
#
|
||||
# Aufruf:
|
||||
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet
|
||||
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply
|
||||
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply --yes
|
||||
#
|
||||
# Voraussetzungen:
|
||||
# - Ausführung im aktivierten venv empfohlen: source .venv/bin/activate
|
||||
# - Qdrant läuft lokal (oder URL/API-Key in ENV), siehe app/core/qdrant.py
|
||||
#
|
||||
# Änderungen:
|
||||
# - 1.0.0: Erster Release.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Set, Tuple
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.parser import parse_markdown_file # nutzt euer bestehendes Parsing/Schema
|
||||
|
||||
|
||||
def read_vault_note_ids(vault_dir: Path) -> Set[str]:
|
||||
note_ids: Set[str] = set()
|
||||
for p in vault_dir.rglob("*.md"):
|
||||
try:
|
||||
parsed = parse_markdown_file(str(p))
|
||||
fm = parsed.frontmatter if hasattr(parsed, "frontmatter") else (parsed.get("frontmatter") or {})
|
||||
nid = fm.get("id") or fm.get("note_id")
|
||||
if nid:
|
||||
note_ids.add(str(nid))
|
||||
except Exception:
|
||||
# still und leise weiter – wir wollen robust sein
|
||||
continue
|
||||
return note_ids
|
||||
|
||||
|
||||
def qdrant_note_ids(client: QdrantClient, notes_col: str) -> Set[str]:
|
||||
ids: Set[str] = set()
|
||||
scroll_filter = None
|
||||
offset = None
|
||||
while True:
|
||||
res = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=scroll_filter,
|
||||
offset=offset,
|
||||
limit=256,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
points, next_offset = res
|
||||
for pt in points:
|
||||
pl = pt.payload or {}
|
||||
nid = pl.get("note_id")
|
||||
if nid:
|
||||
ids.add(str(nid))
|
||||
if next_offset is None:
|
||||
break
|
||||
offset = next_offset
|
||||
return ids
|
||||
|
||||
|
||||
def delete_by_filter(client: QdrantClient, collection: str, flt: rest.Filter) -> int:
|
||||
# Sammel erst IDs via scroll (robuster Überblick), lösche dann über point_ids (batch)
|
||||
to_delete: List[int] = []
|
||||
offset = None
|
||||
while True:
|
||||
pts, next_offset = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=flt,
|
||||
offset=offset,
|
||||
limit=256,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
)
|
||||
for pt in pts:
|
||||
to_delete.append(pt.id)
|
||||
if next_offset is None:
|
||||
break
|
||||
offset = next_offset
|
||||
if not to_delete:
|
||||
return 0
|
||||
client.delete(collection_name=collection, points_selector=rest.PointIdsList(points=to_delete), wait=True)
|
||||
return len(to_delete)
|
||||
|
||||
|
||||
def prune_for_note(
|
||||
client: QdrantClient,
|
||||
cols: Dict[str, str],
|
||||
note_id: str,
|
||||
dry_run: bool = True,
|
||||
) -> Dict[str, int]:
|
||||
stats = {"notes": 0, "chunks": 0, "edges": 0}
|
||||
|
||||
# Notes löschen (Filter auf payload.note_id)
|
||||
f_notes = rest.Filter(
|
||||
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
|
||||
)
|
||||
|
||||
# Chunks löschen (Filter auf payload.note_id)
|
||||
f_chunks = rest.Filter(
|
||||
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
|
||||
)
|
||||
|
||||
# Edges löschen: mehrere Teilmengen erfassen
|
||||
edge_filters = [
|
||||
rest.Filter(must=[rest.FieldCondition(key="source_id", match=rest.MatchValue(value=note_id))]),
|
||||
rest.Filter(must=[rest.FieldCondition(key="target_id", match=rest.MatchValue(value=note_id))]),
|
||||
rest.Filter(must=[rest.FieldCondition(key="source_note_id", match=rest.MatchValue(value=note_id))]),
|
||||
rest.Filter(must=[rest.FieldCondition(key="target_note_id", match=rest.MatchValue(value=note_id))]),
|
||||
]
|
||||
|
||||
if dry_run:
|
||||
# Zähle nur, was *wäre* gelöscht worden
|
||||
def count(flt: rest.Filter, col: str) -> int:
|
||||
total = 0
|
||||
offset = None
|
||||
while True:
|
||||
pts, next_offset = client.scroll(
|
||||
collection_name=col,
|
||||
scroll_filter=flt,
|
||||
offset=offset,
|
||||
limit=256,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
)
|
||||
total += len(pts)
|
||||
if next_offset is None:
|
||||
break
|
||||
offset = next_offset
|
||||
return total
|
||||
|
||||
stats["notes"] = count(f_notes, cols["notes"])
|
||||
stats["chunks"] = count(f_chunks, cols["chunks"])
|
||||
stats["edges"] = sum(count(f, cols["edges"]) for f in edge_filters)
|
||||
return stats
|
||||
|
||||
# tatsächliches Löschen
|
||||
stats["notes"] = delete_by_filter(client, cols["notes"], f_notes)
|
||||
stats["chunks"] = delete_by_filter(client, cols["chunks"], f_chunks)
|
||||
e_deleted = 0
|
||||
for f in edge_filters:
|
||||
e_deleted += delete_by_filter(client, cols["edges"], f)
|
||||
stats["edges"] = e_deleted
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Prune Qdrant data for notes missing in the Vault.")
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Vault-Verzeichnis (Root).")
|
||||
ap.add_argument("--prefix", default="mindnet", help="Collections-Präfix (Default: mindnet).")
|
||||
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter wird nur ein Dry-Run gemacht.")
|
||||
ap.add_argument("--yes", action="store_true", help="Ohne Nachfrage ausführen.")
|
||||
args = ap.parse_args()
|
||||
|
||||
vault_root = Path(args.vault).resolve()
|
||||
if not vault_root.exists():
|
||||
print(f"Vault-Verzeichnis nicht gefunden: {vault_root}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
cfg = QdrantConfig()
|
||||
client = get_client(cfg)
|
||||
cols = collection_names(args.prefix)
|
||||
|
||||
vault_ids = read_vault_note_ids(vault_root)
|
||||
qdrant_ids = qdrant_note_ids(client, cols["notes"])
|
||||
orphans = sorted(qdrant_ids - vault_ids)
|
||||
|
||||
preview = {
|
||||
"mode": "APPLY" if args.apply else "DRY-RUN",
|
||||
"prefix": args.prefix,
|
||||
"vault_root": str(vault_root),
|
||||
"collections": cols,
|
||||
"counts": {
|
||||
"vault_note_ids": len(vault_ids),
|
||||
"qdrant_note_ids": len(qdrant_ids),
|
||||
"orphans": len(orphans),
|
||||
},
|
||||
"orphans_sample": orphans[:20],
|
||||
}
|
||||
print(json.dumps(preview, ensure_ascii=False, indent=2))
|
||||
|
||||
if not orphans:
|
||||
print("Keine verwaisten Einträge gefunden. Nichts zu tun.")
|
||||
return
|
||||
|
||||
if not args.yes:
|
||||
resp = input("\nFortfahren mit dem oben angezeigten Modus? (yes/no): ").strip().lower()
|
||||
if resp not in ("y", "yes", "j", "ja"):
|
||||
print("Abgebrochen.")
|
||||
return
|
||||
|
||||
total_stats = {"notes": 0, "chunks": 0, "edges": 0}
|
||||
for nid in orphans:
|
||||
s = prune_for_note(client, cols, nid, dry_run=(not args.apply))
|
||||
total_stats["notes"] += s["notes"]
|
||||
total_stats["chunks"] += s["chunks"]
|
||||
total_stats["edges"] += s["edges"]
|
||||
|
||||
print(json.dumps({"deleted": total_stats}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
# scripts/resolve_unresolved_references.py
|
||||
"""
|
||||
resolve_unresolved_references.py — Unaufgelöste Wikilinks in Qdrant nachträglich auflösen
|
||||
|
||||
Version: 1.0.0 (2025-09-05)
|
||||
|
||||
Zweck
|
||||
------
|
||||
- Findet Edges in {prefix}_edges mit payload.status=="unresolved" und versucht, den Zielknoten
|
||||
anhand bereits vorhandener Notes in {prefix}_notes aufzulösen.
|
||||
- Aktualisiert die Edges (setzt target_id, entfernt status, setzt resolution), und erzeugt
|
||||
– NUR für Note-Level 'references' – die symmetrische 'backlink'-Kante.
|
||||
|
||||
Warum?
|
||||
------
|
||||
- Beim ersten Import können Links auf (noch) nicht existierende Notizen zeigen.
|
||||
- Sobald die Zielnotiz später existiert, kann dieses Skript die Kanten reparieren.
|
||||
|
||||
Aufruf
|
||||
------
|
||||
# Dry-Run (Standard):
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet
|
||||
|
||||
# Anwenden:
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
||||
|
||||
# Optional: nur X Edges anfassen
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply --limit 500
|
||||
|
||||
Parameter
|
||||
---------
|
||||
--prefix : Collection-Prefix (Default: aus Env COLLECION_PREFIX oder "mindnet")
|
||||
--apply : Änderungen tatsächlich schreiben (ohne --apply = Dry-Run)
|
||||
--limit : Max. Anzahl unaufgelöster Edges, die in diesem Lauf bearbeitet werden (Default: keine Begrenzung)
|
||||
--batch : Upsert-Batchgröße (Default: 512)
|
||||
|
||||
Voraussetzungen / Hinweise
|
||||
--------------------------
|
||||
- Bitte im aktivierten venv laufen lassen (deine Umgebung: `.venv`).
|
||||
- Qdrant-URL/Key/Prefix/Vektor-Dim werden wie üblich aus ENV gelesen (sieh app/core/qdrant.py). # noqa
|
||||
- Nutzt die vorhandenen Utilities:
|
||||
- app/core/qdrant.py (Client/Collections)
|
||||
- app/core/qdrant_points.py (points_for_edges/upsert_batch)
|
||||
- app/core/derive_edges.py (build_note_index/resolve_target)
|
||||
|
||||
Änderungshistorie
|
||||
-----------------
|
||||
1.0.0 Erstveröffentlichung.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from typing import Any, Dict, List, Tuple, Iterable
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names # :contentReference[oaicite:3]{index=3}
|
||||
from app.core.qdrant_points import points_for_edges, upsert_batch # :contentReference[oaicite:4]{index=4}
|
||||
from app.core.derive_edges import build_note_index, resolve_target # :contentReference[oaicite:5]{index=5}
|
||||
|
||||
|
||||
def _scroll(client: QdrantClient, **kwargs):
|
||||
"""
|
||||
Wrapper um qdrant_client.scroll() für unterschiedliche Client-Versionen:
|
||||
neuere: (points, next_offset)
|
||||
ältere: (points, next_page_offset, _)
|
||||
"""
|
||||
res = client.scroll(**kwargs)
|
||||
if isinstance(res, tuple):
|
||||
if len(res) == 2:
|
||||
points, next_off = res
|
||||
else:
|
||||
# ältere Signatur: (points, next_off, _)
|
||||
points, next_off, _ = res[0], res[1], res[2]
|
||||
else:
|
||||
# sehr alte Clients -> konservativ behandeln
|
||||
points, next_off = res, None
|
||||
return points, next_off
|
||||
|
||||
|
||||
def _load_all_notes(client: QdrantClient, notes_col: str) -> List[Dict[str, Any]]:
|
||||
notes: List[Dict[str, Any]] = []
|
||||
next_off = None
|
||||
while True:
|
||||
pts, next_off = _scroll(
|
||||
client,
|
||||
collection_name=notes_col,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1024,
|
||||
offset=next_off,
|
||||
)
|
||||
for p in pts or []:
|
||||
pl = getattr(p, "payload", {}) or {}
|
||||
# Erwartet Felder: note_id, title, path etc. (gemäß Schema) # :contentReference[oaicite:6]{index=6}
|
||||
if pl.get("note_id"):
|
||||
notes.append(pl)
|
||||
if not next_off:
|
||||
break
|
||||
return notes
|
||||
|
||||
|
||||
def _iter_unresolved_edges(client: QdrantClient, edges_col: str) -> Iterable[rest.Record]:
|
||||
"""
|
||||
Liefert alle Edge-Records mit payload.status == 'unresolved' und 'target_label' (string).
|
||||
"""
|
||||
f = rest.Filter(
|
||||
must=[
|
||||
rest.FieldCondition(key="status", match=rest.MatchValue(value="unresolved")),
|
||||
]
|
||||
)
|
||||
next_off = None
|
||||
while True:
|
||||
pts, next_off = _scroll(
|
||||
client,
|
||||
collection_name=edges_col,
|
||||
scroll_filter=f,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1024,
|
||||
offset=next_off,
|
||||
)
|
||||
for p in pts or []:
|
||||
pl = getattr(p, "payload", {}) or {}
|
||||
if isinstance(pl.get("target_label"), str):
|
||||
yield p
|
||||
if not next_off:
|
||||
break
|
||||
|
||||
|
||||
def _make_backlink(source_note_id: str, target_note_id: str, extra: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Baue eine 'backlink'-Edge-Payload source <- target (note-level).
|
||||
"""
|
||||
e = {
|
||||
"kind": "backlink",
|
||||
"source_id": target_note_id,
|
||||
"target_id": source_note_id,
|
||||
}
|
||||
# Metafelder aus dem Original übernehmen (ohne status)
|
||||
copy_keys = ["raw", "alias", "heading", "resolution"]
|
||||
for k in copy_keys:
|
||||
if k in extra:
|
||||
e[k] = extra[k]
|
||||
return e
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (Default: Env/COLLECTION_PREFIX oder 'mindnet')")
|
||||
ap.add_argument("--apply", action="store_true", help="Änderungen schreiben (ohne Flag = Dry-Run)")
|
||||
ap.add_argument("--limit", type=int, default=0, help="Max. Anzahl unaufgelöster Edges bearbeiten (0 = alle)")
|
||||
ap.add_argument("--batch", type=int, default=512, help="Upsert-Batchgröße")
|
||||
args = ap.parse_args()
|
||||
|
||||
# Qdrant-Setup
|
||||
cfg = QdrantConfig.from_env()
|
||||
if args.prefix:
|
||||
cfg.prefix = args.prefix
|
||||
client = get_client(cfg)
|
||||
ensure_collections(client, cfg.prefix, cfg.dim) # sorgt u. a. für 1D-Vektor-Collection bei Edges :contentReference[oaicite:7]{index=7}
|
||||
notes_col, _, edges_col = collection_names(cfg.prefix) # :contentReference[oaicite:8]{index=8}
|
||||
|
||||
# Notes laden & Index bauen
|
||||
notes = _load_all_notes(client, notes_col)
|
||||
idx = build_note_index(notes) # (by_id, by_slug, by_file_slug) :contentReference[oaicite:9]{index=9}
|
||||
|
||||
# Unresolved-Edges scannen
|
||||
to_fix: List[dict] = []
|
||||
backlinks: List[dict] = []
|
||||
processed = 0
|
||||
resolved = 0
|
||||
|
||||
for rec in _iter_unresolved_edges(client, edges_col):
|
||||
if args.limit and processed >= args.limit:
|
||||
break
|
||||
processed += 1
|
||||
|
||||
pl = dict(rec.payload or {})
|
||||
kind = pl.get("kind") or "references"
|
||||
src = pl.get("source_id")
|
||||
tgt_label = pl.get("target_label") or pl.get("target_id") # Fallback
|
||||
|
||||
# Zielauflösung
|
||||
resolved_id, how = resolve_target(str(tgt_label), idx) # :contentReference[oaicite:10]{index=10}
|
||||
if not resolved_id:
|
||||
continue # weiterhin unresolved
|
||||
|
||||
# Edge-Update
|
||||
new_pl = dict(pl)
|
||||
new_pl["target_id"] = resolved_id
|
||||
new_pl["resolution"] = how
|
||||
if "status" in new_pl:
|
||||
del new_pl["status"]
|
||||
# ID stabil lassen -> points_for_edges erzeugt UUID aus edge_id/Fallback :contentReference[oaicite:11]{index=11}
|
||||
if "edge_id" not in new_pl:
|
||||
# stabiler Key aus (kind, src, tgt, evtl. seq)
|
||||
seq = new_pl.get("seq") or new_pl.get("order") or ""
|
||||
new_pl["edge_id"] = f"{kind}:{src}->{resolved_id}#{seq}"
|
||||
|
||||
to_fix.append(new_pl)
|
||||
resolved += 1
|
||||
|
||||
# Nur bei Note-Level references (nicht references_at) -> Backlink erzeugen
|
||||
if kind == "references":
|
||||
extra = {k: new_pl.get(k) for k in ("raw", "alias", "heading")}
|
||||
extra["resolution"] = how
|
||||
backlinks.append(_make_backlink(source_note_id=src, target_note_id=resolved_id, extra=extra))
|
||||
|
||||
# Ergebnis ausgeben
|
||||
summary = {
|
||||
"prefix": cfg.prefix,
|
||||
"scanned_unresolved": processed,
|
||||
"resolved": resolved,
|
||||
"backlinks_to_create": len(backlinks),
|
||||
"apply": bool(args.apply),
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
|
||||
if not args.apply:
|
||||
return
|
||||
|
||||
# Upserts (in Batches)
|
||||
def _batched(items: List[dict], n: int) -> Iterable[List[dict]]:
|
||||
for i in range(0, len(items), n):
|
||||
yield items[i : i + n]
|
||||
|
||||
# 1) Updates für reparierte Edges
|
||||
for chunk in _batched(to_fix, args.batch):
|
||||
col, pts = points_for_edges(cfg.prefix, chunk) # sorgt für Edge-UUID & Dummy-Vector :contentReference[oaicite:12]{index=12}
|
||||
upsert_batch(client, col, pts)
|
||||
|
||||
# 2) Backlinks (nur references)
|
||||
for chunk in _batched(backlinks, args.batch):
|
||||
col, pts = points_for_edges(cfg.prefix, chunk)
|
||||
upsert_batch(client, col, pts)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user