Compare commits

..
Author SHA1 Message Date
Lars e040fcc0dd scripts/audit_edges_vs_expectations.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:34:08 +02:00
Lars 305089fcf6 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:27:18 +02:00
Lars f9c0b5df19 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:26:57 +02:00
Lars 77fa57a2a6 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:19:06 +02:00
Lars dcb02dd194 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:17:30 +02:00
Lars 35338dea64 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:17:06 +02:00
Lars c34df96839 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:15:19 +02:00
Lars 0617163c42 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:15:01 +02:00
Lars 47797ecd29 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:12:52 +02:00
Lars 897d0c9e6d scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:12:33 +02:00
Lars 6c25d76135 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:05:32 +02:00
Lars 22885a4857 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:04:50 +02:00
Lars a807cc8bd1 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 10:13:57 +02:00
Lars c2954b5663 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 09:33:51 +02:00
Lars 8f97f42618 app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 09:12:50 +02:00
Lars 0a116ea3f4 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 18:36:59 +02:00
Lars aa7d0190c8 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 18:04:11 +02:00
Lars af5d0a0c91 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:43:44 +02:00
Lars fb016c6c6a scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:42:57 +02:00
Lars 77732445ea scripts/prune_qdrant_vs_vault.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:37:08 +02:00
Lars 1b833f76ce scripts/export_markdown.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:36:40 +02:00
Lars b394cadf73 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 12:42:51 +02:00
Lars f3b6166daa app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 12:22:05 +02:00
Lars ce720090ec docs/Handbuch.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-07 07:48:59 +02:00
Lars bf2149bbeb docs/Handbuch.md hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-07 07:46:39 +02:00
Lars 80a7db0e54 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-06 15:42:10 +02:00
Lars 47e6d56b21 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-06 14:04:46 +02:00
Lars df33293621 scripts/resolve_unresolved_references.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 19:16:27 +02:00
Lars a202c53594 scripts/make_test_vault.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 15:48:06 +02:00
Lars 4f54e750b2 scripts/make_test_vault.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 15:35:41 +02:00
Lars 5478be60c1 scripts/resolve_unresolved_references.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 12:37:59 +02:00
Lars 3deb99f053 scripts/gc_qdrant_after_vault_scan.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 12:18:23 +02:00
Lars 8df2069c6f scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 12:03:33 +02:00
Lars d790c0ef75 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:53:59 +02:00
11 changed files with 2221 additions and 209 deletions

No files matched your search

+149 -27
View File
@@ -1,37 +1,159 @@
from __future__ import annotations
from typing import Dict, List
from .chunker import Chunk
from .parser import extract_wikilinks
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Modul: app/core/chunk_payload.py
Version: 1.1.3
Datum: 2025-09-09
def make_chunk_payloads(note_meta: Dict, path: str, chunks: List[Chunk]) -> List[Dict]:
res = []
Kurzbeschreibung
----------------
Erzeugt Qdrant-Payloads für Text-Chunks einer Note. Jeder Chunk enthält
den tatsächlichen Text unter dem Schlüssel ``text``. Dadurch kann der
Exporter den vollständigen Body verlässlich aus Chunks rekonstruieren,
falls ``notes.payload.fulltext`` fehlt.
Wesentliche Features
--------------------
- Stabile, idempotente Payload-Struktur für Chunks
- Persistenter Chunk-Text (``text``)
- Extraktion von Wikilinks pro Chunk (``wikilinks`` & ``references``)
- Pfadübernahme (relativ zum Vault, wird vom Aufrufer geliefert)
- Bereinigung leerer Felder (keine ``None``/leere Collections im Payload)
Abhängigkeiten
--------------
- ``app.core.chunker.Chunk`` und ``assemble_chunks`` (nur für CLI-Demo)
- ``app.core.parser.extract_wikilinks`` und ``read_markdown`` (nur CLI-Demo)
Beispiele (CLI – Sichtprüfung)
------------------------------
python3 -m app.core.chunk_payload --from-file ./vault/demo.md --print
"""
from __future__ import annotations
from typing import Dict, List, Any
import argparse
import json
import os
# WICHTIG: Keine direkten Selbst- oder Kreuz-Imports, die Zyklen erzeugen!
# Für die Kernfunktion reicht extract_wikilinks als Typ-unabhängige Hilfsfunktion.
try:
from app.core.parser import extract_wikilinks # nur Funktion, erzeugt keine Zyklen
except Exception: # pragma: no cover
from .parser import extract_wikilinks # type: ignore
# Typ-Hinweis für Chunk (zur Laufzeit nicht notwendig)
try:
from app.core.chunker import Chunk # nur Typ-Annotation / Nutzung im CLI
except Exception: # pragma: no cover
class Chunk: # type: ignore
pass
# ---------------------------------------------------------------------------
# Utils
# ---------------------------------------------------------------------------
def _drop_empty(d: Dict[str, Any]) -> Dict[str, Any]:
"""Entfernt leere/None-Felder aus einem Dict (für saubere Payloads)."""
return {k: v for k, v in d.items() if v not in (None, [], {}, "")}
# ---------------------------------------------------------------------------
# Kernfunktion
# ---------------------------------------------------------------------------
def make_chunk_payloads(note_meta: Dict[str, Any], path: str, chunks: List[Chunk]) -> List[Dict[str, Any]]:
"""
Baut Payloads für alle ``chunks`` der gegebenen Note.
Parameters
----------
note_meta : Dict[str, Any]
Minimale Metadaten der Note (mind. ``id``, ``title``; optional ``type``,
``area``, ``project``, ``tags``, ``lang``).
path : str
Relativer Pfad der Note innerhalb des Vaults (z. B. "area/topic/file.md").
chunks : List[Chunk]
Liste vorsegmentierter Chunks (vgl. app.core.chunker.Chunk).
Returns
-------
List[Dict[str, Any]]
Payload-Objekte, bereit für Qdrant-Upserts.
"""
res: List[Dict[str, Any]] = []
for ch in chunks:
wikilinks = extract_wikilinks(ch.text)
# 'text' kann bei leerem Abschnitt None/'' sein → robust behandeln
text_val = getattr(ch, "text", "") or ""
wikilinks = extract_wikilinks(text_val)
payload = {
"id": ch.id,
"note_id": note_meta["id"],
"note_title": note_meta["title"],
"chunk_index": ch.index,
"char_start": ch.char_start,
"char_end": ch.char_end,
"token_count": ch.token_count,
"type": note_meta.get("type"),
"area": note_meta.get("area"),
"project": note_meta.get("project"),
"tags": note_meta.get("tags", []),
"section_title": ch.section_title,
"section_path": ch.section_path,
"id": getattr(ch, "id", None),
"note_id": note_meta.get("id"),
"note_title": note_meta.get("title"),
"chunk_index": getattr(ch, "index", None),
"char_start": getattr(ch, "char_start", None),
"char_end": getattr(ch, "char_end", None),
"token_count": getattr(ch, "token_count", None),
"section_title": getattr(ch, "section_title", None),
"section_path": getattr(ch, "section_path", None),
"lang": note_meta.get("lang"),
"wikilinks": wikilinks,
"external_links": [], # (optional später ergänzen)
"external_links": [],
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
"neighbors": {
"prev": ch.neighbors_prev,
"next": ch.neighbors_next
"prev": getattr(ch, "neighbors_prev", None),
"next": getattr(ch, "neighbors_next", None),
},
"path": path,
"path": path, # vom Aufrufer relativ geliefert
"text": text_val, # WICHTIG für Export/Rekonstruktion
}
# None/Leere bereinigen
payload = {k: v for k, v in payload.items() if v not in (None, [], {})}
res.append(payload)
res.append(_drop_empty(payload))
return res
# ---------------------------------------------------------------------------
# CLI – nur zur Sichtprüfung / Debug
# ---------------------------------------------------------------------------
def _cli() -> None:
# CLI lädt hier *lazy* die schweren Abhängigkeiten,
# damit die Modul-Initialisierung zirkularfrei bleibt.
try:
from app.core.parser import read_markdown
from app.core.chunker import assemble_chunks
except Exception: # pragma: no cover
from .parser import read_markdown # type: ignore
from .chunker import assemble_chunks # type: ignore
ap = argparse.ArgumentParser(description="Vorschau: Chunk-Payloads erzeugen und anzeigen")
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zu einer Markdown-Datei")
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
args = ap.parse_args()
note = read_markdown(args.src)
note_meta = {
"id": note.frontmatter.get("id") or note.frontmatter.get("note_id"),
"title": note.frontmatter.get("title"),
"type": note.frontmatter.get("type"),
"area": note.frontmatter.get("area"),
"project": note.frontmatter.get("project"),
"tags": note.frontmatter.get("tags", []),
"lang": note.frontmatter.get("lang"),
}
chunks = assemble_chunks(note.frontmatter.get("id"), note.body or "", note.frontmatter.get("type", "concept"))
# Vault-Root heuristisch relativieren (nur für Demo)
vault_root = os.path.dirname(os.path.dirname(args.src)) # heuristisch
rel_path = os.path.relpath(args.src, vault_root).replace("\\", "/").lstrip("/")
payloads = make_chunk_payloads(note_meta, rel_path, chunks)
if args.do_print:
print(json.dumps(payloads, ensure_ascii=False, indent=2))
if __name__ == "__main__": # pragma: no cover
_cli()
+187 -28
View File
@@ -1,39 +1,198 @@
from __future__ import annotations
import os
import hashlib
from typing import Dict
from .parser import ParsedNote
from .parser import extract_wikilinks
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Modul: app/core/note_payload.py
Version: 1.4.0
Datum: 2025-09-09
Kurzbeschreibung
----------------
Erzeugt den Qdrant-Payload für **Notes**. Neben stabiler Hash-Bildung zur
Idempotenz wird der vollständige Body unter ``fulltext`` persistiert und
der Dateipfad relativ zum Vault gespeichert. Das erlaubt eine verlustfreie
Rekonstruktion im Export (erst ``fulltext``, sonst Chunks).
Hash-Steuerung
--------------
- Modus: ``body`` (Default) | ``frontmatter`` | ``body+frontmatter``
* per Funktionsparameter ``hash_mode`` (prio 1)
* oder ENV ``MINDNET_HASH_MODE`` (prio 2)
- Normalisierung: ``canonical`` (Default) | ``none``
* per Funktionsparameter ``hash_normalize`` (prio 1)
* oder ENV ``MINDNET_HASH_NORMALIZE`` (prio 2)
Beispiele (CLI – Sichtprüfung)
------------------------------
python3 -m app.core.note_payload --from-file ./vault/demo.md --vault-root ./vault --print
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from typing import Any, Dict, Optional
try:
from app.core.parser import read_markdown, extract_wikilinks
except Exception: # pragma: no cover
from .parser import read_markdown, extract_wikilinks # type: ignore
# ---------------------------------------------------------------------------
# Hashing
# ---------------------------------------------------------------------------
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
"""Kanonische, stabile JSON-Serialisierung der Frontmatter für Hashbildung."""
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def _normalize_body(body: str, mode: str) -> str:
"""Normalisiert den Body für reproduzierbare Hashes (oder nicht)."""
if mode == "none":
return body if body is not None else ""
# canonical: \r\n→\n, trailing spaces, mehrfach-blankzeilen trimmen
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
# trailing whitespace am Zeilenende entfernen
text = "\n".join(line.rstrip() for line in text.split("\n"))
# keine aggressivere Logik (keine inhaltliche Änderung)
return text
def compute_hash(
*,
body: Optional[str],
frontmatter: Optional[Dict[str, Any]],
mode: Optional[str] = None,
normalize: Optional[str] = None,
) -> str:
"""
Berechnet einen Hex-Hash gemäß ``mode`` und ``normalize``.
mode:
- "body" (Default)
- "frontmatter"
- "body+frontmatter"
normalize:
- "canonical" (Default)
- "none"
"""
mode = (mode or os.environ.get("MINDNET_HASH_MODE", "body")).strip().lower()
normalize = (normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
body = _normalize_body(body or "", normalize)
fm_s = _canon_frontmatter(frontmatter or {})
def sha256_text(text: str) -> str:
h = hashlib.sha256()
h.update(text.encode("utf-8"))
if mode == "frontmatter":
h.update(fm_s.encode("utf-8"))
elif mode == "body+frontmatter":
h.update(body.encode("utf-8"))
h.update(b"\n--FM--\n")
h.update(fm_s.encode("utf-8"))
else: # body
h.update(body.encode("utf-8"))
return h.hexdigest()
def make_note_payload(parsed: ParsedNote, vault_root: str) -> Dict:
fm = parsed.frontmatter
body = parsed.body
rel_path = os.path.relpath(parsed.path, vault_root) if vault_root else parsed.path
payload = {
"note_id": fm.get("id"),
# ---------------------------------------------------------------------------
# Kernfunktion
# ---------------------------------------------------------------------------
def make_note_payload(
parsed: Any,
vault_root: Optional[str] = None,
*,
hash_mode: Optional[str] = None,
hash_normalize: Optional[str] = None,
) -> Dict[str, Any]:
"""
Erzeugt den Payload für eine geparste Note.
Parameters
----------
parsed : Any
Objekt mit Attributen/Keys ``frontmatter``, ``body``, ``path``.
vault_root : Optional[str]
Vault-Wurzel (für Pfad-Relativierung). Wenn ``None``, wird ``path`` unverändert übernommen.
hash_mode : Optional[str]
"body" | "frontmatter" | "body+frontmatter" (überschreibt ENV).
hash_normalize : Optional[str]
"canonical" | "none" (überschreibt ENV).
Returns
-------
Dict[str, Any]
Qdrant-Payload für die Notes-Collection.
"""
# "Duck typing": dict oder Objekt akzeptieren
fm = (
getattr(parsed, "frontmatter", None)
or getattr(parsed, "fm", None)
or getattr(parsed, "front_matter", None)
or (parsed.get("frontmatter") if isinstance(parsed, dict) else {})
) or {}
body = getattr(parsed, "body", None) or (parsed.get("body") if isinstance(parsed, dict) else "") or ""
path = getattr(parsed, "path", None) or (parsed.get("path") if isinstance(parsed, dict) else "") or ""
# Hash gem. Modus bilden
hash_fulltext = compute_hash(body=body, frontmatter=fm, mode=hash_mode, normalize=hash_normalize)
# Pfad relativieren
rel_path = path
try:
if vault_root:
rel = os.path.relpath(path, vault_root)
rel = rel.replace("\\", "/").lstrip("/") # normalisieren
rel_path = rel
except Exception:
pass # Pfad nicht kritisch für Hash/ID
# Optionale Note-Level-Wikilinks (Fallback, wenn Chunks nicht geliefert werden)
note_level_refs = list(dict.fromkeys(extract_wikilinks(body))) if body else []
payload: Dict[str, Any] = {
"note_id": fm.get("id") or fm.get("note_id"),
"title": fm.get("title"),
"type": fm.get("type"),
"status": fm.get("status"),
"created": fm.get("created"),
"updated": fm.get("updated"),
"tags": fm.get("tags") or [],
"priority": fm.get("priority"),
"effort_min": fm.get("effort_min"),
"due": fm.get("due"),
"people": fm.get("people") or [],
"aliases": fm.get("aliases") or [],
"depends_on": fm.get("depends_on") or [],
"assigned_to": fm.get("assigned_to") or [],
"references": list(sorted(set(extract_wikilinks(body)))),
"lang": fm.get("lang") or None,
"path": rel_path.replace("\\", "/"),
"hash_fulltext": sha256_text(body.strip()),
"path": rel_path or fm.get("path"),
"tags": fm.get("tags"),
"hash_fulltext": hash_fulltext,
# Volltext persistieren (verlustfreie Rekonstruktion)
"fulltext": body,
# Fallback-Refs auf Note-Ebene
"references": note_level_refs,
}
# Entferne None-Werte für saubere Payloads
return {k: v for k, v in payload.items() if v not in (None, [])}
for k in ("area", "project", "source", "lang", "slug"):
if k in fm:
payload[k] = fm[k]
return payload
# ---------------------------------------------------------------------------
# CLI – Sichtprüfung
# ---------------------------------------------------------------------------
def _cli() -> None:
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zur Markdown-Datei")
ap.add_argument("--vault-root", dest="vault_root", default=None, help="Vault-Wurzel zur Pfad-Relativierung")
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter"], default=None)
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
args = ap.parse_args()
parsed = read_markdown(args.src)
payload = make_note_payload(parsed, vault_root=args.vault_root,
hash_mode=args.hash_mode, hash_normalize=args.hash_normalize)
if args.do_print:
print(json.dumps(payload, ensure_ascii=False, indent=2))
if __name__ == "__main__": # pragma: no cover
_cli()
+79 -21
View File
@@ -1,19 +1,31 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: app/core/qdrant_points.py
Version: v1.2.0 (2025-09-05)
app/core/qdrant_points.py
Kurzbeschreibung
Qdrant-Points für Notes/Chunks/Edges.
NEU: Edge-ID berücksichtigt optional 'occ' (Occurrence je Match):
"{kind}:{src}->{tgt}#{seq}@{occ}"
Zweck
- Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
- Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
- alt: {'edge_type','src_id','dst_id', ...}
- neu: {'kind','source_id','target_id', ...}
Aufruf
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
Version
- 1.3 (2025-09-08)
Changelog
v1.2.0: Edge-ID mit '@occ' falls vorhanden.
Änderungen (ggü. 1.2)
- points_for_edges() akzeptiert jetzt beide Edge-Schemata.
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
stabile 'edge_id' zurück in die Payload.
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
Aufruf / Verwendung
- Wird von Import-/Backfill-Skripten via:
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
eingebunden. Keine CLI.
Hinweise
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
"""
from __future__ import annotations
@@ -21,13 +33,23 @@ import uuid
from typing import List, Tuple
from qdrant_client.http import models as rest
def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _to_uuid(stable_key: str) -> str:
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
def points_for_note(
prefix: str,
note_payload: dict,
note_vec: List[float] | None,
dim: int,
) -> Tuple[str, List[rest.PointStruct]]:
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
notes_col, _, _ = _names(prefix)
vector = note_vec if note_vec is not None else [0.0] * int(dim)
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
@@ -35,7 +57,18 @@ def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | Non
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
return notes_col, [pt]
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
def points_for_chunks(
prefix: str,
chunk_payloads: List[dict],
vectors: List[List[float]],
) -> Tuple[str, List[rest.PointStruct]]:
"""
Chunks-Collection: erwartet pro Chunk einen Vektor.
Robustheit:
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
"""
_, chunks_col, _ = _names(prefix)
points: List[rest.PointStruct] = []
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
@@ -48,30 +81,55 @@ def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[Lis
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
return chunks_col, points
def _normalize_edge_payload(pl: dict) -> dict:
"""
Sorgt für kompatible Feldnamen.
akzeptiert:
- neu: kind, source_id, target_id, seq?
- alt: edge_type, src_id, dst_id, order?/index?
schreibt zurück: kind, source_id, target_id, seq?
"""
# bereits neu?
kind = pl.get("kind") or pl.get("edge_type") or "edge"
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
seq = pl.get("seq") or pl.get("order") or pl.get("index")
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
pl.setdefault("kind", kind)
pl.setdefault("source_id", source_id)
pl.setdefault("target_id", target_id)
if seq is not None and "seq" not in pl:
pl["seq"] = seq
return pl
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
"""
Edges-Collection mit 1D-Dummy-Vektor (keine Vektorsuche auf Edges).
Stable-ID:
Basis: "{kind}:{src}->{tgt}#{seq}"
Neu: Falls 'occ' vorhanden -> "{…}@{occ}"
Edges-Collection mit 1D-Dummy-Vektor.
- Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
- Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
"""
_, _, edges_col = _names(prefix)
points: List[rest.PointStruct] = []
for pl in edge_payloads:
for raw in edge_payloads:
pl = _normalize_edge_payload(raw)
edge_id = pl.get("edge_id")
if not edge_id:
kind = pl.get("kind", "edge")
s = pl.get("source_id", "unknown-src")
t = pl.get("target_id", "unknown-tgt")
seq = pl.get("seq") or pl.get("order") or ""
occ = pl.get("occ")
base = f"{kind}:{s}->{t}#{seq}"
edge_id = f"{base}@{occ}" if occ is not None else base
seq = pl.get("seq") or ""
edge_id = f"{kind}:{s}->{t}#{seq}"
pl["edge_id"] = edge_id
point_id = _to_uuid(edge_id)
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
return edges_col, points
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
if not points:
return
+190
View File
@@ -0,0 +1,190 @@
# 📘 Mindnet Skript-Handbuch
Dieses Handbuch dokumentiert die implementierten Skripte für das Projekt **mindnet Wissensnetzwerk**.
Es beschreibt Zweck, Parameter und typische Aufrufe. Alle Beispiele sind geprüft und lauffähig.
Alle Skripte liegen in `~/mindnet/scripts/`.
Ausführung erfolgt **im aktivierten venv**:
cd ~/mindnet
source .venv/bin/activate
---
## 1. Vault anlegen (für Tests)
### `make_test_vault.py`
**Zweck:**
Erstellt einen kleinen, nachvollziehbaren Test-Vault mit Markdown-Dateien.
Szenarien: offene Links, nachträgliche Anlage fehlender Notes, geänderte Chunk-Grenzen.
**Parameter:**
- `--out PATH` : Zielverzeichnis (Default: `./test_vault`)
- `--force` : Existierenden Ordner löschen und neu anlegen
- `--with-missing` : Optional – auch die `missing-note` gleich anlegen
**Hinweise für Tests**
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
Kompatibel mit:
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
**Beispiele:**
python3 -m scripts.make_test_vault --out ./test_vault --force
python3 -m scripts.make_test_vault --out ./test_vault --force --with-missing
---
## 2. Import in Qdrant
### `import_markdown.py`
**Zweck:**
Importiert Notes/Chunks/Edges aus einem Vault nach Qdrant.
- prüft Frontmatter
- chunked Inhalte + Embeddings
- erzeugt Edges (`references`, `references_at`, `backlink`)
- erkennt geänderte Dateien automatisch
- optional: Purge der alten Chunks/Edges pro Note
**Parameter:**
- `--vault PATH` : (Pflicht) Vault-Verzeichnis
- `--apply` : Änderungen wirklich durchführen (sonst Dry-Run)
- `--purge-before-upsert` : Vor Upsert alte Chunks/Edges **nur der geänderten Notes** löschen
- `--note-id ID` : Nur eine bestimmte Note importieren
**Beispiele:**
python3 -m scripts.import_markdown --vault ./vault
python3 -m scripts.import_markdown --vault ./vault --apply
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
---
## 3. Konsistenzprüfungen
### `audit_vault_vs_qdrant.py`
**Zweck:**
Vergleicht Vault-Inhalte mit Qdrant.
- Zählt Notes/Chunks/Edges
- Verifiziert Anzahl Wikilinks vs. Edges
- Meldet Mismatches pro Note
**Parameter:**
- `--vault PATH` : Vault-Verzeichnis
- `--prefix STR` : Prefix für Qdrant-Collections (Default: `mindnet`)
**Beispiel:**
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
---
### `validate_edges.py`
**Zweck:**
Prüft die Edges-Collection in Qdrant.
- Zählt Edges pro Typ
- Kontrolliert Invarianten (z. B. jede Reference hat Backlink)
- Listet unresolved Edges auf
**Parameter:**
- `--prefix STR` : Prefix für Qdrant-Collections
- `--details` : Detailausgabe mit Beispielen
**Beispiel:**
python3 -m scripts.validate_edges --prefix mindnet --details
---
## 4. Qdrant zurücksetzen
### `reset_qdrant.py`
**Zweck:**
Setzt die Qdrant-Collections zurück.
- „wipe“: löscht Collections komplett (inkl. Schema)
- „truncate“: löscht nur Inhalte, behält Collections
**Parameter:**
- `--mode {wipe,truncate}` : Aktion
- `--prefix STR` : Prefix der Collections (Default: `mindnet`)
- `--yes` : Automatische Bestätigung
- `--dry-run` : Nur anzeigen, nicht ausführen
**Beispiele:**
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
python3 -m scripts.reset_qdrant --mode wipe --prefix mindnet --yes
---
## 5. Diagnose
### `diagnose_changed.py`
**Zweck:**
Prüft, ob die Datei-Hashes (`sha256`) mit den gespeicherten `hash_fulltext` in Qdrant übereinstimmen.
Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt.
**Parameter:**
- `--vault PATH` : Vault-Verzeichnis
- `--prefix STR` : Prefix (Default: `mindnet`)
- `--note-id ID` : Nur bestimmte Note prüfen
- `--all` : Alle Notes prüfen
**Beispiel:**
python3 -m scripts.diagnose_changed --vault ./vault --prefix mindnet --all
---
## 6. Unresolved Links nachträglich auflösen
### `resolve_unresolved_references.py`
**Zweck:**
Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwischen passende Notizen existieren.
**Parameter:**
- `--prefix STR` : Prefix für Qdrant-Collections
- `--apply` : Änderungen wirklich ausführen (sonst nur Vorschau)
**Beispiele:**
python3 -m scripts.resolve_unresolved_references --prefix mindnet
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
---
# ✅ Typische Workflows
### A) Vault neu importieren
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
python3 -m scripts.import_markdown --vault ./vault --apply
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
### B) Änderungen einspielen (mit Purge für geänderte Notes)
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
### C) Unresolved Links nachziehen
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
python3 -m scripts.validate_edges --prefix mindnet --details
### D) Test-Vault Szenario durchspielen
python3 -m scripts.make_test_vault --out ./test_vault --force
python3 -m scripts.import_markdown --vault ./test_vault --apply
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
+208
View File
@@ -0,0 +1,208 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: audit_edges_vs_expectations.py — Prüfe Kanten in Qdrant gegen Vault-Erwartungen
Version: 1.0.0
Datum: 2025-09-09
Zweck
-----
- Liest Edges/Chunks/Notes aus Qdrant.
- Ermittelt erwartete Kanten-Anzahlen aus dem Vault:
* belongs_to : sollte == #Chunks
* next / prev : je Note (#Chunks_in_Note - 1)
* references : Summe aller Chunk-Wikilinks
* backlink : Summe einzigartiger Wikilinks pro Note (Note-Level)
- Vergleicht IST vs. SOLL und meldet Abweichungen.
ENV/Qdrant
----------
QDRANT_URL, QDRANT_API_KEY (optional), COLLECTION_PREFIX (Default: mindnet)
Aufrufe
-------
# Gesamtaudit
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault
# Mit anderem Prefix
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --prefix mindnet_dev
# Details anzeigen
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --details
"""
from __future__ import annotations
import argparse
import json
import os
import re
from collections import defaultdict, Counter
from typing import Dict, List, Tuple
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from dotenv import load_dotenv
# Projektmodule – nur leichtgewichtige Funktionen
try:
from app.core.parser import read_markdown
except Exception:
# sehr einfacher Fallback für Wikilinks
read_markdown = None
WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
# ------------------------------
# Qdrant Helpers
# ------------------------------
def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _scroll_all(client: QdrantClient, col: str, flt=None, with_payload=True, with_vectors=False, limit=256):
out, next_page = [], None
while True:
pts, next_page = client.scroll(
collection_name=col,
scroll_filter=flt,
with_payload=with_payload,
with_vectors=with_vectors,
limit=limit,
offset=next_page,
)
if not pts:
break
out.extend(pts)
if not next_page:
break
return out
# ------------------------------
# Vault scan
# ------------------------------
def _iter_md(root: str) -> List[str]:
out: List[str] = []
for dp, _, fns in os.walk(root):
for fn in fns:
if fn.lower().endswith(".md"):
p = os.path.join(dp, fn)
if "/.obsidian/" in p.replace("\\", "/"):
continue
out.append(p)
return sorted(out)
def _wikilinks_in_text(text: str) -> List[str]:
return WIKILINK_RE.findall(text or "")
def _wikilinks_per_note(vault_root: str) -> Dict[str, List[str]]:
res: Dict[str, List[str]] = {}
for p in _iter_md(vault_root):
body = ""
try:
if read_markdown:
parsed = read_markdown(p)
body = parsed.body or ""
fm = parsed.frontmatter or {}
nid = fm.get("id") or fm.get("note_id") or os.path.splitext(os.path.basename(p))[0]
else:
with open(p, "r", encoding="utf-8") as f:
txt = f.read()
# sehr einfacher Split: YAML-Frontmatter rausnehmen
if txt.lstrip().startswith("---"):
parts = txt.split("\n---", 1)
body = parts[1] if len(parts) > 1 else txt
else:
body = txt
nid = os.path.splitext(os.path.basename(p))[0]
res[nid] = _wikilinks_in_text(body)
except Exception:
continue
return res
# ------------------------------
# Main Audit
# ------------------------------
def main():
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Vault (für Erwartungswerte)")
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX", "mindnet"), help="Collection-Prefix")
ap.add_argument("--details", action="store_true", help="Detail-Listen ausgeben")
args = ap.parse_args()
client = QdrantClient(url=os.environ.get("QDRANT_URL", "http://127.0.0.1:6333"),
api_key=os.environ.get("QDRANT_API_KEY") or None)
notes_col, chunks_col, edges_col = _names(args.prefix)
# Qdrant laden
notes = _scroll_all(client, notes_col, with_payload=True, with_vectors=False)
chunks = _scroll_all(client, chunks_col, with_payload=True, with_vectors=False)
edges = _scroll_all(client, edges_col, with_payload=True, with_vectors=False)
# --- Ist-Zähler
cnt_kind = Counter()
cnt_scope = Counter()
by_note_chunks: Dict[str, int] = defaultdict(int)
chunk_wikilinks_total = 0
for p in chunks:
pl = p.payload or {}
by_note_chunks[pl.get("note_id")] += 1
wl = pl.get("wikilinks") or []
if isinstance(wl, list):
chunk_wikilinks_total += len(wl)
for p in edges:
pl = p.payload or {}
kind = pl.get("kind") or pl.get("edge_type") or "?"
scope = pl.get("scope") or "?"
cnt_kind[kind] += 1
cnt_scope[f"{kind}:{scope}"] += 1
total_chunks = sum(by_note_chunks.values())
# --- Soll-Zähler aus Vault
wl_per_note = _wikilinks_per_note(args.vault)
backlink_expected = sum(len(set(v)) for v in wl_per_note.values())
next_expected = sum(max(c - 1, 0) for c in by_note_chunks.values())
prev_expected = next_expected # symmetrische Kanten
belongs_to_expected = total_chunks
references_expected = chunk_wikilinks_total # aus Chunk-Payloads
# --- Ergebnis
result = {
"qdrant_counts": dict(cnt_kind),
"qdrant_counts_by_scope": dict(cnt_scope),
"chunks_total": total_chunks,
"by_note_chunks": dict(by_note_chunks),
"vault_expected": {
"belongs_to": belongs_to_expected,
"next": next_expected,
"prev": prev_expected,
"references": references_expected,
"backlink": backlink_expected,
},
"deltas": {
"belongs_to": cnt_kind.get("belongs_to", 0) - belongs_to_expected,
"next": cnt_kind.get("next", 0) - next_expected,
"prev": cnt_kind.get("prev", 0) - prev_expected,
"references": cnt_kind.get("references", 0) - references_expected,
"backlink": cnt_kind.get("backlink", 0) - backlink_expected,
},
"collections": {
"notes": notes_col, "chunks": chunks_col, "edges": edges_col
}
}
print(json.dumps(result, ensure_ascii=False, indent=2))
if args.details:
# optionale Stichproben (z. B. fehlerhafte Kantenarten)
pass
if __name__ == "__main__":
main()
+250
View File
@@ -0,0 +1,250 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: export_markdown.py — Qdrant → Markdown (Obsidian-kompatibel)
Version: 1.4.0
Datum: 2025-09-09
Kurzbeschreibung
----------------
Exportiert Markdown-Notizen aus Qdrant in einen Zielordner (Vault).
Rekonstruiert YAML-Frontmatter und Body.
Body-Rekonstruktions-Priorität (abwärtskompatibel):
1) notes.payload.fulltext (verlustfrei, wenn beim Import gespeichert)
2) ansonsten aus allen zugehörigen Chunks: payload.text → payload.content → payload.raw
in stabiler, sequentieller Reihenfolge (seq/chunk_index/ID-Nummer)
Wichtige Fixes
--------------
- **Pfad-Normalisierung**: erzwingt relative Pfade (führe führende '/' ab, backslashes → slashes),
damit ``--out`` nicht ignoriert wird.
- **`--prefix` (optional)**: Überschreibt COLLECTION_PREFIX; ENV bleibt Default (rückwärtskompatibel).
ENV / Qdrant
------------
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
- QDRANT_API_KEY (optional)
- COLLECTION_PREFIX (Default: mindnet)
Aufruf
------
python3 -m scripts.export_markdown --out ./_exportVault
python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-foo
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
python3 -m scripts.export_markdown --out ./_exportVault --prefix mindnet_dev # optional
Beispiele
---------
COLLECTION_PREFIX=mindnet QDRANT_URL=http://127.0.0.1:6333 \\
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
"""
from __future__ import annotations
import argparse
import json
import os
import re
from typing import Dict, Iterable, List, Optional, Tuple
import yaml
from qdrant_client.http import models as rest
from qdrant_client import QdrantClient
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
# -----------------------------------------------------------------------------
# Utilities
# -----------------------------------------------------------------------------
def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _ensure_dir(path: str) -> None:
d = os.path.dirname(path)
if d and not os.path.isdir(d):
os.makedirs(d, exist_ok=True)
def _normalize_rel_path(p: str) -> str:
"""Pfad relativ halten & normalisieren (slashes, führende / entfernen)."""
p = (p or "").replace("\\", "/")
return p.lstrip("/")
def _to_md(frontmatter: dict, body: str) -> str:
fm = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=True).strip()
return f"---\n{fm}\n---\n{(body or '').rstrip()}\n"
def _scroll_all(
client: QdrantClient,
col: str,
flt: Optional[rest.Filter] = None,
with_payload: bool = True,
with_vectors: bool = False,
limit: int = 256,
):
"""Scrollt durch alle Punkte einer Collection und liefert eine Liste mit Points."""
out = []
next_page = None
while True:
pts, next_page = client.scroll(
collection_name=col,
scroll_filter=flt,
with_payload=with_payload,
with_vectors=with_vectors,
limit=limit,
offset=next_page,
)
if not pts:
break
out.extend(pts)
if not next_page:
break
return out
def _load_chunks_for_note(client: QdrantClient, chunks_col: str, note_id: str) -> List[dict]:
flt = rest.Filter(must=[rest.FieldCondition(
key="note_id",
match=rest.MatchValue(value=note_id),
)])
pts = _scroll_all(client, chunks_col, flt, with_payload=True, with_vectors=False)
# Sortierung: bevorzugt seq → chunk_index → Nummer in id
def _seq(pl: dict) -> Tuple[int, int, int]:
s1 = pl.get("seq", pl.get("chunk_index", -1))
s2 = pl.get("chunk_index", -1)
# Nummer-Anteil aus "noteid#<n>"
s3 = 0
try:
m = re.search(r"#(\\d+)$", pl.get("id") or "")
if m:
s3 = int(m.group(1))
except Exception:
pass
return (int(s1) if isinstance(s1, int) else -1, int(s2) if isinstance(s2, int) else -1, s3)
pts_sorted = sorted(pts, key=lambda p: _seq(p.payload or {}))
return [p.payload or {} for p in pts_sorted]
def _reconstruct_body(note_pl: dict, chunk_payloads: List[dict]) -> str:
# 1) Volltext vorhanden?
fulltext = note_pl.get("fulltext")
if isinstance(fulltext, str) and fulltext.strip():
return fulltext
# 2) Aus Chunks zusammensetzen: text → content → raw
parts: List[str] = []
for ch in chunk_payloads:
text = ch.get("text") or ch.get("content") or ch.get("raw")
if isinstance(text, str) and text.strip():
parts.append(text.rstrip())
return ("\n\n".join(parts)).rstrip() + ("\n" if parts else "")
def _export_one_note(
client: QdrantClient,
prefix: str,
note_pl: dict,
out_root: str,
overwrite: bool,
) -> dict:
notes_col, chunks_col, _ = _names(prefix)
note_id = note_pl.get("note_id") or note_pl.get("id")
# Pfad robust bestimmen und relativ halten
path = note_pl.get("path") or f"{note_id}.md"
path = _normalize_rel_path(path)
out_path = os.path.join(out_root, path).replace("\\", "/")
# Frontmatter aus Payload zurückführen (nur bekannte Felder)
fm: Dict[str, object] = {}
for k in [
"title", "id", "type", "status", "created", "updated", "tags",
"priority", "effort_min", "due", "people", "aliases",
"depends_on", "assigned_to", "lang",
]:
v = note_pl.get(k) if k in note_pl else note_pl.get(f"note_{k}")
if v not in (None, [], ""):
fm[k] = v
# Mindestfelder
if "id" not in fm and note_id:
fm["id"] = note_id
if "title" not in fm and note_pl.get("title"):
fm["title"] = note_pl["title"]
# Body beschaffen
chunks = _load_chunks_for_note(client, chunks_col, note_id)
body = _reconstruct_body(note_pl, chunks)
# Schreiben?
if os.path.exists(out_path) and not overwrite:
return {"note_id": note_id, "path": path, "status": "skip_exists"}
_ensure_dir(out_path)
with open(out_path, "w", encoding="utf-8") as f:
f.write(_to_md(fm, body))
return {"note_id": note_id, "path": path, "status": "written"}
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--out", required=True, help="Zielordner für den Export-Vault")
ap.add_argument("--note-id", help="Nur eine Note exportieren (Note-ID)")
ap.add_argument("--overwrite", action="store_true", help="Bestehende Dateien überschreiben")
ap.add_argument("--prefix", help="(Optional) überschreibt COLLECTION_PREFIX aus ENV")
args = ap.parse_args()
# Qdrant-Konfiguration
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix # abwärtskompatibel: ENV bleibt Default
client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim)
notes_col, _, _ = _names(cfg.prefix)
# Notes holen (optional gefiltert)
flt = None
if args.note_id:
flt = rest.Filter(must=[rest.FieldCondition(
key="note_id",
match=rest.MatchValue(value=args.note_id),
)])
note_pts = _scroll_all(client, notes_col, flt, with_payload=True, with_vectors=False)
if not note_pts:
print(json.dumps({"exported": 0, "out": args.out, "message": "Keine Notes gefunden."}, ensure_ascii=False))
return
results = []
for p in note_pts:
pl = p.payload or {}
try:
res = _export_one_note(client, cfg.prefix, pl, args.out, args.overwrite)
except Exception as e:
res = {"note_id": pl.get("note_id") or pl.get("id"), "error": str(e)}
results.append(res)
print(json.dumps({
"exported": len([r for r in results if r.get("status") == "written"]),
"skipped": len([r for r in results if r.get("status") == "skip_exists"]),
"out": args.out,
"details": results,
}, ensure_ascii=False))
if __name__ == "__main__":
main()
+242
View File
@@ -0,0 +1,242 @@
#!/usr/bin/env python3
# scripts/gc_qdrant_after_vault_scan.py
# Version: 1.0.0 (2025-09-05)
#
# Zweck
# -----
# Garbage-Collector für Qdrant: löscht Einträge (Edges, Chunks, Notes), deren Note-ID
# nicht mehr im Obsidian-Vault vorhanden ist. So bleiben Vektorindex/Graph mit dem Vault konsistent.
#
# Aufrufparameter
# ---------------
# --vault PATH Pfad zum Obsidian-Vault (z. B. ./vault) [erforderlich]
# --mode {edges,content,all}
# edges -> nur Kanten mit source_id/target_id ∉ Vault-Notes löschen
# content -> Chunks & Notes löschen, deren note_id ∉ Vault-Notes (keine Edges)
# all -> zuerst edges, dann content
# --prefix PREFIX Collection-Präfix (Default: aus ENV COLLECTION_PREFIX oder "mindnet")
# --apply Ohne diesen Schalter: Dry-Run (nur Vorschau). Mit Schalter: ausführen.
# --yes Sicherheitsabfrage überspringen (non-interaktiv)
# --batch-size N Größe für In-Filter (Default: 1000)
#
# Hinweise
# --------
# - Benötigt funktionierendes Python-venv mit allen Abhängigkeiten von mindnet (qdrant-client usw.).
# - Läuft gegen die drei Collections {prefix}_notes, {prefix}_chunks, {prefix}_edges (keine anderen Projekte).
# - Nutzt OR-Filter (Filter.should) korrekt, keine minimum_should (Pydantic v2 lässt die Option nicht zu).
#
# Änderungen ggü. vorher
# ----------------------
# - Neu: eigenes GC-Tool; berührt ausschließlich das übergebene Projekt-Präfix.
# - Sichere Dry-Run-Vorschau mit Zählung pro Collection; interaktive Bestätigung.
# - Löscht Edges mittels Filter auf (source_id ∈ MISSING) ODER (target_id ∈ MISSING).
# - Löscht Chunks/Notes mittels Filter auf note_id ∈ MISSING.
#
# Beispiele
# ---------
# DRY RUN: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all
# APPLY: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all --apply
# CI-Modus: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode edges --apply --yes
#
# Kompatibilität / Kontext
# ------------------------
# - Frontmatter-Validierung & Parsing wie im Projekt (parser.read_markdown, validate_required_frontmatter). :contentReference[oaicite:0]{index=0}
# - Note-Payload-Schema (u. a. note_id, hash_fulltext) wie spezifiziert. :contentReference[oaicite:1]{index=1}
# - Collection-Namen & Anlage-Logik über app.core.qdrant (notes/chunks = dim, edges = 1D). :contentReference[oaicite:2]{index=2}
# - Edges nutzen Felder source_id/target_id; Chunks/Notes tragen note_id in payload. :contentReference[oaicite:3]{index=3} :contentReference[oaicite:4]{index=4}
#
from __future__ import annotations
import argparse, os, glob, json, sys
from typing import Iterable, Set, Tuple, List
from dotenv import load_dotenv
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter # :contentReference[oaicite:5]{index=5}
from app.core.qdrant import QdrantConfig, get_client, collection_names # :contentReference[oaicite:6]{index=6}
def iter_note_ids_from_vault(vault_root: str) -> Set[str]:
"""Liest alle Markdown-Dateien im Vault und sammelt valide Frontmatter-IDs."""
paths = [p for p in glob.glob(os.path.join(vault_root, "**", "*.md"), recursive=True)]
out: Set[str] = set()
for p in paths:
try:
parsed = read_markdown(p)
fm = normalize_frontmatter(parsed.frontmatter)
validate_required_frontmatter(fm)
nid = fm.get("id") or fm.get("note_id")
if nid:
out.add(nid)
except Exception:
# ungültige oder Template-Dateien ohne Pflichtfelder ignorieren
continue
return out
def scroll_payload_values(client: QdrantClient, collection: str, field: str, limit: int = 2000) -> Set[str]:
"""Scrollt eine Collection und sammelt alle Payload-Werte eines Feldes (falls vorhanden)."""
out: Set[str] = set()
offset = None
while True:
res = client.scroll(
collection_name=collection,
scroll_filter=None,
with_payload=True,
with_vectors=False,
limit=limit,
offset=offset
)
# qdrant-client (neuere Versionen) geben (points, next_offset)
if isinstance(res, tuple) and len(res) == 2:
points, offset = res
else:
# Fallback, falls sich API signifikant ändert
points, offset = res[0], res[1] if len(res) > 1 else None
for pt in points or []:
pl = getattr(pt, "payload", None) or {}
v = pl.get(field)
if isinstance(v, str):
out.add(v)
if not offset:
break
return out
def chunked(iterable: Iterable[str], n: int) -> Iterable[List[str]]:
buf: List[str] = []
for x in iterable:
buf.append(x)
if len(buf) >= n:
yield buf
buf = []
if buf:
yield buf
def count_matches(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
"""Grober Zähler über scroll (Qdrant Count-API ist optional je Version)."""
total = 0
offset = None
while True:
points, offset = client.scroll(
collection_name=collection,
scroll_filter=filter_,
with_payload=False,
with_vectors=False,
limit=2000,
offset=offset
)
total += len(points or [])
if not offset:
break
return total
def build_filters_for_missing(prefix: str, missing_note_ids: Set[str]) -> Tuple[rest.Filter, rest.Filter, rest.Filter]:
"""
Erzeugt je Collection passende Filter:
- Notes/Chunks: note_id ∈ missing
- Edges: (source_id ∈ missing) ODER (target_id ∈ missing)
"""
notes_col, chunks_col, edges_col = collection_names(prefix) # :contentReference[oaicite:7]{index=7}
# MatchAny statt MatchValue für große Mengen
any_missing = rest.MatchAny(any=list(missing_note_ids))
f_notes = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
f_edges = rest.Filter(should=[
rest.FieldCondition(key="source_id", match=any_missing),
rest.FieldCondition(key="target_id", match=any_missing),
])
return f_notes, f_chunks, f_edges
def preview(client: QdrantClient, prefix: str, missing: Set[str]) -> dict:
"""Erstellt eine Dry-Run-Vorschau (Anzahl Treffer pro Collection)."""
notes_col, chunks_col, edges_col = collection_names(prefix)
f_notes, f_chunks, f_edges = build_filters_for_missing(prefix, missing)
return {
"prefix": prefix,
"collections": {"notes": notes_col, "chunks": chunks_col, "edges": edges_col},
"missing_note_ids_count": len(missing),
"to_delete_counts": {
"edges": count_matches(client, edges_col, f_edges),
"chunks": count_matches(client, chunks_col, f_chunks),
"notes": count_matches(client, notes_col, f_notes),
}
}
def delete_with_filter(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
"""Löscht alle Punkte, die den Filter matchen. Gibt grobe Trefferzahl zurück."""
# erst zählen (für Feedback)
to_del = count_matches(client, collection, filter_)
if to_del == 0:
return 0
client.delete(
collection_name=collection,
points_selector=filter_,
wait=True
)
return to_del
def main():
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (z. B. ./vault)")
ap.add_argument("--mode", choices=["edges", "content", "all"], default="all",
help="Löschmodus (edges=nur Kanten, content=Chunks+Notes, all=beides)")
ap.add_argument("--prefix", help="Collection-Präfix (Default aus ENV COLLECTION_PREFIX)")
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter: Dry-Run")
ap.add_argument("--yes", action="store_true", help="Rückfrage unterdrücken")
ap.add_argument("--batch-size", type=int, default=1000, help="Batch-Größe für MatchAny")
args = ap.parse_args()
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix
client = get_client(cfg)
notes_col, chunks_col, edges_col = collection_names(cfg.prefix)
# 1) Vault lesen → Menge gültiger Note-IDs
vault_root = os.path.abspath(args.vault)
vault_ids = iter_note_ids_from_vault(vault_root)
# 2) Qdrant vorhandene Note-IDs einsammeln
existing_note_ids = scroll_payload_values(client, notes_col, "note_id")
# 3) Differenz ermitteln
missing = existing_note_ids - vault_ids
# 4) Vorschau
summary = preview(client, cfg.prefix, missing)
print(json.dumps({"mode": args.mode, "apply": args.apply, "summary": summary}, ensure_ascii=False, indent=2))
if not missing:
# nichts zu tun
return
if not args.apply:
# Dry-Run Ende
return
if not args.yes:
resp = input("Fortfahren und die oben gezeigten Objekte löschen? (yes/no): ").strip().lower()
if resp not in ("y", "yes"):
print("Abgebrochen.")
return
# 5) Löschen je Modus
f_notes, f_chunks, f_edges = build_filters_for_missing(cfg.prefix, missing)
report = {"deleted": {"edges": 0, "chunks": 0, "notes": 0}}
try:
if args.mode in ("edges", "all"):
report["deleted"]["edges"] = delete_with_filter(client, edges_col, f_edges)
if args.mode in ("content", "all"):
# Reihenfolge: erst chunks, dann notes
report["deleted"]["chunks"] = delete_with_filter(client, chunks_col, f_chunks)
report["deleted"]["notes"] = delete_with_filter(client, notes_col, f_notes)
finally:
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+218 -133
View File
@@ -1,185 +1,270 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: scripts/import_markdown.py
Version: v2.3.1 (2025-09-05)
Script: import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
Version: 3.2.0
Datum: 2025-09-09
Beschreibung
Importiert Markdown-Notizen in Qdrant (Notes, Chunks, Edges).
- Chunking + Embedding (MiniLM 384d, externer Embed-Server)
- Edges direkt beim Import aus Wikilinks ([[…]]) ableiten
- Idempotente UUIDv5-IDs; Collections werden bereitgestellt
Kurzbeschreibung
----------------
Liest Markdown-Dateien aus einem Vault ein und schreibt Notes, Chunks und Edges
idempotent nach Qdrant. Change-Detection basiert standardmäßig auf dem **Body-Hash**.
Neu: Hash-Modus und Normalisierung sind auch per **CLI** steuerbar.
ENV / Qdrant
------------
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
- QDRANT_API_KEY (optional)
- COLLECTION_PREFIX (Default: mindnet)
- VECTOR_DIM (Default: 384)
- MINDNET_HASH_MODE: "body" (Default) | "frontmatter" | "body+frontmatter"
- MINDNET_HASH_NORMALIZE: "canonical" (Default) | "none"
CLI (übersteuert ENV)
---------------------
--hash-mode body|frontmatter|body+frontmatter
--hash-normalize canonical|none
Aufruf
python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id ID] [--embed-note] [--force-replace]
Parameter
--vault Pfad zum Obsidian-Vault (erforderlich)
--apply Ohne Flag: Dry-Run (nur JSON-Zeilen). Mit Flag: schreibt in Qdrant.
--note-id Nur eine spezifische Note-ID verarbeiten
--embed-note Optional: Note-Volltext zusätzlich einbetten
--force-replace Vor Upsert: zugehörige Edges der Quell-Note in Qdrant löschen (harte Ersetzung)
Hinweise
- Im venv laufen: `source .venv/bin/activate`
- Erwartet laufenden Embed-Server (http://127.0.0.1:8990)
- Qdrant via env: QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM
Changelog
v2.3.1: FIX – Für derive_wikilink_edges werden jetzt echte Chunk-Texte übergeben
(chunks_for_links mit {"chunk_id","text"}), damit `references_at` erzeugt werden.
v2.3.0: Umstellung auf app.core.derive_edges; Edge-IDs unterstützen Occurrence.
v2.2.x: Fix Filter-API (pydantic v2) bei Purge; Kleinkosmetik.
------
python3 -m scripts.import_markdown --vault ./vault
python3 -m scripts.import_markdown --vault ./vault --apply
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
python3 -m scripts.import_markdown --vault ./vault --note-id 20250821-foo --apply
python3 -m scripts.import_markdown --vault ./vault --apply --embed-note
# Feingranulare Erkennung (jede Kleinigkeit im Body zählt):
python3 -m scripts.import_markdown --vault ./vault --hash-normalize none
"""
from __future__ import annotations
import argparse, os, glob, json, sys
from dotenv import load_dotenv
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
from app.core.note_payload import make_note_payload
from app.core.validate_note import validate_note_payload
from app.core.chunker import assemble_chunks
from app.core.chunk_payload import make_chunk_payloads
from app.core.embed import embed_texts, embed_one
from app.core.qdrant import QdrantConfig, ensure_collections, get_client
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
from app.core.derive_edges import build_note_index, derive_wikilink_edges # nutzt 'text' je Chunk :contentReference[oaicite:1]{index=1}
import argparse
import json
import os
import sys
from typing import Dict, List, Optional, Tuple
from dotenv import load_dotenv
from qdrant_client.http import models as rest
def iter_md(root: str, exclude_dirs=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")):
files = [p for p in glob.glob(os.path.join(root, "**", "*.md"), recursive=True)]
out = []
for p in files:
pn = p.replace("\\","/")
if any(ex in pn for ex in exclude_dirs):
continue
out.append(p)
return out
from app.core.parser import (
read_markdown,
normalize_frontmatter,
validate_required_frontmatter,
extract_wikilinks,
)
from app.core.note_payload import make_note_payload
from app.core.chunker import assemble_chunks
from app.core.chunk_payload import make_chunk_payloads
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
from app.core.qdrant_points import (
points_for_chunks,
points_for_note,
points_for_edges,
upsert_batch,
)
def purge_note_edges(client, prefix: str, source_note_id: str):
# Löscht Edges, deren source_id == source_note_id ODER (kind==backlink && target_id==source_note_id)
edges_col = f"{prefix}_edges"
f = rest.Filter(
should=[
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_note_id)),
rest.Filter(
must=[
rest.FieldCondition(key="kind", match=rest.MatchValue(value="backlink")),
rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_note_id)),
]
),
]
try:
from app.core.embed import embed_texts, embed_one # optional
except Exception:
embed_texts = None
embed_one = None
# -----------------------------------------------------------------------------
# Hilfsfunktionen
# -----------------------------------------------------------------------------
def iter_md(root: str) -> List[str]:
out: List[str] = []
for dirpath, _, filenames in os.walk(root):
for fn in filenames:
if not fn.lower().endswith(".md"):
continue
p = os.path.join(dirpath, fn)
pn = p.replace("\\", "/")
if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
continue
out.append(p)
return sorted(out)
def collections(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def fetch_existing_note_hash(client, prefix: str, note_id: str) -> Optional[str]:
notes_col, _, _ = collections(prefix)
f = rest.Filter(must=[rest.FieldCondition(
key="note_id",
match=rest.MatchValue(value=note_id),
)])
points, _ = client.scroll(
collection_name=notes_col,
scroll_filter=f,
with_payload=True,
with_vectors=False,
limit=1,
)
# pydantic v2: client.delete(...) statt delete_points(...)
client.delete(collection_name=edges_col, points_selector=f, wait=True)
if not points:
return None
pl = points[0].payload or {}
return pl.get("hash_fulltext")
def main():
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
_, chunks_col, edges_col = collections(prefix)
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
client.delete(collection_name=chunks_col, points_selector=f_chunks, wait=True)
should = [
rest.FieldCondition(key="source_id", match=rest.MatchText(text=f"{note_id}#")),
rest.FieldCondition(key="target_id", match=rest.MatchText(text=f"{note_id}#")),
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=note_id)),
rest.FieldCondition(key="target_id", match=rest.MatchValue(value=note_id)),
]
f_edges = rest.Filter(should=should)
client.delete(collection_name=edges_col, points_selector=f_edges, wait=True)
def _normalize_rel_path(abs_path: str, vault_root: str) -> str:
try:
rel = os.path.relpath(abs_path, vault_root)
except Exception:
rel = abs_path
return rel.replace("\\", "/").lstrip("/")
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
def main() -> None:
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian Vault (z.B. ./vault)")
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant (sonst Dry-Run)")
ap.add_argument("--note-id", help="Nur eine Note-ID verarbeiten")
ap.add_argument("--embed-note", action="store_true", help="Auch Note-Volltext einbetten (optional)")
ap.add_argument("--force-replace", action="store_true", help="Vor Upsert alte Edges der Note löschen")
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
ap.add_argument("--purge-before-upsert", action="store_true",
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
ap.add_argument("--force-replace", action="store_true",
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
# NEU: Hash-Steuerung per CLI
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter"], default=None)
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
args = ap.parse_args()
# Qdrant
cfg = QdrantConfig(
url=os.getenv("QDRANT_URL", "http://127.0.0.1:6333"),
api_key=os.getenv("QDRANT_API_KEY") or None,
prefix=os.getenv("COLLECTION_PREFIX", "mindnet"),
dim=int(os.getenv("VECTOR_DIM","384")),
)
cfg = QdrantConfig.from_env()
client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim)
notes_col, chunks_col, edges_col = collections(cfg.prefix)
root = os.path.abspath(args.vault)
files = iter_md(root)
if not files:
print("Keine Markdown-Dateien gefunden.", file=sys.stderr); sys.exit(2)
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
sys.exit(2)
# --- Note-Index (für robuste Zielauflösung) ---
note_stubs = []
processed = 0
for path in files:
parsed = read_markdown(path)
fm = normalize_frontmatter(parsed.frontmatter)
try:
validate_required_frontmatter(fm)
except Exception:
continue
if args.note_id and fm.get("id") != args.note_id:
continue
rel = os.path.relpath(parsed.path, root).replace("\\","/")
note_stubs.append({"note_id": fm["id"], "title": fm.get("title",""), "path": rel})
note_index = build_note_index(note_stubs)
total_notes = 0
for path in files:
parsed = read_markdown(path)
fm = normalize_frontmatter(parsed.frontmatter)
try:
validate_required_frontmatter(fm)
except Exception:
except Exception as e:
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"}))
continue
if args.note_id and fm.get("id") != args.note_id:
continue
total_notes += 1
processed += 1
# Note-Payload
note_pl = make_note_payload(parsed, vault_root=root)
validate_note_payload(note_pl)
note_pl["fulltext"] = parsed.body # für derive_edges
# Note-Payload (mit expliziten Hash-Parametern)
note_pl = make_note_payload(parsed, vault_root=root,
hash_mode=args.hash_mode, hash_normalize=args.hash_normalize)
if "fulltext" not in (note_pl or {}):
note_pl["fulltext"] = parsed.body or ""
if note_pl.get("path"):
note_pl["path"] = _normalize_rel_path(
os.path.join(root, note_pl["path"]) if not os.path.isabs(note_pl["path"]) else note_pl["path"], root
)
else:
note_pl["path"] = _normalize_rel_path(parsed.path, root)
# Chunks
note_id = note_pl["note_id"]
# Change-Detection
new_hash = note_pl.get("hash_fulltext")
old_hash = None if args.force_replace else fetch_existing_note_hash(client, cfg.prefix, note_id)
changed = args.force_replace or (old_hash != new_hash)
# Chunks + Embeddings (Nullvektor-Fallback)
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
if embed_texts:
vecs = embed_texts([getattr(c, "text", "") for c in chunks])
else:
vecs = [[0.0] * cfg.dim for _ in chunks]
# Embeddings (Chunks)
texts = [ch.text for ch in chunks]
vectors = embed_texts(texts)
# Edges (leichtgewichtig direkt hier ableiten)
# belongs_to / prev/next aus Chunk-Nachbarschaften + Wikilinks als references
edges: List[Dict] = []
for ch in chunk_pls:
cid = ch["id"]
edges.append({"src_id": cid, "dst_id": note_id, "edge_type": "belongs_to", "scope": "chunk"})
nb = ch.get("neighbors") or {}
if nb.get("prev"):
edges.append({"src_id": nb["prev"], "dst_id": cid, "edge_type": "next", "scope": "chunk"})
edges.append({"src_id": cid, "dst_id": nb["prev"], "edge_type": "prev", "scope": "chunk"})
if nb.get("next"):
edges.append({"src_id": cid, "dst_id": nb["next"], "edge_type": "next", "scope": "chunk"})
edges.append({"src_id": nb["next"], "dst_id": cid, "edge_type": "prev", "scope": "chunk"})
for ref in (ch.get("references") or []):
tid = ref.get("target_id")
if tid:
edges.append({"src_id": cid, "dst_id": tid, "edge_type": "references", "scope": "chunk"})
for tid in (note_pl.get("references") or []):
edges.append({"src_id": note_id, "dst_id": tid, "edge_type": "references", "scope": "note"})
edges.append({"src_id": tid, "dst_id": note_id, "edge_type": "backlink", "scope": "note"})
# Dedupe
_uniq = {}
for e in edges:
_uniq[(e["src_id"], e["dst_id"], e["edge_type"], e.get("scope", ""))] = e
edges = list(_uniq.values())
# Optional: Note-Vektor
note_vec = embed_one(parsed.body) if args.embed_note else None
# --- WICHTIG: Chunk-Texte für derive_wikilink_edges ---
chunks_for_links = [
{"chunk_id": (pl.get("chunk_id") or pl.get("id") or f"{fm['id']}#{i+1}"),
"text": chunks[i].text}
for i, pl in enumerate(chunk_pls)
if i < len(chunks)
]
# Kanten (Note- & Chunk-Ebene)
edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index)
# Dry-Run-Ausgabe
print(json.dumps({
"note_id": fm["id"],
"title": fm["title"],
# Zusammenfassung
summary = {
"note_id": note_id,
"title": fm.get("title"),
"chunks": len(chunk_pls),
"edges": len(edges),
"path": note_pl["path"]
}, ensure_ascii=False))
"changed": changed,
"decision": ("apply" if args.apply and changed else
"apply-skip-unchanged" if args.apply and not changed else
"dry-run"),
"path": note_pl["path"],
"hash_mode": args.hash_mode or os.environ.get("MINDNET_HASH_MODE", "body"),
"hash_normalize": args.hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical"),
"hash_old": old_hash,
"hash_new": new_hash,
}
print(json.dumps(summary, ensure_ascii=False))
if args.apply:
if args.force_replace:
purge_note_edges(client, cfg.prefix, fm["id"])
if not args.apply:
continue
# Notes upsert
notes_col, note_pts = points_for_note(cfg.prefix, note_pl, note_vec, cfg.dim)
upsert_batch(client, notes_col, note_pts)
if changed and args.purge_before_upsert:
purge_note_artifacts(client, cfg.prefix, note_id)
# Chunks upsert
chunks_col, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vectors)
upsert_batch(client, chunks_col, chunk_pts)
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
upsert_batch(client, notes_name, note_pts)
# Edges upsert
edges_col, edge_pts = points_for_edges(cfg.prefix, edges)
upsert_batch(client, edges_col, edge_pts)
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
upsert_batch(client, chunks_name, chunk_pts)
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
upsert_batch(client, edges_name, edge_pts)
print(f"Done. Processed notes: {processed}")
print(f"Done. Processed notes: {total_notes}")
if __name__ == "__main__":
main()
+223
View File
@@ -0,0 +1,223 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
make_test_vault.py — erzeugt einen minimalen, nachvollziehbaren Test-Vault
Version: 1.1 (2025-09-05)
Änderungen ggü. 1.0:
- created/updated werden als QUOTED ISO-Strings geschrieben, damit der YAML-Parser
sie nicht in datetime-Objekte konvertiert (Schema verlangt strings).
- Kommentare/Output gestrafft.
Zweck
- Kleiner Obsidian-ähnlicher Vault zum Durchspielen des Importers (Chunks/Edges).
- Szenarien: leere Links, spätere Anlage fehlender Noten, Chunk-Neuaufteilung.
Struktur (Default: ./test_vault)
- 40_concepts/concept-alpha.md
- 20_experiences/exp-one.md (verlinkt auf [[concept-alpha]] und [[missing-note]])
- 20_experiences/exp-two.md (verlinkt auf [[concept-alpha]])
- 30_projects/project-demo.md (verlinkt auf [[concept-alpha]] und [[exp-one]])
Voraussetzungen
- Keine. Rein Dateigenerierung.
Aufruf
python3 -m scripts.make_test_vault [--out ./test_vault] [--force]
Parameter
--out Zielverzeichnis (Standard: ./test_vault)
--force Bestehenden Ordner löschen und neu anlegen.
Hinweise für Tests
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
Kompatibel mit:
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
"""
from __future__ import annotations
import argparse
import os
import shutil
from datetime import datetime, timezone
def iso_now() -> str:
# ISO 8601 mit Offset; als String zurückgeben
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
def q(s: str) -> str:
# Sicheres Quoten für YAML-Frontmatter
# (einfaches doppelt-Quoten reicht hier, da wir keine eingebetteten """ verwenden)
return f"\"{s}\""
TEMPLATE_CONCEPT = """---
title: "Concept Alpha"
id: "concept-alpha"
type: "concept"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/concept","topic/test"]
---
## Beschreibung
Dies ist ein einfaches Konzept für Testzwecke.
## Mögliche Verbindungen
- [[exp-one]]
- [[project-demo]]
"""
TEMPLATE_EXP_ONE = """---
title: "Experience One"
id: "exp-one"
type: "experience"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/experience","topic/test"]
---
## Kontext
Diese Notiz verlinkt auf ein existierendes Konzept und auf eine noch **fehlende** Note.
## Beobachtung
- Bezug zu [[concept-alpha]]
- Offener Verweis auf [[missing-note]]
## Interpretation
Wenn später eine Datei für 'missing-note' angelegt wird, sollte der Importer die Kanten nachziehen.
## Mögliche Verbindungen
- [[concept-alpha]]
- [[missing-note]]
"""
TEMPLATE_EXP_TWO = """---
title: "Experience Two"
id: "exp-two"
type: "experience"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/experience","topic/test"]
---
## Kontext
Zweite Experience, die auf dasselbe Konzept verweist.
## Beobachtung
Verweis auf [[concept-alpha]].
## Mögliche Verbindungen
- [[concept-alpha]]
"""
TEMPLATE_PROJECT = """---
title: "Project Demo"
id: "project-demo"
type: "project"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/project","topic/test"]
---
## Scope
Ein kleines Demo-Projekt, das andere Notizen referenziert.
## Arbeitspakete
- AP-1: Zusammenhang erklären (siehe [[concept-alpha]])
- AP-2: Ereignis berücksichtigen (siehe [[exp-one]])
## Mögliche Verbindungen
- [[concept-alpha]]
- [[exp-one]]
"""
TEMPLATE_MISSING = """---
title: "Missing Note (jetzt angelegt)"
id: "missing-note"
type: "concept"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/concept","topic/test"]
---
## Kontext
Diese Notiz wurde **nachträglich** erstellt, um einen zuvor offenen Link zu schließen.
## Mögliche Verbindungen
- [[exp-one]]
"""
def write_file(path: str, content: str) -> None:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(content)
def build_vault(base: str, include_missing_note: bool) -> None:
created = q(iso_now())
updated = q(iso_now())
# 40_concepts
write_file(
os.path.join(base, "40_concepts", "concept-alpha.md"),
TEMPLATE_CONCEPT.format(created=created, updated=updated),
)
# 20_experiences
write_file(
os.path.join(base, "20_experiences", "exp-one.md"),
TEMPLATE_EXP_ONE.format(created=created, updated=updated),
)
write_file(
os.path.join(base, "20_experiences", "exp-two.md"),
TEMPLATE_EXP_TWO.format(created=created, updated=updated),
)
# 30_projects
write_file(
os.path.join(base, "30_projects", "project-demo.md"),
TEMPLATE_PROJECT.format(created=created, updated=updated),
)
# Optional: zunächst NICHT anlegen (Szenario „leerer Link“),
# später für den 2. Testlauf anlegen.
if include_missing_note:
write_file(
os.path.join(base, "40_concepts", "missing-note.md"),
TEMPLATE_MISSING.format(created=created, updated=updated),
)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="./test_vault", help="Zielverzeichnis für den Test-Vault")
ap.add_argument("--force", action="store_true", help="Falls vorhanden: Verzeichnis löschen und neu anlegen")
ap.add_argument("--with-missing", action="store_true",
help="Auch die 'missing-note' direkt anlegen (Standard: erst im 2. Lauf)")
args = ap.parse_args()
base = os.path.abspath(args.out)
if os.path.exists(base):
if args.force:
shutil.rmtree(base)
else:
print(f"Ziel existiert bereits: {base}\nNutze --force zum Überschreiben.")
return
build_vault(base, include_missing_note=args.with_missing)
print(f"Test-Vault erstellt unter: {base}")
print("Hinweis:")
print(" 1) Zuerst ohne missing-note anlegen (Standard), importieren, Edges prüfen.")
print(" 2) Dann erneut ausführen mit --force --with-missing (missing-note wird erzeugt),")
print(" und Import wiederholen → Edges/Backlinks sollten nachgezogen werden.")
if __name__ == "__main__":
main()
+232
View File
@@ -0,0 +1,232 @@
# scripts/prune_qdrant_vs_vault.py
# -----------------------------------------------------------------------------
# Name: prune_qdrant_vs_vault.py
# Version: 1.0.0 (2025-09-08)
# Zweck: Entfernt verwaiste Qdrant-Einträge (notes/chunks/edges), wenn
# die zugehörigen Markdown-Dateien im Vault nicht mehr existieren.
#
# Was es macht:
# - Liest alle note_id aus dem Vault (Frontmatter: id / note_id).
# - Liest alle note_id aus Qdrant (mindnet_notes).
# - Bildet die Differenz (nur in Qdrant vorhandene, im Vault fehlende).
# - Löscht für jede verwaiste note_id:
# * Notes-Point(s)
# * Alle Chunks der Note
# * Alle Edges, die auf diese Note referenzieren
# (source_id == note_id ODER target_id == note_id ODER
# source_note_id == note_id ODER target_note_id == note_id)
#
# Hinweise:
# - Kein globaler Delete. Nur betroffene note_id.
# - Dry-Run standardmäßig; tatsächliches Löschen erst mit --apply.
# - Interaktive Bestätigung (abschaltbar mit --yes).
#
# Aufruf:
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply --yes
#
# Voraussetzungen:
# - Ausführung im aktivierten venv empfohlen: source .venv/bin/activate
# - Qdrant läuft lokal (oder URL/API-Key in ENV), siehe app/core/qdrant.py
#
# Änderungen:
# - 1.0.0: Erster Release.
# -----------------------------------------------------------------------------
import argparse
import json
import os
import sys
from pathlib import Path
from typing import Dict, List, Set, Tuple
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client, collection_names
from app.core.parser import parse_markdown_file # nutzt euer bestehendes Parsing/Schema
def read_vault_note_ids(vault_dir: Path) -> Set[str]:
note_ids: Set[str] = set()
for p in vault_dir.rglob("*.md"):
try:
parsed = parse_markdown_file(str(p))
fm = parsed.frontmatter if hasattr(parsed, "frontmatter") else (parsed.get("frontmatter") or {})
nid = fm.get("id") or fm.get("note_id")
if nid:
note_ids.add(str(nid))
except Exception:
# still und leise weiter – wir wollen robust sein
continue
return note_ids
def qdrant_note_ids(client: QdrantClient, notes_col: str) -> Set[str]:
ids: Set[str] = set()
scroll_filter = None
offset = None
while True:
res = client.scroll(
collection_name=notes_col,
scroll_filter=scroll_filter,
offset=offset,
limit=256,
with_payload=True,
with_vectors=False,
)
points, next_offset = res
for pt in points:
pl = pt.payload or {}
nid = pl.get("note_id")
if nid:
ids.add(str(nid))
if next_offset is None:
break
offset = next_offset
return ids
def delete_by_filter(client: QdrantClient, collection: str, flt: rest.Filter) -> int:
# Sammel erst IDs via scroll (robuster Überblick), lösche dann über point_ids (batch)
to_delete: List[int] = []
offset = None
while True:
pts, next_offset = client.scroll(
collection_name=collection,
scroll_filter=flt,
offset=offset,
limit=256,
with_payload=False,
with_vectors=False,
)
for pt in pts:
to_delete.append(pt.id)
if next_offset is None:
break
offset = next_offset
if not to_delete:
return 0
client.delete(collection_name=collection, points_selector=rest.PointIdsList(points=to_delete), wait=True)
return len(to_delete)
def prune_for_note(
client: QdrantClient,
cols: Dict[str, str],
note_id: str,
dry_run: bool = True,
) -> Dict[str, int]:
stats = {"notes": 0, "chunks": 0, "edges": 0}
# Notes löschen (Filter auf payload.note_id)
f_notes = rest.Filter(
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
)
# Chunks löschen (Filter auf payload.note_id)
f_chunks = rest.Filter(
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
)
# Edges löschen: mehrere Teilmengen erfassen
edge_filters = [
rest.Filter(must=[rest.FieldCondition(key="source_id", match=rest.MatchValue(value=note_id))]),
rest.Filter(must=[rest.FieldCondition(key="target_id", match=rest.MatchValue(value=note_id))]),
rest.Filter(must=[rest.FieldCondition(key="source_note_id", match=rest.MatchValue(value=note_id))]),
rest.Filter(must=[rest.FieldCondition(key="target_note_id", match=rest.MatchValue(value=note_id))]),
]
if dry_run:
# Zähle nur, was *wäre* gelöscht worden
def count(flt: rest.Filter, col: str) -> int:
total = 0
offset = None
while True:
pts, next_offset = client.scroll(
collection_name=col,
scroll_filter=flt,
offset=offset,
limit=256,
with_payload=False,
with_vectors=False,
)
total += len(pts)
if next_offset is None:
break
offset = next_offset
return total
stats["notes"] = count(f_notes, cols["notes"])
stats["chunks"] = count(f_chunks, cols["chunks"])
stats["edges"] = sum(count(f, cols["edges"]) for f in edge_filters)
return stats
# tatsächliches Löschen
stats["notes"] = delete_by_filter(client, cols["notes"], f_notes)
stats["chunks"] = delete_by_filter(client, cols["chunks"], f_chunks)
e_deleted = 0
for f in edge_filters:
e_deleted += delete_by_filter(client, cols["edges"], f)
stats["edges"] = e_deleted
return stats
def main():
ap = argparse.ArgumentParser(description="Prune Qdrant data for notes missing in the Vault.")
ap.add_argument("--vault", required=True, help="Pfad zum Vault-Verzeichnis (Root).")
ap.add_argument("--prefix", default="mindnet", help="Collections-Präfix (Default: mindnet).")
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter wird nur ein Dry-Run gemacht.")
ap.add_argument("--yes", action="store_true", help="Ohne Nachfrage ausführen.")
args = ap.parse_args()
vault_root = Path(args.vault).resolve()
if not vault_root.exists():
print(f"Vault-Verzeichnis nicht gefunden: {vault_root}", file=sys.stderr)
sys.exit(1)
cfg = QdrantConfig()
client = get_client(cfg)
cols = collection_names(args.prefix)
vault_ids = read_vault_note_ids(vault_root)
qdrant_ids = qdrant_note_ids(client, cols["notes"])
orphans = sorted(qdrant_ids - vault_ids)
preview = {
"mode": "APPLY" if args.apply else "DRY-RUN",
"prefix": args.prefix,
"vault_root": str(vault_root),
"collections": cols,
"counts": {
"vault_note_ids": len(vault_ids),
"qdrant_note_ids": len(qdrant_ids),
"orphans": len(orphans),
},
"orphans_sample": orphans[:20],
}
print(json.dumps(preview, ensure_ascii=False, indent=2))
if not orphans:
print("Keine verwaisten Einträge gefunden. Nichts zu tun.")
return
if not args.yes:
resp = input("\nFortfahren mit dem oben angezeigten Modus? (yes/no): ").strip().lower()
if resp not in ("y", "yes", "j", "ja"):
print("Abgebrochen.")
return
total_stats = {"notes": 0, "chunks": 0, "edges": 0}
for nid in orphans:
s = prune_for_note(client, cols, nid, dry_run=(not args.apply))
total_stats["notes"] += s["notes"]
total_stats["chunks"] += s["chunks"]
total_stats["edges"] += s["edges"]
print(json.dumps({"deleted": total_stats}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python3
# scripts/resolve_unresolved_references.py
"""
resolve_unresolved_references.py — Unaufgelöste Wikilinks in Qdrant nachträglich auflösen
Version: 1.0.0 (2025-09-05)
Zweck
------
- Findet Edges in {prefix}_edges mit payload.status=="unresolved" und versucht, den Zielknoten
anhand bereits vorhandener Notes in {prefix}_notes aufzulösen.
- Aktualisiert die Edges (setzt target_id, entfernt status, setzt resolution), und erzeugt
– NUR für Note-Level 'references' – die symmetrische 'backlink'-Kante.
Warum?
------
- Beim ersten Import können Links auf (noch) nicht existierende Notizen zeigen.
- Sobald die Zielnotiz später existiert, kann dieses Skript die Kanten reparieren.
Aufruf
------
# Dry-Run (Standard):
python3 -m scripts.resolve_unresolved_references --prefix mindnet
# Anwenden:
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
# Optional: nur X Edges anfassen
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply --limit 500
Parameter
---------
--prefix : Collection-Prefix (Default: aus Env COLLECION_PREFIX oder "mindnet")
--apply : Änderungen tatsächlich schreiben (ohne --apply = Dry-Run)
--limit : Max. Anzahl unaufgelöster Edges, die in diesem Lauf bearbeitet werden (Default: keine Begrenzung)
--batch : Upsert-Batchgröße (Default: 512)
Voraussetzungen / Hinweise
--------------------------
- Bitte im aktivierten venv laufen lassen (deine Umgebung: `.venv`).
- Qdrant-URL/Key/Prefix/Vektor-Dim werden wie üblich aus ENV gelesen (sieh app/core/qdrant.py). # noqa
- Nutzt die vorhandenen Utilities:
- app/core/qdrant.py (Client/Collections)
- app/core/qdrant_points.py (points_for_edges/upsert_batch)
- app/core/derive_edges.py (build_note_index/resolve_target)
Änderungshistorie
-----------------
1.0.0 Erstveröffentlichung.
"""
from __future__ import annotations
import argparse
import json
from typing import Any, Dict, List, Tuple, Iterable
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names # :contentReference[oaicite:3]{index=3}
from app.core.qdrant_points import points_for_edges, upsert_batch # :contentReference[oaicite:4]{index=4}
from app.core.derive_edges import build_note_index, resolve_target # :contentReference[oaicite:5]{index=5}
def _scroll(client: QdrantClient, **kwargs):
"""
Wrapper um qdrant_client.scroll() für unterschiedliche Client-Versionen:
neuere: (points, next_offset)
ältere: (points, next_page_offset, _)
"""
res = client.scroll(**kwargs)
if isinstance(res, tuple):
if len(res) == 2:
points, next_off = res
else:
# ältere Signatur: (points, next_off, _)
points, next_off, _ = res[0], res[1], res[2]
else:
# sehr alte Clients -> konservativ behandeln
points, next_off = res, None
return points, next_off
def _load_all_notes(client: QdrantClient, notes_col: str) -> List[Dict[str, Any]]:
notes: List[Dict[str, Any]] = []
next_off = None
while True:
pts, next_off = _scroll(
client,
collection_name=notes_col,
with_payload=True,
with_vectors=False,
limit=1024,
offset=next_off,
)
for p in pts or []:
pl = getattr(p, "payload", {}) or {}
# Erwartet Felder: note_id, title, path etc. (gemäß Schema) # :contentReference[oaicite:6]{index=6}
if pl.get("note_id"):
notes.append(pl)
if not next_off:
break
return notes
def _iter_unresolved_edges(client: QdrantClient, edges_col: str) -> Iterable[rest.Record]:
"""
Liefert alle Edge-Records mit payload.status == 'unresolved' und 'target_label' (string).
"""
f = rest.Filter(
must=[
rest.FieldCondition(key="status", match=rest.MatchValue(value="unresolved")),
]
)
next_off = None
while True:
pts, next_off = _scroll(
client,
collection_name=edges_col,
scroll_filter=f,
with_payload=True,
with_vectors=False,
limit=1024,
offset=next_off,
)
for p in pts or []:
pl = getattr(p, "payload", {}) or {}
if isinstance(pl.get("target_label"), str):
yield p
if not next_off:
break
def _make_backlink(source_note_id: str, target_note_id: str, extra: Dict[str, Any]) -> Dict[str, Any]:
"""
Baue eine 'backlink'-Edge-Payload source <- target (note-level).
"""
e = {
"kind": "backlink",
"source_id": target_note_id,
"target_id": source_note_id,
}
# Metafelder aus dem Original übernehmen (ohne status)
copy_keys = ["raw", "alias", "heading", "resolution"]
for k in copy_keys:
if k in extra:
e[k] = extra[k]
return e
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", help="Collection-Prefix (Default: Env/COLLECTION_PREFIX oder 'mindnet')")
ap.add_argument("--apply", action="store_true", help="Änderungen schreiben (ohne Flag = Dry-Run)")
ap.add_argument("--limit", type=int, default=0, help="Max. Anzahl unaufgelöster Edges bearbeiten (0 = alle)")
ap.add_argument("--batch", type=int, default=512, help="Upsert-Batchgröße")
args = ap.parse_args()
# Qdrant-Setup
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix
client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim) # sorgt u. a. für 1D-Vektor-Collection bei Edges :contentReference[oaicite:7]{index=7}
notes_col, _, edges_col = collection_names(cfg.prefix) # :contentReference[oaicite:8]{index=8}
# Notes laden & Index bauen
notes = _load_all_notes(client, notes_col)
idx = build_note_index(notes) # (by_id, by_slug, by_file_slug) :contentReference[oaicite:9]{index=9}
# Unresolved-Edges scannen
to_fix: List[dict] = []
backlinks: List[dict] = []
processed = 0
resolved = 0
for rec in _iter_unresolved_edges(client, edges_col):
if args.limit and processed >= args.limit:
break
processed += 1
pl = dict(rec.payload or {})
kind = pl.get("kind") or "references"
src = pl.get("source_id")
tgt_label = pl.get("target_label") or pl.get("target_id") # Fallback
# Zielauflösung
resolved_id, how = resolve_target(str(tgt_label), idx) # :contentReference[oaicite:10]{index=10}
if not resolved_id:
continue # weiterhin unresolved
# Edge-Update
new_pl = dict(pl)
new_pl["target_id"] = resolved_id
new_pl["resolution"] = how
if "status" in new_pl:
del new_pl["status"]
# ID stabil lassen -> points_for_edges erzeugt UUID aus edge_id/Fallback :contentReference[oaicite:11]{index=11}
if "edge_id" not in new_pl:
# stabiler Key aus (kind, src, tgt, evtl. seq)
seq = new_pl.get("seq") or new_pl.get("order") or ""
new_pl["edge_id"] = f"{kind}:{src}->{resolved_id}#{seq}"
to_fix.append(new_pl)
resolved += 1
# Nur bei Note-Level references (nicht references_at) -> Backlink erzeugen
if kind == "references":
extra = {k: new_pl.get(k) for k in ("raw", "alias", "heading")}
extra["resolution"] = how
backlinks.append(_make_backlink(source_note_id=src, target_note_id=resolved_id, extra=extra))
# Ergebnis ausgeben
summary = {
"prefix": cfg.prefix,
"scanned_unresolved": processed,
"resolved": resolved,
"backlinks_to_create": len(backlinks),
"apply": bool(args.apply),
}
print(json.dumps(summary, ensure_ascii=False))
if not args.apply:
return
# Upserts (in Batches)
def _batched(items: List[dict], n: int) -> Iterable[List[dict]]:
for i in range(0, len(items), n):
yield items[i : i + n]
# 1) Updates für reparierte Edges
for chunk in _batched(to_fix, args.batch):
col, pts = points_for_edges(cfg.prefix, chunk) # sorgt für Edge-UUID & Dummy-Vector :contentReference[oaicite:12]{index=12}
upsert_batch(client, col, pts)
# 2) Backlinks (nur references)
for chunk in _batched(backlinks, args.batch):
col, pts = points_for_edges(cfg.prefix, chunk)
upsert_batch(client, col, pts)
if __name__ == "__main__":
main()