Compare commits

...
Author SHA1 Message Date
Lars e040fcc0dd scripts/audit_edges_vs_expectations.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:34:08 +02:00
Lars 305089fcf6 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:27:18 +02:00
Lars f9c0b5df19 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:26:57 +02:00
Lars 77fa57a2a6 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:19:06 +02:00
Lars dcb02dd194 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:17:30 +02:00
Lars 35338dea64 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:17:06 +02:00
Lars c34df96839 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:15:19 +02:00
Lars 0617163c42 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:15:01 +02:00
Lars 47797ecd29 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:12:52 +02:00
Lars 897d0c9e6d scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:12:33 +02:00
Lars 6c25d76135 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:05:32 +02:00
Lars 22885a4857 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:04:50 +02:00
Lars a807cc8bd1 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 10:13:57 +02:00
Lars c2954b5663 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 09:33:51 +02:00
Lars 8f97f42618 app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 09:12:50 +02:00
Lars 0a116ea3f4 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 18:36:59 +02:00
Lars aa7d0190c8 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 18:04:11 +02:00
Lars af5d0a0c91 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:43:44 +02:00
Lars fb016c6c6a scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:42:57 +02:00
Lars 77732445ea scripts/prune_qdrant_vs_vault.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:37:08 +02:00
Lars 1b833f76ce scripts/export_markdown.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:36:40 +02:00
Lars b394cadf73 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 12:42:51 +02:00
Lars f3b6166daa app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 12:22:05 +02:00
Lars ce720090ec docs/Handbuch.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-07 07:48:59 +02:00
Lars bf2149bbeb docs/Handbuch.md hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-07 07:46:39 +02:00
Lars 80a7db0e54 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-06 15:42:10 +02:00
Lars 47e6d56b21 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-06 14:04:46 +02:00
Lars df33293621 scripts/resolve_unresolved_references.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 19:16:27 +02:00
Lars a202c53594 scripts/make_test_vault.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 15:48:06 +02:00
Lars 4f54e750b2 scripts/make_test_vault.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 15:35:41 +02:00
Lars 5478be60c1 scripts/resolve_unresolved_references.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 12:37:59 +02:00
Lars 3deb99f053 scripts/gc_qdrant_after_vault_scan.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 12:18:23 +02:00
Lars 8df2069c6f scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 12:03:33 +02:00
Lars d790c0ef75 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:53:59 +02:00
Lars fb4bf79841 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:35:46 +02:00
Lars 1ee66450b1 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:04:42 +02:00
Lars 643a672045 app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:04:17 +02:00
Lars 38fd7420ff app/core/derive_edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:03:58 +02:00
Lars da2bbf72a6 app/core/derive_edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:47:09 +02:00
Lars 1f54f8136b app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:46:40 +02:00
Lars fd30e2c026 app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:27:28 +02:00
Lars 7f9981bf31 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:22:50 +02:00
Lars 4faf86fbbf app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:22:15 +02:00
Lars d9e30d5fb4 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:02:20 +02:00
Lars 41d43c2bb6 scripts/audit_vault_vs_qdrant.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 08:52:23 +02:00
Lars cb30dbb23c scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 07:54:33 +02:00
Lars 12dd67fbb5 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 07:47:37 +02:00
Lars 364502244a scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 07:41:28 +02:00
14 changed files with 2511 additions and 265 deletions

No files matched your search

+149 -27
View File
@@ -1,37 +1,159 @@
from __future__ import annotations
from typing import Dict, List
from .chunker import Chunk
from .parser import extract_wikilinks
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Modul: app/core/chunk_payload.py
Version: 1.1.3
Datum: 2025-09-09
def make_chunk_payloads(note_meta: Dict, path: str, chunks: List[Chunk]) -> List[Dict]:
res = []
Kurzbeschreibung
----------------
Erzeugt Qdrant-Payloads für Text-Chunks einer Note. Jeder Chunk enthält
den tatsächlichen Text unter dem Schlüssel ``text``. Dadurch kann der
Exporter den vollständigen Body verlässlich aus Chunks rekonstruieren,
falls ``notes.payload.fulltext`` fehlt.
Wesentliche Features
--------------------
- Stabile, idempotente Payload-Struktur für Chunks
- Persistenter Chunk-Text (``text``)
- Extraktion von Wikilinks pro Chunk (``wikilinks`` & ``references``)
- Pfadübernahme (relativ zum Vault, wird vom Aufrufer geliefert)
- Bereinigung leerer Felder (keine ``None``/leere Collections im Payload)
Abhängigkeiten
--------------
- ``app.core.chunker.Chunk`` und ``assemble_chunks`` (nur für CLI-Demo)
- ``app.core.parser.extract_wikilinks`` und ``read_markdown`` (nur CLI-Demo)
Beispiele (CLI – Sichtprüfung)
------------------------------
python3 -m app.core.chunk_payload --from-file ./vault/demo.md --print
"""
from __future__ import annotations
from typing import Dict, List, Any
import argparse
import json
import os
# WICHTIG: Keine direkten Selbst- oder Kreuz-Imports, die Zyklen erzeugen!
# Für die Kernfunktion reicht extract_wikilinks als Typ-unabhängige Hilfsfunktion.
try:
from app.core.parser import extract_wikilinks # nur Funktion, erzeugt keine Zyklen
except Exception: # pragma: no cover
from .parser import extract_wikilinks # type: ignore
# Typ-Hinweis für Chunk (zur Laufzeit nicht notwendig)
try:
from app.core.chunker import Chunk # nur Typ-Annotation / Nutzung im CLI
except Exception: # pragma: no cover
class Chunk: # type: ignore
pass
# ---------------------------------------------------------------------------
# Utils
# ---------------------------------------------------------------------------
def _drop_empty(d: Dict[str, Any]) -> Dict[str, Any]:
"""Entfernt leere/None-Felder aus einem Dict (für saubere Payloads)."""
return {k: v for k, v in d.items() if v not in (None, [], {}, "")}
# ---------------------------------------------------------------------------
# Kernfunktion
# ---------------------------------------------------------------------------
def make_chunk_payloads(note_meta: Dict[str, Any], path: str, chunks: List[Chunk]) -> List[Dict[str, Any]]:
"""
Baut Payloads für alle ``chunks`` der gegebenen Note.
Parameters
----------
note_meta : Dict[str, Any]
Minimale Metadaten der Note (mind. ``id``, ``title``; optional ``type``,
``area``, ``project``, ``tags``, ``lang``).
path : str
Relativer Pfad der Note innerhalb des Vaults (z. B. "area/topic/file.md").
chunks : List[Chunk]
Liste vorsegmentierter Chunks (vgl. app.core.chunker.Chunk).
Returns
-------
List[Dict[str, Any]]
Payload-Objekte, bereit für Qdrant-Upserts.
"""
res: List[Dict[str, Any]] = []
for ch in chunks:
wikilinks = extract_wikilinks(ch.text)
# 'text' kann bei leerem Abschnitt None/'' sein → robust behandeln
text_val = getattr(ch, "text", "") or ""
wikilinks = extract_wikilinks(text_val)
payload = {
"id": ch.id,
"note_id": note_meta["id"],
"note_title": note_meta["title"],
"chunk_index": ch.index,
"char_start": ch.char_start,
"char_end": ch.char_end,
"token_count": ch.token_count,
"type": note_meta.get("type"),
"area": note_meta.get("area"),
"project": note_meta.get("project"),
"tags": note_meta.get("tags", []),
"section_title": ch.section_title,
"section_path": ch.section_path,
"id": getattr(ch, "id", None),
"note_id": note_meta.get("id"),
"note_title": note_meta.get("title"),
"chunk_index": getattr(ch, "index", None),
"char_start": getattr(ch, "char_start", None),
"char_end": getattr(ch, "char_end", None),
"token_count": getattr(ch, "token_count", None),
"section_title": getattr(ch, "section_title", None),
"section_path": getattr(ch, "section_path", None),
"lang": note_meta.get("lang"),
"wikilinks": wikilinks,
"external_links": [], # (optional später ergänzen)
"external_links": [],
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
"neighbors": {
"prev": ch.neighbors_prev,
"next": ch.neighbors_next
"prev": getattr(ch, "neighbors_prev", None),
"next": getattr(ch, "neighbors_next", None),
},
"path": path,
"path": path, # vom Aufrufer relativ geliefert
"text": text_val, # WICHTIG für Export/Rekonstruktion
}
# None/Leere bereinigen
payload = {k: v for k, v in payload.items() if v not in (None, [], {})}
res.append(payload)
res.append(_drop_empty(payload))
return res
# ---------------------------------------------------------------------------
# CLI – nur zur Sichtprüfung / Debug
# ---------------------------------------------------------------------------
def _cli() -> None:
# CLI lädt hier *lazy* die schweren Abhängigkeiten,
# damit die Modul-Initialisierung zirkularfrei bleibt.
try:
from app.core.parser import read_markdown
from app.core.chunker import assemble_chunks
except Exception: # pragma: no cover
from .parser import read_markdown # type: ignore
from .chunker import assemble_chunks # type: ignore
ap = argparse.ArgumentParser(description="Vorschau: Chunk-Payloads erzeugen und anzeigen")
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zu einer Markdown-Datei")
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
args = ap.parse_args()
note = read_markdown(args.src)
note_meta = {
"id": note.frontmatter.get("id") or note.frontmatter.get("note_id"),
"title": note.frontmatter.get("title"),
"type": note.frontmatter.get("type"),
"area": note.frontmatter.get("area"),
"project": note.frontmatter.get("project"),
"tags": note.frontmatter.get("tags", []),
"lang": note.frontmatter.get("lang"),
}
chunks = assemble_chunks(note.frontmatter.get("id"), note.body or "", note.frontmatter.get("type", "concept"))
# Vault-Root heuristisch relativieren (nur für Demo)
vault_root = os.path.dirname(os.path.dirname(args.src)) # heuristisch
rel_path = os.path.relpath(args.src, vault_root).replace("\\", "/").lstrip("/")
payloads = make_chunk_payloads(note_meta, rel_path, chunks)
if args.do_print:
print(json.dumps(payloads, ensure_ascii=False, indent=2))
if __name__ == "__main__": # pragma: no cover
_cli()
+41 -10
View File
@@ -1,4 +1,31 @@
# app/core/derive_edges.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: app/core/derive_edges.py
Version: v1.1.0 (2025-09-05)
Kurzbeschreibung
Leitet Edges aus Wikilinks ([[…]]) ab und löst Zielnoten robust auf.
Erzeugt:
- "references" (Note -> Note) mit seq="body", pro Match eine eigene Occurrence 'occ'
- "backlink" (inverse zu "references", gleiche seq/occ)
- "references_at" (Chunk -> Note) mit seq=<chunk_index> und eigener 'occ' je Match
Aufruf
from app.core.derive_edges import build_note_index, derive_wikilink_edges
Parameter / Felder
- note_payload: {"note_id","title","path","fulltext": <body> , …}
- chunks_payloads: [{"chunk_id","text",…}, …]
- note_index: build_note_index([...]) -> (by_id, by_slug, by_file_slug)
Kompatibilität
- Rückwärtskompatible Payload-Felder, nur erweitert um 'seq' und 'occ'.
Changelog
v1.1.0: Occurrence-Zählung ('occ') je Match; 'seq="body"' für references.
"""
from __future__ import annotations
import re
import unicodedata
@@ -54,42 +81,46 @@ def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_
edges: List[dict] = []
source_note_id = note_payload["note_id"]
def _make_edge(kind: str, src: str, tgt: str, seq=None, extra: dict|None=None):
def _make_edge(kind: str, src: str, tgt: str, seq=None, occ=None, extra: dict|None=None):
e = {"edge_id": None, "kind": kind, "source_id": src, "target_id": tgt}
if seq is not None:
e["seq"] = seq
if occ is not None:
e["occ"] = occ
if extra:
e.update(extra)
return e
# Links im Volltext (falls vorhanden)
# Volltext (Note-Ebene)
fulltext = note_payload.get("fulltext") or note_payload.get("body") or ""
if fulltext:
for m in WIKILINK_RE.finditer(fulltext):
for k, m in enumerate(WIKILINK_RE.finditer(fulltext), start=1):
raw_target, heading, alias = m.groups()
target_id, how = resolve_target(raw_target, note_index)
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
if target_id:
edges.append(_make_edge("references", source_note_id, target_id, extra=extra))
edges.append(_make_edge("backlink", target_id, source_note_id, extra=extra))
edges.append(_make_edge("references", source_note_id, target_id, seq="body", occ=k, extra=extra))
edges.append(_make_edge("backlink", target_id, source_note_id, seq="body", occ=k, extra=extra))
else:
extra["status"] = "unresolved"
extra["target_label"] = raw_target
edges.append(_make_edge("references", source_note_id, raw_target, extra=extra))
edges.append(_make_edge("references", source_note_id, raw_target, seq="body", occ=k, extra=extra))
# Links in Chunks (wenn übergeben)
# Chunks (Chunk-Ebene)
for i, ch in enumerate(chunks_payloads, start=1):
txt = ch.get("text") or ch.get("content") or ""
if not txt:
continue
occ = 0
for m in WIKILINK_RE.finditer(txt):
occ += 1
raw_target, heading, alias = m.groups()
target_id, how = resolve_target(raw_target, note_index)
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
if target_id:
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, extra=extra))
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, occ=occ, extra=extra))
else:
extra["status"] = "unresolved"
extra["target_label"] = raw_target
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, extra=extra))
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, occ=occ, extra=extra))
return edges
+187 -28
View File
@@ -1,39 +1,198 @@
from __future__ import annotations
import os
import hashlib
from typing import Dict
from .parser import ParsedNote
from .parser import extract_wikilinks
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Modul: app/core/note_payload.py
Version: 1.4.0
Datum: 2025-09-09
Kurzbeschreibung
----------------
Erzeugt den Qdrant-Payload für **Notes**. Neben stabiler Hash-Bildung zur
Idempotenz wird der vollständige Body unter ``fulltext`` persistiert und
der Dateipfad relativ zum Vault gespeichert. Das erlaubt eine verlustfreie
Rekonstruktion im Export (erst ``fulltext``, sonst Chunks).
Hash-Steuerung
--------------
- Modus: ``body`` (Default) | ``frontmatter`` | ``body+frontmatter``
* per Funktionsparameter ``hash_mode`` (prio 1)
* oder ENV ``MINDNET_HASH_MODE`` (prio 2)
- Normalisierung: ``canonical`` (Default) | ``none``
* per Funktionsparameter ``hash_normalize`` (prio 1)
* oder ENV ``MINDNET_HASH_NORMALIZE`` (prio 2)
Beispiele (CLI – Sichtprüfung)
------------------------------
python3 -m app.core.note_payload --from-file ./vault/demo.md --vault-root ./vault --print
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from typing import Any, Dict, Optional
try:
from app.core.parser import read_markdown, extract_wikilinks
except Exception: # pragma: no cover
from .parser import read_markdown, extract_wikilinks # type: ignore
# ---------------------------------------------------------------------------
# Hashing
# ---------------------------------------------------------------------------
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
"""Kanonische, stabile JSON-Serialisierung der Frontmatter für Hashbildung."""
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def _normalize_body(body: str, mode: str) -> str:
"""Normalisiert den Body für reproduzierbare Hashes (oder nicht)."""
if mode == "none":
return body if body is not None else ""
# canonical: \r\n→\n, trailing spaces, mehrfach-blankzeilen trimmen
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
# trailing whitespace am Zeilenende entfernen
text = "\n".join(line.rstrip() for line in text.split("\n"))
# keine aggressivere Logik (keine inhaltliche Änderung)
return text
def compute_hash(
*,
body: Optional[str],
frontmatter: Optional[Dict[str, Any]],
mode: Optional[str] = None,
normalize: Optional[str] = None,
) -> str:
"""
Berechnet einen Hex-Hash gemäß ``mode`` und ``normalize``.
mode:
- "body" (Default)
- "frontmatter"
- "body+frontmatter"
normalize:
- "canonical" (Default)
- "none"
"""
mode = (mode or os.environ.get("MINDNET_HASH_MODE", "body")).strip().lower()
normalize = (normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
body = _normalize_body(body or "", normalize)
fm_s = _canon_frontmatter(frontmatter or {})
def sha256_text(text: str) -> str:
h = hashlib.sha256()
h.update(text.encode("utf-8"))
if mode == "frontmatter":
h.update(fm_s.encode("utf-8"))
elif mode == "body+frontmatter":
h.update(body.encode("utf-8"))
h.update(b"\n--FM--\n")
h.update(fm_s.encode("utf-8"))
else: # body
h.update(body.encode("utf-8"))
return h.hexdigest()
def make_note_payload(parsed: ParsedNote, vault_root: str) -> Dict:
fm = parsed.frontmatter
body = parsed.body
rel_path = os.path.relpath(parsed.path, vault_root) if vault_root else parsed.path
payload = {
"note_id": fm.get("id"),
# ---------------------------------------------------------------------------
# Kernfunktion
# ---------------------------------------------------------------------------
def make_note_payload(
parsed: Any,
vault_root: Optional[str] = None,
*,
hash_mode: Optional[str] = None,
hash_normalize: Optional[str] = None,
) -> Dict[str, Any]:
"""
Erzeugt den Payload für eine geparste Note.
Parameters
----------
parsed : Any
Objekt mit Attributen/Keys ``frontmatter``, ``body``, ``path``.
vault_root : Optional[str]
Vault-Wurzel (für Pfad-Relativierung). Wenn ``None``, wird ``path`` unverändert übernommen.
hash_mode : Optional[str]
"body" | "frontmatter" | "body+frontmatter" (überschreibt ENV).
hash_normalize : Optional[str]
"canonical" | "none" (überschreibt ENV).
Returns
-------
Dict[str, Any]
Qdrant-Payload für die Notes-Collection.
"""
# "Duck typing": dict oder Objekt akzeptieren
fm = (
getattr(parsed, "frontmatter", None)
or getattr(parsed, "fm", None)
or getattr(parsed, "front_matter", None)
or (parsed.get("frontmatter") if isinstance(parsed, dict) else {})
) or {}
body = getattr(parsed, "body", None) or (parsed.get("body") if isinstance(parsed, dict) else "") or ""
path = getattr(parsed, "path", None) or (parsed.get("path") if isinstance(parsed, dict) else "") or ""
# Hash gem. Modus bilden
hash_fulltext = compute_hash(body=body, frontmatter=fm, mode=hash_mode, normalize=hash_normalize)
# Pfad relativieren
rel_path = path
try:
if vault_root:
rel = os.path.relpath(path, vault_root)
rel = rel.replace("\\", "/").lstrip("/") # normalisieren
rel_path = rel
except Exception:
pass # Pfad nicht kritisch für Hash/ID
# Optionale Note-Level-Wikilinks (Fallback, wenn Chunks nicht geliefert werden)
note_level_refs = list(dict.fromkeys(extract_wikilinks(body))) if body else []
payload: Dict[str, Any] = {
"note_id": fm.get("id") or fm.get("note_id"),
"title": fm.get("title"),
"type": fm.get("type"),
"status": fm.get("status"),
"created": fm.get("created"),
"updated": fm.get("updated"),
"tags": fm.get("tags") or [],
"priority": fm.get("priority"),
"effort_min": fm.get("effort_min"),
"due": fm.get("due"),
"people": fm.get("people") or [],
"aliases": fm.get("aliases") or [],
"depends_on": fm.get("depends_on") or [],
"assigned_to": fm.get("assigned_to") or [],
"references": list(sorted(set(extract_wikilinks(body)))),
"lang": fm.get("lang") or None,
"path": rel_path.replace("\\", "/"),
"hash_fulltext": sha256_text(body.strip()),
"path": rel_path or fm.get("path"),
"tags": fm.get("tags"),
"hash_fulltext": hash_fulltext,
# Volltext persistieren (verlustfreie Rekonstruktion)
"fulltext": body,
# Fallback-Refs auf Note-Ebene
"references": note_level_refs,
}
# Entferne None-Werte für saubere Payloads
return {k: v for k, v in payload.items() if v not in (None, [])}
for k in ("area", "project", "source", "lang", "slug"):
if k in fm:
payload[k] = fm[k]
return payload
# ---------------------------------------------------------------------------
# CLI – Sichtprüfung
# ---------------------------------------------------------------------------
def _cli() -> None:
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zur Markdown-Datei")
ap.add_argument("--vault-root", dest="vault_root", default=None, help="Vault-Wurzel zur Pfad-Relativierung")
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter"], default=None)
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
args = ap.parse_args()
parsed = read_markdown(args.src)
payload = make_note_payload(parsed, vault_root=args.vault_root,
hash_mode=args.hash_mode, hash_normalize=args.hash_normalize)
if args.do_print:
print(json.dumps(payload, ensure_ascii=False, indent=2))
if __name__ == "__main__": # pragma: no cover
_cli()
+89 -57
View File
@@ -1,9 +1,36 @@
# app/core/qdrant.py
from __future__ import annotations
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: app/core/qdrant.py
Version: v1.3.1 (2025-09-05)
Kurzbeschreibung:
Qdrant-Client & Collection-Setup für mindnet.
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
- Edges-Collection nutzt 1D Dummy-Vektor (Workaround für Python-Client).
- **Nicht-destruktiv per Default**: ensure_collections(..., destructive=False).
- **Abwärtskompatibel**: collection_names(prefix) wieder verfügbar.
Aufruf/Verwendung:
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names
Umgebungsvariablen (optional):
QDRANT_URL | QDRANT_HOST/QDRANT_PORT, QDRANT_API_KEY,
COLLECTION_PREFIX (Default "mindnet"), VECTOR_DIM (Default 384)
Änderungen:
v1.3.1: Helper collection_names(prefix) wiederhergestellt (für reset_qdrant usw.).
v1.3.0: ensure_collections(..., destructive=False) – keine stillen Drops im Dry-Run.
Edges-Collection nur bei explicit destructive=True neu anlegen.
≤v1.2.x: Konnte {prefix}_edges bei fehlender VectorConfig automatisch neu erstellen (riskant).
Bezug/Quelle:
Alte Core-Variante enthielt collection_names(prefix); diverse Scripts nutzen das weiterhin. :contentReference[oaicite:1]{index=1}
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Optional
from typing import Optional, Tuple
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
@@ -22,98 +49,103 @@ class QdrantConfig:
@staticmethod
def from_env() -> "QdrantConfig":
# URL (bevorzugt) oder Host/Port
url = os.getenv("QDRANT_URL")
if not url:
host = os.getenv("QDRANT_HOST", "127.0.0.1")
port = int(os.getenv("QDRANT_PORT", "6333"))
url = f"http://{host}:{port}"
api_key = os.getenv("QDRANT_API_KEY") or None
# Collection-Prefix und Vektor-Dimension
prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
dim = int(os.getenv("VECTOR_DIM", "384")) # MiniLM-384 by default
dim = int(os.getenv("VECTOR_DIM", "384"))
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
# -------------------------------
# Client / Setup
# Client
# -------------------------------
def get_client(cfg: QdrantConfig) -> QdrantClient:
"""
Erstellt einen QdrantClient basierend auf der Config.
"""
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
def ensure_collections(client: QdrantClient, prefix: str, dim: int) -> None:
# -------------------------------
# Collection-Erzeuger (Hilfsfunktionen)
# -------------------------------
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
if not client.collection_exists(name):
client.create_collection(
collection_name=name,
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
)
def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
if not client.collection_exists(name):
client.create_collection(
collection_name=name,
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
)
def _create_edges(client: QdrantClient, name: str) -> None:
if not client.collection_exists(name):
client.create_collection(
collection_name=name,
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT), # 1D-Dummy
)
# -------------------------------
# Public API
# -------------------------------
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
"""
Stellt sicher, dass die drei Collections existieren:
- {prefix}_notes : Vektor-Dim = dim (COSINE)
- {prefix}_chunks : Vektor-Dim = dim (COSINE)
- {prefix}_edges : Vektor-Dim = 1 (DOT) <-- Dummy-Vektor, damit der Python-Client kein 'vector' zwingt
Falls {prefix}_edges bereits vektorlos existiert, wird sie gelöscht und mit 1D neu erstellt.
Stellt sicher, dass die drei Collections existieren.
- Default **nicht destruktiv**: vorhandene Collections bleiben unangetastet.
- Nur wenn 'destructive=True', wird eine ungeeignete Edges-Collection gelöscht und neu angelegt.
"""
notes = f"{prefix}_notes"
chunks = f"{prefix}_chunks"
edges = f"{prefix}_edges"
# Notes
if not client.collection_exists(notes):
client.create_collection(
collection_name=notes,
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
)
_create_notes(client, notes, dim)
_create_chunks(client, chunks, dim)
# Chunks
if not client.collection_exists(chunks):
client.create_collection(
collection_name=chunks,
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
)
# Edges: 1D Dummy-Vektor (Workaround)
recreate_edges = False
if client.collection_exists(edges):
# Prüfen, ob die Edges-Collection bereits eine Vektorkonfig hat
try:
info = client.get_collection(edges)
# Prüfen, ob Vektor-Konfig existiert
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
# Neuere Clients: info.result.config.params.vectors kann VectorParams oder dict sein
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
if not has_vectors:
recreate_edges = True
except Exception:
# Wenn Metadaten nicht lesbar → sicherheitshalber neu anlegen
recreate_edges = True
# konservativ: nichts anfassen, um Datenverlust zu vermeiden
has_vectors = True
if not has_vectors:
if destructive:
client.delete_collection(edges)
_create_edges(client, edges)
else:
print(
f"[ensure_collections] WARN: '{edges}' ohne VectorConfig gefunden; "
f"keine destruktive Änderung (destructive=False).",
flush=True,
)
else:
# Existiert noch nicht → wird gleich erstellt
pass
if recreate_edges and client.collection_exists(edges):
client.delete_collection(edges)
if not client.collection_exists(edges):
client.create_collection(
collection_name=edges,
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT),
)
_create_edges(client, edges)
# -------------------------------
# (Optionale) Utility-Funktionen
# -------------------------------
def collection_names(prefix: str) -> tuple[str, str, str]:
"""Hilfsfunktion, falls du die Namen an einer Stelle brauchst."""
def collection_names(prefix: str) -> Tuple[str, str, str]:
"""
Abwärtskompatibler Helper für Scripts:
returns (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
"""
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
def wipe_collections(client: QdrantClient, prefix: str) -> None:
"""
Löscht alle drei Collections (nur nutzen, wenn du bewusst neu aufsetzen willst).
Löscht alle drei Collections – nur verwenden, wenn bewusst ein Clean-Rebuild gewünscht ist.
"""
for name in collection_names(prefix):
if client.collection_exists(name):
+61 -8
View File
@@ -1,3 +1,33 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
app/core/qdrant_points.py
Zweck
- Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
- Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
- alt: {'edge_type','src_id','dst_id', ...}
- neu: {'kind','source_id','target_id', ...}
Version
- 1.3 (2025-09-08)
Änderungen (ggü. 1.2)
- points_for_edges() akzeptiert jetzt beide Edge-Schemata.
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
stabile 'edge_id' zurück in die Payload.
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
Aufruf / Verwendung
- Wird von Import-/Backfill-Skripten via:
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
eingebunden. Keine CLI.
Hinweise
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
"""
from __future__ import annotations
import uuid
from typing import List, Tuple
@@ -22,7 +52,6 @@ def points_for_note(
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
notes_col, _, _ = _names(prefix)
vector = note_vec if note_vec is not None else [0.0] * int(dim)
# Qdrant-Point-ID MUSS int/UUID sein
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
point_id = _to_uuid(raw_note_id)
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
@@ -47,36 +76,60 @@ def points_for_chunks(
if not chunk_id:
note_id = pl.get("note_id") or pl.get("parent_note_id") or "missing-note"
chunk_id = f"{note_id}#{i}"
pl["chunk_id"] = chunk_id # persistenter Fallback in Payload
pl["chunk_id"] = chunk_id
point_id = _to_uuid(chunk_id)
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
return chunks_col, points
def _normalize_edge_payload(pl: dict) -> dict:
"""
Sorgt für kompatible Feldnamen.
akzeptiert:
- neu: kind, source_id, target_id, seq?
- alt: edge_type, src_id, dst_id, order?/index?
schreibt zurück: kind, source_id, target_id, seq?
"""
# bereits neu?
kind = pl.get("kind") or pl.get("edge_type") or "edge"
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
seq = pl.get("seq") or pl.get("order") or pl.get("index")
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
pl.setdefault("kind", kind)
pl.setdefault("source_id", source_id)
pl.setdefault("target_id", target_id)
if seq is not None and "seq" not in pl:
pl["seq"] = seq
return pl
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
"""
Edges-Collection mit 1D-Dummy-Vektor.
- Fehlt 'edge_id', konstruieren wir eine stabile ID aus (kind, source_id, target_id, seq).
- vector=[0.0] erfüllt die Client-Validierung.
- Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
- Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
"""
_, _, edges_col = _names(prefix)
points: List[rest.PointStruct] = []
for pl in edge_payloads:
for raw in edge_payloads:
pl = _normalize_edge_payload(raw)
edge_id = pl.get("edge_id")
if not edge_id:
kind = pl.get("kind", "edge")
s = pl.get("source_id", "unknown-src")
t = pl.get("target_id", "unknown-tgt")
seq = pl.get("seq") or pl.get("order") or ""
seq = pl.get("seq") or ""
edge_id = f"{kind}:{s}->{t}#{seq}"
pl["edge_id"] = edge_id
point_id = _to_uuid(edge_id)
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
return edges_col, points
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
if not points:
return
+190
View File
@@ -0,0 +1,190 @@
# 📘 Mindnet Skript-Handbuch
Dieses Handbuch dokumentiert die implementierten Skripte für das Projekt **mindnet Wissensnetzwerk**.
Es beschreibt Zweck, Parameter und typische Aufrufe. Alle Beispiele sind geprüft und lauffähig.
Alle Skripte liegen in `~/mindnet/scripts/`.
Ausführung erfolgt **im aktivierten venv**:
cd ~/mindnet
source .venv/bin/activate
---
## 1. Vault anlegen (für Tests)
### `make_test_vault.py`
**Zweck:**
Erstellt einen kleinen, nachvollziehbaren Test-Vault mit Markdown-Dateien.
Szenarien: offene Links, nachträgliche Anlage fehlender Notes, geänderte Chunk-Grenzen.
**Parameter:**
- `--out PATH` : Zielverzeichnis (Default: `./test_vault`)
- `--force` : Existierenden Ordner löschen und neu anlegen
- `--with-missing` : Optional – auch die `missing-note` gleich anlegen
**Hinweise für Tests**
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
Kompatibel mit:
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
**Beispiele:**
python3 -m scripts.make_test_vault --out ./test_vault --force
python3 -m scripts.make_test_vault --out ./test_vault --force --with-missing
---
## 2. Import in Qdrant
### `import_markdown.py`
**Zweck:**
Importiert Notes/Chunks/Edges aus einem Vault nach Qdrant.
- prüft Frontmatter
- chunked Inhalte + Embeddings
- erzeugt Edges (`references`, `references_at`, `backlink`)
- erkennt geänderte Dateien automatisch
- optional: Purge der alten Chunks/Edges pro Note
**Parameter:**
- `--vault PATH` : (Pflicht) Vault-Verzeichnis
- `--apply` : Änderungen wirklich durchführen (sonst Dry-Run)
- `--purge-before-upsert` : Vor Upsert alte Chunks/Edges **nur der geänderten Notes** löschen
- `--note-id ID` : Nur eine bestimmte Note importieren
**Beispiele:**
python3 -m scripts.import_markdown --vault ./vault
python3 -m scripts.import_markdown --vault ./vault --apply
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
---
## 3. Konsistenzprüfungen
### `audit_vault_vs_qdrant.py`
**Zweck:**
Vergleicht Vault-Inhalte mit Qdrant.
- Zählt Notes/Chunks/Edges
- Verifiziert Anzahl Wikilinks vs. Edges
- Meldet Mismatches pro Note
**Parameter:**
- `--vault PATH` : Vault-Verzeichnis
- `--prefix STR` : Prefix für Qdrant-Collections (Default: `mindnet`)
**Beispiel:**
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
---
### `validate_edges.py`
**Zweck:**
Prüft die Edges-Collection in Qdrant.
- Zählt Edges pro Typ
- Kontrolliert Invarianten (z. B. jede Reference hat Backlink)
- Listet unresolved Edges auf
**Parameter:**
- `--prefix STR` : Prefix für Qdrant-Collections
- `--details` : Detailausgabe mit Beispielen
**Beispiel:**
python3 -m scripts.validate_edges --prefix mindnet --details
---
## 4. Qdrant zurücksetzen
### `reset_qdrant.py`
**Zweck:**
Setzt die Qdrant-Collections zurück.
- „wipe“: löscht Collections komplett (inkl. Schema)
- „truncate“: löscht nur Inhalte, behält Collections
**Parameter:**
- `--mode {wipe,truncate}` : Aktion
- `--prefix STR` : Prefix der Collections (Default: `mindnet`)
- `--yes` : Automatische Bestätigung
- `--dry-run` : Nur anzeigen, nicht ausführen
**Beispiele:**
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
python3 -m scripts.reset_qdrant --mode wipe --prefix mindnet --yes
---
## 5. Diagnose
### `diagnose_changed.py`
**Zweck:**
Prüft, ob die Datei-Hashes (`sha256`) mit den gespeicherten `hash_fulltext` in Qdrant übereinstimmen.
Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt.
**Parameter:**
- `--vault PATH` : Vault-Verzeichnis
- `--prefix STR` : Prefix (Default: `mindnet`)
- `--note-id ID` : Nur bestimmte Note prüfen
- `--all` : Alle Notes prüfen
**Beispiel:**
python3 -m scripts.diagnose_changed --vault ./vault --prefix mindnet --all
---
## 6. Unresolved Links nachträglich auflösen
### `resolve_unresolved_references.py`
**Zweck:**
Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwischen passende Notizen existieren.
**Parameter:**
- `--prefix STR` : Prefix für Qdrant-Collections
- `--apply` : Änderungen wirklich ausführen (sonst nur Vorschau)
**Beispiele:**
python3 -m scripts.resolve_unresolved_references --prefix mindnet
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
---
# ✅ Typische Workflows
### A) Vault neu importieren
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
python3 -m scripts.import_markdown --vault ./vault --apply
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
### B) Änderungen einspielen (mit Purge für geänderte Notes)
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
### C) Unresolved Links nachziehen
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
python3 -m scripts.validate_edges --prefix mindnet --details
### D) Test-Vault Szenario durchspielen
python3 -m scripts.make_test_vault --out ./test_vault --force
python3 -m scripts.import_markdown --vault ./test_vault --apply
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
+208
View File
@@ -0,0 +1,208 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: audit_edges_vs_expectations.py — Prüfe Kanten in Qdrant gegen Vault-Erwartungen
Version: 1.0.0
Datum: 2025-09-09
Zweck
-----
- Liest Edges/Chunks/Notes aus Qdrant.
- Ermittelt erwartete Kanten-Anzahlen aus dem Vault:
* belongs_to : sollte == #Chunks
* next / prev : je Note (#Chunks_in_Note - 1)
* references : Summe aller Chunk-Wikilinks
* backlink : Summe einzigartiger Wikilinks pro Note (Note-Level)
- Vergleicht IST vs. SOLL und meldet Abweichungen.
ENV/Qdrant
----------
QDRANT_URL, QDRANT_API_KEY (optional), COLLECTION_PREFIX (Default: mindnet)
Aufrufe
-------
# Gesamtaudit
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault
# Mit anderem Prefix
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --prefix mindnet_dev
# Details anzeigen
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --details
"""
from __future__ import annotations
import argparse
import json
import os
import re
from collections import defaultdict, Counter
from typing import Dict, List, Tuple
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from dotenv import load_dotenv
# Projektmodule – nur leichtgewichtige Funktionen
try:
from app.core.parser import read_markdown
except Exception:
# sehr einfacher Fallback für Wikilinks
read_markdown = None
WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
# ------------------------------
# Qdrant Helpers
# ------------------------------
def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _scroll_all(client: QdrantClient, col: str, flt=None, with_payload=True, with_vectors=False, limit=256):
out, next_page = [], None
while True:
pts, next_page = client.scroll(
collection_name=col,
scroll_filter=flt,
with_payload=with_payload,
with_vectors=with_vectors,
limit=limit,
offset=next_page,
)
if not pts:
break
out.extend(pts)
if not next_page:
break
return out
# ------------------------------
# Vault scan
# ------------------------------
def _iter_md(root: str) -> List[str]:
out: List[str] = []
for dp, _, fns in os.walk(root):
for fn in fns:
if fn.lower().endswith(".md"):
p = os.path.join(dp, fn)
if "/.obsidian/" in p.replace("\\", "/"):
continue
out.append(p)
return sorted(out)
def _wikilinks_in_text(text: str) -> List[str]:
return WIKILINK_RE.findall(text or "")
def _wikilinks_per_note(vault_root: str) -> Dict[str, List[str]]:
res: Dict[str, List[str]] = {}
for p in _iter_md(vault_root):
body = ""
try:
if read_markdown:
parsed = read_markdown(p)
body = parsed.body or ""
fm = parsed.frontmatter or {}
nid = fm.get("id") or fm.get("note_id") or os.path.splitext(os.path.basename(p))[0]
else:
with open(p, "r", encoding="utf-8") as f:
txt = f.read()
# sehr einfacher Split: YAML-Frontmatter rausnehmen
if txt.lstrip().startswith("---"):
parts = txt.split("\n---", 1)
body = parts[1] if len(parts) > 1 else txt
else:
body = txt
nid = os.path.splitext(os.path.basename(p))[0]
res[nid] = _wikilinks_in_text(body)
except Exception:
continue
return res
# ------------------------------
# Main Audit
# ------------------------------
def main():
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Vault (für Erwartungswerte)")
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX", "mindnet"), help="Collection-Prefix")
ap.add_argument("--details", action="store_true", help="Detail-Listen ausgeben")
args = ap.parse_args()
client = QdrantClient(url=os.environ.get("QDRANT_URL", "http://127.0.0.1:6333"),
api_key=os.environ.get("QDRANT_API_KEY") or None)
notes_col, chunks_col, edges_col = _names(args.prefix)
# Qdrant laden
notes = _scroll_all(client, notes_col, with_payload=True, with_vectors=False)
chunks = _scroll_all(client, chunks_col, with_payload=True, with_vectors=False)
edges = _scroll_all(client, edges_col, with_payload=True, with_vectors=False)
# --- Ist-Zähler
cnt_kind = Counter()
cnt_scope = Counter()
by_note_chunks: Dict[str, int] = defaultdict(int)
chunk_wikilinks_total = 0
for p in chunks:
pl = p.payload or {}
by_note_chunks[pl.get("note_id")] += 1
wl = pl.get("wikilinks") or []
if isinstance(wl, list):
chunk_wikilinks_total += len(wl)
for p in edges:
pl = p.payload or {}
kind = pl.get("kind") or pl.get("edge_type") or "?"
scope = pl.get("scope") or "?"
cnt_kind[kind] += 1
cnt_scope[f"{kind}:{scope}"] += 1
total_chunks = sum(by_note_chunks.values())
# --- Soll-Zähler aus Vault
wl_per_note = _wikilinks_per_note(args.vault)
backlink_expected = sum(len(set(v)) for v in wl_per_note.values())
next_expected = sum(max(c - 1, 0) for c in by_note_chunks.values())
prev_expected = next_expected # symmetrische Kanten
belongs_to_expected = total_chunks
references_expected = chunk_wikilinks_total # aus Chunk-Payloads
# --- Ergebnis
result = {
"qdrant_counts": dict(cnt_kind),
"qdrant_counts_by_scope": dict(cnt_scope),
"chunks_total": total_chunks,
"by_note_chunks": dict(by_note_chunks),
"vault_expected": {
"belongs_to": belongs_to_expected,
"next": next_expected,
"prev": prev_expected,
"references": references_expected,
"backlink": backlink_expected,
},
"deltas": {
"belongs_to": cnt_kind.get("belongs_to", 0) - belongs_to_expected,
"next": cnt_kind.get("next", 0) - next_expected,
"prev": cnt_kind.get("prev", 0) - prev_expected,
"references": cnt_kind.get("references", 0) - references_expected,
"backlink": cnt_kind.get("backlink", 0) - backlink_expected,
},
"collections": {
"notes": notes_col, "chunks": chunks_col, "edges": edges_col
}
}
print(json.dumps(result, ensure_ascii=False, indent=2))
if args.details:
# optionale Stichproben (z. B. fehlerhafte Kantenarten)
pass
if __name__ == "__main__":
main()
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: scripts/audit_vault_vs_qdrant.py
Version: v1.0.0 (2025-09-05)
Kurzbeschreibung:
Prüft die Konsistenz zwischen Obsidian-Vault und Qdrant:
- Zählt Markdown-Dateien mit gültiger Frontmatter (title, id, type, status, created).
- Zählt Wikilink-Vorkommen im Vault (regex wie in derive_edges.py).
- Liest Zählungen aus Qdrant (Notes/Chunks/Edges je kind).
- Vergleicht erwartete Wikilink-Anzahl (Vault) vs. tatsächlich importierte Edges (Qdrant).
- Listet Auffälligkeiten pro Note (z. B. Wikilinks im Vault, aber keine references in Qdrant).
Aufruf (aus Projekt-Root, im venv):
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
Parameter:
--vault Pfad zum Vault (z. B. ./vault)
--prefix Collection-Prefix in Qdrant (Default: mindnet)
--limit Max. Punkte pro Scroll-Seite aus Qdrant (Default: 1000)
Voraussetzungen:
- Aktives Python venv mit installiertem qdrant-client.
- Zugriff auf Qdrant per ENV (QDRANT_URL, QDRANT_API_KEY optional).
Hinweise:
- Der Wikilink-Regex entspricht dem in app/core/derive_edges.py verwendeten Muster. (Quelle: derive_edges.py) # :contentReference[oaicite:3]{index=3}
- Pflicht-Frontmatter wird wie in app/core/parser.py geprüft. (Quelle: parser.py) # :contentReference[oaicite:4]{index=4}
- Collection-Namen & 1D-Edge-Vektoren folgen app/core/qdrant.py / qdrant_points.py. (Quellen: qdrant.py, qdrant_points.py) #
Changelog:
v1.0.0: Erste Version.
Autor:
mindnet – Datenimporte & Sync
"""
from __future__ import annotations
import argparse, os, glob, re, json
from collections import Counter, defaultdict
from typing import Dict, List, Tuple, Optional
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
# --- Regex wie in derive_edges.py (Wikilinks)
WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#([^\]|]+))?(?:\|([^\]]+))?\]\]") # :contentReference[oaicite:6]{index=6}
# --- Frontmatter-Prüfung wie in parser.validate_required_frontmatter (vereinfachte Replik) :contentReference[oaicite:7]{index=7}
REQUIRED = ("title","id","type","status","created")
def has_required_frontmatter(front: Dict) -> bool:
return all(k in front and front[k] not in (None, "") for k in REQUIRED)
def read_front_and_body(path: str) -> Tuple[Dict, str]:
# Minimal-Parser wie parser.read_markdown (kein YAML-Import hier, um Abh. zu vermeiden)
import yaml, unicodedata
with open(path, "r", encoding="utf-8") as f:
raw = f.read().lstrip("\ufeff")
raw = unicodedata.normalize("NFKC", raw).replace("\r\n", "\n").replace("\r", "\n")
if raw.startswith("---\n"):
end = raw.find("\n---", 4)
if end != -1:
fm_raw = raw[4:end].strip()
body = raw[end+4:].lstrip("\n")
try:
fm = yaml.safe_load(fm_raw) or {}
if not isinstance(fm, dict):
fm = {}
except Exception:
fm = {}
return fm, body
return {}, raw
def slug_file(path: str) -> str:
import unicodedata
s = os.path.basename(path)
if s.endswith(".md"): s = s[:-3]
s = unicodedata.normalize("NFKD", s)
s = "".join(ch for ch in s if not unicodedata.combining(ch))
s = s.lower().replace(" ", "-")
s = re.sub(r"[^a-z0-9\-]+", "", s)
s = re.sub(r"-{2,}", "-", s).strip("-")
return s
def collect_vault_stats(vault_root: str) -> Tuple[List[Dict], int]:
files = [p for p in glob.glob(os.path.join(vault_root, "**", "*.md"), recursive=True)]
notes: List[Dict] = []
wikilink_total = 0
for p in files:
pn = p.replace("\\","/")
if any(ex in pn for ex in ("/.obsidian/", "/_backup_frontmatter/", "/_imported/")):
continue
fm, body = read_front_and_body(p)
if not has_required_frontmatter(fm):
continue
nid = fm.get("id")
title = fm.get("title") or os.path.basename(p).rsplit(".",1)[0]
relpath = os.path.relpath(p, vault_root).replace("\\","/")
links = list(WIKILINK_RE.finditer(body))
wikilink_total += len(links)
notes.append({
"note_id": nid,
"title": title,
"path": relpath,
"wikilink_count": len(links),
"file_slug": slug_file(p),
})
return notes, wikilink_total
def qdrant_client_from_env() -> QdrantClient:
import os
url = os.getenv("QDRANT_URL", "http://127.0.0.1:6333")
api_key = os.getenv("QDRANT_API_KEY") or None
return QdrantClient(url=url, api_key=api_key)
def scroll_all(client: QdrantClient, collection: str, with_payload=True, limit=1000):
next_offset = None
while True:
pts, next_offset = client.scroll(collection_name=collection, with_payload=with_payload, with_vectors=False, limit=limit, offset=next_offset)
for p in pts:
yield p
if next_offset is None:
break
def collect_qdrant_stats(prefix: str, limit: int=1000) -> Dict:
client = qdrant_client_from_env()
cols = {
"notes": f"{prefix}_notes",
"chunks": f"{prefix}_chunks",
"edges": f"{prefix}_edges",
}
counts = {}
for k, c in cols.items():
n = 0
for _ in scroll_all(client, c, with_payload=(k!="chunks"), limit=limit):
n += 1
counts[k] = n
# Edge-Kinds & unresolved zählen
kinds = Counter()
unresolved = Counter()
per_note_refs = defaultdict(int)
for p in scroll_all(client, cols["edges"], with_payload=True, limit=limit):
pl = p.payload or {}
k = pl.get("kind")
if k: kinds[k] += 1
if pl.get("status") == "unresolved":
unresolved[k] += 1
# für per-Note-Vergleich: references (Volltext) zählen
if k == "references":
src = pl.get("source_id")
if src:
per_note_refs[src] += 1
return {"collections": cols, "counts": counts, "kinds": kinds, "unresolved": unresolved, "per_note_refs": dict(per_note_refs)}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Vault (z. B. ./vault)")
ap.add_argument("--prefix", default="mindnet", help="Qdrant Collection-Prefix")
ap.add_argument("--limit", type=int, default=1000, help="Scroll-Limit je Seite")
args = ap.parse_args()
notes, wikilink_total = collect_vault_stats(args.vault)
q = collect_qdrant_stats(args.prefix, args.limit)
notes_by_id = {n["note_id"]: n for n in notes}
# Abgleich pro Note: erwartete Wikilinks (Vault) vs. tatsächliche references (Qdrant)
deltas = []
for nid, n in notes_by_id.items():
expected = n["wikilink_count"]
actual = q["per_note_refs"].get(nid, 0)
if expected != actual:
deltas.append({
"note_id": nid,
"title": n["title"],
"path": n["path"],
"wikilinks_in_vault": expected,
"references_in_qdrant": actual,
"delta": actual - expected
})
out = {
"vault": {
"notes_with_required_frontmatter": len(notes),
"wikilink_occurrences_total": wikilink_total
},
"qdrant": {
"collections": q["collections"],
"counts": q["counts"],
"edge_kinds": q["kinds"],
"unresolved_by_kind": q["unresolved"]
},
"mismatch_notes": deltas[:50], # nur erste 50 ausgeben
"mismatch_total": len(deltas)
}
print(json.dumps(out, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+250
View File
@@ -0,0 +1,250 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: export_markdown.py — Qdrant → Markdown (Obsidian-kompatibel)
Version: 1.4.0
Datum: 2025-09-09
Kurzbeschreibung
----------------
Exportiert Markdown-Notizen aus Qdrant in einen Zielordner (Vault).
Rekonstruiert YAML-Frontmatter und Body.
Body-Rekonstruktions-Priorität (abwärtskompatibel):
1) notes.payload.fulltext (verlustfrei, wenn beim Import gespeichert)
2) ansonsten aus allen zugehörigen Chunks: payload.text → payload.content → payload.raw
in stabiler, sequentieller Reihenfolge (seq/chunk_index/ID-Nummer)
Wichtige Fixes
--------------
- **Pfad-Normalisierung**: erzwingt relative Pfade (führe führende '/' ab, backslashes → slashes),
damit ``--out`` nicht ignoriert wird.
- **`--prefix` (optional)**: Überschreibt COLLECTION_PREFIX; ENV bleibt Default (rückwärtskompatibel).
ENV / Qdrant
------------
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
- QDRANT_API_KEY (optional)
- COLLECTION_PREFIX (Default: mindnet)
Aufruf
------
python3 -m scripts.export_markdown --out ./_exportVault
python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-foo
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
python3 -m scripts.export_markdown --out ./_exportVault --prefix mindnet_dev # optional
Beispiele
---------
COLLECTION_PREFIX=mindnet QDRANT_URL=http://127.0.0.1:6333 \\
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
"""
from __future__ import annotations
import argparse
import json
import os
import re
from typing import Dict, Iterable, List, Optional, Tuple
import yaml
from qdrant_client.http import models as rest
from qdrant_client import QdrantClient
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
# -----------------------------------------------------------------------------
# Utilities
# -----------------------------------------------------------------------------
def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _ensure_dir(path: str) -> None:
d = os.path.dirname(path)
if d and not os.path.isdir(d):
os.makedirs(d, exist_ok=True)
def _normalize_rel_path(p: str) -> str:
"""Pfad relativ halten & normalisieren (slashes, führende / entfernen)."""
p = (p or "").replace("\\", "/")
return p.lstrip("/")
def _to_md(frontmatter: dict, body: str) -> str:
fm = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=True).strip()
return f"---\n{fm}\n---\n{(body or '').rstrip()}\n"
def _scroll_all(
client: QdrantClient,
col: str,
flt: Optional[rest.Filter] = None,
with_payload: bool = True,
with_vectors: bool = False,
limit: int = 256,
):
"""Scrollt durch alle Punkte einer Collection und liefert eine Liste mit Points."""
out = []
next_page = None
while True:
pts, next_page = client.scroll(
collection_name=col,
scroll_filter=flt,
with_payload=with_payload,
with_vectors=with_vectors,
limit=limit,
offset=next_page,
)
if not pts:
break
out.extend(pts)
if not next_page:
break
return out
def _load_chunks_for_note(client: QdrantClient, chunks_col: str, note_id: str) -> List[dict]:
flt = rest.Filter(must=[rest.FieldCondition(
key="note_id",
match=rest.MatchValue(value=note_id),
)])
pts = _scroll_all(client, chunks_col, flt, with_payload=True, with_vectors=False)
# Sortierung: bevorzugt seq → chunk_index → Nummer in id
def _seq(pl: dict) -> Tuple[int, int, int]:
s1 = pl.get("seq", pl.get("chunk_index", -1))
s2 = pl.get("chunk_index", -1)
# Nummer-Anteil aus "noteid#<n>"
s3 = 0
try:
m = re.search(r"#(\\d+)$", pl.get("id") or "")
if m:
s3 = int(m.group(1))
except Exception:
pass
return (int(s1) if isinstance(s1, int) else -1, int(s2) if isinstance(s2, int) else -1, s3)
pts_sorted = sorted(pts, key=lambda p: _seq(p.payload or {}))
return [p.payload or {} for p in pts_sorted]
def _reconstruct_body(note_pl: dict, chunk_payloads: List[dict]) -> str:
# 1) Volltext vorhanden?
fulltext = note_pl.get("fulltext")
if isinstance(fulltext, str) and fulltext.strip():
return fulltext
# 2) Aus Chunks zusammensetzen: text → content → raw
parts: List[str] = []
for ch in chunk_payloads:
text = ch.get("text") or ch.get("content") or ch.get("raw")
if isinstance(text, str) and text.strip():
parts.append(text.rstrip())
return ("\n\n".join(parts)).rstrip() + ("\n" if parts else "")
def _export_one_note(
client: QdrantClient,
prefix: str,
note_pl: dict,
out_root: str,
overwrite: bool,
) -> dict:
notes_col, chunks_col, _ = _names(prefix)
note_id = note_pl.get("note_id") or note_pl.get("id")
# Pfad robust bestimmen und relativ halten
path = note_pl.get("path") or f"{note_id}.md"
path = _normalize_rel_path(path)
out_path = os.path.join(out_root, path).replace("\\", "/")
# Frontmatter aus Payload zurückführen (nur bekannte Felder)
fm: Dict[str, object] = {}
for k in [
"title", "id", "type", "status", "created", "updated", "tags",
"priority", "effort_min", "due", "people", "aliases",
"depends_on", "assigned_to", "lang",
]:
v = note_pl.get(k) if k in note_pl else note_pl.get(f"note_{k}")
if v not in (None, [], ""):
fm[k] = v
# Mindestfelder
if "id" not in fm and note_id:
fm["id"] = note_id
if "title" not in fm and note_pl.get("title"):
fm["title"] = note_pl["title"]
# Body beschaffen
chunks = _load_chunks_for_note(client, chunks_col, note_id)
body = _reconstruct_body(note_pl, chunks)
# Schreiben?
if os.path.exists(out_path) and not overwrite:
return {"note_id": note_id, "path": path, "status": "skip_exists"}
_ensure_dir(out_path)
with open(out_path, "w", encoding="utf-8") as f:
f.write(_to_md(fm, body))
return {"note_id": note_id, "path": path, "status": "written"}
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--out", required=True, help="Zielordner für den Export-Vault")
ap.add_argument("--note-id", help="Nur eine Note exportieren (Note-ID)")
ap.add_argument("--overwrite", action="store_true", help="Bestehende Dateien überschreiben")
ap.add_argument("--prefix", help="(Optional) überschreibt COLLECTION_PREFIX aus ENV")
args = ap.parse_args()
# Qdrant-Konfiguration
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix # abwärtskompatibel: ENV bleibt Default
client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim)
notes_col, _, _ = _names(cfg.prefix)
# Notes holen (optional gefiltert)
flt = None
if args.note_id:
flt = rest.Filter(must=[rest.FieldCondition(
key="note_id",
match=rest.MatchValue(value=args.note_id),
)])
note_pts = _scroll_all(client, notes_col, flt, with_payload=True, with_vectors=False)
if not note_pts:
print(json.dumps({"exported": 0, "out": args.out, "message": "Keine Notes gefunden."}, ensure_ascii=False))
return
results = []
for p in note_pts:
pl = p.payload or {}
try:
res = _export_one_note(client, cfg.prefix, pl, args.out, args.overwrite)
except Exception as e:
res = {"note_id": pl.get("note_id") or pl.get("id"), "error": str(e)}
results.append(res)
print(json.dumps({
"exported": len([r for r in results if r.get("status") == "written"]),
"skipped": len([r for r in results if r.get("status") == "skip_exists"]),
"out": args.out,
"details": results,
}, ensure_ascii=False))
if __name__ == "__main__":
main()
+242
View File
@@ -0,0 +1,242 @@
#!/usr/bin/env python3
# scripts/gc_qdrant_after_vault_scan.py
# Version: 1.0.0 (2025-09-05)
#
# Zweck
# -----
# Garbage-Collector für Qdrant: löscht Einträge (Edges, Chunks, Notes), deren Note-ID
# nicht mehr im Obsidian-Vault vorhanden ist. So bleiben Vektorindex/Graph mit dem Vault konsistent.
#
# Aufrufparameter
# ---------------
# --vault PATH Pfad zum Obsidian-Vault (z. B. ./vault) [erforderlich]
# --mode {edges,content,all}
# edges -> nur Kanten mit source_id/target_id ∉ Vault-Notes löschen
# content -> Chunks & Notes löschen, deren note_id ∉ Vault-Notes (keine Edges)
# all -> zuerst edges, dann content
# --prefix PREFIX Collection-Präfix (Default: aus ENV COLLECTION_PREFIX oder "mindnet")
# --apply Ohne diesen Schalter: Dry-Run (nur Vorschau). Mit Schalter: ausführen.
# --yes Sicherheitsabfrage überspringen (non-interaktiv)
# --batch-size N Größe für In-Filter (Default: 1000)
#
# Hinweise
# --------
# - Benötigt funktionierendes Python-venv mit allen Abhängigkeiten von mindnet (qdrant-client usw.).
# - Läuft gegen die drei Collections {prefix}_notes, {prefix}_chunks, {prefix}_edges (keine anderen Projekte).
# - Nutzt OR-Filter (Filter.should) korrekt, keine minimum_should (Pydantic v2 lässt die Option nicht zu).
#
# Änderungen ggü. vorher
# ----------------------
# - Neu: eigenes GC-Tool; berührt ausschließlich das übergebene Projekt-Präfix.
# - Sichere Dry-Run-Vorschau mit Zählung pro Collection; interaktive Bestätigung.
# - Löscht Edges mittels Filter auf (source_id ∈ MISSING) ODER (target_id ∈ MISSING).
# - Löscht Chunks/Notes mittels Filter auf note_id ∈ MISSING.
#
# Beispiele
# ---------
# DRY RUN: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all
# APPLY: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all --apply
# CI-Modus: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode edges --apply --yes
#
# Kompatibilität / Kontext
# ------------------------
# - Frontmatter-Validierung & Parsing wie im Projekt (parser.read_markdown, validate_required_frontmatter). :contentReference[oaicite:0]{index=0}
# - Note-Payload-Schema (u. a. note_id, hash_fulltext) wie spezifiziert. :contentReference[oaicite:1]{index=1}
# - Collection-Namen & Anlage-Logik über app.core.qdrant (notes/chunks = dim, edges = 1D). :contentReference[oaicite:2]{index=2}
# - Edges nutzen Felder source_id/target_id; Chunks/Notes tragen note_id in payload. :contentReference[oaicite:3]{index=3} :contentReference[oaicite:4]{index=4}
#
from __future__ import annotations
import argparse, os, glob, json, sys
from typing import Iterable, Set, Tuple, List
from dotenv import load_dotenv
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter # :contentReference[oaicite:5]{index=5}
from app.core.qdrant import QdrantConfig, get_client, collection_names # :contentReference[oaicite:6]{index=6}
def iter_note_ids_from_vault(vault_root: str) -> Set[str]:
"""Liest alle Markdown-Dateien im Vault und sammelt valide Frontmatter-IDs."""
paths = [p for p in glob.glob(os.path.join(vault_root, "**", "*.md"), recursive=True)]
out: Set[str] = set()
for p in paths:
try:
parsed = read_markdown(p)
fm = normalize_frontmatter(parsed.frontmatter)
validate_required_frontmatter(fm)
nid = fm.get("id") or fm.get("note_id")
if nid:
out.add(nid)
except Exception:
# ungültige oder Template-Dateien ohne Pflichtfelder ignorieren
continue
return out
def scroll_payload_values(client: QdrantClient, collection: str, field: str, limit: int = 2000) -> Set[str]:
"""Scrollt eine Collection und sammelt alle Payload-Werte eines Feldes (falls vorhanden)."""
out: Set[str] = set()
offset = None
while True:
res = client.scroll(
collection_name=collection,
scroll_filter=None,
with_payload=True,
with_vectors=False,
limit=limit,
offset=offset
)
# qdrant-client (neuere Versionen) geben (points, next_offset)
if isinstance(res, tuple) and len(res) == 2:
points, offset = res
else:
# Fallback, falls sich API signifikant ändert
points, offset = res[0], res[1] if len(res) > 1 else None
for pt in points or []:
pl = getattr(pt, "payload", None) or {}
v = pl.get(field)
if isinstance(v, str):
out.add(v)
if not offset:
break
return out
def chunked(iterable: Iterable[str], n: int) -> Iterable[List[str]]:
buf: List[str] = []
for x in iterable:
buf.append(x)
if len(buf) >= n:
yield buf
buf = []
if buf:
yield buf
def count_matches(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
"""Grober Zähler über scroll (Qdrant Count-API ist optional je Version)."""
total = 0
offset = None
while True:
points, offset = client.scroll(
collection_name=collection,
scroll_filter=filter_,
with_payload=False,
with_vectors=False,
limit=2000,
offset=offset
)
total += len(points or [])
if not offset:
break
return total
def build_filters_for_missing(prefix: str, missing_note_ids: Set[str]) -> Tuple[rest.Filter, rest.Filter, rest.Filter]:
"""
Erzeugt je Collection passende Filter:
- Notes/Chunks: note_id ∈ missing
- Edges: (source_id ∈ missing) ODER (target_id ∈ missing)
"""
notes_col, chunks_col, edges_col = collection_names(prefix) # :contentReference[oaicite:7]{index=7}
# MatchAny statt MatchValue für große Mengen
any_missing = rest.MatchAny(any=list(missing_note_ids))
f_notes = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
f_edges = rest.Filter(should=[
rest.FieldCondition(key="source_id", match=any_missing),
rest.FieldCondition(key="target_id", match=any_missing),
])
return f_notes, f_chunks, f_edges
def preview(client: QdrantClient, prefix: str, missing: Set[str]) -> dict:
"""Erstellt eine Dry-Run-Vorschau (Anzahl Treffer pro Collection)."""
notes_col, chunks_col, edges_col = collection_names(prefix)
f_notes, f_chunks, f_edges = build_filters_for_missing(prefix, missing)
return {
"prefix": prefix,
"collections": {"notes": notes_col, "chunks": chunks_col, "edges": edges_col},
"missing_note_ids_count": len(missing),
"to_delete_counts": {
"edges": count_matches(client, edges_col, f_edges),
"chunks": count_matches(client, chunks_col, f_chunks),
"notes": count_matches(client, notes_col, f_notes),
}
}
def delete_with_filter(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
"""Löscht alle Punkte, die den Filter matchen. Gibt grobe Trefferzahl zurück."""
# erst zählen (für Feedback)
to_del = count_matches(client, collection, filter_)
if to_del == 0:
return 0
client.delete(
collection_name=collection,
points_selector=filter_,
wait=True
)
return to_del
def main():
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (z. B. ./vault)")
ap.add_argument("--mode", choices=["edges", "content", "all"], default="all",
help="Löschmodus (edges=nur Kanten, content=Chunks+Notes, all=beides)")
ap.add_argument("--prefix", help="Collection-Präfix (Default aus ENV COLLECTION_PREFIX)")
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter: Dry-Run")
ap.add_argument("--yes", action="store_true", help="Rückfrage unterdrücken")
ap.add_argument("--batch-size", type=int, default=1000, help="Batch-Größe für MatchAny")
args = ap.parse_args()
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix
client = get_client(cfg)
notes_col, chunks_col, edges_col = collection_names(cfg.prefix)
# 1) Vault lesen → Menge gültiger Note-IDs
vault_root = os.path.abspath(args.vault)
vault_ids = iter_note_ids_from_vault(vault_root)
# 2) Qdrant vorhandene Note-IDs einsammeln
existing_note_ids = scroll_payload_values(client, notes_col, "note_id")
# 3) Differenz ermitteln
missing = existing_note_ids - vault_ids
# 4) Vorschau
summary = preview(client, cfg.prefix, missing)
print(json.dumps({"mode": args.mode, "apply": args.apply, "summary": summary}, ensure_ascii=False, indent=2))
if not missing:
# nichts zu tun
return
if not args.apply:
# Dry-Run Ende
return
if not args.yes:
resp = input("Fortfahren und die oben gezeigten Objekte löschen? (yes/no): ").strip().lower()
if resp not in ("y", "yes"):
print("Abgebrochen.")
return
# 5) Löschen je Modus
f_notes, f_chunks, f_edges = build_filters_for_missing(cfg.prefix, missing)
report = {"deleted": {"edges": 0, "chunks": 0, "notes": 0}}
try:
if args.mode in ("edges", "all"):
report["deleted"]["edges"] = delete_with_filter(client, edges_col, f_edges)
if args.mode in ("content", "all"):
# Reihenfolge: erst chunks, dann notes
report["deleted"]["chunks"] = delete_with_filter(client, chunks_col, f_chunks)
report["deleted"]["notes"] = delete_with_filter(client, notes_col, f_notes)
finally:
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+195 -135
View File
@@ -1,115 +1,159 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: scripts/import_markdown.py
Version: v2.1.1 (2025-09-05)
Kurzbeschreibung:
Importiert Obsidian-Markdown-Notes in Qdrant (Notes/Chunks/Edges).
Leitet Wikilink-Edges (references/backlink/references_at) direkt beim Import
aus Volltext + echten Chunk-Texten ab. Keine Abhängigkeit zu edges.py (Legacy).
Script: import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
Version: 3.2.0
Datum: 2025-09-09
Aufruf (aus Projekt-Root, im venv):
python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id NOTE_ID] [--embed-note]
Kurzbeschreibung
----------------
Liest Markdown-Dateien aus einem Vault ein und schreibt Notes, Chunks und Edges
idempotent nach Qdrant. Change-Detection basiert standardmäßig auf dem **Body-Hash**.
Neu: Hash-Modus und Normalisierung sind auch per **CLI** steuerbar.
Parameter:
--vault Pfad zum Vault (z. B. ./vault)
--apply Führt Upserts in Qdrant aus (ohne Flag = Dry-Run mit JSON-Summaries)
--note-id Bearbeite nur eine konkrete Note-ID
--embed-note Optional: Note-Vektor (Volltext) zusätzlich einbetten
ENV / Qdrant
------------
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
- QDRANT_API_KEY (optional)
- COLLECTION_PREFIX (Default: mindnet)
- VECTOR_DIM (Default: 384)
- MINDNET_HASH_MODE: "body" (Default) | "frontmatter" | "body+frontmatter"
- MINDNET_HASH_NORMALIZE: "canonical" (Default) | "none"
Umgebungsvariablen (optional):
QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)
CLI (übersteuert ENV)
---------------------
--hash-mode body|frontmatter|body+frontmatter
--hash-normalize canonical|none
Exitcodes:
0 = OK, 2 = keine Markdown-Dateien gefunden
Wichtige Hinweise:
- Verwendet ausschließlich app.core.derive_edges für Wikilink-Kanten.
- Chunk-Texte werden an derive_wikilink_edges übergeben; nur so entstehen references_at.
- Edges-IDs sind deterministisch (UUIDv5 in qdrant_points), idempotent.
- Edges-Collection bleibt 1D-Dummy-Vektor (DOT), reine Payload-Nutzung.
Changelog:
v2.1.1: Entfernt Abhängigkeit auf app.core.edges; direkte Nutzung von derive_wikilink_edges;
sichert Übergabe der Chunk-Texte → references_at werden erzeugt.
v2.1.0: Note-Index über gesamten Vault; erste Fassung mit direkter Edge-Ableitung.
v2.0.x: Import Notes/Chunks/Embeddings, Edges via Backfill (separat).
Aufruf
------
python3 -m scripts.import_markdown --vault ./vault
python3 -m scripts.import_markdown --vault ./vault --apply
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
python3 -m scripts.import_markdown --vault ./vault --note-id 20250821-foo --apply
python3 -m scripts.import_markdown --vault ./vault --apply --embed-note
# Feingranulare Erkennung (jede Kleinigkeit im Body zählt):
python3 -m scripts.import_markdown --vault ./vault --hash-normalize none
"""
from __future__ import annotations
import argparse
import os
import glob
import json
import os
import sys
from typing import List, Dict
from typing import Dict, List, Optional, Tuple
from dotenv import load_dotenv
from qdrant_client.http import models as rest
# Core-Module (gemäß Projektstruktur)
from app.core.parser import (
read_markdown,
normalize_frontmatter,
validate_required_frontmatter,
extract_wikilinks,
)
from app.core.note_payload import make_note_payload
from app.core.validate_note import validate_note_payload
from app.core.chunker import assemble_chunks
from app.core.chunk_payload import make_chunk_payloads
from app.core.embed import embed_texts, embed_one
from app.core.qdrant import QdrantConfig, ensure_collections, get_client
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
from app.core.qdrant_points import (
points_for_chunks,
points_for_note,
points_for_edges,
upsert_batch,
)
# Nur Wikilink-Ableitung (keine Kollision mit edges.py):
from app.core.derive_edges import build_note_index, derive_wikilink_edges
try:
from app.core.embed import embed_texts, embed_one # optional
except Exception:
embed_texts = None
embed_one = None
def iter_md(root: str, exclude=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")) -> List[str]:
files = [p for p in glob.glob(os.path.join(root, "**", "*.md"), recursive=True)]
# -----------------------------------------------------------------------------
# Hilfsfunktionen
# -----------------------------------------------------------------------------
def iter_md(root: str) -> List[str]:
out: List[str] = []
for p in files:
pn = p.replace("\\", "/")
if any(ex in pn for ex in exclude):
continue
out.append(p)
return out
for dirpath, _, filenames in os.walk(root):
for fn in filenames:
if not fn.lower().endswith(".md"):
continue
p = os.path.join(dirpath, fn)
pn = p.replace("\\", "/")
if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
continue
out.append(p)
return sorted(out)
def collections(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def fetch_existing_note_hash(client, prefix: str, note_id: str) -> Optional[str]:
notes_col, _, _ = collections(prefix)
f = rest.Filter(must=[rest.FieldCondition(
key="note_id",
match=rest.MatchValue(value=note_id),
)])
points, _ = client.scroll(
collection_name=notes_col,
scroll_filter=f,
with_payload=True,
with_vectors=False,
limit=1,
)
if not points:
return None
pl = points[0].payload or {}
return pl.get("hash_fulltext")
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
_, chunks_col, edges_col = collections(prefix)
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
client.delete(collection_name=chunks_col, points_selector=f_chunks, wait=True)
should = [
rest.FieldCondition(key="source_id", match=rest.MatchText(text=f"{note_id}#")),
rest.FieldCondition(key="target_id", match=rest.MatchText(text=f"{note_id}#")),
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=note_id)),
rest.FieldCondition(key="target_id", match=rest.MatchValue(value=note_id)),
]
f_edges = rest.Filter(should=should)
client.delete(collection_name=edges_col, points_selector=f_edges, wait=True)
def _normalize_rel_path(abs_path: str, vault_root: str) -> str:
try:
rel = os.path.relpath(abs_path, vault_root)
except Exception:
rel = abs_path
return rel.replace("\\", "/").lstrip("/")
def minimal_note_index_payload(abs_path: str, vault_root: str) -> Dict:
"""Nur Felder, die der Resolver braucht (id/title/path)."""
parsed = read_markdown(abs_path)
fm = normalize_frontmatter(parsed.frontmatter)
validate_required_frontmatter(fm)
relpath = os.path.relpath(abs_path, vault_root).replace("\\", "/")
return {
"note_id": fm.get("id") or fm.get("note_id"),
"title": fm.get("title"),
"path": relpath,
}
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
def main():
def main() -> None:
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Obsidian Vault Pfad (z.B. mindnet/vault)")
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant (sonst Dry-Run)")
ap.add_argument("--note-id", help="Nur eine Note-ID verarbeiten")
ap.add_argument("--embed-note", action="store_true", help="Auch Note-Volltext einbetten (optional)")
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
ap.add_argument("--purge-before-upsert", action="store_true",
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
ap.add_argument("--force-replace", action="store_true",
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
# NEU: Hash-Steuerung per CLI
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter"], default=None)
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
args = ap.parse_args()
# Qdrant-Konfiguration (+ Collections sicherstellen)
cfg = QdrantConfig(
url=os.getenv("QDRANT_URL", "http://127.0.0.1:6333"),
api_key=os.getenv("QDRANT_API_KEY") or None,
prefix=os.getenv("COLLECTION_PREFIX", "mindnet"),
dim=int(os.getenv("VECTOR_DIM", "384")),
)
cfg = QdrantConfig.from_env()
client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim)
notes_col, chunks_col, edges_col = collections(cfg.prefix)
root = os.path.abspath(args.vault)
files = iter_md(root)
@@ -117,93 +161,109 @@ def main():
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
sys.exit(2)
# (1) Vorab-Lauf: Note-Index (robuste Auflösung via ID/Titel-Slug/File-Slug)
index_payloads: List[Dict] = []
for path in files:
try:
pl = minimal_note_index_payload(path, root)
if not pl.get("note_id"):
continue
if args.note_id and pl["note_id"] != args.note_id:
continue
index_payloads.append(pl)
except Exception:
# Einzeldefekte nicht fatal
continue
note_index = build_note_index(index_payloads)
# (2) Hauptlauf: Import + direkte Wikilink-Edge-Ableitung
total_notes = 0
processed = 0
for path in files:
parsed = read_markdown(path)
fm = normalize_frontmatter(parsed.frontmatter)
try:
validate_required_frontmatter(fm)
except Exception:
except Exception as e:
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"}))
continue
if args.note_id and fm.get("id") != args.note_id:
continue
total_notes += 1
processed += 1
# Note-Payload (vollständig für notes-Collection)
note_pl = make_note_payload(parsed, vault_root=root)
validate_note_payload(note_pl)
# Note-Payload (mit expliziten Hash-Parametern)
note_pl = make_note_payload(parsed, vault_root=root,
hash_mode=args.hash_mode, hash_normalize=args.hash_normalize)
if "fulltext" not in (note_pl or {}):
note_pl["fulltext"] = parsed.body or ""
if note_pl.get("path"):
note_pl["path"] = _normalize_rel_path(
os.path.join(root, note_pl["path"]) if not os.path.isabs(note_pl["path"]) else note_pl["path"], root
)
else:
note_pl["path"] = _normalize_rel_path(parsed.path, root)
# Chunks aus Body gemäß Chunking-Strategie
note_id = note_pl["note_id"]
# Change-Detection
new_hash = note_pl.get("hash_fulltext")
old_hash = None if args.force_replace else fetch_existing_note_hash(client, cfg.prefix, note_id)
changed = args.force_replace or (old_hash != new_hash)
# Chunks + Embeddings (Nullvektor-Fallback)
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
if embed_texts:
vecs = embed_texts([getattr(c, "text", "") for c in chunks])
else:
vecs = [[0.0] * cfg.dim for _ in chunks]
# Embeddings (Chunks)
texts = [ch.text for ch in chunks]
vectors = embed_texts(texts)
# Edges (leichtgewichtig direkt hier ableiten)
# belongs_to / prev/next aus Chunk-Nachbarschaften + Wikilinks als references
edges: List[Dict] = []
for ch in chunk_pls:
cid = ch["id"]
edges.append({"src_id": cid, "dst_id": note_id, "edge_type": "belongs_to", "scope": "chunk"})
nb = ch.get("neighbors") or {}
if nb.get("prev"):
edges.append({"src_id": nb["prev"], "dst_id": cid, "edge_type": "next", "scope": "chunk"})
edges.append({"src_id": cid, "dst_id": nb["prev"], "edge_type": "prev", "scope": "chunk"})
if nb.get("next"):
edges.append({"src_id": cid, "dst_id": nb["next"], "edge_type": "next", "scope": "chunk"})
edges.append({"src_id": nb["next"], "dst_id": cid, "edge_type": "prev", "scope": "chunk"})
for ref in (ch.get("references") or []):
tid = ref.get("target_id")
if tid:
edges.append({"src_id": cid, "dst_id": tid, "edge_type": "references", "scope": "chunk"})
for tid in (note_pl.get("references") or []):
edges.append({"src_id": note_id, "dst_id": tid, "edge_type": "references", "scope": "note"})
edges.append({"src_id": tid, "dst_id": note_id, "edge_type": "backlink", "scope": "note"})
# Dedupe
_uniq = {}
for e in edges:
_uniq[(e["src_id"], e["dst_id"], e["edge_type"], e.get("scope", ""))] = e
edges = list(_uniq.values())
# Optional: Note-Vektor (z. B. für Doc-Suche/Clustering)
note_vec = None
if args.embed_note:
note_vec = embed_one(parsed.body)
# Edges direkt aus realen Chunks + Note-Index
# ACHTUNG: derive_wikilink_edges erwartet Chunks mit Text → wir bauen eine
# minimalistische Liste {"chunk_id","text"} in gleicher Reihenfolge.
note_pl_for_edges = {
"note_id": fm["id"],
# Zusammenfassung
summary = {
"note_id": note_id,
"title": fm.get("title"),
"path": note_pl["path"],
"fulltext": parsed.body,
}
chunks_for_links = []
for i, pl in enumerate(chunk_pls):
cid = pl.get("chunk_id") or pl.get("id")
txt = chunks[i].text if i < len(chunks) else ""
chunks_for_links.append({"chunk_id": cid, "text": txt})
edges = derive_wikilink_edges(note_pl_for_edges, chunks_for_links, note_index)
# Dry-Run-Summary
print(json.dumps({
"note_id": fm["id"],
"title": fm["title"],
"chunks": len(chunk_pls),
"edges": len(edges),
"changed": changed,
"decision": ("apply" if args.apply and changed else
"apply-skip-unchanged" if args.apply and not changed else
"dry-run"),
"path": note_pl["path"],
}, ensure_ascii=False))
"hash_mode": args.hash_mode or os.environ.get("MINDNET_HASH_MODE", "body"),
"hash_normalize": args.hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical"),
"hash_old": old_hash,
"hash_new": new_hash,
}
print(json.dumps(summary, ensure_ascii=False))
if args.apply:
# Notes upsert (idempotent; UUIDv5)
notes_col, note_pts = points_for_note(cfg.prefix, note_pl, note_vec, cfg.dim)
upsert_batch(client, notes_col, note_pts)
if not args.apply:
continue
# Chunks upsert (idempotent)
chunks_col, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vectors)
upsert_batch(client, chunks_col, chunk_pts)
if changed and args.purge_before_upsert:
purge_note_artifacts(client, cfg.prefix, note_id)
# Edges upsert (idempotent; deterministisches edge_id)
edges_col, edge_pts = points_for_edges(cfg.prefix, edges)
upsert_batch(client, edges_col, edge_pts)
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
upsert_batch(client, notes_name, note_pts)
print(f"Done. Processed notes: {total_notes}")
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
upsert_batch(client, chunks_name, chunk_pts)
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
upsert_batch(client, edges_name, edge_pts)
print(f"Done. Processed notes: {processed}")
if __name__ == "__main__":
+223
View File
@@ -0,0 +1,223 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
make_test_vault.py — erzeugt einen minimalen, nachvollziehbaren Test-Vault
Version: 1.1 (2025-09-05)
Änderungen ggü. 1.0:
- created/updated werden als QUOTED ISO-Strings geschrieben, damit der YAML-Parser
sie nicht in datetime-Objekte konvertiert (Schema verlangt strings).
- Kommentare/Output gestrafft.
Zweck
- Kleiner Obsidian-ähnlicher Vault zum Durchspielen des Importers (Chunks/Edges).
- Szenarien: leere Links, spätere Anlage fehlender Noten, Chunk-Neuaufteilung.
Struktur (Default: ./test_vault)
- 40_concepts/concept-alpha.md
- 20_experiences/exp-one.md (verlinkt auf [[concept-alpha]] und [[missing-note]])
- 20_experiences/exp-two.md (verlinkt auf [[concept-alpha]])
- 30_projects/project-demo.md (verlinkt auf [[concept-alpha]] und [[exp-one]])
Voraussetzungen
- Keine. Rein Dateigenerierung.
Aufruf
python3 -m scripts.make_test_vault [--out ./test_vault] [--force]
Parameter
--out Zielverzeichnis (Standard: ./test_vault)
--force Bestehenden Ordner löschen und neu anlegen.
Hinweise für Tests
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
Kompatibel mit:
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
"""
from __future__ import annotations
import argparse
import os
import shutil
from datetime import datetime, timezone
def iso_now() -> str:
# ISO 8601 mit Offset; als String zurückgeben
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
def q(s: str) -> str:
# Sicheres Quoten für YAML-Frontmatter
# (einfaches doppelt-Quoten reicht hier, da wir keine eingebetteten """ verwenden)
return f"\"{s}\""
TEMPLATE_CONCEPT = """---
title: "Concept Alpha"
id: "concept-alpha"
type: "concept"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/concept","topic/test"]
---
## Beschreibung
Dies ist ein einfaches Konzept für Testzwecke.
## Mögliche Verbindungen
- [[exp-one]]
- [[project-demo]]
"""
TEMPLATE_EXP_ONE = """---
title: "Experience One"
id: "exp-one"
type: "experience"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/experience","topic/test"]
---
## Kontext
Diese Notiz verlinkt auf ein existierendes Konzept und auf eine noch **fehlende** Note.
## Beobachtung
- Bezug zu [[concept-alpha]]
- Offener Verweis auf [[missing-note]]
## Interpretation
Wenn später eine Datei für 'missing-note' angelegt wird, sollte der Importer die Kanten nachziehen.
## Mögliche Verbindungen
- [[concept-alpha]]
- [[missing-note]]
"""
TEMPLATE_EXP_TWO = """---
title: "Experience Two"
id: "exp-two"
type: "experience"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/experience","topic/test"]
---
## Kontext
Zweite Experience, die auf dasselbe Konzept verweist.
## Beobachtung
Verweis auf [[concept-alpha]].
## Mögliche Verbindungen
- [[concept-alpha]]
"""
TEMPLATE_PROJECT = """---
title: "Project Demo"
id: "project-demo"
type: "project"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/project","topic/test"]
---
## Scope
Ein kleines Demo-Projekt, das andere Notizen referenziert.
## Arbeitspakete
- AP-1: Zusammenhang erklären (siehe [[concept-alpha]])
- AP-2: Ereignis berücksichtigen (siehe [[exp-one]])
## Mögliche Verbindungen
- [[concept-alpha]]
- [[exp-one]]
"""
TEMPLATE_MISSING = """---
title: "Missing Note (jetzt angelegt)"
id: "missing-note"
type: "concept"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/concept","topic/test"]
---
## Kontext
Diese Notiz wurde **nachträglich** erstellt, um einen zuvor offenen Link zu schließen.
## Mögliche Verbindungen
- [[exp-one]]
"""
def write_file(path: str, content: str) -> None:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(content)
def build_vault(base: str, include_missing_note: bool) -> None:
created = q(iso_now())
updated = q(iso_now())
# 40_concepts
write_file(
os.path.join(base, "40_concepts", "concept-alpha.md"),
TEMPLATE_CONCEPT.format(created=created, updated=updated),
)
# 20_experiences
write_file(
os.path.join(base, "20_experiences", "exp-one.md"),
TEMPLATE_EXP_ONE.format(created=created, updated=updated),
)
write_file(
os.path.join(base, "20_experiences", "exp-two.md"),
TEMPLATE_EXP_TWO.format(created=created, updated=updated),
)
# 30_projects
write_file(
os.path.join(base, "30_projects", "project-demo.md"),
TEMPLATE_PROJECT.format(created=created, updated=updated),
)
# Optional: zunächst NICHT anlegen (Szenario „leerer Link“),
# später für den 2. Testlauf anlegen.
if include_missing_note:
write_file(
os.path.join(base, "40_concepts", "missing-note.md"),
TEMPLATE_MISSING.format(created=created, updated=updated),
)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="./test_vault", help="Zielverzeichnis für den Test-Vault")
ap.add_argument("--force", action="store_true", help="Falls vorhanden: Verzeichnis löschen und neu anlegen")
ap.add_argument("--with-missing", action="store_true",
help="Auch die 'missing-note' direkt anlegen (Standard: erst im 2. Lauf)")
args = ap.parse_args()
base = os.path.abspath(args.out)
if os.path.exists(base):
if args.force:
shutil.rmtree(base)
else:
print(f"Ziel existiert bereits: {base}\nNutze --force zum Überschreiben.")
return
build_vault(base, include_missing_note=args.with_missing)
print(f"Test-Vault erstellt unter: {base}")
print("Hinweis:")
print(" 1) Zuerst ohne missing-note anlegen (Standard), importieren, Edges prüfen.")
print(" 2) Dann erneut ausführen mit --force --with-missing (missing-note wird erzeugt),")
print(" und Import wiederholen → Edges/Backlinks sollten nachgezogen werden.")
if __name__ == "__main__":
main()
+232
View File
@@ -0,0 +1,232 @@
# scripts/prune_qdrant_vs_vault.py
# -----------------------------------------------------------------------------
# Name: prune_qdrant_vs_vault.py
# Version: 1.0.0 (2025-09-08)
# Zweck: Entfernt verwaiste Qdrant-Einträge (notes/chunks/edges), wenn
# die zugehörigen Markdown-Dateien im Vault nicht mehr existieren.
#
# Was es macht:
# - Liest alle note_id aus dem Vault (Frontmatter: id / note_id).
# - Liest alle note_id aus Qdrant (mindnet_notes).
# - Bildet die Differenz (nur in Qdrant vorhandene, im Vault fehlende).
# - Löscht für jede verwaiste note_id:
# * Notes-Point(s)
# * Alle Chunks der Note
# * Alle Edges, die auf diese Note referenzieren
# (source_id == note_id ODER target_id == note_id ODER
# source_note_id == note_id ODER target_note_id == note_id)
#
# Hinweise:
# - Kein globaler Delete. Nur betroffene note_id.
# - Dry-Run standardmäßig; tatsächliches Löschen erst mit --apply.
# - Interaktive Bestätigung (abschaltbar mit --yes).
#
# Aufruf:
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply --yes
#
# Voraussetzungen:
# - Ausführung im aktivierten venv empfohlen: source .venv/bin/activate
# - Qdrant läuft lokal (oder URL/API-Key in ENV), siehe app/core/qdrant.py
#
# Änderungen:
# - 1.0.0: Erster Release.
# -----------------------------------------------------------------------------
import argparse
import json
import os
import sys
from pathlib import Path
from typing import Dict, List, Set, Tuple
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client, collection_names
from app.core.parser import parse_markdown_file # nutzt euer bestehendes Parsing/Schema
def read_vault_note_ids(vault_dir: Path) -> Set[str]:
note_ids: Set[str] = set()
for p in vault_dir.rglob("*.md"):
try:
parsed = parse_markdown_file(str(p))
fm = parsed.frontmatter if hasattr(parsed, "frontmatter") else (parsed.get("frontmatter") or {})
nid = fm.get("id") or fm.get("note_id")
if nid:
note_ids.add(str(nid))
except Exception:
# still und leise weiter – wir wollen robust sein
continue
return note_ids
def qdrant_note_ids(client: QdrantClient, notes_col: str) -> Set[str]:
ids: Set[str] = set()
scroll_filter = None
offset = None
while True:
res = client.scroll(
collection_name=notes_col,
scroll_filter=scroll_filter,
offset=offset,
limit=256,
with_payload=True,
with_vectors=False,
)
points, next_offset = res
for pt in points:
pl = pt.payload or {}
nid = pl.get("note_id")
if nid:
ids.add(str(nid))
if next_offset is None:
break
offset = next_offset
return ids
def delete_by_filter(client: QdrantClient, collection: str, flt: rest.Filter) -> int:
# Sammel erst IDs via scroll (robuster Überblick), lösche dann über point_ids (batch)
to_delete: List[int] = []
offset = None
while True:
pts, next_offset = client.scroll(
collection_name=collection,
scroll_filter=flt,
offset=offset,
limit=256,
with_payload=False,
with_vectors=False,
)
for pt in pts:
to_delete.append(pt.id)
if next_offset is None:
break
offset = next_offset
if not to_delete:
return 0
client.delete(collection_name=collection, points_selector=rest.PointIdsList(points=to_delete), wait=True)
return len(to_delete)
def prune_for_note(
client: QdrantClient,
cols: Dict[str, str],
note_id: str,
dry_run: bool = True,
) -> Dict[str, int]:
stats = {"notes": 0, "chunks": 0, "edges": 0}
# Notes löschen (Filter auf payload.note_id)
f_notes = rest.Filter(
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
)
# Chunks löschen (Filter auf payload.note_id)
f_chunks = rest.Filter(
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
)
# Edges löschen: mehrere Teilmengen erfassen
edge_filters = [
rest.Filter(must=[rest.FieldCondition(key="source_id", match=rest.MatchValue(value=note_id))]),
rest.Filter(must=[rest.FieldCondition(key="target_id", match=rest.MatchValue(value=note_id))]),
rest.Filter(must=[rest.FieldCondition(key="source_note_id", match=rest.MatchValue(value=note_id))]),
rest.Filter(must=[rest.FieldCondition(key="target_note_id", match=rest.MatchValue(value=note_id))]),
]
if dry_run:
# Zähle nur, was *wäre* gelöscht worden
def count(flt: rest.Filter, col: str) -> int:
total = 0
offset = None
while True:
pts, next_offset = client.scroll(
collection_name=col,
scroll_filter=flt,
offset=offset,
limit=256,
with_payload=False,
with_vectors=False,
)
total += len(pts)
if next_offset is None:
break
offset = next_offset
return total
stats["notes"] = count(f_notes, cols["notes"])
stats["chunks"] = count(f_chunks, cols["chunks"])
stats["edges"] = sum(count(f, cols["edges"]) for f in edge_filters)
return stats
# tatsächliches Löschen
stats["notes"] = delete_by_filter(client, cols["notes"], f_notes)
stats["chunks"] = delete_by_filter(client, cols["chunks"], f_chunks)
e_deleted = 0
for f in edge_filters:
e_deleted += delete_by_filter(client, cols["edges"], f)
stats["edges"] = e_deleted
return stats
def main():
ap = argparse.ArgumentParser(description="Prune Qdrant data for notes missing in the Vault.")
ap.add_argument("--vault", required=True, help="Pfad zum Vault-Verzeichnis (Root).")
ap.add_argument("--prefix", default="mindnet", help="Collections-Präfix (Default: mindnet).")
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter wird nur ein Dry-Run gemacht.")
ap.add_argument("--yes", action="store_true", help="Ohne Nachfrage ausführen.")
args = ap.parse_args()
vault_root = Path(args.vault).resolve()
if not vault_root.exists():
print(f"Vault-Verzeichnis nicht gefunden: {vault_root}", file=sys.stderr)
sys.exit(1)
cfg = QdrantConfig()
client = get_client(cfg)
cols = collection_names(args.prefix)
vault_ids = read_vault_note_ids(vault_root)
qdrant_ids = qdrant_note_ids(client, cols["notes"])
orphans = sorted(qdrant_ids - vault_ids)
preview = {
"mode": "APPLY" if args.apply else "DRY-RUN",
"prefix": args.prefix,
"vault_root": str(vault_root),
"collections": cols,
"counts": {
"vault_note_ids": len(vault_ids),
"qdrant_note_ids": len(qdrant_ids),
"orphans": len(orphans),
},
"orphans_sample": orphans[:20],
}
print(json.dumps(preview, ensure_ascii=False, indent=2))
if not orphans:
print("Keine verwaisten Einträge gefunden. Nichts zu tun.")
return
if not args.yes:
resp = input("\nFortfahren mit dem oben angezeigten Modus? (yes/no): ").strip().lower()
if resp not in ("y", "yes", "j", "ja"):
print("Abgebrochen.")
return
total_stats = {"notes": 0, "chunks": 0, "edges": 0}
for nid in orphans:
s = prune_for_note(client, cols, nid, dry_run=(not args.apply))
total_stats["notes"] += s["notes"]
total_stats["chunks"] += s["chunks"]
total_stats["edges"] += s["edges"]
print(json.dumps({"deleted": total_stats}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python3
# scripts/resolve_unresolved_references.py
"""
resolve_unresolved_references.py — Unaufgelöste Wikilinks in Qdrant nachträglich auflösen
Version: 1.0.0 (2025-09-05)
Zweck
------
- Findet Edges in {prefix}_edges mit payload.status=="unresolved" und versucht, den Zielknoten
anhand bereits vorhandener Notes in {prefix}_notes aufzulösen.
- Aktualisiert die Edges (setzt target_id, entfernt status, setzt resolution), und erzeugt
– NUR für Note-Level 'references' – die symmetrische 'backlink'-Kante.
Warum?
------
- Beim ersten Import können Links auf (noch) nicht existierende Notizen zeigen.
- Sobald die Zielnotiz später existiert, kann dieses Skript die Kanten reparieren.
Aufruf
------
# Dry-Run (Standard):
python3 -m scripts.resolve_unresolved_references --prefix mindnet
# Anwenden:
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
# Optional: nur X Edges anfassen
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply --limit 500
Parameter
---------
--prefix : Collection-Prefix (Default: aus Env COLLECION_PREFIX oder "mindnet")
--apply : Änderungen tatsächlich schreiben (ohne --apply = Dry-Run)
--limit : Max. Anzahl unaufgelöster Edges, die in diesem Lauf bearbeitet werden (Default: keine Begrenzung)
--batch : Upsert-Batchgröße (Default: 512)
Voraussetzungen / Hinweise
--------------------------
- Bitte im aktivierten venv laufen lassen (deine Umgebung: `.venv`).
- Qdrant-URL/Key/Prefix/Vektor-Dim werden wie üblich aus ENV gelesen (sieh app/core/qdrant.py). # noqa
- Nutzt die vorhandenen Utilities:
- app/core/qdrant.py (Client/Collections)
- app/core/qdrant_points.py (points_for_edges/upsert_batch)
- app/core/derive_edges.py (build_note_index/resolve_target)
Änderungshistorie
-----------------
1.0.0 Erstveröffentlichung.
"""
from __future__ import annotations
import argparse
import json
from typing import Any, Dict, List, Tuple, Iterable
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names # :contentReference[oaicite:3]{index=3}
from app.core.qdrant_points import points_for_edges, upsert_batch # :contentReference[oaicite:4]{index=4}
from app.core.derive_edges import build_note_index, resolve_target # :contentReference[oaicite:5]{index=5}
def _scroll(client: QdrantClient, **kwargs):
"""
Wrapper um qdrant_client.scroll() für unterschiedliche Client-Versionen:
neuere: (points, next_offset)
ältere: (points, next_page_offset, _)
"""
res = client.scroll(**kwargs)
if isinstance(res, tuple):
if len(res) == 2:
points, next_off = res
else:
# ältere Signatur: (points, next_off, _)
points, next_off, _ = res[0], res[1], res[2]
else:
# sehr alte Clients -> konservativ behandeln
points, next_off = res, None
return points, next_off
def _load_all_notes(client: QdrantClient, notes_col: str) -> List[Dict[str, Any]]:
notes: List[Dict[str, Any]] = []
next_off = None
while True:
pts, next_off = _scroll(
client,
collection_name=notes_col,
with_payload=True,
with_vectors=False,
limit=1024,
offset=next_off,
)
for p in pts or []:
pl = getattr(p, "payload", {}) or {}
# Erwartet Felder: note_id, title, path etc. (gemäß Schema) # :contentReference[oaicite:6]{index=6}
if pl.get("note_id"):
notes.append(pl)
if not next_off:
break
return notes
def _iter_unresolved_edges(client: QdrantClient, edges_col: str) -> Iterable[rest.Record]:
"""
Liefert alle Edge-Records mit payload.status == 'unresolved' und 'target_label' (string).
"""
f = rest.Filter(
must=[
rest.FieldCondition(key="status", match=rest.MatchValue(value="unresolved")),
]
)
next_off = None
while True:
pts, next_off = _scroll(
client,
collection_name=edges_col,
scroll_filter=f,
with_payload=True,
with_vectors=False,
limit=1024,
offset=next_off,
)
for p in pts or []:
pl = getattr(p, "payload", {}) or {}
if isinstance(pl.get("target_label"), str):
yield p
if not next_off:
break
def _make_backlink(source_note_id: str, target_note_id: str, extra: Dict[str, Any]) -> Dict[str, Any]:
"""
Baue eine 'backlink'-Edge-Payload source <- target (note-level).
"""
e = {
"kind": "backlink",
"source_id": target_note_id,
"target_id": source_note_id,
}
# Metafelder aus dem Original übernehmen (ohne status)
copy_keys = ["raw", "alias", "heading", "resolution"]
for k in copy_keys:
if k in extra:
e[k] = extra[k]
return e
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", help="Collection-Prefix (Default: Env/COLLECTION_PREFIX oder 'mindnet')")
ap.add_argument("--apply", action="store_true", help="Änderungen schreiben (ohne Flag = Dry-Run)")
ap.add_argument("--limit", type=int, default=0, help="Max. Anzahl unaufgelöster Edges bearbeiten (0 = alle)")
ap.add_argument("--batch", type=int, default=512, help="Upsert-Batchgröße")
args = ap.parse_args()
# Qdrant-Setup
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix
client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim) # sorgt u. a. für 1D-Vektor-Collection bei Edges :contentReference[oaicite:7]{index=7}
notes_col, _, edges_col = collection_names(cfg.prefix) # :contentReference[oaicite:8]{index=8}
# Notes laden & Index bauen
notes = _load_all_notes(client, notes_col)
idx = build_note_index(notes) # (by_id, by_slug, by_file_slug) :contentReference[oaicite:9]{index=9}
# Unresolved-Edges scannen
to_fix: List[dict] = []
backlinks: List[dict] = []
processed = 0
resolved = 0
for rec in _iter_unresolved_edges(client, edges_col):
if args.limit and processed >= args.limit:
break
processed += 1
pl = dict(rec.payload or {})
kind = pl.get("kind") or "references"
src = pl.get("source_id")
tgt_label = pl.get("target_label") or pl.get("target_id") # Fallback
# Zielauflösung
resolved_id, how = resolve_target(str(tgt_label), idx) # :contentReference[oaicite:10]{index=10}
if not resolved_id:
continue # weiterhin unresolved
# Edge-Update
new_pl = dict(pl)
new_pl["target_id"] = resolved_id
new_pl["resolution"] = how
if "status" in new_pl:
del new_pl["status"]
# ID stabil lassen -> points_for_edges erzeugt UUID aus edge_id/Fallback :contentReference[oaicite:11]{index=11}
if "edge_id" not in new_pl:
# stabiler Key aus (kind, src, tgt, evtl. seq)
seq = new_pl.get("seq") or new_pl.get("order") or ""
new_pl["edge_id"] = f"{kind}:{src}->{resolved_id}#{seq}"
to_fix.append(new_pl)
resolved += 1
# Nur bei Note-Level references (nicht references_at) -> Backlink erzeugen
if kind == "references":
extra = {k: new_pl.get(k) for k in ("raw", "alias", "heading")}
extra["resolution"] = how
backlinks.append(_make_backlink(source_note_id=src, target_note_id=resolved_id, extra=extra))
# Ergebnis ausgeben
summary = {
"prefix": cfg.prefix,
"scanned_unresolved": processed,
"resolved": resolved,
"backlinks_to_create": len(backlinks),
"apply": bool(args.apply),
}
print(json.dumps(summary, ensure_ascii=False))
if not args.apply:
return
# Upserts (in Batches)
def _batched(items: List[dict], n: int) -> Iterable[List[dict]]:
for i in range(0, len(items), n):
yield items[i : i + n]
# 1) Updates für reparierte Edges
for chunk in _batched(to_fix, args.batch):
col, pts = points_for_edges(cfg.prefix, chunk) # sorgt für Edge-UUID & Dummy-Vector :contentReference[oaicite:12]{index=12}
upsert_batch(client, col, pts)
# 2) Backlinks (nur references)
for chunk in _batched(backlinks, args.batch):
col, pts = points_for_edges(cfg.prefix, chunk)
upsert_batch(client, col, pts)
if __name__ == "__main__":
main()