Compare commits

..
Author SHA1 Message Date
Lars fb4bf79841 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:35:46 +02:00
Lars 1ee66450b1 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:04:42 +02:00
Lars 643a672045 app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:04:17 +02:00
Lars 38fd7420ff app/core/derive_edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:03:58 +02:00
Lars da2bbf72a6 app/core/derive_edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:47:09 +02:00
Lars 1f54f8136b app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:46:40 +02:00
Lars fd30e2c026 app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:27:28 +02:00
Lars 7f9981bf31 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:22:50 +02:00
Lars 4faf86fbbf app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:22:15 +02:00
Lars d9e30d5fb4 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 09:02:20 +02:00
Lars 41d43c2bb6 scripts/audit_vault_vs_qdrant.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 08:52:23 +02:00
Lars cb30dbb23c scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 07:54:33 +02:00
Lars 12dd67fbb5 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 07:47:37 +02:00
Lars 364502244a scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 07:41:28 +02:00
5 changed files with 446 additions and 212 deletions

No files matched your search

+41 -10
View File
@@ -1,4 +1,31 @@
# app/core/derive_edges.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: app/core/derive_edges.py
Version: v1.1.0 (2025-09-05)
Kurzbeschreibung
Leitet Edges aus Wikilinks ([[…]]) ab und löst Zielnoten robust auf.
Erzeugt:
- "references" (Note -> Note) mit seq="body", pro Match eine eigene Occurrence 'occ'
- "backlink" (inverse zu "references", gleiche seq/occ)
- "references_at" (Chunk -> Note) mit seq=<chunk_index> und eigener 'occ' je Match
Aufruf
from app.core.derive_edges import build_note_index, derive_wikilink_edges
Parameter / Felder
- note_payload: {"note_id","title","path","fulltext": <body> , …}
- chunks_payloads: [{"chunk_id","text",…}, …]
- note_index: build_note_index([...]) -> (by_id, by_slug, by_file_slug)
Kompatibilität
- Rückwärtskompatible Payload-Felder, nur erweitert um 'seq' und 'occ'.
Changelog
v1.1.0: Occurrence-Zählung ('occ') je Match; 'seq="body"' für references.
"""
from __future__ import annotations
import re
import unicodedata
@@ -54,42 +81,46 @@ def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_
edges: List[dict] = []
source_note_id = note_payload["note_id"]
def _make_edge(kind: str, src: str, tgt: str, seq=None, extra: dict|None=None):
def _make_edge(kind: str, src: str, tgt: str, seq=None, occ=None, extra: dict|None=None):
e = {"edge_id": None, "kind": kind, "source_id": src, "target_id": tgt}
if seq is not None:
e["seq"] = seq
if occ is not None:
e["occ"] = occ
if extra:
e.update(extra)
return e
# Links im Volltext (falls vorhanden)
# Volltext (Note-Ebene)
fulltext = note_payload.get("fulltext") or note_payload.get("body") or ""
if fulltext:
for m in WIKILINK_RE.finditer(fulltext):
for k, m in enumerate(WIKILINK_RE.finditer(fulltext), start=1):
raw_target, heading, alias = m.groups()
target_id, how = resolve_target(raw_target, note_index)
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
if target_id:
edges.append(_make_edge("references", source_note_id, target_id, extra=extra))
edges.append(_make_edge("backlink", target_id, source_note_id, extra=extra))
edges.append(_make_edge("references", source_note_id, target_id, seq="body", occ=k, extra=extra))
edges.append(_make_edge("backlink", target_id, source_note_id, seq="body", occ=k, extra=extra))
else:
extra["status"] = "unresolved"
extra["target_label"] = raw_target
edges.append(_make_edge("references", source_note_id, raw_target, extra=extra))
edges.append(_make_edge("references", source_note_id, raw_target, seq="body", occ=k, extra=extra))
# Links in Chunks (wenn übergeben)
# Chunks (Chunk-Ebene)
for i, ch in enumerate(chunks_payloads, start=1):
txt = ch.get("text") or ch.get("content") or ""
if not txt:
continue
occ = 0
for m in WIKILINK_RE.finditer(txt):
occ += 1
raw_target, heading, alias = m.groups()
target_id, how = resolve_target(raw_target, note_index)
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
if target_id:
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, extra=extra))
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, occ=occ, extra=extra))
else:
extra["status"] = "unresolved"
extra["target_label"] = raw_target
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, extra=extra))
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, occ=occ, extra=extra))
return edges
+89 -57
View File
@@ -1,9 +1,36 @@
# app/core/qdrant.py
from __future__ import annotations
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: app/core/qdrant.py
Version: v1.3.1 (2025-09-05)
Kurzbeschreibung:
Qdrant-Client & Collection-Setup für mindnet.
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
- Edges-Collection nutzt 1D Dummy-Vektor (Workaround für Python-Client).
- **Nicht-destruktiv per Default**: ensure_collections(..., destructive=False).
- **Abwärtskompatibel**: collection_names(prefix) wieder verfügbar.
Aufruf/Verwendung:
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names
Umgebungsvariablen (optional):
QDRANT_URL | QDRANT_HOST/QDRANT_PORT, QDRANT_API_KEY,
COLLECTION_PREFIX (Default "mindnet"), VECTOR_DIM (Default 384)
Änderungen:
v1.3.1: Helper collection_names(prefix) wiederhergestellt (für reset_qdrant usw.).
v1.3.0: ensure_collections(..., destructive=False) – keine stillen Drops im Dry-Run.
Edges-Collection nur bei explicit destructive=True neu anlegen.
≤v1.2.x: Konnte {prefix}_edges bei fehlender VectorConfig automatisch neu erstellen (riskant).
Bezug/Quelle:
Alte Core-Variante enthielt collection_names(prefix); diverse Scripts nutzen das weiterhin. :contentReference[oaicite:1]{index=1}
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Optional
from typing import Optional, Tuple
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
@@ -22,98 +49,103 @@ class QdrantConfig:
@staticmethod
def from_env() -> "QdrantConfig":
# URL (bevorzugt) oder Host/Port
url = os.getenv("QDRANT_URL")
if not url:
host = os.getenv("QDRANT_HOST", "127.0.0.1")
port = int(os.getenv("QDRANT_PORT", "6333"))
url = f"http://{host}:{port}"
api_key = os.getenv("QDRANT_API_KEY") or None
# Collection-Prefix und Vektor-Dimension
prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
dim = int(os.getenv("VECTOR_DIM", "384")) # MiniLM-384 by default
dim = int(os.getenv("VECTOR_DIM", "384"))
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
# -------------------------------
# Client / Setup
# Client
# -------------------------------
def get_client(cfg: QdrantConfig) -> QdrantClient:
"""
Erstellt einen QdrantClient basierend auf der Config.
"""
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
def ensure_collections(client: QdrantClient, prefix: str, dim: int) -> None:
# -------------------------------
# Collection-Erzeuger (Hilfsfunktionen)
# -------------------------------
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
if not client.collection_exists(name):
client.create_collection(
collection_name=name,
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
)
def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
if not client.collection_exists(name):
client.create_collection(
collection_name=name,
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
)
def _create_edges(client: QdrantClient, name: str) -> None:
if not client.collection_exists(name):
client.create_collection(
collection_name=name,
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT), # 1D-Dummy
)
# -------------------------------
# Public API
# -------------------------------
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
"""
Stellt sicher, dass die drei Collections existieren:
- {prefix}_notes : Vektor-Dim = dim (COSINE)
- {prefix}_chunks : Vektor-Dim = dim (COSINE)
- {prefix}_edges : Vektor-Dim = 1 (DOT) <-- Dummy-Vektor, damit der Python-Client kein 'vector' zwingt
Falls {prefix}_edges bereits vektorlos existiert, wird sie gelöscht und mit 1D neu erstellt.
Stellt sicher, dass die drei Collections existieren.
- Default **nicht destruktiv**: vorhandene Collections bleiben unangetastet.
- Nur wenn 'destructive=True', wird eine ungeeignete Edges-Collection gelöscht und neu angelegt.
"""
notes = f"{prefix}_notes"
chunks = f"{prefix}_chunks"
edges = f"{prefix}_edges"
# Notes
if not client.collection_exists(notes):
client.create_collection(
collection_name=notes,
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
)
_create_notes(client, notes, dim)
_create_chunks(client, chunks, dim)
# Chunks
if not client.collection_exists(chunks):
client.create_collection(
collection_name=chunks,
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
)
# Edges: 1D Dummy-Vektor (Workaround)
recreate_edges = False
if client.collection_exists(edges):
# Prüfen, ob die Edges-Collection bereits eine Vektorkonfig hat
try:
info = client.get_collection(edges)
# Prüfen, ob Vektor-Konfig existiert
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
# Neuere Clients: info.result.config.params.vectors kann VectorParams oder dict sein
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
if not has_vectors:
recreate_edges = True
except Exception:
# Wenn Metadaten nicht lesbar → sicherheitshalber neu anlegen
recreate_edges = True
# konservativ: nichts anfassen, um Datenverlust zu vermeiden
has_vectors = True
if not has_vectors:
if destructive:
client.delete_collection(edges)
_create_edges(client, edges)
else:
print(
f"[ensure_collections] WARN: '{edges}' ohne VectorConfig gefunden; "
f"keine destruktive Änderung (destructive=False).",
flush=True,
)
else:
# Existiert noch nicht → wird gleich erstellt
pass
if recreate_edges and client.collection_exists(edges):
client.delete_collection(edges)
if not client.collection_exists(edges):
client.create_collection(
collection_name=edges,
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT),
)
_create_edges(client, edges)
# -------------------------------
# (Optionale) Utility-Funktionen
# -------------------------------
def collection_names(prefix: str) -> tuple[str, str, str]:
"""Hilfsfunktion, falls du die Namen an einer Stelle brauchst."""
def collection_names(prefix: str) -> Tuple[str, str, str]:
"""
Abwärtskompatibler Helper für Scripts:
returns (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
"""
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
def wipe_collections(client: QdrantClient, prefix: str) -> None:
"""
Löscht alle drei Collections (nur nutzen, wenn du bewusst neu aufsetzen willst).
Löscht alle drei Collections – nur verwenden, wenn bewusst ein Clean-Rebuild gewünscht ist.
"""
for name in collection_names(prefix):
if client.collection_exists(name):
+28 -33
View File
@@ -1,45 +1,41 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: app/core/qdrant_points.py
Version: v1.2.0 (2025-09-05)
Kurzbeschreibung
Qdrant-Points für Notes/Chunks/Edges.
NEU: Edge-ID berücksichtigt optional 'occ' (Occurrence je Match):
"{kind}:{src}->{tgt}#{seq}@{occ}"
Aufruf
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
Changelog
v1.2.0: Edge-ID mit '@occ' falls vorhanden.
"""
from __future__ import annotations
import uuid
from typing import List, Tuple
from qdrant_client.http import models as rest
def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _to_uuid(stable_key: str) -> str:
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
def points_for_note(
prefix: str,
note_payload: dict,
note_vec: List[float] | None,
dim: int,
) -> Tuple[str, List[rest.PointStruct]]:
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
notes_col, _, _ = _names(prefix)
vector = note_vec if note_vec is not None else [0.0] * int(dim)
# Qdrant-Point-ID MUSS int/UUID sein
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
point_id = _to_uuid(raw_note_id)
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
return notes_col, [pt]
def points_for_chunks(
prefix: str,
chunk_payloads: List[dict],
vectors: List[List[float]],
) -> Tuple[str, List[rest.PointStruct]]:
"""
Chunks-Collection: erwartet pro Chunk einen Vektor.
Robustheit:
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
"""
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
_, chunks_col, _ = _names(prefix)
points: List[rest.PointStruct] = []
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
@@ -47,17 +43,17 @@ def points_for_chunks(
if not chunk_id:
note_id = pl.get("note_id") or pl.get("parent_note_id") or "missing-note"
chunk_id = f"{note_id}#{i}"
pl["chunk_id"] = chunk_id # persistenter Fallback in Payload
pl["chunk_id"] = chunk_id
point_id = _to_uuid(chunk_id)
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
return chunks_col, points
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
"""
Edges-Collection mit 1D-Dummy-Vektor.
- Fehlt 'edge_id', konstruieren wir eine stabile ID aus (kind, source_id, target_id, seq).
- vector=[0.0] erfüllt die Client-Validierung.
Edges-Collection mit 1D-Dummy-Vektor (keine Vektorsuche auf Edges).
Stable-ID:
Basis: "{kind}:{src}->{tgt}#{seq}"
Neu: Falls 'occ' vorhanden -> "{…}@{occ}"
"""
_, _, edges_col = _names(prefix)
points: List[rest.PointStruct] = []
@@ -68,15 +64,14 @@ def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[
s = pl.get("source_id", "unknown-src")
t = pl.get("target_id", "unknown-tgt")
seq = pl.get("seq") or pl.get("order") or ""
edge_id = f"{kind}:{s}->{t}#{seq}"
occ = pl.get("occ")
base = f"{kind}:{s}->{t}#{seq}"
edge_id = f"{base}@{occ}" if occ is not None else base
pl["edge_id"] = edge_id
point_id = _to_uuid(edge_id)
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
return edges_col, points
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
if not points:
return
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: scripts/audit_vault_vs_qdrant.py
Version: v1.0.0 (2025-09-05)
Kurzbeschreibung:
Prüft die Konsistenz zwischen Obsidian-Vault und Qdrant:
- Zählt Markdown-Dateien mit gültiger Frontmatter (title, id, type, status, created).
- Zählt Wikilink-Vorkommen im Vault (regex wie in derive_edges.py).
- Liest Zählungen aus Qdrant (Notes/Chunks/Edges je kind).
- Vergleicht erwartete Wikilink-Anzahl (Vault) vs. tatsächlich importierte Edges (Qdrant).
- Listet Auffälligkeiten pro Note (z. B. Wikilinks im Vault, aber keine references in Qdrant).
Aufruf (aus Projekt-Root, im venv):
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
Parameter:
--vault Pfad zum Vault (z. B. ./vault)
--prefix Collection-Prefix in Qdrant (Default: mindnet)
--limit Max. Punkte pro Scroll-Seite aus Qdrant (Default: 1000)
Voraussetzungen:
- Aktives Python venv mit installiertem qdrant-client.
- Zugriff auf Qdrant per ENV (QDRANT_URL, QDRANT_API_KEY optional).
Hinweise:
- Der Wikilink-Regex entspricht dem in app/core/derive_edges.py verwendeten Muster. (Quelle: derive_edges.py) # :contentReference[oaicite:3]{index=3}
- Pflicht-Frontmatter wird wie in app/core/parser.py geprüft. (Quelle: parser.py) # :contentReference[oaicite:4]{index=4}
- Collection-Namen & 1D-Edge-Vektoren folgen app/core/qdrant.py / qdrant_points.py. (Quellen: qdrant.py, qdrant_points.py) #
Changelog:
v1.0.0: Erste Version.
Autor:
mindnet – Datenimporte & Sync
"""
from __future__ import annotations
import argparse, os, glob, re, json
from collections import Counter, defaultdict
from typing import Dict, List, Tuple, Optional
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
# --- Regex wie in derive_edges.py (Wikilinks)
WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#([^\]|]+))?(?:\|([^\]]+))?\]\]") # :contentReference[oaicite:6]{index=6}
# --- Frontmatter-Prüfung wie in parser.validate_required_frontmatter (vereinfachte Replik) :contentReference[oaicite:7]{index=7}
REQUIRED = ("title","id","type","status","created")
def has_required_frontmatter(front: Dict) -> bool:
return all(k in front and front[k] not in (None, "") for k in REQUIRED)
def read_front_and_body(path: str) -> Tuple[Dict, str]:
# Minimal-Parser wie parser.read_markdown (kein YAML-Import hier, um Abh. zu vermeiden)
import yaml, unicodedata
with open(path, "r", encoding="utf-8") as f:
raw = f.read().lstrip("\ufeff")
raw = unicodedata.normalize("NFKC", raw).replace("\r\n", "\n").replace("\r", "\n")
if raw.startswith("---\n"):
end = raw.find("\n---", 4)
if end != -1:
fm_raw = raw[4:end].strip()
body = raw[end+4:].lstrip("\n")
try:
fm = yaml.safe_load(fm_raw) or {}
if not isinstance(fm, dict):
fm = {}
except Exception:
fm = {}
return fm, body
return {}, raw
def slug_file(path: str) -> str:
import unicodedata
s = os.path.basename(path)
if s.endswith(".md"): s = s[:-3]
s = unicodedata.normalize("NFKD", s)
s = "".join(ch for ch in s if not unicodedata.combining(ch))
s = s.lower().replace(" ", "-")
s = re.sub(r"[^a-z0-9\-]+", "", s)
s = re.sub(r"-{2,}", "-", s).strip("-")
return s
def collect_vault_stats(vault_root: str) -> Tuple[List[Dict], int]:
files = [p for p in glob.glob(os.path.join(vault_root, "**", "*.md"), recursive=True)]
notes: List[Dict] = []
wikilink_total = 0
for p in files:
pn = p.replace("\\","/")
if any(ex in pn for ex in ("/.obsidian/", "/_backup_frontmatter/", "/_imported/")):
continue
fm, body = read_front_and_body(p)
if not has_required_frontmatter(fm):
continue
nid = fm.get("id")
title = fm.get("title") or os.path.basename(p).rsplit(".",1)[0]
relpath = os.path.relpath(p, vault_root).replace("\\","/")
links = list(WIKILINK_RE.finditer(body))
wikilink_total += len(links)
notes.append({
"note_id": nid,
"title": title,
"path": relpath,
"wikilink_count": len(links),
"file_slug": slug_file(p),
})
return notes, wikilink_total
def qdrant_client_from_env() -> QdrantClient:
import os
url = os.getenv("QDRANT_URL", "http://127.0.0.1:6333")
api_key = os.getenv("QDRANT_API_KEY") or None
return QdrantClient(url=url, api_key=api_key)
def scroll_all(client: QdrantClient, collection: str, with_payload=True, limit=1000):
next_offset = None
while True:
pts, next_offset = client.scroll(collection_name=collection, with_payload=with_payload, with_vectors=False, limit=limit, offset=next_offset)
for p in pts:
yield p
if next_offset is None:
break
def collect_qdrant_stats(prefix: str, limit: int=1000) -> Dict:
client = qdrant_client_from_env()
cols = {
"notes": f"{prefix}_notes",
"chunks": f"{prefix}_chunks",
"edges": f"{prefix}_edges",
}
counts = {}
for k, c in cols.items():
n = 0
for _ in scroll_all(client, c, with_payload=(k!="chunks"), limit=limit):
n += 1
counts[k] = n
# Edge-Kinds & unresolved zählen
kinds = Counter()
unresolved = Counter()
per_note_refs = defaultdict(int)
for p in scroll_all(client, cols["edges"], with_payload=True, limit=limit):
pl = p.payload or {}
k = pl.get("kind")
if k: kinds[k] += 1
if pl.get("status") == "unresolved":
unresolved[k] += 1
# für per-Note-Vergleich: references (Volltext) zählen
if k == "references":
src = pl.get("source_id")
if src:
per_note_refs[src] += 1
return {"collections": cols, "counts": counts, "kinds": kinds, "unresolved": unresolved, "per_note_refs": dict(per_note_refs)}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Vault (z. B. ./vault)")
ap.add_argument("--prefix", default="mindnet", help="Qdrant Collection-Prefix")
ap.add_argument("--limit", type=int, default=1000, help="Scroll-Limit je Seite")
args = ap.parse_args()
notes, wikilink_total = collect_vault_stats(args.vault)
q = collect_qdrant_stats(args.prefix, args.limit)
notes_by_id = {n["note_id"]: n for n in notes}
# Abgleich pro Note: erwartete Wikilinks (Vault) vs. tatsächliche references (Qdrant)
deltas = []
for nid, n in notes_by_id.items():
expected = n["wikilink_count"]
actual = q["per_note_refs"].get(nid, 0)
if expected != actual:
deltas.append({
"note_id": nid,
"title": n["title"],
"path": n["path"],
"wikilinks_in_vault": expected,
"references_in_qdrant": actual,
"delta": actual - expected
})
out = {
"vault": {
"notes_with_required_frontmatter": len(notes),
"wikilink_occurrences_total": wikilink_total
},
"qdrant": {
"collections": q["collections"],
"counts": q["counts"],
"edge_kinds": q["kinds"],
"unresolved_by_kind": q["unresolved"]
},
"mismatch_notes": deltas[:50], # nur erste 50 ausgeben
"mismatch_total": len(deltas)
}
print(json.dumps(out, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+87 -112
View File
@@ -1,112 +1,95 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name: scripts/import_markdown.py
Version: v2.1.1 (2025-09-05)
Kurzbeschreibung:
Importiert Obsidian-Markdown-Notes in Qdrant (Notes/Chunks/Edges).
Leitet Wikilink-Edges (references/backlink/references_at) direkt beim Import
aus Volltext + echten Chunk-Texten ab. Keine Abhängigkeit zu edges.py (Legacy).
Script: scripts/import_markdown.py
Version: v2.3.1 (2025-09-05)
Aufruf (aus Projekt-Root, im venv):
python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id NOTE_ID] [--embed-note]
Beschreibung
Importiert Markdown-Notizen in Qdrant (Notes, Chunks, Edges).
- Chunking + Embedding (MiniLM 384d, externer Embed-Server)
- Edges direkt beim Import aus Wikilinks ([[…]]) ableiten
- Idempotente UUIDv5-IDs; Collections werden bereitgestellt
Parameter:
--vault Pfad zum Vault (z. B. ./vault)
--apply Führt Upserts in Qdrant aus (ohne Flag = Dry-Run mit JSON-Summaries)
--note-id Bearbeite nur eine konkrete Note-ID
--embed-note Optional: Note-Vektor (Volltext) zusätzlich einbetten
Aufruf
python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id ID] [--embed-note] [--force-replace]
Umgebungsvariablen (optional):
QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)
Parameter
--vault Pfad zum Obsidian-Vault (erforderlich)
--apply Ohne Flag: Dry-Run (nur JSON-Zeilen). Mit Flag: schreibt in Qdrant.
--note-id Nur eine spezifische Note-ID verarbeiten
--embed-note Optional: Note-Volltext zusätzlich einbetten
--force-replace Vor Upsert: zugehörige Edges der Quell-Note in Qdrant löschen (harte Ersetzung)
Exitcodes:
0 = OK, 2 = keine Markdown-Dateien gefunden
Hinweise
- Im venv laufen: `source .venv/bin/activate`
- Erwartet laufenden Embed-Server (http://127.0.0.1:8990)
- Qdrant via env: QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM
Wichtige Hinweise:
- Verwendet ausschließlich app.core.derive_edges für Wikilink-Kanten.
- Chunk-Texte werden an derive_wikilink_edges übergeben; nur so entstehen references_at.
- Edges-IDs sind deterministisch (UUIDv5 in qdrant_points), idempotent.
- Edges-Collection bleibt 1D-Dummy-Vektor (DOT), reine Payload-Nutzung.
Changelog:
v2.1.1: Entfernt Abhängigkeit auf app.core.edges; direkte Nutzung von derive_wikilink_edges;
sichert Übergabe der Chunk-Texte → references_at werden erzeugt.
v2.1.0: Note-Index über gesamten Vault; erste Fassung mit direkter Edge-Ableitung.
v2.0.x: Import Notes/Chunks/Embeddings, Edges via Backfill (separat).
Changelog
v2.3.1: FIX – Für derive_wikilink_edges werden jetzt echte Chunk-Texte übergeben
(chunks_for_links mit {"chunk_id","text"}), damit `references_at` erzeugt werden.
v2.3.0: Umstellung auf app.core.derive_edges; Edge-IDs unterstützen Occurrence.
v2.2.x: Fix Filter-API (pydantic v2) bei Purge; Kleinkosmetik.
"""
from __future__ import annotations
import argparse
import os
import glob
import json
import sys
from typing import List, Dict
from __future__ import annotations
import argparse, os, glob, json, sys
from dotenv import load_dotenv
# Core-Module (gemäß Projektstruktur)
from app.core.parser import (
read_markdown,
normalize_frontmatter,
validate_required_frontmatter,
)
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
from app.core.note_payload import make_note_payload
from app.core.validate_note import validate_note_payload
from app.core.chunker import assemble_chunks
from app.core.chunk_payload import make_chunk_payloads
from app.core.embed import embed_texts, embed_one
from app.core.qdrant import QdrantConfig, ensure_collections, get_client
from app.core.qdrant_points import (
points_for_chunks,
points_for_note,
points_for_edges,
upsert_batch,
)
# Nur Wikilink-Ableitung (keine Kollision mit edges.py):
from app.core.derive_edges import build_note_index, derive_wikilink_edges
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
from app.core.derive_edges import build_note_index, derive_wikilink_edges # nutzt 'text' je Chunk :contentReference[oaicite:1]{index=1}
from qdrant_client.http import models as rest
def iter_md(root: str, exclude=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")) -> List[str]:
def iter_md(root: str, exclude_dirs=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")):
files = [p for p in glob.glob(os.path.join(root, "**", "*.md"), recursive=True)]
out: List[str] = []
out = []
for p in files:
pn = p.replace("\\", "/")
if any(ex in pn for ex in exclude):
pn = p.replace("\\","/")
if any(ex in pn for ex in exclude_dirs):
continue
out.append(p)
return out
def minimal_note_index_payload(abs_path: str, vault_root: str) -> Dict:
"""Nur Felder, die der Resolver braucht (id/title/path)."""
parsed = read_markdown(abs_path)
fm = normalize_frontmatter(parsed.frontmatter)
validate_required_frontmatter(fm)
relpath = os.path.relpath(abs_path, vault_root).replace("\\", "/")
return {
"note_id": fm.get("id") or fm.get("note_id"),
"title": fm.get("title"),
"path": relpath,
}
def purge_note_edges(client, prefix: str, source_note_id: str):
# Löscht Edges, deren source_id == source_note_id ODER (kind==backlink && target_id==source_note_id)
edges_col = f"{prefix}_edges"
f = rest.Filter(
should=[
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_note_id)),
rest.Filter(
must=[
rest.FieldCondition(key="kind", match=rest.MatchValue(value="backlink")),
rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_note_id)),
]
),
]
)
# pydantic v2: client.delete(...) statt delete_points(...)
client.delete(collection_name=edges_col, points_selector=f, wait=True)
def main():
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Obsidian Vault Pfad (z.B. mindnet/vault)")
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian Vault (z.B. ./vault)")
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant (sonst Dry-Run)")
ap.add_argument("--note-id", help="Nur eine Note-ID verarbeiten")
ap.add_argument("--embed-note", action="store_true", help="Auch Note-Volltext einbetten (optional)")
ap.add_argument("--force-replace", action="store_true", help="Vor Upsert alte Edges der Note löschen")
args = ap.parse_args()
# Qdrant-Konfiguration (+ Collections sicherstellen)
# Qdrant
cfg = QdrantConfig(
url=os.getenv("QDRANT_URL", "http://127.0.0.1:6333"),
api_key=os.getenv("QDRANT_API_KEY") or None,
prefix=os.getenv("COLLECTION_PREFIX", "mindnet"),
dim=int(os.getenv("VECTOR_DIM", "384")),
dim=int(os.getenv("VECTOR_DIM","384")),
)
client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim)
@@ -114,26 +97,23 @@ def main():
root = os.path.abspath(args.vault)
files = iter_md(root)
if not files:
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
sys.exit(2)
print("Keine Markdown-Dateien gefunden.", file=sys.stderr); sys.exit(2)
# (1) Vorab-Lauf: Note-Index (robuste Auflösung via ID/Titel-Slug/File-Slug)
index_payloads: List[Dict] = []
# --- Note-Index (für robuste Zielauflösung) ---
note_stubs = []
for path in files:
parsed = read_markdown(path)
fm = normalize_frontmatter(parsed.frontmatter)
try:
pl = minimal_note_index_payload(path, root)
if not pl.get("note_id"):
continue
if args.note_id and pl["note_id"] != args.note_id:
continue
index_payloads.append(pl)
validate_required_frontmatter(fm)
except Exception:
# Einzeldefekte nicht fatal
continue
if args.note_id and fm.get("id") != args.note_id:
continue
rel = os.path.relpath(parsed.path, root).replace("\\","/")
note_stubs.append({"note_id": fm["id"], "title": fm.get("title",""), "path": rel})
note_index = build_note_index(note_stubs)
note_index = build_note_index(index_payloads)
# (2) Hauptlauf: Import + direkte Wikilink-Edge-Ableitung
total_notes = 0
for path in files:
parsed = read_markdown(path)
@@ -147,11 +127,12 @@ def main():
total_notes += 1
# Note-Payload (vollständig für notes-Collection)
# Note-Payload
note_pl = make_note_payload(parsed, vault_root=root)
validate_note_payload(note_pl)
note_pl["fulltext"] = parsed.body # für derive_edges
# Chunks aus Body gemäß Chunking-Strategie
# Chunks
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
@@ -159,52 +140,46 @@ def main():
texts = [ch.text for ch in chunks]
vectors = embed_texts(texts)
# Optional: Note-Vektor (z. B. für Doc-Suche/Clustering)
note_vec = None
if args.embed_note:
note_vec = embed_one(parsed.body)
# Optional: Note-Vektor
note_vec = embed_one(parsed.body) if args.embed_note else None
# Edges direkt aus realen Chunks + Note-Index
# ACHTUNG: derive_wikilink_edges erwartet Chunks mit Text → wir bauen eine
# minimalistische Liste {"chunk_id","text"} in gleicher Reihenfolge.
note_pl_for_edges = {
"note_id": fm["id"],
"title": fm.get("title"),
"path": note_pl["path"],
"fulltext": parsed.body,
}
chunks_for_links = []
for i, pl in enumerate(chunk_pls):
cid = pl.get("chunk_id") or pl.get("id")
txt = chunks[i].text if i < len(chunks) else ""
chunks_for_links.append({"chunk_id": cid, "text": txt})
# --- WICHTIG: Chunk-Texte für derive_wikilink_edges ---
chunks_for_links = [
{"chunk_id": (pl.get("chunk_id") or pl.get("id") or f"{fm['id']}#{i+1}"),
"text": chunks[i].text}
for i, pl in enumerate(chunk_pls)
if i < len(chunks)
]
edges = derive_wikilink_edges(note_pl_for_edges, chunks_for_links, note_index)
# Kanten (Note- & Chunk-Ebene)
edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index)
# Dry-Run-Summary
# Dry-Run-Ausgabe
print(json.dumps({
"note_id": fm["id"],
"title": fm["title"],
"chunks": len(chunk_pls),
"edges": len(edges),
"path": note_pl["path"],
"path": note_pl["path"]
}, ensure_ascii=False))
if args.apply:
# Notes upsert (idempotent; UUIDv5)
if args.force_replace:
purge_note_edges(client, cfg.prefix, fm["id"])
# Notes upsert
notes_col, note_pts = points_for_note(cfg.prefix, note_pl, note_vec, cfg.dim)
upsert_batch(client, notes_col, note_pts)
# Chunks upsert (idempotent)
# Chunks upsert
chunks_col, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vectors)
upsert_batch(client, chunks_col, chunk_pts)
# Edges upsert (idempotent; deterministisches edge_id)
# Edges upsert
edges_col, edge_pts = points_for_edges(cfg.prefix, edges)
upsert_batch(client, edges_col, edge_pts)
print(f"Done. Processed notes: {total_notes}")
if __name__ == "__main__":
main()