Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fb4bf79841 | ||
|
|
1ee66450b1 | ||
|
|
643a672045 | ||
|
|
38fd7420ff | ||
|
|
da2bbf72a6 | ||
|
|
1f54f8136b | ||
|
|
fd30e2c026 | ||
|
|
7f9981bf31 | ||
|
|
4faf86fbbf | ||
|
|
d9e30d5fb4 | ||
|
|
41d43c2bb6 | ||
|
|
cb30dbb23c | ||
|
|
12dd67fbb5 | ||
|
|
364502244a |
No files matched your search
+41
-10
@@ -1,4 +1,31 @@
|
||||
# app/core/derive_edges.py
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: app/core/derive_edges.py
|
||||
Version: v1.1.0 (2025-09-05)
|
||||
|
||||
Kurzbeschreibung
|
||||
Leitet Edges aus Wikilinks ([[…]]) ab und löst Zielnoten robust auf.
|
||||
Erzeugt:
|
||||
- "references" (Note -> Note) mit seq="body", pro Match eine eigene Occurrence 'occ'
|
||||
- "backlink" (inverse zu "references", gleiche seq/occ)
|
||||
- "references_at" (Chunk -> Note) mit seq=<chunk_index> und eigener 'occ' je Match
|
||||
|
||||
Aufruf
|
||||
from app.core.derive_edges import build_note_index, derive_wikilink_edges
|
||||
|
||||
Parameter / Felder
|
||||
- note_payload: {"note_id","title","path","fulltext": <body> , …}
|
||||
- chunks_payloads: [{"chunk_id","text",…}, …]
|
||||
- note_index: build_note_index([...]) -> (by_id, by_slug, by_file_slug)
|
||||
|
||||
Kompatibilität
|
||||
- Rückwärtskompatible Payload-Felder, nur erweitert um 'seq' und 'occ'.
|
||||
|
||||
Changelog
|
||||
v1.1.0: Occurrence-Zählung ('occ') je Match; 'seq="body"' für references.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import re
|
||||
import unicodedata
|
||||
@@ -54,42 +81,46 @@ def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_
|
||||
edges: List[dict] = []
|
||||
source_note_id = note_payload["note_id"]
|
||||
|
||||
def _make_edge(kind: str, src: str, tgt: str, seq=None, extra: dict|None=None):
|
||||
def _make_edge(kind: str, src: str, tgt: str, seq=None, occ=None, extra: dict|None=None):
|
||||
e = {"edge_id": None, "kind": kind, "source_id": src, "target_id": tgt}
|
||||
if seq is not None:
|
||||
e["seq"] = seq
|
||||
if occ is not None:
|
||||
e["occ"] = occ
|
||||
if extra:
|
||||
e.update(extra)
|
||||
return e
|
||||
|
||||
# Links im Volltext (falls vorhanden)
|
||||
# Volltext (Note-Ebene)
|
||||
fulltext = note_payload.get("fulltext") or note_payload.get("body") or ""
|
||||
if fulltext:
|
||||
for m in WIKILINK_RE.finditer(fulltext):
|
||||
for k, m in enumerate(WIKILINK_RE.finditer(fulltext), start=1):
|
||||
raw_target, heading, alias = m.groups()
|
||||
target_id, how = resolve_target(raw_target, note_index)
|
||||
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
|
||||
if target_id:
|
||||
edges.append(_make_edge("references", source_note_id, target_id, extra=extra))
|
||||
edges.append(_make_edge("backlink", target_id, source_note_id, extra=extra))
|
||||
edges.append(_make_edge("references", source_note_id, target_id, seq="body", occ=k, extra=extra))
|
||||
edges.append(_make_edge("backlink", target_id, source_note_id, seq="body", occ=k, extra=extra))
|
||||
else:
|
||||
extra["status"] = "unresolved"
|
||||
extra["target_label"] = raw_target
|
||||
edges.append(_make_edge("references", source_note_id, raw_target, extra=extra))
|
||||
edges.append(_make_edge("references", source_note_id, raw_target, seq="body", occ=k, extra=extra))
|
||||
|
||||
# Links in Chunks (wenn übergeben)
|
||||
# Chunks (Chunk-Ebene)
|
||||
for i, ch in enumerate(chunks_payloads, start=1):
|
||||
txt = ch.get("text") or ch.get("content") or ""
|
||||
if not txt:
|
||||
continue
|
||||
occ = 0
|
||||
for m in WIKILINK_RE.finditer(txt):
|
||||
occ += 1
|
||||
raw_target, heading, alias = m.groups()
|
||||
target_id, how = resolve_target(raw_target, note_index)
|
||||
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
|
||||
if target_id:
|
||||
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, extra=extra))
|
||||
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, occ=occ, extra=extra))
|
||||
else:
|
||||
extra["status"] = "unresolved"
|
||||
extra["target_label"] = raw_target
|
||||
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, extra=extra))
|
||||
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, occ=occ, extra=extra))
|
||||
return edges
|
||||
+89
-57
@@ -1,9 +1,36 @@
|
||||
# app/core/qdrant.py
|
||||
from __future__ import annotations
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: app/core/qdrant.py
|
||||
Version: v1.3.1 (2025-09-05)
|
||||
|
||||
Kurzbeschreibung:
|
||||
Qdrant-Client & Collection-Setup für mindnet.
|
||||
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
|
||||
- Edges-Collection nutzt 1D Dummy-Vektor (Workaround für Python-Client).
|
||||
- **Nicht-destruktiv per Default**: ensure_collections(..., destructive=False).
|
||||
- **Abwärtskompatibel**: collection_names(prefix) wieder verfügbar.
|
||||
|
||||
Aufruf/Verwendung:
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names
|
||||
|
||||
Umgebungsvariablen (optional):
|
||||
QDRANT_URL | QDRANT_HOST/QDRANT_PORT, QDRANT_API_KEY,
|
||||
COLLECTION_PREFIX (Default "mindnet"), VECTOR_DIM (Default 384)
|
||||
|
||||
Änderungen:
|
||||
v1.3.1: Helper collection_names(prefix) wiederhergestellt (für reset_qdrant usw.).
|
||||
v1.3.0: ensure_collections(..., destructive=False) – keine stillen Drops im Dry-Run.
|
||||
Edges-Collection nur bei explicit destructive=True neu anlegen.
|
||||
≤v1.2.x: Konnte {prefix}_edges bei fehlender VectorConfig automatisch neu erstellen (riskant).
|
||||
|
||||
Bezug/Quelle:
|
||||
Alte Core-Variante enthielt collection_names(prefix); diverse Scripts nutzen das weiterhin. :contentReference[oaicite:1]{index=1}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
@@ -22,98 +49,103 @@ class QdrantConfig:
|
||||
|
||||
@staticmethod
|
||||
def from_env() -> "QdrantConfig":
|
||||
# URL (bevorzugt) oder Host/Port
|
||||
url = os.getenv("QDRANT_URL")
|
||||
if not url:
|
||||
host = os.getenv("QDRANT_HOST", "127.0.0.1")
|
||||
port = int(os.getenv("QDRANT_PORT", "6333"))
|
||||
url = f"http://{host}:{port}"
|
||||
api_key = os.getenv("QDRANT_API_KEY") or None
|
||||
|
||||
# Collection-Prefix und Vektor-Dimension
|
||||
prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
|
||||
dim = int(os.getenv("VECTOR_DIM", "384")) # MiniLM-384 by default
|
||||
|
||||
dim = int(os.getenv("VECTOR_DIM", "384"))
|
||||
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
|
||||
|
||||
|
||||
# -------------------------------
|
||||
# Client / Setup
|
||||
# Client
|
||||
# -------------------------------
|
||||
|
||||
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
||||
"""
|
||||
Erstellt einen QdrantClient basierend auf der Config.
|
||||
"""
|
||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
|
||||
|
||||
|
||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int) -> None:
|
||||
# -------------------------------
|
||||
# Collection-Erzeuger (Hilfsfunktionen)
|
||||
# -------------------------------
|
||||
|
||||
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||
)
|
||||
|
||||
def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||
)
|
||||
|
||||
def _create_edges(client: QdrantClient, name: str) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT), # 1D-Dummy
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------
|
||||
# Public API
|
||||
# -------------------------------
|
||||
|
||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
|
||||
"""
|
||||
Stellt sicher, dass die drei Collections existieren:
|
||||
- {prefix}_notes : Vektor-Dim = dim (COSINE)
|
||||
- {prefix}_chunks : Vektor-Dim = dim (COSINE)
|
||||
- {prefix}_edges : Vektor-Dim = 1 (DOT) <-- Dummy-Vektor, damit der Python-Client kein 'vector' zwingt
|
||||
Falls {prefix}_edges bereits vektorlos existiert, wird sie gelöscht und mit 1D neu erstellt.
|
||||
Stellt sicher, dass die drei Collections existieren.
|
||||
- Default **nicht destruktiv**: vorhandene Collections bleiben unangetastet.
|
||||
- Nur wenn 'destructive=True', wird eine ungeeignete Edges-Collection gelöscht und neu angelegt.
|
||||
"""
|
||||
notes = f"{prefix}_notes"
|
||||
chunks = f"{prefix}_chunks"
|
||||
edges = f"{prefix}_edges"
|
||||
|
||||
# Notes
|
||||
if not client.collection_exists(notes):
|
||||
client.create_collection(
|
||||
collection_name=notes,
|
||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||
)
|
||||
_create_notes(client, notes, dim)
|
||||
_create_chunks(client, chunks, dim)
|
||||
|
||||
# Chunks
|
||||
if not client.collection_exists(chunks):
|
||||
client.create_collection(
|
||||
collection_name=chunks,
|
||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||
)
|
||||
|
||||
# Edges: 1D Dummy-Vektor (Workaround)
|
||||
recreate_edges = False
|
||||
if client.collection_exists(edges):
|
||||
# Prüfen, ob die Edges-Collection bereits eine Vektorkonfig hat
|
||||
try:
|
||||
info = client.get_collection(edges)
|
||||
# Prüfen, ob Vektor-Konfig existiert
|
||||
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
|
||||
# Neuere Clients: info.result.config.params.vectors kann VectorParams oder dict sein
|
||||
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
|
||||
if not has_vectors:
|
||||
recreate_edges = True
|
||||
except Exception:
|
||||
# Wenn Metadaten nicht lesbar → sicherheitshalber neu anlegen
|
||||
recreate_edges = True
|
||||
# konservativ: nichts anfassen, um Datenverlust zu vermeiden
|
||||
has_vectors = True
|
||||
|
||||
if not has_vectors:
|
||||
if destructive:
|
||||
client.delete_collection(edges)
|
||||
_create_edges(client, edges)
|
||||
else:
|
||||
print(
|
||||
f"[ensure_collections] WARN: '{edges}' ohne VectorConfig gefunden; "
|
||||
f"keine destruktive Änderung (destructive=False).",
|
||||
flush=True,
|
||||
)
|
||||
else:
|
||||
# Existiert noch nicht → wird gleich erstellt
|
||||
pass
|
||||
|
||||
if recreate_edges and client.collection_exists(edges):
|
||||
client.delete_collection(edges)
|
||||
|
||||
if not client.collection_exists(edges):
|
||||
client.create_collection(
|
||||
collection_name=edges,
|
||||
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT),
|
||||
)
|
||||
_create_edges(client, edges)
|
||||
|
||||
|
||||
# -------------------------------
|
||||
# (Optionale) Utility-Funktionen
|
||||
# -------------------------------
|
||||
|
||||
def collection_names(prefix: str) -> tuple[str, str, str]:
|
||||
"""Hilfsfunktion, falls du die Namen an einer Stelle brauchst."""
|
||||
def collection_names(prefix: str) -> Tuple[str, str, str]:
|
||||
"""
|
||||
Abwärtskompatibler Helper für Scripts:
|
||||
returns (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
||||
"""
|
||||
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
||||
|
||||
|
||||
def wipe_collections(client: QdrantClient, prefix: str) -> None:
|
||||
"""
|
||||
Löscht alle drei Collections (nur nutzen, wenn du bewusst neu aufsetzen willst).
|
||||
Löscht alle drei Collections – nur verwenden, wenn bewusst ein Clean-Rebuild gewünscht ist.
|
||||
"""
|
||||
for name in collection_names(prefix):
|
||||
if client.collection_exists(name):
|
||||
|
||||
+28
-33
@@ -1,45 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: app/core/qdrant_points.py
|
||||
Version: v1.2.0 (2025-09-05)
|
||||
|
||||
Kurzbeschreibung
|
||||
Qdrant-Points für Notes/Chunks/Edges.
|
||||
NEU: Edge-ID berücksichtigt optional 'occ' (Occurrence je Match):
|
||||
"{kind}:{src}->{tgt}#{seq}@{occ}"
|
||||
|
||||
Aufruf
|
||||
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
|
||||
|
||||
Changelog
|
||||
v1.2.0: Edge-ID mit '@occ' falls vorhanden.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from typing import List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
|
||||
def _to_uuid(stable_key: str) -> str:
|
||||
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||
|
||||
|
||||
def points_for_note(
|
||||
prefix: str,
|
||||
note_payload: dict,
|
||||
note_vec: List[float] | None,
|
||||
dim: int,
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
|
||||
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
|
||||
notes_col, _, _ = _names(prefix)
|
||||
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
||||
# Qdrant-Point-ID MUSS int/UUID sein
|
||||
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
||||
point_id = _to_uuid(raw_note_id)
|
||||
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
||||
return notes_col, [pt]
|
||||
|
||||
|
||||
def points_for_chunks(
|
||||
prefix: str,
|
||||
chunk_payloads: List[dict],
|
||||
vectors: List[List[float]],
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Chunks-Collection: erwartet pro Chunk einen Vektor.
|
||||
Robustheit:
|
||||
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
|
||||
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
|
||||
"""
|
||||
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
_, chunks_col, _ = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
||||
@@ -47,17 +43,17 @@ def points_for_chunks(
|
||||
if not chunk_id:
|
||||
note_id = pl.get("note_id") or pl.get("parent_note_id") or "missing-note"
|
||||
chunk_id = f"{note_id}#{i}"
|
||||
pl["chunk_id"] = chunk_id # persistenter Fallback in Payload
|
||||
pl["chunk_id"] = chunk_id
|
||||
point_id = _to_uuid(chunk_id)
|
||||
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
||||
return chunks_col, points
|
||||
|
||||
|
||||
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Edges-Collection mit 1D-Dummy-Vektor.
|
||||
- Fehlt 'edge_id', konstruieren wir eine stabile ID aus (kind, source_id, target_id, seq).
|
||||
- vector=[0.0] erfüllt die Client-Validierung.
|
||||
Edges-Collection mit 1D-Dummy-Vektor (keine Vektorsuche auf Edges).
|
||||
Stable-ID:
|
||||
Basis: "{kind}:{src}->{tgt}#{seq}"
|
||||
Neu: Falls 'occ' vorhanden -> "{…}@{occ}"
|
||||
"""
|
||||
_, _, edges_col = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
@@ -68,15 +64,14 @@ def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[
|
||||
s = pl.get("source_id", "unknown-src")
|
||||
t = pl.get("target_id", "unknown-tgt")
|
||||
seq = pl.get("seq") or pl.get("order") or ""
|
||||
edge_id = f"{kind}:{s}->{t}#{seq}"
|
||||
occ = pl.get("occ")
|
||||
base = f"{kind}:{s}->{t}#{seq}"
|
||||
edge_id = f"{base}@{occ}" if occ is not None else base
|
||||
pl["edge_id"] = edge_id
|
||||
point_id = _to_uuid(edge_id)
|
||||
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
||||
return edges_col, points
|
||||
|
||||
|
||||
|
||||
|
||||
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
|
||||
if not points:
|
||||
return
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: scripts/audit_vault_vs_qdrant.py
|
||||
Version: v1.0.0 (2025-09-05)
|
||||
Kurzbeschreibung:
|
||||
Prüft die Konsistenz zwischen Obsidian-Vault und Qdrant:
|
||||
- Zählt Markdown-Dateien mit gültiger Frontmatter (title, id, type, status, created).
|
||||
- Zählt Wikilink-Vorkommen im Vault (regex wie in derive_edges.py).
|
||||
- Liest Zählungen aus Qdrant (Notes/Chunks/Edges je kind).
|
||||
- Vergleicht erwartete Wikilink-Anzahl (Vault) vs. tatsächlich importierte Edges (Qdrant).
|
||||
- Listet Auffälligkeiten pro Note (z. B. Wikilinks im Vault, aber keine references in Qdrant).
|
||||
|
||||
Aufruf (aus Projekt-Root, im venv):
|
||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
||||
|
||||
Parameter:
|
||||
--vault Pfad zum Vault (z. B. ./vault)
|
||||
--prefix Collection-Prefix in Qdrant (Default: mindnet)
|
||||
--limit Max. Punkte pro Scroll-Seite aus Qdrant (Default: 1000)
|
||||
|
||||
Voraussetzungen:
|
||||
- Aktives Python venv mit installiertem qdrant-client.
|
||||
- Zugriff auf Qdrant per ENV (QDRANT_URL, QDRANT_API_KEY optional).
|
||||
|
||||
Hinweise:
|
||||
- Der Wikilink-Regex entspricht dem in app/core/derive_edges.py verwendeten Muster. (Quelle: derive_edges.py) # :contentReference[oaicite:3]{index=3}
|
||||
- Pflicht-Frontmatter wird wie in app/core/parser.py geprüft. (Quelle: parser.py) # :contentReference[oaicite:4]{index=4}
|
||||
- Collection-Namen & 1D-Edge-Vektoren folgen app/core/qdrant.py / qdrant_points.py. (Quellen: qdrant.py, qdrant_points.py) #
|
||||
|
||||
Changelog:
|
||||
v1.0.0: Erste Version.
|
||||
|
||||
Autor:
|
||||
mindnet – Datenimporte & Sync
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, os, glob, re, json
|
||||
from collections import Counter, defaultdict
|
||||
from typing import Dict, List, Tuple, Optional
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
# --- Regex wie in derive_edges.py (Wikilinks)
|
||||
WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#([^\]|]+))?(?:\|([^\]]+))?\]\]") # :contentReference[oaicite:6]{index=6}
|
||||
|
||||
# --- Frontmatter-Prüfung wie in parser.validate_required_frontmatter (vereinfachte Replik) :contentReference[oaicite:7]{index=7}
|
||||
REQUIRED = ("title","id","type","status","created")
|
||||
|
||||
def has_required_frontmatter(front: Dict) -> bool:
|
||||
return all(k in front and front[k] not in (None, "") for k in REQUIRED)
|
||||
|
||||
def read_front_and_body(path: str) -> Tuple[Dict, str]:
|
||||
# Minimal-Parser wie parser.read_markdown (kein YAML-Import hier, um Abh. zu vermeiden)
|
||||
import yaml, unicodedata
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
raw = f.read().lstrip("\ufeff")
|
||||
raw = unicodedata.normalize("NFKC", raw).replace("\r\n", "\n").replace("\r", "\n")
|
||||
if raw.startswith("---\n"):
|
||||
end = raw.find("\n---", 4)
|
||||
if end != -1:
|
||||
fm_raw = raw[4:end].strip()
|
||||
body = raw[end+4:].lstrip("\n")
|
||||
try:
|
||||
fm = yaml.safe_load(fm_raw) or {}
|
||||
if not isinstance(fm, dict):
|
||||
fm = {}
|
||||
except Exception:
|
||||
fm = {}
|
||||
return fm, body
|
||||
return {}, raw
|
||||
|
||||
def slug_file(path: str) -> str:
|
||||
import unicodedata
|
||||
s = os.path.basename(path)
|
||||
if s.endswith(".md"): s = s[:-3]
|
||||
s = unicodedata.normalize("NFKD", s)
|
||||
s = "".join(ch for ch in s if not unicodedata.combining(ch))
|
||||
s = s.lower().replace(" ", "-")
|
||||
s = re.sub(r"[^a-z0-9\-]+", "", s)
|
||||
s = re.sub(r"-{2,}", "-", s).strip("-")
|
||||
return s
|
||||
|
||||
def collect_vault_stats(vault_root: str) -> Tuple[List[Dict], int]:
|
||||
files = [p for p in glob.glob(os.path.join(vault_root, "**", "*.md"), recursive=True)]
|
||||
notes: List[Dict] = []
|
||||
wikilink_total = 0
|
||||
for p in files:
|
||||
pn = p.replace("\\","/")
|
||||
if any(ex in pn for ex in ("/.obsidian/", "/_backup_frontmatter/", "/_imported/")):
|
||||
continue
|
||||
fm, body = read_front_and_body(p)
|
||||
if not has_required_frontmatter(fm):
|
||||
continue
|
||||
nid = fm.get("id")
|
||||
title = fm.get("title") or os.path.basename(p).rsplit(".",1)[0]
|
||||
relpath = os.path.relpath(p, vault_root).replace("\\","/")
|
||||
links = list(WIKILINK_RE.finditer(body))
|
||||
wikilink_total += len(links)
|
||||
notes.append({
|
||||
"note_id": nid,
|
||||
"title": title,
|
||||
"path": relpath,
|
||||
"wikilink_count": len(links),
|
||||
"file_slug": slug_file(p),
|
||||
})
|
||||
return notes, wikilink_total
|
||||
|
||||
def qdrant_client_from_env() -> QdrantClient:
|
||||
import os
|
||||
url = os.getenv("QDRANT_URL", "http://127.0.0.1:6333")
|
||||
api_key = os.getenv("QDRANT_API_KEY") or None
|
||||
return QdrantClient(url=url, api_key=api_key)
|
||||
|
||||
def scroll_all(client: QdrantClient, collection: str, with_payload=True, limit=1000):
|
||||
next_offset = None
|
||||
while True:
|
||||
pts, next_offset = client.scroll(collection_name=collection, with_payload=with_payload, with_vectors=False, limit=limit, offset=next_offset)
|
||||
for p in pts:
|
||||
yield p
|
||||
if next_offset is None:
|
||||
break
|
||||
|
||||
def collect_qdrant_stats(prefix: str, limit: int=1000) -> Dict:
|
||||
client = qdrant_client_from_env()
|
||||
cols = {
|
||||
"notes": f"{prefix}_notes",
|
||||
"chunks": f"{prefix}_chunks",
|
||||
"edges": f"{prefix}_edges",
|
||||
}
|
||||
counts = {}
|
||||
for k, c in cols.items():
|
||||
n = 0
|
||||
for _ in scroll_all(client, c, with_payload=(k!="chunks"), limit=limit):
|
||||
n += 1
|
||||
counts[k] = n
|
||||
|
||||
# Edge-Kinds & unresolved zählen
|
||||
kinds = Counter()
|
||||
unresolved = Counter()
|
||||
per_note_refs = defaultdict(int)
|
||||
for p in scroll_all(client, cols["edges"], with_payload=True, limit=limit):
|
||||
pl = p.payload or {}
|
||||
k = pl.get("kind")
|
||||
if k: kinds[k] += 1
|
||||
if pl.get("status") == "unresolved":
|
||||
unresolved[k] += 1
|
||||
# für per-Note-Vergleich: references (Volltext) zählen
|
||||
if k == "references":
|
||||
src = pl.get("source_id")
|
||||
if src:
|
||||
per_note_refs[src] += 1
|
||||
|
||||
return {"collections": cols, "counts": counts, "kinds": kinds, "unresolved": unresolved, "per_note_refs": dict(per_note_refs)}
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Vault (z. B. ./vault)")
|
||||
ap.add_argument("--prefix", default="mindnet", help="Qdrant Collection-Prefix")
|
||||
ap.add_argument("--limit", type=int, default=1000, help="Scroll-Limit je Seite")
|
||||
args = ap.parse_args()
|
||||
|
||||
notes, wikilink_total = collect_vault_stats(args.vault)
|
||||
q = collect_qdrant_stats(args.prefix, args.limit)
|
||||
|
||||
notes_by_id = {n["note_id"]: n for n in notes}
|
||||
|
||||
# Abgleich pro Note: erwartete Wikilinks (Vault) vs. tatsächliche references (Qdrant)
|
||||
deltas = []
|
||||
for nid, n in notes_by_id.items():
|
||||
expected = n["wikilink_count"]
|
||||
actual = q["per_note_refs"].get(nid, 0)
|
||||
if expected != actual:
|
||||
deltas.append({
|
||||
"note_id": nid,
|
||||
"title": n["title"],
|
||||
"path": n["path"],
|
||||
"wikilinks_in_vault": expected,
|
||||
"references_in_qdrant": actual,
|
||||
"delta": actual - expected
|
||||
})
|
||||
|
||||
out = {
|
||||
"vault": {
|
||||
"notes_with_required_frontmatter": len(notes),
|
||||
"wikilink_occurrences_total": wikilink_total
|
||||
},
|
||||
"qdrant": {
|
||||
"collections": q["collections"],
|
||||
"counts": q["counts"],
|
||||
"edge_kinds": q["kinds"],
|
||||
"unresolved_by_kind": q["unresolved"]
|
||||
},
|
||||
"mismatch_notes": deltas[:50], # nur erste 50 ausgeben
|
||||
"mismatch_total": len(deltas)
|
||||
}
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+87
-112
@@ -1,112 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: scripts/import_markdown.py
|
||||
Version: v2.1.1 (2025-09-05)
|
||||
Kurzbeschreibung:
|
||||
Importiert Obsidian-Markdown-Notes in Qdrant (Notes/Chunks/Edges).
|
||||
Leitet Wikilink-Edges (references/backlink/references_at) direkt beim Import
|
||||
aus Volltext + echten Chunk-Texten ab. Keine Abhängigkeit zu edges.py (Legacy).
|
||||
Script: scripts/import_markdown.py
|
||||
Version: v2.3.1 (2025-09-05)
|
||||
|
||||
Aufruf (aus Projekt-Root, im venv):
|
||||
python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id NOTE_ID] [--embed-note]
|
||||
Beschreibung
|
||||
Importiert Markdown-Notizen in Qdrant (Notes, Chunks, Edges).
|
||||
- Chunking + Embedding (MiniLM 384d, externer Embed-Server)
|
||||
- Edges direkt beim Import aus Wikilinks ([[…]]) ableiten
|
||||
- Idempotente UUIDv5-IDs; Collections werden bereitgestellt
|
||||
|
||||
Parameter:
|
||||
--vault Pfad zum Vault (z. B. ./vault)
|
||||
--apply Führt Upserts in Qdrant aus (ohne Flag = Dry-Run mit JSON-Summaries)
|
||||
--note-id Bearbeite nur eine konkrete Note-ID
|
||||
--embed-note Optional: Note-Vektor (Volltext) zusätzlich einbetten
|
||||
Aufruf
|
||||
python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id ID] [--embed-note] [--force-replace]
|
||||
|
||||
Umgebungsvariablen (optional):
|
||||
QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)
|
||||
Parameter
|
||||
--vault Pfad zum Obsidian-Vault (erforderlich)
|
||||
--apply Ohne Flag: Dry-Run (nur JSON-Zeilen). Mit Flag: schreibt in Qdrant.
|
||||
--note-id Nur eine spezifische Note-ID verarbeiten
|
||||
--embed-note Optional: Note-Volltext zusätzlich einbetten
|
||||
--force-replace Vor Upsert: zugehörige Edges der Quell-Note in Qdrant löschen (harte Ersetzung)
|
||||
|
||||
Exitcodes:
|
||||
0 = OK, 2 = keine Markdown-Dateien gefunden
|
||||
Hinweise
|
||||
- Im venv laufen: `source .venv/bin/activate`
|
||||
- Erwartet laufenden Embed-Server (http://127.0.0.1:8990)
|
||||
- Qdrant via env: QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM
|
||||
|
||||
Wichtige Hinweise:
|
||||
- Verwendet ausschließlich app.core.derive_edges für Wikilink-Kanten.
|
||||
- Chunk-Texte werden an derive_wikilink_edges übergeben; nur so entstehen references_at.
|
||||
- Edges-IDs sind deterministisch (UUIDv5 in qdrant_points), idempotent.
|
||||
- Edges-Collection bleibt 1D-Dummy-Vektor (DOT), reine Payload-Nutzung.
|
||||
|
||||
Changelog:
|
||||
v2.1.1: Entfernt Abhängigkeit auf app.core.edges; direkte Nutzung von derive_wikilink_edges;
|
||||
sichert Übergabe der Chunk-Texte → references_at werden erzeugt.
|
||||
v2.1.0: Note-Index über gesamten Vault; erste Fassung mit direkter Edge-Ableitung.
|
||||
v2.0.x: Import Notes/Chunks/Embeddings, Edges via Backfill (separat).
|
||||
Changelog
|
||||
v2.3.1: FIX – Für derive_wikilink_edges werden jetzt echte Chunk-Texte übergeben
|
||||
(chunks_for_links mit {"chunk_id","text"}), damit `references_at` erzeugt werden.
|
||||
v2.3.0: Umstellung auf app.core.derive_edges; Edge-IDs unterstützen Occurrence.
|
||||
v2.2.x: Fix Filter-API (pydantic v2) bei Purge; Kleinkosmetik.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import os
|
||||
import glob
|
||||
import json
|
||||
import sys
|
||||
from typing import List, Dict
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse, os, glob, json, sys
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Core-Module (gemäß Projektstruktur)
|
||||
from app.core.parser import (
|
||||
read_markdown,
|
||||
normalize_frontmatter,
|
||||
validate_required_frontmatter,
|
||||
)
|
||||
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.validate_note import validate_note_payload
|
||||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunk_payload import make_chunk_payloads
|
||||
from app.core.embed import embed_texts, embed_one
|
||||
from app.core.qdrant import QdrantConfig, ensure_collections, get_client
|
||||
from app.core.qdrant_points import (
|
||||
points_for_chunks,
|
||||
points_for_note,
|
||||
points_for_edges,
|
||||
upsert_batch,
|
||||
)
|
||||
# Nur Wikilink-Ableitung (keine Kollision mit edges.py):
|
||||
from app.core.derive_edges import build_note_index, derive_wikilink_edges
|
||||
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
|
||||
from app.core.derive_edges import build_note_index, derive_wikilink_edges # nutzt 'text' je Chunk :contentReference[oaicite:1]{index=1}
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
|
||||
def iter_md(root: str, exclude=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")) -> List[str]:
|
||||
def iter_md(root: str, exclude_dirs=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")):
|
||||
files = [p for p in glob.glob(os.path.join(root, "**", "*.md"), recursive=True)]
|
||||
out: List[str] = []
|
||||
out = []
|
||||
for p in files:
|
||||
pn = p.replace("\\", "/")
|
||||
if any(ex in pn for ex in exclude):
|
||||
pn = p.replace("\\","/")
|
||||
if any(ex in pn for ex in exclude_dirs):
|
||||
continue
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
|
||||
def minimal_note_index_payload(abs_path: str, vault_root: str) -> Dict:
|
||||
"""Nur Felder, die der Resolver braucht (id/title/path)."""
|
||||
parsed = read_markdown(abs_path)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
relpath = os.path.relpath(abs_path, vault_root).replace("\\", "/")
|
||||
return {
|
||||
"note_id": fm.get("id") or fm.get("note_id"),
|
||||
"title": fm.get("title"),
|
||||
"path": relpath,
|
||||
}
|
||||
|
||||
def purge_note_edges(client, prefix: str, source_note_id: str):
|
||||
# Löscht Edges, deren source_id == source_note_id ODER (kind==backlink && target_id==source_note_id)
|
||||
edges_col = f"{prefix}_edges"
|
||||
f = rest.Filter(
|
||||
should=[
|
||||
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_note_id)),
|
||||
rest.Filter(
|
||||
must=[
|
||||
rest.FieldCondition(key="kind", match=rest.MatchValue(value="backlink")),
|
||||
rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_note_id)),
|
||||
]
|
||||
),
|
||||
]
|
||||
)
|
||||
# pydantic v2: client.delete(...) statt delete_points(...)
|
||||
client.delete(collection_name=edges_col, points_selector=f, wait=True)
|
||||
|
||||
def main():
|
||||
load_dotenv()
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Obsidian Vault Pfad (z.B. mindnet/vault)")
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian Vault (z.B. ./vault)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant (sonst Dry-Run)")
|
||||
ap.add_argument("--note-id", help="Nur eine Note-ID verarbeiten")
|
||||
ap.add_argument("--embed-note", action="store_true", help="Auch Note-Volltext einbetten (optional)")
|
||||
ap.add_argument("--force-replace", action="store_true", help="Vor Upsert alte Edges der Note löschen")
|
||||
args = ap.parse_args()
|
||||
|
||||
# Qdrant-Konfiguration (+ Collections sicherstellen)
|
||||
# Qdrant
|
||||
cfg = QdrantConfig(
|
||||
url=os.getenv("QDRANT_URL", "http://127.0.0.1:6333"),
|
||||
api_key=os.getenv("QDRANT_API_KEY") or None,
|
||||
prefix=os.getenv("COLLECTION_PREFIX", "mindnet"),
|
||||
dim=int(os.getenv("VECTOR_DIM", "384")),
|
||||
dim=int(os.getenv("VECTOR_DIM","384")),
|
||||
)
|
||||
client = get_client(cfg)
|
||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
@@ -114,26 +97,23 @@ def main():
|
||||
root = os.path.abspath(args.vault)
|
||||
files = iter_md(root)
|
||||
if not files:
|
||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr); sys.exit(2)
|
||||
|
||||
# (1) Vorab-Lauf: Note-Index (robuste Auflösung via ID/Titel-Slug/File-Slug)
|
||||
index_payloads: List[Dict] = []
|
||||
# --- Note-Index (für robuste Zielauflösung) ---
|
||||
note_stubs = []
|
||||
for path in files:
|
||||
parsed = read_markdown(path)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
try:
|
||||
pl = minimal_note_index_payload(path, root)
|
||||
if not pl.get("note_id"):
|
||||
continue
|
||||
if args.note_id and pl["note_id"] != args.note_id:
|
||||
continue
|
||||
index_payloads.append(pl)
|
||||
validate_required_frontmatter(fm)
|
||||
except Exception:
|
||||
# Einzeldefekte nicht fatal
|
||||
continue
|
||||
if args.note_id and fm.get("id") != args.note_id:
|
||||
continue
|
||||
rel = os.path.relpath(parsed.path, root).replace("\\","/")
|
||||
note_stubs.append({"note_id": fm["id"], "title": fm.get("title",""), "path": rel})
|
||||
note_index = build_note_index(note_stubs)
|
||||
|
||||
note_index = build_note_index(index_payloads)
|
||||
|
||||
# (2) Hauptlauf: Import + direkte Wikilink-Edge-Ableitung
|
||||
total_notes = 0
|
||||
for path in files:
|
||||
parsed = read_markdown(path)
|
||||
@@ -147,11 +127,12 @@ def main():
|
||||
|
||||
total_notes += 1
|
||||
|
||||
# Note-Payload (vollständig für notes-Collection)
|
||||
# Note-Payload
|
||||
note_pl = make_note_payload(parsed, vault_root=root)
|
||||
validate_note_payload(note_pl)
|
||||
note_pl["fulltext"] = parsed.body # für derive_edges
|
||||
|
||||
# Chunks aus Body gemäß Chunking-Strategie
|
||||
# Chunks
|
||||
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
|
||||
|
||||
@@ -159,52 +140,46 @@ def main():
|
||||
texts = [ch.text for ch in chunks]
|
||||
vectors = embed_texts(texts)
|
||||
|
||||
# Optional: Note-Vektor (z. B. für Doc-Suche/Clustering)
|
||||
note_vec = None
|
||||
if args.embed_note:
|
||||
note_vec = embed_one(parsed.body)
|
||||
# Optional: Note-Vektor
|
||||
note_vec = embed_one(parsed.body) if args.embed_note else None
|
||||
|
||||
# Edges direkt aus realen Chunks + Note-Index
|
||||
# ACHTUNG: derive_wikilink_edges erwartet Chunks mit Text → wir bauen eine
|
||||
# minimalistische Liste {"chunk_id","text"} in gleicher Reihenfolge.
|
||||
note_pl_for_edges = {
|
||||
"note_id": fm["id"],
|
||||
"title": fm.get("title"),
|
||||
"path": note_pl["path"],
|
||||
"fulltext": parsed.body,
|
||||
}
|
||||
chunks_for_links = []
|
||||
for i, pl in enumerate(chunk_pls):
|
||||
cid = pl.get("chunk_id") or pl.get("id")
|
||||
txt = chunks[i].text if i < len(chunks) else ""
|
||||
chunks_for_links.append({"chunk_id": cid, "text": txt})
|
||||
# --- WICHTIG: Chunk-Texte für derive_wikilink_edges ---
|
||||
chunks_for_links = [
|
||||
{"chunk_id": (pl.get("chunk_id") or pl.get("id") or f"{fm['id']}#{i+1}"),
|
||||
"text": chunks[i].text}
|
||||
for i, pl in enumerate(chunk_pls)
|
||||
if i < len(chunks)
|
||||
]
|
||||
|
||||
edges = derive_wikilink_edges(note_pl_for_edges, chunks_for_links, note_index)
|
||||
# Kanten (Note- & Chunk-Ebene)
|
||||
edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index)
|
||||
|
||||
# Dry-Run-Summary
|
||||
# Dry-Run-Ausgabe
|
||||
print(json.dumps({
|
||||
"note_id": fm["id"],
|
||||
"title": fm["title"],
|
||||
"chunks": len(chunk_pls),
|
||||
"edges": len(edges),
|
||||
"path": note_pl["path"],
|
||||
"path": note_pl["path"]
|
||||
}, ensure_ascii=False))
|
||||
|
||||
if args.apply:
|
||||
# Notes upsert (idempotent; UUIDv5)
|
||||
if args.force_replace:
|
||||
purge_note_edges(client, cfg.prefix, fm["id"])
|
||||
|
||||
# Notes upsert
|
||||
notes_col, note_pts = points_for_note(cfg.prefix, note_pl, note_vec, cfg.dim)
|
||||
upsert_batch(client, notes_col, note_pts)
|
||||
|
||||
# Chunks upsert (idempotent)
|
||||
# Chunks upsert
|
||||
chunks_col, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vectors)
|
||||
upsert_batch(client, chunks_col, chunk_pts)
|
||||
|
||||
# Edges upsert (idempotent; deterministisches edge_id)
|
||||
# Edges upsert
|
||||
edges_col, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||
upsert_batch(client, edges_col, edge_pts)
|
||||
|
||||
print(f"Done. Processed notes: {total_notes}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user