Compare commits
62
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
81c1400ef4 | ||
|
|
298c1fb180 | ||
|
|
f8108bd724 | ||
|
|
65c40b287f | ||
|
|
13e50d4695 | ||
|
|
791cda23af | ||
|
|
466ea64e63 | ||
|
|
f5e6fcc097 | ||
|
|
9ef2e8d397 | ||
|
|
4637091125 | ||
|
|
43a8c80508 | ||
|
|
9362064ad1 | ||
|
|
23952aa3c5 | ||
|
|
79e19c85eb | ||
|
|
62d8299480 | ||
|
|
8f342ea2d3 | ||
|
|
5d5de06290 | ||
|
|
d7e5f398f0 | ||
|
|
5bf5316af5 | ||
|
|
d9fd6d8a8f | ||
|
|
e672eade8b | ||
|
|
ec346905ed | ||
|
|
c9cbbbfcb9 | ||
|
|
93739935b7 | ||
|
|
6a293e842e | ||
|
|
f4abb1d873 | ||
|
|
8f48701ea0 | ||
|
|
6f32c60c47 | ||
|
|
e040fcc0dd | ||
|
|
305089fcf6 | ||
|
|
f9c0b5df19 | ||
|
|
77fa57a2a6 | ||
|
|
dcb02dd194 | ||
|
|
35338dea64 | ||
|
|
c34df96839 | ||
|
|
0617163c42 | ||
|
|
47797ecd29 | ||
|
|
897d0c9e6d | ||
|
|
6c25d76135 | ||
|
|
22885a4857 | ||
|
|
a807cc8bd1 | ||
|
|
c2954b5663 | ||
|
|
8f97f42618 | ||
|
|
0a116ea3f4 | ||
|
|
aa7d0190c8 | ||
|
|
af5d0a0c91 | ||
|
|
fb016c6c6a | ||
|
|
77732445ea | ||
|
|
1b833f76ce | ||
|
|
b394cadf73 | ||
|
|
f3b6166daa | ||
|
|
ce720090ec | ||
|
|
bf2149bbeb | ||
|
|
80a7db0e54 | ||
|
|
47e6d56b21 | ||
|
|
df33293621 | ||
|
|
a202c53594 | ||
|
|
4f54e750b2 | ||
|
|
5478be60c1 | ||
|
|
3deb99f053 | ||
|
|
8df2069c6f | ||
|
|
d790c0ef75 |
No files matched your search
+243
-27
@@ -1,37 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/chunk_payload.py
|
||||
Version: 2.0.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
Erzeugt **Chunk-Payloads** für die Qdrant-Collection `<prefix>_chunks` auf Basis der
|
||||
vom Chunker gelieferten `Chunk`-Objekte. Ziel:
|
||||
- *Verlustfreie Rekonstruktion*: Jeder Chunk enthält seinen **Text** (payload["text"]).
|
||||
- *Schnelle Abfragen*: Wichtige Note-Metadaten werden gespiegelt, um Filter ohne Join zu erlauben.
|
||||
- *Graph-Kompatibilität*: Wikilinks und externe Links werden extrahiert; Nachbarschaften werden übernommen.
|
||||
- *Monitoring*: Token- und Längenmetriken sowie Text-Hash erleichtern Audits und Re-Embeddings.
|
||||
|
||||
Kompatibilität
|
||||
--------------
|
||||
- **Abwärtskompatibel** zur bisherigen `make_chunk_payloads`-Signatur.
|
||||
- Zusätzliche Felder stören bestehende Upserts nicht (Payload ist schema-flexibel).
|
||||
- Erwartet, dass `Chunk` u. a. die Attribute `id`, `index`, `text`, `char_start`, `char_end`,
|
||||
`section_title`, `section_path`, `neighbors_prev`, `neighbors_next` liefert.
|
||||
|
||||
CLI (Schnelltest)
|
||||
-----------------
|
||||
# Preview aus einer Markdown-Datei
|
||||
python3 -m app.core.chunk_payload --from-file ./test_vault/20_experiences/exp-two.md --vault-root ./test_vault
|
||||
|
||||
# Nur IDs & Tokenmengen
|
||||
python3 -m app.core.chunk_payload --from-file ./test_vault/20_experiences/exp-two.md --vault-root ./test_vault --summary
|
||||
|
||||
Felder (Auszug)
|
||||
---------------
|
||||
id : "<note_id>#cNN"
|
||||
scope : "chunk"
|
||||
note_id : "<note_id>"
|
||||
note_title : str
|
||||
note_type : str
|
||||
note_status : str
|
||||
area, project : optional
|
||||
tags : list[str]
|
||||
note_path : str (relativ, Slashes normalisiert)
|
||||
chunk_index : int
|
||||
section_title : str | None
|
||||
section_path : str | None
|
||||
char_start : int | None
|
||||
char_end : int | None
|
||||
char_len : int
|
||||
token_est : int (≈ len(text)/4)
|
||||
neighbors : {"prev": str|None, "next": str|None}
|
||||
text : str (Chunk-Text, **Pflicht**)
|
||||
text_sha256 : str "sha256:<hex>"
|
||||
lang : optional
|
||||
wikilinks : list[str]
|
||||
external_links : list[{"href": str, "label": str|None}]
|
||||
references : list[{"target_id": str, "kind": "wikilink"}]
|
||||
embed_model : optional (durchreichbar)
|
||||
embed_dim : optional
|
||||
embed_version : optional
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Dict, List
|
||||
from .chunker import Chunk
|
||||
from .parser import extract_wikilinks
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
# Paket-Import (normaler Betrieb)
|
||||
from app.core.chunker import Chunk
|
||||
from app.core.parser import extract_wikilinks, read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||
except Exception: # pragma: no cover
|
||||
# Relativ (lokale Tests)
|
||||
from .chunker import Chunk # type: ignore
|
||||
from .parser import extract_wikilinks, read_markdown, normalize_frontmatter, validate_required_frontmatter # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utilities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RE_MD_LINK = re.compile(r"\[([^\]]*)\]\(([^)\s]+)(?:\s+\"([^\"]+)\")?\)")
|
||||
RE_HTTP_SCHEMES = ("http://", "https://", "mailto:", "obsidian://", "tel:")
|
||||
|
||||
def _estimate_tokens(text: str) -> int:
|
||||
"""Grobe Token-Schätzung (≈ 1 Token pro 4 Zeichen)."""
|
||||
return max(0, int(round(len((text or '').strip()) / 4)))
|
||||
|
||||
def _sha256_text(text: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
h.update((text or "").encode("utf-8"))
|
||||
return "sha256:" + h.hexdigest()
|
||||
|
||||
def _normalize_rel_path(path: Optional[str], vault_root: Optional[str]) -> Optional[str]:
|
||||
if not path:
|
||||
return None
|
||||
p = str(path)
|
||||
p = p.replace("\\", "/")
|
||||
if vault_root and os.path.isabs(p):
|
||||
try:
|
||||
p = os.path.relpath(p, vault_root)
|
||||
except Exception:
|
||||
pass
|
||||
p = p.replace("\\", "/").lstrip("/")
|
||||
return p
|
||||
|
||||
def _extract_external_links(text: str) -> List[Dict[str, Optional[str]]]:
|
||||
"""Findet Markdown-Links [label](href "title") mit erlaubten Schemes."""
|
||||
out: List[Dict[str, Optional[str]]] = []
|
||||
if not text:
|
||||
return out
|
||||
for m in RE_MD_LINK.finditer(text):
|
||||
label = (m.group(1) or "").strip() or None
|
||||
href = (m.group(2) or "").strip()
|
||||
title = (m.group(3) or "").strip() or None
|
||||
if any(href.startswith(s) for s in RE_HTTP_SCHEMES):
|
||||
out.append({"href": href, "label": label or title})
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_chunk_payloads(note_meta: Dict, path: str, chunks: List[Chunk]) -> List[Dict]:
|
||||
res = []
|
||||
"""
|
||||
Erzeugt Payload-Dicts für alle Chunks einer Note.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
note_meta : Dict
|
||||
Normalisierte Frontmatter der Note (mind.: id, title, type, status, tags, [area, project, lang]).
|
||||
path : str
|
||||
Pfad zur Note (sollte relativ zum Vault sein; wird hier zur Sicherheit normalisiert).
|
||||
chunks : List[Chunk]
|
||||
Vom Chunker erzeugte Chunks.
|
||||
|
||||
Returns
|
||||
-------
|
||||
List[Dict]
|
||||
Liste von Payloads (ein Eintrag pro Chunk).
|
||||
"""
|
||||
res: List[Dict] = []
|
||||
rel_path = _normalize_rel_path(path, vault_root=None)
|
||||
|
||||
for ch in chunks:
|
||||
wikilinks = extract_wikilinks(ch.text)
|
||||
payload = {
|
||||
"id": ch.id,
|
||||
"note_id": note_meta["id"],
|
||||
"note_title": note_meta["title"],
|
||||
"chunk_index": ch.index,
|
||||
"char_start": ch.char_start,
|
||||
"char_end": ch.char_end,
|
||||
"token_count": ch.token_count,
|
||||
"type": note_meta.get("type"),
|
||||
text: str = getattr(ch, "text", "") or ""
|
||||
wikilinks = extract_wikilinks(text)
|
||||
ext_links = _extract_external_links(text)
|
||||
|
||||
payload: Dict = {
|
||||
"id": getattr(ch, "id", None),
|
||||
"scope": "chunk",
|
||||
"note_id": note_meta.get("id"),
|
||||
"note_title": note_meta.get("title"),
|
||||
# gespiegelt für schnelle Filter:
|
||||
"note_type": note_meta.get("type"),
|
||||
"note_status": note_meta.get("status"),
|
||||
"area": note_meta.get("area"),
|
||||
"project": note_meta.get("project"),
|
||||
"tags": note_meta.get("tags", []),
|
||||
"section_title": ch.section_title,
|
||||
"section_path": ch.section_path,
|
||||
"lang": note_meta.get("lang"),
|
||||
"wikilinks": wikilinks,
|
||||
"external_links": [], # (optional später ergänzen)
|
||||
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
|
||||
"tags": note_meta.get("tags"),
|
||||
# Pfad
|
||||
"note_path": rel_path,
|
||||
"path": rel_path, # Back-compat
|
||||
# Reihenfolge & Section
|
||||
"chunk_index": getattr(ch, "index", None),
|
||||
"section_title": getattr(ch, "section_title", None),
|
||||
"section_path": getattr(ch, "section_path", None),
|
||||
# Position
|
||||
"char_start": getattr(ch, "char_start", None),
|
||||
"char_end": getattr(ch, "char_end", None),
|
||||
"char_len": max(0, int(getattr(ch, "char_end", 0) or 0) - int(getattr(ch, "char_start", 0) or 0)) or len(text),
|
||||
# Nachbarn
|
||||
"neighbors": {
|
||||
"prev": ch.neighbors_prev,
|
||||
"next": ch.neighbors_next
|
||||
"prev": getattr(ch, "neighbors_prev", None),
|
||||
"next": getattr(ch, "neighbors_next", None),
|
||||
},
|
||||
"path": path,
|
||||
# Inhalt & Metrik
|
||||
"text": text,
|
||||
"text_sha256": _sha256_text(text),
|
||||
"token_est": _estimate_tokens(text),
|
||||
# Sprache
|
||||
"lang": note_meta.get("lang"),
|
||||
# Links
|
||||
"wikilinks": wikilinks,
|
||||
"external_links": ext_links,
|
||||
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
|
||||
}
|
||||
# None/Leere bereinigen
|
||||
payload = {k: v for k, v in payload.items() if v not in (None, [], {})}
|
||||
res.append(payload)
|
||||
|
||||
# Entferne Nones/Leeres, aber **text** bleibt (darf leer sein z. B. bei Bild-Only-Chunks)
|
||||
cleaned = {}
|
||||
for k, v in payload.items():
|
||||
if v in (None, [], {}):
|
||||
# immer behalten:
|
||||
if k in ("text", "neighbors"):
|
||||
cleaned[k] = v
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
cleaned[k] = v
|
||||
|
||||
res.append(cleaned)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI zum schnellen Testen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _cli() -> None:
|
||||
ap = argparse.ArgumentParser(description="Chunk-Payloads aus einer einzelnen Markdown-Datei erzeugen")
|
||||
ap.add_argument("--from-file", required=True, help="Pfad zur Markdown-Datei")
|
||||
ap.add_argument("--vault-root", default=None, help="Vault-Wurzel (zur Pfad-Relativierung)")
|
||||
ap.add_argument("--summary", action="store_true", help="Nur kurze Übersicht je Chunk ausgeben")
|
||||
args = ap.parse_args()
|
||||
|
||||
parsed = read_markdown(args.from_file)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
|
||||
# lazy import, um Zyklen zu vermeiden
|
||||
try:
|
||||
from app.core.chunker import assemble_chunks
|
||||
except Exception:
|
||||
from .chunker import assemble_chunks # type: ignore
|
||||
|
||||
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
|
||||
rel = _normalize_rel_path(parsed.path, args.vault_root)
|
||||
|
||||
pls = make_chunk_payloads(fm, rel or parsed.path, chunks)
|
||||
|
||||
if args.summary:
|
||||
out = []
|
||||
for p in pls:
|
||||
out.append({
|
||||
"id": p.get("id"),
|
||||
"chunk_index": p.get("chunk_index"),
|
||||
"token_est": p.get("token_est"),
|
||||
"wikilinks": p.get("wikilinks"),
|
||||
"ext_links": [e.get("href") for e in p.get("external_links", [])],
|
||||
"prev": (p.get("neighbors") or {}).get("prev"),
|
||||
"next": (p.get("neighbors") or {}).get("next"),
|
||||
})
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(json.dumps(pls, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
_cli()
|
||||
+108
-33
@@ -1,45 +1,120 @@
|
||||
from __future__ import annotations
|
||||
from typing import List, Dict
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/edges.py
|
||||
Version: 1.0.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
def deriv_edges_for_note(note_meta: Dict, chunk_payloads: List[Dict]) -> List[Dict]:
|
||||
Zweck
|
||||
-----
|
||||
Zentrale, konsistente Erzeugung von Edge-Payloads im **neuen Schema**:
|
||||
- kind : "belongs_to" | "next" | "prev" | "references" | "backlink"
|
||||
- source_id : ID des Quellknotens (Chunk- oder Note-ID)
|
||||
- target_id : ID des Zielknotens
|
||||
- scope : "chunk" | "note"
|
||||
- note_id : Owner-Note (für performantes Filtern/Löschen)
|
||||
- seq : optional (z. B. Reihenfolge von Vorkommen)
|
||||
|
||||
Hinweise
|
||||
--------
|
||||
- Edges werden dedupliziert (key=(kind,source_id,target_id,scope)).
|
||||
- Für Chunk-Edges wird `note_id` aus dem Chunk-Payload entnommen.
|
||||
- Für Note-Scope-Edges ist `note_id` die Quell-Note-ID.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Dict, List
|
||||
|
||||
def build_edges_for_note(
|
||||
note_id: str,
|
||||
chunk_payloads: List[Dict],
|
||||
note_level_refs: List[str] | None,
|
||||
*,
|
||||
include_note_scope_refs: bool = False,
|
||||
) -> List[Dict]:
|
||||
edges: List[Dict] = []
|
||||
|
||||
# Chunk → Note (belongs_to) + prev/next
|
||||
for idx, ch in enumerate(chunk_payloads):
|
||||
# Chunk-Scope: belongs_to / prev / next / references
|
||||
for ch in chunk_payloads:
|
||||
cid = ch["id"]
|
||||
owner = ch.get("note_id") or note_id
|
||||
# belongs_to
|
||||
edges.append({
|
||||
"src_id": ch["id"], "dst_id": note_meta["id"],
|
||||
"edge_type": "belongs_to", "scope": "chunk"
|
||||
"kind": "belongs_to",
|
||||
"source_id": cid,
|
||||
"target_id": note_id,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
prev_id = ch.get("neighbors",{}).get("prev")
|
||||
next_id = ch.get("neighbors",{}).get("next")
|
||||
# Nachbarn
|
||||
nb = ch.get("neighbors") or {}
|
||||
prev_id = nb.get("prev")
|
||||
next_id = nb.get("next")
|
||||
if prev_id:
|
||||
edges.append({"src_id": ch["id"], "dst_id": prev_id, "edge_type": "next", "scope": "chunk"})
|
||||
edges.append({"src_id": prev_id, "dst_id": ch["id"], "edge_type": "prev", "scope": "chunk"})
|
||||
edges.append({
|
||||
"kind": "prev",
|
||||
"source_id": cid,
|
||||
"target_id": prev_id,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
edges.append({
|
||||
"kind": "next",
|
||||
"source_id": prev_id,
|
||||
"target_id": cid,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
if next_id:
|
||||
edges.append({"src_id": ch["id"], "dst_id": next_id, "edge_type": "next", "scope": "chunk"})
|
||||
edges.append({"src_id": next_id, "dst_id": ch["id"], "edge_type": "prev", "scope": "chunk"})
|
||||
|
||||
# references aus wikilinks (Chunk-Scope)
|
||||
for ref in ch.get("references", []):
|
||||
edges.append({
|
||||
"kind": "next",
|
||||
"source_id": cid,
|
||||
"target_id": next_id,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
edges.append({
|
||||
"kind": "prev",
|
||||
"source_id": next_id,
|
||||
"target_id": cid,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
# references aus Chunk
|
||||
for ref in (ch.get("references") or []):
|
||||
tid = ref.get("target_id")
|
||||
if not tid: continue
|
||||
edges.append({"src_id": ch["id"], "dst_id": tid, "edge_type": "references", "scope": "chunk"})
|
||||
edges.append({"src_id": tid, "dst_id": ch["id"], "edge_type": "backlink", "scope": "chunk"})
|
||||
if not tid:
|
||||
continue
|
||||
edges.append({
|
||||
"kind": "references",
|
||||
"source_id": cid,
|
||||
"target_id": tid,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
|
||||
# depends_on / assigned_to (Note-Scope, aus Frontmatter)
|
||||
for dep in note_meta.get("depends_on", []) or []:
|
||||
edges.append({"src_id": note_meta["id"], "dst_id": dep, "edge_type": "depends_on", "scope": "note"})
|
||||
for ass in note_meta.get("assigned_to", []) or []:
|
||||
edges.append({"src_id": note_meta["id"], "dst_id": ass, "edge_type": "assigned_to", "scope": "note"})
|
||||
|
||||
# Note-Level references (optional: falls du im Note-Payload `references` sammelst)
|
||||
for tid in note_meta.get("references", []) or []:
|
||||
edges.append({"src_id": note_meta["id"], "dst_id": tid, "edge_type": "references", "scope": "note"})
|
||||
edges.append({"src_id": tid, "dst_id": note_meta["id"], "edge_type": "backlink", "scope": "note"})
|
||||
# Note-Scope: backlink (immer); references (optional)
|
||||
unique_refs = list(dict.fromkeys(note_level_refs or []))
|
||||
for tid in unique_refs:
|
||||
if include_note_scope_refs:
|
||||
edges.append({
|
||||
"kind": "references",
|
||||
"source_id": note_id,
|
||||
"target_id": tid,
|
||||
"scope": "note",
|
||||
"note_id": note_id,
|
||||
})
|
||||
edges.append({
|
||||
"kind": "backlink",
|
||||
"source_id": tid,
|
||||
"target_id": note_id,
|
||||
"scope": "note",
|
||||
"note_id": note_id,
|
||||
})
|
||||
|
||||
# Dedupe
|
||||
uniq = {}
|
||||
dedup = {}
|
||||
for e in edges:
|
||||
k = (e["src_id"], e["edge_type"], e["dst_id"], e.get("scope","note"))
|
||||
uniq[k] = e
|
||||
return list(uniq.values())
|
||||
k = (e["kind"], e["source_id"], e["target_id"], e.get("scope", ""))
|
||||
dedup[k] = e
|
||||
return list(dedup.values())
|
||||
+298
-28
@@ -1,39 +1,309 @@
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import hashlib
|
||||
from typing import Dict
|
||||
from .parser import ParsedNote
|
||||
from .parser import extract_wikilinks
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Modul: app/core/note_payload.py
|
||||
# Version: 1.6.3
|
||||
# Datum: 2025-09-09
|
||||
#
|
||||
# Zweck
|
||||
# -----
|
||||
# Erzeugt den Qdrant-Payload für Notes inkl. robuster Hash-Bildung zur
|
||||
# Änderungserkennung. Der vollständige (parsed) Body wird unter "fulltext"
|
||||
# persistiert; der Pfad ist relativ (für verlustfreien Export).
|
||||
#
|
||||
# Steuerung (CLI/ENV, vom Importer durchgereicht)
|
||||
# -----------------------------------------------
|
||||
# - Vergleichsmodus:
|
||||
# --hash-mode body|frontmatter|full
|
||||
# ENV: MINDNET_HASH_MODE oder MINDNET_HASH_COMPARE (Body|Frontmatter|Full)
|
||||
# "full" ist Alias für "body+frontmatter".
|
||||
# - Hash-Quelle:
|
||||
# --hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE)
|
||||
# - Normalisierung:
|
||||
# --hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE)
|
||||
#
|
||||
# Payload-Felder (Auszug)
|
||||
# -----------------------
|
||||
# note_id, title, type, status, created, updated, path, tags,
|
||||
# fulltext, references (Note-Level-Wikilinks),
|
||||
# hash_fulltext (Primärhash), hash_signature (z. B. "body:raw:none:<hex>")
|
||||
#
|
||||
# Hinweise
|
||||
# --------
|
||||
# - Keine Abhängigkeit von FS-Zeitstempeln; nur Inhalte fließen in den Hash ein.
|
||||
# - Abwärtskompatibel: Feldernamen bleiben stabil; zusätzliche Felder stören nicht.
|
||||
|
||||
def sha256_text(text: str) -> str:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
try:
|
||||
from app.core.parser import read_markdown, extract_wikilinks, FRONTMATTER_RE
|
||||
except Exception: # pragma: no cover
|
||||
from .parser import read_markdown, extract_wikilinks, FRONTMATTER_RE # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
|
||||
"""Kanonische JSON-Serialisierung der Frontmatter für Hashbildung."""
|
||||
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
def _normalize_body(body: str, mode: str) -> str:
|
||||
"""Normalisiert den Body für reproduzierbare Hashes (oder nicht)."""
|
||||
if mode == "none":
|
||||
return body if body is not None else ""
|
||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
|
||||
text = "\n".join(line.rstrip() for line in text.split("\n"))
|
||||
return text
|
||||
|
||||
def _resolve_hash_mode(explicit: Optional[str]) -> str:
|
||||
"""
|
||||
Normalisiert den Hash-Modus auf:
|
||||
'body' | 'frontmatter' | 'body+frontmatter'
|
||||
Akzeptiert 'full' als Alias.
|
||||
Berücksichtigt ENV: MINDNET_HASH_MODE oder MINDNET_HASH_COMPARE.
|
||||
"""
|
||||
if explicit:
|
||||
val = explicit.strip().lower()
|
||||
else:
|
||||
val = (os.environ.get("MINDNET_HASH_MODE")
|
||||
or os.environ.get("MINDNET_HASH_COMPARE")
|
||||
or "body").strip().lower()
|
||||
if val in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
||||
return "body+frontmatter"
|
||||
if val in ("frontmatter", "fm"):
|
||||
return "frontmatter"
|
||||
return "body"
|
||||
|
||||
def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, Any]]:
|
||||
"""Liest die Rohdatei und extrahiert Body & Frontmatter ohne Parser-Logik.
|
||||
|
||||
Rückgabe:
|
||||
(body_text, frontmatter_dict)
|
||||
"""
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return "", {}
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
raw = f.read()
|
||||
except Exception:
|
||||
return "", {}
|
||||
# Frontmatter per Regex entfernen
|
||||
m = FRONTMATTER_RE.match(raw)
|
||||
fm = {}
|
||||
if m:
|
||||
fm_txt = m.group(1)
|
||||
try:
|
||||
import yaml # lazy
|
||||
fm = yaml.safe_load(fm_txt) or {}
|
||||
except Exception:
|
||||
fm = {}
|
||||
body = raw[m.end():]
|
||||
else:
|
||||
body = raw
|
||||
return body, fm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hashing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _sha256(s: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
h.update(text.encode("utf-8"))
|
||||
h.update(s.encode("utf-8"))
|
||||
return h.hexdigest()
|
||||
|
||||
def make_note_payload(parsed: ParsedNote, vault_root: str) -> Dict:
|
||||
fm = parsed.frontmatter
|
||||
body = parsed.body
|
||||
rel_path = os.path.relpath(parsed.path, vault_root) if vault_root else parsed.path
|
||||
def compute_hash(
|
||||
*,
|
||||
body: Optional[str],
|
||||
frontmatter: Optional[Dict[str, Any]],
|
||||
mode: Optional[str] = None,
|
||||
normalize: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Berechnet einen Hex-Hash gemäß 'mode' und 'normalize'.
|
||||
|
||||
payload = {
|
||||
"note_id": fm.get("id"),
|
||||
mode: "body" | "frontmatter" | "body+frontmatter"
|
||||
normalize: "canonical" | "none"
|
||||
"""
|
||||
mode = _resolve_hash_mode(mode)
|
||||
normalize = (normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
|
||||
body_n = _normalize_body(body or "", normalize)
|
||||
fm_s = _canon_frontmatter(frontmatter or {})
|
||||
|
||||
if mode == "frontmatter":
|
||||
return _sha256(fm_s)
|
||||
if mode == "body+frontmatter":
|
||||
return _sha256(body_n + "\n--FM--\n" + fm_s)
|
||||
# default: body
|
||||
return _sha256(body_n)
|
||||
|
||||
def compute_hash_set(*, body_parsed: str, body_raw: str, fm: Dict[str, Any], normalize: str) -> Dict[str, str]:
|
||||
"""Berechnet ein Set an Hashes für Monitoring/Debug."""
|
||||
fm_s = _canon_frontmatter(fm or {})
|
||||
bp = _normalize_body(body_parsed or "", normalize)
|
||||
br = _normalize_body(body_raw or "", normalize)
|
||||
return {
|
||||
"frontmatter": _sha256(fm_s),
|
||||
"body_parsed": _sha256(bp),
|
||||
"body_raw": _sha256(br),
|
||||
"full_parsed": _sha256(bp + "\n--FM--\n" + fm_s),
|
||||
"full_raw": _sha256(br + "\n--FM--\n" + fm_s),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kernfunktion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_note_payload(
|
||||
parsed: Any,
|
||||
vault_root: Optional[str] = None,
|
||||
*,
|
||||
hash_mode: Optional[str] = None,
|
||||
hash_normalize: Optional[str] = None,
|
||||
hash_source: Optional[str] = None,
|
||||
file_path: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Erzeugt den Payload für eine geparste Note.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
parsed : Any
|
||||
Objekt mit Attributen/Keys 'frontmatter', 'body', 'path'.
|
||||
vault_root : Optional[str]
|
||||
Vault-Wurzel (für Pfad-Relativierung).
|
||||
hash_mode : Optional[str]
|
||||
"body" | "frontmatter" | "body+frontmatter" | "full" (Alias; überschreibt ENV).
|
||||
hash_normalize : Optional[str]
|
||||
"canonical" | "none" (überschreibt ENV).
|
||||
hash_source : Optional[str]
|
||||
"parsed" (Default) oder "raw". Wenn "raw", wird der Body aus der Rohdatei gelesen.
|
||||
file_path : Optional[str]
|
||||
Pfad zur Markdown-Datei, erforderlich für 'hash_source=raw'.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Dict[str, Any]
|
||||
Qdrant-Payload für die Notes-Collection.
|
||||
"""
|
||||
# dict oder Objekt akzeptieren
|
||||
if isinstance(parsed, dict):
|
||||
fm = parsed.get("frontmatter") or {}
|
||||
body_parsed = parsed.get("body") or ""
|
||||
path = parsed.get("path") or ""
|
||||
else:
|
||||
fm = getattr(parsed, "frontmatter", {}) or {}
|
||||
body_parsed = getattr(parsed, "body", "") or ""
|
||||
path = getattr(parsed, "path", "") or ""
|
||||
|
||||
# Hash-Quelle bestimmen
|
||||
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower()
|
||||
raw_body, raw_fm = ("", {})
|
||||
if src == "raw":
|
||||
raw_body, raw_fm = _read_raw_body_from_file(file_path or path)
|
||||
# Roh-FM ergänzen (nicht überschreiben)
|
||||
if isinstance(raw_fm, dict) and raw_fm:
|
||||
merged_fm = dict(fm)
|
||||
for k, v in raw_fm.items():
|
||||
merged_fm.setdefault(k, v)
|
||||
fm = merged_fm
|
||||
|
||||
normalize = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
|
||||
mode_resolved = _resolve_hash_mode(hash_mode)
|
||||
|
||||
# Hash gemäß Modus/Quelle bilden
|
||||
body_for_hash = raw_body if src == "raw" else body_parsed
|
||||
primary_hash = compute_hash(body=body_for_hash, frontmatter=fm, mode=mode_resolved, normalize=normalize)
|
||||
hash_signature = f"{'full' if mode_resolved=='body+frontmatter' else mode_resolved}:{src}:{normalize}:{primary_hash}"
|
||||
|
||||
# Pfad relativieren
|
||||
rel_path = path
|
||||
try:
|
||||
if vault_root:
|
||||
rel = os.path.relpath(path, vault_root)
|
||||
rel = rel.replace("\\", "/").lstrip("/") # normalisieren
|
||||
rel_path = rel
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Note-Level-Wikilinks (Fallback, wenn Chunks nicht geliefert werden)
|
||||
note_level_refs = list(dict.fromkeys(extract_wikilinks(body_parsed))) if body_parsed else []
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"note_id": fm.get("id") or fm.get("note_id"),
|
||||
"title": fm.get("title"),
|
||||
"type": fm.get("type"),
|
||||
"status": fm.get("status"),
|
||||
"created": fm.get("created"),
|
||||
"updated": fm.get("updated"),
|
||||
"tags": fm.get("tags") or [],
|
||||
"priority": fm.get("priority"),
|
||||
"effort_min": fm.get("effort_min"),
|
||||
"due": fm.get("due"),
|
||||
"people": fm.get("people") or [],
|
||||
"aliases": fm.get("aliases") or [],
|
||||
"depends_on": fm.get("depends_on") or [],
|
||||
"assigned_to": fm.get("assigned_to") or [],
|
||||
"references": list(sorted(set(extract_wikilinks(body)))),
|
||||
"lang": fm.get("lang") or None,
|
||||
"path": rel_path.replace("\\", "/"),
|
||||
"hash_fulltext": sha256_text(body.strip()),
|
||||
"path": rel_path or fm.get("path"),
|
||||
"tags": fm.get("tags"),
|
||||
# Primärer Hash + Signatur (für Vergleich)
|
||||
"hash_fulltext": primary_hash,
|
||||
"hash_signature": hash_signature,
|
||||
# Volltext persistieren (verlustfreie Rekonstruktion) – parsed Body
|
||||
"fulltext": body_parsed,
|
||||
# Fallback-Refs auf Note-Ebene
|
||||
"references": note_level_refs,
|
||||
}
|
||||
# Entferne None-Werte für saubere Payloads
|
||||
return {k: v for k, v in payload.items() if v not in (None, [])}
|
||||
|
||||
for k in ("area", "project", "source", "lang", "slug", "aliases"):
|
||||
if k in fm:
|
||||
payload[k] = fm[k]
|
||||
|
||||
# Optional: gesamtes Hash-Set persistieren (Debug/Monitoring)
|
||||
if os.environ.get("MINDNET_HASH_RECORD_ALL", "false").strip().lower() == "true":
|
||||
payload["hashes"] = compute_hash_set(
|
||||
body_parsed=body_parsed, body_raw=raw_body, fm=fm, normalize=normalize
|
||||
)
|
||||
|
||||
# Optional: Roh-Body-Hash separat (historische Kompatibilität)
|
||||
if os.environ.get("MINDNET_HASH_STORE_RAW", "false").strip().lower() == "true" and src == "raw":
|
||||
try:
|
||||
payload["hash_raw_body"] = compute_hash(
|
||||
body=raw_body, frontmatter=fm, mode="body", normalize="none"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI – Sichtprüfung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _cli() -> None:
|
||||
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
|
||||
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zur Markdown-Datei")
|
||||
ap.add_argument("--vault-root", dest="vault_root", default=None, help="Vault-Wurzel zur Pfad-Relativierung")
|
||||
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter", "full"], default=None)
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
parsed = read_markdown(args.src)
|
||||
payload = make_note_payload(
|
||||
parsed,
|
||||
vault_root=args.vault_root,
|
||||
hash_mode=args.hash_mode,
|
||||
hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source,
|
||||
file_path=args.src,
|
||||
)
|
||||
|
||||
if args.do_print:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
_cli()
|
||||
+31
-59
@@ -2,30 +2,16 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: app/core/qdrant.py
|
||||
Version: v1.3.1 (2025-09-05)
|
||||
Version: v1.4.0 (2025-09-09)
|
||||
|
||||
Kurzbeschreibung:
|
||||
Qdrant-Client & Collection-Setup für mindnet.
|
||||
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
|
||||
- Edges-Collection nutzt 1D Dummy-Vektor (Workaround für Python-Client).
|
||||
- **Nicht-destruktiv per Default**: ensure_collections(..., destructive=False).
|
||||
- **Abwärtskompatibel**: collection_names(prefix) wieder verfügbar.
|
||||
- Edges-Collection nutzt 1D Dummy-Vektor.
|
||||
- NEW: ensure_payload_indexes(...) legt sinnvolle Payload-Indizes an.
|
||||
|
||||
Aufruf/Verwendung:
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names
|
||||
|
||||
Umgebungsvariablen (optional):
|
||||
QDRANT_URL | QDRANT_HOST/QDRANT_PORT, QDRANT_API_KEY,
|
||||
COLLECTION_PREFIX (Default "mindnet"), VECTOR_DIM (Default 384)
|
||||
|
||||
Änderungen:
|
||||
v1.3.1: Helper collection_names(prefix) wiederhergestellt (für reset_qdrant usw.).
|
||||
v1.3.0: ensure_collections(..., destructive=False) – keine stillen Drops im Dry-Run.
|
||||
Edges-Collection nur bei explicit destructive=True neu anlegen.
|
||||
≤v1.2.x: Konnte {prefix}_edges bei fehlender VectorConfig automatisch neu erstellen (riskant).
|
||||
|
||||
Bezug/Quelle:
|
||||
Alte Core-Variante enthielt collection_names(prefix); diverse Scripts nutzen das weiterhin. :contentReference[oaicite:1]{index=1}
|
||||
Aufruf:
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
@@ -36,10 +22,6 @@ from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
|
||||
# -------------------------------
|
||||
# Konfiguration
|
||||
# -------------------------------
|
||||
|
||||
@dataclass
|
||||
class QdrantConfig:
|
||||
url: str
|
||||
@@ -60,18 +42,10 @@ class QdrantConfig:
|
||||
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
|
||||
|
||||
|
||||
# -------------------------------
|
||||
# Client
|
||||
# -------------------------------
|
||||
|
||||
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
|
||||
|
||||
|
||||
# -------------------------------
|
||||
# Collection-Erzeuger (Hilfsfunktionen)
|
||||
# -------------------------------
|
||||
|
||||
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
@@ -94,16 +68,7 @@ def _create_edges(client: QdrantClient, name: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------
|
||||
# Public API
|
||||
# -------------------------------
|
||||
|
||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
|
||||
"""
|
||||
Stellt sicher, dass die drei Collections existieren.
|
||||
- Default **nicht destruktiv**: vorhandene Collections bleiben unangetastet.
|
||||
- Nur wenn 'destructive=True', wird eine ungeeignete Edges-Collection gelöscht und neu angelegt.
|
||||
"""
|
||||
notes = f"{prefix}_notes"
|
||||
chunks = f"{prefix}_chunks"
|
||||
edges = f"{prefix}_edges"
|
||||
@@ -112,41 +77,48 @@ def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive:
|
||||
_create_chunks(client, chunks, dim)
|
||||
|
||||
if client.collection_exists(edges):
|
||||
# Prüfen, ob die Edges-Collection bereits eine Vektorkonfig hat
|
||||
try:
|
||||
info = client.get_collection(edges)
|
||||
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
|
||||
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
|
||||
except Exception:
|
||||
# konservativ: nichts anfassen, um Datenverlust zu vermeiden
|
||||
has_vectors = True
|
||||
|
||||
if not has_vectors:
|
||||
if destructive:
|
||||
client.delete_collection(edges)
|
||||
_create_edges(client, edges)
|
||||
else:
|
||||
print(
|
||||
f"[ensure_collections] WARN: '{edges}' ohne VectorConfig gefunden; "
|
||||
f"keine destruktive Änderung (destructive=False).",
|
||||
flush=True,
|
||||
)
|
||||
print(f"[ensure_collections] WARN: '{edges}' ohne VectorConfig; destructive=False.", flush=True)
|
||||
else:
|
||||
_create_edges(client, edges)
|
||||
|
||||
|
||||
def collection_names(prefix: str) -> Tuple[str, str, str]:
|
||||
"""
|
||||
Abwärtskompatibler Helper für Scripts:
|
||||
returns (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
||||
"""
|
||||
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
||||
|
||||
|
||||
def wipe_collections(client: QdrantClient, prefix: str) -> None:
|
||||
"""
|
||||
Löscht alle drei Collections – nur verwenden, wenn bewusst ein Clean-Rebuild gewünscht ist.
|
||||
"""
|
||||
for name in collection_names(prefix):
|
||||
if client.collection_exists(name):
|
||||
client.delete_collection(name)
|
||||
# -------------------------------
|
||||
# NEW: Payload-Indexing
|
||||
# -------------------------------
|
||||
|
||||
def _safe_create_index(client: QdrantClient, col: str, field: str, schema: rest.PayloadSchemaType):
|
||||
try:
|
||||
client.create_payload_index(
|
||||
collection_name=col,
|
||||
field_name=field,
|
||||
field_schema=schema,
|
||||
)
|
||||
except Exception:
|
||||
# bereits vorhanden oder nicht unterstütztes Schema → ignorieren
|
||||
pass
|
||||
|
||||
def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
|
||||
notes, chunks, edges = collection_names(prefix)
|
||||
# Notes
|
||||
_safe_create_index(client, notes, "note_id", rest.PayloadSchemaType.KEYWORD)
|
||||
# Chunks
|
||||
_safe_create_index(client, chunks, "note_id", rest.PayloadSchemaType.KEYWORD)
|
||||
_safe_create_index(client, chunks, "chunk_index", rest.PayloadSchemaType.INTEGER)
|
||||
# Edges
|
||||
for f in ("kind", "scope", "source_id", "target_id", "note_id"):
|
||||
_safe_create_index(client, edges, f, rest.PayloadSchemaType.KEYWORD)
|
||||
+79
-21
@@ -1,19 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: app/core/qdrant_points.py
|
||||
Version: v1.2.0 (2025-09-05)
|
||||
app/core/qdrant_points.py
|
||||
|
||||
Kurzbeschreibung
|
||||
Qdrant-Points für Notes/Chunks/Edges.
|
||||
NEU: Edge-ID berücksichtigt optional 'occ' (Occurrence je Match):
|
||||
"{kind}:{src}->{tgt}#{seq}@{occ}"
|
||||
Zweck
|
||||
- Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
|
||||
- Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
|
||||
- alt: {'edge_type','src_id','dst_id', ...}
|
||||
- neu: {'kind','source_id','target_id', ...}
|
||||
|
||||
Aufruf
|
||||
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
|
||||
Version
|
||||
- 1.3 (2025-09-08)
|
||||
|
||||
Changelog
|
||||
v1.2.0: Edge-ID mit '@occ' falls vorhanden.
|
||||
Änderungen (ggü. 1.2)
|
||||
- points_for_edges() akzeptiert jetzt beide Edge-Schemata.
|
||||
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
|
||||
stabile 'edge_id' zurück in die Payload.
|
||||
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
|
||||
|
||||
Aufruf / Verwendung
|
||||
- Wird von Import-/Backfill-Skripten via:
|
||||
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
|
||||
eingebunden. Keine CLI.
|
||||
|
||||
Hinweise
|
||||
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
|
||||
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,13 +33,23 @@ import uuid
|
||||
from typing import List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
|
||||
def _to_uuid(stable_key: str) -> str:
|
||||
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||
|
||||
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
|
||||
|
||||
def points_for_note(
|
||||
prefix: str,
|
||||
note_payload: dict,
|
||||
note_vec: List[float] | None,
|
||||
dim: int,
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
|
||||
notes_col, _, _ = _names(prefix)
|
||||
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
||||
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
||||
@@ -35,7 +57,18 @@ def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | Non
|
||||
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
||||
return notes_col, [pt]
|
||||
|
||||
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
|
||||
def points_for_chunks(
|
||||
prefix: str,
|
||||
chunk_payloads: List[dict],
|
||||
vectors: List[List[float]],
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Chunks-Collection: erwartet pro Chunk einen Vektor.
|
||||
Robustheit:
|
||||
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
|
||||
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
|
||||
"""
|
||||
_, chunks_col, _ = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
||||
@@ -48,30 +81,55 @@ def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[Lis
|
||||
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
||||
return chunks_col, points
|
||||
|
||||
|
||||
def _normalize_edge_payload(pl: dict) -> dict:
|
||||
"""
|
||||
Sorgt für kompatible Feldnamen.
|
||||
akzeptiert:
|
||||
- neu: kind, source_id, target_id, seq?
|
||||
- alt: edge_type, src_id, dst_id, order?/index?
|
||||
schreibt zurück: kind, source_id, target_id, seq?
|
||||
"""
|
||||
# bereits neu?
|
||||
kind = pl.get("kind") or pl.get("edge_type") or "edge"
|
||||
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
|
||||
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
|
||||
seq = pl.get("seq") or pl.get("order") or pl.get("index")
|
||||
|
||||
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
|
||||
pl.setdefault("kind", kind)
|
||||
pl.setdefault("source_id", source_id)
|
||||
pl.setdefault("target_id", target_id)
|
||||
if seq is not None and "seq" not in pl:
|
||||
pl["seq"] = seq
|
||||
return pl
|
||||
|
||||
|
||||
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Edges-Collection mit 1D-Dummy-Vektor (keine Vektorsuche auf Edges).
|
||||
Stable-ID:
|
||||
Basis: "{kind}:{src}->{tgt}#{seq}"
|
||||
Neu: Falls 'occ' vorhanden -> "{…}@{occ}"
|
||||
Edges-Collection mit 1D-Dummy-Vektor.
|
||||
- Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
|
||||
- Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
|
||||
"""
|
||||
_, _, edges_col = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for pl in edge_payloads:
|
||||
for raw in edge_payloads:
|
||||
pl = _normalize_edge_payload(raw)
|
||||
|
||||
edge_id = pl.get("edge_id")
|
||||
if not edge_id:
|
||||
kind = pl.get("kind", "edge")
|
||||
s = pl.get("source_id", "unknown-src")
|
||||
t = pl.get("target_id", "unknown-tgt")
|
||||
seq = pl.get("seq") or pl.get("order") or ""
|
||||
occ = pl.get("occ")
|
||||
base = f"{kind}:{s}->{t}#{seq}"
|
||||
edge_id = f"{base}@{occ}" if occ is not None else base
|
||||
seq = pl.get("seq") or ""
|
||||
edge_id = f"{kind}:{s}->{t}#{seq}"
|
||||
pl["edge_id"] = edge_id
|
||||
|
||||
point_id = _to_uuid(edge_id)
|
||||
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
||||
return edges_col, points
|
||||
|
||||
|
||||
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
|
||||
if not points:
|
||||
return
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
# 📘 Mindnet Skript-Handbuch
|
||||
|
||||
Dieses Handbuch dokumentiert die implementierten Skripte für das Projekt **mindnet Wissensnetzwerk**.
|
||||
Es beschreibt Zweck, Parameter und typische Aufrufe. Alle Beispiele sind geprüft und lauffähig.
|
||||
|
||||
Alle Skripte liegen in `~/mindnet/scripts/`.
|
||||
Ausführung erfolgt **im aktivierten venv**:
|
||||
|
||||
cd ~/mindnet
|
||||
source .venv/bin/activate
|
||||
|
||||
---
|
||||
|
||||
## 1. Vault anlegen (für Tests)
|
||||
|
||||
### `make_test_vault.py`
|
||||
|
||||
**Zweck:**
|
||||
Erstellt einen kleinen, nachvollziehbaren Test-Vault mit Markdown-Dateien.
|
||||
Szenarien: offene Links, nachträgliche Anlage fehlender Notes, geänderte Chunk-Grenzen.
|
||||
|
||||
**Parameter:**
|
||||
- `--out PATH` : Zielverzeichnis (Default: `./test_vault`)
|
||||
- `--force` : Existierenden Ordner löschen und neu anlegen
|
||||
- `--with-missing` : Optional – auch die `missing-note` gleich anlegen
|
||||
|
||||
**Hinweise für Tests**
|
||||
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
|
||||
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
|
||||
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
|
||||
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
|
||||
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
|
||||
|
||||
Kompatibel mit:
|
||||
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
|
||||
|
||||
**Beispiele:**
|
||||
|
||||
python3 -m scripts.make_test_vault --out ./test_vault --force
|
||||
python3 -m scripts.make_test_vault --out ./test_vault --force --with-missing
|
||||
|
||||
---
|
||||
|
||||
## 2. Import in Qdrant
|
||||
|
||||
### `import_markdown.py`
|
||||
|
||||
**Zweck:**
|
||||
Importiert Notes/Chunks/Edges aus einem Vault nach Qdrant.
|
||||
- prüft Frontmatter
|
||||
- chunked Inhalte + Embeddings
|
||||
- erzeugt Edges (`references`, `references_at`, `backlink`)
|
||||
- erkennt geänderte Dateien automatisch
|
||||
- optional: Purge der alten Chunks/Edges pro Note
|
||||
|
||||
**Parameter:**
|
||||
- `--vault PATH` : (Pflicht) Vault-Verzeichnis
|
||||
- `--apply` : Änderungen wirklich durchführen (sonst Dry-Run)
|
||||
- `--purge-before-upsert` : Vor Upsert alte Chunks/Edges **nur der geänderten Notes** löschen
|
||||
- `--note-id ID` : Nur eine bestimmte Note importieren
|
||||
|
||||
**Beispiele:**
|
||||
|
||||
python3 -m scripts.import_markdown --vault ./vault
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
||||
|
||||
---
|
||||
|
||||
## 3. Konsistenzprüfungen
|
||||
|
||||
### `audit_vault_vs_qdrant.py`
|
||||
|
||||
**Zweck:**
|
||||
Vergleicht Vault-Inhalte mit Qdrant.
|
||||
- Zählt Notes/Chunks/Edges
|
||||
- Verifiziert Anzahl Wikilinks vs. Edges
|
||||
- Meldet Mismatches pro Note
|
||||
|
||||
**Parameter:**
|
||||
- `--vault PATH` : Vault-Verzeichnis
|
||||
- `--prefix STR` : Prefix für Qdrant-Collections (Default: `mindnet`)
|
||||
|
||||
**Beispiel:**
|
||||
|
||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
||||
|
||||
---
|
||||
|
||||
### `validate_edges.py`
|
||||
|
||||
**Zweck:**
|
||||
Prüft die Edges-Collection in Qdrant.
|
||||
- Zählt Edges pro Typ
|
||||
- Kontrolliert Invarianten (z. B. jede Reference hat Backlink)
|
||||
- Listet unresolved Edges auf
|
||||
|
||||
**Parameter:**
|
||||
- `--prefix STR` : Prefix für Qdrant-Collections
|
||||
- `--details` : Detailausgabe mit Beispielen
|
||||
|
||||
**Beispiel:**
|
||||
|
||||
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||
|
||||
---
|
||||
|
||||
## 4. Qdrant zurücksetzen
|
||||
|
||||
### `reset_qdrant.py`
|
||||
|
||||
**Zweck:**
|
||||
Setzt die Qdrant-Collections zurück.
|
||||
- „wipe“: löscht Collections komplett (inkl. Schema)
|
||||
- „truncate“: löscht nur Inhalte, behält Collections
|
||||
|
||||
**Parameter:**
|
||||
- `--mode {wipe,truncate}` : Aktion
|
||||
- `--prefix STR` : Prefix der Collections (Default: `mindnet`)
|
||||
- `--yes` : Automatische Bestätigung
|
||||
- `--dry-run` : Nur anzeigen, nicht ausführen
|
||||
|
||||
**Beispiele:**
|
||||
|
||||
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
||||
python3 -m scripts.reset_qdrant --mode wipe --prefix mindnet --yes
|
||||
|
||||
---
|
||||
|
||||
## 5. Diagnose
|
||||
|
||||
### `diagnose_changed.py`
|
||||
|
||||
**Zweck:**
|
||||
Prüft, ob die Datei-Hashes (`sha256`) mit den gespeicherten `hash_fulltext` in Qdrant übereinstimmen.
|
||||
Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt.
|
||||
|
||||
**Parameter:**
|
||||
- `--vault PATH` : Vault-Verzeichnis
|
||||
- `--prefix STR` : Prefix (Default: `mindnet`)
|
||||
- `--note-id ID` : Nur bestimmte Note prüfen
|
||||
- `--all` : Alle Notes prüfen
|
||||
|
||||
**Beispiel:**
|
||||
|
||||
python3 -m scripts.diagnose_changed --vault ./vault --prefix mindnet --all
|
||||
|
||||
---
|
||||
|
||||
## 6. Unresolved Links nachträglich auflösen
|
||||
|
||||
### `resolve_unresolved_references.py`
|
||||
|
||||
**Zweck:**
|
||||
Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwischen passende Notizen existieren.
|
||||
|
||||
**Parameter:**
|
||||
- `--prefix STR` : Prefix für Qdrant-Collections
|
||||
- `--apply` : Änderungen wirklich ausführen (sonst nur Vorschau)
|
||||
|
||||
**Beispiele:**
|
||||
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
||||
|
||||
---
|
||||
|
||||
# ✅ Typische Workflows
|
||||
|
||||
### A) Vault neu importieren
|
||||
|
||||
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
||||
|
||||
### B) Änderungen einspielen (mit Purge für geänderte Notes)
|
||||
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
||||
|
||||
### C) Unresolved Links nachziehen
|
||||
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
||||
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||
|
||||
### D) Test-Vault Szenario durchspielen
|
||||
|
||||
python3 -m scripts.make_test_vault --out ./test_vault --force
|
||||
python3 -m scripts.import_markdown --vault ./test_vault --apply
|
||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
|
||||
+106
-116
@@ -13,74 +13,67 @@ Entwickle eine robuste, parser-freundliche **Chunking-Strategie** für Markdown-
|
||||
---
|
||||
|
||||
## 2) Annahmen aus `knowledge_design.md`
|
||||
- **Stabile IDs** & Dateinamen-Schema (z. B. `YYYYMMDD-HHMM-type-slug` bzw. `type-slug`).
|
||||
- **Pflichtfelder** in YAML: `title`, `id`, `type`, `status`, `created`, `tags` (empfohlene Felder ggf. mehr).
|
||||
- **Abschnitts-Konventionen** je `type` (z. B. `## Zusammenfassung`, `## Kontext` …) → ideale **Chunk-Grenzen**.
|
||||
- **Edge-Typen**: `belongs_to`, `references`, `backlink`, `depends_on`, `assigned_to`, `discussed_in`, `authored_by`, `related`. Für Chunking primär `belongs_to` & `references` (+ optionale `prev`/`next`).
|
||||
- **IDs**: deterministische, stabile `id` pro Note (Slug / semantischer Identifier), ggf. `aliases`.
|
||||
- **Dateinamen**: entweder `YYYY-MM-DD_title.md` oder `slug.md`; Pfade sind **relativ** zum Vault.
|
||||
- **Frontmatter-Pflichtfelder**: `id`, `title`, `type`, `created`, `updated` (Details in `knowledge_design.md`).
|
||||
- **Backlink-/Link-Regeln**: Wikilinks `[[id]]` im Body; am Ende optional redaktionelle Sektion „Mögliche Verbindungen“ (erzeugt **keine** Kanten).
|
||||
|
||||
---
|
||||
|
||||
## 3) Chunking-Prinzipien
|
||||
1. **Frontmatter extrahieren** → als strukturiertes `metadata` (nicht in `text` des Chunks).
|
||||
2. **Markdown AST** (oder strukturierter Parser) nutzen:
|
||||
- Primäre Grenzen: **Überschriftenebene H2/H3** (H1 = Dokumenttitel).
|
||||
- Sekundär: **Absätze**, **nummerierte/unnummerierte Listen** als atomare Einheiten, **Codeblöcke** als zusammenhängender Block (nicht splitten).
|
||||
3. **Längensteuerung**: Semantik vor harter Länge. Wenn Abschnitt > Zielgröße, **weich segmentieren** entlang von Absätzen/Listenpunkten.
|
||||
4. **Referenz-Extraktion**: `[[Wikilinks]]` + Markdown-Links → `references`. Backlinks werden downstream generiert.
|
||||
5. **Kontext-Overlap**: Chunks erhalten **Überlappung** über Satz-/Absatzgrenzen, um Kohärenz beim Retrieval zu erhöhen.
|
||||
- **Semantische Schnitte**: Chunks enden an Absätzen, Überschriften, Listenpunkten, Codeblöcken, Zitaten.
|
||||
- **Frontmatter**: YAML **nicht** embedden; stattdessen komplett im Note-Payload speichern.
|
||||
- **Verlustfreiheit**:
|
||||
- Note-Payload: `fulltext` (gesamter Body).
|
||||
- Chunk-Payload: `text` (Originalchunk).
|
||||
Damit kann der Export den Body **verlustfrei** rekonstruieren (erst `fulltext`, sonst Chunks).
|
||||
|
||||
---
|
||||
|
||||
## 4) Optimale Chunkgrößen & Overlaps
|
||||
> Ziel: gutes Recall@k, wenig Fragmentierung, stabile Antwortqualität.
|
||||
- **Zielgröße**: 250–400 Tokens (Richtwert, abhängig vom `type`).
|
||||
- **Maxgröße**: 600–800 Tokens (nur in Ausnahmen).
|
||||
- **Mingröße**: 80–120 Tokens (sonst mit Nachbar verschmelzen).
|
||||
- **Overlap**: 40–60 Tokens, um Kontextverlust zu vermeiden.
|
||||
- **Kontextabhängig**:
|
||||
- `concept`, `thought`, `experience`: 250–350 Tokens
|
||||
- `task`, `project`: eher 200–300 Tokens (mehr Bullet-Struktur)
|
||||
- `journal`: 300–400 Tokens (tagesabschnittsweise)
|
||||
|
||||
| Notiz-Typ (`type`) | Inhalt | Zielgröße (Tokens) | Max. Größe (Tokens) | Overlap (Tokens) | Hinweise |
|
||||
|---|---|---:|---:|---:|---|
|
||||
| `thought` | kurze Thesen/Ideen | 150–250 | 300 | 30–40 | meist 1–2 Chunks |
|
||||
| `experience` | Kontext/Beobachtung/Interpretation | 250–350 | 450 | 40–60 | Trenne entlang der Template-Sektionen |
|
||||
| `journal` | Tagebuch, episodisch | 200–300 | 400 | 30–50 | Tagesabschnitte bündeln |
|
||||
| `task` | knapp, Checklisten | 120–200 | 250 | 20–30 | Listen nicht splitten |
|
||||
| `project` | Scope/WPs/Risiken/Status | 300–450 | 600 | 50–70 | Pro Hauptsektion ein Chunk |
|
||||
| `concept` | definierte Begriffe, Erklärungen | 250–400 | 550 | 40–60 | Definition separat halten |
|
||||
| `source` | Metadaten + Auszüge/Notizen | 200–350 | 500 | 30–50 | Zitate als eigene Chunks (Urheberrecht) |
|
||||
---
|
||||
|
||||
## 4.1 Overlap-Regeln
|
||||
- Overlap **nie mitten im Satz** beenden → mindestens **ein kompletter Satz** wandert in den nächsten Chunk.
|
||||
- An **Absatzgrenzen** ausrichten, wenn möglich.
|
||||
- Bei **Listen** ggf. den vorherigen Punkt als Overlap mitführen (falls < 50 Tokens).
|
||||
- Codeblöcke und Tabellen **nicht splitten**; wenn sie größer als `Max-Tokens` sind, als **eigener Chunk** belassen (Ausnahmefall zulassen).
|
||||
**Daumenregel (zeichenzentriert):** ~4 Zeichen ≈ 1 Token → 800–1600 Zeichen Ziel; Overlap ~120–240 Zeichen (≈ 30–60 Tokens).
|
||||
**Mindestanforderung:** Keine Trennung innerhalb von Sätzen. Sicherstellung, dass immer an Satzenden gechunct wird
|
||||
**Default für mindnet:** **~300 Tokens** + **~50 Tokens Overlap**; per `type` feinjustieren (siehe Tabelle).
|
||||
- Overlap betrifft **nur den Body**; die YAML-Frontmatter bleibt **außerhalb**.
|
||||
- Keine Overlaps, die Codeblöcke oder Tabellen „aufreißen“. Falls nötig: Chunk-Grenze an Blockgrenze ziehen.
|
||||
- Bei reinen Listen: zusammenhängende Bullets gruppieren, ggf. kleinere Overlaps.
|
||||
|
||||
---
|
||||
|
||||
## 5) Semantische Regeln je Strukturelement
|
||||
- **Überschriften (H2/H3)**: new chunk start; speichere `section_path` (z. B. `/Kontext/Beobachtung`).
|
||||
- **Absätze**: Primäre Untereinheiten; wenn Zusammenfassung + Beispiel direkt folgen, zusammen lassen.
|
||||
- **Listen**: Jeder **Listeneintrag** bleibt zusammen mit seinem einleitenden Satz (falls vorhanden). Große Listen → in **logische Blöcke** (5–8 Items) splitten.
|
||||
- **Codeblöcke**: Nicht splitten; ggf. separater Chunk, **aber** im Retrieval **downweighten** (optional).
|
||||
- **Zitate/Blockquotes**: zusammenhalten; Quelle im `references_meta` anreichern (falls Link vorhanden).
|
||||
- **Tabellen**: als ein Chunk; zusätzlich **Plain-Text-Extrakt** (Header + 1–2 Zeilen) in `summary` für Retriever, falls Modelle Tabellen schlechter verarbeiten.
|
||||
- **Überschriften**: vorzugsweise an H2/H3 trennen; H1 meist Dokumenttitel → Teil der Frontmatter.
|
||||
- **Absätze**: primäre Trennkandidaten.
|
||||
- **Listen**: zusammenhängende Gruppe als ein Chunk, wenn sie einen semantischen Block bilden.
|
||||
- **Codeblöcke**: niemals splitten; Code bleibt in einem Chunk.
|
||||
- **Zitate**: ab 2+ Zeilen als eigener Chunk.
|
||||
- **Bilder/Anhänge**: Referenzen im Text belassen; Pfade relativ halten.
|
||||
|
||||
---
|
||||
|
||||
## 6) Normalisierung & Preprocessing
|
||||
- **Nicht embedden:** YAML-Frontmatter (aber spezifische Felder als `metadata.*` mitschreiben: `type`, `tags`, `area`, `project`, `priority`, `people`, `aliases`).
|
||||
- **Bereinigen:** führende/trailing Spaces, Mehrfach-Leerzeilen → 1; Unicode-NFKC; Zeilenenden normalisieren.
|
||||
- **Bewahren:** Markdown-Semantik (Überschriften-Hashes, Listenpräfixe, Codefences) **im Text beibehalten**, damit RAG-Antworten sauber zitieren können.
|
||||
- **Sprachhinweis:** `lang` heuristisch bestimmen (DE/EN/…); in `metadata.lang` ablegen → später für sprachspezifische Embeddings nützlich.
|
||||
- **Stop-Abschnitte:** „Mögliche Verbindungen“ am Ende **separat** chunken (nur Links), um Link-Graph sauber zu extrahieren.
|
||||
- **Zeilenenden**: `\r\n` → `\n`, trailing spaces trimmen (nur für Hashing, nicht für Anzeige).
|
||||
- **Whitespace**: mehrfach-Blankzeilen reduzieren (optional, Hash-Modus beachten).
|
||||
- **Links**: `[[Titel|id]]` zu `(display="Titel", target_id="id")` normalisieren.
|
||||
- **Pfade**: `path` im Payload **zwingend relativ** (Backslashes → Slashes).
|
||||
|
||||
---
|
||||
|
||||
## 7) Edge-Modell (Graph)
|
||||
- **Chunk → Note:** `belongs_to(note_id)` (obligatorisch).
|
||||
- **Chunk → Chunk (gleiche Note):** `prev`, `next` (lineare Leserichtung).
|
||||
- **Note/Chunk → Note:** `references(target_id)` aus `[[Wikilinks]]` & Markdown-Links; optional `related` (schwach).
|
||||
- **Backlink:** systemisch generiert: inverse Kante `backlink`.
|
||||
- **Sektionen:** `has_section(note_id, section_path)` (optional, für Navigations-UI).
|
||||
- **Task-Spezialfälle:** `depends_on`, `assigned_to(person-id)` bleiben **Note-Level** (nicht Chunk-Level).
|
||||
- **Chunk → Note:** `belongs_to(note_id)` (obligatorisch; Edge-Payload enthält zusätzlich `note_id` als Owner-Note).
|
||||
- **Chunk → Chunk (gleiche Note):** `prev`, `next` (symmetrisch für lineare Navigation; beide Richtungen werden erzeugt).
|
||||
- **Chunk → Note:** `references(target_id)` aus `[[Wikilinks]]` oder Markdown-Links im Chunk-Text (Standard; *Chunk-Scope*).
|
||||
- **Note → Note:** `backlink(target_id)` automatisch generiert, dedupliziert pro Note (*Note-Scope*).
|
||||
- **Optional:** `references:note` (Note-Scope-References) per Flag/ENV aktivierbar; **Default = aus**, um Doppelzählungen zu vermeiden.
|
||||
- **Unresolved:** Wenn `target_id` nicht existiert, wird eine Kante mit `status: "unresolved"` angelegt (später durch Resolver/Agent heilbar).
|
||||
|
||||
---
|
||||
|
||||
@@ -100,26 +93,49 @@ Entwickle eine robuste, parser-freundliche **Chunking-Strategie** für Markdown-
|
||||
"project": "project-mindnet",
|
||||
"tags": ["area/mindnet","type/thought","topic/yaml"],
|
||||
"chunk_index": 2,
|
||||
"char_start": 1240,
|
||||
"char_end": 2012,
|
||||
"token_count": 305,
|
||||
"char_start": 1024,
|
||||
"char_end": 1789,
|
||||
"section_title": "Begründung / Details",
|
||||
"section_path": "/Begründung / Details",
|
||||
"lang": "de",
|
||||
"wikilinks": ["concept-vektorsuche-qdrant","source-obsidian-properties"],
|
||||
"external_links": [],
|
||||
"references": [
|
||||
{"target_id": "concept-vektorsuche-qdrant", "kind": "wikilink"}
|
||||
],
|
||||
"neighbors": {"prev": "#c01", "next": "#c03"},
|
||||
"created_at": "2025-09-02T18:35:00+02:00"
|
||||
"section_path": "Begründung / Details",
|
||||
"neighbors": {"prev": "…#c01", "next": "…#c03"},
|
||||
"wikilinks": ["concept-yaml-styleguide"],
|
||||
"references": [{"kind":"wikilink", "target_id":"concept-yaml-styleguide"}],
|
||||
"text": "… ursprünglicher Chunk-Text …"
|
||||
}
|
||||
}
|
||||
|
||||
**Collection-Einstellungen (Empfehlung)**
|
||||
- `distance`: `Cosine` (gemischte Textlängen)
|
||||
- `vectors`: 384–1024 Dim (abh. Embedding-Modell)
|
||||
- `hnsw_ef_construct`: 128–256; `m`: 16–32 (Workload-abhängig)
|
||||
**Point (Note)**
|
||||
|
||||
{
|
||||
"id": "20250902-1830-thought-yaml-recall",
|
||||
"vector": [/* optional Nullvektor oder Zentroid */],
|
||||
"payload": {
|
||||
"note_id": "20250902-1830-thought-yaml-recall",
|
||||
"title": "Gedanke: Einheitliche YAML-Standards steigern Recall",
|
||||
"type": "thought",
|
||||
"status": "draft",
|
||||
"created": "2025-09-02",
|
||||
"updated": "2025-09-02",
|
||||
"path": "10_thoughts/20250902-1830-thought-yaml-recall.md",
|
||||
"tags": ["area/mindnet","type/thought","topic/yaml"],
|
||||
"hash_fulltext": "sha256:…",
|
||||
"fulltext": "… gesamter Body für verlustfreie Rekonstruktion …",
|
||||
"references": ["concept-yaml-styleguide"] // Fallback für Note-Level-Backlinks
|
||||
}
|
||||
}
|
||||
|
||||
**Edges (Payload; neues Schema)**
|
||||
|
||||
{
|
||||
"kind": "references" | "backlink" | "belongs_to" | "prev" | "next",
|
||||
"source_id": "<chunk_id|note_id>",
|
||||
"target_id": "<note_id|chunk_id>",
|
||||
"scope": "chunk" | "note",
|
||||
"note_id": "<owner-note-id>", // neu: für schnelles Filtern/Purge
|
||||
"status": "ok" | "unresolved" // optional
|
||||
}
|
||||
|
||||
**Performance-Hinweis (Qdrant):** Für schnelle Filter/Deletes Payload-Indizes anlegen: **edges** → `kind`, `scope`, `source_id`, `target_id`, `note_id`; **chunks** → `note_id`, `chunk_index`; **notes** → `note_id`.
|
||||
|
||||
---
|
||||
|
||||
@@ -136,79 +152,53 @@ Entwickle eine robuste, parser-freundliche **Chunking-Strategie** für Markdown-
|
||||
blocks = group_blocks(sec, keep_code=True, keep_lists=True)
|
||||
for group in soft_wrap(blocks, target_tokens=target_size(meta["type"]),
|
||||
max_tokens=max_size(meta["type"])):
|
||||
raw_chunks.append(render_markdown(group))
|
||||
raw_chunks.append(render_md(group))
|
||||
|
||||
chunks = apply_overlap(raw_chunks, overlap_tokens=overlap_size(meta["type"]))
|
||||
chunks = trim_whitespace(chunks)
|
||||
chunks = apply_overlap(raw_chunks, overlap_tokens=overlap(meta["type"]))
|
||||
chunks = normalize_chunks(chunks) # trim, fix code fences, etc.
|
||||
|
||||
# link extraction
|
||||
for i, ch in enumerate(chunks):
|
||||
ch.payload.update({
|
||||
"wikilinks": extract_wikilinks(ch.text),
|
||||
"external_links": extract_md_links(ch.text),
|
||||
"references": [{"target_id": t, "kind": "wikilink"} for t in extract_wikilinks(ch.text)],
|
||||
"neighbors": {"prev": f"#c{i-1}" if i>0 else None,
|
||||
"next": f"#c{i+1}" if i<len(chunks)-1 else None}
|
||||
})
|
||||
# IDs & Nachbarn
|
||||
for i, ch in enumerate(chunks, start=1):
|
||||
ch.id = f"{meta['id']}#c{i:02d}"
|
||||
ch.index = i
|
||||
ch.neighbors_prev = chunks[i-2].id if i > 1 else None
|
||||
ch.neighbors_next = chunks[i].id if i < len(chunks) else None
|
||||
|
||||
# attach metadata
|
||||
for idx, ch in enumerate(chunks):
|
||||
ch.id = f'{meta["id"]}#c{idx:02d}'
|
||||
ch.payload |= minimal_payload_from_meta(meta, file_path, idx, ch)
|
||||
# Wikilinks je Chunk
|
||||
for ch in chunks:
|
||||
ch.wikilinks = extract_wikilinks(ch.text)
|
||||
|
||||
return chunks
|
||||
|
||||
**Hinweise zur Implementierung**
|
||||
- **Tokenizer**: gleiche Tokenizer-Funktion für **Längensteuerung** und **Embedding** nutzen (Leak vermeiden).
|
||||
- **Overlap** implementieren auf **Satz-/Absatzebene**; notfalls anhand Token-Fenster (kein Wort-Cut).
|
||||
- **Heuristik** für `target_size`/`overlap_size` per `type` (siehe Tabelle).
|
||||
- **Später**: Re-Chunking ermöglichen (idempotente Re-Runs), Chunk-IDs stabil halten (Index + Hash über `section_path + first_64_tokens`).
|
||||
|
||||
---
|
||||
|
||||
## 10) Einbettung (Embeddings)
|
||||
- **Standard:** nur `chunk.text` embedden (ohne YAML), **aber**:
|
||||
- `aliases`, `title` können als **synthetischer Kontext** *vorangestellt* werden (konkateniert, durch Trennzeichen), falls Recall für Synonyme wichtig ist.
|
||||
- **Mehrsprachigkeit:** pro Chunk die Sprache detektieren; ggf. **mehrsprachige Modelle**/Pipelines verwenden.
|
||||
- **Code-Gewichtung:** optional zweite Vektor-Spur für Code (separate Collection) falls viel Technik.
|
||||
- Modell: MiniLM/all-MiniLM-L6-v2 (384d) oder E5 (besseres Retrieval, ggf. multilingual).
|
||||
- **Note-Embeddings** optional (Zentroid der Chunks).
|
||||
- **Hashing für Change-Detection** (Importer):
|
||||
- `MINDNET_HASH_MODE = body|frontmatter|body+frontmatter` (Default: `body`)
|
||||
- `MINDNET_HASH_NORMALIZE = canonical|none` (Default: `canonical`)
|
||||
|
||||
---
|
||||
|
||||
## 11) Qualitätssicherung
|
||||
- **Unit-Tests**: Gold-Notes mit erwarteten Section-Splits und Link-Graph.
|
||||
- **Metrics**: Retrieval-Eval (nDCG@k, Recall@k) auf internen QA-Fragen.
|
||||
- **Linting**: Pre-commit prüft Pflichtfelder & Link-Integrität (IDs, Dateinamen, Wikilinks).
|
||||
- **Zähler**: `notes/chunks/edges` je Lauf prüfen.
|
||||
- **Edge-Invarianten**:
|
||||
- `belongs_to == #Chunks`
|
||||
- `next == prev == Σ(max(chunks_in_note-1, 0))`
|
||||
- `references (chunk) == Σ(Chunk-Wikilinks)`
|
||||
- `backlink == Σ(eindeutige Wikilinks je Note)`
|
||||
- **Unresolved**: sollten durch Resolver-Jobs über Zeit abnehmen.
|
||||
- **Roundtrip-Test**: Import → Export → Diff; exakt rekonstruierbare Bodies.
|
||||
|
||||
---
|
||||
|
||||
## 12) Deliverables
|
||||
1. **Python-Modul** `mindnet_chunker/` (Parser, Chunker, Link-Extractor, Qdrant-Client).
|
||||
2. **CLI** `mindnet-chunk` (stdin→stdout, oder Datei→NDJSON).
|
||||
3. **Docs**: README mit Schema & Beispielen; Tests + Fixtures.
|
||||
- Code: Parser, Chunker, Importer, Exporter, Edges.
|
||||
- Schemas: `note.schema.json`, Payload-Layouts.
|
||||
- Tests: Mini-Vault (`scripts/make_test_vault.py`), Audit/Validate.
|
||||
|
||||
---
|
||||
|
||||
## 13) Prompt für neues Chatfenster (kopieren & einfügen)
|
||||
|
||||
**Projekt:** mindnet Wissensnetzwerk
|
||||
**Rolle:** Du bist mein Assistent für NLP-Strategien.
|
||||
**Ziel:** Hilf mir, eine robuste Chunking-Strategie für Markdown-Notizen zu entwickeln, die später in Qdrant gespeichert werden.
|
||||
|
||||
**Anforderungen:**
|
||||
- Nutze das YAML-Frontmatter-Schema und die Strukturregeln aus `knowledge_design.md`. (Pflichtfelder, Abschnitts-Konventionen, Edge-Typen)
|
||||
- Definiere optimale Chunkgrößen (Tokens) nach Notiz-Typ, inkl. **Overlap-Empfehlungen**.
|
||||
- Beschreibe, wie **logische Abschnitte** (Überschriften/Absätze/Listen/Code) als Grenzen dienen.
|
||||
- Erkläre das **Edge-Modell** zwischen **Chunks** und **Notizen** (`belongs_to`, `references`, `backlink`, `prev/next`).
|
||||
- Ergebnis: ein **konkretes Konzept + Python-Outline**, das ich direkt implementieren kann (inkl. Datenmodell für Qdrant).
|
||||
|
||||
**Kontext zum System:**
|
||||
- IDs/Dateinamen/Tags gemäß `knowledge_design.md`.
|
||||
- Zielgröße pro Chunk ~**300 Tokens**, **Overlap ~50 Tokens** als Default; per `type` feinjustiert (Tabelle).
|
||||
- YAML wird **nicht** in die Embeddings gemischt; ausgewählte Felder als `payload`.
|
||||
|
||||
Bitte liefere:
|
||||
1) Zusammenfassung der Strategie,
|
||||
2) Tabelle mit Größen/Overlaps,
|
||||
3) Python-Pseudocode für Chunker,
|
||||
4) Qdrant-Payload-Beispiel,
|
||||
5) Test-Plan kurz.
|
||||
*(Unverändert)*
|
||||
+371
-303
@@ -3,9 +3,9 @@ title: "mindnet – Knowledge Design"
|
||||
id: "design-knowledge-architecture"
|
||||
type: "guide"
|
||||
status: "active"
|
||||
version: "1.2.0"
|
||||
version: "1.3.0"
|
||||
created: 2025-09-02
|
||||
updated: 2025-09-02
|
||||
updated: 2025-09-09
|
||||
tags: ["type/guide","area/mindnet","topic/obsidian","topic/knowledge-architecture"]
|
||||
owner: "Ich"
|
||||
---
|
||||
@@ -15,422 +15,490 @@ owner: "Ich"
|
||||
## 1. Gesamtziel, Anspruch & Nutzen
|
||||
|
||||
**Ziel:**
|
||||
*mindnet* ist ein persönliches Wissensnetzwerk, das **alle** relevanten Inhalte – Gedanken, Erfahrungen, Aufgaben, Projekte, Quellen, Personen, Meetings, Tagebuch/Journal, Wendepunkte und Leitbilder – **einheitlich** in Markdown + YAML ablegt. Es bildet einen **Spiegel der Persönlichkeit** (Werte, Überzeugungen, Erlebnisse) und ist zugleich so strukturiert, dass Tools (Dataview, Embeddings, Vektorsuche/Qdrant, Agenten) es **maschinenlesbar** nutzen können.
|
||||
*mindnet* ist ein persönliches Wissensnetzwerk, das **alle** relevanten Informationen über mein Leben, meine Werte, Projekte, Erfahrungen, Pläne und Erkenntnisse **strukturiert** erfasst. Es bleibt **portabel** (Markdown + YAML), ist **parser-freundlich**, und so gestaltet, dass Tools (Obsidian + Dataview, Embeddings/Vektorsuche via Qdrant, Agenten) es **maschinenlesbar** nutzen können.
|
||||
|
||||
**Anspruch an die Struktur:**
|
||||
- **Stabil**: robuste IDs, deterministische Dateinamen, konsistente Feldnamen
|
||||
- **Atomar**: ein Kerninhalt pro Datei (keine Sammelcontainer)
|
||||
- **Erweiterbar**: neue Typen (z. B. `milestone`, `manifesto`) jederzeit möglich
|
||||
- **Portabel**: reines Markdown + YAML → funktioniert in Obsidian, Git und Parsern
|
||||
- **Vernetzbar**: systematische Links/Backlinks erzeugen ein sinnvoll navigierbares Netz
|
||||
- **Parser-freundlich**: wiederkehrende Abschnittsüberschriften; Felder/Enums klar definiert
|
||||
- **Atomar**: eine Kernaussage/ein zentrales Thema pro Datei (keine Sammelcontainer)
|
||||
- **Erweiterbar**: neue Typen jederzeit möglich (ohne Migrationsschmerz)
|
||||
- **Portabel**: pures Markdown + YAML → funktioniert in Obsidian, Git, Parsern
|
||||
- **Vernetzbar**: systematische Links/Backlinks bilden ein navigierbares Wissensnetz
|
||||
- **Parser-freundlich**: wiederkehrende Abschnittsüberschriften, klare Enums
|
||||
|
||||
**Späterer Nutzen / Einsatzzwecke:**
|
||||
- **Schnelles Wiederfinden**: Dataview-Abfragen, Filter, Dashboards
|
||||
- **Selbstreflexion & Entwicklung**: persönliche Werte, Wendepunkte, Kinderentwicklung, Lernpfade
|
||||
- **Automatisierung**: Vektor-Index (Qdrant), Embeddings, LLM-Q&A, Agenten-Workflows
|
||||
- **Langfristiges Gedächtnis**: eigene Lebens-/Projektchronik reproduzierbar pflegen
|
||||
- **Aufgaben-/Projektsteuerung**: im selben System, vernetzt mit Kontextwissen
|
||||
**Nutzen / Einsatzzwecke:**
|
||||
- **Schnelles Wiederfinden**: Dataview-Abfragen, Filter, Dashboards
|
||||
- **LLM-gestütztes Arbeiten**: Q&A mit Kontext-Zitaten aus Chunks (RAG)
|
||||
- **Agenten**: Vorschläge für Verbindungen/Struktur; Auflösen von „unresolved links“
|
||||
- **Projektnavigation**: Aufgabenbeziehungen, Abhängigkeiten
|
||||
- **Langfristiges Gedächtnis**: Lebens-/Projektchronik (rückblickend und planend)
|
||||
- **Automatisierung**: Vektor-Index (Qdrant), Embeddings, LLM-Q&A, Agenten-Workflows
|
||||
|
||||
---
|
||||
|
||||
## 2. YAML-Frontmatter-Schema
|
||||
|
||||
> **Pflichtfelder:** `title`, `id`, `type`, `status`, `created`, `tags`
|
||||
> **Empfohlen:** `updated`, `area`, `project`, `priority`, `aliases` (u. a.)
|
||||
> **Empfohlen:** `updated`, `area`, `project`, `priority`, `aliases`, `lang`, `people`, `depends_on`, `source` u. a.
|
||||
|
||||
### 2.1 Feldübersicht
|
||||
|
||||
| Feld | Typ | Bedeutung | Beispiel(e) |
|
||||
|---------------------|-------------------------------|---------------------------------------------------------------------------|------------------------------------------------------------------------------|
|
||||
| `title` | string | Menschlich lesbarer Titel | `"Vektorsuche in Qdrant"` |
|
||||
| `id` | string (slug/identifier) | **Stabile** technische ID (nie umbenennen) | `"20250902-1710-thought-obsidian-struktur"`, `"concept-persoenliches-leitbild"` |
|
||||
| `type` | enum | Notiz-Typ | `thought`, `experience`, `task`, `project`, `source`, `concept`, `person`, `meeting`, `journal`, `milestone` |
|
||||
| `status` | enum | Lebenszyklus | `idea`, `draft`, `active`, `paused`, `blocked`, `done`, `archived` |
|
||||
| `created` | date/datetime | Erstellzeitpunkt | `2025-09-02` oder `2025-09-02T17:10:00+02:00` |
|
||||
| `updated` | date/datetime | Letzte inhaltliche Änderung | `2025-09-02T18:05:00+02:00` |
|
||||
| `tags` | string[] | Schlagwörter gemäß Taxonomie | `["area/mindnet","topic/obsidian","type/thought"]` |
|
||||
| `area` | enum/string | Lebens-/Arbeitsbereich | `mindnet`, `family`, `karate`, `travel`, `health`, `home-automation`, … |
|
||||
| `role` | string | Rolle/Hut (Filter) | `"father"`, `"researcher"`, `"project-owner"` |
|
||||
| `project` | string | Zugehöriges Projekt (ID oder sprechend) | `"project-mindnet"` |
|
||||
| `priority` | int (1–5) | 1 = hoch, 5 = niedrig | `2` |
|
||||
| `effort_min` | int | Aufwand in Minuten (für `task`) | `90` |
|
||||
| `due` | date | Fälligkeit (Task/Projekt) | `2025-09-10` |
|
||||
| `start` | date | geplanter Start (Task/Projekt) | `2025-09-05` |
|
||||
| `people` | string[] (Wikilinks erlaubt) | Beteiligte Personen | `["Ich","[[person-kathie]]"]` |
|
||||
| `location` | string | Ort/Platz | `"Home Office"`, `"München"` |
|
||||
| `source_url` | url | Kanonische URL (für `source`) | `"https://qdrant.tech/docs/"` |
|
||||
| `authors` | string[] | Autoren (für `source`) | `["Jane Doe"]` |
|
||||
| `published` | date | Veröffentlichungsdatum (für `source`) | `2024-11-20` |
|
||||
| `accessed` | date | Zugriffsdatum (für `source`) | `2025-09-02` |
|
||||
| `aliases` | string[] | Alternative Namen/Titel | `["Manifest","Leitbild"]` |
|
||||
| `related_ids` | string[] | ID-basierte Verknüpfungen | `["concept-persoenliches-leitbild"]` |
|
||||
| `embedding_exclude` | boolean | Von Embeddings/Index ausschließen | `false` |
|
||||
| Feld | Typ | Bedeutung |
|
||||
|--------------------|-------------------------------|---------------------------------------------------------------------------|
|
||||
| `title` | string | Menschlich lesbarer Titel |
|
||||
| `id` | string (slug/identifier) | **Stabile** ID; referenzierbar über Wikilinks |
|
||||
| `type` | enum | `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone`, … |
|
||||
| `status` | enum | `draft` \| `active` \| `done` \| `archived` |
|
||||
| `created` | date | Erstellungsdatum (ISO) |
|
||||
| `updated` | date | Letzte Änderung (ISO) |
|
||||
| `area` | string | Themen-/Lebensbereich (z. B. „mindnet“, „gesundheit“, „familie“) |
|
||||
| `project` | string | Projekt-/Arbeitskontext |
|
||||
| `tags` | list<string> | Taxonomische Tags (präfixiert, s. Abschnitt 4) |
|
||||
| `aliases` | list<string> | Alternative IDs/Titel (Slug-Resolver) |
|
||||
| `lang` | string | Sprache, optional (z. B. `de`, `en`) |
|
||||
| `priority` | enum/int | Priorität, optional (`low`/`med`/`high` oder 1–5) |
|
||||
| `due`, `effort_min`| date/int | Für Aufgaben/Projekte (Fälligkeit, Aufwand in Minuten) |
|
||||
| `people` | list<string> | Beteiligte Personen/IDs |
|
||||
| `depends_on` | list<string> | Abhängigkeiten auf andere IDs |
|
||||
| `source` | string | Quelle (Artikel, Buch, Video, …) |
|
||||
| `slug` | string | Stabiler Pfad-/Dateiname, falls abweichend |
|
||||
|
||||
### 2.2 Konventionen
|
||||
- **IDs:** entweder Zeitstempel-basiert (`YYYYMMDD-HHMM-type-slug`) **oder** stabile, sprechende (`concept-*`, `project-*`, `person-*`).
|
||||
- **Datums-/Zeitformat:** ISO 8601, Zeiten möglichst mit TZ (z. B. `+02:00`).
|
||||
- **Felder:** klein geschrieben; zusammengesetzte Felder in `snake_case`.
|
||||
- `id` ist **menschenlesbar** (Slug), eindeutig im Vault; `aliases` dienen als Fallback-IDs.
|
||||
- Datumsformat: ISO `YYYY-MM-DD`.
|
||||
- Keine duplizierten Tags; Präfixe nutzen (`area/…`, `type/…`, `topic/…`).
|
||||
- Frontmatter nur Fakten/Metadaten, **keine** ausufernden Inhalte (die stehen in den Body-Abschnitten).
|
||||
|
||||
### 2.3 Minimalbeispiel Frontmatter
|
||||
---
|
||||
title: "Kurzer, klarer Gedanke"
|
||||
id: "20250902-1715-thought-klarer-gedanke"
|
||||
type: "thought"
|
||||
status: "idea"
|
||||
created: 2025-09-02T17:15:00+02:00
|
||||
updated: 2025-09-02T17:15:00+02:00
|
||||
tags: ["area/mindnet","type/thought","topic/obsidian"]
|
||||
---
|
||||
|
||||
## 2.4 Enum-Werte
|
||||
title: "Gedanke: YAML-Standards"
|
||||
id: thought-yaml-standards
|
||||
type: thought
|
||||
status: draft
|
||||
created: 2025-09-02
|
||||
updated: 2025-09-09
|
||||
area: mindnet
|
||||
project: project-mindnet
|
||||
tags: [area/mindnet, type/thought, topic/yaml]
|
||||
aliases: [yaml-gedanken]
|
||||
lang: de
|
||||
|
||||
Zur Sicherstellung der Konsistenz werden für bestimmte Felder feste Wertemengen (Enums) verwendet:
|
||||
|
||||
- **type**
|
||||
- `idea` – spontane Gedanken, Inspirationen
|
||||
- `experience` – persönliche Erfahrungen, Reflexionen
|
||||
- `task` – konkrete Aufgaben
|
||||
- `project` – Projekte, bestehend aus mehreren Tasks
|
||||
- `concept` – abstrakte Konzepte, Modelle, Methoden
|
||||
- `milestone` – wichtige Ereignisse oder Wendepunkte
|
||||
- `person` – Personen (z. B. Kontakte, Vorbilder)
|
||||
- `meeting` – Besprechungen, Treffen, Gespräche
|
||||
- `resource` – externe Quellen (Artikel, Bücher, Links)
|
||||
- `journal` – Tagebuchnotizen
|
||||
|
||||
- **status**
|
||||
- `open`
|
||||
- `in-progress`
|
||||
- `done`
|
||||
- `blocked`
|
||||
- `canceled`
|
||||
|
||||
- **priority**
|
||||
- `low`
|
||||
- `medium`
|
||||
- `high`
|
||||
|
||||
- **area** (Beispiele – erweiterbar)
|
||||
- `personal`
|
||||
- `work`
|
||||
- `family`
|
||||
- `karate`
|
||||
- `health`
|
||||
- `finance`
|
||||
### 2.4 Enum-Werte (Beispiele)
|
||||
- `type`: `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone`, `manifesto`
|
||||
- `status`: `draft`, `active`, `done`, `archived`
|
||||
- `priority`: `low`, `med`, `high` oder 1–5
|
||||
|
||||
---
|
||||
|
||||
## 3. Dateinamen-Konventionen
|
||||
|
||||
> Keine Sonderzeichen wie `: / \` im Dateinamen.
|
||||
|
||||
- **Zeitbasierte Notizen (Gedanken, Journal, tagesgebundene Einträge):**
|
||||
`YYYYMMDD-HHMM-<type>-<slug>.md`
|
||||
Beispiel: `20250902-1710-thought-obsidian-struktur.md`
|
||||
|
||||
- **Zeitunabhängige Wissensknoten (Konzepte, Personen, Projekte, Leitbild):**
|
||||
`<type>-<slug>.md`
|
||||
Beispiele: `concept-vektorsuche-qdrant.md`, `project-mindnet.md`, `person-kathie.md`, `concept-persoenliches-leitbild.md`
|
||||
|
||||
- **Ordnerstruktur (Empfehlung):**
|
||||
00_inbox/
|
||||
10_thoughts/
|
||||
20_experiences/
|
||||
30_projects/
|
||||
31_tasks/
|
||||
40_concepts/
|
||||
50_sources/
|
||||
60_people/
|
||||
70_meetings/
|
||||
80_journal/
|
||||
90_templates/
|
||||
_attachments/
|
||||
_meta/
|
||||
- `YYYY-MM-DD_title.md` oder `slug.md`.
|
||||
- Pfade **relativ** zum Vault; keine absoluten Pfade (einheitliche Slashes `/`).
|
||||
- Verzeichnisse nach eigener Ordnung, empfohlen: `10_thoughts/`, `20_concepts/`, `30_projects/`, `40_journal/`, …
|
||||
|
||||
---
|
||||
|
||||
## 4. Tagging-Regeln & Taxonomie
|
||||
|
||||
- **Hierarchische Tags:**
|
||||
- `area/*` – Lebens-/Arbeitsbereiche: `area/mindnet`, `area/family`, `area/karate`, `area/health`, …
|
||||
- `type/*` – optionaler Spiegel von `type`: `type/thought`, `type/experience`, …
|
||||
- `topic/*` – thematische Schwerpunkte: `topic/obsidian`, `topic/yaml`, `topic/qdrant`, `topic/werte`, …
|
||||
- optional: `place/*`, `person/*`, `level/*` (z. B. `person/rouven`, `person/rohan`)
|
||||
- **Regeln:**
|
||||
- Jede Notiz hat **mindestens ein** `area/*`.
|
||||
- `topic/*` sparsam, aber konsistent; ein Mapping in `_meta/tags.md` pflegen.
|
||||
- Möglichst **keine** losen Ein-Wort-Tags ohne Präfix (verhindert Wildwuchs).
|
||||
- Präfixe verwenden: `area/…`, `type/…`, `topic/…`, `status/…`.
|
||||
- Möglichst **keine** losen Ein-Wort-Tags ohne Präfix (verhindert Wildwuchs).
|
||||
- Für Dataview konsistente Namespaces beibehalten (z. B. `topic/llm`, `topic/lernen`).
|
||||
|
||||
---
|
||||
|
||||
## 5. Backlink- & Link-Regeln
|
||||
|
||||
- **Wikilinks** verwenden: `[[stabile-id-oder-dateiname]]`
|
||||
- Im **Fließtext** bei erster Nennung verlinken; am **Ende** die Sektion **„Mögliche Verbindungen“** pflegen:
|
||||
- **Wikilinks**: `[[stabile-id]]` oder `[[Titel|stabile-id]]`
|
||||
- Im **Fließtext** verlinken (Forwardlinks); am **Ende** optional **„Mögliche Verbindungen“** redaktionell pflegen:
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-persoenliches-leitbild]]
|
||||
- [[project-mindnet]]
|
||||
|
||||
- **Linkrichtung:** „aufwärts“ (Detail → Konzept/Leitbild) und „seitwärts“ (verwandte Knoten).
|
||||
- Optional zusätzlich **ID-Relationen** im YAML (`related_ids`) für Parser.
|
||||
- **Linkrichtung:** „aufwärts“ (Detail → Konzept/Leitbild) und „seitwärts“ (verwandte Knoten).
|
||||
- Optional zusätzliche **ID-Relationen** im YAML (`related_ids`) für Parser.
|
||||
- **Unresolved Links:** Links auf noch nicht existierende Ziele sind zulässig; der Importer schreibt `status: unresolved`-Edges (später durch Resolver/Agent heilbar).
|
||||
|
||||
## 5.1 Edge-Typen
|
||||
---
|
||||
|
||||
Beziehungen zwischen Notizen, Chunks und Objekten werden als **Edges** gespeichert. Jeder Edge erhält einen `type` und optionale `meta`-Informationen (z. B. Gewichtung, Richtung).
|
||||
## 5.1 Edge-Typen & Semantik (Graph)
|
||||
|
||||
- **belongs_to**
|
||||
Chunk → Note (ein Chunk gehört zu genau einer Note)
|
||||
Beziehungen werden als **Edges** mit neuem Payload-Schema gespeichert:
|
||||
|
||||
- **references**
|
||||
Note → Note (eine Notiz verweist auf eine andere)
|
||||
kind : "belongs_to" | "next" | "prev" | "references" | "backlink" | "unresolved"
|
||||
source_id : ID des Quellknotens (Chunk- oder Note-ID)
|
||||
target_id : ID des Zielknotens (Note- oder Chunk-ID)
|
||||
scope : "chunk" | "note"
|
||||
note_id : Owner-Note des Edges (für schnelles Filtern/Purge)
|
||||
status : optional, z. B. "unresolved"
|
||||
|
||||
- **backlink**
|
||||
Automatisch generiert: Gegenkante von `references`
|
||||
**Kantenarten**
|
||||
- **belongs_to** — *Chunk → Note*
|
||||
Jeder Chunk gehört zu **genau einer** Note. *Scope:* `chunk`.
|
||||
|
||||
- **depends_on**
|
||||
Task → Task oder Task → Project (Aufgabe hängt von anderer ab)
|
||||
- **prev / next** — *Chunk → Chunk*
|
||||
Lineare Reihenfolge innerhalb derselben Note; **symmetrisch** angelegt (d. h. `prev` und `next` konsistent). *Scope:* `chunk`.
|
||||
|
||||
- **assigned_to**
|
||||
Task → Person (eine Aufgabe ist einer Person zugeordnet)
|
||||
- **references** — *Chunk → Note*
|
||||
Aus `[[Wikilinks]]` oder Markdown-Links im **Chunk-Text** (präzise Herkunft). *Scope:* `chunk`.
|
||||
|
||||
- **discussed_in**
|
||||
Note/Concept → Meeting (ein Thema wurde in einem Meeting behandelt)
|
||||
- **backlink** — *Note → Note*
|
||||
Automatisch generierte Gegenkante zu `references` (pro Note **dedupliziert**). *Scope:* `note`.
|
||||
|
||||
- **authored_by**
|
||||
Note → Person (z. B. externer Autor oder eigene Urheberschaft)
|
||||
- **unresolved**
|
||||
Edge mit `status: "unresolved"`, wenn `target_id` (noch) nicht existiert. Wird später vom Resolver/Agent ersetzt.
|
||||
|
||||
- **related**
|
||||
Generische Relation für schwache Assoziationen
|
||||
- **Optional:** `references:note` — *Note → Note*
|
||||
Zusätzliche Forwardlinks auf Note-Ebene; standardmäßig **deaktiviert**, um Doppelzählungen mit `references` (Chunk-Scope) zu vermeiden. Aktivierung per ENV/Flag möglich.
|
||||
|
||||
**Performanzhinweise (Qdrant)**
|
||||
- Edges werden dedupliziert über `(kind, source_id, target_id, scope)`.
|
||||
- `note_id` ist Pflichtfeld im Edge-Payload (Owner-Note).
|
||||
- Payload-Indizes für Qdrant anlegen:
|
||||
|
||||
edges : kind (KEYWORD), scope (KEYWORD), source_id (KEYWORD), target_id (KEYWORD), note_id (KEYWORD)
|
||||
chunks : note_id (KEYWORD), chunk_index (INTEGER)
|
||||
notes : note_id (KEYWORD)
|
||||
|
||||
---
|
||||
|
||||
## 6. Abschnitts-Konventionen im Body
|
||||
|
||||
Einheitliche Überschriften erleichtern spätere Parser und Abfragen.
|
||||
Einheitliche Überschriften erleichtern Parser, Chunking und Abfragen. Typ-spezifische Empfehlungen:
|
||||
|
||||
- **Gedanke (`thought`)**
|
||||
`## Zusammenfassung` · `## Begründung / Details` · `## Belege / Quellen` · `## Nächste Schritte` · `## Mögliche Verbindungen`
|
||||
|
||||
- **Erfahrung (`experience`)**
|
||||
`## Kontext` · `## Beobachtung` · `## Interpretation` · `## Implikationen` · `## Mögliche Verbindungen`
|
||||
|
||||
- **Aufgabe (`task`)**
|
||||
`## Checkliste` · `## Notizen` · `## Mögliche Verbindungen`
|
||||
`## Checkliste` · `## Notizen` · `## Nächste Schritte` · `## Mögliche Verbindungen`
|
||||
|
||||
- **Projekt (`project`)**
|
||||
`## Scope` · `## Arbeitspakete` · `## Risiken & Gegenmaßnahmen` · `## Status / Fortschritt` · `## Mögliche Verbindungen`
|
||||
|
||||
- **Journal (`journal`)**
|
||||
`## Tageszusammenfassung` · `## Highlights` · `## Gelernt / Notizen` · `## Mögliche Verbindungen`
|
||||
`## Tageszusammenfassung` · `## Highlights` · `## Learnings` · `## Mögliche Verbindungen`
|
||||
|
||||
- **Meeting (`meeting`)**
|
||||
`## Agenda` · `## Notizen` · `## Entscheidungen` · `## Aufgaben` · `## Mögliche Verbindungen`
|
||||
|
||||
---
|
||||
|
||||
## 7. Abbildung persönlicher Inhalte auf Typen
|
||||
|
||||
- **Persönliches Leitbild** → `type: "concept"` (oder optional neuer Typ `manifesto`)
|
||||
Datei: `concept-persoenliches-leitbild.md` (zeitunabhängig)
|
||||
Verlinkt zu prägenden Erfahrungen und Wendepunkten.
|
||||
- **Erfahrungen, die dich geprägt haben** → `type: "experience"`
|
||||
Je **eine** prägende Erfahrung pro Datei; klarer Kontext + Interpretation.
|
||||
- **Wendepunkte im Leben** → `type: "milestone"`
|
||||
Je Wendepunkt ein Knoten; verlinkt zu den auslösenden Erfahrungen.
|
||||
- **Aktuelle Erfahrungen** → `type: "experience"`, `status: "active"`
|
||||
Bei Bedarf später in „prägende“ Erfahrungen überführen.
|
||||
- **Entwicklungsschritte der Kinder** → `type: "journal"` (chronologisch) **oder** `experience` (wenn es prägend ist); Tags `person/<name>`.
|
||||
- **Tagebucheinträge** → `type: "journal"`; ein Eintrag pro Tag; tagesbasierte Dateinamen.
|
||||
|
||||
Damit entsteht ein **netzartiges Persönlichkeitsprofil** mit klaren Pfaden zwischen Werten (Leitbild), Ereignissen (Erfahrungen), Wendepunkten und fortlaufenden Beobachtungen (Journal).
|
||||
**Richtlinien:**
|
||||
- Wähle den Typ nach **Hauptintention** (Definition, Gedanke, Beobachtung, Aufgabe, Projekt, …).
|
||||
- **Konzepte** (`concept`): langlebige Definitionen, Prinzipien, Leitbilder.
|
||||
- **Gedanken** (`thought`): explorative Ideen, noch nicht stabil (können später bei Reife in `concept` überführt werden).
|
||||
- **Erfahrungen** (`experience`): Beobachtungen & Learnings aus der Praxis.
|
||||
- **Aufgaben** (`task`): konkrete, umsetzbare To-dos mit `due`/`effort_min`.
|
||||
- **Projekte** (`project`): Ziele, Deliverables, Arbeitspakete, Abhängigkeiten.
|
||||
- **Journal** (`journal`): tagesbasierte Notizen, Reflexionen, Reiseberichte.
|
||||
- **Personen** (`person`): Kontakte, Profile, Bezüge, Gespräche.
|
||||
- **Meetings** (`meeting`): Agenda, Notizen, Entscheidungen, Folgeaufgaben.
|
||||
- **Meilensteine** (`milestone`): wichtige Wendepunkte im Lebens-/Projektverlauf.
|
||||
|
||||
---
|
||||
|
||||
## 8. Beispiel-Notizen
|
||||
|
||||
### 8.1 Gedanke (`thought`)
|
||||
---
|
||||
title: "Gedanke: Einheitliche YAML-Standards steigern Recall"
|
||||
id: "20250902-1830-thought-yaml-recall"
|
||||
type: "thought"
|
||||
status: "idea"
|
||||
created: 2025-09-02T18:30:00+02:00
|
||||
updated: 2025-09-02T18:30:00+02:00
|
||||
tags: ["area/mindnet","type/thought","topic/yaml","topic/recall"]
|
||||
area: "mindnet"
|
||||
project: "project-mindnet"
|
||||
aliases: []
|
||||
---
|
||||
### 8.1 Gedanke (`thought`) – Beispiel
|
||||
|
||||
**Frontmatter**
|
||||
|
||||
title: "Gedanke: Übungen für tiefe Arbeit"
|
||||
id: thought-deep-work-uebungen
|
||||
type: thought
|
||||
status: active
|
||||
created: 2025-08-21
|
||||
updated: 2025-09-05
|
||||
area: lernen
|
||||
project: project-self-improvement
|
||||
tags: [area/lernen, type/thought, topic/fokus]
|
||||
aliases: [deep-work-ideen]
|
||||
lang: de
|
||||
|
||||
**Body (Ausschnitt)**
|
||||
|
||||
## Zusammenfassung
|
||||
Konsistente YAML-Felder verbessern Filter, Dataview-Queries und Vektor-Retrieval.
|
||||
Kurze tägliche Routinen (z. B. Timer-Blöcke, Ablenkungslog) erhöhen meinen Fokus.
|
||||
|
||||
## Begründung / Details
|
||||
- Parser benötigen stabile Feldnamen (z. B. `type`, `status`).
|
||||
- Einheitliche Werte erleichtern Dashboards und Automationen.
|
||||
- 25–45 Min Timerblöcke mit 5 Min Pause
|
||||
- Ablenkungen sofort notieren
|
||||
- Abschluss-Log am Ende des Tages
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-deep-work]]
|
||||
- [[project-productivity-week]]
|
||||
|
||||
### 8.2 Konzept (`concept`) – Beispiel
|
||||
|
||||
**Frontmatter**
|
||||
|
||||
title: "Konzept: Deep Work"
|
||||
id: concept-deep-work
|
||||
type: concept
|
||||
status: active
|
||||
created: 2024-05-14
|
||||
updated: 2025-08-01
|
||||
area: lernen
|
||||
tags: [area/lernen, type/concept, topic/fokus]
|
||||
aliases: [konzept-tiefe-arbeit]
|
||||
lang: de
|
||||
|
||||
**Body (Ausschnitt)**
|
||||
|
||||
## Definition
|
||||
Tiefe Arbeit = kognitiv anspruchsvolle Arbeit ohne Ablenkung.
|
||||
|
||||
## Belege / Quellen
|
||||
- [[source-obsidian-properties]]
|
||||
- [[concept-vektorsuche-qdrant]]
|
||||
|
||||
## Nächste Schritte
|
||||
- [ ] `tags`-Mapping in `_meta/tags.md` erstellen
|
||||
- [ ] Beispiel-Queries für Dataview hinterlegen
|
||||
- Buch: Cal Newport – Deep Work
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[project-mindnet]]
|
||||
- [[20250902-1730-task-yaml-standard-migration]]
|
||||
- [[thought-deep-work-uebungen]]
|
||||
|
||||
### 8.2 Erfahrung (`experience`)
|
||||
---
|
||||
title: "Erfahrung: Journal → Erfahrung extrahieren erhöht Klarheit"
|
||||
id: "20250902-1840-experience-journal-extraktion"
|
||||
type: "experience"
|
||||
status: "active"
|
||||
created: 2025-09-02T18:40:00+02:00
|
||||
updated: 2025-09-02T18:40:00+02:00
|
||||
tags: ["area/personal","type/experience","topic/journal","topic/reflexion"]
|
||||
area: "personal"
|
||||
project: "project-mindnet"
|
||||
---
|
||||
### 8.3 Aufgabe (`task`) – Beispiel
|
||||
|
||||
## Kontext
|
||||
Wiederkehrende Journaleinträge enthalten Rohmaterial für prägende Einsichten.
|
||||
**Frontmatter**
|
||||
|
||||
## Beobachtung
|
||||
Wird Material regelmäßig zu `experience`-Knoten verdichtet, sind spätere Bezüge einfacher.
|
||||
title: "Task: Wochenplanung erstellen"
|
||||
id: task-week-planning
|
||||
type: task
|
||||
status: active
|
||||
created: 2025-09-07
|
||||
updated: 2025-09-07
|
||||
due: 2025-09-11
|
||||
effort_min: 45
|
||||
area: organisation
|
||||
project: project-productivity-week
|
||||
tags: [area/organisation, type/task, topic/planung]
|
||||
lang: de
|
||||
|
||||
## Interpretation
|
||||
Abstraktion + Verlinkung (z. B. auf [[concept-persoenliches-leitbild]]) erhöhen Nutzen und Wiederfindbarkeit.
|
||||
|
||||
## Implikationen
|
||||
Pro Woche 15–30 Minuten „Verdichtungs-Review“ einplanen.
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[journal-20250902]]
|
||||
- [[concept-persoenliches-leitbild]]
|
||||
|
||||
### 8.3 Aufgabe (`task`)
|
||||
---
|
||||
title: "Task: YAML-Standard in Altbeständen ergänzen"
|
||||
id: "20250902-1850-task-yaml-migration"
|
||||
type: "task"
|
||||
status: "active"
|
||||
created: 2025-09-02
|
||||
updated: 2025-09-02
|
||||
due: 2025-09-10
|
||||
start: 2025-09-03
|
||||
priority: 2
|
||||
effort_min: 120
|
||||
area: "mindnet"
|
||||
project: "project-mindnet"
|
||||
tags: ["area/mindnet","type/task","topic/migration"]
|
||||
depends_on: []
|
||||
---
|
||||
**Body (Ausschnitt)**
|
||||
|
||||
## Checkliste
|
||||
- [ ] Triage: Notiz-Typ bestimmen
|
||||
- [ ] Minimal-YAML ergänzen (Pflichtfelder)
|
||||
- [ ] Dateinamen vereinheitlichen
|
||||
- [ ] „Mögliche Verbindungen“ pflegen
|
||||
|
||||
## Notizen
|
||||
Beginne mit zuletzt genutzten Notizen (Pareto 20/80).
|
||||
- Ziele der Woche definieren
|
||||
- 5 Fokusblöcke eintragen
|
||||
- 2 Lerneinheiten terminieren
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[project-mindnet]]
|
||||
- [[20250902-1830-thought-yaml-recall]]
|
||||
- [[project-productivity-week]]
|
||||
- [[thought-deep-work-uebungen]]
|
||||
|
||||
---
|
||||
|
||||
## 9. Schrittweise Migration bestehender freier Notizen
|
||||
|
||||
**Phase A – Triage (schnell)**
|
||||
1. Sichtung & **Typ** bestimmen (`type`).
|
||||
2. **Minimal-YAML** einfügen (`title`, `id`, `type`, `status=draft`, `created≈Dateidatum`, `tags`).
|
||||
3. **Dateiname** grob anpassen (Schema + Slug).
|
||||
|
||||
**Phase B – Strukturieren (fokussiert)**
|
||||
1. **Abschnitte** nach Typ-Template ergänzen.
|
||||
2. **Zerlegen**: mehrere Gedanken/Ereignisse → je **eigene Datei**, ursprüngliche Datei wird zum **Index** mit Links.
|
||||
3. **Verlinken**: relevante `[[Knoten]]` + **„Mögliche Verbindungen“** pflegen.
|
||||
|
||||
**Phase C – Verfeinern (bei Bedarf)**
|
||||
1. **Status & Felder** pflegen (`status`, `priority`, `due`, `effort_min`).
|
||||
2. **Tag-Harmonisierung**: Mapping-Tabelle pflegen (`_meta/tags.md`).
|
||||
3. **Aliases** ergänzen (Such-/Linkvarianten).
|
||||
|
||||
**Priorisierung:** Starte mit **aktuellen & häufig genutzten** Notizen (Pareto-Prinzip), Altbestände nachziehen, wenn sie wieder gebraucht werden.
|
||||
1. **Frontmatter ergänzen** (min. `id`, `title`, `type`, `status`, `created`, `tags`).
|
||||
2. **Abschnittsstruktur** an Typ anpassen (Konventionen s. Abschnitt 6).
|
||||
3. **Wikilinks** im Body setzen (statt bloßer Erwähnungen).
|
||||
4. **Tags normalisieren** (Präfixe, Kleinschreibung, keine Dubletten).
|
||||
5. **Aliases** angeben, falls Notiz zuvor unter anderem Namen existierte.
|
||||
6. **Testlauf**: Import (Dry-Run), danach Apply, Audit/Validate prüfen.
|
||||
|
||||
---
|
||||
|
||||
## 10. Dataview-Beispiele (direkt nutzbar)
|
||||
|
||||
**Offene Aufgaben dieser Woche (nach Priorität):**
|
||||
|
||||
TABLE due, priority, effort_min
|
||||
FROM "31_tasks"
|
||||
WHERE type = "task" AND status != "done"
|
||||
AND due >= date(today) - dur(1 week)
|
||||
AND due <= date(today) + dur(1 week)
|
||||
SORT priority ASC, due ASC
|
||||
**Aktive Gedanken der letzten 30 Tage**
|
||||
|
||||
**Aktive Projekte mit Meilensteinen:**
|
||||
|
||||
LIST
|
||||
FROM "30_projects"
|
||||
WHERE type = "project" AND status = "active"
|
||||
SORT priority ASC, file.ctime DESC
|
||||
|
||||
**Gedanken im Bereich mindnet, jüngste zuerst:**
|
||||
|
||||
TABLE created, status
|
||||
TABLE title, created, updated, area
|
||||
FROM "10_thoughts"
|
||||
WHERE type = "thought" AND area = "mindnet"
|
||||
SORT created DESC
|
||||
LIMIT 50
|
||||
WHERE type = "thought" AND status = "active" AND created >= date(today) - dur(30 days)
|
||||
SORT updated DESC
|
||||
|
||||
**Quellen ohne `source_url` (Pflegebedarf):**
|
||||
|
||||
TABLE title, created
|
||||
FROM "50_sources"
|
||||
WHERE type = "source" AND (source_url = "" OR !source_url)
|
||||
**Offene Tasks mit Fälligkeit**
|
||||
|
||||
TABLE title, due, effort_min, project, area
|
||||
FROM "30_tasks"
|
||||
WHERE type = "task" AND status = "active" AND due
|
||||
SORT due ASC
|
||||
|
||||
**Konzepte je Topic**
|
||||
|
||||
TABLE title, area, tags
|
||||
FROM "20_concepts"
|
||||
WHERE contains(tags, "type/concept")
|
||||
GROUP BY choice(regexreplace(join(filter(tags, (t) => startswith(t, "topic/")), ", "), "^topic/", ""))
|
||||
|
||||
**Verknüpfungskandidaten (redaktionell, nicht automatisch)**
|
||||
*(Liste am Ende der jeweiligen Notiz in „Mögliche Verbindungen“)*
|
||||
|
||||
---
|
||||
|
||||
## 11. Validierung & Pflege
|
||||
|
||||
- **Wöchentlicher Review-Slot (15–30 Min.):**
|
||||
- Neue Notizen triagieren (`type`, `status`, `area`)
|
||||
- 5 Alt-Notizen migrieren (YAML + Dateiname + Verlinkung)
|
||||
- **Dataview-Checks:** fehlende `source_url`, offene Tasks, aktive Projekte, veraltete `status`.
|
||||
- **Optional (technisch):**
|
||||
- **Pre-commit Hook**: prüft Pflichtfelder (`title`, `id`, `type`, `status`, `created`, `tags`)
|
||||
- **ID-Generator**: `YYYYMMDD-HHMM-type-slug` bei Neuanlage
|
||||
- **Link-Linter**: prüft tote `[[Links]]`
|
||||
- **Linter/Validator** prüfen Pflichtfelder und Wertebereiche (Enums, ISO-Datum).
|
||||
- **Resolver** löst `unresolved`-Edges, wenn neue Notizen/Aliases entstehen.
|
||||
- **Idempotente Re-Imports**: Hash-Steuerung konfigurierbar:
|
||||
- `MINDNET_HASH_MODE`: `body` \| `frontmatter` \| `body+frontmatter` (Default: `body`)
|
||||
- `MINDNET_HASH_NORMALIZE`: `canonical` \| `none` (Default: `canonical`)
|
||||
- **Edge-Invarianten** (Audit):
|
||||
- `belongs_to == #Chunks`
|
||||
- `next == prev == Σ(max(chunks_in_note−1, 0))`
|
||||
- `references (chunk) == Σ(Chunk-Wikilinks)`
|
||||
- `backlink == Σ(eindeutige Wikilinks je Note)`
|
||||
- **Roundtrip-Test**: Import → Export → Diff muss Body verlustfrei rekonstruieren.
|
||||
|
||||
---
|
||||
|
||||
## 12. Versions- & Änderungsrichtlinien
|
||||
## 12. Qdrant-Ablage & Indizes
|
||||
|
||||
- Dieses Dokument pflegt `version` & `updated`.
|
||||
- **Breaking Changes** (z. B. Feldumbenennungen) in `_meta/changelog.md` dokumentieren.
|
||||
- **Migrationen** (Tag-Refactorings, Typänderungen) als Skript oder Batch-Checkliste planen.
|
||||
**Collections**
|
||||
- `{prefix}_notes` – Vektor optional (Nullvektor oder Zentroid), Payload inkl. Frontmatter & `fulltext`
|
||||
- `{prefix}_chunks` – 384d Embedding je Chunk, Payload inkl. `text`, `chunk_index`, `note_id`
|
||||
- `{prefix}_edges` – 1d Dummy-Vektor, Payload im neuen Schema (s. 5.1)
|
||||
|
||||
**IDs & Idempotenz**
|
||||
- Deterministisch/UUIDv5 über stabile Schlüssel (`note_id`, `chunk_id = note_id#cNN`, `edge_id` aus `(kind, source_id, target_id, scope)`)
|
||||
- Re-Imports erzeugen keine Duplikate (Upsert)
|
||||
|
||||
**Payload-Indizes (Performance)**
|
||||
|
||||
notes : note_id (KEYWORD)
|
||||
chunks : note_id (KEYWORD), chunk_index (INTEGER)
|
||||
edges : kind (KEYWORD), scope (KEYWORD), source_id (KEYWORD), target_id (KEYWORD), note_id (KEYWORD)
|
||||
|
||||
**Delete/Purge**
|
||||
- Selektives Löschen in Edges/Chunks per `note_id`-Filter (schnell, sicher).
|
||||
- `--sync-deletes` (geplant): Qdrant→Vault Diffs gezielt entfernen (mit Dry-Run).
|
||||
|
||||
---
|
||||
|
||||
## 13. Typen-Kurzreferenz
|
||||
## 13. LLM/RAG & Agenten – Arbeitsweise (Überblick)
|
||||
|
||||
- `thought` – Gedanke/These
|
||||
- `experience` – Beobachtung/Erfahrung mit Interpretation & Implikationen
|
||||
- `task` – umsetzbare Arbeitseinheit (mit `due`, `effort_min`, `priority`)
|
||||
- `project` – Container für Ziele/Tasks/Meilensteine
|
||||
- `source` – externe Quelle (Artikel, Buch, Video)
|
||||
- `concept` – definierter Begriff / Prinzip / Leitbild
|
||||
- `person` – Bezugsperson/Kontakt
|
||||
- `meeting` – Termin mit Agenda, Notizen, erzeugten Aufgaben
|
||||
- `journal` – Tagebuch-/Reiseeintrag (tagesbasiert)
|
||||
- `milestone` – Wendepunkt / Meilenstein im Lebens-/Projektverlauf
|
||||
**Retriever (Hybrid)**
|
||||
1) **kNN (Chunks)** in Qdrant → Top-k Kandidaten (Semantik).
|
||||
2) **Nachbarschafts-Expand** (1 Hop) über Edges (`belongs_to`, `references`, `backlink`) → zusätzliche Kontexte.
|
||||
3) **Rerank** (optional BM25/LLM-score).
|
||||
4) **Prompt** mit Zitaten/Belegen (Verweis auf `path` + Abschnittstitel).
|
||||
|
||||
**Agentisches Einordnen (neue Inputs)**
|
||||
- **Chunking + Embedding** → **kNN-Kandidaten**
|
||||
- Klassifikation (`type`, `area`, `project`) via Heuristik/LLM
|
||||
- Edge-Vorschläge: `references` (Semantik + geteilte Tags/Area), `belongs_to` (beste Einordnung)
|
||||
- Falls kein Ziel existiert → `unresolved` erzeugen oder **Stub-Note** anlegen
|
||||
- **Human-in-the-loop**: Vorschläge bestätigen, dann Commit via Importer
|
||||
|
||||
**Graph-Layer (optional, empfohlen)**
|
||||
- Zusätzliche SQLite/Postgres-Tabellen `nodes/edges/aliases` für 2–3-Hop-Queries, Pfadsuche, Zählungen.
|
||||
- Importer pflegt Qdrant **und** SQL (UPSERT).
|
||||
- Graph-Features (Degree, Communities, optional PageRank) regelmäßig berechnen und als Felder in `notes.payload` speichern.
|
||||
|
||||
---
|
||||
|
||||
## 14 Erweiterungen
|
||||
## 14. Versions- & Änderungsrichtlinien
|
||||
|
||||
Zukünftige Typen möglich (z. B. decision, recommendation).
|
||||
- **Kleine Änderungen** direkt in der bestehenden Notiz (Frontmatter `updated` anpassen).
|
||||
- **Große Strukturänderungen** (z. B. neuer Typ) als neue Notiz mit Verweis; alte Note verlinken/archivieren (`status: archived`).
|
||||
- **IDs nie recyceln** (Konstanz für Referenzen und Edges).
|
||||
- **Aliases** nutzen, wenn sich Titel/Slug ändert (Resolver kann beides auflösen).
|
||||
- **Commit-Meldungen** im Git prägnant strukturieren:
|
||||
- `feat(note): …` · `fix(frontmatter): …` · `chore(tags): …` · `refactor(structure): …`
|
||||
|
||||
Personalisierung durch zusätzliche Properties (values, goals)
|
||||
---
|
||||
|
||||
## 15. Typen-Kurzreferenz
|
||||
|
||||
- **concept** — Definitionen, Leitbilder, Prinzipien
|
||||
Empfohlene Abschnitte: *Definition*, *Beispiele*, *Belege*, *Mögliche Verbindungen*
|
||||
|
||||
- **thought** — Ideen, Entwürfe, Skizzen
|
||||
Abschnitte: *Zusammenfassung*, *Details*, *Nächste Schritte*, *Mögliche Verbindungen*
|
||||
|
||||
- **experience** — Beobachtungen, Learnings
|
||||
Abschnitte: *Kontext*, *Beobachtung*, *Interpretation*, *Implikationen*, *Mögliche Verbindungen*
|
||||
|
||||
- **task** — konkrete To-dos
|
||||
Felder: `due`, `effort_min`; Abschnitte: *Checkliste*, *Notizen*, *Mögliche Verbindungen*
|
||||
|
||||
- **project** — Ziele, Deliverables, Arbeitspakete
|
||||
Abschnitte: *Scope*, *Arbeitspakete*, *Risiken*, *Status*, *Mögliche Verbindungen*
|
||||
|
||||
- **journal** — tagesbasierte Einträge
|
||||
Abschnitte: *Tageszusammenfassung*, *Highlights*, *Learnings*, *Mögliche Verbindungen*
|
||||
|
||||
- **person** — Kontakte, Profile
|
||||
Felder: `people` (Selbst-/Fremd-ID), *Notizen*, *Mögliche Verbindungen*
|
||||
|
||||
- **meeting** — Termine, Protokolle
|
||||
Abschnitte: *Agenda*, *Notizen*, *Entscheidungen*, *Aufgaben*, *Mögliche Verbindungen*
|
||||
|
||||
- **milestone** — Wendepunkte
|
||||
Abschnitte: *Kontext*, *Bedeutung*, *Verknüpfungen*
|
||||
|
||||
---
|
||||
|
||||
## 16. Erweiterungen
|
||||
|
||||
- Neue Typen (z. B. `decision`, `recommendation`, `resource`) können ohne Bruch ergänzt werden.
|
||||
- Personalisierung durch zusätzliche Properties (`values`, `goals`) in Frontmatter.
|
||||
- Internationale Inhalte: `lang` pflegen, optional **mehrsprachige** Embeddings (z. B. mE5).
|
||||
|
||||
---
|
||||
|
||||
## 17. Anhang – Beispiele & Snippets
|
||||
|
||||
**Beispiel: YAML-Frontmatter für Projekt**
|
||||
|
||||
title: "Projekt: Produktivität Woche 37"
|
||||
id: project-productivity-week
|
||||
type: project
|
||||
status: active
|
||||
created: 2025-09-08
|
||||
updated: 2025-09-09
|
||||
area: organisation
|
||||
tags: [area/organisation, type/project, topic/planung]
|
||||
people: [ich]
|
||||
depends_on: [concept-deep-work]
|
||||
lang: de
|
||||
|
||||
**Beispiel: Redaktions-Sektion (keine Kanten-Erzeugung)**
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-wochenplanung]]
|
||||
- [[thought-deep-work-uebungen]]
|
||||
- [[journal-2025-09-08]]
|
||||
|
||||
**CLI: Import & Audit**
|
||||
|
||||
# Dry-Run Import
|
||||
python3 -m scripts.import_markdown --vault ./vault
|
||||
|
||||
# Apply mit strenger Hash-Erkennung (jede Änderung zählt)
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --hash-normalize none
|
||||
|
||||
# Audit Edges vs. Erwartungen
|
||||
python3 -m scripts.audit_edges_vs_expectations --vault ./vault
|
||||
|
||||
---
|
||||
|
||||
## 18. Glossar
|
||||
|
||||
- **Alias**: alternativer Name/Slug, der auf dieselbe Note verweist
|
||||
- **Backlink**: automatisch erzeugte Gegenkante zu `references` (Note-Scope)
|
||||
- **Chunk**: semantisch sinnvoller Textabschnitt einer Note, inkl. Embedding
|
||||
- **Owner-Note**: Note, zu der ein Edge gehört (`note_id` im Edge-Payload)
|
||||
- **Resolver**: Prozess/Script, das `unresolved`-Edges bei späterer Ziel-Existenz heilt
|
||||
|
||||
---
|
||||
|
||||
## 19. Verweise
|
||||
|
||||
- `chunking_strategy.md` – Details zu Chunkgrößen, Overlaps, Abschnittsregeln
|
||||
- `schemas/note.schema.json` – Validierung von Frontmatter‐Feldern
|
||||
- Import/Export-Skripte: `scripts/import_markdown.py`, `scripts/export_markdown.py`
|
||||
- Qdrant-Client/Setup: `app/core/qdrant.py`, `app/core/qdrant_points.py`
|
||||
|
||||
---
|
||||
|
||||
## 20. Änderungsverlauf
|
||||
|
||||
- **1.3.0 (2025-09-09)**
|
||||
- Edge-Politik präzisiert: `references` (Chunk-Scope), `backlink` (Note-Scope), `references:note` optional.
|
||||
- `note_id` als Pflichtfeld im Edge-Payload (Owner).
|
||||
- Qdrant-Payload-Indizes ergänzt (Performanz).
|
||||
- Beispiele & Dataview-Snippets erweitert.
|
||||
|
||||
- **1.2.0 (2025-09-02)**
|
||||
- Erstfassung des Knowledge-Designs.
|
||||
+79
-27
@@ -2,37 +2,89 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "mindnet_chunk",
|
||||
"type": "object",
|
||||
"required": ["id", "note_id", "chunk_index", "type", "path", "token_count", "lang"],
|
||||
"description": "Chunk-Payload (Qdrant). Kompatibel mit Alt-Feldern und neuen Feldern für Export/Roundtrip.",
|
||||
"required": ["id", "note_id", "chunk_index", "path"],
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"note_id": {"type": "string"},
|
||||
"note_title": {"type": "string"},
|
||||
"chunk_index": {"type": "integer"},
|
||||
"char_start": {"type": "integer"},
|
||||
"char_end": {"type": "integer"},
|
||||
"token_count": {"type": "integer"},
|
||||
"type": {"type": "string"},
|
||||
"area": {"type": "string"},
|
||||
"project": {"type": "string"},
|
||||
"tags": {"type": "array", "items": {"type": "string"}},
|
||||
"section_title": {"type": "string"},
|
||||
"section_path": {"type": "string"},
|
||||
"lang": {"type": "string"},
|
||||
"wikilinks": {"type": "array", "items": {"type": "string"}},
|
||||
"external_links": {"type": "array", "items": {"type": "string"}},
|
||||
"id": { "type": "string" },
|
||||
"scope": { "type": "string", "enum": ["chunk"] },
|
||||
"note_id": { "type": "string" },
|
||||
"note_title": { "type": "string" },
|
||||
"note_type": { "type": "string" },
|
||||
"note_status": { "type": "string" },
|
||||
"type": { "type": "string", "description": "Legacy: früherer Chunk-Typ; kann dem Note-Typ entsprechen" },
|
||||
"area": { "type": "string" },
|
||||
"project": { "type": "string" },
|
||||
"tags": { "type": "array", "items": { "type": "string" } },
|
||||
|
||||
"note_path": { "type": "string" },
|
||||
"path": { "type": "string" },
|
||||
|
||||
"chunk_index": { "type": "integer" },
|
||||
"section_title":{ "type": ["string","null"] },
|
||||
"section_path": { "type": ["string","null"] },
|
||||
|
||||
"char_start": { "type": ["integer","null"] },
|
||||
"char_end": { "type": ["integer","null"] },
|
||||
"char_len": { "type": "integer" },
|
||||
|
||||
"token_count": { "type": "integer", "description": "Legacy: frühere Token-Zahl" },
|
||||
"token_est": { "type": "integer", "description": "Neue grobe Token-Schätzung (≈ len(text)/4)" },
|
||||
|
||||
"neighbors": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prev": { "type": ["string","null"] },
|
||||
"next": { "type": ["string","null"] }
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
|
||||
"text": { "type": "string" },
|
||||
"text_sha256": { "type": "string", "pattern": "^sha256:[0-9a-fA-F]{64}$" },
|
||||
"lang": { "type": "string" },
|
||||
|
||||
"wikilinks": { "type": "array", "items": { "type": "string" } },
|
||||
"external_links": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{ "type": "string" },
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"href": { "type": "string" },
|
||||
"label": { "type": ["string","null"] }
|
||||
},
|
||||
"required": ["href"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"references": {
|
||||
"type": "array",
|
||||
"items": {"type": "object", "properties": {
|
||||
"target_id": {"type": "string"},
|
||||
"kind": {"type": "string"}
|
||||
}, "required": ["target_id", "kind"]}
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"target_id": { "type": "string" },
|
||||
"kind": { "type": "string" }
|
||||
},
|
||||
"required": ["target_id","kind"],
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"neighbors": {"type": "object", "properties": {
|
||||
"prev": {"type": ["string","null"]},
|
||||
"next": {"type": ["string","null"]}
|
||||
}},
|
||||
"path": {"type": "string"},
|
||||
"created_at": {"type": "string"}
|
||||
|
||||
"embed_model": { "type": "string" },
|
||||
"embed_dim": { "type": "integer" },
|
||||
"embed_version": { "type": "integer" },
|
||||
|
||||
"created_at": { "type": "string" }
|
||||
},
|
||||
|
||||
"allOf": [
|
||||
{ "anyOf": [ { "required": ["token_count"] }, { "required": ["token_est"] } ] },
|
||||
{ "anyOf": [ { "required": ["type"] }, { "required": ["note_type"] } ] }
|
||||
],
|
||||
|
||||
"additionalProperties": true
|
||||
}
|
||||
@@ -2,15 +2,30 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "mindnet_edge",
|
||||
"type": "object",
|
||||
"required": ["src_id", "dst_id", "edge_type", "scope"],
|
||||
"description": "Edge-Payload (Qdrant). Unterstützt Legacy (edge_type/src_id/dst_id) und neues Schema (kind/source_id/target_id/note_id/status).",
|
||||
|
||||
"properties": {
|
||||
"src_id": {"type": "string"},
|
||||
"dst_id": {"type": "string"},
|
||||
"edge_type": {"type": "string"},
|
||||
"scope": {"type": "string", "enum": ["note","chunk"]},
|
||||
"weight": {"type": "number"},
|
||||
"meta": {"type": "object"},
|
||||
"created_at": {"type": "string"}
|
||||
"scope": { "type": "string", "enum": ["note","chunk"] },
|
||||
|
||||
"edge_type": { "type": "string", "description": "Legacy: z. B. references/backlink/belongs_to/prev/next" },
|
||||
"src_id": { "type": "string", "description": "Legacy: source_id" },
|
||||
"dst_id": { "type": "string", "description": "Legacy: target_id" },
|
||||
|
||||
"kind": { "type": "string", "description": "Neu: z. B. references/backlink/belongs_to/prev/next" },
|
||||
"source_id": { "type": "string" },
|
||||
"target_id": { "type": "string" },
|
||||
"note_id": { "type": "string", "description": "Owner-Note für diesen Edge (Filter/Purge)" },
|
||||
"status": { "type": "string", "description": "optional, z. B. 'unresolved'" },
|
||||
|
||||
"weight": { "type": "number" },
|
||||
"meta": { "type": "object" },
|
||||
"created_at":{ "type": "string" }
|
||||
},
|
||||
|
||||
"anyOf": [
|
||||
{ "required": ["src_id", "dst_id", "edge_type", "scope"] },
|
||||
{ "required": ["source_id", "target_id", "kind", "scope"] }
|
||||
],
|
||||
|
||||
"additionalProperties": true
|
||||
}
|
||||
+30
-38
@@ -2,46 +2,38 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "mindnet_note",
|
||||
"type": "object",
|
||||
"description": "Note-Payload (Qdrant). Erweitert um fulltext/hash_raw_body für verlustfreien Export.",
|
||||
|
||||
"required": ["note_id", "title", "type", "status", "created", "path"],
|
||||
|
||||
"properties": {
|
||||
"note_id": { "type": "string" },
|
||||
"title": { "type": "string" },
|
||||
"type": { "type": "string" },
|
||||
"status": { "type": "string" },
|
||||
"created": { "type": "string" },
|
||||
"updated": { "type": "string" },
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"area": { "type": "string" },
|
||||
"project": { "type": "string" },
|
||||
"priority": { "type": "number" },
|
||||
"effort_min": { "type": "number" },
|
||||
"due": { "type": "string" },
|
||||
"people": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"aliases": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"references": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"depends_on": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"assigned_to": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"lang": { "type": "string" },
|
||||
"path": { "type": "string" },
|
||||
"hash_fulltext": { "type": "string" }
|
||||
"note_id": { "type": "string" },
|
||||
"title": { "type": "string" },
|
||||
"type": { "type": "string" },
|
||||
"status": { "type": "string" },
|
||||
"created": { "type": "string" },
|
||||
"updated": { "type": "string" },
|
||||
|
||||
"tags": { "type": "array", "items": { "type": "string" } },
|
||||
"area": { "type": "string" },
|
||||
"project": { "type": "string" },
|
||||
"priority": { "type": "number" },
|
||||
"effort_min": { "type": "number" },
|
||||
"due": { "type": "string" },
|
||||
"people": { "type": "array", "items": { "type": "string" } },
|
||||
"aliases": { "type": "array", "items": { "type": "string" } },
|
||||
"lang": { "type": "string" },
|
||||
|
||||
"path": { "type": "string" },
|
||||
|
||||
"references": { "type": "array", "items": { "type": "string" } },
|
||||
"depends_on": { "type": "array", "items": { "type": "string" } },
|
||||
"assigned_to": { "type": "array", "items": { "type": "string" } },
|
||||
|
||||
"hash_fulltext":{ "type": "string" },
|
||||
"hash_raw_body":{ "type": "string" },
|
||||
"fulltext": { "type": "string" }
|
||||
},
|
||||
|
||||
"additionalProperties": true
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: audit_edges_vs_expectations.py — Prüfe Kanten in Qdrant gegen Vault-Erwartungen
|
||||
Version: 1.0.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
Zweck
|
||||
-----
|
||||
- Liest Edges/Chunks/Notes aus Qdrant.
|
||||
- Ermittelt erwartete Kanten-Anzahlen aus dem Vault:
|
||||
* belongs_to : sollte == #Chunks
|
||||
* next / prev : je Note (#Chunks_in_Note - 1)
|
||||
* references : Summe aller Chunk-Wikilinks
|
||||
* backlink : Summe einzigartiger Wikilinks pro Note (Note-Level)
|
||||
- Vergleicht IST vs. SOLL und meldet Abweichungen.
|
||||
|
||||
ENV/Qdrant
|
||||
----------
|
||||
QDRANT_URL, QDRANT_API_KEY (optional), COLLECTION_PREFIX (Default: mindnet)
|
||||
|
||||
Aufrufe
|
||||
-------
|
||||
# Gesamtaudit
|
||||
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault
|
||||
|
||||
# Mit anderem Prefix
|
||||
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --prefix mindnet_dev
|
||||
|
||||
# Details anzeigen
|
||||
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --details
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict, Counter
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Projektmodule – nur leichtgewichtige Funktionen
|
||||
try:
|
||||
from app.core.parser import read_markdown
|
||||
except Exception:
|
||||
# sehr einfacher Fallback für Wikilinks
|
||||
read_markdown = None
|
||||
|
||||
WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||
|
||||
# ------------------------------
|
||||
# Qdrant Helpers
|
||||
# ------------------------------
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def _scroll_all(client: QdrantClient, col: str, flt=None, with_payload=True, with_vectors=False, limit=256):
|
||||
out, next_page = [], None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=col,
|
||||
scroll_filter=flt,
|
||||
with_payload=with_payload,
|
||||
with_vectors=with_vectors,
|
||||
limit=limit,
|
||||
offset=next_page,
|
||||
)
|
||||
if not pts:
|
||||
break
|
||||
out.extend(pts)
|
||||
if not next_page:
|
||||
break
|
||||
return out
|
||||
|
||||
# ------------------------------
|
||||
# Vault scan
|
||||
# ------------------------------
|
||||
|
||||
def _iter_md(root: str) -> List[str]:
|
||||
out: List[str] = []
|
||||
for dp, _, fns in os.walk(root):
|
||||
for fn in fns:
|
||||
if fn.lower().endswith(".md"):
|
||||
p = os.path.join(dp, fn)
|
||||
if "/.obsidian/" in p.replace("\\", "/"):
|
||||
continue
|
||||
out.append(p)
|
||||
return sorted(out)
|
||||
|
||||
def _wikilinks_in_text(text: str) -> List[str]:
|
||||
return WIKILINK_RE.findall(text or "")
|
||||
|
||||
def _wikilinks_per_note(vault_root: str) -> Dict[str, List[str]]:
|
||||
res: Dict[str, List[str]] = {}
|
||||
for p in _iter_md(vault_root):
|
||||
body = ""
|
||||
try:
|
||||
if read_markdown:
|
||||
parsed = read_markdown(p)
|
||||
body = parsed.body or ""
|
||||
fm = parsed.frontmatter or {}
|
||||
nid = fm.get("id") or fm.get("note_id") or os.path.splitext(os.path.basename(p))[0]
|
||||
else:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
txt = f.read()
|
||||
# sehr einfacher Split: YAML-Frontmatter rausnehmen
|
||||
if txt.lstrip().startswith("---"):
|
||||
parts = txt.split("\n---", 1)
|
||||
body = parts[1] if len(parts) > 1 else txt
|
||||
else:
|
||||
body = txt
|
||||
nid = os.path.splitext(os.path.basename(p))[0]
|
||||
res[nid] = _wikilinks_in_text(body)
|
||||
except Exception:
|
||||
continue
|
||||
return res
|
||||
|
||||
# ------------------------------
|
||||
# Main Audit
|
||||
# ------------------------------
|
||||
|
||||
def main():
|
||||
load_dotenv()
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Vault (für Erwartungswerte)")
|
||||
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX", "mindnet"), help="Collection-Prefix")
|
||||
ap.add_argument("--details", action="store_true", help="Detail-Listen ausgeben")
|
||||
args = ap.parse_args()
|
||||
|
||||
client = QdrantClient(url=os.environ.get("QDRANT_URL", "http://127.0.0.1:6333"),
|
||||
api_key=os.environ.get("QDRANT_API_KEY") or None)
|
||||
notes_col, chunks_col, edges_col = _names(args.prefix)
|
||||
|
||||
# Qdrant laden
|
||||
notes = _scroll_all(client, notes_col, with_payload=True, with_vectors=False)
|
||||
chunks = _scroll_all(client, chunks_col, with_payload=True, with_vectors=False)
|
||||
edges = _scroll_all(client, edges_col, with_payload=True, with_vectors=False)
|
||||
|
||||
# --- Ist-Zähler
|
||||
cnt_kind = Counter()
|
||||
cnt_scope = Counter()
|
||||
by_note_chunks: Dict[str, int] = defaultdict(int)
|
||||
chunk_wikilinks_total = 0
|
||||
|
||||
for p in chunks:
|
||||
pl = p.payload or {}
|
||||
by_note_chunks[pl.get("note_id")] += 1
|
||||
wl = pl.get("wikilinks") or []
|
||||
if isinstance(wl, list):
|
||||
chunk_wikilinks_total += len(wl)
|
||||
|
||||
for p in edges:
|
||||
pl = p.payload or {}
|
||||
kind = pl.get("kind") or pl.get("edge_type") or "?"
|
||||
scope = pl.get("scope") or "?"
|
||||
cnt_kind[kind] += 1
|
||||
cnt_scope[f"{kind}:{scope}"] += 1
|
||||
|
||||
total_chunks = sum(by_note_chunks.values())
|
||||
|
||||
# --- Soll-Zähler aus Vault
|
||||
wl_per_note = _wikilinks_per_note(args.vault)
|
||||
backlink_expected = sum(len(set(v)) for v in wl_per_note.values())
|
||||
|
||||
next_expected = sum(max(c - 1, 0) for c in by_note_chunks.values())
|
||||
prev_expected = next_expected # symmetrische Kanten
|
||||
|
||||
belongs_to_expected = total_chunks
|
||||
references_expected = chunk_wikilinks_total # aus Chunk-Payloads
|
||||
|
||||
# --- Ergebnis
|
||||
result = {
|
||||
"qdrant_counts": dict(cnt_kind),
|
||||
"qdrant_counts_by_scope": dict(cnt_scope),
|
||||
"chunks_total": total_chunks,
|
||||
"by_note_chunks": dict(by_note_chunks),
|
||||
"vault_expected": {
|
||||
"belongs_to": belongs_to_expected,
|
||||
"next": next_expected,
|
||||
"prev": prev_expected,
|
||||
"references": references_expected,
|
||||
"backlink": backlink_expected,
|
||||
},
|
||||
"deltas": {
|
||||
"belongs_to": cnt_kind.get("belongs_to", 0) - belongs_to_expected,
|
||||
"next": cnt_kind.get("next", 0) - next_expected,
|
||||
"prev": cnt_kind.get("prev", 0) - prev_expected,
|
||||
"references": cnt_kind.get("references", 0) - references_expected,
|
||||
"backlink": cnt_kind.get("backlink", 0) - backlink_expected,
|
||||
},
|
||||
"collections": {
|
||||
"notes": notes_col, "chunks": chunks_col, "edges": edges_col
|
||||
}
|
||||
}
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
if args.details:
|
||||
# optionale Stichproben (z. B. fehlerhafte Kantenarten)
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/debug_note_payload.py
|
||||
Zeigt, welche app.core.note_payload geladen wird, ruft make_note_payload auf
|
||||
und druckt Typ/Preview der Rückgabe.
|
||||
|
||||
Aufruf:
|
||||
python3 -m scripts.debug_note_payload --file ./test_vault/40_concepts/concept-alpha.md --vault-root ./test_vault
|
||||
"""
|
||||
import argparse, json, os, pprint
|
||||
from app.core import note_payload as np
|
||||
from app.core.parser import read_markdown
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--file", required=True)
|
||||
ap.add_argument("--vault-root", default=None)
|
||||
ap.add_argument("--hash-mode", default=None)
|
||||
ap.add_argument("--hash-normalize", default=None)
|
||||
ap.add_argument("--hash-source", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
print("module_path:", getattr(np, "__file__", "<unknown>"))
|
||||
parsed = read_markdown(args.file)
|
||||
res = np.make_note_payload(parsed, vault_root=args.vault_root,
|
||||
hash_mode=args.hash_mode,
|
||||
hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source,
|
||||
file_path=args.file)
|
||||
print("return_type:", type(res).__name__)
|
||||
if isinstance(res, dict):
|
||||
print("keys:", list(res.keys()))
|
||||
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
preview = repr(res)
|
||||
if len(preview) > 600: preview = preview[:600] + "…"
|
||||
print("preview:", preview)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: export_markdown.py — Qdrant → Markdown (Obsidian-kompatibel)
|
||||
Version: 1.4.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
Exportiert Markdown-Notizen aus Qdrant in einen Zielordner (Vault).
|
||||
Rekonstruiert YAML-Frontmatter und Body.
|
||||
|
||||
Body-Rekonstruktions-Priorität (abwärtskompatibel):
|
||||
1) notes.payload.fulltext (verlustfrei, wenn beim Import gespeichert)
|
||||
2) ansonsten aus allen zugehörigen Chunks: payload.text → payload.content → payload.raw
|
||||
in stabiler, sequentieller Reihenfolge (seq/chunk_index/ID-Nummer)
|
||||
|
||||
Wichtige Fixes
|
||||
--------------
|
||||
- **Pfad-Normalisierung**: erzwingt relative Pfade (führe führende '/' ab, backslashes → slashes),
|
||||
damit ``--out`` nicht ignoriert wird.
|
||||
- **`--prefix` (optional)**: Überschreibt COLLECTION_PREFIX; ENV bleibt Default (rückwärtskompatibel).
|
||||
|
||||
ENV / Qdrant
|
||||
------------
|
||||
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
|
||||
- QDRANT_API_KEY (optional)
|
||||
- COLLECTION_PREFIX (Default: mindnet)
|
||||
|
||||
Aufruf
|
||||
------
|
||||
python3 -m scripts.export_markdown --out ./_exportVault
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-foo
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --prefix mindnet_dev # optional
|
||||
|
||||
Beispiele
|
||||
---------
|
||||
COLLECTION_PREFIX=mindnet QDRANT_URL=http://127.0.0.1:6333 \\
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
import yaml
|
||||
from qdrant_client.http import models as rest
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Utilities
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
|
||||
def _ensure_dir(path: str) -> None:
|
||||
d = os.path.dirname(path)
|
||||
if d and not os.path.isdir(d):
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
|
||||
def _normalize_rel_path(p: str) -> str:
|
||||
"""Pfad relativ halten & normalisieren (slashes, führende / entfernen)."""
|
||||
p = (p or "").replace("\\", "/")
|
||||
return p.lstrip("/")
|
||||
|
||||
|
||||
def _to_md(frontmatter: dict, body: str) -> str:
|
||||
fm = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=True).strip()
|
||||
return f"---\n{fm}\n---\n{(body or '').rstrip()}\n"
|
||||
|
||||
|
||||
def _scroll_all(
|
||||
client: QdrantClient,
|
||||
col: str,
|
||||
flt: Optional[rest.Filter] = None,
|
||||
with_payload: bool = True,
|
||||
with_vectors: bool = False,
|
||||
limit: int = 256,
|
||||
):
|
||||
"""Scrollt durch alle Punkte einer Collection und liefert eine Liste mit Points."""
|
||||
out = []
|
||||
next_page = None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=col,
|
||||
scroll_filter=flt,
|
||||
with_payload=with_payload,
|
||||
with_vectors=with_vectors,
|
||||
limit=limit,
|
||||
offset=next_page,
|
||||
)
|
||||
if not pts:
|
||||
break
|
||||
out.extend(pts)
|
||||
if not next_page:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _load_chunks_for_note(client: QdrantClient, chunks_col: str, note_id: str) -> List[dict]:
|
||||
flt = rest.Filter(must=[rest.FieldCondition(
|
||||
key="note_id",
|
||||
match=rest.MatchValue(value=note_id),
|
||||
)])
|
||||
pts = _scroll_all(client, chunks_col, flt, with_payload=True, with_vectors=False)
|
||||
# Sortierung: bevorzugt seq → chunk_index → Nummer in id
|
||||
def _seq(pl: dict) -> Tuple[int, int, int]:
|
||||
s1 = pl.get("seq", pl.get("chunk_index", -1))
|
||||
s2 = pl.get("chunk_index", -1)
|
||||
# Nummer-Anteil aus "noteid#<n>"
|
||||
s3 = 0
|
||||
try:
|
||||
m = re.search(r"#(\\d+)$", pl.get("id") or "")
|
||||
if m:
|
||||
s3 = int(m.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
return (int(s1) if isinstance(s1, int) else -1, int(s2) if isinstance(s2, int) else -1, s3)
|
||||
|
||||
pts_sorted = sorted(pts, key=lambda p: _seq(p.payload or {}))
|
||||
return [p.payload or {} for p in pts_sorted]
|
||||
|
||||
|
||||
def _reconstruct_body(note_pl: dict, chunk_payloads: List[dict]) -> str:
|
||||
# 1) Volltext vorhanden?
|
||||
fulltext = note_pl.get("fulltext")
|
||||
if isinstance(fulltext, str) and fulltext.strip():
|
||||
return fulltext
|
||||
|
||||
# 2) Aus Chunks zusammensetzen: text → content → raw
|
||||
parts: List[str] = []
|
||||
for ch in chunk_payloads:
|
||||
text = ch.get("text") or ch.get("content") or ch.get("raw")
|
||||
if isinstance(text, str) and text.strip():
|
||||
parts.append(text.rstrip())
|
||||
return ("\n\n".join(parts)).rstrip() + ("\n" if parts else "")
|
||||
|
||||
|
||||
def _export_one_note(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
note_pl: dict,
|
||||
out_root: str,
|
||||
overwrite: bool,
|
||||
) -> dict:
|
||||
notes_col, chunks_col, _ = _names(prefix)
|
||||
note_id = note_pl.get("note_id") or note_pl.get("id")
|
||||
|
||||
# Pfad robust bestimmen und relativ halten
|
||||
path = note_pl.get("path") or f"{note_id}.md"
|
||||
path = _normalize_rel_path(path)
|
||||
out_path = os.path.join(out_root, path).replace("\\", "/")
|
||||
|
||||
# Frontmatter aus Payload zurückführen (nur bekannte Felder)
|
||||
fm: Dict[str, object] = {}
|
||||
for k in [
|
||||
"title", "id", "type", "status", "created", "updated", "tags",
|
||||
"priority", "effort_min", "due", "people", "aliases",
|
||||
"depends_on", "assigned_to", "lang",
|
||||
]:
|
||||
v = note_pl.get(k) if k in note_pl else note_pl.get(f"note_{k}")
|
||||
if v not in (None, [], ""):
|
||||
fm[k] = v
|
||||
|
||||
# Mindestfelder
|
||||
if "id" not in fm and note_id:
|
||||
fm["id"] = note_id
|
||||
if "title" not in fm and note_pl.get("title"):
|
||||
fm["title"] = note_pl["title"]
|
||||
|
||||
# Body beschaffen
|
||||
chunks = _load_chunks_for_note(client, chunks_col, note_id)
|
||||
body = _reconstruct_body(note_pl, chunks)
|
||||
|
||||
# Schreiben?
|
||||
if os.path.exists(out_path) and not overwrite:
|
||||
return {"note_id": note_id, "path": path, "status": "skip_exists"}
|
||||
|
||||
_ensure_dir(out_path)
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
f.write(_to_md(fm, body))
|
||||
|
||||
return {"note_id": note_id, "path": path, "status": "written"}
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Main
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", required=True, help="Zielordner für den Export-Vault")
|
||||
ap.add_argument("--note-id", help="Nur eine Note exportieren (Note-ID)")
|
||||
ap.add_argument("--overwrite", action="store_true", help="Bestehende Dateien überschreiben")
|
||||
ap.add_argument("--prefix", help="(Optional) überschreibt COLLECTION_PREFIX aus ENV")
|
||||
args = ap.parse_args()
|
||||
|
||||
# Qdrant-Konfiguration
|
||||
cfg = QdrantConfig.from_env()
|
||||
if args.prefix:
|
||||
cfg.prefix = args.prefix # abwärtskompatibel: ENV bleibt Default
|
||||
|
||||
client = get_client(cfg)
|
||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
|
||||
notes_col, _, _ = _names(cfg.prefix)
|
||||
|
||||
# Notes holen (optional gefiltert)
|
||||
flt = None
|
||||
if args.note_id:
|
||||
flt = rest.Filter(must=[rest.FieldCondition(
|
||||
key="note_id",
|
||||
match=rest.MatchValue(value=args.note_id),
|
||||
)])
|
||||
|
||||
note_pts = _scroll_all(client, notes_col, flt, with_payload=True, with_vectors=False)
|
||||
if not note_pts:
|
||||
print(json.dumps({"exported": 0, "out": args.out, "message": "Keine Notes gefunden."}, ensure_ascii=False))
|
||||
return
|
||||
|
||||
results = []
|
||||
for p in note_pts:
|
||||
pl = p.payload or {}
|
||||
try:
|
||||
res = _export_one_note(client, cfg.prefix, pl, args.out, args.overwrite)
|
||||
except Exception as e:
|
||||
res = {"note_id": pl.get("note_id") or pl.get("id"), "error": str(e)}
|
||||
results.append(res)
|
||||
|
||||
print(json.dumps({
|
||||
"exported": len([r for r in results if r.get("status") == "written"]),
|
||||
"skipped": len([r for r in results if r.get("status") == "skip_exists"]),
|
||||
"out": args.out,
|
||||
"details": results,
|
||||
}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
# scripts/gc_qdrant_after_vault_scan.py
|
||||
# Version: 1.0.0 (2025-09-05)
|
||||
#
|
||||
# Zweck
|
||||
# -----
|
||||
# Garbage-Collector für Qdrant: löscht Einträge (Edges, Chunks, Notes), deren Note-ID
|
||||
# nicht mehr im Obsidian-Vault vorhanden ist. So bleiben Vektorindex/Graph mit dem Vault konsistent.
|
||||
#
|
||||
# Aufrufparameter
|
||||
# ---------------
|
||||
# --vault PATH Pfad zum Obsidian-Vault (z. B. ./vault) [erforderlich]
|
||||
# --mode {edges,content,all}
|
||||
# edges -> nur Kanten mit source_id/target_id ∉ Vault-Notes löschen
|
||||
# content -> Chunks & Notes löschen, deren note_id ∉ Vault-Notes (keine Edges)
|
||||
# all -> zuerst edges, dann content
|
||||
# --prefix PREFIX Collection-Präfix (Default: aus ENV COLLECTION_PREFIX oder "mindnet")
|
||||
# --apply Ohne diesen Schalter: Dry-Run (nur Vorschau). Mit Schalter: ausführen.
|
||||
# --yes Sicherheitsabfrage überspringen (non-interaktiv)
|
||||
# --batch-size N Größe für In-Filter (Default: 1000)
|
||||
#
|
||||
# Hinweise
|
||||
# --------
|
||||
# - Benötigt funktionierendes Python-venv mit allen Abhängigkeiten von mindnet (qdrant-client usw.).
|
||||
# - Läuft gegen die drei Collections {prefix}_notes, {prefix}_chunks, {prefix}_edges (keine anderen Projekte).
|
||||
# - Nutzt OR-Filter (Filter.should) korrekt, keine minimum_should (Pydantic v2 lässt die Option nicht zu).
|
||||
#
|
||||
# Änderungen ggü. vorher
|
||||
# ----------------------
|
||||
# - Neu: eigenes GC-Tool; berührt ausschließlich das übergebene Projekt-Präfix.
|
||||
# - Sichere Dry-Run-Vorschau mit Zählung pro Collection; interaktive Bestätigung.
|
||||
# - Löscht Edges mittels Filter auf (source_id ∈ MISSING) ODER (target_id ∈ MISSING).
|
||||
# - Löscht Chunks/Notes mittels Filter auf note_id ∈ MISSING.
|
||||
#
|
||||
# Beispiele
|
||||
# ---------
|
||||
# DRY RUN: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all
|
||||
# APPLY: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all --apply
|
||||
# CI-Modus: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode edges --apply --yes
|
||||
#
|
||||
# Kompatibilität / Kontext
|
||||
# ------------------------
|
||||
# - Frontmatter-Validierung & Parsing wie im Projekt (parser.read_markdown, validate_required_frontmatter). :contentReference[oaicite:0]{index=0}
|
||||
# - Note-Payload-Schema (u. a. note_id, hash_fulltext) wie spezifiziert. :contentReference[oaicite:1]{index=1}
|
||||
# - Collection-Namen & Anlage-Logik über app.core.qdrant (notes/chunks = dim, edges = 1D). :contentReference[oaicite:2]{index=2}
|
||||
# - Edges nutzen Felder source_id/target_id; Chunks/Notes tragen note_id in payload. :contentReference[oaicite:3]{index=3} :contentReference[oaicite:4]{index=4}
|
||||
#
|
||||
from __future__ import annotations
|
||||
import argparse, os, glob, json, sys
|
||||
from typing import Iterable, Set, Tuple, List
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter # :contentReference[oaicite:5]{index=5}
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names # :contentReference[oaicite:6]{index=6}
|
||||
|
||||
def iter_note_ids_from_vault(vault_root: str) -> Set[str]:
|
||||
"""Liest alle Markdown-Dateien im Vault und sammelt valide Frontmatter-IDs."""
|
||||
paths = [p for p in glob.glob(os.path.join(vault_root, "**", "*.md"), recursive=True)]
|
||||
out: Set[str] = set()
|
||||
for p in paths:
|
||||
try:
|
||||
parsed = read_markdown(p)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
nid = fm.get("id") or fm.get("note_id")
|
||||
if nid:
|
||||
out.add(nid)
|
||||
except Exception:
|
||||
# ungültige oder Template-Dateien ohne Pflichtfelder ignorieren
|
||||
continue
|
||||
return out
|
||||
|
||||
def scroll_payload_values(client: QdrantClient, collection: str, field: str, limit: int = 2000) -> Set[str]:
|
||||
"""Scrollt eine Collection und sammelt alle Payload-Werte eines Feldes (falls vorhanden)."""
|
||||
out: Set[str] = set()
|
||||
offset = None
|
||||
while True:
|
||||
res = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=None,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=limit,
|
||||
offset=offset
|
||||
)
|
||||
# qdrant-client (neuere Versionen) geben (points, next_offset)
|
||||
if isinstance(res, tuple) and len(res) == 2:
|
||||
points, offset = res
|
||||
else:
|
||||
# Fallback, falls sich API signifikant ändert
|
||||
points, offset = res[0], res[1] if len(res) > 1 else None
|
||||
|
||||
for pt in points or []:
|
||||
pl = getattr(pt, "payload", None) or {}
|
||||
v = pl.get(field)
|
||||
if isinstance(v, str):
|
||||
out.add(v)
|
||||
if not offset:
|
||||
break
|
||||
return out
|
||||
|
||||
def chunked(iterable: Iterable[str], n: int) -> Iterable[List[str]]:
|
||||
buf: List[str] = []
|
||||
for x in iterable:
|
||||
buf.append(x)
|
||||
if len(buf) >= n:
|
||||
yield buf
|
||||
buf = []
|
||||
if buf:
|
||||
yield buf
|
||||
|
||||
def count_matches(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
|
||||
"""Grober Zähler über scroll (Qdrant Count-API ist optional je Version)."""
|
||||
total = 0
|
||||
offset = None
|
||||
while True:
|
||||
points, offset = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=filter_,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
limit=2000,
|
||||
offset=offset
|
||||
)
|
||||
total += len(points or [])
|
||||
if not offset:
|
||||
break
|
||||
return total
|
||||
|
||||
def build_filters_for_missing(prefix: str, missing_note_ids: Set[str]) -> Tuple[rest.Filter, rest.Filter, rest.Filter]:
|
||||
"""
|
||||
Erzeugt je Collection passende Filter:
|
||||
- Notes/Chunks: note_id ∈ missing
|
||||
- Edges: (source_id ∈ missing) ODER (target_id ∈ missing)
|
||||
"""
|
||||
notes_col, chunks_col, edges_col = collection_names(prefix) # :contentReference[oaicite:7]{index=7}
|
||||
# MatchAny statt MatchValue für große Mengen
|
||||
any_missing = rest.MatchAny(any=list(missing_note_ids))
|
||||
|
||||
f_notes = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
|
||||
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
|
||||
f_edges = rest.Filter(should=[
|
||||
rest.FieldCondition(key="source_id", match=any_missing),
|
||||
rest.FieldCondition(key="target_id", match=any_missing),
|
||||
])
|
||||
return f_notes, f_chunks, f_edges
|
||||
|
||||
def preview(client: QdrantClient, prefix: str, missing: Set[str]) -> dict:
|
||||
"""Erstellt eine Dry-Run-Vorschau (Anzahl Treffer pro Collection)."""
|
||||
notes_col, chunks_col, edges_col = collection_names(prefix)
|
||||
f_notes, f_chunks, f_edges = build_filters_for_missing(prefix, missing)
|
||||
|
||||
return {
|
||||
"prefix": prefix,
|
||||
"collections": {"notes": notes_col, "chunks": chunks_col, "edges": edges_col},
|
||||
"missing_note_ids_count": len(missing),
|
||||
"to_delete_counts": {
|
||||
"edges": count_matches(client, edges_col, f_edges),
|
||||
"chunks": count_matches(client, chunks_col, f_chunks),
|
||||
"notes": count_matches(client, notes_col, f_notes),
|
||||
}
|
||||
}
|
||||
|
||||
def delete_with_filter(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
|
||||
"""Löscht alle Punkte, die den Filter matchen. Gibt grobe Trefferzahl zurück."""
|
||||
# erst zählen (für Feedback)
|
||||
to_del = count_matches(client, collection, filter_)
|
||||
if to_del == 0:
|
||||
return 0
|
||||
client.delete(
|
||||
collection_name=collection,
|
||||
points_selector=filter_,
|
||||
wait=True
|
||||
)
|
||||
return to_del
|
||||
|
||||
def main():
|
||||
load_dotenv()
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (z. B. ./vault)")
|
||||
ap.add_argument("--mode", choices=["edges", "content", "all"], default="all",
|
||||
help="Löschmodus (edges=nur Kanten, content=Chunks+Notes, all=beides)")
|
||||
ap.add_argument("--prefix", help="Collection-Präfix (Default aus ENV COLLECTION_PREFIX)")
|
||||
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter: Dry-Run")
|
||||
ap.add_argument("--yes", action="store_true", help="Rückfrage unterdrücken")
|
||||
ap.add_argument("--batch-size", type=int, default=1000, help="Batch-Größe für MatchAny")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
if args.prefix:
|
||||
cfg.prefix = args.prefix
|
||||
client = get_client(cfg)
|
||||
|
||||
notes_col, chunks_col, edges_col = collection_names(cfg.prefix)
|
||||
|
||||
# 1) Vault lesen → Menge gültiger Note-IDs
|
||||
vault_root = os.path.abspath(args.vault)
|
||||
vault_ids = iter_note_ids_from_vault(vault_root)
|
||||
|
||||
# 2) Qdrant vorhandene Note-IDs einsammeln
|
||||
existing_note_ids = scroll_payload_values(client, notes_col, "note_id")
|
||||
|
||||
# 3) Differenz ermitteln
|
||||
missing = existing_note_ids - vault_ids
|
||||
|
||||
# 4) Vorschau
|
||||
summary = preview(client, cfg.prefix, missing)
|
||||
print(json.dumps({"mode": args.mode, "apply": args.apply, "summary": summary}, ensure_ascii=False, indent=2))
|
||||
|
||||
if not missing:
|
||||
# nichts zu tun
|
||||
return
|
||||
|
||||
if not args.apply:
|
||||
# Dry-Run Ende
|
||||
return
|
||||
|
||||
if not args.yes:
|
||||
resp = input("Fortfahren und die oben gezeigten Objekte löschen? (yes/no): ").strip().lower()
|
||||
if resp not in ("y", "yes"):
|
||||
print("Abgebrochen.")
|
||||
return
|
||||
|
||||
# 5) Löschen je Modus
|
||||
f_notes, f_chunks, f_edges = build_filters_for_missing(cfg.prefix, missing)
|
||||
report = {"deleted": {"edges": 0, "chunks": 0, "notes": 0}}
|
||||
|
||||
try:
|
||||
if args.mode in ("edges", "all"):
|
||||
report["deleted"]["edges"] = delete_with_filter(client, edges_col, f_edges)
|
||||
if args.mode in ("content", "all"):
|
||||
# Reihenfolge: erst chunks, dann notes
|
||||
report["deleted"]["chunks"] = delete_with_filter(client, chunks_col, f_chunks)
|
||||
report["deleted"]["notes"] = delete_with_filter(client, notes_col, f_notes)
|
||||
finally:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+404
-131
@@ -1,185 +1,458 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: scripts/import_markdown.py
|
||||
Version: v2.3.1 (2025-09-05)
|
||||
Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
|
||||
Version: 3.5.3
|
||||
Datum: 2025-09-09
|
||||
|
||||
Beschreibung
|
||||
Importiert Markdown-Notizen in Qdrant (Notes, Chunks, Edges).
|
||||
- Chunking + Embedding (MiniLM 384d, externer Embed-Server)
|
||||
- Edges direkt beim Import aus Wikilinks ([[…]]) ableiten
|
||||
- Idempotente UUIDv5-IDs; Collections werden bereitgestellt
|
||||
Neu in 3.5.3
|
||||
------------
|
||||
- **Fallback**, falls app.core.note_payload.make_note_payload(None) / non-dict zurückgibt:
|
||||
Der Importer baut einen minimalen, korrekten Note-Payload selbst (inkl. Hash
|
||||
gemäß --hash-mode/--hash-source/--hash-normalize) und läuft weiter.
|
||||
- Detaillierte Debug-Ausgaben mit Typ/Preview, falls die Rückgabeform abweicht.
|
||||
|
||||
Aufruf
|
||||
python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id ID] [--embed-note] [--force-replace]
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
- Liest Markdown-Dateien ein, erzeugt Notes/Chunks/Edges **idempotent**.
|
||||
- Änderungserkennung (nur Inhalte, keine FS-Zeitstempel) konfigurierbar:
|
||||
* --hash-mode: body | frontmatter | body+frontmatter | full (Alias)
|
||||
- Env: MINDNET_HASH_MODE oder MINDNET_HASH_COMPARE (Body/Frontmatter/Full)
|
||||
* --hash-normalize: canonical | none (Default: canonical)
|
||||
* --hash-source: parsed (Default) | raw
|
||||
* Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true)
|
||||
* Signaturabgleich: Wechselt die Signatur (z. B. body→full, parsed→raw, canonical→none),
|
||||
gilt die Note einmalig als geändert.
|
||||
|
||||
Parameter
|
||||
--vault Pfad zum Obsidian-Vault (erforderlich)
|
||||
--apply Ohne Flag: Dry-Run (nur JSON-Zeilen). Mit Flag: schreibt in Qdrant.
|
||||
--note-id Nur eine spezifische Note-ID verarbeiten
|
||||
--embed-note Optional: Note-Volltext zusätzlich einbetten
|
||||
--force-replace Vor Upsert: zugehörige Edges der Quell-Note in Qdrant löschen (harte Ersetzung)
|
||||
ENV / Qdrant
|
||||
------------
|
||||
- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
|
||||
- COLLECTION_PREFIX (Default: mindnet)
|
||||
- VECTOR_DIM (Default: 384)
|
||||
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
||||
- MINDNET_COMPARE_TEXT: true|false (Default: false)
|
||||
|
||||
Hinweise
|
||||
- Im venv laufen: `source .venv/bin/activate`
|
||||
- Erwartet laufenden Embed-Server (http://127.0.0.1:8990)
|
||||
- Qdrant via env: QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM
|
||||
|
||||
Changelog
|
||||
v2.3.1: FIX – Für derive_wikilink_edges werden jetzt echte Chunk-Texte übergeben
|
||||
(chunks_for_links mit {"chunk_id","text"}), damit `references_at` erzeugt werden.
|
||||
v2.3.0: Umstellung auf app.core.derive_edges; Edge-IDs unterstützen Occurrence.
|
||||
v2.2.x: Fix Filter-API (pydantic v2) bei Purge; Kleinkosmetik.
|
||||
Aufruf-Beispiele
|
||||
----------------
|
||||
python3 -m scripts.import_markdown --vault ./vault
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --hash-source raw --hash-normalize none
|
||||
MINDNET_HASH_COMPARE=Full python3 -m scripts.import_markdown --vault ./vault --apply
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --compare-text
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse, os, glob, json, sys
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.validate_note import validate_note_payload
|
||||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunk_payload import make_chunk_payloads
|
||||
from app.core.embed import embed_texts, embed_one
|
||||
from app.core.qdrant import QdrantConfig, ensure_collections, get_client
|
||||
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
|
||||
from app.core.derive_edges import build_note_index, derive_wikilink_edges # nutzt 'text' je Chunk :contentReference[oaicite:1]{index=1}
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from collections.abc import Mapping
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
def iter_md(root: str, exclude_dirs=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")):
|
||||
files = [p for p in glob.glob(os.path.join(root, "**", "*.md"), recursive=True)]
|
||||
out = []
|
||||
for p in files:
|
||||
pn = p.replace("\\","/")
|
||||
if any(ex in pn for ex in exclude_dirs):
|
||||
continue
|
||||
out.append(p)
|
||||
return out
|
||||
from app.core.parser import (
|
||||
read_markdown,
|
||||
normalize_frontmatter,
|
||||
validate_required_frontmatter,
|
||||
extract_wikilinks,
|
||||
FRONTMATTER_RE,
|
||||
)
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunk_payload import make_chunk_payloads
|
||||
from app.core.edges import build_edges_for_note
|
||||
from app.core.qdrant import (
|
||||
QdrantConfig,
|
||||
get_client,
|
||||
ensure_collections,
|
||||
ensure_payload_indexes,
|
||||
)
|
||||
from app.core.qdrant_points import (
|
||||
points_for_chunks,
|
||||
points_for_note,
|
||||
points_for_edges,
|
||||
upsert_batch,
|
||||
)
|
||||
|
||||
def purge_note_edges(client, prefix: str, source_note_id: str):
|
||||
# Löscht Edges, deren source_id == source_note_id ODER (kind==backlink && target_id==source_note_id)
|
||||
edges_col = f"{prefix}_edges"
|
||||
f = rest.Filter(
|
||||
should=[
|
||||
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_note_id)),
|
||||
rest.Filter(
|
||||
must=[
|
||||
rest.FieldCondition(key="kind", match=rest.MatchValue(value="backlink")),
|
||||
rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_note_id)),
|
||||
]
|
||||
),
|
||||
]
|
||||
try:
|
||||
from app.core.embed import embed_texts # optional
|
||||
except Exception:
|
||||
embed_texts = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def iter_md(root: str) -> List[str]:
|
||||
out: List[str] = []
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
for fn in filenames:
|
||||
if not fn.lower().endswith(".md"):
|
||||
continue
|
||||
p = os.path.join(dirpath, fn)
|
||||
pn = p.replace("\\", "/")
|
||||
if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
|
||||
continue
|
||||
out.append(p)
|
||||
return sorted(out)
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
points, _ = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=f,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1,
|
||||
)
|
||||
# pydantic v2: client.delete(...) statt delete_points(...)
|
||||
client.delete(collection_name=edges_col, points_selector=f, wait=True)
|
||||
if not points:
|
||||
return None
|
||||
return points[0].payload or {}
|
||||
|
||||
def main():
|
||||
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||
_, chunks_col, edges_col = collections(prefix)
|
||||
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
client.delete(collection_name=chunks_col, points_selector=f_chunks, wait=True)
|
||||
f_edges = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
client.delete(collection_name=edges_col, points_selector=f_edges, wait=True)
|
||||
|
||||
def _normalize_rel_path(abs_path: str, vault_root: str) -> str:
|
||||
try:
|
||||
rel = os.path.relpath(abs_path, vault_root)
|
||||
except Exception:
|
||||
rel = abs_path
|
||||
return rel.replace("\\", "/").lstrip("/")
|
||||
|
||||
def _resolve_hash_mode(val: Optional[str]) -> str:
|
||||
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
|
||||
if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
||||
return "body+frontmatter"
|
||||
if v in ("frontmatter", "fm"):
|
||||
return "frontmatter"
|
||||
return "body"
|
||||
|
||||
def _normalize_body(body: str, mode: str) -> str:
|
||||
if mode == "none":
|
||||
return body if body is not None else ""
|
||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
|
||||
text = "\n".join(line.rstrip() for line in text.split("\n"))
|
||||
return text
|
||||
|
||||
def _canon_fm(fm: Dict[str, Any]) -> str:
|
||||
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
def _read_raw_body(file_path: str) -> tuple[str, Dict[str, Any]]:
|
||||
if not os.path.exists(file_path):
|
||||
return "", {}
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
raw = f.read()
|
||||
m = FRONTMATTER_RE.match(raw)
|
||||
fm = {}
|
||||
if m:
|
||||
import yaml
|
||||
try:
|
||||
fm = yaml.safe_load(m.group(1)) or {}
|
||||
except Exception:
|
||||
fm = {}
|
||||
body = raw[m.end():]
|
||||
else:
|
||||
body = raw
|
||||
return body, fm
|
||||
|
||||
def _fallback_note_payload(parsed: Any, vault_root: str, *, hash_mode: Optional[str], hash_normalize: Optional[str],
|
||||
hash_source: Optional[str], file_path: str) -> Dict[str, Any]:
|
||||
"""Baut einen minimalen Note-Payload, falls make_note_payload None liefert."""
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
path = getattr(parsed, "path", file_path)
|
||||
rel_path = _normalize_rel_path(path, vault_root)
|
||||
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower()
|
||||
norm = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
|
||||
mode = _resolve_hash_mode(hash_mode)
|
||||
|
||||
if src == "raw":
|
||||
body_raw, fm_raw = _read_raw_body(file_path)
|
||||
# Roh-FM ergänzen (nicht überschreiben)
|
||||
if fm_raw:
|
||||
merged = dict(fm)
|
||||
for k, v in fm_raw.items():
|
||||
merged.setdefault(k, v)
|
||||
fm = merged
|
||||
body_for_hash = body_raw
|
||||
else:
|
||||
body_for_hash = getattr(parsed, "body", "") or ""
|
||||
|
||||
body_n = _normalize_body(body_for_hash, norm)
|
||||
fm_s = _canon_fm(fm)
|
||||
|
||||
import hashlib
|
||||
def h(s: str) -> str:
|
||||
x = hashlib.sha256()
|
||||
x.update(s.encode("utf-8"))
|
||||
return x.hexdigest()
|
||||
|
||||
if mode == "frontmatter":
|
||||
primary_hash = h(fm_s)
|
||||
elif mode == "body+frontmatter":
|
||||
primary_hash = h(body_n + "\n--FM--\n" + fm_s)
|
||||
else:
|
||||
primary_hash = h(body_n)
|
||||
|
||||
sig = f"{'full' if mode=='body+frontmatter' else mode}:{src}:{norm}:{primary_hash}"
|
||||
|
||||
refs = extract_wikilinks(getattr(parsed, "body", "") or "")
|
||||
|
||||
return {
|
||||
"note_id": fm.get("id") or fm.get("note_id"),
|
||||
"title": fm.get("title"),
|
||||
"type": fm.get("type"),
|
||||
"status": fm.get("status"),
|
||||
"created": fm.get("created"),
|
||||
"updated": fm.get("updated"),
|
||||
"path": rel_path or fm.get("path"),
|
||||
"tags": fm.get("tags"),
|
||||
"hash_fulltext": primary_hash,
|
||||
"hash_signature": sig,
|
||||
"fulltext": getattr(parsed, "body", "") or "",
|
||||
"references": list(dict.fromkeys(refs)),
|
||||
}
|
||||
|
||||
def _coerce_to_dict(obj: Any) -> Optional[Dict[str, Any]]:
|
||||
if obj is None:
|
||||
return None
|
||||
if isinstance(obj, dict):
|
||||
return obj
|
||||
if isinstance(obj, Mapping):
|
||||
try:
|
||||
return dict(obj)
|
||||
except Exception:
|
||||
pass
|
||||
if isinstance(obj, (list, tuple)):
|
||||
for it in obj:
|
||||
d = _coerce_to_dict(it)
|
||||
if isinstance(d, dict):
|
||||
return d
|
||||
return None
|
||||
md = getattr(obj, "model_dump", None)
|
||||
if callable(md):
|
||||
try: return md()
|
||||
except Exception: pass
|
||||
dd = getattr(obj, "dict", None)
|
||||
if callable(dd):
|
||||
try: return dd()
|
||||
except Exception: pass
|
||||
dct = getattr(obj, "__dict__", None)
|
||||
if isinstance(dct, dict):
|
||||
return dict(dct)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
load_dotenv()
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian Vault (z.B. ./vault)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant (sonst Dry-Run)")
|
||||
ap.add_argument("--note-id", help="Nur eine Note-ID verarbeiten")
|
||||
ap.add_argument("--embed-note", action="store_true", help="Auch Note-Volltext einbetten (optional)")
|
||||
ap.add_argument("--force-replace", action="store_true", help="Vor Upsert alte Edges der Note löschen")
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
|
||||
ap.add_argument("--purge-before-upsert", action="store_true",
|
||||
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
|
||||
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
|
||||
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
|
||||
ap.add_argument("--force-replace", action="store_true",
|
||||
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter", "full"], default=None)
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
|
||||
ap.add_argument("--note-scope-refs", action="store_true")
|
||||
ap.add_argument("--debug-hash-diff", action="store_true")
|
||||
ap.add_argument("--compare-text", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
# Qdrant
|
||||
cfg = QdrantConfig(
|
||||
url=os.getenv("QDRANT_URL", "http://127.0.0.1:6333"),
|
||||
api_key=os.getenv("QDRANT_API_KEY") or None,
|
||||
prefix=os.getenv("COLLECTION_PREFIX", "mindnet"),
|
||||
dim=int(os.getenv("VECTOR_DIM","384")),
|
||||
)
|
||||
note_scope_refs_env = (os.environ.get("MINDNET_NOTE_SCOPE_REFS", "false").strip().lower() == "true")
|
||||
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
||||
compare_text = args.compare_text or (os.environ.get("MINDNET_COMPARE_TEXT", "false").strip().lower() == "true")
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
|
||||
root = os.path.abspath(args.vault)
|
||||
files = iter_md(root)
|
||||
if not files:
|
||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr); sys.exit(2)
|
||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# --- Note-Index (für robuste Zielauflösung) ---
|
||||
note_stubs = []
|
||||
processed = 0
|
||||
for path in files:
|
||||
parsed = read_markdown(path)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
# ------------- Parsing -------------
|
||||
try:
|
||||
validate_required_frontmatter(fm)
|
||||
except Exception:
|
||||
parsed = read_markdown(path)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"read_markdown failed: {e}"}))
|
||||
continue
|
||||
if args.note_id and fm.get("id") != args.note_id:
|
||||
if parsed is None:
|
||||
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
|
||||
continue
|
||||
rel = os.path.relpath(parsed.path, root).replace("\\","/")
|
||||
note_stubs.append({"note_id": fm["id"], "title": fm.get("title",""), "path": rel})
|
||||
note_index = build_note_index(note_stubs)
|
||||
|
||||
total_notes = 0
|
||||
for path in files:
|
||||
parsed = read_markdown(path)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
try:
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"}))
|
||||
continue
|
||||
|
||||
if args.note_id and fm.get("id") != args.note_id:
|
||||
continue
|
||||
|
||||
total_notes += 1
|
||||
processed += 1
|
||||
|
||||
# Note-Payload
|
||||
note_pl = make_note_payload(parsed, vault_root=root)
|
||||
validate_note_payload(note_pl)
|
||||
note_pl["fulltext"] = parsed.body # für derive_edges
|
||||
# ------------- Note-Payload -------------
|
||||
try:
|
||||
note_pl_raw = make_note_payload(
|
||||
parsed,
|
||||
vault_root=root,
|
||||
hash_mode=args.hash_mode,
|
||||
hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source,
|
||||
file_path=path,
|
||||
)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": fm.get("id"), "error": f"make_note_payload failed: {e}"}))
|
||||
note_pl_raw = None
|
||||
|
||||
# Chunks
|
||||
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
|
||||
note_pl = _coerce_to_dict(note_pl_raw)
|
||||
if not isinstance(note_pl, dict):
|
||||
# Fallback: baue minimalen Payload
|
||||
note_pl = _fallback_note_payload(
|
||||
parsed,
|
||||
root,
|
||||
hash_mode=args.hash_mode,
|
||||
hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source,
|
||||
file_path=path,
|
||||
)
|
||||
print(json.dumps({
|
||||
"path": path,
|
||||
"note_id": fm.get("id"),
|
||||
"warn": "make_note_payload returned non-dict; used fallback",
|
||||
"returned_type": type(note_pl_raw).__name__ if note_pl_raw is not None else "NoneType"
|
||||
}))
|
||||
|
||||
# Embeddings (Chunks)
|
||||
texts = [ch.text for ch in chunks]
|
||||
vectors = embed_texts(texts)
|
||||
# fulltext + path Sicherheit
|
||||
if not note_pl.get("fulltext"):
|
||||
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
||||
if note_pl.get("path"):
|
||||
note_pl["path"] = _normalize_rel_path(
|
||||
os.path.join(root, note_pl["path"]) if not os.path.isabs(note_pl["path"]) else note_pl["path"], root
|
||||
)
|
||||
else:
|
||||
p_path = getattr(parsed, "path", None) or path
|
||||
note_pl["path"] = _normalize_rel_path(p_path, root)
|
||||
|
||||
# Optional: Note-Vektor
|
||||
note_vec = embed_one(parsed.body) if args.embed_note else None
|
||||
note_id = note_pl.get("note_id") or fm.get("id")
|
||||
if not note_id:
|
||||
print(json.dumps({"path": path, "error": "Missing note_id after payload build (even in fallback)"}))
|
||||
continue
|
||||
|
||||
# --- WICHTIG: Chunk-Texte für derive_wikilink_edges ---
|
||||
chunks_for_links = [
|
||||
{"chunk_id": (pl.get("chunk_id") or pl.get("id") or f"{fm['id']}#{i+1}"),
|
||||
"text": chunks[i].text}
|
||||
for i, pl in enumerate(chunk_pls)
|
||||
if i < len(chunks)
|
||||
]
|
||||
# ------------- Change-Detection -------------
|
||||
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
||||
old_hash = None if not old_payload else old_payload.get("hash_fulltext")
|
||||
old_sig = (old_payload or {}).get("hash_signature")
|
||||
new_hash = note_pl.get("hash_fulltext")
|
||||
new_sig = note_pl.get("hash_signature")
|
||||
|
||||
# Kanten (Note- & Chunk-Ebene)
|
||||
edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index)
|
||||
sig_changed = bool(old_sig) and bool(new_sig) and (old_sig.split(":")[:3] != new_sig.split(":")[:3])
|
||||
hash_changed = (old_hash != new_hash)
|
||||
|
||||
# Dry-Run-Ausgabe
|
||||
print(json.dumps({
|
||||
"note_id": fm["id"],
|
||||
"title": fm["title"],
|
||||
text_changed = False
|
||||
if compare_text:
|
||||
old_text = (old_payload or {}).get("fulltext") or ""
|
||||
new_text = note_pl.get("fulltext") or ""
|
||||
text_changed = (old_text != new_text)
|
||||
|
||||
changed = args.force_replace or sig_changed or hash_changed or text_changed
|
||||
|
||||
# ------------- Chunks / Embeddings / Edges -------------
|
||||
try:
|
||||
chunks = assemble_chunks(fm["id"], getattr(parsed, "body", "") or "", fm.get("type", "concept"))
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {e}"}))
|
||||
continue
|
||||
|
||||
if embed_texts:
|
||||
try:
|
||||
vecs = embed_texts([getattr(c, "text", "") for c in chunks]) # type: ignore[attr-defined]
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
||||
else:
|
||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
||||
|
||||
try:
|
||||
note_refs = note_pl.get("references") or []
|
||||
edges = build_edges_for_note(
|
||||
note_id,
|
||||
chunk_pls,
|
||||
note_refs,
|
||||
include_note_scope_refs=note_scope_refs,
|
||||
)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"build_edges_for_note failed: {e}"}))
|
||||
continue
|
||||
|
||||
# ------------- Summary -------------
|
||||
summary = {
|
||||
"note_id": note_id,
|
||||
"title": fm.get("title"),
|
||||
"chunks": len(chunk_pls),
|
||||
"edges": len(edges),
|
||||
"path": note_pl["path"]
|
||||
}, ensure_ascii=False))
|
||||
"changed": changed,
|
||||
"decision": ("apply" if args.apply and changed else
|
||||
"apply-skip-unchanged" if args.apply and not changed else
|
||||
"dry-run"),
|
||||
"path": note_pl["path"],
|
||||
"hash_mode": args.hash_mode or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE", "body"),
|
||||
"hash_normalize": args.hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical"),
|
||||
"hash_source": args.hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed"),
|
||||
"hash_signature": note_pl.get("hash_signature"),
|
||||
"sig_changed": sig_changed,
|
||||
"hash_changed": hash_changed,
|
||||
"text_changed": text_changed,
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
|
||||
if args.apply:
|
||||
if args.force_replace:
|
||||
purge_note_edges(client, cfg.prefix, fm["id"])
|
||||
# ------------- Upserts -------------
|
||||
if not args.apply:
|
||||
continue
|
||||
|
||||
# Notes upsert
|
||||
notes_col, note_pts = points_for_note(cfg.prefix, note_pl, note_vec, cfg.dim)
|
||||
upsert_batch(client, notes_col, note_pts)
|
||||
try:
|
||||
if changed and args.purge_before_upsert:
|
||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
||||
|
||||
# Chunks upsert
|
||||
chunks_col, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vectors)
|
||||
upsert_batch(client, chunks_col, chunk_pts)
|
||||
try:
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||
upsert_batch(client, chunks_name, chunk_pts)
|
||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||
upsert_batch(client, edges_name, edge_pts)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"upsert failed: {e}"}))
|
||||
|
||||
# Edges upsert
|
||||
edges_col, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||
upsert_batch(client, edges_col, edge_pts)
|
||||
print(f"Done. Processed notes: {processed}")
|
||||
|
||||
print(f"Done. Processed notes: {total_notes}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
make_test_vault.py — erzeugt einen minimalen, nachvollziehbaren Test-Vault
|
||||
|
||||
Version: 1.2 (2025-09-09)
|
||||
Änderungen ggü. 1.1:
|
||||
- Fügt **externe Links** (http/mailto) in `exp-two.md` hinzu, damit der Chunk-Payload-Exporter
|
||||
`external_links` realistisch testen kann.
|
||||
- Kleine Textanpassungen zur stabilen Chunk-Bildung (Absatzstruktur).
|
||||
|
||||
Zweck
|
||||
- Kleiner Obsidian-ähnlicher Vault zum Durchspielen des Importers (Chunks/Edges).
|
||||
- Szenarien: leere Links, spätere Anlage fehlender Noten, Chunk-Neuaufteilung, externe Links.
|
||||
|
||||
Struktur (Default: ./test_vault)
|
||||
- 40_concepts/concept-alpha.md
|
||||
- 20_experiences/exp-one.md (verlinkt auf [[concept-alpha]] und [[missing-note]])
|
||||
- 20_experiences/exp-two.md (verlinkt auf [[concept-alpha]] + externe Links)
|
||||
- 30_projects/project-demo.md (verlinkt auf [[concept-alpha]] und [[exp-one]])
|
||||
|
||||
Voraussetzungen
|
||||
- Keine. Rein Dateigenerierung.
|
||||
|
||||
Aufruf
|
||||
python3 -m scripts.make_test_vault [--out ./test_vault] [--force] [--with-missing]
|
||||
|
||||
Parameter
|
||||
--out Zielverzeichnis (Standard: ./test_vault)
|
||||
--force Bestehenden Ordner löschen und neu anlegen.
|
||||
--with-missing Auch die 'missing-note' direkt anlegen (Standard: erst im 2. Lauf)
|
||||
|
||||
Hinweise für Tests
|
||||
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
|
||||
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
|
||||
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
|
||||
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
|
||||
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
|
||||
4) Prüfe externe Links (verify_chunk_texts.py betrifft nur Text; externe Links siehst du in chunk_payloads).
|
||||
|
||||
Kompatibel mit:
|
||||
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
|
||||
def iso_now() -> str:
|
||||
# ISO 8601 mit Offset; als String zurückgeben
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
def q(s: str) -> str:
|
||||
# Sicheres Quoten für YAML-Frontmatter
|
||||
return f"\"{s}\""
|
||||
|
||||
TEMPLATE_CONCEPT = """---
|
||||
title: "Concept Alpha"
|
||||
id: "concept-alpha"
|
||||
type: "concept"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/concept","topic/test"]
|
||||
---
|
||||
|
||||
## Beschreibung
|
||||
Dies ist ein einfaches Konzept für Testzwecke. Es wird von anderen Notizen referenziert.
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[exp-one]]
|
||||
- [[project-demo]]
|
||||
"""
|
||||
|
||||
TEMPLATE_EXP_ONE = """---
|
||||
title: "Experience One"
|
||||
id: "exp-one"
|
||||
type: "experience"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/experience","topic/test"]
|
||||
---
|
||||
|
||||
## Kontext
|
||||
Diese Notiz verlinkt auf ein existierendes Konzept und auf eine noch **fehlende** Note.
|
||||
|
||||
## Beobachtung
|
||||
- Bezug zu [[concept-alpha]]
|
||||
- Offener Verweis auf [[missing-note]]
|
||||
|
||||
## Interpretation
|
||||
Wenn später eine Datei für 'missing-note' angelegt wird, sollte der Importer die Kanten nachziehen.
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-alpha]]
|
||||
- [[missing-note]]
|
||||
"""
|
||||
|
||||
TEMPLATE_EXP_TWO = """---
|
||||
title: "Experience Two"
|
||||
id: "exp-two"
|
||||
type: "experience"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/experience","topic/test"]
|
||||
---
|
||||
|
||||
## Kontext
|
||||
Zweite Experience, die auf dasselbe Konzept verweist, ergänzt um **externe Links**.
|
||||
|
||||
## Beobachtung
|
||||
Verweis auf [[concept-alpha]]. Zusätzliche Ressourcen:
|
||||
- Link: [Beispielseite](https://example.com "Referenz")
|
||||
- Kontakt: [E-Mail](mailto:info@example.com)
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-alpha]]
|
||||
"""
|
||||
|
||||
TEMPLATE_PROJECT = """---
|
||||
title: "Project Demo"
|
||||
id: "project-demo"
|
||||
type: "project"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/project","topic/test"]
|
||||
---
|
||||
|
||||
## Scope
|
||||
Ein kleines Demo-Projekt, das andere Notizen referenziert.
|
||||
|
||||
## Arbeitspakete
|
||||
- AP-1: Zusammenhang erklären (siehe [[concept-alpha]])
|
||||
- AP-2: Ereignis berücksichtigen (siehe [[exp-one]])
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-alpha]]
|
||||
- [[exp-one]]
|
||||
"""
|
||||
|
||||
TEMPLATE_MISSING = """---
|
||||
title: "Missing Note (jetzt angelegt)"
|
||||
id: "missing-note"
|
||||
type: "concept"
|
||||
status: "active"
|
||||
created: {created}
|
||||
updated: {updated}
|
||||
tags: ["area/mindnet","type/concept","topic/test"]
|
||||
---
|
||||
|
||||
## Kontext
|
||||
Diese Notiz wurde **nachträglich** erstellt, um einen zuvor offenen Link zu schließen.
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[exp-one]]
|
||||
"""
|
||||
|
||||
def write_file(path: str, content: str) -> None:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
def build_vault(base: str, include_missing_note: bool) -> None:
|
||||
created = q(iso_now())
|
||||
updated = q(iso_now())
|
||||
|
||||
# 40_concepts
|
||||
write_file(
|
||||
os.path.join(base, "40_concepts", "concept-alpha.md"),
|
||||
TEMPLATE_CONCEPT.format(created=created, updated=updated),
|
||||
)
|
||||
|
||||
# 20_experiences
|
||||
write_file(
|
||||
os.path.join(base, "20_experiences", "exp-one.md"),
|
||||
TEMPLATE_EXP_ONE.format(created=created, updated=updated),
|
||||
)
|
||||
write_file(
|
||||
os.path.join(base, "20_experiences", "exp-two.md"),
|
||||
TEMPLATE_EXP_TWO.format(created=created, updated=updated),
|
||||
)
|
||||
|
||||
# 30_projects
|
||||
write_file(
|
||||
os.path.join(base, "30_projects", "project-demo.md"),
|
||||
TEMPLATE_PROJECT.format(created=created, updated=updated),
|
||||
)
|
||||
|
||||
# Optional: zunächst NICHT anlegen (Szenario „leerer Link“),
|
||||
# später für den 2. Testlauf anlegen.
|
||||
if include_missing_note:
|
||||
write_file(
|
||||
os.path.join(base, "40_concepts", "missing-note.md"),
|
||||
TEMPLATE_MISSING.format(created=created, updated=updated),
|
||||
)
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="./test_vault", help="Zielverzeichnis für den Test-Vault")
|
||||
ap.add_argument("--force", action="store_true", help="Falls vorhanden: Verzeichnis löschen und neu anlegen")
|
||||
ap.add_argument("--with-missing", action="store_true",
|
||||
help="Auch die 'missing-note' direkt anlegen (Standard: erst im 2. Lauf)")
|
||||
args = ap.parse_args()
|
||||
|
||||
base = os.path.abspath(args.out)
|
||||
if os.path.exists(base):
|
||||
if args.force:
|
||||
shutil.rmtree(base)
|
||||
else:
|
||||
print(f"Ziel existiert bereits: {base}\nNutze --force zum Überschreiben.")
|
||||
return
|
||||
|
||||
build_vault(base, include_missing_note=args.with_missing)
|
||||
print(f"Test-Vault erstellt unter: {base}")
|
||||
print("Hinweis:")
|
||||
print(" 1) Zuerst ohne missing-note anlegen (Standard), importieren, Edges prüfen.")
|
||||
print(" 2) Dann erneut ausführen mit --force --with-missing (missing-note wird erzeugt),")
|
||||
print(" und Import wiederholen → Edges/Backlinks sollten nachgezogen werden.")
|
||||
print(" 3) In exp-two.md sind **externe Links** enthalten → prüfe `external_links` in Chunk-Payloads.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,232 @@
|
||||
# scripts/prune_qdrant_vs_vault.py
|
||||
# -----------------------------------------------------------------------------
|
||||
# Name: prune_qdrant_vs_vault.py
|
||||
# Version: 1.0.0 (2025-09-08)
|
||||
# Zweck: Entfernt verwaiste Qdrant-Einträge (notes/chunks/edges), wenn
|
||||
# die zugehörigen Markdown-Dateien im Vault nicht mehr existieren.
|
||||
#
|
||||
# Was es macht:
|
||||
# - Liest alle note_id aus dem Vault (Frontmatter: id / note_id).
|
||||
# - Liest alle note_id aus Qdrant (mindnet_notes).
|
||||
# - Bildet die Differenz (nur in Qdrant vorhandene, im Vault fehlende).
|
||||
# - Löscht für jede verwaiste note_id:
|
||||
# * Notes-Point(s)
|
||||
# * Alle Chunks der Note
|
||||
# * Alle Edges, die auf diese Note referenzieren
|
||||
# (source_id == note_id ODER target_id == note_id ODER
|
||||
# source_note_id == note_id ODER target_note_id == note_id)
|
||||
#
|
||||
# Hinweise:
|
||||
# - Kein globaler Delete. Nur betroffene note_id.
|
||||
# - Dry-Run standardmäßig; tatsächliches Löschen erst mit --apply.
|
||||
# - Interaktive Bestätigung (abschaltbar mit --yes).
|
||||
#
|
||||
# Aufruf:
|
||||
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet
|
||||
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply
|
||||
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply --yes
|
||||
#
|
||||
# Voraussetzungen:
|
||||
# - Ausführung im aktivierten venv empfohlen: source .venv/bin/activate
|
||||
# - Qdrant läuft lokal (oder URL/API-Key in ENV), siehe app/core/qdrant.py
|
||||
#
|
||||
# Änderungen:
|
||||
# - 1.0.0: Erster Release.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Set, Tuple
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
from app.core.parser import parse_markdown_file # nutzt euer bestehendes Parsing/Schema
|
||||
|
||||
|
||||
def read_vault_note_ids(vault_dir: Path) -> Set[str]:
|
||||
note_ids: Set[str] = set()
|
||||
for p in vault_dir.rglob("*.md"):
|
||||
try:
|
||||
parsed = parse_markdown_file(str(p))
|
||||
fm = parsed.frontmatter if hasattr(parsed, "frontmatter") else (parsed.get("frontmatter") or {})
|
||||
nid = fm.get("id") or fm.get("note_id")
|
||||
if nid:
|
||||
note_ids.add(str(nid))
|
||||
except Exception:
|
||||
# still und leise weiter – wir wollen robust sein
|
||||
continue
|
||||
return note_ids
|
||||
|
||||
|
||||
def qdrant_note_ids(client: QdrantClient, notes_col: str) -> Set[str]:
|
||||
ids: Set[str] = set()
|
||||
scroll_filter = None
|
||||
offset = None
|
||||
while True:
|
||||
res = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=scroll_filter,
|
||||
offset=offset,
|
||||
limit=256,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
points, next_offset = res
|
||||
for pt in points:
|
||||
pl = pt.payload or {}
|
||||
nid = pl.get("note_id")
|
||||
if nid:
|
||||
ids.add(str(nid))
|
||||
if next_offset is None:
|
||||
break
|
||||
offset = next_offset
|
||||
return ids
|
||||
|
||||
|
||||
def delete_by_filter(client: QdrantClient, collection: str, flt: rest.Filter) -> int:
|
||||
# Sammel erst IDs via scroll (robuster Überblick), lösche dann über point_ids (batch)
|
||||
to_delete: List[int] = []
|
||||
offset = None
|
||||
while True:
|
||||
pts, next_offset = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=flt,
|
||||
offset=offset,
|
||||
limit=256,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
)
|
||||
for pt in pts:
|
||||
to_delete.append(pt.id)
|
||||
if next_offset is None:
|
||||
break
|
||||
offset = next_offset
|
||||
if not to_delete:
|
||||
return 0
|
||||
client.delete(collection_name=collection, points_selector=rest.PointIdsList(points=to_delete), wait=True)
|
||||
return len(to_delete)
|
||||
|
||||
|
||||
def prune_for_note(
|
||||
client: QdrantClient,
|
||||
cols: Dict[str, str],
|
||||
note_id: str,
|
||||
dry_run: bool = True,
|
||||
) -> Dict[str, int]:
|
||||
stats = {"notes": 0, "chunks": 0, "edges": 0}
|
||||
|
||||
# Notes löschen (Filter auf payload.note_id)
|
||||
f_notes = rest.Filter(
|
||||
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
|
||||
)
|
||||
|
||||
# Chunks löschen (Filter auf payload.note_id)
|
||||
f_chunks = rest.Filter(
|
||||
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
|
||||
)
|
||||
|
||||
# Edges löschen: mehrere Teilmengen erfassen
|
||||
edge_filters = [
|
||||
rest.Filter(must=[rest.FieldCondition(key="source_id", match=rest.MatchValue(value=note_id))]),
|
||||
rest.Filter(must=[rest.FieldCondition(key="target_id", match=rest.MatchValue(value=note_id))]),
|
||||
rest.Filter(must=[rest.FieldCondition(key="source_note_id", match=rest.MatchValue(value=note_id))]),
|
||||
rest.Filter(must=[rest.FieldCondition(key="target_note_id", match=rest.MatchValue(value=note_id))]),
|
||||
]
|
||||
|
||||
if dry_run:
|
||||
# Zähle nur, was *wäre* gelöscht worden
|
||||
def count(flt: rest.Filter, col: str) -> int:
|
||||
total = 0
|
||||
offset = None
|
||||
while True:
|
||||
pts, next_offset = client.scroll(
|
||||
collection_name=col,
|
||||
scroll_filter=flt,
|
||||
offset=offset,
|
||||
limit=256,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
)
|
||||
total += len(pts)
|
||||
if next_offset is None:
|
||||
break
|
||||
offset = next_offset
|
||||
return total
|
||||
|
||||
stats["notes"] = count(f_notes, cols["notes"])
|
||||
stats["chunks"] = count(f_chunks, cols["chunks"])
|
||||
stats["edges"] = sum(count(f, cols["edges"]) for f in edge_filters)
|
||||
return stats
|
||||
|
||||
# tatsächliches Löschen
|
||||
stats["notes"] = delete_by_filter(client, cols["notes"], f_notes)
|
||||
stats["chunks"] = delete_by_filter(client, cols["chunks"], f_chunks)
|
||||
e_deleted = 0
|
||||
for f in edge_filters:
|
||||
e_deleted += delete_by_filter(client, cols["edges"], f)
|
||||
stats["edges"] = e_deleted
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Prune Qdrant data for notes missing in the Vault.")
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Vault-Verzeichnis (Root).")
|
||||
ap.add_argument("--prefix", default="mindnet", help="Collections-Präfix (Default: mindnet).")
|
||||
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter wird nur ein Dry-Run gemacht.")
|
||||
ap.add_argument("--yes", action="store_true", help="Ohne Nachfrage ausführen.")
|
||||
args = ap.parse_args()
|
||||
|
||||
vault_root = Path(args.vault).resolve()
|
||||
if not vault_root.exists():
|
||||
print(f"Vault-Verzeichnis nicht gefunden: {vault_root}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
cfg = QdrantConfig()
|
||||
client = get_client(cfg)
|
||||
cols = collection_names(args.prefix)
|
||||
|
||||
vault_ids = read_vault_note_ids(vault_root)
|
||||
qdrant_ids = qdrant_note_ids(client, cols["notes"])
|
||||
orphans = sorted(qdrant_ids - vault_ids)
|
||||
|
||||
preview = {
|
||||
"mode": "APPLY" if args.apply else "DRY-RUN",
|
||||
"prefix": args.prefix,
|
||||
"vault_root": str(vault_root),
|
||||
"collections": cols,
|
||||
"counts": {
|
||||
"vault_note_ids": len(vault_ids),
|
||||
"qdrant_note_ids": len(qdrant_ids),
|
||||
"orphans": len(orphans),
|
||||
},
|
||||
"orphans_sample": orphans[:20],
|
||||
}
|
||||
print(json.dumps(preview, ensure_ascii=False, indent=2))
|
||||
|
||||
if not orphans:
|
||||
print("Keine verwaisten Einträge gefunden. Nichts zu tun.")
|
||||
return
|
||||
|
||||
if not args.yes:
|
||||
resp = input("\nFortfahren mit dem oben angezeigten Modus? (yes/no): ").strip().lower()
|
||||
if resp not in ("y", "yes", "j", "ja"):
|
||||
print("Abgebrochen.")
|
||||
return
|
||||
|
||||
total_stats = {"notes": 0, "chunks": 0, "edges": 0}
|
||||
for nid in orphans:
|
||||
s = prune_for_note(client, cols, nid, dry_run=(not args.apply))
|
||||
total_stats["notes"] += s["notes"]
|
||||
total_stats["chunks"] += s["chunks"]
|
||||
total_stats["edges"] += s["edges"]
|
||||
|
||||
print(json.dumps({"deleted": total_stats}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
# scripts/resolve_unresolved_references.py
|
||||
"""
|
||||
resolve_unresolved_references.py — Unaufgelöste Wikilinks in Qdrant nachträglich auflösen
|
||||
|
||||
Version: 1.0.0 (2025-09-05)
|
||||
|
||||
Zweck
|
||||
------
|
||||
- Findet Edges in {prefix}_edges mit payload.status=="unresolved" und versucht, den Zielknoten
|
||||
anhand bereits vorhandener Notes in {prefix}_notes aufzulösen.
|
||||
- Aktualisiert die Edges (setzt target_id, entfernt status, setzt resolution), und erzeugt
|
||||
– NUR für Note-Level 'references' – die symmetrische 'backlink'-Kante.
|
||||
|
||||
Warum?
|
||||
------
|
||||
- Beim ersten Import können Links auf (noch) nicht existierende Notizen zeigen.
|
||||
- Sobald die Zielnotiz später existiert, kann dieses Skript die Kanten reparieren.
|
||||
|
||||
Aufruf
|
||||
------
|
||||
# Dry-Run (Standard):
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet
|
||||
|
||||
# Anwenden:
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
||||
|
||||
# Optional: nur X Edges anfassen
|
||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply --limit 500
|
||||
|
||||
Parameter
|
||||
---------
|
||||
--prefix : Collection-Prefix (Default: aus Env COLLECION_PREFIX oder "mindnet")
|
||||
--apply : Änderungen tatsächlich schreiben (ohne --apply = Dry-Run)
|
||||
--limit : Max. Anzahl unaufgelöster Edges, die in diesem Lauf bearbeitet werden (Default: keine Begrenzung)
|
||||
--batch : Upsert-Batchgröße (Default: 512)
|
||||
|
||||
Voraussetzungen / Hinweise
|
||||
--------------------------
|
||||
- Bitte im aktivierten venv laufen lassen (deine Umgebung: `.venv`).
|
||||
- Qdrant-URL/Key/Prefix/Vektor-Dim werden wie üblich aus ENV gelesen (sieh app/core/qdrant.py). # noqa
|
||||
- Nutzt die vorhandenen Utilities:
|
||||
- app/core/qdrant.py (Client/Collections)
|
||||
- app/core/qdrant_points.py (points_for_edges/upsert_batch)
|
||||
- app/core/derive_edges.py (build_note_index/resolve_target)
|
||||
|
||||
Änderungshistorie
|
||||
-----------------
|
||||
1.0.0 Erstveröffentlichung.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from typing import Any, Dict, List, Tuple, Iterable
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names # :contentReference[oaicite:3]{index=3}
|
||||
from app.core.qdrant_points import points_for_edges, upsert_batch # :contentReference[oaicite:4]{index=4}
|
||||
from app.core.derive_edges import build_note_index, resolve_target # :contentReference[oaicite:5]{index=5}
|
||||
|
||||
|
||||
def _scroll(client: QdrantClient, **kwargs):
|
||||
"""
|
||||
Wrapper um qdrant_client.scroll() für unterschiedliche Client-Versionen:
|
||||
neuere: (points, next_offset)
|
||||
ältere: (points, next_page_offset, _)
|
||||
"""
|
||||
res = client.scroll(**kwargs)
|
||||
if isinstance(res, tuple):
|
||||
if len(res) == 2:
|
||||
points, next_off = res
|
||||
else:
|
||||
# ältere Signatur: (points, next_off, _)
|
||||
points, next_off, _ = res[0], res[1], res[2]
|
||||
else:
|
||||
# sehr alte Clients -> konservativ behandeln
|
||||
points, next_off = res, None
|
||||
return points, next_off
|
||||
|
||||
|
||||
def _load_all_notes(client: QdrantClient, notes_col: str) -> List[Dict[str, Any]]:
|
||||
notes: List[Dict[str, Any]] = []
|
||||
next_off = None
|
||||
while True:
|
||||
pts, next_off = _scroll(
|
||||
client,
|
||||
collection_name=notes_col,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1024,
|
||||
offset=next_off,
|
||||
)
|
||||
for p in pts or []:
|
||||
pl = getattr(p, "payload", {}) or {}
|
||||
# Erwartet Felder: note_id, title, path etc. (gemäß Schema) # :contentReference[oaicite:6]{index=6}
|
||||
if pl.get("note_id"):
|
||||
notes.append(pl)
|
||||
if not next_off:
|
||||
break
|
||||
return notes
|
||||
|
||||
|
||||
def _iter_unresolved_edges(client: QdrantClient, edges_col: str) -> Iterable[rest.Record]:
|
||||
"""
|
||||
Liefert alle Edge-Records mit payload.status == 'unresolved' und 'target_label' (string).
|
||||
"""
|
||||
f = rest.Filter(
|
||||
must=[
|
||||
rest.FieldCondition(key="status", match=rest.MatchValue(value="unresolved")),
|
||||
]
|
||||
)
|
||||
next_off = None
|
||||
while True:
|
||||
pts, next_off = _scroll(
|
||||
client,
|
||||
collection_name=edges_col,
|
||||
scroll_filter=f,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1024,
|
||||
offset=next_off,
|
||||
)
|
||||
for p in pts or []:
|
||||
pl = getattr(p, "payload", {}) or {}
|
||||
if isinstance(pl.get("target_label"), str):
|
||||
yield p
|
||||
if not next_off:
|
||||
break
|
||||
|
||||
|
||||
def _make_backlink(source_note_id: str, target_note_id: str, extra: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Baue eine 'backlink'-Edge-Payload source <- target (note-level).
|
||||
"""
|
||||
e = {
|
||||
"kind": "backlink",
|
||||
"source_id": target_note_id,
|
||||
"target_id": source_note_id,
|
||||
}
|
||||
# Metafelder aus dem Original übernehmen (ohne status)
|
||||
copy_keys = ["raw", "alias", "heading", "resolution"]
|
||||
for k in copy_keys:
|
||||
if k in extra:
|
||||
e[k] = extra[k]
|
||||
return e
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (Default: Env/COLLECTION_PREFIX oder 'mindnet')")
|
||||
ap.add_argument("--apply", action="store_true", help="Änderungen schreiben (ohne Flag = Dry-Run)")
|
||||
ap.add_argument("--limit", type=int, default=0, help="Max. Anzahl unaufgelöster Edges bearbeiten (0 = alle)")
|
||||
ap.add_argument("--batch", type=int, default=512, help="Upsert-Batchgröße")
|
||||
args = ap.parse_args()
|
||||
|
||||
# Qdrant-Setup
|
||||
cfg = QdrantConfig.from_env()
|
||||
if args.prefix:
|
||||
cfg.prefix = args.prefix
|
||||
client = get_client(cfg)
|
||||
ensure_collections(client, cfg.prefix, cfg.dim) # sorgt u. a. für 1D-Vektor-Collection bei Edges :contentReference[oaicite:7]{index=7}
|
||||
notes_col, _, edges_col = collection_names(cfg.prefix) # :contentReference[oaicite:8]{index=8}
|
||||
|
||||
# Notes laden & Index bauen
|
||||
notes = _load_all_notes(client, notes_col)
|
||||
idx = build_note_index(notes) # (by_id, by_slug, by_file_slug) :contentReference[oaicite:9]{index=9}
|
||||
|
||||
# Unresolved-Edges scannen
|
||||
to_fix: List[dict] = []
|
||||
backlinks: List[dict] = []
|
||||
processed = 0
|
||||
resolved = 0
|
||||
|
||||
for rec in _iter_unresolved_edges(client, edges_col):
|
||||
if args.limit and processed >= args.limit:
|
||||
break
|
||||
processed += 1
|
||||
|
||||
pl = dict(rec.payload or {})
|
||||
kind = pl.get("kind") or "references"
|
||||
src = pl.get("source_id")
|
||||
tgt_label = pl.get("target_label") or pl.get("target_id") # Fallback
|
||||
|
||||
# Zielauflösung
|
||||
resolved_id, how = resolve_target(str(tgt_label), idx) # :contentReference[oaicite:10]{index=10}
|
||||
if not resolved_id:
|
||||
continue # weiterhin unresolved
|
||||
|
||||
# Edge-Update
|
||||
new_pl = dict(pl)
|
||||
new_pl["target_id"] = resolved_id
|
||||
new_pl["resolution"] = how
|
||||
if "status" in new_pl:
|
||||
del new_pl["status"]
|
||||
# ID stabil lassen -> points_for_edges erzeugt UUID aus edge_id/Fallback :contentReference[oaicite:11]{index=11}
|
||||
if "edge_id" not in new_pl:
|
||||
# stabiler Key aus (kind, src, tgt, evtl. seq)
|
||||
seq = new_pl.get("seq") or new_pl.get("order") or ""
|
||||
new_pl["edge_id"] = f"{kind}:{src}->{resolved_id}#{seq}"
|
||||
|
||||
to_fix.append(new_pl)
|
||||
resolved += 1
|
||||
|
||||
# Nur bei Note-Level references (nicht references_at) -> Backlink erzeugen
|
||||
if kind == "references":
|
||||
extra = {k: new_pl.get(k) for k in ("raw", "alias", "heading")}
|
||||
extra["resolution"] = how
|
||||
backlinks.append(_make_backlink(source_note_id=src, target_note_id=resolved_id, extra=extra))
|
||||
|
||||
# Ergebnis ausgeben
|
||||
summary = {
|
||||
"prefix": cfg.prefix,
|
||||
"scanned_unresolved": processed,
|
||||
"resolved": resolved,
|
||||
"backlinks_to_create": len(backlinks),
|
||||
"apply": bool(args.apply),
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
|
||||
if not args.apply:
|
||||
return
|
||||
|
||||
# Upserts (in Batches)
|
||||
def _batched(items: List[dict], n: int) -> Iterable[List[dict]]:
|
||||
for i in range(0, len(items), n):
|
||||
yield items[i : i + n]
|
||||
|
||||
# 1) Updates für reparierte Edges
|
||||
for chunk in _batched(to_fix, args.batch):
|
||||
col, pts = points_for_edges(cfg.prefix, chunk) # sorgt für Edge-UUID & Dummy-Vector :contentReference[oaicite:12]{index=12}
|
||||
upsert_batch(client, col, pts)
|
||||
|
||||
# 2) Backlinks (nur references)
|
||||
for chunk in _batched(backlinks, args.batch):
|
||||
col, pts = points_for_edges(cfg.prefix, chunk)
|
||||
upsert_batch(client, col, pts)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: scripts/verify_chunk_texts.py
|
||||
Version: 1.0.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
Kurzbeschreibung
|
||||
---------------
|
||||
Verifiziert, dass in Qdrant für jede Note die zugehörigen Chunks ein Textfeld
|
||||
enthalten und der Body (notes.payload.fulltext) aus den Chunk-Texten sinnvoll
|
||||
rekonstruiert werden kann.
|
||||
|
||||
Prüfungen pro Note:
|
||||
- Alle Chunks vorhanden (>=1).
|
||||
- Jedes Chunk-Payload hat einen nutzbaren Textschlüssel: "text" (bevorzugt), sonst "content", sonst "raw".
|
||||
- Reihenfolge der Chunks wird stabil bestimmt (payload.chunk_index -> Nummer aus chunk_id).
|
||||
- Coverage: Summe der im Fulltext gefundenen Chunk-Textsegmente / len(Fulltext) (Toleranz für Overlaps).
|
||||
-> OK wenn coverage >= 0.90 (konfigurierbar via --min-coverage)
|
||||
|
||||
Ausgabe:
|
||||
- JSON mit Gesamtsummen und Details je Note.
|
||||
|
||||
ENV:
|
||||
- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
|
||||
- COLLECTION_PREFIX (Fallback, wenn --prefix fehlt)
|
||||
|
||||
Beispiele:
|
||||
# Alle Notes prüfen (Prefix aus ENV)
|
||||
python3 -m scripts.verify_chunk_texts
|
||||
|
||||
# Nur eine Note prüfen
|
||||
python3 -m scripts.verify_chunk_texts --note-id concept-alpha
|
||||
|
||||
# Prefix explizit setzen und strengere Coverage verlangen
|
||||
python3 -m scripts.verify_chunk_texts --prefix mindnet --min-coverage 0.95
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, json, os, re, sys
|
||||
from typing import Dict, List, Tuple, Optional
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
def _names(prefix: str) -> Tuple[str,str,str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def _client() -> QdrantClient:
|
||||
url = os.getenv("QDRANT_URL")
|
||||
if not url:
|
||||
host = os.getenv("QDRANT_HOST", "127.0.0.1")
|
||||
port = int(os.getenv("QDRANT_PORT", "6333"))
|
||||
url = f"http://{host}:{port}"
|
||||
api_key = os.getenv("QDRANT_API_KEY") or None
|
||||
return QdrantClient(url=url, api_key=api_key)
|
||||
|
||||
def _chunk_sort_key(p: Dict, pid: str) -> Tuple[int,int,str]:
|
||||
# Primär: payload.chunk_index, sekundär: Nummer am Ende der ID (#cNN oder #NN), sonst 0
|
||||
ci = p.get("chunk_index")
|
||||
n = 0
|
||||
m = re.search(r'#c?(\d+)$', pid or "")
|
||||
if m:
|
||||
try:
|
||||
n = int(m.group(1))
|
||||
except Exception:
|
||||
n = 0
|
||||
return (ci if isinstance(ci, int) else 1_000_000 + n, n, pid)
|
||||
|
||||
def _choose_text(payload: Dict) -> Optional[str]:
|
||||
for k in ("text", "content", "raw"):
|
||||
v = payload.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v
|
||||
return None
|
||||
|
||||
def _coverage(fulltext: str, pieces: List[str]) -> float:
|
||||
"""Berechnet die Abdeckungsquote der Stücke im Fulltext (sequenzielles Matching)."""
|
||||
if not fulltext:
|
||||
return 0.0 if pieces else 1.0
|
||||
cursor = 0
|
||||
covered = 0
|
||||
ft = fulltext
|
||||
for piece in pieces:
|
||||
if not piece:
|
||||
continue
|
||||
# Tolerant gegen Whitespace-Unterschiede: normalisieren nur \r\n→\n
|
||||
p = piece.replace("\r\n", "\n").replace("\r", "\n")
|
||||
idx = ft.find(p, cursor)
|
||||
if idx == -1:
|
||||
# Versuche ein paar Heuristiken: trimmen
|
||||
p2 = p.strip()
|
||||
if p2 and len(p2) > 8:
|
||||
idx = ft.find(p2, cursor)
|
||||
if idx != -1:
|
||||
covered += len(p)
|
||||
cursor = idx + len(p)
|
||||
# sonst: nicht abgedeckt
|
||||
return covered / max(1, len(ft))
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (Default: ENV COLLECTION_PREFIX oder 'mindnet')")
|
||||
ap.add_argument("--note-id", help="Nur eine bestimmte Note prüfen")
|
||||
ap.add_argument("--min-coverage", type=float, default=0.90, help="Mindestabdeckung durch Chunks (Default: 0.90)")
|
||||
args = ap.parse_args()
|
||||
|
||||
prefix = args.prefix or os.getenv("COLLECTION_PREFIX", "mindnet")
|
||||
notes_col, chunks_col, _ = _names(prefix)
|
||||
cli = _client()
|
||||
|
||||
# Notes abrufen (optional filter by note_id)
|
||||
notes_filter = None
|
||||
if args.note_id:
|
||||
notes_filter = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=args.note_id))])
|
||||
|
||||
notes: List[Dict] = []
|
||||
off = None
|
||||
while True:
|
||||
pts, off = cli.scroll(collection_name=notes_col, scroll_filter=notes_filter,
|
||||
with_payload=True, with_vectors=False, limit=256, offset=off)
|
||||
if not pts:
|
||||
break
|
||||
for p in pts:
|
||||
notes.append({"id": p.id, "payload": p.payload or {}})
|
||||
if off is None:
|
||||
break
|
||||
|
||||
results = []
|
||||
total_missing_text = 0
|
||||
total_notes_ok = 0
|
||||
for n in notes:
|
||||
pl = n["payload"]
|
||||
nid = pl.get("note_id") or pl.get("id") or n.get("id")
|
||||
fulltext = pl.get("fulltext") or ""
|
||||
# Chunks der Note holen
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||
chunks = []
|
||||
off = None
|
||||
while True:
|
||||
pts, off = cli.scroll(collection_name=chunks_col, scroll_filter=f,
|
||||
with_payload=True, with_vectors=False, limit=256, offset=off)
|
||||
if not pts:
|
||||
break
|
||||
for p in pts:
|
||||
chunks.append({"id": p.id, "payload": p.payload or {}})
|
||||
if off is None:
|
||||
break
|
||||
# sortieren
|
||||
chunks.sort(key=lambda c: _chunk_sort_key(c["payload"], c["id"]))
|
||||
texts = []
|
||||
missing_text = 0
|
||||
for c in chunks:
|
||||
t = _choose_text(c["payload"])
|
||||
if t is None:
|
||||
missing_text += 1
|
||||
texts.append("")
|
||||
else:
|
||||
texts.append(t)
|
||||
cov = _coverage(fulltext, texts)
|
||||
ok = (missing_text == 0) and (cov >= args.min_coverage or not fulltext)
|
||||
if ok:
|
||||
total_notes_ok += 1
|
||||
total_missing_text += missing_text
|
||||
results.append({
|
||||
"note_id": nid,
|
||||
"title": pl.get("title"),
|
||||
"chunks": len(chunks),
|
||||
"missing_chunk_texts": missing_text,
|
||||
"coverage": round(cov, 4),
|
||||
"has_fulltext": bool(fulltext),
|
||||
"ok": ok
|
||||
})
|
||||
|
||||
out = {
|
||||
"collections": {"notes": notes_col, "chunks": chunks_col},
|
||||
"notes_checked": len(notes),
|
||||
"notes_ok": total_notes_ok,
|
||||
"total_missing_chunk_texts": total_missing_text,
|
||||
"min_coverage": args.min_coverage,
|
||||
"details": results
|
||||
}
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user