Compare commits
39
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0a891cfd95 | ||
|
|
66adc6a354 | ||
|
|
e99f1c2ba0 | ||
|
|
f52a9face8 | ||
|
|
4fcf22131c | ||
|
|
d27324f8dd | ||
|
|
dd86a15555 | ||
|
|
28d173b8fd | ||
|
|
b4b7ea76ab | ||
|
|
6ea452cc3f | ||
|
|
6e0e965883 | ||
|
|
6b4b3ce2ba | ||
|
|
18cdbab885 | ||
|
|
ebca9b4caf | ||
|
|
6945ef8ddc | ||
|
|
f20bf03ee1 | ||
|
|
7cda15553d | ||
|
|
3c67fd5f9b | ||
|
|
0150931df4 | ||
|
|
5ff1b06009 | ||
|
|
5279bcae18 | ||
|
|
daa58e6b27 | ||
|
|
589d021744 | ||
|
|
e58b9c8ada | ||
|
|
507c28e006 | ||
|
|
2d74fe4057 | ||
|
|
e8312268d2 | ||
|
|
f4cd1e51c0 | ||
|
|
60e707e7df | ||
|
|
0d6eb4dac4 | ||
|
|
80d8f986bd | ||
|
|
9ee3bb9007 | ||
|
|
8a07010704 | ||
|
|
314fc8ac94 | ||
|
|
b46e5461a7 | ||
|
|
ada7c1797d | ||
|
|
54197995f5 | ||
|
|
13e510dd9c | ||
|
|
5c937fbffe |
No files matched your search
+31
-12
@@ -1,22 +1,36 @@
|
|||||||
# .gitea/workflows/deploy.yml
|
# .gitea/workflows/deploy.yml
|
||||||
name: Deploy mindnet to llm-node
|
name: Deploy mindnet to llm-node
|
||||||
on: { push: { branches: [ "main" ] } }
|
|
||||||
# on: {}
|
|
||||||
|
|
||||||
concurrency: { group: deploy-mindnet, cancel-in-progress: false }
|
on:
|
||||||
|
push:
|
||||||
|
branches: ["main"]
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: deploy-mindnet
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
deploy:
|
deploy:
|
||||||
runs-on: linux_host
|
runs-on: linux_host
|
||||||
env:
|
env:
|
||||||
DEPLOY_DIRS: "app scripts schemas docker requirements.txt README.md"
|
TARGET_BASE: /home/llmadmin/mindnet
|
||||||
TARGET_BASE: "/home/llmadmin/mindnet"
|
DEPLOY_DIRS: "app scripts schemas docker tests requirements.txt README.md"
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: https://github.com/actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
- name: Ensure target base
|
|
||||||
|
- name: Ensure target base exists
|
||||||
run: install -d "$TARGET_BASE"
|
run: install -d "$TARGET_BASE"
|
||||||
- name: Deploy whitelisted
|
|
||||||
|
- name: Stop API (graceful)
|
||||||
|
continue-on-error: true
|
||||||
|
run: |
|
||||||
|
if systemctl --user list-unit-files | grep -q '^mindnet-api.service'; then
|
||||||
|
systemctl --user stop mindnet-api.service || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Deploy whitelisted (rsync)
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
IFS=' ' read -r -a DIRS <<< "$DEPLOY_DIRS"
|
IFS=' ' read -r -a DIRS <<< "$DEPLOY_DIRS"
|
||||||
@@ -24,22 +38,27 @@ jobs:
|
|||||||
if [ -e "$d" ]; then
|
if [ -e "$d" ]; then
|
||||||
if [ -d "$d" ]; then
|
if [ -d "$d" ]; then
|
||||||
install -d "$TARGET_BASE/$d"
|
install -d "$TARGET_BASE/$d"
|
||||||
rsync -a --delete --exclude='.git' \
|
rsync -a --delete \
|
||||||
--exclude='.env' --exclude='.env.*' --exclude='**/.env*' \
|
--exclude='.git' \
|
||||||
|
--exclude='.env*' \
|
||||||
|
--exclude='.venv' \
|
||||||
|
--exclude='hf_cache' --exclude='hf_cache/**' \
|
||||||
"$d"/ "$TARGET_BASE/$d"/
|
"$d"/ "$TARGET_BASE/$d"/
|
||||||
else
|
else
|
||||||
rsync -a "$d" "$TARGET_BASE/$d"
|
rsync -a "$d" "$TARGET_BASE/$d"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
- name: Python venv (idempotent)
|
|
||||||
|
- name: Python venv & requirements
|
||||||
run: |
|
run: |
|
||||||
cd "$TARGET_BASE"
|
cd "$TARGET_BASE"
|
||||||
[ -d .venv ] || python3 -m venv .venv
|
[ -d .venv ] || python3 -m venv .venv
|
||||||
source .venv/bin/activate
|
source .venv/bin/activate
|
||||||
pip install --upgrade pip
|
pip install --upgrade pip
|
||||||
[ -f requirements.txt ] && pip install -r requirements.txt || true
|
[ -f requirements.txt ] && pip install -r requirements.txt || true
|
||||||
- name: Optional — restart mindnet-api
|
|
||||||
|
- name: Start/Restart mindnet-api
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
run: |
|
run: |
|
||||||
if systemctl --user list-unit-files | grep -q '^mindnet-api.service'; then
|
if systemctl --user list-unit-files | grep -q '^mindnet-api.service'; then
|
||||||
|
|||||||
+235
-217
@@ -2,252 +2,270 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Modul: app/core/chunk_payload.py
|
Modul: app/core/chunk_payload.py
|
||||||
Version: 2.0.0
|
Version: 2.2.0
|
||||||
Datum: 2025-09-09
|
Datum: 2025-10-06
|
||||||
|
|
||||||
Kurzbeschreibung
|
Zweck
|
||||||
----------------
|
-----
|
||||||
Erzeugt **Chunk-Payloads** für die Qdrant-Collection `<prefix>_chunks` auf Basis der
|
Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.0.1.
|
||||||
vom Chunker gelieferten `Chunk`-Objekte. Ziel:
|
Neu: Wenn der Chunker KEIN Overlap im Fenster liefert (== window fehlt / identisch zur Kernpassage),
|
||||||
- *Verlustfreie Rekonstruktion*: Jeder Chunk enthält seinen **Text** (payload["text"]).
|
erzeugen wir FENSTER mit synthetischem Overlap auf Basis chunk_config.get_sizes(note_type)['overlap'].
|
||||||
- *Schnelle Abfragen*: Wichtige Note-Metadaten werden gespiegelt, um Filter ohne Join zu erlauben.
|
|
||||||
- *Graph-Kompatibilität*: Wikilinks und externe Links werden extrahiert; Nachbarschaften werden übernommen.
|
|
||||||
- *Monitoring*: Token- und Längenmetriken sowie Text-Hash erleichtern Audits und Re-Embeddings.
|
|
||||||
|
|
||||||
Kompatibilität
|
Felder (beibehalten aus 2.0.1):
|
||||||
--------------
|
- note_id, chunk_id, id (Alias), chunk_index, seq, path
|
||||||
- **Abwärtskompatibel** zur bisherigen `make_chunk_payloads`-Signatur.
|
- window (mit Overlap), text (ohne linkes Overlap)
|
||||||
- Zusätzliche Felder stören bestehende Upserts nicht (Payload ist schema-flexibel).
|
- start, end (Offsets im gesamten Body)
|
||||||
- Erwartet, dass `Chunk` u. a. die Attribute `id`, `index`, `text`, `char_start`, `char_end`,
|
- overlap_left, overlap_right
|
||||||
`section_title`, `section_path`, `neighbors_prev`, `neighbors_next` liefert.
|
- token_count?, section_title?, section_path?, type?, title?, tags?
|
||||||
|
|
||||||
CLI (Schnelltest)
|
Kompatibilität:
|
||||||
-----------------
|
- 'id' == 'chunk_id' als Alias
|
||||||
# Preview aus einer Markdown-Datei
|
- Pfade bleiben relativ (keine führenden '/'), Backslashes → Slashes
|
||||||
python3 -m app.core.chunk_payload --from-file ./test_vault/20_experiences/exp-two.md --vault-root ./test_vault
|
- Robust für Chunk-Objekte oder Dicts; Fensterquelle: 'window'|'text'|'content'|'raw'
|
||||||
|
|
||||||
# Nur IDs & Tokenmengen
|
Lizenz: MIT (projektintern)
|
||||||
python3 -m app.core.chunk_payload --from-file ./test_vault/20_experiences/exp-two.md --vault-root ./test_vault --summary
|
|
||||||
|
|
||||||
Felder (Auszug)
|
|
||||||
---------------
|
|
||||||
id : "<note_id>#cNN"
|
|
||||||
scope : "chunk"
|
|
||||||
note_id : "<note_id>"
|
|
||||||
note_title : str
|
|
||||||
note_type : str
|
|
||||||
note_status : str
|
|
||||||
area, project : optional
|
|
||||||
tags : list[str]
|
|
||||||
note_path : str (relativ, Slashes normalisiert)
|
|
||||||
chunk_index : int
|
|
||||||
section_title : str | None
|
|
||||||
section_path : str | None
|
|
||||||
char_start : int | None
|
|
||||||
char_end : int | None
|
|
||||||
char_len : int
|
|
||||||
token_est : int (≈ len(text)/4)
|
|
||||||
neighbors : {"prev": str|None, "next": str|None}
|
|
||||||
text : str (Chunk-Text, **Pflicht**)
|
|
||||||
text_sha256 : str "sha256:<hex>"
|
|
||||||
lang : optional
|
|
||||||
wikilinks : list[str]
|
|
||||||
external_links : list[{"href": str, "label": str|None}]
|
|
||||||
references : list[{"target_id": str, "kind": "wikilink"}]
|
|
||||||
embed_model : optional (durchreichbar)
|
|
||||||
embed_dim : optional
|
|
||||||
embed_version : optional
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
from typing import Dict, List, Optional, Tuple
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Paket-Import (normaler Betrieb)
|
# Typgerechtes Overlap aus deiner Konfiguration holen
|
||||||
from app.core.chunker import Chunk
|
from app.core.chunk_config import get_sizes as _get_sizes
|
||||||
from app.core.parser import extract_wikilinks, read_markdown, normalize_frontmatter, validate_required_frontmatter
|
except Exception:
|
||||||
except Exception: # pragma: no cover
|
def _get_sizes(_note_type: str):
|
||||||
# Relativ (lokale Tests)
|
# konservativer Default, falls Import fehlschlägt
|
||||||
from .chunker import Chunk # type: ignore
|
return {"overlap": (40, 60), "target": (250, 350), "max": 500}
|
||||||
from .parser import extract_wikilinks, read_markdown, normalize_frontmatter, validate_required_frontmatter # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ------------------------------- Utils ------------------------------- #
|
||||||
# Utilities
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
RE_MD_LINK = re.compile(r"\[([^\]]*)\]\(([^)\s]+)(?:\s+\"([^\"]+)\")?\)")
|
def _get_attr_or_key(obj: Any, key: str, default=None):
|
||||||
RE_HTTP_SCHEMES = ("http://", "https://", "mailto:", "obsidian://", "tel:")
|
if obj is None:
|
||||||
|
return default
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
return obj.get(key, default)
|
||||||
|
return getattr(obj, key, default)
|
||||||
|
|
||||||
def _estimate_tokens(text: str) -> int:
|
def _as_window_text(chunk: Any) -> str:
|
||||||
"""Grobe Token-Schätzung (≈ 1 Token pro 4 Zeichen)."""
|
"""Fenstertext robust lesen (bevorzugt echte Fenster, sonst Kern)."""
|
||||||
return max(0, int(round(len((text or '').strip()) / 4)))
|
for k in ("window", "text", "content", "raw"):
|
||||||
|
v = _get_attr_or_key(chunk, k, None)
|
||||||
|
if isinstance(v, str) and v:
|
||||||
|
return v
|
||||||
|
return ""
|
||||||
|
|
||||||
def _sha256_text(text: str) -> str:
|
def _to_int(x: Any, default: int = 0) -> int:
|
||||||
h = hashlib.sha256()
|
try:
|
||||||
h.update((text or "").encode("utf-8"))
|
return int(x)
|
||||||
return "sha256:" + h.hexdigest()
|
except Exception:
|
||||||
|
return default
|
||||||
|
|
||||||
def _normalize_rel_path(path: Optional[str], vault_root: Optional[str]) -> Optional[str]:
|
def _normalize_rel_path(p: str) -> str:
|
||||||
if not path:
|
p = (p or "").replace("\\", "/")
|
||||||
return None
|
while p.startswith("/"):
|
||||||
p = str(path)
|
p = p[1:]
|
||||||
p = p.replace("\\", "/")
|
|
||||||
if vault_root and os.path.isabs(p):
|
|
||||||
try:
|
|
||||||
p = os.path.relpath(p, vault_root)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
p = p.replace("\\", "/").lstrip("/")
|
|
||||||
return p
|
return p
|
||||||
|
|
||||||
def _extract_external_links(text: str) -> List[Dict[str, Optional[str]]]:
|
|
||||||
"""Findet Markdown-Links [label](href "title") mit erlaubten Schemes."""
|
|
||||||
out: List[Dict[str, Optional[str]]] = []
|
|
||||||
if not text:
|
|
||||||
return out
|
|
||||||
for m in RE_MD_LINK.finditer(text):
|
|
||||||
label = (m.group(1) or "").strip() or None
|
|
||||||
href = (m.group(2) or "").strip()
|
|
||||||
title = (m.group(3) or "").strip() or None
|
|
||||||
if any(href.startswith(s) for s in RE_HTTP_SCHEMES):
|
|
||||||
out.append({"href": href, "label": label or title})
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
# ---------------------- Overlap & Offsets ---------------------------- #
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int], str]:
|
||||||
# Public API
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def make_chunk_payloads(note_meta: Dict, path: str, chunks: List[Chunk]) -> List[Dict]:
|
|
||||||
"""
|
"""
|
||||||
Erzeugt Payload-Dicts für alle Chunks einer Note.
|
Entfernt linkes Overlap aus echten Fenster-Strings.
|
||||||
|
Rückgabe: (segments, overlaps_left, reconstructed_text)
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
note_meta : Dict
|
|
||||||
Normalisierte Frontmatter der Note (mind.: id, title, type, status, tags, [area, project, lang]).
|
|
||||||
path : str
|
|
||||||
Pfad zur Note (sollte relativ zum Vault sein; wird hier zur Sicherheit normalisiert).
|
|
||||||
chunks : List[Chunk]
|
|
||||||
Vom Chunker erzeugte Chunks.
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
List[Dict]
|
|
||||||
Liste von Payloads (ein Eintrag pro Chunk).
|
|
||||||
"""
|
"""
|
||||||
res: List[Dict] = []
|
segments: List[str] = []
|
||||||
rel_path = _normalize_rel_path(path, vault_root=None)
|
overlaps_left: List[int] = []
|
||||||
|
reconstructed = ""
|
||||||
|
for w in windows:
|
||||||
|
w = w or ""
|
||||||
|
max_k = min(len(w), len(reconstructed))
|
||||||
|
k = 0
|
||||||
|
for cand in range(max_k, -1, -1):
|
||||||
|
if reconstructed.endswith(w[:cand]):
|
||||||
|
k = cand
|
||||||
|
break
|
||||||
|
seg = w[k:]
|
||||||
|
segments.append(seg)
|
||||||
|
overlaps_left.append(k)
|
||||||
|
reconstructed += seg
|
||||||
|
return segments, overlaps_left, reconstructed
|
||||||
|
|
||||||
for ch in chunks:
|
def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
|
||||||
text: str = getattr(ch, "text", "") or ""
|
"""Länge längsten Suffix(a), der Prefix(b) ist."""
|
||||||
wikilinks = extract_wikilinks(text)
|
if not a or not b:
|
||||||
ext_links = _extract_external_links(text)
|
return 0
|
||||||
|
a1 = a[-max_probe:]
|
||||||
payload: Dict = {
|
b1 = b[:max_probe]
|
||||||
"id": getattr(ch, "id", None),
|
n = min(len(a1), len(b1))
|
||||||
"scope": "chunk",
|
for k in range(n, 0, -1):
|
||||||
"note_id": note_meta.get("id"),
|
if a1[-k:] == b1[:k]:
|
||||||
"note_title": note_meta.get("title"),
|
return k
|
||||||
# gespiegelt für schnelle Filter:
|
return 0
|
||||||
"note_type": note_meta.get("type"),
|
|
||||||
"note_status": note_meta.get("status"),
|
|
||||||
"area": note_meta.get("area"),
|
|
||||||
"project": note_meta.get("project"),
|
|
||||||
"tags": note_meta.get("tags"),
|
|
||||||
# Pfad
|
|
||||||
"note_path": rel_path,
|
|
||||||
"path": rel_path, # Back-compat
|
|
||||||
# Reihenfolge & Section
|
|
||||||
"chunk_index": getattr(ch, "index", None),
|
|
||||||
"section_title": getattr(ch, "section_title", None),
|
|
||||||
"section_path": getattr(ch, "section_path", None),
|
|
||||||
# Position
|
|
||||||
"char_start": getattr(ch, "char_start", None),
|
|
||||||
"char_end": getattr(ch, "char_end", None),
|
|
||||||
"char_len": max(0, int(getattr(ch, "char_end", 0) or 0) - int(getattr(ch, "char_start", 0) or 0)) or len(text),
|
|
||||||
# Nachbarn
|
|
||||||
"neighbors": {
|
|
||||||
"prev": getattr(ch, "neighbors_prev", None),
|
|
||||||
"next": getattr(ch, "neighbors_next", None),
|
|
||||||
},
|
|
||||||
# Inhalt & Metrik
|
|
||||||
"text": text,
|
|
||||||
"text_sha256": _sha256_text(text),
|
|
||||||
"token_est": _estimate_tokens(text),
|
|
||||||
# Sprache
|
|
||||||
"lang": note_meta.get("lang"),
|
|
||||||
# Links
|
|
||||||
"wikilinks": wikilinks,
|
|
||||||
"external_links": ext_links,
|
|
||||||
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
|
|
||||||
}
|
|
||||||
|
|
||||||
# Entferne Nones/Leeres, aber **text** bleibt (darf leer sein z. B. bei Bild-Only-Chunks)
|
|
||||||
cleaned = {}
|
|
||||||
for k, v in payload.items():
|
|
||||||
if v in (None, [], {}):
|
|
||||||
# immer behalten:
|
|
||||||
if k in ("text", "neighbors"):
|
|
||||||
cleaned[k] = v
|
|
||||||
else:
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
cleaned[k] = v
|
|
||||||
|
|
||||||
res.append(cleaned)
|
|
||||||
|
|
||||||
return res
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ----------------------------- Public API ---------------------------- #
|
||||||
# CLI zum schnellen Testen
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def _cli() -> None:
|
def make_chunk_payloads(
|
||||||
ap = argparse.ArgumentParser(description="Chunk-Payloads aus einer einzelnen Markdown-Datei erzeugen")
|
frontmatter: Dict[str, Any],
|
||||||
ap.add_argument("--from-file", required=True, help="Pfad zur Markdown-Datei")
|
rel_path: str,
|
||||||
ap.add_argument("--vault-root", default=None, help="Vault-Wurzel (zur Pfad-Relativierung)")
|
chunks: Iterable[Union[Dict[str, Any], Any]],
|
||||||
ap.add_argument("--summary", action="store_true", help="Nur kurze Übersicht je Chunk ausgeben")
|
note_text: Optional[str] = None,
|
||||||
args = ap.parse_args()
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Baut Payloads pro Chunk. Falls Fenster ohne Overlap geliefert werden,
|
||||||
|
erzeugen wir synthetische 'window'-Texte mit typgerechtem Overlap.
|
||||||
|
"""
|
||||||
|
note_id = str(frontmatter.get("id") or "").strip()
|
||||||
|
note_type = str(frontmatter.get("type", "")).lower()
|
||||||
|
note_title = frontmatter.get("title", None)
|
||||||
|
note_tags = frontmatter.get("tags", None)
|
||||||
|
rel_path = _normalize_rel_path(rel_path)
|
||||||
|
|
||||||
parsed = read_markdown(args.from_file)
|
# 1) Rohdaten sammeln (so wie geliefert)
|
||||||
fm = normalize_frontmatter(parsed.frontmatter)
|
chunks_list = list(chunks)
|
||||||
validate_required_frontmatter(fm)
|
raw_windows: List[str] = []
|
||||||
|
seqs: List[int] = []
|
||||||
|
ids_in: List[Optional[str]] = []
|
||||||
|
token_counts: List[Optional[int]] = []
|
||||||
|
section_titles: List[Optional[str]] = []
|
||||||
|
section_paths: List[Optional[str]] = []
|
||||||
|
any_explicit_window = False
|
||||||
|
|
||||||
# lazy import, um Zyklen zu vermeiden
|
for idx, c in enumerate(chunks_list):
|
||||||
try:
|
# Fensterquelle
|
||||||
from app.core.chunker import assemble_chunks
|
w = _get_attr_or_key(c, "window", None)
|
||||||
except Exception:
|
if isinstance(w, str) and w:
|
||||||
from .chunker import assemble_chunks # type: ignore
|
any_explicit_window = True
|
||||||
|
raw_windows.append(w)
|
||||||
|
else:
|
||||||
|
raw_windows.append(_as_window_text(c)) # 'text'|'content'|'raw' als Ersatz
|
||||||
|
# Ordnung
|
||||||
|
seqs.append(_to_int(_get_attr_or_key(c, "seq", _get_attr_or_key(c, "chunk_index", idx)), idx))
|
||||||
|
# IDs, Tokens, Sektionen
|
||||||
|
cid = _get_attr_or_key(c, "chunk_id", _get_attr_or_key(c, "id", None))
|
||||||
|
ids_in.append(str(cid) if isinstance(cid, str) and cid else None)
|
||||||
|
tc = _get_attr_or_key(c, "token_count", None)
|
||||||
|
token_counts.append(_to_int(tc, 0) if tc is not None else None)
|
||||||
|
section_titles.append(_get_attr_or_key(c, "section_title", None))
|
||||||
|
section_paths.append(_get_attr_or_key(c, "section_path", None))
|
||||||
|
|
||||||
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
|
# 2) Segmente & Overlaps bestimmen
|
||||||
rel = _normalize_rel_path(parsed.path, args.vault_root)
|
if any_explicit_window:
|
||||||
|
# Es existieren echte Fenster → dedupe, um Kernsegmente zu finden
|
||||||
pls = make_chunk_payloads(fm, rel or parsed.path, chunks)
|
segments, overlaps_left, recon = _dedupe_windows_to_segments(raw_windows)
|
||||||
|
windows_final = raw_windows[:] # bereits mit Overlap geliefert
|
||||||
if args.summary:
|
|
||||||
out = []
|
|
||||||
for p in pls:
|
|
||||||
out.append({
|
|
||||||
"id": p.get("id"),
|
|
||||||
"chunk_index": p.get("chunk_index"),
|
|
||||||
"token_est": p.get("token_est"),
|
|
||||||
"wikilinks": p.get("wikilinks"),
|
|
||||||
"ext_links": [e.get("href") for e in p.get("external_links", [])],
|
|
||||||
"prev": (p.get("neighbors") or {}).get("prev"),
|
|
||||||
"next": (p.get("neighbors") or {}).get("next"),
|
|
||||||
})
|
|
||||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
|
||||||
else:
|
else:
|
||||||
print(json.dumps(pls, ensure_ascii=False, indent=2))
|
# Keine echten Fenster → Segmente sind identisch zu "Fenstern" (bisher),
|
||||||
|
# wir erzeugen synthetische Fenster mit Overlap gemäß Typ
|
||||||
|
segments = [w or "" for w in raw_windows]
|
||||||
|
overlaps_left = []
|
||||||
|
windows_final = []
|
||||||
|
recon = ""
|
||||||
|
try:
|
||||||
|
overlap_low, overlap_high = tuple(_get_sizes(note_type).get("overlap", (40, 60)))
|
||||||
|
except Exception:
|
||||||
|
overlap_low, overlap_high = (40, 60)
|
||||||
|
overlap_target = int(overlap_low)
|
||||||
|
|
||||||
|
for i, seg in enumerate(segments):
|
||||||
|
if i == 0:
|
||||||
|
# erstes Fenster: kein linker Kontext
|
||||||
|
windows_final.append(seg)
|
||||||
|
overlaps_left.append(0)
|
||||||
|
recon += seg
|
||||||
|
else:
|
||||||
|
# synthetischer linker Kontext = Suffix des bisher rekonstruierten Texts
|
||||||
|
k = min(overlap_target, len(recon))
|
||||||
|
left_ctx = recon[-k:] if k > 0 else ""
|
||||||
|
windows_final.append(left_ctx + seg)
|
||||||
|
overlaps_left.append(k)
|
||||||
|
recon += seg # Rekonstruktion bleibt kerntreu
|
||||||
|
|
||||||
|
# 3) overlap_right bestimmen
|
||||||
|
overlaps_right: List[int] = []
|
||||||
|
for i in range(len(windows_final)):
|
||||||
|
if i + 1 < len(windows_final):
|
||||||
|
ov = _overlap_len_suffix_prefix(windows_final[i], windows_final[i + 1], max_probe=4096)
|
||||||
|
else:
|
||||||
|
ov = 0
|
||||||
|
overlaps_right.append(ov)
|
||||||
|
|
||||||
|
# 4) start/end-Offsets (exakt via note_text, sonst kumulativ)
|
||||||
|
starts: List[int] = [0] * len(segments)
|
||||||
|
ends: List[int] = [0] * len(segments)
|
||||||
|
pos = 0
|
||||||
|
if isinstance(note_text, str) and note_text:
|
||||||
|
search_pos = 0
|
||||||
|
for i, seg in enumerate(segments):
|
||||||
|
if not seg:
|
||||||
|
starts[i] = ends[i] = search_pos
|
||||||
|
continue
|
||||||
|
j = note_text.find(seg, search_pos)
|
||||||
|
if j >= 0:
|
||||||
|
starts[i] = j
|
||||||
|
ends[i] = j + len(seg)
|
||||||
|
search_pos = ends[i]
|
||||||
|
else:
|
||||||
|
# Fallback: kumulativ
|
||||||
|
starts[i] = pos
|
||||||
|
pos += len(seg)
|
||||||
|
ends[i] = pos
|
||||||
|
else:
|
||||||
|
for i, seg in enumerate(segments):
|
||||||
|
starts[i] = pos
|
||||||
|
pos += len(seg)
|
||||||
|
ends[i] = pos
|
||||||
|
|
||||||
|
# 5) Payload-Dicts
|
||||||
|
payloads: List[Dict[str, Any]] = []
|
||||||
|
for i, (win, seg) in enumerate(zip(windows_final, segments)):
|
||||||
|
chunk_id = ids_in[i] or f"{note_id}#{i+1}"
|
||||||
|
pl: Dict[str, Any] = {
|
||||||
|
"note_id": note_id,
|
||||||
|
"chunk_id": chunk_id,
|
||||||
|
"id": chunk_id, # Alias
|
||||||
|
"chunk_index": i,
|
||||||
|
"seq": seqs[i],
|
||||||
|
"path": rel_path,
|
||||||
|
"window": win,
|
||||||
|
"text": seg,
|
||||||
|
"start": starts[i],
|
||||||
|
"end": ends[i],
|
||||||
|
"overlap_left": overlaps_left[i],
|
||||||
|
"overlap_right": overlaps_right[i],
|
||||||
|
}
|
||||||
|
# optionale Metadaten
|
||||||
|
if note_type:
|
||||||
|
pl["type"] = note_type
|
||||||
|
if note_title is not None:
|
||||||
|
pl["title"] = note_title
|
||||||
|
if note_tags is not None:
|
||||||
|
pl["tags"] = note_tags
|
||||||
|
if token_counts[i] is not None:
|
||||||
|
pl["token_count"] = int(token_counts[i])
|
||||||
|
if section_titles[i] is not None:
|
||||||
|
pl["section_title"] = section_titles[i]
|
||||||
|
if section_paths[i] is not None:
|
||||||
|
sp = str(section_paths[i]).replace("\\", "/")
|
||||||
|
pl["section_path"] = sp if sp else "/"
|
||||||
|
payloads.append(pl)
|
||||||
|
|
||||||
|
return payloads
|
||||||
|
|
||||||
|
|
||||||
|
# __main__ Demo (optional)
|
||||||
if __name__ == "__main__": # pragma: no cover
|
if __name__ == "__main__": # pragma: no cover
|
||||||
_cli()
|
fm = {"id": "demo", "title": "Demo", "type": "concept"}
|
||||||
|
# Beispiel ohne echte Fenster → erzeugt synthetische Overlaps
|
||||||
|
chunks = [
|
||||||
|
{"id": "demo#1", "text": "Alpha Beta Gamma"},
|
||||||
|
{"id": "demo#2", "text": "Gamma Delta"},
|
||||||
|
{"id": "demo#3", "text": "Delta Epsilon Zeta"},
|
||||||
|
]
|
||||||
|
pls = make_chunk_payloads(fm, "path/demo.md", chunks, note_text="Alpha Beta Gamma Delta Epsilon Zeta")
|
||||||
|
from pprint import pprint
|
||||||
|
pprint(pls)
|
||||||
|
recon = "".join(p["text"] for p in pls)
|
||||||
|
print("RECON:", recon)
|
||||||
+114
-109
@@ -1,126 +1,131 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Name: app/core/derive_edges.py
|
Modul: app/core/derive_edges.py
|
||||||
Version: v1.1.0 (2025-09-05)
|
Version: 1.4.0
|
||||||
|
Datum: 2025-10-01
|
||||||
|
|
||||||
Kurzbeschreibung
|
Zweck
|
||||||
Leitet Edges aus Wikilinks ([[…]]) ab und löst Zielnoten robust auf.
|
-----
|
||||||
Erzeugt:
|
Robuste Kantenbildung für mindnet (Notes/Chunks):
|
||||||
- "references" (Note -> Note) mit seq="body", pro Match eine eigene Occurrence 'occ'
|
- belongs_to (chunk -> note)
|
||||||
- "backlink" (inverse zu "references", gleiche seq/occ)
|
- next / prev (chunk-Kette)
|
||||||
- "references_at" (Chunk -> Note) mit seq=<chunk_index> und eigener 'occ' je Match
|
- references (chunk-scope) aus Chunk.window/text
|
||||||
|
- optional references/backlink (note-scope)
|
||||||
|
|
||||||
Aufruf
|
Wichtig: Wikilinks werden mit der Parser-Funktion `extract_wikilinks` extrahiert,
|
||||||
from app.core.derive_edges import build_note_index, derive_wikilink_edges
|
damit Varianten wie [[id#anchor]] oder [[id|label]] korrekt auf 'id' reduziert werden.
|
||||||
|
|
||||||
Parameter / Felder
|
Erwartete Chunk-Payload-Felder:
|
||||||
- note_payload: {"note_id","title","path","fulltext": <body> , …}
|
{
|
||||||
- chunks_payloads: [{"chunk_id","text",…}, …]
|
"note_id": "...",
|
||||||
- note_index: build_note_index([...]) -> (by_id, by_slug, by_file_slug)
|
"chunk_id": "...", # Alias "id" ist zulässig
|
||||||
|
"id": "...",
|
||||||
Kompatibilität
|
"chunk_index": int,
|
||||||
- Rückwärtskompatible Payload-Felder, nur erweitert um 'seq' und 'occ'.
|
"seq": int,
|
||||||
|
"window": str,
|
||||||
Changelog
|
"text": str,
|
||||||
v1.1.0: Occurrence-Zählung ('occ') je Match; 'seq="body"' für references.
|
"path": "rel/path.md",
|
||||||
|
...
|
||||||
|
}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import re
|
|
||||||
import unicodedata
|
|
||||||
from typing import Dict, List, Tuple
|
|
||||||
|
|
||||||
# [[Ziel]], [[Ziel|Alias]], [[Ziel#Heading]], [[Ziel#Heading|Alias]]
|
from typing import Dict, List, Optional, Iterable
|
||||||
WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#([^\]|]+))?(?:\|([^\]]+))?\]\]")
|
|
||||||
|
|
||||||
def _slug(s: str) -> str:
|
# WICHTIG: benutze die Parser-Extraktion für saubere Wikilinks
|
||||||
s = s.strip()
|
from app.core.parser import extract_wikilinks
|
||||||
if s.endswith(".md"):
|
|
||||||
s = s[:-3]
|
|
||||||
s = unicodedata.normalize("NFKD", s)
|
|
||||||
s = "".join(ch for ch in s if not unicodedata.combining(ch))
|
|
||||||
s = s.replace("\\", "/")
|
|
||||||
s = s.split("/")[-1]
|
|
||||||
s = s.lower().replace(" ", "-")
|
|
||||||
s = re.sub(r"[^a-z0-9\-]+", "", s)
|
|
||||||
s = re.sub(r"-{2,}", "-", s).strip("-")
|
|
||||||
return s
|
|
||||||
|
|
||||||
def build_note_index(notes_payloads: List[dict]) -> Tuple[Dict[str, dict], Dict[str, dict], Dict[str, dict]]:
|
def _get(d: dict, *keys, default=None):
|
||||||
by_id: Dict[str, dict] = {}
|
for k in keys:
|
||||||
by_slug: Dict[str, dict] = {}
|
if k in d and d[k] is not None:
|
||||||
by_file_slug: Dict[str, dict] = {}
|
return d[k]
|
||||||
for n in notes_payloads:
|
return default
|
||||||
nid = n.get("note_id") or n.get("id")
|
|
||||||
if not nid:
|
|
||||||
continue
|
|
||||||
by_id[nid] = n
|
|
||||||
title = n.get("title", "")
|
|
||||||
path = n.get("path", "")
|
|
||||||
file_slug = _slug(path.split("/")[-1]) if path else ""
|
|
||||||
if title:
|
|
||||||
by_slug[_slug(title)] = n
|
|
||||||
if file_slug:
|
|
||||||
by_file_slug[file_slug] = n
|
|
||||||
return by_id, by_slug, by_file_slug
|
|
||||||
|
|
||||||
def resolve_target(note_like: str, idx: Tuple[Dict[str,dict],Dict[str,dict],Dict[str,dict]]):
|
def _chunk_text_for_refs(chunk: dict) -> str:
|
||||||
by_id, by_slug, by_file_slug = idx
|
# bevorzugt 'window' → dann 'text' → 'content' → 'raw'
|
||||||
key = note_like.strip()
|
return (
|
||||||
if key in by_id:
|
_get(chunk, "window")
|
||||||
return by_id[key]["note_id"], "by_id"
|
or _get(chunk, "text")
|
||||||
s = _slug(key)
|
or _get(chunk, "content")
|
||||||
if s in by_slug:
|
or _get(chunk, "raw")
|
||||||
return by_slug[s]["note_id"], "by_slug"
|
or ""
|
||||||
if s in by_file_slug:
|
)
|
||||||
return by_file_slug[s]["note_id"], "by_file_slug"
|
|
||||||
return None, "unresolved"
|
|
||||||
|
|
||||||
def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_index) -> List[dict]:
|
def _dedupe(seq: Iterable[str]) -> List[str]:
|
||||||
|
seen = set()
|
||||||
|
out: List[str] = []
|
||||||
|
for s in seq:
|
||||||
|
if s not in seen:
|
||||||
|
seen.add(s)
|
||||||
|
out.append(s)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
|
||||||
|
pl = {
|
||||||
|
"kind": kind,
|
||||||
|
"scope": scope, # "chunk" | "note"
|
||||||
|
"source_id": source_id,
|
||||||
|
"target_id": target_id,
|
||||||
|
"note_id": note_id, # Träger/Quelle der Kante (aktuelle Note)
|
||||||
|
}
|
||||||
|
if extra:
|
||||||
|
pl.update(extra)
|
||||||
|
return pl
|
||||||
|
|
||||||
|
def build_edges_for_note(
|
||||||
|
note_id: str,
|
||||||
|
chunks: List[dict],
|
||||||
|
note_level_references: Optional[List[str]] = None,
|
||||||
|
include_note_scope_refs: bool = False,
|
||||||
|
) -> List[dict]:
|
||||||
|
"""
|
||||||
|
Erzeugt Kanten für eine Note.
|
||||||
|
|
||||||
|
- belongs_to: für jeden Chunk (chunk -> note)
|
||||||
|
- next / prev: zwischen aufeinanderfolgenden Chunks
|
||||||
|
- references: pro Chunk aus window/text
|
||||||
|
- optional note-scope references/backlinks: dedupliziert über alle Chunk-Funde + note_level_references
|
||||||
|
"""
|
||||||
edges: List[dict] = []
|
edges: List[dict] = []
|
||||||
source_note_id = note_payload["note_id"]
|
|
||||||
|
|
||||||
def _make_edge(kind: str, src: str, tgt: str, seq=None, occ=None, extra: dict|None=None):
|
# belongs_to
|
||||||
e = {"edge_id": None, "kind": kind, "source_id": src, "target_id": tgt}
|
for ch in chunks:
|
||||||
if seq is not None:
|
cid = _get(ch, "chunk_id", "id")
|
||||||
e["seq"] = seq
|
if not cid:
|
||||||
if occ is not None:
|
|
||||||
e["occ"] = occ
|
|
||||||
if extra:
|
|
||||||
e.update(extra)
|
|
||||||
return e
|
|
||||||
|
|
||||||
# Volltext (Note-Ebene)
|
|
||||||
fulltext = note_payload.get("fulltext") or note_payload.get("body") or ""
|
|
||||||
if fulltext:
|
|
||||||
for k, m in enumerate(WIKILINK_RE.finditer(fulltext), start=1):
|
|
||||||
raw_target, heading, alias = m.groups()
|
|
||||||
target_id, how = resolve_target(raw_target, note_index)
|
|
||||||
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
|
|
||||||
if target_id:
|
|
||||||
edges.append(_make_edge("references", source_note_id, target_id, seq="body", occ=k, extra=extra))
|
|
||||||
edges.append(_make_edge("backlink", target_id, source_note_id, seq="body", occ=k, extra=extra))
|
|
||||||
else:
|
|
||||||
extra["status"] = "unresolved"
|
|
||||||
extra["target_label"] = raw_target
|
|
||||||
edges.append(_make_edge("references", source_note_id, raw_target, seq="body", occ=k, extra=extra))
|
|
||||||
|
|
||||||
# Chunks (Chunk-Ebene)
|
|
||||||
for i, ch in enumerate(chunks_payloads, start=1):
|
|
||||||
txt = ch.get("text") or ch.get("content") or ""
|
|
||||||
if not txt:
|
|
||||||
continue
|
continue
|
||||||
occ = 0
|
edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {"chunk_id": cid}))
|
||||||
for m in WIKILINK_RE.finditer(txt):
|
|
||||||
occ += 1
|
# next/prev
|
||||||
raw_target, heading, alias = m.groups()
|
for i in range(len(chunks) - 1):
|
||||||
target_id, how = resolve_target(raw_target, note_index)
|
a, b = chunks[i], chunks[i + 1]
|
||||||
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
|
a_id = _get(a, "chunk_id", "id")
|
||||||
if target_id:
|
b_id = _get(b, "chunk_id", "id")
|
||||||
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, occ=occ, extra=extra))
|
if not a_id or not b_id:
|
||||||
else:
|
continue
|
||||||
extra["status"] = "unresolved"
|
edges.append(_edge("next", "chunk", a_id, b_id, note_id, {"chunk_id": a_id}))
|
||||||
extra["target_label"] = raw_target
|
edges.append(_edge("prev", "chunk", b_id, a_id, note_id, {"chunk_id": b_id}))
|
||||||
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, occ=occ, extra=extra))
|
|
||||||
|
# references (chunk-scope) – Links aus window bevorzugen (Overlap-fest)
|
||||||
|
refs_all: List[str] = []
|
||||||
|
for ch in chunks:
|
||||||
|
cid = _get(ch, "chunk_id", "id")
|
||||||
|
if not cid:
|
||||||
|
continue
|
||||||
|
txt = _chunk_text_for_refs(ch)
|
||||||
|
refs = extract_wikilinks(txt) # <— Parser-Logik, kompatibel zu deinem System
|
||||||
|
for r in refs:
|
||||||
|
edges.append(_edge("references", "chunk", cid, r, note_id, {"chunk_id": cid, "ref_text": r}))
|
||||||
|
refs_all.extend(refs)
|
||||||
|
|
||||||
|
# optional: note-scope references/backlinks
|
||||||
|
if include_note_scope_refs:
|
||||||
|
refs_note = refs_all[:]
|
||||||
|
if note_level_references:
|
||||||
|
refs_note.extend([r for r in note_level_references if isinstance(r, str) and r])
|
||||||
|
refs_note = _dedupe(refs_note)
|
||||||
|
for r in refs_note:
|
||||||
|
edges.append(_edge("references", "note", note_id, r, note_id))
|
||||||
|
edges.append(_edge("backlink", "note", r, note_id, note_id))
|
||||||
|
|
||||||
return edges
|
return edges
|
||||||
+244
-45
@@ -1,67 +1,266 @@
|
|||||||
from __future__ import annotations
|
#!/usr/bin/env python3
|
||||||
from dataclasses import dataclass
|
# -*- coding: utf-8 -*-
|
||||||
import re
|
"""
|
||||||
import os
|
Modul: app/core/parser.py
|
||||||
import unicodedata
|
Version: 1.7.1 (fault-tolerant, API-kompatibel)
|
||||||
import yaml
|
Datum: 2025-10-01
|
||||||
from typing import Tuple, Dict
|
|
||||||
|
|
||||||
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?", re.DOTALL) # YAML-Frontmatter am Anfang
|
Zweck
|
||||||
|
-----
|
||||||
|
Fehlertolerantes Einlesen von Markdown-Dateien mit YAML-Frontmatter.
|
||||||
|
Kompatibel zur bisherigen Parser-API, aber robust gegenüber Nicht-UTF-8-Dateien:
|
||||||
|
- Versucht nacheinander: utf-8 → utf-8-sig → cp1252 → latin-1.
|
||||||
|
- Bei Fallback wird ein JSON-Warnhinweis auf stdout ausgegeben; der Import bricht NICHT ab.
|
||||||
|
- YAML-Frontmatter wird mit '---' am Anfang und '---' als Abschluss erkannt.
|
||||||
|
- extract_wikilinks() normalisiert [[id#anchor|label]] → 'id'.
|
||||||
|
|
||||||
|
Öffentliche API (kompatibel):
|
||||||
|
- class ParsedNote(frontmatter: dict, body: str, path: str)
|
||||||
|
- read_markdown(path) -> ParsedNote | None
|
||||||
|
- normalize_frontmatter(fm) -> dict
|
||||||
|
- validate_required_frontmatter(fm, required: tuple[str,...]=("id","title")) -> None
|
||||||
|
- extract_wikilinks(text) -> list[str]
|
||||||
|
- FRONTMATTER_RE (Kompatibilitäts-Konstante; Regex für '---'-Zeilen)
|
||||||
|
|
||||||
|
Beispiele
|
||||||
|
---------
|
||||||
|
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||||
|
parsed = read_markdown("./vault/30_projects/project-demo.md")
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
|
validate_required_frontmatter(fm)
|
||||||
|
body = parsed.body
|
||||||
|
|
||||||
|
from app.core.parser import extract_wikilinks
|
||||||
|
links = extract_wikilinks(body)
|
||||||
|
|
||||||
|
Abhängigkeiten
|
||||||
|
--------------
|
||||||
|
- PyYAML (yaml)
|
||||||
|
|
||||||
|
Lizenz: MIT (projektintern)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any, Dict, Optional, Tuple, Iterable, List
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # PyYAML
|
||||||
|
except Exception as e: # pragma: no cover
|
||||||
|
yaml = None # Fehler wird zur Laufzeit geworfen, falls wirklich benötigt
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# Datamodell
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ParsedNote:
|
class ParsedNote:
|
||||||
frontmatter: Dict
|
frontmatter: Dict[str, Any]
|
||||||
body: str
|
body: str
|
||||||
path: str
|
path: str
|
||||||
|
|
||||||
def _strip_bom(text: str) -> str:
|
|
||||||
return text.lstrip("\ufeff")
|
|
||||||
|
|
||||||
def _normalize_text(t: str) -> str:
|
# ---------------------------------------------------------------------
|
||||||
# Unicode-NFKC + vereinheitlichte Zeilenenden + Trim
|
# Frontmatter-Erkennung
|
||||||
t = unicodedata.normalize("NFKC", t)
|
# ---------------------------------------------------------------------
|
||||||
t = t.replace("\r\n", "\n").replace("\r", "\n")
|
|
||||||
return t
|
|
||||||
|
|
||||||
def read_markdown(path: str) -> ParsedNote:
|
# Öffentliche Kompatibilitäts-Konstante: frühere Skripte importieren FRONTMATTER_RE
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
FRONTMATTER_RE = re.compile(r"^\s*---\s*$") # <— public
|
||||||
raw = _strip_bom(f.read())
|
# Zusätzlich interner Alias (falls jemand ihn referenziert)
|
||||||
raw = _normalize_text(raw)
|
FRONTMATTER_END = FRONTMATTER_RE # <— public alias
|
||||||
|
|
||||||
m = FRONTMATTER_RE.match(raw)
|
# interne Namen bleiben bestehen
|
||||||
front, body = {}, raw
|
_FRONTMATTER_HEAD = FRONTMATTER_RE
|
||||||
if m:
|
_FRONTMATTER_END = FRONTMATTER_RE
|
||||||
yaml_block = m.group(1)
|
|
||||||
body = raw[m.end():]
|
|
||||||
|
def _split_frontmatter(text: str) -> Tuple[Dict[str, Any], str]:
|
||||||
|
"""
|
||||||
|
Zerlegt Text in (frontmatter: dict, body: str).
|
||||||
|
Erkennt Frontmatter nur, wenn die erste Zeile '---' ist und später ein zweites '---' folgt.
|
||||||
|
YAML-Fehler im Frontmatter führen NICHT zum Abbruch: es wird dann ein leeres dict benutzt.
|
||||||
|
"""
|
||||||
|
lines = text.splitlines(True) # keep line endings
|
||||||
|
if not lines:
|
||||||
|
return {}, ""
|
||||||
|
|
||||||
|
if not _FRONTMATTER_HEAD.match(lines[0]):
|
||||||
|
# kein Frontmatter-Header → gesamter Text ist Body
|
||||||
|
return {}, text
|
||||||
|
|
||||||
|
end_idx = None
|
||||||
|
# Suche nach nächstem '---' (max. 2000 Zeilen als Sicherheitslimit)
|
||||||
|
for i in range(1, min(len(lines), 2000)):
|
||||||
|
if _FRONTMATTER_END.match(lines[i]):
|
||||||
|
end_idx = i
|
||||||
|
break
|
||||||
|
|
||||||
|
if end_idx is None:
|
||||||
|
# unvollständiger Frontmatter-Block → behandle alles als Body
|
||||||
|
return {}, text
|
||||||
|
|
||||||
|
fm_raw = "".join(lines[1:end_idx])
|
||||||
|
body = "".join(lines[end_idx + 1:])
|
||||||
|
|
||||||
|
data: Dict[str, Any] = {}
|
||||||
|
if yaml is None:
|
||||||
|
raise RuntimeError("PyYAML ist nicht installiert (pip install pyyaml).")
|
||||||
|
|
||||||
|
try:
|
||||||
|
loaded = yaml.safe_load(fm_raw) or {}
|
||||||
|
if isinstance(loaded, dict):
|
||||||
|
data = loaded
|
||||||
|
else:
|
||||||
|
data = {}
|
||||||
|
except Exception as e:
|
||||||
|
# YAML-Fehler nicht fatal machen
|
||||||
|
print(json.dumps({"warn": "frontmatter_yaml_parse_failed", "error": str(e)}))
|
||||||
|
data = {}
|
||||||
|
|
||||||
|
# optionales kosmetisches Trim: eine führende Leerzeile im Body entfernen
|
||||||
|
if body.startswith("\n"):
|
||||||
|
body = body[1:]
|
||||||
|
|
||||||
|
return data, body
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# Robustes Lesen mit Encoding-Fallback
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
_FALLBACK_ENCODINGS: Tuple[str, ...] = ("utf-8", "utf-8-sig", "cp1252", "latin-1")
|
||||||
|
|
||||||
|
|
||||||
|
def _read_text_with_fallback(path: str) -> Tuple[str, str, bool]:
|
||||||
|
"""
|
||||||
|
Liest Datei mit mehreren Decodierungsversuchen.
|
||||||
|
Rückgabe: (text, used_encoding, had_fallback)
|
||||||
|
- had_fallback=True, falls NICHT 'utf-8' verwendet wurde (oder 'utf-8-sig').
|
||||||
|
"""
|
||||||
|
last_err: Optional[str] = None
|
||||||
|
for enc in _FALLBACK_ENCODINGS:
|
||||||
try:
|
try:
|
||||||
front = yaml.safe_load(yaml_block) or {}
|
with io.open(path, "r", encoding=enc, errors="strict") as f:
|
||||||
if not isinstance(front, dict):
|
text = f.read()
|
||||||
raise ValueError("Frontmatter must be a mapping")
|
# 'utf-8-sig' zählt hier als Fallback (weil BOM), aber ist unproblematisch
|
||||||
except yaml.YAMLError as e:
|
return text, enc, (enc != "utf-8")
|
||||||
raise ValueError(f"Invalid YAML frontmatter in {path}: {e}") from e
|
except UnicodeDecodeError as e:
|
||||||
|
last_err = f"{type(e).__name__}: {e}"
|
||||||
|
continue
|
||||||
|
|
||||||
return ParsedNote(frontmatter=front, body=body, path=path)
|
# Letzter, extrem defensiver Fallback: Bytes → UTF-8 mit REPLACE (keine Exception)
|
||||||
|
with open(path, "rb") as fb:
|
||||||
|
raw = fb.read()
|
||||||
|
text = raw.decode("utf-8", errors="replace")
|
||||||
|
print(json.dumps({
|
||||||
|
"path": path,
|
||||||
|
"warn": "encoding_fallback_exhausted",
|
||||||
|
"info": last_err or "unknown"
|
||||||
|
}, ensure_ascii=False))
|
||||||
|
return text, "utf-8(replace)", True
|
||||||
|
|
||||||
RE_WIKILINK = re.compile(r"\[\[([^\]\|#]+)(?:#[^\]]+)?(?:\|[^\]]+)?\]\]") # [[id]] | [[id#anchor]] | [[id|label]]
|
|
||||||
|
|
||||||
def extract_wikilinks(text: str) -> list[str]:
|
# ---------------------------------------------------------------------
|
||||||
return list({m.group(1).strip() for m in RE_WIKILINK.finditer(text)})
|
# Öffentliche API
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
def validate_required_frontmatter(fm: dict, required=("title","id","type","status","created")):
|
def read_markdown(path: str) -> Optional[ParsedNote]:
|
||||||
missing = [k for k in required if k not in fm or fm[k] in (None, "")]
|
"""
|
||||||
|
Liest eine Markdown-Datei fehlertolerant:
|
||||||
|
- Erlaubt verschiedene Encodings (UTF-8 bevorzugt, cp1252/latin-1 als Fallback).
|
||||||
|
- Schlägt NICHT mit UnicodeDecodeError fehl.
|
||||||
|
- Gibt ParsedNote(frontmatter, body, path) zurück oder None, falls die Datei nicht existiert.
|
||||||
|
|
||||||
|
Bei Decoding-Fallback wird ein JSON-Warnhinweis geloggt:
|
||||||
|
{"path": "...", "warn": "encoding_fallback_used", "used": "cp1252"}
|
||||||
|
"""
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return None
|
||||||
|
|
||||||
|
text, enc, had_fb = _read_text_with_fallback(path)
|
||||||
|
if had_fb:
|
||||||
|
print(json.dumps({"path": path, "warn": "encoding_fallback_used", "used": enc}, ensure_ascii=False))
|
||||||
|
|
||||||
|
fm, body = _split_frontmatter(text)
|
||||||
|
return ParsedNote(frontmatter=fm or {}, body=body or "", path=path)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_required_frontmatter(fm: Dict[str, Any],
|
||||||
|
required: Tuple[str, ...] = ("id", "title")) -> None:
|
||||||
|
"""
|
||||||
|
Prüft, ob alle Pflichtfelder vorhanden sind.
|
||||||
|
Default-kompatibel: ('id', 'title'), kann aber vom Aufrufer erweitert werden, z. B.:
|
||||||
|
validate_required_frontmatter(fm, required=("id","title","type","status","created"))
|
||||||
|
|
||||||
|
Hebt ValueError, falls Felder fehlen oder leer sind.
|
||||||
|
"""
|
||||||
|
if fm is None:
|
||||||
|
fm = {}
|
||||||
|
missing = []
|
||||||
|
for k in required:
|
||||||
|
v = fm.get(k)
|
||||||
|
if v is None:
|
||||||
|
missing.append(k)
|
||||||
|
elif isinstance(v, str) and not v.strip():
|
||||||
|
missing.append(k)
|
||||||
if missing:
|
if missing:
|
||||||
raise ValueError(f"Missing required frontmatter fields: {', '.join(missing)}")
|
raise ValueError(f"Missing required frontmatter fields: {', '.join(missing)}")
|
||||||
|
|
||||||
# einfache Plausibilitäten
|
# Plausibilitäten: 'tags' sollte eine Liste sein, wenn vorhanden
|
||||||
if not isinstance(fm.get("tags", []), (list, tuple)):
|
if "tags" in fm and fm["tags"] not in (None, "") and not isinstance(fm["tags"], (list, tuple)):
|
||||||
if "tags" in fm and fm["tags"] not in (None, ""):
|
raise ValueError("frontmatter 'tags' must be a list of strings")
|
||||||
raise ValueError("tags must be a list of strings")
|
|
||||||
|
|
||||||
def normalize_frontmatter(fm: dict) -> dict:
|
|
||||||
# kleinere Normalisierungen ohne die Semantik zu verändern
|
def normalize_frontmatter(fm: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
out = dict(fm)
|
"""
|
||||||
if "tags" in out and isinstance(out["tags"], list):
|
Sanfte Normalisierung ohne Semantikänderung:
|
||||||
out["tags"] = [str(t).strip() for t in out["tags"]]
|
- 'tags' → Liste von Strings (Trim)
|
||||||
|
- 'embedding_exclude' → bool
|
||||||
|
- andere Felder unverändert
|
||||||
|
"""
|
||||||
|
out = dict(fm or {})
|
||||||
|
if "tags" in out:
|
||||||
|
if isinstance(out["tags"], str):
|
||||||
|
out["tags"] = [out["tags"].strip()] if out["tags"].strip() else []
|
||||||
|
elif isinstance(out["tags"], list):
|
||||||
|
out["tags"] = [str(t).strip() for t in out["tags"] if t is not None]
|
||||||
|
else:
|
||||||
|
out["tags"] = [str(out["tags"]).strip()] if out["tags"] not in (None, "") else []
|
||||||
if "embedding_exclude" in out:
|
if "embedding_exclude" in out:
|
||||||
out["embedding_exclude"] = bool(out["embedding_exclude"])
|
out["embedding_exclude"] = bool(out["embedding_exclude"])
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------ Wikilinks ---------------------------- #
|
||||||
|
|
||||||
|
# Basismuster für [[...]]; die Normalisierung (id vor '#', vor '|') macht extract_wikilinks
|
||||||
|
_WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||||
|
|
||||||
|
|
||||||
|
def extract_wikilinks(text: str) -> List[str]:
|
||||||
|
"""
|
||||||
|
Extrahiert Wikilinks wie [[id]], [[id#anchor]], [[id|label]], [[id#anchor|label]].
|
||||||
|
Rückgabe sind NUR die Ziel-IDs (ohne Anchor/Label), führend/folgend getrimmt.
|
||||||
|
Keine aggressive Slug-Normalisierung (die kann später im Resolver erfolgen).
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return []
|
||||||
|
out: List[str] = []
|
||||||
|
for m in _WIKILINK_RE.finditer(text):
|
||||||
|
raw = (m.group(1) or "").strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
# Split an Pipe (Label) → links vor '|'
|
||||||
|
if "|" in raw:
|
||||||
|
raw = raw.split("|", 1)[0].strip()
|
||||||
|
# Split an Anchor
|
||||||
|
if "#" in raw:
|
||||||
|
raw = raw.split("#", 1)[0].strip()
|
||||||
|
if raw:
|
||||||
|
out.append(raw)
|
||||||
|
return out
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Modul: app/graph/service.py
|
||||||
|
Version: 0.1.0
|
||||||
|
Datum: 2025-09-10
|
||||||
|
|
||||||
|
Zweck
|
||||||
|
-----
|
||||||
|
Leichtgewichtiger Graph-Layer über Qdrant:
|
||||||
|
- get_note(note_id)
|
||||||
|
- get_chunks(note_id)
|
||||||
|
- neighbors(source_id, kinds=[...], scope=['note','chunk'], depth=1)
|
||||||
|
- walk_bfs(source_id, kinds, max_depth)
|
||||||
|
- context_for_note(note_id, max_neighbors): heuristische Kontextsammlung
|
||||||
|
|
||||||
|
Hinweise
|
||||||
|
--------
|
||||||
|
- Nutzt die bestehenden Collections <prefix>_notes/_chunks/_edges.
|
||||||
|
- Edges werden über Payload-Felder (`kind`, `source_id`, `target_id`) abgefragt.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List, Dict, Any, Optional, Iterable, Set, Tuple
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def _cols(prefix: str):
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
class GraphService:
|
||||||
|
def __init__(self, cfg: Optional[QdrantConfig] = None, prefix: Optional[str] = None):
|
||||||
|
self.cfg = cfg or QdrantConfig.from_env()
|
||||||
|
if prefix:
|
||||||
|
self.cfg.prefix = prefix
|
||||||
|
self.client = get_client(self.cfg)
|
||||||
|
self.notes_col, self.chunks_col, self.edges_col = _cols(self.cfg.prefix)
|
||||||
|
|
||||||
|
# ------------------------ fetch helpers ------------------------
|
||||||
|
def _scroll(self, col: str, flt: Optional[rest.Filter] = None, limit: int = 256):
|
||||||
|
out = []
|
||||||
|
nextp = None
|
||||||
|
while True:
|
||||||
|
pts, nextp = self.client.scroll(
|
||||||
|
collection_name=col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=limit,
|
||||||
|
offset=nextp,
|
||||||
|
scroll_filter=flt,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
out.extend(pts)
|
||||||
|
if nextp is None:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
# ------------------------ public API ---------------------------
|
||||||
|
def get_note(self, note_id: str) -> Optional[Dict[str, Any]]:
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
pts, _ = self.client.scroll(self.notes_col, with_payload=True, with_vectors=False, limit=1, scroll_filter=f)
|
||||||
|
return (pts[0].payload or None) if pts else None
|
||||||
|
|
||||||
|
def get_chunks(self, note_id: str) -> List[Dict[str, Any]]:
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
pts = self._scroll(self.chunks_col, f)
|
||||||
|
# Sortierung analog Export
|
||||||
|
def key(pl):
|
||||||
|
p = pl.payload or {}
|
||||||
|
s = p.get("seq") or 0
|
||||||
|
ci = p.get("chunk_index") or 0
|
||||||
|
n = 0
|
||||||
|
cid = p.get("chunk_id") or ""
|
||||||
|
if isinstance(cid, str) and "#" in cid:
|
||||||
|
try:
|
||||||
|
n = int(cid.rsplit("#", 1)[-1])
|
||||||
|
except Exception:
|
||||||
|
n = 0
|
||||||
|
return (int(s), int(ci), n)
|
||||||
|
pts_sorted = sorted(pts, key=key)
|
||||||
|
return [p.payload or {} for p in pts_sorted]
|
||||||
|
|
||||||
|
def neighbors(self, source_id: str, kinds: Optional[Iterable[str]] = None,
|
||||||
|
scope: Optional[Iterable[str]] = None, depth: int = 1) -> Dict[str, List[Dict[str, Any]]]:
|
||||||
|
"""
|
||||||
|
Liefert eingehende & ausgehende Nachbarn (nur nach kind gefiltert).
|
||||||
|
depth==1: direkte Kanten.
|
||||||
|
"""
|
||||||
|
kinds = list(kinds) if kinds else None
|
||||||
|
must = [rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_id))]
|
||||||
|
if kinds:
|
||||||
|
must.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
|
||||||
|
f = rest.Filter(must=must)
|
||||||
|
edges = self._scroll(self.edges_col, f)
|
||||||
|
out = {"out": [], "in": []}
|
||||||
|
for e in edges:
|
||||||
|
out["out"].append(e.payload or {})
|
||||||
|
# Inverse Richtung (eingehend)
|
||||||
|
must_in = [rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_id))]
|
||||||
|
if kinds:
|
||||||
|
must_in.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
|
||||||
|
f_in = rest.Filter(must=must_in)
|
||||||
|
edges_in = self._scroll(self.edges_col, f_in)
|
||||||
|
for e in edges_in:
|
||||||
|
out["in"].append(e.payload or {})
|
||||||
|
return out
|
||||||
|
|
||||||
|
def walk_bfs(self, source_id: str, kinds: Iterable[str], max_depth: int = 2) -> Set[str]:
|
||||||
|
visited: Set[str] = {source_id}
|
||||||
|
frontier: Set[str] = {source_id}
|
||||||
|
kinds = list(kinds)
|
||||||
|
for _ in range(max_depth):
|
||||||
|
nxt: Set[str] = set()
|
||||||
|
for s in frontier:
|
||||||
|
neigh = self.neighbors(s, kinds=kinds)
|
||||||
|
for e in neigh["out"]:
|
||||||
|
t = e.get("target_id")
|
||||||
|
if isinstance(t, str) and t not in visited:
|
||||||
|
visited.add(t)
|
||||||
|
nxt.add(t)
|
||||||
|
frontier = nxt
|
||||||
|
if not frontier:
|
||||||
|
break
|
||||||
|
return visited
|
||||||
|
|
||||||
|
def context_for_note(self, note_id: str, kinds: Iterable[str] = ("references","backlink"), max_neighbors: int = 12) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Heuristischer Kontext: eigene Chunks + Nachbarn nach Kantenarten, dedupliziert.
|
||||||
|
"""
|
||||||
|
note = self.get_note(note_id) or {}
|
||||||
|
chunks = self.get_chunks(note_id)
|
||||||
|
neigh = self.neighbors(note_id, kinds=list(kinds))
|
||||||
|
targets = []
|
||||||
|
for e in neigh["out"]:
|
||||||
|
t = e.get("target_id")
|
||||||
|
if isinstance(t, str):
|
||||||
|
targets.append(t)
|
||||||
|
for e in neigh["in"]:
|
||||||
|
s = e.get("source_id")
|
||||||
|
if isinstance(s, str):
|
||||||
|
targets.append(s)
|
||||||
|
# de-dupe
|
||||||
|
seen = set()
|
||||||
|
uniq = []
|
||||||
|
for t in targets:
|
||||||
|
if t not in seen:
|
||||||
|
seen.add(t)
|
||||||
|
uniq.append(t)
|
||||||
|
uniq = uniq[:max_neighbors]
|
||||||
|
neighbor_notes = [self.get_note(t) for t in uniq]
|
||||||
|
return {
|
||||||
|
"note": note,
|
||||||
|
"chunks": chunks,
|
||||||
|
"neighbors": [n for n in neighbor_notes if n],
|
||||||
|
"edges_out": neigh["out"],
|
||||||
|
"edges_in": neigh["in"],
|
||||||
|
}
|
||||||
|
|
||||||
|
# Optional: Mini-CLI
|
||||||
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
import argparse, json
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV)")
|
||||||
|
ap.add_argument("--note-id", required=True)
|
||||||
|
ap.add_argument("--neighbors", action="store_true", help="Nur Nachbarn anzeigen")
|
||||||
|
args = ap.parse_args()
|
||||||
|
svc = GraphService(prefix=args.prefix)
|
||||||
|
if args.neighbors:
|
||||||
|
out = svc.neighbors(args.note_id, kinds=["references","backlink","prev","next","belongs_to"])
|
||||||
|
else:
|
||||||
|
out = svc.context_for_note(args.note_id)
|
||||||
|
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||||
+234
-75
@@ -1,19 +1,42 @@
|
|||||||
# 📘 Mindnet Skript-Handbuch
|
# 📘 Mindnet Skript-Handbuch (aktualisiert)
|
||||||
|
|
||||||
Dieses Handbuch dokumentiert die implementierten Skripte für das Projekt **mindnet Wissensnetzwerk**.
|
Dieses Handbuch dokumentiert die Skripte und Tests für das Projekt **mindnet Wissensnetzwerk**.
|
||||||
Es beschreibt Zweck, Parameter und typische Aufrufe. Alle Beispiele sind geprüft und lauffähig.
|
Es beschreibt Zweck, Parameter, typische Aufrufe sowie aktuelle Verhaltensweisen (Hash-Modi, Parser-Robustheit, `window`/`text`-Trennung in Chunks, Export-Fixes).
|
||||||
|
Alle Beispiele sind so formatiert, dass sie direkt kopierbar sind. **Code/CLI-Beispiele sind eingerückt**, damit die Markdown-Darstellung nicht abbricht.
|
||||||
Alle Skripte liegen in `~/mindnet/scripts/`.
|
|
||||||
Ausführung erfolgt **im aktivierten venv**:
|
|
||||||
|
|
||||||
cd ~/mindnet
|
|
||||||
source .venv/bin/activate
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 1. Vault anlegen (für Tests)
|
## 0) Umgebung & Konfiguration
|
||||||
|
|
||||||
### `make_test_vault.py`
|
**Wichtige ENV-Variablen**
|
||||||
|
|
||||||
|
- `QDRANT_URL` – z. B. `http://localhost:6333`
|
||||||
|
- `QDRANT_API_KEY` – API-Key (falls aktiviert)
|
||||||
|
- `COLLECTION_PREFIX` – Prefix der drei Collections, Standard: `mindnet` → `mindnet_notes`, `mindnet_chunks`, `mindnet_edges`
|
||||||
|
- `VECTOR_DIM` – Embedding-Dimension (Chunks), Standard: `384`
|
||||||
|
|
||||||
|
**Hash-Vergleich & Änderungserkennung**
|
||||||
|
|
||||||
|
- `MINDNET_HASH_COMPARE` – steuert, **welcher Anteil** für „changed“ verglichen wird
|
||||||
|
Werte: `Body` (Standard), `Frontmatter`, `Full`
|
||||||
|
- `MINDNET_HASH_SOURCE` – Quelle: `parsed` (Standard), `raw`, `file`
|
||||||
|
- `MINDNET_HASH_NORMALIZE` – Normalisierung: `canonical` (Standard) oder `none`
|
||||||
|
|
||||||
|
**Beispiel (nur Body berücksichtigen, kanonisch, geparst):**
|
||||||
|
|
||||||
|
export MINDNET_HASH_COMPARE=Body
|
||||||
|
export MINDNET_HASH_SOURCE=parsed
|
||||||
|
export MINDNET_HASH_NORMALIZE=canonical
|
||||||
|
source .venv/bin/activate
|
||||||
|
|
||||||
|
**Baseline-Signaturen (mehrere Modi):**
|
||||||
|
Der Importer kann **Signaturen aller Modi** (Body/Frontmatter/Full) vorhalten, damit spätere Moduswechsel deterministisch sind (keine „Mass-Changes“). Siehe `--baseline-modes` unter **Importer**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1) Vault anlegen (für Tests)
|
||||||
|
|
||||||
|
### `scripts/make_test_vault.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Erstellt einen kleinen, nachvollziehbaren Test-Vault mit Markdown-Dateien.
|
Erstellt einen kleinen, nachvollziehbaren Test-Vault mit Markdown-Dateien.
|
||||||
@@ -22,17 +45,17 @@ Szenarien: offene Links, nachträgliche Anlage fehlender Notes, geänderte Chunk
|
|||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--out PATH` : Zielverzeichnis (Default: `./test_vault`)
|
- `--out PATH` : Zielverzeichnis (Default: `./test_vault`)
|
||||||
- `--force` : Existierenden Ordner löschen und neu anlegen
|
- `--force` : Existierenden Ordner löschen und neu anlegen
|
||||||
- `--with-missing` : Optional – auch die `missing-note` gleich anlegen
|
- `--with-missing` : Optional – auch die `missing-note` gleich anlegen
|
||||||
|
|
||||||
**Hinweise für Tests**
|
**Hinweise für Tests**
|
||||||
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
|
1. Erster Import (dry-run oder `--apply`): Es gibt einen „leeren“ Link `[[missing-note]]`.
|
||||||
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
|
2. Lege danach eine Datei für „missing-note“ an (mit gleicher `id` im YAML) und importiere erneut:
|
||||||
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
|
→ Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (`references` + `backlink`).
|
||||||
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
|
3. Ändere den Body von `exp-one.md` so, dass andere Chunk-Grenzen entstehen und importiere erneut:
|
||||||
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
|
→ Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
|
||||||
|
|
||||||
Kompatibel mit:
|
**Kompatibilität:**
|
||||||
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
|
- `note.schema.json`: `created`/`updated` sind Strings.
|
||||||
|
|
||||||
**Beispiele:**
|
**Beispiele:**
|
||||||
|
|
||||||
@@ -41,41 +64,60 @@ Kompatibel mit:
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 2. Import in Qdrant
|
## 2) Import in Qdrant
|
||||||
|
|
||||||
### `import_markdown.py`
|
### `scripts/import_markdown.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Importiert Notes/Chunks/Edges aus einem Vault nach Qdrant.
|
Importiert Notes/Chunks/Edges aus einem Vault nach Qdrant.
|
||||||
- prüft Frontmatter
|
- validiert Frontmatter
|
||||||
- chunked Inhalte + Embeddings
|
- chunked Inhalte + Embeddings
|
||||||
- erzeugt Edges (`references`, `references_at`, `backlink`)
|
- erzeugt Edges (`references`, `references_at`, `backlink`, `next`, `prev`, `belongs_to`)
|
||||||
- erkennt geänderte Dateien automatisch
|
- erkennt geänderte Dateien **modusgesteuert** (Body/Frontmatter/Full)
|
||||||
- optional: Purge der alten Chunks/Edges pro Note
|
- optional: Purge der alten Chunks/Edges pro Note
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--vault PATH` : (Pflicht) Vault-Verzeichnis
|
- `--vault PATH` : (Pflicht) Vault-Verzeichnis
|
||||||
- `--apply` : Änderungen wirklich durchführen (sonst Dry-Run)
|
- `--apply` : Änderungen wirklich durchführen (ohne: Dry-Run)
|
||||||
- `--purge-before-upsert` : Vor Upsert alte Chunks/Edges **nur der geänderten Notes** löschen
|
- `--purge-before-upsert` : Vor Upsert alte Chunks/Edges **nur der geänderten Notes** löschen
|
||||||
- `--note-id ID` : Nur eine bestimmte Note importieren
|
- `--prefix STR` : überschreibt `COLLECTION_PREFIX`
|
||||||
|
- `--note-id ID` : nur eine bestimmte Note importieren
|
||||||
|
- `--note-scope-refs` : zusätzlich **note-weite** `references` erzeugen (nicht nur chunk-weise)
|
||||||
|
- `--baseline-modes` : berechne & speichere Hash-Signaturen für **alle** Modi (`Body`, `Frontmatter`, `Full`) als Baseline
|
||||||
|
|
||||||
|
**Erwartetes Verhalten:**
|
||||||
|
- Unveränderte Notes → `"changed": false`, Entscheidung: `apply-skip-unchanged` (bei `--apply`)
|
||||||
|
- Geänderte Notes → `"changed": true` (abhängig von `MINDNET_HASH_COMPARE`)
|
||||||
|
- Idempotenz: mehrfacher Lauf ohne Änderungen erzeugt **keine** neuen Punkte
|
||||||
|
- Baseline: Bei `--baseline-modes` erscheinen Felder wie `hash_signature`, `hash_fulltext`, ggf. `hash_frontmatter` und `needs_baseline_for_mode: false` bei Folgeläufen.
|
||||||
|
|
||||||
**Beispiele:**
|
**Beispiele:**
|
||||||
|
|
||||||
python3 -m scripts.import_markdown --vault ./vault
|
# Dry-Run (zeigt Änderungen, schreibt NICHT)
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
python3 -m scripts.import_markdown --vault ./vault --prefix mindnet
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
|
||||||
|
# Apply + gezielte Vorreinigung pro geänderter Note
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --prefix mindnet --apply --purge-before-upsert
|
||||||
|
|
||||||
|
# Baseline-Signaturen aller Modi vorhalten
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --apply --baseline-modes
|
||||||
|
|
||||||
|
**Hash-Modi in der Praxis:**
|
||||||
|
- `MINDNET_HASH_COMPARE=Body` → Änderungen im Body werden erkannt; reine Frontmatter-Änderungen bleiben **ohne Effekt**.
|
||||||
|
- `MINDNET_HASH_COMPARE=Frontmatter` → nur Frontmatter maßgeblich.
|
||||||
|
- `MINDNET_HASH_COMPARE=Full` → beide Anteile zählen.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. Konsistenzprüfungen
|
## 3) Konsistenzprüfungen
|
||||||
|
|
||||||
### `audit_vault_vs_qdrant.py`
|
### `scripts/audit_vault_vs_qdrant.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Vergleicht Vault-Inhalte mit Qdrant.
|
Vergleicht Vault-Inhalte mit Qdrant.
|
||||||
- Zählt Notes/Chunks/Edges
|
- Zählt Notes/Chunks/Edges
|
||||||
- Verifiziert Anzahl Wikilinks vs. Edges
|
- Verifiziert Anzahl Wikilinks vs. Edges
|
||||||
- Meldet Mismatches pro Note
|
- Meldet Deltas (z. B. `references` im Vault vs. in Qdrant)
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--vault PATH` : Vault-Verzeichnis
|
- `--vault PATH` : Vault-Verzeichnis
|
||||||
@@ -85,15 +127,13 @@ Vergleicht Vault-Inhalte mit Qdrant.
|
|||||||
|
|
||||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
||||||
|
|
||||||
---
|
### `scripts/validate_edges.py`
|
||||||
|
|
||||||
### `validate_edges.py`
|
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Prüft die Edges-Collection in Qdrant.
|
Prüft die Edges-Collection in Qdrant.
|
||||||
- Zählt Edges pro Typ
|
- Zählt Edges pro Typ (`references`, `references_at`, `backlink`, `next`, `prev`, `belongs_to`)
|
||||||
- Kontrolliert Invarianten (z. B. jede Reference hat Backlink)
|
- Kontrolliert Invarianten (z. B. jede `references` hat einen `backlink`)
|
||||||
- Listet unresolved Edges auf
|
- Listet `unresolved` Edges auf
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--prefix STR` : Prefix für Qdrant-Collections
|
- `--prefix STR` : Prefix für Qdrant-Collections
|
||||||
@@ -105,35 +145,69 @@ Prüft die Edges-Collection in Qdrant.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4. Qdrant zurücksetzen
|
## 4) Export Qdrant → Markdown (Vault)
|
||||||
|
|
||||||
### `reset_qdrant.py`
|
### `scripts/export_markdown.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Setzt die Qdrant-Collections zurück.
|
Exportiert Notes + rekonstruierten Body in einen Zielordner.
|
||||||
- „wipe“: löscht Collections komplett (inkl. Schema)
|
- Pfade werden **relativ** aus `note.payload.path` abgeleitet (führende `/` entfernt, Backslashes → Slashes; Unicode-NFC empfohlen)
|
||||||
- „truncate“: löscht nur Inhalte, behält Collections
|
- **Body-Rekonstruktion** priorisiert:
|
||||||
|
1. `fulltext` (falls in Note-Payload vorhanden)
|
||||||
|
2. sonst **Chunks**: Sortierung `seq` → `chunk_index` → Nummer in `chunk_id`; **nur `text`** (Overlap bereits entfernt)
|
||||||
|
- `--prefix` (CLI) wird unterstützt (oder ENV `COLLECTION_PREFIX`)
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--mode {wipe,truncate}` : Aktion
|
- `--out PATH` : Ziel-Root (muss existieren)
|
||||||
- `--prefix STR` : Prefix der Collections (Default: `mindnet`)
|
- `--overwrite` : vorhandene Dateien überschreiben
|
||||||
- `--yes` : Automatische Bestätigung
|
- `--note-id ID` : nur eine bestimmte Note exportieren
|
||||||
- `--dry-run` : Nur anzeigen, nicht ausführen
|
- `--prefix STR` : überschreibt `COLLECTION_PREFIX`
|
||||||
|
- `--include-edges {none,footer,yaml}` *(optional, falls aktiviert)*: Edges als YAML-Liste im Frontmatter oder im Footer beilegen
|
||||||
|
|
||||||
**Beispiele:**
|
**Beispiele:**
|
||||||
|
|
||||||
|
export COLLECTION_PREFIX="mindnet"
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault
|
||||||
|
|
||||||
|
# Nur eine bestimmte Note:
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-architektur-ki-trainerassistent-761cfe
|
||||||
|
|
||||||
|
# Existierende Dateien überschreiben:
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5) Qdrant zurücksetzen
|
||||||
|
|
||||||
|
### `scripts/reset_qdrant.py`
|
||||||
|
|
||||||
|
**Zweck:**
|
||||||
|
Setzt die Qdrant-Collections zurück.
|
||||||
|
- `wipe`: löscht Collections komplett (inkl. Schema)
|
||||||
|
- `truncate`: löscht nur Inhalte, behält Collections
|
||||||
|
- `truncate-edges`: löscht nur Punkte in `*_edges`
|
||||||
|
|
||||||
|
**Parameter:**
|
||||||
|
- `--mode {wipe,truncate,truncate-edges}`
|
||||||
|
- `--prefix STR` : Prefix der Collections (Default: `mindnet`)
|
||||||
|
- `--yes` : Automatische Bestätigung
|
||||||
|
- `--dry-run` : Nur anzeigen, nicht ausführen
|
||||||
|
|
||||||
|
**Beispiele:**
|
||||||
|
|
||||||
|
python3 -m scripts.reset_qdrant --mode truncate-edges --prefix mindnet
|
||||||
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
||||||
python3 -m scripts.reset_qdrant --mode wipe --prefix mindnet --yes
|
python3 -m scripts.reset_qdrant --mode wipe --prefix mindnet --yes
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 5. Diagnose
|
## 6) Diagnose & Tools
|
||||||
|
|
||||||
### `diagnose_changed.py`
|
### `scripts/diagnose_changed.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Prüft, ob die Datei-Hashes (`sha256`) mit den gespeicherten `hash_fulltext` in Qdrant übereinstimmen.
|
Prüft, ob Datei-Hashes (`sha256`) mit gespeicherten `hash_*` in Qdrant übereinstimmen.
|
||||||
Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt.
|
Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt oder Moduswechsel geplant sind.
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--vault PATH` : Vault-Verzeichnis
|
- `--vault PATH` : Vault-Verzeichnis
|
||||||
@@ -145,14 +219,11 @@ Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt.
|
|||||||
|
|
||||||
python3 -m scripts.diagnose_changed --vault ./vault --prefix mindnet --all
|
python3 -m scripts.diagnose_changed --vault ./vault --prefix mindnet --all
|
||||||
|
|
||||||
---
|
### `scripts/resolve_unresolved_references.py`
|
||||||
|
|
||||||
## 6. Unresolved Links nachträglich auflösen
|
|
||||||
|
|
||||||
### `resolve_unresolved_references.py`
|
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwischen passende Notizen existieren.
|
Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwischen passende Notizen existieren.
|
||||||
|
Ersetzt `unresolved` durch gültige `references`/`backlink`.
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--prefix STR` : Prefix für Qdrant-Collections
|
- `--prefix STR` : Prefix für Qdrant-Collections
|
||||||
@@ -165,26 +236,114 @@ Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwisc
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# ✅ Typische Workflows
|
## 7) Parser-Verhalten (robust & abwärtskompatibel)
|
||||||
|
|
||||||
### A) Vault neu importieren
|
- Der Parser liest Markdown **fehlertolerant** (UTF-8 mit Fallbacks), extrahiert YAML-Frontmatter und Body.
|
||||||
|
- Keine harten Abhängigkeiten von alten Regex-Symbolen (`FRONTMATTER_RE` entfällt zugunsten interner Marker).
|
||||||
|
- `read_markdown(path)` gibt `(frontmatter_dict, body_str)` zurück.
|
||||||
|
- **Sonderfälle aus Fremd-Tools** (z. B. Silverbullet-Dateien wie `CONFIG.md`, `index.md`) werden **nicht importiert**, sofern sie nicht zum Vault gehören.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8) Chunk-Payload (Fenster vs. Kern)
|
||||||
|
|
||||||
|
- **`window`** = kontextualisierter Fenster-Text (mit linkem Overlap)
|
||||||
|
- **`text`** = **Kerntext** ohne das vom vorherigen Chunk übernommene Overlap
|
||||||
|
- Felder: `start`, `end`, `overlap_left`, `overlap_right`, `chunk_index`, `seq`, `path`, `section_title`, `section_path`, `token_count` (falls vorhanden)
|
||||||
|
- **Fallback (synthetisch):** Falls der Chunker **keinen** Overlap-Fenstertext liefert, erzeugt die Payload-Schicht Fenster mit typgerechtem Overlap aus `chunk_config.get_sizes(type)['overlap']` (unterer Wert).
|
||||||
|
- **Edges** werden aus `window` abgeleitet (um Links an Chunk-Grenzen nicht zu verlieren); der **Export** rekonstruiert den Body aus `text` (Overlap-frei).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9) Edge-Modell & IDs
|
||||||
|
|
||||||
|
- Collections:
|
||||||
|
`*_notes` (optional Vektor / Nullvektor), `*_chunks` (384-d Embeddings), `*_edges` (1-d Dummy-Vektor)
|
||||||
|
- **Deterministische IDs** (UUIDv5 / stabile Keys):
|
||||||
|
- `note_id` stabil aus Frontmatter/Slug
|
||||||
|
- `chunk_id` i. d. R. `note_id#<n>`
|
||||||
|
- `edge_id` stabil aus `(kind, source_id, target_id, seq)`
|
||||||
|
- Kantenarten: `references`, `references_at`, `backlink`, `next`, `prev`, `belongs_to`
|
||||||
|
- `unresolved` wird gekennzeichnet und kann per Resolver später aufgelöst werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10) Tests & Qualitätssicherung
|
||||||
|
|
||||||
|
**Tests unter `tests/`**
|
||||||
|
|
||||||
|
- `check_chunks_window_vs_text.py` – vergleicht je Chunk `window` vs. `text`, schätzt Overlap
|
||||||
|
- `compare_vaults.py` – rekursiver Vault-Vergleich (Body/Frontmatter/All, Key: auto/id/title/filename), Unicode-NFC-robust
|
||||||
|
- `verify_chunks_integrity.py` – prüft, ob Rekonstruktion aus Chunks (`text`) zu `fulltext` bzw. zum Vault-Body passt
|
||||||
|
- `list_md_seen_by_importer.py` – zeigt, welche `.md` der Importer sieht (inkl. Gründe/Filter)
|
||||||
|
- `inspect_one_note.py` – Parser-Diagnose für eine konkrete Datei
|
||||||
|
- `run_e2e_roundtrip.sh` – End-to-End: Reset → Import → Audit → (optional) Hash-Reporter → Export → Vergleich
|
||||||
|
|
||||||
|
**Typische Aufrufe:**
|
||||||
|
|
||||||
|
# Overlap-Qualität (Fenster vs. Kern)
|
||||||
|
python3 tests/check_chunks_window_vs_text.py --prefix "$COLLECTION_PREFIX"
|
||||||
|
|
||||||
|
# Roundtrip-Vergleich (rekursiv)
|
||||||
|
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --focus body
|
||||||
|
|
||||||
|
# Chunks-Rekonstruktion (gegen Vault-Body)
|
||||||
|
python3 tests/verify_chunks_integrity.py --prefix "$COLLECTION_PREFIX" --vault ./vault
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11) Typische Workflows
|
||||||
|
|
||||||
|
**A) Clean-Start mit Test-Vault**
|
||||||
|
|
||||||
|
python3 -m scripts.make_test_vault --out ./test_vault --force
|
||||||
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet
|
||||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet --apply --purge-before-upsert
|
||||||
|
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
|
||||||
|
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||||
|
export COLLECTION_PREFIX="mindnet"
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault
|
||||||
|
|
||||||
### B) Änderungen einspielen (mit Purge für geänderte Notes)
|
**B) Änderungs-Propagation & Idempotenz**
|
||||||
|
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet
|
||||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet --apply --purge-before-upsert
|
||||||
|
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
|
||||||
|
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||||
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet
|
||||||
|
|
||||||
### C) Unresolved Links nachziehen
|
**C) Unresolved Links lösen**
|
||||||
|
|
||||||
|
python3 -m scripts.import_markdown --vault ./test_vault --apply --prefix mindnet
|
||||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
||||||
python3 -m scripts.validate_edges --prefix mindnet --details
|
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||||
|
|
||||||
### D) Test-Vault Szenario durchspielen
|
**D) Export & Roundtrip (Produkt-Vault)**
|
||||||
|
|
||||||
python3 -m scripts.make_test_vault --out ./test_vault --force
|
export COLLECTION_PREFIX="mindnet"
|
||||||
python3 -m scripts.import_markdown --vault ./test_vault --apply
|
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
|
||||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
|
python3 -m scripts.export_markdown --out ./_exportVault --prefix "$COLLECTION_PREFIX"
|
||||||
|
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --focus body
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12) Troubleshooting (kurz)
|
||||||
|
|
||||||
|
- **Importer markiert alle als changed (Moduswechsel):**
|
||||||
|
→ `--baseline-modes` einmalig laufen lassen; sicherstellen: `MINDNET_HASH_COMPARE` passt zum Ziel.
|
||||||
|
- **Edges verschwinden / nur 1 Edge sichtbar:**
|
||||||
|
→ `app/core/qdrant_points.py` muss Edge-Payload normalisieren (Schema Alt/Neu) und `edge_id` eindeutig bilden.
|
||||||
|
- **Export ohne Body / falsches Zielverzeichnis:**
|
||||||
|
→ Exporter nutzt relativierte Pfade und rekonstruiert Body aus `fulltext` oder Chunks (`text`). ENV/CLI `--prefix` beachten.
|
||||||
|
- **Parser-Fehler (UTF-8):**
|
||||||
|
→ Neuer Parser arbeitet mit Fallbacks; mit `tests/inspect_one_note.py` prüfen, was ankommt.
|
||||||
|
- **„Fehlende“ Dateien im Vergleich:**
|
||||||
|
→ Prüfe, ob Dateien **wirklich zum Vault gehören** (z. B. Silverbullet-Artefakte `CONFIG.md`, `index.md` ausblenden).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13) Hinweise zu Silverbullet-Dateien
|
||||||
|
|
||||||
|
- Dateien wie `CONFIG.md` oder `index.md`, die **nicht** zum inhaltlichen Vault gehören, werden **nicht importiert** und erscheinen daher nicht im Export-Roundtrip.
|
||||||
|
- Diese Dateien bitte vom produktiven Vault trennen oder im Vergleich (`tests/compare_vaults.py
|
||||||
+180
-435
@@ -1,504 +1,249 @@
|
|||||||
---
|
# mindnet – Knowledge Design
|
||||||
title: "mindnet – Knowledge Design"
|
**Version:** 1.4.0 (aktualisiert 2025-10-06)
|
||||||
id: "design-knowledge-architecture"
|
|
||||||
type: "guide"
|
|
||||||
status: "active"
|
|
||||||
version: "1.3.0"
|
|
||||||
created: 2025-09-02
|
|
||||||
updated: 2025-09-09
|
|
||||||
tags: ["type/guide","area/mindnet","topic/obsidian","topic/knowledge-architecture"]
|
|
||||||
owner: "Ich"
|
|
||||||
---
|
|
||||||
|
|
||||||
# mindnet – Knowledge Design
|
## 1. Gesamtziel und Systemverständnis
|
||||||
|
|
||||||
## 1. Gesamtziel, Anspruch & Nutzen
|
**mindnet** ist ein lokales Wissensnetzwerk, das alle meine Notizen, Projekte, Gedanken, Erfahrungen und Pläne als Markdown-Dateien speichert, automatisch importiert, semantisch in **Chunks** zerlegt und als **Graph-Struktur in Qdrant** ablegt.
|
||||||
|
Es ist die Grundlage für ein agentenfähiges, lokal betriebenes LLM-System, das meine Daten verknüpft, abfragt und erweitert.
|
||||||
|
|
||||||
**Ziel:**
|
**Zentrale Ziele**
|
||||||
*mindnet* ist ein persönliches Wissensnetzwerk, das **alle** relevanten Informationen über mein Leben, meine Werte, Projekte, Erfahrungen, Pläne und Erkenntnisse **strukturiert** erfasst. Es bleibt **portabel** (Markdown + YAML), ist **parser-freundlich**, und so gestaltet, dass Tools (Obsidian + Dataview, Embeddings/Vektorsuche via Qdrant, Agenten) es **maschinenlesbar** nutzen können.
|
- Vollständige Repräsentation persönlichen Wissens in strukturierter, maschinenlesbarer Form
|
||||||
|
- Reproduzierbare Verarbeitung: Import → Chunking → Embedding → Edges → Graph
|
||||||
|
- Verlustfreie Roundtrip-Fähigkeit (Import ⇄ Export)
|
||||||
|
- Erweiterbarkeit für Agenten, RAG-Retriever und semantische Suche
|
||||||
|
|
||||||
**Anspruch an die Struktur:**
|
**Prinzipien**
|
||||||
- **Stabil**: robuste IDs, deterministische Dateinamen, konsistente Feldnamen
|
- **Deterministisch:** IDs, Hashes und Edges wiederholbar
|
||||||
- **Atomar**: eine Kernaussage/ein zentrales Thema pro Datei (keine Sammelcontainer)
|
- **Portabel:** Markdown + YAML
|
||||||
- **Erweiterbar**: neue Typen jederzeit möglich (ohne Migrationsschmerz)
|
- **Robust:** fehlertoleranter Parser
|
||||||
- **Portabel**: pures Markdown + YAML → funktioniert in Obsidian, Git, Parsern
|
- **Idempotent:** keine Duplikate bei Re-Import
|
||||||
- **Vernetzbar**: systematische Links/Backlinks bilden ein navigierbares Wissensnetz
|
- **Erweiterbar:** neue Typen und Edge-Arten ohne Migration
|
||||||
- **Parser-freundlich**: wiederkehrende Abschnittsüberschriften, klare Enums
|
|
||||||
|
|
||||||
**Nutzen / Einsatzzwecke:**
|
|
||||||
- **Schnelles Wiederfinden**: Dataview-Abfragen, Filter, Dashboards
|
|
||||||
- **LLM-gestütztes Arbeiten**: Q&A mit Kontext-Zitaten aus Chunks (RAG)
|
|
||||||
- **Agenten**: Vorschläge für Verbindungen/Struktur; Auflösen von „unresolved links“
|
|
||||||
- **Projektnavigation**: Aufgabenbeziehungen, Abhängigkeiten
|
|
||||||
- **Langfristiges Gedächtnis**: Lebens-/Projektchronik (rückblickend und planend)
|
|
||||||
- **Automatisierung**: Vektor-Index (Qdrant), Embeddings, LLM-Q&A, Agenten-Workflows
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 2. YAML-Frontmatter-Schema
|
## 2. YAML-Frontmatter-Schema
|
||||||
|
|
||||||
> **Pflichtfelder:** `title`, `id`, `type`, `status`, `created`, `tags`
|
| Feld | Typ | Beschreibung |
|
||||||
> **Empfohlen:** `updated`, `area`, `project`, `priority`, `aliases`, `lang`, `people`, `depends_on`, `source` u. a.
|
|-------|-----|--------------|
|
||||||
|
| `title` | string | Menschlich lesbarer Titel |
|
||||||
|
| `id` | string | Eindeutige Note-ID (Slug) |
|
||||||
|
| `type` | enum | `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone` |
|
||||||
|
| `status` | enum | `draft`, `active`, `done`, `archived` |
|
||||||
|
| `created` | date | ISO-Datum |
|
||||||
|
| `updated` | date | Letzte Änderung |
|
||||||
|
| `area` | string | Themen- oder Lebensbereich |
|
||||||
|
| `project` | string | Zugehöriges Projekt |
|
||||||
|
| `tags` | list | Strukturierte Tags (`area/…`, `topic/…`) |
|
||||||
|
| `depends_on` | list | IDs anderer Notizen |
|
||||||
|
| `assigned_to` | list | Beteiligte Personen |
|
||||||
|
| `embedding_exclude` | bool | Falls `true`, keine Embeddings |
|
||||||
|
| `hash_mode` | enum | Hash-Modus beim Import (`body`, `frontmatter`, `full`) |
|
||||||
|
| `priority` | enum/int | 1–5 oder `low` / `med` / `high` |
|
||||||
|
| `effort_min`, `due` | int/date | Aufwand / Termin |
|
||||||
|
| `aliases` | list | Alternative IDs |
|
||||||
|
| `lang` | string | ISO-Sprachcode |
|
||||||
|
| `source` | string | Herkunft / Referenzquelle |
|
||||||
|
|
||||||
### 2.1 Feldübersicht
|
**Pflichtfelder:** `title`, `id`, `type`, `status`, `created`
|
||||||
|
**Optionale Felder:** alle übrigen
|
||||||
| Feld | Typ | Bedeutung |
|
|
||||||
|--------------------|-------------------------------|---------------------------------------------------------------------------|
|
|
||||||
| `title` | string | Menschlich lesbarer Titel |
|
|
||||||
| `id` | string (slug/identifier) | **Stabile** ID; referenzierbar über Wikilinks |
|
|
||||||
| `type` | enum | `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone`, … |
|
|
||||||
| `status` | enum | `draft` \| `active` \| `done` \| `archived` |
|
|
||||||
| `created` | date | Erstellungsdatum (ISO) |
|
|
||||||
| `updated` | date | Letzte Änderung (ISO) |
|
|
||||||
| `area` | string | Themen-/Lebensbereich (z. B. „mindnet“, „gesundheit“, „familie“) |
|
|
||||||
| `project` | string | Projekt-/Arbeitskontext |
|
|
||||||
| `tags` | list<string> | Taxonomische Tags (präfixiert, s. Abschnitt 4) |
|
|
||||||
| `aliases` | list<string> | Alternative IDs/Titel (Slug-Resolver) |
|
|
||||||
| `lang` | string | Sprache, optional (z. B. `de`, `en`) |
|
|
||||||
| `priority` | enum/int | Priorität, optional (`low`/`med`/`high` oder 1–5) |
|
|
||||||
| `due`, `effort_min`| date/int | Für Aufgaben/Projekte (Fälligkeit, Aufwand in Minuten) |
|
|
||||||
| `people` | list<string> | Beteiligte Personen/IDs |
|
|
||||||
| `depends_on` | list<string> | Abhängigkeiten auf andere IDs |
|
|
||||||
| `source` | string | Quelle (Artikel, Buch, Video, …) |
|
|
||||||
| `slug` | string | Stabiler Pfad-/Dateiname, falls abweichend |
|
|
||||||
|
|
||||||
### 2.2 Konventionen
|
|
||||||
- `id` ist **menschenlesbar** (Slug), eindeutig im Vault; `aliases` dienen als Fallback-IDs.
|
|
||||||
- Datumsformat: ISO `YYYY-MM-DD`.
|
|
||||||
- Keine duplizierten Tags; Präfixe nutzen (`area/…`, `type/…`, `topic/…`).
|
|
||||||
- Frontmatter nur Fakten/Metadaten, **keine** ausufernden Inhalte (die stehen in den Body-Abschnitten).
|
|
||||||
|
|
||||||
### 2.3 Minimalbeispiel Frontmatter
|
|
||||||
|
|
||||||
title: "Gedanke: YAML-Standards"
|
|
||||||
id: thought-yaml-standards
|
|
||||||
type: thought
|
|
||||||
status: draft
|
|
||||||
created: 2025-09-02
|
|
||||||
updated: 2025-09-09
|
|
||||||
area: mindnet
|
|
||||||
project: project-mindnet
|
|
||||||
tags: [area/mindnet, type/thought, topic/yaml]
|
|
||||||
aliases: [yaml-gedanken]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
### 2.4 Enum-Werte (Beispiele)
|
|
||||||
- `type`: `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone`, `manifesto`
|
|
||||||
- `status`: `draft`, `active`, `done`, `archived`
|
|
||||||
- `priority`: `low`, `med`, `high` oder 1–5
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. Dateinamen-Konventionen
|
## 3. Dateinamen und Ordner
|
||||||
- `YYYY-MM-DD_title.md` oder `slug.md`.
|
|
||||||
- Pfade **relativ** zum Vault; keine absoluten Pfade (einheitliche Slashes `/`).
|
- Format: `YYYY-MM-DD_title.md` oder `slug.md`
|
||||||
- Verzeichnisse nach eigener Ordnung, empfohlen: `10_thoughts/`, `20_concepts/`, `30_projects/`, `40_journal/`, …
|
- Strukturierte Ordner:
|
||||||
|
- `10_thoughts/`, `20_concepts/`, `30_projects/`, `40_experiences/`, …
|
||||||
|
- Alle Pfade relativ; Unterstriche statt Leerzeichen
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4. Tagging-Regeln & Taxonomie
|
## 4. Link- und Edge-Design
|
||||||
- Präfixe verwenden: `area/…`, `type/…`, `topic/…`, `status/…`.
|
|
||||||
- Möglichst **keine** losen Ein-Wort-Tags ohne Präfix (verhindert Wildwuchs).
|
**Verlinkungen**
|
||||||
- Für Dataview konsistente Namespaces beibehalten (z. B. `topic/llm`, `topic/lernen`).
|
- Wikilinks: `[[note-id]]` oder `[[Titel|note-id]]`
|
||||||
|
- Automatische Edges aus Text und Frontmatter
|
||||||
|
|
||||||
|
**Edge-Typen in Qdrant**
|
||||||
|
|
||||||
|
| Typ | Bedeutung | Quelle | Scope |
|
||||||
|
|------|------------|--------|--------|
|
||||||
|
| `belongs_to` | Chunk → Note | intern | chunk |
|
||||||
|
| `prev` / `next` | Reihenfolge | intern | chunk |
|
||||||
|
| `references` | explizite Wikilinks | Text | chunk |
|
||||||
|
| `backlink` | Gegenkante zu `references` | abgeleitet | note |
|
||||||
|
| `depends_on` | YAML-Abhängigkeiten | YAML | note |
|
||||||
|
| `assigned_to` | YAML-Zuweisung | YAML | note |
|
||||||
|
| `unresolved` | Ziel fehlt (Stub) | abgeleitet | chunk |
|
||||||
|
|
||||||
|
**Edge-Schlüssel**
|
||||||
|
- `source_id`, `target_id`, `kind`, `scope`, `note_id`
|
||||||
|
- Dedup-Schlüssel: Kombination aus `(kind, source_id, target_id, scope)`
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 5. Backlink- & Link-Regeln
|
## 5. Chunking-Modell
|
||||||
|
|
||||||
- **Wikilinks**: `[[stabile-id]]` oder `[[Titel|stabile-id]]`
|
**Chunk-Datenstruktur**
|
||||||
- Im **Fließtext** verlinken (Forwardlinks); am **Ende** optional **„Mögliche Verbindungen“** redaktionell pflegen:
|
| Feld | Beschreibung |
|
||||||
|
|-------|---------------|
|
||||||
|
| `note_id` | Zugehörige Note |
|
||||||
|
| `chunk_index` | Laufende Nummer |
|
||||||
|
| `text` | Originalabschnitt |
|
||||||
|
| `window` | Text + Overlap links/rechts |
|
||||||
|
| `overlap_left` | Länge des linken Overlaps |
|
||||||
|
| `overlap_right` | Länge des rechten Overlaps |
|
||||||
|
| `embedding` | 384-dimensionaler Vektor |
|
||||||
|
| `tokens` | Tokenanzahl (optional) |
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
**Chunking-Regeln (aus `chunking_strategy.md`)**
|
||||||
- [[concept-persoenliches-leitbild]]
|
- Trennung nach Absätzen, Überschriften, Listen, Tabellen
|
||||||
- [[project-mindnet]]
|
- Token-Zielgröße 350–500 (max 600)
|
||||||
|
- Overlap: 30–40 % für semantische Kohärenz
|
||||||
- **Linkrichtung:** „aufwärts“ (Detail → Konzept/Leitbild) und „seitwärts“ (verwandte Knoten).
|
- Overlap wird in `window` integriert, sodass `text ≠ window`
|
||||||
- Optional zusätzliche **ID-Relationen** im YAML (`related_ids`) für Parser.
|
|
||||||
- **Unresolved Links:** Links auf noch nicht existierende Ziele sind zulässig; der Importer schreibt `status: unresolved`-Edges (später durch Resolver/Agent heilbar).
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 5.1 Edge-Typen & Semantik (Graph)
|
## 6. Hash-Strategie und Änderungsverfolgung
|
||||||
|
|
||||||
Beziehungen werden als **Edges** mit neuem Payload-Schema gespeichert:
|
**Zweck:** Nur bei realer Inhaltsänderung re-indexieren
|
||||||
|
|
||||||
kind : "belongs_to" | "next" | "prev" | "references" | "backlink" | "unresolved"
|
| Umgebungsvariable | Beschreibung |
|
||||||
source_id : ID des Quellknotens (Chunk- oder Note-ID)
|
|--------------------|--------------|
|
||||||
target_id : ID des Zielknotens (Note- oder Chunk-ID)
|
| `MINDNET_HASH_COMPARE` | Vergleichsquelle (`Body`, `Full`, `Frontmatter`) |
|
||||||
scope : "chunk" | "note"
|
| `MINDNET_HASH_SOURCE` | Rohtext vs. geparst (`raw` / `parsed`) |
|
||||||
note_id : Owner-Note des Edges (für schnelles Filtern/Purge)
|
| `MINDNET_HASH_NORMALIZE` | Normalisierung (`canonical` / `none`) |
|
||||||
status : optional, z. B. "unresolved"
|
|
||||||
|
|
||||||
**Kantenarten**
|
**Basismodi**
|
||||||
- **belongs_to** — *Chunk → Note*
|
- `body` → Nur Textkörper
|
||||||
Jeder Chunk gehört zu **genau einer** Note. *Scope:* `chunk`.
|
- `frontmatter` → Nur Metadaten
|
||||||
|
- `full` → Kombination
|
||||||
|
- Hash = SHA-256(canonicalized_input)
|
||||||
|
|
||||||
- **prev / next** — *Chunk → Chunk*
|
Bei Änderung: Re-Import, neue Embeddings, Edges werden aktualisiert.
|
||||||
Lineare Reihenfolge innerhalb derselben Note; **symmetrisch** angelegt (d. h. `prev` und `next` konsistent). *Scope:* `chunk`.
|
|
||||||
|
|
||||||
- **references** — *Chunk → Note*
|
|
||||||
Aus `[[Wikilinks]]` oder Markdown-Links im **Chunk-Text** (präzise Herkunft). *Scope:* `chunk`.
|
|
||||||
|
|
||||||
- **backlink** — *Note → Note*
|
|
||||||
Automatisch generierte Gegenkante zu `references` (pro Note **dedupliziert**). *Scope:* `note`.
|
|
||||||
|
|
||||||
- **unresolved**
|
|
||||||
Edge mit `status: "unresolved"`, wenn `target_id` (noch) nicht existiert. Wird später vom Resolver/Agent ersetzt.
|
|
||||||
|
|
||||||
- **Optional:** `references:note` — *Note → Note*
|
|
||||||
Zusätzliche Forwardlinks auf Note-Ebene; standardmäßig **deaktiviert**, um Doppelzählungen mit `references` (Chunk-Scope) zu vermeiden. Aktivierung per ENV/Flag möglich.
|
|
||||||
|
|
||||||
**Performanzhinweise (Qdrant)**
|
|
||||||
- Edges werden dedupliziert über `(kind, source_id, target_id, scope)`.
|
|
||||||
- `note_id` ist Pflichtfeld im Edge-Payload (Owner-Note).
|
|
||||||
- Payload-Indizes für Qdrant anlegen:
|
|
||||||
|
|
||||||
edges : kind (KEYWORD), scope (KEYWORD), source_id (KEYWORD), target_id (KEYWORD), note_id (KEYWORD)
|
|
||||||
chunks : note_id (KEYWORD), chunk_index (INTEGER)
|
|
||||||
notes : note_id (KEYWORD)
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 6. Abschnitts-Konventionen im Body
|
## 7. Qdrant-Speicherstruktur
|
||||||
|
|
||||||
Einheitliche Überschriften erleichtern Parser, Chunking und Abfragen. Typ-spezifische Empfehlungen:
|
| Collection | Inhalt | Primärfelder | Embedding |
|
||||||
|
|-------------|---------|---------------|------------|
|
||||||
|
| `mindnet_notes` | Notes mit Fulltext & Hash | `note_id` | optional |
|
||||||
|
| `mindnet_chunks` | Chunks mit Text, Window, Embedding | `chunk_id`, `note_id` | 384 d |
|
||||||
|
| `mindnet_edges` | Graphkanten | `edge_id`, `source_id`, `target_id` | — |
|
||||||
|
|
||||||
- **Gedanke (`thought`)**
|
**Upsert-Strategie**
|
||||||
`## Zusammenfassung` · `## Begründung / Details` · `## Belege / Quellen` · `## Nächste Schritte` · `## Mögliche Verbindungen`
|
- Idempotente UUIDv5-IDs
|
||||||
|
- Keine Duplikate durch deterministische Signaturen
|
||||||
|
- `purge-before-upsert` zur Bereinigung
|
||||||
|
|
||||||
- **Erfahrung (`experience`)**
|
**Indizes**
|
||||||
`## Kontext` · `## Beobachtung` · `## Interpretation` · `## Implikationen` · `## Mögliche Verbindungen`
|
- notes: `note_id`, `hash_signature`
|
||||||
|
- chunks: `note_id`, `chunk_index`
|
||||||
- **Aufgabe (`task`)**
|
- edges: `(kind, source_id, target_id, scope)`
|
||||||
`## Checkliste` · `## Notizen` · `## Nächste Schritte` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
- **Projekt (`project`)**
|
|
||||||
`## Scope` · `## Arbeitspakete` · `## Risiken & Gegenmaßnahmen` · `## Status / Fortschritt` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
- **Journal (`journal`)**
|
|
||||||
`## Tageszusammenfassung` · `## Highlights` · `## Learnings` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
- **Meeting (`meeting`)**
|
|
||||||
`## Agenda` · `## Notizen` · `## Entscheidungen` · `## Aufgaben` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 7. Abbildung persönlicher Inhalte auf Typen
|
## 8. Fehler-Toleranz und Parser-Robustheit
|
||||||
|
|
||||||
**Richtlinien:**
|
- UTF-8 mit Fallback: ersetzt ungültige Bytes (`errors="replace"`)
|
||||||
- Wähle den Typ nach **Hauptintention** (Definition, Gedanke, Beobachtung, Aufgabe, Projekt, …).
|
- Ignoriert Nullbytes, BOM, exotische Zeilenenden
|
||||||
- **Konzepte** (`concept`): langlebige Definitionen, Prinzipien, Leitbilder.
|
- Erkennt defekte oder leere YAML-Header und überspringt sie
|
||||||
- **Gedanken** (`thought`): explorative Ideen, noch nicht stabil (können später bei Reife in `concept` überführt werden).
|
- Logt Problemfälle (`read_markdown failed`, `make_note_payload returned non-dict`)
|
||||||
- **Erfahrungen** (`experience`): Beobachtungen & Learnings aus der Praxis.
|
- Bei Fehlern: Import → warn → continue
|
||||||
- **Aufgaben** (`task`): konkrete, umsetzbare To-dos mit `due`/`effort_min`.
|
|
||||||
- **Projekte** (`project`): Ziele, Deliverables, Arbeitspakete, Abhängigkeiten.
|
|
||||||
- **Journal** (`journal`): tagesbasierte Notizen, Reflexionen, Reiseberichte.
|
|
||||||
- **Personen** (`person`): Kontakte, Profile, Bezüge, Gespräche.
|
|
||||||
- **Meetings** (`meeting`): Agenda, Notizen, Entscheidungen, Folgeaufgaben.
|
|
||||||
- **Meilensteine** (`milestone`): wichtige Wendepunkte im Lebens-/Projektverlauf.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 8. Beispiel-Notizen
|
## 9. Tests & Validierung
|
||||||
|
|
||||||
### 8.1 Gedanke (`thought`) – Beispiel
|
**Integrationstests**
|
||||||
|
- Roundtrip: Import → Export → Compare (`compare_vaults.py`)
|
||||||
|
- Chunk-Integrität: `verify_chunks_integrity.py`
|
||||||
|
- Hash-Audit: `hash_reporter.py`
|
||||||
|
- Window-Vergleich: `check_chunks_window_vs_text.py`
|
||||||
|
|
||||||
**Frontmatter**
|
**Erwartete Ergebnisse**
|
||||||
|
- Alle Notes werden erkannt
|
||||||
title: "Gedanke: Übungen für tiefe Arbeit"
|
- `window ≠ text` bei > 60 % der Chunks
|
||||||
id: thought-deep-work-uebungen
|
- Roundtrip-Vergleich liefert „OK“
|
||||||
type: thought
|
- Edge-Anzahl entspricht Modell (belongs_to = #Chunks, next/prev ≈ #Chunks – 1)
|
||||||
status: active
|
|
||||||
created: 2025-08-21
|
|
||||||
updated: 2025-09-05
|
|
||||||
area: lernen
|
|
||||||
project: project-self-improvement
|
|
||||||
tags: [area/lernen, type/thought, topic/fokus]
|
|
||||||
aliases: [deep-work-ideen]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
**Body (Ausschnitt)**
|
|
||||||
|
|
||||||
## Zusammenfassung
|
|
||||||
Kurze tägliche Routinen (z. B. Timer-Blöcke, Ablenkungslog) erhöhen meinen Fokus.
|
|
||||||
|
|
||||||
## Begründung / Details
|
|
||||||
- 25–45 Min Timerblöcke mit 5 Min Pause
|
|
||||||
- Ablenkungen sofort notieren
|
|
||||||
- Abschluss-Log am Ende des Tages
|
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
|
||||||
- [[concept-deep-work]]
|
|
||||||
- [[project-productivity-week]]
|
|
||||||
|
|
||||||
### 8.2 Konzept (`concept`) – Beispiel
|
|
||||||
|
|
||||||
**Frontmatter**
|
|
||||||
|
|
||||||
title: "Konzept: Deep Work"
|
|
||||||
id: concept-deep-work
|
|
||||||
type: concept
|
|
||||||
status: active
|
|
||||||
created: 2024-05-14
|
|
||||||
updated: 2025-08-01
|
|
||||||
area: lernen
|
|
||||||
tags: [area/lernen, type/concept, topic/fokus]
|
|
||||||
aliases: [konzept-tiefe-arbeit]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
**Body (Ausschnitt)**
|
|
||||||
|
|
||||||
## Definition
|
|
||||||
Tiefe Arbeit = kognitiv anspruchsvolle Arbeit ohne Ablenkung.
|
|
||||||
|
|
||||||
## Belege / Quellen
|
|
||||||
- Buch: Cal Newport – Deep Work
|
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
|
||||||
- [[thought-deep-work-uebungen]]
|
|
||||||
|
|
||||||
### 8.3 Aufgabe (`task`) – Beispiel
|
|
||||||
|
|
||||||
**Frontmatter**
|
|
||||||
|
|
||||||
title: "Task: Wochenplanung erstellen"
|
|
||||||
id: task-week-planning
|
|
||||||
type: task
|
|
||||||
status: active
|
|
||||||
created: 2025-09-07
|
|
||||||
updated: 2025-09-07
|
|
||||||
due: 2025-09-11
|
|
||||||
effort_min: 45
|
|
||||||
area: organisation
|
|
||||||
project: project-productivity-week
|
|
||||||
tags: [area/organisation, type/task, topic/planung]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
**Body (Ausschnitt)**
|
|
||||||
|
|
||||||
## Checkliste
|
|
||||||
- Ziele der Woche definieren
|
|
||||||
- 5 Fokusblöcke eintragen
|
|
||||||
- 2 Lerneinheiten terminieren
|
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
|
||||||
- [[project-productivity-week]]
|
|
||||||
- [[thought-deep-work-uebungen]]
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 9. Schrittweise Migration bestehender freier Notizen
|
## 10. Export & Roundtrip-Verhalten
|
||||||
|
|
||||||
1. **Frontmatter ergänzen** (min. `id`, `title`, `type`, `status`, `created`, `tags`).
|
- `export_markdown.py` erzeugt identische Markdown-Dateien
|
||||||
2. **Abschnittsstruktur** an Typ anpassen (Konventionen s. Abschnitt 6).
|
- Schreibt YAML + Body in korrekter Reihenfolge
|
||||||
3. **Wikilinks** im Body setzen (statt bloßer Erwähnungen).
|
- Roundtrip-Prüfung (`compare_vaults.py`) validiert:
|
||||||
4. **Tags normalisieren** (Präfixe, Kleinschreibung, keine Dubletten).
|
- vollständigen Body
|
||||||
5. **Aliases** angeben, falls Notiz zuvor unter anderem Namen existierte.
|
- unveränderte Hash-Signatur
|
||||||
6. **Testlauf**: Import (Dry-Run), danach Apply, Audit/Validate prüfen.
|
- gleiche Chunkzahl
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 10. Dataview-Beispiele (direkt nutzbar)
|
## 11. Graph-Schicht und WP-04-Integration
|
||||||
|
|
||||||
**Aktive Gedanken der letzten 30 Tage**
|
- Qdrant fungiert als Graph-Backend
|
||||||
|
- `graph/service.py` (WP-04): stellt API für Mehrhop-Abfragen bereit
|
||||||
TABLE title, created, updated, area
|
- Unterstützt:
|
||||||
FROM "10_thoughts"
|
- `expand(note_id, hops=2)`
|
||||||
WHERE type = "thought" AND status = "active" AND created >= date(today) - dur(30 days)
|
- `resolve_unresolved()`
|
||||||
SORT updated DESC
|
- `neighbors(kind=…)`
|
||||||
|
- Grundlage für spätere **Retriever-Funktionen** (LLM / Agenten)
|
||||||
**Offene Tasks mit Fälligkeit**
|
|
||||||
|
|
||||||
TABLE title, due, effort_min, project, area
|
|
||||||
FROM "30_tasks"
|
|
||||||
WHERE type = "task" AND status = "active" AND due
|
|
||||||
SORT due ASC
|
|
||||||
|
|
||||||
**Konzepte je Topic**
|
|
||||||
|
|
||||||
TABLE title, area, tags
|
|
||||||
FROM "20_concepts"
|
|
||||||
WHERE contains(tags, "type/concept")
|
|
||||||
GROUP BY choice(regexreplace(join(filter(tags, (t) => startswith(t, "topic/")), ", "), "^topic/", ""))
|
|
||||||
|
|
||||||
**Verknüpfungskandidaten (redaktionell, nicht automatisch)**
|
|
||||||
*(Liste am Ende der jeweiligen Notiz in „Mögliche Verbindungen“)*
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 11. Validierung & Pflege
|
## 12. Agenten- und LLM-Integration
|
||||||
|
|
||||||
- **Linter/Validator** prüfen Pflichtfelder und Wertebereiche (Enums, ISO-Datum).
|
**Retriever-Pipeline**
|
||||||
- **Resolver** löst `unresolved`-Edges, wenn neue Notizen/Aliases entstehen.
|
1. Suche top-k Chunks (cosine distance)
|
||||||
- **Idempotente Re-Imports**: Hash-Steuerung konfigurierbar:
|
2. Erweiterung um Edges (`references`, `backlinks`)
|
||||||
- `MINDNET_HASH_MODE`: `body` \| `frontmatter` \| `body+frontmatter` (Default: `body`)
|
3. Kontext-Assembling via `window`-Text
|
||||||
- `MINDNET_HASH_NORMALIZE`: `canonical` \| `none` (Default: `canonical`)
|
4. Prompt-Generierung
|
||||||
- **Edge-Invarianten** (Audit):
|
|
||||||
- `belongs_to == #Chunks`
|
**Agent-Use-Cases**
|
||||||
- `next == prev == Σ(max(chunks_in_note−1, 0))`
|
- Automatisches Tagging (`type`, `status`)
|
||||||
- `references (chunk) == Σ(Chunk-Wikilinks)`
|
- Vorschläge für neue Edges
|
||||||
- `backlink == Σ(eindeutige Wikilinks je Note)`
|
- Auflösen von `unresolved`-Referenzen
|
||||||
- **Roundtrip-Test**: Import → Export → Diff muss Body verlustfrei rekonstruieren.
|
- Auto-Import externer Quellen (MediaWiki, PDFs, Webseiten)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 12. Qdrant-Ablage & Indizes
|
## 13. Erweiterbarkeit
|
||||||
|
|
||||||
**Collections**
|
- Neue Edge-Typen: `inspired_by`, `relates_to`, `contradicts`
|
||||||
- `{prefix}_notes` – Vektor optional (Nullvektor oder Zentroid), Payload inkl. Frontmatter & `fulltext`
|
- Neue Note-Typen: `decision`, `resource`, `recommendation`
|
||||||
- `{prefix}_chunks` – 384d Embedding je Chunk, Payload inkl. `text`, `chunk_index`, `note_id`
|
- Embedding-Engines austauschbar (MiniLM → LaBSE → E5)
|
||||||
- `{prefix}_edges` – 1d Dummy-Vektor, Payload im neuen Schema (s. 5.1)
|
- Versionierte Chunking-Strategien (`chunk_config.py`)
|
||||||
|
|
||||||
**IDs & Idempotenz**
|
|
||||||
- Deterministisch/UUIDv5 über stabile Schlüssel (`note_id`, `chunk_id = note_id#cNN`, `edge_id` aus `(kind, source_id, target_id, scope)`)
|
|
||||||
- Re-Imports erzeugen keine Duplikate (Upsert)
|
|
||||||
|
|
||||||
**Payload-Indizes (Performance)**
|
|
||||||
|
|
||||||
notes : note_id (KEYWORD)
|
|
||||||
chunks : note_id (KEYWORD), chunk_index (INTEGER)
|
|
||||||
edges : kind (KEYWORD), scope (KEYWORD), source_id (KEYWORD), target_id (KEYWORD), note_id (KEYWORD)
|
|
||||||
|
|
||||||
**Delete/Purge**
|
|
||||||
- Selektives Löschen in Edges/Chunks per `note_id`-Filter (schnell, sicher).
|
|
||||||
- `--sync-deletes` (geplant): Qdrant→Vault Diffs gezielt entfernen (mit Dry-Run).
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 13. LLM/RAG & Agenten – Arbeitsweise (Überblick)
|
## 14. Qualitätssicherung und Best Practices
|
||||||
|
|
||||||
**Retriever (Hybrid)**
|
- Validierung vor Upsert
|
||||||
1) **kNN (Chunks)** in Qdrant → Top-k Kandidaten (Semantik).
|
- Keine Leer-Chunks
|
||||||
2) **Nachbarschafts-Expand** (1 Hop) über Edges (`belongs_to`, `references`, `backlink`) → zusätzliche Kontexte.
|
- Vollständige Metadaten in Notes
|
||||||
3) **Rerank** (optional BM25/LLM-score).
|
- Tests in `tests/` automatisch ausführbar (`run_e2e_roundtrip.sh`)
|
||||||
4) **Prompt** mit Zitaten/Belegen (Verweis auf `path` + Abschnittstitel).
|
- Ergebnis: „Roundtrip OK“ und `verify_chunks_integrity` = OK
|
||||||
|
|
||||||
**Agentisches Einordnen (neue Inputs)**
|
|
||||||
- **Chunking + Embedding** → **kNN-Kandidaten**
|
|
||||||
- Klassifikation (`type`, `area`, `project`) via Heuristik/LLM
|
|
||||||
- Edge-Vorschläge: `references` (Semantik + geteilte Tags/Area), `belongs_to` (beste Einordnung)
|
|
||||||
- Falls kein Ziel existiert → `unresolved` erzeugen oder **Stub-Note** anlegen
|
|
||||||
- **Human-in-the-loop**: Vorschläge bestätigen, dann Commit via Importer
|
|
||||||
|
|
||||||
**Graph-Layer (optional, empfohlen)**
|
|
||||||
- Zusätzliche SQLite/Postgres-Tabellen `nodes/edges/aliases` für 2–3-Hop-Queries, Pfadsuche, Zählungen.
|
|
||||||
- Importer pflegt Qdrant **und** SQL (UPSERT).
|
|
||||||
- Graph-Features (Degree, Communities, optional PageRank) regelmäßig berechnen und als Felder in `notes.payload` speichern.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 14. Versions- & Änderungsrichtlinien
|
## 15. Änderungsverlauf
|
||||||
|
|
||||||
- **Kleine Änderungen** direkt in der bestehenden Notiz (Frontmatter `updated` anpassen).
|
**v1.4.0 (2025-10-06)**
|
||||||
- **Große Strukturänderungen** (z. B. neuer Typ) als neue Notiz mit Verweis; alte Note verlinken/archivieren (`status: archived`).
|
- Neu: `window` vs `text` Feld in `mindnet_chunks`
|
||||||
- **IDs nie recyceln** (Konstanz für Referenzen und Edges).
|
- Neu: Hash-Modi (`body`, `frontmatter`, `full`) mit Env-Steuerung
|
||||||
- **Aliases** nutzen, wenn sich Titel/Slug ändert (Resolver kann beides auflösen).
|
- Neu: Baseline-Modus für Vergleich
|
||||||
- **Commit-Meldungen** im Git prägnant strukturieren:
|
- Parser → fehlertolerant (UTF-8 replace)
|
||||||
- `feat(note): …` · `fix(frontmatter): …` · `chore(tags): …` · `refactor(structure): …`
|
- Roundtrip-Test vollständig integriert
|
||||||
|
- Graph-Service und Mehrhop-Abfragen (WP-04-Vorbereitung)
|
||||||
|
- `purge-before-upsert` und `--note-scope-refs` Parameter ergänzt
|
||||||
|
|
||||||
|
**v1.3.0 (2025-09-09)**
|
||||||
|
- Hash-Normalisierung und Edge-Scope
|
||||||
|
- Roundtrip-Tests, Agenten-Integration
|
||||||
|
|
||||||
|
**v1.2.0 (2025-09-02)**
|
||||||
|
- Grundstruktur des Knowledge-Designs
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 15. Typen-Kurzreferenz
|
|
||||||
|
|
||||||
- **concept** — Definitionen, Leitbilder, Prinzipien
|
|
||||||
Empfohlene Abschnitte: *Definition*, *Beispiele*, *Belege*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **thought** — Ideen, Entwürfe, Skizzen
|
|
||||||
Abschnitte: *Zusammenfassung*, *Details*, *Nächste Schritte*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **experience** — Beobachtungen, Learnings
|
|
||||||
Abschnitte: *Kontext*, *Beobachtung*, *Interpretation*, *Implikationen*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **task** — konkrete To-dos
|
|
||||||
Felder: `due`, `effort_min`; Abschnitte: *Checkliste*, *Notizen*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **project** — Ziele, Deliverables, Arbeitspakete
|
|
||||||
Abschnitte: *Scope*, *Arbeitspakete*, *Risiken*, *Status*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **journal** — tagesbasierte Einträge
|
|
||||||
Abschnitte: *Tageszusammenfassung*, *Highlights*, *Learnings*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **person** — Kontakte, Profile
|
|
||||||
Felder: `people` (Selbst-/Fremd-ID), *Notizen*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **meeting** — Termine, Protokolle
|
|
||||||
Abschnitte: *Agenda*, *Notizen*, *Entscheidungen*, *Aufgaben*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **milestone** — Wendepunkte
|
|
||||||
Abschnitte: *Kontext*, *Bedeutung*, *Verknüpfungen*
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 16. Erweiterungen
|
|
||||||
|
|
||||||
- Neue Typen (z. B. `decision`, `recommendation`, `resource`) können ohne Bruch ergänzt werden.
|
|
||||||
- Personalisierung durch zusätzliche Properties (`values`, `goals`) in Frontmatter.
|
|
||||||
- Internationale Inhalte: `lang` pflegen, optional **mehrsprachige** Embeddings (z. B. mE5).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 17. Anhang – Beispiele & Snippets
|
|
||||||
|
|
||||||
**Beispiel: YAML-Frontmatter für Projekt**
|
|
||||||
|
|
||||||
title: "Projekt: Produktivität Woche 37"
|
|
||||||
id: project-productivity-week
|
|
||||||
type: project
|
|
||||||
status: active
|
|
||||||
created: 2025-09-08
|
|
||||||
updated: 2025-09-09
|
|
||||||
area: organisation
|
|
||||||
tags: [area/organisation, type/project, topic/planung]
|
|
||||||
people: [ich]
|
|
||||||
depends_on: [concept-deep-work]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
**Beispiel: Redaktions-Sektion (keine Kanten-Erzeugung)**
|
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
|
||||||
- [[concept-wochenplanung]]
|
|
||||||
- [[thought-deep-work-uebungen]]
|
|
||||||
- [[journal-2025-09-08]]
|
|
||||||
|
|
||||||
**CLI: Import & Audit**
|
|
||||||
|
|
||||||
# Dry-Run Import
|
|
||||||
python3 -m scripts.import_markdown --vault ./vault
|
|
||||||
|
|
||||||
# Apply mit strenger Hash-Erkennung (jede Änderung zählt)
|
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --hash-normalize none
|
|
||||||
|
|
||||||
# Audit Edges vs. Erwartungen
|
|
||||||
python3 -m scripts.audit_edges_vs_expectations --vault ./vault
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 18. Glossar
|
|
||||||
|
|
||||||
- **Alias**: alternativer Name/Slug, der auf dieselbe Note verweist
|
|
||||||
- **Backlink**: automatisch erzeugte Gegenkante zu `references` (Note-Scope)
|
|
||||||
- **Chunk**: semantisch sinnvoller Textabschnitt einer Note, inkl. Embedding
|
|
||||||
- **Owner-Note**: Note, zu der ein Edge gehört (`note_id` im Edge-Payload)
|
|
||||||
- **Resolver**: Prozess/Script, das `unresolved`-Edges bei späterer Ziel-Existenz heilt
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 19. Verweise
|
|
||||||
|
|
||||||
- `chunking_strategy.md` – Details zu Chunkgrößen, Overlaps, Abschnittsregeln
|
|
||||||
- `schemas/note.schema.json` – Validierung von Frontmatter‐Feldern
|
|
||||||
- Import/Export-Skripte: `scripts/import_markdown.py`, `scripts/export_markdown.py`
|
|
||||||
- Qdrant-Client/Setup: `app/core/qdrant.py`, `app/core/qdrant_points.py`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 20. Änderungsverlauf
|
|
||||||
|
|
||||||
- **1.3.0 (2025-09-09)**
|
|
||||||
- Edge-Politik präzisiert: `references` (Chunk-Scope), `backlink` (Note-Scope), `references:note` optional.
|
|
||||||
- `note_id` als Pflichtfeld im Edge-Payload (Owner).
|
|
||||||
- Qdrant-Payload-Indizes ergänzt (Performanz).
|
|
||||||
- Beispiele & Dataview-Snippets erweitert.
|
|
||||||
|
|
||||||
- **1.2.0 (2025-09-02)**
|
|
||||||
- Erstfassung des Knowledge-Designs.
|
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/debug_qdrant_state.py
|
||||||
|
Zeigt Prefix, Collections und einfache Counts (notes/chunks/edges) + sample note_ids.
|
||||||
|
|
||||||
|
Aufruf:
|
||||||
|
export COLLECTION_PREFIX="mindnet"
|
||||||
|
python3 -m scripts.debug_qdrant_state
|
||||||
|
# oder:
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix mindnet
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, os, json
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def count_points(client, collection: str) -> int:
|
||||||
|
try:
|
||||||
|
res = client.count(collection, exact=True)
|
||||||
|
return int(getattr(res, "count", 0))
|
||||||
|
except Exception:
|
||||||
|
pts, _ = client.scroll(collection, limit=1)
|
||||||
|
return 1 if pts else 0
|
||||||
|
|
||||||
|
def sample_ids(client, collection: str, id_key: str, limit: int = 5):
|
||||||
|
pts, _ = client.scroll(collection, with_payload=True, with_vectors=False, limit=limit)
|
||||||
|
out = []
|
||||||
|
for p in pts or []:
|
||||||
|
pl = p.payload or {}
|
||||||
|
if id_key in pl:
|
||||||
|
out.append(pl[id_key])
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX", "mindnet"))
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
cfg.prefix = args.prefix
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
notes = f"{cfg.prefix}_notes"
|
||||||
|
chunks = f"{cfg.prefix}_chunks"
|
||||||
|
edges = f"{cfg.prefix}_edges"
|
||||||
|
|
||||||
|
print(json.dumps({
|
||||||
|
"prefix": cfg.prefix,
|
||||||
|
"collections": {"notes": notes, "chunks": chunks, "edges": edges},
|
||||||
|
"counts": {
|
||||||
|
"notes": count_points(client, notes),
|
||||||
|
"chunks": count_points(client, chunks),
|
||||||
|
"edges": count_points(client, edges),
|
||||||
|
},
|
||||||
|
"samples": {
|
||||||
|
"notes.note_id": sample_ids(client, notes, "note_id"),
|
||||||
|
"chunks.note_id": sample_ids(client, chunks, "note_id"),
|
||||||
|
"edges.note_id": sample_ids(client, edges, "note_id"),
|
||||||
|
}
|
||||||
|
}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+165
-188
@@ -1,249 +1,226 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Script: export_markdown.py — Qdrant → Markdown (Obsidian-kompatibel)
|
Script: scripts/export_markdown.py — Qdrant → Markdown (Vault)
|
||||||
Version: 1.4.0
|
Version: 1.4.1
|
||||||
Datum: 2025-09-09
|
Datum: 2025-09-10
|
||||||
|
|
||||||
Kurzbeschreibung
|
Funktion
|
||||||
----------------
|
--------
|
||||||
Exportiert Markdown-Notizen aus Qdrant in einen Zielordner (Vault).
|
Exportiert Notes (Frontmatter + Body) aus Qdrant in einen Zielordner. Der Body wird
|
||||||
Rekonstruiert YAML-Frontmatter und Body.
|
bevorzugt aus dem Feld `fulltext` rekonstruiert; falls leer/nicht vorhanden, aus Chunks
|
||||||
|
(Sortierung: seq → chunk_index → Nummer in chunk_id). Pfade werden **relativ** geschrieben.
|
||||||
|
|
||||||
Body-Rekonstruktions-Priorität (abwärtskompatibel):
|
Optionen
|
||||||
1) notes.payload.fulltext (verlustfrei, wenn beim Import gespeichert)
|
--------
|
||||||
2) ansonsten aus allen zugehörigen Chunks: payload.text → payload.content → payload.raw
|
--out PATH Zielordner (erforderlich)
|
||||||
in stabiler, sequentieller Reihenfolge (seq/chunk_index/ID-Nummer)
|
--prefix TEXT Collection-Prefix (CLI überschreibt ENV COLLECTION_PREFIX)
|
||||||
|
--note-id ID Nur eine Note exportieren
|
||||||
|
--overwrite Existierende Dateien überschreiben
|
||||||
|
--include-edges MODE none|yaml|footer (Default: none)
|
||||||
|
--flatten-paths Alle Dateien flach schreiben; Originalpfad in FM: orig_path
|
||||||
|
|
||||||
Wichtige Fixes
|
ENV
|
||||||
--------------
|
---
|
||||||
- **Pfad-Normalisierung**: erzwingt relative Pfade (führe führende '/' ab, backslashes → slashes),
|
COLLECTION_PREFIX, QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
|
||||||
damit ``--out`` nicht ignoriert wird.
|
|
||||||
- **`--prefix` (optional)**: Überschreibt COLLECTION_PREFIX; ENV bleibt Default (rückwärtskompatibel).
|
|
||||||
|
|
||||||
ENV / Qdrant
|
|
||||||
------------
|
|
||||||
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
|
|
||||||
- QDRANT_API_KEY (optional)
|
|
||||||
- COLLECTION_PREFIX (Default: mindnet)
|
|
||||||
|
|
||||||
Aufruf
|
|
||||||
------
|
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault
|
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-foo
|
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault --prefix mindnet_dev # optional
|
|
||||||
|
|
||||||
Beispiele
|
Beispiele
|
||||||
---------
|
---------
|
||||||
COLLECTION_PREFIX=mindnet QDRANT_URL=http://127.0.0.1:6333 \\
|
export COLLECTION_PREFIX="mindnet"
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
python3 -m scripts.export_markdown --out ./_exportVault
|
||||||
"""
|
|
||||||
|
|
||||||
|
# Nur eine Note, Edges als YAML-Feld 'references'
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --note-id concept-alpha --include-edges yaml
|
||||||
|
|
||||||
|
# Flach schreiben mit Überschreiben
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --flatten-paths --overwrite
|
||||||
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import re
|
import json
|
||||||
from typing import Dict, Iterable, List, Optional, Tuple
|
from typing import Dict, List, Optional, Tuple, Any
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
from qdrant_client import QdrantClient
|
|
||||||
|
|
||||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
from app.core.qdrant import ensure_collections # safety
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
# Utilities
|
|
||||||
# -----------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
|
||||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def _norm_rel_path(path: str) -> str:
|
||||||
|
p = (path or "").replace("\\", "/").lstrip("/")
|
||||||
|
return p if p else ""
|
||||||
|
|
||||||
def _ensure_dir(path: str) -> None:
|
def _ensure_dir(path: str) -> None:
|
||||||
d = os.path.dirname(path)
|
d = os.path.dirname(path)
|
||||||
if d and not os.path.isdir(d):
|
if d and not os.path.exists(d):
|
||||||
os.makedirs(d, exist_ok=True)
|
os.makedirs(d, exist_ok=True)
|
||||||
|
|
||||||
|
def _yaml_dump(data: Dict[str, Any]) -> str:
|
||||||
|
return yaml.safe_dump(data, allow_unicode=True, sort_keys=False).strip()
|
||||||
|
|
||||||
def _normalize_rel_path(p: str) -> str:
|
def _frontmatter_block(fm: Dict[str, Any]) -> str:
|
||||||
"""Pfad relativ halten & normalisieren (slashes, führende / entfernen)."""
|
y = _yaml_dump(fm)
|
||||||
p = (p or "").replace("\\", "/")
|
return f"---\n{y}\n---\n"
|
||||||
return p.lstrip("/")
|
|
||||||
|
|
||||||
|
def _scroll_all(client, collection: str, flt: Optional[rest.Filter] = None) -> List[Any]:
|
||||||
def _to_md(frontmatter: dict, body: str) -> str:
|
|
||||||
fm = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=True).strip()
|
|
||||||
return f"---\n{fm}\n---\n{(body or '').rstrip()}\n"
|
|
||||||
|
|
||||||
|
|
||||||
def _scroll_all(
|
|
||||||
client: QdrantClient,
|
|
||||||
col: str,
|
|
||||||
flt: Optional[rest.Filter] = None,
|
|
||||||
with_payload: bool = True,
|
|
||||||
with_vectors: bool = False,
|
|
||||||
limit: int = 256,
|
|
||||||
):
|
|
||||||
"""Scrollt durch alle Punkte einer Collection und liefert eine Liste mit Points."""
|
|
||||||
out = []
|
out = []
|
||||||
next_page = None
|
nextp = None
|
||||||
while True:
|
while True:
|
||||||
pts, next_page = client.scroll(
|
pts, nextp = client.scroll(
|
||||||
collection_name=col,
|
collection_name=collection,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
scroll_filter=flt,
|
scroll_filter=flt,
|
||||||
with_payload=with_payload,
|
offset=nextp,
|
||||||
with_vectors=with_vectors,
|
|
||||||
limit=limit,
|
|
||||||
offset=next_page,
|
|
||||||
)
|
)
|
||||||
if not pts:
|
if not pts:
|
||||||
break
|
break
|
||||||
out.extend(pts)
|
out.extend(pts)
|
||||||
if not next_page:
|
if nextp is None:
|
||||||
break
|
break
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
def _reconstruct_body_from_chunks(chunks: List[Any]) -> str:
|
||||||
|
def seq_key(pl: Dict[str, Any]) -> Tuple[int, int, int]:
|
||||||
|
s = pl.get("seq")
|
||||||
|
ci = pl.get("chunk_index")
|
||||||
|
cid = pl.get("chunk_id") or ""
|
||||||
|
n = 0
|
||||||
|
if isinstance(cid, str) and "#" in cid:
|
||||||
|
try:
|
||||||
|
n = int(cid.rsplit("#", 1)[-1])
|
||||||
|
except Exception:
|
||||||
|
n = 0
|
||||||
|
return (int(s) if isinstance(s, int) else 0,
|
||||||
|
int(ci) if isinstance(ci, int) else 0,
|
||||||
|
n)
|
||||||
|
chunks_sorted = sorted(chunks, key=lambda p: seq_key(p.payload or {}))
|
||||||
|
texts: List[str] = []
|
||||||
|
for p in chunks_sorted:
|
||||||
|
pl = p.payload or {}
|
||||||
|
t = pl.get("text") or pl.get("content") or pl.get("raw") or ""
|
||||||
|
if isinstance(t, str) and t:
|
||||||
|
texts.append(t)
|
||||||
|
return "\n".join(texts).strip()
|
||||||
|
|
||||||
def _load_chunks_for_note(client: QdrantClient, chunks_col: str, note_id: str) -> List[dict]:
|
def _collect_forward_refs_from_edges(edges: List[Any]) -> List[str]:
|
||||||
flt = rest.Filter(must=[rest.FieldCondition(
|
refs = []
|
||||||
key="note_id",
|
for p in edges:
|
||||||
match=rest.MatchValue(value=note_id),
|
pl = p.payload or {}
|
||||||
)])
|
if pl.get("kind") == "references" and isinstance(pl.get("target_id"), str):
|
||||||
pts = _scroll_all(client, chunks_col, flt, with_payload=True, with_vectors=False)
|
refs.append(pl["target_id"])
|
||||||
# Sortierung: bevorzugt seq → chunk_index → Nummer in id
|
# de-dupe, preserve order
|
||||||
def _seq(pl: dict) -> Tuple[int, int, int]:
|
seen = set()
|
||||||
s1 = pl.get("seq", pl.get("chunk_index", -1))
|
out = []
|
||||||
s2 = pl.get("chunk_index", -1)
|
for r in refs:
|
||||||
# Nummer-Anteil aus "noteid#<n>"
|
if r not in seen:
|
||||||
s3 = 0
|
seen.add(r)
|
||||||
try:
|
out.append(r)
|
||||||
m = re.search(r"#(\\d+)$", pl.get("id") or "")
|
return out
|
||||||
if m:
|
|
||||||
s3 = int(m.group(1))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return (int(s1) if isinstance(s1, int) else -1, int(s2) if isinstance(s2, int) else -1, s3)
|
|
||||||
|
|
||||||
pts_sorted = sorted(pts, key=lambda p: _seq(p.payload or {}))
|
# ---------------------------------------------------------------------
|
||||||
return [p.payload or {} for p in pts_sorted]
|
|
||||||
|
|
||||||
|
|
||||||
def _reconstruct_body(note_pl: dict, chunk_payloads: List[dict]) -> str:
|
|
||||||
# 1) Volltext vorhanden?
|
|
||||||
fulltext = note_pl.get("fulltext")
|
|
||||||
if isinstance(fulltext, str) and fulltext.strip():
|
|
||||||
return fulltext
|
|
||||||
|
|
||||||
# 2) Aus Chunks zusammensetzen: text → content → raw
|
|
||||||
parts: List[str] = []
|
|
||||||
for ch in chunk_payloads:
|
|
||||||
text = ch.get("text") or ch.get("content") or ch.get("raw")
|
|
||||||
if isinstance(text, str) and text.strip():
|
|
||||||
parts.append(text.rstrip())
|
|
||||||
return ("\n\n".join(parts)).rstrip() + ("\n" if parts else "")
|
|
||||||
|
|
||||||
|
|
||||||
def _export_one_note(
|
|
||||||
client: QdrantClient,
|
|
||||||
prefix: str,
|
|
||||||
note_pl: dict,
|
|
||||||
out_root: str,
|
|
||||||
overwrite: bool,
|
|
||||||
) -> dict:
|
|
||||||
notes_col, chunks_col, _ = _names(prefix)
|
|
||||||
note_id = note_pl.get("note_id") or note_pl.get("id")
|
|
||||||
|
|
||||||
# Pfad robust bestimmen und relativ halten
|
|
||||||
path = note_pl.get("path") or f"{note_id}.md"
|
|
||||||
path = _normalize_rel_path(path)
|
|
||||||
out_path = os.path.join(out_root, path).replace("\\", "/")
|
|
||||||
|
|
||||||
# Frontmatter aus Payload zurückführen (nur bekannte Felder)
|
|
||||||
fm: Dict[str, object] = {}
|
|
||||||
for k in [
|
|
||||||
"title", "id", "type", "status", "created", "updated", "tags",
|
|
||||||
"priority", "effort_min", "due", "people", "aliases",
|
|
||||||
"depends_on", "assigned_to", "lang",
|
|
||||||
]:
|
|
||||||
v = note_pl.get(k) if k in note_pl else note_pl.get(f"note_{k}")
|
|
||||||
if v not in (None, [], ""):
|
|
||||||
fm[k] = v
|
|
||||||
|
|
||||||
# Mindestfelder
|
|
||||||
if "id" not in fm and note_id:
|
|
||||||
fm["id"] = note_id
|
|
||||||
if "title" not in fm and note_pl.get("title"):
|
|
||||||
fm["title"] = note_pl["title"]
|
|
||||||
|
|
||||||
# Body beschaffen
|
|
||||||
chunks = _load_chunks_for_note(client, chunks_col, note_id)
|
|
||||||
body = _reconstruct_body(note_pl, chunks)
|
|
||||||
|
|
||||||
# Schreiben?
|
|
||||||
if os.path.exists(out_path) and not overwrite:
|
|
||||||
return {"note_id": note_id, "path": path, "status": "skip_exists"}
|
|
||||||
|
|
||||||
_ensure_dir(out_path)
|
|
||||||
with open(out_path, "w", encoding="utf-8") as f:
|
|
||||||
f.write(_to_md(fm, body))
|
|
||||||
|
|
||||||
return {"note_id": note_id, "path": path, "status": "written"}
|
|
||||||
|
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
|
||||||
# Main
|
# Main
|
||||||
# -----------------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
ap.add_argument("--out", required=True, help="Zielordner für den Export-Vault")
|
ap.add_argument("--out", required=True, help="Zielordner für exportierte Markdown-Dateien")
|
||||||
ap.add_argument("--note-id", help="Nur eine Note exportieren (Note-ID)")
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||||
ap.add_argument("--overwrite", action="store_true", help="Bestehende Dateien überschreiben")
|
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID exportieren")
|
||||||
ap.add_argument("--prefix", help="(Optional) überschreibt COLLECTION_PREFIX aus ENV")
|
ap.add_argument("--overwrite", action="store_true", help="Existierende Dateien überschreiben")
|
||||||
|
ap.add_argument("--include-edges", choices=["none", "yaml", "footer"], default="none",
|
||||||
|
help="Forward-Links mit exportieren (aus Edges oder Note-Payload)")
|
||||||
|
ap.add_argument("--flatten-paths", action="store_true", help="Alle Dateien flach schreiben (orig_path in Frontmatter)")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
# Qdrant-Konfiguration
|
|
||||||
cfg = QdrantConfig.from_env()
|
cfg = QdrantConfig.from_env()
|
||||||
if args.prefix:
|
if args.prefix:
|
||||||
cfg.prefix = args.prefix # abwärtskompatibel: ENV bleibt Default
|
cfg.prefix = args.prefix.strip()
|
||||||
|
|
||||||
client = get_client(cfg)
|
client = get_client(cfg)
|
||||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||||
|
|
||||||
notes_col, _, _ = _names(cfg.prefix)
|
out_root = os.path.abspath(args.out)
|
||||||
|
os.makedirs(out_root, exist_ok=True)
|
||||||
|
|
||||||
# Notes holen (optional gefiltert)
|
notes_col, chunks_col, edges_col = collections(cfg.prefix)
|
||||||
|
|
||||||
|
# Filter nach note-id (optional)
|
||||||
flt = None
|
flt = None
|
||||||
if args.note_id:
|
if args.note_id:
|
||||||
flt = rest.Filter(must=[rest.FieldCondition(
|
flt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=args.note_id))])
|
||||||
key="note_id",
|
|
||||||
match=rest.MatchValue(value=args.note_id),
|
|
||||||
)])
|
|
||||||
|
|
||||||
note_pts = _scroll_all(client, notes_col, flt, with_payload=True, with_vectors=False)
|
notes = _scroll_all(client, notes_col, flt)
|
||||||
if not note_pts:
|
total = 0
|
||||||
print(json.dumps({"exported": 0, "out": args.out, "message": "Keine Notes gefunden."}, ensure_ascii=False))
|
for n in notes:
|
||||||
return
|
pl = n.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
rel_path = _norm_rel_path(pl.get("path") or "")
|
||||||
|
if args.flatten_paths or not rel_path:
|
||||||
|
fname = f"{(nid or 'note')}.md"
|
||||||
|
out_path = os.path.join(out_root, fname)
|
||||||
|
else:
|
||||||
|
out_path = os.path.join(out_root, rel_path)
|
||||||
|
out_path = out_path.replace("\\", "/")
|
||||||
|
_ensure_dir(out_path)
|
||||||
|
|
||||||
results = []
|
# Frontmatter aufbauen (nur sinnvolle Felder)
|
||||||
for p in note_pts:
|
fm_fields = ["id","title","type","status","created","updated","tags","area","project","source","lang","slug","aliases"]
|
||||||
pl = p.payload or {}
|
fm: Dict[str, Any] = {}
|
||||||
try:
|
fm["id"] = nid
|
||||||
res = _export_one_note(client, cfg.prefix, pl, args.out, args.overwrite)
|
for k in fm_fields:
|
||||||
except Exception as e:
|
if k == "id":
|
||||||
res = {"note_id": pl.get("note_id") or pl.get("id"), "error": str(e)}
|
continue
|
||||||
results.append(res)
|
if k in pl and pl[k] is not None:
|
||||||
|
fm[k] = pl[k]
|
||||||
|
if args.flatten_paths and rel_path:
|
||||||
|
fm["orig_path"] = rel_path
|
||||||
|
|
||||||
print(json.dumps({
|
# Body ermitteln (fulltext oder Chunks)
|
||||||
"exported": len([r for r in results if r.get("status") == "written"]),
|
body = (pl.get("fulltext") or "").strip()
|
||||||
"skipped": len([r for r in results if r.get("status") == "skip_exists"]),
|
if not body:
|
||||||
"out": args.out,
|
flt_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||||
"details": results,
|
chunks = _scroll_all(client, chunks_col, flt_chunks)
|
||||||
}, ensure_ascii=False))
|
body = _reconstruct_body_from_chunks(chunks)
|
||||||
|
|
||||||
|
# Edges (optional)
|
||||||
|
refs: List[str] = []
|
||||||
|
if args.include_edges != "none":
|
||||||
|
# aus Note-Payload, falls vorhanden
|
||||||
|
if isinstance(pl.get("references"), list) and pl["references"]:
|
||||||
|
refs = [r for r in pl["references"] if isinstance(r, str)]
|
||||||
|
else:
|
||||||
|
flt_edges = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||||
|
edges = _scroll_all(client, edges_col, flt_edges)
|
||||||
|
refs = _collect_forward_refs_from_edges(edges)
|
||||||
|
if args.include_edges == "yaml" and refs:
|
||||||
|
fm["references"] = refs
|
||||||
|
|
||||||
|
# Datei schreiben
|
||||||
|
if (not args.overwrite) and os.path.exists(out_path):
|
||||||
|
print(json.dumps({"note_id": nid, "path": out_path, "decision": "skip-exists"}))
|
||||||
|
continue
|
||||||
|
|
||||||
|
content = _frontmatter_block(fm) + (body + "\n" if body else "")
|
||||||
|
if args.include_edges == "footer" and refs:
|
||||||
|
content += "\n---\nLinks:\n" + "\n".join(f"- [[{r}]]" for r in refs) + "\n"
|
||||||
|
|
||||||
|
with open(out_path, "w", encoding="utf-8") as f:
|
||||||
|
f.write(content)
|
||||||
|
|
||||||
|
print(json.dumps({"note_id": nid, "path": out_path, "decision": "write"}))
|
||||||
|
total += 1
|
||||||
|
|
||||||
|
print(f"Done. Exported notes: {total}")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
+171
-103
@@ -2,60 +2,67 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
|
Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
|
||||||
Version: 3.6.2
|
Version: 3.7.2
|
||||||
Datum: 2025-09-09
|
Datum: 2025-09-30
|
||||||
|
|
||||||
Kurzbeschreibung
|
Kurzbeschreibung
|
||||||
----------------
|
----------------
|
||||||
- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
|
- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
|
||||||
- Robuste Änderungserkennung: Mehrfach-Hashes werden parallel in der Note gespeichert
|
- Änderungserkennung „Option C“: mehrere Hash-Varianten werden parallel in der Note
|
||||||
(Option C). Vergleich erfolgt **modusgenau** anhand von `hashes[<mode>:<source>:<normalize>]`.
|
gespeichert (Feld `hashes` mit Schlüsseln `<mode>:<source>:<normalize>`). Der Vergleich
|
||||||
Ein Wechsel des Vergleichsmodus führt so **nicht** zu Massenänderungen.
|
nutzt NUR den aktuellen Modus-Key — ein Moduswechsel triggert keine Massenänderungen mehr.
|
||||||
- **Fix (v3.6.2):** Bei **erstem Import** (kein Alt-Payload) wird die Note als **geändert**
|
- „Erstimport-Fix“: Bei leerem Qdrant gilt Create-Fall automatisch als geändert.
|
||||||
behandelt → Create (Notes/Chunks/Edges) findet zuverlässig statt.
|
- `--baseline-modes`: fehlende Hash-Varianten still nachtragen (nur Notes upserten).
|
||||||
- Baseline-Modus: Mit `--baseline-modes` werden **fehlende** Hash-Varianten
|
- `--sync-deletes`: gezielte Lösch-Synchronisation (Dry-Run + Apply).
|
||||||
im Feld `hashes` „still“ nachgetragen (Upsert NUR Notes; Legacy-Hashfelder bleiben unangetastet).
|
- `--only-path`: exakt **eine** Datei (Pfad) importieren — nützlich für Diagnosefälle.
|
||||||
|
|
||||||
Konfiguration Hash/Compare
|
Neu in 3.7.1/3.7.2
|
||||||
|
------------------
|
||||||
|
- Chunk-Payloads: `window` (für Embeddings), `text` (überlappungsfrei, verlustfrei rekonstruierbar),
|
||||||
|
`start/end/overlap_*`. Embeddings nutzen `window`.
|
||||||
|
- **3.7.2:** Edges-Fehler führen nicht mehr zum Abbruch der gesamten Note; Note/Chunks werden trotzdem geschrieben.
|
||||||
|
|
||||||
|
Hash/Compare Konfiguration
|
||||||
--------------------------
|
--------------------------
|
||||||
- Vergleichsmodus: `--hash-mode body|frontmatter|full`
|
- Vergleichsmodus:
|
||||||
- oder ENV: `MINDNET_HASH_MODE` / `MINDNET_HASH_COMPARE` (Body|Frontmatter|Full)
|
--hash-mode body|frontmatter|full
|
||||||
- Quelle: `--hash-source parsed|raw` (ENV: `MINDNET_HASH_SOURCE`, Default parsed)
|
oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE
|
||||||
- Normalisierung: `--hash-normalize canonical|none` (ENV: `MINDNET_HASH_NORMALIZE`, Default canonical)
|
- Quelle:
|
||||||
- Optional: `--compare-text` (oder `MINDNET_COMPARE_TEXT=true`) vergleicht zusätzlich den parsed Body-Text direkt.
|
--hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE, Default parsed)
|
||||||
|
- Normalisierung:
|
||||||
|
--hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE, Default canonical)
|
||||||
|
- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich
|
||||||
|
den parsed Body-Text direkt.
|
||||||
|
|
||||||
Weitere ENV / Qdrant
|
Qdrant / ENV
|
||||||
--------------------
|
------------
|
||||||
- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
|
- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
|
||||||
- COLLECTION_PREFIX (Default: mindnet)
|
- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
|
||||||
- VECTOR_DIM (Default: 384)
|
- VECTOR_DIM (Default: 384)
|
||||||
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
||||||
|
|
||||||
Aufruf-Beispiele
|
Beispiele
|
||||||
----------------
|
---------
|
||||||
# Standard (Body, parsed, canonical)
|
# Standard (Body, parsed, canonical)
|
||||||
python3 -m scripts.import_markdown --vault ./vault
|
python3 -m scripts.import_markdown --vault ./vault
|
||||||
|
|
||||||
# Baseline (füllt hashes für parsed:canonical Tripel „still“ auf)
|
# Erstimport nach truncate (Create-Fall)
|
||||||
MINDNET_HASH_SOURCE=parsed MINDNET_HASH_NORMALIZE=canonical \
|
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --baseline-modes
|
|
||||||
|
|
||||||
# Frontmatter-Vergleich ohne Massenänderungen (nach Baseline)
|
# Nur eine Datei (Diagnose)
|
||||||
MINDNET_HASH_COMPARE=Frontmatter \
|
python3 -m scripts.import_markdown --vault ./vault --only-path ./vault/30_projects/project-demo.md --apply
|
||||||
python3 -m scripts.import_markdown --vault ./vault
|
|
||||||
|
|
||||||
# Sehr sensibel (raw + none) und direkten Textvergleich
|
# Sync-Deletes (Dry-Run → Apply)
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --hash-source raw --hash-normalize none --compare-text
|
python3 -m scripts.import_markdown --vault ./vault --sync-deletes
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import difflib
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from typing import Dict, List, Optional, Tuple, Any
|
from typing import Dict, List, Optional, Tuple, Any, Set
|
||||||
from collections.abc import Mapping
|
|
||||||
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
@@ -68,7 +75,10 @@ from app.core.parser import (
|
|||||||
from app.core.note_payload import make_note_payload
|
from app.core.note_payload import make_note_payload
|
||||||
from app.core.chunker import assemble_chunks
|
from app.core.chunker import assemble_chunks
|
||||||
from app.core.chunk_payload import make_chunk_payloads
|
from app.core.chunk_payload import make_chunk_payloads
|
||||||
from app.core.edges import build_edges_for_note
|
try:
|
||||||
|
from app.core.derive_edges import build_edges_for_note
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
from app.core.edges import build_edges_for_note # type: ignore
|
||||||
from app.core.qdrant import (
|
from app.core.qdrant import (
|
||||||
QdrantConfig,
|
QdrantConfig,
|
||||||
get_client,
|
get_client,
|
||||||
@@ -89,7 +99,7 @@ except Exception:
|
|||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
# Helpers
|
# Helper
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
def iter_md(root: str) -> List[str]:
|
def iter_md(root: str) -> List[str]:
|
||||||
@@ -105,9 +115,11 @@ def iter_md(root: str) -> List[str]:
|
|||||||
out.append(p)
|
out.append(p)
|
||||||
return sorted(out)
|
return sorted(out)
|
||||||
|
|
||||||
|
|
||||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
|
||||||
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
||||||
notes_col, _, _ = collections(prefix)
|
notes_col, _, _ = collections(prefix)
|
||||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
@@ -122,37 +134,58 @@ def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[D
|
|||||||
return None
|
return None
|
||||||
return points[0].payload or {}
|
return points[0].payload or {}
|
||||||
|
|
||||||
|
|
||||||
|
def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
||||||
|
notes_col, _, _ = collections(prefix)
|
||||||
|
out: Set[str] = set()
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=notes_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
for p in pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if isinstance(nid, str):
|
||||||
|
out.add(nid)
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||||
"""
|
|
||||||
Löscht alle Chunks & Edges zu einer Note mittels Filter-Selector (kompatibel mit aktuellen Qdrant-Clients).
|
|
||||||
"""
|
|
||||||
_, chunks_col, edges_col = collections(prefix)
|
_, chunks_col, edges_col = collections(prefix)
|
||||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
for col in (chunks_col, edges_col):
|
||||||
|
try:
|
||||||
|
client.delete(
|
||||||
|
collection_name=col,
|
||||||
|
points_selector=rest.FilterSelector(filter=filt),
|
||||||
|
wait=True
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
|
||||||
|
|
||||||
try:
|
|
||||||
client.delete(
|
|
||||||
collection_name=chunks_col,
|
|
||||||
points_selector=rest.FilterSelector(filter=filt),
|
|
||||||
wait=True
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
print(json.dumps({"note_id": note_id, "warn": f"delete chunks via filter failed: {e}"}))
|
|
||||||
|
|
||||||
try:
|
def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
||||||
client.delete(
|
notes_col, chunks_col, edges_col = collections(prefix)
|
||||||
collection_name=edges_col,
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
points_selector=rest.FilterSelector(filter=filt),
|
for col in (edges_col, chunks_col, notes_col):
|
||||||
wait=True
|
try:
|
||||||
)
|
client.delete(
|
||||||
except Exception as e:
|
collection_name=col,
|
||||||
print(json.dumps({"note_id": note_id, "warn": f"delete edges via filter failed: {e}"}))
|
points_selector=rest.FilterSelector(filter=filt),
|
||||||
|
wait=True
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
|
||||||
|
|
||||||
def _normalize_rel_path(abs_path: str, vault_root: str) -> str:
|
|
||||||
try:
|
|
||||||
rel = os.path.relpath(abs_path, vault_root)
|
|
||||||
except Exception:
|
|
||||||
rel = abs_path
|
|
||||||
return rel.replace("\\", "/").lstrip("/")
|
|
||||||
|
|
||||||
def _resolve_mode(val: Optional[str]) -> str:
|
def _resolve_mode(val: Optional[str]) -> str:
|
||||||
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
|
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
|
||||||
@@ -162,6 +195,7 @@ def _resolve_mode(val: Optional[str]) -> str:
|
|||||||
return "frontmatter"
|
return "frontmatter"
|
||||||
return "body"
|
return "body"
|
||||||
|
|
||||||
|
|
||||||
def _env(key: str, default: str) -> str:
|
def _env(key: str, default: str) -> str:
|
||||||
return (os.environ.get(key) or default).strip().lower()
|
return (os.environ.get(key) or default).strip().lower()
|
||||||
|
|
||||||
@@ -178,6 +212,7 @@ def main() -> None:
|
|||||||
ap.add_argument("--purge-before-upsert", action="store_true",
|
ap.add_argument("--purge-before-upsert", action="store_true",
|
||||||
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
|
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
|
||||||
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
|
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
|
||||||
|
ap.add_argument("--only-path", help="Exakt diesen Markdown-Pfad verarbeiten (ignoriert --note-id)")
|
||||||
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
|
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
|
||||||
ap.add_argument("--force-replace", action="store_true",
|
ap.add_argument("--force-replace", action="store_true",
|
||||||
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
|
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
|
||||||
@@ -187,13 +222,16 @@ def main() -> None:
|
|||||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
|
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
|
||||||
help="Quelle für die Hash-Berechnung (Default: parsed)")
|
help="Quelle für die Hash-Berechnung (Default: parsed)")
|
||||||
ap.add_argument("--note-scope-refs", action="store_true",
|
ap.add_argument("--note-scope-refs", action="store_true",
|
||||||
help="(Optional) erzeugt zusätzlich references:note (Default: aus)")
|
help="(Optional) erzeugt zusätzlich references:note/backlink:note (Default: aus)")
|
||||||
ap.add_argument("--debug-hash-diff", action="store_true",
|
ap.add_argument("--debug-hash-diff", action="store_true",
|
||||||
help="Zeigt einen kurzen Diff zwischen altem und neuem Body")
|
help="(reserviert) optionaler Body-Diff")
|
||||||
ap.add_argument("--compare-text", action="store_true",
|
ap.add_argument("--compare-text", action="store_true",
|
||||||
help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
|
help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
|
||||||
ap.add_argument("--baseline-modes", action="store_true",
|
ap.add_argument("--baseline-modes", action="store_true",
|
||||||
help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
|
help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
|
||||||
|
ap.add_argument("--sync-deletes", action="store_true",
|
||||||
|
help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)")
|
||||||
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
mode = _resolve_mode(args.hash_mode) # body|frontmatter|full
|
mode = _resolve_mode(args.hash_mode) # body|frontmatter|full
|
||||||
@@ -204,16 +242,53 @@ def main() -> None:
|
|||||||
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
||||||
|
|
||||||
cfg = QdrantConfig.from_env()
|
cfg = QdrantConfig.from_env()
|
||||||
|
if args.prefix:
|
||||||
|
cfg.prefix = args.prefix.strip()
|
||||||
client = get_client(cfg)
|
client = get_client(cfg)
|
||||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||||
ensure_payload_indexes(client, cfg.prefix)
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
|
|
||||||
root = os.path.abspath(args.vault)
|
root = os.path.abspath(args.vault)
|
||||||
files = iter_md(root)
|
|
||||||
|
# Dateiliste bestimmen
|
||||||
|
if args.only_path:
|
||||||
|
only = os.path.abspath(args.only_path)
|
||||||
|
files = [only]
|
||||||
|
else:
|
||||||
|
files = iter_md(root)
|
||||||
if not files:
|
if not files:
|
||||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
|
||||||
|
# Optional: Sync-Deletes vorab
|
||||||
|
if args.sync_deletes:
|
||||||
|
vault_note_ids: Set[str] = set()
|
||||||
|
for path in files:
|
||||||
|
try:
|
||||||
|
parsed = read_markdown(path)
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
|
nid = fm.get("id")
|
||||||
|
if isinstance(nid, str):
|
||||||
|
vault_note_ids.add(nid)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
qdrant_note_ids = list_qdrant_note_ids(client, cfg.prefix)
|
||||||
|
to_delete = sorted(qdrant_note_ids - vault_note_ids)
|
||||||
|
print(json.dumps({
|
||||||
|
"action": "sync-deletes",
|
||||||
|
"prefix": cfg.prefix,
|
||||||
|
"qdrant_total": len(qdrant_note_ids),
|
||||||
|
"vault_total": len(vault_note_ids),
|
||||||
|
"to_delete_count": len(to_delete),
|
||||||
|
"to_delete": to_delete[:50] + (["…"] if len(to_delete) > 50 else [])
|
||||||
|
}, ensure_ascii=False))
|
||||||
|
if args.apply and to_delete:
|
||||||
|
for nid in to_delete:
|
||||||
|
print(json.dumps({"action": "delete", "note_id": nid, "decision": "apply"}))
|
||||||
|
delete_note_everywhere(client, cfg.prefix, nid)
|
||||||
|
|
||||||
key_current = f"{mode}:{src}:{norm}"
|
key_current = f"{mode}:{src}:{norm}"
|
||||||
|
|
||||||
processed = 0
|
processed = 0
|
||||||
@@ -222,7 +297,7 @@ def main() -> None:
|
|||||||
try:
|
try:
|
||||||
parsed = read_markdown(path)
|
parsed = read_markdown(path)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "error": f"read_markdown failed: {e}"}))
|
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
||||||
continue
|
continue
|
||||||
if parsed is None:
|
if parsed is None:
|
||||||
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
|
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
|
||||||
@@ -232,10 +307,10 @@ def main() -> None:
|
|||||||
fm = normalize_frontmatter(parsed.frontmatter)
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
validate_required_frontmatter(fm)
|
validate_required_frontmatter(fm)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"}))
|
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {type(e).__name__}: {e}"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if args.note_id and fm.get("id") != args.note_id:
|
if args.note_id and not args.only_path and fm.get("id") != args.note_id:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
processed += 1
|
processed += 1
|
||||||
@@ -263,16 +338,9 @@ def main() -> None:
|
|||||||
|
|
||||||
old_hashes = (old_payload or {}).get("hashes") or {}
|
old_hashes = (old_payload or {}).get("hashes") or {}
|
||||||
old_hash_exact = old_hashes.get(key_current)
|
old_hash_exact = old_hashes.get(key_current)
|
||||||
|
|
||||||
# Neu-Hash (aktueller Modus) aus neuem Payload
|
|
||||||
new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
|
new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
|
||||||
|
|
||||||
needs_baseline = (old_hash_exact is None)
|
needs_baseline = (old_hash_exact is None)
|
||||||
|
|
||||||
# Change-Detection:
|
|
||||||
# - CREATE: wenn es KEIN Alt-Payload gibt -> changed=True
|
|
||||||
# - UPDATE: baseline existiert UND Hash differiert
|
|
||||||
# - force/text_changed wie gehabt
|
|
||||||
hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact)
|
hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact)
|
||||||
|
|
||||||
text_changed = False
|
text_changed = False
|
||||||
@@ -282,44 +350,43 @@ def main() -> None:
|
|||||||
text_changed = (old_text != new_text)
|
text_changed = (old_text != new_text)
|
||||||
|
|
||||||
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
||||||
|
|
||||||
# Baseline-only nur, wenn Alt-Payload existiert UND Key fehlt UND keine sonstige Änderung
|
|
||||||
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
||||||
|
|
||||||
# -------- Optional: Chunks / Embeddings / Edges vorbereiten --------
|
# -------- Chunks / Embeddings --------
|
||||||
# CREATE/UPDATE: wir brauchen Chunks/Edges; Baseline-only: nein
|
chunk_pls: List[Dict[str, Any]] = []
|
||||||
chunks = []
|
try:
|
||||||
chunk_pls = []
|
body_text = getattr(parsed, "body", "") or ""
|
||||||
edges = []
|
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
|
||||||
vecs = []
|
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||||
if (args.apply and (changed and (not do_baseline_only))):
|
except Exception as e:
|
||||||
|
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
|
||||||
|
continue
|
||||||
|
|
||||||
|
vecs: List[List[float]] = [[0.0] * cfg.dim for _ in chunk_pls]
|
||||||
|
if embed_texts and chunk_pls:
|
||||||
try:
|
try:
|
||||||
chunks = assemble_chunks(fm["id"], getattr(parsed, "body", "") or "", fm.get("type", "concept"))
|
texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls]
|
||||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
|
vecs = embed_texts(texts_for_embed)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
||||||
continue
|
|
||||||
|
|
||||||
if embed_texts:
|
|
||||||
try:
|
|
||||||
vecs = embed_texts([getattr(c, "text", "") for c in chunks]) # type: ignore[attr-defined]
|
|
||||||
except Exception as e:
|
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
|
||||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
|
||||||
else:
|
|
||||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
|
||||||
|
|
||||||
|
# -------- Edges (robust) --------
|
||||||
|
edges: List[Dict[str, Any]] = []
|
||||||
|
edges_failed = False
|
||||||
|
if changed and (not do_baseline_only):
|
||||||
try:
|
try:
|
||||||
note_refs = note_pl.get("references") or []
|
note_refs = note_pl.get("references") or []
|
||||||
edges = build_edges_for_note(
|
edges = build_edges_for_note(
|
||||||
note_id,
|
note_id,
|
||||||
chunk_pls,
|
chunk_pls,
|
||||||
note_refs,
|
note_level_references=note_refs,
|
||||||
include_note_scope_refs=note_scope_refs,
|
include_note_scope_refs=note_scope_refs,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"build_edges_for_note failed: {e}"}))
|
edges_failed = True
|
||||||
continue
|
edges = []
|
||||||
|
# WICHTIG: Wir brechen NICHT mehr ab — Note & Chunks werden geschrieben.
|
||||||
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
|
||||||
|
|
||||||
# -------- Summary --------
|
# -------- Summary --------
|
||||||
summary = {
|
summary = {
|
||||||
@@ -327,6 +394,7 @@ def main() -> None:
|
|||||||
"title": fm.get("title"),
|
"title": fm.get("title"),
|
||||||
"chunks": len(chunk_pls),
|
"chunks": len(chunk_pls),
|
||||||
"edges": len(edges),
|
"edges": len(edges),
|
||||||
|
"edges_failed": edges_failed,
|
||||||
"changed": changed,
|
"changed": changed,
|
||||||
"needs_baseline_for_mode": needs_baseline,
|
"needs_baseline_for_mode": needs_baseline,
|
||||||
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
||||||
@@ -337,6 +405,7 @@ def main() -> None:
|
|||||||
"hash_mode": mode,
|
"hash_mode": mode,
|
||||||
"hash_normalize": norm,
|
"hash_normalize": norm,
|
||||||
"hash_source": src,
|
"hash_source": src,
|
||||||
|
"prefix": cfg.prefix,
|
||||||
}
|
}
|
||||||
print(json.dumps(summary, ensure_ascii=False))
|
print(json.dumps(summary, ensure_ascii=False))
|
||||||
|
|
||||||
@@ -344,12 +413,10 @@ def main() -> None:
|
|||||||
if not args.apply:
|
if not args.apply:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# BASELINE-ONLY: fehlenden Key nachtragen, ohne legacy Felder anzutasten
|
|
||||||
if do_baseline_only:
|
if do_baseline_only:
|
||||||
merged_hashes = {}
|
merged_hashes = {}
|
||||||
merged_hashes.update(old_hashes)
|
merged_hashes.update(old_hashes)
|
||||||
merged_hashes.update(note_pl.get("hashes") or {})
|
merged_hashes.update(note_pl.get("hashes") or {})
|
||||||
# Legacy-Hashfelder unverändert lassen, falls vorhanden
|
|
||||||
if old_payload:
|
if old_payload:
|
||||||
note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext"))
|
note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext"))
|
||||||
note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature"))
|
note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature"))
|
||||||
@@ -358,7 +425,6 @@ def main() -> None:
|
|||||||
upsert_batch(client, notes_name, note_pts)
|
upsert_batch(client, notes_name, note_pts)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Normale CREATE/UPDATE
|
|
||||||
if not changed:
|
if not changed:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -370,10 +436,12 @@ def main() -> None:
|
|||||||
|
|
||||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||||
upsert_batch(client, notes_name, note_pts)
|
upsert_batch(client, notes_name, note_pts)
|
||||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
if chunk_pls:
|
||||||
upsert_batch(client, chunks_name, chunk_pts)
|
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
upsert_batch(client, chunks_name, chunk_pts)
|
||||||
upsert_batch(client, edges_name, edge_pts)
|
if edges:
|
||||||
|
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||||
|
upsert_batch(client, edges_name, edge_pts)
|
||||||
|
|
||||||
print(f"Done. Processed notes: {processed}")
|
print(f"Done. Processed notes: {processed}")
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Script: scripts/report_hashes.py — Übersicht & Lücken bei Mehrfach-Hashes
|
||||||
|
Version: 1.0.0
|
||||||
|
Datum: 2025-09-10
|
||||||
|
|
||||||
|
Funktion
|
||||||
|
--------
|
||||||
|
Listet je Note die vorhandenen Einträge im Feld `hashes` (Signaturen: <mode>:<source>:<normalize>)
|
||||||
|
und meldet fehlende Soll-Keys. Eignet sich als CI-Check.
|
||||||
|
|
||||||
|
Optionen
|
||||||
|
--------
|
||||||
|
--prefix TEXT Collection-Prefix (CLI überschreibt ENV)
|
||||||
|
--require K [K ...] Zusätzliche Soll-Keys (Default: body|frontmatter|full:parsed:canonical)
|
||||||
|
--fail-on-missing Exitcode 2, wenn fehlende Keys gefunden werden
|
||||||
|
|
||||||
|
Beispiele
|
||||||
|
---------
|
||||||
|
python3 -m scripts.report_hashes --prefix mindnet
|
||||||
|
python3 -m scripts.report_hashes --require frontmatter:raw:none --fail-on-missing
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from typing import List, Dict, Any
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def collections(prefix: str):
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def _scroll_all(client, collection: str):
|
||||||
|
out = []
|
||||||
|
nextp = None
|
||||||
|
while True:
|
||||||
|
pts, nextp = client.scroll(collection_name=collection, with_payload=True, with_vectors=False, limit=256, offset=nextp)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
out.extend(pts)
|
||||||
|
if nextp is None:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||||
|
ap.add_argument("--require", nargs="+", help="Zusätzliche Soll-Keys <mode>:<source>:<normalize>")
|
||||||
|
ap.add_argument("--fail-on-missing", action="store_true", help="Exitcode 2 bei fehlenden Keys")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
if args.prefix:
|
||||||
|
cfg.prefix = args.prefix.strip()
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
notes_col, _, _ = collections(cfg.prefix)
|
||||||
|
pts = _scroll_all(client, notes_col)
|
||||||
|
|
||||||
|
required = set(args.require or [])
|
||||||
|
required |= {
|
||||||
|
"body:parsed:canonical",
|
||||||
|
"frontmatter:parsed:canonical",
|
||||||
|
"full:parsed:canonical",
|
||||||
|
}
|
||||||
|
|
||||||
|
missing_total = 0
|
||||||
|
for p in pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
hashes = pl.get("hashes") or {}
|
||||||
|
present = set(hashes.keys())
|
||||||
|
missing = sorted(list(required - present))
|
||||||
|
obj = {
|
||||||
|
"note_id": nid,
|
||||||
|
"present_count": len(present),
|
||||||
|
"missing": missing,
|
||||||
|
}
|
||||||
|
print(json.dumps(obj, ensure_ascii=False))
|
||||||
|
missing_total += len(missing)
|
||||||
|
|
||||||
|
if args.fail_on_missing and missing_total > 0:
|
||||||
|
raise SystemExit(2)
|
||||||
|
|
||||||
|
print(json.dumps({"summary_missing_total": missing_total}))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/audit_edges_now.py — Zählt Edges nach kind & scope und zeigt Chunk- und Note-Anzahl.
|
||||||
|
"""
|
||||||
|
import os, sys, json
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def scroll_all(c, col, flt=None):
|
||||||
|
out, nextp = [], None
|
||||||
|
while True:
|
||||||
|
pts, nextp = c.scroll(collection_name=col, with_payload=True, with_vectors=False, limit=256, scroll_filter=flt, offset=nextp)
|
||||||
|
if not pts: break
|
||||||
|
out.extend(pts)
|
||||||
|
if nextp is None: break
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
c = get_client(cfg)
|
||||||
|
notes = f"{cfg.prefix}_notes"
|
||||||
|
chunks = f"{cfg.prefix}_chunks"
|
||||||
|
edges = f"{cfg.prefix}_edges"
|
||||||
|
|
||||||
|
es = scroll_all(c, edges)
|
||||||
|
counts = {}
|
||||||
|
by_scope = {}
|
||||||
|
for p in es:
|
||||||
|
pl = p.payload or {}
|
||||||
|
k = pl.get("kind") or "?"
|
||||||
|
s = pl.get("scope") or "?"
|
||||||
|
counts[k] = counts.get(k, 0) + 1
|
||||||
|
key = f"{k}:{s}"
|
||||||
|
by_scope[key] = by_scope.get(key, 0) + 1
|
||||||
|
|
||||||
|
print(json.dumps({
|
||||||
|
"collections": {"notes": notes, "chunks": chunks, "edges": edges},
|
||||||
|
"total_edges": len(es),
|
||||||
|
"counts": counts,
|
||||||
|
"counts_by_scope": by_scope
|
||||||
|
}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/check_chunks_window_vs_text.py
|
||||||
|
Prüft pro Note und global:
|
||||||
|
- ob window == text (sollte bei Overlap normalerweise NEIN sein)
|
||||||
|
- ungefähre linke Overlap-Länge (Suffix von prev.text vs. Prefix von cur.window)
|
||||||
|
- Größen & einfache Statistiken
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 tests/check_chunks_window_vs_text.py --prefix mindnet [--vault ./test_vault]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os, sys, json, argparse
|
||||||
|
from typing import List, Dict, Any, Tuple, Optional
|
||||||
|
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def scroll_chunks_by_note(client, chunks_col: str, note_id: str) -> List[Dict[str, Any]]:
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
out = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=chunks_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
scroll_filter=f,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
out.extend([p.payload or {} for p in pts])
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
# sortieren: seq -> chunk_index -> chunk_id
|
||||||
|
def keyer(pl):
|
||||||
|
return (pl.get("seq", 1<<30), pl.get("chunk_index", 1<<30), str(pl.get("chunk_id", "")))
|
||||||
|
return sorted(out, key=keyer)
|
||||||
|
|
||||||
|
def list_note_ids(client, notes_col: str) -> List[str]:
|
||||||
|
out = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=notes_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
for p in pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if isinstance(nid, str):
|
||||||
|
out.append(nid)
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
return sorted(set(out))
|
||||||
|
|
||||||
|
def common_overlap_len(a_suffix: str, b_prefix: str, max_probe: int = 256) -> int:
|
||||||
|
"""
|
||||||
|
Länge des längsten Suffix von a_suffix, das Prefix von b_prefix ist (bruteforce, begrenzt).
|
||||||
|
"""
|
||||||
|
n = min(len(a_suffix), len(b_prefix), max_probe)
|
||||||
|
if n <= 0:
|
||||||
|
return 0
|
||||||
|
a = a_suffix[-n:]
|
||||||
|
b = b_prefix[:n]
|
||||||
|
# steigere den Match von 0..n
|
||||||
|
for k in range(n, 0, -1):
|
||||||
|
if a[-k:] == b[:k]:
|
||||||
|
return k
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", required=True)
|
||||||
|
ap.add_argument("--vault", help="optional, nur für Vergleichsausgaben")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
cfg.prefix = args.prefix
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
notes_col, chunks_col, _ = collections(cfg.prefix)
|
||||||
|
note_ids = list_note_ids(client, notes_col)
|
||||||
|
if not note_ids:
|
||||||
|
print(json.dumps({"error": "no notes found in qdrant for this prefix"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
global_total = 0
|
||||||
|
global_ident = 0
|
||||||
|
global_left_ov_sum = 0
|
||||||
|
detail = []
|
||||||
|
|
||||||
|
for nid in note_ids:
|
||||||
|
chunks = scroll_chunks_by_note(client, chunks_col, nid)
|
||||||
|
if not chunks:
|
||||||
|
detail.append({"note_id": nid, "chunks": 0})
|
||||||
|
continue
|
||||||
|
ident = 0
|
||||||
|
left_ov_sum = 0
|
||||||
|
for i, ch in enumerate(chunks):
|
||||||
|
txt = (ch.get("text") or "").replace("\r\n", "\n")
|
||||||
|
win = (ch.get("window") or "").replace("\r\n", "\n")
|
||||||
|
if txt == win:
|
||||||
|
ident += 1
|
||||||
|
if i > 0:
|
||||||
|
prev_txt = (chunks[i-1].get("text") or "").replace("\r\n", "\n")
|
||||||
|
left_ov = common_overlap_len(prev_txt, win, max_probe=1024)
|
||||||
|
left_ov_sum += left_ov
|
||||||
|
global_total += len(chunks)
|
||||||
|
global_ident += ident
|
||||||
|
global_left_ov_sum += left_ov_sum
|
||||||
|
detail.append({
|
||||||
|
"note_id": nid,
|
||||||
|
"chunks": len(chunks),
|
||||||
|
"identical_text_window": ident,
|
||||||
|
"identical_share": round(ident / max(1, len(chunks)), 3),
|
||||||
|
"avg_left_overlap_est": round(left_ov_sum / max(1, len(chunks)-1), 2) if len(chunks) > 1 else 0.0
|
||||||
|
})
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"notes": len(note_ids),
|
||||||
|
"chunks_total": global_total,
|
||||||
|
"identical_total": global_ident,
|
||||||
|
"identical_share_global": round(global_ident / max(1, global_total), 3),
|
||||||
|
"avg_left_overlap_est_global": round(global_left_ov_sum / max(1, global_total - len(note_ids)), 2) if global_total > len(note_ids) else 0.0,
|
||||||
|
"details": detail[:100] # capped
|
||||||
|
}
|
||||||
|
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/check_chunks_window_vs_text.py
|
||||||
|
Prüft pro Note und global:
|
||||||
|
- ob window == text (sollte bei Overlap normalerweise NEIN sein)
|
||||||
|
- ungefähre linke Overlap-Länge (Suffix von prev.text vs. Prefix von cur.window)
|
||||||
|
- Größen & einfache Statistiken
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 tests/check_chunks_window_vs_text.py --prefix mindnet [--vault ./test_vault]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os, sys, json, argparse
|
||||||
|
from typing import List, Dict, Any, Tuple, Optional
|
||||||
|
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def scroll_chunks_by_note(client, chunks_col: str, note_id: str) -> List[Dict[str, Any]]:
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
out = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=chunks_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
scroll_filter=f,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
out.extend([p.payload or {} for p in pts])
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
# sortieren: seq -> chunk_index -> chunk_id
|
||||||
|
def keyer(pl):
|
||||||
|
return (pl.get("seq", 1<<30), pl.get("chunk_index", 1<<30), str(pl.get("chunk_id", "")))
|
||||||
|
return sorted(out, key=keyer)
|
||||||
|
|
||||||
|
def list_note_ids(client, notes_col: str) -> List[str]:
|
||||||
|
out = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=notes_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
for p in pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if isinstance(nid, str):
|
||||||
|
out.append(nid)
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
return sorted(set(out))
|
||||||
|
|
||||||
|
def common_overlap_len(a_suffix: str, b_prefix: str, max_probe: int = 256) -> int:
|
||||||
|
"""
|
||||||
|
Länge des längsten Suffix von a_suffix, das Prefix von b_prefix ist (bruteforce, begrenzt).
|
||||||
|
"""
|
||||||
|
n = min(len(a_suffix), len(b_prefix), max_probe)
|
||||||
|
if n <= 0:
|
||||||
|
return 0
|
||||||
|
a = a_suffix[-n:]
|
||||||
|
b = b_prefix[:n]
|
||||||
|
# steigere den Match von 0..n
|
||||||
|
for k in range(n, 0, -1):
|
||||||
|
if a[-k:] == b[:k]:
|
||||||
|
return k
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", required=True)
|
||||||
|
ap.add_argument("--vault", help="optional, nur für Vergleichsausgaben")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
cfg.prefix = args.prefix
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
notes_col, chunks_col, _ = collections(cfg.prefix)
|
||||||
|
note_ids = list_note_ids(client, notes_col)
|
||||||
|
if not note_ids:
|
||||||
|
print(json.dumps({"error": "no notes found in qdrant for this prefix"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
global_total = 0
|
||||||
|
global_ident = 0
|
||||||
|
global_left_ov_sum = 0
|
||||||
|
detail = []
|
||||||
|
|
||||||
|
for nid in note_ids:
|
||||||
|
chunks = scroll_chunks_by_note(client, chunks_col, nid)
|
||||||
|
if not chunks:
|
||||||
|
detail.append({"note_id": nid, "chunks": 0})
|
||||||
|
continue
|
||||||
|
ident = 0
|
||||||
|
left_ov_sum = 0
|
||||||
|
for i, ch in enumerate(chunks):
|
||||||
|
txt = (ch.get("text") or "").replace("\r\n", "\n")
|
||||||
|
win = (ch.get("window") or "").replace("\r\n", "\n")
|
||||||
|
if txt == win:
|
||||||
|
ident += 1
|
||||||
|
if i > 0:
|
||||||
|
prev_txt = (chunks[i-1].get("text") or "").replace("\r\n", "\n")
|
||||||
|
left_ov = common_overlap_len(prev_txt, win, max_probe=1024)
|
||||||
|
left_ov_sum += left_ov
|
||||||
|
global_total += len(chunks)
|
||||||
|
global_ident += ident
|
||||||
|
global_left_ov_sum += left_ov_sum
|
||||||
|
detail.append({
|
||||||
|
"note_id": nid,
|
||||||
|
"chunks": len(chunks),
|
||||||
|
"identical_text_window": ident,
|
||||||
|
"identical_share": round(ident / max(1, len(chunks)), 3),
|
||||||
|
"avg_left_overlap_est": round(left_ov_sum / max(1, len(chunks)-1), 2) if len(chunks) > 1 else 0.0
|
||||||
|
})
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"notes": len(note_ids),
|
||||||
|
"chunks_total": global_total,
|
||||||
|
"identical_total": global_ident,
|
||||||
|
"identical_share_global": round(global_ident / max(1, global_total), 3),
|
||||||
|
"avg_left_overlap_est_global": round(global_left_ov_sum / max(1, global_total - len(note_ids)), 2) if global_total > len(note_ids) else 0.0,
|
||||||
|
"details": detail[:100] # capped
|
||||||
|
}
|
||||||
|
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/compare_vaults.py
|
||||||
|
|
||||||
|
Vergleicht zwei Markdown-Vaults rekursiv:
|
||||||
|
- Schlüsselauswahl: id | title | filename | auto (default: auto = id>title>filename)
|
||||||
|
- Fokus: body | frontmatter | all (default: body)
|
||||||
|
- Unicode: NFC-Normalisierung für IDs/Titel/Dateinamen und Text
|
||||||
|
|
||||||
|
Beispiele:
|
||||||
|
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --focus body
|
||||||
|
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --key id --focus all
|
||||||
|
|
||||||
|
Ausgabe:
|
||||||
|
- Einträge mit 'missing' (nur in src oder nur in dst)
|
||||||
|
- Einträge mit 'diff' (vorhanden in beiden, aber Abweichungen gem. Fokus)
|
||||||
|
- Summary-Objekt am Ende
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import argparse
|
||||||
|
import unicodedata
|
||||||
|
from typing import Dict, Tuple, Optional, Any
|
||||||
|
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
# Wir nutzen denselben Parser wie im Projekt (fehlertolerant)
|
||||||
|
from app.core.parser import read_markdown # erwartet: (frontmatter, body)
|
||||||
|
|
||||||
|
MD_EXTS = {".md", ".markdown"}
|
||||||
|
|
||||||
|
def nfc(s: Optional[str]) -> str:
|
||||||
|
return unicodedata.normalize("NFC", s or "")
|
||||||
|
|
||||||
|
def norm_text(txt: str) -> str:
|
||||||
|
# normalize line endings + strip trailing spaces per line + NFC
|
||||||
|
t = txt.replace("\r\n", "\n").replace("\r", "\n")
|
||||||
|
t = "\n".join(line.rstrip() for line in t.split("\n")).strip()
|
||||||
|
return nfc(t)
|
||||||
|
|
||||||
|
def iter_md_files(root: str):
|
||||||
|
for base, _dirs, files in os.walk(root):
|
||||||
|
for fn in files:
|
||||||
|
ext = os.path.splitext(fn)[1].lower()
|
||||||
|
if ext in MD_EXTS:
|
||||||
|
yield os.path.join(base, fn)
|
||||||
|
|
||||||
|
def key_from(front: Dict[str, Any], path: str, pref: str) -> str:
|
||||||
|
"""
|
||||||
|
Liefert den Vergleichsschlüssel.
|
||||||
|
pref in {"auto","id","title","filename"}
|
||||||
|
"""
|
||||||
|
if pref == "id":
|
||||||
|
return nfc(str(front.get("id", "")).strip())
|
||||||
|
if pref == "title":
|
||||||
|
return nfc(str(front.get("title", "")).strip())
|
||||||
|
if pref == "filename":
|
||||||
|
name = os.path.splitext(os.path.basename(path))[0]
|
||||||
|
return nfc(name.strip())
|
||||||
|
|
||||||
|
# auto: id > title > filename
|
||||||
|
if "id" in front and str(front["id"]).strip():
|
||||||
|
return nfc(str(front["id"]).strip())
|
||||||
|
if "title" in front and str(front["title"]).strip():
|
||||||
|
return nfc(str(front["title"]).strip())
|
||||||
|
name = os.path.splitext(os.path.basename(path))[0]
|
||||||
|
return nfc(name.strip())
|
||||||
|
|
||||||
|
def load_index(root: str, key_pref: str) -> Dict[str, Tuple[str, Dict[str, Any], str]]:
|
||||||
|
"""
|
||||||
|
Baut ein Index-Dict: key -> (path, frontmatter, body)
|
||||||
|
"""
|
||||||
|
idx: Dict[str, Tuple[str, Dict[str, Any], str]] = {}
|
||||||
|
for p in iter_md_files(root):
|
||||||
|
try:
|
||||||
|
fm, body = read_markdown(p)
|
||||||
|
k = key_from(fm or {}, p, key_pref)
|
||||||
|
if not k:
|
||||||
|
# Fallback: filename
|
||||||
|
k = key_from({}, p, "filename")
|
||||||
|
idx[k] = (p, fm or {}, body or "")
|
||||||
|
except Exception as e:
|
||||||
|
# Datei nicht parsebar -> trotzdem indexieren nach Dateiname (leer FM/Body)
|
||||||
|
k = key_from({}, p, key_pref if key_pref != "auto" else "filename")
|
||||||
|
idx[k] = (p, {}, "")
|
||||||
|
return idx
|
||||||
|
|
||||||
|
def compare_entries(src: Tuple[str, Dict[str, Any], str],
|
||||||
|
dst: Tuple[str, Dict[str, Any], str],
|
||||||
|
focus: str) -> Dict[str, Any]:
|
||||||
|
sp, sfm, sbody = src
|
||||||
|
dp, dfm, dbody = dst
|
||||||
|
|
||||||
|
res = {"status": "ok"} # wird 'diff' falls Unterschiede
|
||||||
|
diffs = {}
|
||||||
|
|
||||||
|
if focus in ("frontmatter", "all"):
|
||||||
|
# Vergleiche FM minimal: id und title
|
||||||
|
s_id = nfc(str(sfm.get("id", "")).strip())
|
||||||
|
d_id = nfc(str(dfm.get("id", "")).strip())
|
||||||
|
s_title = nfc(str(sfm.get("title", "")).strip())
|
||||||
|
d_title = nfc(str(dfm.get("title", "")).strip())
|
||||||
|
if s_id != d_id:
|
||||||
|
diffs["frontmatter.id"] = {"src": s_id, "dst": d_id}
|
||||||
|
if s_title != d_title:
|
||||||
|
diffs["frontmatter.title"] = {"src": s_title, "dst": d_title}
|
||||||
|
|
||||||
|
if focus in ("body", "all"):
|
||||||
|
if norm_text(sbody) != norm_text(dbody):
|
||||||
|
diffs["body"] = {"src_len": len(sbody or ""), "dst_len": len(dbody or "")}
|
||||||
|
|
||||||
|
if diffs:
|
||||||
|
res["status"] = "diff"
|
||||||
|
res["diffs"] = diffs
|
||||||
|
res["src_path"] = sp
|
||||||
|
res["dst_path"] = dp
|
||||||
|
return res
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--src", required=True, help="Quell-Vault-Ordner")
|
||||||
|
ap.add_argument("--dst", required=True, help="Export-Vault-Ordner")
|
||||||
|
ap.add_argument("--focus", choices=["body", "frontmatter", "all"], default="body")
|
||||||
|
ap.add_argument("--key", choices=["auto", "id", "title", "filename"], default="auto",
|
||||||
|
help="Vergleichsschlüssel (default: auto=id>title>filename)")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
src_idx = load_index(args.src, args.key)
|
||||||
|
dst_idx = load_index(args.dst, args.key)
|
||||||
|
|
||||||
|
src_keys = set(src_idx.keys())
|
||||||
|
dst_keys = set(dst_idx.keys())
|
||||||
|
|
||||||
|
# Fehlende
|
||||||
|
only_src = sorted(src_keys - dst_keys)
|
||||||
|
only_dst = sorted(dst_keys - src_keys)
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
for k in only_src:
|
||||||
|
print(json.dumps({"note_id": k, "status": "missing", "src": True, "dst": False}, ensure_ascii=False))
|
||||||
|
count += 1
|
||||||
|
for k in only_dst:
|
||||||
|
print(json.dumps({"note_id": k, "status": "missing", "src": False, "dst": True}, ensure_ascii=False))
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
# Vergleiche, wo vorhanden
|
||||||
|
diff_count = 0
|
||||||
|
common = sorted(src_keys & dst_keys)
|
||||||
|
for k in common:
|
||||||
|
res = compare_entries(src_idx[k], dst_idx[k], args.focus)
|
||||||
|
if res["status"] == "diff":
|
||||||
|
res["note_id"] = k
|
||||||
|
print(json.dumps(res, ensure_ascii=False))
|
||||||
|
diff_count += 1
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"summary": "OK" if (count == 0 and diff_count == 0) else "DIFFS",
|
||||||
|
"missing_count": count,
|
||||||
|
"diff_count": diff_count,
|
||||||
|
"total_src": len(src_keys),
|
||||||
|
"total_dst": len(dst_keys),
|
||||||
|
"focus": args.focus,
|
||||||
|
"key": args.key,
|
||||||
|
}
|
||||||
|
print(json.dumps(summary, ensure_ascii=False))
|
||||||
|
# Exit-Code für CI
|
||||||
|
if summary["summary"] != "OK":
|
||||||
|
# non-zero exit optional; hier 0 lassen, damit CLI-Ausgaben sichtbar bleiben
|
||||||
|
pass
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/inspect_one_note.py
|
||||||
|
Diagnose einer einzelnen Markdown-Datei (kein Upsert).
|
||||||
|
Usage:
|
||||||
|
python3 tests/inspect_one_note.py --path ./test_vault/30_projects/project-demo.md
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os, sys, json, argparse
|
||||||
|
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||||
|
from app.core.chunker import assemble_chunks
|
||||||
|
from app.core.chunk_payload import make_chunk_payloads
|
||||||
|
try:
|
||||||
|
from app.core.derive_edges import build_edges_for_note
|
||||||
|
except Exception:
|
||||||
|
from app.core.edges import build_edges_for_note # type: ignore
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--path", required=True)
|
||||||
|
args = ap.parse_args()
|
||||||
|
p = os.path.abspath(args.path)
|
||||||
|
|
||||||
|
try:
|
||||||
|
parsed = read_markdown(p)
|
||||||
|
if parsed is None:
|
||||||
|
print(json.dumps({"path": p, "error": "read_markdown returned None"}))
|
||||||
|
return
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"path": p, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
|
validate_required_frontmatter(fm)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"path": p, "error": f"frontmatter invalid: {type(e).__name__}: {e}"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
body = getattr(parsed, "body", "") or ""
|
||||||
|
chunks = assemble_chunks(fm["id"], body, fm.get("type", "concept"))
|
||||||
|
rel_for_payload = os.path.relpath(p, os.path.join(p, "..", ".."))
|
||||||
|
chunk_pls = make_chunk_payloads(fm, rel_for_payload, chunks, note_text=body)
|
||||||
|
|
||||||
|
edge_error = None
|
||||||
|
edges_count = 0
|
||||||
|
try:
|
||||||
|
edges = build_edges_for_note(fm["id"], chunk_pls, include_note_scope_refs=True)
|
||||||
|
edges_count = len(edges)
|
||||||
|
except Exception as e:
|
||||||
|
edge_error = f"{type(e).__name__}: {e}"
|
||||||
|
|
||||||
|
print(json.dumps({
|
||||||
|
"path": p,
|
||||||
|
"id": fm.get("id"),
|
||||||
|
"title": fm.get("title"),
|
||||||
|
"body_len": len(body),
|
||||||
|
"chunks": len(chunk_pls),
|
||||||
|
"edges": edges_count,
|
||||||
|
"edge_error": edge_error
|
||||||
|
}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/list_md_seen_by_importer.py
|
||||||
|
Zeigt, welche Markdown-Dateien der Importer sehen würde (gleiche Filter & Sortierung).
|
||||||
|
Usage:
|
||||||
|
python3 tests/list_md_seen_by_importer.py --vault ./test_vault
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, os, sys, json
|
||||||
|
|
||||||
|
def iter_md(root: str):
|
||||||
|
out = []
|
||||||
|
for dirpath, _, filenames in os.walk(root):
|
||||||
|
for fn in filenames:
|
||||||
|
p = os.path.join(dirpath, fn)
|
||||||
|
pn = p.replace("\\", "/")
|
||||||
|
item = {
|
||||||
|
"path": os.path.abspath(p),
|
||||||
|
"rel": os.path.relpath(p, root).replace("\\", "/"),
|
||||||
|
"included": False,
|
||||||
|
"reason": "",
|
||||||
|
}
|
||||||
|
if not fn.lower().endswith(".md"):
|
||||||
|
item["reason"] = "skip_ext"
|
||||||
|
elif "/.obsidian/" in pn or "/_backup_frontmatter/" in pn or "/_imported/" in pn:
|
||||||
|
item["reason"] = "skip_special_dir"
|
||||||
|
else:
|
||||||
|
item["included"] = True
|
||||||
|
item["reason"] = "ok"
|
||||||
|
out.append(item)
|
||||||
|
# auch die ausgeschlossenen zeigen:
|
||||||
|
if not item["included"]:
|
||||||
|
print(json.dumps(item, ensure_ascii=False))
|
||||||
|
out_sorted = sorted(out, key=lambda x: x["rel"])
|
||||||
|
for i, it in enumerate(out_sorted):
|
||||||
|
it["order"] = i
|
||||||
|
print(json.dumps(it, ensure_ascii=False))
|
||||||
|
print(json.dumps({"summary":"done","included":len(out_sorted)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--vault", required=True)
|
||||||
|
args = ap.parse_args()
|
||||||
|
root = os.path.abspath(args.vault)
|
||||||
|
iter_md(root)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# tests/run_e2e_roundtrip.sh
|
||||||
|
# End-to-End Testpipeline für mindnet WP-03
|
||||||
|
#
|
||||||
|
# Schritte:
|
||||||
|
# 1) Qdrant truncate (Prefix!)
|
||||||
|
# 2) Import (Apply + Purge) => Create-Fall
|
||||||
|
# 3) Import (Apply - ohne Änderungen) => Idempotenz
|
||||||
|
# 4) Hash-Reporter (Fehlende Hash-Varianten)
|
||||||
|
# 5) Export nach _exportVault
|
||||||
|
# 6) Vergleich Vault vs. Export (Body-Fokus)
|
||||||
|
# 7) Sync-Deletes: Eine Datei temporär „entfernen“, Dry-Run + Apply, dann zurückrollen
|
||||||
|
#
|
||||||
|
# Aufrufe:
|
||||||
|
# ./tests/run_e2e_roundtrip.sh [--vault ./test_vault] [--prefix mindnet]
|
||||||
|
#
|
||||||
|
# ENV:
|
||||||
|
# COLLECTION_PREFIX (Default mindnet)
|
||||||
|
|
||||||
|
VAULT="./test_vault"
|
||||||
|
PREFIX="${COLLECTION_PREFIX:-mindnet}"
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--vault) VAULT="$2"; shift 2;;
|
||||||
|
--prefix) PREFIX="$2"; shift 2;;
|
||||||
|
*) echo "Unbekannte Option: $1" >&2; exit 2;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "== E2E Roundtrip =="
|
||||||
|
echo "Vault: $VAULT"
|
||||||
|
echo "Prefix: $PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
export COLLECTION_PREFIX="$PREFIX"
|
||||||
|
|
||||||
|
echo "-- 1) Qdrant truncate"
|
||||||
|
python3 -m scripts.reset_qdrant --mode truncate --prefix "$PREFIX"
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 2) Import (Create-Fall)"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --apply --purge-before-upsert --prefix "$PREFIX"
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 3) Import (Idempotenz: keine Änderungen erwartet)"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --apply --prefix "$PREFIX"
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 4) Hash-Reporter"
|
||||||
|
python3 -m scripts.report_hashes --prefix "$PREFIX" || true
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 5) Export nach _exportVault"
|
||||||
|
OUT_DIR="_exportVault"
|
||||||
|
rm -rf "$OUT_DIR"
|
||||||
|
python3 -m scripts.export_markdown --out "$OUT_DIR" --prefix "$PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 6) Vergleich Vault vs. Export (Body-Fokus)"
|
||||||
|
python3 tests/compare_vaults.py --src "$VAULT" --dst "$OUT_DIR" --focus body
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 7) Sync-Deletes (simuliert)"
|
||||||
|
# wähle eine Datei und verschiebe sie temporär weg
|
||||||
|
TMP_DIR="$(mktemp -d)"
|
||||||
|
SAMPLE_MD="$(find "$VAULT" -type f -name '*.md' | head -n 1 || true)"
|
||||||
|
if [[ -n "${SAMPLE_MD:-}" ]]; then
|
||||||
|
mv "$SAMPLE_MD" "$TMP_DIR/"
|
||||||
|
echo "Temporär entfernt: $(basename "$SAMPLE_MD")"
|
||||||
|
|
||||||
|
echo " 7a) Dry-Run"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --sync-deletes --prefix "$PREFIX"
|
||||||
|
echo " 7b) Apply"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --sync-deletes --apply --prefix "$PREFIX"
|
||||||
|
|
||||||
|
# zurückrollen
|
||||||
|
mv "$TMP_DIR/$(basename "$SAMPLE_MD")" "$SAMPLE_MD"
|
||||||
|
rm -rf "$TMP_DIR"
|
||||||
|
|
||||||
|
echo " 7c) Re-Import (wieder anlegen)"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --apply --prefix "$PREFIX"
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
|
||||||
|
else
|
||||||
|
echo "Hinweis: Keine MD-Datei im Vault gefunden – Sync-Deletes übersprungen."
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "== E2E Roundtrip abgeschlossen =="
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# tests/test_importer_detect_changes.sh
|
||||||
|
# Prüft Änderungserkennung:
|
||||||
|
# - Body-Change -> wird erkannt bei hash-mode=body
|
||||||
|
# - Frontmatter-Change -> wird NICHT erkannt bei hash-mode=body
|
||||||
|
# - Frontmatter-Change -> wird erkannt bei hash-mode=frontmatter (bzw. MINDNET_HASH_COMPARE=Frontmatter)
|
||||||
|
#
|
||||||
|
# Aufruf:
|
||||||
|
# ./tests/test_importer_detect_changes.sh [--vault ./test_vault] [--prefix mindnet] [--note-id concept-alpha]
|
||||||
|
|
||||||
|
VAULT="./test_vault"
|
||||||
|
PREFIX="${COLLECTION_PREFIX:-mindnet}"
|
||||||
|
NOTE_ID="concept-alpha"
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--vault) VAULT="$2"; shift 2;;
|
||||||
|
--prefix) PREFIX="$2"; shift 2;;
|
||||||
|
--note-id) NOTE_ID="$2"; shift 2;;
|
||||||
|
*) echo "Unbekannte Option: $1" >&2; exit 2;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
export COLLECTION_PREFIX="$PREFIX"
|
||||||
|
|
||||||
|
# Hilfsfunktion: lese Pfad zur Note aus Frontmatter id:
|
||||||
|
PYFIND=$(cat <<'PY'
|
||||||
|
import sys, os, glob, yaml
|
||||||
|
vault = sys.argv[1]
|
||||||
|
nid = sys.argv[2]
|
||||||
|
for p in glob.glob(os.path.join(vault, "**", "*.md"), recursive=True):
|
||||||
|
with open(p, "r", encoding="utf-8") as f:
|
||||||
|
s = f.read()
|
||||||
|
if s.startswith("---"):
|
||||||
|
try:
|
||||||
|
fm_txt, body = s.split("\n---\n", 1)
|
||||||
|
fm = yaml.safe_load(fm_txt.strip("- \n")) or {}
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if fm.get("id") == nid:
|
||||||
|
print(p)
|
||||||
|
break
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
NOTE_PATH=$(python3 -c "$PYFIND" "$VAULT" "$NOTE_ID")
|
||||||
|
if [[ -z "${NOTE_PATH:-}" ]]; then
|
||||||
|
echo "Note mit id=$NOTE_ID nicht gefunden im Vault=$VAULT" >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
echo "Note: $NOTE_PATH"
|
||||||
|
|
||||||
|
# Backup
|
||||||
|
ORIG="$(mktemp)"
|
||||||
|
cp "$NOTE_PATH" "$ORIG"
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
cp "$ORIG" "$NOTE_PATH" || true
|
||||||
|
rm -f "$ORIG"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
# Ausgangszustand importieren (Baseline)
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --apply --prefix "$PREFIX" >/dev/null
|
||||||
|
|
||||||
|
echo "-- Test A: Body-Änderung soll erkannt werden (hash-mode=body)"
|
||||||
|
# Body um eine Zeile erweitern
|
||||||
|
python3 - "$NOTE_PATH" <<'PY'
|
||||||
|
import sys
|
||||||
|
p=sys.argv[1]
|
||||||
|
with open(p,"r+",encoding="utf-8") as f:
|
||||||
|
s=f.read()
|
||||||
|
if s.startswith("---"):
|
||||||
|
fm, body = s.split("\n---\n",1)
|
||||||
|
body = body + "\n\nBodyTest: Änderung A"
|
||||||
|
f.seek(0); f.truncate(); f.write(fm+"\n---\n"+body)
|
||||||
|
PY
|
||||||
|
OUT=$(python3 -m scripts.import_markdown --vault "$VAULT" --prefix "$PREFIX" | grep "\"note_id\": \"$NOTE_ID\"" || true)
|
||||||
|
echo "$OUT"
|
||||||
|
echo "$OUT" | grep -q '"changed": true' || { echo "ERWARTET: changed=true (Body) — FEHLER"; exit 4; }
|
||||||
|
|
||||||
|
echo "-- Test B: Frontmatter-Änderung soll NICHT erkannt werden bei mode=body"
|
||||||
|
# Frontmatter-Feld hinzufügen
|
||||||
|
python3 - "$NOTE_PATH" <<'PY'
|
||||||
|
import sys, yaml, io
|
||||||
|
p=sys.argv[1]
|
||||||
|
with open(p,"r+",encoding="utf-8") as f:
|
||||||
|
s=f.read()
|
||||||
|
if s.startswith("---"):
|
||||||
|
fm_txt, body = s.split("\n---\n",1)
|
||||||
|
fm = yaml.safe_load(fm_txt.strip("- \n")) or {}
|
||||||
|
fm["test_flag"] = True
|
||||||
|
buf = io.StringIO()
|
||||||
|
yaml.safe_dump(fm, buf, allow_unicode=True, sort_keys=False)
|
||||||
|
f.seek(0); f.truncate(); f.write("---\n"+buf.getvalue()+"---\n"+body)
|
||||||
|
PY
|
||||||
|
OUT=$(python3 -m scripts.import_markdown --vault "$VAULT" --prefix "$PREFIX" | grep "\"note_id\": \"$NOTE_ID\"" || true)
|
||||||
|
echo "$OUT"
|
||||||
|
echo "$OUT" | grep -q '"changed": false' || { echo "ERWARTET: changed=false (FM bei mode=body) — FEHLER"; exit 5; }
|
||||||
|
|
||||||
|
echo "-- Test C: Frontmatter-Änderung soll erkannt werden bei mode=frontmatter"
|
||||||
|
OUT=$(MINDNET_HASH_COMPARE=Frontmatter python3 -m scripts.import_markdown --vault "$VAULT" --prefix "$PREFIX" | grep "\"note_id\": \"$NOTE_ID\"" || true)
|
||||||
|
echo "$OUT"
|
||||||
|
echo "$OUT" | grep -q '"changed": true' || { echo "ERWARTET: changed=true (FM bei mode=frontmatter) — FEHLER"; exit 6; }
|
||||||
|
|
||||||
|
echo "✔ Änderungserkennung: OK"
|
||||||
@@ -0,0 +1,193 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Script: tests/verify_chunks_integrity.py
|
||||||
|
Version: 1.0.1
|
||||||
|
Datum: 2025-09-10
|
||||||
|
|
||||||
|
Zweck
|
||||||
|
-----
|
||||||
|
Verifiziert die Text-Integrität der gespeicherten Chunks:
|
||||||
|
1) Rekonstruiert den Body aus den Chunks (Sortierung: seq → chunk_index → # in chunk_id).
|
||||||
|
2) Vergleicht mit dem in Qdrant gespeicherten Note-`fulltext` (falls vorhanden).
|
||||||
|
3) Optional: Vergleicht zusätzlich mit dem Body der zugehörigen Markdown-Datei im Vault.
|
||||||
|
|
||||||
|
Aufrufe
|
||||||
|
-------
|
||||||
|
# Nur gegen Qdrant (fulltext vs. Chunks)
|
||||||
|
python3 tests/verify_chunks_integrity.py --prefix mindnet
|
||||||
|
|
||||||
|
# Zusätzlich gegen den Vault abgleichen (Body der .md-Datei)
|
||||||
|
python3 tests/verify_chunks_integrity.py --prefix mindnet --vault ./test_vault
|
||||||
|
|
||||||
|
# Streng + CI-geeignet (Fehlercode bei Abweichungen):
|
||||||
|
python3 tests/verify_chunks_integrity.py --prefix mindnet --vault ./test_vault --strict --fail-on-mismatch
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
# --- FIX: Projekt-Root in sys.path aufnehmen, damit 'app.*' importierbar ist ---
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
# --------------------------- Helpers ---------------------------
|
||||||
|
|
||||||
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def scroll_all(client, collection: str, flt: Optional[rest.Filter] = None, limit: int = 256) -> List[Any]:
|
||||||
|
out: List[Any] = []
|
||||||
|
nextp = None
|
||||||
|
while True:
|
||||||
|
pts, nextp = client.scroll(
|
||||||
|
collection_name=collection,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=limit,
|
||||||
|
offset=nextp,
|
||||||
|
scroll_filter=flt,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
out.extend(pts)
|
||||||
|
if nextp is None:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
def sort_key_for_chunk_payload(pl: Dict[str, Any]) -> Tuple[int, int, int]:
|
||||||
|
s = pl.get("seq") or 0
|
||||||
|
ci = pl.get("chunk_index") or 0
|
||||||
|
n = 0
|
||||||
|
cid = pl.get("chunk_id") or ""
|
||||||
|
if isinstance(cid, str) and "#" in cid:
|
||||||
|
try:
|
||||||
|
n = int(cid.rsplit("#", 1)[-1])
|
||||||
|
except Exception:
|
||||||
|
n = 0
|
||||||
|
return (int(s), int(ci), int(n))
|
||||||
|
|
||||||
|
def reconstruct_from_chunks(chunks_points: List[Any]) -> Tuple[str, int, int]:
|
||||||
|
"""Gibt (text, total_chunks, chunks_with_text) zurück."""
|
||||||
|
chunks_sorted = sorted(chunks_points, key=lambda p: sort_key_for_chunk_payload(p.payload or {}))
|
||||||
|
texts: List[str] = []
|
||||||
|
have = 0
|
||||||
|
for p in chunks_sorted:
|
||||||
|
pl = p.payload or {}
|
||||||
|
t = pl.get("text") or pl.get("content") or pl.get("raw") or ""
|
||||||
|
if isinstance(t, str) and t:
|
||||||
|
have += 1
|
||||||
|
texts.append(t)
|
||||||
|
return ("\n".join(texts).strip(), len(chunks_sorted), have)
|
||||||
|
|
||||||
|
def normalize(s: str, strict: bool = False) -> str:
|
||||||
|
if s is None:
|
||||||
|
return ""
|
||||||
|
s = s.replace("\r\n", "\n").replace("\r", "\n")
|
||||||
|
if strict:
|
||||||
|
return s
|
||||||
|
lines = [ln.rstrip() for ln in s.strip().split("\n")]
|
||||||
|
return "\n".join(lines).strip()
|
||||||
|
|
||||||
|
def read_vault_body(vault_root: str, rel_path: str) -> Optional[str]:
|
||||||
|
if not rel_path:
|
||||||
|
return None
|
||||||
|
p = os.path.join(vault_root, rel_path.replace("\\", "/").lstrip("/"))
|
||||||
|
if not os.path.exists(p):
|
||||||
|
return None
|
||||||
|
with open(p, "r", encoding="utf-8") as f:
|
||||||
|
s = f.read()
|
||||||
|
if s.startswith("---"):
|
||||||
|
try:
|
||||||
|
fm_txt, body = s.split("\n---\n", 1)
|
||||||
|
except Exception:
|
||||||
|
return s
|
||||||
|
else:
|
||||||
|
body = s
|
||||||
|
return body
|
||||||
|
|
||||||
|
# --------------------------- Main ------------------------------
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||||
|
ap.add_argument("--vault", help="Optional: Vault-Wurzelordner für Abgleich gegen .md")
|
||||||
|
ap.add_argument("--strict", action="store_true", help="Strikter Vergleich (kein Trimmen/Normalisieren)")
|
||||||
|
ap.add_argument("--fail-on-mismatch", action="store_true", help="Exit 1, wenn ein Mismatch gefunden wurde")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
if args.prefix:
|
||||||
|
cfg.prefix = args.prefix.strip()
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
notes_col, chunks_col, _ = collections(cfg.prefix)
|
||||||
|
notes = scroll_all(client, notes_col)
|
||||||
|
|
||||||
|
mismatches = 0
|
||||||
|
total = 0
|
||||||
|
|
||||||
|
for n in notes:
|
||||||
|
total += 1
|
||||||
|
pl = n.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
rel_path = (pl.get("path") or "").replace("\\", "/").lstrip("/")
|
||||||
|
fulltext = (pl.get("fulltext") or "").strip()
|
||||||
|
|
||||||
|
flt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||||
|
chunks = scroll_all(client, chunks_col, flt)
|
||||||
|
recon, cnt, have = reconstruct_from_chunks(chunks)
|
||||||
|
|
||||||
|
norm_recon = normalize(recon, strict=args.strict)
|
||||||
|
norm_full = normalize(fulltext, strict=args.strict)
|
||||||
|
|
||||||
|
match_full = (bool(norm_full) and (norm_recon == norm_full))
|
||||||
|
match_vault = None
|
||||||
|
|
||||||
|
issues: List[str] = []
|
||||||
|
if cnt == 0:
|
||||||
|
issues.append("no_chunks")
|
||||||
|
if have < cnt:
|
||||||
|
issues.append(f"missing_text_in_chunks:{cnt-have}/{cnt}")
|
||||||
|
if norm_full == "":
|
||||||
|
issues.append("note_fulltext_empty")
|
||||||
|
if not match_full:
|
||||||
|
issues.append("reconstructed_vs_fulltext_mismatch")
|
||||||
|
|
||||||
|
if args.vault:
|
||||||
|
body = read_vault_body(args.vault, rel_path) or ""
|
||||||
|
norm_body = normalize(body, strict=args.strict)
|
||||||
|
match_vault = (norm_body == norm_recon)
|
||||||
|
if not match_vault:
|
||||||
|
issues.append("reconstructed_vs_vault_body_mismatch")
|
||||||
|
|
||||||
|
obj = {
|
||||||
|
"note_id": nid,
|
||||||
|
"path": rel_path,
|
||||||
|
"chunks_count": cnt,
|
||||||
|
"chunks_with_text": have,
|
||||||
|
"match_fulltext": match_full,
|
||||||
|
"match_vault": match_vault,
|
||||||
|
"issues": issues,
|
||||||
|
}
|
||||||
|
print(json.dumps(obj, ensure_ascii=False))
|
||||||
|
if ("reconstructed_vs_fulltext_mismatch" in issues) or ("reconstructed_vs_vault_body_mismatch" in issues):
|
||||||
|
mismatches += 1
|
||||||
|
|
||||||
|
summary = {"summary": "OK" if mismatches == 0 else "DIFFS", "mismatch_count": mismatches, "notes": total}
|
||||||
|
print(json.dumps(summary, ensure_ascii=False))
|
||||||
|
if args.fail_on_mismatch and mismatches:
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in new issue
Block a user