Compare commits
143
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
300086fc83 | ||
|
|
688eb262a9 | ||
|
|
bfb6f5dede | ||
|
|
88c0fbe56b | ||
|
|
3d74eff224 | ||
|
|
22d08afe2d | ||
|
|
a97f757e34 | ||
|
|
bbc8f13944 | ||
|
|
f18a40d76c | ||
|
|
c7644a36aa | ||
|
|
f394f31c12 | ||
|
|
0513673c2a | ||
|
|
a73542a391 | ||
|
|
c465797654 | ||
|
|
7ce3f982f2 | ||
|
|
6311456917 | ||
|
|
3c06559c5d | ||
|
|
5c9f2e8c83 | ||
|
|
62902faaca | ||
|
|
d22fd77b22 | ||
|
|
2f86f493d6 | ||
|
|
eafa0fad85 | ||
|
|
2e7c497b69 | ||
|
|
79dcfdcf9a | ||
|
|
487e28a904 | ||
|
|
c492f97a67 | ||
|
|
ec2ff74514 | ||
|
|
c501f8d6e6 | ||
|
|
b4287cbfda | ||
|
|
26724a1db1 | ||
|
|
d6368ee709 | ||
|
|
ea211a5c0b | ||
|
|
9e8b433c95 | ||
|
|
2a1c62aeed | ||
|
|
948d6f4b47 | ||
|
|
725271a7da | ||
|
|
4ea62e6886 | ||
|
|
c01cf3b078 | ||
|
|
c67777a6b9 | ||
|
|
e5e5e7560c | ||
|
|
6e1cd5fbbc | ||
|
|
8bd8f7c0bb | ||
|
|
1c42cd1f78 | ||
|
|
be090a43f2 | ||
|
|
2c81fa0cb5 | ||
|
|
c3815afcd4 | ||
|
|
8800160564 | ||
|
|
bdbc4a1bf7 | ||
|
|
597090bc45 | ||
|
|
af36c410b4 | ||
|
|
4dcd606c10 | ||
|
|
bbd5a7fa48 | ||
|
|
6dc37ccb66 | ||
|
|
b84906283e | ||
|
|
af0dcdf76a | ||
|
|
f1e1cde597 | ||
|
|
a7c5630e5b | ||
|
|
82151d6bc3 | ||
|
|
c93e7ad598 | ||
|
|
290b271cf6 | ||
|
|
a686bdbeaf | ||
|
|
2b84c62875 | ||
|
|
2de786fc64 | ||
|
|
2ddf034983 | ||
|
|
b2043f4f84 | ||
|
|
52eae52061 | ||
|
|
e451ea64ae | ||
|
|
eb6e4028ff | ||
|
|
ea45493941 | ||
|
|
da70f0e00c | ||
|
|
8bf3707511 | ||
|
|
804eeef858 | ||
|
|
36658f12fe | ||
|
|
f0e675ffd8 | ||
|
|
90911c1074 | ||
|
|
1729e0b40d | ||
|
|
70b3169e7f | ||
|
|
315a1c0945 | ||
|
|
b9316c693d | ||
|
|
820e52d869 | ||
|
|
444e8e4206 | ||
|
|
e6f2c8b4e8 | ||
|
|
f6b7652219 | ||
|
|
3cd00ab5cf | ||
|
|
0feae50d90 | ||
|
|
d79c55fc64 | ||
|
|
98a6f5c480 | ||
|
|
0fa4aa7280 | ||
|
|
3b192e2eed | ||
|
|
2794d26181 | ||
|
|
7b56f696d6 | ||
|
|
ead1b8c1bc | ||
|
|
b8126e00c4 | ||
|
|
62f8f4b313 | ||
|
|
21924541dc | ||
|
|
197242ca42 | ||
|
|
7922734dce | ||
|
|
afd7675425 | ||
|
|
8b3b343645 | ||
|
|
6713f342c3 | ||
|
|
79bb8a143c | ||
|
|
d03a181536 | ||
|
|
8e5304c156 | ||
|
|
c487db6534 | ||
|
|
0ca54f429e | ||
|
|
6c62f64784 | ||
|
|
3282f85007 | ||
|
|
b186569750 | ||
|
|
7cde2462ec | ||
|
|
c0b96f7c05 | ||
|
|
3a88caac69 | ||
|
|
31457e9240 | ||
|
|
307335a733 | ||
|
|
4eb5e34ea7 | ||
|
|
f4be219790 | ||
|
|
53591b6f27 | ||
|
|
a14d0bb7cb | ||
|
|
c2802e7cb3 | ||
|
|
c7f2815f1d | ||
|
|
afc90fe992 | ||
|
|
2f9ce824a0 | ||
|
|
a39b2a6950 | ||
|
|
70fa95966c | ||
|
|
10479014d5 | ||
|
|
148e22aa7d | ||
|
|
a9fe6196f7 | ||
|
|
ae20665a66 | ||
|
|
ccbf078463 | ||
|
|
cd1e46784f | ||
|
|
b49d97b8d7 | ||
|
|
3d44de9d87 | ||
|
|
987b3c1770 | ||
|
|
25bc6544c4 | ||
|
|
b241ae753b | ||
|
|
2041771b14 | ||
|
|
b5958a9f63 | ||
|
|
c05dbd4b3b | ||
|
|
44e468fc21 | ||
|
|
5f358d329b | ||
|
|
59fe8154f2 | ||
|
|
d5b2e9bb8f | ||
|
|
3a3907e1b6 | ||
|
|
f66cdc70b2 |
No files matched your search
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: linux_host
|
||||
env:
|
||||
TARGET_BASE: /home/llmadmin/mindnet
|
||||
DEPLOY_DIRS: "app scripts schemas docker tests requirements.txt README.md"
|
||||
DEPLOY_DIRS: "app scripts schemas docker tests config requirements.txt README.md"
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
+105
-247
@@ -1,271 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/chunk_payload.py
|
||||
Version: 2.2.0
|
||||
Datum: 2025-10-06
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.0.1.
|
||||
Neu: Wenn der Chunker KEIN Overlap im Fenster liefert (== window fehlt / identisch zur Kernpassage),
|
||||
erzeugen wir FENSTER mit synthetischem Overlap auf Basis chunk_config.get_sizes(note_type)['overlap'].
|
||||
|
||||
Felder (beibehalten aus 2.0.1):
|
||||
- note_id, chunk_id, id (Alias), chunk_index, seq, path
|
||||
- window (mit Overlap), text (ohne linkes Overlap)
|
||||
- start, end (Offsets im gesamten Body)
|
||||
- overlap_left, overlap_right
|
||||
- token_count?, section_title?, section_path?, type?, title?, tags?
|
||||
|
||||
Kompatibilität:
|
||||
- 'id' == 'chunk_id' als Alias
|
||||
- Pfade bleiben relativ (keine führenden '/'), Backslashes → Slashes
|
||||
- Robust für Chunk-Objekte oder Dicts; Fensterquelle: 'window'|'text'|'content'|'raw'
|
||||
|
||||
Lizenz: MIT (projektintern)
|
||||
app/core/chunk_payload.py (Mindnet V2 — types.yaml authoritative)
|
||||
- neighbors_prev / neighbors_next sind Listen ([], [id]).
|
||||
- retriever_weight / chunk_profile kommen aus types.yaml (Frontmatter wird ignoriert).
|
||||
- Fallbacks: defaults.* in types.yaml; sonst 1.0 / "default".
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, List, Optional
|
||||
import os, yaml
|
||||
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
def _env(n: str, d: Optional[str]=None) -> str:
|
||||
v = os.getenv(n)
|
||||
return v if v is not None else (d or "")
|
||||
|
||||
try:
|
||||
# Typgerechtes Overlap aus deiner Konfiguration holen
|
||||
from app.core.chunk_config import get_sizes as _get_sizes
|
||||
except Exception:
|
||||
def _get_sizes(_note_type: str):
|
||||
# konservativer Default, falls Import fehlschlägt
|
||||
return {"overlap": (40, 60), "target": (250, 350), "max": 500}
|
||||
|
||||
|
||||
# ------------------------------- Utils ------------------------------- #
|
||||
|
||||
def _get_attr_or_key(obj: Any, key: str, default=None):
|
||||
if obj is None:
|
||||
return default
|
||||
if isinstance(obj, dict):
|
||||
return obj.get(key, default)
|
||||
return getattr(obj, key, default)
|
||||
|
||||
def _as_window_text(chunk: Any) -> str:
|
||||
"""Fenstertext robust lesen (bevorzugt echte Fenster, sonst Kern)."""
|
||||
for k in ("window", "text", "content", "raw"):
|
||||
v = _get_attr_or_key(chunk, k, None)
|
||||
if isinstance(v, str) and v:
|
||||
return v
|
||||
return ""
|
||||
|
||||
def _to_int(x: Any, default: int = 0) -> int:
|
||||
def _load_types() -> dict:
|
||||
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||
try:
|
||||
return int(x)
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
except Exception:
|
||||
return default
|
||||
return {}
|
||||
|
||||
def _normalize_rel_path(p: str) -> str:
|
||||
p = (p or "").replace("\\", "/")
|
||||
while p.startswith("/"):
|
||||
p = p[1:]
|
||||
return p
|
||||
def _get_types_map(reg: dict) -> dict:
|
||||
if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
|
||||
return reg["types"]
|
||||
return reg if isinstance(reg, dict) else {}
|
||||
|
||||
def _get_defaults(reg: dict) -> dict:
|
||||
if isinstance(reg, dict) and isinstance(reg.get("defaults"), dict):
|
||||
return reg["defaults"]
|
||||
if isinstance(reg, dict) and isinstance(reg.get("global"), dict):
|
||||
return reg["global"]
|
||||
return {}
|
||||
|
||||
# ---------------------- Overlap & Offsets ---------------------------- #
|
||||
def _as_float(x: Any):
|
||||
try:
|
||||
return float(x)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int], str]:
|
||||
"""
|
||||
Entfernt linkes Overlap aus echten Fenster-Strings.
|
||||
Rückgabe: (segments, overlaps_left, reconstructed_text)
|
||||
"""
|
||||
segments: List[str] = []
|
||||
overlaps_left: List[int] = []
|
||||
reconstructed = ""
|
||||
for w in windows:
|
||||
w = w or ""
|
||||
max_k = min(len(w), len(reconstructed))
|
||||
k = 0
|
||||
for cand in range(max_k, -1, -1):
|
||||
if reconstructed.endswith(w[:cand]):
|
||||
k = cand
|
||||
break
|
||||
seg = w[k:]
|
||||
segments.append(seg)
|
||||
overlaps_left.append(k)
|
||||
reconstructed += seg
|
||||
return segments, overlaps_left, reconstructed
|
||||
def _resolve_chunk_profile(note_type: str, reg: dict) -> str:
|
||||
types = _get_types_map(reg)
|
||||
if isinstance(types, dict):
|
||||
t = types.get(note_type, {})
|
||||
if isinstance(t, dict) and isinstance(t.get("chunk_profile"), str):
|
||||
return t["chunk_profile"]
|
||||
defs = _get_defaults(reg)
|
||||
if isinstance(defs, dict) and isinstance(defs.get("chunk_profile"), str):
|
||||
return defs["chunk_profile"]
|
||||
return "default"
|
||||
|
||||
def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
|
||||
"""Länge längsten Suffix(a), der Prefix(b) ist."""
|
||||
if not a or not b:
|
||||
return 0
|
||||
a1 = a[-max_probe:]
|
||||
b1 = b[:max_probe]
|
||||
n = min(len(a1), len(b1))
|
||||
for k in range(n, 0, -1):
|
||||
if a1[-k:] == b1[:k]:
|
||||
return k
|
||||
return 0
|
||||
def _resolve_retriever_weight(note_type: str, reg: dict) -> float:
|
||||
types = _get_types_map(reg)
|
||||
if isinstance(types, dict):
|
||||
t = types.get(note_type, {})
|
||||
if isinstance(t, dict) and (t.get("retriever_weight") is not None):
|
||||
v = _as_float(t.get("retriever_weight"))
|
||||
if v is not None:
|
||||
return float(v)
|
||||
defs = _get_defaults(reg)
|
||||
if isinstance(defs, dict) and (defs.get("retriever_weight") is not None):
|
||||
v = _as_float(defs.get("retriever_weight"))
|
||||
if v is not None:
|
||||
return float(v)
|
||||
return 1.0
|
||||
|
||||
def _as_list(x):
|
||||
if x is None:
|
||||
return []
|
||||
if isinstance(x, list):
|
||||
return x
|
||||
return [x]
|
||||
|
||||
# ----------------------------- Public API ---------------------------- #
|
||||
def make_chunk_payloads(note: Dict[str, Any],
|
||||
note_path: str,
|
||||
chunks_from_chunker: List[Any],
|
||||
*,
|
||||
note_text: str = "",
|
||||
types_cfg: Optional[dict] = None,
|
||||
file_path: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
fm = (note or {}).get("frontmatter", {}) or {}
|
||||
note_type = fm.get("type") or note.get("type") or "concept"
|
||||
reg = types_cfg if isinstance(types_cfg, dict) else _load_types()
|
||||
|
||||
def make_chunk_payloads(
|
||||
frontmatter: Dict[str, Any],
|
||||
rel_path: str,
|
||||
chunks: Iterable[Union[Dict[str, Any], Any]],
|
||||
note_text: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Baut Payloads pro Chunk. Falls Fenster ohne Overlap geliefert werden,
|
||||
erzeugen wir synthetische 'window'-Texte mit typgerechtem Overlap.
|
||||
"""
|
||||
note_id = str(frontmatter.get("id") or "").strip()
|
||||
note_type = str(frontmatter.get("type", "")).lower()
|
||||
note_title = frontmatter.get("title", None)
|
||||
note_tags = frontmatter.get("tags", None)
|
||||
rel_path = _normalize_rel_path(rel_path)
|
||||
# types.yaml authoritative
|
||||
cp = _resolve_chunk_profile(note_type, reg)
|
||||
rw = _resolve_retriever_weight(note_type, reg)
|
||||
|
||||
# 1) Rohdaten sammeln (so wie geliefert)
|
||||
chunks_list = list(chunks)
|
||||
raw_windows: List[str] = []
|
||||
seqs: List[int] = []
|
||||
ids_in: List[Optional[str]] = []
|
||||
token_counts: List[Optional[int]] = []
|
||||
section_titles: List[Optional[str]] = []
|
||||
section_paths: List[Optional[str]] = []
|
||||
any_explicit_window = False
|
||||
tags = fm.get("tags") or []
|
||||
if isinstance(tags, str):
|
||||
tags = [tags]
|
||||
|
||||
for idx, c in enumerate(chunks_list):
|
||||
# Fensterquelle
|
||||
w = _get_attr_or_key(c, "window", None)
|
||||
if isinstance(w, str) and w:
|
||||
any_explicit_window = True
|
||||
raw_windows.append(w)
|
||||
else:
|
||||
raw_windows.append(_as_window_text(c)) # 'text'|'content'|'raw' als Ersatz
|
||||
# Ordnung
|
||||
seqs.append(_to_int(_get_attr_or_key(c, "seq", _get_attr_or_key(c, "chunk_index", idx)), idx))
|
||||
# IDs, Tokens, Sektionen
|
||||
cid = _get_attr_or_key(c, "chunk_id", _get_attr_or_key(c, "id", None))
|
||||
ids_in.append(str(cid) if isinstance(cid, str) and cid else None)
|
||||
tc = _get_attr_or_key(c, "token_count", None)
|
||||
token_counts.append(_to_int(tc, 0) if tc is not None else None)
|
||||
section_titles.append(_get_attr_or_key(c, "section_title", None))
|
||||
section_paths.append(_get_attr_or_key(c, "section_path", None))
|
||||
out: List[Dict[str, Any]] = []
|
||||
for idx, ch in enumerate(chunks_from_chunker):
|
||||
# Attribute oder Keys (Chunk-Objekt oder Dict)
|
||||
cid = getattr(ch, "id", None) or (ch.get("id") if isinstance(ch, dict) else None)
|
||||
nid = getattr(ch, "note_id", None) or (ch.get("note_id") if isinstance(ch, dict) else fm.get("id"))
|
||||
index = getattr(ch, "index", None) or (ch.get("index") if isinstance(ch, dict) else idx)
|
||||
text = getattr(ch, "text", None) or (ch.get("text") if isinstance(ch, dict) else "")
|
||||
window = getattr(ch, "window", None) or (ch.get("window") if isinstance(ch, dict) else text)
|
||||
prev_id = getattr(ch, "neighbors_prev", None) or (ch.get("neighbors_prev") if isinstance(ch, dict) else None)
|
||||
next_id = getattr(ch, "neighbors_next", None) or (ch.get("neighbors_next") if isinstance(ch, dict) else None)
|
||||
|
||||
# 2) Segmente & Overlaps bestimmen
|
||||
if any_explicit_window:
|
||||
# Es existieren echte Fenster → dedupe, um Kernsegmente zu finden
|
||||
segments, overlaps_left, recon = _dedupe_windows_to_segments(raw_windows)
|
||||
windows_final = raw_windows[:] # bereits mit Overlap geliefert
|
||||
else:
|
||||
# Keine echten Fenster → Segmente sind identisch zu "Fenstern" (bisher),
|
||||
# wir erzeugen synthetische Fenster mit Overlap gemäß Typ
|
||||
segments = [w or "" for w in raw_windows]
|
||||
overlaps_left = []
|
||||
windows_final = []
|
||||
recon = ""
|
||||
try:
|
||||
overlap_low, overlap_high = tuple(_get_sizes(note_type).get("overlap", (40, 60)))
|
||||
except Exception:
|
||||
overlap_low, overlap_high = (40, 60)
|
||||
overlap_target = int(overlap_low)
|
||||
|
||||
for i, seg in enumerate(segments):
|
||||
if i == 0:
|
||||
# erstes Fenster: kein linker Kontext
|
||||
windows_final.append(seg)
|
||||
overlaps_left.append(0)
|
||||
recon += seg
|
||||
else:
|
||||
# synthetischer linker Kontext = Suffix des bisher rekonstruierten Texts
|
||||
k = min(overlap_target, len(recon))
|
||||
left_ctx = recon[-k:] if k > 0 else ""
|
||||
windows_final.append(left_ctx + seg)
|
||||
overlaps_left.append(k)
|
||||
recon += seg # Rekonstruktion bleibt kerntreu
|
||||
|
||||
# 3) overlap_right bestimmen
|
||||
overlaps_right: List[int] = []
|
||||
for i in range(len(windows_final)):
|
||||
if i + 1 < len(windows_final):
|
||||
ov = _overlap_len_suffix_prefix(windows_final[i], windows_final[i + 1], max_probe=4096)
|
||||
else:
|
||||
ov = 0
|
||||
overlaps_right.append(ov)
|
||||
|
||||
# 4) start/end-Offsets (exakt via note_text, sonst kumulativ)
|
||||
starts: List[int] = [0] * len(segments)
|
||||
ends: List[int] = [0] * len(segments)
|
||||
pos = 0
|
||||
if isinstance(note_text, str) and note_text:
|
||||
search_pos = 0
|
||||
for i, seg in enumerate(segments):
|
||||
if not seg:
|
||||
starts[i] = ends[i] = search_pos
|
||||
continue
|
||||
j = note_text.find(seg, search_pos)
|
||||
if j >= 0:
|
||||
starts[i] = j
|
||||
ends[i] = j + len(seg)
|
||||
search_pos = ends[i]
|
||||
else:
|
||||
# Fallback: kumulativ
|
||||
starts[i] = pos
|
||||
pos += len(seg)
|
||||
ends[i] = pos
|
||||
else:
|
||||
for i, seg in enumerate(segments):
|
||||
starts[i] = pos
|
||||
pos += len(seg)
|
||||
ends[i] = pos
|
||||
|
||||
# 5) Payload-Dicts
|
||||
payloads: List[Dict[str, Any]] = []
|
||||
for i, (win, seg) in enumerate(zip(windows_final, segments)):
|
||||
chunk_id = ids_in[i] or f"{note_id}#{i+1}"
|
||||
pl: Dict[str, Any] = {
|
||||
"note_id": note_id,
|
||||
"chunk_id": chunk_id,
|
||||
"id": chunk_id, # Alias
|
||||
"chunk_index": i,
|
||||
"seq": seqs[i],
|
||||
"path": rel_path,
|
||||
"window": win,
|
||||
"text": seg,
|
||||
"start": starts[i],
|
||||
"end": ends[i],
|
||||
"overlap_left": overlaps_left[i],
|
||||
"overlap_right": overlaps_right[i],
|
||||
"note_id": nid,
|
||||
"chunk_id": cid,
|
||||
"index": int(index),
|
||||
"ord": int(index) + 1,
|
||||
"type": note_type,
|
||||
"tags": tags,
|
||||
"text": text,
|
||||
"window": window,
|
||||
"neighbors_prev": _as_list(prev_id),
|
||||
"neighbors_next": _as_list(next_id),
|
||||
"section": getattr(ch, "section", None) or (ch.get("section") if isinstance(ch, dict) else ""),
|
||||
"path": note_path,
|
||||
"source_path": file_path or note_path,
|
||||
"retriever_weight": float(rw),
|
||||
"chunk_profile": cp,
|
||||
}
|
||||
# optionale Metadaten
|
||||
if note_type:
|
||||
pl["type"] = note_type
|
||||
if note_title is not None:
|
||||
pl["title"] = note_title
|
||||
if note_tags is not None:
|
||||
pl["tags"] = note_tags
|
||||
if token_counts[i] is not None:
|
||||
pl["token_count"] = int(token_counts[i])
|
||||
if section_titles[i] is not None:
|
||||
pl["section_title"] = section_titles[i]
|
||||
if section_paths[i] is not None:
|
||||
sp = str(section_paths[i]).replace("\\", "/")
|
||||
pl["section_path"] = sp if sp else "/"
|
||||
payloads.append(pl)
|
||||
# Aufräumen von Alt-Feldern
|
||||
for alias in ("chunk_num", "Chunk_Number"):
|
||||
pl.pop(alias, None)
|
||||
|
||||
return payloads
|
||||
out.append(pl)
|
||||
|
||||
|
||||
# __main__ Demo (optional)
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
fm = {"id": "demo", "title": "Demo", "type": "concept"}
|
||||
# Beispiel ohne echte Fenster → erzeugt synthetische Overlaps
|
||||
chunks = [
|
||||
{"id": "demo#1", "text": "Alpha Beta Gamma"},
|
||||
{"id": "demo#2", "text": "Gamma Delta"},
|
||||
{"id": "demo#3", "text": "Delta Epsilon Zeta"},
|
||||
]
|
||||
pls = make_chunk_payloads(fm, "path/demo.md", chunks, note_text="Alpha Beta Gamma Delta Epsilon Zeta")
|
||||
from pprint import pprint
|
||||
pprint(pls)
|
||||
recon = "".join(p["text"] for p in pls)
|
||||
print("RECON:", recon)
|
||||
return out
|
||||
@@ -3,10 +3,7 @@
|
||||
"""
|
||||
Modul: app/core/derive_edges.py
|
||||
Version: 1.4.0
|
||||
Datum: 2025-10-01
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Robuste Kantenbildung für mindnet (Notes/Chunks):
|
||||
- belongs_to (chunk -> note)
|
||||
- next / prev (chunk-Kette)
|
||||
@@ -15,25 +12,12 @@ Robuste Kantenbildung für mindnet (Notes/Chunks):
|
||||
|
||||
Wichtig: Wikilinks werden mit der Parser-Funktion `extract_wikilinks` extrahiert,
|
||||
damit Varianten wie [[id#anchor]] oder [[id|label]] korrekt auf 'id' reduziert werden.
|
||||
|
||||
Erwartete Chunk-Payload-Felder:
|
||||
{
|
||||
"note_id": "...",
|
||||
"chunk_id": "...", # Alias "id" ist zulässig
|
||||
"id": "...",
|
||||
"chunk_index": int,
|
||||
"seq": int,
|
||||
"window": str,
|
||||
"text": str,
|
||||
"path": "rel/path.md",
|
||||
...
|
||||
}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Optional, Iterable
|
||||
|
||||
# WICHTIG: benutze die Parser-Extraktion für saubere Wikilinks
|
||||
# Parser-Extraktion für saubere Wikilinks
|
||||
from app.core.parser import extract_wikilinks
|
||||
|
||||
def _get(d: dict, *keys, default=None):
|
||||
@@ -113,7 +97,7 @@ def build_edges_for_note(
|
||||
if not cid:
|
||||
continue
|
||||
txt = _chunk_text_for_refs(ch)
|
||||
refs = extract_wikilinks(txt) # <— Parser-Logik, kompatibel zu deinem System
|
||||
refs = extract_wikilinks(txt) # Parser-Logik
|
||||
for r in refs:
|
||||
edges.append(_edge("references", "chunk", cid, r, note_id, {"chunk_id": cid, "ref_text": r}))
|
||||
refs_all.extend(refs)
|
||||
@@ -128,4 +112,9 @@ def build_edges_for_note(
|
||||
edges.append(_edge("references", "note", note_id, r, note_id))
|
||||
edges.append(_edge("backlink", "note", r, note_id, note_id))
|
||||
|
||||
return edges
|
||||
# Dedupe (Schlüssel: kind,source_id,target_id,scope)
|
||||
dedup = {}
|
||||
for e in edges:
|
||||
k = (e["kind"], e["source_id"], e["target_id"], e.get("scope", ""))
|
||||
dedup[k] = e
|
||||
return list(dedup.values())
|
||||
+257
-81
@@ -2,119 +2,295 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/edges.py
|
||||
Version: 1.0.0
|
||||
Datum: 2025-09-09
|
||||
Version: 2.0.0 (V2‑superset, rückwärtskompatibel zu v1 vom 2025‑09‑09)
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Zentrale, konsistente Erzeugung von Edge-Payloads im **neuen Schema**:
|
||||
- kind : "belongs_to" | "next" | "prev" | "references" | "backlink"
|
||||
- source_id : ID des Quellknotens (Chunk- oder Note-ID)
|
||||
- target_id : ID des Zielknotens
|
||||
- scope : "chunk" | "note"
|
||||
- note_id : Owner-Note (für performantes Filtern/Löschen)
|
||||
- seq : optional (z. B. Reihenfolge von Vorkommen)
|
||||
Bewahrt die bestehende Edgelogik (belongs_to, prev/next, references, backlink)
|
||||
und ergänzt V2‑Felder + Typ‑Default‑Kanten gemäß config/types.yaml (edge_defaults).
|
||||
Die Funktion ist **idempotent** und **rückwärtskompatibel** zur bisherigen Signatur.
|
||||
|
||||
Hinweise
|
||||
--------
|
||||
- Edges werden dedupliziert (key=(kind,source_id,target_id,scope)).
|
||||
- Für Chunk-Edges wird `note_id` aus dem Chunk-Payload entnommen.
|
||||
- Für Note-Scope-Edges ist `note_id` die Quell-Note-ID.
|
||||
Kompatibilitätsgarantien (gegenüber v1):
|
||||
- **Input**: akzeptiert identische Chunk‑Payloads wie v1:
|
||||
* `id` (Chunk‑ID), `note_id` (Owner), `neighbors.prev|next` (optional),
|
||||
`references: [{target_id: ...}]` (optional),
|
||||
alternativ: `chunk_id`, `chunk_index|ord`, `window|text`
|
||||
- **Output (v1‑Felder)**: `kind`, `source_id`, `target_id`, `scope`, `note_id`, `edge_id`
|
||||
- **Neu (v2‑Felder)**: `relation`, `src_note_id`, `src_chunk_id?`, `dst_note_id`, `dst_chunk_id?`,
|
||||
`provenance` (`explicit|rule`), `rule_id?`, `confidence?`
|
||||
|
||||
Regeln
|
||||
------
|
||||
- Deduplizierungsschlüssel: (source_id, target_id, relation, rule_id)
|
||||
- Strukturkanten:
|
||||
* belongs_to: 1× pro Chunk
|
||||
* next/prev: Sequenz der Chunks; nutzt bevorzugt neighbors; sonst ord/chunk_index
|
||||
- Explizite Referenzen:
|
||||
* aus Chunk: `references[].target_id` (falls vorhanden)
|
||||
* Fallback: Wikilinks in `window|text`: [[Some Title|some-id]] oder [[some-id]]
|
||||
- Note‑Scope:
|
||||
* backlink immer; references nur, wenn include_note_scope_refs=True
|
||||
- Typ‑Defaults (edge_defaults aus config/types.yaml des **Quell‑Notiztyps**):
|
||||
* Für jede explizite Referenz wird je default‑Relation eine Regel‑Kante erzeugt
|
||||
* rule_id: "type_default:{note_type}:{relation}:v1", provenance="rule"
|
||||
|
||||
Konfiguration
|
||||
-------------
|
||||
- ENV MINDNET_TYPES_FILE (Default: ./config/types.yaml)
|
||||
|
||||
Lizenz/Autor
|
||||
------------
|
||||
- Erstimplementierung v1 (2025‑09‑09) — Projekt Mindnet
|
||||
- Erweiterung v2 (2025‑11‑11) — kompatible Superset‑Implementierung
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Dict, List
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, Iterable, List, Optional, Tuple, Set
|
||||
|
||||
try:
|
||||
import yaml # optional, nur für types.yaml
|
||||
except Exception: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Hilfen: types.yaml laden (edge_defaults)
|
||||
# ------------------------------------------------------------
|
||||
|
||||
def _types_path() -> str:
|
||||
return os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
|
||||
|
||||
def _load_types() -> Dict[str, dict]:
|
||||
p = _types_path()
|
||||
if not os.path.isfile(p) or yaml is None:
|
||||
return {}
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
|
||||
return data["types"]
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def _edge_defaults_for(note_type: Optional[str]) -> List[str]:
|
||||
types = _load_types()
|
||||
t = (note_type or "").strip().lower()
|
||||
cfg = types.get(t) or {}
|
||||
defaults = cfg.get("edge_defaults") or []
|
||||
if isinstance(defaults, str):
|
||||
defaults = [defaults]
|
||||
return [str(x) for x in defaults if isinstance(x, (str, int, float))]
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Wikilink‑Parser (Fallback, wenn ch["references"] fehlt)
|
||||
# ------------------------------------------------------------
|
||||
|
||||
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([a-zA-Z0-9_\-#:. ]+)\]\]")
|
||||
|
||||
def _extract_wikilinks(text: str) -> List[str]:
|
||||
ids: List[str] = []
|
||||
for m in _WIKILINK_RE.finditer(text or ""):
|
||||
ids.append(m.group(1).strip())
|
||||
return ids
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Utility
|
||||
# ------------------------------------------------------------
|
||||
|
||||
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None) -> str:
|
||||
base = f"{kind}:{s}->{t}#{scope}"
|
||||
if rule_id:
|
||||
base += f"|{rule_id}"
|
||||
try:
|
||||
import hashlib
|
||||
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||
except Exception: # pragma: no cover
|
||||
return base
|
||||
|
||||
def _dedupe(edges: List[Dict]) -> List[Dict]:
|
||||
seen: Set[Tuple[str,str,str,str]] = set()
|
||||
out: List[Dict] = []
|
||||
for e in edges:
|
||||
s = str(e.get("source_id") or "")
|
||||
t = str(e.get("target_id") or "")
|
||||
rel = str(e.get("relation") or e.get("kind") or "edge")
|
||||
rule = str(e.get("rule_id") or "")
|
||||
key = (s, t, rel, rule)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(e)
|
||||
return out
|
||||
|
||||
def _first(v: dict, *keys, default=None):
|
||||
for k in keys:
|
||||
if k in v and v[k] is not None:
|
||||
return v[k]
|
||||
return default
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Hauptfunktion
|
||||
# ------------------------------------------------------------
|
||||
|
||||
def build_edges_for_note(
|
||||
note_id: str,
|
||||
chunk_payloads: List[Dict],
|
||||
note_level_refs: List[str] | None,
|
||||
note_level_refs: Optional[List[str]] = None,
|
||||
*,
|
||||
include_note_scope_refs: bool = False,
|
||||
) -> List[Dict]:
|
||||
edges: List[Dict] = []
|
||||
chunks = list(chunk_payloads or [])
|
||||
# Notiztyp aus erstem Chunk ableiten (kompatibel zu existierenden Payloads)
|
||||
note_type = (chunks[0].get("type") if chunks else None) or (chunks[0].get("note_type") if chunks else None)
|
||||
|
||||
# Chunk-Scope: belongs_to / prev / next / references
|
||||
for ch in chunk_payloads:
|
||||
cid = ch["id"]
|
||||
# --- Strukturkanten ------------------------------------------------------
|
||||
# belongs_to
|
||||
for ch in chunks:
|
||||
cid = _first(ch, "id", "chunk_id")
|
||||
if not cid:
|
||||
continue
|
||||
owner = ch.get("note_id") or note_id
|
||||
# belongs_to
|
||||
edges.append({
|
||||
e = {
|
||||
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to:v1"),
|
||||
"kind": "belongs_to",
|
||||
"relation": "belongs_to",
|
||||
"scope": "chunk",
|
||||
"source_id": cid,
|
||||
"target_id": note_id,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
# Nachbarn
|
||||
"note_id": owner, # v1-Kompat
|
||||
# v2
|
||||
"src_note_id": owner,
|
||||
"src_chunk_id": cid,
|
||||
"dst_note_id": note_id,
|
||||
"provenance": "rule",
|
||||
"rule_id": "structure:belongs_to:v1",
|
||||
"confidence": 1.0,
|
||||
}
|
||||
edges.append(e)
|
||||
|
||||
# next/prev — bevorzugt neighbors.prev/next; sonst via ord/chunk_index
|
||||
# Map der Chunks nach Index
|
||||
ordered = list(chunks)
|
||||
def _idx(c):
|
||||
return _first(c, "chunk_index", "ord", default=0)
|
||||
ordered.sort(key=_idx)
|
||||
|
||||
for i, ch in enumerate(ordered):
|
||||
cid = _first(ch, "id", "chunk_id")
|
||||
if not cid:
|
||||
continue
|
||||
owner = ch.get("note_id") or note_id
|
||||
nb = ch.get("neighbors") or {}
|
||||
prev_id = nb.get("prev")
|
||||
next_id = nb.get("next")
|
||||
# Fallback-Reihenfolge
|
||||
if prev_id is None and i > 0:
|
||||
prev_id = _first(ordered[i-1], "id", "chunk_id")
|
||||
if next_id is None and i+1 < len(ordered):
|
||||
next_id = _first(ordered[i+1], "id", "chunk_id")
|
||||
|
||||
if prev_id:
|
||||
edges.append({
|
||||
"kind": "prev",
|
||||
"source_id": cid,
|
||||
"target_id": prev_id,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
"edge_id": _mk_edge_id("prev", cid, prev_id, "chunk", "structure:order:v1"),
|
||||
"kind": "prev", "relation": "prev", "scope": "chunk",
|
||||
"source_id": cid, "target_id": prev_id, "note_id": owner,
|
||||
"src_note_id": owner, "src_chunk_id": cid,
|
||||
"dst_note_id": owner, "dst_chunk_id": prev_id,
|
||||
"provenance": "rule", "rule_id": "structure:order:v1", "confidence": 0.95,
|
||||
})
|
||||
edges.append({
|
||||
"kind": "next",
|
||||
"source_id": prev_id,
|
||||
"target_id": cid,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
if next_id:
|
||||
edges.append({
|
||||
"kind": "next",
|
||||
"source_id": cid,
|
||||
"target_id": next_id,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
edges.append({
|
||||
"kind": "prev",
|
||||
"source_id": next_id,
|
||||
"target_id": cid,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
# references aus Chunk
|
||||
for ref in (ch.get("references") or []):
|
||||
tid = ref.get("target_id")
|
||||
if not tid:
|
||||
continue
|
||||
edges.append({
|
||||
"kind": "references",
|
||||
"source_id": cid,
|
||||
"target_id": tid,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
"edge_id": _mk_edge_id("next", prev_id, cid, "chunk", "structure:order:v1"),
|
||||
"kind": "next", "relation": "next", "scope": "chunk",
|
||||
"source_id": prev_id, "target_id": cid, "note_id": owner,
|
||||
"src_note_id": owner, "src_chunk_id": prev_id,
|
||||
"dst_note_id": owner, "dst_chunk_id": cid,
|
||||
"provenance": "rule", "rule_id": "structure:order:v1", "confidence": 0.95,
|
||||
})
|
||||
|
||||
# Note-Scope: backlink (immer); references (optional)
|
||||
unique_refs = list(dict.fromkeys(note_level_refs or []))
|
||||
# --- Explizite Referenzen (Chunk‑Scope) ---------------------------------
|
||||
explicit_refs: List[Dict] = []
|
||||
for ch in chunks:
|
||||
cid = _first(ch, "id", "chunk_id")
|
||||
if not cid:
|
||||
continue
|
||||
owner = ch.get("note_id") or note_id
|
||||
# 1) bevorzugt vorhandene ch["references"]
|
||||
refs = ch.get("references") or []
|
||||
targets = [r.get("target_id") for r in refs if isinstance(r, dict) and r.get("target_id")]
|
||||
# 2) Fallback: Wikilinks aus Text
|
||||
if not targets:
|
||||
text = _first(ch, "window", "text", default="") or ""
|
||||
targets = _extract_wikilinks(text)
|
||||
for tid in targets:
|
||||
if not isinstance(tid, str) or not tid.strip():
|
||||
continue
|
||||
e = {
|
||||
"edge_id": _mk_edge_id("references", cid, tid, "chunk"),
|
||||
"kind": "references",
|
||||
"relation": "references",
|
||||
"scope": "chunk",
|
||||
"source_id": cid,
|
||||
"target_id": tid,
|
||||
"note_id": owner,
|
||||
# v2
|
||||
"src_note_id": owner,
|
||||
"src_chunk_id": cid,
|
||||
"dst_note_id": tid,
|
||||
"provenance": "explicit",
|
||||
"rule_id": "",
|
||||
"confidence": 1.0,
|
||||
}
|
||||
edges.append(e)
|
||||
explicit_refs.append(e)
|
||||
|
||||
# --- Note‑Scope: references (optional) + backlink (immer) ----------------
|
||||
unique_refs = []
|
||||
if note_level_refs:
|
||||
seen = set()
|
||||
for tid in note_level_refs:
|
||||
if isinstance(tid, str) and tid.strip() and tid not in seen:
|
||||
unique_refs.append(tid); seen.add(tid)
|
||||
|
||||
for tid in unique_refs:
|
||||
if include_note_scope_refs:
|
||||
edges.append({
|
||||
"kind": "references",
|
||||
"source_id": note_id,
|
||||
"target_id": tid,
|
||||
"scope": "note",
|
||||
"note_id": note_id,
|
||||
"edge_id": _mk_edge_id("references", note_id, tid, "note"),
|
||||
"kind": "references", "relation": "references", "scope": "note",
|
||||
"source_id": note_id, "target_id": tid, "note_id": note_id,
|
||||
"src_note_id": note_id, "dst_note_id": tid,
|
||||
"provenance": "explicit", "rule_id": "", "confidence": 1.0,
|
||||
})
|
||||
edges.append({
|
||||
"kind": "backlink",
|
||||
"source_id": tid,
|
||||
"target_id": note_id,
|
||||
"scope": "note",
|
||||
"note_id": note_id,
|
||||
"edge_id": _mk_edge_id("backlink", tid, note_id, "note", "derived:backlink:v1"),
|
||||
"kind": "backlink", "relation": "backlink", "scope": "note",
|
||||
"source_id": tid, "target_id": note_id, "note_id": note_id,
|
||||
"src_note_id": tid, "dst_note_id": note_id,
|
||||
"provenance": "rule", "rule_id": "derived:backlink:v1", "confidence": 0.9,
|
||||
})
|
||||
|
||||
# Dedupe
|
||||
dedup = {}
|
||||
for e in edges:
|
||||
k = (e["kind"], e["source_id"], e["target_id"], e.get("scope", ""))
|
||||
dedup[k] = e
|
||||
return list(dedup.values())
|
||||
# --- Type‑Defaults je expliziter Referenz --------------------------------
|
||||
defaults = [d for d in _edge_defaults_for(note_type) if d and d != "references"]
|
||||
if defaults:
|
||||
for e in explicit_refs + ([ ] if not include_note_scope_refs else []):
|
||||
# wir nutzen die bereits erzeugten explicit‑Edges als Vorlage
|
||||
src = e["source_id"]; tgt = e["target_id"]
|
||||
scope = e.get("scope", "chunk")
|
||||
s_note = e.get("src_note_id") or note_id
|
||||
s_chunk = e.get("src_chunk_id")
|
||||
t_note = e.get("dst_note_id") or tgt
|
||||
for rel in defaults:
|
||||
rule_id = f"type_default:{(note_type or 'unknown')}:{rel}:v1"
|
||||
edges.append({
|
||||
"edge_id": _mk_edge_id(rel, src, tgt, scope, rule_id),
|
||||
"kind": rel, "relation": rel, "scope": scope,
|
||||
"source_id": src, "target_id": tgt, "note_id": s_note,
|
||||
"src_note_id": s_note, "src_chunk_id": s_chunk,
|
||||
"dst_note_id": t_note,
|
||||
"provenance": "rule", "rule_id": rule_id, "confidence": 0.7,
|
||||
})
|
||||
|
||||
# --- Dedupe & Return -----------------------------------------------------
|
||||
return _dedupe(edges)
|
||||
@@ -0,0 +1,94 @@
|
||||
# app/core/edges_writer.py
|
||||
from __future__ import annotations
|
||||
import hashlib
|
||||
from typing import Dict, List, Iterable, Tuple
|
||||
|
||||
try:
|
||||
# Dein Modul mit der Schemadefinition und der Builder-Funktion
|
||||
from app.core.edges import build_edges_for_note # noqa: F401
|
||||
except Exception as e:
|
||||
raise RuntimeError("Konnte app.core.edges nicht importieren. "
|
||||
"Bitte sicherstellen, dass app/core/edges.py vorhanden ist.") from e
|
||||
|
||||
def _edge_uid(kind: str, source_id: str, target_id: str, scope: str) -> str:
|
||||
"""
|
||||
Deterministische, kurze ID für eine Edge.
|
||||
Kollisionen sind praktisch ausgeschlossen (BLAKE2s über den Kanonischen Schlüssel).
|
||||
"""
|
||||
key = f"{kind}|{source_id}|{target_id}|{scope}"
|
||||
return hashlib.blake2s(key.encode("utf-8"), digest_size=12).hexdigest()
|
||||
|
||||
def ensure_edges_collection(qdrant_client, collection: str) -> None:
|
||||
"""
|
||||
Legt die Edge-Collection an, falls sie nicht existiert.
|
||||
Minimal: 1D-Vector (Dummy), Cosine. Payload-only-Collections sind je nach Qdrant-Version heikel.
|
||||
"""
|
||||
from qdrant_client.http import models as qm
|
||||
|
||||
existing = [c.name for c in qdrant_client.get_collections().collections]
|
||||
if collection in existing:
|
||||
return
|
||||
|
||||
qdrant_client.recreate_collection(
|
||||
collection_name=collection,
|
||||
vectors_config=qm.VectorParams(size=1, distance=qm.Distance.COSINE),
|
||||
on_disk_payload=True,
|
||||
)
|
||||
|
||||
def edges_from_note(
|
||||
note_id: str,
|
||||
chunk_payloads: List[Dict],
|
||||
note_level_refs: Iterable[str] | None,
|
||||
*,
|
||||
include_note_scope_refs: bool = False,
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Ruft deinen Edge-Builder auf und gibt die (deduplizierten) Edge-Payloads zurück.
|
||||
Keine Schemaänderung – exakt das aus app/core/edges.py.
|
||||
"""
|
||||
return build_edges_for_note(
|
||||
note_id=note_id,
|
||||
chunk_payloads=chunk_payloads,
|
||||
note_level_refs=list(note_level_refs or []),
|
||||
include_note_scope_refs=include_note_scope_refs,
|
||||
)
|
||||
|
||||
def upsert_edges(
|
||||
qdrant_client,
|
||||
collection: str,
|
||||
edge_payloads: List[Dict],
|
||||
) -> Tuple[int, int]:
|
||||
"""
|
||||
Schreibt Edges als Points in Qdrant.
|
||||
- id: deterministisch aus (kind, source_id, target_id, scope)
|
||||
- vector: [0.0] Dummy
|
||||
- payload: Edge-Dict (unverändert, siehe Schema in app/core/edges.py)
|
||||
Gibt (anzahl_points, anzahl_unique_keys) zurück.
|
||||
"""
|
||||
from qdrant_client.models import PointStruct
|
||||
|
||||
if not edge_payloads:
|
||||
return 0, 0
|
||||
|
||||
points = []
|
||||
seen = set()
|
||||
for e in edge_payloads:
|
||||
key = (e.get("kind"), e.get("source_id"), e.get("target_id"), e.get("scope"))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
eid = _edge_uid(*key)
|
||||
points.append(
|
||||
PointStruct(
|
||||
id=eid,
|
||||
vector=[0.0],
|
||||
payload=e,
|
||||
)
|
||||
)
|
||||
|
||||
if not points:
|
||||
return 0, 0
|
||||
|
||||
ensure_edges_collection(qdrant_client, collection)
|
||||
qdrant_client.upsert(collection_name=collection, points=points)
|
||||
return len(points), len(seen)
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Datei: app/core/env_vars.py
|
||||
Version: 1.1.0 (2025-11-08)
|
||||
|
||||
Zweck
|
||||
Einheitliche Auflösung von ENV-Variablen (Prefix, Qdrant, Embeddings, Hashing)
|
||||
mit Abwärtskompatibilität.
|
||||
|
||||
Grundsatz
|
||||
- Für Qdrant-Funktionen ist 'COLLECTION_PREFIX' der Primärschlüssel.
|
||||
- 'MINDNET_PREFIX' bleibt für App-/UI-/Exporter-Kontexte nutzbar.
|
||||
- Fallbacks sorgen dafür, dass ältere Umgebungen weiter funktionieren.
|
||||
|
||||
Wichtig
|
||||
- Lädt optional eine .env (wenn python-dotenv verfügbar ist).
|
||||
- Überschreibt keine bereits gesetzten OS-Variablen (override=False).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Optional, Dict
|
||||
|
||||
# Optional: .env automatisch laden (ohne Hard-Fail, falls nicht vorhanden)
|
||||
try:
|
||||
from dotenv import load_dotenv, find_dotenv # type: ignore
|
||||
_p = find_dotenv()
|
||||
if _p:
|
||||
load_dotenv(_p, override=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# -------- Prefix-Auflösung --------
|
||||
|
||||
def get_collection_prefix(cli_override: Optional[str] = None) -> str:
|
||||
"""
|
||||
Für Qdrant-relevante Funktionen:
|
||||
1) CLI-Override (--prefix)
|
||||
2) ENV COLLECTION_PREFIX
|
||||
3) ENV MINDNET_PREFIX (Fallback)
|
||||
4) 'mindnet' (Default)
|
||||
"""
|
||||
if cli_override and str(cli_override).strip():
|
||||
return str(cli_override).strip()
|
||||
return (
|
||||
os.getenv("COLLECTION_PREFIX")
|
||||
or os.getenv("MINDNET_PREFIX")
|
||||
or "mindnet"
|
||||
)
|
||||
|
||||
def get_mindnet_prefix(cli_override: Optional[str] = None) -> str:
|
||||
"""
|
||||
Für App-/UI-/Exporter-Kontexte:
|
||||
1) CLI-Override (--prefix)
|
||||
2) ENV MINDNET_PREFIX
|
||||
3) ENV COLLECTION_PREFIX (Fallback)
|
||||
4) 'mindnet'
|
||||
"""
|
||||
if cli_override and str(cli_override).strip():
|
||||
return str(cli_override).strip()
|
||||
return (
|
||||
os.getenv("MINDNET_PREFIX")
|
||||
or os.getenv("COLLECTION_PREFIX")
|
||||
or "mindnet"
|
||||
)
|
||||
|
||||
def get_prefix(cli_override: Optional[str] = None, target: str = "qdrant") -> str:
|
||||
"""
|
||||
Universelle Hülle (abwärtskompatibel):
|
||||
target='qdrant' -> get_collection_prefix
|
||||
target='app' -> get_mindnet_prefix
|
||||
"""
|
||||
if target.lower() == "app":
|
||||
return get_mindnet_prefix(cli_override)
|
||||
return get_collection_prefix(cli_override)
|
||||
|
||||
# -------- Qdrant / Embeddings / Hashing --------
|
||||
|
||||
def get_qdrant_url(default: str = "http://127.0.0.1:6333") -> str:
|
||||
return os.getenv("QDRANT_URL", default)
|
||||
|
||||
def get_qdrant_api_key(default: str = "") -> str:
|
||||
return os.getenv("QDRANT_API_KEY", default)
|
||||
|
||||
def get_vector_dim(default: int = 384) -> int:
|
||||
try:
|
||||
return int(os.getenv("VECTOR_DIM", str(default)))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def get_embed_url(default: Optional[str] = None) -> Optional[str]:
|
||||
return os.getenv("EMBED_URL", default)
|
||||
|
||||
def get_hash_env() -> Dict[str, str]:
|
||||
"""
|
||||
Liefert die Hash-Konfiguration (nur Aggregation; die Auswertung bleibt in den Skripten).
|
||||
"""
|
||||
return {
|
||||
"MINDNET_HASH_COMPARE": os.getenv("MINDNET_HASH_COMPARE", ""),
|
||||
"MINDNET_HASH_SOURCE": os.getenv("MINDNET_HASH_SOURCE", ""),
|
||||
"MINDNET_HASH_NORMALIZE": os.getenv("MINDNET_HASH_NORMALIZE", ""),
|
||||
}
|
||||
+131
-201
@@ -1,229 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Modul: app/core/note_payload.py
|
||||
# Version: 1.7.0
|
||||
# Datum: 2025-09-09
|
||||
"""
|
||||
app/core/note_payload.py (Mindnet V2 — types.yaml authoritative)
|
||||
- retriever_weight und chunk_profile werden aus config/types.yaml gelesen.
|
||||
- Reihenfolge: types.<type> > defaults.* > Fallbacks (1.0 / "default").
|
||||
- Frontmatter-Overrides für diese beiden Felder werden bewusst IGNORIERT.
|
||||
- edge_defaults (falls in types.yaml definiert) werden in die Note-Payload gespiegelt.
|
||||
- MINDNET_TYPES_FILE kann absoluten Pfad liefern, sonst ./config/types.yaml.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, Optional, List
|
||||
import os, yaml, hashlib, datetime
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
# ----------------------- helpers -----------------------
|
||||
|
||||
try:
|
||||
from app.core.parser import read_markdown, extract_wikilinks, FRONTMATTER_RE
|
||||
except Exception: # pragma: no cover
|
||||
from .parser import read_markdown, extract_wikilinks, FRONTMATTER_RE # type: ignore
|
||||
def _env(n: str, d: Optional[str]=None) -> str:
|
||||
v = os.getenv(n)
|
||||
return v if v is not None else (d or "")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
|
||||
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
def _normalize_body(body: str, mode: str) -> str:
|
||||
if mode == "none":
|
||||
return body if body is not None else ""
|
||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
|
||||
text = "\n".join(line.rstrip() for line in text.split("\n"))
|
||||
return text
|
||||
|
||||
def _resolve_hash_mode(explicit: Optional[str]) -> str:
|
||||
if explicit:
|
||||
val = explicit.strip().lower()
|
||||
else:
|
||||
val = (os.environ.get("MINDNET_HASH_MODE")
|
||||
or os.environ.get("MINDNET_HASH_COMPARE")
|
||||
or "body").strip().lower()
|
||||
if val in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
||||
return "full"
|
||||
if val in ("frontmatter", "fm"):
|
||||
return "frontmatter"
|
||||
return "body"
|
||||
|
||||
def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, Any]]:
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return "", {}
|
||||
def _load_types() -> dict:
|
||||
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
raw = f.read()
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
except Exception:
|
||||
return "", {}
|
||||
m = FRONTMATTER_RE.match(raw)
|
||||
fm = {}
|
||||
if m:
|
||||
fm_txt = m.group(1)
|
||||
try:
|
||||
import yaml # lazy
|
||||
fm = yaml.safe_load(fm_txt) or {}
|
||||
except Exception:
|
||||
fm = {}
|
||||
body = raw[m.end():]
|
||||
else:
|
||||
body = raw
|
||||
return body, fm
|
||||
return {}
|
||||
|
||||
def _sha256(s: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
h.update(s.encode("utf-8"))
|
||||
return h.hexdigest()
|
||||
def _get_types_map(reg: dict) -> dict:
|
||||
if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
|
||||
return reg["types"]
|
||||
return reg if isinstance(reg, dict) else {}
|
||||
|
||||
def _hash_for(mode: str, *, body: str, fm: Dict[str, Any], normalize: str) -> str:
|
||||
body_n = _normalize_body(body or "", normalize)
|
||||
fm_s = _canon_frontmatter(fm or {})
|
||||
if mode == "frontmatter":
|
||||
return _sha256(fm_s)
|
||||
if mode == "full":
|
||||
return _sha256(body_n + "\n--FM--\n" + fm_s)
|
||||
# default: body
|
||||
return _sha256(body_n)
|
||||
def _get_defaults(reg: dict) -> dict:
|
||||
if isinstance(reg, dict) and isinstance(reg.get("defaults"), dict):
|
||||
return reg["defaults"]
|
||||
# alias "global" erlaubt
|
||||
if isinstance(reg, dict) and isinstance(reg.get("global"), dict):
|
||||
return reg["global"]
|
||||
return {}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kernfunktion
|
||||
# ---------------------------------------------------------------------------
|
||||
def _resolve_chunk_profile(note_type: str, reg: dict) -> str:
|
||||
types = _get_types_map(reg)
|
||||
if isinstance(types, dict):
|
||||
t = types.get(note_type, {})
|
||||
if isinstance(t, dict) and isinstance(t.get("chunk_profile"), str):
|
||||
return t["chunk_profile"]
|
||||
defs = _get_defaults(reg)
|
||||
if isinstance(defs, dict) and isinstance(defs.get("chunk_profile"), str):
|
||||
return defs["chunk_profile"]
|
||||
return "default"
|
||||
|
||||
def make_note_payload(
|
||||
parsed: Any,
|
||||
vault_root: Optional[str] = None,
|
||||
*,
|
||||
hash_mode: Optional[str] = None,
|
||||
hash_normalize: Optional[str] = None,
|
||||
hash_source: Optional[str] = None,
|
||||
file_path: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Liefert den Note-Payload inkl. Mehrfach-Hashes.
|
||||
- Es werden IMMER die drei Hashes für (body|frontmatter|full) unter
|
||||
'parsed:canonical' erzeugt (Schlüssel: z. B. 'body:parsed:canonical').
|
||||
- Zusätzlich werden – falls die aktuelle Konfig (source/normalize) davon
|
||||
abweicht – die drei Hashes unter den entsprechenden Schlüsseln erzeugt,
|
||||
z. B. 'frontmatter:raw:none'.
|
||||
- 'hash_fulltext' und 'hash_signature' repräsentieren den *aktuellen* Modus.
|
||||
"""
|
||||
# dict oder Objekt akzeptieren
|
||||
if isinstance(parsed, dict):
|
||||
fm = parsed.get("frontmatter") or {}
|
||||
body_parsed = parsed.get("body") or ""
|
||||
path = parsed.get("path") or ""
|
||||
else:
|
||||
fm = getattr(parsed, "frontmatter", {}) or {}
|
||||
body_parsed = getattr(parsed, "body", "") or ""
|
||||
path = getattr(parsed, "path", "") or ""
|
||||
|
||||
# Zielpfad relativieren
|
||||
rel_path = path
|
||||
def _as_float(x: Any) -> Optional[float]:
|
||||
try:
|
||||
if vault_root:
|
||||
rel = os.path.relpath(path, vault_root)
|
||||
rel = rel.replace("\\", "/").lstrip("/")
|
||||
rel_path = rel
|
||||
return float(x)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
# Konfiguration auflösen
|
||||
mode_resolved = _resolve_hash_mode(hash_mode) # body|frontmatter|full
|
||||
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower() # parsed|raw
|
||||
norm = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower() # canonical|none
|
||||
def _resolve_retriever_weight(note_type: str, reg: dict) -> float:
|
||||
types = _get_types_map(reg)
|
||||
if isinstance(types, dict):
|
||||
t = types.get(note_type, {})
|
||||
if isinstance(t, dict) and (t.get("retriever_weight") is not None):
|
||||
v = _as_float(t.get("retriever_weight"))
|
||||
if v is not None:
|
||||
return float(v)
|
||||
defs = _get_defaults(reg)
|
||||
if isinstance(defs, dict) and (defs.get("retriever_weight") is not None):
|
||||
v = _as_float(defs.get("retriever_weight"))
|
||||
if v is not None:
|
||||
return float(v)
|
||||
return 1.0
|
||||
|
||||
# Body-Quelle laden
|
||||
raw_body, raw_fm = ("", {})
|
||||
if src == "raw":
|
||||
raw_body, raw_fm = _read_raw_body_from_file(file_path or path)
|
||||
if isinstance(raw_fm, dict) and raw_fm:
|
||||
merged_fm = dict(fm)
|
||||
for k, v in raw_fm.items():
|
||||
merged_fm.setdefault(k, v)
|
||||
fm = merged_fm
|
||||
body_for_hash = raw_body
|
||||
else:
|
||||
body_for_hash = body_parsed
|
||||
# ------------------- public API -------------------
|
||||
|
||||
# --- 1) Standard-Tripel (parsed:canonical) immer erzeugen ---
|
||||
std_src = "parsed"
|
||||
std_norm = "canonical"
|
||||
std_hashes: Dict[str, str] = {}
|
||||
for m in ("body", "frontmatter", "full"):
|
||||
std_hashes[f"{m}:{std_src}:{std_norm}"] = _hash_for(
|
||||
m, body=body_parsed, fm=fm, normalize=std_norm
|
||||
)
|
||||
def make_note_payload(parsed_note: Any,
|
||||
*,
|
||||
vault_root: str,
|
||||
hash_mode: str = "body",
|
||||
hash_normalize: str = "canonical",
|
||||
hash_source: str = "parsed",
|
||||
file_path: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Erwartet ein Parsed-Objekt mit Attributen:
|
||||
- frontmatter (dict)
|
||||
- body (str)
|
||||
- path (optional)
|
||||
Liefert Note-Payload mit deterministischer note_id und Typ-Werten aus types.yaml.
|
||||
"""
|
||||
fm = (getattr(parsed_note, "frontmatter", None) or {}) if parsed_note else {}
|
||||
title = fm.get("title") or ""
|
||||
note_type = fm.get("type") or "concept"
|
||||
note_id = fm.get("id") or _stable_id_from_path_or_title(file_path or "", title)
|
||||
tags = fm.get("tags") or []
|
||||
if isinstance(tags, str):
|
||||
tags = [tags]
|
||||
|
||||
# Convenience-Felder (für Tools)
|
||||
hash_body = std_hashes["body:parsed:canonical"]
|
||||
hash_frontmatter = std_hashes["frontmatter:parsed:canonical"]
|
||||
hash_full = std_hashes["full:parsed:canonical"]
|
||||
# types.yaml authoritative:
|
||||
reg = _load_types()
|
||||
chunk_profile = _resolve_chunk_profile(note_type, reg)
|
||||
retriever_weight = _resolve_retriever_weight(note_type, reg)
|
||||
# edge_defaults (falls vorhanden)
|
||||
edge_defaults = None
|
||||
types = _get_types_map(reg)
|
||||
if isinstance(types, dict):
|
||||
t = types.get(note_type, {})
|
||||
if isinstance(t, dict) and isinstance(t.get("edge_defaults"), list):
|
||||
edge_defaults = t["edge_defaults"]
|
||||
|
||||
# --- 2) Hashes für die *aktuelle* Konfiguration (falls abweichend) ---
|
||||
cur_hashes: Dict[str, str] = {}
|
||||
if not (src == std_src and norm == std_norm):
|
||||
for m in ("body", "frontmatter", "full"):
|
||||
cur_hashes[f"{m}:{src}:{norm}"] = _hash_for(
|
||||
m, body=body_for_hash, fm=fm, normalize=norm
|
||||
)
|
||||
|
||||
# --- 3) Aktueller Modus für Backwards-Compat Felder ---
|
||||
current_hash = _hash_for(mode_resolved, body=body_for_hash, fm=fm, normalize=norm)
|
||||
hash_signature = f"{mode_resolved}:{src}:{norm}:{current_hash}"
|
||||
|
||||
# Wikilinks (Note-Ebene)
|
||||
refs = list(dict.fromkeys(extract_wikilinks(body_parsed))) if body_parsed else []
|
||||
updated = _ts_to_int(fm.get("updated"))
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"note_id": fm.get("id") or fm.get("note_id"),
|
||||
"title": fm.get("title"),
|
||||
"type": fm.get("type"),
|
||||
"status": fm.get("status"),
|
||||
"created": fm.get("created"),
|
||||
"updated": fm.get("updated"),
|
||||
"path": rel_path or fm.get("path"),
|
||||
"tags": fm.get("tags"),
|
||||
# Volltext für verlustfreien Export
|
||||
"fulltext": body_parsed,
|
||||
# Backwards-Compat:
|
||||
"hash_fulltext": current_hash,
|
||||
"hash_signature": hash_signature,
|
||||
# Option C: Mehrfach-Hashes
|
||||
"hashes": {**std_hashes, **cur_hashes},
|
||||
"hash_body": hash_body,
|
||||
"hash_frontmatter": hash_frontmatter,
|
||||
"hash_full": hash_full,
|
||||
# Fallback-Refs
|
||||
"references": refs,
|
||||
"note_id": note_id,
|
||||
"type": note_type,
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
"updated": updated if updated is not None else 0,
|
||||
"path": file_path or "",
|
||||
"chunk_profile": chunk_profile,
|
||||
"retriever_weight": float(retriever_weight),
|
||||
}
|
||||
|
||||
for k in ("area", "project", "source", "lang", "slug", "aliases"):
|
||||
if k in fm:
|
||||
payload[k] = fm[k]
|
||||
if edge_defaults is not None:
|
||||
payload["edge_defaults"] = edge_defaults
|
||||
|
||||
return payload
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI – Sichtprüfung
|
||||
# ---------------------------------------------------------------------------
|
||||
# ------------------- internal utilities -------------------
|
||||
|
||||
def _cli() -> None:
|
||||
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
|
||||
ap.add_argument("--from-file", dest="src", required=True)
|
||||
ap.add_argument("--vault-root", dest="vault_root", default=None)
|
||||
ap.add_argument("--print", dest="do_print", action="store_true")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None)
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
|
||||
args = ap.parse_args()
|
||||
def _stable_id_from_path_or_title(path: str, title: str) -> str:
|
||||
base = path or title or ""
|
||||
if not base:
|
||||
base = "note"
|
||||
h = hashlib.sha1(base.encode("utf-8")).hexdigest()[:6]
|
||||
# title-sourced; in V2 typischerweise durch Frontmatter id ersetzt
|
||||
return f"auto-{h}"
|
||||
|
||||
parsed = read_markdown(args.src)
|
||||
payload = make_note_payload(
|
||||
parsed,
|
||||
vault_root=args.vault_root,
|
||||
hash_mode=args.hash_mode,
|
||||
hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source,
|
||||
file_path=args.src,
|
||||
)
|
||||
if args.do_print:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
_cli()
|
||||
def _ts_to_int(val: Any) -> Optional[int]:
|
||||
# akzeptiert YYYY-MM-DD oder epoch int; None bei Fehler
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, int):
|
||||
return val
|
||||
if isinstance(val, float):
|
||||
return int(val)
|
||||
if isinstance(val, str):
|
||||
val = val.strip()
|
||||
# YYYY-MM-DD
|
||||
try:
|
||||
dt = datetime.date.fromisoformat(val)
|
||||
return int(datetime.datetime(dt.year, dt.month, dt.day).timestamp())
|
||||
except Exception:
|
||||
pass
|
||||
# epoch string
|
||||
try:
|
||||
return int(val)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
+147
-92
@@ -1,124 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: app/core/qdrant.py
|
||||
Version: v1.4.0 (2025-09-09)
|
||||
app/core/qdrant.py
|
||||
Version: 2.2.0 (2025-11-11)
|
||||
|
||||
Kurzbeschreibung:
|
||||
Qdrant-Client & Collection-Setup für mindnet.
|
||||
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
|
||||
- Edges-Collection nutzt 1D Dummy-Vektor.
|
||||
- NEW: ensure_payload_indexes(...) legt sinnvolle Payload-Indizes an.
|
||||
Aufgabe
|
||||
-------
|
||||
- Zentraler Qdrant-Zugriff (Client, Config)
|
||||
- Collection-Anlage (notes/chunks/edges)
|
||||
- **Payload-Indizes sicherstellen** (idempotent)
|
||||
|
||||
Aufruf:
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
||||
Hinweis
|
||||
-------
|
||||
Diese Datei ist als Drop-in-Ersatz gedacht, falls in deinem Projekt noch keine
|
||||
robuste ensure_payload_indexes()-Implementierung vorliegt. Die Signaturen
|
||||
bleiben kompatibel zu scripts.import_markdown und scripts.reset_qdrant.
|
||||
|
||||
API-Notizen
|
||||
-----------
|
||||
- Payload-Indizes werden mit `create_payload_index` angelegt.
|
||||
- Typen stammen aus `qdrant_client.http.models.PayloadSchemaType`:
|
||||
KEYWORD | TEXT | INTEGER | FLOAT | BOOL | GEO | DATETIME
|
||||
- Für häufige Filterfelder (note_id, kind, scope, type, tags, ...) legen wir
|
||||
Indizes an. Das ist laut Qdrant-Doku Best Practice für performante Filter.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Tuple
|
||||
from typing import Optional, Tuple, Dict, List
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Konfiguration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class QdrantConfig:
|
||||
url: str
|
||||
api_key: Optional[str]
|
||||
prefix: str
|
||||
dim: int
|
||||
host: Optional[str] = None
|
||||
port: Optional[int] = None
|
||||
url: Optional[str] = None
|
||||
api_key: Optional[str] = None
|
||||
prefix: str = "mindnet"
|
||||
dim: int = 384
|
||||
distance: str = "Cosine" # Cosine | Dot | Euclid
|
||||
on_disk_payload: bool = True
|
||||
|
||||
@staticmethod
|
||||
def from_env() -> "QdrantConfig":
|
||||
url = os.getenv("QDRANT_URL")
|
||||
if not url:
|
||||
host = os.getenv("QDRANT_HOST", "127.0.0.1")
|
||||
port = int(os.getenv("QDRANT_PORT", "6333"))
|
||||
url = f"http://{host}:{port}"
|
||||
@classmethod
|
||||
def from_env(cls) -> "QdrantConfig":
|
||||
# Entweder URL ODER Host/Port, API-Key optional
|
||||
url = os.getenv("QDRANT_URL") or None
|
||||
host = os.getenv("QDRANT_HOST") or None
|
||||
port = os.getenv("QDRANT_PORT")
|
||||
port = int(port) if port else None
|
||||
api_key = os.getenv("QDRANT_API_KEY") or None
|
||||
prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
|
||||
dim = int(os.getenv("VECTOR_DIM", "384"))
|
||||
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
|
||||
prefix = os.getenv("COLLECTION_PREFIX") or "mindnet"
|
||||
dim = int(os.getenv("VECTOR_DIM") or 384)
|
||||
distance = os.getenv("DISTANCE", "Cosine")
|
||||
on_disk_payload = (os.getenv("ON_DISK_PAYLOAD", "true").lower() == "true")
|
||||
return cls(
|
||||
host=host, port=port, url=url, api_key=api_key,
|
||||
prefix=prefix, dim=dim, distance=distance, on_disk_payload=on_disk_payload
|
||||
)
|
||||
|
||||
|
||||
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
|
||||
# QdrantClient akzeptiert entweder url=... oder host/port
|
||||
if cfg.url:
|
||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key, timeout=60.0)
|
||||
return QdrantClient(host=cfg.host or "127.0.0.1", port=cfg.port or 6333, api_key=cfg.api_key, timeout=60.0)
|
||||
|
||||
|
||||
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||
)
|
||||
|
||||
def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||
)
|
||||
|
||||
def _create_edges(client: QdrantClient, name: str) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT), # 1D-Dummy
|
||||
)
|
||||
|
||||
|
||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
|
||||
notes = f"{prefix}_notes"
|
||||
chunks = f"{prefix}_chunks"
|
||||
edges = f"{prefix}_edges"
|
||||
|
||||
_create_notes(client, notes, dim)
|
||||
_create_chunks(client, chunks, dim)
|
||||
|
||||
if client.collection_exists(edges):
|
||||
try:
|
||||
info = client.get_collection(edges)
|
||||
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
|
||||
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
|
||||
except Exception:
|
||||
has_vectors = True
|
||||
if not has_vectors:
|
||||
if destructive:
|
||||
client.delete_collection(edges)
|
||||
_create_edges(client, edges)
|
||||
else:
|
||||
print(f"[ensure_collections] WARN: '{edges}' ohne VectorConfig; destructive=False.", flush=True)
|
||||
else:
|
||||
_create_edges(client, edges)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Collections
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def collection_names(prefix: str) -> Tuple[str, str, str]:
|
||||
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
|
||||
# -------------------------------
|
||||
# NEW: Payload-Indexing
|
||||
# -------------------------------
|
||||
def _vector_params(dim: int, distance: str) -> rest.VectorParams:
|
||||
# Distance: "Cosine" | "Dot" | "Euclid"
|
||||
dist = getattr(rest.Distance, distance.capitalize(), rest.Distance.COSINE)
|
||||
return rest.VectorParams(size=dim, distance=dist)
|
||||
|
||||
def _safe_create_index(client: QdrantClient, col: str, field: str, schema: rest.PayloadSchemaType):
|
||||
try:
|
||||
client.create_payload_index(
|
||||
collection_name=col,
|
||||
field_name=field,
|
||||
field_schema=schema,
|
||||
|
||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int) -> None:
|
||||
"""Legt mindnet_notes, mindnet_chunks, mindnet_edges an (falls nicht vorhanden)."""
|
||||
notes, chunks, edges = collection_names(prefix)
|
||||
|
||||
# notes
|
||||
if not client.collection_exists(notes):
|
||||
client.create_collection(
|
||||
collection_name=notes,
|
||||
vectors_config=_vector_params(dim, os.getenv("DISTANCE", "Cosine")),
|
||||
on_disk_payload=True,
|
||||
)
|
||||
except Exception:
|
||||
# bereits vorhanden oder nicht unterstütztes Schema → ignorieren
|
||||
pass
|
||||
# chunks
|
||||
if not client.collection_exists(chunks):
|
||||
client.create_collection(
|
||||
collection_name=chunks,
|
||||
vectors_config=_vector_params(dim, os.getenv("DISTANCE", "Cosine")),
|
||||
on_disk_payload=True,
|
||||
)
|
||||
# edges (Dummy-Vektor, Filter via Payload)
|
||||
if not client.collection_exists(edges):
|
||||
client.create_collection(
|
||||
collection_name=edges,
|
||||
vectors_config=_vector_params(1, "Dot"),
|
||||
on_disk_payload=True,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Payload-Indizes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _ensure_index(client: QdrantClient, collection: str, field: str, schema: rest.PayloadSchemaType) -> None:
|
||||
"""Idempotentes Anlegen eines Payload-Indexes für ein Feld."""
|
||||
try:
|
||||
client.create_payload_index(collection_name=collection, field_name=field, field_schema=schema, wait=True)
|
||||
except Exception as e:
|
||||
# Fehler ignorieren, falls Index bereits existiert oder Server "already indexed" meldet.
|
||||
# Für Debugging ggf. Logging ergänzen.
|
||||
_ = e
|
||||
|
||||
|
||||
def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
|
||||
"""
|
||||
Stellt sicher, dass alle benötigten Payload-Indizes existieren.
|
||||
- notes: note_id(KEYWORD), type(KEYWORD), title(TEXT), updated(INTEGER), tags(KEYWORD)
|
||||
- chunks: note_id(KEYWORD), chunk_id(KEYWORD), index(INTEGER), type(KEYWORD), tags(KEYWORD)
|
||||
- edges: note_id(KEYWORD), kind(KEYWORD), scope(KEYWORD), source_id(KEYWORD), target_id(KEYWORD), chunk_id(KEYWORD)
|
||||
"""
|
||||
notes, chunks, edges = collection_names(prefix)
|
||||
# Notes
|
||||
_safe_create_index(client, notes, "note_id", rest.PayloadSchemaType.KEYWORD)
|
||||
# Chunks
|
||||
_safe_create_index(client, chunks, "note_id", rest.PayloadSchemaType.KEYWORD)
|
||||
_safe_create_index(client, chunks, "chunk_index", rest.PayloadSchemaType.INTEGER)
|
||||
# Edges
|
||||
for f in ("kind", "scope", "source_id", "target_id", "note_id"):
|
||||
_safe_create_index(client, edges, f, rest.PayloadSchemaType.KEYWORD)
|
||||
|
||||
# NOTES
|
||||
for field, schema in [
|
||||
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("type", rest.PayloadSchemaType.KEYWORD),
|
||||
("title", rest.PayloadSchemaType.TEXT),
|
||||
("updated", rest.PayloadSchemaType.INTEGER),
|
||||
("tags", rest.PayloadSchemaType.KEYWORD),
|
||||
]:
|
||||
_ensure_index(client, notes, field, schema)
|
||||
|
||||
# CHUNKS
|
||||
for field, schema in [
|
||||
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("chunk_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("index", rest.PayloadSchemaType.INTEGER),
|
||||
("type", rest.PayloadSchemaType.KEYWORD),
|
||||
("tags", rest.PayloadSchemaType.KEYWORD),
|
||||
]:
|
||||
_ensure_index(client, chunks, field, schema)
|
||||
|
||||
# EDGES
|
||||
for field, schema in [
|
||||
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("kind", rest.PayloadSchemaType.KEYWORD),
|
||||
("scope", rest.PayloadSchemaType.KEYWORD),
|
||||
("source_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("target_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("chunk_id", rest.PayloadSchemaType.KEYWORD),
|
||||
]:
|
||||
_ensure_index(client, edges, field, schema)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QdrantConfig",
|
||||
"get_client",
|
||||
"ensure_collections",
|
||||
"ensure_payload_indexes",
|
||||
"collection_names",
|
||||
]
|
||||
+118
-232
@@ -1,55 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
app/core/qdrant_points.py
|
||||
app/core/qdrant_points.py - robust points helpers for Qdrant
|
||||
|
||||
Zweck
|
||||
- Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
|
||||
- Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
|
||||
- alt: {'edge_type','src_id','dst_id', ...}
|
||||
- neu: {'kind','source_id','target_id', ...}
|
||||
- Single source of truth for building PointStruct for notes/chunks/edges
|
||||
- Backward-compatible payloads for edges
|
||||
- Handles both Single-Vector and Named-Vector collections
|
||||
- Deterministic overrides via ENV to avoid auto-detection traps:
|
||||
* NOTES_VECTOR_NAME, CHUNKS_VECTOR_NAME, EDGES_VECTOR_NAME
|
||||
* MINDNET_VECTOR_NAME (fallback)
|
||||
> Set to a concrete name (e.g. "text") to force Named-Vector with that name
|
||||
> Set to "__single__" (or "single") to force Single-Vector
|
||||
|
||||
Version
|
||||
- 1.3 (2025-09-08)
|
||||
|
||||
Änderungen (ggü. 1.2)
|
||||
- points_for_edges() akzeptiert jetzt beide Edge-Schemata.
|
||||
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
|
||||
stabile 'edge_id' zurück in die Payload.
|
||||
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
|
||||
|
||||
Aufruf / Verwendung
|
||||
- Wird von Import-/Backfill-Skripten via:
|
||||
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
|
||||
eingebunden. Keine CLI.
|
||||
|
||||
Hinweise
|
||||
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
|
||||
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
|
||||
Version: 1.5.0 (2025-11-08)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import uuid
|
||||
from typing import List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
from typing import List, Tuple, Iterable, Optional, Dict, Any
|
||||
|
||||
from qdrant_client.http import models as rest
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
# --------------------- ID helpers ---------------------
|
||||
|
||||
def _to_uuid(stable_key: str) -> str:
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
# --------------------- Points builders ---------------------
|
||||
|
||||
def _to_uuid(stable_key: str) -> str:
|
||||
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||
|
||||
|
||||
def points_for_note(
|
||||
prefix: str,
|
||||
note_payload: dict,
|
||||
note_vec: List[float] | None,
|
||||
dim: int,
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
|
||||
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
|
||||
notes_col, _, _ = _names(prefix)
|
||||
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
||||
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
||||
@@ -57,18 +40,7 @@ def points_for_note(
|
||||
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
||||
return notes_col, [pt]
|
||||
|
||||
|
||||
def points_for_chunks(
|
||||
prefix: str,
|
||||
chunk_payloads: List[dict],
|
||||
vectors: List[List[float]],
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Chunks-Collection: erwartet pro Chunk einen Vektor.
|
||||
Robustheit:
|
||||
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
|
||||
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
|
||||
"""
|
||||
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
_, chunks_col, _ = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
||||
@@ -81,22 +53,12 @@ def points_for_chunks(
|
||||
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
||||
return chunks_col, points
|
||||
|
||||
|
||||
def _normalize_edge_payload(pl: dict) -> dict:
|
||||
"""
|
||||
Sorgt für kompatible Feldnamen.
|
||||
akzeptiert:
|
||||
- neu: kind, source_id, target_id, seq?
|
||||
- alt: edge_type, src_id, dst_id, order?/index?
|
||||
schreibt zurück: kind, source_id, target_id, seq?
|
||||
"""
|
||||
# bereits neu?
|
||||
kind = pl.get("kind") or pl.get("edge_type") or "edge"
|
||||
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
|
||||
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
|
||||
seq = pl.get("seq") or pl.get("order") or pl.get("index")
|
||||
|
||||
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
|
||||
pl.setdefault("kind", kind)
|
||||
pl.setdefault("source_id", source_id)
|
||||
pl.setdefault("target_id", target_id)
|
||||
@@ -104,18 +66,11 @@ def _normalize_edge_payload(pl: dict) -> dict:
|
||||
pl["seq"] = seq
|
||||
return pl
|
||||
|
||||
|
||||
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Edges-Collection mit 1D-Dummy-Vektor.
|
||||
- Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
|
||||
- Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
|
||||
"""
|
||||
_, _, edges_col = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for raw in edge_payloads:
|
||||
pl = _normalize_edge_payload(raw)
|
||||
|
||||
edge_id = pl.get("edge_id")
|
||||
if not edge_id:
|
||||
kind = pl.get("kind", "edge")
|
||||
@@ -124,59 +79,126 @@ def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[
|
||||
seq = pl.get("seq") or ""
|
||||
edge_id = f"{kind}:{s}->{t}#{seq}"
|
||||
pl["edge_id"] = edge_id
|
||||
|
||||
point_id = _to_uuid(edge_id)
|
||||
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
||||
return edges_col, points
|
||||
|
||||
# --------------------- Vector schema & overrides ---------------------
|
||||
|
||||
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
|
||||
def _preferred_name(candidates: List[str]) -> str:
|
||||
for k in ("text", "default", "embedding", "content"):
|
||||
if k in candidates:
|
||||
return k
|
||||
return sorted(candidates)[0]
|
||||
|
||||
def _env_override_for_collection(collection: str) -> Optional[str]:
|
||||
"""
|
||||
Returns:
|
||||
- "__single__" to force single-vector
|
||||
- concrete name (str) to force named-vector with that name
|
||||
- None to auto-detect
|
||||
"""
|
||||
base = os.getenv("MINDNET_VECTOR_NAME")
|
||||
if collection.endswith("_notes"):
|
||||
base = os.getenv("NOTES_VECTOR_NAME", base)
|
||||
elif collection.endswith("_chunks"):
|
||||
base = os.getenv("CHUNKS_VECTOR_NAME", base)
|
||||
elif collection.endswith("_edges"):
|
||||
base = os.getenv("EDGES_VECTOR_NAME", base)
|
||||
|
||||
if not base:
|
||||
return None
|
||||
val = base.strip()
|
||||
if val.lower() in ("__single__", "single"):
|
||||
return "__single__"
|
||||
return val # concrete name
|
||||
|
||||
def _get_vector_schema(client: QdrantClient, collection_name: str) -> dict:
|
||||
"""
|
||||
Return {"kind": "single", "size": int} or {"kind": "named", "names": [...], "primary": str}.
|
||||
"""
|
||||
try:
|
||||
info = client.get_collection(collection_name=collection_name)
|
||||
vecs = getattr(info, "vectors", None)
|
||||
# Single-vector config
|
||||
if hasattr(vecs, "size") and isinstance(vecs.size, int):
|
||||
return {"kind": "single", "size": vecs.size}
|
||||
# Named-vectors config (dict-like in .config)
|
||||
cfg = getattr(vecs, "config", None)
|
||||
if isinstance(cfg, dict) and cfg:
|
||||
names = list(cfg.keys())
|
||||
if names:
|
||||
return {"kind": "named", "names": names, "primary": _preferred_name(names)}
|
||||
except Exception:
|
||||
pass
|
||||
return {"kind": "single", "size": None}
|
||||
|
||||
def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruct]:
|
||||
out: List[rest.PointStruct] = []
|
||||
for pt in points:
|
||||
vec = getattr(pt, "vector", None)
|
||||
if isinstance(vec, dict):
|
||||
if name in vec:
|
||||
out.append(pt)
|
||||
else:
|
||||
# take any existing entry; if empty dict fallback to [0.0]
|
||||
fallback_vec = None
|
||||
try:
|
||||
fallback_vec = list(next(iter(vec.values())))
|
||||
except Exception:
|
||||
fallback_vec = [0.0]
|
||||
out.append(rest.PointStruct(id=pt.id, vector={name: fallback_vec}, payload=pt.payload))
|
||||
elif vec is not None:
|
||||
out.append(rest.PointStruct(id=pt.id, vector={name: vec}, payload=pt.payload))
|
||||
else:
|
||||
out.append(pt)
|
||||
return out
|
||||
|
||||
# --------------------- Qdrant ops ---------------------
|
||||
|
||||
def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointStruct]) -> None:
|
||||
if not points:
|
||||
return
|
||||
|
||||
# 1) ENV overrides come first
|
||||
override = _env_override_for_collection(collection)
|
||||
if override == "__single__":
|
||||
client.upsert(collection_name=collection, points=points, wait=True)
|
||||
return
|
||||
elif isinstance(override, str):
|
||||
client.upsert(collection_name=collection, points=_as_named(points, override), wait=True)
|
||||
return
|
||||
|
||||
# 2) Auto-detect schema
|
||||
schema = _get_vector_schema(client, collection)
|
||||
if schema.get("kind") == "named":
|
||||
name = schema.get("primary") or _preferred_name(schema.get("names") or [])
|
||||
client.upsert(collection_name=collection, points=_as_named(points, name), wait=True)
|
||||
return
|
||||
|
||||
# 3) Fallback single-vector
|
||||
client.upsert(collection_name=collection, points=points, wait=True)
|
||||
|
||||
# --- WP-04 Ergänzungen: Graph/Retriever Hilfsfunktionen ---
|
||||
from typing import Optional, Dict, Any, Iterable
|
||||
from qdrant_client import QdrantClient
|
||||
# --- Optional search helpers ---
|
||||
|
||||
def _filter_any(field: str, values: Iterable[str]) -> rest.Filter:
|
||||
"""Erzeuge OR-Filter: payload[field] == any(values)."""
|
||||
return rest.Filter(
|
||||
should=[
|
||||
rest.FieldCondition(key=field, match=rest.MatchValue(value=v))
|
||||
for v in values
|
||||
]
|
||||
)
|
||||
return rest.Filter(should=[rest.FieldCondition(key=field, match=rest.MatchValue(value=v)) for v in values])
|
||||
|
||||
def _merge_filters(*filters: Optional[rest.Filter]) -> Optional[rest.Filter]:
|
||||
"""Fasst mehrere Filter zu einem AND zusammen (None wird ignoriert)."""
|
||||
fs = [f for f in filters if f is not None]
|
||||
if not fs:
|
||||
return None
|
||||
if len(fs) == 1:
|
||||
return fs[0]
|
||||
# rest.Filter hat must/should; wir kombinieren als must=[...]
|
||||
must = []
|
||||
for f in fs:
|
||||
# Überführe vorhandene Bedingungen in must
|
||||
if getattr(f, "must", None):
|
||||
must.extend(f.must)
|
||||
if getattr(f, "should", None):
|
||||
# "should" als eigene Gruppe beilegen (Qdrant interpretiert OR)
|
||||
must.append(rest.Filter(should=f.should))
|
||||
if getattr(f, "must_not", None):
|
||||
# negative Bedingungen weiterreichen
|
||||
if "must_not" not in locals():
|
||||
pass
|
||||
return rest.Filter(must=must)
|
||||
|
||||
def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter]:
|
||||
"""
|
||||
Einfache Filterumsetzung:
|
||||
- Bei Listenwerten: OR über mehrere MatchValue (field == any(values))
|
||||
- Bei Skalarwerten: Gleichheit (field == value)
|
||||
Für komplexere Filter (z. B. tags ∈ payload.tags) bitte erweitern.
|
||||
"""
|
||||
if not filters:
|
||||
return None
|
||||
parts = []
|
||||
@@ -187,147 +209,11 @@ def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter
|
||||
parts.append(rest.Filter(must=[rest.FieldCondition(key=k, match=rest.MatchValue(value=v))]))
|
||||
return _merge_filters(*parts)
|
||||
|
||||
def search_chunks_by_vector(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
vector: list[float],
|
||||
top: int = 10,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
) -> list[tuple[str, float, dict]]:
|
||||
"""
|
||||
Vektorielle Suche in {prefix}_chunks.
|
||||
Rückgabe: Liste von (point_id, score, payload)
|
||||
"""
|
||||
def search_chunks_by_vector(client: QdrantClient, prefix: str, vector: List[float], top: int = 10, filters: Optional[Dict[str, Any]] = None) -> List[Tuple[str, float, dict]]:
|
||||
_, chunks_col, _ = _names(prefix)
|
||||
flt = _filter_from_dict(filters)
|
||||
res = client.search(
|
||||
collection_name=chunks_col,
|
||||
query_vector=vector,
|
||||
limit=top,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
query_filter=flt,
|
||||
)
|
||||
out: list[tuple[str, float, dict]] = []
|
||||
res = client.search(collection_name=chunks_col, query_vector=vector, limit=top, with_payload=True, with_vectors=False, query_filter=flt)
|
||||
out: List[Tuple[str, float, dict]] = []
|
||||
for r in res:
|
||||
out.append((str(r.id), float(r.score), dict(r.payload or {})))
|
||||
return out
|
||||
|
||||
def get_edges_for_sources(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
source_ids: list[str],
|
||||
edge_types: Optional[list[str]] = None,
|
||||
limit: int = 2048,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Hole Edges aus {prefix}_edges mit source_id ∈ source_ids (und optional kind ∈ edge_types).
|
||||
Liefert Payload-Dicts inkl. edge_id/source_id/target_id/kind/seq (falls vorhanden).
|
||||
"""
|
||||
_, _, edges_col = _names(prefix)
|
||||
f_src = _filter_any("source_id", source_ids)
|
||||
f_kind = _filter_any("kind", edge_types) if edge_types else None
|
||||
flt = _merge_filters(f_src, f_kind)
|
||||
|
||||
collected: list[dict] = []
|
||||
next_page = None
|
||||
while True:
|
||||
points, next_page = client.scroll(
|
||||
collection_name=edges_col,
|
||||
scroll_filter=flt,
|
||||
limit=min(512, limit - len(collected)),
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
offset=next_page,
|
||||
)
|
||||
for p in points:
|
||||
pl = dict(p.payload or {})
|
||||
# füge die deterministische ID hinzu (nützlich für Clients)
|
||||
pl.setdefault("id", str(p.id))
|
||||
collected.append(pl)
|
||||
if len(collected) >= limit:
|
||||
return collected
|
||||
if next_page is None:
|
||||
break
|
||||
return collected
|
||||
|
||||
def get_note_payload(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
note_id: str,
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
Hole eine Note anhand ihres payload.note_id (nicht internal UUID!).
|
||||
"""
|
||||
notes_col, _, _ = _names(prefix)
|
||||
flt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
points, _ = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=flt,
|
||||
limit=1,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
if not points:
|
||||
return None
|
||||
pl = dict(points[0].payload or {})
|
||||
pl.setdefault("id", str(points[0].id))
|
||||
return pl
|
||||
|
||||
def get_neighbor_nodes(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
target_ids: list[str],
|
||||
limit_per_collection: int = 2048,
|
||||
) -> dict[str, dict]:
|
||||
"""
|
||||
Hole Payloads der Zielknoten (Notes/Chunks) zu den angegebenen IDs.
|
||||
IDs sind die stabilen payload-IDs (note_id/chunk_id), nicht internal UUIDs.
|
||||
Rückgabe: Mapping target_id -> payload
|
||||
"""
|
||||
notes_col, chunks_col, _ = _names(prefix)
|
||||
out: dict[str, dict] = {}
|
||||
|
||||
# Notes
|
||||
flt_notes = _filter_any("note_id", target_ids)
|
||||
next_page = None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=flt_notes,
|
||||
limit=256,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
offset=next_page,
|
||||
)
|
||||
for p in pts:
|
||||
pl = dict(p.payload or {})
|
||||
nid = pl.get("note_id")
|
||||
if nid and nid not in out:
|
||||
pl.setdefault("id", str(p.id))
|
||||
out[nid] = pl
|
||||
if next_page is None or len(out) >= limit_per_collection:
|
||||
break
|
||||
|
||||
# Chunks
|
||||
flt_chunks = _filter_any("chunk_id", target_ids)
|
||||
next_page = None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=chunks_col,
|
||||
scroll_filter=flt_chunks,
|
||||
limit=256,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
offset=next_page,
|
||||
)
|
||||
for p in pts:
|
||||
pl = dict(p.payload or {})
|
||||
cid = pl.get("chunk_id")
|
||||
if cid and cid not in out:
|
||||
pl.setdefault("id", str(p.id))
|
||||
out[cid] = pl
|
||||
if next_page is None or len(out) >= limit_per_collection:
|
||||
break
|
||||
|
||||
return out
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/type_registry.py
|
||||
Version: 1.0.0
|
||||
Datum: 2025-11-08
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Lädt eine optionale Typ-Registry (config/types.yaml) und stellt
|
||||
komfortable Zugriffsfunktionen bereit. Die Registry ist *optional*:
|
||||
- Fehlt die Datei oder ist das YAML defekt, wird ein konservativer
|
||||
Default (Typ "concept") verwendet und es wird eine Warnung ausgegeben.
|
||||
- Änderungen an der Datei greifen nach einem Neustart des Prozesses.
|
||||
|
||||
Öffentliche API
|
||||
---------------
|
||||
- load_type_registry(path: str = "config/types.yaml") -> dict
|
||||
- get_type_config(note_type: str, reg: dict) -> dict
|
||||
- resolve_note_type(fm_type: str | None, reg: dict) -> str
|
||||
- effective_chunk_profile(note_type: str, reg: dict) -> str | None
|
||||
- profile_overlap(profile: str | None) -> tuple[int,int] # nur Overlap-Empfehlung
|
||||
|
||||
Hinweis
|
||||
-------
|
||||
Die Registry steuert KEINE Breaking Changes. Ohne Datei/Typ bleibt das
|
||||
Verhalten exakt wie im Release-Stand 20251105.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
import os
|
||||
|
||||
try:
|
||||
import yaml # PyYAML
|
||||
except Exception:
|
||||
yaml = None # wird erst benötigt, wenn eine Datei gelesen werden soll
|
||||
|
||||
# Konservativer Default – bewusst minimal
|
||||
_DEFAULT_REGISTRY: Dict[str, Any] = {
|
||||
"version": "1.0",
|
||||
"types": {
|
||||
"concept": {
|
||||
"chunk_profile": "medium",
|
||||
"edge_defaults": ["references", "related_to"],
|
||||
"retriever_weight": 1.0,
|
||||
}
|
||||
},
|
||||
"_using_defaults": True,
|
||||
"_warning": "types.yaml missing or invalid – using built-in defaults (type=concept).",
|
||||
}
|
||||
|
||||
# Chunk-Profile → Overlap-Empfehlungen (nur für synthetische Fensterbildung)
|
||||
# Die absoluten Chunk-Längen bleiben Aufgabe des Chunkers (assemble_chunks).
|
||||
_PROFILE_TO_OVERLAP: Dict[str, Tuple[int, int]] = {
|
||||
"short": (20, 30),
|
||||
"medium": (40, 60),
|
||||
"long": (60, 80),
|
||||
}
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def load_type_registry(path: str = "config/types.yaml") -> Dict[str, Any]:
|
||||
"""
|
||||
Lädt die Registry aus 'path'. Bei Fehlern wird ein konserviver Default geliefert.
|
||||
Die Rückgabe ist *prozessweit* gecached.
|
||||
"""
|
||||
if not path:
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
if not os.path.isfile(path):
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
if yaml is None:
|
||||
# PyYAML fehlt → auf Default zurückfallen
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
# Minimal validieren
|
||||
if not isinstance(data, dict) or "types" not in data or not isinstance(data["types"], dict):
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
data.setdefault("version", "1.0")
|
||||
data.setdefault("_using_defaults", False)
|
||||
return data
|
||||
except Exception:
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
|
||||
def get_type_config(note_type: Optional[str], reg: Dict[str, Any]) -> Dict[str, Any]:
|
||||
t = (note_type or "concept").strip().lower()
|
||||
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
|
||||
return types.get(t) or types.get("concept") or _DEFAULT_REGISTRY["types"]["concept"]
|
||||
|
||||
|
||||
def resolve_note_type(fm_type: Optional[str], reg: Dict[str, Any]) -> str:
|
||||
"""Liefert einen gültigen Typ (unbekannt → 'concept')."""
|
||||
t = (fm_type or "concept").strip().lower()
|
||||
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
|
||||
return t if t in types else "concept"
|
||||
|
||||
|
||||
def effective_chunk_profile(note_type: Optional[str], reg: Dict[str, Any]) -> Optional[str]:
|
||||
cfg = get_type_config(note_type, reg)
|
||||
prof = cfg.get("chunk_profile")
|
||||
if isinstance(prof, str) and prof.strip():
|
||||
return prof.strip().lower()
|
||||
return None
|
||||
|
||||
|
||||
def profile_overlap(profile: Optional[str]) -> Tuple[int, int]:
|
||||
"""Gibt eine Overlap-Empfehlung (low, high) für das Profil zurück."""
|
||||
if not profile:
|
||||
return _PROFILE_TO_OVERLAP["medium"]
|
||||
return _PROFILE_TO_OVERLAP.get(profile.strip().lower(), _PROFILE_TO_OVERLAP["medium"])
|
||||
@@ -0,0 +1,25 @@
|
||||
version: 1.0
|
||||
defaults:
|
||||
retriever_weight: 0.1
|
||||
chunk_profile: default
|
||||
types:
|
||||
concept:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["references", "related_to"]
|
||||
retriever_weight: 0.35
|
||||
task:
|
||||
chunk_profile: short
|
||||
edge_defaults: ["depends_on", "belongs_to"]
|
||||
retriever_weight: 0.8
|
||||
experience:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["derived_from", "inspired_by"]
|
||||
retriever_weight: 0.9
|
||||
project:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.97
|
||||
profile:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.66
|
||||
@@ -0,0 +1,21 @@
|
||||
# Changelog — WP-03 Type-Registry (2025-11-08)
|
||||
|
||||
## Added
|
||||
- `app/core/type_registry.py` — Loader & Helper (cached), robust Fallbacks.
|
||||
- `config/types.yaml` — Beispiel-Registry (version 1.0).
|
||||
- `scripts/import_markdown.py` v3.8.0 — Registry-Integration:
|
||||
- typvalidierung (unbekannt → `concept`, Warnung)
|
||||
- `chunk_profile` → an `make_chunk_payloads(...)`
|
||||
- `retriever_weight` → persistiert im Note-Payload
|
||||
- `edge_defaults["references"]` schaltet Note-Scope-Refs additiv ein
|
||||
- `app/core/chunk_payload.py` v2.3.0 — optionales `chunk_profile` für
|
||||
synthetische Overlaps (keine Schemaänderung).
|
||||
- `app/core/derive_edges.py` v1.5.0 — Type-Registry aktiviert note-scope
|
||||
References/Backlinks additiv.
|
||||
|
||||
## Changed
|
||||
- Keine Breaking Changes. Alle CLI-Flags, ENV-Variablen und Collection-
|
||||
Prefix-Logik bleiben unverändert.
|
||||
|
||||
## Removed
|
||||
- N/A
|
||||
@@ -0,0 +1,74 @@
|
||||
# Kurzhandbuch — Type-Registry (WP-03)
|
||||
|
||||
## Zweck
|
||||
Die Type-Registry macht den `type:`-Wert im Frontmatter maschinenlesbar
|
||||
und steuert:
|
||||
- Chunk-Profile (indirekt über Overlap bei fensterlosen Chunks),
|
||||
- typabhängige Default-Kanten (hier: Aktivierung von Note-Scope-References),
|
||||
- optionale Retriever-Gewichtungen (`retriever_weight`) für WP‑04.
|
||||
|
||||
## Aktivierung
|
||||
1. Legen Sie `config/types.yaml` an (siehe Beispiel in diesem Paket).
|
||||
2. Starten Sie Import/Server neu. Die Registry wird beim Prozessstart
|
||||
geladen und gecached.
|
||||
|
||||
Fehlt die Datei oder ist sie defekt, bleibt das System voll funktionsfähig
|
||||
(der Typ `concept` wird genutzt; Warnhinweis im Log).
|
||||
|
||||
## Schema (Beispiel)
|
||||
```yaml
|
||||
version: 1.0
|
||||
types:
|
||||
concept:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "related_to"]
|
||||
retriever_weight: 1.0
|
||||
task:
|
||||
chunk_profile: short
|
||||
edge_defaults: ["depends_on", "belongs_to"]
|
||||
retriever_weight: 0.8
|
||||
experience:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["derived_from", "inspired_by"]
|
||||
retriever_weight: 0.9
|
||||
project:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.95
|
||||
```
|
||||
|
||||
## Verhalten
|
||||
- **Chunking**: Die eigentliche Chunk-Länge bleibt weiterhin typabhängig über
|
||||
`chunk_config.get_sizes(type)` gesteuert. Falls der Chunker keine Fenster
|
||||
mit Overlap liefert, werden in `chunk_payload` synthetische Fenster erzeugt.
|
||||
Hier greift das `chunk_profile` mit folgenden Overlap-Empfehlungen:
|
||||
- short → (20,30), medium → (40,60), long → (60,80).
|
||||
- **Edges**: Enthält `edge_defaults` den Eintrag `references`, werden Note-
|
||||
Scope-References/Backlinks additiv aktiviert (kein Abbruch, keine Duplikate).
|
||||
Weitere Kantenarten bleiben unverändert.
|
||||
- **Retriever**: `retriever_weight` wird in der Note gespeichert und kann von
|
||||
WP‑04 im Ranking/Scoring berücksichtigt werden.
|
||||
|
||||
## Tests / Validierung
|
||||
- Dry-Run:
|
||||
```bash
|
||||
python3 -m scripts.import_markdown --vault ./test_vault
|
||||
```
|
||||
- Vollimport + Purge:
|
||||
```bash
|
||||
python3 -m scripts.import_markdown --vault ./test_vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
- Chunks-Qualität (Window vs Text):
|
||||
```bash
|
||||
python3 tests/check_chunks_window_vs_text.py --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
- Roundtrip:
|
||||
```bash
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --prefix "$COLLECTION_PREFIX"
|
||||
python3 tests/compare_vaults.py --src ./test_vault --dst ./_exportVault --focus body
|
||||
```
|
||||
|
||||
## Fallbacks & Robustheit
|
||||
- Registry nicht vorhanden/ungültig → `concept`, Warnung, keine Abbrüche.
|
||||
- Alle bestehenden ENV/Flags bleiben identisch.
|
||||
- Keine Änderungen an Punkt-IDs, Feldern oder Vektoren.
|
||||
+155
-188
@@ -1,249 +1,216 @@
|
||||
# mindnet – Knowledge Design
|
||||
**Version:** 1.4.0 (aktualisiert 2025-10-06)
|
||||
**Version:** 1.5.0 (aktualisiert 2025-11-09)
|
||||
|
||||
## 1. Gesamtziel und Systemverständnis
|
||||
|
||||
**mindnet** ist ein lokales Wissensnetzwerk, das alle meine Notizen, Projekte, Gedanken, Erfahrungen und Pläne als Markdown-Dateien speichert, automatisch importiert, semantisch in **Chunks** zerlegt und als **Graph-Struktur in Qdrant** ablegt.
|
||||
Es ist die Grundlage für ein agentenfähiges, lokal betriebenes LLM-System, das meine Daten verknüpft, abfragt und erweitert.
|
||||
|
||||
**Zentrale Ziele**
|
||||
- Vollständige Repräsentation persönlichen Wissens in strukturierter, maschinenlesbarer Form
|
||||
- Reproduzierbare Verarbeitung: Import → Chunking → Embedding → Edges → Graph
|
||||
- Verlustfreie Roundtrip-Fähigkeit (Import ⇄ Export)
|
||||
- Erweiterbarkeit für Agenten, RAG-Retriever und semantische Suche
|
||||
|
||||
**Prinzipien**
|
||||
- **Deterministisch:** IDs, Hashes und Edges wiederholbar
|
||||
- **Portabel:** Markdown + YAML
|
||||
- **Robust:** fehlertoleranter Parser
|
||||
- **Idempotent:** keine Duplikate bei Re-Import
|
||||
- **Erweiterbar:** neue Typen und Edge-Arten ohne Migration
|
||||
> Dieses Dokument beschreibt das Ziel, die Architektur, die Datenmodelle sowie die Regeln und Prozesse für den Aufbau von **mindnet** – einem persönlichen, lokal betriebenen Wissensnetz, das auf Markdown-Quellen, Graph-Beziehungen und Vektor-Suche basiert. Es soll die Persönlichkeit, Werte und Erfahrungen des Besitzers widerspiegeln und künftig Antworten generieren, die diese Perspektive authentisch berücksichtigen.
|
||||
|
||||
---
|
||||
|
||||
## 2. YAML-Frontmatter-Schema
|
||||
## 1. Ziel von mindnet
|
||||
|
||||
| Feld | Typ | Beschreibung |
|
||||
|-------|-----|--------------|
|
||||
| `title` | string | Menschlich lesbarer Titel |
|
||||
| `id` | string | Eindeutige Note-ID (Slug) |
|
||||
| `type` | enum | `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone` |
|
||||
| `status` | enum | `draft`, `active`, `done`, `archived` |
|
||||
| `created` | date | ISO-Datum |
|
||||
| `updated` | date | Letzte Änderung |
|
||||
| `area` | string | Themen- oder Lebensbereich |
|
||||
| `project` | string | Zugehöriges Projekt |
|
||||
| `tags` | list | Strukturierte Tags (`area/…`, `topic/…`) |
|
||||
| `depends_on` | list | IDs anderer Notizen |
|
||||
| `assigned_to` | list | Beteiligte Personen |
|
||||
| `embedding_exclude` | bool | Falls `true`, keine Embeddings |
|
||||
| `hash_mode` | enum | Hash-Modus beim Import (`body`, `frontmatter`, `full`) |
|
||||
| `priority` | enum/int | 1–5 oder `low` / `med` / `high` |
|
||||
| `effort_min`, `due` | int/date | Aufwand / Termin |
|
||||
| `aliases` | list | Alternative IDs |
|
||||
| `lang` | string | ISO-Sprachcode |
|
||||
| `source` | string | Herkunft / Referenzquelle |
|
||||
**1.1 Zweck**
|
||||
- mindnet dient als persönliches Wissensnetz, das Erfahrungen, Werte, Prinzipien, Ziele und prägenden Ereignisse langfristig strukturiert.
|
||||
- Es ermöglicht spätere Einsichten in Entscheidungen und Entwicklungen und kann in Zukunft für die Familie (z. B. Kinder) eine nachvollziehbare Quelle sein.
|
||||
|
||||
**Pflichtfelder:** `title`, `id`, `type`, `status`, `created`
|
||||
**Optionale Felder:** alle übrigen
|
||||
**1.2 Leitidee**
|
||||
- Statt einer reinen Dokumentablage wird Wissen durch **Knoten (Notes/Chunks)** und **Kanten (Edges)** vernetzt.
|
||||
- Antworten sollen **persönlich** kontextualisiert werden (Perspektive, Konditionierung, Persönlichkeit).
|
||||
|
||||
**1.3 Langfristigkeit**
|
||||
- Stabiler, zukunftsfähiger Entwurf; spätere Massenmigrationen sollen vermieden werden.
|
||||
- Architektur unterstützt schrittweise Erweiterung (neue Quellen, neue Agenten, neue Regeln).
|
||||
|
||||
---
|
||||
|
||||
## 3. Dateinamen und Ordner
|
||||
## 2. Zielarchitektur (High Level)
|
||||
|
||||
- Format: `YYYY-MM-DD_title.md` oder `slug.md`
|
||||
- Strukturierte Ordner:
|
||||
- `10_thoughts/`, `20_concepts/`, `30_projects/`, `40_experiences/`, …
|
||||
- Alle Pfade relativ; Unterstriche statt Leerzeichen
|
||||
- **Datenquellen:** Markdown-Vault (Obsidian-kompatibel), perspektivisch weitere Quellen (MediaWiki, PDFs, Web-Exporte, E-Mail-Auszüge).
|
||||
- **Import-Pipeline:** Parser → Frontmatter/Body → Chunking → Embedding → Qdrant (Notes/Chunks/Edges).
|
||||
- **Speicher:** Qdrant (Vektoren + Payload + Textindex), Dateien bleiben im Filesystem (Quelle ist Referenz).
|
||||
- **Graph-Schicht:** Edges explizit modelliert (Typ, Richtung, Evidenz, Confidence); optionale abgeleitete Edges per Batch-Job.
|
||||
- **Retriever/Composer:** Hybrid-Retrieval (Vektor + Filter + optional Volltext), Komposition zu Antwortkandidaten.
|
||||
- **Erklärbarkeit:** Begründungspfade (Top-Chunks, Graph-Kette, Provenienz).
|
||||
- **Schnittstellen:** REST/CLI, n8n-Glue, lokale LLM-Anbindung (Ollama).
|
||||
|
||||
---
|
||||
|
||||
## 4. Link- und Edge-Design
|
||||
## 3. Collections in Qdrant
|
||||
|
||||
**Verlinkungen**
|
||||
- Wikilinks: `[[note-id]]` oder `[[Titel|note-id]]`
|
||||
- Automatische Edges aus Text und Frontmatter
|
||||
|
||||
**Edge-Typen in Qdrant**
|
||||
|
||||
| Typ | Bedeutung | Quelle | Scope |
|
||||
|------|------------|--------|--------|
|
||||
| `belongs_to` | Chunk → Note | intern | chunk |
|
||||
| `prev` / `next` | Reihenfolge | intern | chunk |
|
||||
| `references` | explizite Wikilinks | Text | chunk |
|
||||
| `backlink` | Gegenkante zu `references` | abgeleitet | note |
|
||||
| `depends_on` | YAML-Abhängigkeiten | YAML | note |
|
||||
| `assigned_to` | YAML-Zuweisung | YAML | note |
|
||||
| `unresolved` | Ziel fehlt (Stub) | abgeleitet | chunk |
|
||||
|
||||
**Edge-Schlüssel**
|
||||
- `source_id`, `target_id`, `kind`, `scope`, `note_id`
|
||||
- Dedup-Schlüssel: Kombination aus `(kind, source_id, target_id, scope)`
|
||||
- `mindnet_notes` – Note-Metadaten (id, title, type, path, timestamps, tags, …).
|
||||
- `mindnet_chunks` – inhaltstragende Segmente inkl. Vektoren und Textindex.
|
||||
- `mindnet_edges` – Beziehungen (src/dst, relation, direction, evidence, confidence, …).
|
||||
- (optional) `mindnet_people` – strukturierte Personenobjekte (Eigen-/Fremdbezug).
|
||||
|
||||
---
|
||||
|
||||
## 5. Chunking-Modell
|
||||
## 4. Identitäten & IDs
|
||||
|
||||
**Chunk-Datenstruktur**
|
||||
| Feld | Beschreibung |
|
||||
|-------|---------------|
|
||||
| `note_id` | Zugehörige Note |
|
||||
| `chunk_index` | Laufende Nummer |
|
||||
| `text` | Originalabschnitt |
|
||||
| `window` | Text + Overlap links/rechts |
|
||||
| `overlap_left` | Länge des linken Overlaps |
|
||||
| `overlap_right` | Länge des rechten Overlaps |
|
||||
| `embedding` | 384-dimensionaler Vektor |
|
||||
| `tokens` | Tokenanzahl (optional) |
|
||||
|
||||
**Chunking-Regeln (aus `chunking_strategy.md`)**
|
||||
- Trennung nach Absätzen, Überschriften, Listen, Tabellen
|
||||
- Token-Zielgröße 350–500 (max 600)
|
||||
- Overlap: 30–40 % für semantische Kohärenz
|
||||
- Overlap wird in `window` integriert, sodass `text ≠ window`
|
||||
- **Deterministische IDs** für Notes (z. B. Namespace-basierte UUIDv5) und stabile, ableitbare IDs für Chunks/Edges.
|
||||
- ID-Stabilität ist Voraussetzung für Idempotenz (Re-Imports ohne Duplikate).
|
||||
|
||||
---
|
||||
|
||||
## 6. Hash-Strategie und Änderungsverfolgung
|
||||
## 5. Frontmatter & Typen
|
||||
|
||||
**Zweck:** Nur bei realer Inhaltsänderung re-indexieren
|
||||
|
||||
| Umgebungsvariable | Beschreibung |
|
||||
|--------------------|--------------|
|
||||
| `MINDNET_HASH_COMPARE` | Vergleichsquelle (`Body`, `Full`, `Frontmatter`) |
|
||||
| `MINDNET_HASH_SOURCE` | Rohtext vs. geparst (`raw` / `parsed`) |
|
||||
| `MINDNET_HASH_NORMALIZE` | Normalisierung (`canonical` / `none`) |
|
||||
|
||||
**Basismodi**
|
||||
- `body` → Nur Textkörper
|
||||
- `frontmatter` → Nur Metadaten
|
||||
- `full` → Kombination
|
||||
- Hash = SHA-256(canonicalized_input)
|
||||
|
||||
Bei Änderung: Re-Import, neue Embeddings, Edges werden aktualisiert.
|
||||
- Pflichtfelder: `id`, `title`, `type`, `created`, `updated`, `tags`, `source_path`.
|
||||
- Note-Typen (Auszug): `experience`, `principle`, `value`, `goal`, `role`, `persona`, `plan`, `project`, `journal`, `reference`, `translation`.
|
||||
- **types.yaml** definiert Regeln/Defaults je Typ und für Relationstypen (siehe Abschnitt 10).
|
||||
|
||||
---
|
||||
|
||||
## 7. Qdrant-Speicherstruktur
|
||||
## 6. Chunking-Strategie
|
||||
|
||||
| Collection | Inhalt | Primärfelder | Embedding |
|
||||
|-------------|---------|---------------|------------|
|
||||
| `mindnet_notes` | Notes mit Fulltext & Hash | `note_id` | optional |
|
||||
| `mindnet_chunks` | Chunks mit Text, Window, Embedding | `chunk_id`, `note_id` | 384 d |
|
||||
| `mindnet_edges` | Graphkanten | `edge_id`, `source_id`, `target_id` | — |
|
||||
|
||||
**Upsert-Strategie**
|
||||
- Idempotente UUIDv5-IDs
|
||||
- Keine Duplikate durch deterministische Signaturen
|
||||
- `purge-before-upsert` zur Bereinigung
|
||||
|
||||
**Indizes**
|
||||
- notes: `note_id`, `hash_signature`
|
||||
- chunks: `note_id`, `chunk_index`
|
||||
- edges: `(kind, source_id, target_id, scope)`
|
||||
- Satz- bis Absatz-Chunks, moderate Überlappung; Meta-Felder verweisen auf Note/Quelle.
|
||||
- Named Vectors (z. B. `content`, optional `title`, `tags`) ermöglichen hybride Scores.
|
||||
|
||||
---
|
||||
|
||||
## 8. Fehler-Toleranz und Parser-Robustheit
|
||||
## 7. Embeddings & Vektoren
|
||||
|
||||
- UTF-8 mit Fallback: ersetzt ungültige Bytes (`errors="replace"`)
|
||||
- Ignoriert Nullbytes, BOM, exotische Zeilenenden
|
||||
- Erkennt defekte oder leere YAML-Header und überspringt sie
|
||||
- Logt Problemfälle (`read_markdown failed`, `make_note_payload returned non-dict`)
|
||||
- Bei Fehlern: Import → warn → continue
|
||||
- Einheitliche Metrik (cosine), reproduzierbares Embedding-Setup.
|
||||
- Konsistenz über Zeit durch Versionsfeld (`embedding_model`, `embedding_version`).
|
||||
|
||||
---
|
||||
|
||||
## 9. Tests & Validierung
|
||||
## 8. Volltext & Filter
|
||||
|
||||
**Integrationstests**
|
||||
- Roundtrip: Import → Export → Compare (`compare_vaults.py`)
|
||||
- Chunk-Integrität: `verify_chunks_integrity.py`
|
||||
- Hash-Audit: `hash_reporter.py`
|
||||
- Window-Vergleich: `check_chunks_window_vs_text.py`
|
||||
|
||||
**Erwartete Ergebnisse**
|
||||
- Alle Notes werden erkannt
|
||||
- `window ≠ text` bei > 60 % der Chunks
|
||||
- Roundtrip-Vergleich liefert „OK“
|
||||
- Edge-Anzahl entspricht Modell (belongs_to = #Chunks, next/prev ≈ #Chunks – 1)
|
||||
- Textindex auf `mindnet_chunks.body` für exakte Term/Operator-Queries.
|
||||
- Payload-Filter zur schnellen Einschränkung (tags, types, time-buckets, visibility u. a.).
|
||||
|
||||
---
|
||||
|
||||
## 10. Export & Roundtrip-Verhalten
|
||||
## 9. Edges – Beziehungen
|
||||
|
||||
- `export_markdown.py` erzeugt identische Markdown-Dateien
|
||||
- Schreibt YAML + Body in korrekter Reihenfolge
|
||||
- Roundtrip-Prüfung (`compare_vaults.py`) validiert:
|
||||
- vollständigen Body
|
||||
- unveränderte Hash-Signatur
|
||||
- gleiche Chunkzahl
|
||||
- **Explizit**: Jede tatsächliche Beziehung wird als Kante gespeichert (kein Workaround).
|
||||
- Felder u. a.: `src_id`, `dst_id`, `relation`, `direction`, `confidence (0–1)`, `evidence` (Chunk-IDs), `created_at`, `updated_at`.
|
||||
- **edge_defaults** liefern **Interpretations-Hints** (Gewichte, Symmetrie, Transitivität) und parametrisieren abgeleitete Edges; sie **ersetzen** keine Originalkanten.
|
||||
|
||||
---
|
||||
|
||||
## 11. Graph-Schicht und WP-04-Integration
|
||||
## 10. Regeln & Policies (types.yaml)
|
||||
|
||||
- Qdrant fungiert als Graph-Backend
|
||||
- `graph/service.py` (WP-04): stellt API für Mehrhop-Abfragen bereit
|
||||
- Unterstützt:
|
||||
- `expand(note_id, hops=2)`
|
||||
- `resolve_unresolved()`
|
||||
- `neighbors(kind=…)`
|
||||
- Grundlage für spätere **Retriever-Funktionen** (LLM / Agenten)
|
||||
- Per Typ/Relation:
|
||||
- Default-Eigenschaften (Gewicht, Directedness, Symmetrie, Transitivität).
|
||||
- Scoring-Hooks für Graph-Kontext im Retriever.
|
||||
- (später) Privacy-Hooks je Relation (Mindest-Visibility).
|
||||
|
||||
---
|
||||
|
||||
## 12. Agenten- und LLM-Integration
|
||||
## 11. Retrieval & Ranking (Komposition)
|
||||
|
||||
**Retriever-Pipeline**
|
||||
1. Suche top-k Chunks (cosine distance)
|
||||
2. Erweiterung um Edges (`references`, `backlinks`)
|
||||
3. Kontext-Assembling via `window`-Text
|
||||
4. Prompt-Generierung
|
||||
|
||||
**Agent-Use-Cases**
|
||||
- Automatisches Tagging (`type`, `status`)
|
||||
- Vorschläge für neue Edges
|
||||
- Auflösen von `unresolved`-Referenzen
|
||||
- Auto-Import externer Quellen (MediaWiki, PDFs, Webseiten)
|
||||
- Hybrid: Vektor-Score + Filter + optional BM25/Volltext.
|
||||
- Zeitgewicht (Decays) und Graph-Kontext werden komponiert (Formel siehe 16.3).
|
||||
- Konfiguration über `.env`/`config.yaml` (gewichts- und halbwertszeit-basiert).
|
||||
|
||||
---
|
||||
|
||||
## 13. Erweiterbarkeit
|
||||
## 12. Erklärbarkeit
|
||||
|
||||
- Neue Edge-Typen: `inspired_by`, `relates_to`, `contradicts`
|
||||
- Neue Note-Typen: `decision`, `resource`, `recommendation`
|
||||
- Embedding-Engines austauschbar (MiniLM → LaBSE → E5)
|
||||
- Versionierte Chunking-Strategien (`chunk_config.py`)
|
||||
- „Warum-Pfad“: Top-Chunks mit Score-Zerlegung (Vektor/Time/Graph), Graph-Kette, Provenienz.
|
||||
- Mehrschichtige Narrative (Kindheit/Erlebnisse, Werte/Prinzipien, Ziele/Leitbild, Gegenwart).
|
||||
|
||||
---
|
||||
|
||||
## 14. Qualitätssicherung und Best Practices
|
||||
## 13. Betrieb & Sicherheit
|
||||
|
||||
- Validierung vor Upsert
|
||||
- Keine Leer-Chunks
|
||||
- Vollständige Metadaten in Notes
|
||||
- Tests in `tests/` automatisch ausführbar (`run_e2e_roundtrip.sh`)
|
||||
- Ergebnis: „Roundtrip OK“ und `verify_chunks_integrity` = OK
|
||||
- Lokal auf Single-Node starten (Backups via Snapshots).
|
||||
- Zugriff nur intern/Reverse-Proxy; Festplattenverschlüsselung OS-seitig.
|
||||
- Migrationspfad zu Qdrant-Distributed (Shards, Replikation) einplanen.
|
||||
|
||||
---
|
||||
|
||||
## 15. Änderungsverlauf
|
||||
## 14. Qualität & Monitoring
|
||||
|
||||
**v1.4.0 (2025-10-06)**
|
||||
- Neu: `window` vs `text` Feld in `mindnet_chunks`
|
||||
- Neu: Hash-Modi (`body`, `frontmatter`, `full`) mit Env-Steuerung
|
||||
- Neu: Baseline-Modus für Vergleich
|
||||
- Parser → fehlertolerant (UTF-8 replace)
|
||||
- Roundtrip-Test vollständig integriert
|
||||
- Graph-Service und Mehrhop-Abfragen (WP-04-Vorbereitung)
|
||||
- `purge-before-upsert` und `--note-scope-refs` Parameter ergänzt
|
||||
|
||||
**v1.3.0 (2025-09-09)**
|
||||
- Hash-Normalisierung und Edge-Scope
|
||||
- Roundtrip-Tests, Agenten-Integration
|
||||
|
||||
**v1.2.0 (2025-09-02)**
|
||||
- Grundstruktur des Knowledge-Designs
|
||||
- Offline-Eval-Sets, Telemetrie (lokal), Drift-Checks, manuelle Reviews.
|
||||
- Kennzahlen: Hit-Rate, Overlap, Quellen-Alter, Erklärungspfad-Nutzung.
|
||||
|
||||
---
|
||||
|
||||
## 15. Offene Punkte
|
||||
|
||||
- Ausdetaillierung `types.yaml` (Relationen, Hooks).
|
||||
- UI-Oberfläche für Erklärpfade und Provenienz.
|
||||
- Batch-Jobs für abgeleitete Edges.
|
||||
|
||||
---
|
||||
|
||||
## 16. Ergänzungen 2025-11-09 (v1.5.0)
|
||||
|
||||
**Kontext:** Basierend auf den neuen Rahmenbedingungen (nur Deutsch; perspektivische externe Abfragen; einheitliche Vertraulichkeit mit Option auf feinere Stufen; Recency-Priorisierung als Default; Regeln später konfigurierbar; zunächst lokale Speicherung mit optionaler Migration; heutige Hardware, später skalierbar; hohe Erklärbarkeit mit mehreren Erklärungsebenen) werden folgende Entscheidungen und Erweiterungen getroffen. Alle bisherigen Inhalte bleiben gültig und werden **nicht** verändert.
|
||||
|
||||
### 16.1 Sprach- und Inhaltsrichtlinie (nur Deutsch)
|
||||
- `lang: "de"` als verpflichtendes Feld in allen Note-, Chunk- und Edge-Payloads.
|
||||
- Für spätere Mehrsprachigkeit **nicht** die Collection splitten, sondern ein Feld `lang` + optional `lang_original` behalten; Translation-Artefakte als eigener `type: translation` (verweisend auf `source_id`).
|
||||
|
||||
### 16.2 Vertraulichkeit & Sichtbarkeit
|
||||
- Heute: einheitliche Stufe `visibility: "internal"`.
|
||||
- Vorausschauend: reservierte Felder in allen Payloads
|
||||
- `visibility`: `"public" | "internal" | "restricted"`
|
||||
- `sensitivity`: Integer 0–3 (0 = unkritisch, 3 = hochsensibel)
|
||||
- `acl`: Liste erlaubter Rollen/Personen-IDs (App-Layer erzwingt Filter vor Query).
|
||||
- Access-Control erfolgt **im Applikations-Layer** mittels Qdrant-Payload-Filtern (`must: [ { key: "visibility", match: { any: [...] }}, ... ]`); Qdrant bleibt „dumm“ in Bezug auf ACL.
|
||||
|
||||
### 16.3 Recency-Modell & Ranking (konfigurierbar)
|
||||
- Default: **Zeitnähe** priorisieren (sanfte Abwertung älterer Inhalte).
|
||||
- Im Retrieval kombinieren wir:
|
||||
1) Vektor-Score (cosine) aus Qdrant
|
||||
2) Zeitgewicht `time_decay = exp( - ln(2) * age_days / half_life_days )`
|
||||
3) Graph-Kontext aus Edges (z. B. Degree, Path-Hops zum Anker „Ich/Persona“)
|
||||
- Konfigurierbare Gewichte (per `.env` oder `config.yaml`):
|
||||
```text
|
||||
FINAL_SCORE = w_vec * norm(vec_score)
|
||||
+ w_time * time_decay
|
||||
+ w_graph * graph_context_score
|
||||
# Vorschlag: w_vec=0.70, w_time=0.20, w_graph=0.10; half_life_days=365
|
||||
```
|
||||
- Für Fragen, die „zeitlos“ sind, kann `half_life_days = inf` gesetzt werden (kein Decay).
|
||||
|
||||
### 16.4 Erklärbarkeit („Warum diese Antwort?“)
|
||||
- Jeder Antwort wird ein **Erklärpfad** mitgegeben (optional im UI):
|
||||
- Top-Chunks (mit Score-Zerlegung w_vec/w_time/w_graph)
|
||||
- **Kausalpfad** im Graphen: `Note → Edge(relation, confidence) → Note …`
|
||||
- **Provenienz** (Quelle, Erstellungszeit, letzte Änderung).
|
||||
- Ein **Multi-Layer-Explanations**-Modul ordnet Beiträge zu: Kindheit/Erlebnisse, Werte & Prinzipien, aktuelle Ziele/Leitbild, Gegenwartskontext.
|
||||
|
||||
### 16.5 Qdrant-Schemata (Ergänzungen, rückwärtskompatibel)
|
||||
- **Collections** bleiben: `mindnet_notes`, `mindnet_chunks`, `mindnet_edges` (+ `mindnet_people` falls noch nicht vorhanden).
|
||||
- Gemeinsame Zusatzfelder (alle Payloads):
|
||||
- `lang: "de"`
|
||||
- `visibility`/`sensitivity`/`acl` (s. 16.2)
|
||||
- `created_at` (ISO-8601), `updated_at` (ISO-8601), `age_days` (App-seitig berechnet)
|
||||
- **Chunks**:
|
||||
- Named-Vectors (z. B. `"content"`, optional `"title"`, `"tags"` für Hybrid-Scores).
|
||||
- `text_index` auf `body` für BM25-ähnliche Volltextsuche (Qdrant-Textindex).
|
||||
- `recency_group`: Jahr-Monat `YYYY-MM` (ermöglicht Pre-Filter für „letzte N Monate“).
|
||||
- **Edges** (explizit, keine Workarounds):
|
||||
- Felder: `src_id`, `dst_id`, `relation`, `direction`, `confidence` (0–1), `evidence` (Chunk-IDs), `created_at`, `updated_at`.
|
||||
- **edge_defaults** bleiben als **Interpretations-Hints**: z. B. `default_weight`, `symmetry`, `transitivity`. Sie ersetzen **nicht** echte Kanten, sondern parametrisieren abgeleitete/zusätzliche Kantenbildung.
|
||||
- Abgeleitete Edges: optionaler Batch-Job erzeugt *ergänzende* Kanten entlang `types.yaml`-Regeln (z. B. Backlinks, „inspired_by“ aus Zitaten), niemals statt der Originalkanten.
|
||||
|
||||
### 16.6 Regeln & Typen (konfigurierbar)
|
||||
- `types.yaml` erweitert um:
|
||||
- Relationstypen mit Default-Eigenschaften (`weight`, `directed`, `symmetric`, `transitive`).
|
||||
- **Scoring-Hooks**: pro Relation optionaler Einfluss auf `graph_context_score`.
|
||||
- **Privacy-Hooks**: pro Relation Minimal-Visibility (z. B. Beziehungen zu Personen sind mindestens `internal`).
|
||||
|
||||
### 16.7 Speicher- & Betriebsstrategie (heute lokal, später skalierbar)
|
||||
- **Heute**: Single-Node Qdrant mit HNSW-Index, Text-Index und Payload-Filtern; regelmäßiges Backup (Snapshots).
|
||||
- **Morgen**: Migrationspfad in **Qdrant-Distributed** (Shards, ggf. Replikation) bei Bedarf; Parameter-Gleichheit sichern (named-vectors, Metrik, HNSW-Param).
|
||||
- **Sicherheit**: Zugriff nur über internes Netz/Reverse-Proxy, Auth vor Qdrant; Festplatten-Verschlüsselung auf OS-Ebene (z. B. LUKS).
|
||||
|
||||
### 16.8 Persona-Layer („meine Essenz“)
|
||||
- Eigenständiger **Persona-Knoten** (z. B. Note `person/lars`), verbunden mit:
|
||||
- Werten/Prinzipien, Leitbild, prägenden Erlebnissen, Zielen, Rollen.
|
||||
- Jede Antwort kann gegen diesen Knoten **kontextualisiert** werden (Graph-Pfadnähe).
|
||||
- Feld `persona_weight` in Chunks/Notes zur Priorisierung persönlicher Relevanz.
|
||||
|
||||
### 16.9 Evaluierung & Qualitätssicherung
|
||||
- **Offline-Sets** aus echten Fragen (inkl. gewünschter Perspektiven) mit erwarteten Top-Quellen.
|
||||
- **Telemetrie** (lokal): Query-Arten, Treffer-Overlaps, Zeitgewicht-Effekte, „Warum-Pfad“ genutzt/ignoriert.
|
||||
- **Drift-Checks**: Anteil sehr alter vs. neuer Quellen in Antworten; manuelle Reviews.
|
||||
|
||||
### 16.10 Offene Punkte / Nächste Schritte
|
||||
1) `types.yaml` um Relationseigenschaften erweitern (s. 16.6).
|
||||
2) Retriever-Komposition inkl. Zeitgewicht (s. 16.3) implementieren.
|
||||
3) Explanations-Pfad inkl. Provenienzoberfläche (s. 16.4) im UI.
|
||||
4) Feld-Erweiterungen in Import-Pipelines (`lang`, `visibility`, Zeitfelder).
|
||||
5) Optional: Batch-Job für abgeleitete Edges nach Regeln (ergänzend, nicht ersetzend).
|
||||
@@ -0,0 +1,192 @@
|
||||
# Mindnet – Fachliche Architektur
|
||||
**Version:** 2.0 • **Datum:** 2025-11-09 • **Autor:** Fachkonzept (gemeinsam mit Owner)
|
||||
|
||||
---
|
||||
|
||||
## 0. Zweck dieses Dokuments
|
||||
Dieses Dokument beschreibt **Fachdomäne, Funktionsräume, Prozesse und Regeln** von *mindnet*. Es ergänzt die technische Architektur und macht die ursprünglichen Ziele, Designentscheidungen, Anti‑Patterns und Umsetzungsstrategien explizit nachvollziehbar.
|
||||
|
||||
---
|
||||
|
||||
## 1. Vision & Leitplanken
|
||||
- **Essenz bewahren:** Entscheidungen, Erlebnisse, Werte, Prinzipien und Konditionierungen so strukturieren, dass sie später erklärbar rekonstruiert werden können.
|
||||
- **Antworten im eigenen Stil:** Ergebnisse sollen die Persönlichkeit des Owners widerspiegeln (Sprache: **Deutsch**).
|
||||
- **Vertraulichkeit:** Aktuell einheitlich, aber mit **Stufen** (public/internal/sensitive) erweiterbar.
|
||||
- **Zeitliche Priorisierung:** Standard: jüngere Inhalte leichter gewichten; Ausnahmen konfigurierbar.
|
||||
- **Lokal zuerst**, später skalierbar.
|
||||
|
||||
---
|
||||
|
||||
## 2. Fachliches Datenmodell
|
||||
### 2.1 Kernobjekte
|
||||
- **Note**: Eintrag im Wissensnetz (z. B. Konzept, Projekt, Erfahrung, Ereignis, Aufgabe).
|
||||
- **Chunk**: semantischer Abschnitt einer Note (Antwortbaustein).
|
||||
- **Edge**: gerichtete Beziehung zwischen Notes/Chunks (mit Typ, Gewicht, Quelle).
|
||||
- **Run**: Importlauf (Audit).
|
||||
- **Config**: aktive Regel‑ und Typkonfigurationen.
|
||||
|
||||
### 2.2 Typen & Semantik (Auszug)
|
||||
- `concept`: Begriff/These/Prinzip.
|
||||
- `experience`: Erlebnis/Beobachtung (zeitlich verankert).
|
||||
- `project`: Vorhaben mit Zielen/Abhängigkeiten.
|
||||
- `task`: Arbeitseinheit.
|
||||
- (erweiterbar: `event`, `decision`, `person`, `place`, `quote` …)
|
||||
|
||||
### 2.3 Relationen (fachlich)
|
||||
- `references`: verweist inhaltlich auf …
|
||||
- `related_to`: thematische Nähe/Assoziation.
|
||||
- `depends_on`: fachliche/kausale Abhängigkeit.
|
||||
- `belongs_to`: Zugehörigkeit (Teil‑von, Kapitel‑zu).
|
||||
- `derived_from`: abgeleitet (z. B. Schluss aus Erfahrung).
|
||||
- `inspired_by`: Impuls/Idee aus Quelle.
|
||||
- `supports` / `contradicts`: argumentative Beziehung.
|
||||
|
||||
---
|
||||
|
||||
## 3. Funktionsräume
|
||||
1. **Kurativer Raum**
|
||||
- Schreiben/Überarbeiten von Notizen (Obsidian Templates, Frontmatter‑Guides).
|
||||
- Plausibilisierung: Pflichtfelder, Dublettencheck, Warnungen (z. B. fehlende Relationen).
|
||||
|
||||
2. **Graphischer Raum**
|
||||
- Relationen setzen (declared), Vorschläge prüfen (inferred).
|
||||
- Subgraph‑Explorer je Frage/Entscheidung.
|
||||
|
||||
3. **Antwort‑Raum**
|
||||
- Query‑Typen: *Faktenabruf*, *Erlebnisbasiert erklären*, *Planung/Entscheidung begründen*.
|
||||
- Erklärpfad: Chunks → Kanten → Zeitfaktoren → Typgewichte.
|
||||
|
||||
4. **Governance‑Raum**
|
||||
- Sichtbarkeit/ACL, Qualitätsflags, Review‑Queues.
|
||||
- Versionsjournal, Import‑Runs, Config‑Historie.
|
||||
|
||||
---
|
||||
|
||||
## 4. Fachliche Regeln
|
||||
### 4.1 Frontmatter‑Schema (Mindeststandard)
|
||||
```yaml
|
||||
id: "{deterministic}"
|
||||
title: "…"
|
||||
type: "concept|experience|project|task|…"
|
||||
created: "YYYY-MM-DD"
|
||||
modified: "YYYY-MM-DD"
|
||||
visibility: "internal" # default
|
||||
retriever_weight: null # optional override
|
||||
chunk_profile: null # optional override
|
||||
tags: ["…"]
|
||||
edge_hints:
|
||||
- relation: references
|
||||
target: "note-id-or-path"
|
||||
- relation: depends_on
|
||||
target: "…"
|
||||
```
|
||||
|
||||
### 4.2 Typregeln (aus types.yaml)
|
||||
- `chunk_profile`, `retriever_weight`, `edge_defaults` werden je Typ **automatisch** angewandt.
|
||||
- Aus `edge_defaults` abgeleitete Kanten werden **persistiert** (`source='rule'`).
|
||||
|
||||
### 4.3 Zeitgewichtung
|
||||
- Default: **Exponential Decay** mit globalem λ; Override per `time_weight_override` möglich.
|
||||
- Erlebnisse (`experience`) erhalten standardmäßig leicht höheren **Base‑Impact** im Score, um Persönlichkeit abzubilden.
|
||||
|
||||
### 4.4 Qualitätsregeln
|
||||
- Jede Note soll **mind. eine Relation** besitzen.
|
||||
- **Konflikte** (contradicts) werden nicht gefiltert, sondern sichtbar gemacht (mit Hinweis).
|
||||
- **Sichtbarkeit**: Antworten aus `sensitive` nur bei expliziter Freigabe.
|
||||
|
||||
---
|
||||
|
||||
## 5. Use Cases (repräsentativ)
|
||||
1. **„Warum habe ich 2019 X entschieden?“**
|
||||
- Abfrage auf `decision` + temporale Nachbarschaft zu `experience`/`principle`.
|
||||
- Ausgabe mit Erklärpfad: *Erlebnis → Prinzip → Projekt → Entscheidung*.
|
||||
|
||||
2. **„Plane ein Trainingsprogramm“**
|
||||
- Query kombiniert Konzepte/Erfahrungen/Projekte → Graph‑Walk + Typgewichtung, priorisiert jüngere Erkenntnisse.
|
||||
|
||||
3. **„Welche Ereignisse haben mein Leitbild geprägt?“**
|
||||
- Subgraph über `derived_from`/`inspired_by` Ketten + Zeitachsen‑Visualisierung.
|
||||
|
||||
---
|
||||
|
||||
## 6. Anti‑Patterns (vermeiden)
|
||||
- **Edges ohne Relation** („Dot‑Kanten“) – verhindern jede sinnvolle Auswertung.
|
||||
- **Nur Vektor‑Suche** ohne Graph / Zeitfaktor – verfälscht Persönlichkeit.
|
||||
- **Freies Tag‑Chaos** ohne Typregeln – mindert Reproduzierbarkeit.
|
||||
- **Heimliche Workarounds** (z. B. Felder mit Mehrdeutigkeit) – erschweren spätere Automatisierung.
|
||||
|
||||
---
|
||||
|
||||
## 7. Umsetzungsstrategie & Roadmap
|
||||
### Phase 1 (Jetzt)
|
||||
- Pflicht‑Frontmatter, deterministische IDs, Ingestion v2 (Chunks/Edges **persistiert**).
|
||||
- types.yaml + retrieval.yaml live‑fähig; Import‑Run‑Audit.
|
||||
- Tests (Parser/Chunker/Edge‑Rules), Regression auf Chunk‑Mengen.
|
||||
|
||||
### Phase 2
|
||||
- Hybrid‑Retrieval mit Graph‑Boosts, Explain‑Traces in API.
|
||||
- Subgraph‑Explorer, Governance‑Views (Review‑Queues).
|
||||
|
||||
### Phase 3
|
||||
- ML‑gestützte Edge‑Vorschläge, temporale Graphanalysen, Reasoning‑Chains.
|
||||
- ACL/Sharing‑Modelle, Export/Archiv‑Pipelines.
|
||||
|
||||
---
|
||||
|
||||
## 8. Beispiel‑Konfigurationen
|
||||
### 8.1 types.yaml (Beispiel – wie technisch)
|
||||
```yaml
|
||||
version: 1.0
|
||||
types:
|
||||
concept:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["references", "related_to"]
|
||||
retriever_weight: 0.33
|
||||
task:
|
||||
chunk_profile: short
|
||||
edge_defaults: ["depends_on", "belongs_to"]
|
||||
retriever_weight: 0.8
|
||||
experience:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["derived_from", "inspired_by"]
|
||||
retriever_weight: 0.9
|
||||
project:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.95
|
||||
```
|
||||
|
||||
### 8.2 retrieval.yaml
|
||||
```yaml
|
||||
relation_weights:
|
||||
references: 1.0
|
||||
related_to: 0.6
|
||||
depends_on: 1.2
|
||||
belongs_to: 0.8
|
||||
derived_from: 1.1
|
||||
inspired_by: 0.8
|
||||
supports: 1.0
|
||||
contradicts: -0.9
|
||||
mixing:
|
||||
alpha: 0.45
|
||||
beta: 0.25
|
||||
gamma: 0.20
|
||||
delta: 0.07
|
||||
epsilon: 0.03
|
||||
time_decay:
|
||||
lambda: 0.0025
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Governance & Nachvollziehbarkeit
|
||||
- **Decision Log** (ADR‑ähnlich): jedes Regel‑/Schema‑Update mit Begründung.
|
||||
- **Import‑Protokolle** versioniert, inkl. geänderter Dateien, Chunk‑/Edge‑Zahlen.
|
||||
- **Explainability Standard** in jeder Antwort.
|
||||
|
||||
---
|
||||
|
||||
## Anhang A: Glossar
|
||||
- **Chunk‑Profil**: Regelset zur Textteilung.
|
||||
- **Rule‑Edge**: automatisch erzeugte Kante auf Basis von Typregeln.
|
||||
- **Declared Edge**: explizit gesetzte Kante aus der Quelle.
|
||||
@@ -0,0 +1,474 @@
|
||||
# Mindnet – Technische Architektur
|
||||
**Version:** 2.0 • **Datum:** 2025-11-09 • **Autor:** Systemarchitektur (gemeinsam mit Owner)
|
||||
|
||||
---
|
||||
|
||||
## 0. Zweck dieses Dokuments
|
||||
Dieses Dokument beschreibt die **technische Architektur** von *mindnet* so präzise, dass ein erfahrener Engineer sie 1:1 implementieren kann. Es konsolidiert die bisherigen Überlegungen, eliminiert redundante/unklare Passagen und ergänzt fehlende Details (Datenmodell, Indizes, Import‑Pipelines, Scoring, Erklärbarkeit, Skalierung, Betrieb). Gültige Inhalte aus früheren Fassungen bleiben erhalten (siehe **Anhang A: Originalinhalt**).
|
||||
|
||||
> **Kernziele:**
|
||||
> - Persönliches Wissensnetz mit langlebiger Semantik (Notizen, Erlebnisse, Entscheidungen).
|
||||
> - Antworten, die **erklärbar** auf Erfahrungen, Konditionierung und Persönlichkeit beruhen.
|
||||
> - **Konfigurierbare Regeln** (Zeitgewichtung, Relationstypen, Typ‑Profile).
|
||||
> - **Lokal** betreibbar, später **skalierbar** (on‑prem → Rechenzentrum/Cloud).
|
||||
> - **Deterministisch**, idempotent, auditierbar.
|
||||
|
||||
---
|
||||
|
||||
## 1. Systemkontext (High Level)
|
||||
- **Quellen:** Obsidian‑Vault (Markdown + Frontmatter), künftig MediaWiki/HTML/PDF/E-Mail/Images.
|
||||
- **Ingestion:** Validator → Parser → Chunker (Profile) → Embedding → Graph‑& Payload‑Indexierung.
|
||||
- **Speicher:** Qdrant als Vektorstore + Payload (JSON), ergänzt um Dateimetadaten.
|
||||
- **APIs:** FastAPI (Read/Write), Tools (mindnet_query, mindnet_subgraph), n8n‑Glue.
|
||||
- **Retrieval/Reasoning:** Hybrid (Vektor + Keyword + Graph‑Walk) mit konfigurierbaren Scoring‑Regeln und **Erklärpfad**.
|
||||
- **Ops:** Deterministische IDs, Versions/Provenance‑Felder, Observability, Tests.
|
||||
|
||||
---
|
||||
|
||||
## 2. Zielarchitektur (Komponenten)
|
||||
### 2.1 Ingestion Layer
|
||||
1. **Frontmatter‑Validator**
|
||||
- Erzwingt Pflichtfelder: `id`, `title`, `type`, `created`, `modified`
|
||||
- Optionale Felder: `tags`, `visibility`, `relevance`, `time_weight_override`, `retriever_weight`, `chunk_profile`, `sources`, `aliases`, `edge_hints`.
|
||||
- Validiert `type` gegen **types.yaml** (Konfiguration).
|
||||
|
||||
2. **Parser**
|
||||
- Extrahiert: reinen Text, Überschriften, Querverweise (`[[Note]]`, Markdown‑Links), Zitate/Quellen.
|
||||
- Normalisiert Unicode, trennt Codeblöcke, Zitate, Aufzählungen.
|
||||
|
||||
3. **Chunker** (profilgesteuert)
|
||||
- Profile: `short` (~300–500 Token), `medium` (~700–1000), `long` (~1200–1600).
|
||||
- Regeln: Absatz‑, Überschriften‑, semantische Grenzen, Überschneidung ~10–15% für Kontextkohärenz.
|
||||
- Chunk‑Header speichert: `{note_id, path, type, idx, section_hierarchy, tokens, chunk_profile}`.
|
||||
|
||||
4. **Embedding**
|
||||
- Standard: MiniLM‑384d (lokal über Ollama/TEI), konfigurierbar (`embedding.model`, `dim`).
|
||||
- Persistiert Vektor + **Payload**.
|
||||
|
||||
5. **Edge‑Builder**
|
||||
- **Declared Edges**: aus expliziten Links/Referenzen in der Quelle.
|
||||
- **Inferred Edges (Rules)**: aus `types.yaml.edge_defaults` **und** Heuristiken (z. B. Abschnittstitel „Bezug“, „siehe auch“).
|
||||
- Beide Ebenen als **Edges** persistiert mit `source='declared' | 'rule'`, **kein bloßer Dot‑Marker**.
|
||||
|
||||
### 2.2 Storage Layer (Qdrant)
|
||||
**Collections (empfohlen):**
|
||||
- `mindnet_notes` (1 Vektor je Note, optional)
|
||||
- `mindnet_chunks` (**primärer Suchraum**)
|
||||
- `mindnet_edges` (Graph als Payload‑Dokumente)
|
||||
- `mindnet_runs` (Import‑Läufe, Audit/Provenance)
|
||||
- `mindnet_configs` (aktive Konfigurationen, z. B. `types.yaml` Schnappschuss)
|
||||
|
||||
**Beispiel‑Schemas:**
|
||||
|
||||
```yaml
|
||||
# mindnet_chunks.payload
|
||||
id: string # deterministic (UUIDv5 über prefix+path+idx)
|
||||
note_id: string # FK mindnet_notes.id
|
||||
type: string # z. B. concept|task|experience|project|event …
|
||||
title: string
|
||||
path: string # Pfad im Vault
|
||||
collection: string # Präfix/Namespace
|
||||
created: string # ISO‑Datum
|
||||
modified: string # ISO‑Datum
|
||||
chunk_idx: int
|
||||
chunk_profile: string # short|medium|long
|
||||
retriever_weight: float # aus types.yaml oder Note‑Override
|
||||
time_weight: float # berechnet (Decay) oder Override
|
||||
section: string # Überschrift/Gliederung
|
||||
text: string # Rohtext des Chunks
|
||||
tokens: int
|
||||
hash: string # content_hash (idempotent)
|
||||
visibility: string # public|internal|sensitive
|
||||
source: { # Provenance
|
||||
repo: string|null,
|
||||
path: string,
|
||||
commit: string|null,
|
||||
import_run_id: string
|
||||
}
|
||||
```
|
||||
|
||||
```yaml
|
||||
# mindnet_edges.payload
|
||||
id: string # deterministic (UUIDv5 über src_id + dst_id + relation + source)
|
||||
src_id: string # id (Note ODER Chunk)
|
||||
dst_id: string # id (Note ODER Chunk)
|
||||
src_kind: string # note|chunk
|
||||
dst_kind: string # note|chunk
|
||||
relation: string # references|related_to|depends_on|belongs_to|derived_from|inspired_by|mentions|contradicts|supports …
|
||||
weight: float # Standard 1.0; typenspezifische Offsets möglich
|
||||
source: string # declared|rule|ml
|
||||
evidence: string|null # Textausschnitt/Link der Ableitung
|
||||
created: string
|
||||
import_run_id: string
|
||||
visibility: string
|
||||
```
|
||||
|
||||
**Indizes (Qdrant):**
|
||||
- Vektorindex auf `mindnet_chunks` (HNSW)
|
||||
- Payload‑Indizes:
|
||||
- `type` (keyword), `visibility` (keyword), `created/modified` (datetime), `note_id` (keyword)
|
||||
- optional: **Full‑Text** auf `text` (Qdrant payload text index) für Hybrid‑Suche
|
||||
|
||||
### 2.3 Retrieval & Scoring
|
||||
**Hybrid‑Pipeline (empfohlen):**
|
||||
1. **Lexikalische Kandidaten** (FTS auf `text`, `title`, `aliases`) – k_l.
|
||||
2. **Vektor‑Kandidaten** (ANN auf Embeddings) – k_v.
|
||||
3. **Graph‑Expansion**: 1–2 Hops über `mindnet_edges` mit relation‑spezifischen Boosts.
|
||||
4. **Re‑Ranking** (finale Score):
|
||||
- `S = α·sim + β·kw + γ·G + δ·time_decay + ε·type_weight + ζ·recency_boost_overrides`
|
||||
- `type_weight` = `retriever_weight` aus **types.yaml** oder Note‑Override.
|
||||
- `time_decay` = `exp(−λ·age_days)`; λ global/konfigurierbar; Override per Frontmatter möglich.
|
||||
- **Erklärbarkeit:** Speichere die Summanden + Top‑Edges als **evidence** im Query‑Log.
|
||||
|
||||
**Relation‑Gewichte (Defaultvorschlag):**
|
||||
```yaml
|
||||
relation_weights:
|
||||
references: 1.0
|
||||
related_to: 0.6
|
||||
depends_on: 1.2
|
||||
belongs_to: 0.8
|
||||
derived_from: 1.1
|
||||
inspired_by: 0.8
|
||||
supports: 1.0
|
||||
contradicts: -0.9
|
||||
```
|
||||
|
||||
### 2.4 Konfiguration (types.yaml + config.yaml)
|
||||
```yaml
|
||||
version: 1.0
|
||||
types:
|
||||
concept: { chunk_profile: medium, edge_defaults: [references, related_to], retriever_weight: 0.33 }
|
||||
task: { chunk_profile: short, edge_defaults: [depends_on, belongs_to], retriever_weight: 0.80 }
|
||||
experience: { chunk_profile: medium, edge_defaults: [derived_from, inspired_by],retriever_weight: 0.90 }
|
||||
project: { chunk_profile: long, edge_defaults: [references, depends_on], retriever_weight: 0.95 }
|
||||
retrieval:
|
||||
alpha: 0.45 # sim
|
||||
beta: 0.25 # keyword
|
||||
gamma: 0.20 # graph
|
||||
delta: 0.07 # time decay
|
||||
epsilon: 0.03 # type weight
|
||||
time_decay:
|
||||
lambda: 0.0025 # ~ Halbwertzeit ≈ 277 Tage
|
||||
visibility_defaults: public
|
||||
```
|
||||
|
||||
### 2.5 Determinismus, Versionierung, Idempotenz
|
||||
- **IDs:** UUIDv5 über `{collection_prefix}:{path}:{chunk_idx}` (Chunks) bzw. `{collection_prefix}:{path}` (Notes).
|
||||
- **content_hash:** SHA256 über normalisierte Inhalte (Unicode NFC, Whitespace‑Policy, Überschriften‑Normalisierung).
|
||||
- **Import‑Lauf (`mindnet_runs`)**: `{run_id, started_at, finished_at, config_hash, files_seen, files_changed, errors}`.
|
||||
- **Idempotenz:** nur `changed` bei `content_hash`‑Abweichung.
|
||||
|
||||
### 2.6 Erklärbarkeit (Audit Trail)
|
||||
- Jede Antwort liefert: Top‑Chunks, Summanden des Scores, **angetretene Kanten** inkl. Relation & Gewicht, Zeitfaktor.
|
||||
- Persistiere Query‑Logs (optional): `{query, filters, topk, decision_trace, timestamp}`.
|
||||
|
||||
### 2.7 Sicherheit & Datenschutz
|
||||
- **visibility**: `public|internal|sensitive`, Query‑Filter standardmäßig `internal`.
|
||||
- Optionale **ACL**‑Schicht: Nutzer/Gruppen, Freigabe je Note/Chunk/Edge.
|
||||
- **Redaktions‑Flags**: `needs_review`, `draft`, `private_journal` (werden standardmäßig ausgeschlossen).
|
||||
|
||||
### 2.8 Skalierung & Betrieb
|
||||
- **Shards/Replicas** (Qdrant): 1/1 lokal; 2/2 in DC/Cloud.
|
||||
- **Snapshots/Backups**: Qdrant Snapshot + Vault‑Repo Snapshot (rsync/borg).
|
||||
- **Observability**: Prometheus‑Exporter (API Latenz, Import‑Dauer, Chunks/Edges pro Lauf), strukturierte Logs (JSON).
|
||||
- **Tests**: deterministische Golden‑Files für Parser/Chunker; Contract‑Tests für API; Offline‑Eval für Retrieval.
|
||||
|
||||
---
|
||||
|
||||
## 3. Implementierungsleitfaden (konkret)
|
||||
### 3.1 Collections anlegen (Qdrant)
|
||||
- `mindnet_chunks(dim=384, distance=cosine)`
|
||||
- `mindnet_notes(dim=384, distance=cosine)` (optional, falls Note‑Level‑Suche gewünscht)
|
||||
- `mindnet_edges(dim=0)` (reiner Payload‑Store; kein Vektor)
|
||||
- Payload‑Indizes: siehe 2.2
|
||||
|
||||
### 3.2 Import‑Pipeline (CLI)
|
||||
1. **Scan** Vault → valide Dateien.
|
||||
2. **Make Note Payload** (inkl. Typ‑Regeln; `retriever_weight`/`chunk_profile` aus **types.yaml**, Override per Frontmatter).
|
||||
3. **Chunken** gemäß Profil + `chunk_idx`.
|
||||
4. **Edges bauen**: declared + rule‑inferred (duplizierte Kanten per ID‑Determinismus verhindern).
|
||||
5. **Upsert**: Notes → Chunks → Edges (in dieser Reihenfolge).
|
||||
6. **Run‑Record** und **Config‑Schnappschuss** persistieren.
|
||||
|
||||
### 3.3 Query‑API (FastAPI)
|
||||
- `POST /query`: text, filters({visibility, types, date_range}), topk, graph_hops, `explain=true`.
|
||||
- `GET /subgraph/{id}`: Nachbarschaft, Relation‑Filter, Depth.
|
||||
- `POST /admin/reindex` / `POST /admin/import`.
|
||||
|
||||
### 3.4 Konfigurierbarkeit
|
||||
- `types.yaml` heiß ladbar (mit Versionsstempel); bei Änderung → `config_hash` & reimport optional.
|
||||
- `retrieval.yaml` für Gewichte, Relation‑Boosts, Zeitdecay.
|
||||
|
||||
---
|
||||
|
||||
## 4. Migrations‑ und Qualitätsregeln
|
||||
- **Keine Workarounds**, die Semantik zerstören (z. B. „Dot‑Edge“ ohne Relation).
|
||||
- **Explizite Kantenpersistenz**: *jede* Regelkante wird als `source='rule'` gespeichert.
|
||||
- **Chunk‑Zahl**: Profiltreue sicherstellen; Regressionstest (z. B. 171 ➝ 59 = **Fehler** oder Profiländerung dokumentieren).
|
||||
- **Provenance Pflicht**: `import_run_id` in Chunks/Edges setzen.
|
||||
|
||||
---
|
||||
|
||||
## 5. Offene Erweiterungen (vordenken)
|
||||
- **ML‑Edge‑Inference** (NER, Coref, TEI‑Reranking).
|
||||
- **Reasoning‑Graph** (Motivationsketten: Kindheitserlebnis → Prinzip → Entscheidung).
|
||||
- **Citations‑Fabric** (autom. Quellenkonsolidierung).
|
||||
- **Temporale Graphabfragen** (Edges mit Zeitintervallen).
|
||||
|
||||
---
|
||||
|
||||
## Anhang A: Originalinhalt (unverändert, falls vorhanden)
|
||||
Der vollständige Originaltext aus `knowledge_design.md` bleibt hier als Referenz erhalten, damit keine implizite Aussage verloren geht.
|
||||
|
||||
|
||||
---
|
||||
|
||||
# mindnet – Knowledge Design
|
||||
**Version:** 1.4.0 (aktualisiert 2025-10-06)
|
||||
|
||||
## 1. Gesamtziel und Systemverständnis
|
||||
|
||||
**mindnet** ist ein lokales Wissensnetzwerk, das alle meine Notizen, Projekte, Gedanken, Erfahrungen und Pläne als Markdown-Dateien speichert, automatisch importiert, semantisch in **Chunks** zerlegt und als **Graph-Struktur in Qdrant** ablegt.
|
||||
Es ist die Grundlage für ein agentenfähiges, lokal betriebenes LLM-System, das meine Daten verknüpft, abfragt und erweitert.
|
||||
|
||||
**Zentrale Ziele**
|
||||
- Vollständige Repräsentation persönlichen Wissens in strukturierter, maschinenlesbarer Form
|
||||
- Reproduzierbare Verarbeitung: Import → Chunking → Embedding → Edges → Graph
|
||||
- Verlustfreie Roundtrip-Fähigkeit (Import ⇄ Export)
|
||||
- Erweiterbarkeit für Agenten, RAG-Retriever und semantische Suche
|
||||
|
||||
**Prinzipien**
|
||||
- **Deterministisch:** IDs, Hashes und Edges wiederholbar
|
||||
- **Portabel:** Markdown + YAML
|
||||
- **Robust:** fehlertoleranter Parser
|
||||
- **Idempotent:** keine Duplikate bei Re-Import
|
||||
- **Erweiterbar:** neue Typen und Edge-Arten ohne Migration
|
||||
|
||||
---
|
||||
|
||||
## 2. YAML-Frontmatter-Schema
|
||||
|
||||
| Feld | Typ | Beschreibung |
|
||||
|-------|-----|--------------|
|
||||
| `title` | string | Menschlich lesbarer Titel |
|
||||
| `id` | string | Eindeutige Note-ID (Slug) |
|
||||
| `type` | enum | `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone` |
|
||||
| `status` | enum | `draft`, `active`, `done`, `archived` |
|
||||
| `created` | date | ISO-Datum |
|
||||
| `updated` | date | Letzte Änderung |
|
||||
| `area` | string | Themen- oder Lebensbereich |
|
||||
| `project` | string | Zugehöriges Projekt |
|
||||
| `tags` | list | Strukturierte Tags (`area/…`, `topic/…`) |
|
||||
| `depends_on` | list | IDs anderer Notizen |
|
||||
| `assigned_to` | list | Beteiligte Personen |
|
||||
| `embedding_exclude` | bool | Falls `true`, keine Embeddings |
|
||||
| `hash_mode` | enum | Hash-Modus beim Import (`body`, `frontmatter`, `full`) |
|
||||
| `priority` | enum/int | 1–5 oder `low` / `med` / `high` |
|
||||
| `effort_min`, `due` | int/date | Aufwand / Termin |
|
||||
| `aliases` | list | Alternative IDs |
|
||||
| `lang` | string | ISO-Sprachcode |
|
||||
| `source` | string | Herkunft / Referenzquelle |
|
||||
|
||||
**Pflichtfelder:** `title`, `id`, `type`, `status`, `created`
|
||||
**Optionale Felder:** alle übrigen
|
||||
|
||||
---
|
||||
|
||||
## 3. Dateinamen und Ordner
|
||||
|
||||
- Format: `YYYY-MM-DD_title.md` oder `slug.md`
|
||||
- Strukturierte Ordner:
|
||||
- `10_thoughts/`, `20_concepts/`, `30_projects/`, `40_experiences/`, …
|
||||
- Alle Pfade relativ; Unterstriche statt Leerzeichen
|
||||
|
||||
---
|
||||
|
||||
## 4. Link- und Edge-Design
|
||||
|
||||
**Verlinkungen**
|
||||
- Wikilinks: `[[note-id]]` oder `[[Titel|note-id]]`
|
||||
- Automatische Edges aus Text und Frontmatter
|
||||
|
||||
**Edge-Typen in Qdrant**
|
||||
|
||||
| Typ | Bedeutung | Quelle | Scope |
|
||||
|------|------------|--------|--------|
|
||||
| `belongs_to` | Chunk → Note | intern | chunk |
|
||||
| `prev` / `next` | Reihenfolge | intern | chunk |
|
||||
| `references` | explizite Wikilinks | Text | chunk |
|
||||
| `backlink` | Gegenkante zu `references` | abgeleitet | note |
|
||||
| `depends_on` | YAML-Abhängigkeiten | YAML | note |
|
||||
| `assigned_to` | YAML-Zuweisung | YAML | note |
|
||||
| `unresolved` | Ziel fehlt (Stub) | abgeleitet | chunk |
|
||||
|
||||
**Edge-Schlüssel**
|
||||
- `source_id`, `target_id`, `kind`, `scope`, `note_id`
|
||||
- Dedup-Schlüssel: Kombination aus `(kind, source_id, target_id, scope)`
|
||||
|
||||
---
|
||||
|
||||
## 5. Chunking-Modell
|
||||
|
||||
**Chunk-Datenstruktur**
|
||||
| Feld | Beschreibung |
|
||||
|-------|---------------|
|
||||
| `note_id` | Zugehörige Note |
|
||||
| `chunk_index` | Laufende Nummer |
|
||||
| `text` | Originalabschnitt |
|
||||
| `window` | Text + Overlap links/rechts |
|
||||
| `overlap_left` | Länge des linken Overlaps |
|
||||
| `overlap_right` | Länge des rechten Overlaps |
|
||||
| `embedding` | 384-dimensionaler Vektor |
|
||||
| `tokens` | Tokenanzahl (optional) |
|
||||
|
||||
**Chunking-Regeln (aus `chunking_strategy.md`)**
|
||||
- Trennung nach Absätzen, Überschriften, Listen, Tabellen
|
||||
- Token-Zielgröße 350–500 (max 600)
|
||||
- Overlap: 30–40 % für semantische Kohärenz
|
||||
- Overlap wird in `window` integriert, sodass `text ≠ window`
|
||||
|
||||
---
|
||||
|
||||
## 6. Hash-Strategie und Änderungsverfolgung
|
||||
|
||||
**Zweck:** Nur bei realer Inhaltsänderung re-indexieren
|
||||
|
||||
| Umgebungsvariable | Beschreibung |
|
||||
|--------------------|--------------|
|
||||
| `MINDNET_HASH_COMPARE` | Vergleichsquelle (`Body`, `Full`, `Frontmatter`) |
|
||||
| `MINDNET_HASH_SOURCE` | Rohtext vs. geparst (`raw` / `parsed`) |
|
||||
| `MINDNET_HASH_NORMALIZE` | Normalisierung (`canonical` / `none`) |
|
||||
|
||||
**Basismodi**
|
||||
- `body` → Nur Textkörper
|
||||
- `frontmatter` → Nur Metadaten
|
||||
- `full` → Kombination
|
||||
- Hash = SHA-256(canonicalized_input)
|
||||
|
||||
Bei Änderung: Re-Import, neue Embeddings, Edges werden aktualisiert.
|
||||
|
||||
---
|
||||
|
||||
## 7. Qdrant-Speicherstruktur
|
||||
|
||||
| Collection | Inhalt | Primärfelder | Embedding |
|
||||
|-------------|---------|---------------|------------|
|
||||
| `mindnet_notes` | Notes mit Fulltext & Hash | `note_id` | optional |
|
||||
| `mindnet_chunks` | Chunks mit Text, Window, Embedding | `chunk_id`, `note_id` | 384 d |
|
||||
| `mindnet_edges` | Graphkanten | `edge_id`, `source_id`, `target_id` | — |
|
||||
|
||||
**Upsert-Strategie**
|
||||
- Idempotente UUIDv5-IDs
|
||||
- Keine Duplikate durch deterministische Signaturen
|
||||
- `purge-before-upsert` zur Bereinigung
|
||||
|
||||
**Indizes**
|
||||
- notes: `note_id`, `hash_signature`
|
||||
- chunks: `note_id`, `chunk_index`
|
||||
- edges: `(kind, source_id, target_id, scope)`
|
||||
|
||||
---
|
||||
|
||||
## 8. Fehler-Toleranz und Parser-Robustheit
|
||||
|
||||
- UTF-8 mit Fallback: ersetzt ungültige Bytes (`errors="replace"`)
|
||||
- Ignoriert Nullbytes, BOM, exotische Zeilenenden
|
||||
- Erkennt defekte oder leere YAML-Header und überspringt sie
|
||||
- Logt Problemfälle (`read_markdown failed`, `make_note_payload returned non-dict`)
|
||||
- Bei Fehlern: Import → warn → continue
|
||||
|
||||
---
|
||||
|
||||
## 9. Tests & Validierung
|
||||
|
||||
**Integrationstests**
|
||||
- Roundtrip: Import → Export → Compare (`compare_vaults.py`)
|
||||
- Chunk-Integrität: `verify_chunks_integrity.py`
|
||||
- Hash-Audit: `hash_reporter.py`
|
||||
- Window-Vergleich: `check_chunks_window_vs_text.py`
|
||||
|
||||
**Erwartete Ergebnisse**
|
||||
- Alle Notes werden erkannt
|
||||
- `window ≠ text` bei > 60 % der Chunks
|
||||
- Roundtrip-Vergleich liefert „OK“
|
||||
- Edge-Anzahl entspricht Modell (belongs_to = #Chunks, next/prev ≈ #Chunks – 1)
|
||||
|
||||
---
|
||||
|
||||
## 10. Export & Roundtrip-Verhalten
|
||||
|
||||
- `export_markdown.py` erzeugt identische Markdown-Dateien
|
||||
- Schreibt YAML + Body in korrekter Reihenfolge
|
||||
- Roundtrip-Prüfung (`compare_vaults.py`) validiert:
|
||||
- vollständigen Body
|
||||
- unveränderte Hash-Signatur
|
||||
- gleiche Chunkzahl
|
||||
|
||||
---
|
||||
|
||||
## 11. Graph-Schicht und WP-04-Integration
|
||||
|
||||
- Qdrant fungiert als Graph-Backend
|
||||
- `graph/service.py` (WP-04): stellt API für Mehrhop-Abfragen bereit
|
||||
- Unterstützt:
|
||||
- `expand(note_id, hops=2)`
|
||||
- `resolve_unresolved()`
|
||||
- `neighbors(kind=…)`
|
||||
- Grundlage für spätere **Retriever-Funktionen** (LLM / Agenten)
|
||||
|
||||
---
|
||||
|
||||
## 12. Agenten- und LLM-Integration
|
||||
|
||||
**Retriever-Pipeline**
|
||||
1. Suche top-k Chunks (cosine distance)
|
||||
2. Erweiterung um Edges (`references`, `backlinks`)
|
||||
3. Kontext-Assembling via `window`-Text
|
||||
4. Prompt-Generierung
|
||||
|
||||
**Agent-Use-Cases**
|
||||
- Automatisches Tagging (`type`, `status`)
|
||||
- Vorschläge für neue Edges
|
||||
- Auflösen von `unresolved`-Referenzen
|
||||
- Auto-Import externer Quellen (MediaWiki, PDFs, Webseiten)
|
||||
|
||||
---
|
||||
|
||||
## 13. Erweiterbarkeit
|
||||
|
||||
- Neue Edge-Typen: `inspired_by`, `relates_to`, `contradicts`
|
||||
- Neue Note-Typen: `decision`, `resource`, `recommendation`
|
||||
- Embedding-Engines austauschbar (MiniLM → LaBSE → E5)
|
||||
- Versionierte Chunking-Strategien (`chunk_config.py`)
|
||||
|
||||
---
|
||||
|
||||
## 14. Qualitätssicherung und Best Practices
|
||||
|
||||
- Validierung vor Upsert
|
||||
- Keine Leer-Chunks
|
||||
- Vollständige Metadaten in Notes
|
||||
- Tests in `tests/` automatisch ausführbar (`run_e2e_roundtrip.sh`)
|
||||
- Ergebnis: „Roundtrip OK“ und `verify_chunks_integrity` = OK
|
||||
|
||||
---
|
||||
|
||||
## 15. Änderungsverlauf
|
||||
|
||||
**v1.4.0 (2025-10-06)**
|
||||
- Neu: `window` vs `text` Feld in `mindnet_chunks`
|
||||
- Neu: Hash-Modi (`body`, `frontmatter`, `full`) mit Env-Steuerung
|
||||
- Neu: Baseline-Modus für Vergleich
|
||||
- Parser → fehlertolerant (UTF-8 replace)
|
||||
- Roundtrip-Test vollständig integriert
|
||||
- Graph-Service und Mehrhop-Abfragen (WP-04-Vorbereitung)
|
||||
- `purge-before-upsert` und `--note-scope-refs` Parameter ergänzt
|
||||
|
||||
**v1.3.0 (2025-09-09)**
|
||||
- Hash-Normalisierung und Edge-Scope
|
||||
- Roundtrip-Tests, Agenten-Integration
|
||||
|
||||
**v1.2.0 (2025-09-02)**
|
||||
- Grundstruktur des Knowledge-Designs
|
||||
|
||||
---
|
||||
@@ -0,0 +1,324 @@
|
||||
# mindnet v2 – Implementierungs-Playbook & Handover (Side‑by‑Side zu v1)
|
||||
*Stand:* 2025-11-10 08:03
|
||||
*Autor:* ChatGPT (Projekt‑Handover)
|
||||
*Zweck:* Vollständige, ausführbare Anleitung zur Weiterentwicklung von **mindnet** auf eine versionierte, testbare v2‑Architektur (neben v1), inklusive Prompt für den Folgethread, Akzeptanzkriterien, Dateiliste, Test‑ und Rollback‑Pläne.
|
||||
|
||||
---
|
||||
|
||||
## 0) Kontext & Zielbild (kurz)
|
||||
- **Mission:** Persönliches Wissensnetz, das langfristig die eigene Persönlichkeit, Erfahrungen und Entscheidungslogik abbildet; später nachvollziehbare Erklärungen für Familie/Nachwelt.
|
||||
- **Heutiger Stand (v1):** `mindnet_notes` in Qdrant vorhanden; **Chunks** und **Edges** bisher nur teilweise/nicht konsistent; Edge‑Defaults („1 Dot“) vorhanden, aber Ziele teils nicht materialisiert. Importskripte: `import_markdown.py`, `edges.py` (historisch).
|
||||
- **Ziel (v2):**
|
||||
- **Collections:** `mindnet_notes_v2`, `mindnet_chunks_v2`, `mindnet_edges_v2` (saubere Payload‑Schemata + Indizes).
|
||||
- **Chunker v2:** Block/Heading‑aware, Ziel 900–1200 Zeichen, Überlappung 120–180, deterministische `chunk_id`s.
|
||||
- **Edge‑Builder v2:** Reihenfolge: **explicit → rule → default_resolved → default**; saubere `provenance`.
|
||||
- **Importer‑Pipeline v2:** Parse → Chunk → Edge → Embeddings → Batch‑Upsert + Payload‑Indexpflege + Snapshots.
|
||||
- **Policies:** Privacy/Recency/Type‑Gewichte als **Konfiguration** (YAML + JSON‑Schema), nicht hart im Code.
|
||||
- **Observability:** FastAPI + OpenTelemetry (Tracing/Metrics); Import‑Reports (CSV/JSON).
|
||||
- **Side‑by‑Side:** v2 **parallel** zu v1; kein Big‑Bang.
|
||||
|
||||
> **Warum so?** Qdrant‑Payload‑Indizes/Filter → schnelle, erklärbare Selektionen; Snapshots → betriebssicheres Rollback; OTel → Nachvollziehbarkeit; YAML/JSON‑Schema → Validierbarkeit & Stabilität.
|
||||
|
||||
---
|
||||
|
||||
## 1) Datenmodelle (Schemata, v2)
|
||||
Die tatsächlichen JSON‑Schemata (2020‑12) werden im Repo abgelegt und in CI validiert.
|
||||
|
||||
### 1.1 note.schema.json
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://example.org/mindnet/note.schema.json",
|
||||
"type": "object",
|
||||
"required": ["id", "title", "type", "privacy", "created", "hash_body"],
|
||||
"properties": {
|
||||
"id": { "type": "string", "pattern": "^[a-zA-Z0-9_-]+$" },
|
||||
"title": { "type": "string", "minLength": 1 },
|
||||
"type": { "type": "string" },
|
||||
"privacy": { "type": "string", "enum": ["public","internal","private"] },
|
||||
"created": { "type": "string", "format": "date-time" },
|
||||
"modified": { "type": "string", "format": "date-time" },
|
||||
"tags": { "type": "array", "items": { "type": "string" } },
|
||||
"lang": { "type": "string", "default": "de" },
|
||||
"source_path": { "type": "string" },
|
||||
"source_collection": { "type": "string" },
|
||||
"hash_body": { "type": "string" },
|
||||
"hash_frontmatter": { "type": "string" },
|
||||
"token_count": { "type": "integer", "minimum": 0 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 1.2 chunk.schema.json
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://example.org/mindnet/chunk.schema.json",
|
||||
"type": "object",
|
||||
"required": ["chunk_id","note_id","text","ord"],
|
||||
"properties": {
|
||||
"chunk_id": { "type": "string", "pattern": "^[a-zA-Z0-9_-]+#c\d{4}$" },
|
||||
"note_id": { "type": "string" },
|
||||
"text": { "type": "string" },
|
||||
"ord": { "type": "integer", "minimum": 0 },
|
||||
"span_char_start": { "type": "integer", "minimum": 0 },
|
||||
"span_char_end": { "type": "integer", "minimum": 0 },
|
||||
"heading_path": { "type": "array", "items": { "type": "string" } },
|
||||
"section_title": { "type": "string" },
|
||||
"tokens_start": { "type": "integer", "minimum": 0 },
|
||||
"tokens_end": { "type": "integer", "minimum": 0 },
|
||||
"embeddings_version": { "type": "string" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 1.3 edge.schema.json
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://example.org/mindnet/edge.schema.json",
|
||||
"type": "object",
|
||||
"required": ["edge_id","src_note_id","relation","provenance"],
|
||||
"properties": {
|
||||
"edge_id": { "type": "string" },
|
||||
"src_note_id": { "type": "string" },
|
||||
"src_chunk_id": { "type": "string" },
|
||||
"dst_note_id": { "type": "string" },
|
||||
"dst_chunk_id": { "type": "string" },
|
||||
"relation": { "type": "string" },
|
||||
"evidence_spans":{ "type": "array", "items": {"type":"object","properties":{"chunk_id":{"type":"string"},"span":[{"type":"integer"},{"type":"integer"}]]} },
|
||||
"provenance": { "type": "string", "enum": ["explicit","rule","default_resolved","default"] },
|
||||
"rule_id": { "type": "string" },
|
||||
"confidence": { "type": "number", "minimum": 0, "maximum": 1 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 1.4 default_edge.schema.json
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://example.org/mindnet/default_edge.schema.json",
|
||||
"type": "object",
|
||||
"required": ["src_note_id","relation","target_kind"],
|
||||
"properties": {
|
||||
"src_note_id": { "type": "string" },
|
||||
"relation": { "type": "string" },
|
||||
"target_kind": { "type": "string" },
|
||||
"when": { "type": "string" },
|
||||
"strength_hint":{ "type": "number", "minimum": 0, "maximum": 1 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2) Qdrant: Collections & Indizes (v2)
|
||||
Anlegen **neben** v1:
|
||||
- `mindnet_notes_v2` (Payload‑Index: `type, privacy, created, modified, tags, source_path`)
|
||||
- `mindnet_chunks_v2` (Payload‑Index: `note_id, ord, heading_path`)
|
||||
- `mindnet_edges_v2` (Payload‑Index: `src_note_id, dst_note_id, relation, provenance`)
|
||||
|
||||
> Snapshots für Backups/Restore einplanen (Runbook siehe unten).
|
||||
|
||||
---
|
||||
|
||||
## 3) Import‑Pipeline v2 – Module & Flags
|
||||
- **Chunker v2** (`chunking/block_chunker.py`): `target_len=1000`, `overlap=150`, `respect_headings=True`, `min_chunk_len=600`, `max_chunk_len=1400`, `chunk_id = f\"{note_id}#c{ordinal:04d}\"`
|
||||
- **Edge‑Builder v2** (`graph/edge_builder_v2.py`): Reihenfolge **explicit → rule → default_resolved → default**, `provenance` korrekt setzen.
|
||||
- **Importer** (`scripts/import_markdown.py` erweitern oder `scripts/import_markdown_v2.py`):
|
||||
Flags: `--schema v2`, `--chunker v2`, `--edges v2`, `--dry-run`, `--apply`, `--prefix "$COLLECTION_PREFIX"`.
|
||||
|
||||
**Beispiel‑Run (dry‑run):**
|
||||
```bash
|
||||
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --dry-run --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
|
||||
**Apply:**
|
||||
```bash
|
||||
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --apply --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4) Policies (konfigurierbar, kein Hard‑Code)
|
||||
- `policies/retrieval.schema.json` (JSON‑Schema) und `policies/retrieval.yaml`, z. B.:
|
||||
```yaml
|
||||
privacy_order: [private, internal, public]
|
||||
recency_boost_half_life_days: 90
|
||||
type_priority:
|
||||
person: 1.2
|
||||
event: 1.1
|
||||
concept: 1.0
|
||||
max_chunks_per_note: 8
|
||||
edge_provenance_weights:
|
||||
explicit: 1.0
|
||||
rule: 0.8
|
||||
default_resolved: 0.6
|
||||
default: 0.2
|
||||
rule_sets:
|
||||
- id: "rules:types-v1"
|
||||
enabled: true
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5) Observability & Reports
|
||||
- **FastAPI + OpenTelemetry**: Traces um Phasen *parse/chunk/edge/upsert*; Metriken (Counter/Histogramme) pro Phase.
|
||||
- **Import‑Report** je Lauf (CSV/JSON): counts für Notes/Chunks/Edges, Fehler, Dauer, Embedding‑Version.
|
||||
|
||||
---
|
||||
|
||||
## 6) Teststrategie
|
||||
- **Gold‑Notizen** (3–5 repräsentative .md) mit erwarteten *Chunk‑Counts* und *Edge‑Sets*.
|
||||
- **Regression:** fixer Soll‑Count (z. B. 171 ± 15 %) über Gesamt‑Vault; keine Duplikate `(src,relation,dst)`.
|
||||
- **Dry‑Run vs Apply**: identische Counts, nur ohne Upserts.
|
||||
- **Filter‑Smoke‑Tests:** Qdrant‑Filter nach `privacy`, `type`, `tags`, `created` funktionieren performant.
|
||||
|
||||
---
|
||||
|
||||
## 7) Akzeptanzkriterien (KPI)
|
||||
1. **Chunk‑Qualität:** ≥ 90 % der Chunks enden an semantischen Grenzen; Ziel‑Länge 900–1200 Zeichen; deterministische IDs.
|
||||
2. **Edge‑Kohärenz:** Keine Duplikate gleicher `(src,relation,dst)`; ≥ 95 % der expliziten Links materialisiert; `default` nur, wenn kein Ziel existiert.
|
||||
3. **Filterbarkeit:** Queries nach `privacy/type/tags/created` performant (Payload‑Index vorhanden).
|
||||
4. **Betriebsfestigkeit:** Telemetrie aktiv; fehlertoleranter Import; Qdrant‑Snapshot nach erfolgreichem Lauf.
|
||||
|
||||
---
|
||||
|
||||
## 8) Roadmap in kleinen, testbaren Schritten
|
||||
|
||||
### Step 0 – Safeguards
|
||||
- Qdrant‑Snapshot v1 erstellen + Restore‑Probe.
|
||||
- ENV: `MINDNET_SCHEMA_VERSION=2` (Importer wird v2 schreiben, v1 bleibt unberührt).
|
||||
|
||||
### Step 1 – Schemata (nur Dateien)
|
||||
- Ablage: `/schemas/note.schema.json`, `/schemas/chunk.schema.json`, `/schemas/edge.schema.json`, `/schemas/default_edge.schema.json`
|
||||
- CI‑Job: `make schema-validate` (jsonschema).
|
||||
- **Abnahme:** Validator grün auf 3 Gold‑Notizen.
|
||||
|
||||
### Step 2 – Qdrant v2‑Collections
|
||||
- Anlegen der 3 v2‑Collections + Payload‑Indizes.
|
||||
- **Abnahme:** Filter‑Query liefert erwartbare Ergebnisse.
|
||||
|
||||
### Step 3 – Chunker v2
|
||||
- Implementierung & Flag `--chunker v2`.
|
||||
- **Abnahme:** Chunk‑Counts ~ alt (≈171 ± 15 %), semantische Schnitte.
|
||||
|
||||
### Step 4 – Edge‑Builder v2
|
||||
- Reihenfolge & `provenance` strikt umsetzen, `default_resolved` integrieren.
|
||||
- **Abnahme:** erwartete Relationensätze auf Gold‑Notizen, keine Duplikate.
|
||||
|
||||
### Step 5 – Importer‑Pipeline v2
|
||||
- `--schema v2` Side‑by‑Side; Batch‑Upserts; vollständige Payloads; Recency‑Boost Konfig **nur** in Policy.
|
||||
- **Abnahme:** Dry‑Run/Apply‑Parität, Reports, Telemetrie‑Ereignisse sichtbar.
|
||||
|
||||
### Step 6 – Observability
|
||||
- OTel‑Instrumentierung + `/import/status` + Reports.
|
||||
- **Abnahme:** Traces & Metriken vorhanden.
|
||||
|
||||
### Step 7 – Snapshots & Runbook
|
||||
- Snapshot nach Erfolg; Restore‑Dokument.
|
||||
- **Abnahme:** Restore‑Probe erfolgreich.
|
||||
|
||||
### Step 8 – Policies
|
||||
- `policies/retrieval.yaml` + JSON‑Schema; A/B‑Test (Policy ändern → Retrieval ändert sich ohne Reimport).
|
||||
- **Abnahme:** Sichtbarer Effekt laut Testfall.
|
||||
|
||||
### Step 9 – Umschalten
|
||||
- Abnahme‑Doku, KPI‑Delta dokumentiert; Alias/Flag‑Switch auf v2.
|
||||
- **Abnahme:** Funktionale Gleichwertigkeit + Qualitätsgewinn.
|
||||
|
||||
---
|
||||
|
||||
## 9) Dateien (neu/ändern)
|
||||
**Neu:**
|
||||
- `schemas/note.schema.json`
|
||||
- `schemas/chunk.schema.json`
|
||||
- `schemas/edge.schema.json`
|
||||
- `schemas/default_edge.schema.json`
|
||||
- `policies/retrieval.schema.json`
|
||||
- `policies/retrieval.yaml`
|
||||
- `chunking/block_chunker.py`
|
||||
- `graph/edge_builder_v2.py`
|
||||
- `docs/OPERATIONS.md` (Snapshots/Restore/Runbook)
|
||||
- `tests/gold_notes/manifest.yaml` (+ 3–5 Notizen Kopien)
|
||||
|
||||
**Änderungen:**
|
||||
- `scripts/import_markdown.py` **oder** `scripts/import_markdown_v2.py` neu (Flags `--schema/--chunker/--edges`).
|
||||
- `edges.py` (falls weiterverwendet): auf **v2‑Edge‑Builder** migrieren oder auslaufen lassen.
|
||||
- `types.yaml`: Regel‑Definitionen klarisieren (IDs, Bedingungen), **nicht** hart in Code.
|
||||
|
||||
---
|
||||
|
||||
## 10) Beispiel‑Kommandos
|
||||
```bash
|
||||
# Step 2: Collections prüfen
|
||||
curl -s http://127.0.0.1:6333/collections | jq
|
||||
|
||||
# Step 3: Chunker v2 (dry-run)
|
||||
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --dry-run --prefix "$COLLECTION_PREFIX"
|
||||
|
||||
# Step 5: Apply + Report
|
||||
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --apply --prefix "$COLLECTION_PREFIX" --report ./reports/import_$(date +%F_%H%M).json
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 11) Rollback
|
||||
- v2‑Collections droppen (wenn Side‑by‑Side).
|
||||
- Aus Snapshot wiederherstellen (v1).
|
||||
- Flags zurück auf v1.
|
||||
|
||||
---
|
||||
|
||||
## 12) Referenzen (offizielle Quellen)
|
||||
- Qdrant – Collections, Payload‑Index, Filter: <https://qdrant.tech/documentation/>
|
||||
- Qdrant – Snapshots/Backup: <https://qdrant.tech/documentation/guides/backup/>
|
||||
- OpenTelemetry (Python/FastAPI): <https://opentelemetry.io/docs/instrumentation/python/>
|
||||
- JSON‑Schema 2020‑12: <https://json-schema.org/>
|
||||
- YAML 1.2: <https://yaml.org/spec/1.2.2/>
|
||||
- Obsidian Frontmatter: <https://help.obsidian.md/Editing+and+formatting/Properties>
|
||||
- Ollama CLI/API (lokale Modelle): <https://github.com/ollama/ollama>
|
||||
|
||||
---
|
||||
|
||||
## 13) **PROMPT für den neuen Chat (bitte exakt so einfügen)**
|
||||
```
|
||||
Rolle: Du bist mein Senior‑Entwickler & Architekt für mindnet. Arbeite strikt in kleinen, testbaren Schritten (Side‑by‑Side v2 neben v1). Liefere komplette Dateien als Downloads. Frage immer zuerst nach den **aktuellsten** Projektdateien, wenn du sie brauchst (z. B. import_markdown.py, edges.py, types.yaml), ändere keine Systemfunktion „workaround‑artig“ ohne die Gesamtwirkung zu prüfen.
|
||||
|
||||
Kontext (Kurzfassung):
|
||||
- v1 hat nur mindnet_notes zuverlässig. Chunks/Edges sind inkonsistent/teilweise leer. Ziel: v2 mit drei Collections (notes/chunks/edges), neuem Chunker v2, Edge‑Builder v2 (explicit → rule → default_resolved → default), Policies (YAML), Observability (OTel), Snapshots. Kein Big‑Bang; v2 parallel zu v1.
|
||||
- Siehe angehängte Datei **mindnet_v2_implementation_playbook.md** (Pfad/Download im Chat). Alles daraus ist verbindlich.
|
||||
|
||||
Deine ersten Aufgaben (Step‑by‑Step):
|
||||
1) **Schemata anlegen (Step 1)**
|
||||
- Erstelle die Dateien:
|
||||
- schemas/note.schema.json
|
||||
- schemas/chunk.schema.json
|
||||
- schemas/edge.schema.json
|
||||
- schemas/default_edge.schema.json
|
||||
- Nutze die in der Playbook‑Datei vorgegebenen Strukturen als Basis.
|
||||
- Liefere zusätzlich ein einfaches `Makefile`‑Ziel `schema-validate` (jsonschema via Python), plus `requirements.txt`.
|
||||
- Output: komplette Dateien als Downloads + kurze Testanleitung.
|
||||
|
||||
2) **Qdrant v2‑Collections (Step 2)**
|
||||
- Erzeuge die Collections + Payload‑Indizes.
|
||||
- Liefere ein kleines Python‑Skript `tools/qdrant_bootstrap_v2.py`, das diese Anlage idempotent durchführt (inkl. Prüf‑Output).
|
||||
|
||||
3) **Chunker v2 (Step 3)**
|
||||
- Implementiere `chunking/block_chunker.py` mit den im Playbook genannten Parametern.
|
||||
- Teste gegen 3 Gold‑Notizen (liefere mini Test‑Harness in `tests/gold_notes/…`).
|
||||
|
||||
Arbeitsweise:
|
||||
- Jede Aufgabe einzeln, mit klaren Akzeptanzkriterien (aus Playbook) und Download‑Artefakten.
|
||||
- Keine parallelen Großumbauten.
|
||||
- Immer deterministische IDs, idempotente Upserts.
|
||||
|
||||
Dateien, die ich dir anfänglich bereitstelle:
|
||||
- **mindnet_v2_implementation_playbook.md** (diese Datei)
|
||||
- **import_markdown.py** (aktuelle Version aus dem Projekt)
|
||||
- **edges.py** (aktuelle Version aus dem Projekt)
|
||||
- **types.yaml** (aktuelle Version)
|
||||
|
||||
Sage mir jeweils, welche Datei du als Nächstes brauchst. Beginne jetzt mit Aufgabe 1 (Schemata).
|
||||
```
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tests/assert_payload_schema.py (compat v1.1)
|
||||
|
||||
Prüft, ob die erwarteten Payload-Indizes (payload_schema) auf den drei Collections vorhanden sind.
|
||||
Kompatibel mit älteren qdrant-client Versionen (ohne with_payload_schema im Wrapper).
|
||||
|
||||
Collections & Pflichtfelder:
|
||||
- mindnet_notes : note_id, type, title, updated, tags
|
||||
- mindnet_chunks : note_id, chunk_id, index, type, tags
|
||||
- mindnet_edges : note_id, kind, scope, source_id, target_id, chunk_id
|
||||
|
||||
Nutzung:
|
||||
python3 -m tests.assert_payload_schema
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
REQUIRED = {
|
||||
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||
"chunks": ["note_id", "chunk_id", "index", "type", "tags"],
|
||||
"edges": ["note_id", "kind", "scope", "source_id", "target_id", "chunk_id"],
|
||||
}
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
return obj if isinstance(obj, dict) else {}
|
||||
|
||||
def _cfg_base_url(cfg) -> str:
|
||||
if getattr(cfg, "url", None):
|
||||
return cfg.url.rstrip('/')
|
||||
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
if api_key:
|
||||
req.add_header("api-key", api_key)
|
||||
req.add_header("Authorization", f"Bearer {api_key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||
# 1) OpenAPI client mit Flag
|
||||
try:
|
||||
oc = getattr(client, "openapi_client", None)
|
||||
if oc is not None and hasattr(oc, "collections_api"):
|
||||
api = oc.collections_api
|
||||
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = _safe_model_dump(info)
|
||||
return (d.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
pass
|
||||
# 2) Wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||
try:
|
||||
info = client.get_collection(collection_name=name)
|
||||
d = _safe_model_dump(info)
|
||||
ps = (d.get("result") or {}).get("payload_schema")
|
||||
if ps is not None:
|
||||
return ps
|
||||
except Exception:
|
||||
pass
|
||||
# 3) Direkter HTTP-Call
|
||||
try:
|
||||
base = _cfg_base_url(cfg)
|
||||
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||
return (raw.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def ensure_fields(schema: Dict[str, dict] | None, required: List[str]) -> Dict[str, bool]:
|
||||
ks = set((schema or {}).keys())
|
||||
return {k: (k in ks) for k in required}
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
names = {"notes": notes, "chunks": chunks, "edges": edges}
|
||||
|
||||
report = {}
|
||||
ok_all = True
|
||||
for kind, col in names.items():
|
||||
sch = get_collection_payload_schema(client, cfg, col)
|
||||
checks = ensure_fields(sch, REQUIRED[kind])
|
||||
ok = all(checks.values())
|
||||
ok_all = ok_all and ok
|
||||
report[kind] = {
|
||||
"collection": col,
|
||||
"ok": ok,
|
||||
"missing": [k for k,v in checks.items() if not v],
|
||||
"present": [k for k,v in checks.items() if v],
|
||||
}
|
||||
|
||||
print(json.dumps({"prefix": cfg.prefix, "ok": ok_all, "report": report}, ensure_ascii=False, indent=2))
|
||||
if not ok_all:
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/diag_payload_indexes.py (compat v1.2)
|
||||
|
||||
Zeigt payload_schema je Collection; kompatibel mit älteren qdrant-client Versionen.
|
||||
Strategie (in dieser Reihenfolge):
|
||||
1) openapi_client.collections_api.get_collection(..., with_payload_schema=True)
|
||||
2) client.get_collection(...)
|
||||
3) HTTP GET /collections/{name}?with_payload_schema=true
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
return obj if isinstance(obj, dict) else {}
|
||||
|
||||
|
||||
def _cfg_base_url(cfg) -> str:
|
||||
if getattr(cfg, "url", None):
|
||||
return cfg.url.rstrip("/")
|
||||
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
|
||||
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
if api_key:
|
||||
req.add_header("api-key", api_key)
|
||||
req.add_header("Authorization", f"Bearer {api_key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
|
||||
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||
# Strategy 1: OpenAPI client with flag
|
||||
try:
|
||||
oc = getattr(client, "openapi_client", None)
|
||||
if oc is not None and hasattr(oc, "collections_api"):
|
||||
api = oc.collections_api
|
||||
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = _safe_model_dump(info)
|
||||
return (d.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
pass
|
||||
# Strategy 2: wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||
try:
|
||||
info = client.get_collection(collection_name=name) # kein with_payload_schema kwarg!
|
||||
d = _safe_model_dump(info)
|
||||
ps = (d.get("result") or {}).get("payload_schema")
|
||||
if ps is not None:
|
||||
return ps
|
||||
except Exception:
|
||||
pass
|
||||
# Strategy 3: direkter HTTP Call
|
||||
try:
|
||||
base = _cfg_base_url(cfg)
|
||||
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||
return (raw.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
cols = [notes, chunks, edges]
|
||||
out = []
|
||||
for name in cols:
|
||||
ps = get_collection_payload_schema(client, cfg, name)
|
||||
# Vektorinfos (optional)
|
||||
try:
|
||||
info = client.get_collection(collection_name=name)
|
||||
d = _safe_model_dump(info)
|
||||
vectors = (d.get("result") or {}).get("vectors")
|
||||
if isinstance(vectors, dict) and "config" in vectors:
|
||||
vectors = vectors["config"]
|
||||
except Exception:
|
||||
vectors = None
|
||||
out.append({"name": name, "vectors": vectors, "payload_schema": ps})
|
||||
print(json.dumps({"prefix": cfg.prefix, "collections": out}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
# scripts/edges_dryrun.py
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Dry-Run: Erzeuge Edges aus einem Vault **ohne** Qdrant-Upsert.
|
||||
- Liest Markdown mit YAML-Frontmatter
|
||||
- Chunking: einfacher Absatz-Chunker (Index + text)
|
||||
- Kanten: nutzt app.core.edges.build_edges_for_note()
|
||||
- Ausgabe: JSON pro Note mit Edge-Counts und 3 Beispiel-Payloads
|
||||
|
||||
Aufruf:
|
||||
python3 -m scripts.edges_dryrun --vault ./vault
|
||||
|
||||
Optional:
|
||||
--include-note-scope-refs # auch Frontmatter-Links (links[].target_id) als Note-Scope-Referenzen
|
||||
|
||||
Voraussetzungen:
|
||||
- app/core/parser.py (read_markdown, normalize_frontmatter)
|
||||
- app/core/edges.py (dieses Modul forwardet zur v2-Implementierung)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter
|
||||
from app.core.edges import build_edges_for_note
|
||||
|
||||
def _iter_markdown(vault: str):
|
||||
for p in Path(vault).rglob("*.md"):
|
||||
if p.name.startswith("."): # ignore hidden
|
||||
continue
|
||||
yield p
|
||||
|
||||
def _simple_chunker(body: str, note_id: str, note_type: str) -> List[Dict]:
|
||||
# Absatzbasiert, minimal ausreichend für Edges (window/text, chunk_id, ord, note_id, type)
|
||||
paras = [s.strip() for s in re.split(r"\n{2,}", body or "") if s.strip()]
|
||||
chunks = []
|
||||
for i, t in enumerate(paras):
|
||||
chunks.append({
|
||||
"chunk_id": f"{note_id}#c{i:04d}",
|
||||
"ord": i,
|
||||
"text": t,
|
||||
"note_id": note_id,
|
||||
"type": note_type,
|
||||
})
|
||||
return chunks
|
||||
|
||||
def _fm_note_refs(fm: Dict) -> List[str]:
|
||||
out = []
|
||||
links = fm.get("links") or []
|
||||
if isinstance(links, list):
|
||||
for e in links:
|
||||
if isinstance(e, dict):
|
||||
tid = e.get("target_id")
|
||||
if isinstance(tid, str) and tid.strip():
|
||||
out.append(tid.strip())
|
||||
return out
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True)
|
||||
ap.add_argument("--include-note-scope-refs", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
vault = args.vault
|
||||
include_note_scope = args.include_note_scope_refs
|
||||
|
||||
report = []
|
||||
for p in _iter_markdown(vault):
|
||||
parsed = read_markdown(str(p))
|
||||
if not parsed:
|
||||
continue
|
||||
fm = normalize_frontmatter(parsed.frontmatter or {})
|
||||
note_id = fm.get("id") or p.stem
|
||||
note_type = (fm.get("type") or "concept").lower()
|
||||
chunks = _simple_chunker(parsed.body, note_id, note_type)
|
||||
note_refs = _fm_note_refs(fm)
|
||||
|
||||
edges = build_edges_for_note(
|
||||
note_id=note_id,
|
||||
chunk_payloads=chunks,
|
||||
note_level_refs=note_refs,
|
||||
include_note_scope_refs=include_note_scope,
|
||||
)
|
||||
kinds = {}
|
||||
for e in edges:
|
||||
key = (e.get("relation") or e.get("kind") or "edge")
|
||||
kinds[key] = kinds.get(key, 0) + 1
|
||||
report.append({
|
||||
"path": str(p),
|
||||
"note_id": note_id,
|
||||
"type": note_type,
|
||||
"chunks": len(chunks),
|
||||
"edges_total": len(edges),
|
||||
"edges_by_kind": kinds,
|
||||
"samples": edges[:3],
|
||||
})
|
||||
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+247
-104
@@ -1,38 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
|
||||
Version: 3.7.2
|
||||
Datum: 2025-09-30
|
||||
scripts/import_markdown.py
|
||||
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
|
||||
- Änderungserkennung „Option C“: mehrere Hash-Varianten werden parallel in der Note
|
||||
gespeichert (Feld `hashes` mit Schlüsseln `<mode>:<source>:<normalize>`). Der Vergleich
|
||||
nutzt NUR den aktuellen Modus-Key — ein Moduswechsel triggert keine Massenänderungen mehr.
|
||||
- „Erstimport-Fix“: Bei leerem Qdrant gilt Create-Fall automatisch als geändert.
|
||||
- `--baseline-modes`: fehlende Hash-Varianten still nachtragen (nur Notes upserten).
|
||||
- `--sync-deletes`: gezielte Lösch-Synchronisation (Dry-Run + Apply).
|
||||
- `--only-path`: exakt **eine** Datei (Pfad) importieren — nützlich für Diagnosefälle.
|
||||
Zweck
|
||||
-----
|
||||
- Liest Markdown-Notizen aus einem Vault ein
|
||||
- Erzeugt Note-Payload, Chunk-Payloads (+ optionale Embeddings) und Edges
|
||||
- Schreibt alles idempotent in Qdrant (Notes, Chunks, Edges)
|
||||
- Integriert eine optionale Type-Registry (types.yaml), um z. B. chunk_profile
|
||||
und retriever_weight pro Notiz-Typ zu steuern.
|
||||
|
||||
Neu in 3.7.1/3.7.2
|
||||
------------------
|
||||
- Chunk-Payloads: `window` (für Embeddings), `text` (überlappungsfrei, verlustfrei rekonstruierbar),
|
||||
`start/end/overlap_*`. Embeddings nutzen `window`.
|
||||
- **3.7.2:** Edges-Fehler führen nicht mehr zum Abbruch der gesamten Note; Note/Chunks werden trotzdem geschrieben.
|
||||
|
||||
Hash/Compare Konfiguration
|
||||
--------------------------
|
||||
- Vergleichsmodus:
|
||||
--hash-mode body|frontmatter|full
|
||||
oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE
|
||||
- Quelle:
|
||||
--hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE, Default parsed)
|
||||
- Normalisierung:
|
||||
--hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE, Default canonical)
|
||||
- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich
|
||||
den parsed Body-Text direkt.
|
||||
Wesentliche Fixes ggü. vorherigen fehlerhaften Ständen
|
||||
------------------------------------------------------
|
||||
- `embed_texts` wird optional importiert und defensiv geprüft (kein NameError mehr)
|
||||
- `effective_chunk_profile` / `effective_retriever_weight` und Registry-Helfer
|
||||
sind VOR `main()` definiert (kein NameError mehr)
|
||||
- `retriever_weight` wird in Note- und Chunk-Payload zuverlässig gesetzt
|
||||
- Robuste Kantenbildung; Fehler bei Edges blockieren Notes/Chunks nicht
|
||||
- Korrekte Verwendung von `scroll_filter` beim Qdrant-Client
|
||||
- `--purge-before-upsert` entfernt alte Chunks/Edges einer Note vor dem Upsert
|
||||
|
||||
Qdrant / ENV
|
||||
------------
|
||||
@@ -40,6 +27,7 @@ Qdrant / ENV
|
||||
- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
|
||||
- VECTOR_DIM (Default: 384)
|
||||
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
||||
- MINDNET_TYPES_FILE: Pfad zu types.yaml (optional; Default: ./types.yaml)
|
||||
|
||||
Beispiele
|
||||
---------
|
||||
@@ -67,6 +55,7 @@ from typing import Dict, List, Optional, Tuple, Any, Set
|
||||
from dotenv import load_dotenv
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
# --- Projekt-Imports ---
|
||||
from app.core.parser import (
|
||||
read_markdown,
|
||||
normalize_frontmatter,
|
||||
@@ -92,14 +81,110 @@ from app.core.qdrant_points import (
|
||||
upsert_batch,
|
||||
)
|
||||
|
||||
# embeddings sind optional (z. B. im reinen Payload-Backfill)
|
||||
try:
|
||||
from app.core.embed import embed_texts # optional
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
embed_texts = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Helper
|
||||
# Type-Registry (types.yaml) – Helper (robust, optional)
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def _env(name: str, default: Optional[str] = None) -> Optional[str]:
|
||||
v = os.getenv(name)
|
||||
return v if v is not None else default
|
||||
|
||||
def _load_json_or_yaml(path: str) -> dict:
|
||||
import io
|
||||
data: dict = {}
|
||||
if not path or not os.path.exists(path):
|
||||
return data
|
||||
try:
|
||||
import yaml # type: ignore
|
||||
with io.open(path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return data
|
||||
except Exception:
|
||||
# YAML evtl. nicht installiert – versuche JSON
|
||||
try:
|
||||
with io.open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return data
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def load_type_registry() -> dict:
|
||||
# Reihenfolge: ENV > ./types.yaml (im aktuellen Arbeitsverzeichnis)
|
||||
p = _env("MINDNET_TYPES_FILE", None)
|
||||
if p and os.path.exists(p):
|
||||
return _load_json_or_yaml(p)
|
||||
fallback = os.path.abspath("./config/types.yaml") if os.path.exists("./config/types.yaml") else os.path.abspath("./types.yaml")
|
||||
return _load_json_or_yaml(fallback)
|
||||
|
||||
def get_type_config(note_type: Optional[str], reg: dict) -> dict:
|
||||
if not reg or not isinstance(reg, dict):
|
||||
return {}
|
||||
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
|
||||
if note_type and isinstance(note_type, str) and note_type in types:
|
||||
return types[note_type] or {}
|
||||
# Fallback: concept
|
||||
return types.get("concept", {}) or {}
|
||||
|
||||
def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
||||
if requested and isinstance(requested, str):
|
||||
return requested
|
||||
# Fallback wenn nichts gesetzt ist
|
||||
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
|
||||
return "concept" if "concept" in types else (requested or "concept")
|
||||
|
||||
def effective_chunk_profile(note_type: str, reg: dict) -> Optional[str]:
|
||||
"""Resolve chunk_profile for type or from defaults/global.
|
||||
Accepts symbolic profiles: short|medium|long|default.
|
||||
"""
|
||||
cfg = get_type_config(note_type, reg)
|
||||
prof = (cfg.get("chunk_profile") if isinstance(cfg, dict) else None)
|
||||
if isinstance(prof, str) and prof:
|
||||
return prof
|
||||
# defaults fallbacks
|
||||
for key in ("defaults", "default", "global"):
|
||||
dcfg = reg.get(key) if isinstance(reg, dict) else None
|
||||
if isinstance(dcfg, dict):
|
||||
dprof = dcfg.get("chunk_profile")
|
||||
if isinstance(dprof, str) and dprof:
|
||||
return dprof
|
||||
return "default"
|
||||
|
||||
def effective_retriever_weight(note_type: str, reg: dict) -> Optional[float]:
|
||||
"""Resolve retriever_weight for type or defaults; returns float.
|
||||
"""
|
||||
cfg = get_type_config(note_type, reg)
|
||||
w = (cfg.get("retriever_weight") if isinstance(cfg, dict) else None)
|
||||
try:
|
||||
if w is not None:
|
||||
return float(w)
|
||||
except Exception:
|
||||
pass
|
||||
# defaults fallbacks
|
||||
for key in ("defaults", "default", "global"):
|
||||
dcfg = reg.get(key) if isinstance(reg, dict) else None
|
||||
if isinstance(dcfg, dict):
|
||||
dw = dcfg.get("retriever_weight")
|
||||
try:
|
||||
if dw is not None:
|
||||
return float(dw)
|
||||
except Exception:
|
||||
pass
|
||||
return 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Sonstige Helper
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def iter_md(root: str) -> List[str]:
|
||||
@@ -115,17 +200,15 @@ def iter_md(root: str) -> List[str]:
|
||||
out.append(p)
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
|
||||
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
points, _ = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=f,
|
||||
scroll_filter=f, # wichtig: scroll_filter (nicht: filter)
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1,
|
||||
@@ -134,7 +217,6 @@ def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[D
|
||||
return None
|
||||
return points[0].payload or {}
|
||||
|
||||
|
||||
def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
out: Set[str] = set()
|
||||
@@ -158,7 +240,6 @@ def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||
_, chunks_col, edges_col = collections(prefix)
|
||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
@@ -172,7 +253,6 @@ def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||
except Exception as e:
|
||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
|
||||
|
||||
|
||||
def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
||||
notes_col, chunks_col, edges_col = collections(prefix)
|
||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
@@ -187,50 +267,56 @@ def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
|
||||
|
||||
|
||||
def _resolve_mode(val: Optional[str]) -> str:
|
||||
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
|
||||
if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
||||
return "full"
|
||||
if v in ("frontmatter", "fm"):
|
||||
return "frontmatter"
|
||||
return "body"
|
||||
# --- Neu: Existenz-Checks für Artefakte (fehlertoleranter Rebuild) ---
|
||||
|
||||
def _has_any_point(client, collection: str, note_id: str) -> bool:
|
||||
"""Prüft, ob es mind. einen Punkt mit note_id in der Collection gibt."""
|
||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
pts, _ = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=filt,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
limit=1,
|
||||
)
|
||||
return bool(pts)
|
||||
|
||||
def _env(key: str, default: str) -> str:
|
||||
return (os.environ.get(key) or default).strip().lower()
|
||||
def artifacts_missing(client, prefix: str, note_id: str) -> Tuple[bool, bool]:
|
||||
"""Gibt (chunks_missing, edges_missing) zurück."""
|
||||
_, chunks_col, edges_col = collections(prefix)
|
||||
chunks_missing = not _has_any_point(client, chunks_col, note_id)
|
||||
edges_missing = not _has_any_point(client, edges_col, note_id)
|
||||
return chunks_missing, edges_missing
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def _resolve_mode(m: Optional[str]) -> str:
|
||||
m = (m or "body").strip().lower()
|
||||
return m if m in {"body", "frontmatter", "full"} else "body"
|
||||
|
||||
def main() -> None:
|
||||
load_dotenv()
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
|
||||
ap.add_argument("--purge-before-upsert", action="store_true",
|
||||
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
|
||||
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
|
||||
ap.add_argument("--only-path", help="Exakt diesen Markdown-Pfad verarbeiten (ignoriert --note-id)")
|
||||
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
|
||||
ap.add_argument("--force-replace", action="store_true",
|
||||
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None,
|
||||
help="Vergleichsmodus (Body | Frontmatter | Full)")
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
|
||||
help="Quelle für die Hash-Berechnung (Default: parsed)")
|
||||
ap.add_argument("--note-scope-refs", action="store_true",
|
||||
help="(Optional) erzeugt zusätzlich references:note/backlink:note (Default: aus)")
|
||||
ap.add_argument("--debug-hash-diff", action="store_true",
|
||||
help="(reserviert) optionaler Body-Diff")
|
||||
ap.add_argument("--compare-text", action="store_true",
|
||||
help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
|
||||
ap.add_argument("--baseline-modes", action="store_true",
|
||||
help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
|
||||
ap.add_argument("--sync-deletes", action="store_true",
|
||||
help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)")
|
||||
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="scripts.import_markdown",
|
||||
description="Importiert Markdown-Notizen in Qdrant (Notes/Chunks/Edges)."
|
||||
)
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Vault (Ordner mit .md-Dateien)")
|
||||
ap.add_argument("--only-path", help="Nur diese Datei verarbeiten (absolut oder relativ)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt nach Qdrant (sonst Dry-Run)")
|
||||
ap.add_argument("--purge-before-upsert", action="store_true", help="Alte Chunks/Edges der Note vorher löschen")
|
||||
ap.add_argument("--force-replace", action="store_true", help="Note/Chunks/Edges unabhängig von Hash neu schreiben")
|
||||
ap.add_argument("--note-id", help="Nur Notes mit dieser ID verarbeiten (Filter)")
|
||||
ap.add_argument("--note-scope-refs", action="store_true", help="Note-scope References/Backlinks erzeugen")
|
||||
ap.add_argument("--hash-mode", help="body|frontmatter|full (Default body)")
|
||||
ap.add_argument("--hash-source", help="parsed|raw (Default parsed)")
|
||||
ap.add_argument("--hash-normalize", help="canonical|none (Default canonical)")
|
||||
ap.add_argument("--compare-text", action="store_true", help="Parsed fulltext zusätzlich direkt vergleichen")
|
||||
ap.add_argument("--baseline-modes", action="store_true", help="Fehlende Hash-Varianten still nachtragen (Notes)")
|
||||
ap.add_argument("--sync-deletes", action="store_true", help="Qdrant->Vault Lösch-Sync (Dry-Run; mit --apply ausführen)")
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||
args = ap.parse_args()
|
||||
|
||||
@@ -241,6 +327,7 @@ def main() -> None:
|
||||
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
||||
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
||||
|
||||
# Qdrant
|
||||
cfg = QdrantConfig.from_env()
|
||||
if args.prefix:
|
||||
cfg.prefix = args.prefix.strip()
|
||||
@@ -248,9 +335,12 @@ def main() -> None:
|
||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
|
||||
# Type-Registry laden (optional)
|
||||
reg = load_type_registry()
|
||||
|
||||
root = os.path.abspath(args.vault)
|
||||
|
||||
# Dateiliste bestimmen
|
||||
# Dateiliste
|
||||
if args.only_path:
|
||||
only = os.path.abspath(args.only_path)
|
||||
files = [only]
|
||||
@@ -293,16 +383,15 @@ def main() -> None:
|
||||
|
||||
processed = 0
|
||||
for path in files:
|
||||
# -------- Parse & Validate --------
|
||||
try:
|
||||
parsed = read_markdown(path)
|
||||
if not parsed:
|
||||
continue
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
||||
continue
|
||||
if parsed is None:
|
||||
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
|
||||
continue
|
||||
|
||||
# --- Frontmatter prüfen ---
|
||||
try:
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
@@ -315,24 +404,54 @@ def main() -> None:
|
||||
|
||||
processed += 1
|
||||
|
||||
# -------- Build new payload (includes 'hashes') --------
|
||||
note_pl = make_note_payload(
|
||||
parsed,
|
||||
vault_root=root,
|
||||
hash_mode=mode,
|
||||
hash_normalize=norm,
|
||||
hash_source=src,
|
||||
file_path=path,
|
||||
)
|
||||
# --- Type-Registry anwenden (chunk_profile / retriever_weight) ---
|
||||
try:
|
||||
note_type = resolve_note_type(fm.get("type"), reg)
|
||||
except Exception:
|
||||
note_type = (fm.get("type") or "concept")
|
||||
fm["type"] = note_type or fm.get("type") or "concept"
|
||||
|
||||
prof = effective_chunk_profile(note_type, reg)
|
||||
if prof:
|
||||
fm["chunk_profile"] = prof
|
||||
|
||||
weight = effective_retriever_weight(note_type, reg)
|
||||
if weight is not None:
|
||||
try:
|
||||
fm["retriever_weight"] = float(weight)
|
||||
except Exception:
|
||||
pass # falls FM string-inkonsistent ist
|
||||
|
||||
# --- Payload aufbauen (inkl. Hashes) ---
|
||||
try:
|
||||
note_pl = make_note_payload(
|
||||
parsed,
|
||||
vault_root=root,
|
||||
hash_mode=mode,
|
||||
hash_normalize=norm,
|
||||
hash_source=src,
|
||||
file_path=path,
|
||||
)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"make_note_payload failed: {type(e).__name__}: {e}"}))
|
||||
continue
|
||||
|
||||
if not note_pl.get("fulltext"):
|
||||
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
||||
|
||||
# retriever_weight sicher in Note-Payload spiegeln (für spätere Filter)
|
||||
if "retriever_weight" not in note_pl and fm.get("retriever_weight") is not None:
|
||||
try:
|
||||
note_pl["retriever_weight"] = float(fm.get("retriever_weight"))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
note_id = note_pl.get("note_id") or fm.get("id")
|
||||
if not note_id:
|
||||
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
|
||||
continue
|
||||
|
||||
# -------- Fetch old payload --------
|
||||
# --- bestehenden Payload laden (zum Diff) ---
|
||||
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
||||
has_old = old_payload is not None
|
||||
|
||||
@@ -352,17 +471,28 @@ def main() -> None:
|
||||
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
||||
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
||||
|
||||
# -------- Chunks / Embeddings --------
|
||||
chunk_pls: List[Dict[str, Any]] = []
|
||||
# --- Chunks + Embeddings vorbereiten ---
|
||||
try:
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||
chunk_pls: List[Dict[str, Any]] = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
|
||||
continue
|
||||
|
||||
vecs: List[List[float]] = [[0.0] * cfg.dim for _ in chunk_pls]
|
||||
# retriever_weight auf Chunk-Payload spiegeln
|
||||
if fm.get("retriever_weight") is not None:
|
||||
try:
|
||||
rw = float(fm.get("retriever_weight"))
|
||||
for pl in chunk_pls:
|
||||
# Feld nur setzen, wenn noch nicht vorhanden
|
||||
if "retriever_weight" not in pl:
|
||||
pl["retriever_weight"] = rw
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Embeddings (fallback: Nullvektoren)
|
||||
vecs: List[List[float]] = [[0.0] * int(cfg.dim) for _ in chunk_pls]
|
||||
if embed_texts and chunk_pls:
|
||||
try:
|
||||
texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls]
|
||||
@@ -370,10 +500,14 @@ def main() -> None:
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
||||
|
||||
# -------- Edges (robust) --------
|
||||
# --- Fehlende Artefakte in Qdrant ermitteln ---
|
||||
chunks_missing, edges_missing = artifacts_missing(client, cfg.prefix, note_id)
|
||||
|
||||
# --- Edges (robust) ---
|
||||
edges: List[Dict[str, Any]] = []
|
||||
edges_failed = False
|
||||
if changed and (not do_baseline_only):
|
||||
should_build_edges = (changed and (not do_baseline_only)) or edges_missing
|
||||
if should_build_edges:
|
||||
try:
|
||||
note_refs = note_pl.get("references") or []
|
||||
edges = build_edges_for_note(
|
||||
@@ -385,10 +519,9 @@ def main() -> None:
|
||||
except Exception as e:
|
||||
edges_failed = True
|
||||
edges = []
|
||||
# WICHTIG: Wir brechen NICHT mehr ab — Note & Chunks werden geschrieben.
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
|
||||
|
||||
# -------- Summary --------
|
||||
# --- Summary (stdout) ---
|
||||
summary = {
|
||||
"note_id": note_id,
|
||||
"title": fm.get("title"),
|
||||
@@ -396,10 +529,12 @@ def main() -> None:
|
||||
"edges": len(edges),
|
||||
"edges_failed": edges_failed,
|
||||
"changed": changed,
|
||||
"chunks_missing": chunks_missing,
|
||||
"edges_missing": edges_missing,
|
||||
"needs_baseline_for_mode": needs_baseline,
|
||||
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
||||
"apply" if args.apply and changed else
|
||||
"apply-skip-unchanged" if args.apply and not changed else
|
||||
"apply" if args.apply and (changed or chunks_missing or edges_missing) else
|
||||
"apply-skip-unchanged" if args.apply and not (changed or chunks_missing or edges_missing) else
|
||||
"dry-run"),
|
||||
"path": note_pl["path"],
|
||||
"hash_mode": mode,
|
||||
@@ -409,7 +544,7 @@ def main() -> None:
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
|
||||
# -------- Writes --------
|
||||
# --- Writes ---
|
||||
if not args.apply:
|
||||
continue
|
||||
|
||||
@@ -425,21 +560,29 @@ def main() -> None:
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
continue
|
||||
|
||||
if not changed:
|
||||
# Wenn nichts geändert und keine Artefakte fehlen → nichts zu tun
|
||||
if not changed and not (chunks_missing or edges_missing):
|
||||
continue
|
||||
|
||||
if args.purge_before_upsert and has_old:
|
||||
# Purge nur bei echten Änderungen (unverändert + fehlende Artefakte ≠ Purge)
|
||||
if args.purge_before_upsert and has_old and changed:
|
||||
try:
|
||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
||||
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
if chunk_pls:
|
||||
# Note nur bei Änderungen neu schreiben
|
||||
if changed:
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
|
||||
# Chunks schreiben, wenn geändert ODER vorher fehlend
|
||||
if chunk_pls and (changed or chunks_missing):
|
||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||
upsert_batch(client, chunks_name, chunk_pts)
|
||||
if edges:
|
||||
|
||||
# Edges schreiben, wenn vorhanden und (geändert ODER vorher fehlend)
|
||||
if edges and (changed or edges_missing):
|
||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||
upsert_batch(client, edges_name, edge_pts)
|
||||
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/payload_dryrun.py (zeigt, was VOR dem Upsert tatsächlich in den Payloads steht)
|
||||
- KEIN Überschreiben der Note-Payload mehr
|
||||
- types.yaml ist maßgeblich (gemäß app/core/note_payload.py & chunk_payload.py)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, os, json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunk_payload import make_chunk_payloads
|
||||
try:
|
||||
from app.core.derive_edges import build_edges_for_note
|
||||
except Exception:
|
||||
from app.core.edges import build_edges_for_note # type: ignore
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True)
|
||||
ap.add_argument("--note-id")
|
||||
ap.add_argument("--with-edges", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = os.path.abspath(args.vault)
|
||||
|
||||
files: List[str] = []
|
||||
for dp, _, fns in os.walk(root):
|
||||
for fn in fns:
|
||||
if fn.lower().endswith(".md"):
|
||||
files.append(os.path.join(dp, fn))
|
||||
files.sort()
|
||||
|
||||
for path in files:
|
||||
parsed = read_markdown(path)
|
||||
if not parsed:
|
||||
continue
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
try:
|
||||
validate_required_frontmatter(fm)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"invalid frontmatter: {e}"}))
|
||||
continue
|
||||
|
||||
if args.note_id and fm.get("id") != args.note_id:
|
||||
continue
|
||||
|
||||
# Note-Payload exakt so, wie der Importer ihn baut (types.yaml maßgeblich)
|
||||
note_pl = make_note_payload(parsed,
|
||||
vault_root=root,
|
||||
hash_mode="body",
|
||||
hash_normalize="canonical",
|
||||
hash_source="parsed",
|
||||
file_path=path)
|
||||
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
chunks = assemble_chunks(fm["id"], body_text, fm.get("type","concept"))
|
||||
|
||||
chunk_note = {
|
||||
"frontmatter": fm,
|
||||
"id": fm.get("id"),
|
||||
"type": fm.get("type"),
|
||||
"title": fm.get("title"),
|
||||
"path": note_pl.get("path") or path,
|
||||
"note_id": note_pl.get("note_id"),
|
||||
"tags": fm.get("tags"),
|
||||
}
|
||||
# make_chunk_payloads bestimmt Werte ebenfalls aus types.yaml (Frontmatter wird ignoriert)
|
||||
chunk_pls = make_chunk_payloads(
|
||||
chunk_note,
|
||||
note_pl["path"],
|
||||
chunks,
|
||||
note_text=body_text,
|
||||
types_cfg=None, # Loader aus Datei; kein Override von außen
|
||||
file_path=path,
|
||||
)
|
||||
|
||||
out = {
|
||||
"note_id": note_pl.get("note_id") or fm.get("id"),
|
||||
"title": fm.get("title"),
|
||||
"type": fm.get("type"),
|
||||
"note_payload": {
|
||||
"retriever_weight": note_pl.get("retriever_weight"),
|
||||
"chunk_profile": note_pl.get("chunk_profile")
|
||||
},
|
||||
"chunks_summary": {
|
||||
"count": len(chunk_pls),
|
||||
"first": [
|
||||
{k: chunk_pls[i].get(k) for k in ("chunk_id","index","ord","retriever_weight","chunk_profile","neighbors_prev","neighbors_next")}
|
||||
for i in range(min(3, len(chunk_pls)))
|
||||
]
|
||||
},
|
||||
"path": note_pl.get("path")
|
||||
}
|
||||
|
||||
if args.with_edges:
|
||||
edges = build_edges_for_note(
|
||||
note_id=note_pl.get("note_id") or fm.get("id"),
|
||||
chunk_payloads=chunk_pls,
|
||||
note_level_refs=note_pl.get("references") or [],
|
||||
include_note_scope_refs=False,
|
||||
)
|
||||
kinds = {}
|
||||
for e in edges:
|
||||
k = (e.get("relation") or e.get("kind") or "edge")
|
||||
kinds[k] = kinds.get(k, 0) + 1
|
||||
out["edges_summary"] = {"total": len(edges), "by_kind": kinds}
|
||||
|
||||
print(json.dumps(out, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+33
-10
@@ -2,28 +2,33 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: scripts/reset_qdrant.py
|
||||
Version: v1.1.1 (2025-09-05)
|
||||
Version: v1.2.0 (2025-11-11)
|
||||
Kurzbeschreibung:
|
||||
Sicheres Zurücksetzen der Qdrant-Collections für EIN Projektpräfix. Das Skript
|
||||
ermittelt zunächst die tatsächlich betroffenen Collections und zeigt eine
|
||||
Übersicht an. Anschließend wird der User um Bestätigung gebeten (interaktive
|
||||
Abfrage), bevor eine der beiden Aktionen ausgeführt wird:
|
||||
Abfrage), bevor eine der Aktionen ausgeführt wird:
|
||||
|
||||
- wipe: Löscht die Collections komplett und legt sie gemäß Projekt-Defaults neu an
|
||||
- truncate: Löscht nur alle Points (Inhalte), behält die Collection-Settings
|
||||
- wipe: Löscht die Collections komplett und legt sie gemäß Projekt-Defaults neu an.
|
||||
**Neu**: Danach werden *immer* die Payload-Indizes & Schema-Ergänzungen
|
||||
über ensure_payload_indexes() idempotent eingerichtet (falls nicht mit --no-indexes deaktiviert).
|
||||
- truncate: Löscht nur alle Points (Inhalte), behält die Collection-Settings.
|
||||
**Neu**: Optional (Default) werden im Anschluss die Payload-Indizes geprüft/ergänzt.
|
||||
|
||||
Aufruf (aus Projekt-Root, im venv):
|
||||
python3 -m scripts.reset_qdrant --mode wipe [--prefix PREFIX] [--yes] [--dry-run]
|
||||
python3 -m scripts.reset_qdrant --mode truncate [--prefix PREFIX] [--yes] [--dry-run]
|
||||
|
||||
Parameter:
|
||||
--mode wipe | truncate (Pflicht)
|
||||
--prefix Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')
|
||||
--yes Keine Rückfrage, direkt ausführen (CI/CD geeignet)
|
||||
--dry-run Nur anzeigen, was passieren würde; nichts ändern
|
||||
--mode wipe | truncate (Pflicht)
|
||||
--prefix Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')
|
||||
--yes Keine Rückfrage, direkt ausführen (CI/CD geeignet)
|
||||
--dry-run Nur anzeigen, was passieren würde; nichts ändern
|
||||
--no-indexes Überspringt den Schritt ensure_payload_indexes() (Standard: Indizes/Schema werden geprüft/ergänzt)
|
||||
|
||||
Umgebungsvariablen (optional):
|
||||
QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)
|
||||
(Zusätzliche ENV für Named Vectors etc. werden innerhalb ensure_collections/ensure_payload_indexes berücksichtigt.)
|
||||
|
||||
Sicherheitsmerkmale:
|
||||
- Betrachtet ausschließlich Collections des angegebenen Präfixes.
|
||||
@@ -34,6 +39,8 @@ Exitcodes:
|
||||
0 = OK, 1 = abgebrochen/keine Aktion, 2 = Verbindungs-/Konfigurationsfehler
|
||||
|
||||
Changelog:
|
||||
v1.2.0: ensure_payload_indexes() nach wipe/truncate standardmäßig ausführen (idempotent); --no-indexes Flag ergänzt.
|
||||
v1.1.1: Stabilisierung & Preview (2025-09-05).
|
||||
v1.1.0: Interaktive Bestätigung, --yes/--dry-run hinzugefügt, Preview der betroffenen Collections.
|
||||
v1.0.0: Wipe/Truncate ohne Bestätigungsabfrage.
|
||||
"""
|
||||
@@ -46,7 +53,13 @@ from typing import List
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names
|
||||
from app.core.qdrant import (
|
||||
QdrantConfig,
|
||||
get_client,
|
||||
ensure_collections,
|
||||
ensure_payload_indexes,
|
||||
collection_names,
|
||||
)
|
||||
|
||||
|
||||
def resolve_existing_collections(client: QdrantClient, prefix: str) -> List[str]:
|
||||
@@ -111,12 +124,13 @@ def wipe_collections(client: QdrantClient, all_col_names: List[str], existing: L
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Wipe oder truncate mindnet-Collections in Qdrant (mit Bestätigung).")
|
||||
ap = argparse.ArgumentParser(description="Wipe oder truncate mindnet-Collections in Qdrant (mit Bestätigung & Index-Setup).")
|
||||
ap.add_argument("--mode", choices=["wipe", "truncate"], required=True,
|
||||
help="wipe = Collections löschen & neu anlegen; truncate = nur Inhalte löschen")
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')")
|
||||
ap.add_argument("--yes", action="store_true", help="Ohne Rückfrage ausführen (CI/CD)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="Nur anzeigen, was passieren würde; nichts ändern")
|
||||
ap.add_argument("--no-indexes", action="store_true", help="Überspringt ensure_payload_indexes()")
|
||||
args = ap.parse_args()
|
||||
|
||||
# Qdrant-Konfiguration
|
||||
@@ -155,13 +169,22 @@ def main():
|
||||
if args.mode == "wipe":
|
||||
wipe_collections(client, all_col_names, existing)
|
||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
if not args.no_indexes:
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
print(f"Wiped & recreated (Prefix={cfg.prefix}): {all_col_names}")
|
||||
if not args.no_indexes:
|
||||
print("Payload-Indizes & Schema-Ergänzungen geprüft/ergänzt (ensure_payload_indexes).")
|
||||
else:
|
||||
if not existing:
|
||||
print("Keine existierenden Collections zum Truncaten gefunden. Beende ohne Aktion.")
|
||||
sys.exit(0)
|
||||
delete_all_points(client, existing)
|
||||
if not args.no_indexes:
|
||||
# Auch nach truncate sicherstellen, dass neue/fehlende Indizes ergänzt werden
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
print(f"Truncated (deleted all points in): {existing}")
|
||||
if not args.no_indexes:
|
||||
print("Payload-Indizes & Schema-Ergänzungen geprüft/ergänzt (ensure_payload_indexes).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tests/assert_payload_schema.py (compat v1.1)
|
||||
|
||||
Prüft, ob die erwarteten Payload-Indizes (payload_schema) auf den drei Collections vorhanden sind.
|
||||
Kompatibel mit älteren qdrant-client Versionen (ohne with_payload_schema im Wrapper).
|
||||
|
||||
Collections & Pflichtfelder:
|
||||
- mindnet_notes : note_id, type, title, updated, tags
|
||||
- mindnet_chunks : note_id, chunk_id, index, type, tags
|
||||
- mindnet_edges : note_id, kind, scope, source_id, target_id, chunk_id
|
||||
|
||||
Nutzung:
|
||||
python3 -m tests.assert_payload_schema
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
REQUIRED = {
|
||||
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||
"chunks": ["note_id", "chunk_id", "index", "type", "tags"],
|
||||
"edges": ["note_id", "kind", "scope", "source_id", "target_id", "chunk_id"],
|
||||
}
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
return obj if isinstance(obj, dict) else {}
|
||||
|
||||
def _cfg_base_url(cfg) -> str:
|
||||
if getattr(cfg, "url", None):
|
||||
return cfg.url.rstrip('/')
|
||||
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
if api_key:
|
||||
req.add_header("api-key", api_key)
|
||||
req.add_header("Authorization", f"Bearer {api_key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||
# 1) OpenAPI client mit Flag
|
||||
try:
|
||||
oc = getattr(client, "openapi_client", None)
|
||||
if oc is not None and hasattr(oc, "collections_api"):
|
||||
api = oc.collections_api
|
||||
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = _safe_model_dump(info)
|
||||
return (d.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
pass
|
||||
# 2) Wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||
try:
|
||||
info = client.get_collection(collection_name=name)
|
||||
d = _safe_model_dump(info)
|
||||
ps = (d.get("result") or {}).get("payload_schema")
|
||||
if ps is not None:
|
||||
return ps
|
||||
except Exception:
|
||||
pass
|
||||
# 3) Direkter HTTP-Call
|
||||
try:
|
||||
base = _cfg_base_url(cfg)
|
||||
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||
return (raw.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def ensure_fields(schema: Dict[str, dict] | None, required: List[str]) -> Dict[str, bool]:
|
||||
ks = set((schema or {}).keys())
|
||||
return {k: (k in ks) for k in required}
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
names = {"notes": notes, "chunks": chunks, "edges": edges}
|
||||
|
||||
report = {}
|
||||
ok_all = True
|
||||
for kind, col in names.items():
|
||||
sch = get_collection_payload_schema(client, cfg, col)
|
||||
checks = ensure_fields(sch, REQUIRED[kind])
|
||||
ok = all(checks.values())
|
||||
ok_all = ok_all and ok
|
||||
report[kind] = {
|
||||
"collection": col,
|
||||
"ok": ok,
|
||||
"missing": [k for k,v in checks.items() if not v],
|
||||
"present": [k for k,v in checks.items() if v],
|
||||
}
|
||||
|
||||
print(json.dumps({"prefix": cfg.prefix, "ok": ok_all, "report": report}, ensure_ascii=False, indent=2))
|
||||
if not ok_all:
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
check_types_registry_edges.py
|
||||
--------------------------------------------------
|
||||
Prüft, ob die in config/types.yaml hinterlegten Default-Kanten
|
||||
pro Note-Typ tatsächlich in der Qdrant *edges*-Collection auftauchen.
|
||||
"""
|
||||
import os, sys, json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Set
|
||||
from dataclasses import dataclass, asdict
|
||||
|
||||
try:
|
||||
import yaml # PyYAML
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"PyYAML not installed: {e}"})); sys.exit(2)
|
||||
|
||||
try:
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest # noqa: F401
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"qdrant-client not installed: {e}"})); sys.exit(2)
|
||||
|
||||
@dataclass
|
||||
class Cfg:
|
||||
url: str
|
||||
api_key: Optional[str]
|
||||
prefix: str
|
||||
notes: str
|
||||
edges: str
|
||||
types_file: Path
|
||||
|
||||
def _env_prefix() -> Optional[str]:
|
||||
for k in ("COLLECTION_PREFIX", "MINDNET_PREFIX"):
|
||||
v = os.environ.get(k, "").strip()
|
||||
if v:
|
||||
return v
|
||||
return None
|
||||
|
||||
def _load_types_yaml(path: Path) -> Dict:
|
||||
if not path.exists():
|
||||
print(json.dumps({"warn": f"types.yaml fehlt: {path}. Fallback: keine Vorgaben."}))
|
||||
return {}
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception as e:
|
||||
print(json.dumps({"warn": f"types.yaml defekt ({path}): {e}. Fallback: keine Vorgaben."}))
|
||||
return {}
|
||||
|
||||
def _cfg_from_env(argv_prefix: Optional[str]) -> Cfg:
|
||||
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
|
||||
api_key = os.environ.get("QDRANT_API_KEY", "").strip() or None
|
||||
prefix = (argv_prefix or _env_prefix() or "mindnet").strip()
|
||||
notes = f"{prefix}_notes"
|
||||
edges = f"{prefix}_edges"
|
||||
types_path = Path(os.environ.get("TYPES_FILE", "config/types.yaml")).resolve()
|
||||
return Cfg(url=url, api_key=api_key, prefix=prefix, notes=notes, edges=edges, types_file=types_path)
|
||||
|
||||
def _mk_client(cfg: Cfg) -> QdrantClient:
|
||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key, timeout=30.0)
|
||||
|
||||
def _scroll_all_notes(client: QdrantClient, notes_col: str) -> Dict[str, Dict]:
|
||||
out = {}
|
||||
offset = None
|
||||
while True:
|
||||
points, offset = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=None, limit=256, offset=offset,
|
||||
with_payload=True, with_vectors=False,
|
||||
)
|
||||
if not points:
|
||||
break
|
||||
for p in points:
|
||||
payload = p.payload or {}
|
||||
nid = payload.get("note_id") or payload.get("id") or payload.get("uid") or payload.get("slug")
|
||||
if not nid:
|
||||
nid = str(p.id)
|
||||
out[str(nid)] = payload
|
||||
return out
|
||||
|
||||
def _scroll_all_edges(client: QdrantClient, edges_col: str) -> List[Tuple[str, dict]]:
|
||||
out = []
|
||||
offset = None
|
||||
while True:
|
||||
points, offset = client.scroll(
|
||||
collection_name=edges_col,
|
||||
scroll_filter=None, limit=256, offset=offset,
|
||||
with_payload=True, with_vectors=False,
|
||||
)
|
||||
if not points:
|
||||
break
|
||||
for p in points:
|
||||
out.append((str(p.id), p.payload or {}))
|
||||
return out
|
||||
|
||||
def _guess_src_note_id(ed_pl: dict) -> Optional[str]:
|
||||
for k in ("src_note_id", "note_id", "src_id", "src"):
|
||||
nid = ed_pl.get(k)
|
||||
if nid:
|
||||
return str(nid)
|
||||
for k in ("src_ref", "from"):
|
||||
nid = ed_pl.get(k)
|
||||
if isinstance(nid, dict):
|
||||
for kk in ("note_id", "id"):
|
||||
if nid.get(kk):
|
||||
return str(nid[kk])
|
||||
elif isinstance(nid, str):
|
||||
return nid
|
||||
return None
|
||||
|
||||
def _edge_type(ed_pl: dict) -> Optional[str]:
|
||||
for k in ("edge_type", "type", "rel", "relation"):
|
||||
if ed_pl.get(k):
|
||||
return str(ed_pl[k])
|
||||
return None
|
||||
|
||||
def _cfg_to_jsonable(cfg: Cfg) -> dict:
|
||||
d = asdict(cfg)
|
||||
d["types_file"] = str(d.get("types_file"))
|
||||
return d
|
||||
|
||||
def main(argv=None):
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (mindnet, etc.)")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
cfg = _cfg_from_env(args.prefix)
|
||||
print(json.dumps({"cfg": _cfg_to_jsonable(cfg)}, ensure_ascii=False))
|
||||
|
||||
tr = _load_types_yaml(cfg.types_file)
|
||||
types_def = (tr.get("types") if isinstance(tr, dict) else {}) or {}
|
||||
print(json.dumps({"types_defined": list(types_def.keys())}, ensure_ascii=False))
|
||||
|
||||
client = _mk_client(cfg)
|
||||
|
||||
notes = _scroll_all_notes(client, cfg.notes)
|
||||
print(json.dumps({"notes_count": len(notes)}, ensure_ascii=False))
|
||||
|
||||
edges = _scroll_all_edges(client, cfg.edges)
|
||||
print(json.dumps({"edges_count": len(edges)}, ensure_ascii=False))
|
||||
|
||||
note_type = {}
|
||||
for nid, pl in notes.items():
|
||||
t = pl.get("type") or "concept"
|
||||
note_type[str(nid)] = str(t)
|
||||
|
||||
seen = {}
|
||||
counts = {}
|
||||
|
||||
for edge_pid, ed_pl in edges:
|
||||
et = _edge_type(ed_pl)
|
||||
if not et:
|
||||
continue
|
||||
src_nid = _guess_src_note_id(ed_pl)
|
||||
if not src_nid:
|
||||
continue
|
||||
t = note_type.get(src_nid)
|
||||
if not t:
|
||||
continue
|
||||
seen.setdefault(t, set()).add(et)
|
||||
counts.setdefault(t, {}).setdefault(et, 0)
|
||||
counts[t][et] += 1
|
||||
|
||||
expected = {}
|
||||
for tname, tdef in types_def.items():
|
||||
eddefs = (tdef or {}).get("edge_defaults") or []
|
||||
expected[tname] = set([str(x) for x in eddefs if x])
|
||||
|
||||
for tname, exp in expected.items():
|
||||
obs = seen.get(tname, set())
|
||||
missing = sorted(list(exp - obs))
|
||||
extra = sorted(list(obs - exp))
|
||||
print(json.dumps({
|
||||
"type": tname,
|
||||
"expected_defaults": sorted(list(exp)),
|
||||
"observed": sorted(list(obs)),
|
||||
"missing": missing,
|
||||
"extra": extra,
|
||||
"counts": counts.get(tname, {}),
|
||||
}, ensure_ascii=False))
|
||||
|
||||
if not expected:
|
||||
print(json.dumps({"warn": "Keine Typ-Defaults in types.yaml gefunden (edge_defaults leer?)."}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,93 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os, json, argparse, random
|
||||
from typing import Any, Dict, List, Tuple, Optional
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
def _get_prefix(explicit: Optional[str]) -> str:
|
||||
return (explicit or os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet").strip()
|
||||
|
||||
def _names(prefix: str) -> Dict[str,str]:
|
||||
return {
|
||||
"notes": f"{prefix}_notes",
|
||||
"chunks": f"{prefix}_chunks",
|
||||
"edges": f"{prefix}_edges",
|
||||
}
|
||||
|
||||
def _scroll(client: QdrantClient, collection: str, *, limit=5, with_payload=True):
|
||||
# Support both modern and older SDKs (filter vs scroll_filter)
|
||||
try:
|
||||
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
|
||||
except TypeError as e:
|
||||
# older signatures might accept scroll_filter kw, but if we passed none it's fine
|
||||
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--host", default=os.environ.get("QDRANT_HOST", "127.0.0.1"))
|
||||
ap.add_argument("--port", default=int(os.environ.get("QDRANT_PORT", "6333")))
|
||||
ap.add_argument("--prefix", default=None)
|
||||
ap.add_argument("--limit", type=int, default=5)
|
||||
ap.add_argument("--note-id", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
prefix = _get_prefix(args.prefix)
|
||||
names = _names(prefix)
|
||||
client = QdrantClient(host=args.host, port=args.port, prefer_grpc=False)
|
||||
|
||||
# Notes sample
|
||||
try:
|
||||
pts, _ = _scroll(client, names["notes"], limit=args.limit, with_payload=True)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"notes scroll failed: {type(e).__name__}: {e}", "collection": names["notes"]}))
|
||||
return
|
||||
|
||||
sample = []
|
||||
for p in pts:
|
||||
pl = (p.payload or {})
|
||||
sample.append({
|
||||
"point_id": str(p.id),
|
||||
"note_id": pl.get("note_id"),
|
||||
"id": pl.get("id"),
|
||||
"title": pl.get("title"),
|
||||
"type": pl.get("type"),
|
||||
"retriever_weight": pl.get("retriever_weight"),
|
||||
"chunk_profile": pl.get("chunk_profile"),
|
||||
})
|
||||
print(json.dumps({"notes_sample": sample}, ensure_ascii=False))
|
||||
|
||||
# Pick a note to fetch chunks
|
||||
target_note_id = args.note_id or None
|
||||
if not target_note_id:
|
||||
for s in sample:
|
||||
if s.get("note_id"):
|
||||
target_note_id = s["note_id"]
|
||||
break
|
||||
|
||||
if not target_note_id:
|
||||
print(json.dumps({"warn": "no note_id available to fetch chunks"}))
|
||||
return
|
||||
|
||||
# Chunks for target note
|
||||
try:
|
||||
# pull a few chunks and filter client-side by note_id
|
||||
pts, _ = _scroll(client, names["chunks"], limit=50, with_payload=True)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"chunks scroll failed: {type(e).__name__}: {e}", "collection": names["chunks"]}))
|
||||
return
|
||||
|
||||
chunks = [{
|
||||
"id": str(p.id),
|
||||
"chunk_id": (p.payload or {}).get("chunk_id"),
|
||||
"note_id": (p.payload or {}).get("note_id"),
|
||||
"type": (p.payload or {}).get("type"),
|
||||
"retriever_weight": (p.payload or {}).get("retriever_weight"),
|
||||
"window_len": len((p.payload or {}).get("window") or ""),
|
||||
"text_len": len((p.payload or {}).get("text") or ""),
|
||||
} for p in pts if (p.payload or {}).get("note_id") == target_note_id][:5]
|
||||
|
||||
print(json.dumps({"target_note_id": target_note_id, "chunk_samples": chunks}, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/diag_payload_indexes.py (v1.1)
|
||||
|
||||
Zeigt payload_schema (Indizes) je Collection.
|
||||
WICHTIG: Einige Qdrant-Versionen liefern payload_schema nur, wenn
|
||||
`with_payload_schema=true` gesetzt wird. Daher setzen wir das Flag explizit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, json
|
||||
from qdrant_client.http import models as rest
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
def compact_schema(ps: dict | None) -> dict:
|
||||
if not isinstance(ps, dict):
|
||||
return {}
|
||||
out = {}
|
||||
for k, v in ps.items():
|
||||
if isinstance(v, dict) and "type" in v:
|
||||
out[k] = v["type"]
|
||||
else:
|
||||
out[k] = v
|
||||
return out
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--raw", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
|
||||
cols = [notes, chunks, edges]
|
||||
result = []
|
||||
for name in cols:
|
||||
info = client.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = info.model_dump() if hasattr(info, "model_dump") else (info.dict() if hasattr(info, "dict") else info)
|
||||
payload_schema = (d.get("result") or {}).get("payload_schema")
|
||||
vectors = (d.get("result") or {}).get("vectors")
|
||||
if isinstance(vectors, dict) and "config" in vectors:
|
||||
vectors = vectors.get("config")
|
||||
if args.raw:
|
||||
result.append({"collection": name, "raw": d})
|
||||
else:
|
||||
result.append({
|
||||
"name": name,
|
||||
"vectors": vectors,
|
||||
"payload_schema": compact_schema(payload_schema),
|
||||
"segments_count": (d.get("result") or {}).get("segments_count"),
|
||||
})
|
||||
print(json.dumps({"prefix": cfg.prefix, "collections": result}, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
dump_edges_payload_sample.py
|
||||
-----------------------------------
|
||||
Zeigt einige Beispiel-Payloads aus der {prefix}_edges Collection,
|
||||
um Feldnamen (edge_type, src_note_id, src_id, etc.) zu verifizieren.
|
||||
|
||||
Nutzung:
|
||||
python3 tests/dump_edges_payload_sample.py --prefix mindnet --n 10
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from qdrant_client import QdrantClient
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"qdrant-client not installed: {e}"}))
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet")
|
||||
ap.add_argument("--n", type=int, default=10)
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
|
||||
api_key = os.environ.get("QDRANT_API_KEY") or None
|
||||
edges = f"{args.prefix}_edges"
|
||||
|
||||
client = QdrantClient(url=url, api_key=api_key, timeout=30)
|
||||
|
||||
left = args.n
|
||||
offset = None
|
||||
while left > 0:
|
||||
limit = min(left, 256)
|
||||
pts, offset = client.scroll(
|
||||
collection_name=edges,
|
||||
scroll_filter=None,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
if not pts:
|
||||
break
|
||||
for p in pts:
|
||||
print(json.dumps({"point_id": str(p.id), "payload_keys": list((p.payload or {}).keys()), "payload": p.payload}, ensure_ascii=False))
|
||||
left -= len(pts)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tests/ensure_indexes_and_show.py (compat v1.0.1)
|
||||
|
||||
Erzwingt ensure_payload_indexes(...) und zeigt danach payload_schema je Collection.
|
||||
Kompatibel mit qdrant-client Versionen ohne 'with_payload_schema' im Wrapper.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_payload_indexes, collection_names
|
||||
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
return obj if isinstance(obj, dict) else {}
|
||||
|
||||
|
||||
def _cfg_base_url(cfg) -> str:
|
||||
if getattr(cfg, "url", None):
|
||||
return cfg.url.rstrip("/")
|
||||
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
|
||||
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
if api_key:
|
||||
req.add_header("api-key", api_key)
|
||||
req.add_header("Authorization", f"Bearer {api_key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
|
||||
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||
# Strategy 1: OpenAPI client with flag
|
||||
try:
|
||||
oc = getattr(client, "openapi_client", None)
|
||||
if oc is not None and hasattr(oc, "collections_api"):
|
||||
api = oc.collections_api
|
||||
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = _safe_model_dump(info)
|
||||
return (d.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
pass
|
||||
# Strategy 2: wrapper
|
||||
try:
|
||||
info = client.get_collection(collection_name=name)
|
||||
d = _safe_model_dump(info)
|
||||
ps = (d.get("result") or {}).get("payload_schema")
|
||||
if ps is not None:
|
||||
return ps
|
||||
except Exception:
|
||||
pass
|
||||
# Strategy 3: direct HTTP
|
||||
try:
|
||||
base = _cfg_base_url(cfg)
|
||||
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||
return (raw.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
res = {}
|
||||
for name in (notes, chunks, edges):
|
||||
res[name] = get_collection_payload_schema(client, cfg, name)
|
||||
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/show_edges_for_note.py
|
||||
|
||||
Zeigt Kanten einer Note (gefiltert nach kind/scope) in einer kompakten Form.
|
||||
Aufrufbeispiele:
|
||||
python3 -m scripts.show_edges_for_note --note-id 20251110-ollama-llm-9f0a12 --kinds references,next,prev --limit 10
|
||||
python3 -m scripts.show_edges_for_note --title "Qdrant Vektordatenbank" --scope note
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse, json, os, sys
|
||||
from typing import Dict, Any, List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def find_note_by_title(client, prefix: str, title: str) -> str | None:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="title", match=rest.MatchText(text=title))])
|
||||
pts, _ = client.scroll(collection_name=notes_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=1)
|
||||
if not pts:
|
||||
return None
|
||||
return pts[0].payload.get("note_id")
|
||||
|
||||
def fetch_edges_for_note(client, prefix: str, note_id: str, kinds: List[str] | None, scope: str | None, limit: int) -> List[Dict[str, Any]]:
|
||||
_, _, edges_col = collections(prefix)
|
||||
must = [rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
|
||||
if scope:
|
||||
must.append(rest.FieldCondition(key="scope", match=rest.MatchValue(value=scope)))
|
||||
if kinds:
|
||||
must.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
|
||||
f = rest.Filter(must=must)
|
||||
pts, _ = client.scroll(collection_name=edges_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=limit)
|
||||
return [p.payload for p in pts]
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--note-id")
|
||||
ap.add_argument("--title")
|
||||
ap.add_argument("--kinds", help="CSV: references,next,prev,belongs_to,backlink")
|
||||
ap.add_argument("--scope", choices=["note","chunk"])
|
||||
ap.add_argument("--limit", type=int, default=25)
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
|
||||
nid = args.note_id
|
||||
if not nid and args.title:
|
||||
nid = find_note_by_title(client, cfg.prefix, args.title)
|
||||
if not nid:
|
||||
print(json.dumps({"error": f"note with title '{args.title}' not found"}))
|
||||
sys.exit(2)
|
||||
if not nid:
|
||||
print(json.dumps({"error": "please provide --note-id or --title"}))
|
||||
sys.exit(2)
|
||||
|
||||
kinds = None
|
||||
if args.kinds:
|
||||
kinds = [s.strip() for s in args.kinds.split(",") if s.strip()]
|
||||
|
||||
edges = fetch_edges_for_note(client, cfg.prefix, nid, kinds, args.scope, args.limit)
|
||||
print(json.dumps({"note_id": nid, "count": len(edges), "edges": edges}, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/test_edges_smoke.py — Progressive Ausgabe
|
||||
|
||||
Prüft edge-Integrität je Note mit Live-Ausgabe:
|
||||
- belongs_to == #Chunks
|
||||
- next == prev == max(#Chunks-1,0)
|
||||
- Duplikat-Edges (Key: (kind,source_id,target_id,scope)) == 0
|
||||
- Zählt references (chunk/note), backlink
|
||||
|
||||
Optionen:
|
||||
--max-notes N : prüft nur die ersten N Notizen
|
||||
--limit L : Scroll-Limit pro Anfrage (Default 256)
|
||||
--flush : jede Zeile sofort flushen
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import os, sys, json, argparse
|
||||
from typing import Dict, Any, List, Tuple, Set
|
||||
from qdrant_client.http import models as rest
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def scroll_iter(client, collection: str, filt: rest.Filter | None, with_payload: bool, limit: int):
|
||||
next_page = None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=filt,
|
||||
with_payload=with_payload,
|
||||
with_vectors=False,
|
||||
limit=limit,
|
||||
offset=next_page,
|
||||
)
|
||||
if not pts:
|
||||
break
|
||||
for p in pts:
|
||||
yield p
|
||||
if next_page is None:
|
||||
break
|
||||
|
||||
def list_notes(client, prefix: str, limit: int, max_notes: int | None) -> List[Dict[str, Any]]:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
out: List[Dict[str, Any]] = []
|
||||
for p in scroll_iter(client, notes_col, None, True, limit):
|
||||
pl = p.payload or {}
|
||||
nid = pl.get("note_id") or pl.get("id")
|
||||
if nid:
|
||||
out.append({"note_id": nid, "title": pl.get("title"), "type": pl.get("type")})
|
||||
if max_notes is not None and len(out) >= max_notes:
|
||||
break
|
||||
return out
|
||||
|
||||
def count_chunks_for_note(client, prefix: str, note_id: str, limit: int) -> int:
|
||||
_, chunks_col, _ = collections(prefix)
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
return sum(1 for _ in scroll_iter(client, chunks_col, f, False, limit))
|
||||
|
||||
def fetch_edges_for_note(client, prefix: str, note_id: str, limit: int) -> List[Dict[str, Any]]:
|
||||
_, _, edges_col = collections(prefix)
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
return [p.payload or {} for p in scroll_iter(client, edges_col, f, True, limit)]
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--max-notes", type=int)
|
||||
ap.add_argument("--limit", type=int, default=256)
|
||||
ap.add_argument("--flush", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
|
||||
notes = list_notes(client, cfg.prefix, args.limit, args.max_notes)
|
||||
total = {"notes": 0, "chunks": 0, "belongs_to": 0, "next": 0, "prev": 0, "refs_chunk": 0, "refs_note": 0, "backlink": 0, "dup_edges": 0}
|
||||
|
||||
for n in notes:
|
||||
nid = n["note_id"]
|
||||
total["notes"] += 1
|
||||
chunk_count = count_chunks_for_note(client, cfg.prefix, nid, args.limit)
|
||||
total["chunks"] += chunk_count
|
||||
|
||||
edges = fetch_edges_for_note(client, cfg.prefix, nid, args.limit)
|
||||
by_kind = {}
|
||||
keys: Set[tuple] = set()
|
||||
dup_count = 0
|
||||
refs_chunk = 0
|
||||
refs_note = 0
|
||||
backlink = 0
|
||||
|
||||
for e in edges:
|
||||
k = e.get("kind")
|
||||
by_kind[k] = by_kind.get(k, 0) + 1
|
||||
t = (e.get("kind"), e.get("source_id"), e.get("target_id"), e.get("scope"))
|
||||
if t in keys:
|
||||
dup_count += 1
|
||||
else:
|
||||
keys.add(t)
|
||||
if k == "references" and e.get("scope") == "chunk":
|
||||
refs_chunk += 1
|
||||
if k == "references" and e.get("scope") == "note":
|
||||
refs_note += 1
|
||||
if k == "backlink":
|
||||
backlink += 1
|
||||
|
||||
total["belongs_to"] += by_kind.get("belongs_to", 0)
|
||||
total["next"] += by_kind.get("next", 0)
|
||||
total["prev"] += by_kind.get("prev", 0)
|
||||
total["refs_chunk"] += refs_chunk
|
||||
total["refs_note"] += refs_note
|
||||
total["backlink"] += backlink
|
||||
total["dup_edges"] += dup_count
|
||||
|
||||
ok_bt = (by_kind.get("belongs_to", 0) == chunk_count)
|
||||
ok_seq = (by_kind.get("next", 0) == max(chunk_count - 1, 0) and by_kind.get("prev", 0) == max(chunk_count - 1, 0))
|
||||
ok_dup = (dup_count == 0)
|
||||
|
||||
line = {
|
||||
"note_id": nid,
|
||||
"title": n.get("title"),
|
||||
"type": n.get("type"),
|
||||
"chunks": chunk_count,
|
||||
"edges_by_kind": by_kind,
|
||||
"checks": {
|
||||
"belongs_to_equals_chunks": ok_bt,
|
||||
"next_prev_match": ok_seq,
|
||||
"no_duplicate_edges": ok_dup,
|
||||
}
|
||||
}
|
||||
print(json.dumps(line, ensure_ascii=False))
|
||||
if args.flush:
|
||||
sys.stdout.flush()
|
||||
|
||||
print(json.dumps({"prefix": cfg.prefix, "summary": total}, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/test_edges_smoke_fast.py — Zählung über /count (sehr schnell)
|
||||
|
||||
Verwendet Qdrant 'count' API je Note/Kind (anstatt scroll), dadurch sehr schnelle Ausführung.
|
||||
Optionen:
|
||||
--max-notes N : prüft nur die ersten N Notizen
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse, json
|
||||
from typing import Dict, Any, List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
|
||||
KINDS = ["belongs_to", "next", "prev", "references", "backlink"]
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def list_note_ids(client, notes_col: str, max_notes: int | None) -> List[Dict[str, Any]]:
|
||||
pts, _ = client.scroll(collection_name=notes_col, with_payload=True, with_vectors=False, limit=max_notes or 1024)
|
||||
out = []
|
||||
for p in pts or []:
|
||||
pl = p.payload or {}
|
||||
nid = pl.get("note_id") or pl.get("id")
|
||||
if nid:
|
||||
out.append({"note_id": nid, "title": pl.get("title"), "type": pl.get("type")})
|
||||
return out
|
||||
|
||||
def count_points(client, col: str, filt: rest.Filter) -> int:
|
||||
res = client.count(collection_name=col, count_filter=filt, exact=True)
|
||||
return int(getattr(res, "count", 0))
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--max-notes", type=int)
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes_col, chunks_col, edges_col = collections(cfg.prefix)
|
||||
notes = list_note_ids(client, notes_col, args.max_notes)
|
||||
|
||||
summary = {"notes": 0, "chunks": 0, "edges": 0}
|
||||
for n in notes:
|
||||
nid = n["note_id"]
|
||||
summary["notes"] += 1
|
||||
|
||||
filt_note = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||
chunk_cnt = count_points(client, chunks_col, filt_note)
|
||||
summary["chunks"] += chunk_cnt
|
||||
|
||||
# counts per kind (edges)
|
||||
by_kind: Dict[str, int] = {}
|
||||
for k in KINDS:
|
||||
f = rest.Filter(must=[
|
||||
rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid)),
|
||||
rest.FieldCondition(key="kind", match=rest.MatchValue(value=k)),
|
||||
])
|
||||
c = count_points(client, edges_col, f)
|
||||
if c:
|
||||
by_kind[k] = c
|
||||
|
||||
summary["edges"] += sum(by_kind.values())
|
||||
line = {"note_id": nid, "title": n.get("title"), "type": n.get("type"), "chunks": chunk_cnt, "edges_by_kind": by_kind}
|
||||
print(json.dumps(line, ensure_ascii=False))
|
||||
|
||||
print(json.dumps({"prefix": cfg.prefix, "summary": summary}, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os, tempfile, textwrap
|
||||
from app.core.type_registry import load_type_registry, resolve_note_type, get_type_config, effective_chunk_profile, profile_overlap
|
||||
|
||||
def test_defaults_when_missing():
|
||||
# ensure default kicks in if file does not exist
|
||||
reg = load_type_registry(os.path.join("config", "does_not_exist.yaml"))
|
||||
assert reg.get("_using_defaults", False) is True
|
||||
t = resolve_note_type(None, reg)
|
||||
assert t == "concept"
|
||||
cfg = get_type_config(t, reg)
|
||||
assert cfg.get("chunk_profile") in ("medium", "long", "short")
|
||||
|
||||
def test_custom_file():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = os.path.join(td, "types.yaml")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(textwrap.dedent('''
|
||||
version: 1.0
|
||||
types:
|
||||
concept:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["references"]
|
||||
retriever_weight: 1.0
|
||||
task:
|
||||
chunk_profile: short
|
||||
edge_defaults: ["depends_on"]
|
||||
retriever_weight: 0.8
|
||||
'''))
|
||||
reg = load_type_registry(p)
|
||||
assert resolve_note_type("task", reg) == "task"
|
||||
assert effective_chunk_profile("task", reg) == "short"
|
||||
assert profile_overlap("short")[0] < profile_overlap("long")[0]
|
||||
Reference in new issue
Block a user