Compare commits
117
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b29ce4a0a6 | ||
|
|
311e598b69 | ||
|
|
413dca770c | ||
|
|
9d9239b11e | ||
|
|
a571c41c61 | ||
|
|
ba1fab9dcf | ||
|
|
5dc9c13a71 | ||
|
|
70821bae5a | ||
|
|
486abf82dd | ||
|
|
19afe3af33 | ||
|
|
113042b7f4 | ||
|
|
f8eebd91cb | ||
|
|
00297fc354 | ||
|
|
5baac082f9 | ||
|
|
611b0aa54e | ||
|
|
110f0dbcf5 | ||
|
|
1688cf0a1e | ||
|
|
13b7c8858a | ||
|
|
b8b6f243d1 | ||
|
|
eaa6e805ce | ||
|
|
05dbb21c58 | ||
|
|
f1f4358dee | ||
|
|
3374c41f07 | ||
|
|
95affe87e5 | ||
|
|
5e86398fcd | ||
|
|
844f0cbe42 | ||
|
|
4d5d7847e5 | ||
|
|
06795feed6 | ||
|
|
a4548a7ee1 | ||
|
|
42216865e2 | ||
|
|
e48bdb2401 | ||
|
|
c67eb4b2f3 | ||
|
|
fafa74b0b8 | ||
|
|
049a39aba3 | ||
|
|
cd26430a61 | ||
|
|
aab010ff17 | ||
|
|
9a7bc32051 | ||
|
|
bfdd8ac6be | ||
|
|
018669fb87 | ||
|
|
fb2de78d62 | ||
|
|
7e8d6c1954 | ||
|
|
d21230c6c3 | ||
|
|
4e98d5bfa1 | ||
|
|
8998269377 | ||
|
|
6e9a227ad2 | ||
|
|
4e7506da5a | ||
|
|
06ad93c77c | ||
|
|
0bb7b34259 | ||
|
|
859c17b49d | ||
|
|
46b26c9624 | ||
|
|
fd215c18e4 | ||
|
|
efa3afc7dc | ||
|
|
e03dd66051 | ||
|
|
3476fe5fae | ||
|
|
3e08c8347e | ||
|
|
c691123d2d | ||
|
|
a4b81535e8 | ||
|
|
a898cf4b65 | ||
|
|
10009a0a43 | ||
|
|
4dfe47ff99 | ||
|
|
bd997e61d6 | ||
|
|
23d0670126 | ||
|
|
12c600edbe | ||
|
|
e4115e705e | ||
|
|
0cbc4a2d72 | ||
|
|
cea6d35729 | ||
|
|
4c56918d8a | ||
|
|
f17263417e | ||
|
|
6f0b463489 | ||
|
|
28c01caa1a | ||
|
|
95b59e9b0a | ||
|
|
bc967f1f6e | ||
|
|
4228b8a74f | ||
|
|
300086fc83 | ||
|
|
688eb262a9 | ||
|
|
bfb6f5dede | ||
|
|
88c0fbe56b | ||
|
|
3d74eff224 | ||
|
|
22d08afe2d | ||
|
|
a97f757e34 | ||
|
|
bbc8f13944 | ||
|
|
f18a40d76c | ||
|
|
c7644a36aa | ||
|
|
f394f31c12 | ||
|
|
0513673c2a | ||
|
|
a73542a391 | ||
|
|
c465797654 | ||
|
|
7ce3f982f2 | ||
|
|
6311456917 | ||
|
|
3c06559c5d | ||
|
|
5c9f2e8c83 | ||
|
|
62902faaca | ||
|
|
d22fd77b22 | ||
|
|
2f86f493d6 | ||
|
|
eafa0fad85 | ||
|
|
2e7c497b69 | ||
|
|
79dcfdcf9a | ||
|
|
487e28a904 | ||
|
|
c492f97a67 | ||
|
|
ec2ff74514 | ||
|
|
c501f8d6e6 | ||
|
|
b4287cbfda | ||
|
|
26724a1db1 | ||
|
|
d6368ee709 | ||
|
|
ea211a5c0b | ||
|
|
9e8b433c95 | ||
|
|
2a1c62aeed | ||
|
|
948d6f4b47 | ||
|
|
725271a7da | ||
|
|
4ea62e6886 | ||
|
|
c01cf3b078 | ||
|
|
c67777a6b9 | ||
|
|
e5e5e7560c | ||
|
|
6e1cd5fbbc | ||
|
|
8bd8f7c0bb | ||
|
|
1c42cd1f78 | ||
|
|
be090a43f2 |
No files matched your search
+111
-162
@@ -1,180 +1,129 @@
|
||||
# chunk_payload.py
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
app/core/chunk_payload.py (Mindnet V2 — types.yaml authoritative)
|
||||
- neighbors_prev / neighbors_next sind Listen ([], [id]).
|
||||
- retriever_weight / chunk_profile kommen aus types.yaml (Frontmatter wird ignoriert).
|
||||
- Fallbacks: defaults.* in types.yaml; sonst 1.0 / "default".
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
import os, json, pathlib, re, yaml, hashlib
|
||||
from typing import Any, Dict, List, Optional
|
||||
import os, yaml
|
||||
|
||||
FRONTMATTER_RE = re.compile(r"^---\s*\n.*?\n---\s*\n?", re.DOTALL)
|
||||
def _env(n: str, d: Optional[str]=None) -> str:
|
||||
v = os.getenv(n)
|
||||
return v if v is not None else (d or "")
|
||||
|
||||
def _as_dict(note: Any) -> Dict[str, Any]:
|
||||
if isinstance(note, dict):
|
||||
return note
|
||||
d: Dict[str, Any] = {}
|
||||
for attr in ("id","note_id","title","path","frontmatter","meta","metadata","body","text","content","raw","markdown","type","chunks"):
|
||||
if hasattr(note, attr):
|
||||
d[attr] = getattr(note, attr)
|
||||
fm = d.get("frontmatter") or d.get("meta") or d.get("metadata") or {}
|
||||
d["frontmatter"] = fm if isinstance(fm, dict) else {}
|
||||
return d
|
||||
def _load_types() -> dict:
|
||||
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[Dict[str,Any]]]:
|
||||
path = kwargs.get("path")
|
||||
types_cfg = kwargs.get("types_config")
|
||||
for a in args:
|
||||
if path is None and isinstance(a, (str, pathlib.Path)):
|
||||
path = str(a)
|
||||
if types_cfg is None and isinstance(a, dict):
|
||||
types_cfg = a
|
||||
return path, types_cfg
|
||||
def _get_types_map(reg: dict) -> dict:
|
||||
if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
|
||||
return reg["types"]
|
||||
return reg if isinstance(reg, dict) else {}
|
||||
|
||||
def _load_types_config(explicit: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
if isinstance(explicit, dict):
|
||||
return explicit
|
||||
for rel in ("config/config.yaml", "config/types.yaml"):
|
||||
p = pathlib.Path(rel)
|
||||
if p.exists():
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
|
||||
return data["types"]
|
||||
return data if isinstance(data, dict) else {}
|
||||
def _get_defaults(reg: dict) -> dict:
|
||||
if isinstance(reg, dict) and isinstance(reg.get("defaults"), dict):
|
||||
return reg["defaults"]
|
||||
if isinstance(reg, dict) and isinstance(reg.get("global"), dict):
|
||||
return reg["global"]
|
||||
return {}
|
||||
|
||||
def _coalesce(*vals):
|
||||
for v in vals:
|
||||
def _as_float(x: Any):
|
||||
try:
|
||||
return float(x)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _resolve_chunk_profile(note_type: str, reg: dict) -> str:
|
||||
types = _get_types_map(reg)
|
||||
if isinstance(types, dict):
|
||||
t = types.get(note_type, {})
|
||||
if isinstance(t, dict) and isinstance(t.get("chunk_profile"), str):
|
||||
return t["chunk_profile"]
|
||||
defs = _get_defaults(reg)
|
||||
if isinstance(defs, dict) and isinstance(defs.get("chunk_profile"), str):
|
||||
return defs["chunk_profile"]
|
||||
return "default"
|
||||
|
||||
def _resolve_retriever_weight(note_type: str, reg: dict) -> float:
|
||||
types = _get_types_map(reg)
|
||||
if isinstance(types, dict):
|
||||
t = types.get(note_type, {})
|
||||
if isinstance(t, dict) and (t.get("retriever_weight") is not None):
|
||||
v = _as_float(t.get("retriever_weight"))
|
||||
if v is not None:
|
||||
return float(v)
|
||||
defs = _get_defaults(reg)
|
||||
if isinstance(defs, dict) and (defs.get("retriever_weight") is not None):
|
||||
v = _as_float(defs.get("retriever_weight"))
|
||||
if v is not None:
|
||||
return v
|
||||
return None
|
||||
return float(v)
|
||||
return 1.0
|
||||
|
||||
def _text_from_note(n: Dict[str, Any], path_hint: Optional[str]) -> str:
|
||||
# häufige Felder
|
||||
cand = [
|
||||
n.get("body"),
|
||||
n.get("text"),
|
||||
n.get("markdown"),
|
||||
n.get("raw"),
|
||||
]
|
||||
content = n.get("content")
|
||||
if isinstance(content, str):
|
||||
cand.append(content)
|
||||
elif isinstance(content, dict):
|
||||
for k in ("text","body","raw","markdown","content"):
|
||||
v = content.get(k)
|
||||
if isinstance(v, str):
|
||||
cand.append(v)
|
||||
for t in cand:
|
||||
if isinstance(t, str) and t.strip():
|
||||
return t
|
||||
|
||||
# Fallback: Datei lesen und Frontmatter entfernen
|
||||
p = n.get("path") or path_hint
|
||||
if p:
|
||||
try:
|
||||
pth = pathlib.Path(p)
|
||||
if pth.exists():
|
||||
txt = pth.read_text(encoding="utf-8", errors="ignore")
|
||||
if txt:
|
||||
return FRONTMATTER_RE.sub("", txt).strip()
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
def _iter_chunks(n: Dict[str, Any], profile: str, fulltext: str) -> List[Dict[str, Any]]:
|
||||
# 1) vorhandene Chunks nehmen, wenn sinnvoll
|
||||
existing = n.get("chunks")
|
||||
out: List[Dict[str, Any]] = []
|
||||
if isinstance(existing, list) and existing:
|
||||
for i, c in enumerate(existing):
|
||||
text = ""
|
||||
if isinstance(c, dict):
|
||||
text = c.get("text") or c.get("body") or c.get("raw") or ""
|
||||
elif isinstance(c, str):
|
||||
text = c
|
||||
if text and text.strip():
|
||||
out.append({"index": i, "text": text})
|
||||
if out:
|
||||
return out
|
||||
|
||||
# 2) Fallback: profilabhängige Bündelung
|
||||
if not isinstance(profile, str):
|
||||
profile = "medium"
|
||||
size = {"short": 600, "medium": 1200, "long": 2400}.get(profile, 1200)
|
||||
if not fulltext:
|
||||
def _as_list(x):
|
||||
if x is None:
|
||||
return []
|
||||
paras = re.split(r"\n{2,}", fulltext)
|
||||
buf = ""
|
||||
chunks: List[str] = []
|
||||
for p in paras:
|
||||
p = (p or "").strip()
|
||||
if not p:
|
||||
continue
|
||||
if len(buf) + (2 if buf else 0) + len(p) <= size:
|
||||
buf = (buf + "\n\n" + p).strip() if buf else p
|
||||
else:
|
||||
if buf: chunks.append(buf)
|
||||
if len(p) <= size:
|
||||
buf = p
|
||||
else:
|
||||
for i in range(0, len(p), size):
|
||||
chunks.append(p[i:i+size])
|
||||
buf = ""
|
||||
if buf: chunks.append(buf)
|
||||
return [{"index": i, "text": c} for i, c in enumerate(chunks)]
|
||||
if isinstance(x, list):
|
||||
return x
|
||||
return [x]
|
||||
|
||||
def make_chunk_payloads(note: Any, *args, **kwargs) -> List[Dict[str, Any]]:
|
||||
n = _as_dict(note)
|
||||
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
|
||||
types_cfg = _load_types_config(types_cfg_explicit)
|
||||
def make_chunk_payloads(note: Dict[str, Any],
|
||||
note_path: str,
|
||||
chunks_from_chunker: List[Any],
|
||||
*,
|
||||
note_text: str = "",
|
||||
types_cfg: Optional[dict] = None,
|
||||
file_path: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
fm = (note or {}).get("frontmatter", {}) or {}
|
||||
note_type = fm.get("type") or note.get("type") or "concept"
|
||||
reg = types_cfg if isinstance(types_cfg, dict) else _load_types()
|
||||
|
||||
fm = n.get("frontmatter") or {}
|
||||
note_type = str(fm.get("type") or n.get("type") or "note")
|
||||
cfg_for_type = types_cfg.get(note_type, {}) if isinstance(types_cfg, dict) else {}
|
||||
# types.yaml authoritative
|
||||
cp = _resolve_chunk_profile(note_type, reg)
|
||||
rw = _resolve_retriever_weight(note_type, reg)
|
||||
|
||||
try:
|
||||
default_rw = float(os.environ.get("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0))
|
||||
except Exception:
|
||||
default_rw = 1.0
|
||||
tags = fm.get("tags") or []
|
||||
if isinstance(tags, str):
|
||||
tags = [tags]
|
||||
|
||||
retriever_weight = _coalesce(fm.get("retriever_weight"), cfg_for_type.get("retriever_weight"), default_rw)
|
||||
try:
|
||||
retriever_weight = float(retriever_weight)
|
||||
except Exception:
|
||||
retriever_weight = default_rw
|
||||
out: List[Dict[str, Any]] = []
|
||||
for idx, ch in enumerate(chunks_from_chunker):
|
||||
# Attribute oder Keys (Chunk-Objekt oder Dict)
|
||||
cid = getattr(ch, "id", None) or (ch.get("id") if isinstance(ch, dict) else None)
|
||||
nid = getattr(ch, "note_id", None) or (ch.get("note_id") if isinstance(ch, dict) else fm.get("id"))
|
||||
index = getattr(ch, "index", None) or (ch.get("index") if isinstance(ch, dict) else idx)
|
||||
text = getattr(ch, "text", None) or (ch.get("text") if isinstance(ch, dict) else "")
|
||||
window = getattr(ch, "window", None) or (ch.get("window") if isinstance(ch, dict) else text)
|
||||
prev_id = getattr(ch, "neighbors_prev", None) or (ch.get("neighbors_prev") if isinstance(ch, dict) else None)
|
||||
next_id = getattr(ch, "neighbors_next", None) or (ch.get("neighbors_next") if isinstance(ch, dict) else None)
|
||||
|
||||
chunk_profile = _coalesce(fm.get("chunk_profile"), cfg_for_type.get("chunk_profile"), os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE","medium"))
|
||||
chunk_profile = chunk_profile if isinstance(chunk_profile, str) else "medium"
|
||||
|
||||
note_id = n.get("note_id") or n.get("id") or fm.get("id")
|
||||
title = n.get("title") or fm.get("title") or ""
|
||||
path = n.get("path") or path_arg
|
||||
if isinstance(path, pathlib.Path):
|
||||
path = str(path)
|
||||
path = path or "" # immer vorhanden
|
||||
|
||||
fulltext = _text_from_note(n, path)
|
||||
chunks = _iter_chunks(n, chunk_profile, fulltext)
|
||||
|
||||
payloads: List[Dict[str, Any]] = []
|
||||
for c in chunks:
|
||||
idx = c.get("index", len(payloads))
|
||||
text = c.get("text") if isinstance(c, dict) else (str(c) if c is not None else "")
|
||||
text = text if isinstance(text, str) else str(text or "")
|
||||
|
||||
key = f"{note_id}|{idx}"
|
||||
h = hashlib.sha1(key.encode("utf-8")).hexdigest()[:12] if note_id else hashlib.sha1(f"{path}|{idx}".encode("utf-8")).hexdigest()[:12]
|
||||
chunk_id = f"{note_id}-{idx:03d}-{h}" if note_id else f"{h}"
|
||||
|
||||
payload = {
|
||||
"note_id": note_id,
|
||||
"chunk_id": chunk_id,
|
||||
"index": idx,
|
||||
"title": title,
|
||||
pl: Dict[str, Any] = {
|
||||
"note_id": nid,
|
||||
"chunk_id": cid,
|
||||
"index": int(index),
|
||||
"ord": int(index) + 1,
|
||||
"type": note_type,
|
||||
"path": path, # <- garantiert vorhanden
|
||||
"text": text, # <- nie leer, sonst werden keine Chunks erzeugt
|
||||
"retriever_weight": retriever_weight,
|
||||
"chunk_profile": chunk_profile,
|
||||
"tags": tags,
|
||||
"text": text,
|
||||
"window": window,
|
||||
"neighbors_prev": _as_list(prev_id),
|
||||
"neighbors_next": _as_list(next_id),
|
||||
"section": getattr(ch, "section", None) or (ch.get("section") if isinstance(ch, dict) else ""),
|
||||
"path": note_path,
|
||||
"source_path": file_path or note_path,
|
||||
"retriever_weight": float(rw),
|
||||
"chunk_profile": cp,
|
||||
}
|
||||
json.loads(json.dumps(payload, ensure_ascii=False))
|
||||
payloads.append(payload)
|
||||
# Aufräumen von Alt-Feldern
|
||||
for alias in ("chunk_num", "Chunk_Number"):
|
||||
pl.pop(alias, None)
|
||||
|
||||
return payloads
|
||||
out.append(pl)
|
||||
|
||||
return out
|
||||
+354
-50
@@ -2,43 +2,45 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/derive_edges.py
|
||||
Version: 1.4.0
|
||||
Datum: 2025-10-01
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Robuste Kantenbildung für mindnet (Notes/Chunks):
|
||||
- belongs_to (chunk -> note)
|
||||
- next / prev (chunk-Kette)
|
||||
- references (chunk-scope) aus Chunk.window/text
|
||||
- optional references/backlink (note-scope)
|
||||
|
||||
Wichtig: Wikilinks werden mit der Parser-Funktion `extract_wikilinks` extrahiert,
|
||||
damit Varianten wie [[id#anchor]] oder [[id|label]] korrekt auf 'id' reduziert werden.
|
||||
|
||||
Erwartete Chunk-Payload-Felder:
|
||||
{
|
||||
"note_id": "...",
|
||||
"chunk_id": "...", # Alias "id" ist zulässig
|
||||
"id": "...",
|
||||
"chunk_index": int,
|
||||
"seq": int,
|
||||
"window": str,
|
||||
"text": str,
|
||||
"path": "rel/path.md",
|
||||
...
|
||||
}
|
||||
Zweck:
|
||||
- Bewahrt bestehende Edgelogik (belongs_to, prev/next, references, backlink)
|
||||
- Ergänzt typenbasierte Default-Kanten (edge_defaults aus config/types.yaml)
|
||||
- Unterstützt "typed inline relations":
|
||||
* [[rel:KIND | Target]]
|
||||
* [[rel:KIND Target]]
|
||||
* rel: KIND [[Target]]
|
||||
- Unterstützt Obsidian-Callouts:
|
||||
* > [!edge] KIND: [[Target]] [[Target2]] ...
|
||||
Kompatibilität:
|
||||
- build_edges_for_note(...) Signatur unverändert
|
||||
- rule_id Werte:
|
||||
* structure:belongs_to
|
||||
* structure:order
|
||||
* explicit:wikilink
|
||||
* inline:rel
|
||||
* callout:edge
|
||||
* edge_defaults:<type>:<relation>
|
||||
* derived:backlink
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Optional, Iterable
|
||||
import os
|
||||
import re
|
||||
from typing import Iterable, List, Optional, Tuple, Set, Dict
|
||||
|
||||
# WICHTIG: benutze die Parser-Extraktion für saubere Wikilinks
|
||||
from app.core.parser import extract_wikilinks
|
||||
try:
|
||||
import yaml # optional, nur für types.yaml
|
||||
except Exception: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Utilities
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _get(d: dict, *keys, default=None):
|
||||
for k in keys:
|
||||
if k in d and d[k] is not None:
|
||||
if isinstance(d, dict) and k in d and d[k] is not None:
|
||||
return d[k]
|
||||
return default
|
||||
|
||||
@@ -52,8 +54,8 @@ def _chunk_text_for_refs(chunk: dict) -> str:
|
||||
or ""
|
||||
)
|
||||
|
||||
def _dedupe(seq: Iterable[str]) -> List[str]:
|
||||
seen = set()
|
||||
def _dedupe_seq(seq: Iterable[str]) -> List[str]:
|
||||
seen: Set[str] = set()
|
||||
out: List[str] = []
|
||||
for s in seq:
|
||||
if s not in seen:
|
||||
@@ -64,15 +66,177 @@ def _dedupe(seq: Iterable[str]) -> List[str]:
|
||||
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
|
||||
pl = {
|
||||
"kind": kind,
|
||||
"scope": scope, # "chunk" | "note"
|
||||
"relation": kind, # Alias (v2)
|
||||
"scope": scope, # "chunk" | "note"
|
||||
"source_id": source_id,
|
||||
"target_id": target_id,
|
||||
"note_id": note_id, # Träger/Quelle der Kante (aktuelle Note)
|
||||
"note_id": note_id, # Träger-Note der Kante
|
||||
}
|
||||
if extra:
|
||||
pl.update(extra)
|
||||
return pl
|
||||
|
||||
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None) -> str:
|
||||
base = f"{kind}:{s}->{t}#{scope}"
|
||||
if rule_id:
|
||||
base += f"|{rule_id}"
|
||||
try:
|
||||
import hashlib
|
||||
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||
except Exception: # pragma: no cover
|
||||
return base
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Typen-Registry (types.yaml)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _env(n: str, default: Optional[str] = None) -> str:
|
||||
v = os.getenv(n)
|
||||
return v if v is not None else (default or "")
|
||||
|
||||
def _load_types_registry() -> dict:
|
||||
"""Lädt die YAML-Registry aus MINDNET_TYPES_FILE oder ./config/types.yaml"""
|
||||
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||
if not os.path.isfile(p) or yaml is None:
|
||||
return {}
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
return data
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def _get_types_map(reg: dict) -> dict:
|
||||
if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
|
||||
return reg["types"]
|
||||
return reg if isinstance(reg, dict) else {}
|
||||
|
||||
def _edge_defaults_for(note_type: Optional[str], reg: dict) -> List[str]:
|
||||
"""
|
||||
Liefert die edge_defaults-Liste für den gegebenen Notiztyp.
|
||||
Fallback-Reihenfolge:
|
||||
1) reg['types'][note_type]['edge_defaults']
|
||||
2) reg['defaults']['edge_defaults'] (oder 'default'/'global')
|
||||
3) []
|
||||
"""
|
||||
types_map = _get_types_map(reg)
|
||||
if note_type and isinstance(types_map, dict):
|
||||
t = types_map.get(note_type)
|
||||
if isinstance(t, dict) and isinstance(t.get("edge_defaults"), list):
|
||||
return [str(x) for x in t["edge_defaults"] if isinstance(x, str)]
|
||||
for key in ("defaults", "default", "global"):
|
||||
v = reg.get(key)
|
||||
if isinstance(v, dict) and isinstance(v.get("edge_defaults"), list):
|
||||
return [str(x) for x in v["edge_defaults"] if isinstance(x, str)]
|
||||
return []
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Parser für Links / Relationen
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
# Normale Wikilinks (Fallback)
|
||||
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([a-zA-Z0-9_\-#:. ]+)\]\]")
|
||||
|
||||
# Getypte Inline-Relationen:
|
||||
# [[rel:KIND | Target]]
|
||||
# [[rel:KIND Target]]
|
||||
_REL_PIPE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s*\|\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||
_REL_SPACE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s+(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||
# rel: KIND [[Target]] (reines Textmuster)
|
||||
_REL_TEXT = re.compile(r"rel\s*:\s*(?P<kind>[a-z_]+)\s*\[\[\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||
|
||||
def _extract_typed_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||
"""
|
||||
Gibt Liste (kind, target) zurück und den Text mit entfernten getypten Relation-Links,
|
||||
damit die generische Wikilink-Erkennung sie nicht doppelt zählt.
|
||||
Unterstützt drei Varianten:
|
||||
- [[rel:KIND | Target]]
|
||||
- [[rel:KIND Target]]
|
||||
- rel: KIND [[Target]]
|
||||
"""
|
||||
pairs: List[Tuple[str,str]] = []
|
||||
def _collect(m):
|
||||
k = (m.group("kind") or "").strip().lower()
|
||||
t = (m.group("target") or "").strip()
|
||||
if k and t:
|
||||
pairs.append((k, t))
|
||||
return "" # Link entfernen
|
||||
|
||||
text = _REL_PIPE.sub(_collect, text)
|
||||
text = _REL_SPACE.sub(_collect, text)
|
||||
text = _REL_TEXT.sub(_collect, text)
|
||||
return pairs, text
|
||||
|
||||
# Obsidian Callout Parser
|
||||
_CALLOUT_START = re.compile(r"^\s*>\s*\[!edge\]\s*(.*)$", re.IGNORECASE)
|
||||
_REL_LINE = re.compile(r"^(?P<kind>[a-z_]+)\s*:\s*(?P<targets>.+?)\s*$", re.IGNORECASE)
|
||||
_WIKILINKS_IN_LINE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||
|
||||
def _extract_callout_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||
"""
|
||||
Findet [!edge]-Callouts und extrahiert (kind, target). Entfernt den gesamten
|
||||
Callout-Block aus dem Text (damit Wikilinks daraus nicht zusätzlich als
|
||||
"references" gezählt werden).
|
||||
"""
|
||||
if not text:
|
||||
return [], text
|
||||
|
||||
lines = text.splitlines()
|
||||
out_pairs: List[Tuple[str,str]] = []
|
||||
keep_lines: List[str] = []
|
||||
i = 0
|
||||
|
||||
while i < len(lines):
|
||||
m = _CALLOUT_START.match(lines[i])
|
||||
if not m:
|
||||
keep_lines.append(lines[i])
|
||||
i += 1
|
||||
continue
|
||||
|
||||
block_lines: List[str] = []
|
||||
first_rest = m.group(1) or ""
|
||||
if first_rest.strip():
|
||||
block_lines.append(first_rest)
|
||||
|
||||
i += 1
|
||||
while i < len(lines) and lines[i].lstrip().startswith('>'):
|
||||
block_lines.append(lines[i].lstrip()[1:].lstrip())
|
||||
i += 1
|
||||
|
||||
for bl in block_lines:
|
||||
mrel = _REL_LINE.match(bl)
|
||||
if not mrel:
|
||||
continue
|
||||
kind = (mrel.group("kind") or "").strip().lower()
|
||||
targets = mrel.group("targets") or ""
|
||||
found = _WIKILINKS_IN_LINE.findall(targets)
|
||||
if found:
|
||||
for t in found:
|
||||
t = t.strip()
|
||||
if t:
|
||||
out_pairs.append((kind, t))
|
||||
else:
|
||||
for raw in re.split(r"[,;]", targets):
|
||||
t = raw.strip()
|
||||
if t:
|
||||
out_pairs.append((kind, t))
|
||||
|
||||
# Callout wird NICHT in keep_lines übernommen
|
||||
continue
|
||||
|
||||
remainder = "\n".join(keep_lines)
|
||||
return out_pairs, remainder
|
||||
|
||||
def _extract_wikilinks(text: str) -> List[str]:
|
||||
ids: List[str] = []
|
||||
for m in _WIKILINK_RE.finditer(text or ""):
|
||||
ids.append(m.group(1).strip())
|
||||
return ids
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Hauptfunktion
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def build_edges_for_note(
|
||||
note_id: str,
|
||||
chunks: List[dict],
|
||||
@@ -84,48 +248,188 @@ def build_edges_for_note(
|
||||
|
||||
- belongs_to: für jeden Chunk (chunk -> note)
|
||||
- next / prev: zwischen aufeinanderfolgenden Chunks
|
||||
- references: pro Chunk aus window/text
|
||||
- references: pro Chunk aus window/text (via Wikilinks)
|
||||
- typed inline relations: [[rel:KIND | Target]] / [[rel:KIND Target]] / rel: KIND [[Target]]
|
||||
- Obsidian Callouts: > [!edge] KIND: [[Target]] [[Target2]]
|
||||
- optional note-scope references/backlinks: dedupliziert über alle Chunk-Funde + note_level_references
|
||||
- typenbasierte Default-Kanten (edge_defaults) je gefundener Referenz
|
||||
"""
|
||||
edges: List[dict] = []
|
||||
|
||||
# belongs_to
|
||||
# Note-Typ (aus erstem Chunk erwartet)
|
||||
note_type = None
|
||||
if chunks:
|
||||
note_type = _get(chunks[0], "type")
|
||||
|
||||
# 1) belongs_to
|
||||
for ch in chunks:
|
||||
cid = _get(ch, "chunk_id", "id")
|
||||
if not cid:
|
||||
continue
|
||||
edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {"chunk_id": cid}))
|
||||
edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to"),
|
||||
"provenance": "rule",
|
||||
"rule_id": "structure:belongs_to",
|
||||
"confidence": 1.0,
|
||||
}))
|
||||
|
||||
# next/prev
|
||||
# 2) next / prev
|
||||
for i in range(len(chunks) - 1):
|
||||
a, b = chunks[i], chunks[i + 1]
|
||||
a_id = _get(a, "chunk_id", "id")
|
||||
b_id = _get(b, "chunk_id", "id")
|
||||
if not a_id or not b_id:
|
||||
continue
|
||||
edges.append(_edge("next", "chunk", a_id, b_id, note_id, {"chunk_id": a_id}))
|
||||
edges.append(_edge("prev", "chunk", b_id, a_id, note_id, {"chunk_id": b_id}))
|
||||
edges.append(_edge("next", "chunk", a_id, b_id, note_id, {
|
||||
"chunk_id": a_id,
|
||||
"edge_id": _mk_edge_id("next", a_id, b_id, "chunk", "structure:order"),
|
||||
"provenance": "rule",
|
||||
"rule_id": "structure:order",
|
||||
"confidence": 0.95,
|
||||
}))
|
||||
edges.append(_edge("prev", "chunk", b_id, a_id, note_id, {
|
||||
"chunk_id": b_id,
|
||||
"edge_id": _mk_edge_id("prev", b_id, a_id, "chunk", "structure:order"),
|
||||
"provenance": "rule",
|
||||
"rule_id": "structure:order",
|
||||
"confidence": 0.95,
|
||||
}))
|
||||
|
||||
# references (chunk-scope) – Links aus window bevorzugen (Overlap-fest)
|
||||
# 3) references + typed inline + callouts + defaults (chunk-scope)
|
||||
reg = _load_types_registry()
|
||||
defaults = _edge_defaults_for(note_type, reg)
|
||||
refs_all: List[str] = []
|
||||
|
||||
for ch in chunks:
|
||||
cid = _get(ch, "chunk_id", "id")
|
||||
if not cid:
|
||||
continue
|
||||
txt = _chunk_text_for_refs(ch)
|
||||
refs = extract_wikilinks(txt) # <— Parser-Logik, kompatibel zu deinem System
|
||||
raw = _chunk_text_for_refs(ch)
|
||||
|
||||
# 3a) typed inline relations
|
||||
typed, remainder = _extract_typed_relations(raw)
|
||||
for kind, target in typed:
|
||||
kind = kind.strip().lower()
|
||||
if not kind or not target:
|
||||
continue
|
||||
edges.append(_edge(kind, "chunk", cid, target, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(kind, cid, target, "chunk", "inline:rel"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "inline:rel",
|
||||
"confidence": 0.95,
|
||||
}))
|
||||
if kind in {"related_to", "similar_to"}:
|
||||
edges.append(_edge(kind, "chunk", target, cid, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(kind, target, cid, "chunk", "inline:rel"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "inline:rel",
|
||||
"confidence": 0.95,
|
||||
}))
|
||||
|
||||
# 3b) callouts
|
||||
call_pairs, remainder2 = _extract_callout_relations(remainder)
|
||||
for kind, target in call_pairs:
|
||||
k = (kind or "").strip().lower()
|
||||
if not k or not target:
|
||||
continue
|
||||
edges.append(_edge(k, "chunk", cid, target, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(k, cid, target, "chunk", "callout:edge"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "callout:edge",
|
||||
"confidence": 0.95,
|
||||
}))
|
||||
if k in {"related_to", "similar_to"}:
|
||||
edges.append(_edge(k, "chunk", target, cid, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(k, target, cid, "chunk", "callout:edge"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "callout:edge",
|
||||
"confidence": 0.95,
|
||||
}))
|
||||
|
||||
# 3c) generische Wikilinks → references (+ defaults je Ref)
|
||||
refs = _extract_wikilinks(remainder2)
|
||||
for r in refs:
|
||||
edges.append(_edge("references", "chunk", cid, r, note_id, {"chunk_id": cid, "ref_text": r}))
|
||||
edges.append(_edge("references", "chunk", cid, r, note_id, {
|
||||
"chunk_id": cid,
|
||||
"ref_text": r,
|
||||
"edge_id": _mk_edge_id("references", cid, r, "chunk", "explicit:wikilink"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "explicit:wikilink",
|
||||
"confidence": 1.0,
|
||||
}))
|
||||
for rel in defaults:
|
||||
if rel == "references":
|
||||
continue
|
||||
edges.append(_edge(rel, "chunk", cid, r, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(rel, cid, r, "chunk", f"edge_defaults:{note_type}:{rel}"),
|
||||
"provenance": "rule",
|
||||
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||
"confidence": 0.7,
|
||||
}))
|
||||
if rel in {"related_to", "similar_to"}:
|
||||
edges.append(_edge(rel, "chunk", r, cid, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(rel, r, cid, "chunk", f"edge_defaults:{note_type}:{rel}"),
|
||||
"provenance": "rule",
|
||||
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||
"confidence": 0.7,
|
||||
}))
|
||||
|
||||
refs_all.extend(refs)
|
||||
|
||||
# optional: note-scope references/backlinks
|
||||
# 4) optional note-scope refs/backlinks (+ defaults)
|
||||
if include_note_scope_refs:
|
||||
refs_note = refs_all[:]
|
||||
refs_note = list(refs_all or [])
|
||||
if note_level_references:
|
||||
refs_note.extend([r for r in note_level_references if isinstance(r, str) and r])
|
||||
refs_note = _dedupe(refs_note)
|
||||
refs_note = _dedupe_seq(refs_note)
|
||||
for r in refs_note:
|
||||
edges.append(_edge("references", "note", note_id, r, note_id))
|
||||
edges.append(_edge("backlink", "note", r, note_id, note_id))
|
||||
edges.append(_edge("references", "note", note_id, r, note_id, {
|
||||
"edge_id": _mk_edge_id("references", note_id, r, "note", "explicit:note_scope"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "explicit:note_scope",
|
||||
"confidence": 1.0,
|
||||
}))
|
||||
edges.append(_edge("backlink", "note", r, note_id, note_id, {
|
||||
"edge_id": _mk_edge_id("backlink", r, note_id, "note", "derived:backlink"),
|
||||
"provenance": "rule",
|
||||
"rule_id": "derived:backlink",
|
||||
"confidence": 0.9,
|
||||
}))
|
||||
for rel in defaults:
|
||||
if rel == "references":
|
||||
continue
|
||||
edges.append(_edge(rel, "note", note_id, r, note_id, {
|
||||
"edge_id": _mk_edge_id(rel, note_id, r, "note", f"edge_defaults:{note_type}:{rel}"),
|
||||
"provenance": "rule",
|
||||
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||
"confidence": 0.7,
|
||||
}))
|
||||
if rel in {"related_to", "similar_to"}:
|
||||
edges.append(_edge(rel, "note", r, note_id, note_id, {
|
||||
"edge_id": _mk_edge_id(rel, r, note_id, "note", f"edge_defaults:{note_type}:{rel}"),
|
||||
"provenance": "rule",
|
||||
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||
"confidence": 0.7,
|
||||
}))
|
||||
|
||||
return edges
|
||||
# 5) De-Dupe (source_id, target_id, relation, rule_id)
|
||||
seen: Set[Tuple[str,str,str,str]] = set()
|
||||
out: List[dict] = []
|
||||
for e in edges:
|
||||
s = str(e.get("source_id") or "")
|
||||
t = str(e.get("target_id") or "")
|
||||
rel = str(e.get("relation") or e.get("kind") or "edge")
|
||||
rule = str(e.get("rule_id") or "")
|
||||
key = (s, t, rel, rule)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(e)
|
||||
return out
|
||||
+257
-81
@@ -2,119 +2,295 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/edges.py
|
||||
Version: 1.0.0
|
||||
Datum: 2025-09-09
|
||||
Version: 2.0.0 (V2‑superset, rückwärtskompatibel zu v1 vom 2025‑09‑09)
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Zentrale, konsistente Erzeugung von Edge-Payloads im **neuen Schema**:
|
||||
- kind : "belongs_to" | "next" | "prev" | "references" | "backlink"
|
||||
- source_id : ID des Quellknotens (Chunk- oder Note-ID)
|
||||
- target_id : ID des Zielknotens
|
||||
- scope : "chunk" | "note"
|
||||
- note_id : Owner-Note (für performantes Filtern/Löschen)
|
||||
- seq : optional (z. B. Reihenfolge von Vorkommen)
|
||||
Bewahrt die bestehende Edgelogik (belongs_to, prev/next, references, backlink)
|
||||
und ergänzt V2‑Felder + Typ‑Default‑Kanten gemäß config/types.yaml (edge_defaults).
|
||||
Die Funktion ist **idempotent** und **rückwärtskompatibel** zur bisherigen Signatur.
|
||||
|
||||
Hinweise
|
||||
--------
|
||||
- Edges werden dedupliziert (key=(kind,source_id,target_id,scope)).
|
||||
- Für Chunk-Edges wird `note_id` aus dem Chunk-Payload entnommen.
|
||||
- Für Note-Scope-Edges ist `note_id` die Quell-Note-ID.
|
||||
Kompatibilitätsgarantien (gegenüber v1):
|
||||
- **Input**: akzeptiert identische Chunk‑Payloads wie v1:
|
||||
* `id` (Chunk‑ID), `note_id` (Owner), `neighbors.prev|next` (optional),
|
||||
`references: [{target_id: ...}]` (optional),
|
||||
alternativ: `chunk_id`, `chunk_index|ord`, `window|text`
|
||||
- **Output (v1‑Felder)**: `kind`, `source_id`, `target_id`, `scope`, `note_id`, `edge_id`
|
||||
- **Neu (v2‑Felder)**: `relation`, `src_note_id`, `src_chunk_id?`, `dst_note_id`, `dst_chunk_id?`,
|
||||
`provenance` (`explicit|rule`), `rule_id?`, `confidence?`
|
||||
|
||||
Regeln
|
||||
------
|
||||
- Deduplizierungsschlüssel: (source_id, target_id, relation, rule_id)
|
||||
- Strukturkanten:
|
||||
* belongs_to: 1× pro Chunk
|
||||
* next/prev: Sequenz der Chunks; nutzt bevorzugt neighbors; sonst ord/chunk_index
|
||||
- Explizite Referenzen:
|
||||
* aus Chunk: `references[].target_id` (falls vorhanden)
|
||||
* Fallback: Wikilinks in `window|text`: [[Some Title|some-id]] oder [[some-id]]
|
||||
- Note‑Scope:
|
||||
* backlink immer; references nur, wenn include_note_scope_refs=True
|
||||
- Typ‑Defaults (edge_defaults aus config/types.yaml des **Quell‑Notiztyps**):
|
||||
* Für jede explizite Referenz wird je default‑Relation eine Regel‑Kante erzeugt
|
||||
* rule_id: "type_default:{note_type}:{relation}:v1", provenance="rule"
|
||||
|
||||
Konfiguration
|
||||
-------------
|
||||
- ENV MINDNET_TYPES_FILE (Default: ./config/types.yaml)
|
||||
|
||||
Lizenz/Autor
|
||||
------------
|
||||
- Erstimplementierung v1 (2025‑09‑09) — Projekt Mindnet
|
||||
- Erweiterung v2 (2025‑11‑11) — kompatible Superset‑Implementierung
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Dict, List
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, Iterable, List, Optional, Tuple, Set
|
||||
|
||||
try:
|
||||
import yaml # optional, nur für types.yaml
|
||||
except Exception: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Hilfen: types.yaml laden (edge_defaults)
|
||||
# ------------------------------------------------------------
|
||||
|
||||
def _types_path() -> str:
|
||||
return os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
|
||||
|
||||
def _load_types() -> Dict[str, dict]:
|
||||
p = _types_path()
|
||||
if not os.path.isfile(p) or yaml is None:
|
||||
return {}
|
||||
try:
|
||||
with open(p, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
|
||||
return data["types"]
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def _edge_defaults_for(note_type: Optional[str]) -> List[str]:
|
||||
types = _load_types()
|
||||
t = (note_type or "").strip().lower()
|
||||
cfg = types.get(t) or {}
|
||||
defaults = cfg.get("edge_defaults") or []
|
||||
if isinstance(defaults, str):
|
||||
defaults = [defaults]
|
||||
return [str(x) for x in defaults if isinstance(x, (str, int, float))]
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Wikilink‑Parser (Fallback, wenn ch["references"] fehlt)
|
||||
# ------------------------------------------------------------
|
||||
|
||||
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([a-zA-Z0-9_\-#:. ]+)\]\]")
|
||||
|
||||
def _extract_wikilinks(text: str) -> List[str]:
|
||||
ids: List[str] = []
|
||||
for m in _WIKILINK_RE.finditer(text or ""):
|
||||
ids.append(m.group(1).strip())
|
||||
return ids
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Utility
|
||||
# ------------------------------------------------------------
|
||||
|
||||
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None) -> str:
|
||||
base = f"{kind}:{s}->{t}#{scope}"
|
||||
if rule_id:
|
||||
base += f"|{rule_id}"
|
||||
try:
|
||||
import hashlib
|
||||
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||
except Exception: # pragma: no cover
|
||||
return base
|
||||
|
||||
def _dedupe(edges: List[Dict]) -> List[Dict]:
|
||||
seen: Set[Tuple[str,str,str,str]] = set()
|
||||
out: List[Dict] = []
|
||||
for e in edges:
|
||||
s = str(e.get("source_id") or "")
|
||||
t = str(e.get("target_id") or "")
|
||||
rel = str(e.get("relation") or e.get("kind") or "edge")
|
||||
rule = str(e.get("rule_id") or "")
|
||||
key = (s, t, rel, rule)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(e)
|
||||
return out
|
||||
|
||||
def _first(v: dict, *keys, default=None):
|
||||
for k in keys:
|
||||
if k in v and v[k] is not None:
|
||||
return v[k]
|
||||
return default
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Hauptfunktion
|
||||
# ------------------------------------------------------------
|
||||
|
||||
def build_edges_for_note(
|
||||
note_id: str,
|
||||
chunk_payloads: List[Dict],
|
||||
note_level_refs: List[str] | None,
|
||||
note_level_refs: Optional[List[str]] = None,
|
||||
*,
|
||||
include_note_scope_refs: bool = False,
|
||||
) -> List[Dict]:
|
||||
edges: List[Dict] = []
|
||||
chunks = list(chunk_payloads or [])
|
||||
# Notiztyp aus erstem Chunk ableiten (kompatibel zu existierenden Payloads)
|
||||
note_type = (chunks[0].get("type") if chunks else None) or (chunks[0].get("note_type") if chunks else None)
|
||||
|
||||
# Chunk-Scope: belongs_to / prev / next / references
|
||||
for ch in chunk_payloads:
|
||||
cid = ch["id"]
|
||||
# --- Strukturkanten ------------------------------------------------------
|
||||
# belongs_to
|
||||
for ch in chunks:
|
||||
cid = _first(ch, "id", "chunk_id")
|
||||
if not cid:
|
||||
continue
|
||||
owner = ch.get("note_id") or note_id
|
||||
# belongs_to
|
||||
edges.append({
|
||||
e = {
|
||||
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to:v1"),
|
||||
"kind": "belongs_to",
|
||||
"relation": "belongs_to",
|
||||
"scope": "chunk",
|
||||
"source_id": cid,
|
||||
"target_id": note_id,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
# Nachbarn
|
||||
"note_id": owner, # v1-Kompat
|
||||
# v2
|
||||
"src_note_id": owner,
|
||||
"src_chunk_id": cid,
|
||||
"dst_note_id": note_id,
|
||||
"provenance": "rule",
|
||||
"rule_id": "structure:belongs_to:v1",
|
||||
"confidence": 1.0,
|
||||
}
|
||||
edges.append(e)
|
||||
|
||||
# next/prev — bevorzugt neighbors.prev/next; sonst via ord/chunk_index
|
||||
# Map der Chunks nach Index
|
||||
ordered = list(chunks)
|
||||
def _idx(c):
|
||||
return _first(c, "chunk_index", "ord", default=0)
|
||||
ordered.sort(key=_idx)
|
||||
|
||||
for i, ch in enumerate(ordered):
|
||||
cid = _first(ch, "id", "chunk_id")
|
||||
if not cid:
|
||||
continue
|
||||
owner = ch.get("note_id") or note_id
|
||||
nb = ch.get("neighbors") or {}
|
||||
prev_id = nb.get("prev")
|
||||
next_id = nb.get("next")
|
||||
# Fallback-Reihenfolge
|
||||
if prev_id is None and i > 0:
|
||||
prev_id = _first(ordered[i-1], "id", "chunk_id")
|
||||
if next_id is None and i+1 < len(ordered):
|
||||
next_id = _first(ordered[i+1], "id", "chunk_id")
|
||||
|
||||
if prev_id:
|
||||
edges.append({
|
||||
"kind": "prev",
|
||||
"source_id": cid,
|
||||
"target_id": prev_id,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
"edge_id": _mk_edge_id("prev", cid, prev_id, "chunk", "structure:order:v1"),
|
||||
"kind": "prev", "relation": "prev", "scope": "chunk",
|
||||
"source_id": cid, "target_id": prev_id, "note_id": owner,
|
||||
"src_note_id": owner, "src_chunk_id": cid,
|
||||
"dst_note_id": owner, "dst_chunk_id": prev_id,
|
||||
"provenance": "rule", "rule_id": "structure:order:v1", "confidence": 0.95,
|
||||
})
|
||||
edges.append({
|
||||
"kind": "next",
|
||||
"source_id": prev_id,
|
||||
"target_id": cid,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
if next_id:
|
||||
edges.append({
|
||||
"kind": "next",
|
||||
"source_id": cid,
|
||||
"target_id": next_id,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
edges.append({
|
||||
"kind": "prev",
|
||||
"source_id": next_id,
|
||||
"target_id": cid,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
})
|
||||
# references aus Chunk
|
||||
for ref in (ch.get("references") or []):
|
||||
tid = ref.get("target_id")
|
||||
if not tid:
|
||||
continue
|
||||
edges.append({
|
||||
"kind": "references",
|
||||
"source_id": cid,
|
||||
"target_id": tid,
|
||||
"scope": "chunk",
|
||||
"note_id": owner,
|
||||
"edge_id": _mk_edge_id("next", prev_id, cid, "chunk", "structure:order:v1"),
|
||||
"kind": "next", "relation": "next", "scope": "chunk",
|
||||
"source_id": prev_id, "target_id": cid, "note_id": owner,
|
||||
"src_note_id": owner, "src_chunk_id": prev_id,
|
||||
"dst_note_id": owner, "dst_chunk_id": cid,
|
||||
"provenance": "rule", "rule_id": "structure:order:v1", "confidence": 0.95,
|
||||
})
|
||||
|
||||
# Note-Scope: backlink (immer); references (optional)
|
||||
unique_refs = list(dict.fromkeys(note_level_refs or []))
|
||||
# --- Explizite Referenzen (Chunk‑Scope) ---------------------------------
|
||||
explicit_refs: List[Dict] = []
|
||||
for ch in chunks:
|
||||
cid = _first(ch, "id", "chunk_id")
|
||||
if not cid:
|
||||
continue
|
||||
owner = ch.get("note_id") or note_id
|
||||
# 1) bevorzugt vorhandene ch["references"]
|
||||
refs = ch.get("references") or []
|
||||
targets = [r.get("target_id") for r in refs if isinstance(r, dict) and r.get("target_id")]
|
||||
# 2) Fallback: Wikilinks aus Text
|
||||
if not targets:
|
||||
text = _first(ch, "window", "text", default="") or ""
|
||||
targets = _extract_wikilinks(text)
|
||||
for tid in targets:
|
||||
if not isinstance(tid, str) or not tid.strip():
|
||||
continue
|
||||
e = {
|
||||
"edge_id": _mk_edge_id("references", cid, tid, "chunk"),
|
||||
"kind": "references",
|
||||
"relation": "references",
|
||||
"scope": "chunk",
|
||||
"source_id": cid,
|
||||
"target_id": tid,
|
||||
"note_id": owner,
|
||||
# v2
|
||||
"src_note_id": owner,
|
||||
"src_chunk_id": cid,
|
||||
"dst_note_id": tid,
|
||||
"provenance": "explicit",
|
||||
"rule_id": "",
|
||||
"confidence": 1.0,
|
||||
}
|
||||
edges.append(e)
|
||||
explicit_refs.append(e)
|
||||
|
||||
# --- Note‑Scope: references (optional) + backlink (immer) ----------------
|
||||
unique_refs = []
|
||||
if note_level_refs:
|
||||
seen = set()
|
||||
for tid in note_level_refs:
|
||||
if isinstance(tid, str) and tid.strip() and tid not in seen:
|
||||
unique_refs.append(tid); seen.add(tid)
|
||||
|
||||
for tid in unique_refs:
|
||||
if include_note_scope_refs:
|
||||
edges.append({
|
||||
"kind": "references",
|
||||
"source_id": note_id,
|
||||
"target_id": tid,
|
||||
"scope": "note",
|
||||
"note_id": note_id,
|
||||
"edge_id": _mk_edge_id("references", note_id, tid, "note"),
|
||||
"kind": "references", "relation": "references", "scope": "note",
|
||||
"source_id": note_id, "target_id": tid, "note_id": note_id,
|
||||
"src_note_id": note_id, "dst_note_id": tid,
|
||||
"provenance": "explicit", "rule_id": "", "confidence": 1.0,
|
||||
})
|
||||
edges.append({
|
||||
"kind": "backlink",
|
||||
"source_id": tid,
|
||||
"target_id": note_id,
|
||||
"scope": "note",
|
||||
"note_id": note_id,
|
||||
"edge_id": _mk_edge_id("backlink", tid, note_id, "note", "derived:backlink:v1"),
|
||||
"kind": "backlink", "relation": "backlink", "scope": "note",
|
||||
"source_id": tid, "target_id": note_id, "note_id": note_id,
|
||||
"src_note_id": tid, "dst_note_id": note_id,
|
||||
"provenance": "rule", "rule_id": "derived:backlink:v1", "confidence": 0.9,
|
||||
})
|
||||
|
||||
# Dedupe
|
||||
dedup = {}
|
||||
for e in edges:
|
||||
k = (e["kind"], e["source_id"], e["target_id"], e.get("scope", ""))
|
||||
dedup[k] = e
|
||||
return list(dedup.values())
|
||||
# --- Type‑Defaults je expliziter Referenz --------------------------------
|
||||
defaults = [d for d in _edge_defaults_for(note_type) if d and d != "references"]
|
||||
if defaults:
|
||||
for e in explicit_refs + ([ ] if not include_note_scope_refs else []):
|
||||
# wir nutzen die bereits erzeugten explicit‑Edges als Vorlage
|
||||
src = e["source_id"]; tgt = e["target_id"]
|
||||
scope = e.get("scope", "chunk")
|
||||
s_note = e.get("src_note_id") or note_id
|
||||
s_chunk = e.get("src_chunk_id")
|
||||
t_note = e.get("dst_note_id") or tgt
|
||||
for rel in defaults:
|
||||
rule_id = f"type_default:{(note_type or 'unknown')}:{rel}:v1"
|
||||
edges.append({
|
||||
"edge_id": _mk_edge_id(rel, src, tgt, scope, rule_id),
|
||||
"kind": rel, "relation": rel, "scope": scope,
|
||||
"source_id": src, "target_id": tgt, "note_id": s_note,
|
||||
"src_note_id": s_note, "src_chunk_id": s_chunk,
|
||||
"dst_note_id": t_note,
|
||||
"provenance": "rule", "rule_id": rule_id, "confidence": 0.7,
|
||||
})
|
||||
|
||||
# --- Dedupe & Return -----------------------------------------------------
|
||||
return _dedupe(edges)
|
||||
+252
-39
@@ -1,90 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
app/core/graph_adapter.py — Adjazenzaufbau & Subgraph-Expansion (WP-04)
|
||||
|
||||
Zweck:
|
||||
Baut aus Qdrant-Edges (Collection: *_edges) einen leichten In-Memory-Graph
|
||||
und liefert Edge-basierte Kennzahlen (In-Degree, Out-Degree, edge_bonus).
|
||||
|
||||
Kompatibilität:
|
||||
Python 3.12+, qdrant-client 1.x
|
||||
- Python 3.12+, qdrant-client 1.x
|
||||
- Wird von app/core/retriever.py im Hybrid-Modus genutzt.
|
||||
- Signaturen bleiben kompatibel zu den bestehenden Tests
|
||||
(tests/test_retriever_edges.py patcht expand()).
|
||||
|
||||
Version:
|
||||
0.1.0 (Erstanlage)
|
||||
Stand:
|
||||
2025-10-07
|
||||
Bezug:
|
||||
- WP-04 Edge-Gewichtungen und Heuristiken
|
||||
- app/core/qdrant_points.py (get_edges_for_sources)
|
||||
Nutzung:
|
||||
from app.core.graph_adapter import expand
|
||||
Änderungsverlauf:
|
||||
0.1.0 (2025-10-07) – Erstanlage.
|
||||
0.3.1 (2025-12-04 – note_id-bewusste Expansion:
|
||||
* Seeds = Note-IDs
|
||||
* Subgraph pflegt zusätzlich Adjazenz auf note_id-Basis,
|
||||
damit der Retriever mit note_id arbeiten kann)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Optional, DefaultDict
|
||||
from collections import defaultdict
|
||||
from qdrant_client import QdrantClient
|
||||
from app.core.qdrant_points import get_edges_for_sources
|
||||
|
||||
EDGE_BASE_WEIGHTS = {
|
||||
"references": 0.20,
|
||||
"belongs_to": 0.10,
|
||||
"next": 0.06,
|
||||
"prev": 0.06,
|
||||
"backlink": 0.04,
|
||||
"references_at": 0.08,
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import collection_names
|
||||
|
||||
# Legacy-Import (wird aktuell nicht mehr verwendet, bleibt aber erhalten,
|
||||
# damit bestehende Importe/Mocks nicht brechen).
|
||||
try: # pragma: no cover
|
||||
from app.core.qdrant_points import get_edges_for_sources # type: ignore
|
||||
except Exception: # pragma: no cover
|
||||
get_edges_for_sources = None # type: ignore
|
||||
|
||||
|
||||
# Basisgewichte je Edge-Typ.
|
||||
# Diese Werte werden mit der in der Edge-Payload hinterlegten "confidence"
|
||||
# multipliziert, falls vorhanden.
|
||||
#
|
||||
# Hinweis:
|
||||
# - Strukturkanten (belongs_to, next/prev) sind schwächer gewichtet.
|
||||
# - Wissenskanten (depends_on, related_to, similar_to, references) erhalten
|
||||
# höhere Basisgewichte, damit sie sich im Retriever bemerkbar machen.
|
||||
EDGE_BASE_WEIGHTS: Dict[str, float] = {
|
||||
# Struktur / Navigationskanten
|
||||
"belongs_to": 0.10,
|
||||
"next": 0.06,
|
||||
"prev": 0.06,
|
||||
"backlink": 0.04,
|
||||
"references_at": 0.08,
|
||||
|
||||
# Wissenskanten
|
||||
"references": 0.20,
|
||||
"depends_on": 0.18,
|
||||
"related_to": 0.15,
|
||||
"similar_to": 0.12,
|
||||
# weitere Typen erhalten per Default 0.0 und wirken nur über centrality
|
||||
}
|
||||
|
||||
|
||||
def _edge_weight(pl: Dict) -> float:
|
||||
"""Berechnet das effektive Edge-Gewicht aus kind + confidence."""
|
||||
kind = pl.get("kind", "edge")
|
||||
base = EDGE_BASE_WEIGHTS.get(kind, 0.0)
|
||||
|
||||
conf_raw = pl.get("confidence", None)
|
||||
try:
|
||||
conf = float(conf_raw) if conf_raw is not None else None
|
||||
except Exception:
|
||||
conf = None
|
||||
|
||||
if conf is None:
|
||||
return base
|
||||
|
||||
# Confidence vorsichtig in [0.0, 1.0] clampen
|
||||
if conf < 0.0:
|
||||
conf = 0.0
|
||||
if conf > 1.0:
|
||||
conf = 1.0
|
||||
|
||||
return base * conf
|
||||
|
||||
|
||||
def _fetch_edges(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
seeds: List[str],
|
||||
edge_types: Optional[List[str]] = None,
|
||||
limit: int = 2048,
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Holt Edges direkt aus der *_edges Collection.
|
||||
|
||||
Filter:
|
||||
- source_id IN seeds ODER target_id IN seeds ODER note_id IN seeds
|
||||
- optional: kind IN edge_types
|
||||
"""
|
||||
if not seeds or limit <= 0:
|
||||
return []
|
||||
|
||||
_, _, edges_col = collection_names(prefix)
|
||||
|
||||
# OR über source_id / target_id / note_id für alle Seeds
|
||||
seed_conditions = []
|
||||
for field in ("source_id", "target_id", "note_id"):
|
||||
for s in seeds:
|
||||
seed_conditions.append(
|
||||
rest.FieldCondition(
|
||||
key=field,
|
||||
match=rest.MatchValue(value=str(s)),
|
||||
)
|
||||
)
|
||||
seeds_filter = rest.Filter(should=seed_conditions) if seed_conditions else None
|
||||
|
||||
# Optional: Filter auf bestimmte Edge-Typen (kind)
|
||||
type_filter = None
|
||||
if edge_types:
|
||||
type_conds = [
|
||||
rest.FieldCondition(
|
||||
key="kind",
|
||||
match=rest.MatchValue(value=str(k)),
|
||||
)
|
||||
for k in edge_types
|
||||
]
|
||||
type_filter = rest.Filter(should=type_conds)
|
||||
|
||||
flt = None
|
||||
must = []
|
||||
if seeds_filter:
|
||||
must.append(seeds_filter)
|
||||
if type_filter:
|
||||
must.append(type_filter)
|
||||
if must:
|
||||
flt = rest.Filter(must=must)
|
||||
|
||||
pts, _ = client.scroll(
|
||||
collection_name=edges_col,
|
||||
scroll_filter=flt,
|
||||
limit=limit,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
|
||||
out: List[Dict] = []
|
||||
for p in pts or []:
|
||||
pl = dict(p.payload or {})
|
||||
if pl:
|
||||
out.append(pl)
|
||||
return out
|
||||
|
||||
|
||||
class Subgraph:
|
||||
"""Leichtgewichtiger Subgraph mit Adjazenzlisten & einfachen Kennzahlen."""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
self.adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||
self.in_degree: DefaultDict[str, int] = defaultdict(int)
|
||||
self.out_degree: DefaultDict[str, int] = defaultdict(int)
|
||||
|
||||
def add_edge(self, e: Dict):
|
||||
src = e["source"]; tgt = e["target"]; kind = e["kind"]
|
||||
def add_edge(self, e: Dict) -> None:
|
||||
"""
|
||||
Fügt eine Kante zum Subgraph hinzu.
|
||||
|
||||
e enthält:
|
||||
- source: Knoten-ID (z. B. source_id)
|
||||
- target: Knoten-ID (z. B. target_id)
|
||||
- kind: Kantentyp
|
||||
- weight: bereits berechnetes Gewicht
|
||||
- note_id: (optional) Kontext-Note dieser Kante
|
||||
|
||||
Wichtig für den Retriever:
|
||||
- Wir pflegen Adjazenz primär unter `source`.
|
||||
- Zusätzlich pflegen wir, falls vorhanden, eine Adjazenz
|
||||
unter `note_id`, damit der Retriever mit note_id arbeiten kann.
|
||||
"""
|
||||
src = e.get("source")
|
||||
tgt = e.get("target")
|
||||
kind = e.get("kind")
|
||||
weight = e.get("weight", EDGE_BASE_WEIGHTS.get(kind, 0.0))
|
||||
self.adj[src].append({"target": tgt, "kind": kind, "weight": weight})
|
||||
owner = e.get("note_id")
|
||||
|
||||
if not src or not tgt:
|
||||
return
|
||||
|
||||
# Primäre Adjazenz (z. B. source_id → target_id)
|
||||
self.adj[src].append(
|
||||
{
|
||||
"target": tgt,
|
||||
"kind": kind,
|
||||
"weight": weight,
|
||||
}
|
||||
)
|
||||
self.out_degree[src] += 1
|
||||
self.in_degree[tgt] += 1
|
||||
|
||||
# Sekundäre Adjazenz auf Note-Ebene:
|
||||
# Falls eine Kontext-Note existiert, bekommt diese ebenfalls
|
||||
# "Ausgangskanten" angerechnet. So kann der Retriever über note_id
|
||||
# edge_bonus(...) und centrality_bonus(...) abfragen.
|
||||
if owner:
|
||||
# same edge unter owner registrieren (falls nicht identisch mit src),
|
||||
# damit edge_bonus(owner) > 0 werden kann.
|
||||
if owner != src:
|
||||
self.adj[owner].append(
|
||||
{
|
||||
"target": tgt,
|
||||
"kind": kind,
|
||||
"weight": weight,
|
||||
}
|
||||
)
|
||||
self.out_degree[owner] += 1
|
||||
|
||||
# Zentralität der Kontext-Note leicht erhöhen
|
||||
if owner != tgt:
|
||||
self.in_degree[owner] += 1
|
||||
|
||||
def aggregate_edge_bonus(self, node_id: str) -> float:
|
||||
"""
|
||||
Summe der ausgehenden Kantengewichte für einen Knoten.
|
||||
"""
|
||||
return sum(edge["weight"] for edge in self.adj.get(node_id, []))
|
||||
|
||||
def edge_bonus(self, node_id: str) -> float:
|
||||
"""
|
||||
Kompatibilitäts-Methode für den Retriever.
|
||||
|
||||
Der Retriever ruft subgraph.edge_bonus(node_id) auf. Intern verwenden
|
||||
wir aggregate_edge_bonus(...), um bestehende Tests nicht zu brechen.
|
||||
"""
|
||||
return self.aggregate_edge_bonus(node_id)
|
||||
|
||||
def centrality_bonus(self, node_id: str) -> float:
|
||||
"""
|
||||
Einfache log-gedämpfte Zentralität auf Basis des In-Degree.
|
||||
Obergrenze: 0.15
|
||||
"""
|
||||
import math
|
||||
# Log-gedämpfter Bonus, hart begrenzt
|
||||
return min(math.log1p(self.in_degree.get(node_id, 0)) / 10.0, 0.15)
|
||||
|
||||
indeg = self.in_degree.get(node_id, 0)
|
||||
if indeg <= 0:
|
||||
return 0.0
|
||||
return min(math.log1p(indeg) / 10.0, 0.15)
|
||||
|
||||
|
||||
def expand(client: QdrantClient, prefix: str, seeds: List[str],
|
||||
depth: int = 1, edge_types: Optional[List[str]] = None) -> Subgraph:
|
||||
def expand(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
seeds: List[str],
|
||||
depth: int = 1,
|
||||
edge_types: Optional[List[str]] = None,
|
||||
) -> Subgraph:
|
||||
"""
|
||||
Expandiert ab Seeds entlang von Edges (bis depth), optional gefiltert nach Typen.
|
||||
Seeds sind stabile payload-IDs (z. B. note_id, chunk_id).
|
||||
Expandiert ab Seeds entlang von Edges (bis `depth`), optional gefiltert
|
||||
nach Edge-Typen.
|
||||
|
||||
Seeds sind stabile payload-IDs (in unserem Fall vor allem note_id).
|
||||
Es werden Edges berücksichtigt, bei denen source_id ODER target_id ODER
|
||||
note_id einem der Seeds entspricht.
|
||||
"""
|
||||
sg = Subgraph()
|
||||
frontier = set(seeds)
|
||||
visited = set()
|
||||
|
||||
for _ in range(max(depth, 0)):
|
||||
max_depth = max(depth, 0)
|
||||
|
||||
for _ in range(max_depth):
|
||||
if not frontier:
|
||||
break
|
||||
|
||||
edges = get_edges_for_sources(client, prefix, list(frontier), edge_types=edge_types, limit=2048)
|
||||
edges_payloads = _fetch_edges(
|
||||
client=client,
|
||||
prefix=prefix,
|
||||
seeds=list(frontier),
|
||||
edge_types=edge_types,
|
||||
limit=2048,
|
||||
)
|
||||
|
||||
next_frontier = set()
|
||||
for pl in edges:
|
||||
for pl in edges_payloads:
|
||||
src = pl.get("source_id")
|
||||
tgt = pl.get("target_id")
|
||||
kind = pl.get("kind", "edge")
|
||||
owner = pl.get("note_id")
|
||||
|
||||
e = {
|
||||
"source": pl.get("source_id"),
|
||||
"target": pl.get("target_id"),
|
||||
"kind": pl.get("kind", "edge"),
|
||||
"weight": pl.get("weight", EDGE_BASE_WEIGHTS.get(pl.get("kind", "edge"), 0.0)),
|
||||
"source": src,
|
||||
"target": tgt,
|
||||
"kind": kind,
|
||||
"weight": _edge_weight(pl),
|
||||
"note_id": owner,
|
||||
}
|
||||
sg.add_edge(e)
|
||||
if e["target"]:
|
||||
next_frontier.add(e["target"])
|
||||
|
||||
if tgt:
|
||||
next_frontier.add(tgt)
|
||||
|
||||
visited |= frontier
|
||||
frontier = next_frontier - visited
|
||||
|
||||
+179
-57
@@ -1,104 +1,226 @@
|
||||
# note_payload.py
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/note_payload.py
|
||||
Version: 2.0.0
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Erzeugt ein robustes Note-Payload. Werte wie `retriever_weight`, `chunk_profile`
|
||||
und `edge_defaults` werden in folgender Priorität bestimmt:
|
||||
1) Frontmatter (Note)
|
||||
2) Typ-Registry (config/types.yaml: types.<type>.*)
|
||||
3) Registry-Defaults (config/types.yaml: defaults.*)
|
||||
4) ENV-Defaults (MINDNET_DEFAULT_RETRIEVER_WEIGHT / MINDNET_DEFAULT_CHUNK_PROFILE)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
import os, json, pathlib, yaml
|
||||
|
||||
def _as_dict(note: Any) -> Dict[str, Any]:
|
||||
if isinstance(note, dict):
|
||||
return note
|
||||
d: Dict[str, Any] = {}
|
||||
for attr in ("id","note_id","title","path","frontmatter","meta","metadata","type","created","modified","date","tags"):
|
||||
if hasattr(note, attr):
|
||||
d[attr] = getattr(note, attr)
|
||||
# Normalisiere Frontmatter
|
||||
fm = d.get("frontmatter") or d.get("meta") or d.get("metadata") or {}
|
||||
d["frontmatter"] = fm if isinstance(fm, dict) else {}
|
||||
return d
|
||||
from typing import Any, Dict, Tuple, Optional
|
||||
import os
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[Dict[str,Any]]]:
|
||||
path = kwargs.get("path")
|
||||
types_cfg = kwargs.get("types_config")
|
||||
# legacy positional: (path, types_config) oder (types_config, ...)
|
||||
for a in args:
|
||||
if path is None and isinstance(a, (str, pathlib.Path)):
|
||||
path = str(a)
|
||||
if types_cfg is None and isinstance(a, dict):
|
||||
types_cfg = a
|
||||
try:
|
||||
import yaml # type: ignore
|
||||
except Exception:
|
||||
yaml = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _as_dict(x) -> Dict[str, Any]:
|
||||
"""Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen."""
|
||||
if isinstance(x, dict):
|
||||
return dict(x)
|
||||
|
||||
out: Dict[str, Any] = {}
|
||||
# bekannte Attribute übernehmen, sofern vorhanden
|
||||
for attr in (
|
||||
"frontmatter",
|
||||
"body",
|
||||
"id",
|
||||
"note_id",
|
||||
"title",
|
||||
"path",
|
||||
"tags",
|
||||
"type",
|
||||
"created",
|
||||
"modified",
|
||||
"date",
|
||||
):
|
||||
if hasattr(x, attr):
|
||||
val = getattr(x, attr)
|
||||
if val is not None:
|
||||
out[attr] = val
|
||||
|
||||
# Fallback: wenn immer noch leer, raw speichern
|
||||
if not out:
|
||||
out["raw"] = str(x)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]:
|
||||
"""Extrahiert optionale Zusatzargumente wie path und types_cfg."""
|
||||
path = kwargs.get("path") or (args[0] if args else None)
|
||||
types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None
|
||||
return path, types_cfg
|
||||
|
||||
def _load_types_config(explicit: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
if isinstance(explicit, dict):
|
||||
return explicit
|
||||
for rel in ("config/config.yaml", "config/types.yaml"):
|
||||
p = pathlib.Path(rel)
|
||||
if p.exists():
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
|
||||
return data["types"]
|
||||
return data if isinstance(data, dict) else {}
|
||||
return {}
|
||||
|
||||
def _coalesce(*vals):
|
||||
for v in vals:
|
||||
if v is not None:
|
||||
return v
|
||||
return None
|
||||
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
"""Liest einen Float-Wert aus der Umgebung, mit robustem Fallback."""
|
||||
try:
|
||||
return float(os.environ.get(name, default))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _ensure_list(x) -> list:
|
||||
if x is None: return []
|
||||
if isinstance(x, list): return [str(i) for i in x]
|
||||
if isinstance(x, (set, tuple)): return [str(i) for i in x]
|
||||
"""Garantiert eine String-Liste."""
|
||||
if x is None:
|
||||
return []
|
||||
if isinstance(x, list):
|
||||
return [str(i) for i in x]
|
||||
if isinstance(x, (set, tuple)):
|
||||
return [str(i) for i in x]
|
||||
return [str(x)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Type-Registry laden
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict:
|
||||
"""Lädt die Type-Registry aus YAML/JSON oder nutzt ein explizit übergebenes Dict."""
|
||||
if explicit_cfg and isinstance(explicit_cfg, dict):
|
||||
return explicit_cfg
|
||||
|
||||
path = os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
|
||||
if not os.path.isfile(path) or yaml is None:
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _cfg_for_type(note_type: str, reg: dict) -> dict:
|
||||
"""Liefert die Konfiguration für einen konkreten Notiztyp aus der Registry."""
|
||||
if not isinstance(reg, dict):
|
||||
return {}
|
||||
types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
|
||||
return types.get(note_type, {}) if isinstance(types, dict) else {}
|
||||
|
||||
|
||||
def _cfg_defaults(reg: dict) -> dict:
|
||||
"""Liefert den Default-Block aus der Registry (defaults/global)."""
|
||||
if not isinstance(reg, dict):
|
||||
return {}
|
||||
for key in ("defaults", "default", "global"):
|
||||
v = reg.get(key)
|
||||
if isinstance(v, dict):
|
||||
return v
|
||||
return {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Haupt-API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
"""
|
||||
Baut das Note-Payload für mindnet_notes auf.
|
||||
|
||||
Erwartete Felder im Payload:
|
||||
- note_id: stabile ID aus Frontmatter (id) oder Note-Objekt
|
||||
- title: Titel der Notiz
|
||||
- type: Notiztyp (z. B. concept, project, journal, ...)
|
||||
- path: Pfad im Vault
|
||||
- retriever_weight: effektives Gewicht für den Retriever
|
||||
- chunk_profile: Profil für Chunking (short|medium|long|default|...)
|
||||
- edge_defaults: Liste von Kanten-Typen, die als Defaults gelten
|
||||
"""
|
||||
n = _as_dict(note)
|
||||
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
|
||||
types_cfg = _load_types_config(types_cfg_explicit)
|
||||
reg = _load_types_config(types_cfg_explicit)
|
||||
|
||||
fm = n.get("frontmatter") or {}
|
||||
note_type = str(fm.get("type") or n.get("type") or "note")
|
||||
cfg_for_type = types_cfg.get(note_type, {}) if isinstance(types_cfg, dict) else {}
|
||||
fm_type = fm.get("type") or n.get("type") or "concept"
|
||||
note_type = str(fm_type)
|
||||
|
||||
cfg_type = _cfg_for_type(note_type, reg)
|
||||
cfg_def = _cfg_defaults(reg)
|
||||
|
||||
# --- retriever_weight: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
|
||||
default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
|
||||
retriever_weight = _coalesce(fm.get("retriever_weight"), cfg_for_type.get("retriever_weight"), default_rw)
|
||||
retriever_weight = fm.get("retriever_weight")
|
||||
if retriever_weight is None:
|
||||
retriever_weight = cfg_type.get(
|
||||
"retriever_weight",
|
||||
cfg_def.get("retriever_weight", default_rw),
|
||||
)
|
||||
try:
|
||||
retriever_weight = float(retriever_weight)
|
||||
except Exception:
|
||||
retriever_weight = default_rw
|
||||
|
||||
chunk_profile = _coalesce(fm.get("chunk_profile"), cfg_for_type.get("chunk_profile"), os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE","medium"))
|
||||
chunk_profile = chunk_profile if isinstance(chunk_profile, str) else "medium"
|
||||
# --- chunk_profile: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
|
||||
chunk_profile = fm.get("chunk_profile")
|
||||
if chunk_profile is None:
|
||||
chunk_profile = cfg_type.get(
|
||||
"chunk_profile",
|
||||
cfg_def.get(
|
||||
"chunk_profile",
|
||||
os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE", "medium"),
|
||||
),
|
||||
)
|
||||
if not isinstance(chunk_profile, str):
|
||||
chunk_profile = "medium"
|
||||
|
||||
edge_defaults = _ensure_list(_coalesce(fm.get("edge_defaults"), cfg_for_type.get("edge_defaults"), []))
|
||||
# --- edge_defaults: Frontmatter > Typ-Config > Registry-Defaults ---
|
||||
edge_defaults = fm.get("edge_defaults")
|
||||
if edge_defaults is None:
|
||||
edge_defaults = cfg_type.get(
|
||||
"edge_defaults",
|
||||
cfg_def.get("edge_defaults", []),
|
||||
)
|
||||
edge_defaults = _ensure_list(edge_defaults)
|
||||
|
||||
# --- Basis-Metadaten (IDs, Titel, Pfad) ---
|
||||
note_id = n.get("note_id") or n.get("id") or fm.get("id")
|
||||
title = n.get("title") or fm.get("title") or ""
|
||||
path = n.get("path") or path_arg
|
||||
if isinstance(path, pathlib.Path):
|
||||
path = str(path)
|
||||
|
||||
payload = {
|
||||
payload: Dict[str, Any] = {
|
||||
"note_id": note_id,
|
||||
"title": title,
|
||||
"type": note_type,
|
||||
"path": path or "", # immer vorhanden
|
||||
"path": path or "",
|
||||
"retriever_weight": retriever_weight,
|
||||
"chunk_profile": chunk_profile,
|
||||
"edge_defaults": edge_defaults,
|
||||
}
|
||||
|
||||
# Tags / Keywords übernehmen
|
||||
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
|
||||
if tags: payload["tags"] = _ensure_list(tags)
|
||||
for k in ("created","modified","date"):
|
||||
v = fm.get(k) or n.get(k)
|
||||
if v: payload[k] = str(v)
|
||||
if tags:
|
||||
payload["tags"] = _ensure_list(tags)
|
||||
|
||||
# Zeitliche Metadaten (sofern vorhanden)
|
||||
for k in ("created", "modified", "date"):
|
||||
v = fm.get(k) or n.get(k)
|
||||
if v:
|
||||
payload[k] = str(v)
|
||||
|
||||
# JSON-Roundtrip zur harten Validierung (ASCII beibehalten)
|
||||
json.loads(json.dumps(payload, ensure_ascii=False))
|
||||
|
||||
return payload
|
||||
+147
-92
@@ -1,124 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: app/core/qdrant.py
|
||||
Version: v1.4.0 (2025-09-09)
|
||||
app/core/qdrant.py
|
||||
Version: 2.2.0 (2025-11-11)
|
||||
|
||||
Kurzbeschreibung:
|
||||
Qdrant-Client & Collection-Setup für mindnet.
|
||||
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
|
||||
- Edges-Collection nutzt 1D Dummy-Vektor.
|
||||
- NEW: ensure_payload_indexes(...) legt sinnvolle Payload-Indizes an.
|
||||
Aufgabe
|
||||
-------
|
||||
- Zentraler Qdrant-Zugriff (Client, Config)
|
||||
- Collection-Anlage (notes/chunks/edges)
|
||||
- **Payload-Indizes sicherstellen** (idempotent)
|
||||
|
||||
Aufruf:
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
||||
Hinweis
|
||||
-------
|
||||
Diese Datei ist als Drop-in-Ersatz gedacht, falls in deinem Projekt noch keine
|
||||
robuste ensure_payload_indexes()-Implementierung vorliegt. Die Signaturen
|
||||
bleiben kompatibel zu scripts.import_markdown und scripts.reset_qdrant.
|
||||
|
||||
API-Notizen
|
||||
-----------
|
||||
- Payload-Indizes werden mit `create_payload_index` angelegt.
|
||||
- Typen stammen aus `qdrant_client.http.models.PayloadSchemaType`:
|
||||
KEYWORD | TEXT | INTEGER | FLOAT | BOOL | GEO | DATETIME
|
||||
- Für häufige Filterfelder (note_id, kind, scope, type, tags, ...) legen wir
|
||||
Indizes an. Das ist laut Qdrant-Doku Best Practice für performante Filter.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Tuple
|
||||
from typing import Optional, Tuple, Dict, List
|
||||
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Konfiguration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class QdrantConfig:
|
||||
url: str
|
||||
api_key: Optional[str]
|
||||
prefix: str
|
||||
dim: int
|
||||
host: Optional[str] = None
|
||||
port: Optional[int] = None
|
||||
url: Optional[str] = None
|
||||
api_key: Optional[str] = None
|
||||
prefix: str = "mindnet"
|
||||
dim: int = 384
|
||||
distance: str = "Cosine" # Cosine | Dot | Euclid
|
||||
on_disk_payload: bool = True
|
||||
|
||||
@staticmethod
|
||||
def from_env() -> "QdrantConfig":
|
||||
url = os.getenv("QDRANT_URL")
|
||||
if not url:
|
||||
host = os.getenv("QDRANT_HOST", "127.0.0.1")
|
||||
port = int(os.getenv("QDRANT_PORT", "6333"))
|
||||
url = f"http://{host}:{port}"
|
||||
@classmethod
|
||||
def from_env(cls) -> "QdrantConfig":
|
||||
# Entweder URL ODER Host/Port, API-Key optional
|
||||
url = os.getenv("QDRANT_URL") or None
|
||||
host = os.getenv("QDRANT_HOST") or None
|
||||
port = os.getenv("QDRANT_PORT")
|
||||
port = int(port) if port else None
|
||||
api_key = os.getenv("QDRANT_API_KEY") or None
|
||||
prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
|
||||
dim = int(os.getenv("VECTOR_DIM", "384"))
|
||||
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
|
||||
prefix = os.getenv("COLLECTION_PREFIX") or "mindnet"
|
||||
dim = int(os.getenv("VECTOR_DIM") or 384)
|
||||
distance = os.getenv("DISTANCE", "Cosine")
|
||||
on_disk_payload = (os.getenv("ON_DISK_PAYLOAD", "true").lower() == "true")
|
||||
return cls(
|
||||
host=host, port=port, url=url, api_key=api_key,
|
||||
prefix=prefix, dim=dim, distance=distance, on_disk_payload=on_disk_payload
|
||||
)
|
||||
|
||||
|
||||
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
|
||||
# QdrantClient akzeptiert entweder url=... oder host/port
|
||||
if cfg.url:
|
||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key, timeout=60.0)
|
||||
return QdrantClient(host=cfg.host or "127.0.0.1", port=cfg.port or 6333, api_key=cfg.api_key, timeout=60.0)
|
||||
|
||||
|
||||
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||
)
|
||||
|
||||
def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||
)
|
||||
|
||||
def _create_edges(client: QdrantClient, name: str) -> None:
|
||||
if not client.collection_exists(name):
|
||||
client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT), # 1D-Dummy
|
||||
)
|
||||
|
||||
|
||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
|
||||
notes = f"{prefix}_notes"
|
||||
chunks = f"{prefix}_chunks"
|
||||
edges = f"{prefix}_edges"
|
||||
|
||||
_create_notes(client, notes, dim)
|
||||
_create_chunks(client, chunks, dim)
|
||||
|
||||
if client.collection_exists(edges):
|
||||
try:
|
||||
info = client.get_collection(edges)
|
||||
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
|
||||
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
|
||||
except Exception:
|
||||
has_vectors = True
|
||||
if not has_vectors:
|
||||
if destructive:
|
||||
client.delete_collection(edges)
|
||||
_create_edges(client, edges)
|
||||
else:
|
||||
print(f"[ensure_collections] WARN: '{edges}' ohne VectorConfig; destructive=False.", flush=True)
|
||||
else:
|
||||
_create_edges(client, edges)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Collections
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def collection_names(prefix: str) -> Tuple[str, str, str]:
|
||||
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
|
||||
# -------------------------------
|
||||
# NEW: Payload-Indexing
|
||||
# -------------------------------
|
||||
def _vector_params(dim: int, distance: str) -> rest.VectorParams:
|
||||
# Distance: "Cosine" | "Dot" | "Euclid"
|
||||
dist = getattr(rest.Distance, distance.capitalize(), rest.Distance.COSINE)
|
||||
return rest.VectorParams(size=dim, distance=dist)
|
||||
|
||||
def _safe_create_index(client: QdrantClient, col: str, field: str, schema: rest.PayloadSchemaType):
|
||||
try:
|
||||
client.create_payload_index(
|
||||
collection_name=col,
|
||||
field_name=field,
|
||||
field_schema=schema,
|
||||
|
||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int) -> None:
|
||||
"""Legt mindnet_notes, mindnet_chunks, mindnet_edges an (falls nicht vorhanden)."""
|
||||
notes, chunks, edges = collection_names(prefix)
|
||||
|
||||
# notes
|
||||
if not client.collection_exists(notes):
|
||||
client.create_collection(
|
||||
collection_name=notes,
|
||||
vectors_config=_vector_params(dim, os.getenv("DISTANCE", "Cosine")),
|
||||
on_disk_payload=True,
|
||||
)
|
||||
except Exception:
|
||||
# bereits vorhanden oder nicht unterstütztes Schema → ignorieren
|
||||
pass
|
||||
# chunks
|
||||
if not client.collection_exists(chunks):
|
||||
client.create_collection(
|
||||
collection_name=chunks,
|
||||
vectors_config=_vector_params(dim, os.getenv("DISTANCE", "Cosine")),
|
||||
on_disk_payload=True,
|
||||
)
|
||||
# edges (Dummy-Vektor, Filter via Payload)
|
||||
if not client.collection_exists(edges):
|
||||
client.create_collection(
|
||||
collection_name=edges,
|
||||
vectors_config=_vector_params(1, "Dot"),
|
||||
on_disk_payload=True,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Payload-Indizes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _ensure_index(client: QdrantClient, collection: str, field: str, schema: rest.PayloadSchemaType) -> None:
|
||||
"""Idempotentes Anlegen eines Payload-Indexes für ein Feld."""
|
||||
try:
|
||||
client.create_payload_index(collection_name=collection, field_name=field, field_schema=schema, wait=True)
|
||||
except Exception as e:
|
||||
# Fehler ignorieren, falls Index bereits existiert oder Server "already indexed" meldet.
|
||||
# Für Debugging ggf. Logging ergänzen.
|
||||
_ = e
|
||||
|
||||
|
||||
def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
|
||||
"""
|
||||
Stellt sicher, dass alle benötigten Payload-Indizes existieren.
|
||||
- notes: note_id(KEYWORD), type(KEYWORD), title(TEXT), updated(INTEGER), tags(KEYWORD)
|
||||
- chunks: note_id(KEYWORD), chunk_id(KEYWORD), index(INTEGER), type(KEYWORD), tags(KEYWORD)
|
||||
- edges: note_id(KEYWORD), kind(KEYWORD), scope(KEYWORD), source_id(KEYWORD), target_id(KEYWORD), chunk_id(KEYWORD)
|
||||
"""
|
||||
notes, chunks, edges = collection_names(prefix)
|
||||
# Notes
|
||||
_safe_create_index(client, notes, "note_id", rest.PayloadSchemaType.KEYWORD)
|
||||
# Chunks
|
||||
_safe_create_index(client, chunks, "note_id", rest.PayloadSchemaType.KEYWORD)
|
||||
_safe_create_index(client, chunks, "chunk_index", rest.PayloadSchemaType.INTEGER)
|
||||
# Edges
|
||||
for f in ("kind", "scope", "source_id", "target_id", "note_id"):
|
||||
_safe_create_index(client, edges, f, rest.PayloadSchemaType.KEYWORD)
|
||||
|
||||
# NOTES
|
||||
for field, schema in [
|
||||
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("type", rest.PayloadSchemaType.KEYWORD),
|
||||
("title", rest.PayloadSchemaType.TEXT),
|
||||
("updated", rest.PayloadSchemaType.INTEGER),
|
||||
("tags", rest.PayloadSchemaType.KEYWORD),
|
||||
]:
|
||||
_ensure_index(client, notes, field, schema)
|
||||
|
||||
# CHUNKS
|
||||
for field, schema in [
|
||||
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("chunk_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("index", rest.PayloadSchemaType.INTEGER),
|
||||
("type", rest.PayloadSchemaType.KEYWORD),
|
||||
("tags", rest.PayloadSchemaType.KEYWORD),
|
||||
]:
|
||||
_ensure_index(client, chunks, field, schema)
|
||||
|
||||
# EDGES
|
||||
for field, schema in [
|
||||
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("kind", rest.PayloadSchemaType.KEYWORD),
|
||||
("scope", rest.PayloadSchemaType.KEYWORD),
|
||||
("source_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("target_id", rest.PayloadSchemaType.KEYWORD),
|
||||
("chunk_id", rest.PayloadSchemaType.KEYWORD),
|
||||
]:
|
||||
_ensure_index(client, edges, field, schema)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QdrantConfig",
|
||||
"get_client",
|
||||
"ensure_collections",
|
||||
"ensure_payload_indexes",
|
||||
"collection_names",
|
||||
]
|
||||
@@ -217,3 +217,84 @@ def search_chunks_by_vector(client: QdrantClient, prefix: str, vector: List[floa
|
||||
for r in res:
|
||||
out.append((str(r.id), float(r.score), dict(r.payload or {})))
|
||||
return out
|
||||
|
||||
|
||||
# --- Edge retrieval helper ---
|
||||
|
||||
def get_edges_for_sources(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
source_ids: Iterable[str],
|
||||
edge_types: Optional[Iterable[str]] = None,
|
||||
limit: int = 2048,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Retrieve edge payloads from the <prefix>_edges collection.
|
||||
|
||||
Args:
|
||||
client: QdrantClient instance.
|
||||
prefix: Mindnet collection prefix (e.g. "mindnet").
|
||||
source_ids: Iterable of source_id values (typically chunk_ids or note_ids).
|
||||
edge_types: Optional iterable of edge kinds (e.g. ["references", "depends_on"]). If None,
|
||||
all kinds are returned.
|
||||
limit: Maximum number of edge payloads to return.
|
||||
|
||||
Returns:
|
||||
A list of edge payload dicts, e.g.:
|
||||
{
|
||||
"note_id": "...",
|
||||
"chunk_id": "...",
|
||||
"kind": "references" | "depends_on" | ...,
|
||||
"scope": "chunk",
|
||||
"source_id": "...",
|
||||
"target_id": "...",
|
||||
"rule_id": "...",
|
||||
"confidence": 0.7,
|
||||
...
|
||||
}
|
||||
"""
|
||||
source_ids = list(source_ids)
|
||||
if not source_ids or limit <= 0:
|
||||
return []
|
||||
|
||||
# Resolve collection name
|
||||
_, _, edges_col = _names(prefix)
|
||||
|
||||
# Build filter: source_id IN source_ids
|
||||
src_filter = _filter_any("source_id", [str(s) for s in source_ids])
|
||||
|
||||
# Optional: kind IN edge_types
|
||||
kind_filter = None
|
||||
if edge_types:
|
||||
kind_filter = _filter_any("kind", [str(k) for k in edge_types])
|
||||
|
||||
flt = _merge_filters(src_filter, kind_filter)
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
next_page = None
|
||||
remaining = int(limit)
|
||||
|
||||
# Use paginated scroll API; we don't need vectors, only payloads.
|
||||
while remaining > 0:
|
||||
batch_limit = min(256, remaining)
|
||||
res, next_page = client.scroll(
|
||||
collection_name=edges_col,
|
||||
scroll_filter=flt,
|
||||
limit=batch_limit,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
offset=next_page,
|
||||
)
|
||||
|
||||
if not res:
|
||||
break
|
||||
|
||||
for r in res:
|
||||
out.append(dict(r.payload or {}))
|
||||
remaining -= 1
|
||||
if remaining <= 0:
|
||||
break
|
||||
|
||||
if next_page is None or remaining <= 0:
|
||||
break
|
||||
|
||||
return out
|
||||
+277
-98
@@ -1,121 +1,300 @@
|
||||
"""
|
||||
app/core/retriever.py — Semantischer/Edge-Aware/Hybrid Retriever (WP-04)
|
||||
|
||||
Zweck:
|
||||
Kandidatenfindung via Vektorsuche in *_chunks, optionale Edge-Expansion
|
||||
und kombiniertes Ranking zur Rückgabe von Top-K Treffern.
|
||||
Erweiterung (0.2.0): Text→Embedding, falls kein query_vector übergeben wurde.
|
||||
|
||||
Kompatibilität:
|
||||
Python 3.12+, qdrant-client 1.x
|
||||
Version:
|
||||
0.2.0 (Text→Embedding ergänzt; bestehendes Verhalten unverändert)
|
||||
Stand:
|
||||
2025-10-07
|
||||
Bezug:
|
||||
- app/core/graph_adapter.py (expand)
|
||||
- app/core/ranking.py (combine_scores)
|
||||
- app/core/qdrant_points.py (search_chunks_by_vector)
|
||||
- app/services/embeddings_client.py (embed_text)
|
||||
- app/models/dto.py (QueryRequest/Response)
|
||||
Änderungsverlauf:
|
||||
0.2.0 (2025-10-07) – Text→Embedding (embed_text_if_needed).
|
||||
0.1.0 (2025-10-07) – Erstanlage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from qdrant_client import QdrantClient
|
||||
from functools import lru_cache
|
||||
from typing import Any, Dict, List, Tuple, Iterable
|
||||
|
||||
from app.models.dto import QueryRequest, QueryResponse, QueryHit
|
||||
from app.core.ranking import combine_scores
|
||||
from app.core.graph_adapter import expand
|
||||
from app.core import qdrant_points as qp
|
||||
from app.config import get_settings
|
||||
from app.services.embeddings_client import embed_text
|
||||
from app.models.dto import QueryRequest, QueryResponse, QueryHit
|
||||
import app.core.qdrant as qdr
|
||||
import app.core.qdrant_points as qp
|
||||
import app.services.embeddings_client as ec
|
||||
import app.core.graph_adapter as ga
|
||||
|
||||
try:
|
||||
import yaml # type: ignore[import]
|
||||
except Exception: # pragma: no cover - Fallback, falls PyYAML nicht installiert ist
|
||||
yaml = None # type: ignore[assignment]
|
||||
|
||||
|
||||
def _vector_from_request(req: QueryRequest) -> List[float]:
|
||||
@lru_cache
|
||||
def _get_scoring_weights() -> Tuple[float, float, float]:
|
||||
"""Liefert (semantic_weight, edge_weight, centrality_weight) für den Retriever.
|
||||
|
||||
Priorität:
|
||||
1. Werte aus config/retriever.yaml (falls vorhanden und gültig),
|
||||
Abschnitt:
|
||||
scoring:
|
||||
semantic_weight: 1.0
|
||||
edge_weight: 0.5
|
||||
centrality_weight: 0.5
|
||||
2. Fallback auf Settings.RETRIEVER_W_* (ENV-basiert).
|
||||
"""
|
||||
Query-Vektor bestimmen:
|
||||
- Falls query_vector gesetzt: unverändert verwenden (Back-compat, Tests).
|
||||
- Sonst, falls query gesetzt: serverseitig einbetten.
|
||||
- Andernfalls: Fehler.
|
||||
settings = get_settings()
|
||||
sem = float(getattr(settings, "RETRIEVER_W_SEM", 1.0))
|
||||
edge = float(getattr(settings, "RETRIEVER_W_EDGE", 0.0))
|
||||
cent = float(getattr(settings, "RETRIEVER_W_CENT", 0.0))
|
||||
|
||||
# YAML-Override, falls konfiguriert
|
||||
config_path = os.getenv("MINDNET_RETRIEVER_CONFIG", "config/retriever.yaml")
|
||||
if yaml is None:
|
||||
return sem, edge, cent
|
||||
try:
|
||||
if os.path.exists(config_path):
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
scoring = data.get("scoring", {}) or {}
|
||||
sem = float(scoring.get("semantic_weight", sem))
|
||||
edge = float(scoring.get("edge_weight", edge))
|
||||
cent = float(scoring.get("centrality_weight", cent))
|
||||
except Exception:
|
||||
# Bei Fehlern in der YAML-Konfiguration defensiv auf Defaults zurückfallen
|
||||
return sem, edge, cent
|
||||
return sem, edge, cent
|
||||
|
||||
|
||||
def _get_client_and_prefix() -> Tuple[Any, str]:
|
||||
"""Liefert (QdrantClient, prefix) basierend auf QdrantConfig.from_env()."""
|
||||
cfg = qdr.QdrantConfig.from_env()
|
||||
client = qdr.get_client(cfg)
|
||||
return client, cfg.prefix
|
||||
|
||||
|
||||
def _get_query_vector(req: QueryRequest) -> List[float]:
|
||||
"""
|
||||
Liefert den Query-Vektor aus dem Request.
|
||||
|
||||
- Falls req.query_vector gesetzt ist, wird dieser unverändert genutzt.
|
||||
- Andernfalls wird req.query über den Embedding-Service in einen Vektor
|
||||
transformiert.
|
||||
"""
|
||||
if req.query_vector:
|
||||
return req.query_vector
|
||||
if req.query:
|
||||
return embed_text(req.query)
|
||||
raise ValueError("query_vector fehlt. Alternativ 'query' (Text) übergeben, wird serverseitig eingebettet.")
|
||||
return list(req.query_vector)
|
||||
|
||||
if not req.query:
|
||||
raise ValueError("QueryRequest benötigt entweder query oder query_vector")
|
||||
|
||||
settings = get_settings()
|
||||
model_name = settings.MODEL_NAME
|
||||
|
||||
# Kompatibel mit Fakes in Unit-Tests (ohne model_name-Parameter)
|
||||
try:
|
||||
return ec.embed_text(req.query, model_name=model_name) # type: ignore[call-arg]
|
||||
except TypeError:
|
||||
# Fallback: einfache Signatur embed_text(text)
|
||||
return ec.embed_text(req.query) # type: ignore[call-arg]
|
||||
|
||||
|
||||
def _semantic_hits(
|
||||
client: Any,
|
||||
prefix: str,
|
||||
vector: List[float],
|
||||
top_k: int,
|
||||
filters: Dict[str, Any] | None = None,
|
||||
) -> List[Tuple[str, float, Dict[str, Any]]]:
|
||||
"""Führt eine semantische Suche über mindnet_chunks aus und liefert Roh-Treffer.
|
||||
|
||||
Rückgabeformat: Liste von (point_id, score, payload)
|
||||
|
||||
Erwartetes Format von qp.search_chunks_by_vector:
|
||||
List[Tuple[str, float, dict]]
|
||||
"""
|
||||
flt = filters or None
|
||||
raw_hits = qp.search_chunks_by_vector(client, prefix, vector, top=top_k, filters=flt)
|
||||
results: List[Tuple[str, float, Dict[str, Any]]] = []
|
||||
for pid, score, payload in raw_hits:
|
||||
results.append((str(pid), float(score), dict(payload or {})))
|
||||
return results
|
||||
|
||||
|
||||
def _get_semantic_score_and_payload(
|
||||
hit: Tuple[str, float, Dict[str, Any]]
|
||||
) -> Tuple[float, Dict[str, Any]]:
|
||||
"""Extrahiert semantic_score und Payload aus einem Raw-Hit."""
|
||||
_, score, payload = hit
|
||||
return float(score), payload or {}
|
||||
|
||||
|
||||
def _compute_total_score(
|
||||
semantic_score: float,
|
||||
payload: Dict[str, Any],
|
||||
edge_bonus: float = 0.0,
|
||||
cent_bonus: float = 0.0,
|
||||
) -> Tuple[float, float, float]:
|
||||
"""Berechnet total_score aus semantic_score, retriever_weight und Graph-Boni.
|
||||
|
||||
Formel (WP-04, konfigurierbar über config/retriever.yaml bzw. ENV):
|
||||
total_score = W_sem * semantic_score * max(retriever_weight, 0.0)
|
||||
+ W_edge * edge_bonus
|
||||
+ W_cent * cent_bonus
|
||||
|
||||
- retriever_weight stammt aus dem Chunk-Payload (types.yaml).
|
||||
- W_sem / W_edge / W_cent kommen aus _get_scoring_weights().
|
||||
"""
|
||||
raw_weight = payload.get("retriever_weight", 1.0)
|
||||
try:
|
||||
weight = float(raw_weight)
|
||||
except (TypeError, ValueError):
|
||||
weight = 1.0
|
||||
if weight < 0.0:
|
||||
weight = 0.0
|
||||
|
||||
sem_w, edge_w, cent_w = _get_scoring_weights()
|
||||
total = (sem_w * float(semantic_score) * weight) + (edge_w * edge_bonus) + (cent_w * cent_bonus)
|
||||
return float(total), float(edge_bonus), float(cent_bonus)
|
||||
|
||||
|
||||
def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
|
||||
"""Extrahiert depth und edge_types aus req.expand, falls vorhanden.
|
||||
|
||||
- Falls expand nicht gesetzt ist: depth=0, edge_types=None (keine Expansion).
|
||||
- Unterstützt sowohl Pydantic-Modelle als auch plain dicts.
|
||||
"""
|
||||
expand = getattr(req, "expand", None)
|
||||
if not expand:
|
||||
return 0, None
|
||||
|
||||
depth = 1
|
||||
edge_types: List[str] | None = None
|
||||
|
||||
# Pydantic-Modell oder Objekt mit Attributen
|
||||
if hasattr(expand, "depth") or hasattr(expand, "edge_types"):
|
||||
depth = int(getattr(expand, "depth", 1) or 1)
|
||||
types_val = getattr(expand, "edge_types", None)
|
||||
if types_val:
|
||||
edge_types = list(types_val)
|
||||
return depth, edge_types
|
||||
|
||||
# Plain dict
|
||||
if isinstance(expand, dict):
|
||||
if "depth" in expand:
|
||||
depth = int(expand.get("depth") or 1)
|
||||
if "edge_types" in expand and expand["edge_types"] is not None:
|
||||
edge_types = list(expand["edge_types"])
|
||||
return depth, edge_types
|
||||
|
||||
return 0, None
|
||||
|
||||
|
||||
def _build_hits_from_semantic(
|
||||
hits: Iterable[Tuple[str, float, Dict[str, Any]]],
|
||||
top_k: int,
|
||||
used_mode: str,
|
||||
subgraph: ga.Subgraph | None = None,
|
||||
) -> QueryResponse:
|
||||
"""Baut aus Raw-Hits und optionalem Subgraph strukturierte QueryHits.
|
||||
|
||||
- Aggregation auf Note-Ebene (note_id),
|
||||
- Berechnung von total_score unter Nutzung von retriever_weight + Graph-Boni,
|
||||
- Rückgabe als QueryResponse mit Latenz.
|
||||
"""
|
||||
t0 = time.time()
|
||||
|
||||
enriched: List[Tuple[str, float, Dict[str, Any], float, float, float]] = []
|
||||
|
||||
for pid, semantic_score, payload in hits:
|
||||
# Graph-Scores, falls Subgraph und stabiler Key vorhanden
|
||||
edge_bonus = 0.0
|
||||
cent_bonus = 0.0
|
||||
if subgraph is not None:
|
||||
node_key = payload.get("chunk_id") or payload.get("note_id")
|
||||
if node_key:
|
||||
try:
|
||||
edge_bonus = float(subgraph.edge_bonus(node_key))
|
||||
except Exception:
|
||||
edge_bonus = 0.0
|
||||
try:
|
||||
cent_bonus = float(subgraph.centrality_bonus(node_key))
|
||||
except Exception:
|
||||
cent_bonus = 0.0
|
||||
|
||||
total, edge_bonus, cent_bonus = _compute_total_score(
|
||||
semantic_score,
|
||||
payload,
|
||||
edge_bonus=edge_bonus,
|
||||
cent_bonus=cent_bonus,
|
||||
)
|
||||
enriched.append((pid, float(semantic_score), payload, total, edge_bonus, cent_bonus))
|
||||
|
||||
# Sortierung nach total_score absteigend
|
||||
enriched_sorted = sorted(enriched, key=lambda h: h[3], reverse=True)
|
||||
limited = enriched_sorted[: max(1, top_k)]
|
||||
|
||||
results: List[QueryHit] = []
|
||||
for pid, semantic_score, payload, total, edge_bonus, cent_bonus in limited:
|
||||
note_id = payload.get("note_id")
|
||||
path = payload.get("path")
|
||||
# mindnet_chunks: section (aktuell genutzt); ältere Stände nutzen ggf. section_title
|
||||
section = payload.get("section") or payload.get("section_title")
|
||||
|
||||
results.append(
|
||||
QueryHit(
|
||||
node_id=str(pid),
|
||||
note_id=note_id,
|
||||
semantic_score=float(semantic_score),
|
||||
edge_bonus=edge_bonus,
|
||||
centrality_bonus=cent_bonus,
|
||||
total_score=total,
|
||||
paths=None,
|
||||
source={
|
||||
"path": path,
|
||||
"section": section,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
dt = int((time.time() - t0) * 1000)
|
||||
return QueryResponse(results=results, used_mode=used_mode, latency_ms=dt)
|
||||
|
||||
|
||||
def semantic_retrieve(req: QueryRequest) -> QueryResponse:
|
||||
"""Nur semantische Kandidaten, keine Edge-Expansion (depth=0)."""
|
||||
t0 = time.time()
|
||||
s = get_settings()
|
||||
client = QdrantClient(url=s.QDRANT_URL, api_key=s.QDRANT_API_KEY)
|
||||
"""Reiner semantischer Retriever (ohne Edge-Expansion)."""
|
||||
client, prefix = _get_client_and_prefix()
|
||||
vector = _get_query_vector(req)
|
||||
top_k = req.top_k or get_settings().RETRIEVER_TOP_K
|
||||
|
||||
q_vec = _vector_from_request(req)
|
||||
raw_hits = qp.search_chunks_by_vector(client, s.COLLECTION_PREFIX, q_vec, top=req.top_k, filters=req.filters)
|
||||
|
||||
results: List[QueryHit] = []
|
||||
for pid, s_score, payload in raw_hits:
|
||||
results.append(QueryHit(
|
||||
node_id=pid,
|
||||
note_id=payload.get("note_id"),
|
||||
semantic_score=float(s_score),
|
||||
edge_bonus=0.0,
|
||||
centrality_bonus=0.0,
|
||||
total_score=float(s_score), # un-normalisiert: ok für quick semantic mode
|
||||
paths=None,
|
||||
source={"path": payload.get("path"), "section": payload.get("section_title")}
|
||||
))
|
||||
dt = int((time.time() - t0) * 1000)
|
||||
return QueryResponse(results=results, used_mode="semantic", latency_ms=dt)
|
||||
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
||||
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="semantic", subgraph=None)
|
||||
|
||||
|
||||
def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
||||
"""Semantik + Edge-Expansion + kombiniertes Ranking."""
|
||||
t0 = time.time()
|
||||
s = get_settings()
|
||||
client = QdrantClient(url=s.QDRANT_URL, api_key=s.QDRANT_API_KEY)
|
||||
"""Hybrid-Retriever: semantische Suche + optionale Edge-Expansion.
|
||||
|
||||
q_vec = _vector_from_request(req)
|
||||
Aktueller Stand (Step 3):
|
||||
- Semantische Suche über mindnet_chunks (wie semantic_retrieve),
|
||||
- Bei expand.depth > 0 wird ein Subgraph aus mindnet_edges aufgebaut,
|
||||
und edge_bonus / centrality_bonus in das Scoring einbezogen.
|
||||
"""
|
||||
client, prefix = _get_client_and_prefix()
|
||||
|
||||
# 1) Semantische Seeds (top_k * 3 für breitere Basis)
|
||||
raw_hits = qp.search_chunks_by_vector(client, s.COLLECTION_PREFIX, q_vec, top=req.top_k * 3, filters=req.filters)
|
||||
id2payload = {pid: payload for (pid, _, payload) in raw_hits}
|
||||
seeds = [pid for (pid, _, _) in raw_hits]
|
||||
# Query-Vektor:
|
||||
# - Falls explizit gesetzt, unverändert nutzen.
|
||||
# - Andernfalls über embed_text erzeugen.
|
||||
if req.query_vector:
|
||||
vector = list(req.query_vector)
|
||||
else:
|
||||
vector = _get_query_vector(req)
|
||||
|
||||
# 2) Edge-Expansion
|
||||
edge_types = req.expand.get("edge_types") if req.expand else None
|
||||
depth = req.expand.get("depth", 1) if req.expand else 1
|
||||
sg = expand(client, s.COLLECTION_PREFIX, seeds, depth=depth, edge_types=edge_types)
|
||||
top_k = req.top_k or get_settings().RETRIEVER_TOP_K
|
||||
|
||||
edge_bonus_map = {pid: sg.aggregate_edge_bonus(pid) for pid in seeds}
|
||||
centrality_map = {pid: sg.centrality_bonus(pid) for pid in seeds}
|
||||
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
||||
|
||||
# 3) Combined Ranking
|
||||
scored = combine_scores(
|
||||
raw_hits, edge_bonus_map, centrality_map,
|
||||
w_sem=s.RETRIEVER_W_SEM, w_edge=s.RETRIEVER_W_EDGE, w_cent=s.RETRIEVER_W_CENT
|
||||
)
|
||||
depth, edge_types = _extract_expand_options(req)
|
||||
subgraph: ga.Subgraph | None = None
|
||||
if depth and depth > 0:
|
||||
# Seeds: stabile IDs aus dem Payload (chunk_id bevorzugt, sonst note_id)
|
||||
seed_ids: List[str] = []
|
||||
for _pid, _score, payload in hits:
|
||||
key = payload.get("chunk_id") or payload.get("note_id")
|
||||
if key and key not in seed_ids:
|
||||
seed_ids.append(key)
|
||||
|
||||
# 4) Antwortobjekte (Chunk-Ebene)
|
||||
results: List[QueryHit] = []
|
||||
for pid, total, e, c, s_score in scored[: req.top_k]:
|
||||
payload = id2payload[pid]
|
||||
results.append(QueryHit(
|
||||
node_id=pid,
|
||||
note_id=payload.get("note_id"),
|
||||
semantic_score=float(s_score),
|
||||
edge_bonus=float(e),
|
||||
centrality_bonus=float(c),
|
||||
total_score=float(total),
|
||||
paths=None,
|
||||
source={"path": payload.get("path"), "section": payload.get("section_title")}
|
||||
))
|
||||
if seed_ids:
|
||||
try:
|
||||
subgraph = ga.expand(client, prefix, seed_ids, depth=depth, edge_types=edge_types)
|
||||
except Exception:
|
||||
# Edge-Expansion ist optional: bei Fehlern weiter ohne Graph-Boni
|
||||
subgraph = None
|
||||
|
||||
dt = int((time.time() - t0) * 1000)
|
||||
return QueryResponse(results=results, used_mode="hybrid", latency_ms=dt)
|
||||
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="hybrid", subgraph=subgraph)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""app/core/retriever_config.py
|
||||
---------------------------------
|
||||
Zentrale Konfiguration für den mindnet-Retriever (WP-04).
|
||||
|
||||
Zweck:
|
||||
- Lädt config/retriever.yaml (falls vorhanden) oder nutzt sinnvolle Defaults.
|
||||
- Bietet einen gecachten Zugriff auf die Retriever-Config für
|
||||
andere Module (z. B. graph_adapter, retriever).
|
||||
|
||||
Hinweis zur Weiterentwicklung (Selbstjustierung):
|
||||
- Die hier definierten Parameter sind so gewählt, dass sie später
|
||||
durch ein Feedback-/Learning-to-Rank-Modell überschrieben werden
|
||||
können, ohne die restliche Architektur anzupassen.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
try:
|
||||
import yaml # type: ignore
|
||||
except Exception: # pragma: no cover - Fallback, falls PyYAML nicht installiert ist.
|
||||
yaml = None # type: ignore
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RetrieverConfig:
|
||||
semantic_scale: float
|
||||
edge_scale: float
|
||||
centrality_scale: float
|
||||
edge_weights: Dict[str, float]
|
||||
|
||||
@lru_cache
|
||||
def get_retriever_config() -> RetrieverConfig:
|
||||
"""Lädt die Retriever-Konfiguration (YAML + Defaults).
|
||||
|
||||
Reihenfolge:
|
||||
1. Defaults (sinnvoll gewählte Startwerte).
|
||||
2. Optional: config/retriever.yaml bzw. Pfad aus ENV
|
||||
MINDNET_RETRIEVER_CONFIG überschreibt die Defaults.
|
||||
|
||||
Die Funktion ist bewusst gecached, da sich die Konfiguration zur
|
||||
Laufzeit üblicherweise nicht ändert. Für dynamisches Nachladen
|
||||
könnte der Cache explizit geleert werden.
|
||||
"""
|
||||
|
||||
# 1) Defaults – bewusst konservativ gewählt.
|
||||
semantic_scale = 1.0
|
||||
edge_scale = 1.0
|
||||
centrality_scale = 1.0
|
||||
|
||||
edge_weights: Dict[str, float] = {
|
||||
# Wissens-Kanten
|
||||
"depends_on": 1.0,
|
||||
"related_to": 0.7,
|
||||
"similar_to": 0.7,
|
||||
"references": 0.5,
|
||||
# Struktur-Kanten
|
||||
"belongs_to": 0.2,
|
||||
"next": 0.1,
|
||||
"prev": 0.1,
|
||||
# Sonstige / technische Kanten
|
||||
"backlink": 0.2,
|
||||
"references_at": 0.2,
|
||||
}
|
||||
|
||||
# 2) Optional: YAML-Konfiguration laden
|
||||
cfg_path_env = os.getenv("MINDNET_RETRIEVER_CONFIG")
|
||||
if cfg_path_env:
|
||||
cfg_path = Path(cfg_path_env)
|
||||
else:
|
||||
# Project-Root = zwei Ebenen über app/core/
|
||||
cfg_path = Path(__file__).resolve().parents[2] / "config" / "retriever.yaml"
|
||||
|
||||
if yaml is not None and cfg_path.exists():
|
||||
try:
|
||||
data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {}
|
||||
except Exception:
|
||||
data = {}
|
||||
|
||||
retr = data.get("retriever") or {}
|
||||
|
||||
# Skalenwerte überschreiben, falls angegeben
|
||||
try:
|
||||
semantic_scale = float(retr.get("semantic_scale", semantic_scale))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
try:
|
||||
edge_scale = float(retr.get("edge_scale", edge_scale))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
try:
|
||||
centrality_scale = float(retr.get("centrality_scale", centrality_scale))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# Edge-Gewichte je Kanten-Typ
|
||||
ew_cfg = retr.get("edge_weights") or {}
|
||||
if isinstance(ew_cfg, dict):
|
||||
for k, v in ew_cfg.items():
|
||||
try:
|
||||
edge_weights[str(k)] = float(v)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
return RetrieverConfig(
|
||||
semantic_scale=semantic_scale,
|
||||
edge_scale=edge_scale,
|
||||
centrality_scale=centrality_scale,
|
||||
edge_weights=edge_weights,
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
version: 1.0
|
||||
|
||||
scoring:
|
||||
# W_sem: skaliert den Term (semantic_score * retriever_weight)
|
||||
# Empfehlung Startwert: 1.0 → Semantik bleibt Hauptsignal
|
||||
semantic_weight: 1.0
|
||||
|
||||
# W_edge: skaliert edge_bonus aus dem Subgraph
|
||||
# Empfehlung: 0.7 → Graph ist deutlich spürbar, aber überstimmt Semantik nicht komplett
|
||||
edge_weight: 0.7
|
||||
|
||||
# W_cent: skaliert centrality_bonus (Knoten-Zentralität im Subgraph)
|
||||
# Empfehlung: 0.5 → zentrale Knoten werden bevorzugt, aber moderat
|
||||
centrality_weight: 0.5
|
||||
|
||||
# Optional, für später (aktuell noch nicht ausgewertet, aber als Blaupause):
|
||||
edge_types:
|
||||
references: 0.20
|
||||
depends_on: 0.18
|
||||
related_to: 0.15
|
||||
similar_to: 0.12
|
||||
belongs_to: 0.10
|
||||
next: 0.06
|
||||
prev: 0.06
|
||||
+67
-9
@@ -1,18 +1,76 @@
|
||||
version: 1.0
|
||||
|
||||
defaults:
|
||||
retriever_weight: 1.0
|
||||
chunk_profile: default
|
||||
edge_defaults: [] # Leerliste → keine impliziten Kanten, wenn nicht gesetzt
|
||||
|
||||
types:
|
||||
# Wissensbausteine
|
||||
concept:
|
||||
chunk_profile: medium
|
||||
retriever_weight: 0.60
|
||||
edge_defaults: ["references", "related_to"]
|
||||
retriever_weight: 0.33
|
||||
task:
|
||||
|
||||
source:
|
||||
chunk_profile: short
|
||||
edge_defaults: ["depends_on", "belongs_to"]
|
||||
retriever_weight: 0.8
|
||||
experience:
|
||||
retriever_weight: 0.50
|
||||
edge_defaults: [] # Quellen sind i. d. R. Ziel von Kanten, erzeugen selbst keine Default-Kanten
|
||||
|
||||
glossary:
|
||||
chunk_profile: short
|
||||
retriever_weight: 0.40
|
||||
edge_defaults: ["related_to"]
|
||||
|
||||
# Person & Persönlichkeit
|
||||
profile: # z. B. "Persönliches Leitbild (Kurzprofil)"
|
||||
chunk_profile: long
|
||||
retriever_weight: 0.66
|
||||
edge_defaults: ["references", "related_to"]
|
||||
|
||||
value: # Werte (z. B. "Mut", "Fairness")
|
||||
chunk_profile: short
|
||||
retriever_weight: 0.90
|
||||
edge_defaults: ["related_to"]
|
||||
|
||||
principle: # Prinzipien/Leitsätze
|
||||
chunk_profile: short
|
||||
retriever_weight: 0.90
|
||||
edge_defaults: ["related_to", "references"]
|
||||
|
||||
experience: # prägende Erlebnisse/Erfahrungen
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["derived_from", "inspired_by"]
|
||||
retriever_weight: 0.9
|
||||
retriever_weight: 0.90
|
||||
edge_defaults: ["related_to", "references"]
|
||||
|
||||
# Ziele, Entscheidungen, Meilensteine
|
||||
goal:
|
||||
chunk_profile: medium
|
||||
retriever_weight: 0.90
|
||||
edge_defaults: ["depends_on", "related_to", "references"]
|
||||
|
||||
decision:
|
||||
chunk_profile: short
|
||||
retriever_weight: 0.90
|
||||
edge_defaults: ["depends_on", "references", "related_to"] # „Worauf stützte sich die Entscheidung?“
|
||||
|
||||
milestone:
|
||||
chunk_profile: short
|
||||
retriever_weight: 0.70
|
||||
edge_defaults: ["related_to", "references"]
|
||||
|
||||
# Operativ: Projekte, Aufgaben, Journal
|
||||
project:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.95
|
||||
retriever_weight: 0.97
|
||||
edge_defaults: ["references", "depends_on"] # Projekt stützt sich auf Konzepte/Quellen und hängt von Bausteinen ab
|
||||
|
||||
task:
|
||||
chunk_profile: short
|
||||
retriever_weight: 0.80
|
||||
edge_defaults: ["depends_on", "references"]
|
||||
|
||||
journal:
|
||||
chunk_profile: medium
|
||||
retriever_weight: 0.80
|
||||
edge_defaults: ["references", "related_to"]
|
||||
@@ -0,0 +1,96 @@
|
||||
# mindnet Retriever (WP-04 / Step 4a) – Kurz-Dokumentation
|
||||
|
||||
## 1. Überblick
|
||||
|
||||
Der Retriever besteht aus:
|
||||
|
||||
- **Semantic Retriever** (`semantic_retrieve`)
|
||||
- **Hybrid Retriever** (`hybrid_retrieve`)
|
||||
- Qdrant-Backend (`*_chunks`, `*_edges`)
|
||||
- optionaler Edge-Expansion über `graph_adapter.expand`
|
||||
|
||||
Die API nutzt das bestehende `/query`-Endpoint (FastAPI), das `QueryRequest` und `QueryResponse` aus `app.models.dto` verwendet.
|
||||
|
||||
## 2. Scoring-Formel
|
||||
|
||||
Pro Treffer werden folgende Komponenten geführt:
|
||||
|
||||
- `semantic_score` – direkt aus Qdrant (Cosine/Euclidean, je nach Collection)
|
||||
- `retriever_weight` – aus Chunk-Payload (`types.yaml` / Frontmatter)
|
||||
- `edge_bonus` – Bonus aus dem Subgraph (z. B. Zahl der eingehenden/ausgehenden Edges, Beziehungstypen)
|
||||
- `centrality_bonus` – Bonus aus einer einfachen Zentralitäts-Heuristik im Subgraph
|
||||
|
||||
Aktuelle Formel (Step 4a):
|
||||
|
||||
```text
|
||||
total_score = semantic_score * max(retriever_weight, 0.0)
|
||||
+ edge_bonus
|
||||
+ centrality_bonus
|
||||
```
|
||||
|
||||
Die Sortierung der Ergebnisse erfolgt absteigend nach `total_score`.
|
||||
|
||||
## 3. Modi
|
||||
|
||||
### 3.1 Semantic Mode
|
||||
|
||||
- Funktion: `semantic_retrieve`
|
||||
- Request-Feld `mode = "semantic"`
|
||||
- Keine Edge-Expansion, kein Graph-Call.
|
||||
- Kandidaten nur aus Vektorsuche in `<prefix>_chunks`.
|
||||
|
||||
### 3.2 Hybrid Mode
|
||||
|
||||
- Funktion: `hybrid_retrieve`
|
||||
- Request-Feld `mode = "hybrid"`
|
||||
- Schritte:
|
||||
1. Semantische Chunk-Suche wie im Semantic Mode
|
||||
2. Falls `expand.depth > 0` gesetzt ist:
|
||||
- Seeds = `chunk_id` (Fallback: `note_id`) der Treffer
|
||||
- Aufruf `graph_adapter.expand(client, prefix, seed_ids, depth, edge_types)`
|
||||
- Ergebnis-Subgraph liefert `edge_bonus` und `centrality_bonus` pro Node
|
||||
3. Scores werden nach obiger Formel kombiniert.
|
||||
|
||||
## 4. Request-Struktur (JSON)
|
||||
|
||||
Beispiel-Hybrid-Request:
|
||||
|
||||
```json
|
||||
{
|
||||
"mode": "hybrid",
|
||||
"query": "embeddings und qdrant",
|
||||
"top_k": 10,
|
||||
"expand": {
|
||||
"depth": 1,
|
||||
"edge_types": ["references", "depends_on"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Minimaler Semantic-Request:
|
||||
|
||||
```json
|
||||
{
|
||||
"mode": "semantic",
|
||||
"query": "karate trainingsplan",
|
||||
"top_k": 5
|
||||
}
|
||||
```
|
||||
|
||||
## 5. Smoke-Test
|
||||
|
||||
Für einen schnellen End-to-End-Test kann folgendes Skript verwendet werden:
|
||||
|
||||
```bash
|
||||
python scripts/test_retriever_smoke.py --query "karate" --mode hybrid --expand-depth 1 --top-k 5
|
||||
```
|
||||
|
||||
Das Skript ruft `/query` auf, wertet Statuscode und JSON-Antwort aus und zeigt die Scores der Top-K-Treffer an.
|
||||
|
||||
## 6. Tests
|
||||
|
||||
Die wichtigsten Tests für den Retriever sind:
|
||||
|
||||
- `tests/test_retriever_basic.py` – Basisfunktion (Semantic + Hybrid)
|
||||
- `tests/test_retriever_weight.py` – Einfluss von `retriever_weight` auf das Ranking
|
||||
- `tests/test_retriever_edges.py` – Einfluss von Edge-/Centrality-Boni im Hybrid-Modus
|
||||
+155
-188
@@ -1,249 +1,216 @@
|
||||
# mindnet – Knowledge Design
|
||||
**Version:** 1.4.0 (aktualisiert 2025-10-06)
|
||||
**Version:** 1.5.0 (aktualisiert 2025-11-09)
|
||||
|
||||
## 1. Gesamtziel und Systemverständnis
|
||||
|
||||
**mindnet** ist ein lokales Wissensnetzwerk, das alle meine Notizen, Projekte, Gedanken, Erfahrungen und Pläne als Markdown-Dateien speichert, automatisch importiert, semantisch in **Chunks** zerlegt und als **Graph-Struktur in Qdrant** ablegt.
|
||||
Es ist die Grundlage für ein agentenfähiges, lokal betriebenes LLM-System, das meine Daten verknüpft, abfragt und erweitert.
|
||||
|
||||
**Zentrale Ziele**
|
||||
- Vollständige Repräsentation persönlichen Wissens in strukturierter, maschinenlesbarer Form
|
||||
- Reproduzierbare Verarbeitung: Import → Chunking → Embedding → Edges → Graph
|
||||
- Verlustfreie Roundtrip-Fähigkeit (Import ⇄ Export)
|
||||
- Erweiterbarkeit für Agenten, RAG-Retriever und semantische Suche
|
||||
|
||||
**Prinzipien**
|
||||
- **Deterministisch:** IDs, Hashes und Edges wiederholbar
|
||||
- **Portabel:** Markdown + YAML
|
||||
- **Robust:** fehlertoleranter Parser
|
||||
- **Idempotent:** keine Duplikate bei Re-Import
|
||||
- **Erweiterbar:** neue Typen und Edge-Arten ohne Migration
|
||||
> Dieses Dokument beschreibt das Ziel, die Architektur, die Datenmodelle sowie die Regeln und Prozesse für den Aufbau von **mindnet** – einem persönlichen, lokal betriebenen Wissensnetz, das auf Markdown-Quellen, Graph-Beziehungen und Vektor-Suche basiert. Es soll die Persönlichkeit, Werte und Erfahrungen des Besitzers widerspiegeln und künftig Antworten generieren, die diese Perspektive authentisch berücksichtigen.
|
||||
|
||||
---
|
||||
|
||||
## 2. YAML-Frontmatter-Schema
|
||||
## 1. Ziel von mindnet
|
||||
|
||||
| Feld | Typ | Beschreibung |
|
||||
|-------|-----|--------------|
|
||||
| `title` | string | Menschlich lesbarer Titel |
|
||||
| `id` | string | Eindeutige Note-ID (Slug) |
|
||||
| `type` | enum | `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone` |
|
||||
| `status` | enum | `draft`, `active`, `done`, `archived` |
|
||||
| `created` | date | ISO-Datum |
|
||||
| `updated` | date | Letzte Änderung |
|
||||
| `area` | string | Themen- oder Lebensbereich |
|
||||
| `project` | string | Zugehöriges Projekt |
|
||||
| `tags` | list | Strukturierte Tags (`area/…`, `topic/…`) |
|
||||
| `depends_on` | list | IDs anderer Notizen |
|
||||
| `assigned_to` | list | Beteiligte Personen |
|
||||
| `embedding_exclude` | bool | Falls `true`, keine Embeddings |
|
||||
| `hash_mode` | enum | Hash-Modus beim Import (`body`, `frontmatter`, `full`) |
|
||||
| `priority` | enum/int | 1–5 oder `low` / `med` / `high` |
|
||||
| `effort_min`, `due` | int/date | Aufwand / Termin |
|
||||
| `aliases` | list | Alternative IDs |
|
||||
| `lang` | string | ISO-Sprachcode |
|
||||
| `source` | string | Herkunft / Referenzquelle |
|
||||
**1.1 Zweck**
|
||||
- mindnet dient als persönliches Wissensnetz, das Erfahrungen, Werte, Prinzipien, Ziele und prägenden Ereignisse langfristig strukturiert.
|
||||
- Es ermöglicht spätere Einsichten in Entscheidungen und Entwicklungen und kann in Zukunft für die Familie (z. B. Kinder) eine nachvollziehbare Quelle sein.
|
||||
|
||||
**Pflichtfelder:** `title`, `id`, `type`, `status`, `created`
|
||||
**Optionale Felder:** alle übrigen
|
||||
**1.2 Leitidee**
|
||||
- Statt einer reinen Dokumentablage wird Wissen durch **Knoten (Notes/Chunks)** und **Kanten (Edges)** vernetzt.
|
||||
- Antworten sollen **persönlich** kontextualisiert werden (Perspektive, Konditionierung, Persönlichkeit).
|
||||
|
||||
**1.3 Langfristigkeit**
|
||||
- Stabiler, zukunftsfähiger Entwurf; spätere Massenmigrationen sollen vermieden werden.
|
||||
- Architektur unterstützt schrittweise Erweiterung (neue Quellen, neue Agenten, neue Regeln).
|
||||
|
||||
---
|
||||
|
||||
## 3. Dateinamen und Ordner
|
||||
## 2. Zielarchitektur (High Level)
|
||||
|
||||
- Format: `YYYY-MM-DD_title.md` oder `slug.md`
|
||||
- Strukturierte Ordner:
|
||||
- `10_thoughts/`, `20_concepts/`, `30_projects/`, `40_experiences/`, …
|
||||
- Alle Pfade relativ; Unterstriche statt Leerzeichen
|
||||
- **Datenquellen:** Markdown-Vault (Obsidian-kompatibel), perspektivisch weitere Quellen (MediaWiki, PDFs, Web-Exporte, E-Mail-Auszüge).
|
||||
- **Import-Pipeline:** Parser → Frontmatter/Body → Chunking → Embedding → Qdrant (Notes/Chunks/Edges).
|
||||
- **Speicher:** Qdrant (Vektoren + Payload + Textindex), Dateien bleiben im Filesystem (Quelle ist Referenz).
|
||||
- **Graph-Schicht:** Edges explizit modelliert (Typ, Richtung, Evidenz, Confidence); optionale abgeleitete Edges per Batch-Job.
|
||||
- **Retriever/Composer:** Hybrid-Retrieval (Vektor + Filter + optional Volltext), Komposition zu Antwortkandidaten.
|
||||
- **Erklärbarkeit:** Begründungspfade (Top-Chunks, Graph-Kette, Provenienz).
|
||||
- **Schnittstellen:** REST/CLI, n8n-Glue, lokale LLM-Anbindung (Ollama).
|
||||
|
||||
---
|
||||
|
||||
## 4. Link- und Edge-Design
|
||||
## 3. Collections in Qdrant
|
||||
|
||||
**Verlinkungen**
|
||||
- Wikilinks: `[[note-id]]` oder `[[Titel|note-id]]`
|
||||
- Automatische Edges aus Text und Frontmatter
|
||||
|
||||
**Edge-Typen in Qdrant**
|
||||
|
||||
| Typ | Bedeutung | Quelle | Scope |
|
||||
|------|------------|--------|--------|
|
||||
| `belongs_to` | Chunk → Note | intern | chunk |
|
||||
| `prev` / `next` | Reihenfolge | intern | chunk |
|
||||
| `references` | explizite Wikilinks | Text | chunk |
|
||||
| `backlink` | Gegenkante zu `references` | abgeleitet | note |
|
||||
| `depends_on` | YAML-Abhängigkeiten | YAML | note |
|
||||
| `assigned_to` | YAML-Zuweisung | YAML | note |
|
||||
| `unresolved` | Ziel fehlt (Stub) | abgeleitet | chunk |
|
||||
|
||||
**Edge-Schlüssel**
|
||||
- `source_id`, `target_id`, `kind`, `scope`, `note_id`
|
||||
- Dedup-Schlüssel: Kombination aus `(kind, source_id, target_id, scope)`
|
||||
- `mindnet_notes` – Note-Metadaten (id, title, type, path, timestamps, tags, …).
|
||||
- `mindnet_chunks` – inhaltstragende Segmente inkl. Vektoren und Textindex.
|
||||
- `mindnet_edges` – Beziehungen (src/dst, relation, direction, evidence, confidence, …).
|
||||
- (optional) `mindnet_people` – strukturierte Personenobjekte (Eigen-/Fremdbezug).
|
||||
|
||||
---
|
||||
|
||||
## 5. Chunking-Modell
|
||||
## 4. Identitäten & IDs
|
||||
|
||||
**Chunk-Datenstruktur**
|
||||
| Feld | Beschreibung |
|
||||
|-------|---------------|
|
||||
| `note_id` | Zugehörige Note |
|
||||
| `chunk_index` | Laufende Nummer |
|
||||
| `text` | Originalabschnitt |
|
||||
| `window` | Text + Overlap links/rechts |
|
||||
| `overlap_left` | Länge des linken Overlaps |
|
||||
| `overlap_right` | Länge des rechten Overlaps |
|
||||
| `embedding` | 384-dimensionaler Vektor |
|
||||
| `tokens` | Tokenanzahl (optional) |
|
||||
|
||||
**Chunking-Regeln (aus `chunking_strategy.md`)**
|
||||
- Trennung nach Absätzen, Überschriften, Listen, Tabellen
|
||||
- Token-Zielgröße 350–500 (max 600)
|
||||
- Overlap: 30–40 % für semantische Kohärenz
|
||||
- Overlap wird in `window` integriert, sodass `text ≠ window`
|
||||
- **Deterministische IDs** für Notes (z. B. Namespace-basierte UUIDv5) und stabile, ableitbare IDs für Chunks/Edges.
|
||||
- ID-Stabilität ist Voraussetzung für Idempotenz (Re-Imports ohne Duplikate).
|
||||
|
||||
---
|
||||
|
||||
## 6. Hash-Strategie und Änderungsverfolgung
|
||||
## 5. Frontmatter & Typen
|
||||
|
||||
**Zweck:** Nur bei realer Inhaltsänderung re-indexieren
|
||||
|
||||
| Umgebungsvariable | Beschreibung |
|
||||
|--------------------|--------------|
|
||||
| `MINDNET_HASH_COMPARE` | Vergleichsquelle (`Body`, `Full`, `Frontmatter`) |
|
||||
| `MINDNET_HASH_SOURCE` | Rohtext vs. geparst (`raw` / `parsed`) |
|
||||
| `MINDNET_HASH_NORMALIZE` | Normalisierung (`canonical` / `none`) |
|
||||
|
||||
**Basismodi**
|
||||
- `body` → Nur Textkörper
|
||||
- `frontmatter` → Nur Metadaten
|
||||
- `full` → Kombination
|
||||
- Hash = SHA-256(canonicalized_input)
|
||||
|
||||
Bei Änderung: Re-Import, neue Embeddings, Edges werden aktualisiert.
|
||||
- Pflichtfelder: `id`, `title`, `type`, `created`, `updated`, `tags`, `source_path`.
|
||||
- Note-Typen (Auszug): `experience`, `principle`, `value`, `goal`, `role`, `persona`, `plan`, `project`, `journal`, `reference`, `translation`.
|
||||
- **types.yaml** definiert Regeln/Defaults je Typ und für Relationstypen (siehe Abschnitt 10).
|
||||
|
||||
---
|
||||
|
||||
## 7. Qdrant-Speicherstruktur
|
||||
## 6. Chunking-Strategie
|
||||
|
||||
| Collection | Inhalt | Primärfelder | Embedding |
|
||||
|-------------|---------|---------------|------------|
|
||||
| `mindnet_notes` | Notes mit Fulltext & Hash | `note_id` | optional |
|
||||
| `mindnet_chunks` | Chunks mit Text, Window, Embedding | `chunk_id`, `note_id` | 384 d |
|
||||
| `mindnet_edges` | Graphkanten | `edge_id`, `source_id`, `target_id` | — |
|
||||
|
||||
**Upsert-Strategie**
|
||||
- Idempotente UUIDv5-IDs
|
||||
- Keine Duplikate durch deterministische Signaturen
|
||||
- `purge-before-upsert` zur Bereinigung
|
||||
|
||||
**Indizes**
|
||||
- notes: `note_id`, `hash_signature`
|
||||
- chunks: `note_id`, `chunk_index`
|
||||
- edges: `(kind, source_id, target_id, scope)`
|
||||
- Satz- bis Absatz-Chunks, moderate Überlappung; Meta-Felder verweisen auf Note/Quelle.
|
||||
- Named Vectors (z. B. `content`, optional `title`, `tags`) ermöglichen hybride Scores.
|
||||
|
||||
---
|
||||
|
||||
## 8. Fehler-Toleranz und Parser-Robustheit
|
||||
## 7. Embeddings & Vektoren
|
||||
|
||||
- UTF-8 mit Fallback: ersetzt ungültige Bytes (`errors="replace"`)
|
||||
- Ignoriert Nullbytes, BOM, exotische Zeilenenden
|
||||
- Erkennt defekte oder leere YAML-Header und überspringt sie
|
||||
- Logt Problemfälle (`read_markdown failed`, `make_note_payload returned non-dict`)
|
||||
- Bei Fehlern: Import → warn → continue
|
||||
- Einheitliche Metrik (cosine), reproduzierbares Embedding-Setup.
|
||||
- Konsistenz über Zeit durch Versionsfeld (`embedding_model`, `embedding_version`).
|
||||
|
||||
---
|
||||
|
||||
## 9. Tests & Validierung
|
||||
## 8. Volltext & Filter
|
||||
|
||||
**Integrationstests**
|
||||
- Roundtrip: Import → Export → Compare (`compare_vaults.py`)
|
||||
- Chunk-Integrität: `verify_chunks_integrity.py`
|
||||
- Hash-Audit: `hash_reporter.py`
|
||||
- Window-Vergleich: `check_chunks_window_vs_text.py`
|
||||
|
||||
**Erwartete Ergebnisse**
|
||||
- Alle Notes werden erkannt
|
||||
- `window ≠ text` bei > 60 % der Chunks
|
||||
- Roundtrip-Vergleich liefert „OK“
|
||||
- Edge-Anzahl entspricht Modell (belongs_to = #Chunks, next/prev ≈ #Chunks – 1)
|
||||
- Textindex auf `mindnet_chunks.body` für exakte Term/Operator-Queries.
|
||||
- Payload-Filter zur schnellen Einschränkung (tags, types, time-buckets, visibility u. a.).
|
||||
|
||||
---
|
||||
|
||||
## 10. Export & Roundtrip-Verhalten
|
||||
## 9. Edges – Beziehungen
|
||||
|
||||
- `export_markdown.py` erzeugt identische Markdown-Dateien
|
||||
- Schreibt YAML + Body in korrekter Reihenfolge
|
||||
- Roundtrip-Prüfung (`compare_vaults.py`) validiert:
|
||||
- vollständigen Body
|
||||
- unveränderte Hash-Signatur
|
||||
- gleiche Chunkzahl
|
||||
- **Explizit**: Jede tatsächliche Beziehung wird als Kante gespeichert (kein Workaround).
|
||||
- Felder u. a.: `src_id`, `dst_id`, `relation`, `direction`, `confidence (0–1)`, `evidence` (Chunk-IDs), `created_at`, `updated_at`.
|
||||
- **edge_defaults** liefern **Interpretations-Hints** (Gewichte, Symmetrie, Transitivität) und parametrisieren abgeleitete Edges; sie **ersetzen** keine Originalkanten.
|
||||
|
||||
---
|
||||
|
||||
## 11. Graph-Schicht und WP-04-Integration
|
||||
## 10. Regeln & Policies (types.yaml)
|
||||
|
||||
- Qdrant fungiert als Graph-Backend
|
||||
- `graph/service.py` (WP-04): stellt API für Mehrhop-Abfragen bereit
|
||||
- Unterstützt:
|
||||
- `expand(note_id, hops=2)`
|
||||
- `resolve_unresolved()`
|
||||
- `neighbors(kind=…)`
|
||||
- Grundlage für spätere **Retriever-Funktionen** (LLM / Agenten)
|
||||
- Per Typ/Relation:
|
||||
- Default-Eigenschaften (Gewicht, Directedness, Symmetrie, Transitivität).
|
||||
- Scoring-Hooks für Graph-Kontext im Retriever.
|
||||
- (später) Privacy-Hooks je Relation (Mindest-Visibility).
|
||||
|
||||
---
|
||||
|
||||
## 12. Agenten- und LLM-Integration
|
||||
## 11. Retrieval & Ranking (Komposition)
|
||||
|
||||
**Retriever-Pipeline**
|
||||
1. Suche top-k Chunks (cosine distance)
|
||||
2. Erweiterung um Edges (`references`, `backlinks`)
|
||||
3. Kontext-Assembling via `window`-Text
|
||||
4. Prompt-Generierung
|
||||
|
||||
**Agent-Use-Cases**
|
||||
- Automatisches Tagging (`type`, `status`)
|
||||
- Vorschläge für neue Edges
|
||||
- Auflösen von `unresolved`-Referenzen
|
||||
- Auto-Import externer Quellen (MediaWiki, PDFs, Webseiten)
|
||||
- Hybrid: Vektor-Score + Filter + optional BM25/Volltext.
|
||||
- Zeitgewicht (Decays) und Graph-Kontext werden komponiert (Formel siehe 16.3).
|
||||
- Konfiguration über `.env`/`config.yaml` (gewichts- und halbwertszeit-basiert).
|
||||
|
||||
---
|
||||
|
||||
## 13. Erweiterbarkeit
|
||||
## 12. Erklärbarkeit
|
||||
|
||||
- Neue Edge-Typen: `inspired_by`, `relates_to`, `contradicts`
|
||||
- Neue Note-Typen: `decision`, `resource`, `recommendation`
|
||||
- Embedding-Engines austauschbar (MiniLM → LaBSE → E5)
|
||||
- Versionierte Chunking-Strategien (`chunk_config.py`)
|
||||
- „Warum-Pfad“: Top-Chunks mit Score-Zerlegung (Vektor/Time/Graph), Graph-Kette, Provenienz.
|
||||
- Mehrschichtige Narrative (Kindheit/Erlebnisse, Werte/Prinzipien, Ziele/Leitbild, Gegenwart).
|
||||
|
||||
---
|
||||
|
||||
## 14. Qualitätssicherung und Best Practices
|
||||
## 13. Betrieb & Sicherheit
|
||||
|
||||
- Validierung vor Upsert
|
||||
- Keine Leer-Chunks
|
||||
- Vollständige Metadaten in Notes
|
||||
- Tests in `tests/` automatisch ausführbar (`run_e2e_roundtrip.sh`)
|
||||
- Ergebnis: „Roundtrip OK“ und `verify_chunks_integrity` = OK
|
||||
- Lokal auf Single-Node starten (Backups via Snapshots).
|
||||
- Zugriff nur intern/Reverse-Proxy; Festplattenverschlüsselung OS-seitig.
|
||||
- Migrationspfad zu Qdrant-Distributed (Shards, Replikation) einplanen.
|
||||
|
||||
---
|
||||
|
||||
## 15. Änderungsverlauf
|
||||
## 14. Qualität & Monitoring
|
||||
|
||||
**v1.4.0 (2025-10-06)**
|
||||
- Neu: `window` vs `text` Feld in `mindnet_chunks`
|
||||
- Neu: Hash-Modi (`body`, `frontmatter`, `full`) mit Env-Steuerung
|
||||
- Neu: Baseline-Modus für Vergleich
|
||||
- Parser → fehlertolerant (UTF-8 replace)
|
||||
- Roundtrip-Test vollständig integriert
|
||||
- Graph-Service und Mehrhop-Abfragen (WP-04-Vorbereitung)
|
||||
- `purge-before-upsert` und `--note-scope-refs` Parameter ergänzt
|
||||
|
||||
**v1.3.0 (2025-09-09)**
|
||||
- Hash-Normalisierung und Edge-Scope
|
||||
- Roundtrip-Tests, Agenten-Integration
|
||||
|
||||
**v1.2.0 (2025-09-02)**
|
||||
- Grundstruktur des Knowledge-Designs
|
||||
- Offline-Eval-Sets, Telemetrie (lokal), Drift-Checks, manuelle Reviews.
|
||||
- Kennzahlen: Hit-Rate, Overlap, Quellen-Alter, Erklärungspfad-Nutzung.
|
||||
|
||||
---
|
||||
|
||||
## 15. Offene Punkte
|
||||
|
||||
- Ausdetaillierung `types.yaml` (Relationen, Hooks).
|
||||
- UI-Oberfläche für Erklärpfade und Provenienz.
|
||||
- Batch-Jobs für abgeleitete Edges.
|
||||
|
||||
---
|
||||
|
||||
## 16. Ergänzungen 2025-11-09 (v1.5.0)
|
||||
|
||||
**Kontext:** Basierend auf den neuen Rahmenbedingungen (nur Deutsch; perspektivische externe Abfragen; einheitliche Vertraulichkeit mit Option auf feinere Stufen; Recency-Priorisierung als Default; Regeln später konfigurierbar; zunächst lokale Speicherung mit optionaler Migration; heutige Hardware, später skalierbar; hohe Erklärbarkeit mit mehreren Erklärungsebenen) werden folgende Entscheidungen und Erweiterungen getroffen. Alle bisherigen Inhalte bleiben gültig und werden **nicht** verändert.
|
||||
|
||||
### 16.1 Sprach- und Inhaltsrichtlinie (nur Deutsch)
|
||||
- `lang: "de"` als verpflichtendes Feld in allen Note-, Chunk- und Edge-Payloads.
|
||||
- Für spätere Mehrsprachigkeit **nicht** die Collection splitten, sondern ein Feld `lang` + optional `lang_original` behalten; Translation-Artefakte als eigener `type: translation` (verweisend auf `source_id`).
|
||||
|
||||
### 16.2 Vertraulichkeit & Sichtbarkeit
|
||||
- Heute: einheitliche Stufe `visibility: "internal"`.
|
||||
- Vorausschauend: reservierte Felder in allen Payloads
|
||||
- `visibility`: `"public" | "internal" | "restricted"`
|
||||
- `sensitivity`: Integer 0–3 (0 = unkritisch, 3 = hochsensibel)
|
||||
- `acl`: Liste erlaubter Rollen/Personen-IDs (App-Layer erzwingt Filter vor Query).
|
||||
- Access-Control erfolgt **im Applikations-Layer** mittels Qdrant-Payload-Filtern (`must: [ { key: "visibility", match: { any: [...] }}, ... ]`); Qdrant bleibt „dumm“ in Bezug auf ACL.
|
||||
|
||||
### 16.3 Recency-Modell & Ranking (konfigurierbar)
|
||||
- Default: **Zeitnähe** priorisieren (sanfte Abwertung älterer Inhalte).
|
||||
- Im Retrieval kombinieren wir:
|
||||
1) Vektor-Score (cosine) aus Qdrant
|
||||
2) Zeitgewicht `time_decay = exp( - ln(2) * age_days / half_life_days )`
|
||||
3) Graph-Kontext aus Edges (z. B. Degree, Path-Hops zum Anker „Ich/Persona“)
|
||||
- Konfigurierbare Gewichte (per `.env` oder `config.yaml`):
|
||||
```text
|
||||
FINAL_SCORE = w_vec * norm(vec_score)
|
||||
+ w_time * time_decay
|
||||
+ w_graph * graph_context_score
|
||||
# Vorschlag: w_vec=0.70, w_time=0.20, w_graph=0.10; half_life_days=365
|
||||
```
|
||||
- Für Fragen, die „zeitlos“ sind, kann `half_life_days = inf` gesetzt werden (kein Decay).
|
||||
|
||||
### 16.4 Erklärbarkeit („Warum diese Antwort?“)
|
||||
- Jeder Antwort wird ein **Erklärpfad** mitgegeben (optional im UI):
|
||||
- Top-Chunks (mit Score-Zerlegung w_vec/w_time/w_graph)
|
||||
- **Kausalpfad** im Graphen: `Note → Edge(relation, confidence) → Note …`
|
||||
- **Provenienz** (Quelle, Erstellungszeit, letzte Änderung).
|
||||
- Ein **Multi-Layer-Explanations**-Modul ordnet Beiträge zu: Kindheit/Erlebnisse, Werte & Prinzipien, aktuelle Ziele/Leitbild, Gegenwartskontext.
|
||||
|
||||
### 16.5 Qdrant-Schemata (Ergänzungen, rückwärtskompatibel)
|
||||
- **Collections** bleiben: `mindnet_notes`, `mindnet_chunks`, `mindnet_edges` (+ `mindnet_people` falls noch nicht vorhanden).
|
||||
- Gemeinsame Zusatzfelder (alle Payloads):
|
||||
- `lang: "de"`
|
||||
- `visibility`/`sensitivity`/`acl` (s. 16.2)
|
||||
- `created_at` (ISO-8601), `updated_at` (ISO-8601), `age_days` (App-seitig berechnet)
|
||||
- **Chunks**:
|
||||
- Named-Vectors (z. B. `"content"`, optional `"title"`, `"tags"` für Hybrid-Scores).
|
||||
- `text_index` auf `body` für BM25-ähnliche Volltextsuche (Qdrant-Textindex).
|
||||
- `recency_group`: Jahr-Monat `YYYY-MM` (ermöglicht Pre-Filter für „letzte N Monate“).
|
||||
- **Edges** (explizit, keine Workarounds):
|
||||
- Felder: `src_id`, `dst_id`, `relation`, `direction`, `confidence` (0–1), `evidence` (Chunk-IDs), `created_at`, `updated_at`.
|
||||
- **edge_defaults** bleiben als **Interpretations-Hints**: z. B. `default_weight`, `symmetry`, `transitivity`. Sie ersetzen **nicht** echte Kanten, sondern parametrisieren abgeleitete/zusätzliche Kantenbildung.
|
||||
- Abgeleitete Edges: optionaler Batch-Job erzeugt *ergänzende* Kanten entlang `types.yaml`-Regeln (z. B. Backlinks, „inspired_by“ aus Zitaten), niemals statt der Originalkanten.
|
||||
|
||||
### 16.6 Regeln & Typen (konfigurierbar)
|
||||
- `types.yaml` erweitert um:
|
||||
- Relationstypen mit Default-Eigenschaften (`weight`, `directed`, `symmetric`, `transitive`).
|
||||
- **Scoring-Hooks**: pro Relation optionaler Einfluss auf `graph_context_score`.
|
||||
- **Privacy-Hooks**: pro Relation Minimal-Visibility (z. B. Beziehungen zu Personen sind mindestens `internal`).
|
||||
|
||||
### 16.7 Speicher- & Betriebsstrategie (heute lokal, später skalierbar)
|
||||
- **Heute**: Single-Node Qdrant mit HNSW-Index, Text-Index und Payload-Filtern; regelmäßiges Backup (Snapshots).
|
||||
- **Morgen**: Migrationspfad in **Qdrant-Distributed** (Shards, ggf. Replikation) bei Bedarf; Parameter-Gleichheit sichern (named-vectors, Metrik, HNSW-Param).
|
||||
- **Sicherheit**: Zugriff nur über internes Netz/Reverse-Proxy, Auth vor Qdrant; Festplatten-Verschlüsselung auf OS-Ebene (z. B. LUKS).
|
||||
|
||||
### 16.8 Persona-Layer („meine Essenz“)
|
||||
- Eigenständiger **Persona-Knoten** (z. B. Note `person/lars`), verbunden mit:
|
||||
- Werten/Prinzipien, Leitbild, prägenden Erlebnissen, Zielen, Rollen.
|
||||
- Jede Antwort kann gegen diesen Knoten **kontextualisiert** werden (Graph-Pfadnähe).
|
||||
- Feld `persona_weight` in Chunks/Notes zur Priorisierung persönlicher Relevanz.
|
||||
|
||||
### 16.9 Evaluierung & Qualitätssicherung
|
||||
- **Offline-Sets** aus echten Fragen (inkl. gewünschter Perspektiven) mit erwarteten Top-Quellen.
|
||||
- **Telemetrie** (lokal): Query-Arten, Treffer-Overlaps, Zeitgewicht-Effekte, „Warum-Pfad“ genutzt/ignoriert.
|
||||
- **Drift-Checks**: Anteil sehr alter vs. neuer Quellen in Antworten; manuelle Reviews.
|
||||
|
||||
### 16.10 Offene Punkte / Nächste Schritte
|
||||
1) `types.yaml` um Relationseigenschaften erweitern (s. 16.6).
|
||||
2) Retriever-Komposition inkl. Zeitgewicht (s. 16.3) implementieren.
|
||||
3) Explanations-Pfad inkl. Provenienzoberfläche (s. 16.4) im UI.
|
||||
4) Feld-Erweiterungen in Import-Pipelines (`lang`, `visibility`, Zeitfelder).
|
||||
5) Optional: Batch-Job für abgeleitete Edges nach Regeln (ergänzend, nicht ersetzend).
|
||||
@@ -0,0 +1,291 @@
|
||||
# Mindnet – Fachliche Architektur (Stand: 2025-11-17)
|
||||
|
||||
> Dieses Dokument beschreibt **was** Mindnet fachlich tut und **warum** – mit Fokus auf die Erzeugung und Nutzung von **Edges** (Kanten) zwischen Notizen/Chunks. Die technische Umsetzung (Module, Flags, Schemas) wird nur dort angerissen, wo es zum Verständnis nötig ist und im technischen Dokument detailliert.
|
||||
|
||||
---
|
||||
|
||||
## 0) Zielbild & Grundprinzip
|
||||
|
||||
Mindnet wandelt Obsidian-Markdown-Notizen in einen **Graph aus Punkten und Kanten** um.
|
||||
Die drei zentralen Artefakt-Sammlungen lauten:
|
||||
|
||||
- `mindnet_notes` – genau **eine** Note pro Markdown-Datei (Metadaten & Hashes)
|
||||
- `mindnet_chunks` – semantische Teilstücke einer Note (Fenster/„Chunks“)
|
||||
- `mindnet_edges` – gerichtete Beziehungen zwischen Knoten (Chunks/Notes)
|
||||
|
||||
Die Import-Pipeline erzeugt diese Artefakte **deterministisch** und **idempotent** (erneute Läufe überschreiben konsistent statt zu duplizieren). Die Import-Schritte sind: *parse → chunk → edge → upsert*. :contentReference[oaicite:0]{index=0}
|
||||
|
||||
---
|
||||
|
||||
## 1) Notizen & Chunks (fachliche Perspektive)
|
||||
|
||||
### 1.1 Notiz (Note)
|
||||
- Repräsentiert eine fachliche Einheit (z. B. *„Vector DB Basics“*, *„KI-Projekt“*).
|
||||
- Trägt Eigenschaften (Titel, Typ, Zeitstempel, optionale Policies).
|
||||
- **Typ** (*type*) steuert u. a. Chunk-Profil und Default-Relationen (siehe §4).
|
||||
- **retriever_weight** und **chunk_profile** werden systematisch an Note **und** Chunks gespiegelt, damit der Retriever beides nutzen kann. :contentReference[oaicite:1]{index=1}
|
||||
|
||||
### 1.2 Chunk
|
||||
- Ausschnitt/Textfenster aus der Note, als eigenständiger Such-Anker.
|
||||
- Jeder Chunk gehört **genau einer** Note.
|
||||
- Chunks bilden eine Sequenz (1…N) – das ermöglicht *next/prev*.
|
||||
|
||||
> **Wichtig:** Chunking-Profile (short/medium/long) kommen aus `types.yaml` (per Note-Typ), können aber lokal überschrieben werden. Die effektiven Werte werden bei der Payload-Erzeugung bestimmt. :contentReference[oaicite:2]{index=2}
|
||||
|
||||
---
|
||||
|
||||
## 2) Edges – fachliche Typen & Bedeutung
|
||||
|
||||
Edges kodieren Beziehungen. Sie sind **gerichtet** und werden in `mindnet_edges` gespeichert.
|
||||
|
||||
### 2.1 Struktur-Kanten
|
||||
- **belongs_to**: *Chunk → Note* (Zuordnung)
|
||||
- **next / prev**: Kettenbeziehungen zwischen aufeinanderfolgenden Chunks derselben Note
|
||||
→ ermöglichen „lineares Weiterlesen“ oder Kontextpfade in Antworten.
|
||||
Diese Kanten entstehen immer, unabhängig von Inhalten. :contentReference[oaicite:3]{index=3}
|
||||
|
||||
### 2.2 Inhalts-Kanten (explizit)
|
||||
- **references (chunk-scope)**: aus dem Textfenster des Chunks extrahierte Links.
|
||||
Standardquelle: **Wikilinks** `[[…]]` – einschließlich Varianten `[[id#anchor]]`, `[[id|label]]`.
|
||||
→ Mindnet reduziert robust auf die Ziel-ID (Titel/ID-Normalisierung). :contentReference[oaicite:4]{index=4}
|
||||
- **Explizite Inline-Relationen**: im Fließtext notierte Relationen, z. B.:
|
||||
rel: similar_to [[Vector DB Basics]]
|
||||
→ werden als Relation **similar_to** mit hoher Confidence materialisiert; *rule_id* `inline:rel`.
|
||||
(Diese Form ist besonders kompakt in Obsidian editierbar.)
|
||||
- **Callout-Edges** (optional):
|
||||
> [!edge] related_to: [[Vector DB Basics]] [[Embeddings 101]]
|
||||
→ erzeugt mehrere **related_to**-Kanten aus einer Zeile; *rule_id* `callout:edge`.
|
||||
(Für kuratierte, sichtbare Linklisten am Notizende.)
|
||||
|
||||
> Alle expliziten Quellen (Wikilink, Inline-Rel, Callout) sind **gleichwertige Primärbelege** und werden als *provenance=explicit* geführt. Wo zutreffend, tragen Edges *rule_id* wie `explicit:wikilink`, `inline:rel`, `callout:edge`. :contentReference[oaicite:5]{index=5}
|
||||
|
||||
### 2.3 Typ-basierte Default-Kanten (regelbasiert)
|
||||
Jede Note hat einen *Typ* (z. B. `concept`, `project`, `profile`). In `config/types.yaml` definieren wir pro Typ **edge_defaults**, z. B.:
|
||||
|
||||
- `concept`: `["references","related_to"]`
|
||||
- `project`: `["references","depends_on"]`
|
||||
|
||||
Regel: **Für jede gefundene explizite Referenz** (s. o.) werden **zusätzliche** Edges nach diesen Defaults erzeugt. Beispiel: Ein *project* mit `edge_defaults=["depends_on"]` erzeugt zu *jedem* explizit referenzierten Ziel **zusätzlich** eine `depends_on`-Kante.
|
||||
Diese Kanten tragen *provenance=rule* und eine **rule_id** der Form
|
||||
`edge_defaults:{note_type}:{relation}` (z. B. `edge_defaults:project:depends_on`). :contentReference[oaicite:6]{index=6}
|
||||
|
||||
---
|
||||
|
||||
## 3) Edge-Payload – Felder & Semantik
|
||||
|
||||
Jede Kante hat mindestens:
|
||||
|
||||
- `kind` – Beziehungsart *(belongs_to, next, prev, references, related_to, depends_on, similar_to, …)*
|
||||
- `scope` – `"chunk"` (Standard) oder `"note"` (optional für Backlinks/Note-Scope)
|
||||
- `source_id`, `target_id` – Quell-/Ziel-Knoten (Chunk- oder Note-ID)
|
||||
- `note_id` – **Owner-Note** (die Note, aus der die Kante stammt; dient der schnellen Löschung/Filterung)
|
||||
|
||||
Erweiterte/abgeleitete Felder (V2-Superset):
|
||||
|
||||
- `provenance` – `"explicit"` (Wikilink/Inline/Callout) oder `"rule"` (Typ-Defaults)
|
||||
- `rule_id` – maschinenlesbare Regelquelle, z. B.
|
||||
- `explicit:wikilink`, `inline:rel`, `callout:edge`
|
||||
- `edge_defaults:{note_type}:{relation}`
|
||||
- `confidence` – 0.0–1.0; Heuristik, z. B. `1.0` für Wikilinks/Backlinks, `0.95` für `inline:rel`, `0.7` für Typ-Defaults.
|
||||
|
||||
**Dedup-Schlüssel** (fachlich): Gleichlautende `(source_id, target_id, kind, scope, rule_id)` werden nicht mehrfach geschrieben. :contentReference[oaicite:7]{index=7}
|
||||
|
||||
---
|
||||
|
||||
## 4) Typ-Registry (`config/types.yaml`)
|
||||
|
||||
### 4.1 Zweck
|
||||
- Steuert **Chunking-Profile** (`short|medium|long`) **pro Typ**
|
||||
- Liefert **retriever_weight** (Note-/Chunk-Gewichtung im Ranking) **pro Typ**
|
||||
- Definiert **edge_defaults** je Typ (s. o.)
|
||||
|
||||
Der Importer lädt die Registry aus `MINDNET_TYPES_FILE` oder nutzt Fallbacks. **Frontmatter-Overrides** (z. B. `retriever_weight`) werden respektiert; am Ende schreibt der Importer die **effektiven** Werte zuverlässig in Note- und Chunk-Payload. :contentReference[oaicite:8]{index=8}
|
||||
|
||||
### 4.2 Beispiel (auslieferungsnah)
|
||||
version: 1.0
|
||||
types:
|
||||
concept:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["references", "related_to"]
|
||||
retriever_weight: 0.35
|
||||
project:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.97
|
||||
profile:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.66
|
||||
|
||||
**Auflösung im Importer**
|
||||
- `effective_chunk_profile(note_type, registry)` → `"short|medium|long"|None`
|
||||
- `effective_retriever_weight(note_type, registry)` → `float|None`
|
||||
- Ergebnis wird in `note_payload` **und** `chunk_payloads` gespiegelt. :contentReference[oaicite:9]{index=9}
|
||||
|
||||
---
|
||||
|
||||
## 5) Wie entstehen Edges aus Markdown?
|
||||
|
||||
### 5.1 Quelle „Wikilinks“ (Primärfall)
|
||||
- Parser extrahiert alle `[[…]]` (Label/Anker werden auf die Ziel-ID reduziert).
|
||||
- Pro Fund entsteht **eine** `references`-Kante (Chunk-Scope).
|
||||
- Zusätzlich: Typ-Defaults (z. B. `depends_on`, `related_to`) pro Ziel. :contentReference[oaicite:10]{index=10}
|
||||
|
||||
### 5.2 Quelle „Inline-Relation“ (Optional, kompakt)
|
||||
- Im Fließtext markierte Relation, z. B.:
|
||||
rel: similar_to [[Vector DB Basics]]
|
||||
- Erzeugt `similar_to`-Kanten; *rule_id* `inline:rel`; `confidence≈0.95`.
|
||||
- Geeignet für sparsame, gut lesbare Referenzierung. (Erfassung in der Chunk-Window-Analyse.) :contentReference[oaicite:11]{index=11}
|
||||
|
||||
### 5.3 Quelle „Callout-Edges“ (Optional, kuratiert)
|
||||
- Klar sichtbare, redaktionell gepflegte Liste am Ende einer Note:
|
||||
> [!edge] related_to: [[Vector DB Basics]] [[Embeddings 101]]
|
||||
- Erzeugt je Ziel eine `related_to`-Kante; *rule_id* `callout:edge`.
|
||||
- Eignet sich für „Leselisten“ und „Siehe auch“. (Zählung als *callout_total* in Reports, wo vorhanden.) :contentReference[oaicite:12]{index=12}
|
||||
|
||||
### 5.4 Struktur-Kanten
|
||||
- `belongs_to` pro Chunk
|
||||
- `next/prev` aus Nachbarbezug (aus Chunk-Sequenz/Neighbors) :contentReference[oaicite:13]{index=13}
|
||||
|
||||
> **Hinweis zu Note-Scope:** Optional können zusätzlich Note-weite *references/backlinks* aus der Gesamtheit aller Chunk-Funde abgeleitet werden (Konfig-Flag/ENV). Das ist für globale Graph-Analysen nützlich, aber im Retrieval seltener erforderlich. :contentReference[oaicite:14]{index=14}
|
||||
|
||||
---
|
||||
|
||||
## 6) Confidence & Provenance – wozu?
|
||||
|
||||
Der Retriever kann Edges gewichten:
|
||||
- **provenance**: *explicit* > *rule*
|
||||
- **confidence**: numerische Feinsteuerung (z. B. Inline-Relation etwas höher, Defaults etwas niedriger)
|
||||
- **retriever_weight (Note/Chunk)**: skaliert die Relevanz des gesamten Knotens
|
||||
|
||||
Eine typische Gewichtung (konfigurierbar) ist:
|
||||
explicit: 1.0, rule: 0.8, default_resolved: 0.6, default: 0.2
|
||||
Damit bevorzugt der Graph **kuratiertes Wissen** (explizit notierte Links) vor „erweiterten“ Default-Ableitungen. :contentReference[oaicite:15]{index=15}
|
||||
|
||||
---
|
||||
|
||||
## 7) Semantik ausgewählter `kind`-Werte
|
||||
|
||||
- `references` – „Nutzt/erwähnt“; neutral, aber stützend für Kontext.
|
||||
- `related_to` – Ähnlichkeit/Verwandtschaft (symmetrisch interpretierbar).
|
||||
- `similar_to` – noch engere Ähnlichkeit; oft aus Inline-Rel (bewusst gesetzt).
|
||||
- `depends_on` – fachliche Abhängigkeit (z. B. „Projekt X hängt von Y ab“).
|
||||
- `belongs_to`, `next`, `prev` – Struktur.
|
||||
|
||||
> Symmetrische Relationen (z. B. `related_to`, `similar_to`) können **explizit** nur einseitig notiert sein, aber im Retriever beidseitig interpretiert werden (z. B. Query-Zeit-Expansion), um Redundanz im Vault zu vermeiden. :contentReference[oaicite:16]{index=16}
|
||||
|
||||
---
|
||||
|
||||
## 8) Frontmatter-Eigenschaften – Rolle & Empfehlung
|
||||
|
||||
Frontmatter-Eigenschaften (Properties) bleiben **minimiert**:
|
||||
- **type** – steuert Typ-Defaults via Registry (Pflicht für differenziertes Verhalten).
|
||||
- **retriever_weight** *(optional)* – kann typbasierte Defaults überschreiben.
|
||||
- **chunk_profile** *(optional)* – kann typbasierte Defaults überschreiben.
|
||||
- **links** *(optional, Bestandskompatibilität)* – wenn vorhanden (z. B. `depends_on: …`), können sie als explizite Kanten materialisiert werden; die Pflege in freiem Text (Inline/Callout) ist jedoch **benutzerfreundlicher** und weniger fehleranfällig.
|
||||
|
||||
Empfehlung: **Links primär über Text (Wikilink/Inline/Callout) pflegen**, nicht als Properties. Die Registry regelt Standards; Properties dienen nur gezielten Ausnahmen. :contentReference[oaicite:17]{index=17}
|
||||
|
||||
---
|
||||
|
||||
## 9) Lösch-/Update-Garantien (Idempotenz)
|
||||
|
||||
- Jede Note hat einen stabilen **note_id** (Frontmatter/Hash).
|
||||
- Vor einem Upsert können *alte Chunks/Edges einer Note* gefiltert gelöscht werden (`note_id`-Filter) – das hält Collections sauber bei Re-Imports.
|
||||
- Es gibt Abkürzungen für „fehlertolerante Rebuilds“: *Existenz-Checks* statt Full-Scan. :contentReference[oaicite:18]{index=18}
|
||||
|
||||
---
|
||||
|
||||
## 10) Beispiel – Von Markdown zu Kanten
|
||||
|
||||
**Markdown (Auszug)**
|
||||
# Relations Showcase
|
||||
…
|
||||
rel: similar_to [[Vector DB Basics]]
|
||||
…
|
||||
[[Embeddings 101]]
|
||||
|
||||
> [!edge] related_to: [[Vector DB Basics]] [[Embeddings 101]]
|
||||
|
||||
**Ergebnis (fachlich)**
|
||||
- `references(Chunk→Vector DB Basics)` + `references(Chunk→Embeddings 101)`
|
||||
- `similar_to(Chunk→Vector DB Basics)` mit `rule_id=inline:rel`, `confidence≈0.95`
|
||||
- `related_to(Chunk→Vector DB Basics)` und `related_to(Chunk→Embeddings 101)`
|
||||
via Callout; `rule_id=callout:edge`
|
||||
- Zusätzliche **Typ-Defaults** je Quell-Typ, z. B. `edge_defaults:concept:related_to` → weitere `related_to`-Kanten
|
||||
|
||||
*Strukturkanten* (`belongs_to`, `next/prev`) fallen wie üblich an. (Die tatsächlichen Zähllogiken erscheinen im Import-Report/Smoke-Tests.) :contentReference[oaicite:19]{index=19}
|
||||
|
||||
---
|
||||
|
||||
## 11) Abfrage-Implikationen (fachlich)
|
||||
|
||||
Retriever und Query-Layer berücksichtigen:
|
||||
1) **Knoten-Gewicht** (`retriever_weight`) aus Typ/Property (Note & Chunk).
|
||||
2) **Edge-Provenance/Confidence** – explizit > regelbasiert; höhere Confidence bevorzugt.
|
||||
3) **Edge-Topologie** – z. B. 1-Hop-Nachbarschaft, Sequenzpfade (`next/prev`), Backlinks (optional).
|
||||
4) **Policy-Filter** – Privacy, Tags, Created usw. via Payload-Indizes.
|
||||
|
||||
Die Abfrage kann so z. B. Ergebnisse bevorzugen, die **explizit** in einem *Projekt* als `depends_on` genannt sind – und nur nachrangig solche, die „typbedingt“ (`edge_defaults`) ähnlich/abhängig sind. :contentReference[oaicite:20]{index=20}
|
||||
|
||||
---
|
||||
|
||||
## 12) Qualitäts- und Testkriterien (fachlich)
|
||||
|
||||
- **Keine Duplikate** gleicher `(src, relation, dst)` in der Collectionsicht (Dedup).
|
||||
- **Explizite Links** werden vollständig materialisiert (*explicit_total*).
|
||||
- **Typ-Defaults** ergänzen, aber überdecken nicht – Counts bleiben plausibel (*defaults_total*).
|
||||
- **Inline/Callout** werden separat quantifiziert (*inline_total*, *callout_total*), damit Redaktionen ihre Pflegewirkung beobachten können. :contentReference[oaicite:21]{index=21}
|
||||
|
||||
---
|
||||
|
||||
## 13) Praxisempfehlungen für den Vault
|
||||
|
||||
1) **Wikilinks** überall, wo eine inhaltliche Nennung stattfindet.
|
||||
2) Für **gezielte Relationen** (Ähnlichkeit/Abhängigkeit) kurze **Inline-Rel**-Zeilen; z. B.:
|
||||
rel: similar_to [[X]]
|
||||
rel: related_to [[Y]] [[Z]]
|
||||
3) Am Notizende **Callout-Edges** für kuratierte Überblicksblöcke:
|
||||
> [!edge] related_to: [[X]] [[Y]]
|
||||
4) **Properties** schlank halten: mindestens `type`, optional `retriever_weight`/`chunk_profile`.
|
||||
5) **types.yaml** pflegen (zentrale Governance): *Welche Typen gibt es? Welche Default-Relationen sind fachlich sinnvoll?*
|
||||
6) **Regelmäßige Smoke-Tests/Reports** prüfen die Kanten-Counts und Provenance-Verteilung. :contentReference[oaicite:22]{index=22}
|
||||
|
||||
---
|
||||
|
||||
## 14) Referenzen (Projektdateien & Leitlinien)
|
||||
|
||||
- Import-Pipeline & Registry-Auflösung (Importer, Flags, Idempotenz): `scripts/import_markdown.py`. :contentReference[oaicite:23]{index=23}
|
||||
- Robuste Kantenbildung (Wikilinks, Struktur, optionale Note-Scope): `app/core/derive_edges.py` (v1.4+). :contentReference[oaicite:24]{index=24}
|
||||
- V2-Edge-Superset (Provenance, rule_id, Confidence, Typ-Defaults): `app/core/edges.py` (v2.0+). :contentReference[oaicite:25]{index=25}
|
||||
- Typ-Registry laden/auflösen (Fallbacks, Effektivwerte): Import-Helper / *Type-Registry*-Hilfsfunktionen. :contentReference[oaicite:26]{index=26}
|
||||
- Test-/Playbook-Leitplanken (Policies, KPIs, Roadmap): *Implementation Playbook*. :contentReference[oaicite:27]{index=27}
|
||||
|
||||
---
|
||||
|
||||
## 15) Anhang – Mini-Beispiel einer Projekt-Note (verkürzt)
|
||||
|
||||
---
|
||||
id: 20251110-ki-trainerassistent-projekt-56d2aa
|
||||
title: KI Trainerassistent (Projekt)
|
||||
type: project
|
||||
retriever_weight: 1.0 # optional Override
|
||||
chunk_profile: default # optional Override
|
||||
---
|
||||
|
||||
# KI Trainerassistent (Projekt)
|
||||
|
||||
Wir nutzen [[Qdrant Vektordatenbank]] und [[Ollama LLM]] …
|
||||
|
||||
rel: depends_on [[Qdrant Vektordatenbank]]
|
||||
rel: related_to [[Ollama LLM]]
|
||||
|
||||
> [!edge] depends_on: [[Qdrant Vektordatenbank]]
|
||||
> [!edge] related_to: [[Ollama LLM]]
|
||||
|
||||
*Erwartung*:
|
||||
- `references` (Wikilinks) auf **beide** Ziele
|
||||
- `depends_on` & `related_to` via **Inline** und **Callout** (provenance=`explicit`, rule_id `inline:rel`/`callout:edge`)
|
||||
- Zusätzlich **Typ-Defaults** aus `types.yaml` für `project` (z. B. weitere `depends_on`) – provenance=`rule`, `rule_id=edge_defaults:project:depends_on`. :contentReference[oaicite:28]{index=28}
|
||||
|
||||
---
|
||||
@@ -0,0 +1,699 @@
|
||||
# Mindnet – Technische Architektur (V2, Stand: 2025-11-xx)
|
||||
|
||||
> **Ziel dieses Dokuments:**
|
||||
> Vollständige, konsolidierte Beschreibung der aktuellen technischen Architektur von **Mindnet V2** – insbesondere der Verarbeitung von Markdown-Notizen, des Chunkings, der Kantenableitung (Edges) inkl. `rule_id` / `confidence` und der Speicherung in Qdrant.
|
||||
> Das Dokument bildet den **aktuellen Implementierungsstand** ab, nicht mehr gültige Annahmen sind bereinigt, neue Konzepte (Typ-Defaults, Inline-Relationen, Edges-Schema) sind integriert.
|
||||
|
||||
---
|
||||
|
||||
## 1. Systemüberblick
|
||||
|
||||
### 1.1 Zielbild
|
||||
|
||||
Mindnet ist ein **persönliches Wissensnetz**. Technisch bedeutet das:
|
||||
|
||||
- Markdown-Notizen in einem Vault (Obsidian-kompatibel) werden
|
||||
- geparst (Frontmatter + Body),
|
||||
- in **Notes**, **Chunks** und **Edges** überführt,
|
||||
- in **Qdrant** als drei Collections abgelegt:
|
||||
- `<prefix>_notes`
|
||||
- `<prefix>_chunks`
|
||||
- `<prefix>_edges`.
|
||||
|
||||
- Das System ist so konzipiert, dass es:
|
||||
- **deterministisch** arbeitet (IDs, Chunks, Kanten),
|
||||
- über `types.yaml` **konfigurierbar** ist,
|
||||
- später von einem LLM-basieren Agenten als **Graph-Retriever** genutzt werden kann.
|
||||
|
||||
---
|
||||
|
||||
### 1.2 Verzeichnisstruktur (Server)
|
||||
|
||||
/home/llmadmin/mindnet/:
|
||||
|
||||
- **/app**
|
||||
- `main.py` – FastAPI-App / Einstieg
|
||||
- **/app/core**
|
||||
- `qdrant.py` – QdrantConfig, Client-Factory, Collection-Management
|
||||
- `qdrant_points.py` – Upserts / Deletes / Queries auf Punktebene
|
||||
- `note_payload.py` – Bau von Note-Payloads
|
||||
- `chunk_payload.py` – Bau von Chunk-Payloads
|
||||
- `chunker.py` – Chunking-Logik
|
||||
- `edges.py` – Datenstruktur & Hilfsfunktionen für Edges
|
||||
- `derive_edges.py` – Ableitung von Edges aus Text und Typen
|
||||
- **/app/models**
|
||||
- `dto.py` – DTOs für API / Services
|
||||
- **/app/routers**
|
||||
- `qdrant_router.py` – API-Layer Richtung Qdrant
|
||||
- **/app/services**
|
||||
- `llm_ollama.py` – Anbindung an Ollama-LLM
|
||||
- `embeddings_client.py` – Embedding-Service (z. B. Ollama / lokaler Dienst)
|
||||
- **/config**
|
||||
- `types.yaml` – Typdefinitionen inkl. Chunk-Profilen und Edge-Defaults
|
||||
- **/scripts**
|
||||
- `import_markdown.py` – Hauptimporter für Vault
|
||||
- `reset_qdrant.py` – Collections löschen/neu anlegen (inkl. Indizes)
|
||||
- `setup_mindnet_collections.py` – Collection-Setup (historisch / Hilfsskript)
|
||||
- diverse Diagnose- und Testskripte, z. B.
|
||||
- `payload_dryrun.py`
|
||||
- `edges_full_check.py`
|
||||
- **/tests**
|
||||
- `test_edges_smoke.py` – einfache Edge-Smoke-Tests
|
||||
- `test_edges_all.py` – umfassendere Checks (Counts, Plausibilität)
|
||||
- `ensure_indexes_and_show.py`, `assert_payload_schema.py` – Schema/Index-Checks
|
||||
- **/vault**
|
||||
- produktives Vault (Obsidian)
|
||||
- **/mindnet_v2_test_vault**
|
||||
- Test-Vault für V2-Funktionen (Chunking, Edges, Typ-Defaults)
|
||||
|
||||
---
|
||||
|
||||
## 2. Datenmodell & Collections
|
||||
|
||||
### 2.1 Note, Chunk, Edge – Begriffe
|
||||
|
||||
- **Note**
|
||||
Entspricht einer Markdown-Datei mit YAML-Frontmatter (`id`, `title`, `type`, …).
|
||||
In Qdrant als ein Punkt in `<prefix>_notes`.
|
||||
|
||||
- **Chunk**
|
||||
Ein Abschnitt einer Note (z. B. Absatz, Überschrifts-Block, Sliding Window).
|
||||
In Qdrant als Punkt in `<prefix>_chunks`.
|
||||
|
||||
- **Edge**
|
||||
Eine gerichtete Kante, die eine Beziehung zwischen:
|
||||
- zwei Chunks,
|
||||
- Chunk ↔ Note,
|
||||
- oder (konzeptionell) Note ↔ Note beschreibt.
|
||||
Gespeichert in `<prefix>_edges`.
|
||||
|
||||
---
|
||||
|
||||
### 2.2 Qdrant Collections (Stand V2)
|
||||
|
||||
**Namenskonvention:**
|
||||
Prefix aus ENV (`COLLECTION_PREFIX`, z. B. `mindnet`):
|
||||
|
||||
- `mindnet_notes`
|
||||
- `mindnet_chunks`
|
||||
- `mindnet_edges`
|
||||
|
||||
Die Collections werden über:
|
||||
|
||||
```bash
|
||||
python3 -m scripts.reset_qdrant --mode wipe --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
|
||||
**vollständig neu angelegt**, inkl. Payload-Indizes (Details in 6.1).
|
||||
|
||||
---
|
||||
|
||||
### 2.3 Notes – Payload & Felder
|
||||
|
||||
Minimale, indexierte Felder (Auszug, Schema laut `diag_payload_indexes`):
|
||||
|
||||
- `note_id` (keyword) – stabile Note-ID (Frontmatter `id`)
|
||||
- `title` (text) – Notiz-Titel
|
||||
- `type` (keyword) – logischer Typ (z. B. `concept`, `project`, `profile`, `journal`, `source`)
|
||||
- `tags` (keyword[]) – Tags aus Frontmatter
|
||||
- `updated` (integer) – Zeitstempel (z. B. `YYYYMMDDHHMMSS` oder epoch; Implementierungsdetail)
|
||||
|
||||
Weitere Felder (nicht alle indexiert, aber im Payload vorhanden):
|
||||
|
||||
- `retriever_weight` (float) – wie stark diese Note/Chunks im Retrieval gewichtet werden
|
||||
- `chunk_profile` (string) – Name des verwendeten Chunk-Profils
|
||||
- `edge_defaults` (string[]) – Liste von Default-Relationen pro Typ (z. B. `["references","related_to"]`)
|
||||
- `confidentiality`, `visibility`, weitere Metadaten aus Frontmatter
|
||||
|
||||
Diese Felder werden in `app/core/note_payload.py` aus Frontmatter + `types.yaml` konstruiert.
|
||||
|
||||
---
|
||||
|
||||
### 2.4 Chunks – Payload & Felder
|
||||
|
||||
Indexierte Kernfelder:
|
||||
|
||||
- `note_id` (keyword) – Referenz auf die übergeordnete Note
|
||||
- `chunk_id` (keyword) – deterministische Chunk-ID:
|
||||
`"{note_id}#c{index:02d}"` (z. B. `20251110-ollama-llm-9f0a12#c02`)
|
||||
- `index` (integer) – 0-basiert; Position in der Note
|
||||
- `type` (keyword) – Note-Typ (vom Note geerbt)
|
||||
- `tags` (keyword[]) – Tags (vom Note geerbt)
|
||||
|
||||
Weitere wichtige Payload-Felder:
|
||||
|
||||
- `ord` (integer) – 1-basiertes Pendant zu `index` (für menschliche Lesbarkeit)
|
||||
- `text` (string) – eigentlicher Chunk-Text (ggf. gereinigt/normalisiert)
|
||||
- `window` (string) – Kontext-Fenster (z. B. Chunk ± Nachbarn)
|
||||
- `retriever_weight` (float) – gültiger Wert aus `types.yaml` (oder Default)
|
||||
- `chunk_profile` (string) – verwendetes Profil, z. B. `short`, `medium`, `long`, `by_heading`
|
||||
- `neighbors_prev` (string[]) – Liste von `chunk_id`s direkter Vorgänger (i. d. R. 0 oder 1 ID)
|
||||
- `neighbors_next` (string[]) – Liste von `chunk_id`s direkter Nachfolger
|
||||
|
||||
Die Konstruktion erfolgt in `app/core/chunk_payload.py` auf Basis der Chunks aus `chunker.py` und der aufgelösten Typ-Defaults.
|
||||
|
||||
---
|
||||
|
||||
### 2.5 Edges – Payload & Felder
|
||||
|
||||
Indexierte Felder:
|
||||
|
||||
- `note_id` (keyword) – Note, zu der der **Quell-Chunk** gehört
|
||||
- `source_id` (keyword) – i. d. R. `chunk_id` des Quell-Chunks (z. B. `2025…#c00`)
|
||||
- `target_id` (keyword) – Zielobjekt (z. B.:
|
||||
- `chunk_id` eines anderen Chunks
|
||||
- Note-Titel (`"Embeddings 101"`) bei **konzeptionellen Referenzen**
|
||||
- `kind` (keyword) – Kantenart:
|
||||
- Struktur: `belongs_to`, `next`, `prev`
|
||||
- Explizit: `references`
|
||||
- Inline: z. B. `similar_to`, `depends_on`, `related_to`
|
||||
- Typ-Defaults: ebenfalls z. B. `depends_on`, `related_to` (aus `edge_defaults`)
|
||||
- `scope` (keyword) – aktuell immer `"chunk"` (Kante wird auf Chunk-Ebene geführt)
|
||||
- `chunk_id` (keyword) – `chunk_id` des Quell-Chunks
|
||||
|
||||
Weitere Payload-Felder:
|
||||
|
||||
- `rule_id` (string) – Herkunftsregel:
|
||||
- Struktur:
|
||||
- `structure:belongs_to`
|
||||
- `structure:next`
|
||||
- `structure:prev`
|
||||
- Explizite Wiki-Links:
|
||||
- `explicit:wikilink`
|
||||
- Inline-Relationen:
|
||||
- `inline:rel`
|
||||
- Typ-Defaults aus `types.yaml`:
|
||||
- `edge_defaults:<type>:<relation>`
|
||||
z. B. `edge_defaults:concept:related_to`
|
||||
- `confidence` (float) – Stärke / Vertrauensgrad:
|
||||
- Struktur-Kanten: `1.0`
|
||||
- Explizite Wiki-Links: `1.0`
|
||||
- Inline-Relationen (`[[rel:… …]]`): z. B. `0.95`
|
||||
- Typ-Defaults: z. B. `0.7`
|
||||
|
||||
Edges werden in `app/core/derive_edges.py` erzeugt und in `mindnet_edges` gespeichert.
|
||||
|
||||
---
|
||||
|
||||
## 3. Typ-Konfiguration (`config/types.yaml`)
|
||||
|
||||
### 3.1 Struktur des Files
|
||||
|
||||
`types.yaml` (vereinfachtes Beispiel):
|
||||
|
||||
```yaml
|
||||
version: 1.0
|
||||
default:
|
||||
retriever_weight: 1.0
|
||||
chunk_profile: default
|
||||
|
||||
types:
|
||||
concept:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["references", "related_to"]
|
||||
retriever_weight: 0.35
|
||||
|
||||
task:
|
||||
chunk_profile: short
|
||||
edge_defaults: ["depends_on", "belongs_to"]
|
||||
retriever_weight: 0.8
|
||||
|
||||
experience:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["derived_from", "inspired_by"]
|
||||
retriever_weight: 0.9
|
||||
|
||||
project:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.97
|
||||
|
||||
profile:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.66
|
||||
```
|
||||
|
||||
### 3.2 Typauflösung – Reihenfolge der Prioritäten
|
||||
|
||||
Für eine Note mit `type` und optionalen Frontmatter Feldern `retriever_weight`, `chunk_profile` gilt aktuell:
|
||||
|
||||
1. **Typ-spezifische Defaults** aus `types.yaml`:
|
||||
- `retriever_weight` ← `types.<type>.retriever_weight` (falls vorhanden)
|
||||
- `chunk_profile` ← `types.<type>.chunk_profile` (falls vorhanden)
|
||||
2. **Globaler Default** `default` in `types.yaml`, falls Typ-Eintrag fehlt.
|
||||
3. **Frontmatter `retriever_weight` / `chunk_profile`** werden aktuell *nicht* mehr als Konfigurationsebene verwendet, sondern eher als historische Relikte. (Dies kann zukünftig wieder aktiviert werden, ist aber im aktuellen Stand bewusst abgeschaltet zugunsten klarer, zentraler Konfiguration über `types.yaml`.)
|
||||
|
||||
Das Ergebnis dieser Auflösung wird im Payload als:
|
||||
|
||||
- `note_payload["retriever_weight"]`
|
||||
- `note_payload["chunk_profile"]`
|
||||
|
||||
gesetzt und **1:1 an alle Chunks vererbt**.
|
||||
|
||||
---
|
||||
|
||||
## 4. Import-Pipeline (Markdown → Qdrant)
|
||||
|
||||
### 4.1 Runbook
|
||||
|
||||
Standard-Import (mit Purge-Before-Upsert):
|
||||
|
||||
```bash
|
||||
export COLLECTION_PREFIX=mindnet
|
||||
export MINDNET_TYPES_FILE="$(pwd)/config/types.yaml"
|
||||
|
||||
python3 -m scripts.import_markdown \
|
||||
--vault ./mindnet_v2_test_vault \
|
||||
--apply \
|
||||
--purge-before-upsert \
|
||||
--prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
|
||||
Nur Sync-Deletes (entfernte Dateien in Qdrant aufräumen):
|
||||
|
||||
```bash
|
||||
python3 -m scripts.import_markdown \
|
||||
--vault ./mindnet_v2_test_vault \
|
||||
--sync-deletes \
|
||||
--apply \
|
||||
--prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4.2 Verarbeitungsschritte
|
||||
|
||||
**1. Discovery & Parsing**
|
||||
|
||||
- `import_markdown.py` traversiert das Vault‐Verzeichnis.
|
||||
- Für jede `.md`:
|
||||
- YAML-Frontmatter extrahieren (id, title, type, created, updated, tags, …).
|
||||
- Body (Markdown) als String.
|
||||
|
||||
**2. Typauflösung & Payload-Bau (Note)**
|
||||
|
||||
- Laden & Auflösen von `types.yaml` (über ENV `MINDNET_TYPES_FILE` oder Default `./config/types.yaml`).
|
||||
- Bestimmung des effektiven Typs (`type` aus Frontmatter; ggf. Fallback).
|
||||
- Anwendung der Typdefaults:
|
||||
- `retriever_weight` und `chunk_profile` werden aus `types.yaml` gelesen.
|
||||
- Konstruktion des Note-Payloads via `make_note_payload` (in `note_payload.py`):
|
||||
- Felder wie `note_id`, `title`, `type`, `tags`, `updated`, `retriever_weight`, `chunk_profile`, `edge_defaults`, …
|
||||
|
||||
**3. Chunking**
|
||||
|
||||
- Übergabe des Note-Bodies + `chunk_profile` an `chunker.py`.
|
||||
- `chunker.py` nutzt `chunk_config.py`, das definierte Profile bereitstellt, z. B.:
|
||||
|
||||
```python
|
||||
CHUNK_PROFILES = {
|
||||
"short": {"max_tokens": 128, "overlap": 32},
|
||||
"medium": {"max_tokens": 256, "overlap": 64},
|
||||
"long": {"max_tokens": 512, "overlap": 128},
|
||||
"by_heading": {"strategy": "by_heading", "max_chars": 2000},
|
||||
"default": {"max_tokens": 256, "overlap": 64},
|
||||
}
|
||||
```
|
||||
|
||||
- Ergebnis: Liste geordneter Chunks mit:
|
||||
- `chunk_id` (`note_id#cXX`),
|
||||
- `index` (0..N-1),
|
||||
- `ord` (1..N),
|
||||
- `text`,
|
||||
- `window` (inkl. Nachbartexte),
|
||||
- `neighbors_prev` / `neighbors_next`.
|
||||
|
||||
**4. Chunk-Payloads**
|
||||
|
||||
- In `chunk_payload.py` werden für jeden Chunk die Payloads gebaut:
|
||||
- Vererbung von `note_id`, `type`, `tags`, `retriever_weight`, `chunk_profile`.
|
||||
- Setzen von `index`, `ord`, `neighbors_prev`, `neighbors_next`.
|
||||
- Persistieren von `text` und `window`.
|
||||
|
||||
**5. Kantenableitung (Edges)**
|
||||
|
||||
- `derive_edges.py` wird mit Note + Chunks + Typinformationen aufgerufen.
|
||||
- Es entstehen mehrere Gruppen von Kanten:
|
||||
1. **Struktur-Kanten**:
|
||||
- `belongs_to`: Chunk → Note
|
||||
- `next`: Chunk[i] → Chunk[i+1]
|
||||
- `prev`: Chunk[i] → Chunk[i-1]
|
||||
2. **Explizite Kanten aus Text**:
|
||||
- Wiki-Links: `[[Target Title]]` → `kind="references"`, `rule_id="explicit:wikilink"`, `confidence=1.0`.
|
||||
- Inline-Relationen: `[[rel:depends_on Embeddings 101]]`:
|
||||
- `kind="depends_on"`
|
||||
- `rule_id="inline:rel"`
|
||||
- `confidence≈0.95`
|
||||
- `target_id="Embeddings 101"` (Titel, nicht ID – Auflösung erfolgt später im Agenten).
|
||||
3. **Typ-basierte Defaults (`edge_defaults`)**:
|
||||
- Aus `types.yaml` je `type`: Liste von Relations-Kinds, z. B.:
|
||||
- `concept.edge_defaults = ["references","related_to"]`
|
||||
- `project.edge_defaults = ["references","depends_on"]`
|
||||
- Für jede explizite Referenz Quelle→Ziel (z. B. `references`, `similar_to`) werden **zusätzliche Kanten** erzeugt, z. B.:
|
||||
- `edge_defaults:project:depends_on`
|
||||
- `edge_defaults:concept:related_to`
|
||||
- `confidence≈0.7`.
|
||||
4. **Deduplication**:
|
||||
- Alle Kanten werden in einem Set anhand eines Keys wie
|
||||
- `(kind, scope, source_id, target_id, rule_id)`
|
||||
dedupliziert (idempotent).
|
||||
|
||||
**6. Upsert nach Qdrant**
|
||||
|
||||
- Punkte (Notes, Chunks, Edges) werden via `qdrant_points.py` in die entsprechenden Collections upserted.
|
||||
- Option `--purge-before-upsert` sorgt dafür, dass alte Punkte mit denselben IDs entfernt werden, bevor neue geschrieben werden (stabiler Reimport).
|
||||
- `--sync-deletes` entfernt Punkte, deren zugehörige Dateien im Vault nicht mehr existieren.
|
||||
|
||||
---
|
||||
|
||||
## 5. Edges im Detail – Semantik & Regeln
|
||||
|
||||
### 5.1 Struktur-Kanten
|
||||
|
||||
Erzeugt für **jede Note** mit >0 Chunks:
|
||||
|
||||
- **belongs_to**
|
||||
- pro Chunk genau eine Kante:
|
||||
- `source_id = chunk_id`
|
||||
- `target_id = note_id` *oder* (konzeptionell) ein Note-Identifikator
|
||||
- `kind = "belongs_to"`
|
||||
- `scope = "chunk"`
|
||||
- `rule_id = "structure:belongs_to"`
|
||||
- `confidence = 1.0`
|
||||
- **next / prev**
|
||||
- für Index i (`0..N-2`):
|
||||
- `next`: `chunk_i → chunk_{i+1}`
|
||||
- `prev`: `chunk_{i+1} → chunk_i`
|
||||
- `kind = "next"` bzw. `"prev"`
|
||||
- `scope = "chunk"`
|
||||
- `rule_id = "structure:next"` / `"structure:prev"`
|
||||
- `confidence = 1.0`
|
||||
|
||||
Tests (`tests.test_edges_smoke` / `tests.test_edges_all`) stellen sicher:
|
||||
|
||||
- `belongs_to` == `#chunks`
|
||||
- `next` == `prev` == `#chunks - 1`
|
||||
- keine Duplikate der Strukturkanten.
|
||||
|
||||
---
|
||||
|
||||
### 5.2 Explizite Wiki-Links (`[[Target Title]]`)
|
||||
|
||||
- Parser in `derive_edges.py` sucht im Chunk-Text nach Mustern:
|
||||
- `[[Some Page]]` (kein `rel:` Präfix)
|
||||
- Erzeugt Kanten:
|
||||
- `kind = "references"`
|
||||
- `scope = "chunk"`
|
||||
- `source_id = chunk_id`
|
||||
- `target_id = "<Titel aus dem Link>"` (z. B. `"Vector DB Basics"`)
|
||||
- `rule_id = "explicit:wikilink"`
|
||||
- `confidence = 1.0`
|
||||
|
||||
Diese Kanten bilden das **explizite Referenznetz** des Nutzers ab (Obsidian-typische Wikilinks).
|
||||
|
||||
---
|
||||
|
||||
### 5.3 Inline-Relationen (`[[rel:… …]]`)
|
||||
|
||||
Aktuell **implementiert und funktional** ist folgende Form:
|
||||
|
||||
- **Format (aktuell unterstützt):**
|
||||
|
||||
```md
|
||||
[[rel:depends_on Embeddings 101]]
|
||||
[[rel:similar_to Vector DB Basics]]
|
||||
[[rel:related_to Qdrant Vektordatenbank]]
|
||||
```
|
||||
|
||||
- Parsing-Strategie:
|
||||
- Link-Inhalt wird gesplittet:
|
||||
- Erster Token: `rel:depends_on`
|
||||
- Rest: Zielbezeichnung (`Embeddings 101`)
|
||||
- `kind` wird aus dem Teil `depends_on` extrahiert (z. B. `similar_to`, `related_to`).
|
||||
- `target_id` wird aus dem Rest (meist ein Titel) abgeleitet.
|
||||
|
||||
- Ergebnis-Kante:
|
||||
- `kind = "<Relation aus rel:…>"`
|
||||
- `scope = "chunk"`
|
||||
- `source_id = chunk_id`
|
||||
- `target_id = "<Ziel-Titel>"`
|
||||
- `rule_id = "inline:rel"`
|
||||
- `confidence ≈ 0.95`
|
||||
|
||||
- **Nicht unterstützt (Stand jetzt):**
|
||||
|
||||
```md
|
||||
rel: depends_on [[Embeddings 101]] [[Vector DB Basics]]
|
||||
```
|
||||
|
||||
Diese Syntax wird derzeit **nicht** erkannt.
|
||||
Multi-Target-Inline-Relationen (`rel: similar_to [[A]] [[B]]`) sind ein **offener Task**.
|
||||
|
||||
---
|
||||
|
||||
### 5.4 Typ-basierte Default-Edges (`edge_defaults`)
|
||||
|
||||
Zweck:
|
||||
|
||||
- Für bestimmte Typen (vor allem `concept`, `project`, `profile`) sollen zusätzliche Kanten automatisch ergänzt werden, um das **Semantiknetz zu verdichten**, ohne dass der Nutzer jede Relation explizit pflegen muss.
|
||||
|
||||
Konfiguration:
|
||||
|
||||
- In `types.yaml` definiert, z. B.:
|
||||
|
||||
```yaml
|
||||
types:
|
||||
concept:
|
||||
edge_defaults: ["references", "related_to"]
|
||||
project:
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
```
|
||||
|
||||
Aktuelle Implementierung:
|
||||
|
||||
- Für jede **explizite** Referenz (Wiki-Link oder Inline-Relation) von Chunk → Ziel:
|
||||
- `kind="references"` (Wiki) oder die Inline-Relation (`depends_on`, `similar_to`, …)
|
||||
- Auf Basis des Note-Typs (`type`) wird pro Default-Relation eine zusätzliche Kante erzeugt:
|
||||
|
||||
Beispiel:
|
||||
|
||||
- Note-Typ: `project`
|
||||
- `edge_defaults: ["references","depends_on"]`
|
||||
- Explizit: `references` von `Edge Index` → `Embeddings 101`
|
||||
- Abgeleitete Kanten:
|
||||
- `kind="references"`, `rule_id="edge_defaults:project:references"`, `confidence=0.7`
|
||||
- `kind="depends_on"`, `rule_id="edge_defaults:project:depends_on"`, `confidence=0.7`
|
||||
|
||||
- Symmetrische Kanten:
|
||||
- Für einige Relations-Typen (z. B. `related_to`) wird zusätzlich eine **spiegelbildliche** Kante Ziel → Quelle erzeugt.
|
||||
- Dies erklärt in Tests z. B. Paare wie:
|
||||
- `related_to (Source=Ollama LLM, Target=Qdrant Vektordatenbank)`
|
||||
- `related_to (Source=Qdrant Vektordatenbank, Target=Ollama LLM)`
|
||||
|
||||
- Deduplication:
|
||||
- Es wird verhindert, dass mehrfach identische Kanten geschrieben werden.
|
||||
- Schlüssel: `(kind, scope, source_id, target_id, rule_id)`.
|
||||
|
||||
---
|
||||
|
||||
### 5.5 `rule_id` & `confidence` – Bedeutung
|
||||
|
||||
**`rule_id`** kodiert **Herkunft** und **Regel**, z. B.:
|
||||
|
||||
- `structure:belongs_to`, `structure:next`, `structure:prev`
|
||||
- `explicit:wikilink`
|
||||
- `inline:rel`
|
||||
- `edge_defaults:<type>:<relation>`
|
||||
|
||||
**`confidence`** dient als späterer Gewichtungsfaktor im Retrieval / Ranking:
|
||||
|
||||
- `1.0` → harte Struktur- oder explizite Kante (vom Nutzer direkt gesetzt)
|
||||
- `0.95` → Inline-Relation (stark, aber nicht so „hart“ wie die reine Referenz)
|
||||
- `0.7` → Typ-Default-Edges (abgeleitete Hypothesen aus Typwissen)
|
||||
|
||||
Ein künftiger LLM-Agent kann daraus z. B.:
|
||||
|
||||
- Kanten mit hoher `confidence` priorisieren,
|
||||
- Default-Edges als schwächere Hinweise betrachten,
|
||||
- Inline-Relationen gezielt als semantische Aussagen über die Beziehung interpretieren.
|
||||
|
||||
---
|
||||
|
||||
## 6. Qdrant-Schema & Indizes
|
||||
|
||||
### 6.1 Erstellung & Indizes
|
||||
|
||||
Die Erstellung erfolgt über:
|
||||
|
||||
```bash
|
||||
python3 -m scripts.reset_qdrant --mode wipe --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
|
||||
Intern:
|
||||
|
||||
- `qdrant.py` / `setup_mindnet_collections.py`:
|
||||
- Collections anlegen, ggf. mit Vektor-Konfiguration (Single-Vector; Named-Vectors sind vorbereitet aber optional).
|
||||
- `ensure_payload_indexes()`:
|
||||
- Legt für jede Collection die relevanten Payload-Indizes an.
|
||||
|
||||
Verifizierbar durch:
|
||||
|
||||
```bash
|
||||
python3 -m tests.ensure_indexes_and_show
|
||||
```
|
||||
|
||||
Beispielauszug (bereits real so gesehen):
|
||||
|
||||
- **mindnet_notes**
|
||||
|
||||
```json
|
||||
{
|
||||
"tags": { "data_type": "keyword", "points": 8 },
|
||||
"note_id":{ "data_type": "keyword", "points": 8 },
|
||||
"title": { "data_type": "text", "points": 8 },
|
||||
"updated":{ "data_type": "integer", "points": 0 },
|
||||
"type": { "data_type": "keyword", "points": 8 }
|
||||
}
|
||||
```
|
||||
|
||||
- **mindnet_chunks**
|
||||
|
||||
```json
|
||||
{
|
||||
"note_id": { "data_type": "keyword", "points": 44 },
|
||||
"chunk_id":{ "data_type": "keyword", "points": 44 },
|
||||
"index": { "data_type": "integer", "points": 44 },
|
||||
"type": { "data_type": "keyword", "points": 44 },
|
||||
"tags": { "data_type": "keyword", "points": 44 }
|
||||
}
|
||||
```
|
||||
|
||||
- **mindnet_edges**
|
||||
|
||||
```json
|
||||
{
|
||||
"scope": { "data_type": "keyword", "points": 135 },
|
||||
"kind": { "data_type": "keyword", "points": 135 },
|
||||
"chunk_id":{ "data_type": "keyword", "points": 135 },
|
||||
"target_id":{ "data_type": "keyword", "points": 135 },
|
||||
"note_id": { "data_type": "keyword", "points": 135 },
|
||||
"source_id":{ "data_type": "keyword", "points": 135 }
|
||||
}
|
||||
```
|
||||
|
||||
### 6.2 Schema-Validierung
|
||||
|
||||
`tests/assert_payload_schema.py` prüft:
|
||||
|
||||
- ob die Minimalfelder (`note_id`, `type`, `title`, `updated`, `tags` etc.) vorhanden sind,
|
||||
- berichtet fehlende Felder.
|
||||
|
||||
Nach Korrektur der Indexanlage sollte:
|
||||
|
||||
- `ok = true` sein, bzw. zumindest alle Kernfelder unter `present` geführt werden.
|
||||
(Im aktuellen Stand werden sie als Indizes geführt; Qdrant speichert zusätzliche Felder ohne explizite Schema-Zeile, was technisch unproblematisch ist.)
|
||||
|
||||
---
|
||||
|
||||
## 7. Nutzung im Retrieval (Ausblick)
|
||||
|
||||
### 7.1 Typbewusstes Chunk-Retrieval
|
||||
|
||||
Ein typischer Query-Pfad eines LLM-Agents:
|
||||
|
||||
1. Query → Embedding
|
||||
2. Suche in `mindnet_chunks` mit Filter, z. B.:
|
||||
|
||||
```python
|
||||
f = rest.Filter(
|
||||
must=[
|
||||
rest.FieldCondition(key="type", match=rest.MatchValue(value="concept")),
|
||||
rest.FieldCondition(key="tags", match=rest.MatchAny(any=["ki","ollama"]))
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
3. Treffer werden nach:
|
||||
- Vektor-Score,
|
||||
- `retriever_weight`,
|
||||
- ggf. Edges (z. B. Projekte mit `depends_on` → Konzepte) gewichtet.
|
||||
|
||||
### 7.2 Graph-orientierte Navigation
|
||||
|
||||
Basierend auf `mindnet_edges` kann der Agent:
|
||||
|
||||
- ausgehend von einem relevanten Chunk:
|
||||
- Struktur-Kanten (`next`, `prev`) zur Kontextvergrößerung nutzen,
|
||||
- `references` / `depends_on` / `related_to` interpretieren,
|
||||
- `edge_defaults` als semantische Ausdehnung (z. B. von `project` zu seinen `concept`-Bausteinen) nutzen.
|
||||
|
||||
Die Bedeutung der Kanten ist dabei durch `rule_id` und `confidence` klar nachvollziehbar.
|
||||
|
||||
---
|
||||
|
||||
## 8. Offene Punkte / Known Limitations
|
||||
|
||||
1. **Multi-Target Inline-Relationen**
|
||||
Syntax wie:
|
||||
|
||||
```md
|
||||
[[rel:similar_to Vector DB Basics]] [[rel:similar_to Embeddings 101]]
|
||||
```
|
||||
funktioniert (zwei separate Inline-Links).
|
||||
**Nicht** unterstützt (Stand jetzt):
|
||||
|
||||
```md
|
||||
[[rel:similar_to Vector DB Basics]] [[Embeddings 101]]
|
||||
rel: similar_to [[Vector DB Basics]] [[Embeddings 101]]
|
||||
```
|
||||
|
||||
→ Offener Task: Parser erweitern, um mehrere Ziele in einer Inline-Relation zu unterstützen.
|
||||
|
||||
2. **Callout-Edges (`[!edge]`)**
|
||||
- Konzeptionell vorgesehen, z. B.:
|
||||
|
||||
```md
|
||||
> [!edge] related_to: [[Vector DB Basics]]
|
||||
```
|
||||
|
||||
- Im aktuellen Implementierungsstand werden Callouts entweder:
|
||||
- gar nicht, oder
|
||||
- nur teilweise verarbeitet (in den aktuellen Test-Runs war `callout_total = 0`).
|
||||
→ Offener Task: Callout-Parser robust implementieren und mit Tests absichern.
|
||||
|
||||
3. **Frontmatter-Overrides für `retriever_weight` / `chunk_profile`**
|
||||
- Aktuell **nicht** wirksam.
|
||||
- Typdefaults aus `types.yaml` sind der **Single Source of Truth**.
|
||||
- Später können kontrollierte Overrides (z. B. für Sonderfälle) eingeführt werden.
|
||||
|
||||
4. **Vektor-Konfiguration für Edges**
|
||||
- `mindnet_edges` wird derzeit ohne Vektoren betrieben (`vectors = null`).
|
||||
- Später könnte z. B. eine semantische Repräsentation von Kanten eingeführt werden (z. B. Embedding von `(source_chunk_text + relation + target_title)`).
|
||||
|
||||
---
|
||||
|
||||
## 9. Zusammenfassung
|
||||
|
||||
- Die Mindnet-V2-Architektur implementiert einen **dreistufigen Speicher**:
|
||||
- Notes (`mindnet_notes`),
|
||||
- Chunks (`mindnet_chunks`),
|
||||
- Edges (`mindnet_edges`).
|
||||
|
||||
- **`types.yaml`** steuert:
|
||||
- `retriever_weight` und `chunk_profile` je Typ (inkl. Default),
|
||||
- `edge_defaults` je Typ (automatische Relations-Ableitung).
|
||||
|
||||
- Das **Chunking** ist profilbasiert (short/medium/long/by_heading) und deterministisch.
|
||||
|
||||
- Die **Kantenableitung** in `derive_edges.py` erzeugt:
|
||||
- strukturelle Kanten (belongs_to / next / prev),
|
||||
- explizite Kanten aus Wiki-Links (`[[Target]]`),
|
||||
- semantische Inline-Relationen (`[[rel:depends_on Target]]`),
|
||||
- typbasierte Default-Kanten (z. B. `edge_defaults:project:depends_on`).
|
||||
|
||||
- `rule_id` und `confidence` machen die Herkunft jeder Kante transparent und ermöglichen später eine **gewichtete Auswertung** durch LLM-basierte Agenten.
|
||||
|
||||
- Qdrant-Collections und Payload-Indizes sind konsistent eingerichtet und durch Tests verifiziert.
|
||||
|
||||
Dieses Dokument bildet damit das aktuelle technische Gesamtbild von Mindnet V2 ab, inkl. aller relevanten Mechanismen rund um Typen, Chunking, Edges und Qdrant-Schema.
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
# mindnet v2 – Implementierungs-Playbook & Handover (Side‑by‑Side zu v1)
|
||||
*Stand:* 2025-11-10 08:03
|
||||
*Autor:* ChatGPT (Projekt‑Handover)
|
||||
*Zweck:* Vollständige, ausführbare Anleitung zur Weiterentwicklung von **mindnet** auf eine versionierte, testbare v2‑Architektur (neben v1), inklusive Prompt für den Folgethread, Akzeptanzkriterien, Dateiliste, Test‑ und Rollback‑Pläne.
|
||||
|
||||
---
|
||||
|
||||
## 0) Kontext & Zielbild (kurz)
|
||||
- **Mission:** Persönliches Wissensnetz, das langfristig die eigene Persönlichkeit, Erfahrungen und Entscheidungslogik abbildet; später nachvollziehbare Erklärungen für Familie/Nachwelt.
|
||||
- **Heutiger Stand (v1):** `mindnet_notes` in Qdrant vorhanden; **Chunks** und **Edges** bisher nur teilweise/nicht konsistent; Edge‑Defaults („1 Dot“) vorhanden, aber Ziele teils nicht materialisiert. Importskripte: `import_markdown.py`, `edges.py` (historisch).
|
||||
- **Ziel (v2):**
|
||||
- **Collections:** `mindnet_notes_v2`, `mindnet_chunks_v2`, `mindnet_edges_v2` (saubere Payload‑Schemata + Indizes).
|
||||
- **Chunker v2:** Block/Heading‑aware, Ziel 900–1200 Zeichen, Überlappung 120–180, deterministische `chunk_id`s.
|
||||
- **Edge‑Builder v2:** Reihenfolge: **explicit → rule → default_resolved → default**; saubere `provenance`.
|
||||
- **Importer‑Pipeline v2:** Parse → Chunk → Edge → Embeddings → Batch‑Upsert + Payload‑Indexpflege + Snapshots.
|
||||
- **Policies:** Privacy/Recency/Type‑Gewichte als **Konfiguration** (YAML + JSON‑Schema), nicht hart im Code.
|
||||
- **Observability:** FastAPI + OpenTelemetry (Tracing/Metrics); Import‑Reports (CSV/JSON).
|
||||
- **Side‑by‑Side:** v2 **parallel** zu v1; kein Big‑Bang.
|
||||
|
||||
> **Warum so?** Qdrant‑Payload‑Indizes/Filter → schnelle, erklärbare Selektionen; Snapshots → betriebssicheres Rollback; OTel → Nachvollziehbarkeit; YAML/JSON‑Schema → Validierbarkeit & Stabilität.
|
||||
|
||||
### 0.1 Aktueller Implementierungsstand (2025-11-18)
|
||||
|
||||
- Es existiert eine produktive mindnet-Instanz mit den Collections:
|
||||
- `${prefix}_notes`
|
||||
- `${prefix}_chunks`
|
||||
- `${prefix}_edges`
|
||||
- Import-Pipeline:
|
||||
- `scripts/import_markdown.py` erzeugt Notes, Chunks und Edges vollständig.
|
||||
- `types.yaml` definiert per Note-Typ:
|
||||
- `retriever_weight`
|
||||
- `chunk_profile`
|
||||
- `edge_defaults` (z. B. `depends_on`, `related_to`).
|
||||
- Edges:
|
||||
- Strukturkanten: `belongs_to`, `next`, `prev`
|
||||
- Explizite Kanten aus Wikilinks: `kind="references"`, `rule_id="explicit:wikilink"`
|
||||
- Typbasierte Default-Kanten aus `edge_defaults`: `rule_id="edge_defaults:<type>:<relation>"`
|
||||
- Inline-Kanten im Format `[[rel:<relation> <Zielnote>]]` mit `rule_id="inline:rel"`
|
||||
- alle Edges tragen eine `confidence` im Bereich `[0,1]` als Hinweis für spätere Ranking-Logiken.
|
||||
- Tests:
|
||||
- `tests/test_edges_smoke.py`, `tests/test_edges_all.py` und `scripts/edges_full_check.py` validieren Counts, Konsistenz (`belongs_to == chunks`, `next/prev`-Ketten, keine Duplikate) und Provenienz-Zählungen (`explicit`, `defaults`, `inline`, `structure`).
|
||||
- Offene Punkte:
|
||||
- Mehrere Ziele in einer Inline-Relation (`rel: similar_to [[A]] [[B]]`) sind noch nicht implementiert.
|
||||
- Retrieval-Service nutzt die neuen Edges und Weights bisher nur teilweise; ein edge-bewusster Retriever ist als nächster Schritt geplant.
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 1) Datenmodelle (Schemata, v2)
|
||||
Die tatsächlichen JSON‑Schemata (2020‑12) werden im Repo abgelegt und in CI validiert.
|
||||
|
||||
### 1.1 note.schema.json
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://example.org/mindnet/note.schema.json",
|
||||
"type": "object",
|
||||
"required": ["id", "title", "type", "privacy", "created", "hash_body"],
|
||||
"properties": {
|
||||
"id": { "type": "string", "pattern": "^[a-zA-Z0-9_-]+$" },
|
||||
"title": { "type": "string", "minLength": 1 },
|
||||
"type": { "type": "string" },
|
||||
"privacy": { "type": "string", "enum": ["public","internal","private"] },
|
||||
"created": { "type": "string", "format": "date-time" },
|
||||
"modified": { "type": "string", "format": "date-time" },
|
||||
"tags": { "type": "array", "items": { "type": "string" } },
|
||||
"lang": { "type": "string", "default": "de" },
|
||||
"source_path": { "type": "string" },
|
||||
"source_collection": { "type": "string" },
|
||||
"hash_body": { "type": "string" },
|
||||
"hash_frontmatter": { "type": "string" },
|
||||
"token_count": { "type": "integer", "minimum": 0 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 1.2 chunk.schema.json
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://example.org/mindnet/chunk.schema.json",
|
||||
"type": "object",
|
||||
"required": ["chunk_id","note_id","text","ord"],
|
||||
"properties": {
|
||||
"chunk_id": { "type": "string", "pattern": "^[a-zA-Z0-9_-]+#c\d{4}$" },
|
||||
"note_id": { "type": "string" },
|
||||
"text": { "type": "string" },
|
||||
"ord": { "type": "integer", "minimum": 0 },
|
||||
"span_char_start": { "type": "integer", "minimum": 0 },
|
||||
"span_char_end": { "type": "integer", "minimum": 0 },
|
||||
"heading_path": { "type": "array", "items": { "type": "string" } },
|
||||
"section_title": { "type": "string" },
|
||||
"tokens_start": { "type": "integer", "minimum": 0 },
|
||||
"tokens_end": { "type": "integer", "minimum": 0 },
|
||||
"embeddings_version": { "type": "string" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 1.3 edge.schema.json
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://example.org/mindnet/edge.schema.json",
|
||||
"type": "object",
|
||||
"required": ["edge_id","src_note_id","relation","provenance"],
|
||||
"properties": {
|
||||
"edge_id": { "type": "string" },
|
||||
"src_note_id": { "type": "string" },
|
||||
"src_chunk_id": { "type": "string" },
|
||||
"dst_note_id": { "type": "string" },
|
||||
"dst_chunk_id": { "type": "string" },
|
||||
"relation": { "type": "string" },
|
||||
"evidence_spans":{ "type": "array", "items": {"type":"object","properties":{"chunk_id":{"type":"string"},"span":[{"type":"integer"},{"type":"integer"}]]} },
|
||||
"provenance": { "type": "string", "enum": ["explicit","rule","default_resolved","default"] },
|
||||
"rule_id": { "type": "string" },
|
||||
"confidence": { "type": "number", "minimum": 0, "maximum": 1 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 1.4 default_edge.schema.json
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://example.org/mindnet/default_edge.schema.json",
|
||||
"type": "object",
|
||||
"required": ["src_note_id","relation","target_kind"],
|
||||
"properties": {
|
||||
"src_note_id": { "type": "string" },
|
||||
"relation": { "type": "string" },
|
||||
"target_kind": { "type": "string" },
|
||||
"when": { "type": "string" },
|
||||
"strength_hint":{ "type": "number", "minimum": 0, "maximum": 1 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2) Qdrant: Collections & Indizes (v2)
|
||||
Anlegen **neben** v1:
|
||||
- `mindnet_notes_v2` (Payload‑Index: `type, privacy, created, modified, tags, source_path`)
|
||||
- `mindnet_chunks_v2` (Payload‑Index: `note_id, ord, heading_path`)
|
||||
- `mindnet_edges_v2` (Payload‑Index: `src_note_id, dst_note_id, relation, provenance`)
|
||||
|
||||
> Snapshots für Backups/Restore einplanen (Runbook siehe unten).
|
||||
|
||||
---
|
||||
|
||||
## 3) Import‑Pipeline v2 – Module & Flags
|
||||
- **Chunker v2** (`chunking/block_chunker.py`): `target_len=1000`, `overlap=150`, `respect_headings=True`, `min_chunk_len=600`, `max_chunk_len=1400`, `chunk_id = f\"{note_id}#c{ordinal:04d}\"`
|
||||
- **Edge‑Builder v2** (`graph/edge_builder_v2.py`): Reihenfolge **explicit → rule → default_resolved → default**, `provenance` korrekt setzen.
|
||||
- **Importer** (`scripts/import_markdown.py` erweitern oder `scripts/import_markdown_v2.py`):
|
||||
Flags: `--schema v2`, `--chunker v2`, `--edges v2`, `--dry-run`, `--apply`, `--prefix "$COLLECTION_PREFIX"`.
|
||||
|
||||
**Beispiel‑Run (dry‑run):**
|
||||
```bash
|
||||
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --dry-run --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
|
||||
**Apply:**
|
||||
```bash
|
||||
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --apply --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4) Policies (konfigurierbar, kein Hard‑Code)
|
||||
- `policies/retrieval.schema.json` (JSON‑Schema) und `policies/retrieval.yaml`, z. B.:
|
||||
```yaml
|
||||
privacy_order: [private, internal, public]
|
||||
recency_boost_half_life_days: 90
|
||||
type_priority:
|
||||
person: 1.2
|
||||
event: 1.1
|
||||
concept: 1.0
|
||||
max_chunks_per_note: 8
|
||||
edge_provenance_weights:
|
||||
explicit: 1.0
|
||||
rule: 0.8
|
||||
default_resolved: 0.6
|
||||
default: 0.2
|
||||
rule_sets:
|
||||
- id: "rules:types-v1"
|
||||
enabled: true
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5) Observability & Reports
|
||||
- **FastAPI + OpenTelemetry**: Traces um Phasen *parse/chunk/edge/upsert*; Metriken (Counter/Histogramme) pro Phase.
|
||||
- **Import‑Report** je Lauf (CSV/JSON): counts für Notes/Chunks/Edges, Fehler, Dauer, Embedding‑Version.
|
||||
|
||||
---
|
||||
|
||||
## 6) Teststrategie
|
||||
- **Gold‑Notizen** (3–5 repräsentative .md) mit erwarteten *Chunk‑Counts* und *Edge‑Sets*.
|
||||
- **Regression:** fixer Soll‑Count (z. B. 171 ± 15 %) über Gesamt‑Vault; keine Duplikate `(src,relation,dst)`.
|
||||
- **Dry‑Run vs Apply**: identische Counts, nur ohne Upserts.
|
||||
- **Filter‑Smoke‑Tests:** Qdrant‑Filter nach `privacy`, `type`, `tags`, `created` funktionieren performant.
|
||||
|
||||
---
|
||||
|
||||
## 7) Akzeptanzkriterien (KPI)
|
||||
1. **Chunk‑Qualität:** ≥ 90 % der Chunks enden an semantischen Grenzen; Ziel‑Länge 900–1200 Zeichen; deterministische IDs.
|
||||
2. **Edge‑Kohärenz:** Keine Duplikate gleicher `(src,relation,dst)`; ≥ 95 % der expliziten Links materialisiert; `default` nur, wenn kein Ziel existiert.
|
||||
3. **Filterbarkeit:** Queries nach `privacy/type/tags/created` performant (Payload‑Index vorhanden).
|
||||
4. **Betriebsfestigkeit:** Telemetrie aktiv; fehlertoleranter Import; Qdrant‑Snapshot nach erfolgreichem Lauf.
|
||||
|
||||
---
|
||||
|
||||
## 8) Roadmap in kleinen, testbaren Schritten
|
||||
|
||||
### Step 0 – Safeguards
|
||||
- Qdrant‑Snapshot v1 erstellen + Restore‑Probe.
|
||||
- ENV: `MINDNET_SCHEMA_VERSION=2` (Importer wird v2 schreiben, v1 bleibt unberührt).
|
||||
|
||||
### Step 1 – Schemata (nur Dateien)
|
||||
- Ablage: `/schemas/note.schema.json`, `/schemas/chunk.schema.json`, `/schemas/edge.schema.json`, `/schemas/default_edge.schema.json`
|
||||
- CI‑Job: `make schema-validate` (jsonschema).
|
||||
- **Abnahme:** Validator grün auf 3 Gold‑Notizen.
|
||||
|
||||
### Step 2 – Qdrant v2‑Collections (Optional)
|
||||
- Anlegen der 3 v2‑Collections + Payload‑Indizes.
|
||||
- **Abnahme:** Filter‑Query liefert erwartbare Ergebnisse.
|
||||
|
||||
### Step 2a – Edge-Pipeline v1 stabilisieren (ERLEDIGT)
|
||||
- Ziel: Vollständige Generierung von Struktur-, expliziten, typbasierten und Inline-Kanten in den bestehenden Collections `${prefix}_notes`, `${prefix}_chunks`, `${prefix}_edges`.
|
||||
- Status: Implementiert und durch `tests/test_edges_smoke.py`, `tests/test_edges_all.py` und `scripts/edges_full_check.py` verifiziert.
|
||||
|
||||
|
||||
### Step 3 – Chunker v2
|
||||
- Implementierung & Flag `--chunker v2`.
|
||||
- **Abnahme:** Chunk‑Counts ~ alt (≈171 ± 15 %), semantische Schnitte.
|
||||
|
||||
### Step 4 – Edge‑Builder v2
|
||||
- Reihenfolge & `provenance` strikt umsetzen, `default_resolved` integrieren.
|
||||
- **Abnahme:** erwartete Relationensätze auf Gold‑Notizen, keine Duplikate.
|
||||
|
||||
### Step 4a – Edge-bewusster Retriever
|
||||
- Implementiere einen Retrieval-Service, der:
|
||||
- Chunks über Vektor-Suche aus `${prefix}_chunks` holt,
|
||||
- dazu passende Nachbar-Chunks (`next/prev`) und typrelevante Notizen (`edge_defaults`, Inline-Relationen, Wikilinks) über `${prefix}_edges` ergänzt,
|
||||
- `retriever_weight` und `confidence` beim Scoring berücksichtigt.
|
||||
- Liefere einen API-Endpoint `/mindnet/query` mit einem einfachen JSON-Response (Chunks + Begründung).
|
||||
- Akzeptanz:
|
||||
- mindestens ein Testfall mit dem Relations-Showcase-Vault,
|
||||
- sichtbarer Einfluss von `edge_defaults` und Inline-Relationen auf die Ergebnisliste.
|
||||
|
||||
|
||||
### Step 5 – Importer‑Pipeline v2
|
||||
- `--schema v2` Side‑by‑Side; Batch‑Upserts; vollständige Payloads; Recency‑Boost Konfig **nur** in Policy.
|
||||
- **Abnahme:** Dry‑Run/Apply‑Parität, Reports, Telemetrie‑Ereignisse sichtbar.
|
||||
|
||||
### Step 6 – Observability
|
||||
- OTel‑Instrumentierung + `/import/status` + Reports.
|
||||
- **Abnahme:** Traces & Metriken vorhanden.
|
||||
|
||||
### Step 7 – Snapshots & Runbook
|
||||
- Snapshot nach Erfolg; Restore‑Dokument.
|
||||
- **Abnahme:** Restore‑Probe erfolgreich.
|
||||
|
||||
### Step 8 – Policies
|
||||
- `policies/retrieval.yaml` + JSON‑Schema; A/B‑Test (Policy ändern → Retrieval ändert sich ohne Reimport).
|
||||
- **Abnahme:** Sichtbarer Effekt laut Testfall.
|
||||
|
||||
### Step 9 – Umschalten
|
||||
- Abnahme‑Doku, KPI‑Delta dokumentiert; Alias/Flag‑Switch auf v2.
|
||||
- **Abnahme:** Funktionale Gleichwertigkeit + Qualitätsgewinn.
|
||||
|
||||
---
|
||||
|
||||
## 9) Dateien (neu/ändern)
|
||||
**Neu:**
|
||||
- `schemas/note.schema.json`
|
||||
- `schemas/chunk.schema.json`
|
||||
- `schemas/edge.schema.json`
|
||||
- `schemas/default_edge.schema.json`
|
||||
- `policies/retrieval.schema.json`
|
||||
- `policies/retrieval.yaml`
|
||||
- `chunking/block_chunker.py`
|
||||
- `graph/edge_builder_v2.py`
|
||||
- `docs/OPERATIONS.md` (Snapshots/Restore/Runbook)
|
||||
- `tests/gold_notes/manifest.yaml` (+ 3–5 Notizen Kopien)
|
||||
|
||||
**Änderungen:**
|
||||
- `scripts/import_markdown.py` **oder** `scripts/import_markdown_v2.py` neu (Flags `--schema/--chunker/--edges`).
|
||||
- `edges.py` (falls weiterverwendet): auf **v2‑Edge‑Builder** migrieren oder auslaufen lassen.
|
||||
- `types.yaml`: Regel‑Definitionen klarisieren (IDs, Bedingungen), **nicht** hart in Code.
|
||||
|
||||
---
|
||||
|
||||
## 10) Beispiel‑Kommandos
|
||||
```bash
|
||||
# Step 2: Collections prüfen
|
||||
curl -s http://127.0.0.1:6333/collections | jq
|
||||
|
||||
# Step 3: Chunker v2 (dry-run)
|
||||
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --dry-run --prefix "$COLLECTION_PREFIX"
|
||||
|
||||
# Step 5: Apply + Report
|
||||
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --apply --prefix "$COLLECTION_PREFIX" --report ./reports/import_$(date +%F_%H%M).json
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 11) Rollback
|
||||
- v2‑Collections droppen (wenn Side‑by‑Side).
|
||||
- Aus Snapshot wiederherstellen (v1).
|
||||
- Flags zurück auf v1.
|
||||
|
||||
---
|
||||
|
||||
## 12) Referenzen (offizielle Quellen)
|
||||
- Qdrant – Collections, Payload‑Index, Filter: <https://qdrant.tech/documentation/>
|
||||
- Qdrant – Snapshots/Backup: <https://qdrant.tech/documentation/guides/backup/>
|
||||
- OpenTelemetry (Python/FastAPI): <https://opentelemetry.io/docs/instrumentation/python/>
|
||||
- JSON‑Schema 2020‑12: <https://json-schema.org/>
|
||||
- YAML 1.2: <https://yaml.org/spec/1.2.2/>
|
||||
- Obsidian Frontmatter: <https://help.obsidian.md/Editing+and+formatting/Properties>
|
||||
- Ollama CLI/API (lokale Modelle): <https://github.com/ollama/ollama>
|
||||
|
||||
---
|
||||
|
||||
## 13) **PROMPT für den neuen Chat (bitte exakt so einfügen)**
|
||||
```
|
||||
Rolle: Du bist mein Senior‑Entwickler & Architekt für mindnet. Arbeite strikt in kleinen, testbaren Schritten (Side‑by‑Side v2 neben v1). Liefere komplette Dateien als Downloads. Frage immer zuerst nach den **aktuellsten** Projektdateien, wenn du sie brauchst (z. B. import_markdown.py, edges.py, types.yaml), ändere keine Systemfunktion „workaround‑artig“ ohne die Gesamtwirkung zu prüfen.
|
||||
|
||||
Kontext (Kurzfassung):
|
||||
- v1 hat nur mindnet_notes zuverlässig. Chunks/Edges sind inkonsistent/teilweise leer. Ziel: v2 mit drei Collections (notes/chunks/edges), neuem Chunker v2, Edge‑Builder v2 (explicit → rule → default_resolved → default), Policies (YAML), Observability (OTel), Snapshots. Kein Big‑Bang; v2 parallel zu v1.
|
||||
- Siehe angehängte Datei **mindnet_v2_implementation_playbook.md** (Pfad/Download im Chat). Alles daraus ist verbindlich.
|
||||
|
||||
Deine ersten Aufgaben (Step‑by‑Step):
|
||||
1) **Schemata anlegen (Step 1)**
|
||||
- Erstelle die Dateien:
|
||||
- schemas/note.schema.json
|
||||
- schemas/chunk.schema.json
|
||||
- schemas/edge.schema.json
|
||||
- schemas/default_edge.schema.json
|
||||
- Nutze die in der Playbook‑Datei vorgegebenen Strukturen als Basis.
|
||||
- Liefere zusätzlich ein einfaches `Makefile`‑Ziel `schema-validate` (jsonschema via Python), plus `requirements.txt`.
|
||||
- Output: komplette Dateien als Downloads + kurze Testanleitung.
|
||||
|
||||
2) **Qdrant v2‑Collections (Step 2)**
|
||||
- Erzeuge die Collections + Payload‑Indizes.
|
||||
- Liefere ein kleines Python‑Skript `tools/qdrant_bootstrap_v2.py`, das diese Anlage idempotent durchführt (inkl. Prüf‑Output).
|
||||
|
||||
3) **Chunker v2 (Step 3)**
|
||||
- Implementiere `chunking/block_chunker.py` mit den im Playbook genannten Parametern.
|
||||
- Teste gegen 3 Gold‑Notizen (liefere mini Test‑Harness in `tests/gold_notes/…`).
|
||||
|
||||
Arbeitsweise:
|
||||
- Jede Aufgabe einzeln, mit klaren Akzeptanzkriterien (aus Playbook) und Download‑Artefakten.
|
||||
- Keine parallelen Großumbauten.
|
||||
- Immer deterministische IDs, idempotente Upserts.
|
||||
|
||||
Dateien, die ich dir anfänglich bereitstelle:
|
||||
- **mindnet_v2_implementation_playbook.md** (diese Datei)
|
||||
- **import_markdown.py** (aktuelle Version aus dem Projekt)
|
||||
- **edges.py** (aktuelle Version aus dem Projekt)
|
||||
- **types.yaml** (aktuelle Version)
|
||||
|
||||
Sage mir jeweils, welche Datei du als Nächstes brauchst. Beginne jetzt mit Aufgabe 1 (Schemata).
|
||||
```
|
||||
@@ -0,0 +1,260 @@
|
||||
# WP-04 / Step 4a – Retriever Scoring & Konfiguration
|
||||
|
||||
Dieses Dokument beschreibt den aktuellen Stand (2025-11-30) der Scoring-Logik des mindnet-Retrievers in WP-04 / Step 4a.
|
||||
Es dient als Referenz für:
|
||||
|
||||
- Nachvollziehbarkeit des Rankings
|
||||
- Konfiguration und Feintuning (ohne Codeänderungen)
|
||||
- spätere Automatisierung / „Selbstjustierung“ des Retrievers
|
||||
|
||||
---
|
||||
|
||||
## 1. Überblick
|
||||
|
||||
Der Retriever kombiniert drei Signalquellen zu einem einheitlichen Score:
|
||||
|
||||
1. **Semantik** – Vektorsuche über `mindnet_chunks`
|
||||
2. **Typ-Gewichtung** – `retriever_weight` pro Note/Chunk (aus `types.yaml`)
|
||||
3. **Graph-Boni** – `edge_bonus` und `centrality_bonus` aus dem Subgraph (Edges)
|
||||
|
||||
Die Berechnung erfolgt in `app/core/retriever.py` und ist über `config/retriever.yaml` konfigurierbar.
|
||||
Die API stellt die Funktionalität über den `/query`-Endpoint (FastAPI) bereit.
|
||||
|
||||
---
|
||||
|
||||
## 2. Datenbasis (Kurzüberblick)
|
||||
|
||||
### 2.1 Notes (`<prefix>_notes`)
|
||||
|
||||
Relevante Felder:
|
||||
|
||||
- note_id: keyword
|
||||
- title: text
|
||||
- type: keyword
|
||||
- retriever_weight: float
|
||||
- chunk_profile: keyword
|
||||
- edge_defaults: keyword[]
|
||||
- tags: keyword[]
|
||||
- path: text
|
||||
- fulltext: text
|
||||
|
||||
### 2.2 Chunks (`<prefix>_chunks`)
|
||||
|
||||
Relevante Felder:
|
||||
|
||||
- note_id: keyword
|
||||
- chunk_id: keyword
|
||||
- text: text
|
||||
- window: text
|
||||
- retriever_weight: float
|
||||
- type: keyword
|
||||
- path: text
|
||||
- section: text
|
||||
- neighbors_prev / neighbors_next
|
||||
- chunk_profile: keyword
|
||||
|
||||
### 2.3 Edges (`<prefix>_edges`)
|
||||
|
||||
Relevante Felder:
|
||||
|
||||
- kind: keyword
|
||||
- source_id: keyword
|
||||
- target_id: keyword
|
||||
- note_id: keyword
|
||||
- confidence: float
|
||||
- rule_id, provenance, edge_id, scope, relation, ref_text
|
||||
|
||||
Edges stammen stets aus der Import-/Edge-Pipeline (WP-03) und bilden die graphbasierte Grundlage des Retrievers.
|
||||
|
||||
---
|
||||
|
||||
## 3. Scoring-Formel (Retriever)
|
||||
|
||||
### 3.1 Eingangsgrößen
|
||||
|
||||
Für jeden Treffer gelten:
|
||||
|
||||
semantic_score – aus der Vektorsuche (Qdrant)
|
||||
retriever_weight – typabhängiges Gewicht (aus types.yaml)
|
||||
edge_bonus – aggregierter Graph-Bonus
|
||||
centrality_bonus – Zentralitätsbonus aus dem Subgraph
|
||||
|
||||
### 3.2 Globale Gewichte (aus retriever.yaml)
|
||||
|
||||
Datei: `config/retriever.yaml`
|
||||
Beispiel:
|
||||
|
||||
version: 1.0
|
||||
|
||||
scoring:
|
||||
semantic_weight: 1.0 # W_sem
|
||||
edge_weight: 0.7 # W_edge
|
||||
centrality_weight: 0.5 # W_cent
|
||||
|
||||
Falls diese Datei fehlt, greifen ENV-basierte Defaults (`RETRIEVER_W_*`).
|
||||
|
||||
### 3.3 Formel
|
||||
|
||||
total_score =
|
||||
W_sem * semantic_score * max(retriever_weight, 0.0)
|
||||
+ W_edge * edge_bonus
|
||||
+ W_cent * centrality_bonus
|
||||
|
||||
Bemerkungen:
|
||||
|
||||
- Negative `retriever_weight` werden abgefangen.
|
||||
- Die rohen `edge_bonus` und `centrality_bonus` werden im API-/Smoke-Output angezeigt.
|
||||
- Die Gewichte aus `retriever.yaml` beeinflussen **nur den total_score**, nicht die Anzeige.
|
||||
|
||||
---
|
||||
|
||||
## 4. Konfigurationsdateien
|
||||
|
||||
### 4.1 Typen-Registry (`types.yaml`)
|
||||
|
||||
Beispielauszug:
|
||||
|
||||
defaults:
|
||||
retriever_weight: 1.0
|
||||
chunk_profile: default
|
||||
|
||||
types:
|
||||
types:
|
||||
concept:
|
||||
retriever_weight: 0.60
|
||||
edge_defaults: ["references", "related_to"]
|
||||
|
||||
project:
|
||||
retriever_weight: 0.97
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
|
||||
journal:
|
||||
retriever_weight: 0.80
|
||||
edge_defaults: ["references", "related_to"]
|
||||
|
||||
source:
|
||||
retriever_weight: 0.50
|
||||
edge_defaults: []
|
||||
|
||||
Diese Werte bestimmen, wie wichtig eine Note/Chunk **prinzipiell** für den Retriever ist.
|
||||
|
||||
### 4.2 Retriever-Konfiguration (`retriever.yaml`)
|
||||
|
||||
Beispiel:
|
||||
|
||||
version: 1.0
|
||||
|
||||
scoring:
|
||||
semantic_weight: 1.0
|
||||
edge_weight: 0.7
|
||||
centrality_weight: 0.5
|
||||
|
||||
# Blueprint für spätere Edge-Feinsteuerung:
|
||||
edge_types:
|
||||
references: 0.20
|
||||
depends_on: 0.18
|
||||
related_to: 0.15
|
||||
similar_to: 0.12
|
||||
belongs_to: 0.10
|
||||
next: 0.06
|
||||
prev: 0.06
|
||||
|
||||
Die Datei befindet sich standardmäßig unter `config/retriever.yaml`.
|
||||
Override per ENV:
|
||||
|
||||
MINDNET_RETRIEVER_CONFIG=/pfad/zur/datei.yaml
|
||||
|
||||
### 4.3 Aktive Gewichte auslesen
|
||||
|
||||
python - << 'PY'
|
||||
from app.core.retriever import _get_scoring_weights
|
||||
print(_get_scoring_weights())
|
||||
PY
|
||||
|
||||
Beispielausgabe:
|
||||
|
||||
(1.0, 0.7, 0.5)
|
||||
|
||||
---
|
||||
|
||||
## 5. Betriebsmodi
|
||||
|
||||
### 5.1 Semantikmodus (semantic)
|
||||
|
||||
Ablauf:
|
||||
|
||||
- Query-Vektor über embed_text()
|
||||
- Qdrant-Suche über mindnet_chunks
|
||||
- kein Subgraph → edge_bonus = centrality = 0
|
||||
- total_score = W_sem * semantic_score * retriever_weight
|
||||
|
||||
### 5.2 Hybridmodus (hybrid)
|
||||
|
||||
Ablauf:
|
||||
|
||||
- Semantik wie oben
|
||||
- falls expand.depth > 0:
|
||||
- Subgraph-Berechnung über ga.expand()
|
||||
- Ermittlung von edge_bonus und centrality_bonus
|
||||
- total_score = kombinierte Formel aus Semantik + Graph
|
||||
|
||||
---
|
||||
|
||||
## 6. Tests
|
||||
|
||||
### Ausführung (Unit & Integration)
|
||||
|
||||
pytest tests/test_retriever_basic.py \
|
||||
tests/test_retriever_weight.py \
|
||||
tests/test_retriever_edges.py
|
||||
|
||||
pytest tests/test_query_unit.py \
|
||||
tests/test_query_text_embed_unit.py
|
||||
|
||||
Tests decken ab:
|
||||
|
||||
- Semantiktreffer
|
||||
- Typ-Gewichte
|
||||
- Edge-Boni
|
||||
- Hybridmodus / FastAPI-Endpunkt
|
||||
|
||||
### Smoke-Test
|
||||
|
||||
python tests/test_retriever_smoke.py \
|
||||
--url "http://127.0.0.1:8001/query" \
|
||||
--query "embeddings" \
|
||||
--mode hybrid \
|
||||
--expand-depth 1 \
|
||||
--top-k 5
|
||||
|
||||
Beispielausgabe:
|
||||
|
||||
[1] note_id=...retriever-design
|
||||
total=1.5054 (semantic=0.4038 edge=1.5910 centrality=0.0000)
|
||||
|
||||
Interpretation:
|
||||
|
||||
- `semantic`, `edge`, `centrality` = Rohwerte
|
||||
- `total` = skaliert mit `retriever.yaml`
|
||||
|
||||
---
|
||||
|
||||
## 7. Betriebs-Hinweise
|
||||
|
||||
### 7.1 Änderung der Gewichte
|
||||
|
||||
- retriever.yaml anpassen
|
||||
- FastAPI/uvicorn neu starten
|
||||
- Smoke-Test prüfen
|
||||
|
||||
### 7.2 Zielbild WP-04
|
||||
|
||||
Der jetzige Stand ermöglicht:
|
||||
|
||||
- transparente, konfigurierbare Score-Berechnung
|
||||
- Kombination aus Semantik, Typwissen und Graphstruktur
|
||||
- spätere agentenbasierte Selbstjustierung
|
||||
- alternative Profile (z. B. „mehr Semantik“, „mehr Graph“)
|
||||
|
||||
Dieses Dokument dient als stabile Referenz für Entwickler, Tests und spätere Automatisierungsschritte.
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tests/assert_payload_schema.py (compat v1.1)
|
||||
|
||||
Prüft, ob die erwarteten Payload-Indizes (payload_schema) auf den drei Collections vorhanden sind.
|
||||
Kompatibel mit älteren qdrant-client Versionen (ohne with_payload_schema im Wrapper).
|
||||
|
||||
Collections & Pflichtfelder:
|
||||
- mindnet_notes : note_id, type, title, updated, tags
|
||||
- mindnet_chunks : note_id, chunk_id, index, type, tags
|
||||
- mindnet_edges : note_id, kind, scope, source_id, target_id, chunk_id
|
||||
|
||||
Nutzung:
|
||||
python3 -m tests.assert_payload_schema
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
REQUIRED = {
|
||||
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||
"chunks": ["note_id", "chunk_id", "index", "type", "tags"],
|
||||
"edges": ["note_id", "kind", "scope", "source_id", "target_id", "chunk_id"],
|
||||
}
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
return obj if isinstance(obj, dict) else {}
|
||||
|
||||
def _cfg_base_url(cfg) -> str:
|
||||
if getattr(cfg, "url", None):
|
||||
return cfg.url.rstrip('/')
|
||||
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
if api_key:
|
||||
req.add_header("api-key", api_key)
|
||||
req.add_header("Authorization", f"Bearer {api_key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||
# 1) OpenAPI client mit Flag
|
||||
try:
|
||||
oc = getattr(client, "openapi_client", None)
|
||||
if oc is not None and hasattr(oc, "collections_api"):
|
||||
api = oc.collections_api
|
||||
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = _safe_model_dump(info)
|
||||
return (d.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
pass
|
||||
# 2) Wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||
try:
|
||||
info = client.get_collection(collection_name=name)
|
||||
d = _safe_model_dump(info)
|
||||
ps = (d.get("result") or {}).get("payload_schema")
|
||||
if ps is not None:
|
||||
return ps
|
||||
except Exception:
|
||||
pass
|
||||
# 3) Direkter HTTP-Call
|
||||
try:
|
||||
base = _cfg_base_url(cfg)
|
||||
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||
return (raw.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def ensure_fields(schema: Dict[str, dict] | None, required: List[str]) -> Dict[str, bool]:
|
||||
ks = set((schema or {}).keys())
|
||||
return {k: (k in ks) for k in required}
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
names = {"notes": notes, "chunks": chunks, "edges": edges}
|
||||
|
||||
report = {}
|
||||
ok_all = True
|
||||
for kind, col in names.items():
|
||||
sch = get_collection_payload_schema(client, cfg, col)
|
||||
checks = ensure_fields(sch, REQUIRED[kind])
|
||||
ok = all(checks.values())
|
||||
ok_all = ok_all and ok
|
||||
report[kind] = {
|
||||
"collection": col,
|
||||
"ok": ok,
|
||||
"missing": [k for k,v in checks.items() if not v],
|
||||
"present": [k for k,v in checks.items() if v],
|
||||
}
|
||||
|
||||
print(json.dumps({"prefix": cfg.prefix, "ok": ok_all, "report": report}, ensure_ascii=False, indent=2))
|
||||
if not ok_all:
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/diag_payload_indexes.py (compat v1.2)
|
||||
|
||||
Zeigt payload_schema je Collection; kompatibel mit älteren qdrant-client Versionen.
|
||||
Strategie (in dieser Reihenfolge):
|
||||
1) openapi_client.collections_api.get_collection(..., with_payload_schema=True)
|
||||
2) client.get_collection(...)
|
||||
3) HTTP GET /collections/{name}?with_payload_schema=true
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
return obj if isinstance(obj, dict) else {}
|
||||
|
||||
|
||||
def _cfg_base_url(cfg) -> str:
|
||||
if getattr(cfg, "url", None):
|
||||
return cfg.url.rstrip("/")
|
||||
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
|
||||
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
if api_key:
|
||||
req.add_header("api-key", api_key)
|
||||
req.add_header("Authorization", f"Bearer {api_key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
|
||||
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||
# Strategy 1: OpenAPI client with flag
|
||||
try:
|
||||
oc = getattr(client, "openapi_client", None)
|
||||
if oc is not None and hasattr(oc, "collections_api"):
|
||||
api = oc.collections_api
|
||||
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = _safe_model_dump(info)
|
||||
return (d.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
pass
|
||||
# Strategy 2: wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||
try:
|
||||
info = client.get_collection(collection_name=name) # kein with_payload_schema kwarg!
|
||||
d = _safe_model_dump(info)
|
||||
ps = (d.get("result") or {}).get("payload_schema")
|
||||
if ps is not None:
|
||||
return ps
|
||||
except Exception:
|
||||
pass
|
||||
# Strategy 3: direkter HTTP Call
|
||||
try:
|
||||
base = _cfg_base_url(cfg)
|
||||
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||
return (raw.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
cols = [notes, chunks, edges]
|
||||
out = []
|
||||
for name in cols:
|
||||
ps = get_collection_payload_schema(client, cfg, name)
|
||||
# Vektorinfos (optional)
|
||||
try:
|
||||
info = client.get_collection(collection_name=name)
|
||||
d = _safe_model_dump(info)
|
||||
vectors = (d.get("result") or {}).get("vectors")
|
||||
if isinstance(vectors, dict) and "config" in vectors:
|
||||
vectors = vectors["config"]
|
||||
except Exception:
|
||||
vectors = None
|
||||
out.append({"name": name, "vectors": vectors, "payload_schema": ps})
|
||||
print(json.dumps({"prefix": cfg.prefix, "collections": out}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
# scripts/edges_dryrun.py
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Dry-Run: Erzeuge Edges aus einem Vault **ohne** Qdrant-Upsert.
|
||||
- Liest Markdown mit YAML-Frontmatter
|
||||
- Chunking: einfacher Absatz-Chunker (Index + text)
|
||||
- Kanten: nutzt app.core.edges.build_edges_for_note()
|
||||
- Ausgabe: JSON pro Note mit Edge-Counts und 3 Beispiel-Payloads
|
||||
|
||||
Aufruf:
|
||||
python3 -m scripts.edges_dryrun --vault ./vault
|
||||
|
||||
Optional:
|
||||
--include-note-scope-refs # auch Frontmatter-Links (links[].target_id) als Note-Scope-Referenzen
|
||||
|
||||
Voraussetzungen:
|
||||
- app/core/parser.py (read_markdown, normalize_frontmatter)
|
||||
- app/core/edges.py (dieses Modul forwardet zur v2-Implementierung)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter
|
||||
from app.core.edges import build_edges_for_note
|
||||
|
||||
def _iter_markdown(vault: str):
|
||||
for p in Path(vault).rglob("*.md"):
|
||||
if p.name.startswith("."): # ignore hidden
|
||||
continue
|
||||
yield p
|
||||
|
||||
def _simple_chunker(body: str, note_id: str, note_type: str) -> List[Dict]:
|
||||
# Absatzbasiert, minimal ausreichend für Edges (window/text, chunk_id, ord, note_id, type)
|
||||
paras = [s.strip() for s in re.split(r"\n{2,}", body or "") if s.strip()]
|
||||
chunks = []
|
||||
for i, t in enumerate(paras):
|
||||
chunks.append({
|
||||
"chunk_id": f"{note_id}#c{i:04d}",
|
||||
"ord": i,
|
||||
"text": t,
|
||||
"note_id": note_id,
|
||||
"type": note_type,
|
||||
})
|
||||
return chunks
|
||||
|
||||
def _fm_note_refs(fm: Dict) -> List[str]:
|
||||
out = []
|
||||
links = fm.get("links") or []
|
||||
if isinstance(links, list):
|
||||
for e in links:
|
||||
if isinstance(e, dict):
|
||||
tid = e.get("target_id")
|
||||
if isinstance(tid, str) and tid.strip():
|
||||
out.append(tid.strip())
|
||||
return out
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True)
|
||||
ap.add_argument("--include-note-scope-refs", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
vault = args.vault
|
||||
include_note_scope = args.include_note_scope_refs
|
||||
|
||||
report = []
|
||||
for p in _iter_markdown(vault):
|
||||
parsed = read_markdown(str(p))
|
||||
if not parsed:
|
||||
continue
|
||||
fm = normalize_frontmatter(parsed.frontmatter or {})
|
||||
note_id = fm.get("id") or p.stem
|
||||
note_type = (fm.get("type") or "concept").lower()
|
||||
chunks = _simple_chunker(parsed.body, note_id, note_type)
|
||||
note_refs = _fm_note_refs(fm)
|
||||
|
||||
edges = build_edges_for_note(
|
||||
note_id=note_id,
|
||||
chunk_payloads=chunks,
|
||||
note_level_refs=note_refs,
|
||||
include_note_scope_refs=include_note_scope,
|
||||
)
|
||||
kinds = {}
|
||||
for e in edges:
|
||||
key = (e.get("relation") or e.get("kind") or "edge")
|
||||
kinds[key] = kinds.get(key, 0) + 1
|
||||
report.append({
|
||||
"path": str(p),
|
||||
"note_id": note_id,
|
||||
"type": note_type,
|
||||
"chunks": len(chunks),
|
||||
"edges_total": len(edges),
|
||||
"edges_by_kind": kinds,
|
||||
"samples": edges[:3],
|
||||
})
|
||||
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/edges_full_check.py
|
||||
Zählt und validiert Kanten in Qdrant. Erkennt folgende Rule-Gruppen:
|
||||
- explicit_total: rule_id startswith "explicit:" (z.B. explicit:wikilink, explicit:note_scope)
|
||||
- callout_total: rule_id == "callout:edge"
|
||||
- inline_total: rule_id startswith "inline:" (z.B. inline:rel)
|
||||
- defaults_total: rule_id startswith "edge_defaults:"
|
||||
- structure: rule_id in {"structure:belongs_to","structure:order"}
|
||||
|
||||
Gibt zusätzlich:
|
||||
- edges_by_kind (aggregiert)
|
||||
- notes/chunks/edges Anzahlen
|
||||
- multi_callout_detected: True, falls ein Chunk mehrere Callout-Ziele der gleichen Relation enthält
|
||||
- per_note_checks: belongs_to == chunks, next == prev == (chunks-1)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from typing import Dict, Any, List, Tuple
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
|
||||
def _count_collection_points(client, name: str) -> int:
|
||||
try:
|
||||
res = client.count(collection_name=name, exact=True)
|
||||
return res.count or 0
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def _scroll_all(client, collection: str) -> List[Any]:
|
||||
pts_all = []
|
||||
offset = None
|
||||
while True:
|
||||
pts, offset = client.scroll(
|
||||
collection_name=collection,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=2048,
|
||||
offset=offset,
|
||||
)
|
||||
pts_all.extend(pts or [])
|
||||
if offset is None:
|
||||
break
|
||||
return pts_all
|
||||
|
||||
|
||||
def _rule_group(rule_id: str) -> str:
|
||||
if not rule_id:
|
||||
return "unknown"
|
||||
if rule_id == "callout:edge":
|
||||
return "callout"
|
||||
if rule_id.startswith("inline:"): # <—— wichtig für "inline:rel"
|
||||
return "inline"
|
||||
if rule_id.startswith("edge_defaults:"):
|
||||
return "defaults"
|
||||
if rule_id.startswith("explicit:"):
|
||||
return "explicit"
|
||||
if rule_id in ("structure:belongs_to", "structure:order"):
|
||||
return "structure"
|
||||
return "other"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
|
||||
col_notes = f"{cfg.prefix}_notes"
|
||||
col_chunks = f"{cfg.prefix}_chunks"
|
||||
col_edges = f"{cfg.prefix}_edges"
|
||||
|
||||
# High-level counts
|
||||
notes_n = _count_collection_points(client, col_notes)
|
||||
chunks_n = _count_collection_points(client, col_chunks)
|
||||
edges_pts = _scroll_all(client, col_edges)
|
||||
edges_n = len(edges_pts)
|
||||
|
||||
# By kind / by rule group
|
||||
by_kind = Counter()
|
||||
group_counts = Counter()
|
||||
callout_buckets: Dict[Tuple[str, str], int] = defaultdict(int) # (chunk_id, kind) -> n targets
|
||||
per_note = defaultdict(lambda: {"chunks": 0, "belongs_to": 0, "next": 0, "prev": 0})
|
||||
|
||||
# Für per_note checks: chunks pro note_id aus mindnet_chunks laden
|
||||
chunks_pts = _scroll_all(client, col_chunks)
|
||||
chunks_by_note = Counter([p.payload.get("note_id") for p in chunks_pts if p.payload])
|
||||
|
||||
for p in edges_pts:
|
||||
pl = p.payload or {}
|
||||
kind = str(pl.get("kind") or pl.get("relation") or "edge")
|
||||
rule_id = str(pl.get("rule_id") or "")
|
||||
note_id = str(pl.get("note_id") or "")
|
||||
chunk_id = str(pl.get("chunk_id") or "")
|
||||
by_kind[kind] += 1
|
||||
|
||||
group = _rule_group(rule_id)
|
||||
group_counts[group] += 1
|
||||
|
||||
# Multi-Callout-Erkennung: mehrere callout-Edges gleicher Relation aus demselben Chunk
|
||||
if group == "callout" and chunk_id and kind:
|
||||
callout_buckets[(chunk_id, kind)] += 1
|
||||
|
||||
# Per-note Strukturchecks
|
||||
if note_id:
|
||||
if kind == "belongs_to":
|
||||
per_note[note_id]["belongs_to"] += 1
|
||||
elif kind == "next":
|
||||
per_note[note_id]["next"] += 1
|
||||
elif kind == "prev":
|
||||
per_note[note_id]["prev"] += 1
|
||||
|
||||
# set chunks count for per_note
|
||||
for n_id, c in chunks_by_note.items():
|
||||
per_note[n_id]["chunks"] = c
|
||||
|
||||
# final checks per note
|
||||
per_note_checks = {}
|
||||
for n_id, stats in per_note.items():
|
||||
c = stats.get("chunks", 0)
|
||||
bt = stats.get("belongs_to", 0)
|
||||
nx = stats.get("next", 0)
|
||||
pv = stats.get("prev", 0)
|
||||
per_note_checks[n_id] = {
|
||||
"chunks": c,
|
||||
"belongs_to": bt,
|
||||
"next": nx,
|
||||
"prev": pv,
|
||||
"checks": {
|
||||
"belongs_to_equals_chunks": (bt == c),
|
||||
"next_prev_match": (nx == pv == max(c - 1, 0)),
|
||||
},
|
||||
}
|
||||
|
||||
multi_callout_detected = any(v > 1 for v in callout_buckets.values())
|
||||
|
||||
out = {
|
||||
"prefix": cfg.prefix,
|
||||
"counts": {
|
||||
"notes": notes_n,
|
||||
"chunks": chunks_n,
|
||||
"edges": edges_n,
|
||||
"edges_by_kind": dict(by_kind),
|
||||
"explicit_total": group_counts.get("explicit", 0),
|
||||
"defaults_total": group_counts.get("defaults", 0),
|
||||
"callout_total": group_counts.get("callout", 0),
|
||||
"inline_total": group_counts.get("inline", 0),
|
||||
"structure_total": group_counts.get("structure", 0),
|
||||
},
|
||||
"per_note_checks": per_note_checks,
|
||||
"multi_callout_detected": bool(multi_callout_detected),
|
||||
"has_duplicates": False, # dedupe passiert beim Upsert
|
||||
}
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
scripts/health_check_mindnet.py
|
||||
|
||||
Health-Check für den mindnet-Retriever-/Query-Endpoint.
|
||||
|
||||
Funktion:
|
||||
- Führt POST-Requests auf /query in verschiedenen Modi aus (standard: semantic + hybrid).
|
||||
- Prüft Status-Code, JSON-Struktur und Anzahl der Treffer.
|
||||
- Kennzeichnet Probleme als:
|
||||
- status="ok"
|
||||
- status="warning" (z.B. Timeout)
|
||||
- status="error" (harte Fehler wie HTTP-Fehler, JSON-Fehler etc.)
|
||||
|
||||
Exit-Code:
|
||||
- Default (tolerant):
|
||||
- overall_status = "ok" (inkl. warnings) → Exit-Code 0
|
||||
- overall_status = "error" → Exit-Code 1
|
||||
- Mit --strict:
|
||||
- warnings werden wie errors behandelt → Exit-Code 1
|
||||
|
||||
Beispiele:
|
||||
|
||||
python scripts/health_check_mindnet.py \
|
||||
--url http://127.0.0.1:8001/query \
|
||||
--query "embeddings" \
|
||||
--top-k 3
|
||||
|
||||
python scripts/health_check_mindnet.py \
|
||||
--url http://127.0.0.1:8001/query \
|
||||
--query "embeddings" \
|
||||
--top-k 3 \
|
||||
--timeout 15 \
|
||||
--modes hybrid
|
||||
|
||||
# Strenger Modus (warnings → Exit-Code 1)
|
||||
python scripts/health_check_mindnet.py --strict
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List
|
||||
|
||||
|
||||
def _post_query(
|
||||
url: str,
|
||||
mode: str,
|
||||
query: str,
|
||||
top_k: int,
|
||||
timeout: float,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Führt einen POST auf den /query-Endpoint aus und gibt das dekodierte JSON zurück.
|
||||
|
||||
Raises:
|
||||
urllib.error.URLError, urllib.error.HTTPError, RuntimeError
|
||||
"""
|
||||
payload = {
|
||||
"mode": mode,
|
||||
"query": query,
|
||||
"top_k": top_k,
|
||||
"expand": {
|
||||
"depth": 1 if mode == "hybrid" else 0,
|
||||
"edge_types": ["references", "belongs_to", "prev", "next"],
|
||||
},
|
||||
"ret": {
|
||||
"with_paths": True,
|
||||
"with_notes": True,
|
||||
"with_chunks": True,
|
||||
},
|
||||
}
|
||||
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
url=url,
|
||||
data=data,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
|
||||
start = time.monotonic()
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
status_code = resp.getcode()
|
||||
body = resp.read().decode("utf-8")
|
||||
end = time.monotonic()
|
||||
|
||||
latency_ms = int((end - start) * 1000)
|
||||
|
||||
if status_code != 200:
|
||||
raise RuntimeError(f"HTTP {status_code}: {body}")
|
||||
|
||||
try:
|
||||
doc = json.loads(body)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"Antwort ist kein gültiges JSON: {exc}") from exc
|
||||
|
||||
if not isinstance(doc, dict):
|
||||
raise RuntimeError("Antwort ist kein JSON-Objekt")
|
||||
|
||||
doc["_latency_ms"] = latency_ms
|
||||
return doc
|
||||
|
||||
|
||||
def _validate_response(
|
||||
mode: str,
|
||||
doc: Dict[str, Any],
|
||||
min_results: int,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Prüft die wichtigsten Invarianten des Query-Responses.
|
||||
|
||||
Rückgabe-Format:
|
||||
|
||||
{
|
||||
"status": "ok" | "warning" | "error",
|
||||
"message": str,
|
||||
"latency_ms": int,
|
||||
"used_mode": str,
|
||||
"result_count": int
|
||||
}
|
||||
"""
|
||||
latency_ms = int(doc.get("_latency_ms", -1))
|
||||
used_mode = str(doc.get("used_mode", "unknown"))
|
||||
results = doc.get("results", [])
|
||||
|
||||
if not isinstance(results, list):
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "results ist kein Array",
|
||||
"latency_ms": latency_ms,
|
||||
"used_mode": used_mode,
|
||||
"result_count": -1,
|
||||
}
|
||||
|
||||
result_count = len(results)
|
||||
|
||||
if result_count < min_results:
|
||||
# Zu wenige Ergebnisse sind aus Sicht eines Health-Checks eher ein Error,
|
||||
# da die Retrieval-Qualität nicht gewährleistet ist.
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"zu wenige Ergebnisse: {result_count} < {min_results}",
|
||||
"latency_ms": latency_ms,
|
||||
"used_mode": used_mode,
|
||||
"result_count": result_count,
|
||||
}
|
||||
|
||||
sample_msg = "OK"
|
||||
if result_count > 0:
|
||||
first = results[0]
|
||||
if not isinstance(first, dict):
|
||||
sample_msg = "first result ist kein Objekt"
|
||||
else:
|
||||
# einfache Plausibilitätschecks (nur Zugriff, kein strikter Typcheck)
|
||||
_ = first.get("note_id")
|
||||
_ = first.get("chunk_id")
|
||||
_ = first.get("total_score")
|
||||
|
||||
status = "ok" if sample_msg == "OK" else "warning"
|
||||
return {
|
||||
"status": status,
|
||||
"message": sample_msg,
|
||||
"latency_ms": latency_ms,
|
||||
"used_mode": used_mode,
|
||||
"result_count": result_count,
|
||||
}
|
||||
|
||||
|
||||
def run_health_check(
|
||||
url: str,
|
||||
query: str,
|
||||
top_k: int,
|
||||
timeout: float,
|
||||
modes: List[str],
|
||||
min_results: int,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Führt den Health-Check für die angegebenen Modi aus.
|
||||
"""
|
||||
report: Dict[str, Any] = {
|
||||
"url": url,
|
||||
"query": query,
|
||||
"top_k": top_k,
|
||||
"timeout_s": timeout,
|
||||
"checks": [],
|
||||
"overall_status": "ok", # wird unten ggf. auf warning oder error gesetzt
|
||||
}
|
||||
|
||||
overall_status = "ok"
|
||||
|
||||
for mode in modes:
|
||||
entry: Dict[str, Any] = {
|
||||
"mode": mode,
|
||||
"status": "error",
|
||||
"message": "",
|
||||
}
|
||||
try:
|
||||
doc = _post_query(url=url, mode=mode, query=query, top_k=top_k, timeout=timeout)
|
||||
validation = _validate_response(mode=mode, doc=doc, min_results=min_results)
|
||||
entry.update(validation)
|
||||
|
||||
except urllib.error.HTTPError as exc:
|
||||
entry["status"] = "error"
|
||||
entry["message"] = f"HTTP-Fehler: {exc}"
|
||||
entry["latency_ms"] = -1
|
||||
entry["used_mode"] = mode
|
||||
entry["result_count"] = -1
|
||||
|
||||
except urllib.error.URLError as exc:
|
||||
# URLError kann u.a. socket.timeout enthalten.
|
||||
lat = -1
|
||||
entry["latency_ms"] = lat
|
||||
entry["used_mode"] = mode
|
||||
entry["result_count"] = -1
|
||||
|
||||
if isinstance(exc.reason, socket.timeout):
|
||||
entry["status"] = "warning"
|
||||
entry["message"] = f"Timeout (URLError/socket.timeout): {exc}"
|
||||
else:
|
||||
entry["status"] = "error"
|
||||
entry["message"] = f"HTTP/Netzwerkfehler: {exc}"
|
||||
|
||||
except socket.timeout as exc:
|
||||
# Direkter Timeout (falls nicht in URLError verpackt)
|
||||
entry["status"] = "warning"
|
||||
entry["message"] = f"Timeout (socket.timeout): {exc}"
|
||||
entry["latency_ms"] = -1
|
||||
entry["used_mode"] = mode
|
||||
entry["result_count"] = -1
|
||||
|
||||
except Exception as exc: # noqa: BLE001
|
||||
entry["status"] = "error"
|
||||
entry["message"] = f"unerwarteter Fehler: {exc}"
|
||||
entry["latency_ms"] = -1
|
||||
entry["used_mode"] = mode
|
||||
entry["result_count"] = -1
|
||||
|
||||
# overall_status bestimmen:
|
||||
# - error > warning > ok
|
||||
if entry["status"] == "error":
|
||||
overall_status = "error"
|
||||
elif entry["status"] == "warning" and overall_status == "ok":
|
||||
overall_status = "warning"
|
||||
|
||||
report["checks"].append(entry)
|
||||
|
||||
report["overall_status"] = overall_status
|
||||
return report
|
||||
|
||||
|
||||
def parse_args(argv: List[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Health-Check für mindnet /query-Retriever",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--url",
|
||||
default="http://127.0.0.1:8001/query",
|
||||
help="Vollständige URL des /query-Endpunkts (Default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--query",
|
||||
default="embeddings",
|
||||
help="Test-Query für den Health-Check (Default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--top-k",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Top-K für die Testabfrage (Default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=5.0,
|
||||
help="Timeout in Sekunden pro Request (Default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--modes",
|
||||
nargs="+",
|
||||
default=["semantic", "hybrid"],
|
||||
help="Zu prüfende Modi (Default: semantic hybrid)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-results",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Minimale Anzahl erwarteter Ergebnisse (Default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help="Warnings als Fehler behandeln (overall_status=warning führt zu Exit-Code 1)",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: List[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
|
||||
report = run_health_check(
|
||||
url=args.url,
|
||||
query=args.query,
|
||||
top_k=args.top_k,
|
||||
timeout=args.timeout,
|
||||
modes=args.modes,
|
||||
min_results=args.min_results,
|
||||
)
|
||||
|
||||
overall = report.get("overall_status", "error")
|
||||
|
||||
# Menschlich lesbare Zusammenfassung:
|
||||
print(f"mindnet health: {overall}")
|
||||
for c in report["checks"]:
|
||||
mode = c.get("mode")
|
||||
status = c.get("status")
|
||||
latency = c.get("latency_ms")
|
||||
result_count = c.get("result_count")
|
||||
msg = c.get("message", "")
|
||||
print(f" - {mode}: {status} (latency={latency} ms, results={result_count}) {msg}")
|
||||
|
||||
# JSON-Ausgabe (z.B. für n8n):
|
||||
print(json.dumps(report, ensure_ascii=False))
|
||||
|
||||
if args.strict:
|
||||
# strict: warning wird wie error behandelt
|
||||
return 0 if overall == "ok" else 1
|
||||
else:
|
||||
# tolerant: nur echte errors führen zu Exit-Code 1
|
||||
return 0 if overall in ("ok", "warning") else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -124,7 +124,7 @@ def load_type_registry() -> dict:
|
||||
p = _env("MINDNET_TYPES_FILE", None)
|
||||
if p and os.path.exists(p):
|
||||
return _load_json_or_yaml(p)
|
||||
fallback = os.path.abspath("./types.yaml")
|
||||
fallback = os.path.abspath("./config/types.yaml") if os.path.exists("./config/types.yaml") else os.path.abspath("./types.yaml")
|
||||
return _load_json_or_yaml(fallback)
|
||||
|
||||
def get_type_config(note_type: Optional[str], reg: dict) -> dict:
|
||||
@@ -144,19 +144,43 @@ def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
||||
return "concept" if "concept" in types else (requested or "concept")
|
||||
|
||||
def effective_chunk_profile(note_type: str, reg: dict) -> Optional[str]:
|
||||
"""Resolve chunk_profile for type or from defaults/global.
|
||||
Accepts symbolic profiles: short|medium|long|default.
|
||||
"""
|
||||
cfg = get_type_config(note_type, reg)
|
||||
prof = cfg.get("chunk_profile")
|
||||
if isinstance(prof, str) and prof in {"short", "medium", "long"}:
|
||||
prof = (cfg.get("chunk_profile") if isinstance(cfg, dict) else None)
|
||||
if isinstance(prof, str) and prof:
|
||||
return prof
|
||||
return None
|
||||
# defaults fallbacks
|
||||
for key in ("defaults", "default", "global"):
|
||||
dcfg = reg.get(key) if isinstance(reg, dict) else None
|
||||
if isinstance(dcfg, dict):
|
||||
dprof = dcfg.get("chunk_profile")
|
||||
if isinstance(dprof, str) and dprof:
|
||||
return dprof
|
||||
return "default"
|
||||
|
||||
def effective_retriever_weight(note_type: str, reg: dict) -> Optional[float]:
|
||||
"""Resolve retriever_weight for type or defaults; returns float.
|
||||
"""
|
||||
cfg = get_type_config(note_type, reg)
|
||||
w = cfg.get("retriever_weight")
|
||||
w = (cfg.get("retriever_weight") if isinstance(cfg, dict) else None)
|
||||
try:
|
||||
return float(w) if w is not None else None
|
||||
if w is not None:
|
||||
return float(w)
|
||||
except Exception:
|
||||
return None
|
||||
pass
|
||||
# defaults fallbacks
|
||||
for key in ("defaults", "default", "global"):
|
||||
dcfg = reg.get(key) if isinstance(reg, dict) else None
|
||||
if isinstance(dcfg, dict):
|
||||
dw = dcfg.get("retriever_weight")
|
||||
try:
|
||||
if dw is not None:
|
||||
return float(dw)
|
||||
except Exception:
|
||||
pass
|
||||
return 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/payload_dryrun.py (zeigt, was VOR dem Upsert tatsächlich in den Payloads steht)
|
||||
- KEIN Überschreiben der Note-Payload mehr
|
||||
- types.yaml ist maßgeblich (gemäß app/core/note_payload.py & chunk_payload.py)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, os, json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunk_payload import make_chunk_payloads
|
||||
try:
|
||||
from app.core.derive_edges import build_edges_for_note
|
||||
except Exception:
|
||||
from app.core.edges import build_edges_for_note # type: ignore
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True)
|
||||
ap.add_argument("--note-id")
|
||||
ap.add_argument("--with-edges", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = os.path.abspath(args.vault)
|
||||
|
||||
files: List[str] = []
|
||||
for dp, _, fns in os.walk(root):
|
||||
for fn in fns:
|
||||
if fn.lower().endswith(".md"):
|
||||
files.append(os.path.join(dp, fn))
|
||||
files.sort()
|
||||
|
||||
for path in files:
|
||||
parsed = read_markdown(path)
|
||||
if not parsed:
|
||||
continue
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
try:
|
||||
validate_required_frontmatter(fm)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"invalid frontmatter: {e}"}))
|
||||
continue
|
||||
|
||||
if args.note_id and fm.get("id") != args.note_id:
|
||||
continue
|
||||
|
||||
# Note-Payload exakt so, wie der Importer ihn baut (types.yaml maßgeblich)
|
||||
note_pl = make_note_payload(parsed,
|
||||
vault_root=root,
|
||||
hash_mode="body",
|
||||
hash_normalize="canonical",
|
||||
hash_source="parsed",
|
||||
file_path=path)
|
||||
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
chunks = assemble_chunks(fm["id"], body_text, fm.get("type","concept"))
|
||||
|
||||
chunk_note = {
|
||||
"frontmatter": fm,
|
||||
"id": fm.get("id"),
|
||||
"type": fm.get("type"),
|
||||
"title": fm.get("title"),
|
||||
"path": note_pl.get("path") or path,
|
||||
"note_id": note_pl.get("note_id"),
|
||||
"tags": fm.get("tags"),
|
||||
}
|
||||
# make_chunk_payloads bestimmt Werte ebenfalls aus types.yaml (Frontmatter wird ignoriert)
|
||||
chunk_pls = make_chunk_payloads(
|
||||
chunk_note,
|
||||
note_pl["path"],
|
||||
chunks,
|
||||
note_text=body_text,
|
||||
types_cfg=None, # Loader aus Datei; kein Override von außen
|
||||
file_path=path,
|
||||
)
|
||||
|
||||
out = {
|
||||
"note_id": note_pl.get("note_id") or fm.get("id"),
|
||||
"title": fm.get("title"),
|
||||
"type": fm.get("type"),
|
||||
"note_payload": {
|
||||
"retriever_weight": note_pl.get("retriever_weight"),
|
||||
"chunk_profile": note_pl.get("chunk_profile")
|
||||
},
|
||||
"chunks_summary": {
|
||||
"count": len(chunk_pls),
|
||||
"first": [
|
||||
{k: chunk_pls[i].get(k) for k in ("chunk_id","index","ord","retriever_weight","chunk_profile","neighbors_prev","neighbors_next")}
|
||||
for i in range(min(3, len(chunk_pls)))
|
||||
]
|
||||
},
|
||||
"path": note_pl.get("path")
|
||||
}
|
||||
|
||||
if args.with_edges:
|
||||
edges = build_edges_for_note(
|
||||
note_id=note_pl.get("note_id") or fm.get("id"),
|
||||
chunk_payloads=chunk_pls,
|
||||
note_level_refs=note_pl.get("references") or [],
|
||||
include_note_scope_refs=False,
|
||||
)
|
||||
kinds = {}
|
||||
for e in edges:
|
||||
k = (e.get("relation") or e.get("kind") or "edge")
|
||||
kinds[k] = kinds.get(k, 0) + 1
|
||||
out["edges_summary"] = {"total": len(edges), "by_kind": kinds}
|
||||
|
||||
print(json.dumps(out, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+33
-10
@@ -2,28 +2,33 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Name: scripts/reset_qdrant.py
|
||||
Version: v1.1.1 (2025-09-05)
|
||||
Version: v1.2.0 (2025-11-11)
|
||||
Kurzbeschreibung:
|
||||
Sicheres Zurücksetzen der Qdrant-Collections für EIN Projektpräfix. Das Skript
|
||||
ermittelt zunächst die tatsächlich betroffenen Collections und zeigt eine
|
||||
Übersicht an. Anschließend wird der User um Bestätigung gebeten (interaktive
|
||||
Abfrage), bevor eine der beiden Aktionen ausgeführt wird:
|
||||
Abfrage), bevor eine der Aktionen ausgeführt wird:
|
||||
|
||||
- wipe: Löscht die Collections komplett und legt sie gemäß Projekt-Defaults neu an
|
||||
- truncate: Löscht nur alle Points (Inhalte), behält die Collection-Settings
|
||||
- wipe: Löscht die Collections komplett und legt sie gemäß Projekt-Defaults neu an.
|
||||
**Neu**: Danach werden *immer* die Payload-Indizes & Schema-Ergänzungen
|
||||
über ensure_payload_indexes() idempotent eingerichtet (falls nicht mit --no-indexes deaktiviert).
|
||||
- truncate: Löscht nur alle Points (Inhalte), behält die Collection-Settings.
|
||||
**Neu**: Optional (Default) werden im Anschluss die Payload-Indizes geprüft/ergänzt.
|
||||
|
||||
Aufruf (aus Projekt-Root, im venv):
|
||||
python3 -m scripts.reset_qdrant --mode wipe [--prefix PREFIX] [--yes] [--dry-run]
|
||||
python3 -m scripts.reset_qdrant --mode truncate [--prefix PREFIX] [--yes] [--dry-run]
|
||||
|
||||
Parameter:
|
||||
--mode wipe | truncate (Pflicht)
|
||||
--prefix Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')
|
||||
--yes Keine Rückfrage, direkt ausführen (CI/CD geeignet)
|
||||
--dry-run Nur anzeigen, was passieren würde; nichts ändern
|
||||
--mode wipe | truncate (Pflicht)
|
||||
--prefix Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')
|
||||
--yes Keine Rückfrage, direkt ausführen (CI/CD geeignet)
|
||||
--dry-run Nur anzeigen, was passieren würde; nichts ändern
|
||||
--no-indexes Überspringt den Schritt ensure_payload_indexes() (Standard: Indizes/Schema werden geprüft/ergänzt)
|
||||
|
||||
Umgebungsvariablen (optional):
|
||||
QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)
|
||||
(Zusätzliche ENV für Named Vectors etc. werden innerhalb ensure_collections/ensure_payload_indexes berücksichtigt.)
|
||||
|
||||
Sicherheitsmerkmale:
|
||||
- Betrachtet ausschließlich Collections des angegebenen Präfixes.
|
||||
@@ -34,6 +39,8 @@ Exitcodes:
|
||||
0 = OK, 1 = abgebrochen/keine Aktion, 2 = Verbindungs-/Konfigurationsfehler
|
||||
|
||||
Changelog:
|
||||
v1.2.0: ensure_payload_indexes() nach wipe/truncate standardmäßig ausführen (idempotent); --no-indexes Flag ergänzt.
|
||||
v1.1.1: Stabilisierung & Preview (2025-09-05).
|
||||
v1.1.0: Interaktive Bestätigung, --yes/--dry-run hinzugefügt, Preview der betroffenen Collections.
|
||||
v1.0.0: Wipe/Truncate ohne Bestätigungsabfrage.
|
||||
"""
|
||||
@@ -46,7 +53,13 @@ from typing import List
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names
|
||||
from app.core.qdrant import (
|
||||
QdrantConfig,
|
||||
get_client,
|
||||
ensure_collections,
|
||||
ensure_payload_indexes,
|
||||
collection_names,
|
||||
)
|
||||
|
||||
|
||||
def resolve_existing_collections(client: QdrantClient, prefix: str) -> List[str]:
|
||||
@@ -111,12 +124,13 @@ def wipe_collections(client: QdrantClient, all_col_names: List[str], existing: L
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Wipe oder truncate mindnet-Collections in Qdrant (mit Bestätigung).")
|
||||
ap = argparse.ArgumentParser(description="Wipe oder truncate mindnet-Collections in Qdrant (mit Bestätigung & Index-Setup).")
|
||||
ap.add_argument("--mode", choices=["wipe", "truncate"], required=True,
|
||||
help="wipe = Collections löschen & neu anlegen; truncate = nur Inhalte löschen")
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')")
|
||||
ap.add_argument("--yes", action="store_true", help="Ohne Rückfrage ausführen (CI/CD)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="Nur anzeigen, was passieren würde; nichts ändern")
|
||||
ap.add_argument("--no-indexes", action="store_true", help="Überspringt ensure_payload_indexes()")
|
||||
args = ap.parse_args()
|
||||
|
||||
# Qdrant-Konfiguration
|
||||
@@ -155,13 +169,22 @@ def main():
|
||||
if args.mode == "wipe":
|
||||
wipe_collections(client, all_col_names, existing)
|
||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
if not args.no_indexes:
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
print(f"Wiped & recreated (Prefix={cfg.prefix}): {all_col_names}")
|
||||
if not args.no_indexes:
|
||||
print("Payload-Indizes & Schema-Ergänzungen geprüft/ergänzt (ensure_payload_indexes).")
|
||||
else:
|
||||
if not existing:
|
||||
print("Keine existierenden Collections zum Truncaten gefunden. Beende ohne Aktion.")
|
||||
sys.exit(0)
|
||||
delete_all_points(client, existing)
|
||||
if not args.no_indexes:
|
||||
# Auch nach truncate sicherstellen, dass neue/fehlende Indizes ergänzt werden
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
print(f"Truncated (deleted all points in): {existing}")
|
||||
if not args.no_indexes:
|
||||
print("Payload-Indizes & Schema-Ergänzungen geprüft/ergänzt (ensure_payload_indexes).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tests/assert_payload_schema.py (compat v1.1)
|
||||
|
||||
Prüft, ob die erwarteten Payload-Indizes (payload_schema) auf den drei Collections vorhanden sind.
|
||||
Kompatibel mit älteren qdrant-client Versionen (ohne with_payload_schema im Wrapper).
|
||||
|
||||
Collections & Pflichtfelder:
|
||||
- mindnet_notes : note_id, type, title, updated, tags
|
||||
- mindnet_chunks : note_id, chunk_id, index, type, tags
|
||||
- mindnet_edges : note_id, kind, scope, source_id, target_id, chunk_id
|
||||
|
||||
Nutzung:
|
||||
python3 -m tests.assert_payload_schema
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
REQUIRED = {
|
||||
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||
"chunks": ["note_id", "chunk_id", "index", "type", "tags"],
|
||||
"edges": ["note_id", "kind", "scope", "source_id", "target_id", "chunk_id"],
|
||||
}
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
return obj if isinstance(obj, dict) else {}
|
||||
|
||||
def _cfg_base_url(cfg) -> str:
|
||||
if getattr(cfg, "url", None):
|
||||
return cfg.url.rstrip('/')
|
||||
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
if api_key:
|
||||
req.add_header("api-key", api_key)
|
||||
req.add_header("Authorization", f"Bearer {api_key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||
# 1) OpenAPI client mit Flag
|
||||
try:
|
||||
oc = getattr(client, "openapi_client", None)
|
||||
if oc is not None and hasattr(oc, "collections_api"):
|
||||
api = oc.collections_api
|
||||
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = _safe_model_dump(info)
|
||||
return (d.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
pass
|
||||
# 2) Wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||
try:
|
||||
info = client.get_collection(collection_name=name)
|
||||
d = _safe_model_dump(info)
|
||||
ps = (d.get("result") or {}).get("payload_schema")
|
||||
if ps is not None:
|
||||
return ps
|
||||
except Exception:
|
||||
pass
|
||||
# 3) Direkter HTTP-Call
|
||||
try:
|
||||
base = _cfg_base_url(cfg)
|
||||
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||
return (raw.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def ensure_fields(schema: Dict[str, dict] | None, required: List[str]) -> Dict[str, bool]:
|
||||
ks = set((schema or {}).keys())
|
||||
return {k: (k in ks) for k in required}
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
names = {"notes": notes, "chunks": chunks, "edges": edges}
|
||||
|
||||
report = {}
|
||||
ok_all = True
|
||||
for kind, col in names.items():
|
||||
sch = get_collection_payload_schema(client, cfg, col)
|
||||
checks = ensure_fields(sch, REQUIRED[kind])
|
||||
ok = all(checks.values())
|
||||
ok_all = ok_all and ok
|
||||
report[kind] = {
|
||||
"collection": col,
|
||||
"ok": ok,
|
||||
"missing": [k for k,v in checks.items() if not v],
|
||||
"present": [k for k,v in checks.items() if v],
|
||||
}
|
||||
|
||||
print(json.dumps({"prefix": cfg.prefix, "ok": ok_all, "report": report}, ensure_ascii=False, indent=2))
|
||||
if not ok_all:
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Kleine Diagnosehilfe für mindnet_edges.
|
||||
|
||||
Nutzung (aus Projektwurzel):
|
||||
|
||||
(.venv) python tests/diag_edges_for_note.py \
|
||||
--note-id 20251130-relshow-embeddings-101
|
||||
|
||||
Optional:
|
||||
|
||||
--prefix mindnet
|
||||
|
||||
Ziel:
|
||||
- Zeigt alle Edges aus der Collection <prefix>_edges,
|
||||
die sich auf die angegebene note_id beziehen:
|
||||
* note_id == NOTE
|
||||
* oder source_id == NOTE
|
||||
* oder target_id == NOTE
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from typing import List
|
||||
|
||||
# Projekt-Root (Ordner mit "app/") zum sys.path hinzufügen,
|
||||
# damit "import app.core.qdrant" funktioniert, auch wenn das Script
|
||||
# direkt aus tests/ gestartet wird.
|
||||
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_ROOT = os.path.dirname(THIS_DIR)
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from qdrant_client.http import models as rest # type: ignore
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
|
||||
def fetch_edges_for_note(note_id: str, prefix: str | None, limit: int = 512) -> List[dict]:
|
||||
cfg = QdrantConfig.from_env()
|
||||
if prefix:
|
||||
cfg.prefix = prefix
|
||||
|
||||
client = get_client(cfg)
|
||||
_, _, edges_col = collection_names(cfg.prefix)
|
||||
|
||||
conditions = []
|
||||
for field in ("note_id", "source_id", "target_id"):
|
||||
conditions.append(
|
||||
rest.FieldCondition(
|
||||
key=field,
|
||||
match=rest.MatchValue(value=str(note_id)),
|
||||
)
|
||||
)
|
||||
|
||||
flt = rest.Filter(
|
||||
should=conditions,
|
||||
)
|
||||
|
||||
points, _ = client.scroll(
|
||||
collection_name=edges_col,
|
||||
scroll_filter=flt,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
out: List[dict] = []
|
||||
for p in points or []:
|
||||
pl = dict(p.payload or {})
|
||||
if pl:
|
||||
out.append(pl)
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--note-id", required=True, help="Note-ID, z. B. 20251130-relshow-embeddings-101")
|
||||
parser.add_argument("--prefix", default=None, help="Qdrant-Prefix (Default: aus ENV/QdrantConfig)")
|
||||
args = parser.parse_args()
|
||||
|
||||
note_id = args.note_id
|
||||
prefix = args.prefix
|
||||
|
||||
edges = fetch_edges_for_note(note_id, prefix)
|
||||
print(f"Prefix: {prefix or '(aus ENV)'}")
|
||||
print(f"Note-ID: {note_id}")
|
||||
print(f"Gefundene Edges: {len(edges)}")
|
||||
print("-" * 60)
|
||||
|
||||
for i, e in enumerate(edges, start=1):
|
||||
print(f"[{i}] kind={e.get('kind')} confidence={e.get('confidence')}")
|
||||
print(f" note_id = {e.get('note_id')}")
|
||||
print(f" source_id= {e.get('source_id')}")
|
||||
print(f" target_id= {e.get('target_id')}")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/diag_payload_indexes.py (v1.1)
|
||||
|
||||
Zeigt payload_schema (Indizes) je Collection.
|
||||
WICHTIG: Einige Qdrant-Versionen liefern payload_schema nur, wenn
|
||||
`with_payload_schema=true` gesetzt wird. Daher setzen wir das Flag explizit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, json
|
||||
from qdrant_client.http import models as rest
|
||||
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||
|
||||
def compact_schema(ps: dict | None) -> dict:
|
||||
if not isinstance(ps, dict):
|
||||
return {}
|
||||
out = {}
|
||||
for k, v in ps.items():
|
||||
if isinstance(v, dict) and "type" in v:
|
||||
out[k] = v["type"]
|
||||
else:
|
||||
out[k] = v
|
||||
return out
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--raw", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
|
||||
cols = [notes, chunks, edges]
|
||||
result = []
|
||||
for name in cols:
|
||||
info = client.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = info.model_dump() if hasattr(info, "model_dump") else (info.dict() if hasattr(info, "dict") else info)
|
||||
payload_schema = (d.get("result") or {}).get("payload_schema")
|
||||
vectors = (d.get("result") or {}).get("vectors")
|
||||
if isinstance(vectors, dict) and "config" in vectors:
|
||||
vectors = vectors.get("config")
|
||||
if args.raw:
|
||||
result.append({"collection": name, "raw": d})
|
||||
else:
|
||||
result.append({
|
||||
"name": name,
|
||||
"vectors": vectors,
|
||||
"payload_schema": compact_schema(payload_schema),
|
||||
"segments_count": (d.get("result") or {}).get("segments_count"),
|
||||
})
|
||||
print(json.dumps({"prefix": cfg.prefix, "collections": result}, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Diagnose für den Graph-Adapter (Subgraph) in mindnet.
|
||||
|
||||
Nutzung (aus Projektwurzel):
|
||||
|
||||
(.venv) python tests/diag_subgraph_for_note.py \
|
||||
--note-id 20251130-relshow-embeddings-101 \
|
||||
--depth 1
|
||||
|
||||
Optional:
|
||||
|
||||
--prefix mindnet
|
||||
--edge-type references --edge-type related_to ...
|
||||
|
||||
Ziel:
|
||||
- Ruft app.core.graph_adapter.expand(...) für eine gegebene note_id auf.
|
||||
- Gibt edge_bonus und centrality_bonus für diese note_id aus.
|
||||
- Zeigt zusätzlich einige Nachbarn und deren Boni.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
# sys.path anpassen, damit "import app...." aus tests/ funktioniert
|
||||
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJECT_ROOT = os.path.dirname(THIS_DIR)
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client # type: ignore
|
||||
import app.core.graph_adapter as ga # type: ignore
|
||||
|
||||
|
||||
def build_subgraph_for_note(
|
||||
note_id: str,
|
||||
prefix: Optional[str],
|
||||
depth: int,
|
||||
edge_types: Optional[List[str]] = None,
|
||||
) -> Tuple[ga.Subgraph, str]:
|
||||
cfg = QdrantConfig.from_env()
|
||||
if prefix:
|
||||
cfg.prefix = prefix
|
||||
|
||||
client = get_client(cfg)
|
||||
sg = ga.expand(client, cfg.prefix, [note_id], depth=depth, edge_types=edge_types)
|
||||
return sg, cfg.prefix
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--note-id",
|
||||
required=True,
|
||||
help="Note-ID, z. B. 20251130-relshow-embeddings-101",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--depth",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Expansionstiefe (Default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prefix",
|
||||
default=None,
|
||||
help="Qdrant-Prefix (Default: aus ENV/QdrantConfig)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--edge-type",
|
||||
action="append",
|
||||
default=None,
|
||||
help="Edge-Typ-Filter (kann mehrfach angegeben werden), z. B. --edge-type references",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
note_id = args.note_id
|
||||
depth = max(0, int(args.depth))
|
||||
prefix = args.prefix
|
||||
edge_types = args.edge_type
|
||||
|
||||
sg, eff_prefix = build_subgraph_for_note(note_id, prefix, depth, edge_types=edge_types)
|
||||
|
||||
print(f"Prefix: {eff_prefix}")
|
||||
print(f"Note-ID (Seed): {note_id}")
|
||||
print(f"Depth: {depth}")
|
||||
print(f"Edge-Types-Filter: {edge_types or '(keiner)'}")
|
||||
print("-" * 60)
|
||||
|
||||
# Boni für die Seed-Note
|
||||
eb = sg.edge_bonus(note_id)
|
||||
cb = sg.centrality_bonus(note_id)
|
||||
print(f"edge_bonus({note_id}) = {eb:.4f}")
|
||||
print(f"centrality_bonus({note_id}) = {cb:.4f}")
|
||||
print("-" * 60)
|
||||
|
||||
# Ein paar weitere Knoten anzeigen (falls vorhanden)
|
||||
print("Beispiele weiterer Knoten mit Boni (max. 10):")
|
||||
printed = 0
|
||||
for node, edges in sg.adj.items():
|
||||
if node == note_id:
|
||||
continue
|
||||
if printed >= 10:
|
||||
break
|
||||
eb2 = sg.edge_bonus(node)
|
||||
cb2 = sg.centrality_bonus(node)
|
||||
print(f"- node={node}")
|
||||
print(f" edge_bonus={eb2:.4f} centrality={cb2:.4f}")
|
||||
printed += 1
|
||||
|
||||
if printed == 0:
|
||||
print("(keine weiteren Knoten im Subgraph registriert)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tests/ensure_indexes_and_show.py (compat v1.0.1)
|
||||
|
||||
Erzwingt ensure_payload_indexes(...) und zeigt danach payload_schema je Collection.
|
||||
Kompatibel mit qdrant-client Versionen ohne 'with_payload_schema' im Wrapper.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_payload_indexes, collection_names
|
||||
|
||||
|
||||
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
return obj if isinstance(obj, dict) else {}
|
||||
|
||||
|
||||
def _cfg_base_url(cfg) -> str:
|
||||
if getattr(cfg, "url", None):
|
||||
return cfg.url.rstrip("/")
|
||||
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
|
||||
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
if api_key:
|
||||
req.add_header("api-key", api_key)
|
||||
req.add_header("Authorization", f"Bearer {api_key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
|
||||
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||
# Strategy 1: OpenAPI client with flag
|
||||
try:
|
||||
oc = getattr(client, "openapi_client", None)
|
||||
if oc is not None and hasattr(oc, "collections_api"):
|
||||
api = oc.collections_api
|
||||
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||
d = _safe_model_dump(info)
|
||||
return (d.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
pass
|
||||
# Strategy 2: wrapper
|
||||
try:
|
||||
info = client.get_collection(collection_name=name)
|
||||
d = _safe_model_dump(info)
|
||||
ps = (d.get("result") or {}).get("payload_schema")
|
||||
if ps is not None:
|
||||
return ps
|
||||
except Exception:
|
||||
pass
|
||||
# Strategy 3: direct HTTP
|
||||
try:
|
||||
base = _cfg_base_url(cfg)
|
||||
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||
return (raw.get("result") or {}).get("payload_schema")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
notes, chunks, edges = collection_names(cfg.prefix)
|
||||
res = {}
|
||||
for name in (notes, chunks, edges):
|
||||
res[name] = get_collection_payload_schema(client, cfg, name)
|
||||
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/test_retriever_smoke.py - einfacher Smoke-Test für den mindnet-Retriever
|
||||
|
||||
Zweck:
|
||||
- Startet eine Query gegen die laufende FastAPI-Instanz
|
||||
- Prüft, ob der /query-Endpunkt antwortet und sinnvolle Treffer liefert
|
||||
- Gibt Top-K-Ergebnisse inkl. Scores auf stdout aus
|
||||
|
||||
Verwendung:
|
||||
1) Stelle sicher, dass der mindnet-Service läuft, z. B.:
|
||||
uvicorn app.main:create_app --factory --reload
|
||||
|
||||
2) Führe dieses Skript aus:
|
||||
python scripts/test_retriever_smoke.py --query "karate" --mode hybrid
|
||||
|
||||
3) Optional:
|
||||
python scripts/test_retriever_smoke.py --query "embeddings" --mode semantic --top-k 5
|
||||
|
||||
Hinweis:
|
||||
- Standard-URL ist http://127.0.0.1:8000
|
||||
- Endpoint ist /query (bestehender Router), der QueryRequest erwartet
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from typing import Any, Dict
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def _build_payload(args: argparse.Namespace) -> Dict[str, Any]:
|
||||
payload: Dict[str, Any] = {
|
||||
"mode": args.mode,
|
||||
"query": args.query,
|
||||
"top_k": args.top_k,
|
||||
}
|
||||
if args.expand_depth is not None and args.expand_depth > 0:
|
||||
expand: Dict[str, Any] = {"depth": args.expand_depth}
|
||||
if args.expand_edge_types:
|
||||
expand["edge_types"] = args.expand_edge_types
|
||||
payload["expand"] = expand
|
||||
return payload
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Smoke-Test für mindnet-Retriever (/query)")
|
||||
parser.add_argument("--url", default="http://127.0.0.1:8000/query", help="Basis-URL des Query-Endpunkts")
|
||||
parser.add_argument("--query", required=True, help="Text-Query für den Retriever")
|
||||
parser.add_argument("--mode", choices=["semantic", "hybrid"], default="hybrid", help="Retrieval-Modus")
|
||||
parser.add_argument("--top-k", type=int, default=5, help="Anzahl der gewünschten Treffer")
|
||||
parser.add_argument("--expand-depth", type=int, default=0, help="Edge-Expansionstiefe (nur hybrid)")
|
||||
parser.add_argument(
|
||||
"--expand-edge-types",
|
||||
nargs="*",
|
||||
default=None,
|
||||
help="Liste von Edge-Typen für die Expansion (z. B. references depends_on)",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
payload = _build_payload(args)
|
||||
|
||||
try:
|
||||
resp = requests.post(args.url, json=payload, timeout=10)
|
||||
except Exception as exc: # pragma: no cover - Netzwerkfehler
|
||||
print(f"[ERROR] HTTP-Request fehlgeschlagen: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if resp.status_code != 200:
|
||||
print(f"[ERROR] Unerwarteter Statuscode: {resp.status_code}", file=sys.stderr)
|
||||
print(resp.text, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as exc: # pragma: no cover - JSON-Fehler
|
||||
print(f"[ERROR] Antwort ist kein gültiges JSON: {exc}", file=sys.stderr)
|
||||
print(resp.text, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("=== mindnet Retriever Smoke-Test ===")
|
||||
print(f"URL: {args.url}")
|
||||
print(f"Mode: {args.mode}")
|
||||
print(f"Query: {args.query!r}")
|
||||
print(f"Top-K: {args.top_k}")
|
||||
print()
|
||||
|
||||
results = data.get("results") or []
|
||||
used_mode = data.get("used_mode")
|
||||
latency_ms = data.get("latency_ms")
|
||||
|
||||
print(f"used_mode = {used_mode}, latency = {latency_ms} ms")
|
||||
print(f"Treffer insgesamt: {len(results)}")
|
||||
print()
|
||||
|
||||
for i, hit in enumerate(results, start=1):
|
||||
note_id = hit.get("note_id")
|
||||
node_id = hit.get("node_id")
|
||||
total = hit.get("total_score")
|
||||
sem = hit.get("semantic_score")
|
||||
edge_bonus = hit.get("edge_bonus")
|
||||
cent_bonus = hit.get("centrality_bonus")
|
||||
src = hit.get("source") or {}
|
||||
path = src.get("path")
|
||||
section = src.get("section")
|
||||
|
||||
print(f"[{i}] note_id={note_id} node_id={node_id}")
|
||||
print(f" total={total:.4f} (semantic={sem:.4f} edge={edge_bonus:.4f} centrality={cent_bonus:.4f})")
|
||||
if path or section:
|
||||
print(f" source: {path or '-'} :: {section or '-'}")
|
||||
print()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/show_edges_for_note.py
|
||||
|
||||
Zeigt Kanten einer Note (gefiltert nach kind/scope) in einer kompakten Form.
|
||||
Aufrufbeispiele:
|
||||
python3 -m scripts.show_edges_for_note --note-id 20251110-ollama-llm-9f0a12 --kinds references,next,prev --limit 10
|
||||
python3 -m scripts.show_edges_for_note --title "Qdrant Vektordatenbank" --scope note
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse, json, os, sys
|
||||
from typing import Dict, Any, List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def find_note_by_title(client, prefix: str, title: str) -> str | None:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="title", match=rest.MatchText(text=title))])
|
||||
pts, _ = client.scroll(collection_name=notes_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=1)
|
||||
if not pts:
|
||||
return None
|
||||
return pts[0].payload.get("note_id")
|
||||
|
||||
def fetch_edges_for_note(client, prefix: str, note_id: str, kinds: List[str] | None, scope: str | None, limit: int) -> List[Dict[str, Any]]:
|
||||
_, _, edges_col = collections(prefix)
|
||||
must = [rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
|
||||
if scope:
|
||||
must.append(rest.FieldCondition(key="scope", match=rest.MatchValue(value=scope)))
|
||||
if kinds:
|
||||
must.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
|
||||
f = rest.Filter(must=must)
|
||||
pts, _ = client.scroll(collection_name=edges_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=limit)
|
||||
return [p.payload for p in pts]
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--note-id")
|
||||
ap.add_argument("--title")
|
||||
ap.add_argument("--kinds", help="CSV: references,next,prev,belongs_to,backlink")
|
||||
ap.add_argument("--scope", choices=["note","chunk"])
|
||||
ap.add_argument("--limit", type=int, default=25)
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
|
||||
nid = args.note_id
|
||||
if not nid and args.title:
|
||||
nid = find_note_by_title(client, cfg.prefix, args.title)
|
||||
if not nid:
|
||||
print(json.dumps({"error": f"note with title '{args.title}' not found"}))
|
||||
sys.exit(2)
|
||||
if not nid:
|
||||
print(json.dumps({"error": "please provide --note-id or --title"}))
|
||||
sys.exit(2)
|
||||
|
||||
kinds = None
|
||||
if args.kinds:
|
||||
kinds = [s.strip() for s in args.kinds.split(",") if s.strip()]
|
||||
|
||||
edges = fetch_edges_for_note(client, cfg.prefix, nid, kinds, args.scope, args.limit)
|
||||
print(json.dumps({"note_id": nid, "count": len(edges), "edges": edges}, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tests/test_edges_all.py
|
||||
Ein knapper Integrationscheck:
|
||||
- Es existieren Notes/Chunks/Edges
|
||||
- Inline-Edges (rule_id startswith "inline:") werden erkannt
|
||||
- Callout-Edges (rule_id == "callout:edge") werden erkannt
|
||||
- Defaults (rule_id startswith "edge_defaults:") werden erkannt
|
||||
- Strukturkanten stimmen (belongs_to == chunks; next == prev == chunks-1)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from typing import Dict, Any, List, Tuple
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
|
||||
|
||||
def _scroll_all(client, collection: str):
|
||||
pts_all = []
|
||||
offset = None
|
||||
while True:
|
||||
pts, offset = client.scroll(
|
||||
collection_name=collection,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=2048,
|
||||
offset=offset,
|
||||
)
|
||||
pts_all.extend(pts or [])
|
||||
if offset is None:
|
||||
break
|
||||
return pts_all
|
||||
|
||||
|
||||
def _rule_group(rule_id: str) -> str:
|
||||
if not rule_id:
|
||||
return "unknown"
|
||||
if rule_id == "callout:edge":
|
||||
return "callout"
|
||||
if rule_id.startswith("inline:"): # <—— wichtig für inline:rel
|
||||
return "inline"
|
||||
if rule_id.startswith("edge_defaults:"):
|
||||
return "defaults"
|
||||
if rule_id.startswith("explicit:"):
|
||||
return "explicit"
|
||||
if rule_id in ("structure:belongs_to", "structure:order"):
|
||||
return "structure"
|
||||
return "other"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
|
||||
col_notes = f"{cfg.prefix}_notes"
|
||||
col_chunks = f"{cfg.prefix}_chunks"
|
||||
col_edges = f"{cfg.prefix}_edges"
|
||||
|
||||
notes_n = client.count(collection_name=col_notes, exact=True).count
|
||||
chunks_pts = _scroll_all(client, col_chunks)
|
||||
edges_pts = _scroll_all(client, col_edges)
|
||||
|
||||
ok = True
|
||||
|
||||
# Basisbedingungen
|
||||
if notes_n == 0 or len(chunks_pts) == 0 or len(edges_pts) == 0:
|
||||
ok = False
|
||||
|
||||
# Gruppen zählen
|
||||
g = Counter(_rule_group((p.payload or {}).get("rule_id", "")) for p in edges_pts)
|
||||
structure = g.get("structure", 0)
|
||||
explicit = g.get("explicit", 0)
|
||||
inline = g.get("inline", 0)
|
||||
callout = g.get("callout", 0)
|
||||
defaults = g.get("defaults", 0)
|
||||
|
||||
if structure == 0:
|
||||
ok = False
|
||||
# mindestens eine der expliziten Varianten vorhanden
|
||||
if (explicit + inline + callout) == 0:
|
||||
ok = False
|
||||
# defaults dürfen 0 sein, wenn types.yaml keine edge_defaults liefert – daher nur Info
|
||||
|
||||
# per-note checks
|
||||
chunks_by_note = Counter([p.payload.get("note_id") for p in chunks_pts if p.payload])
|
||||
belongs = Counter(
|
||||
(p.payload or {}).get("note_id")
|
||||
for p in edges_pts
|
||||
if (p.payload or {}).get("kind") == "belongs_to"
|
||||
)
|
||||
nxt = Counter(
|
||||
(p.payload or {}).get("note_id")
|
||||
for p in edges_pts
|
||||
if (p.payload or {}).get("kind") == "next"
|
||||
)
|
||||
prv = Counter(
|
||||
(p.payload or {}).get("note_id")
|
||||
for p in edges_pts
|
||||
if (p.payload or {}).get("kind") == "prev"
|
||||
)
|
||||
|
||||
for n_id, c in chunks_by_note.items():
|
||||
if belongs.get(n_id, 0) != c:
|
||||
ok = False
|
||||
if (nxt.get(n_id, 0) != max(c - 1, 0)) or (prv.get(n_id, 0) != max(c - 1, 0)):
|
||||
ok = False
|
||||
|
||||
print(json.dumps({"ok": ok, "notes_checked": len(chunks_by_note)}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
tests/test_edges_defaults_smoke.py
|
||||
Aggregiert Edges je Relation (kind) in Qdrant zur schnellen Sichtprüfung.
|
||||
Zeigt, ob typenbasierte edge_defaults-Relationen (z. B. depends_on/related_to) auftauchen.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from collections import Counter
|
||||
import json
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
cl = get_client(cfg)
|
||||
col = f"{cfg.prefix}_edges"
|
||||
pts, _ = cl.scroll(collection_name=col, with_payload=True, with_vectors=False, limit=10000)
|
||||
by_rel = Counter()
|
||||
for p in pts:
|
||||
rel = p.payload.get("kind") or p.payload.get("relation") or "edge"
|
||||
by_rel[rel] += 1
|
||||
print(json.dumps({"prefix": cfg.prefix, "relations": dict(by_rel)}, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
from qdrant_client.http import models as rest
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
|
||||
|
||||
def _rel(pl: dict) -> str:
|
||||
return pl.get("relation") or pl.get("kind") or "edge"
|
||||
|
||||
|
||||
def _scroll(client, col):
|
||||
pts = []
|
||||
next_page = None
|
||||
while True:
|
||||
res, next_page = client.scroll(
|
||||
collection_name=col,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1024,
|
||||
offset=next_page,
|
||||
)
|
||||
pts.extend(res)
|
||||
if next_page is None:
|
||||
break
|
||||
return pts
|
||||
|
||||
|
||||
def main():
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
prefix = os.environ.get("COLLECTION_PREFIX", cfg.prefix)
|
||||
|
||||
cols = {
|
||||
"notes": f"{prefix}_notes",
|
||||
"chunks": f"{prefix}_chunks",
|
||||
"edges": f"{prefix}_edges",
|
||||
}
|
||||
|
||||
# Index: notes -> title/type
|
||||
notes_meta = {}
|
||||
for p in _scroll(client, cols["notes"]):
|
||||
pl = p.payload or {}
|
||||
nid = pl.get("note_id")
|
||||
if nid:
|
||||
notes_meta[nid] = {
|
||||
"title": pl.get("title", ""),
|
||||
"type": pl.get("type", ""),
|
||||
}
|
||||
|
||||
# chunks je note
|
||||
chunks_by_note = defaultdict(int)
|
||||
for p in _scroll(client, cols["chunks"]):
|
||||
pl = p.payload or {}
|
||||
nid = pl.get("note_id")
|
||||
if nid:
|
||||
chunks_by_note[nid] += 1
|
||||
|
||||
# edges je note
|
||||
edges_by_note = defaultdict(list)
|
||||
edges_all = _scroll(client, cols["edges"])
|
||||
for p in edges_all:
|
||||
pl = p.payload or {}
|
||||
nid = pl.get("note_id")
|
||||
if nid:
|
||||
edges_by_note[nid].append(pl)
|
||||
|
||||
# pro note ausgeben
|
||||
summary_edges = Counter()
|
||||
total_chunks = 0
|
||||
for nid in sorted(notes_meta.keys()):
|
||||
meta = notes_meta[nid]
|
||||
chunks = chunks_by_note.get(nid, 0)
|
||||
total_chunks += chunks
|
||||
|
||||
kinds = Counter(_rel(pl) for pl in edges_by_note[nid])
|
||||
summary_edges.update(kinds)
|
||||
|
||||
row = {
|
||||
"note_id": nid,
|
||||
"title": meta["title"],
|
||||
"type": meta["type"],
|
||||
"chunks": chunks,
|
||||
"edges_by_kind": dict(kinds),
|
||||
"checks": {
|
||||
"belongs_to_equals_chunks": (kinds.get("belongs_to", 0) == chunks),
|
||||
"next_prev_match": (kinds.get("next", 0) == kinds.get("prev", 0) == max(0, chunks - 1)),
|
||||
"no_duplicate_edges": _no_dupes(edges_by_note[nid]),
|
||||
},
|
||||
}
|
||||
print(json.dumps(row, ensure_ascii=False))
|
||||
|
||||
# Gesamtsummary
|
||||
total_notes = len(notes_meta)
|
||||
out = {
|
||||
"prefix": prefix,
|
||||
"summary": {
|
||||
"notes": total_notes,
|
||||
"chunks": total_chunks,
|
||||
"belongs_to": summary_edges.get("belongs_to", 0),
|
||||
"next": summary_edges.get("next", 0),
|
||||
"prev": summary_edges.get("prev", 0),
|
||||
"refs_chunk": summary_edges.get("references", 0),
|
||||
"refs_note": summary_edges.get("references_note", 0),
|
||||
"backlink": summary_edges.get("backlink", 0),
|
||||
"dup_edges": 0, # per-Note geprüft
|
||||
},
|
||||
}
|
||||
print(json.dumps(out, ensure_ascii=False))
|
||||
|
||||
|
||||
def _no_dupes(pls):
|
||||
seen = set()
|
||||
for pl in pls:
|
||||
key = (
|
||||
str(pl.get("source_id") or ""),
|
||||
str(pl.get("target_id") or ""),
|
||||
str(pl.get("relation") or pl.get("kind") or ""),
|
||||
str(pl.get("rule_id") or ""),
|
||||
)
|
||||
if key in seen:
|
||||
return False
|
||||
seen.add(key)
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/test_edges_smoke_fast.py — Zählung über /count (sehr schnell)
|
||||
|
||||
Verwendet Qdrant 'count' API je Note/Kind (anstatt scroll), dadurch sehr schnelle Ausführung.
|
||||
Optionen:
|
||||
--max-notes N : prüft nur die ersten N Notizen
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse, json
|
||||
from typing import Dict, Any, List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
from app.core.qdrant import QdrantConfig, get_client
|
||||
|
||||
KINDS = ["belongs_to", "next", "prev", "references", "backlink"]
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
def list_note_ids(client, notes_col: str, max_notes: int | None) -> List[Dict[str, Any]]:
|
||||
pts, _ = client.scroll(collection_name=notes_col, with_payload=True, with_vectors=False, limit=max_notes or 1024)
|
||||
out = []
|
||||
for p in pts or []:
|
||||
pl = p.payload or {}
|
||||
nid = pl.get("note_id") or pl.get("id")
|
||||
if nid:
|
||||
out.append({"note_id": nid, "title": pl.get("title"), "type": pl.get("type")})
|
||||
return out
|
||||
|
||||
def count_points(client, col: str, filt: rest.Filter) -> int:
|
||||
res = client.count(collection_name=col, count_filter=filt, exact=True)
|
||||
return int(getattr(res, "count", 0))
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--max-notes", type=int)
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
notes_col, chunks_col, edges_col = collections(cfg.prefix)
|
||||
notes = list_note_ids(client, notes_col, args.max_notes)
|
||||
|
||||
summary = {"notes": 0, "chunks": 0, "edges": 0}
|
||||
for n in notes:
|
||||
nid = n["note_id"]
|
||||
summary["notes"] += 1
|
||||
|
||||
filt_note = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||
chunk_cnt = count_points(client, chunks_col, filt_note)
|
||||
summary["chunks"] += chunk_cnt
|
||||
|
||||
# counts per kind (edges)
|
||||
by_kind: Dict[str, int] = {}
|
||||
for k in KINDS:
|
||||
f = rest.Filter(must=[
|
||||
rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid)),
|
||||
rest.FieldCondition(key="kind", match=rest.MatchValue(value=k)),
|
||||
])
|
||||
c = count_points(client, edges_col, f)
|
||||
if c:
|
||||
by_kind[k] = c
|
||||
|
||||
summary["edges"] += sum(by_kind.values())
|
||||
line = {"note_id": nid, "title": n.get("title"), "type": n.get("type"), "chunks": chunk_cnt, "edges_by_kind": by_kind}
|
||||
print(json.dumps(line, ensure_ascii=False))
|
||||
|
||||
print(json.dumps({"prefix": cfg.prefix, "summary": summary}, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,64 @@
|
||||
from app.core import retriever as r
|
||||
from app.models.dto import QueryRequest
|
||||
import app.core.qdrant as qdr
|
||||
import app.core.qdrant_points as qp
|
||||
import app.services.embeddings_client as ec
|
||||
|
||||
|
||||
class _DummyClient:
|
||||
"""Minimaler Platzhalter für QdrantClient in Unit-Tests."""
|
||||
pass
|
||||
|
||||
|
||||
def _fake_get_client(cfg):
|
||||
return _DummyClient()
|
||||
|
||||
|
||||
def _fake_embed_text(text: str):
|
||||
# Liefert stabilen Vektor; Größe ist für diesen Test egal,
|
||||
# da search_chunks_by_vector ebenfalls gefaked wird.
|
||||
return [0.0] * 16
|
||||
|
||||
|
||||
def _fake_search_chunks_by_vector(client, prefix, vector, top=10, filters=None):
|
||||
# einfache Hitliste in absteigender Score-Reihenfolge
|
||||
return [
|
||||
("chunk:1", 0.9, {"note_id": "note:1", "path": "a.md", "section_title": "S1"}),
|
||||
("chunk:2", 0.7, {"note_id": "note:2", "path": "b.md", "section_title": "S2"}),
|
||||
("chunk:3", 0.4, {"note_id": "note:3", "path": "c.md", "section_title": "S3"}),
|
||||
]
|
||||
|
||||
|
||||
def test_semantic_retrieve_basic(monkeypatch):
|
||||
# Qdrant- und Embedding-Aufrufe faken
|
||||
monkeypatch.setattr(qdr, "get_client", _fake_get_client)
|
||||
monkeypatch.setattr(ec, "embed_text", _fake_embed_text)
|
||||
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector)
|
||||
|
||||
req = QueryRequest(mode="semantic", query="karate trainingsplan", top_k=2)
|
||||
resp = r.semantic_retrieve(req)
|
||||
|
||||
assert resp.used_mode == "semantic"
|
||||
assert len(resp.results) == 2
|
||||
|
||||
# Scores absteigend sortiert
|
||||
scores = [h.total_score for h in resp.results]
|
||||
assert scores[0] >= scores[1]
|
||||
# Korrekte Zuordnung von note_id und Quelle
|
||||
assert resp.results[0].note_id == "note:1"
|
||||
assert resp.results[0].source["path"] == "a.md"
|
||||
|
||||
|
||||
def test_hybrid_retrieve_basic(monkeypatch):
|
||||
# Qdrant- und Embedding-Aufrufe faken
|
||||
monkeypatch.setattr(qdr, "get_client", _fake_get_client)
|
||||
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector)
|
||||
|
||||
# Im Hybrid-Modus arbeiten wir im Step-1-Stand nur mit query_vector
|
||||
req = QueryRequest(mode="hybrid", query_vector=[0.0] * 16, top_k=2)
|
||||
resp = r.hybrid_retrieve(req)
|
||||
|
||||
assert resp.used_mode == "hybrid"
|
||||
assert len(resp.results) == 2
|
||||
scores = [h.total_score for h in resp.results]
|
||||
assert scores[0] >= scores[1]
|
||||
@@ -0,0 +1,74 @@
|
||||
from app.core import retriever as r
|
||||
from app.models.dto import QueryRequest
|
||||
import app.core.qdrant as qdr
|
||||
import app.core.qdrant_points as qp
|
||||
import app.core.graph_adapter as ga
|
||||
import app.services.embeddings_client as ec
|
||||
|
||||
|
||||
class _DummyClient:
|
||||
pass
|
||||
|
||||
|
||||
def _fake_get_client(cfg):
|
||||
return _DummyClient()
|
||||
|
||||
|
||||
def _fake_embed_text(text: str):
|
||||
# Dimension ist egal, da wir search_chunks_by_vector faken
|
||||
return [0.0] * 384
|
||||
|
||||
|
||||
def _fake_search_chunks_by_vector(client, prefix, vector, top=10, filters=None):
|
||||
# Drei Treffer mit identischem semantic_score und retriever_weight,
|
||||
# aber unterschiedlichen Graph-Boni
|
||||
return [
|
||||
("chunk:1", 0.8, {"note_id": "note:1", "chunk_id": "c1", "path": "a.md", "section_title": "S1"}),
|
||||
("chunk:2", 0.8, {"note_id": "note:2", "chunk_id": "c2", "path": "b.md", "section_title": "S2"}),
|
||||
("chunk:3", 0.8, {"note_id": "note:3", "chunk_id": "c3", "path": "c.md", "section_title": "S3"}),
|
||||
]
|
||||
|
||||
|
||||
class _DummySubgraph:
|
||||
def __init__(self, edge_scores, cent_scores):
|
||||
self._edge = edge_scores
|
||||
self._cent = cent_scores
|
||||
|
||||
def edge_bonus(self, node_id: str) -> float:
|
||||
return self._edge.get(node_id, 0.0)
|
||||
|
||||
def centrality_bonus(self, node_id: str) -> float:
|
||||
return self._cent.get(node_id, 0.0)
|
||||
|
||||
|
||||
def _fake_expand(client, prefix, seeds, depth=1, edge_types=None):
|
||||
# Wir gewichten c2 > c1 > c3
|
||||
edge_scores = {"c1": 0.1, "c2": 0.5, "c3": 0.0}
|
||||
cent_scores = {"c1": 0.05, "c2": 0.2, "c3": 0.0}
|
||||
return _DummySubgraph(edge_scores, cent_scores)
|
||||
|
||||
|
||||
def test_hybrid_retrieve_uses_edge_scores(monkeypatch):
|
||||
# Qdrant-Client, Embeddings, Chunk-Suche und Graph-Expansion faken
|
||||
monkeypatch.setattr(qdr, "get_client", _fake_get_client)
|
||||
monkeypatch.setattr(ec, "embed_text", _fake_embed_text)
|
||||
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector)
|
||||
monkeypatch.setattr(ga, "expand", _fake_expand)
|
||||
|
||||
# expand.depth > 0 triggern, edge_types sind hier egal
|
||||
req = QueryRequest(mode="hybrid", query="karate", top_k=3, expand={"depth": 1, "edge_types": ["references"]})
|
||||
resp = r.hybrid_retrieve(req)
|
||||
|
||||
assert len(resp.results) == 3
|
||||
|
||||
# Da alle semantic_scores und retriever_weight gleich sind,
|
||||
# bestimmt die Summe aus edge_bonus + centrality_bonus das Ranking.
|
||||
first = resp.results[0]
|
||||
second = resp.results[1]
|
||||
third = resp.results[2]
|
||||
|
||||
assert first.note_id == "note:2" # c2 hat höchste Graph-Boni
|
||||
assert second.note_id == "note:1"
|
||||
assert third.note_id == "note:3"
|
||||
|
||||
assert first.total_score > second.total_score > third.total_score
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# test_retriever_smoke.py – End-to-End Smoke-Test für den mindnet Retriever
|
||||
#
|
||||
# Dieser Smoke-Test ist für manuelle End-to-End-Tests gedacht.
|
||||
# Er wird NICHT automatisch von pytest ausgeführt.
|
||||
#
|
||||
# Empfohlener Speicherort: tests/test_retriever_smoke.py
|
||||
#
|
||||
# Aufrufbeispiele:
|
||||
#
|
||||
# python tests/test_retriever_smoke.py --query "embeddings" --mode semantic
|
||||
#
|
||||
# python tests/test_retriever_smoke.py \
|
||||
# --query "embeddings" \
|
||||
# --mode hybrid \
|
||||
# --expand-depth 1 \
|
||||
# --expand-edge-types references depends_on
|
||||
#
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from typing import Any, Dict
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def _build_payload(args: argparse.Namespace) -> Dict[str, Any]:
|
||||
payload: Dict[str, Any] = {
|
||||
"mode": args.mode,
|
||||
"query": args.query,
|
||||
"top_k": args.top_k,
|
||||
}
|
||||
if args.expand_depth is not None and args.expand_depth > 0:
|
||||
expand: Dict[str, Any] = {"depth": args.expand_depth}
|
||||
if args.expand_edge_types:
|
||||
expand["edge_types"] = args.expand_edge_types
|
||||
payload["expand"] = expand
|
||||
return payload
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Smoke-Test für mindnet-Retriever (/query)")
|
||||
parser.add_argument("--url", default="http://127.0.0.1:8000/query", help="URL des Query-Endpunkts")
|
||||
parser.add_argument("--query", required=True, help="Text-Query")
|
||||
parser.add_argument("--mode", choices=["semantic", "hybrid"], default="hybrid")
|
||||
parser.add_argument("--top-k", type=int, default=5)
|
||||
parser.add_argument("--expand-depth", type=int, default=0)
|
||||
parser.add_argument(
|
||||
"--expand-edge-types",
|
||||
nargs="*",
|
||||
default=None,
|
||||
help="Edge-Typen (z. B. references depends_on)",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
payload = _build_payload(args)
|
||||
|
||||
try:
|
||||
resp = requests.post(args.url, json=payload, timeout=10)
|
||||
except Exception as exc:
|
||||
print(f"[ERROR] HTTP-Request fehlgeschlagen: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if resp.status_code != 200:
|
||||
print(f"[ERROR] Statuscode: {resp.status_code}", file=sys.stderr)
|
||||
print(resp.text, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception:
|
||||
print("[ERROR] Ungültiges JSON:", resp.text, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("=== mindnet Retriever Smoke-Test ===")
|
||||
print(f"URL: {args.url}")
|
||||
print(f"Mode: {args.mode}")
|
||||
print(f"Query: {args.query!r}")
|
||||
print(f"Top-K: {args.top_k}")
|
||||
print()
|
||||
|
||||
results = data.get("results") or []
|
||||
print(f"used_mode: {data.get('used_mode')}")
|
||||
print(f"latency: {data.get('latency_ms')} ms")
|
||||
print(f"Treffer: {len(results)}")
|
||||
print()
|
||||
|
||||
for i, hit in enumerate(results, start=1):
|
||||
nid = hit.get("note_id")
|
||||
cid = hit.get("node_id")
|
||||
tot = hit.get("total_score")
|
||||
sem = hit.get("semantic_score")
|
||||
edge = hit.get("edge_bonus")
|
||||
cen = hit.get("centrality_bonus")
|
||||
src = hit.get("source") or {}
|
||||
path = src.get("path")
|
||||
sec = src.get("section")
|
||||
|
||||
print(f"[{i}] note_id={nid} node_id={cid}")
|
||||
try:
|
||||
print(
|
||||
f" total={float(tot):.4f} "
|
||||
f"(semantic={float(sem):.4f} edge={float(edge):.4f} centrality={float(cen):.4f})"
|
||||
)
|
||||
except Exception:
|
||||
print(f" total={tot}, semantic={sem}, edge={edge}, centrality={cen}")
|
||||
|
||||
if path or sec:
|
||||
print(f" source: {path or '-'} :: {sec or '-'}")
|
||||
print()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,48 @@
|
||||
from app.core import retriever as r
|
||||
from app.models.dto import QueryRequest
|
||||
import app.core.qdrant as qdr
|
||||
import app.core.qdrant_points as qp
|
||||
import app.services.embeddings_client as ec
|
||||
|
||||
|
||||
class _DummyClient:
|
||||
pass
|
||||
|
||||
|
||||
def _fake_get_client(cfg):
|
||||
return _DummyClient()
|
||||
|
||||
|
||||
def _fake_embed_text(text: str):
|
||||
# Dimension egal, da wir search_chunks_by_vector faken
|
||||
return [0.0] * 384
|
||||
|
||||
|
||||
def _fake_search_chunks_by_vector_weights(client, prefix, vector, top=10, filters=None):
|
||||
# Zwei Treffer mit unterschiedlichem retriever_weight
|
||||
return [
|
||||
("chunk:low", 0.9, {"note_id": "note:low", "path": "low.md", "section_title": "S1", "retriever_weight": 0.2}),
|
||||
("chunk:high", 0.8, {"note_id": "note:high", "path": "high.md", "section_title": "S2", "retriever_weight": 1.5}),
|
||||
]
|
||||
|
||||
|
||||
def test_retriever_weight_influences_ranking(monkeypatch):
|
||||
# Konfiguration: Qdrant & Embedding faken
|
||||
monkeypatch.setattr(qdr, "get_client", _fake_get_client)
|
||||
monkeypatch.setattr(ec, "embed_text", _fake_embed_text)
|
||||
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector_weights)
|
||||
|
||||
req = QueryRequest(mode="semantic", query="karate", top_k=2)
|
||||
resp = r.semantic_retrieve(req)
|
||||
|
||||
assert len(resp.results) == 2
|
||||
|
||||
first = resp.results[0]
|
||||
second = resp.results[1]
|
||||
|
||||
# Semantic Scores: 0.9 (low), 0.8 (high)
|
||||
# Total Scores: 0.9 * 0.2 = 0.18 vs. 0.8 * 1.5 = 1.2 -> high sollte oben liegen
|
||||
assert first.note_id == "note:high"
|
||||
assert second.note_id == "note:low"
|
||||
|
||||
assert first.total_score > second.total_score
|
||||
Reference in new issue
Block a user