Compare commits
99
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2c81fa0cb5 | ||
|
|
c3815afcd4 | ||
|
|
8800160564 | ||
|
|
bdbc4a1bf7 | ||
|
|
597090bc45 | ||
|
|
af36c410b4 | ||
|
|
4dcd606c10 | ||
|
|
bbd5a7fa48 | ||
|
|
6dc37ccb66 | ||
|
|
b84906283e | ||
|
|
af0dcdf76a | ||
|
|
f1e1cde597 | ||
|
|
a7c5630e5b | ||
|
|
82151d6bc3 | ||
|
|
c93e7ad598 | ||
|
|
290b271cf6 | ||
|
|
a686bdbeaf | ||
|
|
2b84c62875 | ||
|
|
2de786fc64 | ||
|
|
2ddf034983 | ||
|
|
b2043f4f84 | ||
|
|
52eae52061 | ||
|
|
e451ea64ae | ||
|
|
eb6e4028ff | ||
|
|
ea45493941 | ||
|
|
da70f0e00c | ||
|
|
8bf3707511 | ||
|
|
804eeef858 | ||
|
|
36658f12fe | ||
|
|
f0e675ffd8 | ||
|
|
90911c1074 | ||
|
|
1729e0b40d | ||
|
|
70b3169e7f | ||
|
|
315a1c0945 | ||
|
|
b9316c693d | ||
|
|
820e52d869 | ||
|
|
444e8e4206 | ||
|
|
e6f2c8b4e8 | ||
|
|
f6b7652219 | ||
|
|
3cd00ab5cf | ||
|
|
0feae50d90 | ||
|
|
d79c55fc64 | ||
|
|
98a6f5c480 | ||
|
|
0fa4aa7280 | ||
|
|
3b192e2eed | ||
|
|
2794d26181 | ||
|
|
7b56f696d6 | ||
|
|
ead1b8c1bc | ||
|
|
b8126e00c4 | ||
|
|
62f8f4b313 | ||
|
|
21924541dc | ||
|
|
197242ca42 | ||
|
|
7922734dce | ||
|
|
afd7675425 | ||
|
|
8b3b343645 | ||
|
|
6713f342c3 | ||
|
|
79bb8a143c | ||
|
|
d03a181536 | ||
|
|
8e5304c156 | ||
|
|
c487db6534 | ||
|
|
0ca54f429e | ||
|
|
6c62f64784 | ||
|
|
3282f85007 | ||
|
|
b186569750 | ||
|
|
7cde2462ec | ||
|
|
c0b96f7c05 | ||
|
|
3a88caac69 | ||
|
|
31457e9240 | ||
|
|
307335a733 | ||
|
|
4eb5e34ea7 | ||
|
|
f4be219790 | ||
|
|
53591b6f27 | ||
|
|
a14d0bb7cb | ||
|
|
c2802e7cb3 | ||
|
|
c7f2815f1d | ||
|
|
afc90fe992 | ||
|
|
2f9ce824a0 | ||
|
|
a39b2a6950 | ||
|
|
70fa95966c | ||
|
|
10479014d5 | ||
|
|
148e22aa7d | ||
|
|
a9fe6196f7 | ||
|
|
ae20665a66 | ||
|
|
ccbf078463 | ||
|
|
cd1e46784f | ||
|
|
b49d97b8d7 | ||
|
|
3d44de9d87 | ||
|
|
987b3c1770 | ||
|
|
25bc6544c4 | ||
|
|
b241ae753b | ||
|
|
2041771b14 | ||
|
|
b5958a9f63 | ||
|
|
c05dbd4b3b | ||
|
|
44e468fc21 | ||
|
|
5f358d329b | ||
|
|
59fe8154f2 | ||
|
|
d5b2e9bb8f | ||
|
|
3a3907e1b6 | ||
|
|
f66cdc70b2 |
No files matched your search
@@ -14,7 +14,7 @@ jobs:
|
||||
runs-on: linux_host
|
||||
env:
|
||||
TARGET_BASE: /home/llmadmin/mindnet
|
||||
DEPLOY_DIRS: "app scripts schemas docker tests requirements.txt README.md"
|
||||
DEPLOY_DIRS: "app scripts schemas docker tests config requirements.txt README.md"
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
+158
-249
@@ -1,271 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/chunk_payload.py
|
||||
Version: 2.2.0
|
||||
Datum: 2025-10-06
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.0.1.
|
||||
Neu: Wenn der Chunker KEIN Overlap im Fenster liefert (== window fehlt / identisch zur Kernpassage),
|
||||
erzeugen wir FENSTER mit synthetischem Overlap auf Basis chunk_config.get_sizes(note_type)['overlap'].
|
||||
|
||||
Felder (beibehalten aus 2.0.1):
|
||||
- note_id, chunk_id, id (Alias), chunk_index, seq, path
|
||||
- window (mit Overlap), text (ohne linkes Overlap)
|
||||
- start, end (Offsets im gesamten Body)
|
||||
- overlap_left, overlap_right
|
||||
- token_count?, section_title?, section_path?, type?, title?, tags?
|
||||
|
||||
Kompatibilität:
|
||||
- 'id' == 'chunk_id' als Alias
|
||||
- Pfade bleiben relativ (keine führenden '/'), Backslashes → Slashes
|
||||
- Robust für Chunk-Objekte oder Dicts; Fensterquelle: 'window'|'text'|'content'|'raw'
|
||||
|
||||
Lizenz: MIT (projektintern)
|
||||
"""
|
||||
# chunk_payload.py
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
import os, json, pathlib, re, yaml, hashlib
|
||||
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
FRONTMATTER_RE = re.compile(r"^---\s*\n.*?\n---\s*\n?", re.DOTALL)
|
||||
|
||||
try:
|
||||
# Typgerechtes Overlap aus deiner Konfiguration holen
|
||||
from app.core.chunk_config import get_sizes as _get_sizes
|
||||
except Exception:
|
||||
def _get_sizes(_note_type: str):
|
||||
# konservativer Default, falls Import fehlschlägt
|
||||
return {"overlap": (40, 60), "target": (250, 350), "max": 500}
|
||||
def _as_dict(note: Any) -> Dict[str, Any]:
|
||||
if isinstance(note, dict):
|
||||
return note
|
||||
d: Dict[str, Any] = {}
|
||||
for attr in ("id","note_id","title","path","frontmatter","meta","metadata","body","text","content","raw","markdown","type","chunks"):
|
||||
if hasattr(note, attr):
|
||||
d[attr] = getattr(note, attr)
|
||||
fm = d.get("frontmatter") or d.get("meta") or d.get("metadata") or {}
|
||||
d["frontmatter"] = fm if isinstance(fm, dict) else {}
|
||||
return d
|
||||
|
||||
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[Dict[str,Any]]]:
|
||||
path = kwargs.get("path")
|
||||
types_cfg = kwargs.get("types_config")
|
||||
for a in args:
|
||||
if path is None and isinstance(a, (str, pathlib.Path)):
|
||||
path = str(a)
|
||||
if types_cfg is None and isinstance(a, dict):
|
||||
types_cfg = a
|
||||
return path, types_cfg
|
||||
|
||||
# ------------------------------- Utils ------------------------------- #
|
||||
def _load_types_config(explicit: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
if isinstance(explicit, dict):
|
||||
return explicit
|
||||
for rel in ("config/config.yaml", "config/types.yaml"):
|
||||
p = pathlib.Path(rel)
|
||||
if p.exists():
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
|
||||
return data["types"]
|
||||
return data if isinstance(data, dict) else {}
|
||||
return {}
|
||||
|
||||
def _get_attr_or_key(obj: Any, key: str, default=None):
|
||||
if obj is None:
|
||||
return default
|
||||
if isinstance(obj, dict):
|
||||
return obj.get(key, default)
|
||||
return getattr(obj, key, default)
|
||||
|
||||
def _as_window_text(chunk: Any) -> str:
|
||||
"""Fenstertext robust lesen (bevorzugt echte Fenster, sonst Kern)."""
|
||||
for k in ("window", "text", "content", "raw"):
|
||||
v = _get_attr_or_key(chunk, k, None)
|
||||
if isinstance(v, str) and v:
|
||||
def _coalesce(*vals):
|
||||
for v in vals:
|
||||
if v is not None:
|
||||
return v
|
||||
return None
|
||||
|
||||
def _text_from_note(n: Dict[str, Any], path_hint: Optional[str]) -> str:
|
||||
# häufige Felder
|
||||
cand = [
|
||||
n.get("body"),
|
||||
n.get("text"),
|
||||
n.get("markdown"),
|
||||
n.get("raw"),
|
||||
]
|
||||
content = n.get("content")
|
||||
if isinstance(content, str):
|
||||
cand.append(content)
|
||||
elif isinstance(content, dict):
|
||||
for k in ("text","body","raw","markdown","content"):
|
||||
v = content.get(k)
|
||||
if isinstance(v, str):
|
||||
cand.append(v)
|
||||
for t in cand:
|
||||
if isinstance(t, str) and t.strip():
|
||||
return t
|
||||
|
||||
# Fallback: Datei lesen und Frontmatter entfernen
|
||||
p = n.get("path") or path_hint
|
||||
if p:
|
||||
try:
|
||||
pth = pathlib.Path(p)
|
||||
if pth.exists():
|
||||
txt = pth.read_text(encoding="utf-8", errors="ignore")
|
||||
if txt:
|
||||
return FRONTMATTER_RE.sub("", txt).strip()
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
def _to_int(x: Any, default: int = 0) -> int:
|
||||
def _iter_chunks(n: Dict[str, Any], profile: str, fulltext: str) -> List[Dict[str, Any]]:
|
||||
# 1) vorhandene Chunks nehmen, wenn sinnvoll
|
||||
existing = n.get("chunks")
|
||||
out: List[Dict[str, Any]] = []
|
||||
if isinstance(existing, list) and existing:
|
||||
for i, c in enumerate(existing):
|
||||
text = ""
|
||||
if isinstance(c, dict):
|
||||
text = c.get("text") or c.get("body") or c.get("raw") or ""
|
||||
elif isinstance(c, str):
|
||||
text = c
|
||||
if text and text.strip():
|
||||
out.append({"index": i, "text": text})
|
||||
if out:
|
||||
return out
|
||||
|
||||
# 2) Fallback: profilabhängige Bündelung
|
||||
if not isinstance(profile, str):
|
||||
profile = "medium"
|
||||
size = {"short": 600, "medium": 1200, "long": 2400}.get(profile, 1200)
|
||||
if not fulltext:
|
||||
return []
|
||||
paras = re.split(r"\n{2,}", fulltext)
|
||||
buf = ""
|
||||
chunks: List[str] = []
|
||||
for p in paras:
|
||||
p = (p or "").strip()
|
||||
if not p:
|
||||
continue
|
||||
if len(buf) + (2 if buf else 0) + len(p) <= size:
|
||||
buf = (buf + "\n\n" + p).strip() if buf else p
|
||||
else:
|
||||
if buf: chunks.append(buf)
|
||||
if len(p) <= size:
|
||||
buf = p
|
||||
else:
|
||||
for i in range(0, len(p), size):
|
||||
chunks.append(p[i:i+size])
|
||||
buf = ""
|
||||
if buf: chunks.append(buf)
|
||||
return [{"index": i, "text": c} for i, c in enumerate(chunks)]
|
||||
|
||||
def make_chunk_payloads(note: Any, *args, **kwargs) -> List[Dict[str, Any]]:
|
||||
n = _as_dict(note)
|
||||
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
|
||||
types_cfg = _load_types_config(types_cfg_explicit)
|
||||
|
||||
fm = n.get("frontmatter") or {}
|
||||
note_type = str(fm.get("type") or n.get("type") or "note")
|
||||
cfg_for_type = types_cfg.get(note_type, {}) if isinstance(types_cfg, dict) else {}
|
||||
|
||||
try:
|
||||
return int(x)
|
||||
default_rw = float(os.environ.get("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0))
|
||||
except Exception:
|
||||
return default
|
||||
default_rw = 1.0
|
||||
|
||||
def _normalize_rel_path(p: str) -> str:
|
||||
p = (p or "").replace("\\", "/")
|
||||
while p.startswith("/"):
|
||||
p = p[1:]
|
||||
return p
|
||||
retriever_weight = _coalesce(fm.get("retriever_weight"), cfg_for_type.get("retriever_weight"), default_rw)
|
||||
try:
|
||||
retriever_weight = float(retriever_weight)
|
||||
except Exception:
|
||||
retriever_weight = default_rw
|
||||
|
||||
chunk_profile = _coalesce(fm.get("chunk_profile"), cfg_for_type.get("chunk_profile"), os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE","medium"))
|
||||
chunk_profile = chunk_profile if isinstance(chunk_profile, str) else "medium"
|
||||
|
||||
# ---------------------- Overlap & Offsets ---------------------------- #
|
||||
note_id = n.get("note_id") or n.get("id") or fm.get("id")
|
||||
title = n.get("title") or fm.get("title") or ""
|
||||
path = n.get("path") or path_arg
|
||||
if isinstance(path, pathlib.Path):
|
||||
path = str(path)
|
||||
path = path or "" # immer vorhanden
|
||||
|
||||
def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int], str]:
|
||||
"""
|
||||
Entfernt linkes Overlap aus echten Fenster-Strings.
|
||||
Rückgabe: (segments, overlaps_left, reconstructed_text)
|
||||
"""
|
||||
segments: List[str] = []
|
||||
overlaps_left: List[int] = []
|
||||
reconstructed = ""
|
||||
for w in windows:
|
||||
w = w or ""
|
||||
max_k = min(len(w), len(reconstructed))
|
||||
k = 0
|
||||
for cand in range(max_k, -1, -1):
|
||||
if reconstructed.endswith(w[:cand]):
|
||||
k = cand
|
||||
break
|
||||
seg = w[k:]
|
||||
segments.append(seg)
|
||||
overlaps_left.append(k)
|
||||
reconstructed += seg
|
||||
return segments, overlaps_left, reconstructed
|
||||
fulltext = _text_from_note(n, path)
|
||||
chunks = _iter_chunks(n, chunk_profile, fulltext)
|
||||
|
||||
def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
|
||||
"""Länge längsten Suffix(a), der Prefix(b) ist."""
|
||||
if not a or not b:
|
||||
return 0
|
||||
a1 = a[-max_probe:]
|
||||
b1 = b[:max_probe]
|
||||
n = min(len(a1), len(b1))
|
||||
for k in range(n, 0, -1):
|
||||
if a1[-k:] == b1[:k]:
|
||||
return k
|
||||
return 0
|
||||
|
||||
|
||||
# ----------------------------- Public API ---------------------------- #
|
||||
|
||||
def make_chunk_payloads(
|
||||
frontmatter: Dict[str, Any],
|
||||
rel_path: str,
|
||||
chunks: Iterable[Union[Dict[str, Any], Any]],
|
||||
note_text: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Baut Payloads pro Chunk. Falls Fenster ohne Overlap geliefert werden,
|
||||
erzeugen wir synthetische 'window'-Texte mit typgerechtem Overlap.
|
||||
"""
|
||||
note_id = str(frontmatter.get("id") or "").strip()
|
||||
note_type = str(frontmatter.get("type", "")).lower()
|
||||
note_title = frontmatter.get("title", None)
|
||||
note_tags = frontmatter.get("tags", None)
|
||||
rel_path = _normalize_rel_path(rel_path)
|
||||
|
||||
# 1) Rohdaten sammeln (so wie geliefert)
|
||||
chunks_list = list(chunks)
|
||||
raw_windows: List[str] = []
|
||||
seqs: List[int] = []
|
||||
ids_in: List[Optional[str]] = []
|
||||
token_counts: List[Optional[int]] = []
|
||||
section_titles: List[Optional[str]] = []
|
||||
section_paths: List[Optional[str]] = []
|
||||
any_explicit_window = False
|
||||
|
||||
for idx, c in enumerate(chunks_list):
|
||||
# Fensterquelle
|
||||
w = _get_attr_or_key(c, "window", None)
|
||||
if isinstance(w, str) and w:
|
||||
any_explicit_window = True
|
||||
raw_windows.append(w)
|
||||
else:
|
||||
raw_windows.append(_as_window_text(c)) # 'text'|'content'|'raw' als Ersatz
|
||||
# Ordnung
|
||||
seqs.append(_to_int(_get_attr_or_key(c, "seq", _get_attr_or_key(c, "chunk_index", idx)), idx))
|
||||
# IDs, Tokens, Sektionen
|
||||
cid = _get_attr_or_key(c, "chunk_id", _get_attr_or_key(c, "id", None))
|
||||
ids_in.append(str(cid) if isinstance(cid, str) and cid else None)
|
||||
tc = _get_attr_or_key(c, "token_count", None)
|
||||
token_counts.append(_to_int(tc, 0) if tc is not None else None)
|
||||
section_titles.append(_get_attr_or_key(c, "section_title", None))
|
||||
section_paths.append(_get_attr_or_key(c, "section_path", None))
|
||||
|
||||
# 2) Segmente & Overlaps bestimmen
|
||||
if any_explicit_window:
|
||||
# Es existieren echte Fenster → dedupe, um Kernsegmente zu finden
|
||||
segments, overlaps_left, recon = _dedupe_windows_to_segments(raw_windows)
|
||||
windows_final = raw_windows[:] # bereits mit Overlap geliefert
|
||||
else:
|
||||
# Keine echten Fenster → Segmente sind identisch zu "Fenstern" (bisher),
|
||||
# wir erzeugen synthetische Fenster mit Overlap gemäß Typ
|
||||
segments = [w or "" for w in raw_windows]
|
||||
overlaps_left = []
|
||||
windows_final = []
|
||||
recon = ""
|
||||
try:
|
||||
overlap_low, overlap_high = tuple(_get_sizes(note_type).get("overlap", (40, 60)))
|
||||
except Exception:
|
||||
overlap_low, overlap_high = (40, 60)
|
||||
overlap_target = int(overlap_low)
|
||||
|
||||
for i, seg in enumerate(segments):
|
||||
if i == 0:
|
||||
# erstes Fenster: kein linker Kontext
|
||||
windows_final.append(seg)
|
||||
overlaps_left.append(0)
|
||||
recon += seg
|
||||
else:
|
||||
# synthetischer linker Kontext = Suffix des bisher rekonstruierten Texts
|
||||
k = min(overlap_target, len(recon))
|
||||
left_ctx = recon[-k:] if k > 0 else ""
|
||||
windows_final.append(left_ctx + seg)
|
||||
overlaps_left.append(k)
|
||||
recon += seg # Rekonstruktion bleibt kerntreu
|
||||
|
||||
# 3) overlap_right bestimmen
|
||||
overlaps_right: List[int] = []
|
||||
for i in range(len(windows_final)):
|
||||
if i + 1 < len(windows_final):
|
||||
ov = _overlap_len_suffix_prefix(windows_final[i], windows_final[i + 1], max_probe=4096)
|
||||
else:
|
||||
ov = 0
|
||||
overlaps_right.append(ov)
|
||||
|
||||
# 4) start/end-Offsets (exakt via note_text, sonst kumulativ)
|
||||
starts: List[int] = [0] * len(segments)
|
||||
ends: List[int] = [0] * len(segments)
|
||||
pos = 0
|
||||
if isinstance(note_text, str) and note_text:
|
||||
search_pos = 0
|
||||
for i, seg in enumerate(segments):
|
||||
if not seg:
|
||||
starts[i] = ends[i] = search_pos
|
||||
continue
|
||||
j = note_text.find(seg, search_pos)
|
||||
if j >= 0:
|
||||
starts[i] = j
|
||||
ends[i] = j + len(seg)
|
||||
search_pos = ends[i]
|
||||
else:
|
||||
# Fallback: kumulativ
|
||||
starts[i] = pos
|
||||
pos += len(seg)
|
||||
ends[i] = pos
|
||||
else:
|
||||
for i, seg in enumerate(segments):
|
||||
starts[i] = pos
|
||||
pos += len(seg)
|
||||
ends[i] = pos
|
||||
|
||||
# 5) Payload-Dicts
|
||||
payloads: List[Dict[str, Any]] = []
|
||||
for i, (win, seg) in enumerate(zip(windows_final, segments)):
|
||||
chunk_id = ids_in[i] or f"{note_id}#{i+1}"
|
||||
pl: Dict[str, Any] = {
|
||||
for c in chunks:
|
||||
idx = c.get("index", len(payloads))
|
||||
text = c.get("text") if isinstance(c, dict) else (str(c) if c is not None else "")
|
||||
text = text if isinstance(text, str) else str(text or "")
|
||||
|
||||
key = f"{note_id}|{idx}"
|
||||
h = hashlib.sha1(key.encode("utf-8")).hexdigest()[:12] if note_id else hashlib.sha1(f"{path}|{idx}".encode("utf-8")).hexdigest()[:12]
|
||||
chunk_id = f"{note_id}-{idx:03d}-{h}" if note_id else f"{h}"
|
||||
|
||||
payload = {
|
||||
"note_id": note_id,
|
||||
"chunk_id": chunk_id,
|
||||
"id": chunk_id, # Alias
|
||||
"chunk_index": i,
|
||||
"seq": seqs[i],
|
||||
"path": rel_path,
|
||||
"window": win,
|
||||
"text": seg,
|
||||
"start": starts[i],
|
||||
"end": ends[i],
|
||||
"overlap_left": overlaps_left[i],
|
||||
"overlap_right": overlaps_right[i],
|
||||
"index": idx,
|
||||
"title": title,
|
||||
"type": note_type,
|
||||
"path": path, # <- garantiert vorhanden
|
||||
"text": text, # <- nie leer, sonst werden keine Chunks erzeugt
|
||||
"retriever_weight": retriever_weight,
|
||||
"chunk_profile": chunk_profile,
|
||||
}
|
||||
# optionale Metadaten
|
||||
if note_type:
|
||||
pl["type"] = note_type
|
||||
if note_title is not None:
|
||||
pl["title"] = note_title
|
||||
if note_tags is not None:
|
||||
pl["tags"] = note_tags
|
||||
if token_counts[i] is not None:
|
||||
pl["token_count"] = int(token_counts[i])
|
||||
if section_titles[i] is not None:
|
||||
pl["section_title"] = section_titles[i]
|
||||
if section_paths[i] is not None:
|
||||
sp = str(section_paths[i]).replace("\\", "/")
|
||||
pl["section_path"] = sp if sp else "/"
|
||||
payloads.append(pl)
|
||||
json.loads(json.dumps(payload, ensure_ascii=False))
|
||||
payloads.append(payload)
|
||||
|
||||
return payloads
|
||||
|
||||
|
||||
# __main__ Demo (optional)
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
fm = {"id": "demo", "title": "Demo", "type": "concept"}
|
||||
# Beispiel ohne echte Fenster → erzeugt synthetische Overlaps
|
||||
chunks = [
|
||||
{"id": "demo#1", "text": "Alpha Beta Gamma"},
|
||||
{"id": "demo#2", "text": "Gamma Delta"},
|
||||
{"id": "demo#3", "text": "Delta Epsilon Zeta"},
|
||||
]
|
||||
pls = make_chunk_payloads(fm, "path/demo.md", chunks, note_text="Alpha Beta Gamma Delta Epsilon Zeta")
|
||||
from pprint import pprint
|
||||
pprint(pls)
|
||||
recon = "".join(p["text"] for p in pls)
|
||||
print("RECON:", recon)
|
||||
@@ -0,0 +1,94 @@
|
||||
# app/core/edges_writer.py
|
||||
from __future__ import annotations
|
||||
import hashlib
|
||||
from typing import Dict, List, Iterable, Tuple
|
||||
|
||||
try:
|
||||
# Dein Modul mit der Schemadefinition und der Builder-Funktion
|
||||
from app.core.edges import build_edges_for_note # noqa: F401
|
||||
except Exception as e:
|
||||
raise RuntimeError("Konnte app.core.edges nicht importieren. "
|
||||
"Bitte sicherstellen, dass app/core/edges.py vorhanden ist.") from e
|
||||
|
||||
def _edge_uid(kind: str, source_id: str, target_id: str, scope: str) -> str:
|
||||
"""
|
||||
Deterministische, kurze ID für eine Edge.
|
||||
Kollisionen sind praktisch ausgeschlossen (BLAKE2s über den Kanonischen Schlüssel).
|
||||
"""
|
||||
key = f"{kind}|{source_id}|{target_id}|{scope}"
|
||||
return hashlib.blake2s(key.encode("utf-8"), digest_size=12).hexdigest()
|
||||
|
||||
def ensure_edges_collection(qdrant_client, collection: str) -> None:
|
||||
"""
|
||||
Legt die Edge-Collection an, falls sie nicht existiert.
|
||||
Minimal: 1D-Vector (Dummy), Cosine. Payload-only-Collections sind je nach Qdrant-Version heikel.
|
||||
"""
|
||||
from qdrant_client.http import models as qm
|
||||
|
||||
existing = [c.name for c in qdrant_client.get_collections().collections]
|
||||
if collection in existing:
|
||||
return
|
||||
|
||||
qdrant_client.recreate_collection(
|
||||
collection_name=collection,
|
||||
vectors_config=qm.VectorParams(size=1, distance=qm.Distance.COSINE),
|
||||
on_disk_payload=True,
|
||||
)
|
||||
|
||||
def edges_from_note(
|
||||
note_id: str,
|
||||
chunk_payloads: List[Dict],
|
||||
note_level_refs: Iterable[str] | None,
|
||||
*,
|
||||
include_note_scope_refs: bool = False,
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Ruft deinen Edge-Builder auf und gibt die (deduplizierten) Edge-Payloads zurück.
|
||||
Keine Schemaänderung – exakt das aus app/core/edges.py.
|
||||
"""
|
||||
return build_edges_for_note(
|
||||
note_id=note_id,
|
||||
chunk_payloads=chunk_payloads,
|
||||
note_level_refs=list(note_level_refs or []),
|
||||
include_note_scope_refs=include_note_scope_refs,
|
||||
)
|
||||
|
||||
def upsert_edges(
|
||||
qdrant_client,
|
||||
collection: str,
|
||||
edge_payloads: List[Dict],
|
||||
) -> Tuple[int, int]:
|
||||
"""
|
||||
Schreibt Edges als Points in Qdrant.
|
||||
- id: deterministisch aus (kind, source_id, target_id, scope)
|
||||
- vector: [0.0] Dummy
|
||||
- payload: Edge-Dict (unverändert, siehe Schema in app/core/edges.py)
|
||||
Gibt (anzahl_points, anzahl_unique_keys) zurück.
|
||||
"""
|
||||
from qdrant_client.models import PointStruct
|
||||
|
||||
if not edge_payloads:
|
||||
return 0, 0
|
||||
|
||||
points = []
|
||||
seen = set()
|
||||
for e in edge_payloads:
|
||||
key = (e.get("kind"), e.get("source_id"), e.get("target_id"), e.get("scope"))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
eid = _edge_uid(*key)
|
||||
points.append(
|
||||
PointStruct(
|
||||
id=eid,
|
||||
vector=[0.0],
|
||||
payload=e,
|
||||
)
|
||||
)
|
||||
|
||||
if not points:
|
||||
return 0, 0
|
||||
|
||||
ensure_edges_collection(qdrant_client, collection)
|
||||
qdrant_client.upsert(collection_name=collection, points=points)
|
||||
return len(points), len(seen)
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Datei: app/core/env_vars.py
|
||||
Version: 1.1.0 (2025-11-08)
|
||||
|
||||
Zweck
|
||||
Einheitliche Auflösung von ENV-Variablen (Prefix, Qdrant, Embeddings, Hashing)
|
||||
mit Abwärtskompatibilität.
|
||||
|
||||
Grundsatz
|
||||
- Für Qdrant-Funktionen ist 'COLLECTION_PREFIX' der Primärschlüssel.
|
||||
- 'MINDNET_PREFIX' bleibt für App-/UI-/Exporter-Kontexte nutzbar.
|
||||
- Fallbacks sorgen dafür, dass ältere Umgebungen weiter funktionieren.
|
||||
|
||||
Wichtig
|
||||
- Lädt optional eine .env (wenn python-dotenv verfügbar ist).
|
||||
- Überschreibt keine bereits gesetzten OS-Variablen (override=False).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Optional, Dict
|
||||
|
||||
# Optional: .env automatisch laden (ohne Hard-Fail, falls nicht vorhanden)
|
||||
try:
|
||||
from dotenv import load_dotenv, find_dotenv # type: ignore
|
||||
_p = find_dotenv()
|
||||
if _p:
|
||||
load_dotenv(_p, override=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# -------- Prefix-Auflösung --------
|
||||
|
||||
def get_collection_prefix(cli_override: Optional[str] = None) -> str:
|
||||
"""
|
||||
Für Qdrant-relevante Funktionen:
|
||||
1) CLI-Override (--prefix)
|
||||
2) ENV COLLECTION_PREFIX
|
||||
3) ENV MINDNET_PREFIX (Fallback)
|
||||
4) 'mindnet' (Default)
|
||||
"""
|
||||
if cli_override and str(cli_override).strip():
|
||||
return str(cli_override).strip()
|
||||
return (
|
||||
os.getenv("COLLECTION_PREFIX")
|
||||
or os.getenv("MINDNET_PREFIX")
|
||||
or "mindnet"
|
||||
)
|
||||
|
||||
def get_mindnet_prefix(cli_override: Optional[str] = None) -> str:
|
||||
"""
|
||||
Für App-/UI-/Exporter-Kontexte:
|
||||
1) CLI-Override (--prefix)
|
||||
2) ENV MINDNET_PREFIX
|
||||
3) ENV COLLECTION_PREFIX (Fallback)
|
||||
4) 'mindnet'
|
||||
"""
|
||||
if cli_override and str(cli_override).strip():
|
||||
return str(cli_override).strip()
|
||||
return (
|
||||
os.getenv("MINDNET_PREFIX")
|
||||
or os.getenv("COLLECTION_PREFIX")
|
||||
or "mindnet"
|
||||
)
|
||||
|
||||
def get_prefix(cli_override: Optional[str] = None, target: str = "qdrant") -> str:
|
||||
"""
|
||||
Universelle Hülle (abwärtskompatibel):
|
||||
target='qdrant' -> get_collection_prefix
|
||||
target='app' -> get_mindnet_prefix
|
||||
"""
|
||||
if target.lower() == "app":
|
||||
return get_mindnet_prefix(cli_override)
|
||||
return get_collection_prefix(cli_override)
|
||||
|
||||
# -------- Qdrant / Embeddings / Hashing --------
|
||||
|
||||
def get_qdrant_url(default: str = "http://127.0.0.1:6333") -> str:
|
||||
return os.getenv("QDRANT_URL", default)
|
||||
|
||||
def get_qdrant_api_key(default: str = "") -> str:
|
||||
return os.getenv("QDRANT_API_KEY", default)
|
||||
|
||||
def get_vector_dim(default: int = 384) -> int:
|
||||
try:
|
||||
return int(os.getenv("VECTOR_DIM", str(default)))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def get_embed_url(default: Optional[str] = None) -> Optional[str]:
|
||||
return os.getenv("EMBED_URL", default)
|
||||
|
||||
def get_hash_env() -> Dict[str, str]:
|
||||
"""
|
||||
Liefert die Hash-Konfiguration (nur Aggregation; die Auswertung bleibt in den Skripten).
|
||||
"""
|
||||
return {
|
||||
"MINDNET_HASH_COMPARE": os.getenv("MINDNET_HASH_COMPARE", ""),
|
||||
"MINDNET_HASH_SOURCE": os.getenv("MINDNET_HASH_SOURCE", ""),
|
||||
"MINDNET_HASH_NORMALIZE": os.getenv("MINDNET_HASH_NORMALIZE", ""),
|
||||
}
|
||||
+81
-206
@@ -1,229 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Modul: app/core/note_payload.py
|
||||
# Version: 1.7.0
|
||||
# Datum: 2025-09-09
|
||||
|
||||
# note_payload.py
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
import os, json, pathlib, yaml
|
||||
|
||||
try:
|
||||
from app.core.parser import read_markdown, extract_wikilinks, FRONTMATTER_RE
|
||||
except Exception: # pragma: no cover
|
||||
from .parser import read_markdown, extract_wikilinks, FRONTMATTER_RE # type: ignore
|
||||
def _as_dict(note: Any) -> Dict[str, Any]:
|
||||
if isinstance(note, dict):
|
||||
return note
|
||||
d: Dict[str, Any] = {}
|
||||
for attr in ("id","note_id","title","path","frontmatter","meta","metadata","type","created","modified","date","tags"):
|
||||
if hasattr(note, attr):
|
||||
d[attr] = getattr(note, attr)
|
||||
# Normalisiere Frontmatter
|
||||
fm = d.get("frontmatter") or d.get("meta") or d.get("metadata") or {}
|
||||
d["frontmatter"] = fm if isinstance(fm, dict) else {}
|
||||
return d
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[Dict[str,Any]]]:
|
||||
path = kwargs.get("path")
|
||||
types_cfg = kwargs.get("types_config")
|
||||
# legacy positional: (path, types_config) oder (types_config, ...)
|
||||
for a in args:
|
||||
if path is None and isinstance(a, (str, pathlib.Path)):
|
||||
path = str(a)
|
||||
if types_cfg is None and isinstance(a, dict):
|
||||
types_cfg = a
|
||||
return path, types_cfg
|
||||
|
||||
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
|
||||
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
def _load_types_config(explicit: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
if isinstance(explicit, dict):
|
||||
return explicit
|
||||
for rel in ("config/config.yaml", "config/types.yaml"):
|
||||
p = pathlib.Path(rel)
|
||||
if p.exists():
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
|
||||
return data["types"]
|
||||
return data if isinstance(data, dict) else {}
|
||||
return {}
|
||||
|
||||
def _normalize_body(body: str, mode: str) -> str:
|
||||
if mode == "none":
|
||||
return body if body is not None else ""
|
||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
|
||||
text = "\n".join(line.rstrip() for line in text.split("\n"))
|
||||
return text
|
||||
def _coalesce(*vals):
|
||||
for v in vals:
|
||||
if v is not None:
|
||||
return v
|
||||
return None
|
||||
|
||||
def _resolve_hash_mode(explicit: Optional[str]) -> str:
|
||||
if explicit:
|
||||
val = explicit.strip().lower()
|
||||
else:
|
||||
val = (os.environ.get("MINDNET_HASH_MODE")
|
||||
or os.environ.get("MINDNET_HASH_COMPARE")
|
||||
or "body").strip().lower()
|
||||
if val in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
||||
return "full"
|
||||
if val in ("frontmatter", "fm"):
|
||||
return "frontmatter"
|
||||
return "body"
|
||||
|
||||
def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, Any]]:
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return "", {}
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
raw = f.read()
|
||||
return float(os.environ.get(name, default))
|
||||
except Exception:
|
||||
return "", {}
|
||||
m = FRONTMATTER_RE.match(raw)
|
||||
fm = {}
|
||||
if m:
|
||||
fm_txt = m.group(1)
|
||||
try:
|
||||
import yaml # lazy
|
||||
fm = yaml.safe_load(fm_txt) or {}
|
||||
except Exception:
|
||||
fm = {}
|
||||
body = raw[m.end():]
|
||||
else:
|
||||
body = raw
|
||||
return body, fm
|
||||
return default
|
||||
|
||||
def _sha256(s: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
h.update(s.encode("utf-8"))
|
||||
return h.hexdigest()
|
||||
def _ensure_list(x) -> list:
|
||||
if x is None: return []
|
||||
if isinstance(x, list): return [str(i) for i in x]
|
||||
if isinstance(x, (set, tuple)): return [str(i) for i in x]
|
||||
return [str(x)]
|
||||
|
||||
def _hash_for(mode: str, *, body: str, fm: Dict[str, Any], normalize: str) -> str:
|
||||
body_n = _normalize_body(body or "", normalize)
|
||||
fm_s = _canon_frontmatter(fm or {})
|
||||
if mode == "frontmatter":
|
||||
return _sha256(fm_s)
|
||||
if mode == "full":
|
||||
return _sha256(body_n + "\n--FM--\n" + fm_s)
|
||||
# default: body
|
||||
return _sha256(body_n)
|
||||
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||
n = _as_dict(note)
|
||||
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
|
||||
types_cfg = _load_types_config(types_cfg_explicit)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kernfunktion
|
||||
# ---------------------------------------------------------------------------
|
||||
fm = n.get("frontmatter") or {}
|
||||
note_type = str(fm.get("type") or n.get("type") or "note")
|
||||
cfg_for_type = types_cfg.get(note_type, {}) if isinstance(types_cfg, dict) else {}
|
||||
|
||||
def make_note_payload(
|
||||
parsed: Any,
|
||||
vault_root: Optional[str] = None,
|
||||
*,
|
||||
hash_mode: Optional[str] = None,
|
||||
hash_normalize: Optional[str] = None,
|
||||
hash_source: Optional[str] = None,
|
||||
file_path: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Liefert den Note-Payload inkl. Mehrfach-Hashes.
|
||||
- Es werden IMMER die drei Hashes für (body|frontmatter|full) unter
|
||||
'parsed:canonical' erzeugt (Schlüssel: z. B. 'body:parsed:canonical').
|
||||
- Zusätzlich werden – falls die aktuelle Konfig (source/normalize) davon
|
||||
abweicht – die drei Hashes unter den entsprechenden Schlüsseln erzeugt,
|
||||
z. B. 'frontmatter:raw:none'.
|
||||
- 'hash_fulltext' und 'hash_signature' repräsentieren den *aktuellen* Modus.
|
||||
"""
|
||||
# dict oder Objekt akzeptieren
|
||||
if isinstance(parsed, dict):
|
||||
fm = parsed.get("frontmatter") or {}
|
||||
body_parsed = parsed.get("body") or ""
|
||||
path = parsed.get("path") or ""
|
||||
else:
|
||||
fm = getattr(parsed, "frontmatter", {}) or {}
|
||||
body_parsed = getattr(parsed, "body", "") or ""
|
||||
path = getattr(parsed, "path", "") or ""
|
||||
|
||||
# Zielpfad relativieren
|
||||
rel_path = path
|
||||
default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
|
||||
retriever_weight = _coalesce(fm.get("retriever_weight"), cfg_for_type.get("retriever_weight"), default_rw)
|
||||
try:
|
||||
if vault_root:
|
||||
rel = os.path.relpath(path, vault_root)
|
||||
rel = rel.replace("\\", "/").lstrip("/")
|
||||
rel_path = rel
|
||||
retriever_weight = float(retriever_weight)
|
||||
except Exception:
|
||||
pass
|
||||
retriever_weight = default_rw
|
||||
|
||||
# Konfiguration auflösen
|
||||
mode_resolved = _resolve_hash_mode(hash_mode) # body|frontmatter|full
|
||||
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower() # parsed|raw
|
||||
norm = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower() # canonical|none
|
||||
chunk_profile = _coalesce(fm.get("chunk_profile"), cfg_for_type.get("chunk_profile"), os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE","medium"))
|
||||
chunk_profile = chunk_profile if isinstance(chunk_profile, str) else "medium"
|
||||
|
||||
# Body-Quelle laden
|
||||
raw_body, raw_fm = ("", {})
|
||||
if src == "raw":
|
||||
raw_body, raw_fm = _read_raw_body_from_file(file_path or path)
|
||||
if isinstance(raw_fm, dict) and raw_fm:
|
||||
merged_fm = dict(fm)
|
||||
for k, v in raw_fm.items():
|
||||
merged_fm.setdefault(k, v)
|
||||
fm = merged_fm
|
||||
body_for_hash = raw_body
|
||||
else:
|
||||
body_for_hash = body_parsed
|
||||
edge_defaults = _ensure_list(_coalesce(fm.get("edge_defaults"), cfg_for_type.get("edge_defaults"), []))
|
||||
|
||||
# --- 1) Standard-Tripel (parsed:canonical) immer erzeugen ---
|
||||
std_src = "parsed"
|
||||
std_norm = "canonical"
|
||||
std_hashes: Dict[str, str] = {}
|
||||
for m in ("body", "frontmatter", "full"):
|
||||
std_hashes[f"{m}:{std_src}:{std_norm}"] = _hash_for(
|
||||
m, body=body_parsed, fm=fm, normalize=std_norm
|
||||
)
|
||||
note_id = n.get("note_id") or n.get("id") or fm.get("id")
|
||||
title = n.get("title") or fm.get("title") or ""
|
||||
path = n.get("path") or path_arg
|
||||
if isinstance(path, pathlib.Path):
|
||||
path = str(path)
|
||||
|
||||
# Convenience-Felder (für Tools)
|
||||
hash_body = std_hashes["body:parsed:canonical"]
|
||||
hash_frontmatter = std_hashes["frontmatter:parsed:canonical"]
|
||||
hash_full = std_hashes["full:parsed:canonical"]
|
||||
|
||||
# --- 2) Hashes für die *aktuelle* Konfiguration (falls abweichend) ---
|
||||
cur_hashes: Dict[str, str] = {}
|
||||
if not (src == std_src and norm == std_norm):
|
||||
for m in ("body", "frontmatter", "full"):
|
||||
cur_hashes[f"{m}:{src}:{norm}"] = _hash_for(
|
||||
m, body=body_for_hash, fm=fm, normalize=norm
|
||||
)
|
||||
|
||||
# --- 3) Aktueller Modus für Backwards-Compat Felder ---
|
||||
current_hash = _hash_for(mode_resolved, body=body_for_hash, fm=fm, normalize=norm)
|
||||
hash_signature = f"{mode_resolved}:{src}:{norm}:{current_hash}"
|
||||
|
||||
# Wikilinks (Note-Ebene)
|
||||
refs = list(dict.fromkeys(extract_wikilinks(body_parsed))) if body_parsed else []
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"note_id": fm.get("id") or fm.get("note_id"),
|
||||
"title": fm.get("title"),
|
||||
"type": fm.get("type"),
|
||||
"status": fm.get("status"),
|
||||
"created": fm.get("created"),
|
||||
"updated": fm.get("updated"),
|
||||
"path": rel_path or fm.get("path"),
|
||||
"tags": fm.get("tags"),
|
||||
# Volltext für verlustfreien Export
|
||||
"fulltext": body_parsed,
|
||||
# Backwards-Compat:
|
||||
"hash_fulltext": current_hash,
|
||||
"hash_signature": hash_signature,
|
||||
# Option C: Mehrfach-Hashes
|
||||
"hashes": {**std_hashes, **cur_hashes},
|
||||
"hash_body": hash_body,
|
||||
"hash_frontmatter": hash_frontmatter,
|
||||
"hash_full": hash_full,
|
||||
# Fallback-Refs
|
||||
"references": refs,
|
||||
payload = {
|
||||
"note_id": note_id,
|
||||
"title": title,
|
||||
"type": note_type,
|
||||
"path": path or "", # immer vorhanden
|
||||
"retriever_weight": retriever_weight,
|
||||
"chunk_profile": chunk_profile,
|
||||
"edge_defaults": edge_defaults,
|
||||
}
|
||||
|
||||
for k in ("area", "project", "source", "lang", "slug", "aliases"):
|
||||
if k in fm:
|
||||
payload[k] = fm[k]
|
||||
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
|
||||
if tags: payload["tags"] = _ensure_list(tags)
|
||||
for k in ("created","modified","date"):
|
||||
v = fm.get(k) or n.get(k)
|
||||
if v: payload[k] = str(v)
|
||||
|
||||
json.loads(json.dumps(payload, ensure_ascii=False))
|
||||
return payload
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI – Sichtprüfung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _cli() -> None:
|
||||
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
|
||||
ap.add_argument("--from-file", dest="src", required=True)
|
||||
ap.add_argument("--vault-root", dest="vault_root", default=None)
|
||||
ap.add_argument("--print", dest="do_print", action="store_true")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None)
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
parsed = read_markdown(args.src)
|
||||
payload = make_note_payload(
|
||||
parsed,
|
||||
vault_root=args.vault_root,
|
||||
hash_mode=args.hash_mode,
|
||||
hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source,
|
||||
file_path=args.src,
|
||||
)
|
||||
if args.do_print:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
_cli()
|
||||
+118
-232
@@ -1,55 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
app/core/qdrant_points.py
|
||||
app/core/qdrant_points.py - robust points helpers for Qdrant
|
||||
|
||||
Zweck
|
||||
- Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
|
||||
- Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
|
||||
- alt: {'edge_type','src_id','dst_id', ...}
|
||||
- neu: {'kind','source_id','target_id', ...}
|
||||
- Single source of truth for building PointStruct for notes/chunks/edges
|
||||
- Backward-compatible payloads for edges
|
||||
- Handles both Single-Vector and Named-Vector collections
|
||||
- Deterministic overrides via ENV to avoid auto-detection traps:
|
||||
* NOTES_VECTOR_NAME, CHUNKS_VECTOR_NAME, EDGES_VECTOR_NAME
|
||||
* MINDNET_VECTOR_NAME (fallback)
|
||||
> Set to a concrete name (e.g. "text") to force Named-Vector with that name
|
||||
> Set to "__single__" (or "single") to force Single-Vector
|
||||
|
||||
Version
|
||||
- 1.3 (2025-09-08)
|
||||
|
||||
Änderungen (ggü. 1.2)
|
||||
- points_for_edges() akzeptiert jetzt beide Edge-Schemata.
|
||||
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
|
||||
stabile 'edge_id' zurück in die Payload.
|
||||
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
|
||||
|
||||
Aufruf / Verwendung
|
||||
- Wird von Import-/Backfill-Skripten via:
|
||||
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
|
||||
eingebunden. Keine CLI.
|
||||
|
||||
Hinweise
|
||||
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
|
||||
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
|
||||
Version: 1.5.0 (2025-11-08)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import uuid
|
||||
from typing import List, Tuple
|
||||
from qdrant_client.http import models as rest
|
||||
from typing import List, Tuple, Iterable, Optional, Dict, Any
|
||||
|
||||
from qdrant_client.http import models as rest
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
# --------------------- ID helpers ---------------------
|
||||
|
||||
def _to_uuid(stable_key: str) -> str:
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||
|
||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
# --------------------- Points builders ---------------------
|
||||
|
||||
def _to_uuid(stable_key: str) -> str:
|
||||
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||
|
||||
|
||||
def points_for_note(
|
||||
prefix: str,
|
||||
note_payload: dict,
|
||||
note_vec: List[float] | None,
|
||||
dim: int,
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
|
||||
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
|
||||
notes_col, _, _ = _names(prefix)
|
||||
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
||||
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
||||
@@ -57,18 +40,7 @@ def points_for_note(
|
||||
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
||||
return notes_col, [pt]
|
||||
|
||||
|
||||
def points_for_chunks(
|
||||
prefix: str,
|
||||
chunk_payloads: List[dict],
|
||||
vectors: List[List[float]],
|
||||
) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Chunks-Collection: erwartet pro Chunk einen Vektor.
|
||||
Robustheit:
|
||||
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
|
||||
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
|
||||
"""
|
||||
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
_, chunks_col, _ = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
||||
@@ -81,22 +53,12 @@ def points_for_chunks(
|
||||
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
||||
return chunks_col, points
|
||||
|
||||
|
||||
def _normalize_edge_payload(pl: dict) -> dict:
|
||||
"""
|
||||
Sorgt für kompatible Feldnamen.
|
||||
akzeptiert:
|
||||
- neu: kind, source_id, target_id, seq?
|
||||
- alt: edge_type, src_id, dst_id, order?/index?
|
||||
schreibt zurück: kind, source_id, target_id, seq?
|
||||
"""
|
||||
# bereits neu?
|
||||
kind = pl.get("kind") or pl.get("edge_type") or "edge"
|
||||
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
|
||||
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
|
||||
seq = pl.get("seq") or pl.get("order") or pl.get("index")
|
||||
|
||||
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
|
||||
pl.setdefault("kind", kind)
|
||||
pl.setdefault("source_id", source_id)
|
||||
pl.setdefault("target_id", target_id)
|
||||
@@ -104,18 +66,11 @@ def _normalize_edge_payload(pl: dict) -> dict:
|
||||
pl["seq"] = seq
|
||||
return pl
|
||||
|
||||
|
||||
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
||||
"""
|
||||
Edges-Collection mit 1D-Dummy-Vektor.
|
||||
- Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
|
||||
- Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
|
||||
"""
|
||||
_, _, edges_col = _names(prefix)
|
||||
points: List[rest.PointStruct] = []
|
||||
for raw in edge_payloads:
|
||||
pl = _normalize_edge_payload(raw)
|
||||
|
||||
edge_id = pl.get("edge_id")
|
||||
if not edge_id:
|
||||
kind = pl.get("kind", "edge")
|
||||
@@ -124,59 +79,126 @@ def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[
|
||||
seq = pl.get("seq") or ""
|
||||
edge_id = f"{kind}:{s}->{t}#{seq}"
|
||||
pl["edge_id"] = edge_id
|
||||
|
||||
point_id = _to_uuid(edge_id)
|
||||
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
||||
return edges_col, points
|
||||
|
||||
# --------------------- Vector schema & overrides ---------------------
|
||||
|
||||
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
|
||||
def _preferred_name(candidates: List[str]) -> str:
|
||||
for k in ("text", "default", "embedding", "content"):
|
||||
if k in candidates:
|
||||
return k
|
||||
return sorted(candidates)[0]
|
||||
|
||||
def _env_override_for_collection(collection: str) -> Optional[str]:
|
||||
"""
|
||||
Returns:
|
||||
- "__single__" to force single-vector
|
||||
- concrete name (str) to force named-vector with that name
|
||||
- None to auto-detect
|
||||
"""
|
||||
base = os.getenv("MINDNET_VECTOR_NAME")
|
||||
if collection.endswith("_notes"):
|
||||
base = os.getenv("NOTES_VECTOR_NAME", base)
|
||||
elif collection.endswith("_chunks"):
|
||||
base = os.getenv("CHUNKS_VECTOR_NAME", base)
|
||||
elif collection.endswith("_edges"):
|
||||
base = os.getenv("EDGES_VECTOR_NAME", base)
|
||||
|
||||
if not base:
|
||||
return None
|
||||
val = base.strip()
|
||||
if val.lower() in ("__single__", "single"):
|
||||
return "__single__"
|
||||
return val # concrete name
|
||||
|
||||
def _get_vector_schema(client: QdrantClient, collection_name: str) -> dict:
|
||||
"""
|
||||
Return {"kind": "single", "size": int} or {"kind": "named", "names": [...], "primary": str}.
|
||||
"""
|
||||
try:
|
||||
info = client.get_collection(collection_name=collection_name)
|
||||
vecs = getattr(info, "vectors", None)
|
||||
# Single-vector config
|
||||
if hasattr(vecs, "size") and isinstance(vecs.size, int):
|
||||
return {"kind": "single", "size": vecs.size}
|
||||
# Named-vectors config (dict-like in .config)
|
||||
cfg = getattr(vecs, "config", None)
|
||||
if isinstance(cfg, dict) and cfg:
|
||||
names = list(cfg.keys())
|
||||
if names:
|
||||
return {"kind": "named", "names": names, "primary": _preferred_name(names)}
|
||||
except Exception:
|
||||
pass
|
||||
return {"kind": "single", "size": None}
|
||||
|
||||
def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruct]:
|
||||
out: List[rest.PointStruct] = []
|
||||
for pt in points:
|
||||
vec = getattr(pt, "vector", None)
|
||||
if isinstance(vec, dict):
|
||||
if name in vec:
|
||||
out.append(pt)
|
||||
else:
|
||||
# take any existing entry; if empty dict fallback to [0.0]
|
||||
fallback_vec = None
|
||||
try:
|
||||
fallback_vec = list(next(iter(vec.values())))
|
||||
except Exception:
|
||||
fallback_vec = [0.0]
|
||||
out.append(rest.PointStruct(id=pt.id, vector={name: fallback_vec}, payload=pt.payload))
|
||||
elif vec is not None:
|
||||
out.append(rest.PointStruct(id=pt.id, vector={name: vec}, payload=pt.payload))
|
||||
else:
|
||||
out.append(pt)
|
||||
return out
|
||||
|
||||
# --------------------- Qdrant ops ---------------------
|
||||
|
||||
def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointStruct]) -> None:
|
||||
if not points:
|
||||
return
|
||||
|
||||
# 1) ENV overrides come first
|
||||
override = _env_override_for_collection(collection)
|
||||
if override == "__single__":
|
||||
client.upsert(collection_name=collection, points=points, wait=True)
|
||||
return
|
||||
elif isinstance(override, str):
|
||||
client.upsert(collection_name=collection, points=_as_named(points, override), wait=True)
|
||||
return
|
||||
|
||||
# 2) Auto-detect schema
|
||||
schema = _get_vector_schema(client, collection)
|
||||
if schema.get("kind") == "named":
|
||||
name = schema.get("primary") or _preferred_name(schema.get("names") or [])
|
||||
client.upsert(collection_name=collection, points=_as_named(points, name), wait=True)
|
||||
return
|
||||
|
||||
# 3) Fallback single-vector
|
||||
client.upsert(collection_name=collection, points=points, wait=True)
|
||||
|
||||
# --- WP-04 Ergänzungen: Graph/Retriever Hilfsfunktionen ---
|
||||
from typing import Optional, Dict, Any, Iterable
|
||||
from qdrant_client import QdrantClient
|
||||
# --- Optional search helpers ---
|
||||
|
||||
def _filter_any(field: str, values: Iterable[str]) -> rest.Filter:
|
||||
"""Erzeuge OR-Filter: payload[field] == any(values)."""
|
||||
return rest.Filter(
|
||||
should=[
|
||||
rest.FieldCondition(key=field, match=rest.MatchValue(value=v))
|
||||
for v in values
|
||||
]
|
||||
)
|
||||
return rest.Filter(should=[rest.FieldCondition(key=field, match=rest.MatchValue(value=v)) for v in values])
|
||||
|
||||
def _merge_filters(*filters: Optional[rest.Filter]) -> Optional[rest.Filter]:
|
||||
"""Fasst mehrere Filter zu einem AND zusammen (None wird ignoriert)."""
|
||||
fs = [f for f in filters if f is not None]
|
||||
if not fs:
|
||||
return None
|
||||
if len(fs) == 1:
|
||||
return fs[0]
|
||||
# rest.Filter hat must/should; wir kombinieren als must=[...]
|
||||
must = []
|
||||
for f in fs:
|
||||
# Überführe vorhandene Bedingungen in must
|
||||
if getattr(f, "must", None):
|
||||
must.extend(f.must)
|
||||
if getattr(f, "should", None):
|
||||
# "should" als eigene Gruppe beilegen (Qdrant interpretiert OR)
|
||||
must.append(rest.Filter(should=f.should))
|
||||
if getattr(f, "must_not", None):
|
||||
# negative Bedingungen weiterreichen
|
||||
if "must_not" not in locals():
|
||||
pass
|
||||
return rest.Filter(must=must)
|
||||
|
||||
def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter]:
|
||||
"""
|
||||
Einfache Filterumsetzung:
|
||||
- Bei Listenwerten: OR über mehrere MatchValue (field == any(values))
|
||||
- Bei Skalarwerten: Gleichheit (field == value)
|
||||
Für komplexere Filter (z. B. tags ∈ payload.tags) bitte erweitern.
|
||||
"""
|
||||
if not filters:
|
||||
return None
|
||||
parts = []
|
||||
@@ -187,147 +209,11 @@ def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter
|
||||
parts.append(rest.Filter(must=[rest.FieldCondition(key=k, match=rest.MatchValue(value=v))]))
|
||||
return _merge_filters(*parts)
|
||||
|
||||
def search_chunks_by_vector(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
vector: list[float],
|
||||
top: int = 10,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
) -> list[tuple[str, float, dict]]:
|
||||
"""
|
||||
Vektorielle Suche in {prefix}_chunks.
|
||||
Rückgabe: Liste von (point_id, score, payload)
|
||||
"""
|
||||
def search_chunks_by_vector(client: QdrantClient, prefix: str, vector: List[float], top: int = 10, filters: Optional[Dict[str, Any]] = None) -> List[Tuple[str, float, dict]]:
|
||||
_, chunks_col, _ = _names(prefix)
|
||||
flt = _filter_from_dict(filters)
|
||||
res = client.search(
|
||||
collection_name=chunks_col,
|
||||
query_vector=vector,
|
||||
limit=top,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
query_filter=flt,
|
||||
)
|
||||
out: list[tuple[str, float, dict]] = []
|
||||
res = client.search(collection_name=chunks_col, query_vector=vector, limit=top, with_payload=True, with_vectors=False, query_filter=flt)
|
||||
out: List[Tuple[str, float, dict]] = []
|
||||
for r in res:
|
||||
out.append((str(r.id), float(r.score), dict(r.payload or {})))
|
||||
return out
|
||||
|
||||
def get_edges_for_sources(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
source_ids: list[str],
|
||||
edge_types: Optional[list[str]] = None,
|
||||
limit: int = 2048,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Hole Edges aus {prefix}_edges mit source_id ∈ source_ids (und optional kind ∈ edge_types).
|
||||
Liefert Payload-Dicts inkl. edge_id/source_id/target_id/kind/seq (falls vorhanden).
|
||||
"""
|
||||
_, _, edges_col = _names(prefix)
|
||||
f_src = _filter_any("source_id", source_ids)
|
||||
f_kind = _filter_any("kind", edge_types) if edge_types else None
|
||||
flt = _merge_filters(f_src, f_kind)
|
||||
|
||||
collected: list[dict] = []
|
||||
next_page = None
|
||||
while True:
|
||||
points, next_page = client.scroll(
|
||||
collection_name=edges_col,
|
||||
scroll_filter=flt,
|
||||
limit=min(512, limit - len(collected)),
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
offset=next_page,
|
||||
)
|
||||
for p in points:
|
||||
pl = dict(p.payload or {})
|
||||
# füge die deterministische ID hinzu (nützlich für Clients)
|
||||
pl.setdefault("id", str(p.id))
|
||||
collected.append(pl)
|
||||
if len(collected) >= limit:
|
||||
return collected
|
||||
if next_page is None:
|
||||
break
|
||||
return collected
|
||||
|
||||
def get_note_payload(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
note_id: str,
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
Hole eine Note anhand ihres payload.note_id (nicht internal UUID!).
|
||||
"""
|
||||
notes_col, _, _ = _names(prefix)
|
||||
flt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
points, _ = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=flt,
|
||||
limit=1,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
if not points:
|
||||
return None
|
||||
pl = dict(points[0].payload or {})
|
||||
pl.setdefault("id", str(points[0].id))
|
||||
return pl
|
||||
|
||||
def get_neighbor_nodes(
|
||||
client: QdrantClient,
|
||||
prefix: str,
|
||||
target_ids: list[str],
|
||||
limit_per_collection: int = 2048,
|
||||
) -> dict[str, dict]:
|
||||
"""
|
||||
Hole Payloads der Zielknoten (Notes/Chunks) zu den angegebenen IDs.
|
||||
IDs sind die stabilen payload-IDs (note_id/chunk_id), nicht internal UUIDs.
|
||||
Rückgabe: Mapping target_id -> payload
|
||||
"""
|
||||
notes_col, chunks_col, _ = _names(prefix)
|
||||
out: dict[str, dict] = {}
|
||||
|
||||
# Notes
|
||||
flt_notes = _filter_any("note_id", target_ids)
|
||||
next_page = None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=flt_notes,
|
||||
limit=256,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
offset=next_page,
|
||||
)
|
||||
for p in pts:
|
||||
pl = dict(p.payload or {})
|
||||
nid = pl.get("note_id")
|
||||
if nid and nid not in out:
|
||||
pl.setdefault("id", str(p.id))
|
||||
out[nid] = pl
|
||||
if next_page is None or len(out) >= limit_per_collection:
|
||||
break
|
||||
|
||||
# Chunks
|
||||
flt_chunks = _filter_any("chunk_id", target_ids)
|
||||
next_page = None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=chunks_col,
|
||||
scroll_filter=flt_chunks,
|
||||
limit=256,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
offset=next_page,
|
||||
)
|
||||
for p in pts:
|
||||
pl = dict(p.payload or {})
|
||||
cid = pl.get("chunk_id")
|
||||
if cid and cid not in out:
|
||||
pl.setdefault("id", str(p.id))
|
||||
out[cid] = pl
|
||||
if next_page is None or len(out) >= limit_per_collection:
|
||||
break
|
||||
|
||||
return out
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/type_registry.py
|
||||
Version: 1.0.0
|
||||
Datum: 2025-11-08
|
||||
|
||||
Zweck
|
||||
-----
|
||||
Lädt eine optionale Typ-Registry (config/types.yaml) und stellt
|
||||
komfortable Zugriffsfunktionen bereit. Die Registry ist *optional*:
|
||||
- Fehlt die Datei oder ist das YAML defekt, wird ein konservativer
|
||||
Default (Typ "concept") verwendet und es wird eine Warnung ausgegeben.
|
||||
- Änderungen an der Datei greifen nach einem Neustart des Prozesses.
|
||||
|
||||
Öffentliche API
|
||||
---------------
|
||||
- load_type_registry(path: str = "config/types.yaml") -> dict
|
||||
- get_type_config(note_type: str, reg: dict) -> dict
|
||||
- resolve_note_type(fm_type: str | None, reg: dict) -> str
|
||||
- effective_chunk_profile(note_type: str, reg: dict) -> str | None
|
||||
- profile_overlap(profile: str | None) -> tuple[int,int] # nur Overlap-Empfehlung
|
||||
|
||||
Hinweis
|
||||
-------
|
||||
Die Registry steuert KEINE Breaking Changes. Ohne Datei/Typ bleibt das
|
||||
Verhalten exakt wie im Release-Stand 20251105.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
import os
|
||||
|
||||
try:
|
||||
import yaml # PyYAML
|
||||
except Exception:
|
||||
yaml = None # wird erst benötigt, wenn eine Datei gelesen werden soll
|
||||
|
||||
# Konservativer Default – bewusst minimal
|
||||
_DEFAULT_REGISTRY: Dict[str, Any] = {
|
||||
"version": "1.0",
|
||||
"types": {
|
||||
"concept": {
|
||||
"chunk_profile": "medium",
|
||||
"edge_defaults": ["references", "related_to"],
|
||||
"retriever_weight": 1.0,
|
||||
}
|
||||
},
|
||||
"_using_defaults": True,
|
||||
"_warning": "types.yaml missing or invalid – using built-in defaults (type=concept).",
|
||||
}
|
||||
|
||||
# Chunk-Profile → Overlap-Empfehlungen (nur für synthetische Fensterbildung)
|
||||
# Die absoluten Chunk-Längen bleiben Aufgabe des Chunkers (assemble_chunks).
|
||||
_PROFILE_TO_OVERLAP: Dict[str, Tuple[int, int]] = {
|
||||
"short": (20, 30),
|
||||
"medium": (40, 60),
|
||||
"long": (60, 80),
|
||||
}
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def load_type_registry(path: str = "config/types.yaml") -> Dict[str, Any]:
|
||||
"""
|
||||
Lädt die Registry aus 'path'. Bei Fehlern wird ein konserviver Default geliefert.
|
||||
Die Rückgabe ist *prozessweit* gecached.
|
||||
"""
|
||||
if not path:
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
if not os.path.isfile(path):
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
if yaml is None:
|
||||
# PyYAML fehlt → auf Default zurückfallen
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
# Minimal validieren
|
||||
if not isinstance(data, dict) or "types" not in data or not isinstance(data["types"], dict):
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
data.setdefault("version", "1.0")
|
||||
data.setdefault("_using_defaults", False)
|
||||
return data
|
||||
except Exception:
|
||||
return dict(_DEFAULT_REGISTRY)
|
||||
|
||||
|
||||
def get_type_config(note_type: Optional[str], reg: Dict[str, Any]) -> Dict[str, Any]:
|
||||
t = (note_type or "concept").strip().lower()
|
||||
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
|
||||
return types.get(t) or types.get("concept") or _DEFAULT_REGISTRY["types"]["concept"]
|
||||
|
||||
|
||||
def resolve_note_type(fm_type: Optional[str], reg: Dict[str, Any]) -> str:
|
||||
"""Liefert einen gültigen Typ (unbekannt → 'concept')."""
|
||||
t = (fm_type or "concept").strip().lower()
|
||||
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
|
||||
return t if t in types else "concept"
|
||||
|
||||
|
||||
def effective_chunk_profile(note_type: Optional[str], reg: Dict[str, Any]) -> Optional[str]:
|
||||
cfg = get_type_config(note_type, reg)
|
||||
prof = cfg.get("chunk_profile")
|
||||
if isinstance(prof, str) and prof.strip():
|
||||
return prof.strip().lower()
|
||||
return None
|
||||
|
||||
|
||||
def profile_overlap(profile: Optional[str]) -> Tuple[int, int]:
|
||||
"""Gibt eine Overlap-Empfehlung (low, high) für das Profil zurück."""
|
||||
if not profile:
|
||||
return _PROFILE_TO_OVERLAP["medium"]
|
||||
return _PROFILE_TO_OVERLAP.get(profile.strip().lower(), _PROFILE_TO_OVERLAP["medium"])
|
||||
@@ -0,0 +1,18 @@
|
||||
version: 1.0
|
||||
types:
|
||||
concept:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["references", "related_to"]
|
||||
retriever_weight: 0.33
|
||||
task:
|
||||
chunk_profile: short
|
||||
edge_defaults: ["depends_on", "belongs_to"]
|
||||
retriever_weight: 0.8
|
||||
experience:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["derived_from", "inspired_by"]
|
||||
retriever_weight: 0.9
|
||||
project:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.95
|
||||
@@ -0,0 +1,21 @@
|
||||
# Changelog — WP-03 Type-Registry (2025-11-08)
|
||||
|
||||
## Added
|
||||
- `app/core/type_registry.py` — Loader & Helper (cached), robust Fallbacks.
|
||||
- `config/types.yaml` — Beispiel-Registry (version 1.0).
|
||||
- `scripts/import_markdown.py` v3.8.0 — Registry-Integration:
|
||||
- typvalidierung (unbekannt → `concept`, Warnung)
|
||||
- `chunk_profile` → an `make_chunk_payloads(...)`
|
||||
- `retriever_weight` → persistiert im Note-Payload
|
||||
- `edge_defaults["references"]` schaltet Note-Scope-Refs additiv ein
|
||||
- `app/core/chunk_payload.py` v2.3.0 — optionales `chunk_profile` für
|
||||
synthetische Overlaps (keine Schemaänderung).
|
||||
- `app/core/derive_edges.py` v1.5.0 — Type-Registry aktiviert note-scope
|
||||
References/Backlinks additiv.
|
||||
|
||||
## Changed
|
||||
- Keine Breaking Changes. Alle CLI-Flags, ENV-Variablen und Collection-
|
||||
Prefix-Logik bleiben unverändert.
|
||||
|
||||
## Removed
|
||||
- N/A
|
||||
@@ -0,0 +1,74 @@
|
||||
# Kurzhandbuch — Type-Registry (WP-03)
|
||||
|
||||
## Zweck
|
||||
Die Type-Registry macht den `type:`-Wert im Frontmatter maschinenlesbar
|
||||
und steuert:
|
||||
- Chunk-Profile (indirekt über Overlap bei fensterlosen Chunks),
|
||||
- typabhängige Default-Kanten (hier: Aktivierung von Note-Scope-References),
|
||||
- optionale Retriever-Gewichtungen (`retriever_weight`) für WP‑04.
|
||||
|
||||
## Aktivierung
|
||||
1. Legen Sie `config/types.yaml` an (siehe Beispiel in diesem Paket).
|
||||
2. Starten Sie Import/Server neu. Die Registry wird beim Prozessstart
|
||||
geladen und gecached.
|
||||
|
||||
Fehlt die Datei oder ist sie defekt, bleibt das System voll funktionsfähig
|
||||
(der Typ `concept` wird genutzt; Warnhinweis im Log).
|
||||
|
||||
## Schema (Beispiel)
|
||||
```yaml
|
||||
version: 1.0
|
||||
types:
|
||||
concept:
|
||||
chunk_profile: long
|
||||
edge_defaults: ["references", "related_to"]
|
||||
retriever_weight: 1.0
|
||||
task:
|
||||
chunk_profile: short
|
||||
edge_defaults: ["depends_on", "belongs_to"]
|
||||
retriever_weight: 0.8
|
||||
experience:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["derived_from", "inspired_by"]
|
||||
retriever_weight: 0.9
|
||||
project:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
retriever_weight: 0.95
|
||||
```
|
||||
|
||||
## Verhalten
|
||||
- **Chunking**: Die eigentliche Chunk-Länge bleibt weiterhin typabhängig über
|
||||
`chunk_config.get_sizes(type)` gesteuert. Falls der Chunker keine Fenster
|
||||
mit Overlap liefert, werden in `chunk_payload` synthetische Fenster erzeugt.
|
||||
Hier greift das `chunk_profile` mit folgenden Overlap-Empfehlungen:
|
||||
- short → (20,30), medium → (40,60), long → (60,80).
|
||||
- **Edges**: Enthält `edge_defaults` den Eintrag `references`, werden Note-
|
||||
Scope-References/Backlinks additiv aktiviert (kein Abbruch, keine Duplikate).
|
||||
Weitere Kantenarten bleiben unverändert.
|
||||
- **Retriever**: `retriever_weight` wird in der Note gespeichert und kann von
|
||||
WP‑04 im Ranking/Scoring berücksichtigt werden.
|
||||
|
||||
## Tests / Validierung
|
||||
- Dry-Run:
|
||||
```bash
|
||||
python3 -m scripts.import_markdown --vault ./test_vault
|
||||
```
|
||||
- Vollimport + Purge:
|
||||
```bash
|
||||
python3 -m scripts.import_markdown --vault ./test_vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
- Chunks-Qualität (Window vs Text):
|
||||
```bash
|
||||
python3 tests/check_chunks_window_vs_text.py --prefix "$COLLECTION_PREFIX"
|
||||
```
|
||||
- Roundtrip:
|
||||
```bash
|
||||
python3 -m scripts.export_markdown --out ./_exportVault --prefix "$COLLECTION_PREFIX"
|
||||
python3 tests/compare_vaults.py --src ./test_vault --dst ./_exportVault --focus body
|
||||
```
|
||||
|
||||
## Fallbacks & Robustheit
|
||||
- Registry nicht vorhanden/ungültig → `concept`, Warnung, keine Abbrüche.
|
||||
- Alle bestehenden ENV/Flags bleiben identisch.
|
||||
- Keine Änderungen an Punkt-IDs, Feldern oder Vektoren.
|
||||
+223
-104
@@ -1,38 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
|
||||
Version: 3.7.2
|
||||
Datum: 2025-09-30
|
||||
scripts/import_markdown.py
|
||||
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
|
||||
- Änderungserkennung „Option C“: mehrere Hash-Varianten werden parallel in der Note
|
||||
gespeichert (Feld `hashes` mit Schlüsseln `<mode>:<source>:<normalize>`). Der Vergleich
|
||||
nutzt NUR den aktuellen Modus-Key — ein Moduswechsel triggert keine Massenänderungen mehr.
|
||||
- „Erstimport-Fix“: Bei leerem Qdrant gilt Create-Fall automatisch als geändert.
|
||||
- `--baseline-modes`: fehlende Hash-Varianten still nachtragen (nur Notes upserten).
|
||||
- `--sync-deletes`: gezielte Lösch-Synchronisation (Dry-Run + Apply).
|
||||
- `--only-path`: exakt **eine** Datei (Pfad) importieren — nützlich für Diagnosefälle.
|
||||
Zweck
|
||||
-----
|
||||
- Liest Markdown-Notizen aus einem Vault ein
|
||||
- Erzeugt Note-Payload, Chunk-Payloads (+ optionale Embeddings) und Edges
|
||||
- Schreibt alles idempotent in Qdrant (Notes, Chunks, Edges)
|
||||
- Integriert eine optionale Type-Registry (types.yaml), um z. B. chunk_profile
|
||||
und retriever_weight pro Notiz-Typ zu steuern.
|
||||
|
||||
Neu in 3.7.1/3.7.2
|
||||
------------------
|
||||
- Chunk-Payloads: `window` (für Embeddings), `text` (überlappungsfrei, verlustfrei rekonstruierbar),
|
||||
`start/end/overlap_*`. Embeddings nutzen `window`.
|
||||
- **3.7.2:** Edges-Fehler führen nicht mehr zum Abbruch der gesamten Note; Note/Chunks werden trotzdem geschrieben.
|
||||
|
||||
Hash/Compare Konfiguration
|
||||
--------------------------
|
||||
- Vergleichsmodus:
|
||||
--hash-mode body|frontmatter|full
|
||||
oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE
|
||||
- Quelle:
|
||||
--hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE, Default parsed)
|
||||
- Normalisierung:
|
||||
--hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE, Default canonical)
|
||||
- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich
|
||||
den parsed Body-Text direkt.
|
||||
Wesentliche Fixes ggü. vorherigen fehlerhaften Ständen
|
||||
------------------------------------------------------
|
||||
- `embed_texts` wird optional importiert und defensiv geprüft (kein NameError mehr)
|
||||
- `effective_chunk_profile` / `effective_retriever_weight` und Registry-Helfer
|
||||
sind VOR `main()` definiert (kein NameError mehr)
|
||||
- `retriever_weight` wird in Note- und Chunk-Payload zuverlässig gesetzt
|
||||
- Robuste Kantenbildung; Fehler bei Edges blockieren Notes/Chunks nicht
|
||||
- Korrekte Verwendung von `scroll_filter` beim Qdrant-Client
|
||||
- `--purge-before-upsert` entfernt alte Chunks/Edges einer Note vor dem Upsert
|
||||
|
||||
Qdrant / ENV
|
||||
------------
|
||||
@@ -40,6 +27,7 @@ Qdrant / ENV
|
||||
- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
|
||||
- VECTOR_DIM (Default: 384)
|
||||
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
||||
- MINDNET_TYPES_FILE: Pfad zu types.yaml (optional; Default: ./types.yaml)
|
||||
|
||||
Beispiele
|
||||
---------
|
||||
@@ -67,6 +55,7 @@ from typing import Dict, List, Optional, Tuple, Any, Set
|
||||
from dotenv import load_dotenv
|
||||
from qdrant_client.http import models as rest
|
||||
|
||||
# --- Projekt-Imports ---
|
||||
from app.core.parser import (
|
||||
read_markdown,
|
||||
normalize_frontmatter,
|
||||
@@ -92,14 +81,86 @@ from app.core.qdrant_points import (
|
||||
upsert_batch,
|
||||
)
|
||||
|
||||
# embeddings sind optional (z. B. im reinen Payload-Backfill)
|
||||
try:
|
||||
from app.core.embed import embed_texts # optional
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
embed_texts = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Helper
|
||||
# Type-Registry (types.yaml) – Helper (robust, optional)
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def _env(name: str, default: Optional[str] = None) -> Optional[str]:
|
||||
v = os.getenv(name)
|
||||
return v if v is not None else default
|
||||
|
||||
def _load_json_or_yaml(path: str) -> dict:
|
||||
import io
|
||||
data: dict = {}
|
||||
if not path or not os.path.exists(path):
|
||||
return data
|
||||
try:
|
||||
import yaml # type: ignore
|
||||
with io.open(path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return data
|
||||
except Exception:
|
||||
# YAML evtl. nicht installiert – versuche JSON
|
||||
try:
|
||||
with io.open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return data
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def load_type_registry() -> dict:
|
||||
# Reihenfolge: ENV > ./types.yaml (im aktuellen Arbeitsverzeichnis)
|
||||
p = _env("MINDNET_TYPES_FILE", None)
|
||||
if p and os.path.exists(p):
|
||||
return _load_json_or_yaml(p)
|
||||
fallback = os.path.abspath("./types.yaml")
|
||||
return _load_json_or_yaml(fallback)
|
||||
|
||||
def get_type_config(note_type: Optional[str], reg: dict) -> dict:
|
||||
if not reg or not isinstance(reg, dict):
|
||||
return {}
|
||||
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
|
||||
if note_type and isinstance(note_type, str) and note_type in types:
|
||||
return types[note_type] or {}
|
||||
# Fallback: concept
|
||||
return types.get("concept", {}) or {}
|
||||
|
||||
def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
||||
if requested and isinstance(requested, str):
|
||||
return requested
|
||||
# Fallback wenn nichts gesetzt ist
|
||||
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
|
||||
return "concept" if "concept" in types else (requested or "concept")
|
||||
|
||||
def effective_chunk_profile(note_type: str, reg: dict) -> Optional[str]:
|
||||
cfg = get_type_config(note_type, reg)
|
||||
prof = cfg.get("chunk_profile")
|
||||
if isinstance(prof, str) and prof in {"short", "medium", "long"}:
|
||||
return prof
|
||||
return None
|
||||
|
||||
def effective_retriever_weight(note_type: str, reg: dict) -> Optional[float]:
|
||||
cfg = get_type_config(note_type, reg)
|
||||
w = cfg.get("retriever_weight")
|
||||
try:
|
||||
return float(w) if w is not None else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Sonstige Helper
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def iter_md(root: str) -> List[str]:
|
||||
@@ -115,17 +176,15 @@ def iter_md(root: str) -> List[str]:
|
||||
out.append(p)
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||
|
||||
|
||||
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
points, _ = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=f,
|
||||
scroll_filter=f, # wichtig: scroll_filter (nicht: filter)
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1,
|
||||
@@ -134,7 +193,6 @@ def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[D
|
||||
return None
|
||||
return points[0].payload or {}
|
||||
|
||||
|
||||
def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
out: Set[str] = set()
|
||||
@@ -158,7 +216,6 @@ def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||
_, chunks_col, edges_col = collections(prefix)
|
||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
@@ -172,7 +229,6 @@ def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||
except Exception as e:
|
||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
|
||||
|
||||
|
||||
def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
||||
notes_col, chunks_col, edges_col = collections(prefix)
|
||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
@@ -187,50 +243,56 @@ def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
|
||||
|
||||
|
||||
def _resolve_mode(val: Optional[str]) -> str:
|
||||
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
|
||||
if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
||||
return "full"
|
||||
if v in ("frontmatter", "fm"):
|
||||
return "frontmatter"
|
||||
return "body"
|
||||
# --- Neu: Existenz-Checks für Artefakte (fehlertoleranter Rebuild) ---
|
||||
|
||||
def _has_any_point(client, collection: str, note_id: str) -> bool:
|
||||
"""Prüft, ob es mind. einen Punkt mit note_id in der Collection gibt."""
|
||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
pts, _ = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=filt,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
limit=1,
|
||||
)
|
||||
return bool(pts)
|
||||
|
||||
def _env(key: str, default: str) -> str:
|
||||
return (os.environ.get(key) or default).strip().lower()
|
||||
def artifacts_missing(client, prefix: str, note_id: str) -> Tuple[bool, bool]:
|
||||
"""Gibt (chunks_missing, edges_missing) zurück."""
|
||||
_, chunks_col, edges_col = collections(prefix)
|
||||
chunks_missing = not _has_any_point(client, chunks_col, note_id)
|
||||
edges_missing = not _has_any_point(client, edges_col, note_id)
|
||||
return chunks_missing, edges_missing
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
def _resolve_mode(m: Optional[str]) -> str:
|
||||
m = (m or "body").strip().lower()
|
||||
return m if m in {"body", "frontmatter", "full"} else "body"
|
||||
|
||||
def main() -> None:
|
||||
load_dotenv()
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
|
||||
ap.add_argument("--purge-before-upsert", action="store_true",
|
||||
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
|
||||
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
|
||||
ap.add_argument("--only-path", help="Exakt diesen Markdown-Pfad verarbeiten (ignoriert --note-id)")
|
||||
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
|
||||
ap.add_argument("--force-replace", action="store_true",
|
||||
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None,
|
||||
help="Vergleichsmodus (Body | Frontmatter | Full)")
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
|
||||
help="Quelle für die Hash-Berechnung (Default: parsed)")
|
||||
ap.add_argument("--note-scope-refs", action="store_true",
|
||||
help="(Optional) erzeugt zusätzlich references:note/backlink:note (Default: aus)")
|
||||
ap.add_argument("--debug-hash-diff", action="store_true",
|
||||
help="(reserviert) optionaler Body-Diff")
|
||||
ap.add_argument("--compare-text", action="store_true",
|
||||
help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
|
||||
ap.add_argument("--baseline-modes", action="store_true",
|
||||
help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
|
||||
ap.add_argument("--sync-deletes", action="store_true",
|
||||
help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)")
|
||||
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="scripts.import_markdown",
|
||||
description="Importiert Markdown-Notizen in Qdrant (Notes/Chunks/Edges)."
|
||||
)
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Vault (Ordner mit .md-Dateien)")
|
||||
ap.add_argument("--only-path", help="Nur diese Datei verarbeiten (absolut oder relativ)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt nach Qdrant (sonst Dry-Run)")
|
||||
ap.add_argument("--purge-before-upsert", action="store_true", help="Alte Chunks/Edges der Note vorher löschen")
|
||||
ap.add_argument("--force-replace", action="store_true", help="Note/Chunks/Edges unabhängig von Hash neu schreiben")
|
||||
ap.add_argument("--note-id", help="Nur Notes mit dieser ID verarbeiten (Filter)")
|
||||
ap.add_argument("--note-scope-refs", action="store_true", help="Note-scope References/Backlinks erzeugen")
|
||||
ap.add_argument("--hash-mode", help="body|frontmatter|full (Default body)")
|
||||
ap.add_argument("--hash-source", help="parsed|raw (Default parsed)")
|
||||
ap.add_argument("--hash-normalize", help="canonical|none (Default canonical)")
|
||||
ap.add_argument("--compare-text", action="store_true", help="Parsed fulltext zusätzlich direkt vergleichen")
|
||||
ap.add_argument("--baseline-modes", action="store_true", help="Fehlende Hash-Varianten still nachtragen (Notes)")
|
||||
ap.add_argument("--sync-deletes", action="store_true", help="Qdrant->Vault Lösch-Sync (Dry-Run; mit --apply ausführen)")
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||
args = ap.parse_args()
|
||||
|
||||
@@ -241,6 +303,7 @@ def main() -> None:
|
||||
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
||||
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
||||
|
||||
# Qdrant
|
||||
cfg = QdrantConfig.from_env()
|
||||
if args.prefix:
|
||||
cfg.prefix = args.prefix.strip()
|
||||
@@ -248,9 +311,12 @@ def main() -> None:
|
||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
|
||||
# Type-Registry laden (optional)
|
||||
reg = load_type_registry()
|
||||
|
||||
root = os.path.abspath(args.vault)
|
||||
|
||||
# Dateiliste bestimmen
|
||||
# Dateiliste
|
||||
if args.only_path:
|
||||
only = os.path.abspath(args.only_path)
|
||||
files = [only]
|
||||
@@ -293,16 +359,15 @@ def main() -> None:
|
||||
|
||||
processed = 0
|
||||
for path in files:
|
||||
# -------- Parse & Validate --------
|
||||
try:
|
||||
parsed = read_markdown(path)
|
||||
if not parsed:
|
||||
continue
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
||||
continue
|
||||
if parsed is None:
|
||||
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
|
||||
continue
|
||||
|
||||
# --- Frontmatter prüfen ---
|
||||
try:
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
@@ -315,24 +380,54 @@ def main() -> None:
|
||||
|
||||
processed += 1
|
||||
|
||||
# -------- Build new payload (includes 'hashes') --------
|
||||
note_pl = make_note_payload(
|
||||
parsed,
|
||||
vault_root=root,
|
||||
hash_mode=mode,
|
||||
hash_normalize=norm,
|
||||
hash_source=src,
|
||||
file_path=path,
|
||||
)
|
||||
# --- Type-Registry anwenden (chunk_profile / retriever_weight) ---
|
||||
try:
|
||||
note_type = resolve_note_type(fm.get("type"), reg)
|
||||
except Exception:
|
||||
note_type = (fm.get("type") or "concept")
|
||||
fm["type"] = note_type or fm.get("type") or "concept"
|
||||
|
||||
prof = effective_chunk_profile(note_type, reg)
|
||||
if prof:
|
||||
fm["chunk_profile"] = prof
|
||||
|
||||
weight = effective_retriever_weight(note_type, reg)
|
||||
if weight is not None:
|
||||
try:
|
||||
fm["retriever_weight"] = float(weight)
|
||||
except Exception:
|
||||
pass # falls FM string-inkonsistent ist
|
||||
|
||||
# --- Payload aufbauen (inkl. Hashes) ---
|
||||
try:
|
||||
note_pl = make_note_payload(
|
||||
parsed,
|
||||
vault_root=root,
|
||||
hash_mode=mode,
|
||||
hash_normalize=norm,
|
||||
hash_source=src,
|
||||
file_path=path,
|
||||
)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"make_note_payload failed: {type(e).__name__}: {e}"}))
|
||||
continue
|
||||
|
||||
if not note_pl.get("fulltext"):
|
||||
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
||||
|
||||
# retriever_weight sicher in Note-Payload spiegeln (für spätere Filter)
|
||||
if "retriever_weight" not in note_pl and fm.get("retriever_weight") is not None:
|
||||
try:
|
||||
note_pl["retriever_weight"] = float(fm.get("retriever_weight"))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
note_id = note_pl.get("note_id") or fm.get("id")
|
||||
if not note_id:
|
||||
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
|
||||
continue
|
||||
|
||||
# -------- Fetch old payload --------
|
||||
# --- bestehenden Payload laden (zum Diff) ---
|
||||
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
||||
has_old = old_payload is not None
|
||||
|
||||
@@ -352,17 +447,28 @@ def main() -> None:
|
||||
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
||||
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
||||
|
||||
# -------- Chunks / Embeddings --------
|
||||
chunk_pls: List[Dict[str, Any]] = []
|
||||
# --- Chunks + Embeddings vorbereiten ---
|
||||
try:
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||
chunk_pls: List[Dict[str, Any]] = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
|
||||
continue
|
||||
|
||||
vecs: List[List[float]] = [[0.0] * cfg.dim for _ in chunk_pls]
|
||||
# retriever_weight auf Chunk-Payload spiegeln
|
||||
if fm.get("retriever_weight") is not None:
|
||||
try:
|
||||
rw = float(fm.get("retriever_weight"))
|
||||
for pl in chunk_pls:
|
||||
# Feld nur setzen, wenn noch nicht vorhanden
|
||||
if "retriever_weight" not in pl:
|
||||
pl["retriever_weight"] = rw
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Embeddings (fallback: Nullvektoren)
|
||||
vecs: List[List[float]] = [[0.0] * int(cfg.dim) for _ in chunk_pls]
|
||||
if embed_texts and chunk_pls:
|
||||
try:
|
||||
texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls]
|
||||
@@ -370,10 +476,14 @@ def main() -> None:
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
||||
|
||||
# -------- Edges (robust) --------
|
||||
# --- Fehlende Artefakte in Qdrant ermitteln ---
|
||||
chunks_missing, edges_missing = artifacts_missing(client, cfg.prefix, note_id)
|
||||
|
||||
# --- Edges (robust) ---
|
||||
edges: List[Dict[str, Any]] = []
|
||||
edges_failed = False
|
||||
if changed and (not do_baseline_only):
|
||||
should_build_edges = (changed and (not do_baseline_only)) or edges_missing
|
||||
if should_build_edges:
|
||||
try:
|
||||
note_refs = note_pl.get("references") or []
|
||||
edges = build_edges_for_note(
|
||||
@@ -385,10 +495,9 @@ def main() -> None:
|
||||
except Exception as e:
|
||||
edges_failed = True
|
||||
edges = []
|
||||
# WICHTIG: Wir brechen NICHT mehr ab — Note & Chunks werden geschrieben.
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
|
||||
|
||||
# -------- Summary --------
|
||||
# --- Summary (stdout) ---
|
||||
summary = {
|
||||
"note_id": note_id,
|
||||
"title": fm.get("title"),
|
||||
@@ -396,10 +505,12 @@ def main() -> None:
|
||||
"edges": len(edges),
|
||||
"edges_failed": edges_failed,
|
||||
"changed": changed,
|
||||
"chunks_missing": chunks_missing,
|
||||
"edges_missing": edges_missing,
|
||||
"needs_baseline_for_mode": needs_baseline,
|
||||
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
||||
"apply" if args.apply and changed else
|
||||
"apply-skip-unchanged" if args.apply and not changed else
|
||||
"apply" if args.apply and (changed or chunks_missing or edges_missing) else
|
||||
"apply-skip-unchanged" if args.apply and not (changed or chunks_missing or edges_missing) else
|
||||
"dry-run"),
|
||||
"path": note_pl["path"],
|
||||
"hash_mode": mode,
|
||||
@@ -409,7 +520,7 @@ def main() -> None:
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
|
||||
# -------- Writes --------
|
||||
# --- Writes ---
|
||||
if not args.apply:
|
||||
continue
|
||||
|
||||
@@ -425,21 +536,29 @@ def main() -> None:
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
continue
|
||||
|
||||
if not changed:
|
||||
# Wenn nichts geändert und keine Artefakte fehlen → nichts zu tun
|
||||
if not changed and not (chunks_missing or edges_missing):
|
||||
continue
|
||||
|
||||
if args.purge_before_upsert and has_old:
|
||||
# Purge nur bei echten Änderungen (unverändert + fehlende Artefakte ≠ Purge)
|
||||
if args.purge_before_upsert and has_old and changed:
|
||||
try:
|
||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
||||
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
if chunk_pls:
|
||||
# Note nur bei Änderungen neu schreiben
|
||||
if changed:
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
|
||||
# Chunks schreiben, wenn geändert ODER vorher fehlend
|
||||
if chunk_pls and (changed or chunks_missing):
|
||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||
upsert_batch(client, chunks_name, chunk_pts)
|
||||
if edges:
|
||||
|
||||
# Edges schreiben, wenn vorhanden und (geändert ODER vorher fehlend)
|
||||
if edges and (changed or edges_missing):
|
||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||
upsert_batch(client, edges_name, edge_pts)
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
check_types_registry_edges.py
|
||||
--------------------------------------------------
|
||||
Prüft, ob die in config/types.yaml hinterlegten Default-Kanten
|
||||
pro Note-Typ tatsächlich in der Qdrant *edges*-Collection auftauchen.
|
||||
"""
|
||||
import os, sys, json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Set
|
||||
from dataclasses import dataclass, asdict
|
||||
|
||||
try:
|
||||
import yaml # PyYAML
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"PyYAML not installed: {e}"})); sys.exit(2)
|
||||
|
||||
try:
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as rest # noqa: F401
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"qdrant-client not installed: {e}"})); sys.exit(2)
|
||||
|
||||
@dataclass
|
||||
class Cfg:
|
||||
url: str
|
||||
api_key: Optional[str]
|
||||
prefix: str
|
||||
notes: str
|
||||
edges: str
|
||||
types_file: Path
|
||||
|
||||
def _env_prefix() -> Optional[str]:
|
||||
for k in ("COLLECTION_PREFIX", "MINDNET_PREFIX"):
|
||||
v = os.environ.get(k, "").strip()
|
||||
if v:
|
||||
return v
|
||||
return None
|
||||
|
||||
def _load_types_yaml(path: Path) -> Dict:
|
||||
if not path.exists():
|
||||
print(json.dumps({"warn": f"types.yaml fehlt: {path}. Fallback: keine Vorgaben."}))
|
||||
return {}
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
except Exception as e:
|
||||
print(json.dumps({"warn": f"types.yaml defekt ({path}): {e}. Fallback: keine Vorgaben."}))
|
||||
return {}
|
||||
|
||||
def _cfg_from_env(argv_prefix: Optional[str]) -> Cfg:
|
||||
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
|
||||
api_key = os.environ.get("QDRANT_API_KEY", "").strip() or None
|
||||
prefix = (argv_prefix or _env_prefix() or "mindnet").strip()
|
||||
notes = f"{prefix}_notes"
|
||||
edges = f"{prefix}_edges"
|
||||
types_path = Path(os.environ.get("TYPES_FILE", "config/types.yaml")).resolve()
|
||||
return Cfg(url=url, api_key=api_key, prefix=prefix, notes=notes, edges=edges, types_file=types_path)
|
||||
|
||||
def _mk_client(cfg: Cfg) -> QdrantClient:
|
||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key, timeout=30.0)
|
||||
|
||||
def _scroll_all_notes(client: QdrantClient, notes_col: str) -> Dict[str, Dict]:
|
||||
out = {}
|
||||
offset = None
|
||||
while True:
|
||||
points, offset = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=None, limit=256, offset=offset,
|
||||
with_payload=True, with_vectors=False,
|
||||
)
|
||||
if not points:
|
||||
break
|
||||
for p in points:
|
||||
payload = p.payload or {}
|
||||
nid = payload.get("note_id") or payload.get("id") or payload.get("uid") or payload.get("slug")
|
||||
if not nid:
|
||||
nid = str(p.id)
|
||||
out[str(nid)] = payload
|
||||
return out
|
||||
|
||||
def _scroll_all_edges(client: QdrantClient, edges_col: str) -> List[Tuple[str, dict]]:
|
||||
out = []
|
||||
offset = None
|
||||
while True:
|
||||
points, offset = client.scroll(
|
||||
collection_name=edges_col,
|
||||
scroll_filter=None, limit=256, offset=offset,
|
||||
with_payload=True, with_vectors=False,
|
||||
)
|
||||
if not points:
|
||||
break
|
||||
for p in points:
|
||||
out.append((str(p.id), p.payload or {}))
|
||||
return out
|
||||
|
||||
def _guess_src_note_id(ed_pl: dict) -> Optional[str]:
|
||||
for k in ("src_note_id", "note_id", "src_id", "src"):
|
||||
nid = ed_pl.get(k)
|
||||
if nid:
|
||||
return str(nid)
|
||||
for k in ("src_ref", "from"):
|
||||
nid = ed_pl.get(k)
|
||||
if isinstance(nid, dict):
|
||||
for kk in ("note_id", "id"):
|
||||
if nid.get(kk):
|
||||
return str(nid[kk])
|
||||
elif isinstance(nid, str):
|
||||
return nid
|
||||
return None
|
||||
|
||||
def _edge_type(ed_pl: dict) -> Optional[str]:
|
||||
for k in ("edge_type", "type", "rel", "relation"):
|
||||
if ed_pl.get(k):
|
||||
return str(ed_pl[k])
|
||||
return None
|
||||
|
||||
def _cfg_to_jsonable(cfg: Cfg) -> dict:
|
||||
d = asdict(cfg)
|
||||
d["types_file"] = str(d.get("types_file"))
|
||||
return d
|
||||
|
||||
def main(argv=None):
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (mindnet, etc.)")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
cfg = _cfg_from_env(args.prefix)
|
||||
print(json.dumps({"cfg": _cfg_to_jsonable(cfg)}, ensure_ascii=False))
|
||||
|
||||
tr = _load_types_yaml(cfg.types_file)
|
||||
types_def = (tr.get("types") if isinstance(tr, dict) else {}) or {}
|
||||
print(json.dumps({"types_defined": list(types_def.keys())}, ensure_ascii=False))
|
||||
|
||||
client = _mk_client(cfg)
|
||||
|
||||
notes = _scroll_all_notes(client, cfg.notes)
|
||||
print(json.dumps({"notes_count": len(notes)}, ensure_ascii=False))
|
||||
|
||||
edges = _scroll_all_edges(client, cfg.edges)
|
||||
print(json.dumps({"edges_count": len(edges)}, ensure_ascii=False))
|
||||
|
||||
note_type = {}
|
||||
for nid, pl in notes.items():
|
||||
t = pl.get("type") or "concept"
|
||||
note_type[str(nid)] = str(t)
|
||||
|
||||
seen = {}
|
||||
counts = {}
|
||||
|
||||
for edge_pid, ed_pl in edges:
|
||||
et = _edge_type(ed_pl)
|
||||
if not et:
|
||||
continue
|
||||
src_nid = _guess_src_note_id(ed_pl)
|
||||
if not src_nid:
|
||||
continue
|
||||
t = note_type.get(src_nid)
|
||||
if not t:
|
||||
continue
|
||||
seen.setdefault(t, set()).add(et)
|
||||
counts.setdefault(t, {}).setdefault(et, 0)
|
||||
counts[t][et] += 1
|
||||
|
||||
expected = {}
|
||||
for tname, tdef in types_def.items():
|
||||
eddefs = (tdef or {}).get("edge_defaults") or []
|
||||
expected[tname] = set([str(x) for x in eddefs if x])
|
||||
|
||||
for tname, exp in expected.items():
|
||||
obs = seen.get(tname, set())
|
||||
missing = sorted(list(exp - obs))
|
||||
extra = sorted(list(obs - exp))
|
||||
print(json.dumps({
|
||||
"type": tname,
|
||||
"expected_defaults": sorted(list(exp)),
|
||||
"observed": sorted(list(obs)),
|
||||
"missing": missing,
|
||||
"extra": extra,
|
||||
"counts": counts.get(tname, {}),
|
||||
}, ensure_ascii=False))
|
||||
|
||||
if not expected:
|
||||
print(json.dumps({"warn": "Keine Typ-Defaults in types.yaml gefunden (edge_defaults leer?)."}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,93 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os, json, argparse, random
|
||||
from typing import Any, Dict, List, Tuple, Optional
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
def _get_prefix(explicit: Optional[str]) -> str:
|
||||
return (explicit or os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet").strip()
|
||||
|
||||
def _names(prefix: str) -> Dict[str,str]:
|
||||
return {
|
||||
"notes": f"{prefix}_notes",
|
||||
"chunks": f"{prefix}_chunks",
|
||||
"edges": f"{prefix}_edges",
|
||||
}
|
||||
|
||||
def _scroll(client: QdrantClient, collection: str, *, limit=5, with_payload=True):
|
||||
# Support both modern and older SDKs (filter vs scroll_filter)
|
||||
try:
|
||||
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
|
||||
except TypeError as e:
|
||||
# older signatures might accept scroll_filter kw, but if we passed none it's fine
|
||||
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--host", default=os.environ.get("QDRANT_HOST", "127.0.0.1"))
|
||||
ap.add_argument("--port", default=int(os.environ.get("QDRANT_PORT", "6333")))
|
||||
ap.add_argument("--prefix", default=None)
|
||||
ap.add_argument("--limit", type=int, default=5)
|
||||
ap.add_argument("--note-id", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
prefix = _get_prefix(args.prefix)
|
||||
names = _names(prefix)
|
||||
client = QdrantClient(host=args.host, port=args.port, prefer_grpc=False)
|
||||
|
||||
# Notes sample
|
||||
try:
|
||||
pts, _ = _scroll(client, names["notes"], limit=args.limit, with_payload=True)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"notes scroll failed: {type(e).__name__}: {e}", "collection": names["notes"]}))
|
||||
return
|
||||
|
||||
sample = []
|
||||
for p in pts:
|
||||
pl = (p.payload or {})
|
||||
sample.append({
|
||||
"point_id": str(p.id),
|
||||
"note_id": pl.get("note_id"),
|
||||
"id": pl.get("id"),
|
||||
"title": pl.get("title"),
|
||||
"type": pl.get("type"),
|
||||
"retriever_weight": pl.get("retriever_weight"),
|
||||
"chunk_profile": pl.get("chunk_profile"),
|
||||
})
|
||||
print(json.dumps({"notes_sample": sample}, ensure_ascii=False))
|
||||
|
||||
# Pick a note to fetch chunks
|
||||
target_note_id = args.note_id or None
|
||||
if not target_note_id:
|
||||
for s in sample:
|
||||
if s.get("note_id"):
|
||||
target_note_id = s["note_id"]
|
||||
break
|
||||
|
||||
if not target_note_id:
|
||||
print(json.dumps({"warn": "no note_id available to fetch chunks"}))
|
||||
return
|
||||
|
||||
# Chunks for target note
|
||||
try:
|
||||
# pull a few chunks and filter client-side by note_id
|
||||
pts, _ = _scroll(client, names["chunks"], limit=50, with_payload=True)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"chunks scroll failed: {type(e).__name__}: {e}", "collection": names["chunks"]}))
|
||||
return
|
||||
|
||||
chunks = [{
|
||||
"id": str(p.id),
|
||||
"chunk_id": (p.payload or {}).get("chunk_id"),
|
||||
"note_id": (p.payload or {}).get("note_id"),
|
||||
"type": (p.payload or {}).get("type"),
|
||||
"retriever_weight": (p.payload or {}).get("retriever_weight"),
|
||||
"window_len": len((p.payload or {}).get("window") or ""),
|
||||
"text_len": len((p.payload or {}).get("text") or ""),
|
||||
} for p in pts if (p.payload or {}).get("note_id") == target_note_id][:5]
|
||||
|
||||
print(json.dumps({"target_note_id": target_note_id, "chunk_samples": chunks}, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
dump_edges_payload_sample.py
|
||||
-----------------------------------
|
||||
Zeigt einige Beispiel-Payloads aus der {prefix}_edges Collection,
|
||||
um Feldnamen (edge_type, src_note_id, src_id, etc.) zu verifizieren.
|
||||
|
||||
Nutzung:
|
||||
python3 tests/dump_edges_payload_sample.py --prefix mindnet --n 10
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from qdrant_client import QdrantClient
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": f"qdrant-client not installed: {e}"}))
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet")
|
||||
ap.add_argument("--n", type=int, default=10)
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
|
||||
api_key = os.environ.get("QDRANT_API_KEY") or None
|
||||
edges = f"{args.prefix}_edges"
|
||||
|
||||
client = QdrantClient(url=url, api_key=api_key, timeout=30)
|
||||
|
||||
left = args.n
|
||||
offset = None
|
||||
while left > 0:
|
||||
limit = min(left, 256)
|
||||
pts, offset = client.scroll(
|
||||
collection_name=edges,
|
||||
scroll_filter=None,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
)
|
||||
if not pts:
|
||||
break
|
||||
for p in pts:
|
||||
print(json.dumps({"point_id": str(p.id), "payload_keys": list((p.payload or {}).keys()), "payload": p.payload}, ensure_ascii=False))
|
||||
left -= len(pts)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os, tempfile, textwrap
|
||||
from app.core.type_registry import load_type_registry, resolve_note_type, get_type_config, effective_chunk_profile, profile_overlap
|
||||
|
||||
def test_defaults_when_missing():
|
||||
# ensure default kicks in if file does not exist
|
||||
reg = load_type_registry(os.path.join("config", "does_not_exist.yaml"))
|
||||
assert reg.get("_using_defaults", False) is True
|
||||
t = resolve_note_type(None, reg)
|
||||
assert t == "concept"
|
||||
cfg = get_type_config(t, reg)
|
||||
assert cfg.get("chunk_profile") in ("medium", "long", "short")
|
||||
|
||||
def test_custom_file():
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = os.path.join(td, "types.yaml")
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(textwrap.dedent('''
|
||||
version: 1.0
|
||||
types:
|
||||
concept:
|
||||
chunk_profile: medium
|
||||
edge_defaults: ["references"]
|
||||
retriever_weight: 1.0
|
||||
task:
|
||||
chunk_profile: short
|
||||
edge_defaults: ["depends_on"]
|
||||
retriever_weight: 0.8
|
||||
'''))
|
||||
reg = load_type_registry(p)
|
||||
assert resolve_note_type("task", reg) == "task"
|
||||
assert effective_chunk_profile("task", reg) == "short"
|
||||
assert profile_overlap("short")[0] < profile_overlap("long")[0]
|
||||
Reference in new issue
Block a user