Compare commits

..
Author SHA1 Message Date
Lars 2c81fa0cb5 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-09 10:42:55 +01:00
Lars c3815afcd4 app/core/edges_writer.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-09 10:39:16 +01:00
Lars 8800160564 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-09 10:17:24 +01:00
Lars bdbc4a1bf7 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-09 10:17:04 +01:00
Lars 597090bc45 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-09 10:11:34 +01:00
Lars af36c410b4 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-09 10:11:19 +01:00
Lars 4dcd606c10 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-09 09:51:05 +01:00
Lars bbd5a7fa48 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-09 09:15:48 +01:00
Lars 6dc37ccb66 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 22:06:21 +01:00
Lars b84906283e config/types.yaml aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 4s
2025-11-08 21:57:50 +01:00
Lars af0dcdf76a config/types.yaml aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:55:10 +01:00
Lars f1e1cde597 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:53:49 +01:00
Lars a7c5630e5b app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:48:01 +01:00
Lars 82151d6bc3 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:46:49 +01:00
Lars c93e7ad598 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:40:13 +01:00
Lars 290b271cf6 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:36:53 +01:00
Lars a686bdbeaf Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:31:24 +01:00
Lars 2b84c62875 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:26:42 +01:00
Lars 2de786fc64 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:22:17 +01:00
Lars 2ddf034983 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:21:57 +01:00
Lars b2043f4f84 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:19:18 +01:00
Lars 52eae52061 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 21:11:03 +01:00
Lars e451ea64ae Dateien nach "tests" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 20:52:06 +01:00
Lars eb6e4028ff Dateien nach "tests" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 20:49:30 +01:00
Lars ea45493941 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 18:22:28 +01:00
Lars da70f0e00c Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 18:13:07 +01:00
Lars 8bf3707511 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 18:06:33 +01:00
Lars 804eeef858 Dateien nach "tests" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 18:02:33 +01:00
Lars 36658f12fe Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 18:02:15 +01:00
Lars f0e675ffd8 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 18:02:01 +01:00
Lars 90911c1074 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:59:08 +01:00
Lars 1729e0b40d Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:55:56 +01:00
Lars 70b3169e7f Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:55:39 +01:00
Lars 315a1c0945 Dateien nach "tests" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:55:00 +01:00
Lars b9316c693d Dateien nach "tests" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 4s
2025-11-08 17:46:52 +01:00
Lars 820e52d869 Dateien nach "tests" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 17:43:42 +01:00
Lars 444e8e4206 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:36:31 +01:00
Lars e6f2c8b4e8 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:31:14 +01:00
Lars f6b7652219 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:29:25 +01:00
Lars 3cd00ab5cf Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:26:40 +01:00
Lars 0feae50d90 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:25:48 +01:00
Lars d79c55fc64 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 17:12:03 +01:00
Lars 98a6f5c480 Dateien nach "config" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 17:11:44 +01:00
Lars 0fa4aa7280 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 4s
2025-11-08 17:11:17 +01:00
Lars 3b192e2eed Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 17:00:27 +01:00
Lars 2794d26181 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:59:00 +01:00
Lars 7b56f696d6 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:58:47 +01:00
Lars ead1b8c1bc Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 4s
2025-11-08 16:48:02 +01:00
Lars b8126e00c4 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:44:55 +01:00
Lars 62f8f4b313 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:42:33 +01:00
Lars 21924541dc Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:40:35 +01:00
Lars 197242ca42 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:38:24 +01:00
Lars 7922734dce Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:32:42 +01:00
Lars afd7675425 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:28:27 +01:00
Lars 8b3b343645 Dateien nach "app/core" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 4s
2025-11-08 16:27:47 +01:00
Lars 6713f342c3 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:24:03 +01:00
Lars 79bb8a143c Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:20:57 +01:00
Lars d03a181536 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:18:31 +01:00
Lars 8e5304c156 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 16:16:17 +01:00
Lars c487db6534 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 16:02:18 +01:00
Lars 0ca54f429e Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:56:13 +01:00
Lars 6c62f64784 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:38:21 +01:00
Lars 3282f85007 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:35:00 +01:00
Lars b186569750 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:32:08 +01:00
Lars 7cde2462ec Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:29:32 +01:00
Lars c0b96f7c05 Dateien nach "tests" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:25:45 +01:00
Lars 3a88caac69 Dateien nach "scripts" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:25:18 +01:00
Lars 31457e9240 Dateien nach "docs" hochladen
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:24:38 +01:00
Lars 307335a733 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:22:44 +01:00
Lars 4eb5e34ea7 app/core/derive_edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:21:17 +01:00
Lars f4be219790 app/core/type_registry.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 15:20:53 +01:00
Lars 53591b6f27 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 14:25:31 +01:00
Lars a14d0bb7cb app/core/derive_edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 14:25:01 +01:00
Lars c2802e7cb3 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 14:24:40 +01:00
Lars c7f2815f1d config/types.yaml aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 14:24:22 +01:00
Lars afc90fe992 app/core/type_registry.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 14:23:53 +01:00
Lars 2f9ce824a0 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 12:48:31 +01:00
Lars a39b2a6950 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 11:53:06 +01:00
Lars 70fa95966c app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 11:46:15 +01:00
Lars 10479014d5 app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 11:45:59 +01:00
Lars 148e22aa7d scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 11:37:23 +01:00
Lars a9fe6196f7 app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 11:34:47 +01:00
Lars ae20665a66 app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 11:34:25 +01:00
Lars ccbf078463 app/core/env_vars.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 11:32:47 +01:00
Lars cd1e46784f app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 11:25:03 +01:00
Lars b49d97b8d7 app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 11:20:29 +01:00
Lars 3d44de9d87 app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 11:06:57 +01:00
Lars 987b3c1770 app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 10:17:13 +01:00
Lars 25bc6544c4 app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 09:43:13 +01:00
Lars b241ae753b app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-08 09:42:54 +01:00
Lars 2041771b14 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 4s
2025-11-08 08:31:49 +01:00
Lars b5958a9f63 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 08:00:15 +01:00
Lars c05dbd4b3b app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 07:59:47 +01:00
Lars 44e468fc21 app/core/derive_edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-11-08 07:59:24 +01:00
Lars 5f358d329b app/core/type_registry.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-07 13:28:41 +01:00
Lars 59fe8154f2 config/types.yaml aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 4s
2025-11-07 09:38:52 +01:00
Lars d5b2e9bb8f .gitea/workflows/deploy.yml aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-07 09:37:02 +01:00
Lars 3a3907e1b6 config/types.yaml hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-11-07 09:36:25 +01:00
Lars f66cdc70b2 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 5s
2025-11-07 09:30:20 +01:00
15 changed files with 1383 additions and 792 deletions

No files matched your search

+1 -1
View File
@@ -14,7 +14,7 @@ jobs:
runs-on: linux_host
env:
TARGET_BASE: /home/llmadmin/mindnet
DEPLOY_DIRS: "app scripts schemas docker tests requirements.txt README.md"
DEPLOY_DIRS: "app scripts schemas docker tests config requirements.txt README.md"
steps:
- name: Checkout
+158 -249
View File
@@ -1,271 +1,180 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Modul: app/core/chunk_payload.py
Version: 2.2.0
Datum: 2025-10-06
Zweck
-----
Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.0.1.
Neu: Wenn der Chunker KEIN Overlap im Fenster liefert (== window fehlt / identisch zur Kernpassage),
erzeugen wir FENSTER mit synthetischem Overlap auf Basis chunk_config.get_sizes(note_type)['overlap'].
Felder (beibehalten aus 2.0.1):
- note_id, chunk_id, id (Alias), chunk_index, seq, path
- window (mit Overlap), text (ohne linkes Overlap)
- start, end (Offsets im gesamten Body)
- overlap_left, overlap_right
- token_count?, section_title?, section_path?, type?, title?, tags?
Kompatibilität:
- 'id' == 'chunk_id' als Alias
- Pfade bleiben relativ (keine führenden '/'), Backslashes → Slashes
- Robust für Chunk-Objekte oder Dicts; Fensterquelle: 'window'|'text'|'content'|'raw'
Lizenz: MIT (projektintern)
"""
# chunk_payload.py
from __future__ import annotations
from typing import Any, Dict, List, Optional, Tuple
import os, json, pathlib, re, yaml, hashlib
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
FRONTMATTER_RE = re.compile(r"^---\s*\n.*?\n---\s*\n?", re.DOTALL)
try:
# Typgerechtes Overlap aus deiner Konfiguration holen
from app.core.chunk_config import get_sizes as _get_sizes
except Exception:
def _get_sizes(_note_type: str):
# konservativer Default, falls Import fehlschlägt
return {"overlap": (40, 60), "target": (250, 350), "max": 500}
def _as_dict(note: Any) -> Dict[str, Any]:
if isinstance(note, dict):
return note
d: Dict[str, Any] = {}
for attr in ("id","note_id","title","path","frontmatter","meta","metadata","body","text","content","raw","markdown","type","chunks"):
if hasattr(note, attr):
d[attr] = getattr(note, attr)
fm = d.get("frontmatter") or d.get("meta") or d.get("metadata") or {}
d["frontmatter"] = fm if isinstance(fm, dict) else {}
return d
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[Dict[str,Any]]]:
path = kwargs.get("path")
types_cfg = kwargs.get("types_config")
for a in args:
if path is None and isinstance(a, (str, pathlib.Path)):
path = str(a)
if types_cfg is None and isinstance(a, dict):
types_cfg = a
return path, types_cfg
# ------------------------------- Utils ------------------------------- #
def _load_types_config(explicit: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
if isinstance(explicit, dict):
return explicit
for rel in ("config/config.yaml", "config/types.yaml"):
p = pathlib.Path(rel)
if p.exists():
with p.open("r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
return data["types"]
return data if isinstance(data, dict) else {}
return {}
def _get_attr_or_key(obj: Any, key: str, default=None):
if obj is None:
return default
if isinstance(obj, dict):
return obj.get(key, default)
return getattr(obj, key, default)
def _as_window_text(chunk: Any) -> str:
"""Fenstertext robust lesen (bevorzugt echte Fenster, sonst Kern)."""
for k in ("window", "text", "content", "raw"):
v = _get_attr_or_key(chunk, k, None)
if isinstance(v, str) and v:
def _coalesce(*vals):
for v in vals:
if v is not None:
return v
return None
def _text_from_note(n: Dict[str, Any], path_hint: Optional[str]) -> str:
# häufige Felder
cand = [
n.get("body"),
n.get("text"),
n.get("markdown"),
n.get("raw"),
]
content = n.get("content")
if isinstance(content, str):
cand.append(content)
elif isinstance(content, dict):
for k in ("text","body","raw","markdown","content"):
v = content.get(k)
if isinstance(v, str):
cand.append(v)
for t in cand:
if isinstance(t, str) and t.strip():
return t
# Fallback: Datei lesen und Frontmatter entfernen
p = n.get("path") or path_hint
if p:
try:
pth = pathlib.Path(p)
if pth.exists():
txt = pth.read_text(encoding="utf-8", errors="ignore")
if txt:
return FRONTMATTER_RE.sub("", txt).strip()
except Exception:
pass
return ""
def _to_int(x: Any, default: int = 0) -> int:
def _iter_chunks(n: Dict[str, Any], profile: str, fulltext: str) -> List[Dict[str, Any]]:
# 1) vorhandene Chunks nehmen, wenn sinnvoll
existing = n.get("chunks")
out: List[Dict[str, Any]] = []
if isinstance(existing, list) and existing:
for i, c in enumerate(existing):
text = ""
if isinstance(c, dict):
text = c.get("text") or c.get("body") or c.get("raw") or ""
elif isinstance(c, str):
text = c
if text and text.strip():
out.append({"index": i, "text": text})
if out:
return out
# 2) Fallback: profilabhängige Bündelung
if not isinstance(profile, str):
profile = "medium"
size = {"short": 600, "medium": 1200, "long": 2400}.get(profile, 1200)
if not fulltext:
return []
paras = re.split(r"\n{2,}", fulltext)
buf = ""
chunks: List[str] = []
for p in paras:
p = (p or "").strip()
if not p:
continue
if len(buf) + (2 if buf else 0) + len(p) <= size:
buf = (buf + "\n\n" + p).strip() if buf else p
else:
if buf: chunks.append(buf)
if len(p) <= size:
buf = p
else:
for i in range(0, len(p), size):
chunks.append(p[i:i+size])
buf = ""
if buf: chunks.append(buf)
return [{"index": i, "text": c} for i, c in enumerate(chunks)]
def make_chunk_payloads(note: Any, *args, **kwargs) -> List[Dict[str, Any]]:
n = _as_dict(note)
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
types_cfg = _load_types_config(types_cfg_explicit)
fm = n.get("frontmatter") or {}
note_type = str(fm.get("type") or n.get("type") or "note")
cfg_for_type = types_cfg.get(note_type, {}) if isinstance(types_cfg, dict) else {}
try:
return int(x)
default_rw = float(os.environ.get("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0))
except Exception:
return default
default_rw = 1.0
def _normalize_rel_path(p: str) -> str:
p = (p or "").replace("\\", "/")
while p.startswith("/"):
p = p[1:]
return p
retriever_weight = _coalesce(fm.get("retriever_weight"), cfg_for_type.get("retriever_weight"), default_rw)
try:
retriever_weight = float(retriever_weight)
except Exception:
retriever_weight = default_rw
chunk_profile = _coalesce(fm.get("chunk_profile"), cfg_for_type.get("chunk_profile"), os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE","medium"))
chunk_profile = chunk_profile if isinstance(chunk_profile, str) else "medium"
# ---------------------- Overlap & Offsets ---------------------------- #
note_id = n.get("note_id") or n.get("id") or fm.get("id")
title = n.get("title") or fm.get("title") or ""
path = n.get("path") or path_arg
if isinstance(path, pathlib.Path):
path = str(path)
path = path or "" # immer vorhanden
def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int], str]:
"""
Entfernt linkes Overlap aus echten Fenster-Strings.
Rückgabe: (segments, overlaps_left, reconstructed_text)
"""
segments: List[str] = []
overlaps_left: List[int] = []
reconstructed = ""
for w in windows:
w = w or ""
max_k = min(len(w), len(reconstructed))
k = 0
for cand in range(max_k, -1, -1):
if reconstructed.endswith(w[:cand]):
k = cand
break
seg = w[k:]
segments.append(seg)
overlaps_left.append(k)
reconstructed += seg
return segments, overlaps_left, reconstructed
fulltext = _text_from_note(n, path)
chunks = _iter_chunks(n, chunk_profile, fulltext)
def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
"""Länge längsten Suffix(a), der Prefix(b) ist."""
if not a or not b:
return 0
a1 = a[-max_probe:]
b1 = b[:max_probe]
n = min(len(a1), len(b1))
for k in range(n, 0, -1):
if a1[-k:] == b1[:k]:
return k
return 0
# ----------------------------- Public API ---------------------------- #
def make_chunk_payloads(
frontmatter: Dict[str, Any],
rel_path: str,
chunks: Iterable[Union[Dict[str, Any], Any]],
note_text: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""
Baut Payloads pro Chunk. Falls Fenster ohne Overlap geliefert werden,
erzeugen wir synthetische 'window'-Texte mit typgerechtem Overlap.
"""
note_id = str(frontmatter.get("id") or "").strip()
note_type = str(frontmatter.get("type", "")).lower()
note_title = frontmatter.get("title", None)
note_tags = frontmatter.get("tags", None)
rel_path = _normalize_rel_path(rel_path)
# 1) Rohdaten sammeln (so wie geliefert)
chunks_list = list(chunks)
raw_windows: List[str] = []
seqs: List[int] = []
ids_in: List[Optional[str]] = []
token_counts: List[Optional[int]] = []
section_titles: List[Optional[str]] = []
section_paths: List[Optional[str]] = []
any_explicit_window = False
for idx, c in enumerate(chunks_list):
# Fensterquelle
w = _get_attr_or_key(c, "window", None)
if isinstance(w, str) and w:
any_explicit_window = True
raw_windows.append(w)
else:
raw_windows.append(_as_window_text(c)) # 'text'|'content'|'raw' als Ersatz
# Ordnung
seqs.append(_to_int(_get_attr_or_key(c, "seq", _get_attr_or_key(c, "chunk_index", idx)), idx))
# IDs, Tokens, Sektionen
cid = _get_attr_or_key(c, "chunk_id", _get_attr_or_key(c, "id", None))
ids_in.append(str(cid) if isinstance(cid, str) and cid else None)
tc = _get_attr_or_key(c, "token_count", None)
token_counts.append(_to_int(tc, 0) if tc is not None else None)
section_titles.append(_get_attr_or_key(c, "section_title", None))
section_paths.append(_get_attr_or_key(c, "section_path", None))
# 2) Segmente & Overlaps bestimmen
if any_explicit_window:
# Es existieren echte Fenster → dedupe, um Kernsegmente zu finden
segments, overlaps_left, recon = _dedupe_windows_to_segments(raw_windows)
windows_final = raw_windows[:] # bereits mit Overlap geliefert
else:
# Keine echten Fenster → Segmente sind identisch zu "Fenstern" (bisher),
# wir erzeugen synthetische Fenster mit Overlap gemäß Typ
segments = [w or "" for w in raw_windows]
overlaps_left = []
windows_final = []
recon = ""
try:
overlap_low, overlap_high = tuple(_get_sizes(note_type).get("overlap", (40, 60)))
except Exception:
overlap_low, overlap_high = (40, 60)
overlap_target = int(overlap_low)
for i, seg in enumerate(segments):
if i == 0:
# erstes Fenster: kein linker Kontext
windows_final.append(seg)
overlaps_left.append(0)
recon += seg
else:
# synthetischer linker Kontext = Suffix des bisher rekonstruierten Texts
k = min(overlap_target, len(recon))
left_ctx = recon[-k:] if k > 0 else ""
windows_final.append(left_ctx + seg)
overlaps_left.append(k)
recon += seg # Rekonstruktion bleibt kerntreu
# 3) overlap_right bestimmen
overlaps_right: List[int] = []
for i in range(len(windows_final)):
if i + 1 < len(windows_final):
ov = _overlap_len_suffix_prefix(windows_final[i], windows_final[i + 1], max_probe=4096)
else:
ov = 0
overlaps_right.append(ov)
# 4) start/end-Offsets (exakt via note_text, sonst kumulativ)
starts: List[int] = [0] * len(segments)
ends: List[int] = [0] * len(segments)
pos = 0
if isinstance(note_text, str) and note_text:
search_pos = 0
for i, seg in enumerate(segments):
if not seg:
starts[i] = ends[i] = search_pos
continue
j = note_text.find(seg, search_pos)
if j >= 0:
starts[i] = j
ends[i] = j + len(seg)
search_pos = ends[i]
else:
# Fallback: kumulativ
starts[i] = pos
pos += len(seg)
ends[i] = pos
else:
for i, seg in enumerate(segments):
starts[i] = pos
pos += len(seg)
ends[i] = pos
# 5) Payload-Dicts
payloads: List[Dict[str, Any]] = []
for i, (win, seg) in enumerate(zip(windows_final, segments)):
chunk_id = ids_in[i] or f"{note_id}#{i+1}"
pl: Dict[str, Any] = {
for c in chunks:
idx = c.get("index", len(payloads))
text = c.get("text") if isinstance(c, dict) else (str(c) if c is not None else "")
text = text if isinstance(text, str) else str(text or "")
key = f"{note_id}|{idx}"
h = hashlib.sha1(key.encode("utf-8")).hexdigest()[:12] if note_id else hashlib.sha1(f"{path}|{idx}".encode("utf-8")).hexdigest()[:12]
chunk_id = f"{note_id}-{idx:03d}-{h}" if note_id else f"{h}"
payload = {
"note_id": note_id,
"chunk_id": chunk_id,
"id": chunk_id, # Alias
"chunk_index": i,
"seq": seqs[i],
"path": rel_path,
"window": win,
"text": seg,
"start": starts[i],
"end": ends[i],
"overlap_left": overlaps_left[i],
"overlap_right": overlaps_right[i],
"index": idx,
"title": title,
"type": note_type,
"path": path, # <- garantiert vorhanden
"text": text, # <- nie leer, sonst werden keine Chunks erzeugt
"retriever_weight": retriever_weight,
"chunk_profile": chunk_profile,
}
# optionale Metadaten
if note_type:
pl["type"] = note_type
if note_title is not None:
pl["title"] = note_title
if note_tags is not None:
pl["tags"] = note_tags
if token_counts[i] is not None:
pl["token_count"] = int(token_counts[i])
if section_titles[i] is not None:
pl["section_title"] = section_titles[i]
if section_paths[i] is not None:
sp = str(section_paths[i]).replace("\\", "/")
pl["section_path"] = sp if sp else "/"
payloads.append(pl)
json.loads(json.dumps(payload, ensure_ascii=False))
payloads.append(payload)
return payloads
# __main__ Demo (optional)
if __name__ == "__main__": # pragma: no cover
fm = {"id": "demo", "title": "Demo", "type": "concept"}
# Beispiel ohne echte Fenster → erzeugt synthetische Overlaps
chunks = [
{"id": "demo#1", "text": "Alpha Beta Gamma"},
{"id": "demo#2", "text": "Gamma Delta"},
{"id": "demo#3", "text": "Delta Epsilon Zeta"},
]
pls = make_chunk_payloads(fm, "path/demo.md", chunks, note_text="Alpha Beta Gamma Delta Epsilon Zeta")
from pprint import pprint
pprint(pls)
recon = "".join(p["text"] for p in pls)
print("RECON:", recon)
+94
View File
@@ -0,0 +1,94 @@
# app/core/edges_writer.py
from __future__ import annotations
import hashlib
from typing import Dict, List, Iterable, Tuple
try:
# Dein Modul mit der Schemadefinition und der Builder-Funktion
from app.core.edges import build_edges_for_note # noqa: F401
except Exception as e:
raise RuntimeError("Konnte app.core.edges nicht importieren. "
"Bitte sicherstellen, dass app/core/edges.py vorhanden ist.") from e
def _edge_uid(kind: str, source_id: str, target_id: str, scope: str) -> str:
"""
Deterministische, kurze ID für eine Edge.
Kollisionen sind praktisch ausgeschlossen (BLAKE2s über den Kanonischen Schlüssel).
"""
key = f"{kind}|{source_id}|{target_id}|{scope}"
return hashlib.blake2s(key.encode("utf-8"), digest_size=12).hexdigest()
def ensure_edges_collection(qdrant_client, collection: str) -> None:
"""
Legt die Edge-Collection an, falls sie nicht existiert.
Minimal: 1D-Vector (Dummy), Cosine. Payload-only-Collections sind je nach Qdrant-Version heikel.
"""
from qdrant_client.http import models as qm
existing = [c.name for c in qdrant_client.get_collections().collections]
if collection in existing:
return
qdrant_client.recreate_collection(
collection_name=collection,
vectors_config=qm.VectorParams(size=1, distance=qm.Distance.COSINE),
on_disk_payload=True,
)
def edges_from_note(
note_id: str,
chunk_payloads: List[Dict],
note_level_refs: Iterable[str] | None,
*,
include_note_scope_refs: bool = False,
) -> List[Dict]:
"""
Ruft deinen Edge-Builder auf und gibt die (deduplizierten) Edge-Payloads zurück.
Keine Schemaänderung – exakt das aus app/core/edges.py.
"""
return build_edges_for_note(
note_id=note_id,
chunk_payloads=chunk_payloads,
note_level_refs=list(note_level_refs or []),
include_note_scope_refs=include_note_scope_refs,
)
def upsert_edges(
qdrant_client,
collection: str,
edge_payloads: List[Dict],
) -> Tuple[int, int]:
"""
Schreibt Edges als Points in Qdrant.
- id: deterministisch aus (kind, source_id, target_id, scope)
- vector: [0.0] Dummy
- payload: Edge-Dict (unverändert, siehe Schema in app/core/edges.py)
Gibt (anzahl_points, anzahl_unique_keys) zurück.
"""
from qdrant_client.models import PointStruct
if not edge_payloads:
return 0, 0
points = []
seen = set()
for e in edge_payloads:
key = (e.get("kind"), e.get("source_id"), e.get("target_id"), e.get("scope"))
if key in seen:
continue
seen.add(key)
eid = _edge_uid(*key)
points.append(
PointStruct(
id=eid,
vector=[0.0],
payload=e,
)
)
if not points:
return 0, 0
ensure_edges_collection(qdrant_client, collection)
qdrant_client.upsert(collection_name=collection, points=points)
return len(points), len(seen)
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Datei: app/core/env_vars.py
Version: 1.1.0 (2025-11-08)
Zweck
Einheitliche Auflösung von ENV-Variablen (Prefix, Qdrant, Embeddings, Hashing)
mit Abwärtskompatibilität.
Grundsatz
- Für Qdrant-Funktionen ist 'COLLECTION_PREFIX' der Primärschlüssel.
- 'MINDNET_PREFIX' bleibt für App-/UI-/Exporter-Kontexte nutzbar.
- Fallbacks sorgen dafür, dass ältere Umgebungen weiter funktionieren.
Wichtig
- Lädt optional eine .env (wenn python-dotenv verfügbar ist).
- Überschreibt keine bereits gesetzten OS-Variablen (override=False).
"""
from __future__ import annotations
import os
from typing import Optional, Dict
# Optional: .env automatisch laden (ohne Hard-Fail, falls nicht vorhanden)
try:
from dotenv import load_dotenv, find_dotenv # type: ignore
_p = find_dotenv()
if _p:
load_dotenv(_p, override=False)
except Exception:
pass
# -------- Prefix-Auflösung --------
def get_collection_prefix(cli_override: Optional[str] = None) -> str:
"""
Für Qdrant-relevante Funktionen:
1) CLI-Override (--prefix)
2) ENV COLLECTION_PREFIX
3) ENV MINDNET_PREFIX (Fallback)
4) 'mindnet' (Default)
"""
if cli_override and str(cli_override).strip():
return str(cli_override).strip()
return (
os.getenv("COLLECTION_PREFIX")
or os.getenv("MINDNET_PREFIX")
or "mindnet"
)
def get_mindnet_prefix(cli_override: Optional[str] = None) -> str:
"""
Für App-/UI-/Exporter-Kontexte:
1) CLI-Override (--prefix)
2) ENV MINDNET_PREFIX
3) ENV COLLECTION_PREFIX (Fallback)
4) 'mindnet'
"""
if cli_override and str(cli_override).strip():
return str(cli_override).strip()
return (
os.getenv("MINDNET_PREFIX")
or os.getenv("COLLECTION_PREFIX")
or "mindnet"
)
def get_prefix(cli_override: Optional[str] = None, target: str = "qdrant") -> str:
"""
Universelle Hülle (abwärtskompatibel):
target='qdrant' -> get_collection_prefix
target='app' -> get_mindnet_prefix
"""
if target.lower() == "app":
return get_mindnet_prefix(cli_override)
return get_collection_prefix(cli_override)
# -------- Qdrant / Embeddings / Hashing --------
def get_qdrant_url(default: str = "http://127.0.0.1:6333") -> str:
return os.getenv("QDRANT_URL", default)
def get_qdrant_api_key(default: str = "") -> str:
return os.getenv("QDRANT_API_KEY", default)
def get_vector_dim(default: int = 384) -> int:
try:
return int(os.getenv("VECTOR_DIM", str(default)))
except Exception:
return default
def get_embed_url(default: Optional[str] = None) -> Optional[str]:
return os.getenv("EMBED_URL", default)
def get_hash_env() -> Dict[str, str]:
"""
Liefert die Hash-Konfiguration (nur Aggregation; die Auswertung bleibt in den Skripten).
"""
return {
"MINDNET_HASH_COMPARE": os.getenv("MINDNET_HASH_COMPARE", ""),
"MINDNET_HASH_SOURCE": os.getenv("MINDNET_HASH_SOURCE", ""),
"MINDNET_HASH_NORMALIZE": os.getenv("MINDNET_HASH_NORMALIZE", ""),
}
+81 -206
View File
@@ -1,229 +1,104 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Modul: app/core/note_payload.py
# Version: 1.7.0
# Datum: 2025-09-09
# note_payload.py
from __future__ import annotations
import argparse
import hashlib
import json
import os
from typing import Any, Dict, Optional, Tuple
import os, json, pathlib, yaml
try:
from app.core.parser import read_markdown, extract_wikilinks, FRONTMATTER_RE
except Exception: # pragma: no cover
from .parser import read_markdown, extract_wikilinks, FRONTMATTER_RE # type: ignore
def _as_dict(note: Any) -> Dict[str, Any]:
if isinstance(note, dict):
return note
d: Dict[str, Any] = {}
for attr in ("id","note_id","title","path","frontmatter","meta","metadata","type","created","modified","date","tags"):
if hasattr(note, attr):
d[attr] = getattr(note, attr)
# Normalisiere Frontmatter
fm = d.get("frontmatter") or d.get("meta") or d.get("metadata") or {}
d["frontmatter"] = fm if isinstance(fm, dict) else {}
return d
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[Dict[str,Any]]]:
path = kwargs.get("path")
types_cfg = kwargs.get("types_config")
# legacy positional: (path, types_config) oder (types_config, ...)
for a in args:
if path is None and isinstance(a, (str, pathlib.Path)):
path = str(a)
if types_cfg is None and isinstance(a, dict):
types_cfg = a
return path, types_cfg
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def _load_types_config(explicit: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
if isinstance(explicit, dict):
return explicit
for rel in ("config/config.yaml", "config/types.yaml"):
p = pathlib.Path(rel)
if p.exists():
with p.open("r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
return data["types"]
return data if isinstance(data, dict) else {}
return {}
def _normalize_body(body: str, mode: str) -> str:
if mode == "none":
return body if body is not None else ""
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
text = "\n".join(line.rstrip() for line in text.split("\n"))
return text
def _coalesce(*vals):
for v in vals:
if v is not None:
return v
return None
def _resolve_hash_mode(explicit: Optional[str]) -> str:
if explicit:
val = explicit.strip().lower()
else:
val = (os.environ.get("MINDNET_HASH_MODE")
or os.environ.get("MINDNET_HASH_COMPARE")
or "body").strip().lower()
if val in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
return "full"
if val in ("frontmatter", "fm"):
return "frontmatter"
return "body"
def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, Any]]:
if not file_path or not os.path.exists(file_path):
return "", {}
def _env_float(name: str, default: float) -> float:
try:
with open(file_path, "r", encoding="utf-8") as f:
raw = f.read()
return float(os.environ.get(name, default))
except Exception:
return "", {}
m = FRONTMATTER_RE.match(raw)
fm = {}
if m:
fm_txt = m.group(1)
try:
import yaml # lazy
fm = yaml.safe_load(fm_txt) or {}
except Exception:
fm = {}
body = raw[m.end():]
else:
body = raw
return body, fm
return default
def _sha256(s: str) -> str:
h = hashlib.sha256()
h.update(s.encode("utf-8"))
return h.hexdigest()
def _ensure_list(x) -> list:
if x is None: return []
if isinstance(x, list): return [str(i) for i in x]
if isinstance(x, (set, tuple)): return [str(i) for i in x]
return [str(x)]
def _hash_for(mode: str, *, body: str, fm: Dict[str, Any], normalize: str) -> str:
body_n = _normalize_body(body or "", normalize)
fm_s = _canon_frontmatter(fm or {})
if mode == "frontmatter":
return _sha256(fm_s)
if mode == "full":
return _sha256(body_n + "\n--FM--\n" + fm_s)
# default: body
return _sha256(body_n)
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
n = _as_dict(note)
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
types_cfg = _load_types_config(types_cfg_explicit)
# ---------------------------------------------------------------------------
# Kernfunktion
# ---------------------------------------------------------------------------
fm = n.get("frontmatter") or {}
note_type = str(fm.get("type") or n.get("type") or "note")
cfg_for_type = types_cfg.get(note_type, {}) if isinstance(types_cfg, dict) else {}
def make_note_payload(
parsed: Any,
vault_root: Optional[str] = None,
*,
hash_mode: Optional[str] = None,
hash_normalize: Optional[str] = None,
hash_source: Optional[str] = None,
file_path: Optional[str] = None,
) -> Dict[str, Any]:
"""
Liefert den Note-Payload inkl. Mehrfach-Hashes.
- Es werden IMMER die drei Hashes für (body|frontmatter|full) unter
'parsed:canonical' erzeugt (Schlüssel: z. B. 'body:parsed:canonical').
- Zusätzlich werden – falls die aktuelle Konfig (source/normalize) davon
abweicht – die drei Hashes unter den entsprechenden Schlüsseln erzeugt,
z. B. 'frontmatter:raw:none'.
- 'hash_fulltext' und 'hash_signature' repräsentieren den *aktuellen* Modus.
"""
# dict oder Objekt akzeptieren
if isinstance(parsed, dict):
fm = parsed.get("frontmatter") or {}
body_parsed = parsed.get("body") or ""
path = parsed.get("path") or ""
else:
fm = getattr(parsed, "frontmatter", {}) or {}
body_parsed = getattr(parsed, "body", "") or ""
path = getattr(parsed, "path", "") or ""
# Zielpfad relativieren
rel_path = path
default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
retriever_weight = _coalesce(fm.get("retriever_weight"), cfg_for_type.get("retriever_weight"), default_rw)
try:
if vault_root:
rel = os.path.relpath(path, vault_root)
rel = rel.replace("\\", "/").lstrip("/")
rel_path = rel
retriever_weight = float(retriever_weight)
except Exception:
pass
retriever_weight = default_rw
# Konfiguration auflösen
mode_resolved = _resolve_hash_mode(hash_mode) # body|frontmatter|full
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower() # parsed|raw
norm = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower() # canonical|none
chunk_profile = _coalesce(fm.get("chunk_profile"), cfg_for_type.get("chunk_profile"), os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE","medium"))
chunk_profile = chunk_profile if isinstance(chunk_profile, str) else "medium"
# Body-Quelle laden
raw_body, raw_fm = ("", {})
if src == "raw":
raw_body, raw_fm = _read_raw_body_from_file(file_path or path)
if isinstance(raw_fm, dict) and raw_fm:
merged_fm = dict(fm)
for k, v in raw_fm.items():
merged_fm.setdefault(k, v)
fm = merged_fm
body_for_hash = raw_body
else:
body_for_hash = body_parsed
edge_defaults = _ensure_list(_coalesce(fm.get("edge_defaults"), cfg_for_type.get("edge_defaults"), []))
# --- 1) Standard-Tripel (parsed:canonical) immer erzeugen ---
std_src = "parsed"
std_norm = "canonical"
std_hashes: Dict[str, str] = {}
for m in ("body", "frontmatter", "full"):
std_hashes[f"{m}:{std_src}:{std_norm}"] = _hash_for(
m, body=body_parsed, fm=fm, normalize=std_norm
)
note_id = n.get("note_id") or n.get("id") or fm.get("id")
title = n.get("title") or fm.get("title") or ""
path = n.get("path") or path_arg
if isinstance(path, pathlib.Path):
path = str(path)
# Convenience-Felder (für Tools)
hash_body = std_hashes["body:parsed:canonical"]
hash_frontmatter = std_hashes["frontmatter:parsed:canonical"]
hash_full = std_hashes["full:parsed:canonical"]
# --- 2) Hashes für die *aktuelle* Konfiguration (falls abweichend) ---
cur_hashes: Dict[str, str] = {}
if not (src == std_src and norm == std_norm):
for m in ("body", "frontmatter", "full"):
cur_hashes[f"{m}:{src}:{norm}"] = _hash_for(
m, body=body_for_hash, fm=fm, normalize=norm
)
# --- 3) Aktueller Modus für Backwards-Compat Felder ---
current_hash = _hash_for(mode_resolved, body=body_for_hash, fm=fm, normalize=norm)
hash_signature = f"{mode_resolved}:{src}:{norm}:{current_hash}"
# Wikilinks (Note-Ebene)
refs = list(dict.fromkeys(extract_wikilinks(body_parsed))) if body_parsed else []
payload: Dict[str, Any] = {
"note_id": fm.get("id") or fm.get("note_id"),
"title": fm.get("title"),
"type": fm.get("type"),
"status": fm.get("status"),
"created": fm.get("created"),
"updated": fm.get("updated"),
"path": rel_path or fm.get("path"),
"tags": fm.get("tags"),
# Volltext für verlustfreien Export
"fulltext": body_parsed,
# Backwards-Compat:
"hash_fulltext": current_hash,
"hash_signature": hash_signature,
# Option C: Mehrfach-Hashes
"hashes": {**std_hashes, **cur_hashes},
"hash_body": hash_body,
"hash_frontmatter": hash_frontmatter,
"hash_full": hash_full,
# Fallback-Refs
"references": refs,
payload = {
"note_id": note_id,
"title": title,
"type": note_type,
"path": path or "", # immer vorhanden
"retriever_weight": retriever_weight,
"chunk_profile": chunk_profile,
"edge_defaults": edge_defaults,
}
for k in ("area", "project", "source", "lang", "slug", "aliases"):
if k in fm:
payload[k] = fm[k]
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
if tags: payload["tags"] = _ensure_list(tags)
for k in ("created","modified","date"):
v = fm.get(k) or n.get(k)
if v: payload[k] = str(v)
json.loads(json.dumps(payload, ensure_ascii=False))
return payload
# ---------------------------------------------------------------------------
# CLI – Sichtprüfung
# ---------------------------------------------------------------------------
def _cli() -> None:
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
ap.add_argument("--from-file", dest="src", required=True)
ap.add_argument("--vault-root", dest="vault_root", default=None)
ap.add_argument("--print", dest="do_print", action="store_true")
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None)
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
args = ap.parse_args()
parsed = read_markdown(args.src)
payload = make_note_payload(
parsed,
vault_root=args.vault_root,
hash_mode=args.hash_mode,
hash_normalize=args.hash_normalize,
hash_source=args.hash_source,
file_path=args.src,
)
if args.do_print:
print(json.dumps(payload, ensure_ascii=False, indent=2))
if __name__ == "__main__": # pragma: no cover
_cli()
+118 -232
View File
@@ -1,55 +1,38 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
app/core/qdrant_points.py
app/core/qdrant_points.py - robust points helpers for Qdrant
Zweck
- Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
- Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
- alt: {'edge_type','src_id','dst_id', ...}
- neu: {'kind','source_id','target_id', ...}
- Single source of truth for building PointStruct for notes/chunks/edges
- Backward-compatible payloads for edges
- Handles both Single-Vector and Named-Vector collections
- Deterministic overrides via ENV to avoid auto-detection traps:
* NOTES_VECTOR_NAME, CHUNKS_VECTOR_NAME, EDGES_VECTOR_NAME
* MINDNET_VECTOR_NAME (fallback)
> Set to a concrete name (e.g. "text") to force Named-Vector with that name
> Set to "__single__" (or "single") to force Single-Vector
Version
- 1.3 (2025-09-08)
Änderungen (ggü. 1.2)
- points_for_edges() akzeptiert jetzt beide Edge-Schemata.
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
stabile 'edge_id' zurück in die Payload.
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
Aufruf / Verwendung
- Wird von Import-/Backfill-Skripten via:
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
eingebunden. Keine CLI.
Hinweise
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
Version: 1.5.0 (2025-11-08)
"""
from __future__ import annotations
import os
import uuid
from typing import List, Tuple
from qdrant_client.http import models as rest
from typing import List, Tuple, Iterable, Optional, Dict, Any
from qdrant_client.http import models as rest
from qdrant_client import QdrantClient
# --------------------- ID helpers ---------------------
def _to_uuid(stable_key: str) -> str:
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
# --------------------- Points builders ---------------------
def _to_uuid(stable_key: str) -> str:
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
def points_for_note(
prefix: str,
note_payload: dict,
note_vec: List[float] | None,
dim: int,
) -> Tuple[str, List[rest.PointStruct]]:
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
notes_col, _, _ = _names(prefix)
vector = note_vec if note_vec is not None else [0.0] * int(dim)
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
@@ -57,18 +40,7 @@ def points_for_note(
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
return notes_col, [pt]
def points_for_chunks(
prefix: str,
chunk_payloads: List[dict],
vectors: List[List[float]],
) -> Tuple[str, List[rest.PointStruct]]:
"""
Chunks-Collection: erwartet pro Chunk einen Vektor.
Robustheit:
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
"""
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
_, chunks_col, _ = _names(prefix)
points: List[rest.PointStruct] = []
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
@@ -81,22 +53,12 @@ def points_for_chunks(
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
return chunks_col, points
def _normalize_edge_payload(pl: dict) -> dict:
"""
Sorgt für kompatible Feldnamen.
akzeptiert:
- neu: kind, source_id, target_id, seq?
- alt: edge_type, src_id, dst_id, order?/index?
schreibt zurück: kind, source_id, target_id, seq?
"""
# bereits neu?
kind = pl.get("kind") or pl.get("edge_type") or "edge"
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
seq = pl.get("seq") or pl.get("order") or pl.get("index")
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
pl.setdefault("kind", kind)
pl.setdefault("source_id", source_id)
pl.setdefault("target_id", target_id)
@@ -104,18 +66,11 @@ def _normalize_edge_payload(pl: dict) -> dict:
pl["seq"] = seq
return pl
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
"""
Edges-Collection mit 1D-Dummy-Vektor.
- Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
- Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
"""
_, _, edges_col = _names(prefix)
points: List[rest.PointStruct] = []
for raw in edge_payloads:
pl = _normalize_edge_payload(raw)
edge_id = pl.get("edge_id")
if not edge_id:
kind = pl.get("kind", "edge")
@@ -124,59 +79,126 @@ def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[
seq = pl.get("seq") or ""
edge_id = f"{kind}:{s}->{t}#{seq}"
pl["edge_id"] = edge_id
point_id = _to_uuid(edge_id)
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
return edges_col, points
# --------------------- Vector schema & overrides ---------------------
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
def _preferred_name(candidates: List[str]) -> str:
for k in ("text", "default", "embedding", "content"):
if k in candidates:
return k
return sorted(candidates)[0]
def _env_override_for_collection(collection: str) -> Optional[str]:
"""
Returns:
- "__single__" to force single-vector
- concrete name (str) to force named-vector with that name
- None to auto-detect
"""
base = os.getenv("MINDNET_VECTOR_NAME")
if collection.endswith("_notes"):
base = os.getenv("NOTES_VECTOR_NAME", base)
elif collection.endswith("_chunks"):
base = os.getenv("CHUNKS_VECTOR_NAME", base)
elif collection.endswith("_edges"):
base = os.getenv("EDGES_VECTOR_NAME", base)
if not base:
return None
val = base.strip()
if val.lower() in ("__single__", "single"):
return "__single__"
return val # concrete name
def _get_vector_schema(client: QdrantClient, collection_name: str) -> dict:
"""
Return {"kind": "single", "size": int} or {"kind": "named", "names": [...], "primary": str}.
"""
try:
info = client.get_collection(collection_name=collection_name)
vecs = getattr(info, "vectors", None)
# Single-vector config
if hasattr(vecs, "size") and isinstance(vecs.size, int):
return {"kind": "single", "size": vecs.size}
# Named-vectors config (dict-like in .config)
cfg = getattr(vecs, "config", None)
if isinstance(cfg, dict) and cfg:
names = list(cfg.keys())
if names:
return {"kind": "named", "names": names, "primary": _preferred_name(names)}
except Exception:
pass
return {"kind": "single", "size": None}
def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruct]:
out: List[rest.PointStruct] = []
for pt in points:
vec = getattr(pt, "vector", None)
if isinstance(vec, dict):
if name in vec:
out.append(pt)
else:
# take any existing entry; if empty dict fallback to [0.0]
fallback_vec = None
try:
fallback_vec = list(next(iter(vec.values())))
except Exception:
fallback_vec = [0.0]
out.append(rest.PointStruct(id=pt.id, vector={name: fallback_vec}, payload=pt.payload))
elif vec is not None:
out.append(rest.PointStruct(id=pt.id, vector={name: vec}, payload=pt.payload))
else:
out.append(pt)
return out
# --------------------- Qdrant ops ---------------------
def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointStruct]) -> None:
if not points:
return
# 1) ENV overrides come first
override = _env_override_for_collection(collection)
if override == "__single__":
client.upsert(collection_name=collection, points=points, wait=True)
return
elif isinstance(override, str):
client.upsert(collection_name=collection, points=_as_named(points, override), wait=True)
return
# 2) Auto-detect schema
schema = _get_vector_schema(client, collection)
if schema.get("kind") == "named":
name = schema.get("primary") or _preferred_name(schema.get("names") or [])
client.upsert(collection_name=collection, points=_as_named(points, name), wait=True)
return
# 3) Fallback single-vector
client.upsert(collection_name=collection, points=points, wait=True)
# --- WP-04 Ergänzungen: Graph/Retriever Hilfsfunktionen ---
from typing import Optional, Dict, Any, Iterable
from qdrant_client import QdrantClient
# --- Optional search helpers ---
def _filter_any(field: str, values: Iterable[str]) -> rest.Filter:
"""Erzeuge OR-Filter: payload[field] == any(values)."""
return rest.Filter(
should=[
rest.FieldCondition(key=field, match=rest.MatchValue(value=v))
for v in values
]
)
return rest.Filter(should=[rest.FieldCondition(key=field, match=rest.MatchValue(value=v)) for v in values])
def _merge_filters(*filters: Optional[rest.Filter]) -> Optional[rest.Filter]:
"""Fasst mehrere Filter zu einem AND zusammen (None wird ignoriert)."""
fs = [f for f in filters if f is not None]
if not fs:
return None
if len(fs) == 1:
return fs[0]
# rest.Filter hat must/should; wir kombinieren als must=[...]
must = []
for f in fs:
# Überführe vorhandene Bedingungen in must
if getattr(f, "must", None):
must.extend(f.must)
if getattr(f, "should", None):
# "should" als eigene Gruppe beilegen (Qdrant interpretiert OR)
must.append(rest.Filter(should=f.should))
if getattr(f, "must_not", None):
# negative Bedingungen weiterreichen
if "must_not" not in locals():
pass
return rest.Filter(must=must)
def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter]:
"""
Einfache Filterumsetzung:
- Bei Listenwerten: OR über mehrere MatchValue (field == any(values))
- Bei Skalarwerten: Gleichheit (field == value)
Für komplexere Filter (z. B. tags ∈ payload.tags) bitte erweitern.
"""
if not filters:
return None
parts = []
@@ -187,147 +209,11 @@ def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter
parts.append(rest.Filter(must=[rest.FieldCondition(key=k, match=rest.MatchValue(value=v))]))
return _merge_filters(*parts)
def search_chunks_by_vector(
client: QdrantClient,
prefix: str,
vector: list[float],
top: int = 10,
filters: Optional[Dict[str, Any]] = None,
) -> list[tuple[str, float, dict]]:
"""
Vektorielle Suche in {prefix}_chunks.
Rückgabe: Liste von (point_id, score, payload)
"""
def search_chunks_by_vector(client: QdrantClient, prefix: str, vector: List[float], top: int = 10, filters: Optional[Dict[str, Any]] = None) -> List[Tuple[str, float, dict]]:
_, chunks_col, _ = _names(prefix)
flt = _filter_from_dict(filters)
res = client.search(
collection_name=chunks_col,
query_vector=vector,
limit=top,
with_payload=True,
with_vectors=False,
query_filter=flt,
)
out: list[tuple[str, float, dict]] = []
res = client.search(collection_name=chunks_col, query_vector=vector, limit=top, with_payload=True, with_vectors=False, query_filter=flt)
out: List[Tuple[str, float, dict]] = []
for r in res:
out.append((str(r.id), float(r.score), dict(r.payload or {})))
return out
def get_edges_for_sources(
client: QdrantClient,
prefix: str,
source_ids: list[str],
edge_types: Optional[list[str]] = None,
limit: int = 2048,
) -> list[dict]:
"""
Hole Edges aus {prefix}_edges mit source_id ∈ source_ids (und optional kind ∈ edge_types).
Liefert Payload-Dicts inkl. edge_id/source_id/target_id/kind/seq (falls vorhanden).
"""
_, _, edges_col = _names(prefix)
f_src = _filter_any("source_id", source_ids)
f_kind = _filter_any("kind", edge_types) if edge_types else None
flt = _merge_filters(f_src, f_kind)
collected: list[dict] = []
next_page = None
while True:
points, next_page = client.scroll(
collection_name=edges_col,
scroll_filter=flt,
limit=min(512, limit - len(collected)),
with_payload=True,
with_vectors=False,
offset=next_page,
)
for p in points:
pl = dict(p.payload or {})
# füge die deterministische ID hinzu (nützlich für Clients)
pl.setdefault("id", str(p.id))
collected.append(pl)
if len(collected) >= limit:
return collected
if next_page is None:
break
return collected
def get_note_payload(
client: QdrantClient,
prefix: str,
note_id: str,
) -> Optional[dict]:
"""
Hole eine Note anhand ihres payload.note_id (nicht internal UUID!).
"""
notes_col, _, _ = _names(prefix)
flt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
points, _ = client.scroll(
collection_name=notes_col,
scroll_filter=flt,
limit=1,
with_payload=True,
with_vectors=False,
)
if not points:
return None
pl = dict(points[0].payload or {})
pl.setdefault("id", str(points[0].id))
return pl
def get_neighbor_nodes(
client: QdrantClient,
prefix: str,
target_ids: list[str],
limit_per_collection: int = 2048,
) -> dict[str, dict]:
"""
Hole Payloads der Zielknoten (Notes/Chunks) zu den angegebenen IDs.
IDs sind die stabilen payload-IDs (note_id/chunk_id), nicht internal UUIDs.
Rückgabe: Mapping target_id -> payload
"""
notes_col, chunks_col, _ = _names(prefix)
out: dict[str, dict] = {}
# Notes
flt_notes = _filter_any("note_id", target_ids)
next_page = None
while True:
pts, next_page = client.scroll(
collection_name=notes_col,
scroll_filter=flt_notes,
limit=256,
with_payload=True,
with_vectors=False,
offset=next_page,
)
for p in pts:
pl = dict(p.payload or {})
nid = pl.get("note_id")
if nid and nid not in out:
pl.setdefault("id", str(p.id))
out[nid] = pl
if next_page is None or len(out) >= limit_per_collection:
break
# Chunks
flt_chunks = _filter_any("chunk_id", target_ids)
next_page = None
while True:
pts, next_page = client.scroll(
collection_name=chunks_col,
scroll_filter=flt_chunks,
limit=256,
with_payload=True,
with_vectors=False,
offset=next_page,
)
for p in pts:
pl = dict(p.payload or {})
cid = pl.get("chunk_id")
if cid and cid not in out:
pl.setdefault("id", str(p.id))
out[cid] = pl
if next_page is None or len(out) >= limit_per_collection:
break
return out
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Modul: app/core/type_registry.py
Version: 1.0.0
Datum: 2025-11-08
Zweck
-----
Lädt eine optionale Typ-Registry (config/types.yaml) und stellt
komfortable Zugriffsfunktionen bereit. Die Registry ist *optional*:
- Fehlt die Datei oder ist das YAML defekt, wird ein konservativer
Default (Typ "concept") verwendet und es wird eine Warnung ausgegeben.
- Änderungen an der Datei greifen nach einem Neustart des Prozesses.
Öffentliche API
---------------
- load_type_registry(path: str = "config/types.yaml") -> dict
- get_type_config(note_type: str, reg: dict) -> dict
- resolve_note_type(fm_type: str | None, reg: dict) -> str
- effective_chunk_profile(note_type: str, reg: dict) -> str | None
- profile_overlap(profile: str | None) -> tuple[int,int] # nur Overlap-Empfehlung
Hinweis
-------
Die Registry steuert KEINE Breaking Changes. Ohne Datei/Typ bleibt das
Verhalten exakt wie im Release-Stand 20251105.
"""
from __future__ import annotations
from functools import lru_cache
from typing import Dict, Any, Optional, Tuple
import os
try:
import yaml # PyYAML
except Exception:
yaml = None # wird erst benötigt, wenn eine Datei gelesen werden soll
# Konservativer Default – bewusst minimal
_DEFAULT_REGISTRY: Dict[str, Any] = {
"version": "1.0",
"types": {
"concept": {
"chunk_profile": "medium",
"edge_defaults": ["references", "related_to"],
"retriever_weight": 1.0,
}
},
"_using_defaults": True,
"_warning": "types.yaml missing or invalid – using built-in defaults (type=concept).",
}
# Chunk-Profile → Overlap-Empfehlungen (nur für synthetische Fensterbildung)
# Die absoluten Chunk-Längen bleiben Aufgabe des Chunkers (assemble_chunks).
_PROFILE_TO_OVERLAP: Dict[str, Tuple[int, int]] = {
"short": (20, 30),
"medium": (40, 60),
"long": (60, 80),
}
@lru_cache(maxsize=1)
def load_type_registry(path: str = "config/types.yaml") -> Dict[str, Any]:
"""
Lädt die Registry aus 'path'. Bei Fehlern wird ein konserviver Default geliefert.
Die Rückgabe ist *prozessweit* gecached.
"""
if not path:
return dict(_DEFAULT_REGISTRY)
if not os.path.isfile(path):
return dict(_DEFAULT_REGISTRY)
if yaml is None:
# PyYAML fehlt → auf Default zurückfallen
return dict(_DEFAULT_REGISTRY)
try:
with open(path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
# Minimal validieren
if not isinstance(data, dict) or "types" not in data or not isinstance(data["types"], dict):
return dict(_DEFAULT_REGISTRY)
data.setdefault("version", "1.0")
data.setdefault("_using_defaults", False)
return data
except Exception:
return dict(_DEFAULT_REGISTRY)
def get_type_config(note_type: Optional[str], reg: Dict[str, Any]) -> Dict[str, Any]:
t = (note_type or "concept").strip().lower()
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
return types.get(t) or types.get("concept") or _DEFAULT_REGISTRY["types"]["concept"]
def resolve_note_type(fm_type: Optional[str], reg: Dict[str, Any]) -> str:
"""Liefert einen gültigen Typ (unbekannt → 'concept')."""
t = (fm_type or "concept").strip().lower()
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
return t if t in types else "concept"
def effective_chunk_profile(note_type: Optional[str], reg: Dict[str, Any]) -> Optional[str]:
cfg = get_type_config(note_type, reg)
prof = cfg.get("chunk_profile")
if isinstance(prof, str) and prof.strip():
return prof.strip().lower()
return None
def profile_overlap(profile: Optional[str]) -> Tuple[int, int]:
"""Gibt eine Overlap-Empfehlung (low, high) für das Profil zurück."""
if not profile:
return _PROFILE_TO_OVERLAP["medium"]
return _PROFILE_TO_OVERLAP.get(profile.strip().lower(), _PROFILE_TO_OVERLAP["medium"])
+18
View File
@@ -0,0 +1,18 @@
version: 1.0
types:
concept:
chunk_profile: medium
edge_defaults: ["references", "related_to"]
retriever_weight: 0.33
task:
chunk_profile: short
edge_defaults: ["depends_on", "belongs_to"]
retriever_weight: 0.8
experience:
chunk_profile: medium
edge_defaults: ["derived_from", "inspired_by"]
retriever_weight: 0.9
project:
chunk_profile: long
edge_defaults: ["references", "depends_on"]
retriever_weight: 0.95
+21
View File
@@ -0,0 +1,21 @@
# Changelog — WP-03 Type-Registry (2025-11-08)
## Added
- `app/core/type_registry.py` — Loader & Helper (cached), robust Fallbacks.
- `config/types.yaml` — Beispiel-Registry (version 1.0).
- `scripts/import_markdown.py` v3.8.0 — Registry-Integration:
- typvalidierung (unbekannt → `concept`, Warnung)
- `chunk_profile` → an `make_chunk_payloads(...)`
- `retriever_weight` → persistiert im Note-Payload
- `edge_defaults["references"]` schaltet Note-Scope-Refs additiv ein
- `app/core/chunk_payload.py` v2.3.0 — optionales `chunk_profile` für
synthetische Overlaps (keine Schemaänderung).
- `app/core/derive_edges.py` v1.5.0 — Type-Registry aktiviert note-scope
References/Backlinks additiv.
## Changed
- Keine Breaking Changes. Alle CLI-Flags, ENV-Variablen und Collection-
Prefix-Logik bleiben unverändert.
## Removed
- N/A
+74
View File
@@ -0,0 +1,74 @@
# Kurzhandbuch — Type-Registry (WP-03)
## Zweck
Die Type-Registry macht den `type:`-Wert im Frontmatter maschinenlesbar
und steuert:
- Chunk-Profile (indirekt über Overlap bei fensterlosen Chunks),
- typabhängige Default-Kanten (hier: Aktivierung von Note-Scope-References),
- optionale Retriever-Gewichtungen (`retriever_weight`) für WP‑04.
## Aktivierung
1. Legen Sie `config/types.yaml` an (siehe Beispiel in diesem Paket).
2. Starten Sie Import/Server neu. Die Registry wird beim Prozessstart
geladen und gecached.
Fehlt die Datei oder ist sie defekt, bleibt das System voll funktionsfähig
(der Typ `concept` wird genutzt; Warnhinweis im Log).
## Schema (Beispiel)
```yaml
version: 1.0
types:
concept:
chunk_profile: long
edge_defaults: ["references", "related_to"]
retriever_weight: 1.0
task:
chunk_profile: short
edge_defaults: ["depends_on", "belongs_to"]
retriever_weight: 0.8
experience:
chunk_profile: medium
edge_defaults: ["derived_from", "inspired_by"]
retriever_weight: 0.9
project:
chunk_profile: medium
edge_defaults: ["references", "depends_on"]
retriever_weight: 0.95
```
## Verhalten
- **Chunking**: Die eigentliche Chunk-Länge bleibt weiterhin typabhängig über
`chunk_config.get_sizes(type)` gesteuert. Falls der Chunker keine Fenster
mit Overlap liefert, werden in `chunk_payload` synthetische Fenster erzeugt.
Hier greift das `chunk_profile` mit folgenden Overlap-Empfehlungen:
- short → (20,30), medium → (40,60), long → (60,80).
- **Edges**: Enthält `edge_defaults` den Eintrag `references`, werden Note-
Scope-References/Backlinks additiv aktiviert (kein Abbruch, keine Duplikate).
Weitere Kantenarten bleiben unverändert.
- **Retriever**: `retriever_weight` wird in der Note gespeichert und kann von
WP‑04 im Ranking/Scoring berücksichtigt werden.
## Tests / Validierung
- Dry-Run:
```bash
python3 -m scripts.import_markdown --vault ./test_vault
```
- Vollimport + Purge:
```bash
python3 -m scripts.import_markdown --vault ./test_vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
```
- Chunks-Qualität (Window vs Text):
```bash
python3 tests/check_chunks_window_vs_text.py --prefix "$COLLECTION_PREFIX"
```
- Roundtrip:
```bash
python3 -m scripts.export_markdown --out ./_exportVault --prefix "$COLLECTION_PREFIX"
python3 tests/compare_vaults.py --src ./test_vault --dst ./_exportVault --focus body
```
## Fallbacks & Robustheit
- Registry nicht vorhanden/ungültig → `concept`, Warnung, keine Abbrüche.
- Alle bestehenden ENV/Flags bleiben identisch.
- Keine Änderungen an Punkt-IDs, Feldern oder Vektoren.
+223 -104
View File
@@ -1,38 +1,25 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
Version: 3.7.2
Datum: 2025-09-30
scripts/import_markdown.py
Kurzbeschreibung
----------------
- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
- Änderungserkennung „Option C“: mehrere Hash-Varianten werden parallel in der Note
gespeichert (Feld `hashes` mit Schlüsseln `<mode>:<source>:<normalize>`). Der Vergleich
nutzt NUR den aktuellen Modus-Key — ein Moduswechsel triggert keine Massenänderungen mehr.
- „Erstimport-Fix“: Bei leerem Qdrant gilt Create-Fall automatisch als geändert.
- `--baseline-modes`: fehlende Hash-Varianten still nachtragen (nur Notes upserten).
- `--sync-deletes`: gezielte Lösch-Synchronisation (Dry-Run + Apply).
- `--only-path`: exakt **eine** Datei (Pfad) importieren — nützlich für Diagnosefälle.
Zweck
-----
- Liest Markdown-Notizen aus einem Vault ein
- Erzeugt Note-Payload, Chunk-Payloads (+ optionale Embeddings) und Edges
- Schreibt alles idempotent in Qdrant (Notes, Chunks, Edges)
- Integriert eine optionale Type-Registry (types.yaml), um z. B. chunk_profile
und retriever_weight pro Notiz-Typ zu steuern.
Neu in 3.7.1/3.7.2
------------------
- Chunk-Payloads: `window` (für Embeddings), `text` (überlappungsfrei, verlustfrei rekonstruierbar),
`start/end/overlap_*`. Embeddings nutzen `window`.
- **3.7.2:** Edges-Fehler führen nicht mehr zum Abbruch der gesamten Note; Note/Chunks werden trotzdem geschrieben.
Hash/Compare Konfiguration
--------------------------
- Vergleichsmodus:
--hash-mode body|frontmatter|full
oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE
- Quelle:
--hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE, Default parsed)
- Normalisierung:
--hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE, Default canonical)
- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich
den parsed Body-Text direkt.
Wesentliche Fixes ggü. vorherigen fehlerhaften Ständen
------------------------------------------------------
- `embed_texts` wird optional importiert und defensiv geprüft (kein NameError mehr)
- `effective_chunk_profile` / `effective_retriever_weight` und Registry-Helfer
sind VOR `main()` definiert (kein NameError mehr)
- `retriever_weight` wird in Note- und Chunk-Payload zuverlässig gesetzt
- Robuste Kantenbildung; Fehler bei Edges blockieren Notes/Chunks nicht
- Korrekte Verwendung von `scroll_filter` beim Qdrant-Client
- `--purge-before-upsert` entfernt alte Chunks/Edges einer Note vor dem Upsert
Qdrant / ENV
------------
@@ -40,6 +27,7 @@ Qdrant / ENV
- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
- VECTOR_DIM (Default: 384)
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
- MINDNET_TYPES_FILE: Pfad zu types.yaml (optional; Default: ./types.yaml)
Beispiele
---------
@@ -67,6 +55,7 @@ from typing import Dict, List, Optional, Tuple, Any, Set
from dotenv import load_dotenv
from qdrant_client.http import models as rest
# --- Projekt-Imports ---
from app.core.parser import (
read_markdown,
normalize_frontmatter,
@@ -92,14 +81,86 @@ from app.core.qdrant_points import (
upsert_batch,
)
# embeddings sind optional (z. B. im reinen Payload-Backfill)
try:
from app.core.embed import embed_texts # optional
except Exception:
except Exception: # pragma: no cover
embed_texts = None
# ---------------------------------------------------------------------
# Helper
# Type-Registry (types.yaml) – Helper (robust, optional)
# ---------------------------------------------------------------------
def _env(name: str, default: Optional[str] = None) -> Optional[str]:
v = os.getenv(name)
return v if v is not None else default
def _load_json_or_yaml(path: str) -> dict:
import io
data: dict = {}
if not path or not os.path.exists(path):
return data
try:
import yaml # type: ignore
with io.open(path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
if not isinstance(data, dict):
return {}
return data
except Exception:
# YAML evtl. nicht installiert – versuche JSON
try:
with io.open(path, "r", encoding="utf-8") as f:
data = json.load(f)
if not isinstance(data, dict):
return {}
return data
except Exception:
return {}
def load_type_registry() -> dict:
# Reihenfolge: ENV > ./types.yaml (im aktuellen Arbeitsverzeichnis)
p = _env("MINDNET_TYPES_FILE", None)
if p and os.path.exists(p):
return _load_json_or_yaml(p)
fallback = os.path.abspath("./types.yaml")
return _load_json_or_yaml(fallback)
def get_type_config(note_type: Optional[str], reg: dict) -> dict:
if not reg or not isinstance(reg, dict):
return {}
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
if note_type and isinstance(note_type, str) and note_type in types:
return types[note_type] or {}
# Fallback: concept
return types.get("concept", {}) or {}
def resolve_note_type(requested: Optional[str], reg: dict) -> str:
if requested and isinstance(requested, str):
return requested
# Fallback wenn nichts gesetzt ist
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
return "concept" if "concept" in types else (requested or "concept")
def effective_chunk_profile(note_type: str, reg: dict) -> Optional[str]:
cfg = get_type_config(note_type, reg)
prof = cfg.get("chunk_profile")
if isinstance(prof, str) and prof in {"short", "medium", "long"}:
return prof
return None
def effective_retriever_weight(note_type: str, reg: dict) -> Optional[float]:
cfg = get_type_config(note_type, reg)
w = cfg.get("retriever_weight")
try:
return float(w) if w is not None else None
except Exception:
return None
# ---------------------------------------------------------------------
# Sonstige Helper
# ---------------------------------------------------------------------
def iter_md(root: str) -> List[str]:
@@ -115,17 +176,15 @@ def iter_md(root: str) -> List[str]:
out.append(p)
return sorted(out)
def collections(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
notes_col, _, _ = collections(prefix)
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
points, _ = client.scroll(
collection_name=notes_col,
scroll_filter=f,
scroll_filter=f, # wichtig: scroll_filter (nicht: filter)
with_payload=True,
with_vectors=False,
limit=1,
@@ -134,7 +193,6 @@ def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[D
return None
return points[0].payload or {}
def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
notes_col, _, _ = collections(prefix)
out: Set[str] = set()
@@ -158,7 +216,6 @@ def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
break
return out
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
_, chunks_col, edges_col = collections(prefix)
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
@@ -172,7 +229,6 @@ def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
except Exception as e:
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
notes_col, chunks_col, edges_col = collections(prefix)
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
@@ -187,50 +243,56 @@ def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
def _resolve_mode(val: Optional[str]) -> str:
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
return "full"
if v in ("frontmatter", "fm"):
return "frontmatter"
return "body"
# --- Neu: Existenz-Checks für Artefakte (fehlertoleranter Rebuild) ---
def _has_any_point(client, collection: str, note_id: str) -> bool:
"""Prüft, ob es mind. einen Punkt mit note_id in der Collection gibt."""
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
pts, _ = client.scroll(
collection_name=collection,
scroll_filter=filt,
with_payload=False,
with_vectors=False,
limit=1,
)
return bool(pts)
def _env(key: str, default: str) -> str:
return (os.environ.get(key) or default).strip().lower()
def artifacts_missing(client, prefix: str, note_id: str) -> Tuple[bool, bool]:
"""Gibt (chunks_missing, edges_missing) zurück."""
_, chunks_col, edges_col = collections(prefix)
chunks_missing = not _has_any_point(client, chunks_col, note_id)
edges_missing = not _has_any_point(client, edges_col, note_id)
return chunks_missing, edges_missing
# ---------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------
def _resolve_mode(m: Optional[str]) -> str:
m = (m or "body").strip().lower()
return m if m in {"body", "frontmatter", "full"} else "body"
def main() -> None:
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
ap.add_argument("--purge-before-upsert", action="store_true",
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
ap.add_argument("--only-path", help="Exakt diesen Markdown-Pfad verarbeiten (ignoriert --note-id)")
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
ap.add_argument("--force-replace", action="store_true",
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None,
help="Vergleichsmodus (Body | Frontmatter | Full)")
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
help="Quelle für die Hash-Berechnung (Default: parsed)")
ap.add_argument("--note-scope-refs", action="store_true",
help="(Optional) erzeugt zusätzlich references:note/backlink:note (Default: aus)")
ap.add_argument("--debug-hash-diff", action="store_true",
help="(reserviert) optionaler Body-Diff")
ap.add_argument("--compare-text", action="store_true",
help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
ap.add_argument("--baseline-modes", action="store_true",
help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
ap.add_argument("--sync-deletes", action="store_true",
help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)")
ap = argparse.ArgumentParser(
prog="scripts.import_markdown",
description="Importiert Markdown-Notizen in Qdrant (Notes/Chunks/Edges)."
)
ap.add_argument("--vault", required=True, help="Pfad zum Vault (Ordner mit .md-Dateien)")
ap.add_argument("--only-path", help="Nur diese Datei verarbeiten (absolut oder relativ)")
ap.add_argument("--apply", action="store_true", help="Schreibt nach Qdrant (sonst Dry-Run)")
ap.add_argument("--purge-before-upsert", action="store_true", help="Alte Chunks/Edges der Note vorher löschen")
ap.add_argument("--force-replace", action="store_true", help="Note/Chunks/Edges unabhängig von Hash neu schreiben")
ap.add_argument("--note-id", help="Nur Notes mit dieser ID verarbeiten (Filter)")
ap.add_argument("--note-scope-refs", action="store_true", help="Note-scope References/Backlinks erzeugen")
ap.add_argument("--hash-mode", help="body|frontmatter|full (Default body)")
ap.add_argument("--hash-source", help="parsed|raw (Default parsed)")
ap.add_argument("--hash-normalize", help="canonical|none (Default canonical)")
ap.add_argument("--compare-text", action="store_true", help="Parsed fulltext zusätzlich direkt vergleichen")
ap.add_argument("--baseline-modes", action="store_true", help="Fehlende Hash-Varianten still nachtragen (Notes)")
ap.add_argument("--sync-deletes", action="store_true", help="Qdrant->Vault Lösch-Sync (Dry-Run; mit --apply ausführen)")
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
args = ap.parse_args()
@@ -241,6 +303,7 @@ def main() -> None:
note_scope_refs = args.note_scope_refs or note_scope_refs_env
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
# Qdrant
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix.strip()
@@ -248,9 +311,12 @@ def main() -> None:
ensure_collections(client, cfg.prefix, cfg.dim)
ensure_payload_indexes(client, cfg.prefix)
# Type-Registry laden (optional)
reg = load_type_registry()
root = os.path.abspath(args.vault)
# Dateiliste bestimmen
# Dateiliste
if args.only_path:
only = os.path.abspath(args.only_path)
files = [only]
@@ -293,16 +359,15 @@ def main() -> None:
processed = 0
for path in files:
# -------- Parse & Validate --------
try:
parsed = read_markdown(path)
if not parsed:
continue
except Exception as e:
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
continue
if parsed is None:
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
continue
# --- Frontmatter prüfen ---
try:
fm = normalize_frontmatter(parsed.frontmatter)
validate_required_frontmatter(fm)
@@ -315,24 +380,54 @@ def main() -> None:
processed += 1
# -------- Build new payload (includes 'hashes') --------
note_pl = make_note_payload(
parsed,
vault_root=root,
hash_mode=mode,
hash_normalize=norm,
hash_source=src,
file_path=path,
)
# --- Type-Registry anwenden (chunk_profile / retriever_weight) ---
try:
note_type = resolve_note_type(fm.get("type"), reg)
except Exception:
note_type = (fm.get("type") or "concept")
fm["type"] = note_type or fm.get("type") or "concept"
prof = effective_chunk_profile(note_type, reg)
if prof:
fm["chunk_profile"] = prof
weight = effective_retriever_weight(note_type, reg)
if weight is not None:
try:
fm["retriever_weight"] = float(weight)
except Exception:
pass # falls FM string-inkonsistent ist
# --- Payload aufbauen (inkl. Hashes) ---
try:
note_pl = make_note_payload(
parsed,
vault_root=root,
hash_mode=mode,
hash_normalize=norm,
hash_source=src,
file_path=path,
)
except Exception as e:
print(json.dumps({"path": path, "error": f"make_note_payload failed: {type(e).__name__}: {e}"}))
continue
if not note_pl.get("fulltext"):
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
# retriever_weight sicher in Note-Payload spiegeln (für spätere Filter)
if "retriever_weight" not in note_pl and fm.get("retriever_weight") is not None:
try:
note_pl["retriever_weight"] = float(fm.get("retriever_weight"))
except Exception:
pass
note_id = note_pl.get("note_id") or fm.get("id")
if not note_id:
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
continue
# -------- Fetch old payload --------
# --- bestehenden Payload laden (zum Diff) ---
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
has_old = old_payload is not None
@@ -352,17 +447,28 @@ def main() -> None:
changed = args.force_replace or (not has_old) or hash_changed or text_changed
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
# -------- Chunks / Embeddings --------
chunk_pls: List[Dict[str, Any]] = []
# --- Chunks + Embeddings vorbereiten ---
try:
body_text = getattr(parsed, "body", "") or ""
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
chunk_pls: List[Dict[str, Any]] = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
except Exception as e:
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
continue
vecs: List[List[float]] = [[0.0] * cfg.dim for _ in chunk_pls]
# retriever_weight auf Chunk-Payload spiegeln
if fm.get("retriever_weight") is not None:
try:
rw = float(fm.get("retriever_weight"))
for pl in chunk_pls:
# Feld nur setzen, wenn noch nicht vorhanden
if "retriever_weight" not in pl:
pl["retriever_weight"] = rw
except Exception:
pass
# Embeddings (fallback: Nullvektoren)
vecs: List[List[float]] = [[0.0] * int(cfg.dim) for _ in chunk_pls]
if embed_texts and chunk_pls:
try:
texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls]
@@ -370,10 +476,14 @@ def main() -> None:
except Exception as e:
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
# -------- Edges (robust) --------
# --- Fehlende Artefakte in Qdrant ermitteln ---
chunks_missing, edges_missing = artifacts_missing(client, cfg.prefix, note_id)
# --- Edges (robust) ---
edges: List[Dict[str, Any]] = []
edges_failed = False
if changed and (not do_baseline_only):
should_build_edges = (changed and (not do_baseline_only)) or edges_missing
if should_build_edges:
try:
note_refs = note_pl.get("references") or []
edges = build_edges_for_note(
@@ -385,10 +495,9 @@ def main() -> None:
except Exception as e:
edges_failed = True
edges = []
# WICHTIG: Wir brechen NICHT mehr ab — Note & Chunks werden geschrieben.
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
# -------- Summary --------
# --- Summary (stdout) ---
summary = {
"note_id": note_id,
"title": fm.get("title"),
@@ -396,10 +505,12 @@ def main() -> None:
"edges": len(edges),
"edges_failed": edges_failed,
"changed": changed,
"chunks_missing": chunks_missing,
"edges_missing": edges_missing,
"needs_baseline_for_mode": needs_baseline,
"decision": ("baseline-only" if args.apply and do_baseline_only else
"apply" if args.apply and changed else
"apply-skip-unchanged" if args.apply and not changed else
"apply" if args.apply and (changed or chunks_missing or edges_missing) else
"apply-skip-unchanged" if args.apply and not (changed or chunks_missing or edges_missing) else
"dry-run"),
"path": note_pl["path"],
"hash_mode": mode,
@@ -409,7 +520,7 @@ def main() -> None:
}
print(json.dumps(summary, ensure_ascii=False))
# -------- Writes --------
# --- Writes ---
if not args.apply:
continue
@@ -425,21 +536,29 @@ def main() -> None:
upsert_batch(client, notes_name, note_pts)
continue
if not changed:
# Wenn nichts geändert und keine Artefakte fehlen → nichts zu tun
if not changed and not (chunks_missing or edges_missing):
continue
if args.purge_before_upsert and has_old:
# Purge nur bei echten Änderungen (unverändert + fehlende Artefakte ≠ Purge)
if args.purge_before_upsert and has_old and changed:
try:
purge_note_artifacts(client, cfg.prefix, note_id)
except Exception as e:
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
upsert_batch(client, notes_name, note_pts)
if chunk_pls:
# Note nur bei Änderungen neu schreiben
if changed:
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
upsert_batch(client, notes_name, note_pts)
# Chunks schreiben, wenn geändert ODER vorher fehlend
if chunk_pls and (changed or chunks_missing):
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
upsert_batch(client, chunks_name, chunk_pts)
if edges:
# Edges schreiben, wenn vorhanden und (geändert ODER vorher fehlend)
if edges and (changed or edges_missing):
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
upsert_batch(client, edges_name, edge_pts)
+190
View File
@@ -0,0 +1,190 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
check_types_registry_edges.py
--------------------------------------------------
Prüft, ob die in config/types.yaml hinterlegten Default-Kanten
pro Note-Typ tatsächlich in der Qdrant *edges*-Collection auftauchen.
"""
import os, sys, json
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Set
from dataclasses import dataclass, asdict
try:
import yaml # PyYAML
except Exception as e:
print(json.dumps({"error": f"PyYAML not installed: {e}"})); sys.exit(2)
try:
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest # noqa: F401
except Exception as e:
print(json.dumps({"error": f"qdrant-client not installed: {e}"})); sys.exit(2)
@dataclass
class Cfg:
url: str
api_key: Optional[str]
prefix: str
notes: str
edges: str
types_file: Path
def _env_prefix() -> Optional[str]:
for k in ("COLLECTION_PREFIX", "MINDNET_PREFIX"):
v = os.environ.get(k, "").strip()
if v:
return v
return None
def _load_types_yaml(path: Path) -> Dict:
if not path.exists():
print(json.dumps({"warn": f"types.yaml fehlt: {path}. Fallback: keine Vorgaben."}))
return {}
try:
with path.open("r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
return data if isinstance(data, dict) else {}
except Exception as e:
print(json.dumps({"warn": f"types.yaml defekt ({path}): {e}. Fallback: keine Vorgaben."}))
return {}
def _cfg_from_env(argv_prefix: Optional[str]) -> Cfg:
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
api_key = os.environ.get("QDRANT_API_KEY", "").strip() or None
prefix = (argv_prefix or _env_prefix() or "mindnet").strip()
notes = f"{prefix}_notes"
edges = f"{prefix}_edges"
types_path = Path(os.environ.get("TYPES_FILE", "config/types.yaml")).resolve()
return Cfg(url=url, api_key=api_key, prefix=prefix, notes=notes, edges=edges, types_file=types_path)
def _mk_client(cfg: Cfg) -> QdrantClient:
return QdrantClient(url=cfg.url, api_key=cfg.api_key, timeout=30.0)
def _scroll_all_notes(client: QdrantClient, notes_col: str) -> Dict[str, Dict]:
out = {}
offset = None
while True:
points, offset = client.scroll(
collection_name=notes_col,
scroll_filter=None, limit=256, offset=offset,
with_payload=True, with_vectors=False,
)
if not points:
break
for p in points:
payload = p.payload or {}
nid = payload.get("note_id") or payload.get("id") or payload.get("uid") or payload.get("slug")
if not nid:
nid = str(p.id)
out[str(nid)] = payload
return out
def _scroll_all_edges(client: QdrantClient, edges_col: str) -> List[Tuple[str, dict]]:
out = []
offset = None
while True:
points, offset = client.scroll(
collection_name=edges_col,
scroll_filter=None, limit=256, offset=offset,
with_payload=True, with_vectors=False,
)
if not points:
break
for p in points:
out.append((str(p.id), p.payload or {}))
return out
def _guess_src_note_id(ed_pl: dict) -> Optional[str]:
for k in ("src_note_id", "note_id", "src_id", "src"):
nid = ed_pl.get(k)
if nid:
return str(nid)
for k in ("src_ref", "from"):
nid = ed_pl.get(k)
if isinstance(nid, dict):
for kk in ("note_id", "id"):
if nid.get(kk):
return str(nid[kk])
elif isinstance(nid, str):
return nid
return None
def _edge_type(ed_pl: dict) -> Optional[str]:
for k in ("edge_type", "type", "rel", "relation"):
if ed_pl.get(k):
return str(ed_pl[k])
return None
def _cfg_to_jsonable(cfg: Cfg) -> dict:
d = asdict(cfg)
d["types_file"] = str(d.get("types_file"))
return d
def main(argv=None):
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", help="Collection-Prefix (mindnet, etc.)")
args = ap.parse_args(argv)
cfg = _cfg_from_env(args.prefix)
print(json.dumps({"cfg": _cfg_to_jsonable(cfg)}, ensure_ascii=False))
tr = _load_types_yaml(cfg.types_file)
types_def = (tr.get("types") if isinstance(tr, dict) else {}) or {}
print(json.dumps({"types_defined": list(types_def.keys())}, ensure_ascii=False))
client = _mk_client(cfg)
notes = _scroll_all_notes(client, cfg.notes)
print(json.dumps({"notes_count": len(notes)}, ensure_ascii=False))
edges = _scroll_all_edges(client, cfg.edges)
print(json.dumps({"edges_count": len(edges)}, ensure_ascii=False))
note_type = {}
for nid, pl in notes.items():
t = pl.get("type") or "concept"
note_type[str(nid)] = str(t)
seen = {}
counts = {}
for edge_pid, ed_pl in edges:
et = _edge_type(ed_pl)
if not et:
continue
src_nid = _guess_src_note_id(ed_pl)
if not src_nid:
continue
t = note_type.get(src_nid)
if not t:
continue
seen.setdefault(t, set()).add(et)
counts.setdefault(t, {}).setdefault(et, 0)
counts[t][et] += 1
expected = {}
for tname, tdef in types_def.items():
eddefs = (tdef or {}).get("edge_defaults") or []
expected[tname] = set([str(x) for x in eddefs if x])
for tname, exp in expected.items():
obs = seen.get(tname, set())
missing = sorted(list(exp - obs))
extra = sorted(list(obs - exp))
print(json.dumps({
"type": tname,
"expected_defaults": sorted(list(exp)),
"observed": sorted(list(obs)),
"missing": missing,
"extra": extra,
"counts": counts.get(tname, {}),
}, ensure_ascii=False))
if not expected:
print(json.dumps({"warn": "Keine Typ-Defaults in types.yaml gefunden (edge_defaults leer?)."}))
if __name__ == "__main__":
main()
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os, json, argparse, random
from typing import Any, Dict, List, Tuple, Optional
from qdrant_client import QdrantClient
def _get_prefix(explicit: Optional[str]) -> str:
return (explicit or os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet").strip()
def _names(prefix: str) -> Dict[str,str]:
return {
"notes": f"{prefix}_notes",
"chunks": f"{prefix}_chunks",
"edges": f"{prefix}_edges",
}
def _scroll(client: QdrantClient, collection: str, *, limit=5, with_payload=True):
# Support both modern and older SDKs (filter vs scroll_filter)
try:
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
except TypeError as e:
# older signatures might accept scroll_filter kw, but if we passed none it's fine
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--host", default=os.environ.get("QDRANT_HOST", "127.0.0.1"))
ap.add_argument("--port", default=int(os.environ.get("QDRANT_PORT", "6333")))
ap.add_argument("--prefix", default=None)
ap.add_argument("--limit", type=int, default=5)
ap.add_argument("--note-id", default=None)
args = ap.parse_args()
prefix = _get_prefix(args.prefix)
names = _names(prefix)
client = QdrantClient(host=args.host, port=args.port, prefer_grpc=False)
# Notes sample
try:
pts, _ = _scroll(client, names["notes"], limit=args.limit, with_payload=True)
except Exception as e:
print(json.dumps({"error": f"notes scroll failed: {type(e).__name__}: {e}", "collection": names["notes"]}))
return
sample = []
for p in pts:
pl = (p.payload or {})
sample.append({
"point_id": str(p.id),
"note_id": pl.get("note_id"),
"id": pl.get("id"),
"title": pl.get("title"),
"type": pl.get("type"),
"retriever_weight": pl.get("retriever_weight"),
"chunk_profile": pl.get("chunk_profile"),
})
print(json.dumps({"notes_sample": sample}, ensure_ascii=False))
# Pick a note to fetch chunks
target_note_id = args.note_id or None
if not target_note_id:
for s in sample:
if s.get("note_id"):
target_note_id = s["note_id"]
break
if not target_note_id:
print(json.dumps({"warn": "no note_id available to fetch chunks"}))
return
# Chunks for target note
try:
# pull a few chunks and filter client-side by note_id
pts, _ = _scroll(client, names["chunks"], limit=50, with_payload=True)
except Exception as e:
print(json.dumps({"error": f"chunks scroll failed: {type(e).__name__}: {e}", "collection": names["chunks"]}))
return
chunks = [{
"id": str(p.id),
"chunk_id": (p.payload or {}).get("chunk_id"),
"note_id": (p.payload or {}).get("note_id"),
"type": (p.payload or {}).get("type"),
"retriever_weight": (p.payload or {}).get("retriever_weight"),
"window_len": len((p.payload or {}).get("window") or ""),
"text_len": len((p.payload or {}).get("text") or ""),
} for p in pts if (p.payload or {}).get("note_id") == target_note_id][:5]
print(json.dumps({"target_note_id": target_note_id, "chunk_samples": chunks}, ensure_ascii=False))
if __name__ == "__main__":
main()
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
dump_edges_payload_sample.py
-----------------------------------
Zeigt einige Beispiel-Payloads aus der {prefix}_edges Collection,
um Feldnamen (edge_type, src_note_id, src_id, etc.) zu verifizieren.
Nutzung:
python3 tests/dump_edges_payload_sample.py --prefix mindnet --n 10
"""
import os
import sys
import json
from pathlib import Path
try:
from qdrant_client import QdrantClient
except Exception as e:
print(json.dumps({"error": f"qdrant-client not installed: {e}"}))
sys.exit(2)
def main(argv=None):
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet")
ap.add_argument("--n", type=int, default=10)
args = ap.parse_args(argv)
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
api_key = os.environ.get("QDRANT_API_KEY") or None
edges = f"{args.prefix}_edges"
client = QdrantClient(url=url, api_key=api_key, timeout=30)
left = args.n
offset = None
while left > 0:
limit = min(left, 256)
pts, offset = client.scroll(
collection_name=edges,
scroll_filter=None,
limit=limit,
offset=offset,
with_payload=True,
with_vectors=False,
)
if not pts:
break
for p in pts:
print(json.dumps({"point_id": str(p.id), "payload_keys": list((p.payload or {}).keys()), "payload": p.payload}, ensure_ascii=False))
left -= len(pts)
if __name__ == "__main__":
main()
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os, tempfile, textwrap
from app.core.type_registry import load_type_registry, resolve_note_type, get_type_config, effective_chunk_profile, profile_overlap
def test_defaults_when_missing():
# ensure default kicks in if file does not exist
reg = load_type_registry(os.path.join("config", "does_not_exist.yaml"))
assert reg.get("_using_defaults", False) is True
t = resolve_note_type(None, reg)
assert t == "concept"
cfg = get_type_config(t, reg)
assert cfg.get("chunk_profile") in ("medium", "long", "short")
def test_custom_file():
with tempfile.TemporaryDirectory() as td:
p = os.path.join(td, "types.yaml")
with open(p, "w", encoding="utf-8") as f:
f.write(textwrap.dedent('''
version: 1.0
types:
concept:
chunk_profile: medium
edge_defaults: ["references"]
retriever_weight: 1.0
task:
chunk_profile: short
edge_defaults: ["depends_on"]
retriever_weight: 0.8
'''))
reg = load_type_registry(p)
assert resolve_note_type("task", reg) == "task"
assert effective_chunk_profile("task", reg) == "short"
assert profile_overlap("short")[0] < profile_overlap("long")[0]