Compare commits
461
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4f171e5245 | ||
|
|
d48b671314 | ||
|
|
b9d73af062 | ||
|
|
21904e62ee | ||
|
|
f9af30b195 | ||
|
|
531e3790b3 | ||
|
|
7745b71832 | ||
|
|
b39663408c | ||
|
|
9c4696e9d5 | ||
|
|
cf5e53f341 | ||
|
|
e512f768ff | ||
|
|
c3ef65d069 | ||
|
|
90f0be6baf | ||
|
|
eba51eba0f | ||
|
|
fbaa07ba88 | ||
|
|
d0ecc5316f | ||
|
|
9ee39277ae | ||
|
|
9d3fd21c88 | ||
|
|
d7217672a3 | ||
|
|
e39b685b1e | ||
|
|
5b9399361d | ||
|
|
7631b93900 | ||
|
|
f98eb8fc16 | ||
|
|
ab60106fb9 | ||
|
|
caab41a760 | ||
|
|
78d8ab11ed | ||
|
|
d49da1b5fe | ||
|
|
27497fd6be | ||
|
|
466ce3ca82 | ||
|
|
8b5af5ec65 | ||
|
|
34cbc0c0eb | ||
|
|
52ff4bbff5 | ||
|
|
9e7104e362 | ||
|
|
882ae5368f | ||
|
|
12a424ce23 | ||
|
|
c3230761e8 | ||
|
|
f5104008c2 | ||
|
|
013e8c6d0c | ||
|
|
603bba533b | ||
|
|
e202a8d88b | ||
|
|
c56a31c6bc | ||
|
|
b73b18bf9a | ||
|
|
0f351252b2 | ||
|
|
d7d7411fa0 | ||
|
|
45af7f074c | ||
|
|
70ab118612 | ||
|
|
ba0cdea9d0 | ||
|
|
09500ff764 | ||
|
|
b29ce4a0a6 | ||
|
|
311e598b69 | ||
|
|
413dca770c | ||
|
|
9d9239b11e | ||
|
|
a571c41c61 | ||
|
|
ba1fab9dcf | ||
|
|
5dc9c13a71 | ||
|
|
70821bae5a | ||
|
|
486abf82dd | ||
|
|
19afe3af33 | ||
|
|
113042b7f4 | ||
|
|
f8eebd91cb | ||
|
|
00297fc354 | ||
|
|
5baac082f9 | ||
|
|
611b0aa54e | ||
|
|
110f0dbcf5 | ||
|
|
1688cf0a1e | ||
|
|
13b7c8858a | ||
|
|
b8b6f243d1 | ||
|
|
eaa6e805ce | ||
|
|
05dbb21c58 | ||
|
|
f1f4358dee | ||
|
|
3374c41f07 | ||
|
|
95affe87e5 | ||
|
|
5e86398fcd | ||
|
|
844f0cbe42 | ||
|
|
4d5d7847e5 | ||
|
|
06795feed6 | ||
|
|
a4548a7ee1 | ||
|
|
42216865e2 | ||
|
|
e48bdb2401 | ||
|
|
c67eb4b2f3 | ||
|
|
fafa74b0b8 | ||
|
|
049a39aba3 | ||
|
|
cd26430a61 | ||
|
|
aab010ff17 | ||
|
|
9a7bc32051 | ||
|
|
bfdd8ac6be | ||
|
|
018669fb87 | ||
|
|
fb2de78d62 | ||
|
|
7e8d6c1954 | ||
|
|
d21230c6c3 | ||
|
|
4e98d5bfa1 | ||
|
|
8998269377 | ||
|
|
6e9a227ad2 | ||
|
|
4e7506da5a | ||
|
|
06ad93c77c | ||
|
|
0bb7b34259 | ||
|
|
859c17b49d | ||
|
|
46b26c9624 | ||
|
|
fd215c18e4 | ||
|
|
efa3afc7dc | ||
|
|
e03dd66051 | ||
|
|
3476fe5fae | ||
|
|
3e08c8347e | ||
|
|
c691123d2d | ||
|
|
a4b81535e8 | ||
|
|
a898cf4b65 | ||
|
|
10009a0a43 | ||
|
|
4dfe47ff99 | ||
|
|
bd997e61d6 | ||
|
|
23d0670126 | ||
|
|
12c600edbe | ||
|
|
e4115e705e | ||
|
|
0cbc4a2d72 | ||
|
|
cea6d35729 | ||
|
|
4c56918d8a | ||
|
|
f17263417e | ||
|
|
6f0b463489 | ||
|
|
28c01caa1a | ||
|
|
95b59e9b0a | ||
|
|
bc967f1f6e | ||
|
|
4228b8a74f | ||
|
|
300086fc83 | ||
|
|
688eb262a9 | ||
|
|
bfb6f5dede | ||
|
|
88c0fbe56b | ||
|
|
3d74eff224 | ||
|
|
22d08afe2d | ||
|
|
a97f757e34 | ||
|
|
bbc8f13944 | ||
|
|
f18a40d76c | ||
|
|
c7644a36aa | ||
|
|
f394f31c12 | ||
|
|
0513673c2a | ||
|
|
a73542a391 | ||
|
|
c465797654 | ||
|
|
7ce3f982f2 | ||
|
|
6311456917 | ||
|
|
3c06559c5d | ||
|
|
5c9f2e8c83 | ||
|
|
62902faaca | ||
|
|
d22fd77b22 | ||
|
|
2f86f493d6 | ||
|
|
eafa0fad85 | ||
|
|
2e7c497b69 | ||
|
|
79dcfdcf9a | ||
|
|
487e28a904 | ||
|
|
c492f97a67 | ||
|
|
ec2ff74514 | ||
|
|
c501f8d6e6 | ||
|
|
b4287cbfda | ||
|
|
26724a1db1 | ||
|
|
d6368ee709 | ||
|
|
ea211a5c0b | ||
|
|
9e8b433c95 | ||
|
|
2a1c62aeed | ||
|
|
948d6f4b47 | ||
|
|
725271a7da | ||
|
|
4ea62e6886 | ||
|
|
c01cf3b078 | ||
|
|
c67777a6b9 | ||
|
|
e5e5e7560c | ||
|
|
6e1cd5fbbc | ||
|
|
8bd8f7c0bb | ||
|
|
1c42cd1f78 | ||
|
|
be090a43f2 | ||
|
|
2c81fa0cb5 | ||
|
|
c3815afcd4 | ||
|
|
8800160564 | ||
|
|
bdbc4a1bf7 | ||
|
|
597090bc45 | ||
|
|
af36c410b4 | ||
|
|
4dcd606c10 | ||
|
|
bbd5a7fa48 | ||
|
|
6dc37ccb66 | ||
|
|
b84906283e | ||
|
|
af0dcdf76a | ||
|
|
f1e1cde597 | ||
|
|
a7c5630e5b | ||
|
|
82151d6bc3 | ||
|
|
c93e7ad598 | ||
|
|
290b271cf6 | ||
|
|
a686bdbeaf | ||
|
|
2b84c62875 | ||
|
|
2de786fc64 | ||
|
|
2ddf034983 | ||
|
|
b2043f4f84 | ||
|
|
52eae52061 | ||
|
|
e451ea64ae | ||
|
|
eb6e4028ff | ||
|
|
ea45493941 | ||
|
|
da70f0e00c | ||
|
|
8bf3707511 | ||
|
|
804eeef858 | ||
|
|
36658f12fe | ||
|
|
f0e675ffd8 | ||
|
|
90911c1074 | ||
|
|
1729e0b40d | ||
|
|
70b3169e7f | ||
|
|
315a1c0945 | ||
|
|
b9316c693d | ||
|
|
820e52d869 | ||
|
|
444e8e4206 | ||
|
|
e6f2c8b4e8 | ||
|
|
f6b7652219 | ||
|
|
3cd00ab5cf | ||
|
|
0feae50d90 | ||
|
|
d79c55fc64 | ||
|
|
98a6f5c480 | ||
|
|
0fa4aa7280 | ||
|
|
3b192e2eed | ||
|
|
2794d26181 | ||
|
|
7b56f696d6 | ||
|
|
ead1b8c1bc | ||
|
|
b8126e00c4 | ||
|
|
62f8f4b313 | ||
|
|
21924541dc | ||
|
|
197242ca42 | ||
|
|
7922734dce | ||
|
|
afd7675425 | ||
|
|
8b3b343645 | ||
|
|
6713f342c3 | ||
|
|
79bb8a143c | ||
|
|
d03a181536 | ||
|
|
8e5304c156 | ||
|
|
c487db6534 | ||
|
|
0ca54f429e | ||
|
|
6c62f64784 | ||
|
|
3282f85007 | ||
|
|
b186569750 | ||
|
|
7cde2462ec | ||
|
|
c0b96f7c05 | ||
|
|
3a88caac69 | ||
|
|
31457e9240 | ||
|
|
307335a733 | ||
|
|
4eb5e34ea7 | ||
|
|
f4be219790 | ||
|
|
53591b6f27 | ||
|
|
a14d0bb7cb | ||
|
|
c2802e7cb3 | ||
|
|
c7f2815f1d | ||
|
|
afc90fe992 | ||
|
|
2f9ce824a0 | ||
|
|
a39b2a6950 | ||
|
|
70fa95966c | ||
|
|
10479014d5 | ||
|
|
148e22aa7d | ||
|
|
a9fe6196f7 | ||
|
|
ae20665a66 | ||
|
|
ccbf078463 | ||
|
|
cd1e46784f | ||
|
|
b49d97b8d7 | ||
|
|
3d44de9d87 | ||
|
|
987b3c1770 | ||
|
|
25bc6544c4 | ||
|
|
b241ae753b | ||
|
|
2041771b14 | ||
|
|
b5958a9f63 | ||
|
|
c05dbd4b3b | ||
|
|
44e468fc21 | ||
|
|
5f358d329b | ||
|
|
59fe8154f2 | ||
|
|
d5b2e9bb8f | ||
|
|
3a3907e1b6 | ||
|
|
f66cdc70b2 | ||
|
|
e299a497a7 | ||
|
|
eefb29d2ad | ||
|
|
668c485037 | ||
|
|
c83d5bb970 | ||
|
|
8638b3c95b | ||
|
|
c9b55c16f7 | ||
|
|
a3b9d53d06 | ||
|
|
215915cee2 | ||
|
|
df9271ed81 | ||
|
|
97e73a0fa0 | ||
|
|
e2ebe98e7f | ||
|
|
345e30de71 | ||
|
|
0084160e12 | ||
|
|
7280c111f0 | ||
|
|
763de11d2b | ||
|
|
9c0de3eeb2 | ||
|
|
9c81e9fbf1 | ||
|
|
40603f1652 | ||
|
|
4a34da31ad | ||
|
|
6e394faab1 | ||
|
|
25d0895cac | ||
|
|
f5f93cfb12 | ||
|
|
cf1ebe9d6f | ||
|
|
dd0d4ff8b1 | ||
|
|
98103f3270 | ||
|
|
a740ebb42e | ||
|
|
7c79f820cb | ||
|
|
f3b2326304 | ||
|
|
4eded247c5 | ||
|
|
3ffb857181 | ||
|
|
ec92906399 | ||
|
|
87671640ea | ||
|
|
c725e984bb | ||
|
|
8f05b57744 | ||
|
|
120b0147f2 | ||
|
|
ae695b4b15 | ||
|
|
89fee685c3 | ||
|
|
23eb9a21c8 | ||
|
|
a009efe91c | ||
|
|
dbd3111adf | ||
|
|
2b35e0c07a | ||
|
|
ba3354df03 | ||
|
|
7c8c19f416 | ||
|
|
35f83241cb | ||
|
|
5297f9b47e | ||
|
|
cbc50ec4eb | ||
|
|
198cc291e2 | ||
|
|
cb8c3a6ea6 | ||
|
|
66f597bbba | ||
|
|
fe237c8b4b | ||
|
|
ec7b08a625 | ||
|
|
9dd728579d | ||
|
|
0a6ca00f64 | ||
|
|
6e856ce05f | ||
|
|
1b8b78cfe9 | ||
|
|
49508d9f87 | ||
|
|
63a660223e | ||
|
|
2cbc2e2f43 | ||
|
|
eb2ad44755 | ||
|
|
c3979142a6 | ||
|
|
1a30de6bfc | ||
|
|
8c0df47966 | ||
|
|
4c177378ae | ||
|
|
a2db175819 | ||
|
|
2d0f050524 | ||
|
|
2d2b78edb3 | ||
|
|
627fa80f3e | ||
|
|
96cbbb093e | ||
|
|
6551308f14 | ||
|
|
9ccc2badd4 | ||
|
|
07a83bdeb7 | ||
|
|
224dfb6588 | ||
|
|
d8d6c68ac2 | ||
|
|
3e369e5464 | ||
|
|
abaaba97f9 | ||
|
|
bd49467d26 | ||
|
|
b44524e1a9 | ||
|
|
0319667f8e | ||
|
|
4c3ad1f83c | ||
|
|
a7a845187a | ||
|
|
e14a9a11d2 | ||
|
|
d7ce757d65 | ||
|
|
6f5447c316 | ||
|
|
727b67811c | ||
|
|
a85fb68015 | ||
|
|
2dfe7328cc | ||
|
|
cfc0a0ca8f | ||
|
|
2b1c7fe20d | ||
|
|
d5e1d9c09b | ||
|
|
554540451e | ||
|
|
c10134c22b | ||
|
|
9175b3a045 | ||
|
|
0ecae44888 | ||
|
|
a9c2a10fb5 | ||
|
|
e11094e5fa | ||
|
|
38913dc3e1 | ||
|
|
dd2b7932a2 | ||
|
|
2051e187ba | ||
|
|
ca817a8cc4 | ||
|
|
2d815f5902 | ||
|
|
0abdc647c9 | ||
|
|
fded2db73e | ||
|
|
2274c9faf5 | ||
|
|
280b236490 | ||
|
|
ef2c144cdd | ||
|
|
634cf0ff1b | ||
|
|
2dc3d5c169 | ||
|
|
fcad1fdfaa | ||
|
|
5c6cdfb5e1 | ||
|
|
1d6ba761a8 | ||
|
|
893ba6c8f4 | ||
|
|
a835eff14b | ||
|
|
029b4d8a6e | ||
|
|
0053b45691 | ||
|
|
f46812d1d2 | ||
|
|
8495df961b | ||
|
|
abc8be1077 | ||
|
|
84baf27f7e | ||
|
|
1a08d904f9 | ||
|
|
31ca9a95d7 | ||
|
|
ade91b26c3 | ||
|
|
d797957abc | ||
|
|
104ddddb96 | ||
|
|
c8c4c7fa00 | ||
|
|
3111a26229 | ||
|
|
26ef257e06 | ||
|
|
6864ea7b53 | ||
|
|
c2ecc15d89 | ||
|
|
9bbcc0952b | ||
|
|
179ec4d38b | ||
|
|
02a61cdf0d | ||
|
|
6a84b4e5aa | ||
|
|
497a9b9d58 | ||
|
|
c5972be618 | ||
|
|
43e77c4c6b | ||
|
|
d66985e597 | ||
|
|
ff4bed99c9 | ||
|
|
2b4c1b83bd | ||
|
|
f85aa8f604 | ||
|
|
e5ca3c4177 | ||
|
|
0f2f8208e3 | ||
|
|
ba17516814 | ||
|
|
c0979b69e2 | ||
|
|
4b5c33bb90 | ||
|
|
1ea4d53061 | ||
|
|
17b548e9b0 | ||
|
|
5755eb1f08 | ||
|
|
e2f15324d7 | ||
|
|
b0b0802a56 | ||
|
|
e14cc8b758 | ||
|
|
55c6a5058c | ||
|
|
52d5d01337 | ||
|
|
93ceff7f17 | ||
|
|
0a891cfd95 | ||
|
|
66adc6a354 | ||
|
|
e99f1c2ba0 | ||
|
|
f52a9face8 | ||
|
|
4fcf22131c | ||
|
|
d27324f8dd | ||
|
|
dd86a15555 | ||
|
|
28d173b8fd | ||
|
|
b4b7ea76ab | ||
|
|
6ea452cc3f | ||
|
|
6e0e965883 | ||
|
|
6b4b3ce2ba | ||
|
|
18cdbab885 | ||
|
|
ebca9b4caf | ||
|
|
6945ef8ddc | ||
|
|
f20bf03ee1 | ||
|
|
7cda15553d | ||
|
|
3c67fd5f9b | ||
|
|
0150931df4 | ||
|
|
5ff1b06009 | ||
|
|
5279bcae18 | ||
|
|
daa58e6b27 | ||
|
|
589d021744 | ||
|
|
e58b9c8ada | ||
|
|
507c28e006 | ||
|
|
2d74fe4057 | ||
|
|
e8312268d2 | ||
|
|
f4cd1e51c0 | ||
|
|
60e707e7df | ||
|
|
0d6eb4dac4 | ||
|
|
80d8f986bd | ||
|
|
9ee3bb9007 | ||
|
|
8a07010704 | ||
|
|
314fc8ac94 | ||
|
|
b46e5461a7 | ||
|
|
ada7c1797d | ||
|
|
54197995f5 | ||
|
|
13e510dd9c | ||
|
|
5c937fbffe | ||
|
|
58bce0ea19 | ||
|
|
3e888cc669 | ||
|
|
78446c74d3 | ||
|
|
51f08f193c | ||
|
|
4872374a6e |
No files matched your search
+31
-12
@@ -1,22 +1,36 @@
|
|||||||
# .gitea/workflows/deploy.yml
|
# .gitea/workflows/deploy.yml
|
||||||
name: Deploy mindnet to llm-node
|
name: Deploy mindnet to llm-node
|
||||||
on: { push: { branches: [ "main" ] } }
|
|
||||||
# on: {}
|
|
||||||
|
|
||||||
concurrency: { group: deploy-mindnet, cancel-in-progress: false }
|
on:
|
||||||
|
push:
|
||||||
|
branches: ["main"]
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: deploy-mindnet
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
deploy:
|
deploy:
|
||||||
runs-on: linux_host
|
runs-on: linux_host
|
||||||
env:
|
env:
|
||||||
DEPLOY_DIRS: "app scripts schemas docker requirements.txt README.md"
|
TARGET_BASE: /home/llmadmin/mindnet
|
||||||
TARGET_BASE: "/home/llmadmin/mindnet"
|
DEPLOY_DIRS: "app scripts schemas docker tests config requirements.txt README.md"
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: https://github.com/actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
- name: Ensure target base
|
|
||||||
|
- name: Ensure target base exists
|
||||||
run: install -d "$TARGET_BASE"
|
run: install -d "$TARGET_BASE"
|
||||||
- name: Deploy whitelisted
|
|
||||||
|
- name: Stop API (graceful)
|
||||||
|
continue-on-error: true
|
||||||
|
run: |
|
||||||
|
if systemctl --user list-unit-files | grep -q '^mindnet-api.service'; then
|
||||||
|
systemctl --user stop mindnet-api.service || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Deploy whitelisted (rsync)
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
IFS=' ' read -r -a DIRS <<< "$DEPLOY_DIRS"
|
IFS=' ' read -r -a DIRS <<< "$DEPLOY_DIRS"
|
||||||
@@ -24,22 +38,27 @@ jobs:
|
|||||||
if [ -e "$d" ]; then
|
if [ -e "$d" ]; then
|
||||||
if [ -d "$d" ]; then
|
if [ -d "$d" ]; then
|
||||||
install -d "$TARGET_BASE/$d"
|
install -d "$TARGET_BASE/$d"
|
||||||
rsync -a --delete --exclude='.git' \
|
rsync -a --delete \
|
||||||
--exclude='.env' --exclude='.env.*' --exclude='**/.env*' \
|
--exclude='.git' \
|
||||||
|
--exclude='.env*' \
|
||||||
|
--exclude='.venv' \
|
||||||
|
--exclude='hf_cache' --exclude='hf_cache/**' \
|
||||||
"$d"/ "$TARGET_BASE/$d"/
|
"$d"/ "$TARGET_BASE/$d"/
|
||||||
else
|
else
|
||||||
rsync -a "$d" "$TARGET_BASE/$d"
|
rsync -a "$d" "$TARGET_BASE/$d"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
- name: Python venv (idempotent)
|
|
||||||
|
- name: Python venv & requirements
|
||||||
run: |
|
run: |
|
||||||
cd "$TARGET_BASE"
|
cd "$TARGET_BASE"
|
||||||
[ -d .venv ] || python3 -m venv .venv
|
[ -d .venv ] || python3 -m venv .venv
|
||||||
source .venv/bin/activate
|
source .venv/bin/activate
|
||||||
pip install --upgrade pip
|
pip install --upgrade pip
|
||||||
[ -f requirements.txt ] && pip install -r requirements.txt || true
|
[ -f requirements.txt ] && pip install -r requirements.txt || true
|
||||||
- name: Optional — restart mindnet-api
|
|
||||||
|
- name: Start/Restart mindnet-api
|
||||||
continue-on-error: true
|
continue-on-error: true
|
||||||
run: |
|
run: |
|
||||||
if systemctl --user list-unit-files | grep -q '^mindnet-api.service'; then
|
if systemctl --user list-unit-files | grep -q '^mindnet-api.service'; then
|
||||||
|
|||||||
Vendored
+3
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"CodeGPT.apiKey": "Google AI Studio"
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Architecture Snapshot v2.2.1
|
||||||
|
**Stand:** Nach Abschluss von WP-04c
|
||||||
|
**Kontext:** Wichtig für WP-05 Entwicklung.
|
||||||
|
|
||||||
|
## 1. Neue Komponenten (Seit v2.2.0)
|
||||||
|
|
||||||
|
### A. Feedback Service (`app/services/feedback_service.py`)
|
||||||
|
- **Zweck:** Logging von Trainingsdaten für späteres Self-Tuning (WP-08).
|
||||||
|
- **Storage:** Lokale JSONL-Dateien in `data/logs/` (Append-Only).
|
||||||
|
- `search_history.jsonl`: Query + Snapshot der Ergebnisse (Scores).
|
||||||
|
- `feedback.jsonl`: User-Rating zu spezifischer `node_id`.
|
||||||
|
|
||||||
|
### B. Explanation Layer (`app/core/retriever.py`)
|
||||||
|
- **Logik:** Der Retriever berechnet nicht nur Scores, sondern generiert `Explanation`-Objekte.
|
||||||
|
- **Graph:** `Subgraph` (in `graph_adapter.py`) führt jetzt auch `reverse_adj` (Incoming Edges), um zu erklären, warum ein Knoten wichtig ist ("Referenziert von...").
|
||||||
|
|
||||||
|
### C. DTOs (`app/models/dto.py`)
|
||||||
|
Das Datenmodell wurde massiv erweitert. Wichtige Klassen für WP-05:
|
||||||
|
- `QueryResponse`: Enthält jetzt `query_id` (UUID).
|
||||||
|
- `QueryHit`: Enthält optional `explanation` (Typ `Explanation`).
|
||||||
|
- `FeedbackRequest`: Für den Feedback-Loop.
|
||||||
|
|
||||||
|
## 2. Implikationen für WP-05 (Chat)
|
||||||
|
|
||||||
|
1. **Logging:** Auch der neue `/chat` Endpoint sollte idealerweise die `query_id` loggen oder nutzen, um Konsistenz zu wahren.
|
||||||
|
2. **DTO-Nutzung:** Der Chat-Service wird intern den Retriever aufrufen. Er muss mit den `QueryHit`-Objekten arbeiten, um den Kontext für das LLM zu bauen.
|
||||||
|
3. **Config:** Die Persönlichkeit wird in `config/prompts.yaml` definiert (Late Binding), nicht im Python-Code hardcodiert.
|
||||||
@@ -0,0 +1,465 @@
|
|||||||
|
# mindnet v2.2 – Dokumentationsrahmenplan
|
||||||
|
**Datei:** `docs/mindnet_docs_master_plan_v2.2.md`
|
||||||
|
**Stand:** 2025-12-06
|
||||||
|
**Bezug:** Programmplan mindnet v2.2 :contentReference[oaicite:0]{index=0}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zweck dieses Rahmenplans
|
||||||
|
|
||||||
|
Dieser Rahmenplan definiert:
|
||||||
|
|
||||||
|
- die **Zieldokumente** der mindnet v2.2 Dokumentation,
|
||||||
|
- deren **Dateinamen, Zielgruppen, Inhalte und Quellen**,
|
||||||
|
- die **Zuordnung** zu Workpackages (WP) und Release-Tags,
|
||||||
|
- die **Regeln zum Informations-Erhalt** (keine inhaltlichen Verluste),
|
||||||
|
- die **empfohlene Bearbeitungsreihenfolge**.
|
||||||
|
|
||||||
|
Alle hier beschriebenen Zieldokumente werden als **eigene Markdown-Dateien** gepflegt und sind Teil von WP-14 _Review / Refactoring / Dokumentation_ (Phase E). :contentReference[oaicite:1]{index=1}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Prinzipien zur Sicherung der Informationen
|
||||||
|
|
||||||
|
Um zu verhindern, dass Inhalte oder Ergebnisse verloren gehen, gelten folgende Prinzipien:
|
||||||
|
|
||||||
|
1. **Programmplan v2.2 als Masterreferenz**
|
||||||
|
- Programmauftrag, Vision, Phasenmodell und WP-Status werden **nicht verändert**, sondern nur referenziert.
|
||||||
|
- `Programmplan_V2.2.md` bleibt das führende Dokument für Ziele, WPs, Ampelstatus. :contentReference[oaicite:2]{index=2}
|
||||||
|
|
||||||
|
2. **Keine unkontrollierte Löschung von Inhalten**
|
||||||
|
- Bestehende Dokumente (`knowledge_design.md`, `chunking_strategy.md`, `mindnet_technical_architecture.md`, `Handbuch.md`, `TYPE_REGISTRY_MANUAL.md`, `docs_mindnet_retriever.md`, `wp04_retriever_scoring.md`) werden **nicht gelöscht**, sondern:
|
||||||
|
- vollständig archiviert,
|
||||||
|
- in die neuen Zieldokumente **überführt**,
|
||||||
|
- oder **explizit** in den Anhängen (I) als „legacy / historisch“ verlinkt.
|
||||||
|
|
||||||
|
3. **Explizite Quellenzuordnung („Building Blocks“)**
|
||||||
|
- Für jede Sektion eines Zieldokuments wird angegeben, aus **welchen bestehenden Dateien und Abschnitten** der Inhalt stammt.
|
||||||
|
- Neue Inhalte (z. B. User Guide H) werden klar als „neu, v2.2“ gekennzeichnet.
|
||||||
|
|
||||||
|
4. **Ist-Stand vs. Roadmap trennen**
|
||||||
|
- Implementierter Stand v2.2: WP01–WP04a (🟢) mit Importer, Chunking, Edge-Pipeline, Typ-Registry und hybrider Retriever.
|
||||||
|
- Geplante Features (Self-Healing, Virtual Schema Layer, Explanation Layer, Feedback-Logging, MCP-Agenten) werden als **Roadmap-Funktionen** kenntlich gemacht (Phase B–D, WPs 05–13).
|
||||||
|
|
||||||
|
5. **Versionierbarkeit**
|
||||||
|
- Alle Zieldokumente tragen im Kopf eine **Version** und **Stand-Datum** sowie optional einen Verweis auf den letzten Gitea-Tag (z. B. `mindnet_v2.2_docs_initial`, `WP04a-Retriever_final`). :contentReference[oaicite:6]{index=6}
|
||||||
|
- Änderungen werden im Abschnitt „Changelog“ der jeweiligen Datei notiert.
|
||||||
|
|
||||||
|
6. **Keine funktionalen Annahmen ohne Verifikation**
|
||||||
|
- Wo das Verhalten nicht eindeutig aus Code-/Handbuch-/WP-Dokumenten hervorgeht, wird der Punkt zunächst als _offene Frage_ markiert und nicht als Tatsache dokumentiert.
|
||||||
|
- Verifizierte Funktionen (z. B. `import_markdown`, Edge-Schema, `types.yaml`, `/query`, Scoring-Formel) werden exakt gemäß der bestehenden Dokumente beschrieben.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Zieldokumente der mindnet v2.2 Dokumentation
|
||||||
|
|
||||||
|
### 3.1 Übersichtstabelle
|
||||||
|
|
||||||
|
| ID | Zieldatei | Rolle/Zielgruppe | Hauptquellen (Building Blocks) |
|
||||||
|
|----|------------------------------------------|---------------------------------------------|--------------------------------|
|
||||||
|
| A | `mindnet_overview_v2.2.md` | Einstieg für alle Rollen | Programmplan v2.2, Knowledge Design |
|
||||||
|
| B | `mindnet_functional_architecture_v2.2.md`| Architekt:innen, Mindmaster | Knowledge Design, Functional Architecture, Programmplan |
|
||||||
|
| C | `mindnet_technical_architecture_v2.2.md` | Entwickler:innen, Tech-Architekt:innen | mindnet_technical_architecture, chunking_strategy, Handbuch |
|
||||||
|
| D | `mindnet_knowledge_design_manual_v2.2.md`| Vault-Autor:innen, Mindmaster, Analyst:innen| knowledge_design, TYPE_REGISTRY_MANUAL, Functional Architecture |
|
||||||
|
| E | `mindnet_pipeline_playbook_v2.2.md` | Dev/Ops, Tech-Lead | mindnet_v2_implementation_playbook, Handbuch, chunking_strategy, docs_mindnet_retriever, wp04_retriever_scoring |
|
||||||
|
| F | `mindnet_developer_guide_v2.2.md` | Entwickler:innen (Code, API, Tests) | Technical Architecture, Handbuch, docs_mindnet_retriever |
|
||||||
|
| G | `mindnet_admin_guide_v2.2.md` | Admins, Betreiber | Handbuch, Implementation Playbook, Programmplan (Governance) |
|
||||||
|
| H | `mindnet_user_guide_v2.2.md` | Endnutzer:innen, Agent-User | Knowledge Design, Programmplan (Use Cases), spätere MCP-WPs |
|
||||||
|
| I | `mindnet_appendices_v2.2.md` | Alle Rollen (Referenzen, Tabellen) | TYPE_REGISTRY_MANUAL, chunking_strategy, WP-Übersicht, Edge-Referenz |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Detailpläne pro Zieldokument (Inhaltsverzeichnis & Building Blocks)
|
||||||
|
|
||||||
|
### A) `mindnet_overview_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Einstieg in mindnet für alle Rollen (Owner, Mindmaster, Developer, Admin, Agenten-Entwickler).
|
||||||
|
- Auf 3–5 Seiten: _Was ist mindnet? Wie ist es gedacht? Wie lese ich die Dokumentation?_ :contentReference[oaicite:17]{index=17}
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Einführung**
|
||||||
|
1. Programmauftrag & Vision (v2.2)
|
||||||
|
2. Was mindnet ist – und was nicht
|
||||||
|
2. **Drei Ebenen von mindnet**
|
||||||
|
1. Content-Ebene (Vault, Notes, Chunks)
|
||||||
|
2. Semantische Ebene (Edges, Typen, Retriever)
|
||||||
|
3. Identitäts- / Persönlichkeits-Ebene (Werte, Entscheidungen, Profile – Roadmap)
|
||||||
|
3. **End-to-End-Überblick**
|
||||||
|
1. Datenfluss: Vault → Import → Qdrant → Retriever → Antwort
|
||||||
|
2. WP-Phasenmodell (A–E) und aktueller Stand (WP01–04a grün)
|
||||||
|
4. **Rollen & Perspektiven**
|
||||||
|
1. Mindmaster / Owner
|
||||||
|
2. Autor:in (Vault-Notes)
|
||||||
|
3. Developer / Architect
|
||||||
|
4. Administrator:in
|
||||||
|
5. Agenten & LLMs (MCP-Perspektive)
|
||||||
|
5. **Leserführung durch die Dokumentation**
|
||||||
|
1. Welche Datei für welche Rolle?
|
||||||
|
2. Wie mit Ist-Stand vs. Roadmap umgehen?
|
||||||
|
6. **Glossar zentraler Begriffe**
|
||||||
|
- mindnet, Vault, Note, Chunk, Edge, Type, Identity, Self-Healing, Self-Tuning, Explanation Layer, Virtual Schema Layer, etc.
|
||||||
|
|
||||||
|
#### Building Blocks (Quellen-Mapping)
|
||||||
|
|
||||||
|
- Programmauftrag & Vision → `Programmplan_V2.2.md`, Kapitel 1–2. :contentReference[oaicite:18]{index=18}
|
||||||
|
- Phasenmodell & WPs → `Programmplan_V2.1.md`/v2.2, Kapitel 5–9 (angepasst auf v2.2-Stand).
|
||||||
|
- High-Level-Architektur → `knowledge_design.md`, Kapitel 1–3 (Ziel, Zielarchitektur, Collections). :contentReference[oaicite:20]{index=20}
|
||||||
|
- Rollenmodell → Ableitung aus Knowledge Design + Programmplan (Owner, Agenten, MCP-WPs).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B) `mindnet_functional_architecture_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Fachliche Architektur von mindnet (Konzeptmodell).
|
||||||
|
- Für Architekt:innen, Mindmaster, Knowledge-Designer.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Ziel & Einordnung**
|
||||||
|
1. Bezug zu Programmauftrag & WPs (WP01–WP07, WP10–12)
|
||||||
|
2. **Kernkonzepte**
|
||||||
|
1. Notes & Note-Typen (concept, experience, project, decision, value, principle, person, event, …)
|
||||||
|
2. Chunks & Sections
|
||||||
|
3. Edges (Struktur, explizit, inline, Default-Kanten)
|
||||||
|
4. Typen-Registry & Virtual Schema Layer (fachliche Sicht, Roadmap)
|
||||||
|
3. **Graph-Modell**
|
||||||
|
1. Knotenarten (Notes, Chunks, Identity Nodes, People)
|
||||||
|
2. Kantenarten (belongs_to, references, similar_to, depends_on, related_to, next/prev, …)
|
||||||
|
3. Scope & Provenienz (chunk vs. note, rule_id, confidence)
|
||||||
|
4. **End-to-End fachlicher Datenfluss**
|
||||||
|
1. Erfassung von Wissen (Note erstellen)
|
||||||
|
2. Import & Graphaufbau
|
||||||
|
3. Retrieval & Begründungspfade
|
||||||
|
4. Self-Healing & Retroactive Edges (WP06/07 – Roadmap)
|
||||||
|
5. **Szenarien & Use Cases**
|
||||||
|
1. Wissensspeicherung (z. B. Leitbild, Projekt, Erfahrung)
|
||||||
|
2. Entscheidungen & Why-Layer (WP04b – Roadmap)
|
||||||
|
3. Persönlichkeits-/Identitätsmodell (WP10–12 – Roadmap)
|
||||||
|
6. **Rollenmodell (fachliche Sicht)**
|
||||||
|
1. Mindmaster
|
||||||
|
2. Content-Autor:innen
|
||||||
|
3. Analyst:innen (Graph-Leser)
|
||||||
|
7. **Designprinzipien (fachlich)**
|
||||||
|
- Late Binding, Virtual Schema, Self-Healing, Minimalinvasives Schreiben etc.
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `knowledge_design.md` – Kapitel 1–4, 6–7 (Ziel, Architektur, Collections, Identitäten, Prinzipien). :contentReference[oaicite:28]{index=28}
|
||||||
|
- `mindnet_functional_architecture.md` – alle Kapitel (Kernkonzepte, Edgetypen, Provenienz, Rollen). :contentReference[oaicite:29]{index=29}
|
||||||
|
- `chunking_strategy.md` – Edges, Datenmodell, Graph-Sicht. :contentReference[oaicite:30]{index=30}
|
||||||
|
- Programmplan – WPs 01–07, 10–12 für Roadmap-Features.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### C) `mindnet_technical_architecture_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Technische Architektur des Systems: Collections, Payload-Felder, Services, Schemata.
|
||||||
|
- Für Entwickler:innen und Tech-Architekt:innen.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Ziel & Scope**
|
||||||
|
1. Abgrenzung zur funktionalen Architektur
|
||||||
|
2. Bezug zu WPs 02–04a, 05 (Schema, Importer, Retriever)
|
||||||
|
2. **Systemübersicht**
|
||||||
|
1. Komponentenübersicht (Importer, Qdrant, Retriever-API, Embedding-Service, n8n/MCP-Ausblick)
|
||||||
|
2. Ports, Dienste, Hauptverzeichnisse
|
||||||
|
3. **Datenmodell in Qdrant**
|
||||||
|
1. Notes (`mindnet_notes`)
|
||||||
|
2. Chunks (`mindnet_chunks`)
|
||||||
|
3. Edges (`mindnet_edges`)
|
||||||
|
4. Optionale Collections (`mindnet_people`, zukünftige)
|
||||||
|
4. **Typ-Registry & Konfiguration**
|
||||||
|
1. `config/types.yaml` – Aufbau, Defaults, Typauflösung (Ist-Stand: Frontmatter-Overrides deaktiviert)
|
||||||
|
2. `config/retriever.yaml` – Scoringgewichte, Edge-Typ-Gewichte :contentReference[oaicite:35]{index=35}
|
||||||
|
5. **Import-Pipeline (technische Sicht)**
|
||||||
|
1. Discovery & Parsing (`import_markdown.py`)
|
||||||
|
2. Hash-/Idempotenz-Mechanismus (ENV-Variablen, Baseline-Modi)
|
||||||
|
3. Chunking (`chunker.py`, `chunk_payload.py`)
|
||||||
|
4. Edge-Erzeugung (`derive_edges.py`)
|
||||||
|
5. Roundtrip & Tests (Export & Vault-Vergleich)
|
||||||
|
6. **Retriever-Architektur**
|
||||||
|
1. API (FastAPI-Endpunkt `/query`)
|
||||||
|
2. Semantik- und Hybridmodus
|
||||||
|
3. Subgraph & Graph-API (expand)
|
||||||
|
4. Scoring-Formel und Komponenten (semantik, edges, centrality) :contentReference[oaicite:37]{index=37}
|
||||||
|
7. **Offene technische Punkte / Roadmap**
|
||||||
|
1. Explanation Layer (WP04b)
|
||||||
|
2. Feedback-Logging (WP04c)
|
||||||
|
3. Virtual Schema Layer (WP05)
|
||||||
|
4. Self-Healing & Self-Tuning (WP06–08)
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `mindnet_technical_architecture.md` – Kerntext. :contentReference[oaicite:39]{index=39}
|
||||||
|
- `chunking_strategy.md` – Datenmodell & Chunk-Details. :contentReference[oaicite:40]{index=40}
|
||||||
|
- `Handbuch.md` – ENV, Skripte, Runbooks. :contentReference[oaicite:41]{index=41}
|
||||||
|
- `wp04_retriever_scoring.md` / `docs_mindnet_retriever.md` – Retriever-Details. :contentReference[oaicite:42]{index=42}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### D) `mindnet_knowledge_design_manual_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Konkrete Anleitung für das Schreiben und Strukturieren von Notes & Chunks im Vault.
|
||||||
|
- Für dich als Owner/Mindmaster und alle, die Notes erstellen.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Zweck & Scope**
|
||||||
|
1. Rolle des Vaults als „Source of Truth“
|
||||||
|
2. **Note-Struktur & Frontmatter**
|
||||||
|
1. Pflicht- und optionale Felder (id, title, type, tags, status, created/updated, …)
|
||||||
|
2. Empfehlungen für IDs, Pfade und Namenskonventionen
|
||||||
|
3. **Note-Typen & Typ-Registry**
|
||||||
|
1. Übersicht der Typen (concept, project, experience, decision, value, principle, person, event, …)
|
||||||
|
2. Typ-spezifische Einsatzempfehlungen
|
||||||
|
3. Zusammenspiel mit `types.yaml` (retriever_weight, chunk_profile, edge_defaults)
|
||||||
|
4. **Edges & Referenzen in Notes**
|
||||||
|
1. Wikilinks (`[[Note]]`) → `references`
|
||||||
|
2. Inline-Relationen (`[[rel:depends_on XYZ]]`, `similar_to`, `related_to`)
|
||||||
|
3. Callout-Edges (`> [!edge] …`)
|
||||||
|
4. Strukturkanten (Ordner, belongs_to)
|
||||||
|
5. **Best Practices**
|
||||||
|
1. Beispiele für gut modellierte Notes (Projekt, Entscheidung, Erfahrung)
|
||||||
|
2. Typische Anti-Patterns und wie man sie vermeidet
|
||||||
|
6. **Langfristige Stabilität**
|
||||||
|
1. Virtual Schema Layer – was bedeutet das für Note-Autoren? (Wenig Änderungen in bestehenden Notes, Logik in Configs) :contentReference[oaicite:46]{index=46}
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `knowledge_design.md` – Kerntext + Beispiele. :contentReference[oaicite:47]{index=47}
|
||||||
|
- `TYPE_REGISTRY_MANUAL.md` – Typ-Registry & Verhalten. :contentReference[oaicite:48]{index=48}
|
||||||
|
- `mindnet_functional_architecture.md` – ergänzende Konzepte & Edges. :contentReference[oaicite:49]{index=49}
|
||||||
|
- `chunking_strategy.md` – Inline-Relationen, Edges, Datenmodell. :contentReference[oaicite:50]{index=50}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### E) `mindnet_pipeline_playbook_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- End-to-End-Playbook für die gesamte Pipeline:
|
||||||
|
- Importer,
|
||||||
|
- Chunking,
|
||||||
|
- Edge-Erzeugung,
|
||||||
|
- Retriever & Scoring,
|
||||||
|
- (perspektivisch) Explanation Layer, Self-Healing, Self-Tuning.
|
||||||
|
- Für Dev/Ops und technische Leads.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Zweck & Einordnung**
|
||||||
|
1. Bezug zu WPs 02–04a (Ist) und 04b–08 (Roadmap)
|
||||||
|
2. **Import-Pipeline (Runbook-Sicht)**
|
||||||
|
1. Inputs (Vault, Config-Files, ENV)
|
||||||
|
2. Standardläufe (Initialimport, inkrementelle Updates, Sync-Deletes)
|
||||||
|
3. Baseline-Builds & Hash-Modi
|
||||||
|
3. **Chunking & Payload-Aufbau**
|
||||||
|
1. Chunk-Profile, Overlaps, Fenster
|
||||||
|
2. Feldbesetzung (`text`, `window`, `section_title`, `neighbors`, …)
|
||||||
|
4. **Edge-Erzeugung**
|
||||||
|
1. Strukturkanten (belongs_to, next, prev)
|
||||||
|
2. Explizite Edges (Wikilinks, Inline-Relationen, Callouts)
|
||||||
|
3. Typ-Default-Kanten aus `types.yaml`
|
||||||
|
5. **Retriever & Scoring**
|
||||||
|
1. Konfiguration (`retriever.yaml`, ENV)
|
||||||
|
2. Scoring-Formel und Parameter
|
||||||
|
3. Semantik- vs. Hybridmodus
|
||||||
|
4. Test- und Smoke-Suites
|
||||||
|
6. **Quality Gates & Tests**
|
||||||
|
1. Roundtrip-Tests (Vault → Qdrant → Export)
|
||||||
|
2. Chunk-Integrität und Fenster-Checks
|
||||||
|
3. Retriever-Regressionstests
|
||||||
|
7. **Ausblick: Self-Healing, Retroactive Edges, Self-Tuning**
|
||||||
|
1. Geplante Batch-Jobs und Feedback-Pipelines (WP06–08)
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `mindnet_v2_implementation_playbook.md` – Policies, Teststrategie, KPIs. :contentReference[oaicite:57]{index=57}
|
||||||
|
- `Handbuch.md` – Skripte & Test-Runbooks. :contentReference[oaicite:58]{index=58}
|
||||||
|
- `chunking_strategy.md` – Chunk- & Edge-Strategie. :contentReference[oaicite:59]{index=59}
|
||||||
|
- `docs_mindnet_retriever.md` / `wp04_retriever_scoring.md` – Retriever-Praktiken & Tests. :contentReference[oaicite:60]{index=60}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### F) `mindnet_developer_guide_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Konzentrierte Sicht für Entwickler:innen:
|
||||||
|
- wie das Projekt strukturiert ist,
|
||||||
|
- wie man es lokal aufsetzt,
|
||||||
|
- wie man neue Features implementiert/testet.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Projektstruktur**
|
||||||
|
1. Verzeichnisbaum (app/, scripts/, tests/, config/, …)
|
||||||
|
2. **Lokale Entwicklungsumgebung**
|
||||||
|
1. Abhängigkeiten, venv, ENV
|
||||||
|
2. Starten von Services (Qdrant, API, ggf. ollama)
|
||||||
|
3. **APIs & Module**
|
||||||
|
1. Importer-Skripte
|
||||||
|
2. Core-Module (Chunker, Edge-Engine, Retriever)
|
||||||
|
3. FastAPI-Endpunkte (`/query`, evtl. Admin-Endpoints)
|
||||||
|
4. **Tests & Debugging**
|
||||||
|
1. pytest-Suites
|
||||||
|
2. Logging & Diagnoseskripte
|
||||||
|
5. **Guidelines für Erweiterungen**
|
||||||
|
1. Neue Typen & Edge-Regeln
|
||||||
|
2. Änderungen am Datenmodell
|
||||||
|
3. Umgang mit Schema-Änderungen (Virtual Schema Layer)
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `mindnet_technical_architecture.md` – Modul- & API-Beschreibungen. :contentReference[oaicite:63]{index=63}
|
||||||
|
- `Handbuch.md` – Skripte & Beispiele. :contentReference[oaicite:64]{index=64}
|
||||||
|
- `docs_mindnet_retriever.md` – API-Beispiele für den Retriever. :contentReference[oaicite:65]{index=65}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### G) `mindnet_admin_guide_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Betriebshandbuch für mindnet:
|
||||||
|
- Installation,
|
||||||
|
- Konfiguration,
|
||||||
|
- Backup/Restore,
|
||||||
|
- Monitoring,
|
||||||
|
- Troubleshooting.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Zielgruppe & Scope**
|
||||||
|
2. **Initial Setup**
|
||||||
|
1. Voraussetzungen (Ubuntu, Docker/Qdrant, Python-Umgebung)
|
||||||
|
2. ENV-Variablen & Config-Files (types.yaml, retriever.yaml)
|
||||||
|
3. **Betrieb im Alltag**
|
||||||
|
1. Regelmäßige Imports
|
||||||
|
2. Umgang mit Fehlermeldungen im Importer
|
||||||
|
3. Health-Checks für Qdrant & API
|
||||||
|
4. **Update-Prozess**
|
||||||
|
1. Vorgehen bei Schema-/Code-Updates
|
||||||
|
2. Umgang mit neuen Typen und Policies
|
||||||
|
5. **Backup & Restore**
|
||||||
|
1. Qdrant-Snapshots
|
||||||
|
2. Vault-Backup (Obsidian)
|
||||||
|
3. Konsistenzprüfungen
|
||||||
|
6. **Monitoring & Logging**
|
||||||
|
1. typische Logquellen (Importer, API, n8n)
|
||||||
|
7. **Governance & Sicherheit**
|
||||||
|
1. Bezug zur Programm-Governance (WP14, MCP-Sicherheit)
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `Handbuch.md` – Betriebsskripte & ENV. :contentReference[oaicite:69]{index=69}
|
||||||
|
- `mindnet_v2_implementation_playbook.md` – Policies/Resilienz. :contentReference[oaicite:70]{index=70}
|
||||||
|
- Programmplan – Governance / WP14. :contentReference[oaicite:71]{index=71}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### H) `mindnet_user_guide_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Anleitung für Nutzer:innen (inkl. späterer Chat-Agenten-User):
|
||||||
|
- wie man mindnet im Alltag nutzt,
|
||||||
|
- wie man Fragen stellt,
|
||||||
|
- wie man neue Gedanken einbringt.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Was mindnet für dich tut**
|
||||||
|
1. Persönliches Gedächtnis
|
||||||
|
2. KI-Zwilling als Gesprächspartner :contentReference[oaicite:72]{index=72}
|
||||||
|
2. **Zugänge zu mindnet**
|
||||||
|
1. Obsidian-Vault
|
||||||
|
2. Chat-Agent (lokal, n8n-Workflows)
|
||||||
|
3. (später) Interview-Assistent, MCP-Tools
|
||||||
|
3. **Gute Fragen an mindnet**
|
||||||
|
1. Beispiel-Queries (Wissensabruf, Entscheidungen, Reflexionen)
|
||||||
|
4. **Neue Inhalte eingeben**
|
||||||
|
1. Notes in Obsidian erstellen
|
||||||
|
2. Typen und Links bewusst setzen (vereinfachte Version aus D)
|
||||||
|
5. **Ergebnisse interpretieren**
|
||||||
|
1. Was bedeutet ein Treffer-Set?
|
||||||
|
2. Wie sind Scores zu lesen? (nur Ist-Stand, Explanation Layer als Ausblick)
|
||||||
|
6. **Feedback & Lernen**
|
||||||
|
1. Manuelles Feedback (gut/schlecht) – Roadmap WP04c/08
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- Programmplan – Vision & Use-Cases. :contentReference[oaicite:76]{index=76}
|
||||||
|
- Knowledge Design – Autorensicht (stark vereinfacht). :contentReference[oaicite:77]{index=77}
|
||||||
|
- Retriever-Dokumente – Scoring & Interpretation. :contentReference[oaicite:78]{index=78}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### I) `mindnet_appendices_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Referenzen, Tabellen, technische Details, die Hauptdokumente nicht überfrachten sollen.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Typenreferenz (Auszug aus `types.yaml`)**
|
||||||
|
- Tabellenform: Typ, chunk_profile, edge_defaults, retriever_weight
|
||||||
|
2. **Edge-Typen & Bedeutungen**
|
||||||
|
- belongs_to, references, depends_on, similar_to, related_to, next, prev, …
|
||||||
|
3. **Datenmodelle (JSON-Skizzen)**
|
||||||
|
- Note, Chunk, Edge-Payload (aktuell implementierter Stand)
|
||||||
|
4. **Relevante ENV-Variablen & Konfigurationen**
|
||||||
|
- Hash-Settings, COLLECTION_PREFIX, MINDNET_TYPES_FILE, MINDNET_RETRIEVER_CONFIG etc.
|
||||||
|
5. **WP-Übersicht & Status**
|
||||||
|
- kompakte Tabelle aller WPs mit Status (🟢/🟡) und Verweisen auf relevante Dokumente.
|
||||||
|
6. **Historische Dokumente**
|
||||||
|
- Verweis auf alte Versionen (`mindnet_functional_architecture.md`, ältere Playbooks), falls diese nicht vollständig eingearbeitet werden.
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `TYPE_REGISTRY_MANUAL.md` – Typen-Referenz. :contentReference[oaicite:84]{index=84}
|
||||||
|
- `chunking_strategy.md` – JSON-Beispiele. :contentReference[oaicite:85]{index=85}
|
||||||
|
- `mindnet_technical_architecture.md` – Felder & Prioritäten. :contentReference[oaicite:86]{index=86}
|
||||||
|
- Programmplan – WP-Tabellen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Bearbeitungsreihenfolge (Empfehlung)
|
||||||
|
|
||||||
|
Aus Sicht der Informationssicherung und Konsistenz ist folgende Reihenfolge sinnvoll:
|
||||||
|
|
||||||
|
1. **C – `mindnet_technical_architecture_v2.2.md`**
|
||||||
|
- Begründung:
|
||||||
|
- bildet den **aktuellen implementierten Stand** von WP02–WP04a am präzisesten ab,
|
||||||
|
- ist stark an konkrete Felder, Collections und Skripte gebunden (hier ist der Ist-Stand am eindeutigsten belegbar),
|
||||||
|
- ist die Grundlage für Developer Guide, Playbook und Admin Guide.
|
||||||
|
|
||||||
|
2. **E – `mindnet_pipeline_playbook_v2.2.md`**
|
||||||
|
- Konsolidiert Handbuch + Implementation-Playbook + WP04-Dokumente.
|
||||||
|
|
||||||
|
3. **D – `mindnet_knowledge_design_manual_v2.2.md`**
|
||||||
|
- Stellt sicher, dass das Wissensdesign stabil dokumentiert ist, bevor Overview/User Guide entstehen.
|
||||||
|
|
||||||
|
4. **A – `mindnet_overview_v2.2.md`**
|
||||||
|
- Kann sich dann sauber auf die finalisierte fachliche/technische Sicht stützen.
|
||||||
|
|
||||||
|
5. **F, G, H, I**
|
||||||
|
- Developer/Admin/User Guides & Appendices, basierend auf den konsolidierten Kern-Dokumenten.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Nächster konkreter Schritt
|
||||||
|
|
||||||
|
Als nächstes finalisieren wir gemäß dieser Reihenfolge das Dokument:
|
||||||
|
|
||||||
|
> **C – `mindnet_technical_architecture_v2.2.md`**
|
||||||
|
|
||||||
|
Dabei gehen wir wie folgt vor:
|
||||||
|
|
||||||
|
1. Bestehendes `mindnet_technical_architecture.md` **vollständig** als Basis übernehmen. :contentReference[oaicite:89]{index=89}
|
||||||
|
2. Fehlende Details aus `chunking_strategy.md`, `Handbuch.md`, `TYPE_REGISTRY_MANUAL.md` und `wp04_retriever_scoring.md` integrieren (ohne Informationsverlust).
|
||||||
|
3. Ist-Stand vs. Roadmap klar kennzeichnen (WP04b–08, WP05–07).
|
||||||
|
|
||||||
|
Dieses Vorgehen hält die technische Wahrheit konsistent und liefert eine stabile Basis für alle nachfolgenden Dokumente.
|
||||||
@@ -0,0 +1,526 @@
|
|||||||
|
# mindnet v2.2 — Programmplan
|
||||||
|
**Version:** 2.3.0 (Post-WP05 RAG Integration)
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** Aktiv
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Programmauftrag
|
||||||
|
|
||||||
|
mindnet v2.2 entwickelt ein persönliches, wachsendes KI-Gedächtnis, das:
|
||||||
|
|
||||||
|
- Wissen, Erfahrungen, Werte und Entscheidungen speichert,
|
||||||
|
- diese Informationen semantisch verknüpft und rekonstruierbar macht,
|
||||||
|
- einen KI-Zwilling aufbaut, der ähnlich argumentiert, entscheidet und reflektiert wie du,
|
||||||
|
- über mehrere Kanäle gefüttert wird:
|
||||||
|
- Obsidian-Markdown (primäre Quelle),
|
||||||
|
- Chat-basierter Agent (RAG-Chat aktiv),
|
||||||
|
- später: Interview-Assistent (strukturierte Dialogerfassung),
|
||||||
|
- automatisch neue Zusammenhänge erkennt und vernetzt (Edges, Typen, Hinweise),
|
||||||
|
- sich durch Rückmeldungen (Feedback) selbst verbessert (Self-Tuning).
|
||||||
|
|
||||||
|
Langfristig soll mindnet als **digitales Gegenüber** funktionieren, das:
|
||||||
|
|
||||||
|
- dich versteht,
|
||||||
|
- deine Denkweise reflektiert,
|
||||||
|
- deine Werte kennt und verwendet,
|
||||||
|
- Entscheidungen begründen kann („Warum?“),
|
||||||
|
- Erinnerungen einordnet,
|
||||||
|
- und für deine Nachkommen als dialogfähiger Gesprächspartner zur Verfügung steht.
|
||||||
|
|
||||||
|
mindnet ist **kein statisches Wissensarchiv**, sondern ein **lebendes, lernendes Gedächtnismodell** mit Fokus auf:
|
||||||
|
|
||||||
|
- persönliche Perspektive,
|
||||||
|
- erklärbare Begründungspfade (Why-Layer),
|
||||||
|
- kontinuierliche Erweiterbarkeit,
|
||||||
|
- robuste technische Basis (lokal, nachvollziehbar, versioniert).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Vision
|
||||||
|
|
||||||
|
> „Ein persönliches semantisches Gedächtnis, das mit mir wächst, meine Persönlichkeit spiegelt und mir als intelligenter, erklärbarer, lernender Begleiter dient.“
|
||||||
|
|
||||||
|
Kernprinzipien der Vision:
|
||||||
|
|
||||||
|
- **Erklärbarkeit:**
|
||||||
|
Jede Antwort muss über Edges, Scores, Werte-Bezüge und Herkunftsnotizen begründbar sein. Das System kann Entscheidungen auf zugrundeliegende `[DECISION]`-Notizen zurückführen.
|
||||||
|
|
||||||
|
- **Wachstum:**
|
||||||
|
Das System arbeitet von Anfang an mit unvollständigen Daten, kann aber schrittweise dichter werden, ohne dass alte Notizen massenhaft manuell angepasst werden müssen.
|
||||||
|
|
||||||
|
- **Flexibilität (Late Binding):**
|
||||||
|
Semantik wird überwiegend in Konfiguration (z. B. `types.yaml`, `prompts.yaml`, Policies) festgelegt. Die Persönlichkeit entsteht durch das Prompt-Design, nicht durch Hardcoding.
|
||||||
|
|
||||||
|
- **Autonomie & Self-Healing:**
|
||||||
|
mindnet schlägt fehlende Typen, Relationen und Edges vor (z. B. aus Inline-Relationen, Edge-Defaults, Ähnlichkeitsbeziehungen) und baut damit einen „self-healing graph“ auf.
|
||||||
|
|
||||||
|
- **Lernen & Self-Tuning:**
|
||||||
|
Feedback zu Antworten (gut/schlecht, relevant/nicht relevant) fließt in Score-Gewichte, Policies und ggf. Edge-Struktur ein.
|
||||||
|
|
||||||
|
- **Persönlichkeit:**
|
||||||
|
Entscheidungen werden wert- und erfahrungsbasiert begründet; das System agiert als KI-Zwilling durch Nutzung lokaler LLMs (z.B. Phi-3/Mistral).
|
||||||
|
|
||||||
|
- **Incremental Growth:**
|
||||||
|
Das System muss mit wenigen, heterogenen Notizen starten und sich fortlaufend verdichten können – ohne Retro-Massenmigrationen im Vault.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Programmziele
|
||||||
|
|
||||||
|
### 3.1 Kurzfristig (Abgeschlossen / Laufend)
|
||||||
|
|
||||||
|
- Automatische Sammlung von Wissen aus Markdown-Vault.
|
||||||
|
- Aufbau eines stabilen Graph-Speichers in Qdrant (`*_notes`, `*_chunks`, `*_edges`).
|
||||||
|
- Semantischer und graphbasierter Retriever (WP-04a abgeschlossen).
|
||||||
|
- **Erklärbarkeit:** Why-Layer liefert Begründungen zu Treffern (WP-04b abgeschlossen).
|
||||||
|
- **Feedback-Loop:** Systematisches Logging von Suche und Bewertung (WP-04c abgeschlossen).
|
||||||
|
- **RAG-Chat:** KI antwortet in natürlicher Sprache auf Basis von Wissen und Persönlichkeit (WP-05 abgeschlossen).
|
||||||
|
- Technische Basis: FastAPI, Qdrant, Ollama (Local LLM).
|
||||||
|
- Automatisierte Erkennung von Beziehungen:
|
||||||
|
- Wikilinks, Inline-Relationen, Callout-Edges, Typ-Defaults.
|
||||||
|
- Schrittweises Lernen über Feedback (Score-Tuning, noch „manuell“ konfiguriert).
|
||||||
|
- „Mitwachsendes“ Schema ohne Obsidian-Umstrukturierungen:
|
||||||
|
- Neues Wissen kann sofort erfasst werden,
|
||||||
|
- bestehende Notizen bleiben gültig (Virtual Schema Layer).
|
||||||
|
|
||||||
|
### 3.2 Mittelfristig (Nächste Schritte)
|
||||||
|
|
||||||
|
- **Decision Engine (WP-06):** Das System berät aktiv bei Entscheidungen, indem es `type: value` und `type: principle` Notizen gegen eine Fragestellung abwägt.
|
||||||
|
- **Chat Interface (WP-10):** Ablösung der Terminal-Interaktion durch ein Web-Frontend (Streamlit/React) für bessere UX und einfacheres Feedback-Geben.
|
||||||
|
- Interview-Assistent erstellt neue Notes automatisch (strukturierte Dialoge → Markdown).
|
||||||
|
- mindnet erzeugt Vorschläge für neue Notes & Edges und bietet einen „Vernetzungs-Assistenten“ für manuell angelegte Notizen.
|
||||||
|
- Agenten können über MCP-Tools (`mindnet_query`, `mindnet_chat`) auf mindnet zugreifen.
|
||||||
|
|
||||||
|
### 3.3 Langfristig
|
||||||
|
|
||||||
|
- KI-Zwilling, der deinen Stil, deine Werte und deine typische Denkweise repräsentiert.
|
||||||
|
- Weitgehend autonomes Self-Tuning der Wissensstruktur (auf Basis gespeicherter Feedback-Daten & Policies).
|
||||||
|
- Lebenslanges Gedächtnis, das
|
||||||
|
- konsistent bleibt,
|
||||||
|
- wächst,
|
||||||
|
- sich weiterentwickelt,
|
||||||
|
- und deine Geschichte für deine Nachkommen erzählbar und erlebbar macht.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Architekturprinzipien
|
||||||
|
|
||||||
|
Die folgenden Prinzipien steuern alle Workpackages und Entscheidungen:
|
||||||
|
|
||||||
|
1. **Late Binding (späte Semantik)**
|
||||||
|
- Struktur und Interpretation werden in Konfigurationen (z. B. `types.yaml`, `prompts.yaml`, Policies) definiert, nicht direkt in den Vault-Dateien.
|
||||||
|
- Die "Persönlichkeit" des Chats ist ein Prompt-Template, kein Code.
|
||||||
|
|
||||||
|
2. **Virtual Schema Layer**
|
||||||
|
- Typen, Regeln, Policies, Edge-Defaults werden im Schema-Layer beschrieben.
|
||||||
|
- Markdown-Dateien benötigen nur minimale, robuste Angaben (Titel, Typ, optionale Properties).
|
||||||
|
|
||||||
|
3. **Self-Healing Graph**
|
||||||
|
- Der Graph wird regelmäßig analysiert (Edges, Centrality, fehlende Links).
|
||||||
|
- Automatisierte Jobs ergänzen fehlende Kanten.
|
||||||
|
|
||||||
|
4. **Deterministische IDs**
|
||||||
|
- Notes, Chunks und Edges erhalten stabile IDs (z. B. UUIDv5).
|
||||||
|
- Der Graph ist jederzeit wiederaufbaubar (Import-Pipeline idempotent).
|
||||||
|
|
||||||
|
5. **Full Explainability & Context Enrichment**
|
||||||
|
- Jeder Score, jede Beziehung muss nachvollziehbar sein.
|
||||||
|
- Dem LLM werden Metadaten (`[DECISION]`, `[PROJECT]`) injiziert, um Halluzinationen zu vermeiden und logische Schlüsse zu ermöglichen.
|
||||||
|
|
||||||
|
6. **Persistence First**
|
||||||
|
- Obsidian bleibt die primäre Quelle der Wahrheit.
|
||||||
|
- mindnet ergänzt, schlägt Änderungen vor, schreibt aber nur kontrolliert.
|
||||||
|
|
||||||
|
7. **Minimalinvasives Schreiben**
|
||||||
|
- Automatische Veränderungen an Markdown-Dateien erfolgen ausschließlich nach expliziter Zustimmung.
|
||||||
|
|
||||||
|
8. **Incremental Growth & Early Value**
|
||||||
|
- Das System muss bereits mit wenigen Notizen und einem kleinen Vault sinnvolle Antworten liefern.
|
||||||
|
- Feedback-Mechanismen werden früh eingeführt.
|
||||||
|
|
||||||
|
9. **Observability & Testbarkeit**
|
||||||
|
- Jeder Importlauf, jede Retriever-Anfrage und jede Policy-Änderung soll prüfbar sein.
|
||||||
|
|
||||||
|
10. **Local First & Privacy**
|
||||||
|
- Nutzung lokaler LLMs (Ollama/Phi-3) für Inference. Keine Daten verlassen den Server.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Programmstruktur (Phasenmodell)
|
||||||
|
|
||||||
|
Phase A – Fundament & Import (Fertig)
|
||||||
|
Phase B – Semantik, Graph & Lernen (Fertig)
|
||||||
|
Phase C – Persönlichkeitsmodell & KI-Zwilling (Laufend)
|
||||||
|
Phase D – Agenten, MCP & Interaktion (Startend)
|
||||||
|
Phase E – Review, Refactoring, Dokumentation
|
||||||
|
|
||||||
|
Alle Workpackages sind einer Phase zugeordnet. WP-01 bis WP-05 sind bereits erfolgreich abgeschlossen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Workpackages – detaillierte Übersicht
|
||||||
|
|
||||||
|
### Legende Aufwand / Komplexität
|
||||||
|
|
||||||
|
- Aufwand: Niedrig / Mittel / Hoch
|
||||||
|
- Komplexität: Niedrig / Mittel / Hoch
|
||||||
|
|
||||||
|
Die Status-Ampel zeigt den aktuellen Stand gemäß Programmfortschritt.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-01 – Wissensdesign (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** A
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Grundlegendes Schema für mindnet festlegen (`knowledge_design.md`), inkl.:
|
||||||
|
- Note-Typen (z. B. `concept`, `experience`, `project`, `decision`, `value`).
|
||||||
|
- Standard-Properties und Grundstruktur von Edges.
|
||||||
|
- Abbildung zentraler Lebens-Bereiche.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- `knowledge_design.md` und `types.yaml` definieren die Zielarchitektur.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-02 – Chunking & Hash-Strategie (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** A
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Festlegen und implementieren einer robusten Chunking-Strategie und einer Hash-Logik zur Änderungserkennung.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- Implementierte Chunker-Logik mit Overlap.
|
||||||
|
- Stabile Hash-Strategie zur sicheren Änderungserkennung.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-03 – Import-Pipeline & Edge-System v2 (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** A/B
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Durchgängige Import-Pipeline von Markdown → Notes/Chunks/Edges in Qdrant mit deterministischen IDs.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- Vollständige E2E-Import-Pipeline.
|
||||||
|
- Vollständiges Edge-Modell (Strukturkanten, Wikilinks, Inline-Relationen).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-04a – Retriever & Graph-Scoring (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** B
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Aufbau eines hybriden Retrievers, der Semantik, Typwissen und Graph-Informationen kombiniert.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- Hybrides Scoring-Modell (Semantic + Edge + Centrality).
|
||||||
|
- Konfiguration über `retriever.yaml`.
|
||||||
|
- `/query` Endpoint aktiv.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-04b – Explanation Layer ("Why-Layer") (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** B
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Treffererklärungen liefern, die verständlich machen, warum ein Ergebnis gewählt wurde.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- **DTO-Erweiterung:** `QueryHit` enthält `explanation`, `breakdown` und `reason`.
|
||||||
|
- **Logic:** Graph-Adapter unterstützt `incoming_edges` (Reverse Lookup) für Authority-Erklärungen.
|
||||||
|
- **Output:** `/query` Endpoint liefert auf Wunsch (`explain: true`) menschenlesbare Begründungen ("Verweist auf...").
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-04c – Feedback Logging & Bewertungsdaten (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** B
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Grundlage für Self-Tuning schaffen durch systematisches Logging.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- **Data Flywheel:** Logging von Queries und User-Feedback in lokalen JSONL-Dateien.
|
||||||
|
- **Search Log:** Speichert Query + Score-Snapshot in `data/logs/search_history.jsonl`.
|
||||||
|
- **Feedback Log:** Speichert User-Ratings in `data/logs/feedback.jsonl`.
|
||||||
|
- **Traceability:** Durchgängige `query_id` von Request bis Feedback.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-05 – Persönlichkeitsmodell & RAG-Chat (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** C
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Aufbau eines RAG-Systems, das in natürlicher Sprache antwortet und Kontext versteht.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- **Infrastruktur:** Integration von Ollama (Phi-3 Mini) in den Service-Layer.
|
||||||
|
- **Router:** `/chat` Endpoint mit "Hybrid Retrieval Enforcement".
|
||||||
|
- **Context Enrichment:** Injection von Metadaten (`[TYPE]`, `[SCORE]`) in den Prompt, damit kleine Modelle (SLMs) komplexe Zusammenhänge ("Warum?") verstehen.
|
||||||
|
- **Config:** `prompts.yaml` steuert System-Prompt und RAG-Template.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-05b – Advanced Chat (Optional)
|
||||||
|
|
||||||
|
**Phase:** C
|
||||||
|
**Status:** ⚪ optional
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Erweiterung des Chats um Gedächtnis (History) und einfache Tools.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Implementierung von `ChatHistory` (vergangene Nachrichten in den Kontext).
|
||||||
|
- Einfache Tool-Nutzung (z.B. "Suche in Web").
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-06 – Decision Engine (geplant)
|
||||||
|
|
||||||
|
**Phase:** C
|
||||||
|
**Status:** 🟡 geplant (Priorität A)
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Entscheidungsunterstützung auf Basis von Wissen, Persönlichkeit und Zielen – inklusive nachvollziehbarer Begründung.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Erweiterung des RAG-Kontexts um gezieltes Nachladen von `type: value` und `type: principle`.
|
||||||
|
- Prompt-Engineering für "Trade-off Analyse" (Pro/Contra basierend auf Werten).
|
||||||
|
- Output-Formatierung als Entscheidungsvorlage.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Hoch
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-07 – Interview-Assistent (geplant)
|
||||||
|
|
||||||
|
**Phase:** C
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Dialogbasierter Erfassungs-Assistent, der strukturierte Interviews führt und daraus konsistente Markdown-Notizen generiert.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Design von Interview-Flows.
|
||||||
|
- Konvertierung von Dialogtranskripten in typisierte Notes.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Niedrig/Mittel
|
||||||
|
- Komplexität: Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-08 – Self-Tuning v1/v2 (geplant)
|
||||||
|
|
||||||
|
**Phase:** B/C
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Aufbau eines Self-Tuning-Mechanismus, der auf Basis von Feedback-Daten (WP-04c) Vorschläge für Retriever- und Policy-Anpassungen macht.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Auswertung der JSONL-Feedback-Daten.
|
||||||
|
- Regel-basierte Anpassungs-Vorschläge für `retriever.yaml` und Typ-Prioritäten.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Hoch
|
||||||
|
- Komplexität: Hoch
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-09 – Vault-Onboarding & Migration (geplant)
|
||||||
|
|
||||||
|
**Phase:** B
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Sicherstellen, dass bestehende und neue Obsidian-Vaults schrittweise in mindnet integriert werden können – ohne Massenumbau.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Tools zur Analyse des Vault-Status.
|
||||||
|
- Empfehlungen für minimale Anpassungen.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Niedrig/Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-10 – Chat-Interface & Writeback (geplant)
|
||||||
|
|
||||||
|
**Phase:** D
|
||||||
|
**Status:** 🟡 geplant (Priorität B - "Quick Win")
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Ablösung der Terminal-Interaktion durch ein grafisches Interface.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Technologie: Streamlit oder einfaches HTML/JS Frontend.
|
||||||
|
- Funktionen: Chat-Verlauf, Anzeige der Quellen (mit Scores), Daumen-hoch/runter für WP-04c Feedback.
|
||||||
|
- Funktionen für Q&A sowie Vorschlag neuer Notes/Edges.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Hoch (durch Writeback) / Mittel (für reines Frontend)
|
||||||
|
- Komplexität: Hoch
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-11 – Knowledge-Builder & Vernetzungs-Assistent (geplant)
|
||||||
|
|
||||||
|
**Phase:** D
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Assistent, der manuell erstellte oder importierte Notizen analysiert und Vorschläge für Typen, Edges und Einordnung macht.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Hoch
|
||||||
|
- Komplexität: Hoch
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-12 – Knowledge Rewriter (Soft Mode, geplant)
|
||||||
|
|
||||||
|
**Phase:** C/D
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Optionaler Assistent, der vorhandene Notes „aufräumt“, zusammenfasst oder reorganisiert – ausschließlich mit expliziter Freigabe.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Niedrig/Mittel
|
||||||
|
- Komplexität: Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-13 – MCP-Integration & Agenten-Layer (geplant)
|
||||||
|
|
||||||
|
**Phase:** D
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
mindnet als MCP-Server bereitstellen, damit Agenten/LLMs standardisierte Tools nutzen können.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- MCP-Server mit Tools (`mindnet_query`, `mindnet_explain`, etc.).
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-14 – Review / Refactoring / Dokumentation (geplant)
|
||||||
|
|
||||||
|
**Phase:** E
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Aufräumen, dokumentieren, stabilisieren – insbesondere für Onboarding Dritter und Langfrist-Betrieb.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Niedrig/Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Abhängigkeiten (vereinfacht, aktualisiert)
|
||||||
|
|
||||||
|
WP01 → WP02 → WP03 → WP04a
|
||||||
|
WP04a → WP04b → WP04c → WP08
|
||||||
|
WP03 → WP05 → WP06 → WP07
|
||||||
|
WP03 → WP09
|
||||||
|
WP01/WP03 → WP10 → WP11 → WP12
|
||||||
|
WP03/WP04 → WP13
|
||||||
|
Alles → WP14
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Laufzeit- & Komplexitätsindikatoren (aktualisiert)
|
||||||
|
|
||||||
|
| WP | Aufwand | Komplexität |
|
||||||
|
|-------|----------------|------------------|
|
||||||
|
| WP04b | Mittel | Hoch |
|
||||||
|
| WP04c | Mittel | Mittel/Hoch |
|
||||||
|
| WP05 | Mittel | Mittel |
|
||||||
|
| WP06 | Mittel | Hoch |
|
||||||
|
| WP07 | Niedrig/Mittel | Mittel |
|
||||||
|
| WP08 | Hoch | Hoch |
|
||||||
|
| WP09 | Mittel | Niedrig/Mittel |
|
||||||
|
| WP10 | Hoch | Hoch |
|
||||||
|
| WP11 | Hoch | Hoch |
|
||||||
|
| WP12 | Niedrig/Mittel | Mittel |
|
||||||
|
| WP13 | Mittel | Mittel |
|
||||||
|
| WP14 | Mittel | Niedrig/Mittel |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Programmfortschritt (Ampel, aktualisiert)
|
||||||
|
|
||||||
|
| WP | Status |
|
||||||
|
|-------|--------|
|
||||||
|
| WP01 | 🟢 |
|
||||||
|
| WP02 | 🟢 |
|
||||||
|
| WP03 | 🟢 |
|
||||||
|
| WP04a | 🟢 |
|
||||||
|
| WP04b | 🟢 |
|
||||||
|
| WP04c | 🟢 |
|
||||||
|
| WP05 | 🟢 |
|
||||||
|
| WP05b | ⚪ |
|
||||||
|
| WP06 | 🟡 |
|
||||||
|
| WP07 | 🟡 |
|
||||||
|
| WP08 | 🟡 |
|
||||||
|
| WP09 | 🟡 |
|
||||||
|
| WP10 | 🟡 |
|
||||||
|
| WP11 | 🟡 |
|
||||||
|
| WP12 | 🟡 |
|
||||||
|
| WP13 | 🟡 |
|
||||||
|
| WP14 | 🟡 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Governance & Versionierung
|
||||||
|
|
||||||
|
- Versionierung über Gitea (Branches, Tags, Releases).
|
||||||
|
- Feature-Branch Workflow mit "Sync First" Strategie.
|
||||||
|
- Jede wesentliche Änderung an Architektur/Schema erhält einen eigenen Commit.
|
||||||
|
- Jedes Workpackage erhält ein eigenes Chat-Fenster mit dediziertem Prompt (WP-Hand-Over).
|
||||||
|
- Programmleitung verantwortet Konsistenz und Priorisierung.
|
||||||
|
- **Modell-Governance:** Das verwendete LLM (z.B. `phi3:mini`) wird in der `.env` und `config.py` festgeschrieben. Updates erfordern Tests gegen die "Why-Layer" Referenzfragen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. Executive Summary
|
||||||
|
|
||||||
|
mindnet v2.2 ist so aufgesetzt, dass:
|
||||||
|
|
||||||
|
- du **schrittweise** Wissen erfassen kannst (Obsidian + Chat + später Interview-Assistent),
|
||||||
|
- die Struktur **mitwächst**, ohne Retro-Massenarbeit im Vault,
|
||||||
|
- ein **hybrider Retriever** qualitativ hochwertige, erklärbare Antworten liefert,
|
||||||
|
- ein **Self-Healing- und Self-Tuning-Mechanismus** vorbereitet ist (durch WP-04c Feedback-Daten),
|
||||||
|
- ein **Persönlichkeitsmodell** und eine **Decision Engine** entstehen,
|
||||||
|
- langfristig ein **KI-Zwilling** aufgebaut wird, der deine Werte, Erfahrungen und Denkweise spiegelt,
|
||||||
|
- die technische Architektur (FastAPI, Qdrant, YAML-Policies, MCP-Integration) lokal, nachvollziehbar und erweiterbar bleibt.
|
||||||
|
|
||||||
|
Dieser Programmplan bildet die konsolidierte Grundlage (v2.3.0) für alle weiteren Arbeiten.
|
||||||
@@ -0,0 +1,307 @@
|
|||||||
|
# mindnet v2 — Dokumentationsreview (vollständig & konsolidiert)
|
||||||
|
|
||||||
|
Dieses Dokument fasst alle notwendigen Anpassungen der mindnet-v2 Dokumentation zusammen.
|
||||||
|
Der Fokus liegt ausschließlich auf Aspekten, die durch die aktuellen Arbeiten verändert wurden:
|
||||||
|
|
||||||
|
- neue Edge-Mechanik
|
||||||
|
- neuer Importer
|
||||||
|
- neues Verhalten von note_payload & chunk_payload
|
||||||
|
- Default-Typregeln aus `types.yaml`
|
||||||
|
- Inline- und Callout-Relations
|
||||||
|
- WP04a – Einführung der Retriever-Schicht
|
||||||
|
- neue Diagnose- und Testskripte
|
||||||
|
|
||||||
|
Es handelt sich um eine strukturelle Analyse.
|
||||||
|
Nichts davon überschreibt bestehende Inhalte — es markiert nur, **was angepasst werden muss**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 1) knowledge_design.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 1.1 Neue Edge-Arten dokumentieren
|
||||||
|
Es müssen vier neue Kategorien beschrieben werden:
|
||||||
|
|
||||||
|
1. **Inline-Edges**
|
||||||
|
|
||||||
|
Beispiel:
|
||||||
|
`[ [rel:depends_on Embeddings 101] ]`
|
||||||
|
(Leerzeichen hier zur Demonstration; im echten Dokument ohne Leerzeichen)
|
||||||
|
|
||||||
|
2. **Callout-Edges**
|
||||||
|
|
||||||
|
> [!edge] related_to: [[Vector DB Basics]] [[Embeddings 101]]
|
||||||
|
|
||||||
|
3. **Typbasierte Default-Kanten**
|
||||||
|
Aus `types.yaml` → `edge_defaults`
|
||||||
|
|
||||||
|
4. **Strukturkanten**
|
||||||
|
- belongs_to
|
||||||
|
- next
|
||||||
|
- prev
|
||||||
|
|
||||||
|
### 1.2 Neue Edge-Felder ergänzen
|
||||||
|
Jeder Edge enthält jetzt:
|
||||||
|
|
||||||
|
- `rule_id`
|
||||||
|
- `confidence`
|
||||||
|
- `scope = "chunk"`
|
||||||
|
- deterministischer `edge_id`
|
||||||
|
|
||||||
|
Diese Felder müssen vollständig erklärt werden.
|
||||||
|
|
||||||
|
### 1.3 Chunk-basierte Kanten
|
||||||
|
Neu: **Alle Kanten entstehen ausschließlich zwischen Chunks**, nie zwischen Notes.
|
||||||
|
Notes dienen nur noch als Metadatencontainer.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 2) TYPE_REGISTRY_MANUAL.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 2.1 Typauflösung dokumentieren
|
||||||
|
Die Regeln sind jetzt:
|
||||||
|
|
||||||
|
- Typ aus Frontmatter → höchste Priorität
|
||||||
|
- Falls nicht gesetzt: Typ aus Dateipfad ableiten
|
||||||
|
- Falls unbekannt: Fallback auf `types.default`
|
||||||
|
- Typ definiert:
|
||||||
|
- retriever_weight
|
||||||
|
- chunk_profile
|
||||||
|
- edge_defaults
|
||||||
|
|
||||||
|
Diese müssen klar dokumentiert werden.
|
||||||
|
|
||||||
|
### 2.2 Neue Fallback-Regeln
|
||||||
|
Wenn Werte fehlen, werden sie ersetzt durch:
|
||||||
|
|
||||||
|
types.default.retriever_weight
|
||||||
|
types.default.chunk_profile
|
||||||
|
types.default.edge_defaults
|
||||||
|
|
||||||
|
### 2.3 Beispiele aktualisieren
|
||||||
|
Alle Beispiele müssen auf die neue Inline-Syntax angepasst werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 3) wp04_retriever_scoring.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 3.1 Neue Scoring-Formel
|
||||||
|
total_score =
|
||||||
|
semantic_weight * semantic_score
|
||||||
|
+ edge_weight * edge_bonus
|
||||||
|
+ centrality_weight * centrality_bonus
|
||||||
|
+ type_weight * retriever_weight
|
||||||
|
|
||||||
|
### 3.2 Bedeutung von `confidence`
|
||||||
|
confidence wirkt multiplikativ bei edge_bonus.
|
||||||
|
Inline > Callout > Default (Standardwerte im Code hinterlegt).
|
||||||
|
|
||||||
|
### 3.3 Pfadbegründungen
|
||||||
|
Der Retriever gibt optional `paths[]` zurück:
|
||||||
|
|
||||||
|
paths: [
|
||||||
|
{
|
||||||
|
"via": "depends_on",
|
||||||
|
"confidence": 0.7,
|
||||||
|
"hops": ["Note A", "Chunk A#01", "Note B"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
Diese Funktion müssen dokumentiert werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 4) mindnet_v2_implementation_playbook.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 4.1 Neuer Import-Prozess vollständig dokumentieren
|
||||||
|
Der Importer läuft jetzt so:
|
||||||
|
|
||||||
|
1. Markdown lesen
|
||||||
|
2. Frontmatter extrahieren
|
||||||
|
3. Typauflösung via `types.yaml`
|
||||||
|
4. Note-Payload generieren
|
||||||
|
5. Chunking anwenden
|
||||||
|
6. Inline-Kanten finden
|
||||||
|
7. Callout-Kanten finden
|
||||||
|
8. Default-Edges erzeugen
|
||||||
|
9. Strukturkanten erzeugen
|
||||||
|
10. Chunks in Qdrant upserten
|
||||||
|
11. Edges upserten
|
||||||
|
12. Diagnose laufen lassen:
|
||||||
|
|
||||||
|
python3 -m scripts.payload_dryrun
|
||||||
|
python3 -m scripts.edges_full_check
|
||||||
|
|
||||||
|
### 4.2 Erweiterte Testpipeline
|
||||||
|
Neu zu dokumentieren:
|
||||||
|
|
||||||
|
tests.test_edges_all
|
||||||
|
scripts.edges_full_check
|
||||||
|
scripts.payload_dryrun
|
||||||
|
|
||||||
|
Diese Testschritte sind jetzt Standardanforderung nach jeder Änderung.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 5) mindnet_technical_architecture.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 5.1 Aktualisiertes Schema für Qdrant Collections
|
||||||
|
|
||||||
|
#### Notes:
|
||||||
|
note_id
|
||||||
|
title
|
||||||
|
type
|
||||||
|
fulltext
|
||||||
|
retriever_weight
|
||||||
|
chunk_profile
|
||||||
|
edge_defaults
|
||||||
|
tags
|
||||||
|
updated
|
||||||
|
|
||||||
|
#### Chunks:
|
||||||
|
chunk_id
|
||||||
|
note_id
|
||||||
|
text
|
||||||
|
window
|
||||||
|
ord
|
||||||
|
retriever_weight
|
||||||
|
chunk_profile
|
||||||
|
neighbors_prev
|
||||||
|
neighbors_next
|
||||||
|
|
||||||
|
#### Edges:
|
||||||
|
edge_id
|
||||||
|
source_id
|
||||||
|
target_id
|
||||||
|
kind
|
||||||
|
scope = "chunk"
|
||||||
|
rule_id
|
||||||
|
confidence
|
||||||
|
|
||||||
|
### 5.2 Dokumentation der Edge-Pipeline
|
||||||
|
Neu beschreiben:
|
||||||
|
|
||||||
|
- Inline-Parser
|
||||||
|
- Callout-Parser
|
||||||
|
- Default-Edge-Resolver
|
||||||
|
- Strukturkanten
|
||||||
|
- Normalisierung für Qdrant
|
||||||
|
- deterministische ID-Bildung
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 6) mindnet_functional_architecture.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 6.1 Inline-Relations – einzig gültige Syntax
|
||||||
|
Nur diese Syntax erzeugt Kanten:
|
||||||
|
|
||||||
|
[[rel:depends_on Embeddings 101]]
|
||||||
|
|
||||||
|
Nicht unterstützt:
|
||||||
|
|
||||||
|
rel:depends_on [[Embeddings 101]]
|
||||||
|
|
||||||
|
Dies muss klar dokumentiert werden.
|
||||||
|
|
||||||
|
### 6.2 Prioritäten der Kantenerzeugung
|
||||||
|
Gültige Reihenfolge:
|
||||||
|
|
||||||
|
1. Inline
|
||||||
|
2. Callout
|
||||||
|
3. Typdefaults
|
||||||
|
4. Strukturkanten
|
||||||
|
|
||||||
|
### 6.3 Relaunch der Chunk-zentrierten Funktionslogik
|
||||||
|
Alle Beispiele müssen angepasst werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 7) chunking_strategy.md
|
||||||
|
## Erforderliche Änderungen
|
||||||
|
|
||||||
|
### 7.1 Neues Verhalten der Chunks
|
||||||
|
- `ord` ist jetzt Sortierkriterium
|
||||||
|
- `window` wird über chunk_config erzeugt
|
||||||
|
- Chunkprofile kommen ausschließlich aus `types.yaml`
|
||||||
|
|
||||||
|
### 7.2 Best Practices erneuern
|
||||||
|
Empfehlung:
|
||||||
|
|
||||||
|
- keine manuelle Chunkprofile mehr im Frontmatter setzen
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 8) docs_mindnet_retriever.md
|
||||||
|
## Erforderliche Änderungen
|
||||||
|
|
||||||
|
### 8.1 Neue Ausgabeparameter dokumentieren
|
||||||
|
Retriever liefert jetzt:
|
||||||
|
|
||||||
|
semantic_score
|
||||||
|
edge_bonus
|
||||||
|
centrality_bonus
|
||||||
|
type_score
|
||||||
|
total_score
|
||||||
|
paths[]
|
||||||
|
|
||||||
|
### 8.2 Graph-Expansion
|
||||||
|
Dokumentieren:
|
||||||
|
|
||||||
|
- Seed-Auswahl
|
||||||
|
- Depth-Limit
|
||||||
|
- Edge-Filterung
|
||||||
|
- Wirkung von `confidence` auf edge_bonus
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 9) Handbuch.md
|
||||||
|
## Erforderliche Änderungen
|
||||||
|
|
||||||
|
### 9.1 Neuer Import-Workflow
|
||||||
|
Erläuterung der Parser:
|
||||||
|
|
||||||
|
- Inline-Parser
|
||||||
|
- Callout-Parser
|
||||||
|
- Default-Kanten
|
||||||
|
- Struktur-Kanten
|
||||||
|
|
||||||
|
### 9.2 Best Practices
|
||||||
|
Ergänzen:
|
||||||
|
|
||||||
|
- Typenpflege erfolgt nur in `types.yaml`
|
||||||
|
- Tests immer nach Änderungen ausführen
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Gesamtübersicht der notwendigen Anpassungen
|
||||||
|
|
||||||
|
| Dokument | Anpassungsbedarf |
|
||||||
|
|---------|------------------|
|
||||||
|
| knowledge_design.md | neue Edge-Arten, Felder, Chunk-Edges |
|
||||||
|
| TYPE_REGISTRY_MANUAL.md | Typauflösung, neue Felder, neue Fallbacks |
|
||||||
|
| wp04_retriever_scoring.md | neue Score-Formel, Pfade, confidence |
|
||||||
|
| implementation_playbook.md | neuer Importprozess, Tests |
|
||||||
|
| technical_architecture.md | aktualisierte Schemas |
|
||||||
|
| functional_architecture.md | Syntax + Prioritäten |
|
||||||
|
| chunking_strategy.md | ord, window, neue Profile |
|
||||||
|
| docs_mindnet_retriever.md | neue API-Parameter |
|
||||||
|
| Handbuch.md | Parser + Best Practices |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Empfehlung zur Umsetzung
|
||||||
|
1. Dieses Review-Dokument als Master-Referenz verwenden.
|
||||||
|
2. Jedes Dokument strikt nur an den beschriebenen Stellen anpassen.
|
||||||
|
3. Überprüfen, ob Wann/Wo Typen, Kanten und Parser erklärt werden.
|
||||||
|
4. Nach jeder Anpassung Testlauf:
|
||||||
|
|
||||||
|
python3 -m scripts.payload_dryrun
|
||||||
|
python3 -m scripts.edges_full_check
|
||||||
|
|
||||||
|
5. Danach Playbook + Programmplan updaten.
|
||||||
|
|
||||||
@@ -0,0 +1,322 @@
|
|||||||
|
# mindnet – Änderungs- und Ergänzungsleitfaden für bestehende Dokumentation
|
||||||
|
# (Stand: Tag Wp04a-Retriever_final)
|
||||||
|
|
||||||
|
Dieser Leitfaden listet **alle notwendigen Änderungen und Ergänzungen** für die
|
||||||
|
bestehenden Dokumente auf. Er umfasst ausschließlich Inhalte, die durch die
|
||||||
|
tatsächliche Implementierung von WP-04a (Retriever, Hybrid Scoring, Graph-Integration,
|
||||||
|
retriever.yaml, Explainability-Vorbereitung, Centrality, Edge-Bonus, Query-API)
|
||||||
|
betroffen sind.
|
||||||
|
|
||||||
|
Er dient dazu, dass ein anderer Chat alle Dokumente vollständig und korrekt
|
||||||
|
überarbeiten kann, ohne Interpretationsspielraum oder fehlende Informationen.
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 1. Änderungen für: knowledge_design.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 1.1 Ergänzen: Rolle von retriever_weight pro Note-Type
|
||||||
|
Die Implementierung nutzt `retriever_weight`, das im Importer aus `types.yaml`
|
||||||
|
pro Note-Type übernommen und auf *Chunk-Ebene* vererbt wird.
|
||||||
|
|
||||||
|
**Ergänzen:**
|
||||||
|
- Erklärung, dass `retriever_weight` ein konzeptionelles Typ-Attribut ist,
|
||||||
|
das im hybriden Scoring als Multiplikator wirkt.
|
||||||
|
- Klarstellung, dass `retriever_weight` *nicht* durch Markdown-Frontmatter
|
||||||
|
überschrieben werden muss.
|
||||||
|
- Dokumentation, dass Qdrant-Payload dieses Feld für Notes und Chunks enthält.
|
||||||
|
|
||||||
|
## 1.2 Ergänzen: Edge-Typen, die im Retriever genutzt werden
|
||||||
|
Der Retriever unterstützt aktuell:
|
||||||
|
- `references`
|
||||||
|
- `related_to`
|
||||||
|
- `depends_on`
|
||||||
|
- `similar_to`
|
||||||
|
- `belongs_to`
|
||||||
|
- sowie generierte Strukturkanten `next` / `prev`.
|
||||||
|
|
||||||
|
**Ergänzen:**
|
||||||
|
- Welche Edges für den Graph-Bonus ausgewertet werden.
|
||||||
|
- Wie `confidence` aus mindnet_edges genutzt wird.
|
||||||
|
|
||||||
|
## 1.3 Ergänzen: Definition des lokalen Subgraphen
|
||||||
|
Die reale Retrieval-Logik nutzt:
|
||||||
|
- Seed-Notes (aus semantisch Top-K)
|
||||||
|
- Expansionstiefe `depth`
|
||||||
|
- Edge-Typ-Filter
|
||||||
|
- Bonusberechnung pro Knoten
|
||||||
|
|
||||||
|
Dies ist im ursprünglichen Dokument nicht beschrieben und muss hinzugefügt werden.
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 2. Änderungen für: chunking_strategy.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 2.1 Ergänzen: Chunk-Payload-Felder, die der Retriever nutzt
|
||||||
|
Die aktuelle Strategy beschreibt Chunks korrekt, es fehlt jedoch:
|
||||||
|
|
||||||
|
**Ergänzen:**
|
||||||
|
- `retriever_weight` (vererbt vom Note-Type)
|
||||||
|
- `chunk_profile` (aktuell ungenutzt im Retriever, aber Teil der Payload)
|
||||||
|
- Hinweise, dass der Retriever nur aus *Chunks* semantische Seeds bildet,
|
||||||
|
das Ranking aber auf *Notes* aggregiert wird.
|
||||||
|
|
||||||
|
## 2.2 Ergänzen: Section/Window-Felder und ihre Relevanz
|
||||||
|
Die Felder existieren im Qdrant, werden aber im retriever.yaml nicht verändert.
|
||||||
|
|
||||||
|
**Hinweis ergänzen:**
|
||||||
|
- Der Retriever nutzt aktuell `text` und `window`; kein Einfluss der Abschnittslogik.
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 3. Änderungen für: TYPE_REGISTRY_MANUAL.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
Dieses Dokument muss **aktualisiert** werden, da WP-04a folgende Mechanismen nutzt:
|
||||||
|
|
||||||
|
## 3.1 Ergänzen: Bedeutung von retriever_weight
|
||||||
|
- Definition als quantitatives Typ-Merkmal.
|
||||||
|
- Erklärung, dass es in Qdrant gespeichert wird.
|
||||||
|
- Erklärung des Einflusses auf Semantikscore:
|
||||||
|
- Formel:
|
||||||
|
|
||||||
|
final_semantic = semantic_score * retriever_weight
|
||||||
|
|
||||||
|
- Hinweis, dass in future WP-08 das Gewicht durch Feedback lernbar wird.
|
||||||
|
|
||||||
|
## 3.2 Ergänzen: Neue Edge-Typen und ihre Standard-Konfiguration
|
||||||
|
- `similar_to`
|
||||||
|
- `belongs_to`
|
||||||
|
- Strukturkanten `next` / `prev`
|
||||||
|
|
||||||
|
Diese sind im Importer bereits aktiv, aber nicht ausreichend dokumentiert.
|
||||||
|
|
||||||
|
## 3.3 Ergänzen: Edge-Konfidenzen
|
||||||
|
Die automatische Edge-Ableitung generiert `confidence`.
|
||||||
|
Retriever nutzt diesen Wert linear im Edge-Bonus.
|
||||||
|
|
||||||
|
**Dokumentation muss ergänzt werden.**
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 4. Änderungen für: docs_mindnet_retriever.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
Dieses Dokument ist am stärksten betroffen und muss auf den Stand der tatsächlichen
|
||||||
|
Implementierung aktualisiert werden.
|
||||||
|
|
||||||
|
## 4.1 Aktualisieren: Gesamtscoring-Formel
|
||||||
|
Bisher: Semantik + einfache Typgewichtung.
|
||||||
|
|
||||||
|
Neu implementiert:
|
||||||
|
|
||||||
|
total_score =
|
||||||
|
semantic_weight * semantic_score
|
||||||
|
+ edge_weight * edge_bonus
|
||||||
|
+ centrality_weight * centrality_bonus
|
||||||
|
|
||||||
|
Diese Formel muss vollständig dokumentiert werden.
|
||||||
|
|
||||||
|
## 4.2 Aktualisieren: Einführung retriever.yaml
|
||||||
|
Dokumentation benötigt:
|
||||||
|
|
||||||
|
- Speicherort: `app/config/retriever.yaml`
|
||||||
|
- Default-Werte
|
||||||
|
- Abschnittsweise Beschreibung:
|
||||||
|
- semantic_weight
|
||||||
|
- edge_weight
|
||||||
|
- centrality_weight
|
||||||
|
- Hinweis: Änderungen wirken ohne Codeänderung.
|
||||||
|
|
||||||
|
## 4.3 Ergänzen: Beschreibung Graph-Expansion
|
||||||
|
Der Retriever nutzt:
|
||||||
|
|
||||||
|
- Expand-depth (default 1)
|
||||||
|
- Edge-Typ-Filter
|
||||||
|
- Aggregation pro Node
|
||||||
|
- Bonus-Skalierung
|
||||||
|
|
||||||
|
Diese Logik ist neu und muss detailliert erklärt werden.
|
||||||
|
|
||||||
|
## 4.4 Ergänzen: Beschreibung centrality_bonus
|
||||||
|
Aktuell:
|
||||||
|
- proportionale Verteilung anhand Knotendichte
|
||||||
|
- Normalisierung über lokales Subgraph-Maß
|
||||||
|
|
||||||
|
Dokumentation muss Minimum enthalten:
|
||||||
|
- wie centrality gewichtet wird
|
||||||
|
- welche Grenze existieren
|
||||||
|
|
||||||
|
## 4.5 Ergänzen: Explainability-Schnittstelle (WP-04b Vorbereitung)
|
||||||
|
Die Implementierung erzeugt intern:
|
||||||
|
- edge_bonus pro Node
|
||||||
|
- centrality_bonus pro Node
|
||||||
|
- Liste der Edges, die beteiligt waren
|
||||||
|
|
||||||
|
Diese Werte sind abrufbar und müssen dokumentiert werden.
|
||||||
|
|
||||||
|
## 4.6 Ergänzen: Fehlertoleranzen & Timeout (Health Check)
|
||||||
|
Der Retriever kann:
|
||||||
|
- Fallback auf Vektor-Suche (pure semantic)
|
||||||
|
- Fallback bei fehlendem expand
|
||||||
|
|
||||||
|
Diese Sicherheitsmechanismen müssen dokumentiert werden.
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 5. Änderungen für: mindnet_technical_architecture.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 5.1 Ergänzen: Komponentenübersicht Retrieval-Layer
|
||||||
|
Eine neue Abbildung oder Abschnitt muss hinzu:
|
||||||
|
|
||||||
|
- QueryRequest-Parsing
|
||||||
|
- Embedding-Lookup oder Vector-Passthrough
|
||||||
|
- Semantic Seed Search (Qdrant)
|
||||||
|
- Graph-Expansion (mindnet_edges)
|
||||||
|
- Node-/Note-Aggregation
|
||||||
|
- Score-Combination
|
||||||
|
- Response-Ausgabe via FastAPI
|
||||||
|
|
||||||
|
## 5.2 Ergänzen: Datenstruktur mindnet_edges
|
||||||
|
Vor WP-04 nicht vollständig dokumentiert.
|
||||||
|
|
||||||
|
Benötigt:
|
||||||
|
|
||||||
|
- Felder:
|
||||||
|
- edge_id
|
||||||
|
- kind
|
||||||
|
- source_id
|
||||||
|
- target_id
|
||||||
|
- confidence
|
||||||
|
- rule_id
|
||||||
|
- Wie Edges im Retriever ausgewertet werden.
|
||||||
|
|
||||||
|
## 5.3 Ergänzen: retriever.yaml als Konfigurationsquelle
|
||||||
|
Architekturdiagramm muss updated werden:
|
||||||
|
- Retriever nutzt YAML
|
||||||
|
- Keine Hardcodierung im Python
|
||||||
|
|
||||||
|
## 5.4 Ergänzen: Health Check Script
|
||||||
|
Technischer Abschnitt muss dokumentieren:
|
||||||
|
|
||||||
|
- health_check_mindnet.py
|
||||||
|
- Semantik:
|
||||||
|
- semantic query
|
||||||
|
- hybrid query
|
||||||
|
- Fehlercodes
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 6. Änderungen für: mindnet_functional_architecture.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 6.1 Ergänzen: Funktionale Rolle des Retrievers
|
||||||
|
Retriever ist jetzt:
|
||||||
|
|
||||||
|
- Teil des Functional Layers
|
||||||
|
- Grundlage für Agenten / Decision Engine
|
||||||
|
- Hauptquelle für Begründungen
|
||||||
|
|
||||||
|
## 6.2 Ergänzen: Einbindung des Explainability-Layers
|
||||||
|
Funktional gehört dies zwischen:
|
||||||
|
- Retriever
|
||||||
|
- Antwortgenerierung (LLM)
|
||||||
|
|
||||||
|
Der Explainability-Layer formuliert Gründe:
|
||||||
|
- Welche Edges?
|
||||||
|
- Welche Scores?
|
||||||
|
- Warum diese Note?
|
||||||
|
|
||||||
|
## 6.3 Ergänzen: Self-Tuning als neuen Funktionsblock (WP-08)
|
||||||
|
Muss aufgenommen werden:
|
||||||
|
|
||||||
|
- Logging von Feedback
|
||||||
|
- Bewertung
|
||||||
|
- Tuning der retriever.yaml
|
||||||
|
- Human-in-the-loop Freigabe
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 7. Änderungen für: wp04_retriever_scoring.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
Dieses Dokument ist **direkt WP-04** und muss am stärksten überarbeitet werden.
|
||||||
|
|
||||||
|
## 7.1 Aktualisieren: Finale Scoring-Formel
|
||||||
|
Die Formel muss exakt so dokumentiert werden, wie inzwischen implementiert.
|
||||||
|
|
||||||
|
## 7.2 Ergänzen: Bonusberechnung
|
||||||
|
Die bisherige Planung wurde implementiert, aber der Text muss aktualisiert werden:
|
||||||
|
|
||||||
|
- Bonusberechnung je Edge
|
||||||
|
- confidence-Einfluss
|
||||||
|
- Node-Aggregation
|
||||||
|
- Note-Aggregation
|
||||||
|
|
||||||
|
## 7.3 Ergänzen: Centrality
|
||||||
|
Dokument ist zu ergänzen um:
|
||||||
|
|
||||||
|
- Definition
|
||||||
|
- Berechnung
|
||||||
|
- Normalisierung
|
||||||
|
- Gewichtung über retriever.yaml
|
||||||
|
|
||||||
|
## 7.4 Ergänzen: Teststrategie
|
||||||
|
Die finale Testbasis muss dokumentiert werden:
|
||||||
|
|
||||||
|
- Unit-Tests
|
||||||
|
- Hybrid-Tests
|
||||||
|
- API-Tests
|
||||||
|
- Smoke-Test
|
||||||
|
- Diagnose-Skripte
|
||||||
|
|
||||||
|
## 7.5 Ergänzen: Einschränkungen & Offene Punkte
|
||||||
|
Hinweis für WP-04b:
|
||||||
|
- Explainability fehlt noch
|
||||||
|
- Edge-Type Weights noch nicht implementiert
|
||||||
|
- Self-Tuning in WP-08
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 8. Änderungen für: mindnet_v2_implementation_playbook.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 8.1 Ergänzen: Arbeitsablauf mit Retriever
|
||||||
|
Einrichtung, Nutzung und Test sollte ergänzt werden:
|
||||||
|
|
||||||
|
- Starten von FastAPI
|
||||||
|
- Testen des Retrievers via health_check
|
||||||
|
- Import-Workflow (neue Vaults → Qdrant → Retriever)
|
||||||
|
|
||||||
|
## 8.2 Ergänzen: Konfigurationsverwaltung
|
||||||
|
Einbindung in Gitea-Workflows:
|
||||||
|
- retriever.yaml versionieren
|
||||||
|
- Tags/Branches für Funktionsstände setzen
|
||||||
|
|
||||||
|
## 8.3 Ergänzen: Self-Tuning-Prozess (nur Rahmen)
|
||||||
|
Der Playbook-Abschnitt soll die späteren Mechanismen vorbeschreiben.
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 9. Änderungen für: Handbuch.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 9.1 Ergänzen: Einfache Erklärung des Retrievers
|
||||||
|
Das Handbuch benötigt eine verständliche, nicht-technische Beschreibung:
|
||||||
|
|
||||||
|
- Was macht der Retriever?
|
||||||
|
- Warum kommt ein Ergebnis zustande?
|
||||||
|
- Welche Rolle spielen Edges?
|
||||||
|
- Wie kann der Nutzer die Gewichtung beeinflussen (YAML)?
|
||||||
|
|
||||||
|
## 9.2 Ergänzen: Troubleshooting
|
||||||
|
Aufnehmen:
|
||||||
|
- „Ergebnisse wirken unpassend“ → Prüfe retriever.yaml
|
||||||
|
- „Edge-Bonus fehlt“ → Prüfe mindnet_edges
|
||||||
|
- health_check-Skript nutzen
|
||||||
|
|
||||||
|
## 9.3 Ergänzen: API-Nutzung für Endanwender
|
||||||
|
Einfacher Abschnitt:
|
||||||
|
- Wie man `/query` testet
|
||||||
|
- Beispiele für semantic & hybrid queries
|
||||||
@@ -33,3 +33,7 @@ python3 scripts/import_markdown.py --vault /path/to/Obsidian
|
|||||||
- `POST /qdrant/query` → semantic search over chunks with optional filters
|
- `POST /qdrant/query` → semantic search over chunks with optional filters
|
||||||
|
|
||||||
See `scripts/quick_test.sh` for a runnable example.
|
See `scripts/quick_test.sh` for a runnable example.
|
||||||
|
|
||||||
|
>Anmerkung:
|
||||||
|
Diese Datei ist veraltet
|
||||||
|
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
"""
|
||||||
|
app — mindnet API package
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Markiert 'app/' als Python-Paket, damit 'from app.main import create_app'
|
||||||
|
in Tests und Skripten funktioniert.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Hinweise:
|
||||||
|
Keine Logik – nur Paketinitialisierung.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
||||||
+24
-1
@@ -1,9 +1,16 @@
|
|||||||
"""
|
"""
|
||||||
Version 0.1
|
app/config.py — zentrale Konfiguration (ENV → Settings)
|
||||||
|
|
||||||
|
Version:
|
||||||
|
0.3.1 (WP-05: Switch default to Mistral for CPU inference)
|
||||||
|
Stand:
|
||||||
|
2025-12-08
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import os
|
import os
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
class Settings:
|
class Settings:
|
||||||
# Qdrant
|
# Qdrant
|
||||||
@@ -12,10 +19,26 @@ class Settings:
|
|||||||
COLLECTION_PREFIX: str = os.getenv("MINDNET_PREFIX", "mindnet")
|
COLLECTION_PREFIX: str = os.getenv("MINDNET_PREFIX", "mindnet")
|
||||||
VECTOR_SIZE: int = int(os.getenv("MINDNET_VECTOR_SIZE", "384"))
|
VECTOR_SIZE: int = int(os.getenv("MINDNET_VECTOR_SIZE", "384"))
|
||||||
DISTANCE: str = os.getenv("MINDNET_DISTANCE", "Cosine")
|
DISTANCE: str = os.getenv("MINDNET_DISTANCE", "Cosine")
|
||||||
|
|
||||||
# Embeddings
|
# Embeddings
|
||||||
MODEL_NAME: str = os.getenv("MINDNET_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
|
MODEL_NAME: str = os.getenv("MINDNET_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
|
||||||
|
|
||||||
|
# WP-05 LLM / Ollama
|
||||||
|
OLLAMA_URL: str = os.getenv("MINDNET_OLLAMA_URL", "http://127.0.0.1:11434")
|
||||||
|
# ÄNDERUNG: Standard auf 'mistral' gesetzt, da bereits lokal vorhanden
|
||||||
|
LLM_MODEL: str = os.getenv("MINDNET_LLM_MODEL", "phi3:mini")
|
||||||
|
PROMPTS_PATH: str = os.getenv("MINDNET_PROMPTS_PATH", "config/prompts.yaml")
|
||||||
|
|
||||||
# API
|
# API
|
||||||
DEBUG: bool = os.getenv("DEBUG", "false").lower() == "true"
|
DEBUG: bool = os.getenv("DEBUG", "false").lower() == "true"
|
||||||
|
|
||||||
|
# WP-04 Retriever Defaults
|
||||||
|
RETRIEVER_W_SEM: float = float(os.getenv("MINDNET_WP04_W_SEM", "0.70"))
|
||||||
|
RETRIEVER_W_EDGE: float = float(os.getenv("MINDNET_WP04_W_EDGE", "0.25"))
|
||||||
|
RETRIEVER_W_CENT: float = float(os.getenv("MINDNET_WP04_W_CENT", "0.05"))
|
||||||
|
RETRIEVER_TOP_K: int = int(os.getenv("MINDNET_WP04_TOP_K", "10"))
|
||||||
|
RETRIEVER_EXPAND_DEPTH: int = int(os.getenv("MINDNET_WP04_EXPAND_DEPTH", "1"))
|
||||||
|
RETRIEVER_EDGE_TYPES: str = os.getenv("MINDNET_WP04_EDGE_TYPES", "references,belongs_to,prev,next")
|
||||||
|
|
||||||
@lru_cache
|
@lru_cache
|
||||||
def get_settings() -> Settings:
|
def get_settings() -> Settings:
|
||||||
|
|||||||
+115
-239
@@ -1,253 +1,129 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Modul: app/core/chunk_payload.py
|
app/core/chunk_payload.py (Mindnet V2 — types.yaml authoritative)
|
||||||
Version: 2.0.0
|
- neighbors_prev / neighbors_next sind Listen ([], [id]).
|
||||||
Datum: 2025-09-09
|
- retriever_weight / chunk_profile kommen aus types.yaml (Frontmatter wird ignoriert).
|
||||||
|
- Fallbacks: defaults.* in types.yaml; sonst 1.0 / "default".
|
||||||
Kurzbeschreibung
|
|
||||||
----------------
|
|
||||||
Erzeugt **Chunk-Payloads** für die Qdrant-Collection `<prefix>_chunks` auf Basis der
|
|
||||||
vom Chunker gelieferten `Chunk`-Objekte. Ziel:
|
|
||||||
- *Verlustfreie Rekonstruktion*: Jeder Chunk enthält seinen **Text** (payload["text"]).
|
|
||||||
- *Schnelle Abfragen*: Wichtige Note-Metadaten werden gespiegelt, um Filter ohne Join zu erlauben.
|
|
||||||
- *Graph-Kompatibilität*: Wikilinks und externe Links werden extrahiert; Nachbarschaften werden übernommen.
|
|
||||||
- *Monitoring*: Token- und Längenmetriken sowie Text-Hash erleichtern Audits und Re-Embeddings.
|
|
||||||
|
|
||||||
Kompatibilität
|
|
||||||
--------------
|
|
||||||
- **Abwärtskompatibel** zur bisherigen `make_chunk_payloads`-Signatur.
|
|
||||||
- Zusätzliche Felder stören bestehende Upserts nicht (Payload ist schema-flexibel).
|
|
||||||
- Erwartet, dass `Chunk` u. a. die Attribute `id`, `index`, `text`, `char_start`, `char_end`,
|
|
||||||
`section_title`, `section_path`, `neighbors_prev`, `neighbors_next` liefert.
|
|
||||||
|
|
||||||
CLI (Schnelltest)
|
|
||||||
-----------------
|
|
||||||
# Preview aus einer Markdown-Datei
|
|
||||||
python3 -m app.core.chunk_payload --from-file ./test_vault/20_experiences/exp-two.md --vault-root ./test_vault
|
|
||||||
|
|
||||||
# Nur IDs & Tokenmengen
|
|
||||||
python3 -m app.core.chunk_payload --from-file ./test_vault/20_experiences/exp-two.md --vault-root ./test_vault --summary
|
|
||||||
|
|
||||||
Felder (Auszug)
|
|
||||||
---------------
|
|
||||||
id : "<note_id>#cNN"
|
|
||||||
scope : "chunk"
|
|
||||||
note_id : "<note_id>"
|
|
||||||
note_title : str
|
|
||||||
note_type : str
|
|
||||||
note_status : str
|
|
||||||
area, project : optional
|
|
||||||
tags : list[str]
|
|
||||||
note_path : str (relativ, Slashes normalisiert)
|
|
||||||
chunk_index : int
|
|
||||||
section_title : str | None
|
|
||||||
section_path : str | None
|
|
||||||
char_start : int | None
|
|
||||||
char_end : int | None
|
|
||||||
char_len : int
|
|
||||||
token_est : int (≈ len(text)/4)
|
|
||||||
neighbors : {"prev": str|None, "next": str|None}
|
|
||||||
text : str (Chunk-Text, **Pflicht**)
|
|
||||||
text_sha256 : str "sha256:<hex>"
|
|
||||||
lang : optional
|
|
||||||
wikilinks : list[str]
|
|
||||||
external_links : list[{"href": str, "label": str|None}]
|
|
||||||
references : list[{"target_id": str, "kind": "wikilink"}]
|
|
||||||
embed_model : optional (durchreichbar)
|
|
||||||
embed_dim : optional
|
|
||||||
embed_version : optional
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
import os, yaml
|
||||||
|
|
||||||
import argparse
|
def _env(n: str, d: Optional[str]=None) -> str:
|
||||||
import hashlib
|
v = os.getenv(n)
|
||||||
import json
|
return v if v is not None else (d or "")
|
||||||
import os
|
|
||||||
import re
|
|
||||||
from typing import Dict, List, Optional, Tuple
|
|
||||||
|
|
||||||
try:
|
def _load_types() -> dict:
|
||||||
# Paket-Import (normaler Betrieb)
|
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||||
from app.core.chunker import Chunk
|
|
||||||
from app.core.parser import extract_wikilinks, read_markdown, normalize_frontmatter, validate_required_frontmatter
|
|
||||||
except Exception: # pragma: no cover
|
|
||||||
# Relativ (lokale Tests)
|
|
||||||
from .chunker import Chunk # type: ignore
|
|
||||||
from .parser import extract_wikilinks, read_markdown, normalize_frontmatter, validate_required_frontmatter # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Utilities
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
RE_MD_LINK = re.compile(r"\[([^\]]*)\]\(([^)\s]+)(?:\s+\"([^\"]+)\")?\)")
|
|
||||||
RE_HTTP_SCHEMES = ("http://", "https://", "mailto:", "obsidian://", "tel:")
|
|
||||||
|
|
||||||
def _estimate_tokens(text: str) -> int:
|
|
||||||
"""Grobe Token-Schätzung (≈ 1 Token pro 4 Zeichen)."""
|
|
||||||
return max(0, int(round(len((text or '').strip()) / 4)))
|
|
||||||
|
|
||||||
def _sha256_text(text: str) -> str:
|
|
||||||
h = hashlib.sha256()
|
|
||||||
h.update((text or "").encode("utf-8"))
|
|
||||||
return "sha256:" + h.hexdigest()
|
|
||||||
|
|
||||||
def _normalize_rel_path(path: Optional[str], vault_root: Optional[str]) -> Optional[str]:
|
|
||||||
if not path:
|
|
||||||
return None
|
|
||||||
p = str(path)
|
|
||||||
p = p.replace("\\", "/")
|
|
||||||
if vault_root and os.path.isabs(p):
|
|
||||||
try:
|
|
||||||
p = os.path.relpath(p, vault_root)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
p = p.replace("\\", "/").lstrip("/")
|
|
||||||
return p
|
|
||||||
|
|
||||||
def _extract_external_links(text: str) -> List[Dict[str, Optional[str]]]:
|
|
||||||
"""Findet Markdown-Links [label](href "title") mit erlaubten Schemes."""
|
|
||||||
out: List[Dict[str, Optional[str]]] = []
|
|
||||||
if not text:
|
|
||||||
return out
|
|
||||||
for m in RE_MD_LINK.finditer(text):
|
|
||||||
label = (m.group(1) or "").strip() or None
|
|
||||||
href = (m.group(2) or "").strip()
|
|
||||||
title = (m.group(3) or "").strip() or None
|
|
||||||
if any(href.startswith(s) for s in RE_HTTP_SCHEMES):
|
|
||||||
out.append({"href": href, "label": label or title})
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Public API
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def make_chunk_payloads(note_meta: Dict, path: str, chunks: List[Chunk]) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
Erzeugt Payload-Dicts für alle Chunks einer Note.
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
note_meta : Dict
|
|
||||||
Normalisierte Frontmatter der Note (mind.: id, title, type, status, tags, [area, project, lang]).
|
|
||||||
path : str
|
|
||||||
Pfad zur Note (sollte relativ zum Vault sein; wird hier zur Sicherheit normalisiert).
|
|
||||||
chunks : List[Chunk]
|
|
||||||
Vom Chunker erzeugte Chunks.
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
List[Dict]
|
|
||||||
Liste von Payloads (ein Eintrag pro Chunk).
|
|
||||||
"""
|
|
||||||
res: List[Dict] = []
|
|
||||||
rel_path = _normalize_rel_path(path, vault_root=None)
|
|
||||||
|
|
||||||
for ch in chunks:
|
|
||||||
text: str = getattr(ch, "text", "") or ""
|
|
||||||
wikilinks = extract_wikilinks(text)
|
|
||||||
ext_links = _extract_external_links(text)
|
|
||||||
|
|
||||||
payload: Dict = {
|
|
||||||
"id": getattr(ch, "id", None),
|
|
||||||
"scope": "chunk",
|
|
||||||
"note_id": note_meta.get("id"),
|
|
||||||
"note_title": note_meta.get("title"),
|
|
||||||
# gespiegelt für schnelle Filter:
|
|
||||||
"note_type": note_meta.get("type"),
|
|
||||||
"note_status": note_meta.get("status"),
|
|
||||||
"area": note_meta.get("area"),
|
|
||||||
"project": note_meta.get("project"),
|
|
||||||
"tags": note_meta.get("tags"),
|
|
||||||
# Pfad
|
|
||||||
"note_path": rel_path,
|
|
||||||
"path": rel_path, # Back-compat
|
|
||||||
# Reihenfolge & Section
|
|
||||||
"chunk_index": getattr(ch, "index", None),
|
|
||||||
"section_title": getattr(ch, "section_title", None),
|
|
||||||
"section_path": getattr(ch, "section_path", None),
|
|
||||||
# Position
|
|
||||||
"char_start": getattr(ch, "char_start", None),
|
|
||||||
"char_end": getattr(ch, "char_end", None),
|
|
||||||
"char_len": max(0, int(getattr(ch, "char_end", 0) or 0) - int(getattr(ch, "char_start", 0) or 0)) or len(text),
|
|
||||||
# Nachbarn
|
|
||||||
"neighbors": {
|
|
||||||
"prev": getattr(ch, "neighbors_prev", None),
|
|
||||||
"next": getattr(ch, "neighbors_next", None),
|
|
||||||
},
|
|
||||||
# Inhalt & Metrik
|
|
||||||
"text": text,
|
|
||||||
"text_sha256": _sha256_text(text),
|
|
||||||
"token_est": _estimate_tokens(text),
|
|
||||||
# Sprache
|
|
||||||
"lang": note_meta.get("lang"),
|
|
||||||
# Links
|
|
||||||
"wikilinks": wikilinks,
|
|
||||||
"external_links": ext_links,
|
|
||||||
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
|
|
||||||
}
|
|
||||||
|
|
||||||
# Entferne Nones/Leeres, aber **text** bleibt (darf leer sein z. B. bei Bild-Only-Chunks)
|
|
||||||
cleaned = {}
|
|
||||||
for k, v in payload.items():
|
|
||||||
if v in (None, [], {}):
|
|
||||||
# immer behalten:
|
|
||||||
if k in ("text", "neighbors"):
|
|
||||||
cleaned[k] = v
|
|
||||||
else:
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
cleaned[k] = v
|
|
||||||
|
|
||||||
res.append(cleaned)
|
|
||||||
|
|
||||||
return res
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# CLI zum schnellen Testen
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def _cli() -> None:
|
|
||||||
ap = argparse.ArgumentParser(description="Chunk-Payloads aus einer einzelnen Markdown-Datei erzeugen")
|
|
||||||
ap.add_argument("--from-file", required=True, help="Pfad zur Markdown-Datei")
|
|
||||||
ap.add_argument("--vault-root", default=None, help="Vault-Wurzel (zur Pfad-Relativierung)")
|
|
||||||
ap.add_argument("--summary", action="store_true", help="Nur kurze Übersicht je Chunk ausgeben")
|
|
||||||
args = ap.parse_args()
|
|
||||||
|
|
||||||
parsed = read_markdown(args.from_file)
|
|
||||||
fm = normalize_frontmatter(parsed.frontmatter)
|
|
||||||
validate_required_frontmatter(fm)
|
|
||||||
|
|
||||||
# lazy import, um Zyklen zu vermeiden
|
|
||||||
try:
|
try:
|
||||||
from app.core.chunker import assemble_chunks
|
with open(p, "r", encoding="utf-8") as f:
|
||||||
|
return yaml.safe_load(f) or {}
|
||||||
except Exception:
|
except Exception:
|
||||||
from .chunker import assemble_chunks # type: ignore
|
return {}
|
||||||
|
|
||||||
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
|
def _get_types_map(reg: dict) -> dict:
|
||||||
rel = _normalize_rel_path(parsed.path, args.vault_root)
|
if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
|
||||||
|
return reg["types"]
|
||||||
|
return reg if isinstance(reg, dict) else {}
|
||||||
|
|
||||||
pls = make_chunk_payloads(fm, rel or parsed.path, chunks)
|
def _get_defaults(reg: dict) -> dict:
|
||||||
|
if isinstance(reg, dict) and isinstance(reg.get("defaults"), dict):
|
||||||
|
return reg["defaults"]
|
||||||
|
if isinstance(reg, dict) and isinstance(reg.get("global"), dict):
|
||||||
|
return reg["global"]
|
||||||
|
return {}
|
||||||
|
|
||||||
if args.summary:
|
def _as_float(x: Any):
|
||||||
out = []
|
try:
|
||||||
for p in pls:
|
return float(x)
|
||||||
out.append({
|
except Exception:
|
||||||
"id": p.get("id"),
|
return None
|
||||||
"chunk_index": p.get("chunk_index"),
|
|
||||||
"token_est": p.get("token_est"),
|
|
||||||
"wikilinks": p.get("wikilinks"),
|
|
||||||
"ext_links": [e.get("href") for e in p.get("external_links", [])],
|
|
||||||
"prev": (p.get("neighbors") or {}).get("prev"),
|
|
||||||
"next": (p.get("neighbors") or {}).get("next"),
|
|
||||||
})
|
|
||||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
|
||||||
else:
|
|
||||||
print(json.dumps(pls, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
|
def _resolve_chunk_profile(note_type: str, reg: dict) -> str:
|
||||||
|
types = _get_types_map(reg)
|
||||||
|
if isinstance(types, dict):
|
||||||
|
t = types.get(note_type, {})
|
||||||
|
if isinstance(t, dict) and isinstance(t.get("chunk_profile"), str):
|
||||||
|
return t["chunk_profile"]
|
||||||
|
defs = _get_defaults(reg)
|
||||||
|
if isinstance(defs, dict) and isinstance(defs.get("chunk_profile"), str):
|
||||||
|
return defs["chunk_profile"]
|
||||||
|
return "default"
|
||||||
|
|
||||||
if __name__ == "__main__": # pragma: no cover
|
def _resolve_retriever_weight(note_type: str, reg: dict) -> float:
|
||||||
_cli()
|
types = _get_types_map(reg)
|
||||||
|
if isinstance(types, dict):
|
||||||
|
t = types.get(note_type, {})
|
||||||
|
if isinstance(t, dict) and (t.get("retriever_weight") is not None):
|
||||||
|
v = _as_float(t.get("retriever_weight"))
|
||||||
|
if v is not None:
|
||||||
|
return float(v)
|
||||||
|
defs = _get_defaults(reg)
|
||||||
|
if isinstance(defs, dict) and (defs.get("retriever_weight") is not None):
|
||||||
|
v = _as_float(defs.get("retriever_weight"))
|
||||||
|
if v is not None:
|
||||||
|
return float(v)
|
||||||
|
return 1.0
|
||||||
|
|
||||||
|
def _as_list(x):
|
||||||
|
if x is None:
|
||||||
|
return []
|
||||||
|
if isinstance(x, list):
|
||||||
|
return x
|
||||||
|
return [x]
|
||||||
|
|
||||||
|
def make_chunk_payloads(note: Dict[str, Any],
|
||||||
|
note_path: str,
|
||||||
|
chunks_from_chunker: List[Any],
|
||||||
|
*,
|
||||||
|
note_text: str = "",
|
||||||
|
types_cfg: Optional[dict] = None,
|
||||||
|
file_path: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||||
|
fm = (note or {}).get("frontmatter", {}) or {}
|
||||||
|
note_type = fm.get("type") or note.get("type") or "concept"
|
||||||
|
reg = types_cfg if isinstance(types_cfg, dict) else _load_types()
|
||||||
|
|
||||||
|
# types.yaml authoritative
|
||||||
|
cp = _resolve_chunk_profile(note_type, reg)
|
||||||
|
rw = _resolve_retriever_weight(note_type, reg)
|
||||||
|
|
||||||
|
tags = fm.get("tags") or []
|
||||||
|
if isinstance(tags, str):
|
||||||
|
tags = [tags]
|
||||||
|
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for idx, ch in enumerate(chunks_from_chunker):
|
||||||
|
# Attribute oder Keys (Chunk-Objekt oder Dict)
|
||||||
|
cid = getattr(ch, "id", None) or (ch.get("id") if isinstance(ch, dict) else None)
|
||||||
|
nid = getattr(ch, "note_id", None) or (ch.get("note_id") if isinstance(ch, dict) else fm.get("id"))
|
||||||
|
index = getattr(ch, "index", None) or (ch.get("index") if isinstance(ch, dict) else idx)
|
||||||
|
text = getattr(ch, "text", None) or (ch.get("text") if isinstance(ch, dict) else "")
|
||||||
|
window = getattr(ch, "window", None) or (ch.get("window") if isinstance(ch, dict) else text)
|
||||||
|
prev_id = getattr(ch, "neighbors_prev", None) or (ch.get("neighbors_prev") if isinstance(ch, dict) else None)
|
||||||
|
next_id = getattr(ch, "neighbors_next", None) or (ch.get("neighbors_next") if isinstance(ch, dict) else None)
|
||||||
|
|
||||||
|
pl: Dict[str, Any] = {
|
||||||
|
"note_id": nid,
|
||||||
|
"chunk_id": cid,
|
||||||
|
"index": int(index),
|
||||||
|
"ord": int(index) + 1,
|
||||||
|
"type": note_type,
|
||||||
|
"tags": tags,
|
||||||
|
"text": text,
|
||||||
|
"window": window,
|
||||||
|
"neighbors_prev": _as_list(prev_id),
|
||||||
|
"neighbors_next": _as_list(next_id),
|
||||||
|
"section": getattr(ch, "section", None) or (ch.get("section") if isinstance(ch, dict) else ""),
|
||||||
|
"path": note_path,
|
||||||
|
"source_path": file_path or note_path,
|
||||||
|
"retriever_weight": float(rw),
|
||||||
|
"chunk_profile": cp,
|
||||||
|
}
|
||||||
|
# Aufräumen von Alt-Feldern
|
||||||
|
for alias in ("chunk_num", "Chunk_Number"):
|
||||||
|
pl.pop(alias, None)
|
||||||
|
|
||||||
|
out.append(pl)
|
||||||
|
|
||||||
|
return out
|
||||||
+418
-109
@@ -1,126 +1,435 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Name: app/core/derive_edges.py
|
Modul: app/core/derive_edges.py
|
||||||
Version: v1.1.0 (2025-09-05)
|
Zweck:
|
||||||
|
- Bewahrt bestehende Edgelogik (belongs_to, prev/next, references, backlink)
|
||||||
Kurzbeschreibung
|
- Ergänzt typenbasierte Default-Kanten (edge_defaults aus config/types.yaml)
|
||||||
Leitet Edges aus Wikilinks ([[…]]) ab und löst Zielnoten robust auf.
|
- Unterstützt "typed inline relations":
|
||||||
Erzeugt:
|
* [[rel:KIND | Target]]
|
||||||
- "references" (Note -> Note) mit seq="body", pro Match eine eigene Occurrence 'occ'
|
* [[rel:KIND Target]]
|
||||||
- "backlink" (inverse zu "references", gleiche seq/occ)
|
* rel: KIND [[Target]]
|
||||||
- "references_at" (Chunk -> Note) mit seq=<chunk_index> und eigener 'occ' je Match
|
- Unterstützt Obsidian-Callouts:
|
||||||
|
* > [!edge] KIND: [[Target]] [[Target2]] ...
|
||||||
Aufruf
|
Kompatibilität:
|
||||||
from app.core.derive_edges import build_note_index, derive_wikilink_edges
|
- build_edges_for_note(...) Signatur unverändert
|
||||||
|
- rule_id Werte:
|
||||||
Parameter / Felder
|
* structure:belongs_to
|
||||||
- note_payload: {"note_id","title","path","fulltext": <body> , …}
|
* structure:order
|
||||||
- chunks_payloads: [{"chunk_id","text",…}, …]
|
* explicit:wikilink
|
||||||
- note_index: build_note_index([...]) -> (by_id, by_slug, by_file_slug)
|
* inline:rel
|
||||||
|
* callout:edge
|
||||||
Kompatibilität
|
* edge_defaults:<type>:<relation>
|
||||||
- Rückwärtskompatible Payload-Felder, nur erweitert um 'seq' und 'occ'.
|
* derived:backlink
|
||||||
|
|
||||||
Changelog
|
|
||||||
v1.1.0: Occurrence-Zählung ('occ') je Match; 'seq="body"' für references.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import unicodedata
|
from typing import Iterable, List, Optional, Tuple, Set, Dict
|
||||||
from typing import Dict, List, Tuple
|
|
||||||
|
|
||||||
# [[Ziel]], [[Ziel|Alias]], [[Ziel#Heading]], [[Ziel#Heading|Alias]]
|
try:
|
||||||
WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#([^\]|]+))?(?:\|([^\]]+))?\]\]")
|
import yaml # optional, nur für types.yaml
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
yaml = None
|
||||||
|
|
||||||
def _slug(s: str) -> str:
|
# --------------------------------------------------------------------------- #
|
||||||
s = s.strip()
|
# Utilities
|
||||||
if s.endswith(".md"):
|
# --------------------------------------------------------------------------- #
|
||||||
s = s[:-3]
|
|
||||||
s = unicodedata.normalize("NFKD", s)
|
|
||||||
s = "".join(ch for ch in s if not unicodedata.combining(ch))
|
|
||||||
s = s.replace("\\", "/")
|
|
||||||
s = s.split("/")[-1]
|
|
||||||
s = s.lower().replace(" ", "-")
|
|
||||||
s = re.sub(r"[^a-z0-9\-]+", "", s)
|
|
||||||
s = re.sub(r"-{2,}", "-", s).strip("-")
|
|
||||||
return s
|
|
||||||
|
|
||||||
def build_note_index(notes_payloads: List[dict]) -> Tuple[Dict[str, dict], Dict[str, dict], Dict[str, dict]]:
|
def _get(d: dict, *keys, default=None):
|
||||||
by_id: Dict[str, dict] = {}
|
for k in keys:
|
||||||
by_slug: Dict[str, dict] = {}
|
if isinstance(d, dict) and k in d and d[k] is not None:
|
||||||
by_file_slug: Dict[str, dict] = {}
|
return d[k]
|
||||||
for n in notes_payloads:
|
return default
|
||||||
nid = n.get("note_id") or n.get("id")
|
|
||||||
if not nid:
|
def _chunk_text_for_refs(chunk: dict) -> str:
|
||||||
|
# bevorzugt 'window' → dann 'text' → 'content' → 'raw'
|
||||||
|
return (
|
||||||
|
_get(chunk, "window")
|
||||||
|
or _get(chunk, "text")
|
||||||
|
or _get(chunk, "content")
|
||||||
|
or _get(chunk, "raw")
|
||||||
|
or ""
|
||||||
|
)
|
||||||
|
|
||||||
|
def _dedupe_seq(seq: Iterable[str]) -> List[str]:
|
||||||
|
seen: Set[str] = set()
|
||||||
|
out: List[str] = []
|
||||||
|
for s in seq:
|
||||||
|
if s not in seen:
|
||||||
|
seen.add(s)
|
||||||
|
out.append(s)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
|
||||||
|
pl = {
|
||||||
|
"kind": kind,
|
||||||
|
"relation": kind, # Alias (v2)
|
||||||
|
"scope": scope, # "chunk" | "note"
|
||||||
|
"source_id": source_id,
|
||||||
|
"target_id": target_id,
|
||||||
|
"note_id": note_id, # Träger-Note der Kante
|
||||||
|
}
|
||||||
|
if extra:
|
||||||
|
pl.update(extra)
|
||||||
|
return pl
|
||||||
|
|
||||||
|
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None) -> str:
|
||||||
|
base = f"{kind}:{s}->{t}#{scope}"
|
||||||
|
if rule_id:
|
||||||
|
base += f"|{rule_id}"
|
||||||
|
try:
|
||||||
|
import hashlib
|
||||||
|
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
return base
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Typen-Registry (types.yaml)
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
def _env(n: str, default: Optional[str] = None) -> str:
|
||||||
|
v = os.getenv(n)
|
||||||
|
return v if v is not None else (default or "")
|
||||||
|
|
||||||
|
def _load_types_registry() -> dict:
|
||||||
|
"""Lädt die YAML-Registry aus MINDNET_TYPES_FILE oder ./config/types.yaml"""
|
||||||
|
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||||
|
if not os.path.isfile(p) or yaml is None:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
with open(p, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def _get_types_map(reg: dict) -> dict:
|
||||||
|
if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
|
||||||
|
return reg["types"]
|
||||||
|
return reg if isinstance(reg, dict) else {}
|
||||||
|
|
||||||
|
def _edge_defaults_for(note_type: Optional[str], reg: dict) -> List[str]:
|
||||||
|
"""
|
||||||
|
Liefert die edge_defaults-Liste für den gegebenen Notiztyp.
|
||||||
|
Fallback-Reihenfolge:
|
||||||
|
1) reg['types'][note_type]['edge_defaults']
|
||||||
|
2) reg['defaults']['edge_defaults'] (oder 'default'/'global')
|
||||||
|
3) []
|
||||||
|
"""
|
||||||
|
types_map = _get_types_map(reg)
|
||||||
|
if note_type and isinstance(types_map, dict):
|
||||||
|
t = types_map.get(note_type)
|
||||||
|
if isinstance(t, dict) and isinstance(t.get("edge_defaults"), list):
|
||||||
|
return [str(x) for x in t["edge_defaults"] if isinstance(x, str)]
|
||||||
|
for key in ("defaults", "default", "global"):
|
||||||
|
v = reg.get(key)
|
||||||
|
if isinstance(v, dict) and isinstance(v.get("edge_defaults"), list):
|
||||||
|
return [str(x) for x in v["edge_defaults"] if isinstance(x, str)]
|
||||||
|
return []
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Parser für Links / Relationen
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
# Normale Wikilinks (Fallback)
|
||||||
|
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([a-zA-Z0-9_\-#:. ]+)\]\]")
|
||||||
|
|
||||||
|
# Getypte Inline-Relationen:
|
||||||
|
# [[rel:KIND | Target]]
|
||||||
|
# [[rel:KIND Target]]
|
||||||
|
_REL_PIPE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s*\|\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||||
|
_REL_SPACE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s+(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||||
|
# rel: KIND [[Target]] (reines Textmuster)
|
||||||
|
_REL_TEXT = re.compile(r"rel\s*:\s*(?P<kind>[a-z_]+)\s*\[\[\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||||
|
|
||||||
|
def _extract_typed_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||||
|
"""
|
||||||
|
Gibt Liste (kind, target) zurück und den Text mit entfernten getypten Relation-Links,
|
||||||
|
damit die generische Wikilink-Erkennung sie nicht doppelt zählt.
|
||||||
|
Unterstützt drei Varianten:
|
||||||
|
- [[rel:KIND | Target]]
|
||||||
|
- [[rel:KIND Target]]
|
||||||
|
- rel: KIND [[Target]]
|
||||||
|
"""
|
||||||
|
pairs: List[Tuple[str,str]] = []
|
||||||
|
def _collect(m):
|
||||||
|
k = (m.group("kind") or "").strip().lower()
|
||||||
|
t = (m.group("target") or "").strip()
|
||||||
|
if k and t:
|
||||||
|
pairs.append((k, t))
|
||||||
|
return "" # Link entfernen
|
||||||
|
|
||||||
|
text = _REL_PIPE.sub(_collect, text)
|
||||||
|
text = _REL_SPACE.sub(_collect, text)
|
||||||
|
text = _REL_TEXT.sub(_collect, text)
|
||||||
|
return pairs, text
|
||||||
|
|
||||||
|
# Obsidian Callout Parser
|
||||||
|
_CALLOUT_START = re.compile(r"^\s*>\s*\[!edge\]\s*(.*)$", re.IGNORECASE)
|
||||||
|
_REL_LINE = re.compile(r"^(?P<kind>[a-z_]+)\s*:\s*(?P<targets>.+?)\s*$", re.IGNORECASE)
|
||||||
|
_WIKILINKS_IN_LINE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||||
|
|
||||||
|
def _extract_callout_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||||
|
"""
|
||||||
|
Findet [!edge]-Callouts und extrahiert (kind, target). Entfernt den gesamten
|
||||||
|
Callout-Block aus dem Text (damit Wikilinks daraus nicht zusätzlich als
|
||||||
|
"references" gezählt werden).
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return [], text
|
||||||
|
|
||||||
|
lines = text.splitlines()
|
||||||
|
out_pairs: List[Tuple[str,str]] = []
|
||||||
|
keep_lines: List[str] = []
|
||||||
|
i = 0
|
||||||
|
|
||||||
|
while i < len(lines):
|
||||||
|
m = _CALLOUT_START.match(lines[i])
|
||||||
|
if not m:
|
||||||
|
keep_lines.append(lines[i])
|
||||||
|
i += 1
|
||||||
continue
|
continue
|
||||||
by_id[nid] = n
|
|
||||||
title = n.get("title", "")
|
|
||||||
path = n.get("path", "")
|
|
||||||
file_slug = _slug(path.split("/")[-1]) if path else ""
|
|
||||||
if title:
|
|
||||||
by_slug[_slug(title)] = n
|
|
||||||
if file_slug:
|
|
||||||
by_file_slug[file_slug] = n
|
|
||||||
return by_id, by_slug, by_file_slug
|
|
||||||
|
|
||||||
def resolve_target(note_like: str, idx: Tuple[Dict[str,dict],Dict[str,dict],Dict[str,dict]]):
|
block_lines: List[str] = []
|
||||||
by_id, by_slug, by_file_slug = idx
|
first_rest = m.group(1) or ""
|
||||||
key = note_like.strip()
|
if first_rest.strip():
|
||||||
if key in by_id:
|
block_lines.append(first_rest)
|
||||||
return by_id[key]["note_id"], "by_id"
|
|
||||||
s = _slug(key)
|
|
||||||
if s in by_slug:
|
|
||||||
return by_slug[s]["note_id"], "by_slug"
|
|
||||||
if s in by_file_slug:
|
|
||||||
return by_file_slug[s]["note_id"], "by_file_slug"
|
|
||||||
return None, "unresolved"
|
|
||||||
|
|
||||||
def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_index) -> List[dict]:
|
i += 1
|
||||||
|
while i < len(lines) and lines[i].lstrip().startswith('>'):
|
||||||
|
block_lines.append(lines[i].lstrip()[1:].lstrip())
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
for bl in block_lines:
|
||||||
|
mrel = _REL_LINE.match(bl)
|
||||||
|
if not mrel:
|
||||||
|
continue
|
||||||
|
kind = (mrel.group("kind") or "").strip().lower()
|
||||||
|
targets = mrel.group("targets") or ""
|
||||||
|
found = _WIKILINKS_IN_LINE.findall(targets)
|
||||||
|
if found:
|
||||||
|
for t in found:
|
||||||
|
t = t.strip()
|
||||||
|
if t:
|
||||||
|
out_pairs.append((kind, t))
|
||||||
|
else:
|
||||||
|
for raw in re.split(r"[,;]", targets):
|
||||||
|
t = raw.strip()
|
||||||
|
if t:
|
||||||
|
out_pairs.append((kind, t))
|
||||||
|
|
||||||
|
# Callout wird NICHT in keep_lines übernommen
|
||||||
|
continue
|
||||||
|
|
||||||
|
remainder = "\n".join(keep_lines)
|
||||||
|
return out_pairs, remainder
|
||||||
|
|
||||||
|
def _extract_wikilinks(text: str) -> List[str]:
|
||||||
|
ids: List[str] = []
|
||||||
|
for m in _WIKILINK_RE.finditer(text or ""):
|
||||||
|
ids.append(m.group(1).strip())
|
||||||
|
return ids
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Hauptfunktion
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
def build_edges_for_note(
|
||||||
|
note_id: str,
|
||||||
|
chunks: List[dict],
|
||||||
|
note_level_references: Optional[List[str]] = None,
|
||||||
|
include_note_scope_refs: bool = False,
|
||||||
|
) -> List[dict]:
|
||||||
|
"""
|
||||||
|
Erzeugt Kanten für eine Note.
|
||||||
|
|
||||||
|
- belongs_to: für jeden Chunk (chunk -> note)
|
||||||
|
- next / prev: zwischen aufeinanderfolgenden Chunks
|
||||||
|
- references: pro Chunk aus window/text (via Wikilinks)
|
||||||
|
- typed inline relations: [[rel:KIND | Target]] / [[rel:KIND Target]] / rel: KIND [[Target]]
|
||||||
|
- Obsidian Callouts: > [!edge] KIND: [[Target]] [[Target2]]
|
||||||
|
- optional note-scope references/backlinks: dedupliziert über alle Chunk-Funde + note_level_references
|
||||||
|
- typenbasierte Default-Kanten (edge_defaults) je gefundener Referenz
|
||||||
|
"""
|
||||||
edges: List[dict] = []
|
edges: List[dict] = []
|
||||||
source_note_id = note_payload["note_id"]
|
|
||||||
|
|
||||||
def _make_edge(kind: str, src: str, tgt: str, seq=None, occ=None, extra: dict|None=None):
|
# Note-Typ (aus erstem Chunk erwartet)
|
||||||
e = {"edge_id": None, "kind": kind, "source_id": src, "target_id": tgt}
|
note_type = None
|
||||||
if seq is not None:
|
if chunks:
|
||||||
e["seq"] = seq
|
note_type = _get(chunks[0], "type")
|
||||||
if occ is not None:
|
|
||||||
e["occ"] = occ
|
|
||||||
if extra:
|
|
||||||
e.update(extra)
|
|
||||||
return e
|
|
||||||
|
|
||||||
# Volltext (Note-Ebene)
|
# 1) belongs_to
|
||||||
fulltext = note_payload.get("fulltext") or note_payload.get("body") or ""
|
for ch in chunks:
|
||||||
if fulltext:
|
cid = _get(ch, "chunk_id", "id")
|
||||||
for k, m in enumerate(WIKILINK_RE.finditer(fulltext), start=1):
|
if not cid:
|
||||||
raw_target, heading, alias = m.groups()
|
|
||||||
target_id, how = resolve_target(raw_target, note_index)
|
|
||||||
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
|
|
||||||
if target_id:
|
|
||||||
edges.append(_make_edge("references", source_note_id, target_id, seq="body", occ=k, extra=extra))
|
|
||||||
edges.append(_make_edge("backlink", target_id, source_note_id, seq="body", occ=k, extra=extra))
|
|
||||||
else:
|
|
||||||
extra["status"] = "unresolved"
|
|
||||||
extra["target_label"] = raw_target
|
|
||||||
edges.append(_make_edge("references", source_note_id, raw_target, seq="body", occ=k, extra=extra))
|
|
||||||
|
|
||||||
# Chunks (Chunk-Ebene)
|
|
||||||
for i, ch in enumerate(chunks_payloads, start=1):
|
|
||||||
txt = ch.get("text") or ch.get("content") or ""
|
|
||||||
if not txt:
|
|
||||||
continue
|
continue
|
||||||
occ = 0
|
edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {
|
||||||
for m in WIKILINK_RE.finditer(txt):
|
"chunk_id": cid,
|
||||||
occ += 1
|
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to"),
|
||||||
raw_target, heading, alias = m.groups()
|
"provenance": "rule",
|
||||||
target_id, how = resolve_target(raw_target, note_index)
|
"rule_id": "structure:belongs_to",
|
||||||
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
|
"confidence": 1.0,
|
||||||
if target_id:
|
}))
|
||||||
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, occ=occ, extra=extra))
|
|
||||||
else:
|
# 2) next / prev
|
||||||
extra["status"] = "unresolved"
|
for i in range(len(chunks) - 1):
|
||||||
extra["target_label"] = raw_target
|
a, b = chunks[i], chunks[i + 1]
|
||||||
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, occ=occ, extra=extra))
|
a_id = _get(a, "chunk_id", "id")
|
||||||
return edges
|
b_id = _get(b, "chunk_id", "id")
|
||||||
|
if not a_id or not b_id:
|
||||||
|
continue
|
||||||
|
edges.append(_edge("next", "chunk", a_id, b_id, note_id, {
|
||||||
|
"chunk_id": a_id,
|
||||||
|
"edge_id": _mk_edge_id("next", a_id, b_id, "chunk", "structure:order"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": "structure:order",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
edges.append(_edge("prev", "chunk", b_id, a_id, note_id, {
|
||||||
|
"chunk_id": b_id,
|
||||||
|
"edge_id": _mk_edge_id("prev", b_id, a_id, "chunk", "structure:order"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": "structure:order",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
|
||||||
|
# 3) references + typed inline + callouts + defaults (chunk-scope)
|
||||||
|
reg = _load_types_registry()
|
||||||
|
defaults = _edge_defaults_for(note_type, reg)
|
||||||
|
refs_all: List[str] = []
|
||||||
|
|
||||||
|
for ch in chunks:
|
||||||
|
cid = _get(ch, "chunk_id", "id")
|
||||||
|
if not cid:
|
||||||
|
continue
|
||||||
|
raw = _chunk_text_for_refs(ch)
|
||||||
|
|
||||||
|
# 3a) typed inline relations
|
||||||
|
typed, remainder = _extract_typed_relations(raw)
|
||||||
|
for kind, target in typed:
|
||||||
|
kind = kind.strip().lower()
|
||||||
|
if not kind or not target:
|
||||||
|
continue
|
||||||
|
edges.append(_edge(kind, "chunk", cid, target, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(kind, cid, target, "chunk", "inline:rel"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "inline:rel",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
if kind in {"related_to", "similar_to"}:
|
||||||
|
edges.append(_edge(kind, "chunk", target, cid, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(kind, target, cid, "chunk", "inline:rel"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "inline:rel",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
|
||||||
|
# 3b) callouts
|
||||||
|
call_pairs, remainder2 = _extract_callout_relations(remainder)
|
||||||
|
for kind, target in call_pairs:
|
||||||
|
k = (kind or "").strip().lower()
|
||||||
|
if not k or not target:
|
||||||
|
continue
|
||||||
|
edges.append(_edge(k, "chunk", cid, target, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(k, cid, target, "chunk", "callout:edge"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "callout:edge",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
if k in {"related_to", "similar_to"}:
|
||||||
|
edges.append(_edge(k, "chunk", target, cid, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(k, target, cid, "chunk", "callout:edge"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "callout:edge",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
|
||||||
|
# 3c) generische Wikilinks → references (+ defaults je Ref)
|
||||||
|
refs = _extract_wikilinks(remainder2)
|
||||||
|
for r in refs:
|
||||||
|
edges.append(_edge("references", "chunk", cid, r, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"ref_text": r,
|
||||||
|
"edge_id": _mk_edge_id("references", cid, r, "chunk", "explicit:wikilink"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "explicit:wikilink",
|
||||||
|
"confidence": 1.0,
|
||||||
|
}))
|
||||||
|
for rel in defaults:
|
||||||
|
if rel == "references":
|
||||||
|
continue
|
||||||
|
edges.append(_edge(rel, "chunk", cid, r, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(rel, cid, r, "chunk", f"edge_defaults:{note_type}:{rel}"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||||
|
"confidence": 0.7,
|
||||||
|
}))
|
||||||
|
if rel in {"related_to", "similar_to"}:
|
||||||
|
edges.append(_edge(rel, "chunk", r, cid, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(rel, r, cid, "chunk", f"edge_defaults:{note_type}:{rel}"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||||
|
"confidence": 0.7,
|
||||||
|
}))
|
||||||
|
|
||||||
|
refs_all.extend(refs)
|
||||||
|
|
||||||
|
# 4) optional note-scope refs/backlinks (+ defaults)
|
||||||
|
if include_note_scope_refs:
|
||||||
|
refs_note = list(refs_all or [])
|
||||||
|
if note_level_references:
|
||||||
|
refs_note.extend([r for r in note_level_references if isinstance(r, str) and r])
|
||||||
|
refs_note = _dedupe_seq(refs_note)
|
||||||
|
for r in refs_note:
|
||||||
|
edges.append(_edge("references", "note", note_id, r, note_id, {
|
||||||
|
"edge_id": _mk_edge_id("references", note_id, r, "note", "explicit:note_scope"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "explicit:note_scope",
|
||||||
|
"confidence": 1.0,
|
||||||
|
}))
|
||||||
|
edges.append(_edge("backlink", "note", r, note_id, note_id, {
|
||||||
|
"edge_id": _mk_edge_id("backlink", r, note_id, "note", "derived:backlink"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": "derived:backlink",
|
||||||
|
"confidence": 0.9,
|
||||||
|
}))
|
||||||
|
for rel in defaults:
|
||||||
|
if rel == "references":
|
||||||
|
continue
|
||||||
|
edges.append(_edge(rel, "note", note_id, r, note_id, {
|
||||||
|
"edge_id": _mk_edge_id(rel, note_id, r, "note", f"edge_defaults:{note_type}:{rel}"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||||
|
"confidence": 0.7,
|
||||||
|
}))
|
||||||
|
if rel in {"related_to", "similar_to"}:
|
||||||
|
edges.append(_edge(rel, "note", r, note_id, note_id, {
|
||||||
|
"edge_id": _mk_edge_id(rel, r, note_id, "note", f"edge_defaults:{note_type}:{rel}"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||||
|
"confidence": 0.7,
|
||||||
|
}))
|
||||||
|
|
||||||
|
# 5) De-Dupe (source_id, target_id, relation, rule_id)
|
||||||
|
seen: Set[Tuple[str,str,str,str]] = set()
|
||||||
|
out: List[dict] = []
|
||||||
|
for e in edges:
|
||||||
|
s = str(e.get("source_id") or "")
|
||||||
|
t = str(e.get("target_id") or "")
|
||||||
|
rel = str(e.get("relation") or e.get("kind") or "edge")
|
||||||
|
rule = str(e.get("rule_id") or "")
|
||||||
|
key = (s, t, rel, rule)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
out.append(e)
|
||||||
|
return out
|
||||||
+257
-81
@@ -2,119 +2,295 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Modul: app/core/edges.py
|
Modul: app/core/edges.py
|
||||||
Version: 1.0.0
|
Version: 2.0.0 (V2‑superset, rückwärtskompatibel zu v1 vom 2025‑09‑09)
|
||||||
Datum: 2025-09-09
|
|
||||||
|
|
||||||
Zweck
|
Zweck
|
||||||
-----
|
-----
|
||||||
Zentrale, konsistente Erzeugung von Edge-Payloads im **neuen Schema**:
|
Bewahrt die bestehende Edgelogik (belongs_to, prev/next, references, backlink)
|
||||||
- kind : "belongs_to" | "next" | "prev" | "references" | "backlink"
|
und ergänzt V2‑Felder + Typ‑Default‑Kanten gemäß config/types.yaml (edge_defaults).
|
||||||
- source_id : ID des Quellknotens (Chunk- oder Note-ID)
|
Die Funktion ist **idempotent** und **rückwärtskompatibel** zur bisherigen Signatur.
|
||||||
- target_id : ID des Zielknotens
|
|
||||||
- scope : "chunk" | "note"
|
|
||||||
- note_id : Owner-Note (für performantes Filtern/Löschen)
|
|
||||||
- seq : optional (z. B. Reihenfolge von Vorkommen)
|
|
||||||
|
|
||||||
Hinweise
|
Kompatibilitätsgarantien (gegenüber v1):
|
||||||
--------
|
- **Input**: akzeptiert identische Chunk‑Payloads wie v1:
|
||||||
- Edges werden dedupliziert (key=(kind,source_id,target_id,scope)).
|
* `id` (Chunk‑ID), `note_id` (Owner), `neighbors.prev|next` (optional),
|
||||||
- Für Chunk-Edges wird `note_id` aus dem Chunk-Payload entnommen.
|
`references: [{target_id: ...}]` (optional),
|
||||||
- Für Note-Scope-Edges ist `note_id` die Quell-Note-ID.
|
alternativ: `chunk_id`, `chunk_index|ord`, `window|text`
|
||||||
|
- **Output (v1‑Felder)**: `kind`, `source_id`, `target_id`, `scope`, `note_id`, `edge_id`
|
||||||
|
- **Neu (v2‑Felder)**: `relation`, `src_note_id`, `src_chunk_id?`, `dst_note_id`, `dst_chunk_id?`,
|
||||||
|
`provenance` (`explicit|rule`), `rule_id?`, `confidence?`
|
||||||
|
|
||||||
|
Regeln
|
||||||
|
------
|
||||||
|
- Deduplizierungsschlüssel: (source_id, target_id, relation, rule_id)
|
||||||
|
- Strukturkanten:
|
||||||
|
* belongs_to: 1× pro Chunk
|
||||||
|
* next/prev: Sequenz der Chunks; nutzt bevorzugt neighbors; sonst ord/chunk_index
|
||||||
|
- Explizite Referenzen:
|
||||||
|
* aus Chunk: `references[].target_id` (falls vorhanden)
|
||||||
|
* Fallback: Wikilinks in `window|text`: [[Some Title|some-id]] oder [[some-id]]
|
||||||
|
- Note‑Scope:
|
||||||
|
* backlink immer; references nur, wenn include_note_scope_refs=True
|
||||||
|
- Typ‑Defaults (edge_defaults aus config/types.yaml des **Quell‑Notiztyps**):
|
||||||
|
* Für jede explizite Referenz wird je default‑Relation eine Regel‑Kante erzeugt
|
||||||
|
* rule_id: "type_default:{note_type}:{relation}:v1", provenance="rule"
|
||||||
|
|
||||||
|
Konfiguration
|
||||||
|
-------------
|
||||||
|
- ENV MINDNET_TYPES_FILE (Default: ./config/types.yaml)
|
||||||
|
|
||||||
|
Lizenz/Autor
|
||||||
|
------------
|
||||||
|
- Erstimplementierung v1 (2025‑09‑09) — Projekt Mindnet
|
||||||
|
- Erweiterung v2 (2025‑11‑11) — kompatible Superset‑Implementierung
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
from typing import Dict, List
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Dict, Iterable, List, Optional, Tuple, Set
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # optional, nur für types.yaml
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
yaml = None
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Hilfen: types.yaml laden (edge_defaults)
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
|
def _types_path() -> str:
|
||||||
|
return os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
|
||||||
|
|
||||||
|
def _load_types() -> Dict[str, dict]:
|
||||||
|
p = _types_path()
|
||||||
|
if not os.path.isfile(p) or yaml is None:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
with open(p, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
|
||||||
|
return data["types"]
|
||||||
|
return data if isinstance(data, dict) else {}
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def _edge_defaults_for(note_type: Optional[str]) -> List[str]:
|
||||||
|
types = _load_types()
|
||||||
|
t = (note_type or "").strip().lower()
|
||||||
|
cfg = types.get(t) or {}
|
||||||
|
defaults = cfg.get("edge_defaults") or []
|
||||||
|
if isinstance(defaults, str):
|
||||||
|
defaults = [defaults]
|
||||||
|
return [str(x) for x in defaults if isinstance(x, (str, int, float))]
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Wikilink‑Parser (Fallback, wenn ch["references"] fehlt)
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
|
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([a-zA-Z0-9_\-#:. ]+)\]\]")
|
||||||
|
|
||||||
|
def _extract_wikilinks(text: str) -> List[str]:
|
||||||
|
ids: List[str] = []
|
||||||
|
for m in _WIKILINK_RE.finditer(text or ""):
|
||||||
|
ids.append(m.group(1).strip())
|
||||||
|
return ids
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Utility
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
|
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None) -> str:
|
||||||
|
base = f"{kind}:{s}->{t}#{scope}"
|
||||||
|
if rule_id:
|
||||||
|
base += f"|{rule_id}"
|
||||||
|
try:
|
||||||
|
import hashlib
|
||||||
|
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
return base
|
||||||
|
|
||||||
|
def _dedupe(edges: List[Dict]) -> List[Dict]:
|
||||||
|
seen: Set[Tuple[str,str,str,str]] = set()
|
||||||
|
out: List[Dict] = []
|
||||||
|
for e in edges:
|
||||||
|
s = str(e.get("source_id") or "")
|
||||||
|
t = str(e.get("target_id") or "")
|
||||||
|
rel = str(e.get("relation") or e.get("kind") or "edge")
|
||||||
|
rule = str(e.get("rule_id") or "")
|
||||||
|
key = (s, t, rel, rule)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
out.append(e)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _first(v: dict, *keys, default=None):
|
||||||
|
for k in keys:
|
||||||
|
if k in v and v[k] is not None:
|
||||||
|
return v[k]
|
||||||
|
return default
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Hauptfunktion
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
def build_edges_for_note(
|
def build_edges_for_note(
|
||||||
note_id: str,
|
note_id: str,
|
||||||
chunk_payloads: List[Dict],
|
chunk_payloads: List[Dict],
|
||||||
note_level_refs: List[str] | None,
|
note_level_refs: Optional[List[str]] = None,
|
||||||
*,
|
*,
|
||||||
include_note_scope_refs: bool = False,
|
include_note_scope_refs: bool = False,
|
||||||
) -> List[Dict]:
|
) -> List[Dict]:
|
||||||
edges: List[Dict] = []
|
edges: List[Dict] = []
|
||||||
|
chunks = list(chunk_payloads or [])
|
||||||
|
# Notiztyp aus erstem Chunk ableiten (kompatibel zu existierenden Payloads)
|
||||||
|
note_type = (chunks[0].get("type") if chunks else None) or (chunks[0].get("note_type") if chunks else None)
|
||||||
|
|
||||||
# Chunk-Scope: belongs_to / prev / next / references
|
# --- Strukturkanten ------------------------------------------------------
|
||||||
for ch in chunk_payloads:
|
# belongs_to
|
||||||
cid = ch["id"]
|
for ch in chunks:
|
||||||
|
cid = _first(ch, "id", "chunk_id")
|
||||||
|
if not cid:
|
||||||
|
continue
|
||||||
owner = ch.get("note_id") or note_id
|
owner = ch.get("note_id") or note_id
|
||||||
# belongs_to
|
e = {
|
||||||
edges.append({
|
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to:v1"),
|
||||||
"kind": "belongs_to",
|
"kind": "belongs_to",
|
||||||
|
"relation": "belongs_to",
|
||||||
|
"scope": "chunk",
|
||||||
"source_id": cid,
|
"source_id": cid,
|
||||||
"target_id": note_id,
|
"target_id": note_id,
|
||||||
"scope": "chunk",
|
"note_id": owner, # v1-Kompat
|
||||||
"note_id": owner,
|
# v2
|
||||||
})
|
"src_note_id": owner,
|
||||||
# Nachbarn
|
"src_chunk_id": cid,
|
||||||
|
"dst_note_id": note_id,
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": "structure:belongs_to:v1",
|
||||||
|
"confidence": 1.0,
|
||||||
|
}
|
||||||
|
edges.append(e)
|
||||||
|
|
||||||
|
# next/prev — bevorzugt neighbors.prev/next; sonst via ord/chunk_index
|
||||||
|
# Map der Chunks nach Index
|
||||||
|
ordered = list(chunks)
|
||||||
|
def _idx(c):
|
||||||
|
return _first(c, "chunk_index", "ord", default=0)
|
||||||
|
ordered.sort(key=_idx)
|
||||||
|
|
||||||
|
for i, ch in enumerate(ordered):
|
||||||
|
cid = _first(ch, "id", "chunk_id")
|
||||||
|
if not cid:
|
||||||
|
continue
|
||||||
|
owner = ch.get("note_id") or note_id
|
||||||
nb = ch.get("neighbors") or {}
|
nb = ch.get("neighbors") or {}
|
||||||
prev_id = nb.get("prev")
|
prev_id = nb.get("prev")
|
||||||
next_id = nb.get("next")
|
next_id = nb.get("next")
|
||||||
|
# Fallback-Reihenfolge
|
||||||
|
if prev_id is None and i > 0:
|
||||||
|
prev_id = _first(ordered[i-1], "id", "chunk_id")
|
||||||
|
if next_id is None and i+1 < len(ordered):
|
||||||
|
next_id = _first(ordered[i+1], "id", "chunk_id")
|
||||||
|
|
||||||
if prev_id:
|
if prev_id:
|
||||||
edges.append({
|
edges.append({
|
||||||
"kind": "prev",
|
"edge_id": _mk_edge_id("prev", cid, prev_id, "chunk", "structure:order:v1"),
|
||||||
"source_id": cid,
|
"kind": "prev", "relation": "prev", "scope": "chunk",
|
||||||
"target_id": prev_id,
|
"source_id": cid, "target_id": prev_id, "note_id": owner,
|
||||||
"scope": "chunk",
|
"src_note_id": owner, "src_chunk_id": cid,
|
||||||
"note_id": owner,
|
"dst_note_id": owner, "dst_chunk_id": prev_id,
|
||||||
|
"provenance": "rule", "rule_id": "structure:order:v1", "confidence": 0.95,
|
||||||
})
|
})
|
||||||
edges.append({
|
edges.append({
|
||||||
"kind": "next",
|
"edge_id": _mk_edge_id("next", prev_id, cid, "chunk", "structure:order:v1"),
|
||||||
"source_id": prev_id,
|
"kind": "next", "relation": "next", "scope": "chunk",
|
||||||
"target_id": cid,
|
"source_id": prev_id, "target_id": cid, "note_id": owner,
|
||||||
"scope": "chunk",
|
"src_note_id": owner, "src_chunk_id": prev_id,
|
||||||
"note_id": owner,
|
"dst_note_id": owner, "dst_chunk_id": cid,
|
||||||
})
|
"provenance": "rule", "rule_id": "structure:order:v1", "confidence": 0.95,
|
||||||
if next_id:
|
|
||||||
edges.append({
|
|
||||||
"kind": "next",
|
|
||||||
"source_id": cid,
|
|
||||||
"target_id": next_id,
|
|
||||||
"scope": "chunk",
|
|
||||||
"note_id": owner,
|
|
||||||
})
|
|
||||||
edges.append({
|
|
||||||
"kind": "prev",
|
|
||||||
"source_id": next_id,
|
|
||||||
"target_id": cid,
|
|
||||||
"scope": "chunk",
|
|
||||||
"note_id": owner,
|
|
||||||
})
|
|
||||||
# references aus Chunk
|
|
||||||
for ref in (ch.get("references") or []):
|
|
||||||
tid = ref.get("target_id")
|
|
||||||
if not tid:
|
|
||||||
continue
|
|
||||||
edges.append({
|
|
||||||
"kind": "references",
|
|
||||||
"source_id": cid,
|
|
||||||
"target_id": tid,
|
|
||||||
"scope": "chunk",
|
|
||||||
"note_id": owner,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
# Note-Scope: backlink (immer); references (optional)
|
# --- Explizite Referenzen (Chunk‑Scope) ---------------------------------
|
||||||
unique_refs = list(dict.fromkeys(note_level_refs or []))
|
explicit_refs: List[Dict] = []
|
||||||
|
for ch in chunks:
|
||||||
|
cid = _first(ch, "id", "chunk_id")
|
||||||
|
if not cid:
|
||||||
|
continue
|
||||||
|
owner = ch.get("note_id") or note_id
|
||||||
|
# 1) bevorzugt vorhandene ch["references"]
|
||||||
|
refs = ch.get("references") or []
|
||||||
|
targets = [r.get("target_id") for r in refs if isinstance(r, dict) and r.get("target_id")]
|
||||||
|
# 2) Fallback: Wikilinks aus Text
|
||||||
|
if not targets:
|
||||||
|
text = _first(ch, "window", "text", default="") or ""
|
||||||
|
targets = _extract_wikilinks(text)
|
||||||
|
for tid in targets:
|
||||||
|
if not isinstance(tid, str) or not tid.strip():
|
||||||
|
continue
|
||||||
|
e = {
|
||||||
|
"edge_id": _mk_edge_id("references", cid, tid, "chunk"),
|
||||||
|
"kind": "references",
|
||||||
|
"relation": "references",
|
||||||
|
"scope": "chunk",
|
||||||
|
"source_id": cid,
|
||||||
|
"target_id": tid,
|
||||||
|
"note_id": owner,
|
||||||
|
# v2
|
||||||
|
"src_note_id": owner,
|
||||||
|
"src_chunk_id": cid,
|
||||||
|
"dst_note_id": tid,
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "",
|
||||||
|
"confidence": 1.0,
|
||||||
|
}
|
||||||
|
edges.append(e)
|
||||||
|
explicit_refs.append(e)
|
||||||
|
|
||||||
|
# --- Note‑Scope: references (optional) + backlink (immer) ----------------
|
||||||
|
unique_refs = []
|
||||||
|
if note_level_refs:
|
||||||
|
seen = set()
|
||||||
|
for tid in note_level_refs:
|
||||||
|
if isinstance(tid, str) and tid.strip() and tid not in seen:
|
||||||
|
unique_refs.append(tid); seen.add(tid)
|
||||||
|
|
||||||
for tid in unique_refs:
|
for tid in unique_refs:
|
||||||
if include_note_scope_refs:
|
if include_note_scope_refs:
|
||||||
edges.append({
|
edges.append({
|
||||||
"kind": "references",
|
"edge_id": _mk_edge_id("references", note_id, tid, "note"),
|
||||||
"source_id": note_id,
|
"kind": "references", "relation": "references", "scope": "note",
|
||||||
"target_id": tid,
|
"source_id": note_id, "target_id": tid, "note_id": note_id,
|
||||||
"scope": "note",
|
"src_note_id": note_id, "dst_note_id": tid,
|
||||||
"note_id": note_id,
|
"provenance": "explicit", "rule_id": "", "confidence": 1.0,
|
||||||
})
|
})
|
||||||
edges.append({
|
edges.append({
|
||||||
"kind": "backlink",
|
"edge_id": _mk_edge_id("backlink", tid, note_id, "note", "derived:backlink:v1"),
|
||||||
"source_id": tid,
|
"kind": "backlink", "relation": "backlink", "scope": "note",
|
||||||
"target_id": note_id,
|
"source_id": tid, "target_id": note_id, "note_id": note_id,
|
||||||
"scope": "note",
|
"src_note_id": tid, "dst_note_id": note_id,
|
||||||
"note_id": note_id,
|
"provenance": "rule", "rule_id": "derived:backlink:v1", "confidence": 0.9,
|
||||||
})
|
})
|
||||||
|
|
||||||
# Dedupe
|
# --- Type‑Defaults je expliziter Referenz --------------------------------
|
||||||
dedup = {}
|
defaults = [d for d in _edge_defaults_for(note_type) if d and d != "references"]
|
||||||
for e in edges:
|
if defaults:
|
||||||
k = (e["kind"], e["source_id"], e["target_id"], e.get("scope", ""))
|
for e in explicit_refs + ([ ] if not include_note_scope_refs else []):
|
||||||
dedup[k] = e
|
# wir nutzen die bereits erzeugten explicit‑Edges als Vorlage
|
||||||
return list(dedup.values())
|
src = e["source_id"]; tgt = e["target_id"]
|
||||||
|
scope = e.get("scope", "chunk")
|
||||||
|
s_note = e.get("src_note_id") or note_id
|
||||||
|
s_chunk = e.get("src_chunk_id")
|
||||||
|
t_note = e.get("dst_note_id") or tgt
|
||||||
|
for rel in defaults:
|
||||||
|
rule_id = f"type_default:{(note_type or 'unknown')}:{rel}:v1"
|
||||||
|
edges.append({
|
||||||
|
"edge_id": _mk_edge_id(rel, src, tgt, scope, rule_id),
|
||||||
|
"kind": rel, "relation": rel, "scope": scope,
|
||||||
|
"source_id": src, "target_id": tgt, "note_id": s_note,
|
||||||
|
"src_note_id": s_note, "src_chunk_id": s_chunk,
|
||||||
|
"dst_note_id": t_note,
|
||||||
|
"provenance": "rule", "rule_id": rule_id, "confidence": 0.7,
|
||||||
|
})
|
||||||
|
|
||||||
|
# --- Dedupe & Return -----------------------------------------------------
|
||||||
|
return _dedupe(edges)
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
# app/core/edges_writer.py
|
||||||
|
from __future__ import annotations
|
||||||
|
import hashlib
|
||||||
|
from typing import Dict, List, Iterable, Tuple
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Dein Modul mit der Schemadefinition und der Builder-Funktion
|
||||||
|
from app.core.edges import build_edges_for_note # noqa: F401
|
||||||
|
except Exception as e:
|
||||||
|
raise RuntimeError("Konnte app.core.edges nicht importieren. "
|
||||||
|
"Bitte sicherstellen, dass app/core/edges.py vorhanden ist.") from e
|
||||||
|
|
||||||
|
def _edge_uid(kind: str, source_id: str, target_id: str, scope: str) -> str:
|
||||||
|
"""
|
||||||
|
Deterministische, kurze ID für eine Edge.
|
||||||
|
Kollisionen sind praktisch ausgeschlossen (BLAKE2s über den Kanonischen Schlüssel).
|
||||||
|
"""
|
||||||
|
key = f"{kind}|{source_id}|{target_id}|{scope}"
|
||||||
|
return hashlib.blake2s(key.encode("utf-8"), digest_size=12).hexdigest()
|
||||||
|
|
||||||
|
def ensure_edges_collection(qdrant_client, collection: str) -> None:
|
||||||
|
"""
|
||||||
|
Legt die Edge-Collection an, falls sie nicht existiert.
|
||||||
|
Minimal: 1D-Vector (Dummy), Cosine. Payload-only-Collections sind je nach Qdrant-Version heikel.
|
||||||
|
"""
|
||||||
|
from qdrant_client.http import models as qm
|
||||||
|
|
||||||
|
existing = [c.name for c in qdrant_client.get_collections().collections]
|
||||||
|
if collection in existing:
|
||||||
|
return
|
||||||
|
|
||||||
|
qdrant_client.recreate_collection(
|
||||||
|
collection_name=collection,
|
||||||
|
vectors_config=qm.VectorParams(size=1, distance=qm.Distance.COSINE),
|
||||||
|
on_disk_payload=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def edges_from_note(
|
||||||
|
note_id: str,
|
||||||
|
chunk_payloads: List[Dict],
|
||||||
|
note_level_refs: Iterable[str] | None,
|
||||||
|
*,
|
||||||
|
include_note_scope_refs: bool = False,
|
||||||
|
) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Ruft deinen Edge-Builder auf und gibt die (deduplizierten) Edge-Payloads zurück.
|
||||||
|
Keine Schemaänderung – exakt das aus app/core/edges.py.
|
||||||
|
"""
|
||||||
|
return build_edges_for_note(
|
||||||
|
note_id=note_id,
|
||||||
|
chunk_payloads=chunk_payloads,
|
||||||
|
note_level_refs=list(note_level_refs or []),
|
||||||
|
include_note_scope_refs=include_note_scope_refs,
|
||||||
|
)
|
||||||
|
|
||||||
|
def upsert_edges(
|
||||||
|
qdrant_client,
|
||||||
|
collection: str,
|
||||||
|
edge_payloads: List[Dict],
|
||||||
|
) -> Tuple[int, int]:
|
||||||
|
"""
|
||||||
|
Schreibt Edges als Points in Qdrant.
|
||||||
|
- id: deterministisch aus (kind, source_id, target_id, scope)
|
||||||
|
- vector: [0.0] Dummy
|
||||||
|
- payload: Edge-Dict (unverändert, siehe Schema in app/core/edges.py)
|
||||||
|
Gibt (anzahl_points, anzahl_unique_keys) zurück.
|
||||||
|
"""
|
||||||
|
from qdrant_client.models import PointStruct
|
||||||
|
|
||||||
|
if not edge_payloads:
|
||||||
|
return 0, 0
|
||||||
|
|
||||||
|
points = []
|
||||||
|
seen = set()
|
||||||
|
for e in edge_payloads:
|
||||||
|
key = (e.get("kind"), e.get("source_id"), e.get("target_id"), e.get("scope"))
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
eid = _edge_uid(*key)
|
||||||
|
points.append(
|
||||||
|
PointStruct(
|
||||||
|
id=eid,
|
||||||
|
vector=[0.0],
|
||||||
|
payload=e,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not points:
|
||||||
|
return 0, 0
|
||||||
|
|
||||||
|
ensure_edges_collection(qdrant_client, collection)
|
||||||
|
qdrant_client.upsert(collection_name=collection, points=points)
|
||||||
|
return len(points), len(seen)
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Datei: app/core/env_vars.py
|
||||||
|
Version: 1.1.0 (2025-11-08)
|
||||||
|
|
||||||
|
Zweck
|
||||||
|
Einheitliche Auflösung von ENV-Variablen (Prefix, Qdrant, Embeddings, Hashing)
|
||||||
|
mit Abwärtskompatibilität.
|
||||||
|
|
||||||
|
Grundsatz
|
||||||
|
- Für Qdrant-Funktionen ist 'COLLECTION_PREFIX' der Primärschlüssel.
|
||||||
|
- 'MINDNET_PREFIX' bleibt für App-/UI-/Exporter-Kontexte nutzbar.
|
||||||
|
- Fallbacks sorgen dafür, dass ältere Umgebungen weiter funktionieren.
|
||||||
|
|
||||||
|
Wichtig
|
||||||
|
- Lädt optional eine .env (wenn python-dotenv verfügbar ist).
|
||||||
|
- Überschreibt keine bereits gesetzten OS-Variablen (override=False).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from typing import Optional, Dict
|
||||||
|
|
||||||
|
# Optional: .env automatisch laden (ohne Hard-Fail, falls nicht vorhanden)
|
||||||
|
try:
|
||||||
|
from dotenv import load_dotenv, find_dotenv # type: ignore
|
||||||
|
_p = find_dotenv()
|
||||||
|
if _p:
|
||||||
|
load_dotenv(_p, override=False)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# -------- Prefix-Auflösung --------
|
||||||
|
|
||||||
|
def get_collection_prefix(cli_override: Optional[str] = None) -> str:
|
||||||
|
"""
|
||||||
|
Für Qdrant-relevante Funktionen:
|
||||||
|
1) CLI-Override (--prefix)
|
||||||
|
2) ENV COLLECTION_PREFIX
|
||||||
|
3) ENV MINDNET_PREFIX (Fallback)
|
||||||
|
4) 'mindnet' (Default)
|
||||||
|
"""
|
||||||
|
if cli_override and str(cli_override).strip():
|
||||||
|
return str(cli_override).strip()
|
||||||
|
return (
|
||||||
|
os.getenv("COLLECTION_PREFIX")
|
||||||
|
or os.getenv("MINDNET_PREFIX")
|
||||||
|
or "mindnet"
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_mindnet_prefix(cli_override: Optional[str] = None) -> str:
|
||||||
|
"""
|
||||||
|
Für App-/UI-/Exporter-Kontexte:
|
||||||
|
1) CLI-Override (--prefix)
|
||||||
|
2) ENV MINDNET_PREFIX
|
||||||
|
3) ENV COLLECTION_PREFIX (Fallback)
|
||||||
|
4) 'mindnet'
|
||||||
|
"""
|
||||||
|
if cli_override and str(cli_override).strip():
|
||||||
|
return str(cli_override).strip()
|
||||||
|
return (
|
||||||
|
os.getenv("MINDNET_PREFIX")
|
||||||
|
or os.getenv("COLLECTION_PREFIX")
|
||||||
|
or "mindnet"
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_prefix(cli_override: Optional[str] = None, target: str = "qdrant") -> str:
|
||||||
|
"""
|
||||||
|
Universelle Hülle (abwärtskompatibel):
|
||||||
|
target='qdrant' -> get_collection_prefix
|
||||||
|
target='app' -> get_mindnet_prefix
|
||||||
|
"""
|
||||||
|
if target.lower() == "app":
|
||||||
|
return get_mindnet_prefix(cli_override)
|
||||||
|
return get_collection_prefix(cli_override)
|
||||||
|
|
||||||
|
# -------- Qdrant / Embeddings / Hashing --------
|
||||||
|
|
||||||
|
def get_qdrant_url(default: str = "http://127.0.0.1:6333") -> str:
|
||||||
|
return os.getenv("QDRANT_URL", default)
|
||||||
|
|
||||||
|
def get_qdrant_api_key(default: str = "") -> str:
|
||||||
|
return os.getenv("QDRANT_API_KEY", default)
|
||||||
|
|
||||||
|
def get_vector_dim(default: int = 384) -> int:
|
||||||
|
try:
|
||||||
|
return int(os.getenv("VECTOR_DIM", str(default)))
|
||||||
|
except Exception:
|
||||||
|
return default
|
||||||
|
|
||||||
|
def get_embed_url(default: Optional[str] = None) -> Optional[str]:
|
||||||
|
return os.getenv("EMBED_URL", default)
|
||||||
|
|
||||||
|
def get_hash_env() -> Dict[str, str]:
|
||||||
|
"""
|
||||||
|
Liefert die Hash-Konfiguration (nur Aggregation; die Auswertung bleibt in den Skripten).
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"MINDNET_HASH_COMPARE": os.getenv("MINDNET_HASH_COMPARE", ""),
|
||||||
|
"MINDNET_HASH_SOURCE": os.getenv("MINDNET_HASH_SOURCE", ""),
|
||||||
|
"MINDNET_HASH_NORMALIZE": os.getenv("MINDNET_HASH_NORMALIZE", ""),
|
||||||
|
}
|
||||||
@@ -0,0 +1,256 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
app/core/graph_adapter.py — Adjazenzaufbau & Subgraph-Expansion
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Baut aus Qdrant-Edges (Collection: *_edges) einen leichten In-Memory-Graph.
|
||||||
|
|
||||||
|
Kompatibilität:
|
||||||
|
- WP-04a: Liefert Scores (edge_bonus, centrality).
|
||||||
|
- WP-04b: Liefert jetzt auch Struktur-Daten für Erklärungen (Reverse-Lookup).
|
||||||
|
|
||||||
|
Version:
|
||||||
|
0.4.0 (Update für WP-04b: Reverse Adjacency für Explainability)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Dict, List, Optional, DefaultDict, Any
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
from app.core.qdrant import collection_names
|
||||||
|
|
||||||
|
# Legacy-Import Fallback
|
||||||
|
try: # pragma: no cover
|
||||||
|
from app.core.qdrant_points import get_edges_for_sources # type: ignore
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
get_edges_for_sources = None # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
# Basisgewichte je Edge-Typ (WP-04a Config)
|
||||||
|
EDGE_BASE_WEIGHTS: Dict[str, float] = {
|
||||||
|
# Struktur
|
||||||
|
"belongs_to": 0.10,
|
||||||
|
"next": 0.06,
|
||||||
|
"prev": 0.06,
|
||||||
|
"backlink": 0.04,
|
||||||
|
"references_at": 0.08,
|
||||||
|
|
||||||
|
# Wissen
|
||||||
|
"references": 0.20,
|
||||||
|
"depends_on": 0.18,
|
||||||
|
"related_to": 0.15,
|
||||||
|
"similar_to": 0.12,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _edge_weight(pl: Dict) -> float:
|
||||||
|
"""Berechnet das effektive Edge-Gewicht aus kind + confidence."""
|
||||||
|
kind = pl.get("kind", "edge")
|
||||||
|
base = EDGE_BASE_WEIGHTS.get(kind, 0.0)
|
||||||
|
|
||||||
|
conf_raw = pl.get("confidence", None)
|
||||||
|
try:
|
||||||
|
conf = float(conf_raw) if conf_raw is not None else None
|
||||||
|
except Exception:
|
||||||
|
conf = None
|
||||||
|
|
||||||
|
if conf is None:
|
||||||
|
return base
|
||||||
|
|
||||||
|
if conf < 0.0: conf = 0.0
|
||||||
|
if conf > 1.0: conf = 1.0
|
||||||
|
|
||||||
|
return base * conf
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_edges(
|
||||||
|
client: QdrantClient,
|
||||||
|
prefix: str,
|
||||||
|
seeds: List[str],
|
||||||
|
edge_types: Optional[List[str]] = None,
|
||||||
|
limit: int = 2048,
|
||||||
|
) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Holt Edges direkt aus der *_edges Collection.
|
||||||
|
Filter: source_id IN seeds OR target_id IN seeds OR note_id IN seeds
|
||||||
|
"""
|
||||||
|
if not seeds or limit <= 0:
|
||||||
|
return []
|
||||||
|
|
||||||
|
_, _, edges_col = collection_names(prefix)
|
||||||
|
|
||||||
|
seed_conditions = []
|
||||||
|
for field in ("source_id", "target_id", "note_id"):
|
||||||
|
for s in seeds:
|
||||||
|
seed_conditions.append(
|
||||||
|
rest.FieldCondition(key=field, match=rest.MatchValue(value=str(s)))
|
||||||
|
)
|
||||||
|
seeds_filter = rest.Filter(should=seed_conditions) if seed_conditions else None
|
||||||
|
|
||||||
|
type_filter = None
|
||||||
|
if edge_types:
|
||||||
|
type_conds = [
|
||||||
|
rest.FieldCondition(key="kind", match=rest.MatchValue(value=str(k)))
|
||||||
|
for k in edge_types
|
||||||
|
]
|
||||||
|
type_filter = rest.Filter(should=type_conds)
|
||||||
|
|
||||||
|
must = []
|
||||||
|
if seeds_filter: must.append(seeds_filter)
|
||||||
|
if type_filter: must.append(type_filter)
|
||||||
|
|
||||||
|
flt = rest.Filter(must=must) if must else None
|
||||||
|
|
||||||
|
pts, _ = client.scroll(
|
||||||
|
collection_name=edges_col,
|
||||||
|
scroll_filter=flt,
|
||||||
|
limit=limit,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
out: List[Dict] = []
|
||||||
|
for p in pts or []:
|
||||||
|
pl = dict(p.payload or {})
|
||||||
|
if pl:
|
||||||
|
out.append(pl)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class Subgraph:
|
||||||
|
"""Leichtgewichtiger Subgraph mit Adjazenzlisten & Kennzahlen."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
# Forward: source -> [targets]
|
||||||
|
self.adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||||
|
# Reverse: target -> [sources] (Neu für WP-04b Explanation)
|
||||||
|
self.reverse_adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||||
|
|
||||||
|
self.in_degree: DefaultDict[str, int] = defaultdict(int)
|
||||||
|
self.out_degree: DefaultDict[str, int] = defaultdict(int)
|
||||||
|
|
||||||
|
def add_edge(self, e: Dict) -> None:
|
||||||
|
"""
|
||||||
|
Fügt eine Kante hinzu und aktualisiert Forward/Reverse Indizes.
|
||||||
|
e muss enthalten: source, target, kind, weight.
|
||||||
|
"""
|
||||||
|
src = e.get("source")
|
||||||
|
tgt = e.get("target")
|
||||||
|
kind = e.get("kind")
|
||||||
|
weight = e.get("weight", EDGE_BASE_WEIGHTS.get(kind, 0.0))
|
||||||
|
owner = e.get("note_id")
|
||||||
|
|
||||||
|
if not src or not tgt:
|
||||||
|
return
|
||||||
|
|
||||||
|
# 1. Primäre Adjazenz (Forward)
|
||||||
|
edge_data = {"target": tgt, "kind": kind, "weight": weight}
|
||||||
|
self.adj[src].append(edge_data)
|
||||||
|
self.out_degree[src] += 1
|
||||||
|
self.in_degree[tgt] += 1
|
||||||
|
|
||||||
|
# 2. Reverse Adjazenz (Neu für Explanation)
|
||||||
|
# Wir speichern, woher die Kante kam.
|
||||||
|
rev_data = {"source": src, "kind": kind, "weight": weight}
|
||||||
|
self.reverse_adj[tgt].append(rev_data)
|
||||||
|
|
||||||
|
# 3. Kontext-Note Handling (Forward & Reverse)
|
||||||
|
# Wenn eine Kante "im Kontext einer Note" (owner) definiert ist,
|
||||||
|
# schreiben wir sie der Note gut, damit der Retriever Scores auf Note-Ebene findet.
|
||||||
|
if owner and owner != src:
|
||||||
|
# Forward: Owner -> Target
|
||||||
|
self.adj[owner].append(edge_data)
|
||||||
|
self.out_degree[owner] += 1
|
||||||
|
|
||||||
|
# Reverse: Target wird vom Owner referenziert (indirekt)
|
||||||
|
if owner != tgt:
|
||||||
|
rev_owner_data = {"source": owner, "kind": kind, "weight": weight, "via_context": True}
|
||||||
|
self.reverse_adj[tgt].append(rev_owner_data)
|
||||||
|
self.in_degree[owner] += 1 # Leichter Centrality Boost für den Owner
|
||||||
|
|
||||||
|
def aggregate_edge_bonus(self, node_id: str) -> float:
|
||||||
|
"""Summe der ausgehenden Kantengewichte (Hub-Score)."""
|
||||||
|
return sum(edge["weight"] for edge in self.adj.get(node_id, []))
|
||||||
|
|
||||||
|
def edge_bonus(self, node_id: str) -> float:
|
||||||
|
"""API für Retriever (WP-04a Kompatibilität)."""
|
||||||
|
return self.aggregate_edge_bonus(node_id)
|
||||||
|
|
||||||
|
def centrality_bonus(self, node_id: str) -> float:
|
||||||
|
"""Log-gedämpfte Zentralität (In-Degree)."""
|
||||||
|
import math
|
||||||
|
indeg = self.in_degree.get(node_id, 0)
|
||||||
|
if indeg <= 0:
|
||||||
|
return 0.0
|
||||||
|
return min(math.log1p(indeg) / 10.0, 0.15)
|
||||||
|
|
||||||
|
# --- WP-04b Explanation Helpers ---
|
||||||
|
|
||||||
|
def get_outgoing_edges(self, node_id: str) -> List[Dict[str, Any]]:
|
||||||
|
"""Liefert Liste aller Ziele, auf die dieser Knoten zeigt."""
|
||||||
|
return self.adj.get(node_id, [])
|
||||||
|
|
||||||
|
def get_incoming_edges(self, node_id: str) -> List[Dict[str, Any]]:
|
||||||
|
"""Liefert Liste aller Quellen, die auf diesen Knoten zeigen."""
|
||||||
|
return self.reverse_adj.get(node_id, [])
|
||||||
|
|
||||||
|
|
||||||
|
def expand(
|
||||||
|
client: QdrantClient,
|
||||||
|
prefix: str,
|
||||||
|
seeds: List[str],
|
||||||
|
depth: int = 1,
|
||||||
|
edge_types: Optional[List[str]] = None,
|
||||||
|
) -> Subgraph:
|
||||||
|
"""
|
||||||
|
Expandiert ab Seeds entlang von Edges (bis `depth`).
|
||||||
|
"""
|
||||||
|
sg = Subgraph()
|
||||||
|
frontier = set(seeds)
|
||||||
|
visited = set()
|
||||||
|
|
||||||
|
max_depth = max(depth, 0)
|
||||||
|
|
||||||
|
for _ in range(max_depth):
|
||||||
|
if not frontier:
|
||||||
|
break
|
||||||
|
|
||||||
|
edges_payloads = _fetch_edges(
|
||||||
|
client=client,
|
||||||
|
prefix=prefix,
|
||||||
|
seeds=list(frontier),
|
||||||
|
edge_types=edge_types,
|
||||||
|
limit=2048,
|
||||||
|
)
|
||||||
|
|
||||||
|
next_frontier = set()
|
||||||
|
for pl in edges_payloads:
|
||||||
|
src = pl.get("source_id")
|
||||||
|
tgt = pl.get("target_id")
|
||||||
|
|
||||||
|
# Skip invalid edges
|
||||||
|
if not src or not tgt:
|
||||||
|
continue
|
||||||
|
|
||||||
|
e = {
|
||||||
|
"source": src,
|
||||||
|
"target": tgt,
|
||||||
|
"kind": pl.get("kind", "edge"),
|
||||||
|
"weight": _edge_weight(pl),
|
||||||
|
"note_id": pl.get("note_id"),
|
||||||
|
}
|
||||||
|
sg.add_edge(e)
|
||||||
|
|
||||||
|
# Nur weitersuchen, wenn Target noch nicht besucht
|
||||||
|
if tgt and tgt not in visited:
|
||||||
|
next_frontier.add(tgt)
|
||||||
|
|
||||||
|
visited |= frontier
|
||||||
|
frontier = next_frontier - visited
|
||||||
|
|
||||||
|
return sg
|
||||||
+189
-272
@@ -1,309 +1,226 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
# Modul: app/core/note_payload.py
|
"""
|
||||||
# Version: 1.6.3
|
Modul: app/core/note_payload.py
|
||||||
# Datum: 2025-09-09
|
Version: 2.0.0
|
||||||
#
|
|
||||||
# Zweck
|
Zweck
|
||||||
# -----
|
-----
|
||||||
# Erzeugt den Qdrant-Payload für Notes inkl. robuster Hash-Bildung zur
|
Erzeugt ein robustes Note-Payload. Werte wie `retriever_weight`, `chunk_profile`
|
||||||
# Änderungserkennung. Der vollständige (parsed) Body wird unter "fulltext"
|
und `edge_defaults` werden in folgender Priorität bestimmt:
|
||||||
# persistiert; der Pfad ist relativ (für verlustfreien Export).
|
1) Frontmatter (Note)
|
||||||
#
|
2) Typ-Registry (config/types.yaml: types.<type>.*)
|
||||||
# Steuerung (CLI/ENV, vom Importer durchgereicht)
|
3) Registry-Defaults (config/types.yaml: defaults.*)
|
||||||
# -----------------------------------------------
|
4) ENV-Defaults (MINDNET_DEFAULT_RETRIEVER_WEIGHT / MINDNET_DEFAULT_CHUNK_PROFILE)
|
||||||
# - Vergleichsmodus:
|
"""
|
||||||
# --hash-mode body|frontmatter|full
|
|
||||||
# ENV: MINDNET_HASH_MODE oder MINDNET_HASH_COMPARE (Body|Frontmatter|Full)
|
|
||||||
# "full" ist Alias für "body+frontmatter".
|
|
||||||
# - Hash-Quelle:
|
|
||||||
# --hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE)
|
|
||||||
# - Normalisierung:
|
|
||||||
# --hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE)
|
|
||||||
#
|
|
||||||
# Payload-Felder (Auszug)
|
|
||||||
# -----------------------
|
|
||||||
# note_id, title, type, status, created, updated, path, tags,
|
|
||||||
# fulltext, references (Note-Level-Wikilinks),
|
|
||||||
# hash_fulltext (Primärhash), hash_signature (z. B. "body:raw:none:<hex>")
|
|
||||||
#
|
|
||||||
# Hinweise
|
|
||||||
# --------
|
|
||||||
# - Keine Abhängigkeit von FS-Zeitstempeln; nur Inhalte fließen in den Hash ein.
|
|
||||||
# - Abwärtskompatibel: Feldernamen bleiben stabil; zusätzliche Felder stören nicht.
|
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
from typing import Any, Dict, Tuple, Optional
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
from typing import Any, Dict, Optional, Tuple
|
import json
|
||||||
|
import pathlib
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from app.core.parser import read_markdown, extract_wikilinks, FRONTMATTER_RE
|
import yaml # type: ignore
|
||||||
except Exception: # pragma: no cover
|
except Exception:
|
||||||
from .parser import read_markdown, extract_wikilinks, FRONTMATTER_RE # type: ignore
|
yaml = None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Helpers
|
# Helper
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
|
def _as_dict(x) -> Dict[str, Any]:
|
||||||
"""Kanonische JSON-Serialisierung der Frontmatter für Hashbildung."""
|
"""Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen."""
|
||||||
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
if isinstance(x, dict):
|
||||||
|
return dict(x)
|
||||||
|
|
||||||
def _normalize_body(body: str, mode: str) -> str:
|
out: Dict[str, Any] = {}
|
||||||
"""Normalisiert den Body für reproduzierbare Hashes (oder nicht)."""
|
# bekannte Attribute übernehmen, sofern vorhanden
|
||||||
if mode == "none":
|
for attr in (
|
||||||
return body if body is not None else ""
|
"frontmatter",
|
||||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
|
"body",
|
||||||
text = "\n".join(line.rstrip() for line in text.split("\n"))
|
"id",
|
||||||
return text
|
"note_id",
|
||||||
|
"title",
|
||||||
|
"path",
|
||||||
|
"tags",
|
||||||
|
"type",
|
||||||
|
"created",
|
||||||
|
"modified",
|
||||||
|
"date",
|
||||||
|
):
|
||||||
|
if hasattr(x, attr):
|
||||||
|
val = getattr(x, attr)
|
||||||
|
if val is not None:
|
||||||
|
out[attr] = val
|
||||||
|
|
||||||
def _resolve_hash_mode(explicit: Optional[str]) -> str:
|
# Fallback: wenn immer noch leer, raw speichern
|
||||||
"""
|
if not out:
|
||||||
Normalisiert den Hash-Modus auf:
|
out["raw"] = str(x)
|
||||||
'body' | 'frontmatter' | 'body+frontmatter'
|
|
||||||
Akzeptiert 'full' als Alias.
|
|
||||||
Berücksichtigt ENV: MINDNET_HASH_MODE oder MINDNET_HASH_COMPARE.
|
|
||||||
"""
|
|
||||||
if explicit:
|
|
||||||
val = explicit.strip().lower()
|
|
||||||
else:
|
|
||||||
val = (os.environ.get("MINDNET_HASH_MODE")
|
|
||||||
or os.environ.get("MINDNET_HASH_COMPARE")
|
|
||||||
or "body").strip().lower()
|
|
||||||
if val in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
|
||||||
return "body+frontmatter"
|
|
||||||
if val in ("frontmatter", "fm"):
|
|
||||||
return "frontmatter"
|
|
||||||
return "body"
|
|
||||||
|
|
||||||
def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, Any]]:
|
return out
|
||||||
"""Liest die Rohdatei und extrahiert Body & Frontmatter ohne Parser-Logik.
|
|
||||||
|
|
||||||
Rückgabe:
|
|
||||||
(body_text, frontmatter_dict)
|
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]:
|
||||||
"""
|
"""Extrahiert optionale Zusatzargumente wie path und types_cfg."""
|
||||||
if not file_path or not os.path.exists(file_path):
|
path = kwargs.get("path") or (args[0] if args else None)
|
||||||
return "", {}
|
types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None
|
||||||
|
return path, types_cfg
|
||||||
|
|
||||||
|
|
||||||
|
def _env_float(name: str, default: float) -> float:
|
||||||
|
"""Liest einen Float-Wert aus der Umgebung, mit robustem Fallback."""
|
||||||
try:
|
try:
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
return float(os.environ.get(name, default))
|
||||||
raw = f.read()
|
|
||||||
except Exception:
|
except Exception:
|
||||||
return "", {}
|
return default
|
||||||
# Frontmatter per Regex entfernen
|
|
||||||
m = FRONTMATTER_RE.match(raw)
|
|
||||||
fm = {}
|
def _ensure_list(x) -> list:
|
||||||
if m:
|
"""Garantiert eine String-Liste."""
|
||||||
fm_txt = m.group(1)
|
if x is None:
|
||||||
try:
|
return []
|
||||||
import yaml # lazy
|
if isinstance(x, list):
|
||||||
fm = yaml.safe_load(fm_txt) or {}
|
return [str(i) for i in x]
|
||||||
except Exception:
|
if isinstance(x, (set, tuple)):
|
||||||
fm = {}
|
return [str(i) for i in x]
|
||||||
body = raw[m.end():]
|
return [str(x)]
|
||||||
else:
|
|
||||||
body = raw
|
|
||||||
return body, fm
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Hashing
|
# Type-Registry laden
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def _sha256(s: str) -> str:
|
def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict:
|
||||||
h = hashlib.sha256()
|
"""Lädt die Type-Registry aus YAML/JSON oder nutzt ein explizit übergebenes Dict."""
|
||||||
h.update(s.encode("utf-8"))
|
if explicit_cfg and isinstance(explicit_cfg, dict):
|
||||||
return h.hexdigest()
|
return explicit_cfg
|
||||||
|
|
||||||
def compute_hash(
|
path = os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
|
||||||
*,
|
if not os.path.isfile(path) or yaml is None:
|
||||||
body: Optional[str],
|
return {}
|
||||||
frontmatter: Optional[Dict[str, Any]],
|
|
||||||
mode: Optional[str] = None,
|
|
||||||
normalize: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Berechnet einen Hex-Hash gemäß 'mode' und 'normalize'.
|
|
||||||
|
|
||||||
mode: "body" | "frontmatter" | "body+frontmatter"
|
|
||||||
normalize: "canonical" | "none"
|
|
||||||
"""
|
|
||||||
mode = _resolve_hash_mode(mode)
|
|
||||||
normalize = (normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
|
|
||||||
body_n = _normalize_body(body or "", normalize)
|
|
||||||
fm_s = _canon_frontmatter(frontmatter or {})
|
|
||||||
|
|
||||||
if mode == "frontmatter":
|
|
||||||
return _sha256(fm_s)
|
|
||||||
if mode == "body+frontmatter":
|
|
||||||
return _sha256(body_n + "\n--FM--\n" + fm_s)
|
|
||||||
# default: body
|
|
||||||
return _sha256(body_n)
|
|
||||||
|
|
||||||
def compute_hash_set(*, body_parsed: str, body_raw: str, fm: Dict[str, Any], normalize: str) -> Dict[str, str]:
|
|
||||||
"""Berechnet ein Set an Hashes für Monitoring/Debug."""
|
|
||||||
fm_s = _canon_frontmatter(fm or {})
|
|
||||||
bp = _normalize_body(body_parsed or "", normalize)
|
|
||||||
br = _normalize_body(body_raw or "", normalize)
|
|
||||||
return {
|
|
||||||
"frontmatter": _sha256(fm_s),
|
|
||||||
"body_parsed": _sha256(bp),
|
|
||||||
"body_raw": _sha256(br),
|
|
||||||
"full_parsed": _sha256(bp + "\n--FM--\n" + fm_s),
|
|
||||||
"full_raw": _sha256(br + "\n--FM--\n" + fm_s),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Kernfunktion
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def make_note_payload(
|
|
||||||
parsed: Any,
|
|
||||||
vault_root: Optional[str] = None,
|
|
||||||
*,
|
|
||||||
hash_mode: Optional[str] = None,
|
|
||||||
hash_normalize: Optional[str] = None,
|
|
||||||
hash_source: Optional[str] = None,
|
|
||||||
file_path: Optional[str] = None,
|
|
||||||
) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Erzeugt den Payload für eine geparste Note.
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
parsed : Any
|
|
||||||
Objekt mit Attributen/Keys 'frontmatter', 'body', 'path'.
|
|
||||||
vault_root : Optional[str]
|
|
||||||
Vault-Wurzel (für Pfad-Relativierung).
|
|
||||||
hash_mode : Optional[str]
|
|
||||||
"body" | "frontmatter" | "body+frontmatter" | "full" (Alias; überschreibt ENV).
|
|
||||||
hash_normalize : Optional[str]
|
|
||||||
"canonical" | "none" (überschreibt ENV).
|
|
||||||
hash_source : Optional[str]
|
|
||||||
"parsed" (Default) oder "raw". Wenn "raw", wird der Body aus der Rohdatei gelesen.
|
|
||||||
file_path : Optional[str]
|
|
||||||
Pfad zur Markdown-Datei, erforderlich für 'hash_source=raw'.
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
Dict[str, Any]
|
|
||||||
Qdrant-Payload für die Notes-Collection.
|
|
||||||
"""
|
|
||||||
# dict oder Objekt akzeptieren
|
|
||||||
if isinstance(parsed, dict):
|
|
||||||
fm = parsed.get("frontmatter") or {}
|
|
||||||
body_parsed = parsed.get("body") or ""
|
|
||||||
path = parsed.get("path") or ""
|
|
||||||
else:
|
|
||||||
fm = getattr(parsed, "frontmatter", {}) or {}
|
|
||||||
body_parsed = getattr(parsed, "body", "") or ""
|
|
||||||
path = getattr(parsed, "path", "") or ""
|
|
||||||
|
|
||||||
# Hash-Quelle bestimmen
|
|
||||||
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower()
|
|
||||||
raw_body, raw_fm = ("", {})
|
|
||||||
if src == "raw":
|
|
||||||
raw_body, raw_fm = _read_raw_body_from_file(file_path or path)
|
|
||||||
# Roh-FM ergänzen (nicht überschreiben)
|
|
||||||
if isinstance(raw_fm, dict) and raw_fm:
|
|
||||||
merged_fm = dict(fm)
|
|
||||||
for k, v in raw_fm.items():
|
|
||||||
merged_fm.setdefault(k, v)
|
|
||||||
fm = merged_fm
|
|
||||||
|
|
||||||
normalize = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
|
|
||||||
mode_resolved = _resolve_hash_mode(hash_mode)
|
|
||||||
|
|
||||||
# Hash gemäß Modus/Quelle bilden
|
|
||||||
body_for_hash = raw_body if src == "raw" else body_parsed
|
|
||||||
primary_hash = compute_hash(body=body_for_hash, frontmatter=fm, mode=mode_resolved, normalize=normalize)
|
|
||||||
hash_signature = f"{'full' if mode_resolved=='body+frontmatter' else mode_resolved}:{src}:{normalize}:{primary_hash}"
|
|
||||||
|
|
||||||
# Pfad relativieren
|
|
||||||
rel_path = path
|
|
||||||
try:
|
try:
|
||||||
if vault_root:
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
rel = os.path.relpath(path, vault_root)
|
data = yaml.safe_load(f) or {}
|
||||||
rel = rel.replace("\\", "/").lstrip("/") # normalisieren
|
return data if isinstance(data, dict) else {}
|
||||||
rel_path = rel
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
return {}
|
||||||
|
|
||||||
# Note-Level-Wikilinks (Fallback, wenn Chunks nicht geliefert werden)
|
|
||||||
note_level_refs = list(dict.fromkeys(extract_wikilinks(body_parsed))) if body_parsed else []
|
def _cfg_for_type(note_type: str, reg: dict) -> dict:
|
||||||
|
"""Liefert die Konfiguration für einen konkreten Notiztyp aus der Registry."""
|
||||||
|
if not isinstance(reg, dict):
|
||||||
|
return {}
|
||||||
|
types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
|
||||||
|
return types.get(note_type, {}) if isinstance(types, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _cfg_defaults(reg: dict) -> dict:
|
||||||
|
"""Liefert den Default-Block aus der Registry (defaults/global)."""
|
||||||
|
if not isinstance(reg, dict):
|
||||||
|
return {}
|
||||||
|
for key in ("defaults", "default", "global"):
|
||||||
|
v = reg.get(key)
|
||||||
|
if isinstance(v, dict):
|
||||||
|
return v
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Haupt-API
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Baut das Note-Payload für mindnet_notes auf.
|
||||||
|
|
||||||
|
Erwartete Felder im Payload:
|
||||||
|
- note_id: stabile ID aus Frontmatter (id) oder Note-Objekt
|
||||||
|
- title: Titel der Notiz
|
||||||
|
- type: Notiztyp (z. B. concept, project, journal, ...)
|
||||||
|
- path: Pfad im Vault
|
||||||
|
- retriever_weight: effektives Gewicht für den Retriever
|
||||||
|
- chunk_profile: Profil für Chunking (short|medium|long|default|...)
|
||||||
|
- edge_defaults: Liste von Kanten-Typen, die als Defaults gelten
|
||||||
|
"""
|
||||||
|
n = _as_dict(note)
|
||||||
|
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
|
||||||
|
reg = _load_types_config(types_cfg_explicit)
|
||||||
|
|
||||||
|
fm = n.get("frontmatter") or {}
|
||||||
|
fm_type = fm.get("type") or n.get("type") or "concept"
|
||||||
|
note_type = str(fm_type)
|
||||||
|
|
||||||
|
cfg_type = _cfg_for_type(note_type, reg)
|
||||||
|
cfg_def = _cfg_defaults(reg)
|
||||||
|
|
||||||
|
# --- retriever_weight: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
|
||||||
|
default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
|
||||||
|
retriever_weight = fm.get("retriever_weight")
|
||||||
|
if retriever_weight is None:
|
||||||
|
retriever_weight = cfg_type.get(
|
||||||
|
"retriever_weight",
|
||||||
|
cfg_def.get("retriever_weight", default_rw),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
retriever_weight = float(retriever_weight)
|
||||||
|
except Exception:
|
||||||
|
retriever_weight = default_rw
|
||||||
|
|
||||||
|
# --- chunk_profile: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
|
||||||
|
chunk_profile = fm.get("chunk_profile")
|
||||||
|
if chunk_profile is None:
|
||||||
|
chunk_profile = cfg_type.get(
|
||||||
|
"chunk_profile",
|
||||||
|
cfg_def.get(
|
||||||
|
"chunk_profile",
|
||||||
|
os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE", "medium"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if not isinstance(chunk_profile, str):
|
||||||
|
chunk_profile = "medium"
|
||||||
|
|
||||||
|
# --- edge_defaults: Frontmatter > Typ-Config > Registry-Defaults ---
|
||||||
|
edge_defaults = fm.get("edge_defaults")
|
||||||
|
if edge_defaults is None:
|
||||||
|
edge_defaults = cfg_type.get(
|
||||||
|
"edge_defaults",
|
||||||
|
cfg_def.get("edge_defaults", []),
|
||||||
|
)
|
||||||
|
edge_defaults = _ensure_list(edge_defaults)
|
||||||
|
|
||||||
|
# --- Basis-Metadaten (IDs, Titel, Pfad) ---
|
||||||
|
note_id = n.get("note_id") or n.get("id") or fm.get("id")
|
||||||
|
title = n.get("title") or fm.get("title") or ""
|
||||||
|
path = n.get("path") or path_arg
|
||||||
|
if isinstance(path, pathlib.Path):
|
||||||
|
path = str(path)
|
||||||
|
|
||||||
payload: Dict[str, Any] = {
|
payload: Dict[str, Any] = {
|
||||||
"note_id": fm.get("id") or fm.get("note_id"),
|
"note_id": note_id,
|
||||||
"title": fm.get("title"),
|
"title": title,
|
||||||
"type": fm.get("type"),
|
"type": note_type,
|
||||||
"status": fm.get("status"),
|
"path": path or "",
|
||||||
"created": fm.get("created"),
|
"retriever_weight": retriever_weight,
|
||||||
"updated": fm.get("updated"),
|
"chunk_profile": chunk_profile,
|
||||||
"path": rel_path or fm.get("path"),
|
"edge_defaults": edge_defaults,
|
||||||
"tags": fm.get("tags"),
|
|
||||||
# Primärer Hash + Signatur (für Vergleich)
|
|
||||||
"hash_fulltext": primary_hash,
|
|
||||||
"hash_signature": hash_signature,
|
|
||||||
# Volltext persistieren (verlustfreie Rekonstruktion) – parsed Body
|
|
||||||
"fulltext": body_parsed,
|
|
||||||
# Fallback-Refs auf Note-Ebene
|
|
||||||
"references": note_level_refs,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for k in ("area", "project", "source", "lang", "slug", "aliases"):
|
# Tags / Keywords übernehmen
|
||||||
if k in fm:
|
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
|
||||||
payload[k] = fm[k]
|
if tags:
|
||||||
|
payload["tags"] = _ensure_list(tags)
|
||||||
|
|
||||||
# Optional: gesamtes Hash-Set persistieren (Debug/Monitoring)
|
# Zeitliche Metadaten (sofern vorhanden)
|
||||||
if os.environ.get("MINDNET_HASH_RECORD_ALL", "false").strip().lower() == "true":
|
for k in ("created", "modified", "date"):
|
||||||
payload["hashes"] = compute_hash_set(
|
v = fm.get(k) or n.get(k)
|
||||||
body_parsed=body_parsed, body_raw=raw_body, fm=fm, normalize=normalize
|
if v:
|
||||||
)
|
payload[k] = str(v)
|
||||||
|
|
||||||
# Optional: Roh-Body-Hash separat (historische Kompatibilität)
|
# JSON-Roundtrip zur harten Validierung (ASCII beibehalten)
|
||||||
if os.environ.get("MINDNET_HASH_STORE_RAW", "false").strip().lower() == "true" and src == "raw":
|
json.loads(json.dumps(payload, ensure_ascii=False))
|
||||||
try:
|
|
||||||
payload["hash_raw_body"] = compute_hash(
|
|
||||||
body=raw_body, frontmatter=fm, mode="body", normalize="none"
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# CLI – Sichtprüfung
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def _cli() -> None:
|
|
||||||
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
|
|
||||||
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zur Markdown-Datei")
|
|
||||||
ap.add_argument("--vault-root", dest="vault_root", default=None, help="Vault-Wurzel zur Pfad-Relativierung")
|
|
||||||
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
|
|
||||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter", "full"], default=None)
|
|
||||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
|
||||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
|
|
||||||
args = ap.parse_args()
|
|
||||||
|
|
||||||
parsed = read_markdown(args.src)
|
|
||||||
payload = make_note_payload(
|
|
||||||
parsed,
|
|
||||||
vault_root=args.vault_root,
|
|
||||||
hash_mode=args.hash_mode,
|
|
||||||
hash_normalize=args.hash_normalize,
|
|
||||||
hash_source=args.hash_source,
|
|
||||||
file_path=args.src,
|
|
||||||
)
|
|
||||||
|
|
||||||
if args.do_print:
|
|
||||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": # pragma: no cover
|
|
||||||
_cli()
|
|
||||||
+244
-45
@@ -1,67 +1,266 @@
|
|||||||
from __future__ import annotations
|
#!/usr/bin/env python3
|
||||||
from dataclasses import dataclass
|
# -*- coding: utf-8 -*-
|
||||||
import re
|
"""
|
||||||
import os
|
Modul: app/core/parser.py
|
||||||
import unicodedata
|
Version: 1.7.1 (fault-tolerant, API-kompatibel)
|
||||||
import yaml
|
Datum: 2025-10-01
|
||||||
from typing import Tuple, Dict
|
|
||||||
|
|
||||||
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?", re.DOTALL) # YAML-Frontmatter am Anfang
|
Zweck
|
||||||
|
-----
|
||||||
|
Fehlertolerantes Einlesen von Markdown-Dateien mit YAML-Frontmatter.
|
||||||
|
Kompatibel zur bisherigen Parser-API, aber robust gegenüber Nicht-UTF-8-Dateien:
|
||||||
|
- Versucht nacheinander: utf-8 → utf-8-sig → cp1252 → latin-1.
|
||||||
|
- Bei Fallback wird ein JSON-Warnhinweis auf stdout ausgegeben; der Import bricht NICHT ab.
|
||||||
|
- YAML-Frontmatter wird mit '---' am Anfang und '---' als Abschluss erkannt.
|
||||||
|
- extract_wikilinks() normalisiert [[id#anchor|label]] → 'id'.
|
||||||
|
|
||||||
|
Öffentliche API (kompatibel):
|
||||||
|
- class ParsedNote(frontmatter: dict, body: str, path: str)
|
||||||
|
- read_markdown(path) -> ParsedNote | None
|
||||||
|
- normalize_frontmatter(fm) -> dict
|
||||||
|
- validate_required_frontmatter(fm, required: tuple[str,...]=("id","title")) -> None
|
||||||
|
- extract_wikilinks(text) -> list[str]
|
||||||
|
- FRONTMATTER_RE (Kompatibilitäts-Konstante; Regex für '---'-Zeilen)
|
||||||
|
|
||||||
|
Beispiele
|
||||||
|
---------
|
||||||
|
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||||
|
parsed = read_markdown("./vault/30_projects/project-demo.md")
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
|
validate_required_frontmatter(fm)
|
||||||
|
body = parsed.body
|
||||||
|
|
||||||
|
from app.core.parser import extract_wikilinks
|
||||||
|
links = extract_wikilinks(body)
|
||||||
|
|
||||||
|
Abhängigkeiten
|
||||||
|
--------------
|
||||||
|
- PyYAML (yaml)
|
||||||
|
|
||||||
|
Lizenz: MIT (projektintern)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any, Dict, Optional, Tuple, Iterable, List
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # PyYAML
|
||||||
|
except Exception as e: # pragma: no cover
|
||||||
|
yaml = None # Fehler wird zur Laufzeit geworfen, falls wirklich benötigt
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# Datamodell
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ParsedNote:
|
class ParsedNote:
|
||||||
frontmatter: Dict
|
frontmatter: Dict[str, Any]
|
||||||
body: str
|
body: str
|
||||||
path: str
|
path: str
|
||||||
|
|
||||||
def _strip_bom(text: str) -> str:
|
|
||||||
return text.lstrip("\ufeff")
|
|
||||||
|
|
||||||
def _normalize_text(t: str) -> str:
|
# ---------------------------------------------------------------------
|
||||||
# Unicode-NFKC + vereinheitlichte Zeilenenden + Trim
|
# Frontmatter-Erkennung
|
||||||
t = unicodedata.normalize("NFKC", t)
|
# ---------------------------------------------------------------------
|
||||||
t = t.replace("\r\n", "\n").replace("\r", "\n")
|
|
||||||
return t
|
|
||||||
|
|
||||||
def read_markdown(path: str) -> ParsedNote:
|
# Öffentliche Kompatibilitäts-Konstante: frühere Skripte importieren FRONTMATTER_RE
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
FRONTMATTER_RE = re.compile(r"^\s*---\s*$") # <— public
|
||||||
raw = _strip_bom(f.read())
|
# Zusätzlich interner Alias (falls jemand ihn referenziert)
|
||||||
raw = _normalize_text(raw)
|
FRONTMATTER_END = FRONTMATTER_RE # <— public alias
|
||||||
|
|
||||||
m = FRONTMATTER_RE.match(raw)
|
# interne Namen bleiben bestehen
|
||||||
front, body = {}, raw
|
_FRONTMATTER_HEAD = FRONTMATTER_RE
|
||||||
if m:
|
_FRONTMATTER_END = FRONTMATTER_RE
|
||||||
yaml_block = m.group(1)
|
|
||||||
body = raw[m.end():]
|
|
||||||
|
def _split_frontmatter(text: str) -> Tuple[Dict[str, Any], str]:
|
||||||
|
"""
|
||||||
|
Zerlegt Text in (frontmatter: dict, body: str).
|
||||||
|
Erkennt Frontmatter nur, wenn die erste Zeile '---' ist und später ein zweites '---' folgt.
|
||||||
|
YAML-Fehler im Frontmatter führen NICHT zum Abbruch: es wird dann ein leeres dict benutzt.
|
||||||
|
"""
|
||||||
|
lines = text.splitlines(True) # keep line endings
|
||||||
|
if not lines:
|
||||||
|
return {}, ""
|
||||||
|
|
||||||
|
if not _FRONTMATTER_HEAD.match(lines[0]):
|
||||||
|
# kein Frontmatter-Header → gesamter Text ist Body
|
||||||
|
return {}, text
|
||||||
|
|
||||||
|
end_idx = None
|
||||||
|
# Suche nach nächstem '---' (max. 2000 Zeilen als Sicherheitslimit)
|
||||||
|
for i in range(1, min(len(lines), 2000)):
|
||||||
|
if _FRONTMATTER_END.match(lines[i]):
|
||||||
|
end_idx = i
|
||||||
|
break
|
||||||
|
|
||||||
|
if end_idx is None:
|
||||||
|
# unvollständiger Frontmatter-Block → behandle alles als Body
|
||||||
|
return {}, text
|
||||||
|
|
||||||
|
fm_raw = "".join(lines[1:end_idx])
|
||||||
|
body = "".join(lines[end_idx + 1:])
|
||||||
|
|
||||||
|
data: Dict[str, Any] = {}
|
||||||
|
if yaml is None:
|
||||||
|
raise RuntimeError("PyYAML ist nicht installiert (pip install pyyaml).")
|
||||||
|
|
||||||
|
try:
|
||||||
|
loaded = yaml.safe_load(fm_raw) or {}
|
||||||
|
if isinstance(loaded, dict):
|
||||||
|
data = loaded
|
||||||
|
else:
|
||||||
|
data = {}
|
||||||
|
except Exception as e:
|
||||||
|
# YAML-Fehler nicht fatal machen
|
||||||
|
print(json.dumps({"warn": "frontmatter_yaml_parse_failed", "error": str(e)}))
|
||||||
|
data = {}
|
||||||
|
|
||||||
|
# optionales kosmetisches Trim: eine führende Leerzeile im Body entfernen
|
||||||
|
if body.startswith("\n"):
|
||||||
|
body = body[1:]
|
||||||
|
|
||||||
|
return data, body
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# Robustes Lesen mit Encoding-Fallback
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
_FALLBACK_ENCODINGS: Tuple[str, ...] = ("utf-8", "utf-8-sig", "cp1252", "latin-1")
|
||||||
|
|
||||||
|
|
||||||
|
def _read_text_with_fallback(path: str) -> Tuple[str, str, bool]:
|
||||||
|
"""
|
||||||
|
Liest Datei mit mehreren Decodierungsversuchen.
|
||||||
|
Rückgabe: (text, used_encoding, had_fallback)
|
||||||
|
- had_fallback=True, falls NICHT 'utf-8' verwendet wurde (oder 'utf-8-sig').
|
||||||
|
"""
|
||||||
|
last_err: Optional[str] = None
|
||||||
|
for enc in _FALLBACK_ENCODINGS:
|
||||||
try:
|
try:
|
||||||
front = yaml.safe_load(yaml_block) or {}
|
with io.open(path, "r", encoding=enc, errors="strict") as f:
|
||||||
if not isinstance(front, dict):
|
text = f.read()
|
||||||
raise ValueError("Frontmatter must be a mapping")
|
# 'utf-8-sig' zählt hier als Fallback (weil BOM), aber ist unproblematisch
|
||||||
except yaml.YAMLError as e:
|
return text, enc, (enc != "utf-8")
|
||||||
raise ValueError(f"Invalid YAML frontmatter in {path}: {e}") from e
|
except UnicodeDecodeError as e:
|
||||||
|
last_err = f"{type(e).__name__}: {e}"
|
||||||
|
continue
|
||||||
|
|
||||||
return ParsedNote(frontmatter=front, body=body, path=path)
|
# Letzter, extrem defensiver Fallback: Bytes → UTF-8 mit REPLACE (keine Exception)
|
||||||
|
with open(path, "rb") as fb:
|
||||||
|
raw = fb.read()
|
||||||
|
text = raw.decode("utf-8", errors="replace")
|
||||||
|
print(json.dumps({
|
||||||
|
"path": path,
|
||||||
|
"warn": "encoding_fallback_exhausted",
|
||||||
|
"info": last_err or "unknown"
|
||||||
|
}, ensure_ascii=False))
|
||||||
|
return text, "utf-8(replace)", True
|
||||||
|
|
||||||
RE_WIKILINK = re.compile(r"\[\[([^\]\|#]+)(?:#[^\]]+)?(?:\|[^\]]+)?\]\]") # [[id]] | [[id#anchor]] | [[id|label]]
|
|
||||||
|
|
||||||
def extract_wikilinks(text: str) -> list[str]:
|
# ---------------------------------------------------------------------
|
||||||
return list({m.group(1).strip() for m in RE_WIKILINK.finditer(text)})
|
# Öffentliche API
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
def validate_required_frontmatter(fm: dict, required=("title","id","type","status","created")):
|
def read_markdown(path: str) -> Optional[ParsedNote]:
|
||||||
missing = [k for k in required if k not in fm or fm[k] in (None, "")]
|
"""
|
||||||
|
Liest eine Markdown-Datei fehlertolerant:
|
||||||
|
- Erlaubt verschiedene Encodings (UTF-8 bevorzugt, cp1252/latin-1 als Fallback).
|
||||||
|
- Schlägt NICHT mit UnicodeDecodeError fehl.
|
||||||
|
- Gibt ParsedNote(frontmatter, body, path) zurück oder None, falls die Datei nicht existiert.
|
||||||
|
|
||||||
|
Bei Decoding-Fallback wird ein JSON-Warnhinweis geloggt:
|
||||||
|
{"path": "...", "warn": "encoding_fallback_used", "used": "cp1252"}
|
||||||
|
"""
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return None
|
||||||
|
|
||||||
|
text, enc, had_fb = _read_text_with_fallback(path)
|
||||||
|
if had_fb:
|
||||||
|
print(json.dumps({"path": path, "warn": "encoding_fallback_used", "used": enc}, ensure_ascii=False))
|
||||||
|
|
||||||
|
fm, body = _split_frontmatter(text)
|
||||||
|
return ParsedNote(frontmatter=fm or {}, body=body or "", path=path)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_required_frontmatter(fm: Dict[str, Any],
|
||||||
|
required: Tuple[str, ...] = ("id", "title")) -> None:
|
||||||
|
"""
|
||||||
|
Prüft, ob alle Pflichtfelder vorhanden sind.
|
||||||
|
Default-kompatibel: ('id', 'title'), kann aber vom Aufrufer erweitert werden, z. B.:
|
||||||
|
validate_required_frontmatter(fm, required=("id","title","type","status","created"))
|
||||||
|
|
||||||
|
Hebt ValueError, falls Felder fehlen oder leer sind.
|
||||||
|
"""
|
||||||
|
if fm is None:
|
||||||
|
fm = {}
|
||||||
|
missing = []
|
||||||
|
for k in required:
|
||||||
|
v = fm.get(k)
|
||||||
|
if v is None:
|
||||||
|
missing.append(k)
|
||||||
|
elif isinstance(v, str) and not v.strip():
|
||||||
|
missing.append(k)
|
||||||
if missing:
|
if missing:
|
||||||
raise ValueError(f"Missing required frontmatter fields: {', '.join(missing)}")
|
raise ValueError(f"Missing required frontmatter fields: {', '.join(missing)}")
|
||||||
|
|
||||||
# einfache Plausibilitäten
|
# Plausibilitäten: 'tags' sollte eine Liste sein, wenn vorhanden
|
||||||
if not isinstance(fm.get("tags", []), (list, tuple)):
|
if "tags" in fm and fm["tags"] not in (None, "") and not isinstance(fm["tags"], (list, tuple)):
|
||||||
if "tags" in fm and fm["tags"] not in (None, ""):
|
raise ValueError("frontmatter 'tags' must be a list of strings")
|
||||||
raise ValueError("tags must be a list of strings")
|
|
||||||
|
|
||||||
def normalize_frontmatter(fm: dict) -> dict:
|
|
||||||
# kleinere Normalisierungen ohne die Semantik zu verändern
|
def normalize_frontmatter(fm: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
out = dict(fm)
|
"""
|
||||||
if "tags" in out and isinstance(out["tags"], list):
|
Sanfte Normalisierung ohne Semantikänderung:
|
||||||
out["tags"] = [str(t).strip() for t in out["tags"]]
|
- 'tags' → Liste von Strings (Trim)
|
||||||
|
- 'embedding_exclude' → bool
|
||||||
|
- andere Felder unverändert
|
||||||
|
"""
|
||||||
|
out = dict(fm or {})
|
||||||
|
if "tags" in out:
|
||||||
|
if isinstance(out["tags"], str):
|
||||||
|
out["tags"] = [out["tags"].strip()] if out["tags"].strip() else []
|
||||||
|
elif isinstance(out["tags"], list):
|
||||||
|
out["tags"] = [str(t).strip() for t in out["tags"] if t is not None]
|
||||||
|
else:
|
||||||
|
out["tags"] = [str(out["tags"]).strip()] if out["tags"] not in (None, "") else []
|
||||||
if "embedding_exclude" in out:
|
if "embedding_exclude" in out:
|
||||||
out["embedding_exclude"] = bool(out["embedding_exclude"])
|
out["embedding_exclude"] = bool(out["embedding_exclude"])
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------ Wikilinks ---------------------------- #
|
||||||
|
|
||||||
|
# Basismuster für [[...]]; die Normalisierung (id vor '#', vor '|') macht extract_wikilinks
|
||||||
|
_WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||||
|
|
||||||
|
|
||||||
|
def extract_wikilinks(text: str) -> List[str]:
|
||||||
|
"""
|
||||||
|
Extrahiert Wikilinks wie [[id]], [[id#anchor]], [[id|label]], [[id#anchor|label]].
|
||||||
|
Rückgabe sind NUR die Ziel-IDs (ohne Anchor/Label), führend/folgend getrimmt.
|
||||||
|
Keine aggressive Slug-Normalisierung (die kann später im Resolver erfolgen).
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return []
|
||||||
|
out: List[str] = []
|
||||||
|
for m in _WIKILINK_RE.finditer(text):
|
||||||
|
raw = (m.group(1) or "").strip()
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
# Split an Pipe (Label) → links vor '|'
|
||||||
|
if "|" in raw:
|
||||||
|
raw = raw.split("|", 1)[0].strip()
|
||||||
|
# Split an Anchor
|
||||||
|
if "#" in raw:
|
||||||
|
raw = raw.split("#", 1)[0].strip()
|
||||||
|
if raw:
|
||||||
|
out.append(raw)
|
||||||
|
return out
|
||||||
+147
-92
@@ -1,124 +1,179 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Name: app/core/qdrant.py
|
app/core/qdrant.py
|
||||||
Version: v1.4.0 (2025-09-09)
|
Version: 2.2.0 (2025-11-11)
|
||||||
|
|
||||||
Kurzbeschreibung:
|
Aufgabe
|
||||||
Qdrant-Client & Collection-Setup für mindnet.
|
-------
|
||||||
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
|
- Zentraler Qdrant-Zugriff (Client, Config)
|
||||||
- Edges-Collection nutzt 1D Dummy-Vektor.
|
- Collection-Anlage (notes/chunks/edges)
|
||||||
- NEW: ensure_payload_indexes(...) legt sinnvolle Payload-Indizes an.
|
- **Payload-Indizes sicherstellen** (idempotent)
|
||||||
|
|
||||||
Aufruf:
|
Hinweis
|
||||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
-------
|
||||||
|
Diese Datei ist als Drop-in-Ersatz gedacht, falls in deinem Projekt noch keine
|
||||||
|
robuste ensure_payload_indexes()-Implementierung vorliegt. Die Signaturen
|
||||||
|
bleiben kompatibel zu scripts.import_markdown und scripts.reset_qdrant.
|
||||||
|
|
||||||
|
API-Notizen
|
||||||
|
-----------
|
||||||
|
- Payload-Indizes werden mit `create_payload_index` angelegt.
|
||||||
|
- Typen stammen aus `qdrant_client.http.models.PayloadSchemaType`:
|
||||||
|
KEYWORD | TEXT | INTEGER | FLOAT | BOOL | GEO | DATETIME
|
||||||
|
- Für häufige Filterfelder (note_id, kind, scope, type, tags, ...) legen wir
|
||||||
|
Indizes an. Das ist laut Qdrant-Doku Best Practice für performante Filter.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple, Dict, List
|
||||||
|
|
||||||
from qdrant_client import QdrantClient
|
from qdrant_client import QdrantClient
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Konfiguration
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class QdrantConfig:
|
class QdrantConfig:
|
||||||
url: str
|
host: Optional[str] = None
|
||||||
api_key: Optional[str]
|
port: Optional[int] = None
|
||||||
prefix: str
|
url: Optional[str] = None
|
||||||
dim: int
|
api_key: Optional[str] = None
|
||||||
|
prefix: str = "mindnet"
|
||||||
|
dim: int = 384
|
||||||
|
distance: str = "Cosine" # Cosine | Dot | Euclid
|
||||||
|
on_disk_payload: bool = True
|
||||||
|
|
||||||
@staticmethod
|
@classmethod
|
||||||
def from_env() -> "QdrantConfig":
|
def from_env(cls) -> "QdrantConfig":
|
||||||
url = os.getenv("QDRANT_URL")
|
# Entweder URL ODER Host/Port, API-Key optional
|
||||||
if not url:
|
url = os.getenv("QDRANT_URL") or None
|
||||||
host = os.getenv("QDRANT_HOST", "127.0.0.1")
|
host = os.getenv("QDRANT_HOST") or None
|
||||||
port = int(os.getenv("QDRANT_PORT", "6333"))
|
port = os.getenv("QDRANT_PORT")
|
||||||
url = f"http://{host}:{port}"
|
port = int(port) if port else None
|
||||||
api_key = os.getenv("QDRANT_API_KEY") or None
|
api_key = os.getenv("QDRANT_API_KEY") or None
|
||||||
prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
|
prefix = os.getenv("COLLECTION_PREFIX") or "mindnet"
|
||||||
dim = int(os.getenv("VECTOR_DIM", "384"))
|
dim = int(os.getenv("VECTOR_DIM") or 384)
|
||||||
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
|
distance = os.getenv("DISTANCE", "Cosine")
|
||||||
|
on_disk_payload = (os.getenv("ON_DISK_PAYLOAD", "true").lower() == "true")
|
||||||
|
return cls(
|
||||||
|
host=host, port=port, url=url, api_key=api_key,
|
||||||
|
prefix=prefix, dim=dim, distance=distance, on_disk_payload=on_disk_payload
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
||||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
|
# QdrantClient akzeptiert entweder url=... oder host/port
|
||||||
|
if cfg.url:
|
||||||
|
return QdrantClient(url=cfg.url, api_key=cfg.api_key, timeout=60.0)
|
||||||
|
return QdrantClient(host=cfg.host or "127.0.0.1", port=cfg.port or 6333, api_key=cfg.api_key, timeout=60.0)
|
||||||
|
|
||||||
|
|
||||||
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
|
# ---------------------------------------------------------------------------
|
||||||
if not client.collection_exists(name):
|
# Collections
|
||||||
client.create_collection(
|
# ---------------------------------------------------------------------------
|
||||||
collection_name=name,
|
|
||||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
|
|
||||||
if not client.collection_exists(name):
|
|
||||||
client.create_collection(
|
|
||||||
collection_name=name,
|
|
||||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _create_edges(client: QdrantClient, name: str) -> None:
|
|
||||||
if not client.collection_exists(name):
|
|
||||||
client.create_collection(
|
|
||||||
collection_name=name,
|
|
||||||
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT), # 1D-Dummy
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
|
|
||||||
notes = f"{prefix}_notes"
|
|
||||||
chunks = f"{prefix}_chunks"
|
|
||||||
edges = f"{prefix}_edges"
|
|
||||||
|
|
||||||
_create_notes(client, notes, dim)
|
|
||||||
_create_chunks(client, chunks, dim)
|
|
||||||
|
|
||||||
if client.collection_exists(edges):
|
|
||||||
try:
|
|
||||||
info = client.get_collection(edges)
|
|
||||||
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
|
|
||||||
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
|
|
||||||
except Exception:
|
|
||||||
has_vectors = True
|
|
||||||
if not has_vectors:
|
|
||||||
if destructive:
|
|
||||||
client.delete_collection(edges)
|
|
||||||
_create_edges(client, edges)
|
|
||||||
else:
|
|
||||||
print(f"[ensure_collections] WARN: '{edges}' ohne VectorConfig; destructive=False.", flush=True)
|
|
||||||
else:
|
|
||||||
_create_edges(client, edges)
|
|
||||||
|
|
||||||
|
|
||||||
def collection_names(prefix: str) -> Tuple[str, str, str]:
|
def collection_names(prefix: str) -> Tuple[str, str, str]:
|
||||||
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
|
||||||
# -------------------------------
|
def _vector_params(dim: int, distance: str) -> rest.VectorParams:
|
||||||
# NEW: Payload-Indexing
|
# Distance: "Cosine" | "Dot" | "Euclid"
|
||||||
# -------------------------------
|
dist = getattr(rest.Distance, distance.capitalize(), rest.Distance.COSINE)
|
||||||
|
return rest.VectorParams(size=dim, distance=dist)
|
||||||
|
|
||||||
def _safe_create_index(client: QdrantClient, col: str, field: str, schema: rest.PayloadSchemaType):
|
|
||||||
try:
|
def ensure_collections(client: QdrantClient, prefix: str, dim: int) -> None:
|
||||||
client.create_payload_index(
|
"""Legt mindnet_notes, mindnet_chunks, mindnet_edges an (falls nicht vorhanden)."""
|
||||||
collection_name=col,
|
notes, chunks, edges = collection_names(prefix)
|
||||||
field_name=field,
|
|
||||||
field_schema=schema,
|
# notes
|
||||||
|
if not client.collection_exists(notes):
|
||||||
|
client.create_collection(
|
||||||
|
collection_name=notes,
|
||||||
|
vectors_config=_vector_params(dim, os.getenv("DISTANCE", "Cosine")),
|
||||||
|
on_disk_payload=True,
|
||||||
)
|
)
|
||||||
except Exception:
|
# chunks
|
||||||
# bereits vorhanden oder nicht unterstütztes Schema → ignorieren
|
if not client.collection_exists(chunks):
|
||||||
pass
|
client.create_collection(
|
||||||
|
collection_name=chunks,
|
||||||
|
vectors_config=_vector_params(dim, os.getenv("DISTANCE", "Cosine")),
|
||||||
|
on_disk_payload=True,
|
||||||
|
)
|
||||||
|
# edges (Dummy-Vektor, Filter via Payload)
|
||||||
|
if not client.collection_exists(edges):
|
||||||
|
client.create_collection(
|
||||||
|
collection_name=edges,
|
||||||
|
vectors_config=_vector_params(1, "Dot"),
|
||||||
|
on_disk_payload=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Payload-Indizes
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _ensure_index(client: QdrantClient, collection: str, field: str, schema: rest.PayloadSchemaType) -> None:
|
||||||
|
"""Idempotentes Anlegen eines Payload-Indexes für ein Feld."""
|
||||||
|
try:
|
||||||
|
client.create_payload_index(collection_name=collection, field_name=field, field_schema=schema, wait=True)
|
||||||
|
except Exception as e:
|
||||||
|
# Fehler ignorieren, falls Index bereits existiert oder Server "already indexed" meldet.
|
||||||
|
# Für Debugging ggf. Logging ergänzen.
|
||||||
|
_ = e
|
||||||
|
|
||||||
|
|
||||||
def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
|
def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
|
||||||
|
"""
|
||||||
|
Stellt sicher, dass alle benötigten Payload-Indizes existieren.
|
||||||
|
- notes: note_id(KEYWORD), type(KEYWORD), title(TEXT), updated(INTEGER), tags(KEYWORD)
|
||||||
|
- chunks: note_id(KEYWORD), chunk_id(KEYWORD), index(INTEGER), type(KEYWORD), tags(KEYWORD)
|
||||||
|
- edges: note_id(KEYWORD), kind(KEYWORD), scope(KEYWORD), source_id(KEYWORD), target_id(KEYWORD), chunk_id(KEYWORD)
|
||||||
|
"""
|
||||||
notes, chunks, edges = collection_names(prefix)
|
notes, chunks, edges = collection_names(prefix)
|
||||||
# Notes
|
|
||||||
_safe_create_index(client, notes, "note_id", rest.PayloadSchemaType.KEYWORD)
|
# NOTES
|
||||||
# Chunks
|
for field, schema in [
|
||||||
_safe_create_index(client, chunks, "note_id", rest.PayloadSchemaType.KEYWORD)
|
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
_safe_create_index(client, chunks, "chunk_index", rest.PayloadSchemaType.INTEGER)
|
("type", rest.PayloadSchemaType.KEYWORD),
|
||||||
# Edges
|
("title", rest.PayloadSchemaType.TEXT),
|
||||||
for f in ("kind", "scope", "source_id", "target_id", "note_id"):
|
("updated", rest.PayloadSchemaType.INTEGER),
|
||||||
_safe_create_index(client, edges, f, rest.PayloadSchemaType.KEYWORD)
|
("tags", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
]:
|
||||||
|
_ensure_index(client, notes, field, schema)
|
||||||
|
|
||||||
|
# CHUNKS
|
||||||
|
for field, schema in [
|
||||||
|
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("chunk_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("index", rest.PayloadSchemaType.INTEGER),
|
||||||
|
("type", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("tags", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
]:
|
||||||
|
_ensure_index(client, chunks, field, schema)
|
||||||
|
|
||||||
|
# EDGES
|
||||||
|
for field, schema in [
|
||||||
|
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("kind", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("scope", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("source_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("target_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("chunk_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
]:
|
||||||
|
_ensure_index(client, edges, field, schema)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"QdrantConfig",
|
||||||
|
"get_client",
|
||||||
|
"ensure_collections",
|
||||||
|
"ensure_payload_indexes",
|
||||||
|
"collection_names",
|
||||||
|
]
|
||||||
+233
-69
@@ -1,55 +1,38 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
app/core/qdrant_points.py
|
app/core/qdrant_points.py - robust points helpers for Qdrant
|
||||||
|
|
||||||
Zweck
|
- Single source of truth for building PointStruct for notes/chunks/edges
|
||||||
- Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
|
- Backward-compatible payloads for edges
|
||||||
- Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
|
- Handles both Single-Vector and Named-Vector collections
|
||||||
- alt: {'edge_type','src_id','dst_id', ...}
|
- Deterministic overrides via ENV to avoid auto-detection traps:
|
||||||
- neu: {'kind','source_id','target_id', ...}
|
* NOTES_VECTOR_NAME, CHUNKS_VECTOR_NAME, EDGES_VECTOR_NAME
|
||||||
|
* MINDNET_VECTOR_NAME (fallback)
|
||||||
|
> Set to a concrete name (e.g. "text") to force Named-Vector with that name
|
||||||
|
> Set to "__single__" (or "single") to force Single-Vector
|
||||||
|
|
||||||
Version
|
Version: 1.5.0 (2025-11-08)
|
||||||
- 1.3 (2025-09-08)
|
|
||||||
|
|
||||||
Änderungen (ggü. 1.2)
|
|
||||||
- points_for_edges() akzeptiert jetzt beide Edge-Schemata.
|
|
||||||
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
|
|
||||||
stabile 'edge_id' zurück in die Payload.
|
|
||||||
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
|
|
||||||
|
|
||||||
Aufruf / Verwendung
|
|
||||||
- Wird von Import-/Backfill-Skripten via:
|
|
||||||
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
|
|
||||||
eingebunden. Keine CLI.
|
|
||||||
|
|
||||||
Hinweise
|
|
||||||
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
|
|
||||||
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
from typing import List, Tuple
|
from typing import List, Tuple, Iterable, Optional, Dict, Any
|
||||||
from qdrant_client.http import models as rest
|
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
|
||||||
|
# --------------------- ID helpers ---------------------
|
||||||
|
|
||||||
|
def _to_uuid(stable_key: str) -> str:
|
||||||
|
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||||
|
|
||||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
# --------------------- Points builders ---------------------
|
||||||
|
|
||||||
def _to_uuid(stable_key: str) -> str:
|
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
|
||||||
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
|
|
||||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
|
||||||
|
|
||||||
|
|
||||||
def points_for_note(
|
|
||||||
prefix: str,
|
|
||||||
note_payload: dict,
|
|
||||||
note_vec: List[float] | None,
|
|
||||||
dim: int,
|
|
||||||
) -> Tuple[str, List[rest.PointStruct]]:
|
|
||||||
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
|
|
||||||
notes_col, _, _ = _names(prefix)
|
notes_col, _, _ = _names(prefix)
|
||||||
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
||||||
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
||||||
@@ -57,18 +40,7 @@ def points_for_note(
|
|||||||
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
||||||
return notes_col, [pt]
|
return notes_col, [pt]
|
||||||
|
|
||||||
|
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
|
||||||
def points_for_chunks(
|
|
||||||
prefix: str,
|
|
||||||
chunk_payloads: List[dict],
|
|
||||||
vectors: List[List[float]],
|
|
||||||
) -> Tuple[str, List[rest.PointStruct]]:
|
|
||||||
"""
|
|
||||||
Chunks-Collection: erwartet pro Chunk einen Vektor.
|
|
||||||
Robustheit:
|
|
||||||
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
|
|
||||||
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
|
|
||||||
"""
|
|
||||||
_, chunks_col, _ = _names(prefix)
|
_, chunks_col, _ = _names(prefix)
|
||||||
points: List[rest.PointStruct] = []
|
points: List[rest.PointStruct] = []
|
||||||
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
||||||
@@ -81,22 +53,12 @@ def points_for_chunks(
|
|||||||
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
||||||
return chunks_col, points
|
return chunks_col, points
|
||||||
|
|
||||||
|
|
||||||
def _normalize_edge_payload(pl: dict) -> dict:
|
def _normalize_edge_payload(pl: dict) -> dict:
|
||||||
"""
|
|
||||||
Sorgt für kompatible Feldnamen.
|
|
||||||
akzeptiert:
|
|
||||||
- neu: kind, source_id, target_id, seq?
|
|
||||||
- alt: edge_type, src_id, dst_id, order?/index?
|
|
||||||
schreibt zurück: kind, source_id, target_id, seq?
|
|
||||||
"""
|
|
||||||
# bereits neu?
|
|
||||||
kind = pl.get("kind") or pl.get("edge_type") or "edge"
|
kind = pl.get("kind") or pl.get("edge_type") or "edge"
|
||||||
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
|
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
|
||||||
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
|
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
|
||||||
seq = pl.get("seq") or pl.get("order") or pl.get("index")
|
seq = pl.get("seq") or pl.get("order") or pl.get("index")
|
||||||
|
|
||||||
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
|
|
||||||
pl.setdefault("kind", kind)
|
pl.setdefault("kind", kind)
|
||||||
pl.setdefault("source_id", source_id)
|
pl.setdefault("source_id", source_id)
|
||||||
pl.setdefault("target_id", target_id)
|
pl.setdefault("target_id", target_id)
|
||||||
@@ -104,18 +66,11 @@ def _normalize_edge_payload(pl: dict) -> dict:
|
|||||||
pl["seq"] = seq
|
pl["seq"] = seq
|
||||||
return pl
|
return pl
|
||||||
|
|
||||||
|
|
||||||
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
||||||
"""
|
|
||||||
Edges-Collection mit 1D-Dummy-Vektor.
|
|
||||||
- Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
|
|
||||||
- Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
|
|
||||||
"""
|
|
||||||
_, _, edges_col = _names(prefix)
|
_, _, edges_col = _names(prefix)
|
||||||
points: List[rest.PointStruct] = []
|
points: List[rest.PointStruct] = []
|
||||||
for raw in edge_payloads:
|
for raw in edge_payloads:
|
||||||
pl = _normalize_edge_payload(raw)
|
pl = _normalize_edge_payload(raw)
|
||||||
|
|
||||||
edge_id = pl.get("edge_id")
|
edge_id = pl.get("edge_id")
|
||||||
if not edge_id:
|
if not edge_id:
|
||||||
kind = pl.get("kind", "edge")
|
kind = pl.get("kind", "edge")
|
||||||
@@ -124,13 +79,222 @@ def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[
|
|||||||
seq = pl.get("seq") or ""
|
seq = pl.get("seq") or ""
|
||||||
edge_id = f"{kind}:{s}->{t}#{seq}"
|
edge_id = f"{kind}:{s}->{t}#{seq}"
|
||||||
pl["edge_id"] = edge_id
|
pl["edge_id"] = edge_id
|
||||||
|
|
||||||
point_id = _to_uuid(edge_id)
|
point_id = _to_uuid(edge_id)
|
||||||
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
||||||
return edges_col, points
|
return edges_col, points
|
||||||
|
|
||||||
|
# --------------------- Vector schema & overrides ---------------------
|
||||||
|
|
||||||
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
|
def _preferred_name(candidates: List[str]) -> str:
|
||||||
|
for k in ("text", "default", "embedding", "content"):
|
||||||
|
if k in candidates:
|
||||||
|
return k
|
||||||
|
return sorted(candidates)[0]
|
||||||
|
|
||||||
|
def _env_override_for_collection(collection: str) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Returns:
|
||||||
|
- "__single__" to force single-vector
|
||||||
|
- concrete name (str) to force named-vector with that name
|
||||||
|
- None to auto-detect
|
||||||
|
"""
|
||||||
|
base = os.getenv("MINDNET_VECTOR_NAME")
|
||||||
|
if collection.endswith("_notes"):
|
||||||
|
base = os.getenv("NOTES_VECTOR_NAME", base)
|
||||||
|
elif collection.endswith("_chunks"):
|
||||||
|
base = os.getenv("CHUNKS_VECTOR_NAME", base)
|
||||||
|
elif collection.endswith("_edges"):
|
||||||
|
base = os.getenv("EDGES_VECTOR_NAME", base)
|
||||||
|
|
||||||
|
if not base:
|
||||||
|
return None
|
||||||
|
val = base.strip()
|
||||||
|
if val.lower() in ("__single__", "single"):
|
||||||
|
return "__single__"
|
||||||
|
return val # concrete name
|
||||||
|
|
||||||
|
def _get_vector_schema(client: QdrantClient, collection_name: str) -> dict:
|
||||||
|
"""
|
||||||
|
Return {"kind": "single", "size": int} or {"kind": "named", "names": [...], "primary": str}.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=collection_name)
|
||||||
|
vecs = getattr(info, "vectors", None)
|
||||||
|
# Single-vector config
|
||||||
|
if hasattr(vecs, "size") and isinstance(vecs.size, int):
|
||||||
|
return {"kind": "single", "size": vecs.size}
|
||||||
|
# Named-vectors config (dict-like in .config)
|
||||||
|
cfg = getattr(vecs, "config", None)
|
||||||
|
if isinstance(cfg, dict) and cfg:
|
||||||
|
names = list(cfg.keys())
|
||||||
|
if names:
|
||||||
|
return {"kind": "named", "names": names, "primary": _preferred_name(names)}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"kind": "single", "size": None}
|
||||||
|
|
||||||
|
def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruct]:
|
||||||
|
out: List[rest.PointStruct] = []
|
||||||
|
for pt in points:
|
||||||
|
vec = getattr(pt, "vector", None)
|
||||||
|
if isinstance(vec, dict):
|
||||||
|
if name in vec:
|
||||||
|
out.append(pt)
|
||||||
|
else:
|
||||||
|
# take any existing entry; if empty dict fallback to [0.0]
|
||||||
|
fallback_vec = None
|
||||||
|
try:
|
||||||
|
fallback_vec = list(next(iter(vec.values())))
|
||||||
|
except Exception:
|
||||||
|
fallback_vec = [0.0]
|
||||||
|
out.append(rest.PointStruct(id=pt.id, vector={name: fallback_vec}, payload=pt.payload))
|
||||||
|
elif vec is not None:
|
||||||
|
out.append(rest.PointStruct(id=pt.id, vector={name: vec}, payload=pt.payload))
|
||||||
|
else:
|
||||||
|
out.append(pt)
|
||||||
|
return out
|
||||||
|
|
||||||
|
# --------------------- Qdrant ops ---------------------
|
||||||
|
|
||||||
|
def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointStruct]) -> None:
|
||||||
if not points:
|
if not points:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# 1) ENV overrides come first
|
||||||
|
override = _env_override_for_collection(collection)
|
||||||
|
if override == "__single__":
|
||||||
|
client.upsert(collection_name=collection, points=points, wait=True)
|
||||||
|
return
|
||||||
|
elif isinstance(override, str):
|
||||||
|
client.upsert(collection_name=collection, points=_as_named(points, override), wait=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 2) Auto-detect schema
|
||||||
|
schema = _get_vector_schema(client, collection)
|
||||||
|
if schema.get("kind") == "named":
|
||||||
|
name = schema.get("primary") or _preferred_name(schema.get("names") or [])
|
||||||
|
client.upsert(collection_name=collection, points=_as_named(points, name), wait=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 3) Fallback single-vector
|
||||||
client.upsert(collection_name=collection, points=points, wait=True)
|
client.upsert(collection_name=collection, points=points, wait=True)
|
||||||
|
|
||||||
|
# --- Optional search helpers ---
|
||||||
|
|
||||||
|
def _filter_any(field: str, values: Iterable[str]) -> rest.Filter:
|
||||||
|
return rest.Filter(should=[rest.FieldCondition(key=field, match=rest.MatchValue(value=v)) for v in values])
|
||||||
|
|
||||||
|
def _merge_filters(*filters: Optional[rest.Filter]) -> Optional[rest.Filter]:
|
||||||
|
fs = [f for f in filters if f is not None]
|
||||||
|
if not fs:
|
||||||
|
return None
|
||||||
|
if len(fs) == 1:
|
||||||
|
return fs[0]
|
||||||
|
must = []
|
||||||
|
for f in fs:
|
||||||
|
if getattr(f, "must", None):
|
||||||
|
must.extend(f.must)
|
||||||
|
if getattr(f, "should", None):
|
||||||
|
must.append(rest.Filter(should=f.should))
|
||||||
|
return rest.Filter(must=must)
|
||||||
|
|
||||||
|
def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter]:
|
||||||
|
if not filters:
|
||||||
|
return None
|
||||||
|
parts = []
|
||||||
|
for k, v in filters.items():
|
||||||
|
if isinstance(v, (list, tuple, set)):
|
||||||
|
parts.append(_filter_any(k, [str(x) for x in v]))
|
||||||
|
else:
|
||||||
|
parts.append(rest.Filter(must=[rest.FieldCondition(key=k, match=rest.MatchValue(value=v))]))
|
||||||
|
return _merge_filters(*parts)
|
||||||
|
|
||||||
|
def search_chunks_by_vector(client: QdrantClient, prefix: str, vector: List[float], top: int = 10, filters: Optional[Dict[str, Any]] = None) -> List[Tuple[str, float, dict]]:
|
||||||
|
_, chunks_col, _ = _names(prefix)
|
||||||
|
flt = _filter_from_dict(filters)
|
||||||
|
res = client.search(collection_name=chunks_col, query_vector=vector, limit=top, with_payload=True, with_vectors=False, query_filter=flt)
|
||||||
|
out: List[Tuple[str, float, dict]] = []
|
||||||
|
for r in res:
|
||||||
|
out.append((str(r.id), float(r.score), dict(r.payload or {})))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# --- Edge retrieval helper ---
|
||||||
|
|
||||||
|
def get_edges_for_sources(
|
||||||
|
client: QdrantClient,
|
||||||
|
prefix: str,
|
||||||
|
source_ids: Iterable[str],
|
||||||
|
edge_types: Optional[Iterable[str]] = None,
|
||||||
|
limit: int = 2048,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Retrieve edge payloads from the <prefix>_edges collection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: QdrantClient instance.
|
||||||
|
prefix: Mindnet collection prefix (e.g. "mindnet").
|
||||||
|
source_ids: Iterable of source_id values (typically chunk_ids or note_ids).
|
||||||
|
edge_types: Optional iterable of edge kinds (e.g. ["references", "depends_on"]). If None,
|
||||||
|
all kinds are returned.
|
||||||
|
limit: Maximum number of edge payloads to return.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A list of edge payload dicts, e.g.:
|
||||||
|
{
|
||||||
|
"note_id": "...",
|
||||||
|
"chunk_id": "...",
|
||||||
|
"kind": "references" | "depends_on" | ...,
|
||||||
|
"scope": "chunk",
|
||||||
|
"source_id": "...",
|
||||||
|
"target_id": "...",
|
||||||
|
"rule_id": "...",
|
||||||
|
"confidence": 0.7,
|
||||||
|
...
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
source_ids = list(source_ids)
|
||||||
|
if not source_ids or limit <= 0:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Resolve collection name
|
||||||
|
_, _, edges_col = _names(prefix)
|
||||||
|
|
||||||
|
# Build filter: source_id IN source_ids
|
||||||
|
src_filter = _filter_any("source_id", [str(s) for s in source_ids])
|
||||||
|
|
||||||
|
# Optional: kind IN edge_types
|
||||||
|
kind_filter = None
|
||||||
|
if edge_types:
|
||||||
|
kind_filter = _filter_any("kind", [str(k) for k in edge_types])
|
||||||
|
|
||||||
|
flt = _merge_filters(src_filter, kind_filter)
|
||||||
|
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
next_page = None
|
||||||
|
remaining = int(limit)
|
||||||
|
|
||||||
|
# Use paginated scroll API; we don't need vectors, only payloads.
|
||||||
|
while remaining > 0:
|
||||||
|
batch_limit = min(256, remaining)
|
||||||
|
res, next_page = client.scroll(
|
||||||
|
collection_name=edges_col,
|
||||||
|
scroll_filter=flt,
|
||||||
|
limit=batch_limit,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not res:
|
||||||
|
break
|
||||||
|
|
||||||
|
for r in res:
|
||||||
|
out.append(dict(r.payload or {}))
|
||||||
|
remaining -= 1
|
||||||
|
if remaining <= 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
if next_page is None or remaining <= 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
return out
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
"""
|
||||||
|
app/core/ranking.py — Kombiniertes Scoring (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Zusammenführen von semantischem Score (normalisiert), Edge-Bonus und
|
||||||
|
Centrality-Bonus in einen Gesamtscore für die Ergebnisreihung.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
WP-04 Ranking-Formel (w_sem, w_edge, w_cent)
|
||||||
|
Nutzung:
|
||||||
|
from app.core.ranking import combine_scores
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List, Tuple, Dict
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_scores(values: List[float]) -> List[float]:
|
||||||
|
"""Min-Max-Normalisierung über die Kandidatenmenge (Fallback 0.5 bei Konstanz)."""
|
||||||
|
if not values:
|
||||||
|
return values
|
||||||
|
lo, hi = min(values), max(values)
|
||||||
|
if hi - lo < 1e-9:
|
||||||
|
return [0.5] * len(values)
|
||||||
|
return [(v - lo) / (hi - lo) for v in values]
|
||||||
|
|
||||||
|
|
||||||
|
def combine_scores(
|
||||||
|
hits: List[Tuple[str, float, dict]], # (id, semantic_score, payload)
|
||||||
|
edge_bonus_map: Dict[str, float],
|
||||||
|
centrality_map: Dict[str, float],
|
||||||
|
w_sem: float = 0.70,
|
||||||
|
w_edge: float = 0.25,
|
||||||
|
w_cent: float = 0.05,
|
||||||
|
) -> List[Tuple[str, float, float, float, float]]:
|
||||||
|
"""
|
||||||
|
Liefert Liste von (point_id, total_score, edge_bonus, centrality_bonus, raw_semantic_score),
|
||||||
|
absteigend nach total_score sortiert.
|
||||||
|
"""
|
||||||
|
sem = [h[1] for h in hits]
|
||||||
|
sem_n = normalize_scores(sem)
|
||||||
|
out = []
|
||||||
|
for (pid, s, payload), s_norm in zip(hits, sem_n):
|
||||||
|
e = edge_bonus_map.get(pid, 0.0)
|
||||||
|
c = centrality_map.get(pid, 0.0)
|
||||||
|
total = w_sem * s_norm + w_edge * e + w_cent * c
|
||||||
|
out.append((pid, total, e, c, s))
|
||||||
|
out.sort(key=lambda t: t[1], reverse=True)
|
||||||
|
return out
|
||||||
@@ -0,0 +1,333 @@
|
|||||||
|
"""
|
||||||
|
app/core/retriever.py — Hybrider Such-Algorithmus
|
||||||
|
|
||||||
|
Version:
|
||||||
|
0.5.2 (WP-05 Fix: Pass content in QueryHit source)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from functools import lru_cache
|
||||||
|
from typing import Any, Dict, List, Tuple, Iterable, Optional
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
|
from app.models.dto import (
|
||||||
|
QueryRequest,
|
||||||
|
QueryResponse,
|
||||||
|
QueryHit,
|
||||||
|
Explanation,
|
||||||
|
ScoreBreakdown,
|
||||||
|
Reason,
|
||||||
|
EdgeDTO
|
||||||
|
)
|
||||||
|
import app.core.qdrant as qdr
|
||||||
|
import app.core.qdrant_points as qp
|
||||||
|
import app.services.embeddings_client as ec
|
||||||
|
import app.core.graph_adapter as ga
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # type: ignore[import]
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
yaml = None # type: ignore[assignment]
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache
|
||||||
|
def _get_scoring_weights() -> Tuple[float, float, float]:
|
||||||
|
"""Liefert (semantic_weight, edge_weight, centrality_weight) für den Retriever."""
|
||||||
|
settings = get_settings()
|
||||||
|
sem = float(getattr(settings, "RETRIEVER_W_SEM", 1.0))
|
||||||
|
edge = float(getattr(settings, "RETRIEVER_W_EDGE", 0.0))
|
||||||
|
cent = float(getattr(settings, "RETRIEVER_W_CENT", 0.0))
|
||||||
|
|
||||||
|
config_path = os.getenv("MINDNET_RETRIEVER_CONFIG", "config/retriever.yaml")
|
||||||
|
if yaml is None:
|
||||||
|
return sem, edge, cent
|
||||||
|
try:
|
||||||
|
if os.path.exists(config_path):
|
||||||
|
with open(config_path, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
scoring = data.get("scoring", {}) or {}
|
||||||
|
sem = float(scoring.get("semantic_weight", sem))
|
||||||
|
edge = float(scoring.get("edge_weight", edge))
|
||||||
|
cent = float(scoring.get("centrality_weight", cent))
|
||||||
|
except Exception:
|
||||||
|
return sem, edge, cent
|
||||||
|
return sem, edge, cent
|
||||||
|
|
||||||
|
|
||||||
|
def _get_client_and_prefix() -> Tuple[Any, str]:
|
||||||
|
"""Liefert (QdrantClient, prefix)."""
|
||||||
|
cfg = qdr.QdrantConfig.from_env()
|
||||||
|
client = qdr.get_client(cfg)
|
||||||
|
return client, cfg.prefix
|
||||||
|
|
||||||
|
|
||||||
|
def _get_query_vector(req: QueryRequest) -> List[float]:
|
||||||
|
"""Liefert den Query-Vektor aus dem Request."""
|
||||||
|
if req.query_vector:
|
||||||
|
return list(req.query_vector)
|
||||||
|
|
||||||
|
if not req.query:
|
||||||
|
raise ValueError("QueryRequest benötigt entweder query oder query_vector")
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
model_name = settings.MODEL_NAME
|
||||||
|
|
||||||
|
try:
|
||||||
|
return ec.embed_text(req.query, model_name=model_name)
|
||||||
|
except TypeError:
|
||||||
|
return ec.embed_text(req.query)
|
||||||
|
|
||||||
|
|
||||||
|
def _semantic_hits(
|
||||||
|
client: Any,
|
||||||
|
prefix: str,
|
||||||
|
vector: List[float],
|
||||||
|
top_k: int,
|
||||||
|
filters: Dict[str, Any] | None = None,
|
||||||
|
) -> List[Tuple[str, float, Dict[str, Any]]]:
|
||||||
|
"""Führt eine semantische Suche aus."""
|
||||||
|
flt = filters or None
|
||||||
|
raw_hits = qp.search_chunks_by_vector(client, prefix, vector, top=top_k, filters=flt)
|
||||||
|
results: List[Tuple[str, float, Dict[str, Any]]] = []
|
||||||
|
for pid, score, payload in raw_hits:
|
||||||
|
results.append((str(pid), float(score), dict(payload or {})))
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_total_score(
|
||||||
|
semantic_score: float,
|
||||||
|
payload: Dict[str, Any],
|
||||||
|
edge_bonus: float = 0.0,
|
||||||
|
cent_bonus: float = 0.0,
|
||||||
|
) -> Tuple[float, float, float]:
|
||||||
|
"""Berechnet total_score."""
|
||||||
|
raw_weight = payload.get("retriever_weight", 1.0)
|
||||||
|
try:
|
||||||
|
weight = float(raw_weight)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
weight = 1.0
|
||||||
|
if weight < 0.0:
|
||||||
|
weight = 0.0
|
||||||
|
|
||||||
|
sem_w, edge_w, cent_w = _get_scoring_weights()
|
||||||
|
total = (sem_w * float(semantic_score) * weight) + (edge_w * edge_bonus) + (cent_w * cent_bonus)
|
||||||
|
return float(total), float(edge_bonus), float(cent_bonus)
|
||||||
|
|
||||||
|
|
||||||
|
# --- WP-04b Explanation Logic ---
|
||||||
|
|
||||||
|
def _build_explanation(
|
||||||
|
semantic_score: float,
|
||||||
|
payload: Dict[str, Any],
|
||||||
|
edge_bonus: float,
|
||||||
|
cent_bonus: float,
|
||||||
|
subgraph: Optional[ga.Subgraph],
|
||||||
|
node_key: Optional[str]
|
||||||
|
) -> Explanation:
|
||||||
|
"""Erstellt ein Explanation-Objekt."""
|
||||||
|
sem_w, edge_w, cent_w = _get_scoring_weights()
|
||||||
|
|
||||||
|
try:
|
||||||
|
type_weight = float(payload.get("retriever_weight", 1.0))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
type_weight = 1.0
|
||||||
|
|
||||||
|
note_type = payload.get("type", "unknown")
|
||||||
|
|
||||||
|
breakdown = ScoreBreakdown(
|
||||||
|
semantic_contribution=(sem_w * semantic_score * type_weight),
|
||||||
|
edge_contribution=(edge_w * edge_bonus),
|
||||||
|
centrality_contribution=(cent_w * cent_bonus),
|
||||||
|
raw_semantic=semantic_score,
|
||||||
|
raw_edge_bonus=edge_bonus,
|
||||||
|
raw_centrality=cent_bonus,
|
||||||
|
node_weight=type_weight
|
||||||
|
)
|
||||||
|
|
||||||
|
reasons: List[Reason] = []
|
||||||
|
edges_dto: List[EdgeDTO] = []
|
||||||
|
|
||||||
|
if semantic_score > 0.85:
|
||||||
|
reasons.append(Reason(kind="semantic", message="Sehr hohe textuelle Übereinstimmung.", score_impact=breakdown.semantic_contribution))
|
||||||
|
elif semantic_score > 0.70:
|
||||||
|
reasons.append(Reason(kind="semantic", message="Gute textuelle Übereinstimmung.", score_impact=breakdown.semantic_contribution))
|
||||||
|
|
||||||
|
if type_weight != 1.0:
|
||||||
|
msg = "Bevorzugt" if type_weight > 1.0 else "Leicht abgewertet"
|
||||||
|
reasons.append(Reason(kind="type", message=f"{msg} aufgrund des Typs '{note_type}'.", score_impact=(sem_w * semantic_score * (type_weight - 1.0))))
|
||||||
|
|
||||||
|
if subgraph and node_key and edge_bonus > 0:
|
||||||
|
if hasattr(subgraph, "get_outgoing_edges"):
|
||||||
|
outgoing = subgraph.get_outgoing_edges(node_key)
|
||||||
|
for edge in outgoing:
|
||||||
|
target = edge.get("target", "Unknown")
|
||||||
|
kind = edge.get("kind", "edge")
|
||||||
|
weight = edge.get("weight", 0.0)
|
||||||
|
if weight > 0.05:
|
||||||
|
edges_dto.append(EdgeDTO(id=f"{node_key}->{target}:{kind}", kind=kind, source=node_key, target=target, weight=weight, direction="out"))
|
||||||
|
|
||||||
|
if hasattr(subgraph, "get_incoming_edges"):
|
||||||
|
incoming = subgraph.get_incoming_edges(node_key)
|
||||||
|
for edge in incoming:
|
||||||
|
src = edge.get("source", "Unknown")
|
||||||
|
kind = edge.get("kind", "edge")
|
||||||
|
weight = edge.get("weight", 0.0)
|
||||||
|
if weight > 0.05:
|
||||||
|
edges_dto.append(EdgeDTO(id=f"{src}->{node_key}:{kind}", kind=kind, source=src, target=node_key, weight=weight, direction="in"))
|
||||||
|
|
||||||
|
all_edges = sorted(edges_dto, key=lambda e: e.weight, reverse=True)
|
||||||
|
for top_edge in all_edges[:3]:
|
||||||
|
impact = edge_w * top_edge.weight
|
||||||
|
dir_txt = "Verweist auf" if top_edge.direction == "out" else "Referenziert von"
|
||||||
|
tgt_txt = top_edge.target if top_edge.direction == "out" else top_edge.source
|
||||||
|
reasons.append(Reason(kind="edge", message=f"{dir_txt} '{tgt_txt}' via '{top_edge.kind}'", score_impact=impact, details={"kind": top_edge.kind}))
|
||||||
|
|
||||||
|
if cent_bonus > 0.01:
|
||||||
|
reasons.append(Reason(kind="centrality", message="Knoten liegt zentral im Kontext.", score_impact=breakdown.centrality_contribution))
|
||||||
|
|
||||||
|
return Explanation(breakdown=breakdown, reasons=reasons, related_edges=edges_dto if edges_dto else None)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
|
||||||
|
"""Extrahiert depth und edge_types."""
|
||||||
|
expand = getattr(req, "expand", None)
|
||||||
|
if not expand:
|
||||||
|
return 0, None
|
||||||
|
|
||||||
|
depth = 1
|
||||||
|
edge_types: List[str] | None = None
|
||||||
|
|
||||||
|
if hasattr(expand, "depth") or hasattr(expand, "edge_types"):
|
||||||
|
depth = int(getattr(expand, "depth", 1) or 1)
|
||||||
|
types_val = getattr(expand, "edge_types", None)
|
||||||
|
if types_val:
|
||||||
|
edge_types = list(types_val)
|
||||||
|
return depth, edge_types
|
||||||
|
|
||||||
|
if isinstance(expand, dict):
|
||||||
|
if "depth" in expand:
|
||||||
|
depth = int(expand.get("depth") or 1)
|
||||||
|
if "edge_types" in expand and expand["edge_types"] is not None:
|
||||||
|
edge_types = list(expand["edge_types"])
|
||||||
|
return depth, edge_types
|
||||||
|
|
||||||
|
return 0, None
|
||||||
|
|
||||||
|
|
||||||
|
def _build_hits_from_semantic(
|
||||||
|
hits: Iterable[Tuple[str, float, Dict[str, Any]]],
|
||||||
|
top_k: int,
|
||||||
|
used_mode: str,
|
||||||
|
subgraph: ga.Subgraph | None = None,
|
||||||
|
explain: bool = False,
|
||||||
|
) -> QueryResponse:
|
||||||
|
"""Baut strukturierte QueryHits."""
|
||||||
|
t0 = time.time()
|
||||||
|
enriched: List[Tuple[str, float, Dict[str, Any], float, float, float]] = []
|
||||||
|
|
||||||
|
for pid, semantic_score, payload in hits:
|
||||||
|
edge_bonus = 0.0
|
||||||
|
cent_bonus = 0.0
|
||||||
|
node_key = payload.get("chunk_id") or payload.get("note_id")
|
||||||
|
|
||||||
|
if subgraph is not None and node_key:
|
||||||
|
try:
|
||||||
|
edge_bonus = float(subgraph.edge_bonus(node_key))
|
||||||
|
except Exception:
|
||||||
|
edge_bonus = 0.0
|
||||||
|
try:
|
||||||
|
cent_bonus = float(subgraph.centrality_bonus(node_key))
|
||||||
|
except Exception:
|
||||||
|
cent_bonus = 0.0
|
||||||
|
|
||||||
|
total, edge_bonus, cent_bonus = _compute_total_score(semantic_score, payload, edge_bonus=edge_bonus, cent_bonus=cent_bonus)
|
||||||
|
enriched.append((pid, float(semantic_score), payload, total, edge_bonus, cent_bonus))
|
||||||
|
|
||||||
|
enriched_sorted = sorted(enriched, key=lambda h: h[3], reverse=True)
|
||||||
|
limited = enriched_sorted[: max(1, top_k)]
|
||||||
|
|
||||||
|
results: List[QueryHit] = []
|
||||||
|
for pid, semantic_score, payload, total, edge_bonus, cent_bonus in limited:
|
||||||
|
explanation_obj = None
|
||||||
|
if explain:
|
||||||
|
explanation_obj = _build_explanation(
|
||||||
|
semantic_score=float(semantic_score),
|
||||||
|
payload=payload,
|
||||||
|
edge_bonus=edge_bonus,
|
||||||
|
cent_bonus=cent_bonus,
|
||||||
|
subgraph=subgraph,
|
||||||
|
node_key=payload.get("chunk_id") or payload.get("note_id")
|
||||||
|
)
|
||||||
|
|
||||||
|
# FIX: Hier holen wir jetzt den Textinhalt (text, content oder page_content) aus dem Payload
|
||||||
|
text_content = payload.get("page_content") or payload.get("text") or payload.get("content")
|
||||||
|
|
||||||
|
results.append(QueryHit(
|
||||||
|
node_id=str(pid),
|
||||||
|
note_id=payload.get("note_id"),
|
||||||
|
semantic_score=float(semantic_score),
|
||||||
|
edge_bonus=edge_bonus,
|
||||||
|
centrality_bonus=cent_bonus,
|
||||||
|
total_score=total,
|
||||||
|
paths=None,
|
||||||
|
source={
|
||||||
|
"path": payload.get("path"),
|
||||||
|
"section": payload.get("section") or payload.get("section_title"),
|
||||||
|
"text": text_content # WICHTIG: Inhalt durchreichen
|
||||||
|
},
|
||||||
|
explanation=explanation_obj
|
||||||
|
))
|
||||||
|
|
||||||
|
dt = int((time.time() - t0) * 1000)
|
||||||
|
return QueryResponse(results=results, used_mode=used_mode, latency_ms=dt)
|
||||||
|
|
||||||
|
|
||||||
|
def semantic_retrieve(req: QueryRequest) -> QueryResponse:
|
||||||
|
"""Reiner semantischer Retriever."""
|
||||||
|
client, prefix = _get_client_and_prefix()
|
||||||
|
vector = _get_query_vector(req)
|
||||||
|
top_k = req.top_k or get_settings().RETRIEVER_TOP_K
|
||||||
|
|
||||||
|
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
||||||
|
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="semantic", subgraph=None, explain=req.explain)
|
||||||
|
|
||||||
|
|
||||||
|
def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
||||||
|
"""Hybrid-Retriever: semantische Suche + optionale Edge-Expansion."""
|
||||||
|
client, prefix = _get_client_and_prefix()
|
||||||
|
if req.query_vector:
|
||||||
|
vector = list(req.query_vector)
|
||||||
|
else:
|
||||||
|
vector = _get_query_vector(req)
|
||||||
|
|
||||||
|
top_k = req.top_k or get_settings().RETRIEVER_TOP_K
|
||||||
|
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
||||||
|
|
||||||
|
depth, edge_types = _extract_expand_options(req)
|
||||||
|
subgraph: ga.Subgraph | None = None
|
||||||
|
if depth and depth > 0:
|
||||||
|
seed_ids: List[str] = []
|
||||||
|
for _pid, _score, payload in hits:
|
||||||
|
key = payload.get("chunk_id") or payload.get("note_id")
|
||||||
|
if key and key not in seed_ids:
|
||||||
|
seed_ids.append(key)
|
||||||
|
if seed_ids:
|
||||||
|
try:
|
||||||
|
subgraph = ga.expand(client, prefix, seed_ids, depth=depth, edge_types=edge_types)
|
||||||
|
except Exception:
|
||||||
|
subgraph = None
|
||||||
|
|
||||||
|
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="hybrid", subgraph=subgraph, explain=req.explain)
|
||||||
|
|
||||||
|
|
||||||
|
class Retriever:
|
||||||
|
"""
|
||||||
|
Wrapper-Klasse für WP-05 (Chat).
|
||||||
|
"""
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def search(self, request: QueryRequest) -> QueryResponse:
|
||||||
|
return hybrid_retrieve(request)
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""app/core/retriever_config.py
|
||||||
|
---------------------------------
|
||||||
|
Zentrale Konfiguration für den mindnet-Retriever (WP-04).
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
- Lädt config/retriever.yaml (falls vorhanden) oder nutzt sinnvolle Defaults.
|
||||||
|
- Bietet einen gecachten Zugriff auf die Retriever-Config für
|
||||||
|
andere Module (z. B. graph_adapter, retriever).
|
||||||
|
|
||||||
|
Hinweis zur Weiterentwicklung (Selbstjustierung):
|
||||||
|
- Die hier definierten Parameter sind so gewählt, dass sie später
|
||||||
|
durch ein Feedback-/Learning-to-Rank-Modell überschrieben werden
|
||||||
|
können, ohne die restliche Architektur anzupassen.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # type: ignore
|
||||||
|
except Exception: # pragma: no cover - Fallback, falls PyYAML nicht installiert ist.
|
||||||
|
yaml = None # type: ignore
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RetrieverConfig:
|
||||||
|
semantic_scale: float
|
||||||
|
edge_scale: float
|
||||||
|
centrality_scale: float
|
||||||
|
edge_weights: Dict[str, float]
|
||||||
|
|
||||||
|
@lru_cache
|
||||||
|
def get_retriever_config() -> RetrieverConfig:
|
||||||
|
"""Lädt die Retriever-Konfiguration (YAML + Defaults).
|
||||||
|
|
||||||
|
Reihenfolge:
|
||||||
|
1. Defaults (sinnvoll gewählte Startwerte).
|
||||||
|
2. Optional: config/retriever.yaml bzw. Pfad aus ENV
|
||||||
|
MINDNET_RETRIEVER_CONFIG überschreibt die Defaults.
|
||||||
|
|
||||||
|
Die Funktion ist bewusst gecached, da sich die Konfiguration zur
|
||||||
|
Laufzeit üblicherweise nicht ändert. Für dynamisches Nachladen
|
||||||
|
könnte der Cache explizit geleert werden.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# 1) Defaults – bewusst konservativ gewählt.
|
||||||
|
semantic_scale = 1.0
|
||||||
|
edge_scale = 1.0
|
||||||
|
centrality_scale = 1.0
|
||||||
|
|
||||||
|
edge_weights: Dict[str, float] = {
|
||||||
|
# Wissens-Kanten
|
||||||
|
"depends_on": 1.0,
|
||||||
|
"related_to": 0.7,
|
||||||
|
"similar_to": 0.7,
|
||||||
|
"references": 0.5,
|
||||||
|
# Struktur-Kanten
|
||||||
|
"belongs_to": 0.2,
|
||||||
|
"next": 0.1,
|
||||||
|
"prev": 0.1,
|
||||||
|
# Sonstige / technische Kanten
|
||||||
|
"backlink": 0.2,
|
||||||
|
"references_at": 0.2,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 2) Optional: YAML-Konfiguration laden
|
||||||
|
cfg_path_env = os.getenv("MINDNET_RETRIEVER_CONFIG")
|
||||||
|
if cfg_path_env:
|
||||||
|
cfg_path = Path(cfg_path_env)
|
||||||
|
else:
|
||||||
|
# Project-Root = zwei Ebenen über app/core/
|
||||||
|
cfg_path = Path(__file__).resolve().parents[2] / "config" / "retriever.yaml"
|
||||||
|
|
||||||
|
if yaml is not None and cfg_path.exists():
|
||||||
|
try:
|
||||||
|
data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {}
|
||||||
|
except Exception:
|
||||||
|
data = {}
|
||||||
|
|
||||||
|
retr = data.get("retriever") or {}
|
||||||
|
|
||||||
|
# Skalenwerte überschreiben, falls angegeben
|
||||||
|
try:
|
||||||
|
semantic_scale = float(retr.get("semantic_scale", semantic_scale))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
edge_scale = float(retr.get("edge_scale", edge_scale))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
centrality_scale = float(retr.get("centrality_scale", centrality_scale))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Edge-Gewichte je Kanten-Typ
|
||||||
|
ew_cfg = retr.get("edge_weights") or {}
|
||||||
|
if isinstance(ew_cfg, dict):
|
||||||
|
for k, v in ew_cfg.items():
|
||||||
|
try:
|
||||||
|
edge_weights[str(k)] = float(v)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
return RetrieverConfig(
|
||||||
|
semantic_scale=semantic_scale,
|
||||||
|
edge_scale=edge_scale,
|
||||||
|
centrality_scale=centrality_scale,
|
||||||
|
edge_weights=edge_weights,
|
||||||
|
)
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Modul: app/core/type_registry.py
|
||||||
|
Version: 1.0.0
|
||||||
|
Datum: 2025-11-08
|
||||||
|
|
||||||
|
Zweck
|
||||||
|
-----
|
||||||
|
Lädt eine optionale Typ-Registry (config/types.yaml) und stellt
|
||||||
|
komfortable Zugriffsfunktionen bereit. Die Registry ist *optional*:
|
||||||
|
- Fehlt die Datei oder ist das YAML defekt, wird ein konservativer
|
||||||
|
Default (Typ "concept") verwendet und es wird eine Warnung ausgegeben.
|
||||||
|
- Änderungen an der Datei greifen nach einem Neustart des Prozesses.
|
||||||
|
|
||||||
|
Öffentliche API
|
||||||
|
---------------
|
||||||
|
- load_type_registry(path: str = "config/types.yaml") -> dict
|
||||||
|
- get_type_config(note_type: str, reg: dict) -> dict
|
||||||
|
- resolve_note_type(fm_type: str | None, reg: dict) -> str
|
||||||
|
- effective_chunk_profile(note_type: str, reg: dict) -> str | None
|
||||||
|
- profile_overlap(profile: str | None) -> tuple[int,int] # nur Overlap-Empfehlung
|
||||||
|
|
||||||
|
Hinweis
|
||||||
|
-------
|
||||||
|
Die Registry steuert KEINE Breaking Changes. Ohne Datei/Typ bleibt das
|
||||||
|
Verhalten exakt wie im Release-Stand 20251105.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from functools import lru_cache
|
||||||
|
from typing import Dict, Any, Optional, Tuple
|
||||||
|
import os
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # PyYAML
|
||||||
|
except Exception:
|
||||||
|
yaml = None # wird erst benötigt, wenn eine Datei gelesen werden soll
|
||||||
|
|
||||||
|
# Konservativer Default – bewusst minimal
|
||||||
|
_DEFAULT_REGISTRY: Dict[str, Any] = {
|
||||||
|
"version": "1.0",
|
||||||
|
"types": {
|
||||||
|
"concept": {
|
||||||
|
"chunk_profile": "medium",
|
||||||
|
"edge_defaults": ["references", "related_to"],
|
||||||
|
"retriever_weight": 1.0,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_using_defaults": True,
|
||||||
|
"_warning": "types.yaml missing or invalid – using built-in defaults (type=concept).",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Chunk-Profile → Overlap-Empfehlungen (nur für synthetische Fensterbildung)
|
||||||
|
# Die absoluten Chunk-Längen bleiben Aufgabe des Chunkers (assemble_chunks).
|
||||||
|
_PROFILE_TO_OVERLAP: Dict[str, Tuple[int, int]] = {
|
||||||
|
"short": (20, 30),
|
||||||
|
"medium": (40, 60),
|
||||||
|
"long": (60, 80),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def load_type_registry(path: str = "config/types.yaml") -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Lädt die Registry aus 'path'. Bei Fehlern wird ein konserviver Default geliefert.
|
||||||
|
Die Rückgabe ist *prozessweit* gecached.
|
||||||
|
"""
|
||||||
|
if not path:
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
|
||||||
|
if not os.path.isfile(path):
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
|
||||||
|
if yaml is None:
|
||||||
|
# PyYAML fehlt → auf Default zurückfallen
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
# Minimal validieren
|
||||||
|
if not isinstance(data, dict) or "types" not in data or not isinstance(data["types"], dict):
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
data.setdefault("version", "1.0")
|
||||||
|
data.setdefault("_using_defaults", False)
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
|
||||||
|
|
||||||
|
def get_type_config(note_type: Optional[str], reg: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
t = (note_type or "concept").strip().lower()
|
||||||
|
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
|
||||||
|
return types.get(t) or types.get("concept") or _DEFAULT_REGISTRY["types"]["concept"]
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_note_type(fm_type: Optional[str], reg: Dict[str, Any]) -> str:
|
||||||
|
"""Liefert einen gültigen Typ (unbekannt → 'concept')."""
|
||||||
|
t = (fm_type or "concept").strip().lower()
|
||||||
|
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
|
||||||
|
return t if t in types else "concept"
|
||||||
|
|
||||||
|
|
||||||
|
def effective_chunk_profile(note_type: Optional[str], reg: Dict[str, Any]) -> Optional[str]:
|
||||||
|
cfg = get_type_config(note_type, reg)
|
||||||
|
prof = cfg.get("chunk_profile")
|
||||||
|
if isinstance(prof, str) and prof.strip():
|
||||||
|
return prof.strip().lower()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def profile_overlap(profile: Optional[str]) -> Tuple[int, int]:
|
||||||
|
"""Gibt eine Overlap-Empfehlung (low, high) für das Profil zurück."""
|
||||||
|
if not profile:
|
||||||
|
return _PROFILE_TO_OVERLAP["medium"]
|
||||||
|
return _PROFILE_TO_OVERLAP.get(profile.strip().lower(), _PROFILE_TO_OVERLAP["medium"])
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Modul: app/graph/service.py
|
||||||
|
Version: 0.1.0
|
||||||
|
Datum: 2025-09-10
|
||||||
|
|
||||||
|
Zweck
|
||||||
|
-----
|
||||||
|
Leichtgewichtiger Graph-Layer über Qdrant:
|
||||||
|
- get_note(note_id)
|
||||||
|
- get_chunks(note_id)
|
||||||
|
- neighbors(source_id, kinds=[...], scope=['note','chunk'], depth=1)
|
||||||
|
- walk_bfs(source_id, kinds, max_depth)
|
||||||
|
- context_for_note(note_id, max_neighbors): heuristische Kontextsammlung
|
||||||
|
|
||||||
|
Hinweise
|
||||||
|
--------
|
||||||
|
- Nutzt die bestehenden Collections <prefix>_notes/_chunks/_edges.
|
||||||
|
- Edges werden über Payload-Felder (`kind`, `source_id`, `target_id`) abgefragt.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List, Dict, Any, Optional, Iterable, Set, Tuple
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def _cols(prefix: str):
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
class GraphService:
|
||||||
|
def __init__(self, cfg: Optional[QdrantConfig] = None, prefix: Optional[str] = None):
|
||||||
|
self.cfg = cfg or QdrantConfig.from_env()
|
||||||
|
if prefix:
|
||||||
|
self.cfg.prefix = prefix
|
||||||
|
self.client = get_client(self.cfg)
|
||||||
|
self.notes_col, self.chunks_col, self.edges_col = _cols(self.cfg.prefix)
|
||||||
|
|
||||||
|
# ------------------------ fetch helpers ------------------------
|
||||||
|
def _scroll(self, col: str, flt: Optional[rest.Filter] = None, limit: int = 256):
|
||||||
|
out = []
|
||||||
|
nextp = None
|
||||||
|
while True:
|
||||||
|
pts, nextp = self.client.scroll(
|
||||||
|
collection_name=col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=limit,
|
||||||
|
offset=nextp,
|
||||||
|
scroll_filter=flt,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
out.extend(pts)
|
||||||
|
if nextp is None:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
# ------------------------ public API ---------------------------
|
||||||
|
def get_note(self, note_id: str) -> Optional[Dict[str, Any]]:
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
pts, _ = self.client.scroll(self.notes_col, with_payload=True, with_vectors=False, limit=1, scroll_filter=f)
|
||||||
|
return (pts[0].payload or None) if pts else None
|
||||||
|
|
||||||
|
def get_chunks(self, note_id: str) -> List[Dict[str, Any]]:
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
pts = self._scroll(self.chunks_col, f)
|
||||||
|
# Sortierung analog Export
|
||||||
|
def key(pl):
|
||||||
|
p = pl.payload or {}
|
||||||
|
s = p.get("seq") or 0
|
||||||
|
ci = p.get("chunk_index") or 0
|
||||||
|
n = 0
|
||||||
|
cid = p.get("chunk_id") or ""
|
||||||
|
if isinstance(cid, str) and "#" in cid:
|
||||||
|
try:
|
||||||
|
n = int(cid.rsplit("#", 1)[-1])
|
||||||
|
except Exception:
|
||||||
|
n = 0
|
||||||
|
return (int(s), int(ci), n)
|
||||||
|
pts_sorted = sorted(pts, key=key)
|
||||||
|
return [p.payload or {} for p in pts_sorted]
|
||||||
|
|
||||||
|
def neighbors(self, source_id: str, kinds: Optional[Iterable[str]] = None,
|
||||||
|
scope: Optional[Iterable[str]] = None, depth: int = 1) -> Dict[str, List[Dict[str, Any]]]:
|
||||||
|
"""
|
||||||
|
Liefert eingehende & ausgehende Nachbarn (nur nach kind gefiltert).
|
||||||
|
depth==1: direkte Kanten.
|
||||||
|
"""
|
||||||
|
kinds = list(kinds) if kinds else None
|
||||||
|
must = [rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_id))]
|
||||||
|
if kinds:
|
||||||
|
must.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
|
||||||
|
f = rest.Filter(must=must)
|
||||||
|
edges = self._scroll(self.edges_col, f)
|
||||||
|
out = {"out": [], "in": []}
|
||||||
|
for e in edges:
|
||||||
|
out["out"].append(e.payload or {})
|
||||||
|
# Inverse Richtung (eingehend)
|
||||||
|
must_in = [rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_id))]
|
||||||
|
if kinds:
|
||||||
|
must_in.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
|
||||||
|
f_in = rest.Filter(must=must_in)
|
||||||
|
edges_in = self._scroll(self.edges_col, f_in)
|
||||||
|
for e in edges_in:
|
||||||
|
out["in"].append(e.payload or {})
|
||||||
|
return out
|
||||||
|
|
||||||
|
def walk_bfs(self, source_id: str, kinds: Iterable[str], max_depth: int = 2) -> Set[str]:
|
||||||
|
visited: Set[str] = {source_id}
|
||||||
|
frontier: Set[str] = {source_id}
|
||||||
|
kinds = list(kinds)
|
||||||
|
for _ in range(max_depth):
|
||||||
|
nxt: Set[str] = set()
|
||||||
|
for s in frontier:
|
||||||
|
neigh = self.neighbors(s, kinds=kinds)
|
||||||
|
for e in neigh["out"]:
|
||||||
|
t = e.get("target_id")
|
||||||
|
if isinstance(t, str) and t not in visited:
|
||||||
|
visited.add(t)
|
||||||
|
nxt.add(t)
|
||||||
|
frontier = nxt
|
||||||
|
if not frontier:
|
||||||
|
break
|
||||||
|
return visited
|
||||||
|
|
||||||
|
def context_for_note(self, note_id: str, kinds: Iterable[str] = ("references","backlink"), max_neighbors: int = 12) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Heuristischer Kontext: eigene Chunks + Nachbarn nach Kantenarten, dedupliziert.
|
||||||
|
"""
|
||||||
|
note = self.get_note(note_id) or {}
|
||||||
|
chunks = self.get_chunks(note_id)
|
||||||
|
neigh = self.neighbors(note_id, kinds=list(kinds))
|
||||||
|
targets = []
|
||||||
|
for e in neigh["out"]:
|
||||||
|
t = e.get("target_id")
|
||||||
|
if isinstance(t, str):
|
||||||
|
targets.append(t)
|
||||||
|
for e in neigh["in"]:
|
||||||
|
s = e.get("source_id")
|
||||||
|
if isinstance(s, str):
|
||||||
|
targets.append(s)
|
||||||
|
# de-dupe
|
||||||
|
seen = set()
|
||||||
|
uniq = []
|
||||||
|
for t in targets:
|
||||||
|
if t not in seen:
|
||||||
|
seen.add(t)
|
||||||
|
uniq.append(t)
|
||||||
|
uniq = uniq[:max_neighbors]
|
||||||
|
neighbor_notes = [self.get_note(t) for t in uniq]
|
||||||
|
return {
|
||||||
|
"note": note,
|
||||||
|
"chunks": chunks,
|
||||||
|
"neighbors": [n for n in neighbor_notes if n],
|
||||||
|
"edges_out": neigh["out"],
|
||||||
|
"edges_in": neigh["in"],
|
||||||
|
}
|
||||||
|
|
||||||
|
# Optional: Mini-CLI
|
||||||
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
import argparse, json
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV)")
|
||||||
|
ap.add_argument("--note-id", required=True)
|
||||||
|
ap.add_argument("--neighbors", action="store_true", help="Nur Nachbarn anzeigen")
|
||||||
|
args = ap.parse_args()
|
||||||
|
svc = GraphService(prefix=args.prefix)
|
||||||
|
if args.neighbors:
|
||||||
|
out = svc.neighbors(args.note_id, kinds=["references","backlink","prev","next","belongs_to"])
|
||||||
|
else:
|
||||||
|
out = svc.context_for_note(args.note_id)
|
||||||
|
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||||
+26
-5
@@ -1,16 +1,26 @@
|
|||||||
"""
|
"""
|
||||||
Version 0.3
|
app/main.py — mindnet API bootstrap
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from .config import get_settings
|
from .config import get_settings
|
||||||
from .routers.embed_router import router as embed_router
|
from .routers.embed_router import router as embed_router
|
||||||
from .routers.qdrant_router import router as qdrant_router
|
from .routers.qdrant_router import router as qdrant_router
|
||||||
|
|
||||||
|
from .routers.query import router as query_router
|
||||||
|
from .routers.graph import router as graph_router
|
||||||
|
from .routers.tools import router as tools_router
|
||||||
|
from .routers.feedback import router as feedback_router
|
||||||
|
# NEU: Chat Router (WP-05)
|
||||||
|
from .routers.chat import router as chat_router
|
||||||
|
|
||||||
|
try:
|
||||||
|
from .routers.admin import router as admin_router
|
||||||
|
except Exception:
|
||||||
|
admin_router = None
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
app = FastAPI(title="mindnet API", version="0.1.0")
|
app = FastAPI(title="mindnet API", version="0.5.0") # Version bump WP-05
|
||||||
s = get_settings()
|
s = get_settings()
|
||||||
|
|
||||||
@app.get("/healthz")
|
@app.get("/healthz")
|
||||||
@@ -19,7 +29,18 @@ def create_app() -> FastAPI:
|
|||||||
|
|
||||||
app.include_router(embed_router)
|
app.include_router(embed_router)
|
||||||
app.include_router(qdrant_router)
|
app.include_router(qdrant_router)
|
||||||
|
|
||||||
|
app.include_router(query_router, prefix="/query", tags=["query"])
|
||||||
|
app.include_router(graph_router, prefix="/graph", tags=["graph"])
|
||||||
|
app.include_router(tools_router, prefix="/tools", tags=["tools"])
|
||||||
|
app.include_router(feedback_router, prefix="/feedback", tags=["feedback"])
|
||||||
|
|
||||||
|
# NEU: Chat Endpoint
|
||||||
|
app.include_router(chat_router, prefix="/chat", tags=["chat"])
|
||||||
|
|
||||||
|
if admin_router:
|
||||||
|
app.include_router(admin_router, prefix="/admin", tags=["admin"])
|
||||||
|
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
app = create_app()
|
||||||
app = create_app()
|
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""
|
||||||
|
app/models/__init__.py — Package-Init für mindnet DTOs (WP-04)
|
||||||
|
Version: 0.1.0 • Stand: 2025-10-07
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .dto import (
|
||||||
|
EdgeKind,
|
||||||
|
NodeDTO,
|
||||||
|
EdgeDTO,
|
||||||
|
QueryRequest,
|
||||||
|
QueryHit,
|
||||||
|
QueryResponse,
|
||||||
|
GraphResponse,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"EdgeKind",
|
||||||
|
"NodeDTO",
|
||||||
|
"EdgeDTO",
|
||||||
|
"QueryRequest",
|
||||||
|
"QueryHit",
|
||||||
|
"QueryResponse",
|
||||||
|
"GraphResponse",
|
||||||
|
]
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
"""
|
||||||
|
app/models/dto.py — Pydantic-Modelle (DTOs) für WP-04/WP-05 Endpunkte
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Laufzeit-Modelle für FastAPI (Requests/Responses).
|
||||||
|
WP-05 Update: Chat-Modelle.
|
||||||
|
|
||||||
|
Version:
|
||||||
|
0.4.0 (Update für WP-05 Chat)
|
||||||
|
Stand:
|
||||||
|
2025-12-08
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from typing import List, Literal, Optional, Dict, Any
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
EdgeKind = Literal["references", "references_at", "backlink", "next", "prev", "belongs_to", "depends_on", "related_to", "similar_to"]
|
||||||
|
|
||||||
|
|
||||||
|
# --- Basis-DTOs ---
|
||||||
|
|
||||||
|
class NodeDTO(BaseModel):
|
||||||
|
"""Darstellung eines Knotens (Note oder Chunk) im API-Graph."""
|
||||||
|
id: str
|
||||||
|
type: Literal["note", "chunk"]
|
||||||
|
title: Optional[str] = None
|
||||||
|
note_id: Optional[str] = None
|
||||||
|
tags: Optional[List[str]] = None
|
||||||
|
in_degree: Optional[int] = None
|
||||||
|
out_degree: Optional[int] = None
|
||||||
|
score: Optional[float] = None
|
||||||
|
section_title: Optional[str] = None
|
||||||
|
section_path: Optional[str] = None
|
||||||
|
path: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class EdgeDTO(BaseModel):
|
||||||
|
"""Darstellung einer Kante im API-Graph."""
|
||||||
|
id: str
|
||||||
|
kind: str
|
||||||
|
source: str
|
||||||
|
target: str
|
||||||
|
weight: float
|
||||||
|
direction: Literal["out", "in", "undirected"] = "out"
|
||||||
|
|
||||||
|
|
||||||
|
# --- Request Models ---
|
||||||
|
|
||||||
|
class QueryRequest(BaseModel):
|
||||||
|
"""
|
||||||
|
Request für /query.
|
||||||
|
"""
|
||||||
|
mode: Literal["semantic", "edge", "hybrid"] = "hybrid"
|
||||||
|
query: Optional[str] = None
|
||||||
|
query_vector: Optional[List[float]] = None
|
||||||
|
top_k: int = 10
|
||||||
|
expand: Dict = {"depth": 1, "edge_types": ["references", "belongs_to", "prev", "next", "depends_on", "related_to"]}
|
||||||
|
filters: Optional[Dict] = None
|
||||||
|
ret: Dict = {"with_paths": True, "with_notes": True, "with_chunks": True}
|
||||||
|
explain: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class FeedbackRequest(BaseModel):
|
||||||
|
"""
|
||||||
|
User-Feedback zu einem spezifischen Treffer.
|
||||||
|
"""
|
||||||
|
query_id: str = Field(..., description="ID der ursprünglichen Suche")
|
||||||
|
node_id: str = Field(..., description="ID des bewerteten Treffers")
|
||||||
|
score: int = Field(..., ge=0, le=1, description="1 (Positiv) oder 0 (Negativ/Irrelevant)")
|
||||||
|
comment: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class ChatRequest(BaseModel):
|
||||||
|
"""
|
||||||
|
WP-05: Request für /chat.
|
||||||
|
"""
|
||||||
|
message: str = Field(..., description="Die Nachricht des Users")
|
||||||
|
conversation_id: Optional[str] = Field(None, description="Optional: ID für Chat-Verlauf (noch nicht implementiert)")
|
||||||
|
# RAG Parameter (Override defaults)
|
||||||
|
top_k: int = 5
|
||||||
|
explain: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
# --- WP-04b Explanation Models ---
|
||||||
|
|
||||||
|
class ScoreBreakdown(BaseModel):
|
||||||
|
"""Aufschlüsselung der Score-Komponenten."""
|
||||||
|
semantic_contribution: float
|
||||||
|
edge_contribution: float
|
||||||
|
centrality_contribution: float
|
||||||
|
raw_semantic: float
|
||||||
|
raw_edge_bonus: float
|
||||||
|
raw_centrality: float
|
||||||
|
node_weight: float
|
||||||
|
|
||||||
|
|
||||||
|
class Reason(BaseModel):
|
||||||
|
"""Ein semantischer Grund für das Ranking."""
|
||||||
|
kind: Literal["semantic", "edge", "type", "centrality"]
|
||||||
|
message: str
|
||||||
|
score_impact: Optional[float] = None
|
||||||
|
details: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
|
|
||||||
|
class Explanation(BaseModel):
|
||||||
|
"""Container für alle Erklärungsdaten eines Treffers."""
|
||||||
|
breakdown: ScoreBreakdown
|
||||||
|
reasons: List[Reason]
|
||||||
|
related_edges: Optional[List[EdgeDTO]] = None
|
||||||
|
|
||||||
|
|
||||||
|
# --- Response Models ---
|
||||||
|
|
||||||
|
class QueryHit(BaseModel):
|
||||||
|
"""Einzelnes Trefferobjekt für /query."""
|
||||||
|
node_id: str
|
||||||
|
note_id: Optional[str]
|
||||||
|
semantic_score: float
|
||||||
|
edge_bonus: float
|
||||||
|
centrality_bonus: float
|
||||||
|
total_score: float
|
||||||
|
paths: Optional[List[List[Dict]]] = None
|
||||||
|
source: Optional[Dict] = None
|
||||||
|
explanation: Optional[Explanation] = None
|
||||||
|
|
||||||
|
|
||||||
|
class QueryResponse(BaseModel):
|
||||||
|
"""Antwortstruktur für /query."""
|
||||||
|
query_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||||
|
results: List[QueryHit]
|
||||||
|
used_mode: str
|
||||||
|
latency_ms: int
|
||||||
|
|
||||||
|
|
||||||
|
class GraphResponse(BaseModel):
|
||||||
|
"""Antwortstruktur für /graph/{note_id}."""
|
||||||
|
center_note_id: str
|
||||||
|
nodes: List[NodeDTO]
|
||||||
|
edges: List[EdgeDTO]
|
||||||
|
stats: Dict[str, int]
|
||||||
|
|
||||||
|
|
||||||
|
class ChatResponse(BaseModel):
|
||||||
|
"""
|
||||||
|
WP-05: Antwortstruktur für /chat.
|
||||||
|
"""
|
||||||
|
query_id: str = Field(..., description="Traceability ID (dieselbe wie für Search)")
|
||||||
|
answer: str = Field(..., description="Generierte Antwort vom LLM")
|
||||||
|
sources: List[QueryHit] = Field(..., description="Die für die Antwort genutzten Quellen")
|
||||||
|
latency_ms: int
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
"""
|
||||||
|
app/models/__init__.py — Package-Init für mindnet DTOs
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Stellt zentrale API-/DTO-Modelle als Import-Fassade bereit.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, Pydantic 2.x
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage für WP-04)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
Handbuch / Knowledge Design (WP-03/04)
|
||||||
|
Nutzung:
|
||||||
|
from app.models import QueryRequest, QueryResponse, GraphResponse
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .dto import (
|
||||||
|
EdgeKind,
|
||||||
|
NodeDTO,
|
||||||
|
EdgeDTO,
|
||||||
|
QueryRequest,
|
||||||
|
QueryHit,
|
||||||
|
QueryResponse,
|
||||||
|
GraphResponse,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"EdgeKind",
|
||||||
|
"NodeDTO",
|
||||||
|
"EdgeDTO",
|
||||||
|
"QueryRequest",
|
||||||
|
"QueryHit",
|
||||||
|
"QueryResponse",
|
||||||
|
"GraphResponse",
|
||||||
|
]
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
"""
|
||||||
|
app/routers/admin.py — Admin-/Monitoring-Endpunkte (optional)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Liefert einfache Kennzahlen zu Collections (Counts) und Config.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, FastAPI 0.110+, qdrant-client 1.x
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
- Qdrant Collections: *_notes, *_chunks, *_edges
|
||||||
|
Nutzung:
|
||||||
|
app.include_router(admin.router, prefix="/admin", tags=["admin"])
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
@router.get("/stats")
|
||||||
|
def stats():
|
||||||
|
s = get_settings()
|
||||||
|
client = QdrantClient(url=s.QDRANT_URL, api_key=s.QDRANT_API_KEY)
|
||||||
|
notes = f"{s.COLLECTION_PREFIX}_notes"
|
||||||
|
chunks = f"{s.COLLECTION_PREFIX}_chunks"
|
||||||
|
edges = f"{s.COLLECTION_PREFIX}_edges"
|
||||||
|
|
||||||
|
def _count(col: str) -> int:
|
||||||
|
try:
|
||||||
|
return client.count(collection_name=col, exact=True).count
|
||||||
|
except Exception:
|
||||||
|
return -1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"collections": {
|
||||||
|
"notes": {"name": notes, "count": _count(notes)},
|
||||||
|
"chunks": {"name": chunks, "count": _count(chunks)},
|
||||||
|
"edges": {"name": edges, "count": _count(edges)},
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"qdrant": s.QDRANT_URL,
|
||||||
|
"prefix": s.COLLECTION_PREFIX,
|
||||||
|
"vector_size": s.VECTOR_SIZE,
|
||||||
|
"distance": s.DISTANCE,
|
||||||
|
"retriever": {
|
||||||
|
"w_sem": s.RETRIEVER_W_SEM,
|
||||||
|
"w_edge": s.RETRIEVER_W_EDGE,
|
||||||
|
"w_cent": s.RETRIEVER_W_CENT,
|
||||||
|
"top_k": s.RETRIEVER_TOP_K,
|
||||||
|
"expand_depth": s.RETRIEVER_EXPAND_DEPTH,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
"""
|
||||||
|
app/routers/chat.py — RAG Endpunkt (WP-05 Final Audit Version)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Verbindet Retrieval mit LLM-Generation.
|
||||||
|
Enriched Context: Fügt Typen und Metadaten in den Prompt ein,
|
||||||
|
damit das LLM komplexe Zusammenhänge (z.B. Decisions) versteht.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException, Depends
|
||||||
|
from typing import List
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from app.models.dto import ChatRequest, ChatResponse, QueryRequest, QueryHit
|
||||||
|
from app.services.llm_service import LLMService
|
||||||
|
from app.core.retriever import Retriever
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def get_llm_service():
|
||||||
|
return LLMService()
|
||||||
|
|
||||||
|
def get_retriever():
|
||||||
|
return Retriever()
|
||||||
|
|
||||||
|
def _build_enriched_context(hits: List[QueryHit]) -> str:
|
||||||
|
"""
|
||||||
|
Baut einen 'Rich Context' String.
|
||||||
|
Statt nur Text, injizieren wir Metadaten (Typ, Tags), damit das LLM
|
||||||
|
die semantische Rolle des Schnipsels versteht.
|
||||||
|
"""
|
||||||
|
context_parts = []
|
||||||
|
|
||||||
|
for i, hit in enumerate(hits, 1):
|
||||||
|
source = hit.source or {}
|
||||||
|
|
||||||
|
# 1. Content extrahieren (Robust: prüft alle üblichen Felder)
|
||||||
|
content = (
|
||||||
|
source.get("text") or
|
||||||
|
source.get("content") or
|
||||||
|
source.get("page_content") or
|
||||||
|
source.get("chunk_text") or
|
||||||
|
"[Kein Textinhalt verfügbar]"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2. Metadaten für "Context Intelligence"
|
||||||
|
title = hit.note_id or "Unbekannte Notiz"
|
||||||
|
# Typ in Großbuchstaben (z.B. "DECISION"), damit das LLM es als Signal erkennt
|
||||||
|
note_type = source.get("type", "unknown").upper()
|
||||||
|
|
||||||
|
# 3. Formatierung als strukturiertes Dokument für das LLM
|
||||||
|
entry = (
|
||||||
|
f"### QUELLE {i}: {title}\n"
|
||||||
|
f"TYP: [{note_type}] (Score: {hit.total_score:.2f})\n"
|
||||||
|
f"INHALT:\n{content}\n"
|
||||||
|
)
|
||||||
|
context_parts.append(entry)
|
||||||
|
|
||||||
|
return "\n\n".join(context_parts)
|
||||||
|
|
||||||
|
@router.post("/", response_model=ChatResponse)
|
||||||
|
async def chat_endpoint(
|
||||||
|
request: ChatRequest,
|
||||||
|
llm: LLMService = Depends(get_llm_service),
|
||||||
|
retriever: Retriever = Depends(get_retriever)
|
||||||
|
):
|
||||||
|
start_time = time.time()
|
||||||
|
query_id = str(uuid.uuid4())
|
||||||
|
|
||||||
|
logger.info(f"Chat request [{query_id}]: {request.message[:50]}...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 1. Retrieval (Hybrid erzwingen für Graph-Nutzung)
|
||||||
|
query_req = QueryRequest(
|
||||||
|
query=request.message,
|
||||||
|
mode="hybrid", # WICHTIG: Hybrid Mode für Graph-Nachbarn
|
||||||
|
top_k=request.top_k,
|
||||||
|
explain=request.explain
|
||||||
|
)
|
||||||
|
|
||||||
|
retrieve_result = await retriever.search(query_req)
|
||||||
|
hits = retrieve_result.results
|
||||||
|
|
||||||
|
# 2. Context Building (Enriched)
|
||||||
|
if not hits:
|
||||||
|
logger.info(f"[{query_id}] No hits found.")
|
||||||
|
context_str = "Keine relevanten Notizen gefunden."
|
||||||
|
else:
|
||||||
|
context_str = _build_enriched_context(hits)
|
||||||
|
|
||||||
|
# 3. Generation
|
||||||
|
logger.info(f"[{query_id}] Context built with {len(hits)} chunks. Sending to LLM...")
|
||||||
|
answer_text = await llm.generate_rag_response(
|
||||||
|
query=request.message,
|
||||||
|
context_str=context_str
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4. Response
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
logger.info(f"[{query_id}] Completed in {duration_ms}ms")
|
||||||
|
|
||||||
|
return ChatResponse(
|
||||||
|
query_id=retrieve_result.query_id,
|
||||||
|
answer=answer_text,
|
||||||
|
sources=hits,
|
||||||
|
latency_ms=duration_ms
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error in chat endpoint: {e}", exc_info=True)
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
"""
|
||||||
|
app/routers/feedback.py
|
||||||
|
Endpunkt für User-Feedback (WP-04c).
|
||||||
|
"""
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
from app.models.dto import FeedbackRequest
|
||||||
|
from app.services.feedback_service import log_feedback
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
@router.post("", status_code=201)
|
||||||
|
def post_feedback(fb: FeedbackRequest):
|
||||||
|
"""
|
||||||
|
Nimmt Feedback entgegen (z.B. Daumen hoch für einen Treffer).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
log_feedback(fb)
|
||||||
|
return {"status": "recorded", "query_id": fb.query_id}
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
"""
|
||||||
|
app/routers/graph.py — Graph-Endpunkte (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Liefert die Nachbarschaft einer Note/ID als JSON-Graph (Nodes/Edges/Stats).
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, FastAPI 0.110+, qdrant-client 1.x
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
- app/core/graph_adapter.py
|
||||||
|
- app/models/dto.py
|
||||||
|
Nutzung:
|
||||||
|
app.include_router(graph.router, prefix="/graph", tags=["graph"])
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List, Optional
|
||||||
|
from fastapi import APIRouter, Query
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from app.models.dto import GraphResponse, NodeDTO, EdgeDTO
|
||||||
|
from app.core.graph_adapter import expand
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
@router.get("/{note_id}", response_model=GraphResponse)
|
||||||
|
def get_graph(note_id: str, depth: int = 1, edge_types: Optional[List[str]] = Query(None)) -> GraphResponse:
|
||||||
|
s = get_settings()
|
||||||
|
client = QdrantClient(url=s.QDRANT_URL, api_key=s.QDRANT_API_KEY)
|
||||||
|
|
||||||
|
sg = expand(client, s.COLLECTION_PREFIX, [note_id], depth=depth, edge_types=edge_types)
|
||||||
|
|
||||||
|
# Seed-Node (Payload optional später ergänzen über get_note_payload)
|
||||||
|
nodes = [NodeDTO(id=note_id, type="note",
|
||||||
|
title=None, tags=[],
|
||||||
|
in_degree=sg.in_degree.get(note_id, 0),
|
||||||
|
out_degree=sg.out_degree.get(note_id, 0))]
|
||||||
|
|
||||||
|
# direkte Out-Kanten des Seeds
|
||||||
|
edges = []
|
||||||
|
for e in sg.adj.get(note_id, []):
|
||||||
|
edges.append(EdgeDTO(
|
||||||
|
id=f"{note_id}->{e['target']}:{e['kind']}",
|
||||||
|
kind=e["kind"], source=note_id, target=e["target"],
|
||||||
|
weight=float(e["weight"]), direction="out"
|
||||||
|
))
|
||||||
|
|
||||||
|
return GraphResponse(center_note_id=note_id, nodes=nodes, edges=edges,
|
||||||
|
stats={"node_count": len(nodes), "edge_count": len(edges)})
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
"""
|
||||||
|
app/routers/query.py — Query-Endpunkte (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Stellt POST /query bereit und ruft den passenden Retriever-Modus auf.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, FastAPI 0.110+
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
- app/core/retriever.py
|
||||||
|
- app/models/dto.py
|
||||||
|
Nutzung:
|
||||||
|
app.include_router(query.router, prefix="/query", tags=["query"])
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.2.0 (2025-12-07) - Update für WP04c Feedback
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
from fastapi import APIRouter, HTTPException, BackgroundTasks
|
||||||
|
from app.models.dto import QueryRequest, QueryResponse
|
||||||
|
from app.core.retriever import hybrid_retrieve, semantic_retrieve
|
||||||
|
# NEU:
|
||||||
|
from app.services.feedback_service import log_search
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
@router.post("", response_model=QueryResponse)
|
||||||
|
def post_query(req: QueryRequest, background_tasks: BackgroundTasks) -> QueryResponse:
|
||||||
|
try:
|
||||||
|
if req.mode == "semantic":
|
||||||
|
res = semantic_retrieve(req)
|
||||||
|
else:
|
||||||
|
res = hybrid_retrieve(req)
|
||||||
|
|
||||||
|
# WP-04c: Logging im Hintergrund (bremst Antwort nicht)
|
||||||
|
background_tasks.add_task(log_search, req, res)
|
||||||
|
|
||||||
|
return res
|
||||||
|
except ValueError as e:
|
||||||
|
raise HTTPException(status_code=400, detail=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(status_code=500, detail=f"query failed: {e}")
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
"""
|
||||||
|
app/routers/tools.py — Tool-Definitionen für Ollama/n8n/MCP (read-only)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Liefert Funktions-Schemas (OpenAI-/Ollama-kompatibles Tool-JSON) für:
|
||||||
|
- mindnet_query -> POST /query
|
||||||
|
- mindnet_subgraph -> GET /graph/{note_id}
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, FastAPI 0.110+
|
||||||
|
Version:
|
||||||
|
0.1.1 (query ODER query_vector möglich)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Nutzung:
|
||||||
|
app.include_router(tools.router, prefix="/tools", tags=["tools"])
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.1 (2025-10-07) – mindnet_query: oneOf(query, query_vector).
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
TOOLS = {
|
||||||
|
"tools": [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "mindnet_query",
|
||||||
|
"description": "Hybrid-Retrieval über mindnet (Semantik + Edges).",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"oneOf": [
|
||||||
|
{"required": ["query"]},
|
||||||
|
{"required": ["query_vector"]}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"query": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Freitext-Query; wird serverseitig in 384-d Embedding konvertiert."
|
||||||
|
},
|
||||||
|
"query_vector": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "number"},
|
||||||
|
"description": "Direkter 384-d Query-Vektor (optional)."
|
||||||
|
},
|
||||||
|
"top_k": {"type":"integer","default":10,"minimum":1,"maximum":50},
|
||||||
|
"expand_depth": {"type":"integer","default":1,"minimum":0,"maximum":3},
|
||||||
|
"edge_types": {
|
||||||
|
"type":"array","items":{"type":"string"},
|
||||||
|
"default": ["references","belongs_to","prev","next"]
|
||||||
|
},
|
||||||
|
"filters": {"type":"object","description":"payload-Filter (tags etc.)"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "mindnet_subgraph",
|
||||||
|
"description": "Gibt die Nachbarschaft (Edges) einer Note/Seed-ID zurück.",
|
||||||
|
"parameters": {
|
||||||
|
"type":"object",
|
||||||
|
"properties": {
|
||||||
|
"note_id":{"type":"string"},
|
||||||
|
"depth":{"type":"integer","default":1,"minimum":0,"maximum":3},
|
||||||
|
"edge_types":{
|
||||||
|
"type":"array","items":{"type":"string"},
|
||||||
|
"default":["references","belongs_to","prev","next","backlink"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required":["note_id"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
@router.get("/ollama")
|
||||||
|
def get_ollama_tools():
|
||||||
|
return TOOLS
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""
|
||||||
|
app/services/embeddings_client.py — Text→Embedding (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Liefert 384-d Embeddings für Textqueries (lazy load, einmal pro Prozess).
|
||||||
|
Standard: Sentence-Transformers (MODEL_NAME aus app.config.Settings).
|
||||||
|
Hinweis: Kein Netz-Zugriff; nutzt lokal installierte Modelle.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, sentence-transformers 5.x
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
- app/core/retriever.py (nutzt embed_text_if_needed)
|
||||||
|
- app/config.py (MODEL_NAME, VECTOR_SIZE)
|
||||||
|
Nutzung:
|
||||||
|
from app.services.embeddings_client import embed_text
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List
|
||||||
|
from functools import lru_cache
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
# Lazy import, damit Testläufe ohne Modell-Laden schnell sind
|
||||||
|
def _load_model():
|
||||||
|
from sentence_transformers import SentenceTransformer # import hier, nicht top-level
|
||||||
|
s = get_settings()
|
||||||
|
return SentenceTransformer(s.MODEL_NAME, device="cpu")
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _cached_model():
|
||||||
|
return _load_model()
|
||||||
|
|
||||||
|
def embed_text(text: str) -> List[float]:
|
||||||
|
"""
|
||||||
|
Erzeugt einen 384-d Vektor (oder laut Settings.VECTOR_SIZE) für den gegebenen Text.
|
||||||
|
"""
|
||||||
|
if not text or not text.strip():
|
||||||
|
raise ValueError("embed_text: leerer Text")
|
||||||
|
model = _cached_model()
|
||||||
|
vec = model.encode([text], normalize_embeddings=True)[0]
|
||||||
|
return vec.astype(float).tolist()
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""
|
||||||
|
app/services/feedback_service.py
|
||||||
|
Service zum Loggen von Suchanfragen und Feedback (WP-04c).
|
||||||
|
Speichert Daten als JSONL für späteres Self-Tuning (WP-08).
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, Any, List
|
||||||
|
from app.models.dto import QueryRequest, QueryResponse, FeedbackRequest
|
||||||
|
|
||||||
|
# Pfad für Logs (lokal auf dem Beelink/PC)
|
||||||
|
LOG_DIR = Path("data/logs")
|
||||||
|
SEARCH_LOG_FILE = LOG_DIR / "search_history.jsonl"
|
||||||
|
FEEDBACK_LOG_FILE = LOG_DIR / "feedback.jsonl"
|
||||||
|
|
||||||
|
def _ensure_log_dir():
|
||||||
|
if not LOG_DIR.exists():
|
||||||
|
os.makedirs(LOG_DIR, exist_ok=True)
|
||||||
|
|
||||||
|
def log_search(req: QueryRequest, res: QueryResponse):
|
||||||
|
"""
|
||||||
|
Speichert den "Snapshot" der Suche.
|
||||||
|
WICHTIG: Wir speichern die Scores (Breakdown), damit wir später wissen,
|
||||||
|
warum das System so entschieden hat.
|
||||||
|
"""
|
||||||
|
_ensure_log_dir()
|
||||||
|
|
||||||
|
# Wir reduzieren die Datenmenge etwas (z.B. keine vollen Texte)
|
||||||
|
hits_summary = []
|
||||||
|
for hit in res.results:
|
||||||
|
# Falls Explanation an war, speichern wir den Breakdown, sonst die Scores
|
||||||
|
breakdown = None
|
||||||
|
if hit.explanation and hit.explanation.breakdown:
|
||||||
|
breakdown = hit.explanation.breakdown.model_dump()
|
||||||
|
|
||||||
|
hits_summary.append({
|
||||||
|
"node_id": hit.node_id,
|
||||||
|
"note_id": hit.note_id,
|
||||||
|
"total_score": hit.total_score,
|
||||||
|
"breakdown": breakdown, # Wichtig für Training!
|
||||||
|
"rank_semantic": hit.semantic_score,
|
||||||
|
"rank_edge": hit.edge_bonus
|
||||||
|
})
|
||||||
|
|
||||||
|
entry = {
|
||||||
|
"timestamp": time.time(),
|
||||||
|
"query_id": res.query_id,
|
||||||
|
"query_text": req.query,
|
||||||
|
"mode": req.mode,
|
||||||
|
"top_k": req.top_k,
|
||||||
|
"hits": hits_summary
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(SEARCH_LOG_FILE, "a", encoding="utf-8") as f:
|
||||||
|
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"ERROR logging search: {e}")
|
||||||
|
|
||||||
|
def log_feedback(fb: FeedbackRequest):
|
||||||
|
"""
|
||||||
|
Speichert das User-Feedback.
|
||||||
|
"""
|
||||||
|
_ensure_log_dir()
|
||||||
|
|
||||||
|
entry = {
|
||||||
|
"timestamp": time.time(),
|
||||||
|
"query_id": fb.query_id,
|
||||||
|
"node_id": fb.node_id,
|
||||||
|
"score": fb.score,
|
||||||
|
"comment": fb.comment
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(FEEDBACK_LOG_FILE, "a", encoding="utf-8") as f:
|
||||||
|
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"ERROR logging feedback: {e}")
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
"""
|
||||||
|
app/services/llm_ollama.py — Ollama-Integration & Prompt-Bau (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Prompt-Template & (optionaler) lokaler Aufruf von Ollama. Der Aufruf ist
|
||||||
|
bewusst gekapselt und kann gefahrlos deaktiviert bleiben, bis ihr ein
|
||||||
|
konkretes Modell konfigurieren wollt.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
WP-04/05 Kontextbereitstellung für LLM
|
||||||
|
Nutzung:
|
||||||
|
from app.services.llm_ollama import build_prompt, call_ollama
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List, Dict, Optional
|
||||||
|
import subprocess
|
||||||
|
import json
|
||||||
|
|
||||||
|
PROMPT_TEMPLATE = """System: You are a helpful expert.
|
||||||
|
User: {question}
|
||||||
|
|
||||||
|
Context (ranked):
|
||||||
|
{contexts}
|
||||||
|
|
||||||
|
Task: Answer precisely. At the end, list sources (note title + section) and important edge paths.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def build_context_block(items: List[Dict]) -> str:
|
||||||
|
"""Formatiert Top-K-Kontexte (Chunks) für den Prompt."""
|
||||||
|
lines = []
|
||||||
|
for i, it in enumerate(items, 1):
|
||||||
|
note = it.get("note_title", "") or it.get("note_id", "")
|
||||||
|
sec = it.get("section", "") or it.get("section_title", "")
|
||||||
|
sc = it.get("score", 0)
|
||||||
|
txt = it.get("text", "") or it.get("body", "") or ""
|
||||||
|
lines.append(f"{i}) {note} — {sec} [score={sc:.2f}]\n{txt}\n")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def build_prompt(question: str, contexts: List[Dict]) -> str:
|
||||||
|
"""Setzt Frage + Kontexte in ein konsistentes Template."""
|
||||||
|
return PROMPT_TEMPLATE.format(question=question, contexts=build_context_block(contexts))
|
||||||
|
|
||||||
|
|
||||||
|
def call_ollama(prompt: str, model: str = "llama3.1:8b", timeout_s: int = 120) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Optionaler lokaler Aufruf von `ollama run`.
|
||||||
|
Rückgabe: generierter Text oder None bei Fehler/Abbruch.
|
||||||
|
Hinweis: Nur nutzen, wenn Ollama lokal installiert/konfiguriert ist.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(
|
||||||
|
["ollama", "run", model],
|
||||||
|
input=prompt.encode("utf-8"),
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
timeout=timeout_s,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
out = proc.stdout.decode("utf-8", errors="replace")
|
||||||
|
# viele ollama Builds streamen JSON-Zeilen; robust extrahieren:
|
||||||
|
try:
|
||||||
|
# Falls JSONL, letztes "response" zusammenfassen
|
||||||
|
texts = []
|
||||||
|
for line in out.splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
obj = json.loads(line)
|
||||||
|
if "response" in obj:
|
||||||
|
texts.append(obj["response"])
|
||||||
|
except Exception:
|
||||||
|
texts.append(line)
|
||||||
|
return "".join(texts).strip()
|
||||||
|
except Exception:
|
||||||
|
return out.strip()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""
|
||||||
|
app/services/llm_service.py — LLM Client (Ollama)
|
||||||
|
|
||||||
|
Version:
|
||||||
|
0.1.2 (WP-05 Fix: Increased Timeout for CPU Inference)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import yaml
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class LLMService:
|
||||||
|
def __init__(self):
|
||||||
|
self.settings = get_settings()
|
||||||
|
self.prompts = self._load_prompts()
|
||||||
|
# FIX: Timeout auf 120 Sekunden erhöht für CPU-Only Server
|
||||||
|
self.client = httpx.AsyncClient(base_url=self.settings.OLLAMA_URL, timeout=120.0)
|
||||||
|
|
||||||
|
def _load_prompts(self) -> dict:
|
||||||
|
"""Lädt Prompts aus der konfigurierten YAML-Datei."""
|
||||||
|
path = Path(self.settings.PROMPTS_PATH)
|
||||||
|
if not path.exists():
|
||||||
|
logger.warning(f"Prompt config not found at {path}, using defaults.")
|
||||||
|
return {
|
||||||
|
"system_prompt": "You are a helpful AI assistant.",
|
||||||
|
"rag_template": "Context: {context_str}\nQuestion: {query}"
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
return yaml.safe_load(f)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to load prompts: {e}")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
async def generate_rag_response(self, query: str, context_str: str) -> str:
|
||||||
|
"""
|
||||||
|
Generiert eine Antwort basierend auf Query und Kontext.
|
||||||
|
"""
|
||||||
|
system_prompt = self.prompts.get("system_prompt", "")
|
||||||
|
template = self.prompts.get("rag_template", "{context_str}\n\n{query}")
|
||||||
|
|
||||||
|
# Template füllen
|
||||||
|
final_prompt = template.format(context_str=context_str, query=query)
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"model": self.settings.LLM_MODEL,
|
||||||
|
"system": system_prompt,
|
||||||
|
"prompt": final_prompt,
|
||||||
|
"stream": False,
|
||||||
|
"options": {
|
||||||
|
"temperature": 0.7,
|
||||||
|
# Kleinerer Context spart Rechenzeit, falls 4096 zu viel ist
|
||||||
|
"num_ctx": 2048
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await self.client.post("/api/generate", json=payload)
|
||||||
|
|
||||||
|
if response.status_code != 200:
|
||||||
|
error_msg = response.text
|
||||||
|
logger.error(f"Ollama API Error ({response.status_code}): {error_msg}")
|
||||||
|
return f"Fehler vom LLM (Modell '{self.settings.LLM_MODEL}' vorhanden?): {error_msg}"
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
return data.get("response", "")
|
||||||
|
|
||||||
|
except httpx.ReadTimeout:
|
||||||
|
return "Timeout: Das Modell braucht zu lange zum Antworten (>120s). Hardware-Limit erreicht?"
|
||||||
|
except httpx.ConnectError:
|
||||||
|
return "Verbindungsfehler: Ist Ollama gestartet (Port 11434)?"
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"LLM Service Exception: {e}")
|
||||||
|
return f"Interner Fehler: {str(e)}"
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
await self.client.aclose()
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# config/prompts.yaml — Final V2.1
|
||||||
|
# Optimiert für Phi-3 Mini (Small Language Model)
|
||||||
|
|
||||||
|
system_prompt: |
|
||||||
|
Du bist 'mindnet', das KI-Gedächtnis.
|
||||||
|
|
||||||
|
DEINE REGELN:
|
||||||
|
1. Deine Antwort muss zu 100% auf dem KONTEXT basieren. Erfinde nichts.
|
||||||
|
2. Wenn eine Quelle den Typ [DECISION] hat, ist sie die wichtigste Quelle für das "Warum".
|
||||||
|
3. Nenne konkrete technische Details aus dem Text (z.B. genannte Features, Gründe), statt nur allgemein zu antworten.
|
||||||
|
4. Antworte auf Deutsch.
|
||||||
|
|
||||||
|
rag_template: |
|
||||||
|
QUELLEN (WISSEN):
|
||||||
|
=========================================
|
||||||
|
{context_str}
|
||||||
|
=========================================
|
||||||
|
|
||||||
|
FRAGE:
|
||||||
|
{query}
|
||||||
|
|
||||||
|
ANWEISUNG:
|
||||||
|
Beantworte die Frage basierend auf den Quellen.
|
||||||
|
Nenne die spezifischen Gründe, die im Text stehen (besonders aus [DECISION] Quellen).
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# config/prompts.yaml — Persönlichkeit & RAG-Strategie
|
||||||
|
# Version: 2.0 (Audit Update)
|
||||||
|
|
||||||
|
system_prompt: |
|
||||||
|
Du bist 'mindnet', ein persönliches KI-Gedächtnis und der Digitale Zwilling deines Erschaffers ("User").
|
||||||
|
|
||||||
|
DEINE PERSÖNLICHKEIT & WERTE:
|
||||||
|
1. Pragmatismus: Du bevorzugst funktionierende Lösungen über theoretische Perfektion.
|
||||||
|
2. Transparenz: Du erfindest keine Fakten. Wenn Informationen im Kontext fehlen, sagst du das klar.
|
||||||
|
3. Vernetztes Denken: Du suchst aktiv nach Verbindungen zwischen den bereitgestellten Notizen.
|
||||||
|
4. Erklärbarkeit: Wenn du eine Aussage machst, beziehst du dich implizit auf die Quelle (z.B. "Wie in Projekt X definiert...").
|
||||||
|
|
||||||
|
DEINE AUFGABE:
|
||||||
|
Beantworte die Frage des Users ausschließlich basierend auf dem untenstehenden KONTEXT.
|
||||||
|
Der Kontext besteht aus Auszügen verschiedener Notizen. Achte besonders auf den [TYPE] der Notiz:
|
||||||
|
- [DECISION] erklärt das "Warum".
|
||||||
|
- [PROJECT] erklärt das "Was" und "Wann".
|
||||||
|
- [CONCEPT] liefert Definitionen.
|
||||||
|
- [VALUE] definiert die moralische/strategische Ausrichtung.
|
||||||
|
|
||||||
|
rag_template: |
|
||||||
|
HINTERGRUNDWISSEN (KONTEXT):
|
||||||
|
=========================================
|
||||||
|
{context_str}
|
||||||
|
=========================================
|
||||||
|
|
||||||
|
FRAGE DES USERS:
|
||||||
|
{query}
|
||||||
|
|
||||||
|
ANWEISUNG:
|
||||||
|
Analysiere die Quellen oben. Synthetisiere eine Antwort, die die Frage präzise beantwortet.
|
||||||
|
Nutze Markdown für Struktur (Fettgedrucktes für Wichtiges).
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
system_prompt: |
|
||||||
|
Du bist 'mindnet', ein persönliches KI-Gedächtnis.
|
||||||
|
|
||||||
|
DEINE REGELN:
|
||||||
|
1. Antworte NUR basierend auf dem untenstehenden KONTEXT.
|
||||||
|
2. Achte auf den TYP der Quelle:
|
||||||
|
- [DECISION] enthält Begründungen (Warum?).
|
||||||
|
- [PROJECT] enthält Ziele (Was?).
|
||||||
|
- [CONCEPT] enthält Definitionen.
|
||||||
|
3. Sei präzise. Nenne konkrete technische Gründe, wenn sie im Text stehen.
|
||||||
|
4. Halluziniere nicht.
|
||||||
|
|
||||||
|
rag_template: |
|
||||||
|
KONTEXT (WISSEN):
|
||||||
|
=========================================
|
||||||
|
{context_str}
|
||||||
|
=========================================
|
||||||
|
|
||||||
|
FRAGE:
|
||||||
|
{query}
|
||||||
|
|
||||||
|
ANWEISUNG:
|
||||||
|
Analysiere den Kontext. Wenn eine [DECISION] Quelle dabei ist, nutze deren Inhalt für die Begründung.
|
||||||
|
Antworte kurz und präzise auf Deutsch.
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
version: 1.0
|
||||||
|
|
||||||
|
scoring:
|
||||||
|
# W_sem: skaliert den Term (semantic_score * retriever_weight)
|
||||||
|
# Empfehlung Startwert: 1.0 → Semantik bleibt Hauptsignal
|
||||||
|
semantic_weight: 1.0
|
||||||
|
|
||||||
|
# W_edge: skaliert edge_bonus aus dem Subgraph
|
||||||
|
# Empfehlung: 0.7 → Graph ist deutlich spürbar, aber überstimmt Semantik nicht komplett
|
||||||
|
edge_weight: 0.7
|
||||||
|
|
||||||
|
# W_cent: skaliert centrality_bonus (Knoten-Zentralität im Subgraph)
|
||||||
|
# Empfehlung: 0.5 → zentrale Knoten werden bevorzugt, aber moderat
|
||||||
|
centrality_weight: 0.5
|
||||||
|
|
||||||
|
# Optional, für später (aktuell noch nicht ausgewertet, aber als Blaupause):
|
||||||
|
edge_types:
|
||||||
|
references: 0.20
|
||||||
|
depends_on: 0.18
|
||||||
|
related_to: 0.15
|
||||||
|
similar_to: 0.12
|
||||||
|
belongs_to: 0.10
|
||||||
|
next: 0.06
|
||||||
|
prev: 0.06
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
version: 1.0
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
retriever_weight: 1.0
|
||||||
|
chunk_profile: default
|
||||||
|
edge_defaults: [] # Leerliste → keine impliziten Kanten, wenn nicht gesetzt
|
||||||
|
|
||||||
|
types:
|
||||||
|
# Wissensbausteine
|
||||||
|
concept:
|
||||||
|
chunk_profile: medium
|
||||||
|
retriever_weight: 0.60
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
|
||||||
|
source:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.50
|
||||||
|
edge_defaults: [] # Quellen sind i. d. R. Ziel von Kanten, erzeugen selbst keine Default-Kanten
|
||||||
|
|
||||||
|
glossary:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.40
|
||||||
|
edge_defaults: ["related_to"]
|
||||||
|
|
||||||
|
# Person & Persönlichkeit
|
||||||
|
profile: # z. B. "Persönliches Leitbild (Kurzprofil)"
|
||||||
|
chunk_profile: long
|
||||||
|
retriever_weight: 0.66
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
|
||||||
|
value: # Werte (z. B. "Mut", "Fairness")
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["related_to"]
|
||||||
|
|
||||||
|
principle: # Prinzipien/Leitsätze
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["related_to", "references"]
|
||||||
|
|
||||||
|
experience: # prägende Erlebnisse/Erfahrungen
|
||||||
|
chunk_profile: medium
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["related_to", "references"]
|
||||||
|
|
||||||
|
# Ziele, Entscheidungen, Meilensteine
|
||||||
|
goal:
|
||||||
|
chunk_profile: medium
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["depends_on", "related_to", "references"]
|
||||||
|
|
||||||
|
decision:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["depends_on", "references", "related_to"] # „Worauf stützte sich die Entscheidung?“
|
||||||
|
|
||||||
|
milestone:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.70
|
||||||
|
edge_defaults: ["related_to", "references"]
|
||||||
|
|
||||||
|
# Operativ: Projekte, Aufgaben, Journal
|
||||||
|
project:
|
||||||
|
chunk_profile: long
|
||||||
|
retriever_weight: 0.97
|
||||||
|
edge_defaults: ["references", "depends_on"] # Projekt stützt sich auf Konzepte/Quellen und hängt von Bausteinen ab
|
||||||
|
|
||||||
|
task:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.80
|
||||||
|
edge_defaults: ["depends_on", "references"]
|
||||||
|
|
||||||
|
journal:
|
||||||
|
chunk_profile: medium
|
||||||
|
retriever_weight: 0.80
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# Changelog — WP-03 Type-Registry (2025-11-08)
|
||||||
|
|
||||||
|
## Added
|
||||||
|
- `app/core/type_registry.py` — Loader & Helper (cached), robust Fallbacks.
|
||||||
|
- `config/types.yaml` — Beispiel-Registry (version 1.0).
|
||||||
|
- `scripts/import_markdown.py` v3.8.0 — Registry-Integration:
|
||||||
|
- typvalidierung (unbekannt → `concept`, Warnung)
|
||||||
|
- `chunk_profile` → an `make_chunk_payloads(...)`
|
||||||
|
- `retriever_weight` → persistiert im Note-Payload
|
||||||
|
- `edge_defaults["references"]` schaltet Note-Scope-Refs additiv ein
|
||||||
|
- `app/core/chunk_payload.py` v2.3.0 — optionales `chunk_profile` für
|
||||||
|
synthetische Overlaps (keine Schemaänderung).
|
||||||
|
- `app/core/derive_edges.py` v1.5.0 — Type-Registry aktiviert note-scope
|
||||||
|
References/Backlinks additiv.
|
||||||
|
|
||||||
|
## Changed
|
||||||
|
- Keine Breaking Changes. Alle CLI-Flags, ENV-Variablen und Collection-
|
||||||
|
Prefix-Logik bleiben unverändert.
|
||||||
|
|
||||||
|
## Removed
|
||||||
|
- N/A
|
||||||
+234
-75
@@ -1,19 +1,42 @@
|
|||||||
# 📘 Mindnet Skript-Handbuch
|
# 📘 Mindnet Skript-Handbuch (aktualisiert)
|
||||||
|
|
||||||
Dieses Handbuch dokumentiert die implementierten Skripte für das Projekt **mindnet Wissensnetzwerk**.
|
Dieses Handbuch dokumentiert die Skripte und Tests für das Projekt **mindnet Wissensnetzwerk**.
|
||||||
Es beschreibt Zweck, Parameter und typische Aufrufe. Alle Beispiele sind geprüft und lauffähig.
|
Es beschreibt Zweck, Parameter, typische Aufrufe sowie aktuelle Verhaltensweisen (Hash-Modi, Parser-Robustheit, `window`/`text`-Trennung in Chunks, Export-Fixes).
|
||||||
|
Alle Beispiele sind so formatiert, dass sie direkt kopierbar sind. **Code/CLI-Beispiele sind eingerückt**, damit die Markdown-Darstellung nicht abbricht.
|
||||||
Alle Skripte liegen in `~/mindnet/scripts/`.
|
|
||||||
Ausführung erfolgt **im aktivierten venv**:
|
|
||||||
|
|
||||||
cd ~/mindnet
|
|
||||||
source .venv/bin/activate
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 1. Vault anlegen (für Tests)
|
## 0) Umgebung & Konfiguration
|
||||||
|
|
||||||
### `make_test_vault.py`
|
**Wichtige ENV-Variablen**
|
||||||
|
|
||||||
|
- `QDRANT_URL` – z. B. `http://localhost:6333`
|
||||||
|
- `QDRANT_API_KEY` – API-Key (falls aktiviert)
|
||||||
|
- `COLLECTION_PREFIX` – Prefix der drei Collections, Standard: `mindnet` → `mindnet_notes`, `mindnet_chunks`, `mindnet_edges`
|
||||||
|
- `VECTOR_DIM` – Embedding-Dimension (Chunks), Standard: `384`
|
||||||
|
|
||||||
|
**Hash-Vergleich & Änderungserkennung**
|
||||||
|
|
||||||
|
- `MINDNET_HASH_COMPARE` – steuert, **welcher Anteil** für „changed“ verglichen wird
|
||||||
|
Werte: `Body` (Standard), `Frontmatter`, `Full`
|
||||||
|
- `MINDNET_HASH_SOURCE` – Quelle: `parsed` (Standard), `raw`, `file`
|
||||||
|
- `MINDNET_HASH_NORMALIZE` – Normalisierung: `canonical` (Standard) oder `none`
|
||||||
|
|
||||||
|
**Beispiel (nur Body berücksichtigen, kanonisch, geparst):**
|
||||||
|
|
||||||
|
export MINDNET_HASH_COMPARE=Body
|
||||||
|
export MINDNET_HASH_SOURCE=parsed
|
||||||
|
export MINDNET_HASH_NORMALIZE=canonical
|
||||||
|
source .venv/bin/activate
|
||||||
|
|
||||||
|
**Baseline-Signaturen (mehrere Modi):**
|
||||||
|
Der Importer kann **Signaturen aller Modi** (Body/Frontmatter/Full) vorhalten, damit spätere Moduswechsel deterministisch sind (keine „Mass-Changes“). Siehe `--baseline-modes` unter **Importer**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1) Vault anlegen (für Tests)
|
||||||
|
|
||||||
|
### `scripts/make_test_vault.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Erstellt einen kleinen, nachvollziehbaren Test-Vault mit Markdown-Dateien.
|
Erstellt einen kleinen, nachvollziehbaren Test-Vault mit Markdown-Dateien.
|
||||||
@@ -22,17 +45,17 @@ Szenarien: offene Links, nachträgliche Anlage fehlender Notes, geänderte Chunk
|
|||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--out PATH` : Zielverzeichnis (Default: `./test_vault`)
|
- `--out PATH` : Zielverzeichnis (Default: `./test_vault`)
|
||||||
- `--force` : Existierenden Ordner löschen und neu anlegen
|
- `--force` : Existierenden Ordner löschen und neu anlegen
|
||||||
- `--with-missing` : Optional – auch die `missing-note` gleich anlegen
|
- `--with-missing` : Optional – auch die `missing-note` gleich anlegen
|
||||||
|
|
||||||
**Hinweise für Tests**
|
**Hinweise für Tests**
|
||||||
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
|
1. Erster Import (dry-run oder `--apply`): Es gibt einen „leeren“ Link `[[missing-note]]`.
|
||||||
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
|
2. Lege danach eine Datei für „missing-note“ an (mit gleicher `id` im YAML) und importiere erneut:
|
||||||
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
|
→ Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (`references` + `backlink`).
|
||||||
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
|
3. Ändere den Body von `exp-one.md` so, dass andere Chunk-Grenzen entstehen und importiere erneut:
|
||||||
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
|
→ Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
|
||||||
|
|
||||||
Kompatibel mit:
|
**Kompatibilität:**
|
||||||
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
|
- `note.schema.json`: `created`/`updated` sind Strings.
|
||||||
|
|
||||||
**Beispiele:**
|
**Beispiele:**
|
||||||
|
|
||||||
@@ -41,41 +64,60 @@ Kompatibel mit:
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 2. Import in Qdrant
|
## 2) Import in Qdrant
|
||||||
|
|
||||||
### `import_markdown.py`
|
### `scripts/import_markdown.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Importiert Notes/Chunks/Edges aus einem Vault nach Qdrant.
|
Importiert Notes/Chunks/Edges aus einem Vault nach Qdrant.
|
||||||
- prüft Frontmatter
|
- validiert Frontmatter
|
||||||
- chunked Inhalte + Embeddings
|
- chunked Inhalte + Embeddings
|
||||||
- erzeugt Edges (`references`, `references_at`, `backlink`)
|
- erzeugt Edges (`references`, `references_at`, `backlink`, `next`, `prev`, `belongs_to`)
|
||||||
- erkennt geänderte Dateien automatisch
|
- erkennt geänderte Dateien **modusgesteuert** (Body/Frontmatter/Full)
|
||||||
- optional: Purge der alten Chunks/Edges pro Note
|
- optional: Purge der alten Chunks/Edges pro Note
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--vault PATH` : (Pflicht) Vault-Verzeichnis
|
- `--vault PATH` : (Pflicht) Vault-Verzeichnis
|
||||||
- `--apply` : Änderungen wirklich durchführen (sonst Dry-Run)
|
- `--apply` : Änderungen wirklich durchführen (ohne: Dry-Run)
|
||||||
- `--purge-before-upsert` : Vor Upsert alte Chunks/Edges **nur der geänderten Notes** löschen
|
- `--purge-before-upsert` : Vor Upsert alte Chunks/Edges **nur der geänderten Notes** löschen
|
||||||
- `--note-id ID` : Nur eine bestimmte Note importieren
|
- `--prefix STR` : überschreibt `COLLECTION_PREFIX`
|
||||||
|
- `--note-id ID` : nur eine bestimmte Note importieren
|
||||||
|
- `--note-scope-refs` : zusätzlich **note-weite** `references` erzeugen (nicht nur chunk-weise)
|
||||||
|
- `--baseline-modes` : berechne & speichere Hash-Signaturen für **alle** Modi (`Body`, `Frontmatter`, `Full`) als Baseline
|
||||||
|
|
||||||
|
**Erwartetes Verhalten:**
|
||||||
|
- Unveränderte Notes → `"changed": false`, Entscheidung: `apply-skip-unchanged` (bei `--apply`)
|
||||||
|
- Geänderte Notes → `"changed": true` (abhängig von `MINDNET_HASH_COMPARE`)
|
||||||
|
- Idempotenz: mehrfacher Lauf ohne Änderungen erzeugt **keine** neuen Punkte
|
||||||
|
- Baseline: Bei `--baseline-modes` erscheinen Felder wie `hash_signature`, `hash_fulltext`, ggf. `hash_frontmatter` und `needs_baseline_for_mode: false` bei Folgeläufen.
|
||||||
|
|
||||||
**Beispiele:**
|
**Beispiele:**
|
||||||
|
|
||||||
python3 -m scripts.import_markdown --vault ./vault
|
# Dry-Run (zeigt Änderungen, schreibt NICHT)
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
python3 -m scripts.import_markdown --vault ./vault --prefix mindnet
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
|
||||||
|
# Apply + gezielte Vorreinigung pro geänderter Note
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --prefix mindnet --apply --purge-before-upsert
|
||||||
|
|
||||||
|
# Baseline-Signaturen aller Modi vorhalten
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --apply --baseline-modes
|
||||||
|
|
||||||
|
**Hash-Modi in der Praxis:**
|
||||||
|
- `MINDNET_HASH_COMPARE=Body` → Änderungen im Body werden erkannt; reine Frontmatter-Änderungen bleiben **ohne Effekt**.
|
||||||
|
- `MINDNET_HASH_COMPARE=Frontmatter` → nur Frontmatter maßgeblich.
|
||||||
|
- `MINDNET_HASH_COMPARE=Full` → beide Anteile zählen.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. Konsistenzprüfungen
|
## 3) Konsistenzprüfungen
|
||||||
|
|
||||||
### `audit_vault_vs_qdrant.py`
|
### `scripts/audit_vault_vs_qdrant.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Vergleicht Vault-Inhalte mit Qdrant.
|
Vergleicht Vault-Inhalte mit Qdrant.
|
||||||
- Zählt Notes/Chunks/Edges
|
- Zählt Notes/Chunks/Edges
|
||||||
- Verifiziert Anzahl Wikilinks vs. Edges
|
- Verifiziert Anzahl Wikilinks vs. Edges
|
||||||
- Meldet Mismatches pro Note
|
- Meldet Deltas (z. B. `references` im Vault vs. in Qdrant)
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--vault PATH` : Vault-Verzeichnis
|
- `--vault PATH` : Vault-Verzeichnis
|
||||||
@@ -85,15 +127,13 @@ Vergleicht Vault-Inhalte mit Qdrant.
|
|||||||
|
|
||||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
||||||
|
|
||||||
---
|
### `scripts/validate_edges.py`
|
||||||
|
|
||||||
### `validate_edges.py`
|
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Prüft die Edges-Collection in Qdrant.
|
Prüft die Edges-Collection in Qdrant.
|
||||||
- Zählt Edges pro Typ
|
- Zählt Edges pro Typ (`references`, `references_at`, `backlink`, `next`, `prev`, `belongs_to`)
|
||||||
- Kontrolliert Invarianten (z. B. jede Reference hat Backlink)
|
- Kontrolliert Invarianten (z. B. jede `references` hat einen `backlink`)
|
||||||
- Listet unresolved Edges auf
|
- Listet `unresolved` Edges auf
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--prefix STR` : Prefix für Qdrant-Collections
|
- `--prefix STR` : Prefix für Qdrant-Collections
|
||||||
@@ -105,35 +145,69 @@ Prüft die Edges-Collection in Qdrant.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4. Qdrant zurücksetzen
|
## 4) Export Qdrant → Markdown (Vault)
|
||||||
|
|
||||||
### `reset_qdrant.py`
|
### `scripts/export_markdown.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Setzt die Qdrant-Collections zurück.
|
Exportiert Notes + rekonstruierten Body in einen Zielordner.
|
||||||
- „wipe“: löscht Collections komplett (inkl. Schema)
|
- Pfade werden **relativ** aus `note.payload.path` abgeleitet (führende `/` entfernt, Backslashes → Slashes; Unicode-NFC empfohlen)
|
||||||
- „truncate“: löscht nur Inhalte, behält Collections
|
- **Body-Rekonstruktion** priorisiert:
|
||||||
|
1. `fulltext` (falls in Note-Payload vorhanden)
|
||||||
|
2. sonst **Chunks**: Sortierung `seq` → `chunk_index` → Nummer in `chunk_id`; **nur `text`** (Overlap bereits entfernt)
|
||||||
|
- `--prefix` (CLI) wird unterstützt (oder ENV `COLLECTION_PREFIX`)
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--mode {wipe,truncate}` : Aktion
|
- `--out PATH` : Ziel-Root (muss existieren)
|
||||||
- `--prefix STR` : Prefix der Collections (Default: `mindnet`)
|
- `--overwrite` : vorhandene Dateien überschreiben
|
||||||
- `--yes` : Automatische Bestätigung
|
- `--note-id ID` : nur eine bestimmte Note exportieren
|
||||||
- `--dry-run` : Nur anzeigen, nicht ausführen
|
- `--prefix STR` : überschreibt `COLLECTION_PREFIX`
|
||||||
|
- `--include-edges {none,footer,yaml}` *(optional, falls aktiviert)*: Edges als YAML-Liste im Frontmatter oder im Footer beilegen
|
||||||
|
|
||||||
**Beispiele:**
|
**Beispiele:**
|
||||||
|
|
||||||
|
export COLLECTION_PREFIX="mindnet"
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault
|
||||||
|
|
||||||
|
# Nur eine bestimmte Note:
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-architektur-ki-trainerassistent-761cfe
|
||||||
|
|
||||||
|
# Existierende Dateien überschreiben:
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5) Qdrant zurücksetzen
|
||||||
|
|
||||||
|
### `scripts/reset_qdrant.py`
|
||||||
|
|
||||||
|
**Zweck:**
|
||||||
|
Setzt die Qdrant-Collections zurück.
|
||||||
|
- `wipe`: löscht Collections komplett (inkl. Schema)
|
||||||
|
- `truncate`: löscht nur Inhalte, behält Collections
|
||||||
|
- `truncate-edges`: löscht nur Punkte in `*_edges`
|
||||||
|
|
||||||
|
**Parameter:**
|
||||||
|
- `--mode {wipe,truncate,truncate-edges}`
|
||||||
|
- `--prefix STR` : Prefix der Collections (Default: `mindnet`)
|
||||||
|
- `--yes` : Automatische Bestätigung
|
||||||
|
- `--dry-run` : Nur anzeigen, nicht ausführen
|
||||||
|
|
||||||
|
**Beispiele:**
|
||||||
|
|
||||||
|
python3 -m scripts.reset_qdrant --mode truncate-edges --prefix mindnet
|
||||||
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
||||||
python3 -m scripts.reset_qdrant --mode wipe --prefix mindnet --yes
|
python3 -m scripts.reset_qdrant --mode wipe --prefix mindnet --yes
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 5. Diagnose
|
## 6) Diagnose & Tools
|
||||||
|
|
||||||
### `diagnose_changed.py`
|
### `scripts/diagnose_changed.py`
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Prüft, ob die Datei-Hashes (`sha256`) mit den gespeicherten `hash_fulltext` in Qdrant übereinstimmen.
|
Prüft, ob Datei-Hashes (`sha256`) mit gespeicherten `hash_*` in Qdrant übereinstimmen.
|
||||||
Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt.
|
Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt oder Moduswechsel geplant sind.
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--vault PATH` : Vault-Verzeichnis
|
- `--vault PATH` : Vault-Verzeichnis
|
||||||
@@ -145,14 +219,11 @@ Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt.
|
|||||||
|
|
||||||
python3 -m scripts.diagnose_changed --vault ./vault --prefix mindnet --all
|
python3 -m scripts.diagnose_changed --vault ./vault --prefix mindnet --all
|
||||||
|
|
||||||
---
|
### `scripts/resolve_unresolved_references.py`
|
||||||
|
|
||||||
## 6. Unresolved Links nachträglich auflösen
|
|
||||||
|
|
||||||
### `resolve_unresolved_references.py`
|
|
||||||
|
|
||||||
**Zweck:**
|
**Zweck:**
|
||||||
Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwischen passende Notizen existieren.
|
Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwischen passende Notizen existieren.
|
||||||
|
Ersetzt `unresolved` durch gültige `references`/`backlink`.
|
||||||
|
|
||||||
**Parameter:**
|
**Parameter:**
|
||||||
- `--prefix STR` : Prefix für Qdrant-Collections
|
- `--prefix STR` : Prefix für Qdrant-Collections
|
||||||
@@ -165,26 +236,114 @@ Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwisc
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# ✅ Typische Workflows
|
## 7) Parser-Verhalten (robust & abwärtskompatibel)
|
||||||
|
|
||||||
### A) Vault neu importieren
|
- Der Parser liest Markdown **fehlertolerant** (UTF-8 mit Fallbacks), extrahiert YAML-Frontmatter und Body.
|
||||||
|
- Keine harten Abhängigkeiten von alten Regex-Symbolen (`FRONTMATTER_RE` entfällt zugunsten interner Marker).
|
||||||
|
- `read_markdown(path)` gibt `(frontmatter_dict, body_str)` zurück.
|
||||||
|
- **Sonderfälle aus Fremd-Tools** (z. B. Silverbullet-Dateien wie `CONFIG.md`, `index.md`) werden **nicht importiert**, sofern sie nicht zum Vault gehören.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8) Chunk-Payload (Fenster vs. Kern)
|
||||||
|
|
||||||
|
- **`window`** = kontextualisierter Fenster-Text (mit linkem Overlap)
|
||||||
|
- **`text`** = **Kerntext** ohne das vom vorherigen Chunk übernommene Overlap
|
||||||
|
- Felder: `start`, `end`, `overlap_left`, `overlap_right`, `chunk_index`, `seq`, `path`, `section_title`, `section_path`, `token_count` (falls vorhanden)
|
||||||
|
- **Fallback (synthetisch):** Falls der Chunker **keinen** Overlap-Fenstertext liefert, erzeugt die Payload-Schicht Fenster mit typgerechtem Overlap aus `chunk_config.get_sizes(type)['overlap']` (unterer Wert).
|
||||||
|
- **Edges** werden aus `window` abgeleitet (um Links an Chunk-Grenzen nicht zu verlieren); der **Export** rekonstruiert den Body aus `text` (Overlap-frei).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9) Edge-Modell & IDs
|
||||||
|
|
||||||
|
- Collections:
|
||||||
|
`*_notes` (optional Vektor / Nullvektor), `*_chunks` (384-d Embeddings), `*_edges` (1-d Dummy-Vektor)
|
||||||
|
- **Deterministische IDs** (UUIDv5 / stabile Keys):
|
||||||
|
- `note_id` stabil aus Frontmatter/Slug
|
||||||
|
- `chunk_id` i. d. R. `note_id#<n>`
|
||||||
|
- `edge_id` stabil aus `(kind, source_id, target_id, seq)`
|
||||||
|
- Kantenarten: `references`, `references_at`, `backlink`, `next`, `prev`, `belongs_to`
|
||||||
|
- `unresolved` wird gekennzeichnet und kann per Resolver später aufgelöst werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10) Tests & Qualitätssicherung
|
||||||
|
|
||||||
|
**Tests unter `tests/`**
|
||||||
|
|
||||||
|
- `check_chunks_window_vs_text.py` – vergleicht je Chunk `window` vs. `text`, schätzt Overlap
|
||||||
|
- `compare_vaults.py` – rekursiver Vault-Vergleich (Body/Frontmatter/All, Key: auto/id/title/filename), Unicode-NFC-robust
|
||||||
|
- `verify_chunks_integrity.py` – prüft, ob Rekonstruktion aus Chunks (`text`) zu `fulltext` bzw. zum Vault-Body passt
|
||||||
|
- `list_md_seen_by_importer.py` – zeigt, welche `.md` der Importer sieht (inkl. Gründe/Filter)
|
||||||
|
- `inspect_one_note.py` – Parser-Diagnose für eine konkrete Datei
|
||||||
|
- `run_e2e_roundtrip.sh` – End-to-End: Reset → Import → Audit → (optional) Hash-Reporter → Export → Vergleich
|
||||||
|
|
||||||
|
**Typische Aufrufe:**
|
||||||
|
|
||||||
|
# Overlap-Qualität (Fenster vs. Kern)
|
||||||
|
python3 tests/check_chunks_window_vs_text.py --prefix "$COLLECTION_PREFIX"
|
||||||
|
|
||||||
|
# Roundtrip-Vergleich (rekursiv)
|
||||||
|
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --focus body
|
||||||
|
|
||||||
|
# Chunks-Rekonstruktion (gegen Vault-Body)
|
||||||
|
python3 tests/verify_chunks_integrity.py --prefix "$COLLECTION_PREFIX" --vault ./vault
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11) Typische Workflows
|
||||||
|
|
||||||
|
**A) Clean-Start mit Test-Vault**
|
||||||
|
|
||||||
|
python3 -m scripts.make_test_vault --out ./test_vault --force
|
||||||
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet
|
||||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet --apply --purge-before-upsert
|
||||||
|
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
|
||||||
|
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||||
|
export COLLECTION_PREFIX="mindnet"
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault
|
||||||
|
|
||||||
### B) Änderungen einspielen (mit Purge für geänderte Notes)
|
**B) Änderungs-Propagation & Idempotenz**
|
||||||
|
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet
|
||||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet --apply --purge-before-upsert
|
||||||
|
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
|
||||||
|
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||||
|
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet
|
||||||
|
|
||||||
### C) Unresolved Links nachziehen
|
**C) Unresolved Links lösen**
|
||||||
|
|
||||||
|
python3 -m scripts.import_markdown --vault ./test_vault --apply --prefix mindnet
|
||||||
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
|
||||||
python3 -m scripts.validate_edges --prefix mindnet --details
|
python3 -m scripts.validate_edges --prefix mindnet --details
|
||||||
|
|
||||||
### D) Test-Vault Szenario durchspielen
|
**D) Export & Roundtrip (Produkt-Vault)**
|
||||||
|
|
||||||
python3 -m scripts.make_test_vault --out ./test_vault --force
|
export COLLECTION_PREFIX="mindnet"
|
||||||
python3 -m scripts.import_markdown --vault ./test_vault --apply
|
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
|
||||||
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
|
python3 -m scripts.export_markdown --out ./_exportVault --prefix "$COLLECTION_PREFIX"
|
||||||
|
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --focus body
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12) Troubleshooting (kurz)
|
||||||
|
|
||||||
|
- **Importer markiert alle als changed (Moduswechsel):**
|
||||||
|
→ `--baseline-modes` einmalig laufen lassen; sicherstellen: `MINDNET_HASH_COMPARE` passt zum Ziel.
|
||||||
|
- **Edges verschwinden / nur 1 Edge sichtbar:**
|
||||||
|
→ `app/core/qdrant_points.py` muss Edge-Payload normalisieren (Schema Alt/Neu) und `edge_id` eindeutig bilden.
|
||||||
|
- **Export ohne Body / falsches Zielverzeichnis:**
|
||||||
|
→ Exporter nutzt relativierte Pfade und rekonstruiert Body aus `fulltext` oder Chunks (`text`). ENV/CLI `--prefix` beachten.
|
||||||
|
- **Parser-Fehler (UTF-8):**
|
||||||
|
→ Neuer Parser arbeitet mit Fallbacks; mit `tests/inspect_one_note.py` prüfen, was ankommt.
|
||||||
|
- **„Fehlende“ Dateien im Vergleich:**
|
||||||
|
→ Prüfe, ob Dateien **wirklich zum Vault gehören** (z. B. Silverbullet-Artefakte `CONFIG.md`, `index.md` ausblenden).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13) Hinweise zu Silverbullet-Dateien
|
||||||
|
|
||||||
|
- Dateien wie `CONFIG.md` oder `index.md`, die **nicht** zum inhaltlichen Vault gehören, werden **nicht importiert** und erscheinen daher nicht im Export-Roundtrip.
|
||||||
|
- Diese Dateien bitte vom produktiven Vault trennen oder im Vergleich (`tests/compare_vaults.py
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
# mindnet v2.2 – Knowledge Design Manual
|
||||||
|
**Datei:** `docs/mindnet_knowledge_design_manual_v2.2.md`
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** **FINAL** (Integrierter Stand WP01–WP05)
|
||||||
|
**Quellen:** `knowledge_design.md`, `TYPE_REGISTRY_MANUAL.md`, `chunking_strategy.md`, `mindnet_functional_architecture.md`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zweck & Scope
|
||||||
|
|
||||||
|
Dieses Handbuch ist die **primäre Arbeitsanweisung** für dich als Mindmaster (Owner) und alle Autoren, die Inhalte im mindnet-Vault erstellen.
|
||||||
|
|
||||||
|
### 1.1 Zielsetzung
|
||||||
|
Mindnet ist mehr als eine Dokumentablage. Es ist ein vernetztes System, das deine Persönlichkeit, Entscheidungen und Erfahrungen abbildet.
|
||||||
|
Seit Version 2.2 verfügt Mindnet über:
|
||||||
|
* **Explanation Engine:** Das System erklärt, warum es Notizen findet (über Edges).
|
||||||
|
* **RAG-Chat (KI-Zwilling):** Das System antwortet in natürlicher Sprache. **Wie** du schreibst, bestimmt, **wie schlau** die KI antwortet.
|
||||||
|
|
||||||
|
### 1.2 Der Vault als „Source of Truth“
|
||||||
|
Die Markdown-Dateien in deinem Vault sind die **einzige Quelle der Wahrheit**.
|
||||||
|
* Mindnet importiert diesen Zustand deterministisch in die Datenbank (Qdrant).
|
||||||
|
* Änderungen an IDs, Typen oder Inhalten müssen **immer** im Markdown erfolgen, niemals direkt in der Datenbank.
|
||||||
|
* Was nicht im Markdown steht (z. B. implizites Wissen im Kopf), existiert für das System nicht.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Note-Struktur & Frontmatter
|
||||||
|
|
||||||
|
Jede Notiz benötigt einen YAML-Header (Frontmatter) am Dateianfang. Dieser Block macht die Notiz maschinenlesbar.
|
||||||
|
|
||||||
|
### 2.1 Pflichtfelder
|
||||||
|
Jede Datei muss mindestens folgende Felder enthalten, um korrekt verarbeitet zu werden:
|
||||||
|
|
||||||
|
---
|
||||||
|
id: 20251110-projekt-alpha-8a9b2c # Eindeutige Kennung
|
||||||
|
title: Projekt Alpha # Sprechender Titel (wird in Suchergebnissen angezeigt)
|
||||||
|
type: project # Steuert Verarbeitung & Vernetzung (siehe Kap. 3)
|
||||||
|
status: active # Status (z.B. draft, active, archived)
|
||||||
|
created: 2025-11-10 # Erstellungsdatum (ISO 8601 oder YYYY-MM-DD)
|
||||||
|
updated: 2025-12-07 # Letzte Änderung
|
||||||
|
tags: [ki, entwicklung] # Taxonomie für Filterung
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2.2 Optionale Felder
|
||||||
|
Diese Felder sind technisch nicht zwingend, aber für bestimmte Typen sinnvoll:
|
||||||
|
|
||||||
|
lang: de # Sprache (Default: de)
|
||||||
|
aliases: [Alpha Projekt, Project A] # Synonyme für die Suche
|
||||||
|
visibility: internal # internal (default), public, private
|
||||||
|
|
||||||
|
> **Hinweis:** Felder wie `retriever_weight` oder `chunk_profile` sollten **nicht** mehr manuell im Frontmatter gesetzt werden. Diese werden zentral über den `type` gesteuert (siehe Kap. 3), um die Wartbarkeit zu sichern.
|
||||||
|
|
||||||
|
### 2.3 Empfehlungen für IDs und Pfade
|
||||||
|
|
||||||
|
**Die ID (Identifikator):**
|
||||||
|
* Muss global eindeutig und **stabil** sein.
|
||||||
|
* Darf sich nicht ändern, wenn die Datei umbenannt oder verschoben wird.
|
||||||
|
* **Empfehlung:** `YYYYMMDD-slug-hash` (z. B. `20231027-vektor-db-a1b2`). Dies garantiert Eindeutigkeit und Chronologie.
|
||||||
|
|
||||||
|
**Dateinamen & Pfade:**
|
||||||
|
* Pfade dienen der menschlichen Ordnung (Ordnerstruktur), sind für Mindnet aber sekundär.
|
||||||
|
* Dateinamen sollten dem Titel entsprechen (`Projekt Alpha.md`) oder dem Slug (`projekt-alpha.md`).
|
||||||
|
* Sonderzeichen vermeiden (außer Bindestrich und Unterstrich).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Note-Typen & Typ-Registry
|
||||||
|
|
||||||
|
Der `type` ist der wichtigste Hebel im Knowledge Design. Er verknüpft die Notiz mit der Konfiguration in `config/types.yaml`.
|
||||||
|
|
||||||
|
### 3.1 Übersicht der Kern-Typen
|
||||||
|
|
||||||
|
Mindnet unterscheidet verschiedene Wissensarten. Wähle den Typ, der die **Rolle** der Notiz am besten beschreibt:
|
||||||
|
|
||||||
|
| Typ | Beschreibung & Einsatzzweck | Wichtigkeit für Chat |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| **`concept`** | Fachbegriffe, Theorien. Zeitloses Wissen. | Mittel (Basiswissen) |
|
||||||
|
| **`project`** | Ein Vorhaben mit Ziel, Dauer und Aufgaben. | Hoch (Kontext) |
|
||||||
|
| **`experience`** | Persönliche Erfahrung, Lektion oder Erkenntnis. | Sehr Hoch (Persönlichkeit) |
|
||||||
|
| **`decision`** | Eine bewusst getroffene Entscheidung (ADR). | **Kritisch** (Begründung "Warum") |
|
||||||
|
| **`value`** | Ein persönlicher Wert oder ein Prinzip. | **Kritisch** (Moralischer Kompass) |
|
||||||
|
| **`person`** | Eine reale Person (Netzwerk, Autor). | Niedrig |
|
||||||
|
| **`journal`** | Zeitbezogener Log-Eintrag, Daily Note. | Mittel (Historie) |
|
||||||
|
| **`source`** | Externe Quelle (Buch, PDF, Artikel). | Niedrig (Faktenbasis) |
|
||||||
|
|
||||||
|
### 3.2 Zusammenspiel mit `types.yaml`
|
||||||
|
|
||||||
|
Der `type` steuert im Hintergrund drei technische Mechanismen:
|
||||||
|
|
||||||
|
1. **`retriever_weight` (Wichtigkeit):**
|
||||||
|
* Ein `concept` (0.6) wiegt weniger als ein `project` (0.97) oder eine `decision` (1.0).
|
||||||
|
* **Warum?** Bei einer Suche nach "Datenbank" soll Mindnet bevorzugt deine *Entscheidung* ("Warum wir X nutzen") anzeigen.
|
||||||
|
2. **`chunk_profile` (Textzerlegung):**
|
||||||
|
* `journal` (short): Wird fein zerlegt.
|
||||||
|
* `project` (long): Längere Kontext-Fenster.
|
||||||
|
3. **`edge_defaults` (Automatische Vernetzung):**
|
||||||
|
* Mindnet ergänzt automatisch Kanten.
|
||||||
|
* Beispiel: Ein Link in einem `project` wird automatisch als `depends_on` (Abhängigkeit) interpretiert.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Edges & Referenzen in Notes
|
||||||
|
|
||||||
|
Um aus isolierten Dateien ein **Netzwerk** zu machen, nutzen wir Verlinkungen. In Version 2.2 sind diese Verlinkungen die Basis für den **Hybrid Retriever** (Suche über Nachbarn).
|
||||||
|
|
||||||
|
### 4.1 Wikilinks (Die Basis-Referenz)
|
||||||
|
Der klassische Obsidian-Link.
|
||||||
|
|
||||||
|
Wir nutzen [[Qdrant]] als Vektordatenbank.
|
||||||
|
|
||||||
|
* **Bedeutung:** "Diese Notiz erwähnt Qdrant."
|
||||||
|
* **Edge-Typ:** `references`
|
||||||
|
|
||||||
|
### 4.2 Inline-Relationen (Semantische Verknüpfung)
|
||||||
|
Dies ist die **mächtigste** Methode. Du sagst dem System explizit, **wie** Dinge zusammenhängen.
|
||||||
|
|
||||||
|
Daher [[rel:depends_on Qdrant]].
|
||||||
|
Dieses Konzept ist [[rel:similar_to Pinecone]].
|
||||||
|
|
||||||
|
* **Syntax:** `[[rel:RELATION ZIEL]]`.
|
||||||
|
* **Gültige Relationen:**
|
||||||
|
* `depends_on`: Hängt ab von / Benötigt. (Trigger für hohe Graph-Relevanz).
|
||||||
|
* `similar_to`: Ähnelt / Ist vergleichbar mit.
|
||||||
|
* `related_to`: Hat zu tun mit (allgemein).
|
||||||
|
* `caused_by`: Wurde verursacht durch.
|
||||||
|
* `solves`: Löst (Problem).
|
||||||
|
|
||||||
|
### 4.3 Callout-Edges (Kuratierte Listen)
|
||||||
|
Für Zusammenfassungen oder "Siehe auch"-Blöcke am Ende einer Notiz.
|
||||||
|
|
||||||
|
> [!edge] related_to: [[Vector Embeddings]] [[AI Agents]]
|
||||||
|
|
||||||
|
* **Funktion:** Erzeugt `related_to`-Kanten zu allen genannten Zielen in dieser Zeile.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Best Practices & Beispiele
|
||||||
|
|
||||||
|
### 5.1 Beispiel: Projekt-Notiz
|
||||||
|
Projekte profitieren von `depends_on`, um Abhängigkeiten zu klären.
|
||||||
|
|
||||||
|
---
|
||||||
|
id: 20251115-proj-mindnet
|
||||||
|
title: Mindnet Implementierung
|
||||||
|
type: project
|
||||||
|
status: active
|
||||||
|
---
|
||||||
|
|
||||||
|
# Mindnet Implementierung
|
||||||
|
|
||||||
|
Wir bauen ein persönliches Wissensnetz.
|
||||||
|
|
||||||
|
## Tech Stack
|
||||||
|
Wir nutzen [[rel:depends_on Qdrant]] für die Vektorsuche und [[rel:depends_on FastAPI]] für das Backend.
|
||||||
|
|
||||||
|
## Architektur
|
||||||
|
Das Konzept basiert auf [[RAG Architecture]]. (Automatisch 'depends_on' durch Typ-Default).
|
||||||
|
|
||||||
|
### 5.2 Beispiel: Entscheidung (Decision Record)
|
||||||
|
Entscheidungen sind hoch gewichtet (`retriever_weight: 1.0`).
|
||||||
|
|
||||||
|
---
|
||||||
|
id: 20251120-adr-vektordb
|
||||||
|
title: ADR: Wahl von Qdrant
|
||||||
|
type: decision
|
||||||
|
status: final
|
||||||
|
tags: [architektur, db]
|
||||||
|
---
|
||||||
|
|
||||||
|
# Entscheidung: Qdrant
|
||||||
|
|
||||||
|
Wir haben uns für Qdrant entschieden.
|
||||||
|
|
||||||
|
## Alternativen
|
||||||
|
Wir haben auch [[rel:similar_to Pinecone]] und [[rel:similar_to Weaviate]] betrachtet.
|
||||||
|
|
||||||
|
## Begründung
|
||||||
|
Qdrant erlaubt lokalen Betrieb und [[rel:solves Payload Filtering Requirements]].
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Langfristige Stabilität & Virtual Schema Layer
|
||||||
|
|
||||||
|
Mindnet ist auf Langlebigkeit ausgelegt.
|
||||||
|
|
||||||
|
### 6.1 Das "Virtual Schema" Prinzip
|
||||||
|
Wir vermeiden es, Logik in den Markdown-Dateien hart zu kodieren.
|
||||||
|
* Statt in jeder Notiz zu schreiben `chunk_size: 500`, schreiben wir nur `type: essay`.
|
||||||
|
* Wie groß ein Chunk für ein Essay ist, definieren wir in der Konfiguration (`types.yaml`).
|
||||||
|
|
||||||
|
### 6.2 Was bedeutet das für dich?
|
||||||
|
* Du kannst dich auf den Inhalt konzentrieren.
|
||||||
|
* Wenn wir in Zukunft (WP08) basierend auf Feedback lernen, dass "Projekte" noch wichtiger sind, ändern wir **eine Zeile** in der Konfiguration, und das gesamte System passt sich beim nächsten Import an.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Schreiben für den KI-Zwilling (New in v2.2)
|
||||||
|
|
||||||
|
Damit der **RAG-Chat (WP05)** gute Antworten liefert, beachte diese Regeln:
|
||||||
|
|
||||||
|
1. **Atomare Konzepte:**
|
||||||
|
* Der Chatbot baut seine Antwort aus mehreren kleinen Text-Stücken ("Chunks") zusammen.
|
||||||
|
* Schreibe so, dass ein Absatz auch für sich allein verständlich ist.
|
||||||
|
2. **Explizite Entscheidungen:**
|
||||||
|
* Wenn du eine Meinung hast ("Tool X ist schlecht"), schreibe sie nicht in einen Nebensatz.
|
||||||
|
* Erstelle eine Notiz `type: experience` oder `decision` ("Warum Tool X nicht geeignet ist").
|
||||||
|
* Die KI sucht gezielt nach `[DECISION]`-Typen, um "Warum"-Fragen zu beantworten.
|
||||||
|
3. **Werte definieren:**
|
||||||
|
* Erstelle Notizen mit `type: value` (z.B. "Datenschutz First").
|
||||||
|
* Die KI nutzt diese, um bei Konflikten ("Soll ich Cloud oder Lokal nutzen?") in deinem Sinne zu argumentieren.
|
||||||
|
4. **Verlinken ist Pflicht:**
|
||||||
|
* Der Chatbot nutzt **Hybrid Search**. Er findet Notizen nur, wenn sie über Kanten verbunden sind.
|
||||||
|
* Eine isolierte Notiz (ohne Links) ist für die KI fast unsichtbar.
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
# Mindnet v2.2 – Overview & Einstieg
|
||||||
|
**Datei:** `docs/mindnet_overview_v2.2.md`
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** **FINAL** (Post-WP05 Release)
|
||||||
|
**Version:** 2.3.0
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Einführung: Was ist Mindnet?
|
||||||
|
|
||||||
|
**Mindnet** ist ein persönliches, lokales KI-Gedächtnis. Es ist der Versuch, einen **Digitalen Zwilling** deines Wissens und deiner Persönlichkeit zu erschaffen.
|
||||||
|
|
||||||
|
Anders als herkömmliche Notiz-Apps (wie Obsidian oder Evernote), die Texte nur passiv speichern, ist Mindnet ein **aktives System**:
|
||||||
|
* Es **versteht** Zusammenhänge über einen Wissensgraphen.
|
||||||
|
* Es **begründet** Antworten ("Warum ist das so?").
|
||||||
|
* Es **antwortet** im Dialog als Persona (RAG-Chat), basierend auf deinen Werten.
|
||||||
|
|
||||||
|
### Die Vision
|
||||||
|
> „Ein System, das nicht nur speichert, was ich weiß, sondern auch wie ich denke.“
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Die drei Ebenen des Systems
|
||||||
|
|
||||||
|
Mindnet arbeitet auf drei Schichten, die aufeinander aufbauen:
|
||||||
|
|
||||||
|
### Ebene 1: Content (Das Gedächtnis)
|
||||||
|
* **Quelle:** Dein lokaler Obsidian-Vault (Markdown).
|
||||||
|
* **Funktion:** Speicherung von Fakten, Projekten und Logs.
|
||||||
|
* **Technik:** Import-Pipeline, Chunking, Vektor-Datenbank (Qdrant).
|
||||||
|
* **Status:** 🟢 Live (WP01–WP03).
|
||||||
|
|
||||||
|
### Ebene 2: Semantik (Das Verstehen)
|
||||||
|
* **Funktion:** Verknüpfung von isolierten Notizen zu einem Netzwerk.
|
||||||
|
* **Logik:** "Projekt A *hängt ab von* Entscheidung B".
|
||||||
|
* **Technik:** Hybrider Retriever (Graph + Vektor), Explanation Engine.
|
||||||
|
* **Status:** 🟢 Live (WP04).
|
||||||
|
|
||||||
|
### Ebene 3: Identität (Die Persönlichkeit)
|
||||||
|
* **Funktion:** Interaktion und Bewertung. Das System nimmt eine Haltung ein.
|
||||||
|
* **Logik:** "Ich empfehle Lösung X, weil sie unserem Wert 'Datensparsamkeit' entspricht."
|
||||||
|
* **Technik:** RAG-Chat, LLM (Phi-3), Prompt Engineering, Feedback Loop.
|
||||||
|
* **Status:** 🟢 Live (WP05).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. End-to-End Architektur
|
||||||
|
|
||||||
|
Der Datenfluss in Mindnet ist zyklisch ("Data Flywheel"):
|
||||||
|
|
||||||
|
1. **Input:** Du schreibst Notizen in Obsidian.
|
||||||
|
2. **Ingest:** Ein Python-Skript importiert, zerlegt (Chunking) und vernetzt (Edges) die Daten in Qdrant.
|
||||||
|
3. **Retrieval:** Bei einer Frage sucht das System semantisch (Text) und graph-basiert (Nachbarn).
|
||||||
|
4. **Generation:** Ein lokales LLM (Ollama) formuliert die Antwort, angereichert mit Kontext-Metadaten.
|
||||||
|
5. **Feedback:** Du bewertest die Antwort. Das System lernt (langfristig) daraus.
|
||||||
|
|
||||||
|
**Tech-Stack:**
|
||||||
|
* **Backend:** Python 3.12, FastAPI.
|
||||||
|
* **Datenbank:** Qdrant (Vektor & Graph).
|
||||||
|
* **KI:** Ollama (Phi-3 Mini / Mistral) – 100% lokal.
|
||||||
|
* **Frontend:** Terminal (aktuell) / Web-UI (geplant WP10).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Dokumentations-Wegweiser
|
||||||
|
|
||||||
|
Wo findest du was?
|
||||||
|
|
||||||
|
| Wenn du... | ...lies dieses Dokument |
|
||||||
|
| :--- | :--- |
|
||||||
|
| **...wissen willst, wie man Notizen schreibt.** | `mindnet_knowledge_design_manual_v2.2.md` |
|
||||||
|
| **...das System installieren oder betreiben musst.** | `mindnet_admin_guide_v2.2.md` |
|
||||||
|
| **...am Python-Code entwickeln willst.** | `mindnet_developer_guide_v2.2.md` |
|
||||||
|
| **...die Pipeline (Import -> RAG) verstehen willst.** | `mindnet_pipeline_playbook_v2.2.md` |
|
||||||
|
| **...die genaue JSON-Struktur oder APIs suchst.** | `mindnet_technical_architecture.md` |
|
||||||
|
| **...verstehen willst, was fachlich passiert.** | `mindnet_functional_architecture.md` |
|
||||||
|
| **...den aktuellen Projektstatus suchst.** | `Programmplan_V2.2.md` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Rollen im System
|
||||||
|
|
||||||
|
* **Mindmaster (User/Owner):** Du. Du erstellst Inhalte, stellst Fragen und gibst Feedback. Du definierst die Werte (`type: value`).
|
||||||
|
* **Mindnet (Der Agent):** Der digitale Zwilling. Er agiert als pragmatischer, transparenter Assistent im Chat.
|
||||||
|
* **Administrator:** Verantwortlich für Docker-Container, Backups und LLM-Ressourcen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Aktueller Fokus
|
||||||
|
|
||||||
|
Wir befinden uns im Übergang von **Phase C (Persönlichkeit)** zu **Phase D (Interaktion)**.
|
||||||
|
Das "Gehirn" (WP05) ist fertig. Als Nächstes folgen die **Decision Engine (WP06)** für komplexe Entscheidungen und das **Frontend (WP10)** für bessere Usability.
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
# Kurzhandbuch — Type-Registry (WP-03)
|
||||||
|
|
||||||
|
## Zweck
|
||||||
|
Die Type-Registry macht den `type:`-Wert im Frontmatter maschinenlesbar
|
||||||
|
und steuert:
|
||||||
|
- Chunk-Profile (indirekt über Overlap bei fensterlosen Chunks),
|
||||||
|
- typabhängige Default-Kanten (hier: Aktivierung von Note-Scope-References),
|
||||||
|
- optionale Retriever-Gewichtungen (`retriever_weight`) für WP‑04.
|
||||||
|
|
||||||
|
## Aktivierung
|
||||||
|
1. Legen Sie `config/types.yaml` an (siehe Beispiel in diesem Paket).
|
||||||
|
2. Starten Sie Import/Server neu. Die Registry wird beim Prozessstart
|
||||||
|
geladen und gecached.
|
||||||
|
|
||||||
|
Fehlt die Datei oder ist sie defekt, bleibt das System voll funktionsfähig
|
||||||
|
(der Typ `concept` wird genutzt; Warnhinweis im Log).
|
||||||
|
|
||||||
|
## Schema (Beispiel)
|
||||||
|
```yaml
|
||||||
|
version: 1.0
|
||||||
|
types:
|
||||||
|
concept:
|
||||||
|
chunk_profile: long
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
retriever_weight: 1.0
|
||||||
|
task:
|
||||||
|
chunk_profile: short
|
||||||
|
edge_defaults: ["depends_on", "belongs_to"]
|
||||||
|
retriever_weight: 0.8
|
||||||
|
experience:
|
||||||
|
chunk_profile: medium
|
||||||
|
edge_defaults: ["derived_from", "inspired_by"]
|
||||||
|
retriever_weight: 0.9
|
||||||
|
project:
|
||||||
|
chunk_profile: medium
|
||||||
|
edge_defaults: ["references", "depends_on"]
|
||||||
|
retriever_weight: 0.95
|
||||||
|
```
|
||||||
|
|
||||||
|
## Verhalten
|
||||||
|
- **Chunking**: Die eigentliche Chunk-Länge bleibt weiterhin typabhängig über
|
||||||
|
`chunk_config.get_sizes(type)` gesteuert. Falls der Chunker keine Fenster
|
||||||
|
mit Overlap liefert, werden in `chunk_payload` synthetische Fenster erzeugt.
|
||||||
|
Hier greift das `chunk_profile` mit folgenden Overlap-Empfehlungen:
|
||||||
|
- short → (20,30), medium → (40,60), long → (60,80).
|
||||||
|
- **Edges**: Enthält `edge_defaults` den Eintrag `references`, werden Note-
|
||||||
|
Scope-References/Backlinks additiv aktiviert (kein Abbruch, keine Duplikate).
|
||||||
|
Weitere Kantenarten bleiben unverändert.
|
||||||
|
- **Retriever**: `retriever_weight` wird in der Note gespeichert und kann von
|
||||||
|
WP‑04 im Ranking/Scoring berücksichtigt werden.
|
||||||
|
|
||||||
|
## Tests / Validierung
|
||||||
|
- Dry-Run:
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.import_markdown --vault ./test_vault
|
||||||
|
```
|
||||||
|
- Vollimport + Purge:
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.import_markdown --vault ./test_vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
|
||||||
|
```
|
||||||
|
- Chunks-Qualität (Window vs Text):
|
||||||
|
```bash
|
||||||
|
python3 tests/check_chunks_window_vs_text.py --prefix "$COLLECTION_PREFIX"
|
||||||
|
```
|
||||||
|
- Roundtrip:
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --prefix "$COLLECTION_PREFIX"
|
||||||
|
python3 tests/compare_vaults.py --src ./test_vault --dst ./_exportVault --focus body
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fallbacks & Robustheit
|
||||||
|
- Registry nicht vorhanden/ungültig → `concept`, Warnung, keine Abbrüche.
|
||||||
|
- Alle bestehenden ENV/Flags bleiben identisch.
|
||||||
|
- Keine Änderungen an Punkt-IDs, Feldern oder Vektoren.
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
# Mindnet v2.2 – Admin Guide
|
||||||
|
**Datei:** `docs/mindnet_admin_guide_v2.2.md`
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** **FINAL** (Inkl. RAG & LLM Ops)
|
||||||
|
**Quellen:** `Handbuch.md`, `mindnet_developer_guide_v2.2.md`.
|
||||||
|
|
||||||
|
> Dieses Handbuch richtet sich an **Administratoren**. Es beschreibt Installation, Konfiguration, Backup-Strategien, Monitoring und den sicheren Betrieb der Mindnet-Instanz (API + DB + LLM).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zielgruppe & Scope
|
||||||
|
|
||||||
|
Mindnet ist als **Single-Node System** konzipiert, das lokal (z.B. Laptop) oder auf einem privaten Server (Homelab, Intranet) läuft.
|
||||||
|
Wir unterscheiden strikt zwischen:
|
||||||
|
* **Production (Port 8001):** Stabiler `main` Branch.
|
||||||
|
* **Development (Port 8002):** Experimentelle Feature-Branches.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Initial Setup & Installation
|
||||||
|
|
||||||
|
### 2.1 Systemvoraussetzungen
|
||||||
|
* **OS:** Linux (Ubuntu 22.04+ empfohlen) oder macOS.
|
||||||
|
* **Runtime:** Python 3.10+, Docker (für Qdrant), Ollama (für LLM).
|
||||||
|
* **Hardware:**
|
||||||
|
* CPU: 4+ Cores empfohlen (für Import & Inference).
|
||||||
|
* RAM: Min. 8GB empfohlen (4GB System + 4GB für Phi-3/Qdrant).
|
||||||
|
* Disk: SSD empfohlen für Qdrant-Performance.
|
||||||
|
|
||||||
|
### 2.2 Installation (Code)
|
||||||
|
# 1. Repository klonen
|
||||||
|
git clone <repo-url> /home/llmadmin/mindnet
|
||||||
|
cd /home/llmadmin/mindnet
|
||||||
|
|
||||||
|
# 2. Umgebung einrichten
|
||||||
|
python3 -m venv .venv
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
# 3. Verzeichnisse anlegen
|
||||||
|
mkdir -p logs qdrant_storage data/logs
|
||||||
|
|
||||||
|
### 2.3 Qdrant Setup (Docker)
|
||||||
|
Wir nutzen Qdrant als Vektor-Datenbank. Persistenz ist wichtig.
|
||||||
|
docker run -d \
|
||||||
|
--name mindnet_qdrant \
|
||||||
|
--restart always \
|
||||||
|
-p 6333:6333 \
|
||||||
|
-v $(pwd)/qdrant_storage:/qdrant/storage \
|
||||||
|
qdrant/qdrant
|
||||||
|
|
||||||
|
### 2.4 Ollama Setup (LLM Service)
|
||||||
|
Mindnet benötigt einen lokalen LLM-Server für den Chat.
|
||||||
|
# 1. Installieren (Linux Script)
|
||||||
|
curl -fsSL https://ollama.com/install.sh | sh
|
||||||
|
|
||||||
|
# 2. Modell laden (Phi-3 Mini für CPU-Performance)
|
||||||
|
ollama pull phi3:mini
|
||||||
|
|
||||||
|
# 3. Testen
|
||||||
|
curl http://localhost:11434/api/generate -d '{"model": "phi3:mini", "prompt":"Hi"}'
|
||||||
|
|
||||||
|
### 2.5 Konfiguration (ENV)
|
||||||
|
Erstelle eine `.env` Datei im Root-Verzeichnis.
|
||||||
|
|
||||||
|
# Qdrant Verbindung
|
||||||
|
QDRANT_URL="http://localhost:6333"
|
||||||
|
|
||||||
|
# Mindnet Core Settings
|
||||||
|
COLLECTION_PREFIX="mindnet"
|
||||||
|
MINDNET_TYPES_FILE="./config/types.yaml"
|
||||||
|
MINDNET_PROMPTS_PATH="./config/prompts.yaml"
|
||||||
|
|
||||||
|
# LLM Settings
|
||||||
|
MINDNET_LLM_MODEL="phi3:mini"
|
||||||
|
MINDNET_OLLAMA_URL="http://127.0.0.1:11434"
|
||||||
|
|
||||||
|
### 2.6 Deployment via Systemd
|
||||||
|
Mindnet wird als Systemdienst gestartet. Ollama läuft meist als eigener Dienst (`ollama.service`).
|
||||||
|
|
||||||
|
**Production Service (`/etc/systemd/system/mindnet-prod.service`):**
|
||||||
|
* Läuft auf Port 8001.
|
||||||
|
* Autostart (`enabled`).
|
||||||
|
* Restart Policy: `always`.
|
||||||
|
* Abhängigkeit: Sollte nach `docker` und `ollama` starten.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Betrieb im Alltag
|
||||||
|
|
||||||
|
### 3.1 Regelmäßige Importe
|
||||||
|
Der Vault-Zustand sollte regelmäßig (z.B. stündlich per Cronjob) nach Qdrant synchronisiert werden.
|
||||||
|
|
||||||
|
**Cronjob-Beispiel (stündlich):**
|
||||||
|
0 * * * * cd /home/llmadmin/mindnet && .venv/bin/python3 -m scripts.import_markdown --vault /path/to/vault --prefix "mindnet" --apply --purge-before-upsert --sync-deletes >> ./logs/import.log 2>&1
|
||||||
|
|
||||||
|
### 3.2 Health-Checks
|
||||||
|
Prüfe regelmäßig, ob alle drei Komponenten (API, DB, LLM) laufen.
|
||||||
|
|
||||||
|
**Status prüfen:**
|
||||||
|
sudo systemctl status mindnet-prod
|
||||||
|
sudo systemctl status ollama
|
||||||
|
|
||||||
|
**Logischer Smoke-Test:**
|
||||||
|
python3 scripts/test_retriever_smoke.py --mode hybrid --url http://localhost:8001/query
|
||||||
|
|
||||||
|
### 3.3 Logs & Monitoring
|
||||||
|
* **Technische Fehler (API):** `journalctl -u mindnet-prod -f`
|
||||||
|
* **LLM Fehler (Ollama):** `journalctl -u ollama -f`
|
||||||
|
* **Fachliche Logs:** `data/logs/search_history.jsonl`
|
||||||
|
|
||||||
|
**Troubleshooting Chat:**
|
||||||
|
* Wenn `/chat` in den Timeout läuft (>300s): Prüfe, ob `phi3:mini` geladen ist und ob der Server überlastet ist.
|
||||||
|
* Wenn `/chat` halluziniert: Prüfe `config/prompts.yaml` und ob der Import aktuell ist.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Update-Prozess
|
||||||
|
|
||||||
|
Wenn neue Versionen ausgerollt werden (Deployment):
|
||||||
|
|
||||||
|
1. **Code aktualisieren:**
|
||||||
|
cd /home/llmadmin/mindnet
|
||||||
|
git pull origin main
|
||||||
|
2. **Dependencies prüfen:**
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
3. **Dienst neustarten (Zwingend!):**
|
||||||
|
sudo systemctl restart mindnet-prod
|
||||||
|
4. **Schema-Migration (falls nötig):**
|
||||||
|
python3 -m scripts.import_markdown ... --apply
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Backup & Restore
|
||||||
|
|
||||||
|
Datensicherheit ruht auf drei Säulen: Vault (Source), Qdrant (Index), JSONL-Logs (Lern-Daten).
|
||||||
|
|
||||||
|
### 5.1 Vault-Backup (Priorität 1)
|
||||||
|
Der Markdown-Vault ist die **Single Source of Truth**. Er muss klassisch gesichert werden (Git/NAS).
|
||||||
|
|
||||||
|
### 5.2 Qdrant-Snapshots (Priorität 2)
|
||||||
|
Für schnelle Wiederherstellung des Suchindex.
|
||||||
|
docker stop mindnet_qdrant
|
||||||
|
tar -czf qdrant_backup_$(date +%F).tar.gz ./qdrant_storage
|
||||||
|
docker start mindnet_qdrant
|
||||||
|
|
||||||
|
### 5.3 Log-Daten (Priorität 3)
|
||||||
|
Sichere den Ordner `data/logs/`. Verlust dieser Daten bedeutet, dass das System vergisst, welche Antworten Nutzer hilfreich fanden.
|
||||||
|
|
||||||
|
### 5.4 Notfall-Wiederherstellung (Rebuild)
|
||||||
|
Wenn die Datenbank korrupt ist:
|
||||||
|
# 1. DB komplett leeren (Wipe)
|
||||||
|
python3 -m scripts.reset_qdrant --mode wipe --prefix "mindnet" --yes
|
||||||
|
# 2. Alles neu importieren
|
||||||
|
python3 -m scripts.import_markdown --vault /path/to/vault --prefix "mindnet" --apply
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Governance & Sicherheit
|
||||||
|
|
||||||
|
### 6.1 Zugriffsschutz
|
||||||
|
Mindnet hat aktuell **keine integrierte Authentifizierung**.
|
||||||
|
* **API:** Muss hinter einem Reverse Proxy (Nginx) mit Basic Auth laufen.
|
||||||
|
* **Qdrant:** Sollte via Firewall (ufw) auf `127.0.0.1` beschränkt sein.
|
||||||
|
* **Ollama:** Standardmäßig hört Ollama nur auf `localhost`. Das ist sicher.
|
||||||
|
|
||||||
|
### 6.2 Typen-Governance
|
||||||
|
Änderungen an der `types.yaml` (z.B. neue Gewichte) wirken global.
|
||||||
|
* **Prozess:** Änderungen sollten getestet werden (Smoke-Test), bevor sie produktiv gehen.
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
# Mindnet v2.2 – Appendices & Referenzen
|
||||||
|
**Datei:** `docs/mindnet_appendices_v2.2.md`
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** **FINAL** (Integrierter Stand WP01–WP05)
|
||||||
|
**Quellen:** `TYPE_REGISTRY_MANUAL.md`, `chunking_strategy.md`, `mindnet_technical_architecture.md`, `Handbuch.md`.
|
||||||
|
|
||||||
|
> Dieses Dokument bündelt Tabellen, Schemata und technische Referenzen, die in den Prozess-Dokumenten (Playbook, Guides) den Lesefluss stören würden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang A: Typ-Registry Referenz (Default-Werte)
|
||||||
|
|
||||||
|
Diese Tabelle zeigt die Standard-Konfiguration der `types.yaml` (Stand v2.2).
|
||||||
|
|
||||||
|
| Typ (`type`) | Chunk Profile | Retriever Weight | Edge Defaults (Auto-Kanten) | Beschreibung |
|
||||||
|
| :--- | :--- | :--- | :--- | :--- |
|
||||||
|
| **concept** | `medium` | 0.60 | `references`, `related_to` | Abstrakte Begriffe, Theorien. |
|
||||||
|
| **project** | `long` | 0.97 | `references`, `depends_on` | Aktive Vorhaben. Hohe Priorität. |
|
||||||
|
| **decision** | `long` | 1.00 | `caused_by`, `references` | Entscheidungen (ADRs). Höchste Prio. |
|
||||||
|
| **experience** | `medium` | 0.90 | `derived_from`, `inspired_by` | Persönliche Learnings. |
|
||||||
|
| **journal** | `short` | 0.80 | `related_to` | Zeitgebundene Logs. Fein granular. |
|
||||||
|
| **person** | `short` | 0.50 | `related_to` | Personen-Profile. |
|
||||||
|
| **source** | `long` | 0.50 | *(keine)* | Externe Quellen (Bücher, PDFs). |
|
||||||
|
| **event** | `short` | 0.60 | `related_to` | Meetings, Konferenzen. |
|
||||||
|
| **value** | `medium` | 1.00 | `related_to` | Persönliche Werte/Prinzipien. |
|
||||||
|
| **default** | `medium` | 1.00 | `references` | Fallback, wenn Typ unbekannt. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang B: Edge-Typen & Semantik
|
||||||
|
|
||||||
|
Referenz aller implementierten Kantenarten (`kind`).
|
||||||
|
|
||||||
|
| Kind | Herkunft (Primär) | Symmetrisch? | Bedeutung & Nutzung |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| `references` | Wikilink `[[...]]` | Nein | Standard-Verweis. "Erwähnt X". |
|
||||||
|
| `belongs_to` | Struktur (Auto) | Nein | Verknüpft Chunk mit seiner Note. |
|
||||||
|
| `next` / `prev` | Struktur (Auto) | Ja (Logisch) | Definiert Lesereihenfolge der Chunks. |
|
||||||
|
| `depends_on` | Inline / Default | Nein | Harte Abhängigkeit. "Braucht X um zu funktionieren". |
|
||||||
|
| `related_to` | Callout / Default | Ja (Oft) | Thematische Nähe. "Siehe auch X". |
|
||||||
|
| `similar_to` | Inline | Ja | Inhaltliche Ähnlichkeit. "Ist wie X". |
|
||||||
|
| `caused_by` | Inline | Nein | Kausalität. "X ist der Grund für Y". |
|
||||||
|
| `solves` | Inline | Nein | Lösung. "Tool X löst Problem Y". |
|
||||||
|
| `derived_from` | Default (Exp) | Nein | Herkunft. "Erkenntnis stammt aus Quelle X". |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang C: Datenmodelle (JSON Payloads)
|
||||||
|
|
||||||
|
Dies sind die Felder, die effektiv in Qdrant gespeichert werden.
|
||||||
|
|
||||||
|
### C.1 Note Payload (`mindnet_notes`)
|
||||||
|
{
|
||||||
|
"note_id": "string (keyword)", // UUIDv5 oder Slug
|
||||||
|
"title": "string (text)", // Titel aus Frontmatter
|
||||||
|
"type": "string (keyword)", // Typ (z.B. 'project')
|
||||||
|
"retriever_weight": "float", // Numerische Wichtigkeit (0.0-1.0)
|
||||||
|
"chunk_profile": "string", // Genutztes Profil (z.B. 'long')
|
||||||
|
"edge_defaults": ["string"], // Liste der aktiven Default-Kanten
|
||||||
|
"tags": ["string"], // Liste von Tags
|
||||||
|
"created": "string (iso-date)", // Erstellungsdatum
|
||||||
|
"updated": "integer", // Timestamp
|
||||||
|
"fulltext": "string (no-index)" // Gesamter Text (für Recovery)
|
||||||
|
}
|
||||||
|
|
||||||
|
### C.2 Chunk Payload (`mindnet_chunks`)
|
||||||
|
{
|
||||||
|
"chunk_id": "string (keyword)", // Format: {note_id}#c{index}
|
||||||
|
"note_id": "string (keyword)", // FK zur Note
|
||||||
|
"text": "string (text)", // Reintext für Anzeige (ohne Overlap)
|
||||||
|
"window": "string (text)", // Text + Overlap (für Embedding)
|
||||||
|
"ord": "integer", // Laufende Nummer (1..N)
|
||||||
|
"retriever_weight": "float", // Kopie aus Note (für Query-Speed)
|
||||||
|
"neighbors_prev": ["string"], // ID des Vorgängers
|
||||||
|
"neighbors_next": ["string"] // ID des Nachfolgers
|
||||||
|
}
|
||||||
|
|
||||||
|
### C.3 Edge Payload (`mindnet_edges`)
|
||||||
|
{
|
||||||
|
"edge_id": "string (keyword)", // Deterministischer Hash
|
||||||
|
"source_id": "string (keyword)", // Chunk-ID (Start)
|
||||||
|
"target_id": "string (keyword)", // Chunk-ID oder Note-Titel (Ziel)
|
||||||
|
"kind": "string (keyword)", // z.B. 'depends_on'
|
||||||
|
"scope": "string (keyword)", // Immer 'chunk' in v2.2
|
||||||
|
"rule_id": "string (keyword)", // Traceability: 'inline:rel', 'explicit:wikilink'
|
||||||
|
"confidence": "float", // 0.0 - 1.0
|
||||||
|
"note_id": "string (keyword)" // Owner Note ID (für Löschung)
|
||||||
|
}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang D: Environment Variablen
|
||||||
|
|
||||||
|
Diese Variablen steuern das Verhalten der Skripte und Container.
|
||||||
|
|
||||||
|
| Variable | Default | Beschreibung |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| `QDRANT_URL` | `http://localhost:6333` | URL zur Vektor-DB. |
|
||||||
|
| `QDRANT_API_KEY` | *(leer)* | API-Key für Absicherung (optional). |
|
||||||
|
| `COLLECTION_PREFIX` | `mindnet` | Namensraum für Collections (`{prefix}_notes` etc). |
|
||||||
|
| `MINDNET_TYPES_FILE` | `config/types.yaml` | Pfad zur Typ-Registry. |
|
||||||
|
| `MINDNET_RETRIEVER_CONFIG`| `config/retriever.yaml`| Pfad zur Scoring-Konfiguration. |
|
||||||
|
| `MINDNET_PROMPTS_PATH` | `config/prompts.yaml` | Pfad zu LLM-Prompts (Neu in v2.2). |
|
||||||
|
| `MINDNET_LLM_MODEL` | `phi3:mini` | Name des Ollama-Modells (Neu in v2.2). |
|
||||||
|
| `MINDNET_OLLAMA_URL` | `http://127.0.0.1:11434`| URL zum LLM-Server (Neu in v2.2). |
|
||||||
|
| `MINDNET_HASH_COMPARE` | `Body` | Vergleichsmodus für Import (`Body`, `Frontmatter`, `Full`). |
|
||||||
|
| `MINDNET_HASH_SOURCE` | `parsed` | Quelle für Hash (`parsed`, `raw`, `file`). |
|
||||||
|
| `VECTOR_DIM` | `384` | Dimension der Embeddings (Modellabhängig). |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang E: Glossar
|
||||||
|
|
||||||
|
* **Callout-Edge:** Kante via `> [!edge]`.
|
||||||
|
* **Chunking:** Zerlegung von Notizen in kleinere Einheiten.
|
||||||
|
* **Confidence:** Vertrauenswürdigkeit einer Kante (0.0-1.0).
|
||||||
|
* **Explanation Layer:** Komponente, die Scores und Graphen als Begründung liefert.
|
||||||
|
* **Feedback Loop:** Prozess des Loggens und Auswertens von User-Reaktionen.
|
||||||
|
* **Idempotenz:** Mehrfache Ausführung liefert gleiches Ergebnis.
|
||||||
|
* **Inline-Edge:** Kante via `[[rel:type Ziel]]`.
|
||||||
|
* **RAG (Retrieval Augmented Generation):** Kombination aus Suche und Text-Generierung.
|
||||||
|
* **Retriever:** Suchmaschine (FastAPI).
|
||||||
|
* **Vault:** Ordner mit Markdown-Dateien.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang F: Workpackage Status (v2.3.0)
|
||||||
|
|
||||||
|
Aktueller Implementierungsstand der Module.
|
||||||
|
|
||||||
|
| WP | Titel | Status | Anmerkung |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| **WP01** | Knowledge Design | 🟢 Live | Typen, Frontmatter definiert. |
|
||||||
|
| **WP02** | Chunking Strategy | 🟢 Live | Profilbasiertes Chunking aktiv. |
|
||||||
|
| **WP03** | Edge Logic / Import | 🟢 Live | Neue Importer-Pipeline mit Provenance. |
|
||||||
|
| **WP04a**| Retriever Scoring | 🟢 Live | Hybrider Score (Semantik + Graph). |
|
||||||
|
| **WP04b**| Explanation Layer | 🟢 Live | API liefert Reasons & Breakdown. |
|
||||||
|
| **WP04c**| Feedback Loop | 🟢 Live | Logging (JSONL) & Traceability aktiv. |
|
||||||
|
| **WP05** | Persönlichkeit / Chat | 🟢 Live | RAG-Chat mit Context Enrichment. |
|
||||||
|
| **WP06** | Decision Engine | 🟡 Geplant | Nächster Schritt (Logik). |
|
||||||
|
| **WP08** | Self-Tuning | 🔴 Geplant | Auto-Adjustment der Gewichte. |
|
||||||
|
| **WP10** | Chat Interface | 🟡 Geplant | Nächster Schritt (Frontend). |
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
# Mindnet v2.2 – Entwickler-Workflow
|
||||||
|
**Datei:** `DEV_WORKFLOW.md`
|
||||||
|
**Stand:** 2025-12-08 (Aktualisiert: Systemd Services & Sync-First)
|
||||||
|
|
||||||
|
Dieses Handbuch beschreibt den Entwicklungszyklus zwischen **Windows PC** (IDE), **Raspberry Pi** (Gitea) und **Beelink** (Runtime/Server).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Die Architektur
|
||||||
|
|
||||||
|
* **Windows 11 (VS Code):** Hier schreibst du Code. **Nie** direkt auf `main` arbeiten!
|
||||||
|
* **Raspberry Pi (Gitea):** Der Tresor. Speichert den Code und verwaltet Versionen.
|
||||||
|
* **Beelink (Runtime):** Hier läuft die Software. Wir nutzen **Systemd-Services**:
|
||||||
|
* **PROD (`mindnet-prod`):** Läuft stabil auf `main`. Ordner: `~/mindnet`. **Port 8001**.
|
||||||
|
* **DEV (`mindnet-dev`):** Deine Spielwiese. Ordner: `~/mindnet_dev`. **Port 8002**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Der Zyklus: Von der Idee zum Release
|
||||||
|
|
||||||
|
### Phase 1: Entwicklung (Windows / VS Code)
|
||||||
|
|
||||||
|
Hier erstellst du die neue Funktion in einer sicheren Umgebung.
|
||||||
|
|
||||||
|
1. **Basis aktualisieren (WICHTIG!):**
|
||||||
|
Bevor du startest, muss dein lokales `main` auf dem Stand des Servers sein.
|
||||||
|
* Klicke unten links auf den aktuellen Branch und wähle **`main`**.
|
||||||
|
* Klicke links im Menü "Source Control" auf die **drei Punkte (...)** -> **Pull** (oder das Synchronisieren-Symbol).
|
||||||
|
* *Erst jetzt hast du alle Dateien!*
|
||||||
|
|
||||||
|
2. **Branch erstellen:**
|
||||||
|
* Klicke wieder unten links auf `main`.
|
||||||
|
* Wähle `+ Create new branch...`.
|
||||||
|
* Gib den Namen ein: `feature/was-ich-tue` (z.B. `feature/wp05-chat`).
|
||||||
|
* Drücke **Enter**.
|
||||||
|
|
||||||
|
3. **Sicherheits-Check:**
|
||||||
|
* Steht unten links jetzt dein Feature-Branch? **Nur dann darfst du Code ändern!**
|
||||||
|
|
||||||
|
4. **Coden:**
|
||||||
|
* Nimm deine Änderungen vor.
|
||||||
|
|
||||||
|
5. **Sichern & Hochladen:**
|
||||||
|
* **Source Control** Icon (Gabel-Symbol) -> Nachricht eingeben -> **Commit**.
|
||||||
|
* **Publish Branch** (oder "Sync Changes"), um den Branch zum Raspberry Pi (Gitea) zu senden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 2: Testen (Beelink / `mindnet_dev`)
|
||||||
|
|
||||||
|
Hier prüfst du, ob dein neuer Code auf dem echten Server läuft.
|
||||||
|
|
||||||
|
1. **Einloggen:**
|
||||||
|
`ssh user@<BEELINK-IP>`
|
||||||
|
|
||||||
|
2. **In die Werkstatt gehen:**
|
||||||
|
`cd /home/llmadmin/mindnet_dev`
|
||||||
|
|
||||||
|
3. **Code holen:**
|
||||||
|
```bash
|
||||||
|
git fetch
|
||||||
|
# Tipp: 'git branch -r' zeigt alle verfügbaren Branches an
|
||||||
|
git checkout feature/wp05-chat
|
||||||
|
git pull
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **Umgebung vorbereiten (bei Bedarf):**
|
||||||
|
```bash
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -r requirements.txt # Nur nötig bei neuen Paketen
|
||||||
|
```
|
||||||
|
|
||||||
|
5. **Test-Server aktualisieren (WICHTIG):**
|
||||||
|
Der Code ist da, aber der Prozess im RAM ist noch alt. Wir müssen den Service neustarten.
|
||||||
|
|
||||||
|
**Option A: Standard (Als Service laufen lassen)**
|
||||||
|
Ideal, wenn du nur testen willst, ob es läuft.
|
||||||
|
```bash
|
||||||
|
sudo systemctl restart mindnet-dev
|
||||||
|
|
||||||
|
# Logs prüfen (um Fehler zu sehen):
|
||||||
|
journalctl -u mindnet-dev -f
|
||||||
|
```
|
||||||
|
|
||||||
|
**Option B: Manuell Debuggen (Direct Output)**
|
||||||
|
Ideal, wenn du Print-Ausgaben direkt sehen willst oder der Service dauernd crasht.
|
||||||
|
```bash
|
||||||
|
# 1. Service stoppen (wichtig, sonst ist Port 8002 belegt!)
|
||||||
|
sudo systemctl stop mindnet-dev
|
||||||
|
|
||||||
|
# 2. Manuell starten
|
||||||
|
uvicorn app.main:app --host 0.0.0.0 --port 8002 --env-file .env
|
||||||
|
|
||||||
|
# ... Testen ...
|
||||||
|
|
||||||
|
# 3. Wenn fertig: Service wieder anschalten (Optional)
|
||||||
|
# Strg+C drücken
|
||||||
|
sudo systemctl start mindnet-dev
|
||||||
|
```
|
||||||
|
|
||||||
|
6. **Validieren:**
|
||||||
|
Führe deine Tests in einem **zweiten Terminal** aus:
|
||||||
|
```bash
|
||||||
|
# Beispiel für Smoke-Test gegen Dev-Port
|
||||||
|
python3 tests/test_chat_smoke.py --url http://localhost:8002/chat
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 3: Release (Gitea / Raspberry Pi)
|
||||||
|
|
||||||
|
Wenn der Test auf Port 8002 erfolgreich war:
|
||||||
|
|
||||||
|
1. Öffne Gitea im Browser.
|
||||||
|
2. Erstelle einen **Pull Request** (Dein Branch -> `main`).
|
||||||
|
3. **Merge** den Pull Request.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 4: Deployment (Beelink / `mindnet`)
|
||||||
|
|
||||||
|
Jetzt bringen wir die Änderung in das Live-System (Port 8001).
|
||||||
|
|
||||||
|
1. **Deploy-Agent (Automatisch):**
|
||||||
|
Wenn dein `deploy.yml` aktiv ist, passiert das jetzt von selbst!
|
||||||
|
|
||||||
|
2. **Manuell (Falls nötig):**
|
||||||
|
```bash
|
||||||
|
cd /home/llmadmin/mindnet
|
||||||
|
git pull origin main
|
||||||
|
|
||||||
|
# Produktions-Service neustarten
|
||||||
|
sudo systemctl restart mindnet-prod
|
||||||
|
|
||||||
|
# Kurz prüfen, ob er läuft
|
||||||
|
sudo systemctl status mindnet-prod
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 5: Aufräumen (Optional aber empfohlen)
|
||||||
|
|
||||||
|
Damit das Chaos nicht wächst, löschen wir den fertigen Branch.
|
||||||
|
|
||||||
|
1. **Gitea:** Im Pull Request auf "Delete Branch" klicken.
|
||||||
|
2. **Beelink (Dev):**
|
||||||
|
```bash
|
||||||
|
cd ~/mindnet_dev
|
||||||
|
git checkout main
|
||||||
|
git pull
|
||||||
|
git branch -d feature/wp05-chat
|
||||||
|
```
|
||||||
|
*Hinweis: Der `mindnet-dev` Service läuft jetzt mit dem Code von `main`, was okay ist.*
|
||||||
|
3. **VS Code:**
|
||||||
|
* Auf `main` wechseln.
|
||||||
|
* Sync drücken.
|
||||||
|
* `F1` -> `Git: Delete Branch` -> Branch auswählen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Cheat Sheet: Die wichtigsten Befehle
|
||||||
|
|
||||||
|
| Wo? | Befehl | Was tut es? |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| **VS Code** | `Sync (auf main)` | **WICHTIG:** Holt neuesten Code vom Server. |
|
||||||
|
| **Beelink** | `git fetch` | Aktualisiert Liste der Remote-Branches. |
|
||||||
|
| **Beelink** | `git checkout <name>` | Wechsle Branch. |
|
||||||
|
| **Beelink** | `git pull` | Aktualisiere aktuellen Branch. |
|
||||||
|
| **Beelink** | `sudo systemctl restart mindnet-dev` | **Neustart Dev-Server (Port 8002).** |
|
||||||
|
| **Beelink** | `sudo systemctl stop mindnet-dev` | **Stoppt Dev-Server (macht Port frei).** |
|
||||||
|
| **Beelink** | `sudo systemctl restart mindnet-prod`| **Neustart Prod-Server (Port 8001).** |
|
||||||
|
| **Beelink** | `journalctl -u mindnet-dev -f` | **Live-Logs vom Dev-Server sehen.** |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Troubleshooting
|
||||||
|
|
||||||
|
**"Port 8002 already in use"**
|
||||||
|
* Du willst `uvicorn` manuell starten, aber der Service läuft noch.
|
||||||
|
* **Lösung:** `sudo systemctl stop mindnet-dev`
|
||||||
|
|
||||||
|
**"Hilfe, in meinem neuen Branch fehlen Dateien!"**
|
||||||
|
* Das passiert, wenn du beim Erstellen nicht aktuell warst.
|
||||||
|
* **Lösung:**
|
||||||
|
```bash
|
||||||
|
git checkout feature/mein-kaputter-branch
|
||||||
|
git merge main
|
||||||
@@ -0,0 +1,192 @@
|
|||||||
|
# Mindnet v2.2 – Developer Guide
|
||||||
|
**Datei:** `docs/mindnet_developer_guide_v2.2.md`
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** **FINAL** (Inkl. RAG & LLM Setup)
|
||||||
|
**Quellen:** `mindnet_technical_architecture.md`, `Handbuch.md`, `DEV_WORKFLOW.md`.
|
||||||
|
|
||||||
|
> **Zielgruppe:** Entwickler:innen.
|
||||||
|
> **Zweck:** Anleitung zum Aufsetzen der Entwicklungsumgebung, Verständnis der Modulstruktur und Durchführung von Tests.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Projektstruktur (Aktualisiert)
|
||||||
|
|
||||||
|
Der Code ist modular in `app` (Logik), `scripts` (CLI) und `config` (Steuerung) getrennt.
|
||||||
|
|
||||||
|
mindnet/
|
||||||
|
├── app/
|
||||||
|
│ ├── core/ # Kernlogik
|
||||||
|
│ │ ├── chunker.py # Text-Zerlegung
|
||||||
|
│ │ ├── derive_edges.py # Edge-Erzeugung (WP03 Logik)
|
||||||
|
│ │ ├── retriever.py # Scoring & Hybrid Search
|
||||||
|
│ │ ├── qdrant.py # DB-Verbindung
|
||||||
|
│ │ └── ...
|
||||||
|
│ ├── models/ # Pydantic DTOs
|
||||||
|
│ │ └── dto.py # Zentrale DTO-Definition
|
||||||
|
│ ├── routers/ # FastAPI Endpoints
|
||||||
|
│ │ ├── query.py # Suche
|
||||||
|
│ │ ├── chat.py # RAG-Chat (WP05)
|
||||||
|
│ │ ├── feedback.py # Feedback (WP04c)
|
||||||
|
│ │ └── ...
|
||||||
|
│ ├── services/ # Interne & Externe Dienste
|
||||||
|
│ │ ├── llm_service.py # Ollama Client (WP05)
|
||||||
|
│ │ ├── feedback_service.py # Logging (JSONL Writer)
|
||||||
|
│ │ └── embeddings_client.py
|
||||||
|
│ └── main.py # Entrypoint der API
|
||||||
|
├── config/ # YAML-Konfigurationen (Single Source of Truth)
|
||||||
|
│ ├── types.yaml # Import-Regeln
|
||||||
|
│ ├── prompts.yaml # LLM Prompts & RAG Templates (WP05)
|
||||||
|
│ └── retriever.yaml # Scoring-Regeln
|
||||||
|
├── data/
|
||||||
|
│ └── logs/ # Lokale Logs (search_history.jsonl, feedback.jsonl)
|
||||||
|
├── scripts/ # CLI-Tools (Import, Diagnose, Reset)
|
||||||
|
├── tests/ # Pytest Suite & Smoke Scripts
|
||||||
|
└── vault/ # Dein lokaler Markdown-Content (Git-ignored)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Lokales Setup (Development)
|
||||||
|
|
||||||
|
### 2.1 Voraussetzungen
|
||||||
|
* **Python:** 3.10 oder höher.
|
||||||
|
* **Docker:** Für Qdrant.
|
||||||
|
* **Ollama:** Für lokale LLM-Inference (erforderlich für `/chat`).
|
||||||
|
* **Vault:** Ein Ordner mit Markdown-Dateien (z.B. `./mindnet_v2_test_vault` für Tests).
|
||||||
|
|
||||||
|
### 2.2 Installation
|
||||||
|
# 1. Repository klonen & Verzeichnis wechseln
|
||||||
|
git clone <repo> mindnet
|
||||||
|
cd mindnet
|
||||||
|
|
||||||
|
# 2. Virtual Environment erstellen
|
||||||
|
python3 -m venv .venv
|
||||||
|
source .venv/bin/activate
|
||||||
|
|
||||||
|
# 3. Abhängigkeiten installieren
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
# 4. Ollama Setup (Modell laden)
|
||||||
|
# Wir nutzen Phi-3 Mini für schnelle CPU-Inference
|
||||||
|
ollama pull phi3:mini
|
||||||
|
|
||||||
|
### 2.3 Konfiguration (Environment)
|
||||||
|
Erstelle eine `.env` Datei im Root-Verzeichnis.
|
||||||
|
|
||||||
|
# Qdrant Verbindung
|
||||||
|
QDRANT_URL="http://localhost:6333"
|
||||||
|
QDRANT_API_KEY="" # Leer lassen für lokal
|
||||||
|
|
||||||
|
# Mindnet Core Settings
|
||||||
|
COLLECTION_PREFIX="mindnet_dev"
|
||||||
|
MINDNET_TYPES_FILE="./config/types.yaml"
|
||||||
|
MINDNET_RETRIEVER_CONFIG="./config/retriever.yaml"
|
||||||
|
|
||||||
|
# LLM / RAG Settings (WP05)
|
||||||
|
MINDNET_LLM_MODEL="phi3:mini"
|
||||||
|
MINDNET_OLLAMA_URL="http://127.0.0.1:11434"
|
||||||
|
MINDNET_PROMPTS_PATH="./config/prompts.yaml"
|
||||||
|
|
||||||
|
# Import-Strategie
|
||||||
|
MINDNET_HASH_COMPARE="Body"
|
||||||
|
MINDNET_HASH_SOURCE="parsed"
|
||||||
|
|
||||||
|
### 2.4 Dienste starten (Systemd bevorzugt)
|
||||||
|
Auf dem Entwicklungsserver (Beelink) nutzen wir Systemd.
|
||||||
|
|
||||||
|
# Starten / Neustarten
|
||||||
|
sudo systemctl restart mindnet-dev
|
||||||
|
|
||||||
|
# Logs prüfen
|
||||||
|
journalctl -u mindnet-dev -f
|
||||||
|
|
||||||
|
Falls du lokal auf Windows entwickelst:
|
||||||
|
# 1. Qdrant starten
|
||||||
|
docker run -p 6333:6333 qdrant/qdrant
|
||||||
|
# 2. Ollama starten
|
||||||
|
ollama serve
|
||||||
|
# 3. API starten
|
||||||
|
uvicorn app.main:app --host 0.0.0.0 --port 8002 --env-file .env --reload
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Core-Module & Entwicklung
|
||||||
|
|
||||||
|
### 3.1 Der Importer (`scripts.import_markdown`)
|
||||||
|
Dies ist das komplexeste Modul.
|
||||||
|
* **Einstieg:** `scripts/import_markdown.py` -> `main()`.
|
||||||
|
* **Idempotenz:** Der Importer muss mehrfach laufen können, ohne Duplikate zu erzeugen. Wir nutzen deterministische IDs (UUIDv5).
|
||||||
|
* **Debugging:** Nutze `--dry-run` oder `scripts/payload_dryrun.py`.
|
||||||
|
|
||||||
|
### 3.2 Edge-Logik (`app.core.derive_edges`)
|
||||||
|
Hier wird entschieden, welche Kanten entstehen.
|
||||||
|
* **Rule-ID:** Vergib zwingend eine eindeutige `rule_id` (z.B. `custom:my_rule`), damit die Herkunft für die Explanation nachvollziehbar bleibt.
|
||||||
|
|
||||||
|
### 3.3 Der Retriever & Chat (`app.core.retriever` / `app.routers.chat`)
|
||||||
|
Hier passiert das Scoring und die Generation.
|
||||||
|
* **Hybrid Search:** Der Chat-Endpoint erzwingt `mode="hybrid"`.
|
||||||
|
* **Context Enrichment:** In `_build_enriched_context` (chat.py) werden Metadaten (Typ, Score) in den Prompt injiziert. Achte darauf, dass neue Typen hier ggf. berücksichtigt werden, falls sie spezielle Behandlung brauchen (aktuell generisch gelöst).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Tests & Debugging
|
||||||
|
|
||||||
|
Wir unterscheiden drei Test-Ebenen. Ein Pull Request sollte alle passieren.
|
||||||
|
|
||||||
|
### 4.1 Unit Tests (Pytest)
|
||||||
|
Für isolierte Logik (Parsing, Scoring).
|
||||||
|
pytest tests/test_retriever_basic.py
|
||||||
|
pytest tests/test_chunking.py
|
||||||
|
pytest tests/test_edges_all.py
|
||||||
|
|
||||||
|
### 4.2 Integration / Pipeline Tests
|
||||||
|
Prüfen den Datenfluss von Markdown bis Qdrant-JSON.
|
||||||
|
* **Payload Dryrun:** Prüft JSON-Schema Konformität.
|
||||||
|
python3 -m scripts.payload_dryrun --vault ./mindnet_v2_test_vault
|
||||||
|
|
||||||
|
* **Edge Checks:** Prüft Graph-Invarianten.
|
||||||
|
python3 -m scripts.edges_full_check
|
||||||
|
|
||||||
|
### 4.3 Smoke Tests (E2E)
|
||||||
|
Prüfen das laufende System (API) gegen eine echte Qdrant-Instanz und Ollama.
|
||||||
|
|
||||||
|
# 1. Retriever Test (Hybrid + Explanation)
|
||||||
|
python scripts/test_retriever_smoke.py --query "Test" --mode hybrid --top-k 5 --explain
|
||||||
|
|
||||||
|
# 2. Chat / RAG Test (WP05)
|
||||||
|
# Prüft die gesamte Kette: Suche -> Kontext -> LLM -> Antwort
|
||||||
|
python tests/test_chat_wp05.py
|
||||||
|
|
||||||
|
# 3. Feedback Test (WP04c)
|
||||||
|
python tests/test_feedback_smoke.py --url http://localhost:8002/query
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Guidelines für Erweiterungen
|
||||||
|
|
||||||
|
### 5.1 Neuen Note-Typ hinzufügen
|
||||||
|
* Bearbeite `config/types.yaml`.
|
||||||
|
* Definiere `chunk_profile`, `retriever_weight` und `edge_defaults`.
|
||||||
|
* Führe einen **Re-Import** durch.
|
||||||
|
|
||||||
|
### 5.2 Persönlichkeit anpassen (Prompt Engineering)
|
||||||
|
* Bearbeite `config/prompts.yaml`.
|
||||||
|
* Änderungen sind sofort aktiv (kein Neustart nötig, da pro Request geladen, sofern nicht gecached).
|
||||||
|
* **Prompt-Regel:** Kleine Modelle (Phi-3) brauchen strikte Anweisungen (z.B. "Achte auf [DECISION]").
|
||||||
|
|
||||||
|
### 5.3 Neue API-Endpunkte
|
||||||
|
* Erstelle einen neuen Router in `app/routers/`.
|
||||||
|
* Registriere den Router in `app/main.py`.
|
||||||
|
* Denke an **Traceability**: Generiere oder durchschleife `query_id`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Nützliche Einzeiler
|
||||||
|
|
||||||
|
**DB komplett zurücksetzen (Vorsicht!):**
|
||||||
|
python3 -m scripts.reset_qdrant --mode wipe --prefix "mindnet_dev" --yes
|
||||||
|
|
||||||
|
**Einen einzelnen File inspizieren (Parser-Sicht):**
|
||||||
|
python3 tests/inspect_one_note.py --file ./vault/MeinFile.md
|
||||||
|
|
||||||
|
**Live-Logs sehen (Beelink):**
|
||||||
|
journalctl -u mindnet-dev -f
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
# mindnet Retriever (WP-04 / Step 4a) – Kurz-Dokumentation
|
||||||
|
|
||||||
|
## 1. Überblick
|
||||||
|
|
||||||
|
Der Retriever besteht aus:
|
||||||
|
|
||||||
|
- **Semantic Retriever** (`semantic_retrieve`)
|
||||||
|
- **Hybrid Retriever** (`hybrid_retrieve`)
|
||||||
|
- Qdrant-Backend (`*_chunks`, `*_edges`)
|
||||||
|
- optionaler Edge-Expansion über `graph_adapter.expand`
|
||||||
|
|
||||||
|
Die API nutzt das bestehende `/query`-Endpoint (FastAPI), das `QueryRequest` und `QueryResponse` aus `app.models.dto` verwendet.
|
||||||
|
|
||||||
|
## 2. Scoring-Formel
|
||||||
|
|
||||||
|
Pro Treffer werden folgende Komponenten geführt:
|
||||||
|
|
||||||
|
- `semantic_score` – direkt aus Qdrant (Cosine/Euclidean, je nach Collection)
|
||||||
|
- `retriever_weight` – aus Chunk-Payload (`types.yaml` / Frontmatter)
|
||||||
|
- `edge_bonus` – Bonus aus dem Subgraph (z. B. Zahl der eingehenden/ausgehenden Edges, Beziehungstypen)
|
||||||
|
- `centrality_bonus` – Bonus aus einer einfachen Zentralitäts-Heuristik im Subgraph
|
||||||
|
|
||||||
|
Aktuelle Formel (Step 4a):
|
||||||
|
|
||||||
|
```text
|
||||||
|
total_score = semantic_score * max(retriever_weight, 0.0)
|
||||||
|
+ edge_bonus
|
||||||
|
+ centrality_bonus
|
||||||
|
```
|
||||||
|
|
||||||
|
Die Sortierung der Ergebnisse erfolgt absteigend nach `total_score`.
|
||||||
|
|
||||||
|
## 3. Modi
|
||||||
|
|
||||||
|
### 3.1 Semantic Mode
|
||||||
|
|
||||||
|
- Funktion: `semantic_retrieve`
|
||||||
|
- Request-Feld `mode = "semantic"`
|
||||||
|
- Keine Edge-Expansion, kein Graph-Call.
|
||||||
|
- Kandidaten nur aus Vektorsuche in `<prefix>_chunks`.
|
||||||
|
|
||||||
|
### 3.2 Hybrid Mode
|
||||||
|
|
||||||
|
- Funktion: `hybrid_retrieve`
|
||||||
|
- Request-Feld `mode = "hybrid"`
|
||||||
|
- Schritte:
|
||||||
|
1. Semantische Chunk-Suche wie im Semantic Mode
|
||||||
|
2. Falls `expand.depth > 0` gesetzt ist:
|
||||||
|
- Seeds = `chunk_id` (Fallback: `note_id`) der Treffer
|
||||||
|
- Aufruf `graph_adapter.expand(client, prefix, seed_ids, depth, edge_types)`
|
||||||
|
- Ergebnis-Subgraph liefert `edge_bonus` und `centrality_bonus` pro Node
|
||||||
|
3. Scores werden nach obiger Formel kombiniert.
|
||||||
|
|
||||||
|
## 4. Request-Struktur (JSON)
|
||||||
|
|
||||||
|
Beispiel-Hybrid-Request:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mode": "hybrid",
|
||||||
|
"query": "embeddings und qdrant",
|
||||||
|
"top_k": 10,
|
||||||
|
"expand": {
|
||||||
|
"depth": 1,
|
||||||
|
"edge_types": ["references", "depends_on"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Minimaler Semantic-Request:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mode": "semantic",
|
||||||
|
"query": "karate trainingsplan",
|
||||||
|
"top_k": 5
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. Smoke-Test
|
||||||
|
|
||||||
|
Für einen schnellen End-to-End-Test kann folgendes Skript verwendet werden:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python scripts/test_retriever_smoke.py --query "karate" --mode hybrid --expand-depth 1 --top-k 5
|
||||||
|
```
|
||||||
|
|
||||||
|
Das Skript ruft `/query` auf, wertet Statuscode und JSON-Antwort aus und zeigt die Scores der Top-K-Treffer an.
|
||||||
|
|
||||||
|
## 6. Tests
|
||||||
|
|
||||||
|
Die wichtigsten Tests für den Retriever sind:
|
||||||
|
|
||||||
|
- `tests/test_retriever_basic.py` – Basisfunktion (Semantic + Hybrid)
|
||||||
|
- `tests/test_retriever_weight.py` – Einfluss von `retriever_weight` auf das Ranking
|
||||||
|
- `tests/test_retriever_edges.py` – Einfluss von Edge-/Centrality-Boni im Hybrid-Modus
|
||||||
+141
-429
@@ -1,504 +1,216 @@
|
|||||||
---
|
# mindnet – Knowledge Design
|
||||||
title: "mindnet – Knowledge Design"
|
**Version:** 1.5.0 (aktualisiert 2025-11-09)
|
||||||
id: "design-knowledge-architecture"
|
|
||||||
type: "guide"
|
|
||||||
status: "active"
|
|
||||||
version: "1.3.0"
|
|
||||||
created: 2025-09-02
|
|
||||||
updated: 2025-09-09
|
|
||||||
tags: ["type/guide","area/mindnet","topic/obsidian","topic/knowledge-architecture"]
|
|
||||||
owner: "Ich"
|
|
||||||
---
|
|
||||||
|
|
||||||
# mindnet – Knowledge Design
|
> Dieses Dokument beschreibt das Ziel, die Architektur, die Datenmodelle sowie die Regeln und Prozesse für den Aufbau von **mindnet** – einem persönlichen, lokal betriebenen Wissensnetz, das auf Markdown-Quellen, Graph-Beziehungen und Vektor-Suche basiert. Es soll die Persönlichkeit, Werte und Erfahrungen des Besitzers widerspiegeln und künftig Antworten generieren, die diese Perspektive authentisch berücksichtigen.
|
||||||
|
|
||||||
## 1. Gesamtziel, Anspruch & Nutzen
|
|
||||||
|
|
||||||
**Ziel:**
|
|
||||||
*mindnet* ist ein persönliches Wissensnetzwerk, das **alle** relevanten Informationen über mein Leben, meine Werte, Projekte, Erfahrungen, Pläne und Erkenntnisse **strukturiert** erfasst. Es bleibt **portabel** (Markdown + YAML), ist **parser-freundlich**, und so gestaltet, dass Tools (Obsidian + Dataview, Embeddings/Vektorsuche via Qdrant, Agenten) es **maschinenlesbar** nutzen können.
|
|
||||||
|
|
||||||
**Anspruch an die Struktur:**
|
|
||||||
- **Stabil**: robuste IDs, deterministische Dateinamen, konsistente Feldnamen
|
|
||||||
- **Atomar**: eine Kernaussage/ein zentrales Thema pro Datei (keine Sammelcontainer)
|
|
||||||
- **Erweiterbar**: neue Typen jederzeit möglich (ohne Migrationsschmerz)
|
|
||||||
- **Portabel**: pures Markdown + YAML → funktioniert in Obsidian, Git, Parsern
|
|
||||||
- **Vernetzbar**: systematische Links/Backlinks bilden ein navigierbares Wissensnetz
|
|
||||||
- **Parser-freundlich**: wiederkehrende Abschnittsüberschriften, klare Enums
|
|
||||||
|
|
||||||
**Nutzen / Einsatzzwecke:**
|
|
||||||
- **Schnelles Wiederfinden**: Dataview-Abfragen, Filter, Dashboards
|
|
||||||
- **LLM-gestütztes Arbeiten**: Q&A mit Kontext-Zitaten aus Chunks (RAG)
|
|
||||||
- **Agenten**: Vorschläge für Verbindungen/Struktur; Auflösen von „unresolved links“
|
|
||||||
- **Projektnavigation**: Aufgabenbeziehungen, Abhängigkeiten
|
|
||||||
- **Langfristiges Gedächtnis**: Lebens-/Projektchronik (rückblickend und planend)
|
|
||||||
- **Automatisierung**: Vektor-Index (Qdrant), Embeddings, LLM-Q&A, Agenten-Workflows
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 2. YAML-Frontmatter-Schema
|
## 1. Ziel von mindnet
|
||||||
|
|
||||||
> **Pflichtfelder:** `title`, `id`, `type`, `status`, `created`, `tags`
|
**1.1 Zweck**
|
||||||
> **Empfohlen:** `updated`, `area`, `project`, `priority`, `aliases`, `lang`, `people`, `depends_on`, `source` u. a.
|
- mindnet dient als persönliches Wissensnetz, das Erfahrungen, Werte, Prinzipien, Ziele und prägenden Ereignisse langfristig strukturiert.
|
||||||
|
- Es ermöglicht spätere Einsichten in Entscheidungen und Entwicklungen und kann in Zukunft für die Familie (z. B. Kinder) eine nachvollziehbare Quelle sein.
|
||||||
|
|
||||||
### 2.1 Feldübersicht
|
**1.2 Leitidee**
|
||||||
|
- Statt einer reinen Dokumentablage wird Wissen durch **Knoten (Notes/Chunks)** und **Kanten (Edges)** vernetzt.
|
||||||
|
- Antworten sollen **persönlich** kontextualisiert werden (Perspektive, Konditionierung, Persönlichkeit).
|
||||||
|
|
||||||
| Feld | Typ | Bedeutung |
|
**1.3 Langfristigkeit**
|
||||||
|--------------------|-------------------------------|---------------------------------------------------------------------------|
|
- Stabiler, zukunftsfähiger Entwurf; spätere Massenmigrationen sollen vermieden werden.
|
||||||
| `title` | string | Menschlich lesbarer Titel |
|
- Architektur unterstützt schrittweise Erweiterung (neue Quellen, neue Agenten, neue Regeln).
|
||||||
| `id` | string (slug/identifier) | **Stabile** ID; referenzierbar über Wikilinks |
|
|
||||||
| `type` | enum | `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone`, … |
|
|
||||||
| `status` | enum | `draft` \| `active` \| `done` \| `archived` |
|
|
||||||
| `created` | date | Erstellungsdatum (ISO) |
|
|
||||||
| `updated` | date | Letzte Änderung (ISO) |
|
|
||||||
| `area` | string | Themen-/Lebensbereich (z. B. „mindnet“, „gesundheit“, „familie“) |
|
|
||||||
| `project` | string | Projekt-/Arbeitskontext |
|
|
||||||
| `tags` | list<string> | Taxonomische Tags (präfixiert, s. Abschnitt 4) |
|
|
||||||
| `aliases` | list<string> | Alternative IDs/Titel (Slug-Resolver) |
|
|
||||||
| `lang` | string | Sprache, optional (z. B. `de`, `en`) |
|
|
||||||
| `priority` | enum/int | Priorität, optional (`low`/`med`/`high` oder 1–5) |
|
|
||||||
| `due`, `effort_min`| date/int | Für Aufgaben/Projekte (Fälligkeit, Aufwand in Minuten) |
|
|
||||||
| `people` | list<string> | Beteiligte Personen/IDs |
|
|
||||||
| `depends_on` | list<string> | Abhängigkeiten auf andere IDs |
|
|
||||||
| `source` | string | Quelle (Artikel, Buch, Video, …) |
|
|
||||||
| `slug` | string | Stabiler Pfad-/Dateiname, falls abweichend |
|
|
||||||
|
|
||||||
### 2.2 Konventionen
|
|
||||||
- `id` ist **menschenlesbar** (Slug), eindeutig im Vault; `aliases` dienen als Fallback-IDs.
|
|
||||||
- Datumsformat: ISO `YYYY-MM-DD`.
|
|
||||||
- Keine duplizierten Tags; Präfixe nutzen (`area/…`, `type/…`, `topic/…`).
|
|
||||||
- Frontmatter nur Fakten/Metadaten, **keine** ausufernden Inhalte (die stehen in den Body-Abschnitten).
|
|
||||||
|
|
||||||
### 2.3 Minimalbeispiel Frontmatter
|
|
||||||
|
|
||||||
title: "Gedanke: YAML-Standards"
|
|
||||||
id: thought-yaml-standards
|
|
||||||
type: thought
|
|
||||||
status: draft
|
|
||||||
created: 2025-09-02
|
|
||||||
updated: 2025-09-09
|
|
||||||
area: mindnet
|
|
||||||
project: project-mindnet
|
|
||||||
tags: [area/mindnet, type/thought, topic/yaml]
|
|
||||||
aliases: [yaml-gedanken]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
### 2.4 Enum-Werte (Beispiele)
|
|
||||||
- `type`: `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone`, `manifesto`
|
|
||||||
- `status`: `draft`, `active`, `done`, `archived`
|
|
||||||
- `priority`: `low`, `med`, `high` oder 1–5
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. Dateinamen-Konventionen
|
## 2. Zielarchitektur (High Level)
|
||||||
- `YYYY-MM-DD_title.md` oder `slug.md`.
|
|
||||||
- Pfade **relativ** zum Vault; keine absoluten Pfade (einheitliche Slashes `/`).
|
- **Datenquellen:** Markdown-Vault (Obsidian-kompatibel), perspektivisch weitere Quellen (MediaWiki, PDFs, Web-Exporte, E-Mail-Auszüge).
|
||||||
- Verzeichnisse nach eigener Ordnung, empfohlen: `10_thoughts/`, `20_concepts/`, `30_projects/`, `40_journal/`, …
|
- **Import-Pipeline:** Parser → Frontmatter/Body → Chunking → Embedding → Qdrant (Notes/Chunks/Edges).
|
||||||
|
- **Speicher:** Qdrant (Vektoren + Payload + Textindex), Dateien bleiben im Filesystem (Quelle ist Referenz).
|
||||||
|
- **Graph-Schicht:** Edges explizit modelliert (Typ, Richtung, Evidenz, Confidence); optionale abgeleitete Edges per Batch-Job.
|
||||||
|
- **Retriever/Composer:** Hybrid-Retrieval (Vektor + Filter + optional Volltext), Komposition zu Antwortkandidaten.
|
||||||
|
- **Erklärbarkeit:** Begründungspfade (Top-Chunks, Graph-Kette, Provenienz).
|
||||||
|
- **Schnittstellen:** REST/CLI, n8n-Glue, lokale LLM-Anbindung (Ollama).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4. Tagging-Regeln & Taxonomie
|
## 3. Collections in Qdrant
|
||||||
- Präfixe verwenden: `area/…`, `type/…`, `topic/…`, `status/…`.
|
|
||||||
- Möglichst **keine** losen Ein-Wort-Tags ohne Präfix (verhindert Wildwuchs).
|
- `mindnet_notes` – Note-Metadaten (id, title, type, path, timestamps, tags, …).
|
||||||
- Für Dataview konsistente Namespaces beibehalten (z. B. `topic/llm`, `topic/lernen`).
|
- `mindnet_chunks` – inhaltstragende Segmente inkl. Vektoren und Textindex.
|
||||||
|
- `mindnet_edges` – Beziehungen (src/dst, relation, direction, evidence, confidence, …).
|
||||||
|
- (optional) `mindnet_people` – strukturierte Personenobjekte (Eigen-/Fremdbezug).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 5. Backlink- & Link-Regeln
|
## 4. Identitäten & IDs
|
||||||
|
|
||||||
- **Wikilinks**: `[[stabile-id]]` oder `[[Titel|stabile-id]]`
|
- **Deterministische IDs** für Notes (z. B. Namespace-basierte UUIDv5) und stabile, ableitbare IDs für Chunks/Edges.
|
||||||
- Im **Fließtext** verlinken (Forwardlinks); am **Ende** optional **„Mögliche Verbindungen“** redaktionell pflegen:
|
- ID-Stabilität ist Voraussetzung für Idempotenz (Re-Imports ohne Duplikate).
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
|
||||||
- [[concept-persoenliches-leitbild]]
|
|
||||||
- [[project-mindnet]]
|
|
||||||
|
|
||||||
- **Linkrichtung:** „aufwärts“ (Detail → Konzept/Leitbild) und „seitwärts“ (verwandte Knoten).
|
|
||||||
- Optional zusätzliche **ID-Relationen** im YAML (`related_ids`) für Parser.
|
|
||||||
- **Unresolved Links:** Links auf noch nicht existierende Ziele sind zulässig; der Importer schreibt `status: unresolved`-Edges (später durch Resolver/Agent heilbar).
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 5.1 Edge-Typen & Semantik (Graph)
|
## 5. Frontmatter & Typen
|
||||||
|
|
||||||
Beziehungen werden als **Edges** mit neuem Payload-Schema gespeichert:
|
- Pflichtfelder: `id`, `title`, `type`, `created`, `updated`, `tags`, `source_path`.
|
||||||
|
- Note-Typen (Auszug): `experience`, `principle`, `value`, `goal`, `role`, `persona`, `plan`, `project`, `journal`, `reference`, `translation`.
|
||||||
kind : "belongs_to" | "next" | "prev" | "references" | "backlink" | "unresolved"
|
- **types.yaml** definiert Regeln/Defaults je Typ und für Relationstypen (siehe Abschnitt 10).
|
||||||
source_id : ID des Quellknotens (Chunk- oder Note-ID)
|
|
||||||
target_id : ID des Zielknotens (Note- oder Chunk-ID)
|
|
||||||
scope : "chunk" | "note"
|
|
||||||
note_id : Owner-Note des Edges (für schnelles Filtern/Purge)
|
|
||||||
status : optional, z. B. "unresolved"
|
|
||||||
|
|
||||||
**Kantenarten**
|
|
||||||
- **belongs_to** — *Chunk → Note*
|
|
||||||
Jeder Chunk gehört zu **genau einer** Note. *Scope:* `chunk`.
|
|
||||||
|
|
||||||
- **prev / next** — *Chunk → Chunk*
|
|
||||||
Lineare Reihenfolge innerhalb derselben Note; **symmetrisch** angelegt (d. h. `prev` und `next` konsistent). *Scope:* `chunk`.
|
|
||||||
|
|
||||||
- **references** — *Chunk → Note*
|
|
||||||
Aus `[[Wikilinks]]` oder Markdown-Links im **Chunk-Text** (präzise Herkunft). *Scope:* `chunk`.
|
|
||||||
|
|
||||||
- **backlink** — *Note → Note*
|
|
||||||
Automatisch generierte Gegenkante zu `references` (pro Note **dedupliziert**). *Scope:* `note`.
|
|
||||||
|
|
||||||
- **unresolved**
|
|
||||||
Edge mit `status: "unresolved"`, wenn `target_id` (noch) nicht existiert. Wird später vom Resolver/Agent ersetzt.
|
|
||||||
|
|
||||||
- **Optional:** `references:note` — *Note → Note*
|
|
||||||
Zusätzliche Forwardlinks auf Note-Ebene; standardmäßig **deaktiviert**, um Doppelzählungen mit `references` (Chunk-Scope) zu vermeiden. Aktivierung per ENV/Flag möglich.
|
|
||||||
|
|
||||||
**Performanzhinweise (Qdrant)**
|
|
||||||
- Edges werden dedupliziert über `(kind, source_id, target_id, scope)`.
|
|
||||||
- `note_id` ist Pflichtfeld im Edge-Payload (Owner-Note).
|
|
||||||
- Payload-Indizes für Qdrant anlegen:
|
|
||||||
|
|
||||||
edges : kind (KEYWORD), scope (KEYWORD), source_id (KEYWORD), target_id (KEYWORD), note_id (KEYWORD)
|
|
||||||
chunks : note_id (KEYWORD), chunk_index (INTEGER)
|
|
||||||
notes : note_id (KEYWORD)
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 6. Abschnitts-Konventionen im Body
|
## 6. Chunking-Strategie
|
||||||
|
|
||||||
Einheitliche Überschriften erleichtern Parser, Chunking und Abfragen. Typ-spezifische Empfehlungen:
|
- Satz- bis Absatz-Chunks, moderate Überlappung; Meta-Felder verweisen auf Note/Quelle.
|
||||||
|
- Named Vectors (z. B. `content`, optional `title`, `tags`) ermöglichen hybride Scores.
|
||||||
- **Gedanke (`thought`)**
|
|
||||||
`## Zusammenfassung` · `## Begründung / Details` · `## Belege / Quellen` · `## Nächste Schritte` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
- **Erfahrung (`experience`)**
|
|
||||||
`## Kontext` · `## Beobachtung` · `## Interpretation` · `## Implikationen` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
- **Aufgabe (`task`)**
|
|
||||||
`## Checkliste` · `## Notizen` · `## Nächste Schritte` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
- **Projekt (`project`)**
|
|
||||||
`## Scope` · `## Arbeitspakete` · `## Risiken & Gegenmaßnahmen` · `## Status / Fortschritt` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
- **Journal (`journal`)**
|
|
||||||
`## Tageszusammenfassung` · `## Highlights` · `## Learnings` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
- **Meeting (`meeting`)**
|
|
||||||
`## Agenda` · `## Notizen` · `## Entscheidungen` · `## Aufgaben` · `## Mögliche Verbindungen`
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 7. Abbildung persönlicher Inhalte auf Typen
|
## 7. Embeddings & Vektoren
|
||||||
|
|
||||||
**Richtlinien:**
|
- Einheitliche Metrik (cosine), reproduzierbares Embedding-Setup.
|
||||||
- Wähle den Typ nach **Hauptintention** (Definition, Gedanke, Beobachtung, Aufgabe, Projekt, …).
|
- Konsistenz über Zeit durch Versionsfeld (`embedding_model`, `embedding_version`).
|
||||||
- **Konzepte** (`concept`): langlebige Definitionen, Prinzipien, Leitbilder.
|
|
||||||
- **Gedanken** (`thought`): explorative Ideen, noch nicht stabil (können später bei Reife in `concept` überführt werden).
|
|
||||||
- **Erfahrungen** (`experience`): Beobachtungen & Learnings aus der Praxis.
|
|
||||||
- **Aufgaben** (`task`): konkrete, umsetzbare To-dos mit `due`/`effort_min`.
|
|
||||||
- **Projekte** (`project`): Ziele, Deliverables, Arbeitspakete, Abhängigkeiten.
|
|
||||||
- **Journal** (`journal`): tagesbasierte Notizen, Reflexionen, Reiseberichte.
|
|
||||||
- **Personen** (`person`): Kontakte, Profile, Bezüge, Gespräche.
|
|
||||||
- **Meetings** (`meeting`): Agenda, Notizen, Entscheidungen, Folgeaufgaben.
|
|
||||||
- **Meilensteine** (`milestone`): wichtige Wendepunkte im Lebens-/Projektverlauf.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 8. Beispiel-Notizen
|
## 8. Volltext & Filter
|
||||||
|
|
||||||
### 8.1 Gedanke (`thought`) – Beispiel
|
- Textindex auf `mindnet_chunks.body` für exakte Term/Operator-Queries.
|
||||||
|
- Payload-Filter zur schnellen Einschränkung (tags, types, time-buckets, visibility u. a.).
|
||||||
**Frontmatter**
|
|
||||||
|
|
||||||
title: "Gedanke: Übungen für tiefe Arbeit"
|
|
||||||
id: thought-deep-work-uebungen
|
|
||||||
type: thought
|
|
||||||
status: active
|
|
||||||
created: 2025-08-21
|
|
||||||
updated: 2025-09-05
|
|
||||||
area: lernen
|
|
||||||
project: project-self-improvement
|
|
||||||
tags: [area/lernen, type/thought, topic/fokus]
|
|
||||||
aliases: [deep-work-ideen]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
**Body (Ausschnitt)**
|
|
||||||
|
|
||||||
## Zusammenfassung
|
|
||||||
Kurze tägliche Routinen (z. B. Timer-Blöcke, Ablenkungslog) erhöhen meinen Fokus.
|
|
||||||
|
|
||||||
## Begründung / Details
|
|
||||||
- 25–45 Min Timerblöcke mit 5 Min Pause
|
|
||||||
- Ablenkungen sofort notieren
|
|
||||||
- Abschluss-Log am Ende des Tages
|
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
|
||||||
- [[concept-deep-work]]
|
|
||||||
- [[project-productivity-week]]
|
|
||||||
|
|
||||||
### 8.2 Konzept (`concept`) – Beispiel
|
|
||||||
|
|
||||||
**Frontmatter**
|
|
||||||
|
|
||||||
title: "Konzept: Deep Work"
|
|
||||||
id: concept-deep-work
|
|
||||||
type: concept
|
|
||||||
status: active
|
|
||||||
created: 2024-05-14
|
|
||||||
updated: 2025-08-01
|
|
||||||
area: lernen
|
|
||||||
tags: [area/lernen, type/concept, topic/fokus]
|
|
||||||
aliases: [konzept-tiefe-arbeit]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
**Body (Ausschnitt)**
|
|
||||||
|
|
||||||
## Definition
|
|
||||||
Tiefe Arbeit = kognitiv anspruchsvolle Arbeit ohne Ablenkung.
|
|
||||||
|
|
||||||
## Belege / Quellen
|
|
||||||
- Buch: Cal Newport – Deep Work
|
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
|
||||||
- [[thought-deep-work-uebungen]]
|
|
||||||
|
|
||||||
### 8.3 Aufgabe (`task`) – Beispiel
|
|
||||||
|
|
||||||
**Frontmatter**
|
|
||||||
|
|
||||||
title: "Task: Wochenplanung erstellen"
|
|
||||||
id: task-week-planning
|
|
||||||
type: task
|
|
||||||
status: active
|
|
||||||
created: 2025-09-07
|
|
||||||
updated: 2025-09-07
|
|
||||||
due: 2025-09-11
|
|
||||||
effort_min: 45
|
|
||||||
area: organisation
|
|
||||||
project: project-productivity-week
|
|
||||||
tags: [area/organisation, type/task, topic/planung]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
**Body (Ausschnitt)**
|
|
||||||
|
|
||||||
## Checkliste
|
|
||||||
- Ziele der Woche definieren
|
|
||||||
- 5 Fokusblöcke eintragen
|
|
||||||
- 2 Lerneinheiten terminieren
|
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
|
||||||
- [[project-productivity-week]]
|
|
||||||
- [[thought-deep-work-uebungen]]
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 9. Schrittweise Migration bestehender freier Notizen
|
## 9. Edges – Beziehungen
|
||||||
|
|
||||||
1. **Frontmatter ergänzen** (min. `id`, `title`, `type`, `status`, `created`, `tags`).
|
- **Explizit**: Jede tatsächliche Beziehung wird als Kante gespeichert (kein Workaround).
|
||||||
2. **Abschnittsstruktur** an Typ anpassen (Konventionen s. Abschnitt 6).
|
- Felder u. a.: `src_id`, `dst_id`, `relation`, `direction`, `confidence (0–1)`, `evidence` (Chunk-IDs), `created_at`, `updated_at`.
|
||||||
3. **Wikilinks** im Body setzen (statt bloßer Erwähnungen).
|
- **edge_defaults** liefern **Interpretations-Hints** (Gewichte, Symmetrie, Transitivität) und parametrisieren abgeleitete Edges; sie **ersetzen** keine Originalkanten.
|
||||||
4. **Tags normalisieren** (Präfixe, Kleinschreibung, keine Dubletten).
|
|
||||||
5. **Aliases** angeben, falls Notiz zuvor unter anderem Namen existierte.
|
|
||||||
6. **Testlauf**: Import (Dry-Run), danach Apply, Audit/Validate prüfen.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 10. Dataview-Beispiele (direkt nutzbar)
|
## 10. Regeln & Policies (types.yaml)
|
||||||
|
|
||||||
**Aktive Gedanken der letzten 30 Tage**
|
- Per Typ/Relation:
|
||||||
|
- Default-Eigenschaften (Gewicht, Directedness, Symmetrie, Transitivität).
|
||||||
TABLE title, created, updated, area
|
- Scoring-Hooks für Graph-Kontext im Retriever.
|
||||||
FROM "10_thoughts"
|
- (später) Privacy-Hooks je Relation (Mindest-Visibility).
|
||||||
WHERE type = "thought" AND status = "active" AND created >= date(today) - dur(30 days)
|
|
||||||
SORT updated DESC
|
|
||||||
|
|
||||||
**Offene Tasks mit Fälligkeit**
|
|
||||||
|
|
||||||
TABLE title, due, effort_min, project, area
|
|
||||||
FROM "30_tasks"
|
|
||||||
WHERE type = "task" AND status = "active" AND due
|
|
||||||
SORT due ASC
|
|
||||||
|
|
||||||
**Konzepte je Topic**
|
|
||||||
|
|
||||||
TABLE title, area, tags
|
|
||||||
FROM "20_concepts"
|
|
||||||
WHERE contains(tags, "type/concept")
|
|
||||||
GROUP BY choice(regexreplace(join(filter(tags, (t) => startswith(t, "topic/")), ", "), "^topic/", ""))
|
|
||||||
|
|
||||||
**Verknüpfungskandidaten (redaktionell, nicht automatisch)**
|
|
||||||
*(Liste am Ende der jeweiligen Notiz in „Mögliche Verbindungen“)*
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 11. Validierung & Pflege
|
## 11. Retrieval & Ranking (Komposition)
|
||||||
|
|
||||||
- **Linter/Validator** prüfen Pflichtfelder und Wertebereiche (Enums, ISO-Datum).
|
- Hybrid: Vektor-Score + Filter + optional BM25/Volltext.
|
||||||
- **Resolver** löst `unresolved`-Edges, wenn neue Notizen/Aliases entstehen.
|
- Zeitgewicht (Decays) und Graph-Kontext werden komponiert (Formel siehe 16.3).
|
||||||
- **Idempotente Re-Imports**: Hash-Steuerung konfigurierbar:
|
- Konfiguration über `.env`/`config.yaml` (gewichts- und halbwertszeit-basiert).
|
||||||
- `MINDNET_HASH_MODE`: `body` \| `frontmatter` \| `body+frontmatter` (Default: `body`)
|
|
||||||
- `MINDNET_HASH_NORMALIZE`: `canonical` \| `none` (Default: `canonical`)
|
|
||||||
- **Edge-Invarianten** (Audit):
|
|
||||||
- `belongs_to == #Chunks`
|
|
||||||
- `next == prev == Σ(max(chunks_in_note−1, 0))`
|
|
||||||
- `references (chunk) == Σ(Chunk-Wikilinks)`
|
|
||||||
- `backlink == Σ(eindeutige Wikilinks je Note)`
|
|
||||||
- **Roundtrip-Test**: Import → Export → Diff muss Body verlustfrei rekonstruieren.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 12. Qdrant-Ablage & Indizes
|
## 12. Erklärbarkeit
|
||||||
|
|
||||||
**Collections**
|
- „Warum-Pfad“: Top-Chunks mit Score-Zerlegung (Vektor/Time/Graph), Graph-Kette, Provenienz.
|
||||||
- `{prefix}_notes` – Vektor optional (Nullvektor oder Zentroid), Payload inkl. Frontmatter & `fulltext`
|
- Mehrschichtige Narrative (Kindheit/Erlebnisse, Werte/Prinzipien, Ziele/Leitbild, Gegenwart).
|
||||||
- `{prefix}_chunks` – 384d Embedding je Chunk, Payload inkl. `text`, `chunk_index`, `note_id`
|
|
||||||
- `{prefix}_edges` – 1d Dummy-Vektor, Payload im neuen Schema (s. 5.1)
|
|
||||||
|
|
||||||
**IDs & Idempotenz**
|
|
||||||
- Deterministisch/UUIDv5 über stabile Schlüssel (`note_id`, `chunk_id = note_id#cNN`, `edge_id` aus `(kind, source_id, target_id, scope)`)
|
|
||||||
- Re-Imports erzeugen keine Duplikate (Upsert)
|
|
||||||
|
|
||||||
**Payload-Indizes (Performance)**
|
|
||||||
|
|
||||||
notes : note_id (KEYWORD)
|
|
||||||
chunks : note_id (KEYWORD), chunk_index (INTEGER)
|
|
||||||
edges : kind (KEYWORD), scope (KEYWORD), source_id (KEYWORD), target_id (KEYWORD), note_id (KEYWORD)
|
|
||||||
|
|
||||||
**Delete/Purge**
|
|
||||||
- Selektives Löschen in Edges/Chunks per `note_id`-Filter (schnell, sicher).
|
|
||||||
- `--sync-deletes` (geplant): Qdrant→Vault Diffs gezielt entfernen (mit Dry-Run).
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 13. LLM/RAG & Agenten – Arbeitsweise (Überblick)
|
## 13. Betrieb & Sicherheit
|
||||||
|
|
||||||
**Retriever (Hybrid)**
|
- Lokal auf Single-Node starten (Backups via Snapshots).
|
||||||
1) **kNN (Chunks)** in Qdrant → Top-k Kandidaten (Semantik).
|
- Zugriff nur intern/Reverse-Proxy; Festplattenverschlüsselung OS-seitig.
|
||||||
2) **Nachbarschafts-Expand** (1 Hop) über Edges (`belongs_to`, `references`, `backlink`) → zusätzliche Kontexte.
|
- Migrationspfad zu Qdrant-Distributed (Shards, Replikation) einplanen.
|
||||||
3) **Rerank** (optional BM25/LLM-score).
|
|
||||||
4) **Prompt** mit Zitaten/Belegen (Verweis auf `path` + Abschnittstitel).
|
|
||||||
|
|
||||||
**Agentisches Einordnen (neue Inputs)**
|
|
||||||
- **Chunking + Embedding** → **kNN-Kandidaten**
|
|
||||||
- Klassifikation (`type`, `area`, `project`) via Heuristik/LLM
|
|
||||||
- Edge-Vorschläge: `references` (Semantik + geteilte Tags/Area), `belongs_to` (beste Einordnung)
|
|
||||||
- Falls kein Ziel existiert → `unresolved` erzeugen oder **Stub-Note** anlegen
|
|
||||||
- **Human-in-the-loop**: Vorschläge bestätigen, dann Commit via Importer
|
|
||||||
|
|
||||||
**Graph-Layer (optional, empfohlen)**
|
|
||||||
- Zusätzliche SQLite/Postgres-Tabellen `nodes/edges/aliases` für 2–3-Hop-Queries, Pfadsuche, Zählungen.
|
|
||||||
- Importer pflegt Qdrant **und** SQL (UPSERT).
|
|
||||||
- Graph-Features (Degree, Communities, optional PageRank) regelmäßig berechnen und als Felder in `notes.payload` speichern.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 14. Versions- & Änderungsrichtlinien
|
## 14. Qualität & Monitoring
|
||||||
|
|
||||||
- **Kleine Änderungen** direkt in der bestehenden Notiz (Frontmatter `updated` anpassen).
|
- Offline-Eval-Sets, Telemetrie (lokal), Drift-Checks, manuelle Reviews.
|
||||||
- **Große Strukturänderungen** (z. B. neuer Typ) als neue Notiz mit Verweis; alte Note verlinken/archivieren (`status: archived`).
|
- Kennzahlen: Hit-Rate, Overlap, Quellen-Alter, Erklärungspfad-Nutzung.
|
||||||
- **IDs nie recyceln** (Konstanz für Referenzen und Edges).
|
|
||||||
- **Aliases** nutzen, wenn sich Titel/Slug ändert (Resolver kann beides auflösen).
|
|
||||||
- **Commit-Meldungen** im Git prägnant strukturieren:
|
|
||||||
- `feat(note): …` · `fix(frontmatter): …` · `chore(tags): …` · `refactor(structure): …`
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 15. Typen-Kurzreferenz
|
## 15. Offene Punkte
|
||||||
|
|
||||||
- **concept** — Definitionen, Leitbilder, Prinzipien
|
- Ausdetaillierung `types.yaml` (Relationen, Hooks).
|
||||||
Empfohlene Abschnitte: *Definition*, *Beispiele*, *Belege*, *Mögliche Verbindungen*
|
- UI-Oberfläche für Erklärpfade und Provenienz.
|
||||||
|
- Batch-Jobs für abgeleitete Edges.
|
||||||
- **thought** — Ideen, Entwürfe, Skizzen
|
|
||||||
Abschnitte: *Zusammenfassung*, *Details*, *Nächste Schritte*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **experience** — Beobachtungen, Learnings
|
|
||||||
Abschnitte: *Kontext*, *Beobachtung*, *Interpretation*, *Implikationen*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **task** — konkrete To-dos
|
|
||||||
Felder: `due`, `effort_min`; Abschnitte: *Checkliste*, *Notizen*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **project** — Ziele, Deliverables, Arbeitspakete
|
|
||||||
Abschnitte: *Scope*, *Arbeitspakete*, *Risiken*, *Status*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **journal** — tagesbasierte Einträge
|
|
||||||
Abschnitte: *Tageszusammenfassung*, *Highlights*, *Learnings*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **person** — Kontakte, Profile
|
|
||||||
Felder: `people` (Selbst-/Fremd-ID), *Notizen*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **meeting** — Termine, Protokolle
|
|
||||||
Abschnitte: *Agenda*, *Notizen*, *Entscheidungen*, *Aufgaben*, *Mögliche Verbindungen*
|
|
||||||
|
|
||||||
- **milestone** — Wendepunkte
|
|
||||||
Abschnitte: *Kontext*, *Bedeutung*, *Verknüpfungen*
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 16. Erweiterungen
|
## 16. Ergänzungen 2025-11-09 (v1.5.0)
|
||||||
|
|
||||||
- Neue Typen (z. B. `decision`, `recommendation`, `resource`) können ohne Bruch ergänzt werden.
|
**Kontext:** Basierend auf den neuen Rahmenbedingungen (nur Deutsch; perspektivische externe Abfragen; einheitliche Vertraulichkeit mit Option auf feinere Stufen; Recency-Priorisierung als Default; Regeln später konfigurierbar; zunächst lokale Speicherung mit optionaler Migration; heutige Hardware, später skalierbar; hohe Erklärbarkeit mit mehreren Erklärungsebenen) werden folgende Entscheidungen und Erweiterungen getroffen. Alle bisherigen Inhalte bleiben gültig und werden **nicht** verändert.
|
||||||
- Personalisierung durch zusätzliche Properties (`values`, `goals`) in Frontmatter.
|
|
||||||
- Internationale Inhalte: `lang` pflegen, optional **mehrsprachige** Embeddings (z. B. mE5).
|
|
||||||
|
|
||||||
---
|
### 16.1 Sprach- und Inhaltsrichtlinie (nur Deutsch)
|
||||||
|
- `lang: "de"` als verpflichtendes Feld in allen Note-, Chunk- und Edge-Payloads.
|
||||||
|
- Für spätere Mehrsprachigkeit **nicht** die Collection splitten, sondern ein Feld `lang` + optional `lang_original` behalten; Translation-Artefakte als eigener `type: translation` (verweisend auf `source_id`).
|
||||||
|
|
||||||
## 17. Anhang – Beispiele & Snippets
|
### 16.2 Vertraulichkeit & Sichtbarkeit
|
||||||
|
- Heute: einheitliche Stufe `visibility: "internal"`.
|
||||||
|
- Vorausschauend: reservierte Felder in allen Payloads
|
||||||
|
- `visibility`: `"public" | "internal" | "restricted"`
|
||||||
|
- `sensitivity`: Integer 0–3 (0 = unkritisch, 3 = hochsensibel)
|
||||||
|
- `acl`: Liste erlaubter Rollen/Personen-IDs (App-Layer erzwingt Filter vor Query).
|
||||||
|
- Access-Control erfolgt **im Applikations-Layer** mittels Qdrant-Payload-Filtern (`must: [ { key: "visibility", match: { any: [...] }}, ... ]`); Qdrant bleibt „dumm“ in Bezug auf ACL.
|
||||||
|
|
||||||
**Beispiel: YAML-Frontmatter für Projekt**
|
### 16.3 Recency-Modell & Ranking (konfigurierbar)
|
||||||
|
- Default: **Zeitnähe** priorisieren (sanfte Abwertung älterer Inhalte).
|
||||||
|
- Im Retrieval kombinieren wir:
|
||||||
|
1) Vektor-Score (cosine) aus Qdrant
|
||||||
|
2) Zeitgewicht `time_decay = exp( - ln(2) * age_days / half_life_days )`
|
||||||
|
3) Graph-Kontext aus Edges (z. B. Degree, Path-Hops zum Anker „Ich/Persona“)
|
||||||
|
- Konfigurierbare Gewichte (per `.env` oder `config.yaml`):
|
||||||
|
```text
|
||||||
|
FINAL_SCORE = w_vec * norm(vec_score)
|
||||||
|
+ w_time * time_decay
|
||||||
|
+ w_graph * graph_context_score
|
||||||
|
# Vorschlag: w_vec=0.70, w_time=0.20, w_graph=0.10; half_life_days=365
|
||||||
|
```
|
||||||
|
- Für Fragen, die „zeitlos“ sind, kann `half_life_days = inf` gesetzt werden (kein Decay).
|
||||||
|
|
||||||
title: "Projekt: Produktivität Woche 37"
|
### 16.4 Erklärbarkeit („Warum diese Antwort?“)
|
||||||
id: project-productivity-week
|
- Jeder Antwort wird ein **Erklärpfad** mitgegeben (optional im UI):
|
||||||
type: project
|
- Top-Chunks (mit Score-Zerlegung w_vec/w_time/w_graph)
|
||||||
status: active
|
- **Kausalpfad** im Graphen: `Note → Edge(relation, confidence) → Note …`
|
||||||
created: 2025-09-08
|
- **Provenienz** (Quelle, Erstellungszeit, letzte Änderung).
|
||||||
updated: 2025-09-09
|
- Ein **Multi-Layer-Explanations**-Modul ordnet Beiträge zu: Kindheit/Erlebnisse, Werte & Prinzipien, aktuelle Ziele/Leitbild, Gegenwartskontext.
|
||||||
area: organisation
|
|
||||||
tags: [area/organisation, type/project, topic/planung]
|
|
||||||
people: [ich]
|
|
||||||
depends_on: [concept-deep-work]
|
|
||||||
lang: de
|
|
||||||
|
|
||||||
**Beispiel: Redaktions-Sektion (keine Kanten-Erzeugung)**
|
### 16.5 Qdrant-Schemata (Ergänzungen, rückwärtskompatibel)
|
||||||
|
- **Collections** bleiben: `mindnet_notes`, `mindnet_chunks`, `mindnet_edges` (+ `mindnet_people` falls noch nicht vorhanden).
|
||||||
|
- Gemeinsame Zusatzfelder (alle Payloads):
|
||||||
|
- `lang: "de"`
|
||||||
|
- `visibility`/`sensitivity`/`acl` (s. 16.2)
|
||||||
|
- `created_at` (ISO-8601), `updated_at` (ISO-8601), `age_days` (App-seitig berechnet)
|
||||||
|
- **Chunks**:
|
||||||
|
- Named-Vectors (z. B. `"content"`, optional `"title"`, `"tags"` für Hybrid-Scores).
|
||||||
|
- `text_index` auf `body` für BM25-ähnliche Volltextsuche (Qdrant-Textindex).
|
||||||
|
- `recency_group`: Jahr-Monat `YYYY-MM` (ermöglicht Pre-Filter für „letzte N Monate“).
|
||||||
|
- **Edges** (explizit, keine Workarounds):
|
||||||
|
- Felder: `src_id`, `dst_id`, `relation`, `direction`, `confidence` (0–1), `evidence` (Chunk-IDs), `created_at`, `updated_at`.
|
||||||
|
- **edge_defaults** bleiben als **Interpretations-Hints**: z. B. `default_weight`, `symmetry`, `transitivity`. Sie ersetzen **nicht** echte Kanten, sondern parametrisieren abgeleitete/zusätzliche Kantenbildung.
|
||||||
|
- Abgeleitete Edges: optionaler Batch-Job erzeugt *ergänzende* Kanten entlang `types.yaml`-Regeln (z. B. Backlinks, „inspired_by“ aus Zitaten), niemals statt der Originalkanten.
|
||||||
|
|
||||||
## Mögliche Verbindungen
|
### 16.6 Regeln & Typen (konfigurierbar)
|
||||||
- [[concept-wochenplanung]]
|
- `types.yaml` erweitert um:
|
||||||
- [[thought-deep-work-uebungen]]
|
- Relationstypen mit Default-Eigenschaften (`weight`, `directed`, `symmetric`, `transitive`).
|
||||||
- [[journal-2025-09-08]]
|
- **Scoring-Hooks**: pro Relation optionaler Einfluss auf `graph_context_score`.
|
||||||
|
- **Privacy-Hooks**: pro Relation Minimal-Visibility (z. B. Beziehungen zu Personen sind mindestens `internal`).
|
||||||
|
|
||||||
**CLI: Import & Audit**
|
### 16.7 Speicher- & Betriebsstrategie (heute lokal, später skalierbar)
|
||||||
|
- **Heute**: Single-Node Qdrant mit HNSW-Index, Text-Index und Payload-Filtern; regelmäßiges Backup (Snapshots).
|
||||||
|
- **Morgen**: Migrationspfad in **Qdrant-Distributed** (Shards, ggf. Replikation) bei Bedarf; Parameter-Gleichheit sichern (named-vectors, Metrik, HNSW-Param).
|
||||||
|
- **Sicherheit**: Zugriff nur über internes Netz/Reverse-Proxy, Auth vor Qdrant; Festplatten-Verschlüsselung auf OS-Ebene (z. B. LUKS).
|
||||||
|
|
||||||
# Dry-Run Import
|
### 16.8 Persona-Layer („meine Essenz“)
|
||||||
python3 -m scripts.import_markdown --vault ./vault
|
- Eigenständiger **Persona-Knoten** (z. B. Note `person/lars`), verbunden mit:
|
||||||
|
- Werten/Prinzipien, Leitbild, prägenden Erlebnissen, Zielen, Rollen.
|
||||||
|
- Jede Antwort kann gegen diesen Knoten **kontextualisiert** werden (Graph-Pfadnähe).
|
||||||
|
- Feld `persona_weight` in Chunks/Notes zur Priorisierung persönlicher Relevanz.
|
||||||
|
|
||||||
# Apply mit strenger Hash-Erkennung (jede Änderung zählt)
|
### 16.9 Evaluierung & Qualitätssicherung
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --hash-normalize none
|
- **Offline-Sets** aus echten Fragen (inkl. gewünschter Perspektiven) mit erwarteten Top-Quellen.
|
||||||
|
- **Telemetrie** (lokal): Query-Arten, Treffer-Overlaps, Zeitgewicht-Effekte, „Warum-Pfad“ genutzt/ignoriert.
|
||||||
|
- **Drift-Checks**: Anteil sehr alter vs. neuer Quellen in Antworten; manuelle Reviews.
|
||||||
|
|
||||||
# Audit Edges vs. Erwartungen
|
### 16.10 Offene Punkte / Nächste Schritte
|
||||||
python3 -m scripts.audit_edges_vs_expectations --vault ./vault
|
1) `types.yaml` um Relationseigenschaften erweitern (s. 16.6).
|
||||||
|
2) Retriever-Komposition inkl. Zeitgewicht (s. 16.3) implementieren.
|
||||||
---
|
3) Explanations-Pfad inkl. Provenienzoberfläche (s. 16.4) im UI.
|
||||||
|
4) Feld-Erweiterungen in Import-Pipelines (`lang`, `visibility`, Zeitfelder).
|
||||||
## 18. Glossar
|
5) Optional: Batch-Job für abgeleitete Edges nach Regeln (ergänzend, nicht ersetzend).
|
||||||
|
|
||||||
- **Alias**: alternativer Name/Slug, der auf dieselbe Note verweist
|
|
||||||
- **Backlink**: automatisch erzeugte Gegenkante zu `references` (Note-Scope)
|
|
||||||
- **Chunk**: semantisch sinnvoller Textabschnitt einer Note, inkl. Embedding
|
|
||||||
- **Owner-Note**: Note, zu der ein Edge gehört (`note_id` im Edge-Payload)
|
|
||||||
- **Resolver**: Prozess/Script, das `unresolved`-Edges bei späterer Ziel-Existenz heilt
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 19. Verweise
|
|
||||||
|
|
||||||
- `chunking_strategy.md` – Details zu Chunkgrößen, Overlaps, Abschnittsregeln
|
|
||||||
- `schemas/note.schema.json` – Validierung von Frontmatter‐Feldern
|
|
||||||
- Import/Export-Skripte: `scripts/import_markdown.py`, `scripts/export_markdown.py`
|
|
||||||
- Qdrant-Client/Setup: `app/core/qdrant.py`, `app/core/qdrant_points.py`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 20. Änderungsverlauf
|
|
||||||
|
|
||||||
- **1.3.0 (2025-09-09)**
|
|
||||||
- Edge-Politik präzisiert: `references` (Chunk-Scope), `backlink` (Note-Scope), `references:note` optional.
|
|
||||||
- `note_id` als Pflichtfeld im Edge-Payload (Owner).
|
|
||||||
- Qdrant-Payload-Indizes ergänzt (Performanz).
|
|
||||||
- Beispiele & Dataview-Snippets erweitert.
|
|
||||||
|
|
||||||
- **1.2.0 (2025-09-02)**
|
|
||||||
- Erstfassung des Knowledge-Designs.
|
|
||||||
@@ -0,0 +1,294 @@
|
|||||||
|
# Mindnet v2.2 – Fachliche Architektur
|
||||||
|
**Datei:** `docs/mindnet_functional_architecture_v2.2.md`
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** **FINAL** (Integrierter Stand WP01–WP05)
|
||||||
|
|
||||||
|
> Dieses Dokument beschreibt **was** Mindnet fachlich tut und **warum** – mit Fokus auf die Erzeugung und Nutzung von **Edges** (Kanten), die Logik des Retrievers und den neuen **RAG-Chat** (Persönlichkeit). Die technische Umsetzung wird im technischen Dokument detailliert.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0) Zielbild & Grundprinzip
|
||||||
|
|
||||||
|
Mindnet wandelt Obsidian-Markdown-Notizen in einen **dynamischen Graphen aus Punkten und Kanten** um.
|
||||||
|
Die drei zentralen Artefakt-Sammlungen lauten:
|
||||||
|
|
||||||
|
- `mindnet_notes` – genau **eine** Note pro Markdown-Datei (Metadaten & Hashes)
|
||||||
|
- `mindnet_chunks` – semantische Teilstücke einer Note (Fenster/„Chunks“)
|
||||||
|
- `mindnet_edges` – gerichtete Beziehungen zwischen Knoten (Chunks/Notes)
|
||||||
|
|
||||||
|
Die Import-Pipeline erzeugt diese Artefakte **deterministisch** und **idempotent** (erneute Läufe überschreiben konsistent statt zu duplizieren). Die Import-Schritte sind: *parse → chunk → edge → upsert*.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1) Notizen & Chunks (fachliche Perspektive)
|
||||||
|
|
||||||
|
### 1.1 Notiz (Note)
|
||||||
|
- Repräsentiert eine fachliche Einheit (z. B. *„Vector DB Basics“*, *„KI-Projekt“*).
|
||||||
|
- Trägt Eigenschaften (Titel, Typ, Zeitstempel, optionale Policies).
|
||||||
|
- **Typ** (*type*) steuert u. a. Chunk-Profil und Default-Relationen (siehe §4).
|
||||||
|
- **retriever_weight** und **chunk_profile** werden systematisch an Note **und** Chunks gespiegelt, damit der Retriever beides nutzen kann.
|
||||||
|
|
||||||
|
### 1.2 Chunk
|
||||||
|
- Ausschnitt/Textfenster aus der Note, als eigenständiger Such-Anker.
|
||||||
|
- Jeder Chunk gehört **genau einer** Note.
|
||||||
|
- Chunks bilden eine Sequenz (1…N) – das ermöglicht *next/prev*.
|
||||||
|
- **Neu in v2.2:** Alle Kanten entstehen ausschließlich zwischen Chunks (Scope="chunk"), nie zwischen Notes direkt. Notes dienen nur noch als Metadatencontainer.
|
||||||
|
|
||||||
|
> **Wichtig:** Chunking-Profile (short/medium/long) kommen aus `types.yaml` (per Note-Typ), können aber lokal überschrieben werden. Die effektiven Werte werden bei der Payload-Erzeugung bestimmt.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2) Edges – fachliche Typen & Bedeutung
|
||||||
|
|
||||||
|
Edges kodieren Beziehungen. Sie sind **gerichtet** und werden in `mindnet_edges` gespeichert. Die Erzeugung folgt einer strengen Priorität: **Inline > Callout > Defaults > Struktur**.
|
||||||
|
|
||||||
|
### 2.1 Struktur-Kanten (Das Skelett)
|
||||||
|
- **belongs_to**: *Chunk → Note* (Zuordnung)
|
||||||
|
- **next / prev**: Kettenbeziehungen zwischen aufeinanderfolgenden Chunks derselben Note.
|
||||||
|
→ ermöglichen „lineares Weiterlesen“ oder Kontextpfade in Antworten.
|
||||||
|
Diese Kanten entstehen immer, unabhängig von Inhalten.
|
||||||
|
|
||||||
|
### 2.2 Inhalts-Kanten (explizit)
|
||||||
|
Hier unterscheidet v2.2 präzise zwischen verschiedenen Quellen der Evidenz:
|
||||||
|
|
||||||
|
1. **Explizite Inline-Relationen (Höchste Priorität):**
|
||||||
|
Im Fließtext notierte, semantisch qualifizierte Relationen.
|
||||||
|
* Syntax: `[[rel:depends_on Embeddings 101]]`
|
||||||
|
* Fachliche Aussage: "Hängt ab von", "Ähnlich zu".
|
||||||
|
* *rule_id*: `inline:rel`
|
||||||
|
* *confidence*: ~0.95
|
||||||
|
|
||||||
|
2. **Callout-Edges (Kuratierte Listen):**
|
||||||
|
Redaktionell gepflegte Listen am Ende einer Notiz (z.B. "Siehe auch").
|
||||||
|
* Syntax: `> [!edge] related_to: [[Vector DB Basics]]`
|
||||||
|
* *rule_id*: `callout:edge`
|
||||||
|
* *confidence*: ~0.90
|
||||||
|
|
||||||
|
3. **Wikilinks (Standard-Referenzen):**
|
||||||
|
Der klassische Obsidian-Link.
|
||||||
|
* Syntax: `[[Vector DB Basics]]`
|
||||||
|
* Typ: `references`
|
||||||
|
* *rule_id*: `explicit:wikilink`
|
||||||
|
* *confidence*: 1.0
|
||||||
|
|
||||||
|
> Alle expliziten Quellen (Wikilink, Inline-Rel, Callout) sind **Primärbelege**.
|
||||||
|
|
||||||
|
### 2.3 Typ-basierte Default-Kanten (Regelbasiert)
|
||||||
|
Jede Note hat einen *Typ* (z. B. `concept`, `project`, `profile`). In `config/types.yaml` definieren wir pro Typ **edge_defaults**, z. B.:
|
||||||
|
|
||||||
|
- `concept`: `["references","related_to"]`
|
||||||
|
- `project`: `["references","depends_on"]`
|
||||||
|
|
||||||
|
Regel: **Für jede gefundene explizite Referenz** (s. o.) werden **zusätzliche** Edges nach diesen Defaults erzeugt.
|
||||||
|
Beispiel: Ein *project* mit `edge_defaults=["depends_on"]` erzeugt zu *jedem* explizit referenzierten Ziel **zusätzlich** eine `depends_on`-Kante.
|
||||||
|
Diese Kanten tragen *provenance=rule* und eine **rule_id** der Form `edge_defaults:{note_type}:{relation}` sowie eine geringere Confidence (~0.7).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3) Edge-Payload – Felder & Semantik
|
||||||
|
|
||||||
|
Jede Kante hat mindestens:
|
||||||
|
|
||||||
|
- `kind` – Beziehungsart *(belongs_to, next, prev, references, related_to, depends_on, similar_to, …)*
|
||||||
|
- `scope` – `"chunk"` (Standard in v2.2)
|
||||||
|
- `source_id`, `target_id` – Quell-/Ziel-Knoten (Chunk-IDs oder Note-Titel bei unresolved Targets)
|
||||||
|
- `note_id` – **Owner-Note** (die Note, aus der die Kante stammt)
|
||||||
|
|
||||||
|
Erweiterte/abgeleitete Felder (WP03 Superset):
|
||||||
|
|
||||||
|
- `provenance` – `"explicit"` (Wikilink/Inline/Callout) oder `"rule"` (Typ-Defaults)
|
||||||
|
- `rule_id` – maschinenlesbare Regelquelle (z.B. `inline:rel`, `edge_defaults:project:depends_on`)
|
||||||
|
- `confidence` – 0.0–1.0; Heuristik zur Gewichtung im Scoring.
|
||||||
|
|
||||||
|
**Dedup-Schlüssel** (fachlich): Gleichlautende `(source_id, target_id, kind, scope, rule_id)` werden nicht mehrfach geschrieben.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4) Typ-Registry (`config/types.yaml`)
|
||||||
|
|
||||||
|
### 4.1 Zweck
|
||||||
|
- Steuert **Chunking-Profile** (`short|medium|long`) **pro Typ**
|
||||||
|
- Liefert **retriever_weight** (Note-/Chunk-Gewichtung im Ranking) **pro Typ**
|
||||||
|
- Definiert **edge_defaults** je Typ (s. o.)
|
||||||
|
|
||||||
|
Der Importer lädt die Registry aus `MINDNET_TYPES_FILE` oder nutzt Fallbacks. **Frontmatter-Overrides** für Profile werden in v2.2 weitgehend ignoriert zugunsten einer zentralen Governance in der YAML-Datei.
|
||||||
|
|
||||||
|
### 4.2 Beispiel (auslieferungsnah)
|
||||||
|
version: 1.0
|
||||||
|
types:
|
||||||
|
concept:
|
||||||
|
chunk_profile: medium
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
retriever_weight: 0.60
|
||||||
|
project:
|
||||||
|
chunk_profile: long
|
||||||
|
edge_defaults: ["references", "depends_on"]
|
||||||
|
retriever_weight: 0.97
|
||||||
|
|
||||||
|
**Auflösung im Importer**
|
||||||
|
- `effective_chunk_profile(note_type, registry)` → `"short|medium|long"|None`
|
||||||
|
- `effective_retriever_weight(note_type, registry)` → `float|None`
|
||||||
|
- Ergebnis wird in `note_payload` **und** `chunk_payloads` gespiegelt.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5) Der Retriever (Funktionaler Layer)
|
||||||
|
|
||||||
|
Der Retriever ist in v2.2 der zentrale Zugangspunkt. Er realisiert die **Hybride Suche**.
|
||||||
|
|
||||||
|
### 5.1 Scoring-Modell
|
||||||
|
Die Relevanz eines Treffers ergibt sich nicht mehr nur aus Textähnlichkeit, sondern aus einer gewichteten Formel:
|
||||||
|
|
||||||
|
total_score =
|
||||||
|
semantic_weight * semantic_score
|
||||||
|
+ edge_weight * edge_bonus
|
||||||
|
+ centrality_weight * centrality_bonus
|
||||||
|
+ type_weight * retriever_weight
|
||||||
|
|
||||||
|
* **semantic_score:** Vektorähnlichkeit (Qdrant).
|
||||||
|
* **retriever_weight:** Wichtigkeit des Typs (z.B. Projekte > Quellen).
|
||||||
|
* **edge_bonus:** Belohnung für Chunks, die im Kontext der Anfrage stark vernetzt sind (Summe der Confidence eingehender relevanter Kanten).
|
||||||
|
* **centrality_bonus:** Belohnung für Knoten, die zentrale Hubs im lokalen Graphen darstellen.
|
||||||
|
|
||||||
|
### 5.2 Erklärbarkeit (Explainability) – WP04b
|
||||||
|
Der Retriever ist keine Blackbox mehr. Er liefert auf Wunsch (`explain=True`) eine strukturierte Begründung (`Explanation`-Objekt).
|
||||||
|
|
||||||
|
**Die "Warum"-Logik:**
|
||||||
|
1. **Semantik:** Prüfung der Cosine-Similarity ("Sehr hohe textuelle Übereinstimmung").
|
||||||
|
2. **Typ:** Prüfung des `retriever_weight` ("Bevorzugt, da Entscheidung").
|
||||||
|
3. **Graph (Kontext):**
|
||||||
|
* **Hub (Outgoing):** Worauf verweist dieser Treffer? ("Verweist auf Qdrant").
|
||||||
|
* **Authority (Incoming):** Wer verweist auf diesen Treffer? ("Wird referenziert von Projekt Alpha").
|
||||||
|
|
||||||
|
Die API gibt diese Analysen als menschenlesbare Sätze (`reasons`) und als Datenstruktur (`score_breakdown`) zurück.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6) Der RAG-Chat & Persönlichkeit (WP05)
|
||||||
|
|
||||||
|
Seit WP-05 kann Mindnet nicht nur suchen, sondern als **KI-Zwilling** antworten.
|
||||||
|
|
||||||
|
### 6.1 Context Intelligence (Der "Enriched Context")
|
||||||
|
Kleine Sprachmodelle (SLMs wie Phi-3) scheitern oft an komplexen Zusammenhängen. Mindnet löst dies durch **explizite Metadaten-Injection** in den Prompt.
|
||||||
|
Dem LLM wird nicht nur der Text eines Chunks gezeigt, sondern auch sein Typ und Score:
|
||||||
|
* *"Hier ist eine Notiz vom Typ `[DECISION]`. Sie erklärt, warum wir etwas tun."*
|
||||||
|
* *"Hier ist eine Notiz vom Typ `[PROJECT]`. Sie erklärt, was wir tun."*
|
||||||
|
|
||||||
|
Dadurch kann das System Fragen wie *"Warum nutzen wir X?"* korrekt beantworten, indem es die Begründung aus der Decision-Notiz zieht, selbst wenn die Frage auf das Projekt abzielte.
|
||||||
|
|
||||||
|
### 6.2 Persönlichkeit (Late Binding)
|
||||||
|
Die "Persönlichkeit" von Mindnet (Stil, Werte, Tonfall) ist **nicht** im Python-Code verankert, sondern in `config/prompts.yaml`.
|
||||||
|
* **System Prompt:** Definiert die Rolle ("Ich bin dein digitales Gedächtnis...").
|
||||||
|
* **Werte:** Pragmatismus, Transparenz (kein Halluzinieren), Vernetztes Denken.
|
||||||
|
* **Flexibilität:** Die Persönlichkeit kann jederzeit angepasst werden (z.B. "Sei kritischer"), ohne das System neu zu starten.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7) Feedback & Lernen – WP04c
|
||||||
|
|
||||||
|
Das System verfügt nun über ein **Kurzzeitgedächtnis für Interaktionen**, das die Basis für zukünftiges Lernen bildet.
|
||||||
|
|
||||||
|
### 7.1 Der Feedback-Loop
|
||||||
|
1. **Suche (Situation):**
|
||||||
|
Wenn ein Nutzer eine Anfrage stellt, loggt Mindnet die "Situation":
|
||||||
|
* Den Query-Text.
|
||||||
|
* Die Top-K Ergebnisse.
|
||||||
|
* Den exakten `score_breakdown` zu diesem Zeitpunkt.
|
||||||
|
* Eine eindeutige `query_id` wird generiert.
|
||||||
|
|
||||||
|
2. **Bewertung (Label):**
|
||||||
|
Der Nutzer (oder ein Agent) bewertet einen spezifischen Treffer (`node_id`) positiv oder negativ (Score 1-5).
|
||||||
|
Dieses Feedback wird unter Referenzierung der `query_id` gespeichert.
|
||||||
|
|
||||||
|
3. **Lernen (Perspektive WP08):**
|
||||||
|
Durch die Verknüpfung von Situation und Bewertung entstehen Trainingsdaten. Mindnet kann später analysieren: *"Nutzer bevorzugen Treffer mit hohem Edge-Bonus, auch wenn der Text weniger passt."* -> Konsequenz: `edge_weight` erhöhen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8) Confidence & Provenance – wozu?
|
||||||
|
|
||||||
|
Der Retriever kann Edges gewichten:
|
||||||
|
- **provenance**: *explicit* > *rule*
|
||||||
|
- **confidence**: numerische Feinsteuerung
|
||||||
|
- **retriever_weight (Note/Chunk)**: skaliert die Relevanz des gesamten Knotens
|
||||||
|
|
||||||
|
Eine typische Gewichtung (konfigurierbar in `retriever.yaml`) ist:
|
||||||
|
`explicit: 1.0`, `rule: 0.8`. Damit bevorzugt der Graph **kuratiertes Wissen** (explizit notierte Links) vor „erweiterten“ Default-Ableitungen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9) Semantik ausgewählter `kind`-Werte
|
||||||
|
|
||||||
|
- `references` – „Nutzt/erwähnt“; neutral, aber stützend für Kontext.
|
||||||
|
- `related_to` – Ähnlichkeit/Verwandtschaft (symmetrisch interpretierbar).
|
||||||
|
- `similar_to` – noch engere Ähnlichkeit; oft aus Inline-Rel (bewusst gesetzt).
|
||||||
|
- `depends_on` – fachliche Abhängigkeit (z. B. „Projekt X hängt von Y ab“).
|
||||||
|
- `belongs_to`, `next`, `prev` – Struktur.
|
||||||
|
|
||||||
|
> Symmetrische Relationen (z. B. `related_to`, `similar_to`) können **explizit** nur einseitig notiert sein, aber im Retriever beidseitig interpretiert werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10) Frontmatter-Eigenschaften – Rolle & Empfehlung
|
||||||
|
|
||||||
|
Frontmatter-Eigenschaften (Properties) bleiben **minimiert**:
|
||||||
|
- **type** – steuert Typ-Defaults via Registry (Pflicht für differenziertes Verhalten).
|
||||||
|
- **tags** – für klassische Filterung.
|
||||||
|
- **retriever_weight / chunk_profile** – Sollten **nicht** manuell gesetzt werden (Best Practice: `types.yaml` nutzen).
|
||||||
|
- **links** – Veraltet. Bitte Links primär über Text (Wikilink/Inline/Callout) pflegen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11) Lösch-/Update-Garantien (Idempotenz)
|
||||||
|
|
||||||
|
- Jede Note hat einen stabilen **note_id** (Frontmatter/Hash).
|
||||||
|
- Vor einem Upsert können *alte Chunks/Edges einer Note* gefiltert gelöscht werden (`note_id`-Filter) – das hält Collections sauber bei Re-Imports.
|
||||||
|
- Die Import-Skripte unterstützen Modi wie `--purge-before-upsert` und `--sync-deletes`, um den Graph sauber zu halten.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12) Beispiel – Von Markdown zu Kanten (v2.2)
|
||||||
|
|
||||||
|
**Markdown (Auszug)**
|
||||||
|
# Relations Showcase
|
||||||
|
…
|
||||||
|
Wir nutzen [[rel:depends_on Qdrant Vektordatenbank]] für die Suche.
|
||||||
|
…
|
||||||
|
Siehe auch: [[Embeddings 101]]
|
||||||
|
|
||||||
|
> [!edge] related_to: [[Vector DB Basics]]
|
||||||
|
|
||||||
|
**Ergebnis (fachlich)**
|
||||||
|
1. `depends_on(Chunk→Qdrant)` mit `rule_id=inline:rel`, `confidence≈0.95`.
|
||||||
|
2. `references(Chunk→Embeddings 101)` mit `rule_id=explicit:wikilink`, `confidence=1.0`.
|
||||||
|
3. `related_to(Chunk→Vector DB Basics)` via Callout; `rule_id=callout:edge`, `confidence≈0.90`.
|
||||||
|
4. **Typ-Defaults:** Falls die Note vom Typ `project` ist, entstehen zusätzlich `depends_on`-Kanten zu den Zielen aus (2) und (3).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13) Referenzen (Projektdateien & Leitlinien)
|
||||||
|
|
||||||
|
- Import-Pipeline & Registry-Auflösung: `scripts/import_markdown.py`.
|
||||||
|
- Kantenbildung (V2-Logic): `app/core/derive_edges.py`.
|
||||||
|
- Typ-Registry: `config/types.yaml` & `TYPE_REGISTRY_MANUAL.md`.
|
||||||
|
- Retriever-Scoring & Explanation: `app/core/retriever.py`.
|
||||||
|
- Persönlichkeit & Chat: `config/prompts.yaml` & `app/routers/chat.py`.
|
||||||
|
- Logging Service: `app/services/feedback_service.py`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 14) Workpackage Status (v2.2.1)
|
||||||
|
|
||||||
|
Aktueller Implementierungsstand der Module.
|
||||||
|
|
||||||
|
| WP | Titel | Status | Anmerkung |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| **WP01** | Knowledge Design | 🟢 Live | Typen, Frontmatter definiert. |
|
||||||
|
| **WP02** | Chunking Strategy | 🟢 Live | Profilbasiertes Chunking aktiv. |
|
||||||
|
| **WP03** | Edge Logic / Import | 🟢 Live | Neue Importer-Pipeline mit Provenance. |
|
||||||
|
| **WP04a**| Retriever Scoring | 🟢 Live | Hybrider Score (Semantik + Graph). |
|
||||||
|
| **WP04b**| Explanation Layer | 🟢 Live | API liefert Reasons & Breakdown. |
|
||||||
|
| **WP04c**| Feedback Loop | 🟢 Live | Logging (JSONL) & Traceability aktiv. |
|
||||||
|
| **WP05** | Persönlichkeit / Chat | 🟢 Live | RAG-Integration mit Context Enrichment. |
|
||||||
|
| **WP06** | Decision Engine | 🟡 Geplant | Nächster Schritt. |
|
||||||
|
| **WP08** | Self-Tuning | 🔴 Geplant | Auto-Adjustment der Gewichte. |
|
||||||
@@ -0,0 +1,295 @@
|
|||||||
|
# Mindnet v2.2 – Technische Architektur
|
||||||
|
**Datei:** `docs/mindnet_technical_architecture_v2.2.md`
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** **FINAL** (Integrierter Stand WP01–WP05)
|
||||||
|
**Quellen:** `Programmplan_V2.2.md`, `Handbuch.md`, `chunking_strategy.md`, `wp04_retriever_scoring.md`.
|
||||||
|
|
||||||
|
> **Ziel dieses Dokuments:**
|
||||||
|
> Vollständige Beschreibung der technischen Architektur inkl. Graph-Datenbank, Retrieval-Logik und der **neuen RAG-Komponenten (LLM/Chat)**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Systemüberblick
|
||||||
|
|
||||||
|
### 1.1 Architektur-Zielbild
|
||||||
|
Mindnet ist ein **lokales RAG-System (Retrieval Augmented Generation)**.
|
||||||
|
1. **Source:** Markdown-Notizen in einem Vault (Obsidian-kompatibel).
|
||||||
|
2. **Pipeline:** Ein Python-Importer transformiert diese in **Notes**, **Chunks** und **Edges**.
|
||||||
|
3. **Storage:**
|
||||||
|
* **Qdrant:** Vektor-Datenbank für Graph und Semantik (Collections: notes, chunks, edges).
|
||||||
|
* **Local Files (JSONL):** Append-Only Logs für Feedback und Search-History (Data Flywheel).
|
||||||
|
4. **Service:** Eine FastAPI-Anwendung stellt Endpunkte für **Semantische** und **Hybride Suche** sowie **Feedback** bereit.
|
||||||
|
5. **Inference:** Lokales LLM (Ollama: Phi-3 Mini) für RAG-Chat und Antwortgenerierung.
|
||||||
|
|
||||||
|
Das System arbeitet **deterministisch** (stabile IDs) und ist **konfigurationsgetrieben** (`types.yaml`, `retriever.yaml`, `prompts.yaml`).
|
||||||
|
|
||||||
|
### 1.2 Verzeichnisstruktur & Komponenten (Post-WP05)
|
||||||
|
|
||||||
|
/mindnet/
|
||||||
|
├── app/
|
||||||
|
│ ├── main.py # FastAPI Einstiegspunkt
|
||||||
|
│ ├── core/
|
||||||
|
│ │ ├── qdrant.py # Client-Factory & Connection
|
||||||
|
│ │ ├── qdrant_points.py # Low-Level Point Operations (Upsert/Delete)
|
||||||
|
│ │ ├── note_payload.py # Bau der Note-Objekte
|
||||||
|
│ │ ├── chunk_payload.py # Bau der Chunk-Objekte
|
||||||
|
│ │ ├── chunker.py # Text-Zerlegung (Profiling)
|
||||||
|
│ │ ├── edges.py # Edge-Datenstrukturen
|
||||||
|
│ │ ├── derive_edges.py # Logik der Kantenableitung (WP03)
|
||||||
|
│ │ ├── graph_adapter.py # Subgraph & Reverse-Lookup (WP04b)
|
||||||
|
│ │ └── retriever.py # Scoring, Expansion & Explanation (WP04a/b)
|
||||||
|
│ ├── models/ # Pydantic DTOs (QueryHit, Explanation, FeedbackRequest, ChatRequest)
|
||||||
|
│ ├── routers/
|
||||||
|
│ │ ├── query.py # Such-Endpunkt
|
||||||
|
│ │ ├── chat.py # RAG-Chat Endpunkt (WP05)
|
||||||
|
│ │ ├── feedback.py # Feedback-Endpunkt (WP04c)
|
||||||
|
│ │ └── ...
|
||||||
|
│ ├── services/
|
||||||
|
│ │ ├── llm_service.py # Ollama Client (WP05)
|
||||||
|
│ │ ├── feedback_service.py # JSONL Logging (WP04c)
|
||||||
|
│ │ └── embeddings_client.py
|
||||||
|
├── config/
|
||||||
|
│ ├── types.yaml # Typ-Definitionen (Import-Zeit)
|
||||||
|
│ ├── retriever.yaml # Scoring-Gewichte (Laufzeit)
|
||||||
|
│ └── prompts.yaml # LLM System-Prompts & Templates (WP05)
|
||||||
|
├── data/
|
||||||
|
│ └── logs/ # Lokale JSONL-Logs (WP04c)
|
||||||
|
├── scripts/
|
||||||
|
│ ├── import_markdown.py # Haupt-Importer CLI
|
||||||
|
│ ├── payload_dryrun.py # Diagnose: JSON-Generierung ohne DB
|
||||||
|
│ └── edges_full_check.py # Diagnose: Graph-Integrität
|
||||||
|
└── tests/ # Pytest Suite
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Datenmodell & Collections (Qdrant)
|
||||||
|
|
||||||
|
Das Datenmodell verteilt sich auf drei Collections, definiert durch `COLLECTION_PREFIX` (Standard: `mindnet`).
|
||||||
|
|
||||||
|
### 2.1 Notes Collection (`<prefix>_notes`)
|
||||||
|
Repräsentiert die Metadaten einer Datei.
|
||||||
|
* **Zweck:** Filterung, Metadaten-Haltung, Vererbung von Eigenschaften an Chunks.
|
||||||
|
* **Schema (Payload):**
|
||||||
|
|
||||||
|
| Feld | Datentyp | Beschreibung | Herkunft |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| `note_id` | Keyword | Stabile ID (UUIDv5 oder Slug). | Frontmatter |
|
||||||
|
| `title` | Text | Titel der Notiz. | Frontmatter |
|
||||||
|
| `type` | Keyword | Logischer Typ (z.B. `concept`). | `types.yaml` Resolver |
|
||||||
|
| `retriever_weight` | Float | Wichtigkeit im Retrieval. | `types.yaml` |
|
||||||
|
| `chunk_profile` | Keyword | Genutztes Chunking-Profil. | `types.yaml` |
|
||||||
|
| `edge_defaults` | List[Str] | Aktive Default-Relationen. | `types.yaml` |
|
||||||
|
| `tags` | List[Kw] | Tags zur Filterung. | Frontmatter |
|
||||||
|
| `updated` | Integer | Zeitstempel (z.B. YYYYMMDD...). | File Stats |
|
||||||
|
| `fulltext` | Text | Gesamter Inhalt (für Export/Rekonstruktion). | Body |
|
||||||
|
|
||||||
|
### 2.2 Chunks Collection (`<prefix>_chunks`)
|
||||||
|
Die atomaren Sucheinheiten.
|
||||||
|
* **Zweck:** Vektorsuche (Embeddings), Granulares Ergebnis.
|
||||||
|
* **Schema (Payload):**
|
||||||
|
|
||||||
|
| Feld | Datentyp | Beschreibung |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| `chunk_id` | Keyword | Deterministisch: `{note_id}#c{index:02d}`. |
|
||||||
|
| `note_id` | Keyword | Referenz zur Note. |
|
||||||
|
| `text` | Text | **Reiner Inhalt** (ohne Overlap). Anzeige-Text. |
|
||||||
|
| `window` | Text | **Kontext-Fenster** (mit Overlap). Embedding-Basis. |
|
||||||
|
| `ord` | Integer | Sortierreihenfolge (1..N). |
|
||||||
|
| `retriever_weight` | Float | Vererbt von Note. |
|
||||||
|
| `chunk_profile` | Keyword | Vererbt von Note. |
|
||||||
|
| `neighbors_prev` | List[Str] | ID des Vorgänger-Chunks. |
|
||||||
|
| `neighbors_next` | List[Str] | ID des Nachfolger-Chunks. |
|
||||||
|
|
||||||
|
### 2.3 Edges Collection (`<prefix>_edges`)
|
||||||
|
Gerichtete Kanten. Massiv erweitert in WP03 für Provenienz-Tracking.
|
||||||
|
* **Zweck:** Graph-Traversal, Kontext-Anreicherung.
|
||||||
|
* **Schema (Payload):**
|
||||||
|
|
||||||
|
| Feld | Datentyp | Beschreibung | Wertebereich (Bsp.) |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| `edge_id` | Keyword | Hash aus (src, dst, kind). | |
|
||||||
|
| `source_id` | Keyword | ID des Start-Chunks. | |
|
||||||
|
| `target_id` | Keyword | ID des Ziel-Chunks (oder Note-Titel). | |
|
||||||
|
| `note_id` | Keyword | Note, die diese Kante definiert. | |
|
||||||
|
| `kind` | Keyword | Art der Beziehung. | `references`, `depends_on`, `next` |
|
||||||
|
| `scope` | Keyword | Geltungsbereich. | Immer `"chunk"` (v2.2). |
|
||||||
|
| `rule_id` | Keyword | Herkunftsregel. | `explicit:wikilink`, `inline:rel` |
|
||||||
|
| `confidence` | Float | Vertrauenswürdigkeit (0.0-1.0). | 1.0, 0.95, 0.7 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Konfiguration
|
||||||
|
|
||||||
|
Die Logik ist ausgelagert in YAML-Dateien.
|
||||||
|
|
||||||
|
### 3.1 Typ-Registry (`config/types.yaml`)
|
||||||
|
Steuert den Import-Prozess.
|
||||||
|
* **Priorität:** Frontmatter > Pfad > Default.
|
||||||
|
* **Inhalt (Beispiel):**
|
||||||
|
|
||||||
|
types:
|
||||||
|
concept:
|
||||||
|
chunk_profile: medium
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
retriever_weight: 0.60
|
||||||
|
|
||||||
|
### 3.2 Retriever-Config (`config/retriever.yaml`)
|
||||||
|
Steuert das Scoring zur Laufzeit (WP04a).
|
||||||
|
* **Inhalt (Beispiel):**
|
||||||
|
|
||||||
|
scoring:
|
||||||
|
semantic_weight: 1.0
|
||||||
|
edge_weight: 0.7
|
||||||
|
centrality_weight: 0.5
|
||||||
|
|
||||||
|
### 3.3 Prompts (`config/prompts.yaml`)
|
||||||
|
Steuert die LLM-Persönlichkeit (WP05).
|
||||||
|
* **Inhalt (Beispiel):**
|
||||||
|
|
||||||
|
system_prompt: |
|
||||||
|
Du bist 'mindnet', das KI-Gedächtnis.
|
||||||
|
rag_template: |
|
||||||
|
QUELLEN (WISSEN): {context_str}
|
||||||
|
|
||||||
|
### 3.4 Environment (`.env`)
|
||||||
|
Erweiterung für LLM-Steuerung:
|
||||||
|
|
||||||
|
MINDNET_LLM_MODEL=phi3:mini
|
||||||
|
MINDNET_OLLAMA_URL=http://127.0.0.1:11434
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Import-Pipeline (Markdown → Qdrant)
|
||||||
|
|
||||||
|
Das Skript `scripts/import_markdown.py` orchestriert den Prozess.
|
||||||
|
|
||||||
|
### 4.1 Verarbeitungsschritte
|
||||||
|
|
||||||
|
1. **Discovery & Parsing:**
|
||||||
|
* Einlesen der `.md` Dateien. Hash-Vergleich (Body/Frontmatter) zur Erkennung von Änderungen.
|
||||||
|
2. **Typauflösung:**
|
||||||
|
* Laden der `types.yaml`. Bestimmen des effektiven Typs und der `edge_defaults`.
|
||||||
|
3. **Chunking:**
|
||||||
|
* Zerlegung via `chunker.py` basierend auf `chunk_profile` (z.B. `by_heading`, `short`, `long`).
|
||||||
|
* Trennung von `text` (Kern) und `window` (Embedding-Kontext).
|
||||||
|
4. **Kantenableitung (Edge Derivation):**
|
||||||
|
Die `derive_edges.py` erzeugt Kanten in strikter Reihenfolge:
|
||||||
|
1. **Inline-Edges:** `[[rel:depends_on X]]` → `kind=depends_on`, `rule_id=inline:rel`, `conf=0.95`.
|
||||||
|
2. **Callout-Edges:** `> [!edge] related_to: [[X]]` → `kind=related_to`, `rule_id=callout:edge`, `conf=0.90`.
|
||||||
|
3. **Explizite Referenzen:** `[[X]]` → `kind=references`, `rule_id=explicit:wikilink`, `conf=1.0`.
|
||||||
|
4. **Typ-Defaults:** Für jede Referenz werden Zusatzkanten gemäß `edge_defaults` erzeugt (z.B. `project` -> `depends_on`). `rule_id=edge_defaults:...`, `conf=0.7`.
|
||||||
|
5. **Struktur:** `belongs_to`, `next`, `prev` (automatisch).
|
||||||
|
5. **Upsert:**
|
||||||
|
* Schreiben in Qdrant. Nutzung von `--purge-before-upsert` für saubere Updates.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Retriever-Architektur & Scoring
|
||||||
|
|
||||||
|
Der Retriever (`app/core/retriever.py`) unterstützt zwei Modi. Für den Chat wird **zwingend** der Hybrid-Modus genutzt.
|
||||||
|
|
||||||
|
### 5.1 Betriebsmodi
|
||||||
|
* **Semantic:** Reine Vektorsuche. Schnell.
|
||||||
|
* **Hybrid:** Vektorsuche + Graph-Expansion (Tiefe N) + Re-Ranking.
|
||||||
|
|
||||||
|
### 5.2 Scoring-Formel (WP04a)
|
||||||
|
Die Relevanzberechnung ist nun eine gewichtete Summe:
|
||||||
|
|
||||||
|
$$
|
||||||
|
TotalScore = (W_{sem} \cdot S_{sem} \cdot \max(W_{type}, 0)) + (W_{edge} \cdot B_{edge}) + (W_{cent} \cdot B_{cent})
|
||||||
|
$$
|
||||||
|
|
||||||
|
* **Komponenten:**
|
||||||
|
* $S_{sem}$: Semantic Score (Cosine Similarity).
|
||||||
|
* $W_{type}$: `retriever_weight` (aus Note/Chunk Payload).
|
||||||
|
* $B_{edge}$: Edge-Bonus (Summe der Konfidenzen eingehender relevanter Kanten).
|
||||||
|
* $B_{cent}$: Centrality-Bonus (im lokalen Subgraphen).
|
||||||
|
* **Gewichte ($W$):** Stammen aus `retriever.yaml`.
|
||||||
|
|
||||||
|
### 5.3 Explanation Layer (WP04b)
|
||||||
|
Der Retriever kann Ergebnisse erklären (`explain=True`).
|
||||||
|
* **Logik:**
|
||||||
|
* Berechnung des `ScoreBreakdown` (Anteile von Semantik, Graph, Typ).
|
||||||
|
* Analyse des lokalen Subgraphen mittels `graph_adapter.py`.
|
||||||
|
* **Incoming Edges (Authority):** Wer zeigt auf diesen Treffer? (z.B. "Referenziert von...")
|
||||||
|
* **Outgoing Edges (Hub):** Worauf zeigt dieser Treffer? (z.B. "Verweist auf...")
|
||||||
|
* **Output:** `QueryHit` enthält ein `explanation` Objekt mit menschenlesbaren `reasons` und `related_edges`.
|
||||||
|
|
||||||
|
### 5.4 Graph-Expansion
|
||||||
|
Der Hybrid-Modus lädt dynamisch die Nachbarschaft der Top-K Vektor-Treffer ("Seeds") über `graph_adapter.expand`. Dies baut einen temporären `NetworkX`-artigen Graphen im Speicher (Klasse `Subgraph`), auf dem Boni berechnet werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. RAG & Chat Architektur (WP05)
|
||||||
|
|
||||||
|
Der Flow für eine Chat-Anfrage (`/chat`) ist eine Pipeline aus vier Schritten.
|
||||||
|
|
||||||
|
### 6.1 Schritt 1: Intent & Retrieval
|
||||||
|
* Der `ChatRouter` ruft den `Retriever` im **Hybrid-Modus** auf.
|
||||||
|
* Dies stellt sicher, dass logisch verknüpfte Notizen (z.B. Entscheidungen zu einem Projekt) gefunden werden.
|
||||||
|
|
||||||
|
### 6.2 Schritt 2: Context Enrichment (Das "Context Intelligence" Pattern)
|
||||||
|
* Der Router (`_build_enriched_context`) reichert die Chunks mit Metadaten an:
|
||||||
|
* **Typ-Injection:** `[DECISION]`, `[PROJECT]`, `[CONCEPT]`.
|
||||||
|
* **Score-Transparenz:** `(Score: 0.75)`.
|
||||||
|
* **Formatierung:** Markdown-Header pro Quelle.
|
||||||
|
* *Ziel:* Kleine Modelle (SLMs wie Phi-3) benötigen diese expliziten Signale, um komplexe Zusammenhänge ("Warum nutzen wir X?") aus den Texten abzuleiten.
|
||||||
|
|
||||||
|
### 6.3 Schritt 3: Generation (LLM Service)
|
||||||
|
* **Service:** `app/services/llm_service.py` nutzt `httpx` (async).
|
||||||
|
* **Backend:** Ollama (lokal).
|
||||||
|
* **Modell:** `phi3:mini` (3.8B Parameter).
|
||||||
|
* **Timeout:** Erhöht auf 300s für CPU-Inference Sicherheit.
|
||||||
|
|
||||||
|
### 6.4 Schritt 4: Response & Traceability
|
||||||
|
* Die Antwort enthält die generierte Nachricht **und** die verwendeten Quellen (`sources`).
|
||||||
|
* Eine `query_id` wird generiert und durchgereicht, um späteres Feedback (WP04c) zu ermöglichen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Feedback & Logging Architektur (WP04c)
|
||||||
|
|
||||||
|
Mindnet implementiert ein "Data Flywheel" zur späteren Optimierung (Self-Tuning).
|
||||||
|
|
||||||
|
### 7.1 Komponenten
|
||||||
|
* **Feedback Service (`app/services/feedback_service.py`):** Kapselt die Schreibzugriffe.
|
||||||
|
* **Storage:** Lokales Dateisystem (`data/logs/`), Format JSONL (Line-delimited JSON).
|
||||||
|
|
||||||
|
### 7.2 Log-Dateien
|
||||||
|
1. **`search_history.jsonl`**:
|
||||||
|
* Speichert jede Anfrage an `/query` und `/chat`.
|
||||||
|
* Enthält: `query_id`, `query_text`, `timestamp`, `hits` (inkl. `score_breakdown` Snapshots).
|
||||||
|
* Zweck: Trainingsdaten ("Was hat das System gesehen?").
|
||||||
|
2. **`feedback.jsonl`**:
|
||||||
|
* Speichert User-Reaktionen an `/feedback`.
|
||||||
|
* Enthält: `query_id`, `node_id`, `score` (1-5), `comment`.
|
||||||
|
* Zweck: Labels ("War es gut?").
|
||||||
|
|
||||||
|
### 7.3 Traceability
|
||||||
|
Die `query_id` (UUIDv4) wird im `/query` Response generiert und muss vom Client beim `/feedback` Aufruf mitgesendet werden. Dies ermöglicht den Join zwischen Snapshot und Bewertung.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Indizes & Performance
|
||||||
|
|
||||||
|
Damit Qdrant performant bleibt, sind Payload-Indizes essenziell.
|
||||||
|
|
||||||
|
**Erforderliche Indizes:**
|
||||||
|
* **Notes:** `note_id`, `type`, `tags`.
|
||||||
|
* **Chunks:** `note_id`, `chunk_id`.
|
||||||
|
* **Edges:** `source_id`, `target_id`, `kind`, `scope`, `note_id`.
|
||||||
|
|
||||||
|
Validierung erfolgt über `tests/ensure_indexes_and_show.py`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Offene Punkte / Known Limitations
|
||||||
|
|
||||||
|
1. **Multi-Target Inline-Relations:**
|
||||||
|
* `rel: similar_to [[A]] [[B]]` wird aktuell parser-seitig nicht unterstützt.
|
||||||
|
* Workaround: Zwei separate Inline-Links `[[rel:similar_to A]] [[rel:similar_to B]]`.
|
||||||
|
2. **Unresolved Targets:**
|
||||||
|
* Kanten zu Notizen, die noch nicht existieren, werden mit `target_id="Titel"` angelegt.
|
||||||
|
* Heilung durch `scripts/resolve_unresolved_references.py` möglich.
|
||||||
|
3. **Vektor-Konfiguration für Edges:**
|
||||||
|
* `mindnet_edges` hat aktuell keine Vektoren (`vectors = null`). Eine semantische Suche *auf Kanten* ist noch nicht möglich.
|
||||||
@@ -0,0 +1,365 @@
|
|||||||
|
# mindnet v2 – Implementierungs-Playbook & Handover (Side‑by‑Side zu v1)
|
||||||
|
*Stand:* 2025-11-10 08:03
|
||||||
|
*Autor:* ChatGPT (Projekt‑Handover)
|
||||||
|
*Zweck:* Vollständige, ausführbare Anleitung zur Weiterentwicklung von **mindnet** auf eine versionierte, testbare v2‑Architektur (neben v1), inklusive Prompt für den Folgethread, Akzeptanzkriterien, Dateiliste, Test‑ und Rollback‑Pläne.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0) Kontext & Zielbild (kurz)
|
||||||
|
- **Mission:** Persönliches Wissensnetz, das langfristig die eigene Persönlichkeit, Erfahrungen und Entscheidungslogik abbildet; später nachvollziehbare Erklärungen für Familie/Nachwelt.
|
||||||
|
- **Heutiger Stand (v1):** `mindnet_notes` in Qdrant vorhanden; **Chunks** und **Edges** bisher nur teilweise/nicht konsistent; Edge‑Defaults („1 Dot“) vorhanden, aber Ziele teils nicht materialisiert. Importskripte: `import_markdown.py`, `edges.py` (historisch).
|
||||||
|
- **Ziel (v2):**
|
||||||
|
- **Collections:** `mindnet_notes_v2`, `mindnet_chunks_v2`, `mindnet_edges_v2` (saubere Payload‑Schemata + Indizes).
|
||||||
|
- **Chunker v2:** Block/Heading‑aware, Ziel 900–1200 Zeichen, Überlappung 120–180, deterministische `chunk_id`s.
|
||||||
|
- **Edge‑Builder v2:** Reihenfolge: **explicit → rule → default_resolved → default**; saubere `provenance`.
|
||||||
|
- **Importer‑Pipeline v2:** Parse → Chunk → Edge → Embeddings → Batch‑Upsert + Payload‑Indexpflege + Snapshots.
|
||||||
|
- **Policies:** Privacy/Recency/Type‑Gewichte als **Konfiguration** (YAML + JSON‑Schema), nicht hart im Code.
|
||||||
|
- **Observability:** FastAPI + OpenTelemetry (Tracing/Metrics); Import‑Reports (CSV/JSON).
|
||||||
|
- **Side‑by‑Side:** v2 **parallel** zu v1; kein Big‑Bang.
|
||||||
|
|
||||||
|
> **Warum so?** Qdrant‑Payload‑Indizes/Filter → schnelle, erklärbare Selektionen; Snapshots → betriebssicheres Rollback; OTel → Nachvollziehbarkeit; YAML/JSON‑Schema → Validierbarkeit & Stabilität.
|
||||||
|
|
||||||
|
### 0.1 Aktueller Implementierungsstand (2025-11-18)
|
||||||
|
|
||||||
|
- Es existiert eine produktive mindnet-Instanz mit den Collections:
|
||||||
|
- `${prefix}_notes`
|
||||||
|
- `${prefix}_chunks`
|
||||||
|
- `${prefix}_edges`
|
||||||
|
- Import-Pipeline:
|
||||||
|
- `scripts/import_markdown.py` erzeugt Notes, Chunks und Edges vollständig.
|
||||||
|
- `types.yaml` definiert per Note-Typ:
|
||||||
|
- `retriever_weight`
|
||||||
|
- `chunk_profile`
|
||||||
|
- `edge_defaults` (z. B. `depends_on`, `related_to`).
|
||||||
|
- Edges:
|
||||||
|
- Strukturkanten: `belongs_to`, `next`, `prev`
|
||||||
|
- Explizite Kanten aus Wikilinks: `kind="references"`, `rule_id="explicit:wikilink"`
|
||||||
|
- Typbasierte Default-Kanten aus `edge_defaults`: `rule_id="edge_defaults:<type>:<relation>"`
|
||||||
|
- Inline-Kanten im Format `[[rel:<relation> <Zielnote>]]` mit `rule_id="inline:rel"`
|
||||||
|
- alle Edges tragen eine `confidence` im Bereich `[0,1]` als Hinweis für spätere Ranking-Logiken.
|
||||||
|
- Tests:
|
||||||
|
- `tests/test_edges_smoke.py`, `tests/test_edges_all.py` und `scripts/edges_full_check.py` validieren Counts, Konsistenz (`belongs_to == chunks`, `next/prev`-Ketten, keine Duplikate) und Provenienz-Zählungen (`explicit`, `defaults`, `inline`, `structure`).
|
||||||
|
- Offene Punkte:
|
||||||
|
- Mehrere Ziele in einer Inline-Relation (`rel: similar_to [[A]] [[B]]`) sind noch nicht implementiert.
|
||||||
|
- Retrieval-Service nutzt die neuen Edges und Weights bisher nur teilweise; ein edge-bewusster Retriever ist als nächster Schritt geplant.
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1) Datenmodelle (Schemata, v2)
|
||||||
|
Die tatsächlichen JSON‑Schemata (2020‑12) werden im Repo abgelegt und in CI validiert.
|
||||||
|
|
||||||
|
### 1.1 note.schema.json
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://example.org/mindnet/note.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["id", "title", "type", "privacy", "created", "hash_body"],
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string", "pattern": "^[a-zA-Z0-9_-]+$" },
|
||||||
|
"title": { "type": "string", "minLength": 1 },
|
||||||
|
"type": { "type": "string" },
|
||||||
|
"privacy": { "type": "string", "enum": ["public","internal","private"] },
|
||||||
|
"created": { "type": "string", "format": "date-time" },
|
||||||
|
"modified": { "type": "string", "format": "date-time" },
|
||||||
|
"tags": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"lang": { "type": "string", "default": "de" },
|
||||||
|
"source_path": { "type": "string" },
|
||||||
|
"source_collection": { "type": "string" },
|
||||||
|
"hash_body": { "type": "string" },
|
||||||
|
"hash_frontmatter": { "type": "string" },
|
||||||
|
"token_count": { "type": "integer", "minimum": 0 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.2 chunk.schema.json
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://example.org/mindnet/chunk.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["chunk_id","note_id","text","ord"],
|
||||||
|
"properties": {
|
||||||
|
"chunk_id": { "type": "string", "pattern": "^[a-zA-Z0-9_-]+#c\d{4}$" },
|
||||||
|
"note_id": { "type": "string" },
|
||||||
|
"text": { "type": "string" },
|
||||||
|
"ord": { "type": "integer", "minimum": 0 },
|
||||||
|
"span_char_start": { "type": "integer", "minimum": 0 },
|
||||||
|
"span_char_end": { "type": "integer", "minimum": 0 },
|
||||||
|
"heading_path": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"section_title": { "type": "string" },
|
||||||
|
"tokens_start": { "type": "integer", "minimum": 0 },
|
||||||
|
"tokens_end": { "type": "integer", "minimum": 0 },
|
||||||
|
"embeddings_version": { "type": "string" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.3 edge.schema.json
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://example.org/mindnet/edge.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["edge_id","src_note_id","relation","provenance"],
|
||||||
|
"properties": {
|
||||||
|
"edge_id": { "type": "string" },
|
||||||
|
"src_note_id": { "type": "string" },
|
||||||
|
"src_chunk_id": { "type": "string" },
|
||||||
|
"dst_note_id": { "type": "string" },
|
||||||
|
"dst_chunk_id": { "type": "string" },
|
||||||
|
"relation": { "type": "string" },
|
||||||
|
"evidence_spans":{ "type": "array", "items": {"type":"object","properties":{"chunk_id":{"type":"string"},"span":[{"type":"integer"},{"type":"integer"}]]} },
|
||||||
|
"provenance": { "type": "string", "enum": ["explicit","rule","default_resolved","default"] },
|
||||||
|
"rule_id": { "type": "string" },
|
||||||
|
"confidence": { "type": "number", "minimum": 0, "maximum": 1 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.4 default_edge.schema.json
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://example.org/mindnet/default_edge.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["src_note_id","relation","target_kind"],
|
||||||
|
"properties": {
|
||||||
|
"src_note_id": { "type": "string" },
|
||||||
|
"relation": { "type": "string" },
|
||||||
|
"target_kind": { "type": "string" },
|
||||||
|
"when": { "type": "string" },
|
||||||
|
"strength_hint":{ "type": "number", "minimum": 0, "maximum": 1 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2) Qdrant: Collections & Indizes (v2)
|
||||||
|
Anlegen **neben** v1:
|
||||||
|
- `mindnet_notes_v2` (Payload‑Index: `type, privacy, created, modified, tags, source_path`)
|
||||||
|
- `mindnet_chunks_v2` (Payload‑Index: `note_id, ord, heading_path`)
|
||||||
|
- `mindnet_edges_v2` (Payload‑Index: `src_note_id, dst_note_id, relation, provenance`)
|
||||||
|
|
||||||
|
> Snapshots für Backups/Restore einplanen (Runbook siehe unten).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3) Import‑Pipeline v2 – Module & Flags
|
||||||
|
- **Chunker v2** (`chunking/block_chunker.py`): `target_len=1000`, `overlap=150`, `respect_headings=True`, `min_chunk_len=600`, `max_chunk_len=1400`, `chunk_id = f\"{note_id}#c{ordinal:04d}\"`
|
||||||
|
- **Edge‑Builder v2** (`graph/edge_builder_v2.py`): Reihenfolge **explicit → rule → default_resolved → default**, `provenance` korrekt setzen.
|
||||||
|
- **Importer** (`scripts/import_markdown.py` erweitern oder `scripts/import_markdown_v2.py`):
|
||||||
|
Flags: `--schema v2`, `--chunker v2`, `--edges v2`, `--dry-run`, `--apply`, `--prefix "$COLLECTION_PREFIX"`.
|
||||||
|
|
||||||
|
**Beispiel‑Run (dry‑run):**
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --dry-run --prefix "$COLLECTION_PREFIX"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Apply:**
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --apply --prefix "$COLLECTION_PREFIX"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4) Policies (konfigurierbar, kein Hard‑Code)
|
||||||
|
- `policies/retrieval.schema.json` (JSON‑Schema) und `policies/retrieval.yaml`, z. B.:
|
||||||
|
```yaml
|
||||||
|
privacy_order: [private, internal, public]
|
||||||
|
recency_boost_half_life_days: 90
|
||||||
|
type_priority:
|
||||||
|
person: 1.2
|
||||||
|
event: 1.1
|
||||||
|
concept: 1.0
|
||||||
|
max_chunks_per_note: 8
|
||||||
|
edge_provenance_weights:
|
||||||
|
explicit: 1.0
|
||||||
|
rule: 0.8
|
||||||
|
default_resolved: 0.6
|
||||||
|
default: 0.2
|
||||||
|
rule_sets:
|
||||||
|
- id: "rules:types-v1"
|
||||||
|
enabled: true
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5) Observability & Reports
|
||||||
|
- **FastAPI + OpenTelemetry**: Traces um Phasen *parse/chunk/edge/upsert*; Metriken (Counter/Histogramme) pro Phase.
|
||||||
|
- **Import‑Report** je Lauf (CSV/JSON): counts für Notes/Chunks/Edges, Fehler, Dauer, Embedding‑Version.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6) Teststrategie
|
||||||
|
- **Gold‑Notizen** (3–5 repräsentative .md) mit erwarteten *Chunk‑Counts* und *Edge‑Sets*.
|
||||||
|
- **Regression:** fixer Soll‑Count (z. B. 171 ± 15 %) über Gesamt‑Vault; keine Duplikate `(src,relation,dst)`.
|
||||||
|
- **Dry‑Run vs Apply**: identische Counts, nur ohne Upserts.
|
||||||
|
- **Filter‑Smoke‑Tests:** Qdrant‑Filter nach `privacy`, `type`, `tags`, `created` funktionieren performant.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7) Akzeptanzkriterien (KPI)
|
||||||
|
1. **Chunk‑Qualität:** ≥ 90 % der Chunks enden an semantischen Grenzen; Ziel‑Länge 900–1200 Zeichen; deterministische IDs.
|
||||||
|
2. **Edge‑Kohärenz:** Keine Duplikate gleicher `(src,relation,dst)`; ≥ 95 % der expliziten Links materialisiert; `default` nur, wenn kein Ziel existiert.
|
||||||
|
3. **Filterbarkeit:** Queries nach `privacy/type/tags/created` performant (Payload‑Index vorhanden).
|
||||||
|
4. **Betriebsfestigkeit:** Telemetrie aktiv; fehlertoleranter Import; Qdrant‑Snapshot nach erfolgreichem Lauf.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8) Roadmap in kleinen, testbaren Schritten
|
||||||
|
|
||||||
|
### Step 0 – Safeguards
|
||||||
|
- Qdrant‑Snapshot v1 erstellen + Restore‑Probe.
|
||||||
|
- ENV: `MINDNET_SCHEMA_VERSION=2` (Importer wird v2 schreiben, v1 bleibt unberührt).
|
||||||
|
|
||||||
|
### Step 1 – Schemata (nur Dateien)
|
||||||
|
- Ablage: `/schemas/note.schema.json`, `/schemas/chunk.schema.json`, `/schemas/edge.schema.json`, `/schemas/default_edge.schema.json`
|
||||||
|
- CI‑Job: `make schema-validate` (jsonschema).
|
||||||
|
- **Abnahme:** Validator grün auf 3 Gold‑Notizen.
|
||||||
|
|
||||||
|
### Step 2 – Qdrant v2‑Collections (Optional)
|
||||||
|
- Anlegen der 3 v2‑Collections + Payload‑Indizes.
|
||||||
|
- **Abnahme:** Filter‑Query liefert erwartbare Ergebnisse.
|
||||||
|
|
||||||
|
### Step 2a – Edge-Pipeline v1 stabilisieren (ERLEDIGT)
|
||||||
|
- Ziel: Vollständige Generierung von Struktur-, expliziten, typbasierten und Inline-Kanten in den bestehenden Collections `${prefix}_notes`, `${prefix}_chunks`, `${prefix}_edges`.
|
||||||
|
- Status: Implementiert und durch `tests/test_edges_smoke.py`, `tests/test_edges_all.py` und `scripts/edges_full_check.py` verifiziert.
|
||||||
|
|
||||||
|
|
||||||
|
### Step 3 – Chunker v2
|
||||||
|
- Implementierung & Flag `--chunker v2`.
|
||||||
|
- **Abnahme:** Chunk‑Counts ~ alt (≈171 ± 15 %), semantische Schnitte.
|
||||||
|
|
||||||
|
### Step 4 – Edge‑Builder v2
|
||||||
|
- Reihenfolge & `provenance` strikt umsetzen, `default_resolved` integrieren.
|
||||||
|
- **Abnahme:** erwartete Relationensätze auf Gold‑Notizen, keine Duplikate.
|
||||||
|
|
||||||
|
### Step 4a – Edge-bewusster Retriever
|
||||||
|
- Implementiere einen Retrieval-Service, der:
|
||||||
|
- Chunks über Vektor-Suche aus `${prefix}_chunks` holt,
|
||||||
|
- dazu passende Nachbar-Chunks (`next/prev`) und typrelevante Notizen (`edge_defaults`, Inline-Relationen, Wikilinks) über `${prefix}_edges` ergänzt,
|
||||||
|
- `retriever_weight` und `confidence` beim Scoring berücksichtigt.
|
||||||
|
- Liefere einen API-Endpoint `/mindnet/query` mit einem einfachen JSON-Response (Chunks + Begründung).
|
||||||
|
- Akzeptanz:
|
||||||
|
- mindestens ein Testfall mit dem Relations-Showcase-Vault,
|
||||||
|
- sichtbarer Einfluss von `edge_defaults` und Inline-Relationen auf die Ergebnisliste.
|
||||||
|
|
||||||
|
|
||||||
|
### Step 5 – Importer‑Pipeline v2
|
||||||
|
- `--schema v2` Side‑by‑Side; Batch‑Upserts; vollständige Payloads; Recency‑Boost Konfig **nur** in Policy.
|
||||||
|
- **Abnahme:** Dry‑Run/Apply‑Parität, Reports, Telemetrie‑Ereignisse sichtbar.
|
||||||
|
|
||||||
|
### Step 6 – Observability
|
||||||
|
- OTel‑Instrumentierung + `/import/status` + Reports.
|
||||||
|
- **Abnahme:** Traces & Metriken vorhanden.
|
||||||
|
|
||||||
|
### Step 7 – Snapshots & Runbook
|
||||||
|
- Snapshot nach Erfolg; Restore‑Dokument.
|
||||||
|
- **Abnahme:** Restore‑Probe erfolgreich.
|
||||||
|
|
||||||
|
### Step 8 – Policies
|
||||||
|
- `policies/retrieval.yaml` + JSON‑Schema; A/B‑Test (Policy ändern → Retrieval ändert sich ohne Reimport).
|
||||||
|
- **Abnahme:** Sichtbarer Effekt laut Testfall.
|
||||||
|
|
||||||
|
### Step 9 – Umschalten
|
||||||
|
- Abnahme‑Doku, KPI‑Delta dokumentiert; Alias/Flag‑Switch auf v2.
|
||||||
|
- **Abnahme:** Funktionale Gleichwertigkeit + Qualitätsgewinn.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9) Dateien (neu/ändern)
|
||||||
|
**Neu:**
|
||||||
|
- `schemas/note.schema.json`
|
||||||
|
- `schemas/chunk.schema.json`
|
||||||
|
- `schemas/edge.schema.json`
|
||||||
|
- `schemas/default_edge.schema.json`
|
||||||
|
- `policies/retrieval.schema.json`
|
||||||
|
- `policies/retrieval.yaml`
|
||||||
|
- `chunking/block_chunker.py`
|
||||||
|
- `graph/edge_builder_v2.py`
|
||||||
|
- `docs/OPERATIONS.md` (Snapshots/Restore/Runbook)
|
||||||
|
- `tests/gold_notes/manifest.yaml` (+ 3–5 Notizen Kopien)
|
||||||
|
|
||||||
|
**Änderungen:**
|
||||||
|
- `scripts/import_markdown.py` **oder** `scripts/import_markdown_v2.py` neu (Flags `--schema/--chunker/--edges`).
|
||||||
|
- `edges.py` (falls weiterverwendet): auf **v2‑Edge‑Builder** migrieren oder auslaufen lassen.
|
||||||
|
- `types.yaml`: Regel‑Definitionen klarisieren (IDs, Bedingungen), **nicht** hart in Code.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10) Beispiel‑Kommandos
|
||||||
|
```bash
|
||||||
|
# Step 2: Collections prüfen
|
||||||
|
curl -s http://127.0.0.1:6333/collections | jq
|
||||||
|
|
||||||
|
# Step 3: Chunker v2 (dry-run)
|
||||||
|
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --dry-run --prefix "$COLLECTION_PREFIX"
|
||||||
|
|
||||||
|
# Step 5: Apply + Report
|
||||||
|
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --apply --prefix "$COLLECTION_PREFIX" --report ./reports/import_$(date +%F_%H%M).json
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11) Rollback
|
||||||
|
- v2‑Collections droppen (wenn Side‑by‑Side).
|
||||||
|
- Aus Snapshot wiederherstellen (v1).
|
||||||
|
- Flags zurück auf v1.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12) Referenzen (offizielle Quellen)
|
||||||
|
- Qdrant – Collections, Payload‑Index, Filter: <https://qdrant.tech/documentation/>
|
||||||
|
- Qdrant – Snapshots/Backup: <https://qdrant.tech/documentation/guides/backup/>
|
||||||
|
- OpenTelemetry (Python/FastAPI): <https://opentelemetry.io/docs/instrumentation/python/>
|
||||||
|
- JSON‑Schema 2020‑12: <https://json-schema.org/>
|
||||||
|
- YAML 1.2: <https://yaml.org/spec/1.2.2/>
|
||||||
|
- Obsidian Frontmatter: <https://help.obsidian.md/Editing+and+formatting/Properties>
|
||||||
|
- Ollama CLI/API (lokale Modelle): <https://github.com/ollama/ollama>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13) **PROMPT für den neuen Chat (bitte exakt so einfügen)**
|
||||||
|
```
|
||||||
|
Rolle: Du bist mein Senior‑Entwickler & Architekt für mindnet. Arbeite strikt in kleinen, testbaren Schritten (Side‑by‑Side v2 neben v1). Liefere komplette Dateien als Downloads. Frage immer zuerst nach den **aktuellsten** Projektdateien, wenn du sie brauchst (z. B. import_markdown.py, edges.py, types.yaml), ändere keine Systemfunktion „workaround‑artig“ ohne die Gesamtwirkung zu prüfen.
|
||||||
|
|
||||||
|
Kontext (Kurzfassung):
|
||||||
|
- v1 hat nur mindnet_notes zuverlässig. Chunks/Edges sind inkonsistent/teilweise leer. Ziel: v2 mit drei Collections (notes/chunks/edges), neuem Chunker v2, Edge‑Builder v2 (explicit → rule → default_resolved → default), Policies (YAML), Observability (OTel), Snapshots. Kein Big‑Bang; v2 parallel zu v1.
|
||||||
|
- Siehe angehängte Datei **mindnet_v2_implementation_playbook.md** (Pfad/Download im Chat). Alles daraus ist verbindlich.
|
||||||
|
|
||||||
|
Deine ersten Aufgaben (Step‑by‑Step):
|
||||||
|
1) **Schemata anlegen (Step 1)**
|
||||||
|
- Erstelle die Dateien:
|
||||||
|
- schemas/note.schema.json
|
||||||
|
- schemas/chunk.schema.json
|
||||||
|
- schemas/edge.schema.json
|
||||||
|
- schemas/default_edge.schema.json
|
||||||
|
- Nutze die in der Playbook‑Datei vorgegebenen Strukturen als Basis.
|
||||||
|
- Liefere zusätzlich ein einfaches `Makefile`‑Ziel `schema-validate` (jsonschema via Python), plus `requirements.txt`.
|
||||||
|
- Output: komplette Dateien als Downloads + kurze Testanleitung.
|
||||||
|
|
||||||
|
2) **Qdrant v2‑Collections (Step 2)**
|
||||||
|
- Erzeuge die Collections + Payload‑Indizes.
|
||||||
|
- Liefere ein kleines Python‑Skript `tools/qdrant_bootstrap_v2.py`, das diese Anlage idempotent durchführt (inkl. Prüf‑Output).
|
||||||
|
|
||||||
|
3) **Chunker v2 (Step 3)**
|
||||||
|
- Implementiere `chunking/block_chunker.py` mit den im Playbook genannten Parametern.
|
||||||
|
- Teste gegen 3 Gold‑Notizen (liefere mini Test‑Harness in `tests/gold_notes/…`).
|
||||||
|
|
||||||
|
Arbeitsweise:
|
||||||
|
- Jede Aufgabe einzeln, mit klaren Akzeptanzkriterien (aus Playbook) und Download‑Artefakten.
|
||||||
|
- Keine parallelen Großumbauten.
|
||||||
|
- Immer deterministische IDs, idempotente Upserts.
|
||||||
|
|
||||||
|
Dateien, die ich dir anfänglich bereitstelle:
|
||||||
|
- **mindnet_v2_implementation_playbook.md** (diese Datei)
|
||||||
|
- **import_markdown.py** (aktuelle Version aus dem Projekt)
|
||||||
|
- **edges.py** (aktuelle Version aus dem Projekt)
|
||||||
|
- **types.yaml** (aktuelle Version)
|
||||||
|
|
||||||
|
Sage mir jeweils, welche Datei du als Nächstes brauchst. Beginne jetzt mit Aufgabe 1 (Schemata).
|
||||||
|
```
|
||||||
@@ -0,0 +1,198 @@
|
|||||||
|
# mindnet v2.2 – Pipeline Playbook
|
||||||
|
**Datei:** `docs/mindnet_pipeline_playbook_v2.2.md`
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** **FINAL** (Inkl. WP05 RAG Pipeline)
|
||||||
|
**Quellen:** `mindnet_v2_implementation_playbook.md`, `Handbuch.md`, `chunking_strategy.md`, `docs_mindnet_retriever.md`, `mindnet_admin_guide_v2.2.md`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zweck & Einordnung
|
||||||
|
|
||||||
|
Dieses Playbook ist das zentrale operative Handbuch für die **mindnet-Pipeline**. Es beschreibt, wie Daten vom Markdown-Vault in den Wissensgraphen (Qdrant) gelangen, wie der Retriever betrieben wird und wie die **RAG-Generierung** funktioniert.
|
||||||
|
|
||||||
|
**Zielgruppe:** Dev/Ops, Tech-Leads.
|
||||||
|
**Scope:**
|
||||||
|
* **Ist-Stand (WP01–WP05):** Import, Chunking, Edge-Erzeugung, Hybrider Retriever, RAG-Chat, Feedback Loop.
|
||||||
|
* **Roadmap (Ausblick):** Technische Skizzen für Self-Healing (WP06) und Self-Tuning (WP08).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Die Import-Pipeline (Runbook)
|
||||||
|
|
||||||
|
Der Import ist der kritischste Prozess ("Data Ingestion"). Er muss **deterministisch** und **idempotent** sein. Wir nutzen `scripts/import_markdown.py` als zentralen Entrypoint.
|
||||||
|
|
||||||
|
### 2.1 Der 12-Schritte-Prozess
|
||||||
|
Gemäß WP03-Spezifikation läuft der Import intern wie folgt ab:
|
||||||
|
|
||||||
|
1. **Markdown lesen:** Rekursives Scannen des Vaults.
|
||||||
|
2. **Frontmatter extrahieren:** Validierung von Pflichtfeldern (`id`, `type`, `title`).
|
||||||
|
3. **Typauflösung:** Bestimmung des `type` via `types.yaml` (Prio: Frontmatter > Pfad > Default).
|
||||||
|
4. **Note-Payload generieren:** Erstellen des JSON-Objekts für `mindnet_notes`.
|
||||||
|
5. **Chunking anwenden:** Zerlegung des Textes basierend auf dem `chunk_profile` des Typs.
|
||||||
|
6. **Inline-Kanten finden:** Parsing von `[[rel:...]]` im Fließtext.
|
||||||
|
7. **Callout-Kanten finden:** Parsing von `> [!edge]` Blöcken.
|
||||||
|
8. **Default-Edges erzeugen:** Anwendung der `edge_defaults` aus der Typ-Registry.
|
||||||
|
9. **Strukturkanten erzeugen:** `belongs_to` (Chunk->Note), `next`/`prev` (Sequenz).
|
||||||
|
10. **Chunks upserten:** Schreiben in Qdrant (`mindnet_chunks`).
|
||||||
|
11. **Edges upserten:** Schreiben in Qdrant (`mindnet_edges`).
|
||||||
|
12. **Diagnose:** Automatischer Check der Integrität nach dem Lauf.
|
||||||
|
|
||||||
|
### 2.2 Standard-Betrieb (Inkrementell)
|
||||||
|
Für regelmäßige Updates (z.B. Cronjob). Erkennt Änderungen via Hash.
|
||||||
|
|
||||||
|
export QDRANT_URL="http://localhost:6333"
|
||||||
|
export COLLECTION_PREFIX="mindnet"
|
||||||
|
|
||||||
|
# Import starten (Apply = Schreiben, Purge = Sauberer Upsert)
|
||||||
|
# Nutzt das Venv der Produktionsumgebung
|
||||||
|
/home/llmadmin/mindnet/.venv/bin/python3 -m scripts.import_markdown \
|
||||||
|
--vault ./vault \
|
||||||
|
--prefix "$COLLECTION_PREFIX" \
|
||||||
|
--apply \
|
||||||
|
--purge-before-upsert \
|
||||||
|
--sync-deletes
|
||||||
|
|
||||||
|
* `--apply`: Ohne dieses Flag läuft ein Dry-Run (nur Simulation).
|
||||||
|
* `--purge-before-upsert`: Löscht vor dem Schreiben einer Note ihre alten Chunks/Edges. Essentiell, um "Geister-Chunks" zu vermeiden, wenn Text gekürzt wurde.
|
||||||
|
* `--sync-deletes`: Entfernt Notizen aus Qdrant, die im Vault gelöscht wurden.
|
||||||
|
|
||||||
|
### 2.3 Deployment & Restart (Systemd)
|
||||||
|
Nach einem Import oder Code-Update muss der API-Prozess neu gestartet werden.
|
||||||
|
|
||||||
|
# Neustart des Produktions-Services
|
||||||
|
sudo systemctl restart mindnet-prod
|
||||||
|
|
||||||
|
# Prüfung
|
||||||
|
sudo systemctl status mindnet-prod
|
||||||
|
|
||||||
|
### 2.4 Full Rebuild (Clean Slate)
|
||||||
|
Notwendig bei Änderungen an `types.yaml` (z.B. neue Chunk-Größen) oder Embedding-Modellen.
|
||||||
|
|
||||||
|
# 1. Qdrant Collections löschen und neu anlegen (Wipe inkl. Schema)
|
||||||
|
python3 -m scripts.reset_qdrant --mode wipe --prefix "mindnet" --yes
|
||||||
|
|
||||||
|
# 2. Vollständiger Import aller Dateien
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --prefix "mindnet" --apply
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Chunking & Payload-Aufbau
|
||||||
|
|
||||||
|
Das Chunking ist profilbasiert und typgesteuert.
|
||||||
|
|
||||||
|
### 3.1 Chunk-Profile
|
||||||
|
In `types.yaml` definiert. Standard-Profile (in `chunk_config.py` implementiert):
|
||||||
|
* `short`: Max 128 Tokens (z.B. für Logs, Chats).
|
||||||
|
* `medium`: Max 256 Tokens (z.B. für Konzepte).
|
||||||
|
* `long`: Max 512 Tokens (z.B. für Essays, Projekte).
|
||||||
|
* `by_heading`: Trennt strikt an Überschriften.
|
||||||
|
|
||||||
|
### 3.2 Payload-Felder
|
||||||
|
Jeder Chunk erhält zwei Text-Felder:
|
||||||
|
* `text`: Der reine Inhalt des Chunks (ohne Overlap). Wird dem Nutzer angezeigt.
|
||||||
|
* `window`: Der Inhalt plus Overlap zu Vorgänger/Nachfolger. Wird für das Embedding genutzt (besserer Kontext).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Edge-Erzeugung (Die V2-Logik)
|
||||||
|
|
||||||
|
In v2.2 entstehen Kanten nach strenger Priorität.
|
||||||
|
|
||||||
|
### 4.1 Prioritäten & Provenance
|
||||||
|
Der Importer setzt `provenance`, `rule_id` und `confidence` automatisch:
|
||||||
|
|
||||||
|
| Priorität | Quelle | Syntax (Bsp.) | Rule ID | Confidence |
|
||||||
|
| :--- | :--- | :--- | :--- | :--- |
|
||||||
|
| **1** | Inline | `[[rel:depends_on X]]` | `inline:rel` | ~0.95 |
|
||||||
|
| **2** | Callout | `> [!edge] related_to: [[X]]` | `callout:edge` | ~0.90 |
|
||||||
|
| **3** | Wikilink | `[[X]]` | `explicit:wikilink` | 1.00 |
|
||||||
|
| **4** | Default | *(via types.yaml)* | `edge_defaults:...` | ~0.70 |
|
||||||
|
| **5** | Struktur | *(automatisch)* | `structure:...` | 1.00 |
|
||||||
|
|
||||||
|
### 4.2 Typ-Defaults
|
||||||
|
Wenn in `types.yaml` für einen Typ `edge_defaults` definiert sind, werden diese **additiv** zu expliziten Links erzeugt.
|
||||||
|
* *Beispiel:* Note Typ `project` verlinkt `[[Tool A]]`.
|
||||||
|
* *Ergebnis:* Kante `references` (explizit) UND Kante `depends_on` (Default).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Retriever, Chat & Generation (RAG Pipeline)
|
||||||
|
|
||||||
|
Der Datenfluss endet nicht beim Finden. Er geht weiter bis zur Antwort.
|
||||||
|
|
||||||
|
### 5.1 Retrieval (Hybrid)
|
||||||
|
total_score =
|
||||||
|
semantic_weight * semantic_score
|
||||||
|
+ edge_weight * edge_bonus
|
||||||
|
+ centrality_weight * centrality_bonus
|
||||||
|
+ type_weight * retriever_weight
|
||||||
|
|
||||||
|
Der `/chat` Endpunkt nutzt **Hybrid Retrieval** (Semantic + Graph), um auch logisch verbundene, aber textlich unterschiedliche Notizen zu finden (z.B. Decisions zu einem Projekt).
|
||||||
|
|
||||||
|
### 5.2 Context Enrichment (Das "Context Intelligence" Pattern)
|
||||||
|
Bevor der Text an das LLM geht, reichert der Router (`chat.py`) ihn an.
|
||||||
|
* **Metadaten-Injection:** `[DECISION]`, `[PROJECT]`, `[SCORE: 0.9]`.
|
||||||
|
* **Zweck:** Ermöglicht kleinen Modellen (Phi-3) das Erkennen von logischen Rollen ("Warum?" vs "Was?").
|
||||||
|
|
||||||
|
### 5.3 Generation (LLM)
|
||||||
|
* **Engine:** Ollama (lokal).
|
||||||
|
* **Modell:** `phi3:mini` (Standard).
|
||||||
|
* **Prompting:** Gesteuert über `config/prompts.yaml`.
|
||||||
|
|
||||||
|
### 5.4 Explanation Mode (WP04b)
|
||||||
|
Wird `/query` mit `explain=True` aufgerufen, führt der Retriever eine Post-Processing-Analyse durch und liefert `reasons` ("Verweist auf...", "Hoher Typ-Bonus").
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Feedback & Lernen (WP04c)
|
||||||
|
|
||||||
|
Das System schreibt kontinuierlich Logs ("Data Flywheel"):
|
||||||
|
* `data/logs/search_history.jsonl`: Trainingsdaten (Query + Ergebnisse + Breakdown).
|
||||||
|
* `data/logs/feedback.jsonl`: Labels (User-Rating zur `query_id`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Quality Gates & Tests
|
||||||
|
|
||||||
|
Diese Tests garantieren die Stabilität der Pipeline.
|
||||||
|
|
||||||
|
### 7.1 Pflicht-Tests vor Commit
|
||||||
|
|
||||||
|
1. **Payload Dryrun (Schema-Check):**
|
||||||
|
Simuliert Import, prüft JSON-Schema Konformität.
|
||||||
|
|
||||||
|
python3 -m scripts.payload_dryrun --vault ./test_vault
|
||||||
|
|
||||||
|
2. **Full Edge Check (Graph-Integrität):**
|
||||||
|
Prüft Invarianten (z.B. `next` muss reziprok zu `prev` sein).
|
||||||
|
|
||||||
|
python3 -m scripts.edges_full_check
|
||||||
|
|
||||||
|
### 7.2 Smoke-Test (E2E)
|
||||||
|
Prüft am laufenden System (Prod oder Dev), ob Semantik, Graph und Feedback funktionieren.
|
||||||
|
|
||||||
|
# Retriever Test
|
||||||
|
python scripts/test_retriever_smoke.py --mode hybrid --top-k 5
|
||||||
|
|
||||||
|
# Chat Test (Neu WP05)
|
||||||
|
python tests/test_chat_wp05.py
|
||||||
|
|
||||||
|
# Feedback Test
|
||||||
|
python tests/test_feedback_smoke.py --url http://localhost:8001/query
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Ausblick & Roadmap (Technische Skizzen)
|
||||||
|
|
||||||
|
Wie entwickeln wir die Pipeline weiter?
|
||||||
|
|
||||||
|
### 8.1 WP-06: Decision Engine (Skizze)
|
||||||
|
**Ziel:** Aktive Entscheidungsberatung.
|
||||||
|
**Erweiterung:** Der Chat-Router lädt bei Entscheidungfragen gezielt `type: value` Notizen nach, um Optionen gegen Werte abzuwägen.
|
||||||
|
|
||||||
|
### 8.2 WP-08: Self-Tuning (Skizze)
|
||||||
|
**Ziel:** Die Gewichte in `retriever.yaml` basierend auf `feedback.jsonl` optimieren.
|
||||||
|
**Ansatz:** Ein Offline-Learning-Skript `scripts/optimize_weights.py`.
|
||||||
|
1. **Load:** Liest `search_history.jsonl` und joint mit `feedback.jsonl` via `query_id`.
|
||||||
|
2. **Analyze:** Korrelation Scores vs. User-Rating.
|
||||||
|
3. **Optimize:** Vorschlag neuer Gewichte für `retriever.yaml`.
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
# Mindnet v2.2 – User Guide
|
||||||
|
**Datei:** `docs/mindnet_user_guide_v2.2.md`
|
||||||
|
**Stand:** 2025-12-08
|
||||||
|
**Status:** **FINAL** (Inkl. RAG & Chat)
|
||||||
|
**Quellen:** `knowledge_design.md`, `wp04_retriever_scoring.md`, `Programmplan_V2.2.md`, `Handbuch.md`.
|
||||||
|
|
||||||
|
> **Willkommen bei Mindnet.**
|
||||||
|
> Dies ist dein persönliches Wissensnetzwerk. Im Gegensatz zu einer normalen Suche (wie Google Drive), die nur Texte findet, "denkt" Mindnet mit. Es erklärt dir, warum es etwas gefunden hat, und kann dir im Chat Fragen beantworten.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Was Mindnet für dich tut
|
||||||
|
|
||||||
|
Mindnet ist ein **assoziatives Gedächtnis**. Es verbindet deine Notizen zu einem Graphen.
|
||||||
|
|
||||||
|
### 1.1 Mehr als nur Stichworte
|
||||||
|
Wenn du nach "Projekt Alpha" suchst, findet Mindnet nicht nur das Dokument mit diesem Titel. Es findet auch:
|
||||||
|
* **Abhängigkeiten:** "Technologie X wird benötigt" (weil sie verlinkt ist).
|
||||||
|
* **Entscheidungen:** "Warum nutzen wir X?" (weil eine Decision-Note existiert).
|
||||||
|
* **Ähnliches:** "Projekt Beta war ähnlich" (weil es strukturell verwandt ist).
|
||||||
|
|
||||||
|
### 1.2 Dein KI-Zwilling (Vision)
|
||||||
|
Mindnet speichert nicht nur Fakten, sondern deine **Perspektive**. Durch die Gewichtung von Typen (z.B. "Prinzipien sind wichtiger als Quellen") lernt das System, Antworten so zu priorisieren, wie du es tun würdest.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Gute Fragen an Mindnet stellen
|
||||||
|
|
||||||
|
Mindnet nutzt eine **Hybride Suche**. Das heißt, es schaut auf deine Worte (Semantik) und auf das Netzwerk (Graph).
|
||||||
|
|
||||||
|
### 2.1 Frage-Muster
|
||||||
|
Um gute Antworten zu erhalten, formuliere deine Anfragen präzise:
|
||||||
|
|
||||||
|
* **Faktensuche:** "Wie installiere ich Qdrant?"
|
||||||
|
* *Mindnet sucht:* Chunks mit technischer Anleitung.
|
||||||
|
* **Zusammenhänge:** "Womit hängt Projekt Alpha zusammen?"
|
||||||
|
* *Mindnet sucht:* Den Projekt-Knoten und folgt den Kanten (`depends_on`, `related_to`).
|
||||||
|
* **Entscheidungen:** "Warum nutzen wir Vektordatenbanken?"
|
||||||
|
* *Mindnet sucht:* Notizen vom Typ `decision` oder `principle`.
|
||||||
|
|
||||||
|
### 2.2 Tipps für bessere Ergebnisse
|
||||||
|
1. **Nutze deine Sprache:** Verwende die Begriffe, die du auch in deinen Notizen benutzt hast.
|
||||||
|
2. **Kontext geben:** Statt nur "Qdrant" zu tippen, frage "Qdrant Setup für Mindnet", um den Kontext einzuschränken.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Ergebnisse interpretieren (Explanation Layer)
|
||||||
|
|
||||||
|
Mindnet liefert nicht einfach nur Treffer. Es liefert eine **Begründung** (Explanation). Achte auf diese Hinweise in der Antwort:
|
||||||
|
|
||||||
|
### 3.1 Die Gründe ("Reasons")
|
||||||
|
Das System sagt dir in natürlicher Sprache, warum ein Treffer relevant ist:
|
||||||
|
* *"Hohe textuelle Übereinstimmung."* (Semantik war stark)
|
||||||
|
* *"Bevorzugt aufgrund des Typs 'decision'."* (Typ war wichtig)
|
||||||
|
* *"Verweist auf 'Projekt X' via 'depends_on'."* (Dieser Treffer ist eine wichtige Grundlage für deine Suche)
|
||||||
|
* *"Wird referenziert von 'Wichtige Notiz Y'."* (Dieser Treffer ist eine Autorität im Netzwerk)
|
||||||
|
|
||||||
|
### 3.2 Der Score Breakdown
|
||||||
|
Wenn du es genau wissen willst, schau auf die Aufschlüsselung:
|
||||||
|
Score = (Text) + (Typ-Bonus) + (Vernetzungs-Bonus)
|
||||||
|
|
||||||
|
Ein Treffer mit niedrigem Text-Score kann trotzdem auf Platz 1 landen, wenn er extrem gut vernetzt ist ("Hidden Champion").
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Neues Wissen beisteuern (Authoring)
|
||||||
|
|
||||||
|
Mindnet lebt von deinem Input. Du musst kein Techniker sein, um gutes Wissen zu designen. Du schreibst einfach Markdown.
|
||||||
|
|
||||||
|
### 4.1 Die Goldene Regel: "Verlinke semantisch"
|
||||||
|
Statt einfach nur `[[Link]]` zu schreiben, versuche zu sagen, *wie* es zusammenhängt.
|
||||||
|
* Hängt es davon ab? -> `[[rel:depends_on Ziel]]`
|
||||||
|
* Ist es ähnlich? -> `[[rel:similar_to Ziel]]`
|
||||||
|
* Ist es eine Folge davon? -> `[[rel:caused_by Ziel]]`
|
||||||
|
|
||||||
|
### 4.2 Struktur hilft dem Chunker
|
||||||
|
Mindnet zerlegt deinen Text in Häppchen ("Chunks"). Hilf dabei:
|
||||||
|
* Verwende **Überschriften** (##), um Themen zu trennen.
|
||||||
|
* Schreibe **Absätze**, die einen Gedanken fassen.
|
||||||
|
|
||||||
|
### 4.3 Typen nutzen
|
||||||
|
Setze im Frontmatter deiner Notizen den richtigen Typ (z.B. `type: project`, `type: decision`). Das steuert automatisch, wie wichtig die Notiz ist.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Feedback & Lernen
|
||||||
|
|
||||||
|
Mindnet wird schlauer, wenn du es pflegst.
|
||||||
|
|
||||||
|
### 5.1 Feedback geben (Data Flywheel)
|
||||||
|
Das System zeichnet nun auf, welche Ergebnisse es liefert (`search_history`).
|
||||||
|
Du kannst Treffer bewerten (1-5 Sterne oder Daumen hoch/runter).
|
||||||
|
* **Was passiert damit?** Mindnet speichert "Situation" (Query) und "Reaktion" (Rating).
|
||||||
|
* **Wozu?** In Zukunft analysiert das System diese Daten, um zu lernen: "Aha, der Nutzer mag Entscheidungen lieber als Quellen". Es passt seine Gewichte dann selbstständig an (Self-Tuning).
|
||||||
|
|
||||||
|
### 5.2 Ergebnis fehlt?
|
||||||
|
* Existiert die Notiz im Vault?
|
||||||
|
* Ist sie isoliert (keine Links)? Isolierte Notizen haben keinen Graph-Bonus. **Verlinke sie!**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Chat mit Mindnet (Neu in v2.2)
|
||||||
|
|
||||||
|
Neben der Suche (`/query`) gibt es jetzt den Chat (`/chat`).
|
||||||
|
|
||||||
|
### 6.1 Wann Chat, wann Suche?
|
||||||
|
* **Suche:** Wenn du ein konkretes Dokument oder einen Link brauchst.
|
||||||
|
* **Chat:** Wenn du eine Frage hast, die sich aus mehreren Dokumenten zusammensetzt.
|
||||||
|
* *Frage:* "Was ist der Status von Projekt X und welche Risiken gibt es?"
|
||||||
|
* *Antwort:* Mindnet liest die Projekt-Notiz UND verknüpfte Risiko-Notizen und fasst sie zusammen.
|
||||||
|
|
||||||
|
### 6.2 Die Persönlichkeit
|
||||||
|
Der Chatbot agiert als dein "Digitaler Zwilling". Er versucht:
|
||||||
|
1. **Pragmatisch** zu sein (Lösungen statt Theorie).
|
||||||
|
2. **Transparent** zu sein (er sagt, wenn er etwas nicht weiß).
|
||||||
|
3. **Wertebasiert** zu handeln (er bevorzugt Lösungen, die deinen `type: value` Notizen entsprechen).
|
||||||
|
|
||||||
|
### 6.3 Quellen-Check
|
||||||
|
Der Chatbot halluziniert nicht (oder sehr selten). Unter jeder Antwort listet er die **Quellen** auf, die er genutzt hat.
|
||||||
|
* Prüfe immer: Hat er die richtige `[DECISION]`-Notiz gefunden?
|
||||||
|
* Falls nein: Füge in deinem Vault einen Link (`[[rel:depends_on]]`) hinzu, damit er den Zusammenhang beim nächsten Mal sieht.
|
||||||
@@ -0,0 +1,260 @@
|
|||||||
|
# WP-04 / Step 4a – Retriever Scoring & Konfiguration
|
||||||
|
|
||||||
|
Dieses Dokument beschreibt den aktuellen Stand (2025-11-30) der Scoring-Logik des mindnet-Retrievers in WP-04 / Step 4a.
|
||||||
|
Es dient als Referenz für:
|
||||||
|
|
||||||
|
- Nachvollziehbarkeit des Rankings
|
||||||
|
- Konfiguration und Feintuning (ohne Codeänderungen)
|
||||||
|
- spätere Automatisierung / „Selbstjustierung“ des Retrievers
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Überblick
|
||||||
|
|
||||||
|
Der Retriever kombiniert drei Signalquellen zu einem einheitlichen Score:
|
||||||
|
|
||||||
|
1. **Semantik** – Vektorsuche über `mindnet_chunks`
|
||||||
|
2. **Typ-Gewichtung** – `retriever_weight` pro Note/Chunk (aus `types.yaml`)
|
||||||
|
3. **Graph-Boni** – `edge_bonus` und `centrality_bonus` aus dem Subgraph (Edges)
|
||||||
|
|
||||||
|
Die Berechnung erfolgt in `app/core/retriever.py` und ist über `config/retriever.yaml` konfigurierbar.
|
||||||
|
Die API stellt die Funktionalität über den `/query`-Endpoint (FastAPI) bereit.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Datenbasis (Kurzüberblick)
|
||||||
|
|
||||||
|
### 2.1 Notes (`<prefix>_notes`)
|
||||||
|
|
||||||
|
Relevante Felder:
|
||||||
|
|
||||||
|
- note_id: keyword
|
||||||
|
- title: text
|
||||||
|
- type: keyword
|
||||||
|
- retriever_weight: float
|
||||||
|
- chunk_profile: keyword
|
||||||
|
- edge_defaults: keyword[]
|
||||||
|
- tags: keyword[]
|
||||||
|
- path: text
|
||||||
|
- fulltext: text
|
||||||
|
|
||||||
|
### 2.2 Chunks (`<prefix>_chunks`)
|
||||||
|
|
||||||
|
Relevante Felder:
|
||||||
|
|
||||||
|
- note_id: keyword
|
||||||
|
- chunk_id: keyword
|
||||||
|
- text: text
|
||||||
|
- window: text
|
||||||
|
- retriever_weight: float
|
||||||
|
- type: keyword
|
||||||
|
- path: text
|
||||||
|
- section: text
|
||||||
|
- neighbors_prev / neighbors_next
|
||||||
|
- chunk_profile: keyword
|
||||||
|
|
||||||
|
### 2.3 Edges (`<prefix>_edges`)
|
||||||
|
|
||||||
|
Relevante Felder:
|
||||||
|
|
||||||
|
- kind: keyword
|
||||||
|
- source_id: keyword
|
||||||
|
- target_id: keyword
|
||||||
|
- note_id: keyword
|
||||||
|
- confidence: float
|
||||||
|
- rule_id, provenance, edge_id, scope, relation, ref_text
|
||||||
|
|
||||||
|
Edges stammen stets aus der Import-/Edge-Pipeline (WP-03) und bilden die graphbasierte Grundlage des Retrievers.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Scoring-Formel (Retriever)
|
||||||
|
|
||||||
|
### 3.1 Eingangsgrößen
|
||||||
|
|
||||||
|
Für jeden Treffer gelten:
|
||||||
|
|
||||||
|
semantic_score – aus der Vektorsuche (Qdrant)
|
||||||
|
retriever_weight – typabhängiges Gewicht (aus types.yaml)
|
||||||
|
edge_bonus – aggregierter Graph-Bonus
|
||||||
|
centrality_bonus – Zentralitätsbonus aus dem Subgraph
|
||||||
|
|
||||||
|
### 3.2 Globale Gewichte (aus retriever.yaml)
|
||||||
|
|
||||||
|
Datei: `config/retriever.yaml`
|
||||||
|
Beispiel:
|
||||||
|
|
||||||
|
version: 1.0
|
||||||
|
|
||||||
|
scoring:
|
||||||
|
semantic_weight: 1.0 # W_sem
|
||||||
|
edge_weight: 0.7 # W_edge
|
||||||
|
centrality_weight: 0.5 # W_cent
|
||||||
|
|
||||||
|
Falls diese Datei fehlt, greifen ENV-basierte Defaults (`RETRIEVER_W_*`).
|
||||||
|
|
||||||
|
### 3.3 Formel
|
||||||
|
|
||||||
|
total_score =
|
||||||
|
W_sem * semantic_score * max(retriever_weight, 0.0)
|
||||||
|
+ W_edge * edge_bonus
|
||||||
|
+ W_cent * centrality_bonus
|
||||||
|
|
||||||
|
Bemerkungen:
|
||||||
|
|
||||||
|
- Negative `retriever_weight` werden abgefangen.
|
||||||
|
- Die rohen `edge_bonus` und `centrality_bonus` werden im API-/Smoke-Output angezeigt.
|
||||||
|
- Die Gewichte aus `retriever.yaml` beeinflussen **nur den total_score**, nicht die Anzeige.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Konfigurationsdateien
|
||||||
|
|
||||||
|
### 4.1 Typen-Registry (`types.yaml`)
|
||||||
|
|
||||||
|
Beispielauszug:
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
retriever_weight: 1.0
|
||||||
|
chunk_profile: default
|
||||||
|
|
||||||
|
types:
|
||||||
|
types:
|
||||||
|
concept:
|
||||||
|
retriever_weight: 0.60
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
|
||||||
|
project:
|
||||||
|
retriever_weight: 0.97
|
||||||
|
edge_defaults: ["references", "depends_on"]
|
||||||
|
|
||||||
|
journal:
|
||||||
|
retriever_weight: 0.80
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
|
||||||
|
source:
|
||||||
|
retriever_weight: 0.50
|
||||||
|
edge_defaults: []
|
||||||
|
|
||||||
|
Diese Werte bestimmen, wie wichtig eine Note/Chunk **prinzipiell** für den Retriever ist.
|
||||||
|
|
||||||
|
### 4.2 Retriever-Konfiguration (`retriever.yaml`)
|
||||||
|
|
||||||
|
Beispiel:
|
||||||
|
|
||||||
|
version: 1.0
|
||||||
|
|
||||||
|
scoring:
|
||||||
|
semantic_weight: 1.0
|
||||||
|
edge_weight: 0.7
|
||||||
|
centrality_weight: 0.5
|
||||||
|
|
||||||
|
# Blueprint für spätere Edge-Feinsteuerung:
|
||||||
|
edge_types:
|
||||||
|
references: 0.20
|
||||||
|
depends_on: 0.18
|
||||||
|
related_to: 0.15
|
||||||
|
similar_to: 0.12
|
||||||
|
belongs_to: 0.10
|
||||||
|
next: 0.06
|
||||||
|
prev: 0.06
|
||||||
|
|
||||||
|
Die Datei befindet sich standardmäßig unter `config/retriever.yaml`.
|
||||||
|
Override per ENV:
|
||||||
|
|
||||||
|
MINDNET_RETRIEVER_CONFIG=/pfad/zur/datei.yaml
|
||||||
|
|
||||||
|
### 4.3 Aktive Gewichte auslesen
|
||||||
|
|
||||||
|
python - << 'PY'
|
||||||
|
from app.core.retriever import _get_scoring_weights
|
||||||
|
print(_get_scoring_weights())
|
||||||
|
PY
|
||||||
|
|
||||||
|
Beispielausgabe:
|
||||||
|
|
||||||
|
(1.0, 0.7, 0.5)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Betriebsmodi
|
||||||
|
|
||||||
|
### 5.1 Semantikmodus (semantic)
|
||||||
|
|
||||||
|
Ablauf:
|
||||||
|
|
||||||
|
- Query-Vektor über embed_text()
|
||||||
|
- Qdrant-Suche über mindnet_chunks
|
||||||
|
- kein Subgraph → edge_bonus = centrality = 0
|
||||||
|
- total_score = W_sem * semantic_score * retriever_weight
|
||||||
|
|
||||||
|
### 5.2 Hybridmodus (hybrid)
|
||||||
|
|
||||||
|
Ablauf:
|
||||||
|
|
||||||
|
- Semantik wie oben
|
||||||
|
- falls expand.depth > 0:
|
||||||
|
- Subgraph-Berechnung über ga.expand()
|
||||||
|
- Ermittlung von edge_bonus und centrality_bonus
|
||||||
|
- total_score = kombinierte Formel aus Semantik + Graph
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Tests
|
||||||
|
|
||||||
|
### Ausführung (Unit & Integration)
|
||||||
|
|
||||||
|
pytest tests/test_retriever_basic.py \
|
||||||
|
tests/test_retriever_weight.py \
|
||||||
|
tests/test_retriever_edges.py
|
||||||
|
|
||||||
|
pytest tests/test_query_unit.py \
|
||||||
|
tests/test_query_text_embed_unit.py
|
||||||
|
|
||||||
|
Tests decken ab:
|
||||||
|
|
||||||
|
- Semantiktreffer
|
||||||
|
- Typ-Gewichte
|
||||||
|
- Edge-Boni
|
||||||
|
- Hybridmodus / FastAPI-Endpunkt
|
||||||
|
|
||||||
|
### Smoke-Test
|
||||||
|
|
||||||
|
python tests/test_retriever_smoke.py \
|
||||||
|
--url "http://127.0.0.1:8001/query" \
|
||||||
|
--query "embeddings" \
|
||||||
|
--mode hybrid \
|
||||||
|
--expand-depth 1 \
|
||||||
|
--top-k 5
|
||||||
|
|
||||||
|
Beispielausgabe:
|
||||||
|
|
||||||
|
[1] note_id=...retriever-design
|
||||||
|
total=1.5054 (semantic=0.4038 edge=1.5910 centrality=0.0000)
|
||||||
|
|
||||||
|
Interpretation:
|
||||||
|
|
||||||
|
- `semantic`, `edge`, `centrality` = Rohwerte
|
||||||
|
- `total` = skaliert mit `retriever.yaml`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Betriebs-Hinweise
|
||||||
|
|
||||||
|
### 7.1 Änderung der Gewichte
|
||||||
|
|
||||||
|
- retriever.yaml anpassen
|
||||||
|
- FastAPI/uvicorn neu starten
|
||||||
|
- Smoke-Test prüfen
|
||||||
|
|
||||||
|
### 7.2 Zielbild WP-04
|
||||||
|
|
||||||
|
Der jetzige Stand ermöglicht:
|
||||||
|
|
||||||
|
- transparente, konfigurierbare Score-Berechnung
|
||||||
|
- Kombination aus Semantik, Typwissen und Graphstruktur
|
||||||
|
- spätere agentenbasierte Selbstjustierung
|
||||||
|
- alternative Profile (z. B. „mehr Semantik“, „mehr Graph“)
|
||||||
|
|
||||||
|
Dieses Dokument dient als stabile Referenz für Entwickler, Tests und spätere Automatisierungsschritte.
|
||||||
|
|
||||||
+34
-8
@@ -1,9 +1,35 @@
|
|||||||
|
# --- API Server ---
|
||||||
|
fastapi>=0.116.1
|
||||||
|
uvicorn[standard]>=0.35.0
|
||||||
|
# httpx wird oft für Tests benötigt
|
||||||
|
httpx>=0.28.1
|
||||||
|
|
||||||
fastapi
|
# --- Datenbank ---
|
||||||
uvicorn[standard]
|
qdrant-client>=1.15.1
|
||||||
qdrant-client
|
|
||||||
sentence-transformers
|
# --- Datenmodelle & Validierung ---
|
||||||
torch
|
pydantic>=2.11.7
|
||||||
python-frontmatter
|
numpy>=2.3.2
|
||||||
tqdm
|
|
||||||
requests
|
# --- Markdown & Parsing (Hier fehlten Pakete!) ---
|
||||||
|
python-frontmatter>=1.1.0
|
||||||
|
# WICHTIG: Das fehlte und verursachte den Fehler
|
||||||
|
markdown-it-py>=3.0.0
|
||||||
|
# WICHTIG: Für types.yaml und retriever.yaml
|
||||||
|
PyYAML>=6.0.2
|
||||||
|
python-slugify>=8.0.4
|
||||||
|
|
||||||
|
# --- KI & Embeddings ---
|
||||||
|
sentence-transformers>=5.1.0
|
||||||
|
# Torch wird meist durch sentence-transformers geholt,
|
||||||
|
# aber wir listen es explizit für Stabilität
|
||||||
|
torch>=2.0.0
|
||||||
|
|
||||||
|
# --- Utilities ---
|
||||||
|
# WICHTIG: Damit .env Dateien gelesen werden
|
||||||
|
python-dotenv>=1.1.1
|
||||||
|
requests>=2.32.5
|
||||||
|
tqdm>=4.67.1
|
||||||
|
|
||||||
|
# --- Testing ---
|
||||||
|
pytest>=8.4.2
|
||||||
+35
-29
@@ -1,39 +1,45 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
"title": "mindnet_note",
|
"title": "mindnet note payload",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"description": "Note-Payload (Qdrant). Erweitert um fulltext/hash_raw_body für verlustfreien Export.",
|
|
||||||
|
|
||||||
"required": ["note_id", "title", "type", "status", "created", "path"],
|
|
||||||
|
|
||||||
"properties": {
|
"properties": {
|
||||||
"note_id": { "type": "string" },
|
"note_id": { "type": "string" },
|
||||||
"title": { "type": "string" },
|
"title": { "type": ["string","null"] },
|
||||||
"type": { "type": "string" },
|
"type": { "type": ["string","null"] },
|
||||||
"status": { "type": "string" },
|
"status": { "type": ["string","null"] },
|
||||||
"created": { "type": "string" },
|
"created": { "type": ["string","null"] },
|
||||||
"updated": { "type": "string" },
|
"updated": { "type": ["string","null"] },
|
||||||
|
"path": { "type": ["string","null"] },
|
||||||
|
"tags": { "type": ["array","null"], "items": { "type": "string" } },
|
||||||
|
"area": { "type": ["string","null"] },
|
||||||
|
"project": { "type": ["string","null"] },
|
||||||
|
"source": { "type": ["string","null"] },
|
||||||
|
"lang": { "type": ["string","null"] },
|
||||||
|
"slug": { "type": ["string","null"] },
|
||||||
|
"aliases": { "type": ["array","null"], "items": { "type": "string" } },
|
||||||
|
|
||||||
"tags": { "type": "array", "items": { "type": "string" } },
|
"fulltext": { "type": ["string","null"] },
|
||||||
"area": { "type": "string" },
|
"references": { "type": ["array","null"], "items": { "type": "string" } },
|
||||||
"project": { "type": "string" },
|
|
||||||
"priority": { "type": "number" },
|
|
||||||
"effort_min": { "type": "number" },
|
|
||||||
"due": { "type": "string" },
|
|
||||||
"people": { "type": "array", "items": { "type": "string" } },
|
|
||||||
"aliases": { "type": "array", "items": { "type": "string" } },
|
|
||||||
"lang": { "type": "string" },
|
|
||||||
|
|
||||||
"path": { "type": "string" },
|
"hash_fulltext": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
|
||||||
|
"hash_signature": { "type": ["string","null"] },
|
||||||
|
|
||||||
"references": { "type": "array", "items": { "type": "string" } },
|
"hash_body": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
|
||||||
"depends_on": { "type": "array", "items": { "type": "string" } },
|
"hash_frontmatter": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
|
||||||
"assigned_to": { "type": "array", "items": { "type": "string" } },
|
"hash_full": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
|
||||||
|
|
||||||
"hash_fulltext":{ "type": "string" },
|
"hashes": {
|
||||||
"hash_raw_body":{ "type": "string" },
|
"type": ["object","null"],
|
||||||
"fulltext": { "type": "string" }
|
"description": "Mapping: <mode>:<source>:<normalize> -> sha256 hex",
|
||||||
|
"patternProperties": {
|
||||||
|
"^(body|frontmatter|full):(parsed|raw):(canonical|none)$": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^[a-f0-9]{64}$"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
},
|
},
|
||||||
|
"required": ["note_id"],
|
||||||
"additionalProperties": true
|
"additionalProperties": true
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/assert_payload_schema.py (compat v1.1)
|
||||||
|
|
||||||
|
Prüft, ob die erwarteten Payload-Indizes (payload_schema) auf den drei Collections vorhanden sind.
|
||||||
|
Kompatibel mit älteren qdrant-client Versionen (ohne with_payload_schema im Wrapper).
|
||||||
|
|
||||||
|
Collections & Pflichtfelder:
|
||||||
|
- mindnet_notes : note_id, type, title, updated, tags
|
||||||
|
- mindnet_chunks : note_id, chunk_id, index, type, tags
|
||||||
|
- mindnet_edges : note_id, kind, scope, source_id, target_id, chunk_id
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 -m tests.assert_payload_schema
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
REQUIRED = {
|
||||||
|
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||||
|
"chunks": ["note_id", "chunk_id", "index", "type", "tags"],
|
||||||
|
"edges": ["note_id", "kind", "scope", "source_id", "target_id", "chunk_id"],
|
||||||
|
}
|
||||||
|
|
||||||
|
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||||
|
if hasattr(obj, "model_dump"):
|
||||||
|
return obj.model_dump()
|
||||||
|
if hasattr(obj, "dict"):
|
||||||
|
return obj.dict()
|
||||||
|
return obj if isinstance(obj, dict) else {}
|
||||||
|
|
||||||
|
def _cfg_base_url(cfg) -> str:
|
||||||
|
if getattr(cfg, "url", None):
|
||||||
|
return cfg.url.rstrip('/')
|
||||||
|
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||||
|
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||||
|
return f"http://{host}:{port}"
|
||||||
|
|
||||||
|
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||||
|
req = urllib.request.Request(url, method="GET")
|
||||||
|
if api_key:
|
||||||
|
req.add_header("api-key", api_key)
|
||||||
|
req.add_header("Authorization", f"Bearer {api_key}")
|
||||||
|
req.add_header("Accept", "application/json")
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = resp.read()
|
||||||
|
return json.loads(data.decode("utf-8"))
|
||||||
|
|
||||||
|
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||||
|
# 1) OpenAPI client mit Flag
|
||||||
|
try:
|
||||||
|
oc = getattr(client, "openapi_client", None)
|
||||||
|
if oc is not None and hasattr(oc, "collections_api"):
|
||||||
|
api = oc.collections_api
|
||||||
|
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
return (d.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# 2) Wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
ps = (d.get("result") or {}).get("payload_schema")
|
||||||
|
if ps is not None:
|
||||||
|
return ps
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# 3) Direkter HTTP-Call
|
||||||
|
try:
|
||||||
|
base = _cfg_base_url(cfg)
|
||||||
|
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||||
|
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||||
|
return (raw.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def ensure_fields(schema: Dict[str, dict] | None, required: List[str]) -> Dict[str, bool]:
|
||||||
|
ks = set((schema or {}).keys())
|
||||||
|
return {k: (k in ks) for k in required}
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
names = {"notes": notes, "chunks": chunks, "edges": edges}
|
||||||
|
|
||||||
|
report = {}
|
||||||
|
ok_all = True
|
||||||
|
for kind, col in names.items():
|
||||||
|
sch = get_collection_payload_schema(client, cfg, col)
|
||||||
|
checks = ensure_fields(sch, REQUIRED[kind])
|
||||||
|
ok = all(checks.values())
|
||||||
|
ok_all = ok_all and ok
|
||||||
|
report[kind] = {
|
||||||
|
"collection": col,
|
||||||
|
"ok": ok,
|
||||||
|
"missing": [k for k,v in checks.items() if not v],
|
||||||
|
"present": [k for k,v in checks.items() if v],
|
||||||
|
}
|
||||||
|
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "ok": ok_all, "report": report}, ensure_ascii=False, indent=2))
|
||||||
|
if not ok_all:
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/debug_qdrant_state.py
|
||||||
|
Zeigt Prefix, Collections und einfache Counts (notes/chunks/edges) + sample note_ids.
|
||||||
|
|
||||||
|
Aufruf:
|
||||||
|
export COLLECTION_PREFIX="mindnet"
|
||||||
|
python3 -m scripts.debug_qdrant_state
|
||||||
|
# oder:
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix mindnet
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, os, json
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def count_points(client, collection: str) -> int:
|
||||||
|
try:
|
||||||
|
res = client.count(collection, exact=True)
|
||||||
|
return int(getattr(res, "count", 0))
|
||||||
|
except Exception:
|
||||||
|
pts, _ = client.scroll(collection, limit=1)
|
||||||
|
return 1 if pts else 0
|
||||||
|
|
||||||
|
def sample_ids(client, collection: str, id_key: str, limit: int = 5):
|
||||||
|
pts, _ = client.scroll(collection, with_payload=True, with_vectors=False, limit=limit)
|
||||||
|
out = []
|
||||||
|
for p in pts or []:
|
||||||
|
pl = p.payload or {}
|
||||||
|
if id_key in pl:
|
||||||
|
out.append(pl[id_key])
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX", "mindnet"))
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
cfg.prefix = args.prefix
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
notes = f"{cfg.prefix}_notes"
|
||||||
|
chunks = f"{cfg.prefix}_chunks"
|
||||||
|
edges = f"{cfg.prefix}_edges"
|
||||||
|
|
||||||
|
print(json.dumps({
|
||||||
|
"prefix": cfg.prefix,
|
||||||
|
"collections": {"notes": notes, "chunks": chunks, "edges": edges},
|
||||||
|
"counts": {
|
||||||
|
"notes": count_points(client, notes),
|
||||||
|
"chunks": count_points(client, chunks),
|
||||||
|
"edges": count_points(client, edges),
|
||||||
|
},
|
||||||
|
"samples": {
|
||||||
|
"notes.note_id": sample_ids(client, notes, "note_id"),
|
||||||
|
"chunks.note_id": sample_ids(client, chunks, "note_id"),
|
||||||
|
"edges.note_id": sample_ids(client, edges, "note_id"),
|
||||||
|
}
|
||||||
|
}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/diag_payload_indexes.py (compat v1.2)
|
||||||
|
|
||||||
|
Zeigt payload_schema je Collection; kompatibel mit älteren qdrant-client Versionen.
|
||||||
|
Strategie (in dieser Reihenfolge):
|
||||||
|
1) openapi_client.collections_api.get_collection(..., with_payload_schema=True)
|
||||||
|
2) client.get_collection(...)
|
||||||
|
3) HTTP GET /collections/{name}?with_payload_schema=true
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||||
|
if hasattr(obj, "model_dump"):
|
||||||
|
return obj.model_dump()
|
||||||
|
if hasattr(obj, "dict"):
|
||||||
|
return obj.dict()
|
||||||
|
return obj if isinstance(obj, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _cfg_base_url(cfg) -> str:
|
||||||
|
if getattr(cfg, "url", None):
|
||||||
|
return cfg.url.rstrip("/")
|
||||||
|
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||||
|
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||||
|
return f"http://{host}:{port}"
|
||||||
|
|
||||||
|
|
||||||
|
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||||
|
req = urllib.request.Request(url, method="GET")
|
||||||
|
if api_key:
|
||||||
|
req.add_header("api-key", api_key)
|
||||||
|
req.add_header("Authorization", f"Bearer {api_key}")
|
||||||
|
req.add_header("Accept", "application/json")
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = resp.read()
|
||||||
|
return json.loads(data.decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||||
|
# Strategy 1: OpenAPI client with flag
|
||||||
|
try:
|
||||||
|
oc = getattr(client, "openapi_client", None)
|
||||||
|
if oc is not None and hasattr(oc, "collections_api"):
|
||||||
|
api = oc.collections_api
|
||||||
|
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
return (d.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Strategy 2: wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name) # kein with_payload_schema kwarg!
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
ps = (d.get("result") or {}).get("payload_schema")
|
||||||
|
if ps is not None:
|
||||||
|
return ps
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Strategy 3: direkter HTTP Call
|
||||||
|
try:
|
||||||
|
base = _cfg_base_url(cfg)
|
||||||
|
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||||
|
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||||
|
return (raw.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
cols = [notes, chunks, edges]
|
||||||
|
out = []
|
||||||
|
for name in cols:
|
||||||
|
ps = get_collection_payload_schema(client, cfg, name)
|
||||||
|
# Vektorinfos (optional)
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
vectors = (d.get("result") or {}).get("vectors")
|
||||||
|
if isinstance(vectors, dict) and "config" in vectors:
|
||||||
|
vectors = vectors["config"]
|
||||||
|
except Exception:
|
||||||
|
vectors = None
|
||||||
|
out.append({"name": name, "vectors": vectors, "payload_schema": ps})
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "collections": out}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# scripts/edges_dryrun.py
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Dry-Run: Erzeuge Edges aus einem Vault **ohne** Qdrant-Upsert.
|
||||||
|
- Liest Markdown mit YAML-Frontmatter
|
||||||
|
- Chunking: einfacher Absatz-Chunker (Index + text)
|
||||||
|
- Kanten: nutzt app.core.edges.build_edges_for_note()
|
||||||
|
- Ausgabe: JSON pro Note mit Edge-Counts und 3 Beispiel-Payloads
|
||||||
|
|
||||||
|
Aufruf:
|
||||||
|
python3 -m scripts.edges_dryrun --vault ./vault
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
--include-note-scope-refs # auch Frontmatter-Links (links[].target_id) als Note-Scope-Referenzen
|
||||||
|
|
||||||
|
Voraussetzungen:
|
||||||
|
- app/core/parser.py (read_markdown, normalize_frontmatter)
|
||||||
|
- app/core/edges.py (dieses Modul forwardet zur v2-Implementierung)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, List, Optional
|
||||||
|
|
||||||
|
from app.core.parser import read_markdown, normalize_frontmatter
|
||||||
|
from app.core.edges import build_edges_for_note
|
||||||
|
|
||||||
|
def _iter_markdown(vault: str):
|
||||||
|
for p in Path(vault).rglob("*.md"):
|
||||||
|
if p.name.startswith("."): # ignore hidden
|
||||||
|
continue
|
||||||
|
yield p
|
||||||
|
|
||||||
|
def _simple_chunker(body: str, note_id: str, note_type: str) -> List[Dict]:
|
||||||
|
# Absatzbasiert, minimal ausreichend für Edges (window/text, chunk_id, ord, note_id, type)
|
||||||
|
paras = [s.strip() for s in re.split(r"\n{2,}", body or "") if s.strip()]
|
||||||
|
chunks = []
|
||||||
|
for i, t in enumerate(paras):
|
||||||
|
chunks.append({
|
||||||
|
"chunk_id": f"{note_id}#c{i:04d}",
|
||||||
|
"ord": i,
|
||||||
|
"text": t,
|
||||||
|
"note_id": note_id,
|
||||||
|
"type": note_type,
|
||||||
|
})
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
def _fm_note_refs(fm: Dict) -> List[str]:
|
||||||
|
out = []
|
||||||
|
links = fm.get("links") or []
|
||||||
|
if isinstance(links, list):
|
||||||
|
for e in links:
|
||||||
|
if isinstance(e, dict):
|
||||||
|
tid = e.get("target_id")
|
||||||
|
if isinstance(tid, str) and tid.strip():
|
||||||
|
out.append(tid.strip())
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--vault", required=True)
|
||||||
|
ap.add_argument("--include-note-scope-refs", action="store_true")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
vault = args.vault
|
||||||
|
include_note_scope = args.include_note_scope_refs
|
||||||
|
|
||||||
|
report = []
|
||||||
|
for p in _iter_markdown(vault):
|
||||||
|
parsed = read_markdown(str(p))
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter or {})
|
||||||
|
note_id = fm.get("id") or p.stem
|
||||||
|
note_type = (fm.get("type") or "concept").lower()
|
||||||
|
chunks = _simple_chunker(parsed.body, note_id, note_type)
|
||||||
|
note_refs = _fm_note_refs(fm)
|
||||||
|
|
||||||
|
edges = build_edges_for_note(
|
||||||
|
note_id=note_id,
|
||||||
|
chunk_payloads=chunks,
|
||||||
|
note_level_refs=note_refs,
|
||||||
|
include_note_scope_refs=include_note_scope,
|
||||||
|
)
|
||||||
|
kinds = {}
|
||||||
|
for e in edges:
|
||||||
|
key = (e.get("relation") or e.get("kind") or "edge")
|
||||||
|
kinds[key] = kinds.get(key, 0) + 1
|
||||||
|
report.append({
|
||||||
|
"path": str(p),
|
||||||
|
"note_id": note_id,
|
||||||
|
"type": note_type,
|
||||||
|
"chunks": len(chunks),
|
||||||
|
"edges_total": len(edges),
|
||||||
|
"edges_by_kind": kinds,
|
||||||
|
"samples": edges[:3],
|
||||||
|
})
|
||||||
|
|
||||||
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/edges_full_check.py
|
||||||
|
Zählt und validiert Kanten in Qdrant. Erkennt folgende Rule-Gruppen:
|
||||||
|
- explicit_total: rule_id startswith "explicit:" (z.B. explicit:wikilink, explicit:note_scope)
|
||||||
|
- callout_total: rule_id == "callout:edge"
|
||||||
|
- inline_total: rule_id startswith "inline:" (z.B. inline:rel)
|
||||||
|
- defaults_total: rule_id startswith "edge_defaults:"
|
||||||
|
- structure: rule_id in {"structure:belongs_to","structure:order"}
|
||||||
|
|
||||||
|
Gibt zusätzlich:
|
||||||
|
- edges_by_kind (aggregiert)
|
||||||
|
- notes/chunks/edges Anzahlen
|
||||||
|
- multi_callout_detected: True, falls ein Chunk mehrere Callout-Ziele der gleichen Relation enthält
|
||||||
|
- per_note_checks: belongs_to == chunks, next == prev == (chunks-1)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from typing import Dict, Any, List, Tuple
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
|
||||||
|
def _count_collection_points(client, name: str) -> int:
|
||||||
|
try:
|
||||||
|
res = client.count(collection_name=name, exact=True)
|
||||||
|
return res.count or 0
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _scroll_all(client, collection: str) -> List[Any]:
|
||||||
|
pts_all = []
|
||||||
|
offset = None
|
||||||
|
while True:
|
||||||
|
pts, offset = client.scroll(
|
||||||
|
collection_name=collection,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=2048,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
pts_all.extend(pts or [])
|
||||||
|
if offset is None:
|
||||||
|
break
|
||||||
|
return pts_all
|
||||||
|
|
||||||
|
|
||||||
|
def _rule_group(rule_id: str) -> str:
|
||||||
|
if not rule_id:
|
||||||
|
return "unknown"
|
||||||
|
if rule_id == "callout:edge":
|
||||||
|
return "callout"
|
||||||
|
if rule_id.startswith("inline:"): # <—— wichtig für "inline:rel"
|
||||||
|
return "inline"
|
||||||
|
if rule_id.startswith("edge_defaults:"):
|
||||||
|
return "defaults"
|
||||||
|
if rule_id.startswith("explicit:"):
|
||||||
|
return "explicit"
|
||||||
|
if rule_id in ("structure:belongs_to", "structure:order"):
|
||||||
|
return "structure"
|
||||||
|
return "other"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
col_notes = f"{cfg.prefix}_notes"
|
||||||
|
col_chunks = f"{cfg.prefix}_chunks"
|
||||||
|
col_edges = f"{cfg.prefix}_edges"
|
||||||
|
|
||||||
|
# High-level counts
|
||||||
|
notes_n = _count_collection_points(client, col_notes)
|
||||||
|
chunks_n = _count_collection_points(client, col_chunks)
|
||||||
|
edges_pts = _scroll_all(client, col_edges)
|
||||||
|
edges_n = len(edges_pts)
|
||||||
|
|
||||||
|
# By kind / by rule group
|
||||||
|
by_kind = Counter()
|
||||||
|
group_counts = Counter()
|
||||||
|
callout_buckets: Dict[Tuple[str, str], int] = defaultdict(int) # (chunk_id, kind) -> n targets
|
||||||
|
per_note = defaultdict(lambda: {"chunks": 0, "belongs_to": 0, "next": 0, "prev": 0})
|
||||||
|
|
||||||
|
# Für per_note checks: chunks pro note_id aus mindnet_chunks laden
|
||||||
|
chunks_pts = _scroll_all(client, col_chunks)
|
||||||
|
chunks_by_note = Counter([p.payload.get("note_id") for p in chunks_pts if p.payload])
|
||||||
|
|
||||||
|
for p in edges_pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
kind = str(pl.get("kind") or pl.get("relation") or "edge")
|
||||||
|
rule_id = str(pl.get("rule_id") or "")
|
||||||
|
note_id = str(pl.get("note_id") or "")
|
||||||
|
chunk_id = str(pl.get("chunk_id") or "")
|
||||||
|
by_kind[kind] += 1
|
||||||
|
|
||||||
|
group = _rule_group(rule_id)
|
||||||
|
group_counts[group] += 1
|
||||||
|
|
||||||
|
# Multi-Callout-Erkennung: mehrere callout-Edges gleicher Relation aus demselben Chunk
|
||||||
|
if group == "callout" and chunk_id and kind:
|
||||||
|
callout_buckets[(chunk_id, kind)] += 1
|
||||||
|
|
||||||
|
# Per-note Strukturchecks
|
||||||
|
if note_id:
|
||||||
|
if kind == "belongs_to":
|
||||||
|
per_note[note_id]["belongs_to"] += 1
|
||||||
|
elif kind == "next":
|
||||||
|
per_note[note_id]["next"] += 1
|
||||||
|
elif kind == "prev":
|
||||||
|
per_note[note_id]["prev"] += 1
|
||||||
|
|
||||||
|
# set chunks count for per_note
|
||||||
|
for n_id, c in chunks_by_note.items():
|
||||||
|
per_note[n_id]["chunks"] = c
|
||||||
|
|
||||||
|
# final checks per note
|
||||||
|
per_note_checks = {}
|
||||||
|
for n_id, stats in per_note.items():
|
||||||
|
c = stats.get("chunks", 0)
|
||||||
|
bt = stats.get("belongs_to", 0)
|
||||||
|
nx = stats.get("next", 0)
|
||||||
|
pv = stats.get("prev", 0)
|
||||||
|
per_note_checks[n_id] = {
|
||||||
|
"chunks": c,
|
||||||
|
"belongs_to": bt,
|
||||||
|
"next": nx,
|
||||||
|
"prev": pv,
|
||||||
|
"checks": {
|
||||||
|
"belongs_to_equals_chunks": (bt == c),
|
||||||
|
"next_prev_match": (nx == pv == max(c - 1, 0)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
multi_callout_detected = any(v > 1 for v in callout_buckets.values())
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"prefix": cfg.prefix,
|
||||||
|
"counts": {
|
||||||
|
"notes": notes_n,
|
||||||
|
"chunks": chunks_n,
|
||||||
|
"edges": edges_n,
|
||||||
|
"edges_by_kind": dict(by_kind),
|
||||||
|
"explicit_total": group_counts.get("explicit", 0),
|
||||||
|
"defaults_total": group_counts.get("defaults", 0),
|
||||||
|
"callout_total": group_counts.get("callout", 0),
|
||||||
|
"inline_total": group_counts.get("inline", 0),
|
||||||
|
"structure_total": group_counts.get("structure", 0),
|
||||||
|
},
|
||||||
|
"per_note_checks": per_note_checks,
|
||||||
|
"multi_callout_detected": bool(multi_callout_detected),
|
||||||
|
"has_duplicates": False, # dedupe passiert beim Upsert
|
||||||
|
}
|
||||||
|
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+165
-188
@@ -1,249 +1,226 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Script: export_markdown.py — Qdrant → Markdown (Obsidian-kompatibel)
|
Script: scripts/export_markdown.py — Qdrant → Markdown (Vault)
|
||||||
Version: 1.4.0
|
Version: 1.4.1
|
||||||
Datum: 2025-09-09
|
Datum: 2025-09-10
|
||||||
|
|
||||||
Kurzbeschreibung
|
Funktion
|
||||||
----------------
|
--------
|
||||||
Exportiert Markdown-Notizen aus Qdrant in einen Zielordner (Vault).
|
Exportiert Notes (Frontmatter + Body) aus Qdrant in einen Zielordner. Der Body wird
|
||||||
Rekonstruiert YAML-Frontmatter und Body.
|
bevorzugt aus dem Feld `fulltext` rekonstruiert; falls leer/nicht vorhanden, aus Chunks
|
||||||
|
(Sortierung: seq → chunk_index → Nummer in chunk_id). Pfade werden **relativ** geschrieben.
|
||||||
|
|
||||||
Body-Rekonstruktions-Priorität (abwärtskompatibel):
|
Optionen
|
||||||
1) notes.payload.fulltext (verlustfrei, wenn beim Import gespeichert)
|
--------
|
||||||
2) ansonsten aus allen zugehörigen Chunks: payload.text → payload.content → payload.raw
|
--out PATH Zielordner (erforderlich)
|
||||||
in stabiler, sequentieller Reihenfolge (seq/chunk_index/ID-Nummer)
|
--prefix TEXT Collection-Prefix (CLI überschreibt ENV COLLECTION_PREFIX)
|
||||||
|
--note-id ID Nur eine Note exportieren
|
||||||
|
--overwrite Existierende Dateien überschreiben
|
||||||
|
--include-edges MODE none|yaml|footer (Default: none)
|
||||||
|
--flatten-paths Alle Dateien flach schreiben; Originalpfad in FM: orig_path
|
||||||
|
|
||||||
Wichtige Fixes
|
ENV
|
||||||
--------------
|
---
|
||||||
- **Pfad-Normalisierung**: erzwingt relative Pfade (führe führende '/' ab, backslashes → slashes),
|
COLLECTION_PREFIX, QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
|
||||||
damit ``--out`` nicht ignoriert wird.
|
|
||||||
- **`--prefix` (optional)**: Überschreibt COLLECTION_PREFIX; ENV bleibt Default (rückwärtskompatibel).
|
|
||||||
|
|
||||||
ENV / Qdrant
|
|
||||||
------------
|
|
||||||
- QDRANT_URL (oder QDRANT_HOST/QDRANT_PORT)
|
|
||||||
- QDRANT_API_KEY (optional)
|
|
||||||
- COLLECTION_PREFIX (Default: mindnet)
|
|
||||||
|
|
||||||
Aufruf
|
|
||||||
------
|
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault
|
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-foo
|
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault --prefix mindnet_dev # optional
|
|
||||||
|
|
||||||
Beispiele
|
Beispiele
|
||||||
---------
|
---------
|
||||||
COLLECTION_PREFIX=mindnet QDRANT_URL=http://127.0.0.1:6333 \\
|
export COLLECTION_PREFIX="mindnet"
|
||||||
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
|
python3 -m scripts.export_markdown --out ./_exportVault
|
||||||
"""
|
|
||||||
|
|
||||||
|
# Nur eine Note, Edges als YAML-Feld 'references'
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --note-id concept-alpha --include-edges yaml
|
||||||
|
|
||||||
|
# Flach schreiben mit Überschreiben
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --flatten-paths --overwrite
|
||||||
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import re
|
import json
|
||||||
from typing import Dict, Iterable, List, Optional, Tuple
|
from typing import Dict, List, Optional, Tuple, Any
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
from qdrant_client import QdrantClient
|
|
||||||
|
|
||||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
from app.core.qdrant import ensure_collections # safety
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
# Utilities
|
|
||||||
# -----------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
|
||||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def _norm_rel_path(path: str) -> str:
|
||||||
|
p = (path or "").replace("\\", "/").lstrip("/")
|
||||||
|
return p if p else ""
|
||||||
|
|
||||||
def _ensure_dir(path: str) -> None:
|
def _ensure_dir(path: str) -> None:
|
||||||
d = os.path.dirname(path)
|
d = os.path.dirname(path)
|
||||||
if d and not os.path.isdir(d):
|
if d and not os.path.exists(d):
|
||||||
os.makedirs(d, exist_ok=True)
|
os.makedirs(d, exist_ok=True)
|
||||||
|
|
||||||
|
def _yaml_dump(data: Dict[str, Any]) -> str:
|
||||||
|
return yaml.safe_dump(data, allow_unicode=True, sort_keys=False).strip()
|
||||||
|
|
||||||
def _normalize_rel_path(p: str) -> str:
|
def _frontmatter_block(fm: Dict[str, Any]) -> str:
|
||||||
"""Pfad relativ halten & normalisieren (slashes, führende / entfernen)."""
|
y = _yaml_dump(fm)
|
||||||
p = (p or "").replace("\\", "/")
|
return f"---\n{y}\n---\n"
|
||||||
return p.lstrip("/")
|
|
||||||
|
|
||||||
|
def _scroll_all(client, collection: str, flt: Optional[rest.Filter] = None) -> List[Any]:
|
||||||
def _to_md(frontmatter: dict, body: str) -> str:
|
|
||||||
fm = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=True).strip()
|
|
||||||
return f"---\n{fm}\n---\n{(body or '').rstrip()}\n"
|
|
||||||
|
|
||||||
|
|
||||||
def _scroll_all(
|
|
||||||
client: QdrantClient,
|
|
||||||
col: str,
|
|
||||||
flt: Optional[rest.Filter] = None,
|
|
||||||
with_payload: bool = True,
|
|
||||||
with_vectors: bool = False,
|
|
||||||
limit: int = 256,
|
|
||||||
):
|
|
||||||
"""Scrollt durch alle Punkte einer Collection und liefert eine Liste mit Points."""
|
|
||||||
out = []
|
out = []
|
||||||
next_page = None
|
nextp = None
|
||||||
while True:
|
while True:
|
||||||
pts, next_page = client.scroll(
|
pts, nextp = client.scroll(
|
||||||
collection_name=col,
|
collection_name=collection,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
scroll_filter=flt,
|
scroll_filter=flt,
|
||||||
with_payload=with_payload,
|
offset=nextp,
|
||||||
with_vectors=with_vectors,
|
|
||||||
limit=limit,
|
|
||||||
offset=next_page,
|
|
||||||
)
|
)
|
||||||
if not pts:
|
if not pts:
|
||||||
break
|
break
|
||||||
out.extend(pts)
|
out.extend(pts)
|
||||||
if not next_page:
|
if nextp is None:
|
||||||
break
|
break
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
def _reconstruct_body_from_chunks(chunks: List[Any]) -> str:
|
||||||
|
def seq_key(pl: Dict[str, Any]) -> Tuple[int, int, int]:
|
||||||
|
s = pl.get("seq")
|
||||||
|
ci = pl.get("chunk_index")
|
||||||
|
cid = pl.get("chunk_id") or ""
|
||||||
|
n = 0
|
||||||
|
if isinstance(cid, str) and "#" in cid:
|
||||||
|
try:
|
||||||
|
n = int(cid.rsplit("#", 1)[-1])
|
||||||
|
except Exception:
|
||||||
|
n = 0
|
||||||
|
return (int(s) if isinstance(s, int) else 0,
|
||||||
|
int(ci) if isinstance(ci, int) else 0,
|
||||||
|
n)
|
||||||
|
chunks_sorted = sorted(chunks, key=lambda p: seq_key(p.payload or {}))
|
||||||
|
texts: List[str] = []
|
||||||
|
for p in chunks_sorted:
|
||||||
|
pl = p.payload or {}
|
||||||
|
t = pl.get("text") or pl.get("content") or pl.get("raw") or ""
|
||||||
|
if isinstance(t, str) and t:
|
||||||
|
texts.append(t)
|
||||||
|
return "\n".join(texts).strip()
|
||||||
|
|
||||||
def _load_chunks_for_note(client: QdrantClient, chunks_col: str, note_id: str) -> List[dict]:
|
def _collect_forward_refs_from_edges(edges: List[Any]) -> List[str]:
|
||||||
flt = rest.Filter(must=[rest.FieldCondition(
|
refs = []
|
||||||
key="note_id",
|
for p in edges:
|
||||||
match=rest.MatchValue(value=note_id),
|
pl = p.payload or {}
|
||||||
)])
|
if pl.get("kind") == "references" and isinstance(pl.get("target_id"), str):
|
||||||
pts = _scroll_all(client, chunks_col, flt, with_payload=True, with_vectors=False)
|
refs.append(pl["target_id"])
|
||||||
# Sortierung: bevorzugt seq → chunk_index → Nummer in id
|
# de-dupe, preserve order
|
||||||
def _seq(pl: dict) -> Tuple[int, int, int]:
|
seen = set()
|
||||||
s1 = pl.get("seq", pl.get("chunk_index", -1))
|
out = []
|
||||||
s2 = pl.get("chunk_index", -1)
|
for r in refs:
|
||||||
# Nummer-Anteil aus "noteid#<n>"
|
if r not in seen:
|
||||||
s3 = 0
|
seen.add(r)
|
||||||
try:
|
out.append(r)
|
||||||
m = re.search(r"#(\\d+)$", pl.get("id") or "")
|
return out
|
||||||
if m:
|
|
||||||
s3 = int(m.group(1))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return (int(s1) if isinstance(s1, int) else -1, int(s2) if isinstance(s2, int) else -1, s3)
|
|
||||||
|
|
||||||
pts_sorted = sorted(pts, key=lambda p: _seq(p.payload or {}))
|
# ---------------------------------------------------------------------
|
||||||
return [p.payload or {} for p in pts_sorted]
|
|
||||||
|
|
||||||
|
|
||||||
def _reconstruct_body(note_pl: dict, chunk_payloads: List[dict]) -> str:
|
|
||||||
# 1) Volltext vorhanden?
|
|
||||||
fulltext = note_pl.get("fulltext")
|
|
||||||
if isinstance(fulltext, str) and fulltext.strip():
|
|
||||||
return fulltext
|
|
||||||
|
|
||||||
# 2) Aus Chunks zusammensetzen: text → content → raw
|
|
||||||
parts: List[str] = []
|
|
||||||
for ch in chunk_payloads:
|
|
||||||
text = ch.get("text") or ch.get("content") or ch.get("raw")
|
|
||||||
if isinstance(text, str) and text.strip():
|
|
||||||
parts.append(text.rstrip())
|
|
||||||
return ("\n\n".join(parts)).rstrip() + ("\n" if parts else "")
|
|
||||||
|
|
||||||
|
|
||||||
def _export_one_note(
|
|
||||||
client: QdrantClient,
|
|
||||||
prefix: str,
|
|
||||||
note_pl: dict,
|
|
||||||
out_root: str,
|
|
||||||
overwrite: bool,
|
|
||||||
) -> dict:
|
|
||||||
notes_col, chunks_col, _ = _names(prefix)
|
|
||||||
note_id = note_pl.get("note_id") or note_pl.get("id")
|
|
||||||
|
|
||||||
# Pfad robust bestimmen und relativ halten
|
|
||||||
path = note_pl.get("path") or f"{note_id}.md"
|
|
||||||
path = _normalize_rel_path(path)
|
|
||||||
out_path = os.path.join(out_root, path).replace("\\", "/")
|
|
||||||
|
|
||||||
# Frontmatter aus Payload zurückführen (nur bekannte Felder)
|
|
||||||
fm: Dict[str, object] = {}
|
|
||||||
for k in [
|
|
||||||
"title", "id", "type", "status", "created", "updated", "tags",
|
|
||||||
"priority", "effort_min", "due", "people", "aliases",
|
|
||||||
"depends_on", "assigned_to", "lang",
|
|
||||||
]:
|
|
||||||
v = note_pl.get(k) if k in note_pl else note_pl.get(f"note_{k}")
|
|
||||||
if v not in (None, [], ""):
|
|
||||||
fm[k] = v
|
|
||||||
|
|
||||||
# Mindestfelder
|
|
||||||
if "id" not in fm and note_id:
|
|
||||||
fm["id"] = note_id
|
|
||||||
if "title" not in fm and note_pl.get("title"):
|
|
||||||
fm["title"] = note_pl["title"]
|
|
||||||
|
|
||||||
# Body beschaffen
|
|
||||||
chunks = _load_chunks_for_note(client, chunks_col, note_id)
|
|
||||||
body = _reconstruct_body(note_pl, chunks)
|
|
||||||
|
|
||||||
# Schreiben?
|
|
||||||
if os.path.exists(out_path) and not overwrite:
|
|
||||||
return {"note_id": note_id, "path": path, "status": "skip_exists"}
|
|
||||||
|
|
||||||
_ensure_dir(out_path)
|
|
||||||
with open(out_path, "w", encoding="utf-8") as f:
|
|
||||||
f.write(_to_md(fm, body))
|
|
||||||
|
|
||||||
return {"note_id": note_id, "path": path, "status": "written"}
|
|
||||||
|
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
|
||||||
# Main
|
# Main
|
||||||
# -----------------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
ap.add_argument("--out", required=True, help="Zielordner für den Export-Vault")
|
ap.add_argument("--out", required=True, help="Zielordner für exportierte Markdown-Dateien")
|
||||||
ap.add_argument("--note-id", help="Nur eine Note exportieren (Note-ID)")
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||||
ap.add_argument("--overwrite", action="store_true", help="Bestehende Dateien überschreiben")
|
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID exportieren")
|
||||||
ap.add_argument("--prefix", help="(Optional) überschreibt COLLECTION_PREFIX aus ENV")
|
ap.add_argument("--overwrite", action="store_true", help="Existierende Dateien überschreiben")
|
||||||
|
ap.add_argument("--include-edges", choices=["none", "yaml", "footer"], default="none",
|
||||||
|
help="Forward-Links mit exportieren (aus Edges oder Note-Payload)")
|
||||||
|
ap.add_argument("--flatten-paths", action="store_true", help="Alle Dateien flach schreiben (orig_path in Frontmatter)")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
# Qdrant-Konfiguration
|
|
||||||
cfg = QdrantConfig.from_env()
|
cfg = QdrantConfig.from_env()
|
||||||
if args.prefix:
|
if args.prefix:
|
||||||
cfg.prefix = args.prefix # abwärtskompatibel: ENV bleibt Default
|
cfg.prefix = args.prefix.strip()
|
||||||
|
|
||||||
client = get_client(cfg)
|
client = get_client(cfg)
|
||||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||||
|
|
||||||
notes_col, _, _ = _names(cfg.prefix)
|
out_root = os.path.abspath(args.out)
|
||||||
|
os.makedirs(out_root, exist_ok=True)
|
||||||
|
|
||||||
# Notes holen (optional gefiltert)
|
notes_col, chunks_col, edges_col = collections(cfg.prefix)
|
||||||
|
|
||||||
|
# Filter nach note-id (optional)
|
||||||
flt = None
|
flt = None
|
||||||
if args.note_id:
|
if args.note_id:
|
||||||
flt = rest.Filter(must=[rest.FieldCondition(
|
flt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=args.note_id))])
|
||||||
key="note_id",
|
|
||||||
match=rest.MatchValue(value=args.note_id),
|
|
||||||
)])
|
|
||||||
|
|
||||||
note_pts = _scroll_all(client, notes_col, flt, with_payload=True, with_vectors=False)
|
notes = _scroll_all(client, notes_col, flt)
|
||||||
if not note_pts:
|
total = 0
|
||||||
print(json.dumps({"exported": 0, "out": args.out, "message": "Keine Notes gefunden."}, ensure_ascii=False))
|
for n in notes:
|
||||||
return
|
pl = n.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
rel_path = _norm_rel_path(pl.get("path") or "")
|
||||||
|
if args.flatten_paths or not rel_path:
|
||||||
|
fname = f"{(nid or 'note')}.md"
|
||||||
|
out_path = os.path.join(out_root, fname)
|
||||||
|
else:
|
||||||
|
out_path = os.path.join(out_root, rel_path)
|
||||||
|
out_path = out_path.replace("\\", "/")
|
||||||
|
_ensure_dir(out_path)
|
||||||
|
|
||||||
results = []
|
# Frontmatter aufbauen (nur sinnvolle Felder)
|
||||||
for p in note_pts:
|
fm_fields = ["id","title","type","status","created","updated","tags","area","project","source","lang","slug","aliases"]
|
||||||
pl = p.payload or {}
|
fm: Dict[str, Any] = {}
|
||||||
try:
|
fm["id"] = nid
|
||||||
res = _export_one_note(client, cfg.prefix, pl, args.out, args.overwrite)
|
for k in fm_fields:
|
||||||
except Exception as e:
|
if k == "id":
|
||||||
res = {"note_id": pl.get("note_id") or pl.get("id"), "error": str(e)}
|
continue
|
||||||
results.append(res)
|
if k in pl and pl[k] is not None:
|
||||||
|
fm[k] = pl[k]
|
||||||
|
if args.flatten_paths and rel_path:
|
||||||
|
fm["orig_path"] = rel_path
|
||||||
|
|
||||||
print(json.dumps({
|
# Body ermitteln (fulltext oder Chunks)
|
||||||
"exported": len([r for r in results if r.get("status") == "written"]),
|
body = (pl.get("fulltext") or "").strip()
|
||||||
"skipped": len([r for r in results if r.get("status") == "skip_exists"]),
|
if not body:
|
||||||
"out": args.out,
|
flt_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||||
"details": results,
|
chunks = _scroll_all(client, chunks_col, flt_chunks)
|
||||||
}, ensure_ascii=False))
|
body = _reconstruct_body_from_chunks(chunks)
|
||||||
|
|
||||||
|
# Edges (optional)
|
||||||
|
refs: List[str] = []
|
||||||
|
if args.include_edges != "none":
|
||||||
|
# aus Note-Payload, falls vorhanden
|
||||||
|
if isinstance(pl.get("references"), list) and pl["references"]:
|
||||||
|
refs = [r for r in pl["references"] if isinstance(r, str)]
|
||||||
|
else:
|
||||||
|
flt_edges = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||||
|
edges = _scroll_all(client, edges_col, flt_edges)
|
||||||
|
refs = _collect_forward_refs_from_edges(edges)
|
||||||
|
if args.include_edges == "yaml" and refs:
|
||||||
|
fm["references"] = refs
|
||||||
|
|
||||||
|
# Datei schreiben
|
||||||
|
if (not args.overwrite) and os.path.exists(out_path):
|
||||||
|
print(json.dumps({"note_id": nid, "path": out_path, "decision": "skip-exists"}))
|
||||||
|
continue
|
||||||
|
|
||||||
|
content = _frontmatter_block(fm) + (body + "\n" if body else "")
|
||||||
|
if args.include_edges == "footer" and refs:
|
||||||
|
content += "\n---\nLinks:\n" + "\n".join(f"- [[{r}]]" for r in refs) + "\n"
|
||||||
|
|
||||||
|
with open(out_path, "w", encoding="utf-8") as f:
|
||||||
|
f.write(content)
|
||||||
|
|
||||||
|
print(json.dumps({"note_id": nid, "path": out_path, "decision": "write"}))
|
||||||
|
total += 1
|
||||||
|
|
||||||
|
print(f"Done. Exported notes: {total}")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1,340 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
scripts/health_check_mindnet.py
|
||||||
|
|
||||||
|
Health-Check für den mindnet-Retriever-/Query-Endpoint.
|
||||||
|
|
||||||
|
Funktion:
|
||||||
|
- Führt POST-Requests auf /query in verschiedenen Modi aus (standard: semantic + hybrid).
|
||||||
|
- Prüft Status-Code, JSON-Struktur und Anzahl der Treffer.
|
||||||
|
- Kennzeichnet Probleme als:
|
||||||
|
- status="ok"
|
||||||
|
- status="warning" (z.B. Timeout)
|
||||||
|
- status="error" (harte Fehler wie HTTP-Fehler, JSON-Fehler etc.)
|
||||||
|
|
||||||
|
Exit-Code:
|
||||||
|
- Default (tolerant):
|
||||||
|
- overall_status = "ok" (inkl. warnings) → Exit-Code 0
|
||||||
|
- overall_status = "error" → Exit-Code 1
|
||||||
|
- Mit --strict:
|
||||||
|
- warnings werden wie errors behandelt → Exit-Code 1
|
||||||
|
|
||||||
|
Beispiele:
|
||||||
|
|
||||||
|
python scripts/health_check_mindnet.py \
|
||||||
|
--url http://127.0.0.1:8001/query \
|
||||||
|
--query "embeddings" \
|
||||||
|
--top-k 3
|
||||||
|
|
||||||
|
python scripts/health_check_mindnet.py \
|
||||||
|
--url http://127.0.0.1:8001/query \
|
||||||
|
--query "embeddings" \
|
||||||
|
--top-k 3 \
|
||||||
|
--timeout 15 \
|
||||||
|
--modes hybrid
|
||||||
|
|
||||||
|
# Strenger Modus (warnings → Exit-Code 1)
|
||||||
|
python scripts/health_check_mindnet.py --strict
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
|
||||||
|
def _post_query(
|
||||||
|
url: str,
|
||||||
|
mode: str,
|
||||||
|
query: str,
|
||||||
|
top_k: int,
|
||||||
|
timeout: float,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Führt einen POST auf den /query-Endpoint aus und gibt das dekodierte JSON zurück.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
urllib.error.URLError, urllib.error.HTTPError, RuntimeError
|
||||||
|
"""
|
||||||
|
payload = {
|
||||||
|
"mode": mode,
|
||||||
|
"query": query,
|
||||||
|
"top_k": top_k,
|
||||||
|
"expand": {
|
||||||
|
"depth": 1 if mode == "hybrid" else 0,
|
||||||
|
"edge_types": ["references", "belongs_to", "prev", "next"],
|
||||||
|
},
|
||||||
|
"ret": {
|
||||||
|
"with_paths": True,
|
||||||
|
"with_notes": True,
|
||||||
|
"with_chunks": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
data = json.dumps(payload).encode("utf-8")
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url=url,
|
||||||
|
data=data,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
|
||||||
|
start = time.monotonic()
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
|
status_code = resp.getcode()
|
||||||
|
body = resp.read().decode("utf-8")
|
||||||
|
end = time.monotonic()
|
||||||
|
|
||||||
|
latency_ms = int((end - start) * 1000)
|
||||||
|
|
||||||
|
if status_code != 200:
|
||||||
|
raise RuntimeError(f"HTTP {status_code}: {body}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
doc = json.loads(body)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise RuntimeError(f"Antwort ist kein gültiges JSON: {exc}") from exc
|
||||||
|
|
||||||
|
if not isinstance(doc, dict):
|
||||||
|
raise RuntimeError("Antwort ist kein JSON-Objekt")
|
||||||
|
|
||||||
|
doc["_latency_ms"] = latency_ms
|
||||||
|
return doc
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_response(
|
||||||
|
mode: str,
|
||||||
|
doc: Dict[str, Any],
|
||||||
|
min_results: int,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Prüft die wichtigsten Invarianten des Query-Responses.
|
||||||
|
|
||||||
|
Rückgabe-Format:
|
||||||
|
|
||||||
|
{
|
||||||
|
"status": "ok" | "warning" | "error",
|
||||||
|
"message": str,
|
||||||
|
"latency_ms": int,
|
||||||
|
"used_mode": str,
|
||||||
|
"result_count": int
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
latency_ms = int(doc.get("_latency_ms", -1))
|
||||||
|
used_mode = str(doc.get("used_mode", "unknown"))
|
||||||
|
results = doc.get("results", [])
|
||||||
|
|
||||||
|
if not isinstance(results, list):
|
||||||
|
return {
|
||||||
|
"status": "error",
|
||||||
|
"message": "results ist kein Array",
|
||||||
|
"latency_ms": latency_ms,
|
||||||
|
"used_mode": used_mode,
|
||||||
|
"result_count": -1,
|
||||||
|
}
|
||||||
|
|
||||||
|
result_count = len(results)
|
||||||
|
|
||||||
|
if result_count < min_results:
|
||||||
|
# Zu wenige Ergebnisse sind aus Sicht eines Health-Checks eher ein Error,
|
||||||
|
# da die Retrieval-Qualität nicht gewährleistet ist.
|
||||||
|
return {
|
||||||
|
"status": "error",
|
||||||
|
"message": f"zu wenige Ergebnisse: {result_count} < {min_results}",
|
||||||
|
"latency_ms": latency_ms,
|
||||||
|
"used_mode": used_mode,
|
||||||
|
"result_count": result_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
sample_msg = "OK"
|
||||||
|
if result_count > 0:
|
||||||
|
first = results[0]
|
||||||
|
if not isinstance(first, dict):
|
||||||
|
sample_msg = "first result ist kein Objekt"
|
||||||
|
else:
|
||||||
|
# einfache Plausibilitätschecks (nur Zugriff, kein strikter Typcheck)
|
||||||
|
_ = first.get("note_id")
|
||||||
|
_ = first.get("chunk_id")
|
||||||
|
_ = first.get("total_score")
|
||||||
|
|
||||||
|
status = "ok" if sample_msg == "OK" else "warning"
|
||||||
|
return {
|
||||||
|
"status": status,
|
||||||
|
"message": sample_msg,
|
||||||
|
"latency_ms": latency_ms,
|
||||||
|
"used_mode": used_mode,
|
||||||
|
"result_count": result_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run_health_check(
|
||||||
|
url: str,
|
||||||
|
query: str,
|
||||||
|
top_k: int,
|
||||||
|
timeout: float,
|
||||||
|
modes: List[str],
|
||||||
|
min_results: int,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Führt den Health-Check für die angegebenen Modi aus.
|
||||||
|
"""
|
||||||
|
report: Dict[str, Any] = {
|
||||||
|
"url": url,
|
||||||
|
"query": query,
|
||||||
|
"top_k": top_k,
|
||||||
|
"timeout_s": timeout,
|
||||||
|
"checks": [],
|
||||||
|
"overall_status": "ok", # wird unten ggf. auf warning oder error gesetzt
|
||||||
|
}
|
||||||
|
|
||||||
|
overall_status = "ok"
|
||||||
|
|
||||||
|
for mode in modes:
|
||||||
|
entry: Dict[str, Any] = {
|
||||||
|
"mode": mode,
|
||||||
|
"status": "error",
|
||||||
|
"message": "",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
doc = _post_query(url=url, mode=mode, query=query, top_k=top_k, timeout=timeout)
|
||||||
|
validation = _validate_response(mode=mode, doc=doc, min_results=min_results)
|
||||||
|
entry.update(validation)
|
||||||
|
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
entry["status"] = "error"
|
||||||
|
entry["message"] = f"HTTP-Fehler: {exc}"
|
||||||
|
entry["latency_ms"] = -1
|
||||||
|
entry["used_mode"] = mode
|
||||||
|
entry["result_count"] = -1
|
||||||
|
|
||||||
|
except urllib.error.URLError as exc:
|
||||||
|
# URLError kann u.a. socket.timeout enthalten.
|
||||||
|
lat = -1
|
||||||
|
entry["latency_ms"] = lat
|
||||||
|
entry["used_mode"] = mode
|
||||||
|
entry["result_count"] = -1
|
||||||
|
|
||||||
|
if isinstance(exc.reason, socket.timeout):
|
||||||
|
entry["status"] = "warning"
|
||||||
|
entry["message"] = f"Timeout (URLError/socket.timeout): {exc}"
|
||||||
|
else:
|
||||||
|
entry["status"] = "error"
|
||||||
|
entry["message"] = f"HTTP/Netzwerkfehler: {exc}"
|
||||||
|
|
||||||
|
except socket.timeout as exc:
|
||||||
|
# Direkter Timeout (falls nicht in URLError verpackt)
|
||||||
|
entry["status"] = "warning"
|
||||||
|
entry["message"] = f"Timeout (socket.timeout): {exc}"
|
||||||
|
entry["latency_ms"] = -1
|
||||||
|
entry["used_mode"] = mode
|
||||||
|
entry["result_count"] = -1
|
||||||
|
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
entry["status"] = "error"
|
||||||
|
entry["message"] = f"unerwarteter Fehler: {exc}"
|
||||||
|
entry["latency_ms"] = -1
|
||||||
|
entry["used_mode"] = mode
|
||||||
|
entry["result_count"] = -1
|
||||||
|
|
||||||
|
# overall_status bestimmen:
|
||||||
|
# - error > warning > ok
|
||||||
|
if entry["status"] == "error":
|
||||||
|
overall_status = "error"
|
||||||
|
elif entry["status"] == "warning" and overall_status == "ok":
|
||||||
|
overall_status = "warning"
|
||||||
|
|
||||||
|
report["checks"].append(entry)
|
||||||
|
|
||||||
|
report["overall_status"] = overall_status
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args(argv: List[str]) -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Health-Check für mindnet /query-Retriever",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--url",
|
||||||
|
default="http://127.0.0.1:8001/query",
|
||||||
|
help="Vollständige URL des /query-Endpunkts (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--query",
|
||||||
|
default="embeddings",
|
||||||
|
help="Test-Query für den Health-Check (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--top-k",
|
||||||
|
type=int,
|
||||||
|
default=3,
|
||||||
|
help="Top-K für die Testabfrage (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--timeout",
|
||||||
|
type=float,
|
||||||
|
default=5.0,
|
||||||
|
help="Timeout in Sekunden pro Request (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--modes",
|
||||||
|
nargs="+",
|
||||||
|
default=["semantic", "hybrid"],
|
||||||
|
help="Zu prüfende Modi (Default: semantic hybrid)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--min-results",
|
||||||
|
type=int,
|
||||||
|
default=1,
|
||||||
|
help="Minimale Anzahl erwarteter Ergebnisse (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--strict",
|
||||||
|
action="store_true",
|
||||||
|
help="Warnings als Fehler behandeln (overall_status=warning führt zu Exit-Code 1)",
|
||||||
|
)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: List[str]) -> int:
|
||||||
|
args = parse_args(argv)
|
||||||
|
|
||||||
|
report = run_health_check(
|
||||||
|
url=args.url,
|
||||||
|
query=args.query,
|
||||||
|
top_k=args.top_k,
|
||||||
|
timeout=args.timeout,
|
||||||
|
modes=args.modes,
|
||||||
|
min_results=args.min_results,
|
||||||
|
)
|
||||||
|
|
||||||
|
overall = report.get("overall_status", "error")
|
||||||
|
|
||||||
|
# Menschlich lesbare Zusammenfassung:
|
||||||
|
print(f"mindnet health: {overall}")
|
||||||
|
for c in report["checks"]:
|
||||||
|
mode = c.get("mode")
|
||||||
|
status = c.get("status")
|
||||||
|
latency = c.get("latency_ms")
|
||||||
|
result_count = c.get("result_count")
|
||||||
|
msg = c.get("message", "")
|
||||||
|
print(f" - {mode}: {status} (latency={latency} ms, results={result_count}) {msg}")
|
||||||
|
|
||||||
|
# JSON-Ausgabe (z.B. für n8n):
|
||||||
|
print(json.dumps(report, ensure_ascii=False))
|
||||||
|
|
||||||
|
if args.strict:
|
||||||
|
# strict: warning wird wie error behandelt
|
||||||
|
return 0 if overall == "ok" else 1
|
||||||
|
else:
|
||||||
|
# tolerant: nur echte errors führen zu Exit-Code 1
|
||||||
|
return 0 if overall in ("ok", "warning") else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main(sys.argv[1:]))
|
||||||
+409
-274
@@ -1,68 +1,73 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
|
scripts/import_markdown.py
|
||||||
Version: 3.5.3
|
|
||||||
Datum: 2025-09-09
|
|
||||||
|
|
||||||
Neu in 3.5.3
|
Zweck
|
||||||
------------
|
-----
|
||||||
- **Fallback**, falls app.core.note_payload.make_note_payload(None) / non-dict zurückgibt:
|
- Liest Markdown-Notizen aus einem Vault ein
|
||||||
Der Importer baut einen minimalen, korrekten Note-Payload selbst (inkl. Hash
|
- Erzeugt Note-Payload, Chunk-Payloads (+ optionale Embeddings) und Edges
|
||||||
gemäß --hash-mode/--hash-source/--hash-normalize) und läuft weiter.
|
- Schreibt alles idempotent in Qdrant (Notes, Chunks, Edges)
|
||||||
- Detaillierte Debug-Ausgaben mit Typ/Preview, falls die Rückgabeform abweicht.
|
- Integriert eine optionale Type-Registry (types.yaml), um z. B. chunk_profile
|
||||||
|
und retriever_weight pro Notiz-Typ zu steuern.
|
||||||
|
|
||||||
Kurzbeschreibung
|
Wesentliche Fixes ggü. vorherigen fehlerhaften Ständen
|
||||||
----------------
|
------------------------------------------------------
|
||||||
- Liest Markdown-Dateien ein, erzeugt Notes/Chunks/Edges **idempotent**.
|
- `embed_texts` wird optional importiert und defensiv geprüft (kein NameError mehr)
|
||||||
- Änderungserkennung (nur Inhalte, keine FS-Zeitstempel) konfigurierbar:
|
- `effective_chunk_profile` / `effective_retriever_weight` und Registry-Helfer
|
||||||
* --hash-mode: body | frontmatter | body+frontmatter | full (Alias)
|
sind VOR `main()` definiert (kein NameError mehr)
|
||||||
- Env: MINDNET_HASH_MODE oder MINDNET_HASH_COMPARE (Body/Frontmatter/Full)
|
- `retriever_weight` wird in Note- und Chunk-Payload zuverlässig gesetzt
|
||||||
* --hash-normalize: canonical | none (Default: canonical)
|
- Robuste Kantenbildung; Fehler bei Edges blockieren Notes/Chunks nicht
|
||||||
* --hash-source: parsed (Default) | raw
|
- Korrekte Verwendung von `scroll_filter` beim Qdrant-Client
|
||||||
* Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true)
|
- `--purge-before-upsert` entfernt alte Chunks/Edges einer Note vor dem Upsert
|
||||||
* Signaturabgleich: Wechselt die Signatur (z. B. body→full, parsed→raw, canonical→none),
|
|
||||||
gilt die Note einmalig als geändert.
|
|
||||||
|
|
||||||
ENV / Qdrant
|
Qdrant / ENV
|
||||||
------------
|
------------
|
||||||
- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
|
- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
|
||||||
- COLLECTION_PREFIX (Default: mindnet)
|
- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
|
||||||
- VECTOR_DIM (Default: 384)
|
- VECTOR_DIM (Default: 384)
|
||||||
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
||||||
- MINDNET_COMPARE_TEXT: true|false (Default: false)
|
- MINDNET_TYPES_FILE: Pfad zu types.yaml (optional; Default: ./types.yaml)
|
||||||
|
|
||||||
Aufruf-Beispiele
|
Beispiele
|
||||||
----------------
|
---------
|
||||||
|
# Standard (Body, parsed, canonical)
|
||||||
python3 -m scripts.import_markdown --vault ./vault
|
python3 -m scripts.import_markdown --vault ./vault
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --hash-source raw --hash-normalize none
|
|
||||||
MINDNET_HASH_COMPARE=Full python3 -m scripts.import_markdown --vault ./vault --apply
|
# Erstimport nach truncate (Create-Fall)
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --compare-text
|
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
|
||||||
|
|
||||||
|
# Nur eine Datei (Diagnose)
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --only-path ./vault/30_projects/project-demo.md --apply
|
||||||
|
|
||||||
|
# Sync-Deletes (Dry-Run → Apply)
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --sync-deletes
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import difflib
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from typing import Dict, List, Optional, Tuple, Any
|
from typing import Dict, List, Optional, Tuple, Any, Set
|
||||||
from collections.abc import Mapping
|
|
||||||
|
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
# --- Projekt-Imports ---
|
||||||
from app.core.parser import (
|
from app.core.parser import (
|
||||||
read_markdown,
|
read_markdown,
|
||||||
normalize_frontmatter,
|
normalize_frontmatter,
|
||||||
validate_required_frontmatter,
|
validate_required_frontmatter,
|
||||||
extract_wikilinks,
|
|
||||||
FRONTMATTER_RE,
|
|
||||||
)
|
)
|
||||||
from app.core.note_payload import make_note_payload
|
from app.core.note_payload import make_note_payload
|
||||||
from app.core.chunker import assemble_chunks
|
from app.core.chunker import assemble_chunks
|
||||||
from app.core.chunk_payload import make_chunk_payloads
|
from app.core.chunk_payload import make_chunk_payloads
|
||||||
from app.core.edges import build_edges_for_note
|
try:
|
||||||
|
from app.core.derive_edges import build_edges_for_note
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
from app.core.edges import build_edges_for_note # type: ignore
|
||||||
from app.core.qdrant import (
|
from app.core.qdrant import (
|
||||||
QdrantConfig,
|
QdrantConfig,
|
||||||
get_client,
|
get_client,
|
||||||
@@ -76,14 +81,110 @@ from app.core.qdrant_points import (
|
|||||||
upsert_batch,
|
upsert_batch,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# embeddings sind optional (z. B. im reinen Payload-Backfill)
|
||||||
try:
|
try:
|
||||||
from app.core.embed import embed_texts # optional
|
from app.core.embed import embed_texts # optional
|
||||||
except Exception:
|
except Exception: # pragma: no cover
|
||||||
embed_texts = None
|
embed_texts = None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
# Helpers
|
# Type-Registry (types.yaml) – Helper (robust, optional)
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _env(name: str, default: Optional[str] = None) -> Optional[str]:
|
||||||
|
v = os.getenv(name)
|
||||||
|
return v if v is not None else default
|
||||||
|
|
||||||
|
def _load_json_or_yaml(path: str) -> dict:
|
||||||
|
import io
|
||||||
|
data: dict = {}
|
||||||
|
if not path or not os.path.exists(path):
|
||||||
|
return data
|
||||||
|
try:
|
||||||
|
import yaml # type: ignore
|
||||||
|
with io.open(path, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return {}
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
# YAML evtl. nicht installiert – versuche JSON
|
||||||
|
try:
|
||||||
|
with io.open(path, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return {}
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def load_type_registry() -> dict:
|
||||||
|
# Reihenfolge: ENV > ./types.yaml (im aktuellen Arbeitsverzeichnis)
|
||||||
|
p = _env("MINDNET_TYPES_FILE", None)
|
||||||
|
if p and os.path.exists(p):
|
||||||
|
return _load_json_or_yaml(p)
|
||||||
|
fallback = os.path.abspath("./config/types.yaml") if os.path.exists("./config/types.yaml") else os.path.abspath("./types.yaml")
|
||||||
|
return _load_json_or_yaml(fallback)
|
||||||
|
|
||||||
|
def get_type_config(note_type: Optional[str], reg: dict) -> dict:
|
||||||
|
if not reg or not isinstance(reg, dict):
|
||||||
|
return {}
|
||||||
|
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
|
||||||
|
if note_type and isinstance(note_type, str) and note_type in types:
|
||||||
|
return types[note_type] or {}
|
||||||
|
# Fallback: concept
|
||||||
|
return types.get("concept", {}) or {}
|
||||||
|
|
||||||
|
def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
||||||
|
if requested and isinstance(requested, str):
|
||||||
|
return requested
|
||||||
|
# Fallback wenn nichts gesetzt ist
|
||||||
|
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
|
||||||
|
return "concept" if "concept" in types else (requested or "concept")
|
||||||
|
|
||||||
|
def effective_chunk_profile(note_type: str, reg: dict) -> Optional[str]:
|
||||||
|
"""Resolve chunk_profile for type or from defaults/global.
|
||||||
|
Accepts symbolic profiles: short|medium|long|default.
|
||||||
|
"""
|
||||||
|
cfg = get_type_config(note_type, reg)
|
||||||
|
prof = (cfg.get("chunk_profile") if isinstance(cfg, dict) else None)
|
||||||
|
if isinstance(prof, str) and prof:
|
||||||
|
return prof
|
||||||
|
# defaults fallbacks
|
||||||
|
for key in ("defaults", "default", "global"):
|
||||||
|
dcfg = reg.get(key) if isinstance(reg, dict) else None
|
||||||
|
if isinstance(dcfg, dict):
|
||||||
|
dprof = dcfg.get("chunk_profile")
|
||||||
|
if isinstance(dprof, str) and dprof:
|
||||||
|
return dprof
|
||||||
|
return "default"
|
||||||
|
|
||||||
|
def effective_retriever_weight(note_type: str, reg: dict) -> Optional[float]:
|
||||||
|
"""Resolve retriever_weight for type or defaults; returns float.
|
||||||
|
"""
|
||||||
|
cfg = get_type_config(note_type, reg)
|
||||||
|
w = (cfg.get("retriever_weight") if isinstance(cfg, dict) else None)
|
||||||
|
try:
|
||||||
|
if w is not None:
|
||||||
|
return float(w)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# defaults fallbacks
|
||||||
|
for key in ("defaults", "default", "global"):
|
||||||
|
dcfg = reg.get(key) if isinstance(reg, dict) else None
|
||||||
|
if isinstance(dcfg, dict):
|
||||||
|
dw = dcfg.get("retriever_weight")
|
||||||
|
try:
|
||||||
|
if dw is not None:
|
||||||
|
return float(dw)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return 1.0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# Sonstige Helper
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
def iter_md(root: str) -> List[str]:
|
def iter_md(root: str) -> List[str]:
|
||||||
@@ -107,7 +208,7 @@ def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[D
|
|||||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
points, _ = client.scroll(
|
points, _ = client.scroll(
|
||||||
collection_name=notes_col,
|
collection_name=notes_col,
|
||||||
scroll_filter=f,
|
scroll_filter=f, # wichtig: scroll_filter (nicht: filter)
|
||||||
with_payload=True,
|
with_payload=True,
|
||||||
with_vectors=False,
|
with_vectors=False,
|
||||||
limit=1,
|
limit=1,
|
||||||
@@ -116,262 +217,250 @@ def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[D
|
|||||||
return None
|
return None
|
||||||
return points[0].payload or {}
|
return points[0].payload or {}
|
||||||
|
|
||||||
|
def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
||||||
|
notes_col, _, _ = collections(prefix)
|
||||||
|
out: Set[str] = set()
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=notes_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
for p in pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if isinstance(nid, str):
|
||||||
|
out.add(nid)
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||||
_, chunks_col, edges_col = collections(prefix)
|
_, chunks_col, edges_col = collections(prefix)
|
||||||
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
client.delete(collection_name=chunks_col, points_selector=f_chunks, wait=True)
|
for col in (chunks_col, edges_col):
|
||||||
f_edges = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
|
||||||
client.delete(collection_name=edges_col, points_selector=f_edges, wait=True)
|
|
||||||
|
|
||||||
def _normalize_rel_path(abs_path: str, vault_root: str) -> str:
|
|
||||||
try:
|
|
||||||
rel = os.path.relpath(abs_path, vault_root)
|
|
||||||
except Exception:
|
|
||||||
rel = abs_path
|
|
||||||
return rel.replace("\\", "/").lstrip("/")
|
|
||||||
|
|
||||||
def _resolve_hash_mode(val: Optional[str]) -> str:
|
|
||||||
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
|
|
||||||
if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
|
||||||
return "body+frontmatter"
|
|
||||||
if v in ("frontmatter", "fm"):
|
|
||||||
return "frontmatter"
|
|
||||||
return "body"
|
|
||||||
|
|
||||||
def _normalize_body(body: str, mode: str) -> str:
|
|
||||||
if mode == "none":
|
|
||||||
return body if body is not None else ""
|
|
||||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
|
|
||||||
text = "\n".join(line.rstrip() for line in text.split("\n"))
|
|
||||||
return text
|
|
||||||
|
|
||||||
def _canon_fm(fm: Dict[str, Any]) -> str:
|
|
||||||
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
|
||||||
|
|
||||||
def _read_raw_body(file_path: str) -> tuple[str, Dict[str, Any]]:
|
|
||||||
if not os.path.exists(file_path):
|
|
||||||
return "", {}
|
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
|
||||||
raw = f.read()
|
|
||||||
m = FRONTMATTER_RE.match(raw)
|
|
||||||
fm = {}
|
|
||||||
if m:
|
|
||||||
import yaml
|
|
||||||
try:
|
try:
|
||||||
fm = yaml.safe_load(m.group(1)) or {}
|
client.delete(
|
||||||
except Exception:
|
collection_name=col,
|
||||||
fm = {}
|
points_selector=rest.FilterSelector(filter=filt),
|
||||||
body = raw[m.end():]
|
wait=True
|
||||||
else:
|
)
|
||||||
body = raw
|
except Exception as e:
|
||||||
return body, fm
|
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
|
||||||
|
|
||||||
def _fallback_note_payload(parsed: Any, vault_root: str, *, hash_mode: Optional[str], hash_normalize: Optional[str],
|
def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
||||||
hash_source: Optional[str], file_path: str) -> Dict[str, Any]:
|
notes_col, chunks_col, edges_col = collections(prefix)
|
||||||
"""Baut einen minimalen Note-Payload, falls make_note_payload None liefert."""
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
fm = normalize_frontmatter(parsed.frontmatter)
|
for col in (edges_col, chunks_col, notes_col):
|
||||||
path = getattr(parsed, "path", file_path)
|
|
||||||
rel_path = _normalize_rel_path(path, vault_root)
|
|
||||||
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower()
|
|
||||||
norm = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
|
|
||||||
mode = _resolve_hash_mode(hash_mode)
|
|
||||||
|
|
||||||
if src == "raw":
|
|
||||||
body_raw, fm_raw = _read_raw_body(file_path)
|
|
||||||
# Roh-FM ergänzen (nicht überschreiben)
|
|
||||||
if fm_raw:
|
|
||||||
merged = dict(fm)
|
|
||||||
for k, v in fm_raw.items():
|
|
||||||
merged.setdefault(k, v)
|
|
||||||
fm = merged
|
|
||||||
body_for_hash = body_raw
|
|
||||||
else:
|
|
||||||
body_for_hash = getattr(parsed, "body", "") or ""
|
|
||||||
|
|
||||||
body_n = _normalize_body(body_for_hash, norm)
|
|
||||||
fm_s = _canon_fm(fm)
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
def h(s: str) -> str:
|
|
||||||
x = hashlib.sha256()
|
|
||||||
x.update(s.encode("utf-8"))
|
|
||||||
return x.hexdigest()
|
|
||||||
|
|
||||||
if mode == "frontmatter":
|
|
||||||
primary_hash = h(fm_s)
|
|
||||||
elif mode == "body+frontmatter":
|
|
||||||
primary_hash = h(body_n + "\n--FM--\n" + fm_s)
|
|
||||||
else:
|
|
||||||
primary_hash = h(body_n)
|
|
||||||
|
|
||||||
sig = f"{'full' if mode=='body+frontmatter' else mode}:{src}:{norm}:{primary_hash}"
|
|
||||||
|
|
||||||
refs = extract_wikilinks(getattr(parsed, "body", "") or "")
|
|
||||||
|
|
||||||
return {
|
|
||||||
"note_id": fm.get("id") or fm.get("note_id"),
|
|
||||||
"title": fm.get("title"),
|
|
||||||
"type": fm.get("type"),
|
|
||||||
"status": fm.get("status"),
|
|
||||||
"created": fm.get("created"),
|
|
||||||
"updated": fm.get("updated"),
|
|
||||||
"path": rel_path or fm.get("path"),
|
|
||||||
"tags": fm.get("tags"),
|
|
||||||
"hash_fulltext": primary_hash,
|
|
||||||
"hash_signature": sig,
|
|
||||||
"fulltext": getattr(parsed, "body", "") or "",
|
|
||||||
"references": list(dict.fromkeys(refs)),
|
|
||||||
}
|
|
||||||
|
|
||||||
def _coerce_to_dict(obj: Any) -> Optional[Dict[str, Any]]:
|
|
||||||
if obj is None:
|
|
||||||
return None
|
|
||||||
if isinstance(obj, dict):
|
|
||||||
return obj
|
|
||||||
if isinstance(obj, Mapping):
|
|
||||||
try:
|
try:
|
||||||
return dict(obj)
|
client.delete(
|
||||||
except Exception:
|
collection_name=col,
|
||||||
pass
|
points_selector=rest.FilterSelector(filter=filt),
|
||||||
if isinstance(obj, (list, tuple)):
|
wait=True
|
||||||
for it in obj:
|
)
|
||||||
d = _coerce_to_dict(it)
|
except Exception as e:
|
||||||
if isinstance(d, dict):
|
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
|
||||||
return d
|
|
||||||
return None
|
|
||||||
md = getattr(obj, "model_dump", None)
|
# --- Neu: Existenz-Checks für Artefakte (fehlertoleranter Rebuild) ---
|
||||||
if callable(md):
|
|
||||||
try: return md()
|
def _has_any_point(client, collection: str, note_id: str) -> bool:
|
||||||
except Exception: pass
|
"""Prüft, ob es mind. einen Punkt mit note_id in der Collection gibt."""
|
||||||
dd = getattr(obj, "dict", None)
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
if callable(dd):
|
pts, _ = client.scroll(
|
||||||
try: return dd()
|
collection_name=collection,
|
||||||
except Exception: pass
|
scroll_filter=filt,
|
||||||
dct = getattr(obj, "__dict__", None)
|
with_payload=False,
|
||||||
if isinstance(dct, dict):
|
with_vectors=False,
|
||||||
return dict(dct)
|
limit=1,
|
||||||
return None
|
)
|
||||||
|
return bool(pts)
|
||||||
|
|
||||||
|
def artifacts_missing(client, prefix: str, note_id: str) -> Tuple[bool, bool]:
|
||||||
|
"""Gibt (chunks_missing, edges_missing) zurück."""
|
||||||
|
_, chunks_col, edges_col = collections(prefix)
|
||||||
|
chunks_missing = not _has_any_point(client, chunks_col, note_id)
|
||||||
|
edges_missing = not _has_any_point(client, edges_col, note_id)
|
||||||
|
return chunks_missing, edges_missing
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
# Main
|
# Main
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_mode(m: Optional[str]) -> str:
|
||||||
|
m = (m or "body").strip().lower()
|
||||||
|
return m if m in {"body", "frontmatter", "full"} else "body"
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
ap = argparse.ArgumentParser()
|
|
||||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
|
ap = argparse.ArgumentParser(
|
||||||
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
|
prog="scripts.import_markdown",
|
||||||
ap.add_argument("--purge-before-upsert", action="store_true",
|
description="Importiert Markdown-Notizen in Qdrant (Notes/Chunks/Edges)."
|
||||||
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
|
)
|
||||||
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
|
ap.add_argument("--vault", required=True, help="Pfad zum Vault (Ordner mit .md-Dateien)")
|
||||||
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
|
ap.add_argument("--only-path", help="Nur diese Datei verarbeiten (absolut oder relativ)")
|
||||||
ap.add_argument("--force-replace", action="store_true",
|
ap.add_argument("--apply", action="store_true", help="Schreibt nach Qdrant (sonst Dry-Run)")
|
||||||
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
|
ap.add_argument("--purge-before-upsert", action="store_true", help="Alte Chunks/Edges der Note vorher löschen")
|
||||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter", "full"], default=None)
|
ap.add_argument("--force-replace", action="store_true", help="Note/Chunks/Edges unabhängig von Hash neu schreiben")
|
||||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
ap.add_argument("--note-id", help="Nur Notes mit dieser ID verarbeiten (Filter)")
|
||||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
|
ap.add_argument("--note-scope-refs", action="store_true", help="Note-scope References/Backlinks erzeugen")
|
||||||
ap.add_argument("--note-scope-refs", action="store_true")
|
ap.add_argument("--hash-mode", help="body|frontmatter|full (Default body)")
|
||||||
ap.add_argument("--debug-hash-diff", action="store_true")
|
ap.add_argument("--hash-source", help="parsed|raw (Default parsed)")
|
||||||
ap.add_argument("--compare-text", action="store_true")
|
ap.add_argument("--hash-normalize", help="canonical|none (Default canonical)")
|
||||||
|
ap.add_argument("--compare-text", action="store_true", help="Parsed fulltext zusätzlich direkt vergleichen")
|
||||||
|
ap.add_argument("--baseline-modes", action="store_true", help="Fehlende Hash-Varianten still nachtragen (Notes)")
|
||||||
|
ap.add_argument("--sync-deletes", action="store_true", help="Qdrant->Vault Lösch-Sync (Dry-Run; mit --apply ausführen)")
|
||||||
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
note_scope_refs_env = (os.environ.get("MINDNET_NOTE_SCOPE_REFS", "false").strip().lower() == "true")
|
mode = _resolve_mode(args.hash_mode) # body|frontmatter|full
|
||||||
|
src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed") # parsed|raw
|
||||||
|
norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical") # canonical|none
|
||||||
|
note_scope_refs_env = (_env("MINDNET_NOTE_SCOPE_REFS", "false") == "true")
|
||||||
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
||||||
compare_text = args.compare_text or (os.environ.get("MINDNET_COMPARE_TEXT", "false").strip().lower() == "true")
|
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
||||||
|
|
||||||
|
# Qdrant
|
||||||
cfg = QdrantConfig.from_env()
|
cfg = QdrantConfig.from_env()
|
||||||
|
if args.prefix:
|
||||||
|
cfg.prefix = args.prefix.strip()
|
||||||
client = get_client(cfg)
|
client = get_client(cfg)
|
||||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||||
ensure_payload_indexes(client, cfg.prefix)
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
|
|
||||||
|
# Type-Registry laden (optional)
|
||||||
|
reg = load_type_registry()
|
||||||
|
|
||||||
root = os.path.abspath(args.vault)
|
root = os.path.abspath(args.vault)
|
||||||
files = iter_md(root)
|
|
||||||
|
# Dateiliste
|
||||||
|
if args.only_path:
|
||||||
|
only = os.path.abspath(args.only_path)
|
||||||
|
files = [only]
|
||||||
|
else:
|
||||||
|
files = iter_md(root)
|
||||||
if not files:
|
if not files:
|
||||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
|
||||||
|
# Optional: Sync-Deletes vorab
|
||||||
|
if args.sync_deletes:
|
||||||
|
vault_note_ids: Set[str] = set()
|
||||||
|
for path in files:
|
||||||
|
try:
|
||||||
|
parsed = read_markdown(path)
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
|
nid = fm.get("id")
|
||||||
|
if isinstance(nid, str):
|
||||||
|
vault_note_ids.add(nid)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
qdrant_note_ids = list_qdrant_note_ids(client, cfg.prefix)
|
||||||
|
to_delete = sorted(qdrant_note_ids - vault_note_ids)
|
||||||
|
print(json.dumps({
|
||||||
|
"action": "sync-deletes",
|
||||||
|
"prefix": cfg.prefix,
|
||||||
|
"qdrant_total": len(qdrant_note_ids),
|
||||||
|
"vault_total": len(vault_note_ids),
|
||||||
|
"to_delete_count": len(to_delete),
|
||||||
|
"to_delete": to_delete[:50] + (["…"] if len(to_delete) > 50 else [])
|
||||||
|
}, ensure_ascii=False))
|
||||||
|
if args.apply and to_delete:
|
||||||
|
for nid in to_delete:
|
||||||
|
print(json.dumps({"action": "delete", "note_id": nid, "decision": "apply"}))
|
||||||
|
delete_note_everywhere(client, cfg.prefix, nid)
|
||||||
|
|
||||||
|
key_current = f"{mode}:{src}:{norm}"
|
||||||
|
|
||||||
processed = 0
|
processed = 0
|
||||||
for path in files:
|
for path in files:
|
||||||
# ------------- Parsing -------------
|
|
||||||
try:
|
try:
|
||||||
parsed = read_markdown(path)
|
parsed = read_markdown(path)
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "error": f"read_markdown failed: {e}"}))
|
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
||||||
continue
|
|
||||||
if parsed is None:
|
|
||||||
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# --- Frontmatter prüfen ---
|
||||||
try:
|
try:
|
||||||
fm = normalize_frontmatter(parsed.frontmatter)
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
validate_required_frontmatter(fm)
|
validate_required_frontmatter(fm)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"}))
|
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {type(e).__name__}: {e}"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if args.note_id and fm.get("id") != args.note_id:
|
if args.note_id and not args.only_path and fm.get("id") != args.note_id:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
processed += 1
|
processed += 1
|
||||||
|
|
||||||
# ------------- Note-Payload -------------
|
# --- Type-Registry anwenden (chunk_profile / retriever_weight) ---
|
||||||
try:
|
try:
|
||||||
note_pl_raw = make_note_payload(
|
note_type = resolve_note_type(fm.get("type"), reg)
|
||||||
|
except Exception:
|
||||||
|
note_type = (fm.get("type") or "concept")
|
||||||
|
fm["type"] = note_type or fm.get("type") or "concept"
|
||||||
|
|
||||||
|
prof = effective_chunk_profile(note_type, reg)
|
||||||
|
if prof:
|
||||||
|
fm["chunk_profile"] = prof
|
||||||
|
|
||||||
|
weight = effective_retriever_weight(note_type, reg)
|
||||||
|
if weight is not None:
|
||||||
|
try:
|
||||||
|
fm["retriever_weight"] = float(weight)
|
||||||
|
except Exception:
|
||||||
|
pass # falls FM string-inkonsistent ist
|
||||||
|
|
||||||
|
# --- Payload aufbauen (inkl. Hashes) ---
|
||||||
|
try:
|
||||||
|
note_pl = make_note_payload(
|
||||||
parsed,
|
parsed,
|
||||||
vault_root=root,
|
vault_root=root,
|
||||||
hash_mode=args.hash_mode,
|
hash_mode=mode,
|
||||||
hash_normalize=args.hash_normalize,
|
hash_normalize=norm,
|
||||||
hash_source=args.hash_source,
|
hash_source=src,
|
||||||
file_path=path,
|
file_path=path,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": fm.get("id"), "error": f"make_note_payload failed: {e}"}))
|
print(json.dumps({"path": path, "error": f"make_note_payload failed: {type(e).__name__}: {e}"}))
|
||||||
note_pl_raw = None
|
continue
|
||||||
|
|
||||||
note_pl = _coerce_to_dict(note_pl_raw)
|
|
||||||
if not isinstance(note_pl, dict):
|
|
||||||
# Fallback: baue minimalen Payload
|
|
||||||
note_pl = _fallback_note_payload(
|
|
||||||
parsed,
|
|
||||||
root,
|
|
||||||
hash_mode=args.hash_mode,
|
|
||||||
hash_normalize=args.hash_normalize,
|
|
||||||
hash_source=args.hash_source,
|
|
||||||
file_path=path,
|
|
||||||
)
|
|
||||||
print(json.dumps({
|
|
||||||
"path": path,
|
|
||||||
"note_id": fm.get("id"),
|
|
||||||
"warn": "make_note_payload returned non-dict; used fallback",
|
|
||||||
"returned_type": type(note_pl_raw).__name__ if note_pl_raw is not None else "NoneType"
|
|
||||||
}))
|
|
||||||
|
|
||||||
# fulltext + path Sicherheit
|
|
||||||
if not note_pl.get("fulltext"):
|
if not note_pl.get("fulltext"):
|
||||||
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
||||||
if note_pl.get("path"):
|
|
||||||
note_pl["path"] = _normalize_rel_path(
|
# retriever_weight sicher in Note-Payload spiegeln (für spätere Filter)
|
||||||
os.path.join(root, note_pl["path"]) if not os.path.isabs(note_pl["path"]) else note_pl["path"], root
|
if "retriever_weight" not in note_pl and fm.get("retriever_weight") is not None:
|
||||||
)
|
try:
|
||||||
else:
|
note_pl["retriever_weight"] = float(fm.get("retriever_weight"))
|
||||||
p_path = getattr(parsed, "path", None) or path
|
except Exception:
|
||||||
note_pl["path"] = _normalize_rel_path(p_path, root)
|
pass
|
||||||
|
|
||||||
note_id = note_pl.get("note_id") or fm.get("id")
|
note_id = note_pl.get("note_id") or fm.get("id")
|
||||||
if not note_id:
|
if not note_id:
|
||||||
print(json.dumps({"path": path, "error": "Missing note_id after payload build (even in fallback)"}))
|
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# ------------- Change-Detection -------------
|
# --- bestehenden Payload laden (zum Diff) ---
|
||||||
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
||||||
old_hash = None if not old_payload else old_payload.get("hash_fulltext")
|
has_old = old_payload is not None
|
||||||
old_sig = (old_payload or {}).get("hash_signature")
|
|
||||||
new_hash = note_pl.get("hash_fulltext")
|
|
||||||
new_sig = note_pl.get("hash_signature")
|
|
||||||
|
|
||||||
sig_changed = bool(old_sig) and bool(new_sig) and (old_sig.split(":")[:3] != new_sig.split(":")[:3])
|
old_hashes = (old_payload or {}).get("hashes") or {}
|
||||||
hash_changed = (old_hash != new_hash)
|
old_hash_exact = old_hashes.get(key_current)
|
||||||
|
new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
|
||||||
|
needs_baseline = (old_hash_exact is None)
|
||||||
|
|
||||||
|
hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact)
|
||||||
|
|
||||||
text_changed = False
|
text_changed = False
|
||||||
if compare_text:
|
if compare_text:
|
||||||
@@ -379,77 +468,123 @@ def main() -> None:
|
|||||||
new_text = note_pl.get("fulltext") or ""
|
new_text = note_pl.get("fulltext") or ""
|
||||||
text_changed = (old_text != new_text)
|
text_changed = (old_text != new_text)
|
||||||
|
|
||||||
changed = args.force_replace or sig_changed or hash_changed or text_changed
|
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
||||||
|
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
||||||
|
|
||||||
# ------------- Chunks / Embeddings / Edges -------------
|
# --- Chunks + Embeddings vorbereiten ---
|
||||||
try:
|
try:
|
||||||
chunks = assemble_chunks(fm["id"], getattr(parsed, "body", "") or "", fm.get("type", "concept"))
|
body_text = getattr(parsed, "body", "") or ""
|
||||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
|
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
|
||||||
|
chunk_pls: List[Dict[str, Any]] = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if embed_texts:
|
# retriever_weight auf Chunk-Payload spiegeln
|
||||||
|
if fm.get("retriever_weight") is not None:
|
||||||
try:
|
try:
|
||||||
vecs = embed_texts([getattr(c, "text", "") for c in chunks]) # type: ignore[attr-defined]
|
rw = float(fm.get("retriever_weight"))
|
||||||
|
for pl in chunk_pls:
|
||||||
|
# Feld nur setzen, wenn noch nicht vorhanden
|
||||||
|
if "retriever_weight" not in pl:
|
||||||
|
pl["retriever_weight"] = rw
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Embeddings (fallback: Nullvektoren)
|
||||||
|
vecs: List[List[float]] = [[0.0] * int(cfg.dim) for _ in chunk_pls]
|
||||||
|
if embed_texts and chunk_pls:
|
||||||
|
try:
|
||||||
|
texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls]
|
||||||
|
vecs = embed_texts(texts_for_embed)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
||||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
|
||||||
else:
|
|
||||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
|
||||||
|
|
||||||
try:
|
# --- Fehlende Artefakte in Qdrant ermitteln ---
|
||||||
note_refs = note_pl.get("references") or []
|
chunks_missing, edges_missing = artifacts_missing(client, cfg.prefix, note_id)
|
||||||
edges = build_edges_for_note(
|
|
||||||
note_id,
|
|
||||||
chunk_pls,
|
|
||||||
note_refs,
|
|
||||||
include_note_scope_refs=note_scope_refs,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"build_edges_for_note failed: {e}"}))
|
|
||||||
continue
|
|
||||||
|
|
||||||
# ------------- Summary -------------
|
# --- Edges (robust) ---
|
||||||
|
edges: List[Dict[str, Any]] = []
|
||||||
|
edges_failed = False
|
||||||
|
should_build_edges = (changed and (not do_baseline_only)) or edges_missing
|
||||||
|
if should_build_edges:
|
||||||
|
try:
|
||||||
|
note_refs = note_pl.get("references") or []
|
||||||
|
edges = build_edges_for_note(
|
||||||
|
note_id,
|
||||||
|
chunk_pls,
|
||||||
|
note_level_references=note_refs,
|
||||||
|
include_note_scope_refs=note_scope_refs,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
edges_failed = True
|
||||||
|
edges = []
|
||||||
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
|
||||||
|
|
||||||
|
# --- Summary (stdout) ---
|
||||||
summary = {
|
summary = {
|
||||||
"note_id": note_id,
|
"note_id": note_id,
|
||||||
"title": fm.get("title"),
|
"title": fm.get("title"),
|
||||||
"chunks": len(chunk_pls),
|
"chunks": len(chunk_pls),
|
||||||
"edges": len(edges),
|
"edges": len(edges),
|
||||||
|
"edges_failed": edges_failed,
|
||||||
"changed": changed,
|
"changed": changed,
|
||||||
"decision": ("apply" if args.apply and changed else
|
"chunks_missing": chunks_missing,
|
||||||
"apply-skip-unchanged" if args.apply and not changed else
|
"edges_missing": edges_missing,
|
||||||
|
"needs_baseline_for_mode": needs_baseline,
|
||||||
|
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
||||||
|
"apply" if args.apply and (changed or chunks_missing or edges_missing) else
|
||||||
|
"apply-skip-unchanged" if args.apply and not (changed or chunks_missing or edges_missing) else
|
||||||
"dry-run"),
|
"dry-run"),
|
||||||
"path": note_pl["path"],
|
"path": note_pl["path"],
|
||||||
"hash_mode": args.hash_mode or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE", "body"),
|
"hash_mode": mode,
|
||||||
"hash_normalize": args.hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical"),
|
"hash_normalize": norm,
|
||||||
"hash_source": args.hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed"),
|
"hash_source": src,
|
||||||
"hash_signature": note_pl.get("hash_signature"),
|
"prefix": cfg.prefix,
|
||||||
"sig_changed": sig_changed,
|
|
||||||
"hash_changed": hash_changed,
|
|
||||||
"text_changed": text_changed,
|
|
||||||
}
|
}
|
||||||
print(json.dumps(summary, ensure_ascii=False))
|
print(json.dumps(summary, ensure_ascii=False))
|
||||||
|
|
||||||
# ------------- Upserts -------------
|
# --- Writes ---
|
||||||
if not args.apply:
|
if not args.apply:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
if do_baseline_only:
|
||||||
if changed and args.purge_before_upsert:
|
merged_hashes = {}
|
||||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
merged_hashes.update(old_hashes)
|
||||||
except Exception as e:
|
merged_hashes.update(note_pl.get("hashes") or {})
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
if old_payload:
|
||||||
|
note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext"))
|
||||||
try:
|
note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature"))
|
||||||
|
note_pl["hashes"] = merged_hashes
|
||||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||||
upsert_batch(client, notes_name, note_pts)
|
upsert_batch(client, notes_name, note_pts)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Wenn nichts geändert und keine Artefakte fehlen → nichts zu tun
|
||||||
|
if not changed and not (chunks_missing or edges_missing):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Purge nur bei echten Änderungen (unverändert + fehlende Artefakte ≠ Purge)
|
||||||
|
if args.purge_before_upsert and has_old and changed:
|
||||||
|
try:
|
||||||
|
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
||||||
|
|
||||||
|
# Note nur bei Änderungen neu schreiben
|
||||||
|
if changed:
|
||||||
|
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||||
|
upsert_batch(client, notes_name, note_pts)
|
||||||
|
|
||||||
|
# Chunks schreiben, wenn geändert ODER vorher fehlend
|
||||||
|
if chunk_pls and (changed or chunks_missing):
|
||||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||||
upsert_batch(client, chunks_name, chunk_pts)
|
upsert_batch(client, chunks_name, chunk_pts)
|
||||||
|
|
||||||
|
# Edges schreiben, wenn vorhanden und (geändert ODER vorher fehlend)
|
||||||
|
if edges and (changed or edges_missing):
|
||||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||||
upsert_batch(client, edges_name, edge_pts)
|
upsert_batch(client, edges_name, edge_pts)
|
||||||
except Exception as e:
|
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"upsert failed: {e}"}))
|
|
||||||
|
|
||||||
print(f"Done. Processed notes: {processed}")
|
print(f"Done. Processed notes: {processed}")
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/ollama_tool_runner.py — Minimaler Tool-Caller für Ollama + mindnet
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Führt den Tool-Call-Loop für mindnet_query / mindnet_subgraph aus.
|
||||||
|
Nutzt /tools/ollama (Schema) und deine FastAPI (/query, /graph).
|
||||||
|
ENV:
|
||||||
|
OLLAMA=http://127.0.0.1:11434
|
||||||
|
API_BASE=http://127.0.0.1:8000
|
||||||
|
MODEL=llama3.1
|
||||||
|
Nutzung:
|
||||||
|
python3 scripts/ollama_tool_runner.py "Frage an den Agenten"
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
import os, sys, json, requests
|
||||||
|
|
||||||
|
OLLAMA = os.getenv("OLLAMA", "http://127.0.0.1:11434")
|
||||||
|
API_BASE = os.getenv("API_BASE", "http://127.0.0.1:8000")
|
||||||
|
MODEL = os.getenv("MODEL", "llama3.1")
|
||||||
|
|
||||||
|
def load_tools():
|
||||||
|
r = requests.get(f"{API_BASE}/tools/ollama", timeout=30)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()["tools"]
|
||||||
|
|
||||||
|
def call_model(messages, tools):
|
||||||
|
payload = {
|
||||||
|
"model": MODEL,
|
||||||
|
"messages": messages,
|
||||||
|
"tools": tools,
|
||||||
|
"stream": False,
|
||||||
|
"tool_choice": "auto",
|
||||||
|
}
|
||||||
|
r = requests.post(f"{OLLAMA}/api/chat", json=payload, timeout=180)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def execute_tool(name: str, arguments: dict):
|
||||||
|
if name == "mindnet_query":
|
||||||
|
# mappe expand_depth/edge_types auf /query-Body
|
||||||
|
body = {
|
||||||
|
"mode": "hybrid",
|
||||||
|
"top_k": int(arguments.get("top_k", 10)),
|
||||||
|
"expand": {
|
||||||
|
"depth": int(arguments.get("expand_depth", 1)),
|
||||||
|
"edge_types": arguments.get("edge_types") or ["references","belongs_to","prev","next"],
|
||||||
|
},
|
||||||
|
"filters": arguments.get("filters"),
|
||||||
|
}
|
||||||
|
if "query_vector" in arguments:
|
||||||
|
body["query_vector"] = arguments["query_vector"]
|
||||||
|
if "query" in arguments:
|
||||||
|
body["query"] = arguments["query"]
|
||||||
|
r = requests.post(f"{API_BASE}/query", json=body, timeout=120)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
if name == "mindnet_subgraph":
|
||||||
|
params = {
|
||||||
|
"depth": int(arguments.get("depth", 1)),
|
||||||
|
}
|
||||||
|
edge_types = arguments.get("edge_types")
|
||||||
|
if edge_types:
|
||||||
|
# mehrfacher Query-Param 'edge_types' wird von requests automatisch gebaut
|
||||||
|
params["edge_types"] = edge_types
|
||||||
|
note_id = arguments["note_id"]
|
||||||
|
r = requests.get(f"{API_BASE}/graph/{note_id}", params=params, timeout=60)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
raise ValueError(f"Unbekanntes Tool: {name}")
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print("Usage: python3 scripts/ollama_tool_runner.py \"Deine Frage\"")
|
||||||
|
return 2
|
||||||
|
|
||||||
|
user_prompt = sys.argv[1]
|
||||||
|
tools = load_tools()
|
||||||
|
messages = [{"role": "user", "content": user_prompt}]
|
||||||
|
|
||||||
|
while True:
|
||||||
|
resp = call_model(messages, tools)
|
||||||
|
msg = resp.get("message") or {}
|
||||||
|
tool_calls = msg.get("tool_calls") or []
|
||||||
|
|
||||||
|
if not tool_calls:
|
||||||
|
# finale Antwort
|
||||||
|
print("\n=== Antwort ===\n")
|
||||||
|
print(msg.get("content", "").strip())
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# Führe Toolcalls nacheinander aus
|
||||||
|
for tc in tool_calls:
|
||||||
|
name = tc["function"]["name"]
|
||||||
|
args = tc["function"].get("arguments") or {}
|
||||||
|
if isinstance(args, str):
|
||||||
|
try: args = json.loads(args)
|
||||||
|
except Exception: args = {"raw": args}
|
||||||
|
|
||||||
|
print(f"[tool] {name}({args})")
|
||||||
|
result = execute_tool(name, args)
|
||||||
|
|
||||||
|
# Übergib Toolresultat ans Modell zurück
|
||||||
|
messages.append({"role": "tool", "content": json.dumps(result), "name": name})
|
||||||
|
|
||||||
|
# füttere auch die originale Antwort (falls enthalten) in den Verlauf
|
||||||
|
if msg.get("content"):
|
||||||
|
messages.append({"role": "assistant", "content": msg["content"]})
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/payload_dryrun.py (zeigt, was VOR dem Upsert tatsächlich in den Payloads steht)
|
||||||
|
- KEIN Überschreiben der Note-Payload mehr
|
||||||
|
- types.yaml ist maßgeblich (gemäß app/core/note_payload.py & chunk_payload.py)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, os, json
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||||
|
from app.core.note_payload import make_note_payload
|
||||||
|
from app.core.chunker import assemble_chunks
|
||||||
|
from app.core.chunk_payload import make_chunk_payloads
|
||||||
|
try:
|
||||||
|
from app.core.derive_edges import build_edges_for_note
|
||||||
|
except Exception:
|
||||||
|
from app.core.edges import build_edges_for_note # type: ignore
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--vault", required=True)
|
||||||
|
ap.add_argument("--note-id")
|
||||||
|
ap.add_argument("--with-edges", action="store_true")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
root = os.path.abspath(args.vault)
|
||||||
|
|
||||||
|
files: List[str] = []
|
||||||
|
for dp, _, fns in os.walk(root):
|
||||||
|
for fn in fns:
|
||||||
|
if fn.lower().endswith(".md"):
|
||||||
|
files.append(os.path.join(dp, fn))
|
||||||
|
files.sort()
|
||||||
|
|
||||||
|
for path in files:
|
||||||
|
parsed = read_markdown(path)
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
|
try:
|
||||||
|
validate_required_frontmatter(fm)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"path": path, "error": f"invalid frontmatter: {e}"}))
|
||||||
|
continue
|
||||||
|
|
||||||
|
if args.note_id and fm.get("id") != args.note_id:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Note-Payload exakt so, wie der Importer ihn baut (types.yaml maßgeblich)
|
||||||
|
note_pl = make_note_payload(parsed,
|
||||||
|
vault_root=root,
|
||||||
|
hash_mode="body",
|
||||||
|
hash_normalize="canonical",
|
||||||
|
hash_source="parsed",
|
||||||
|
file_path=path)
|
||||||
|
|
||||||
|
body_text = getattr(parsed, "body", "") or ""
|
||||||
|
chunks = assemble_chunks(fm["id"], body_text, fm.get("type","concept"))
|
||||||
|
|
||||||
|
chunk_note = {
|
||||||
|
"frontmatter": fm,
|
||||||
|
"id": fm.get("id"),
|
||||||
|
"type": fm.get("type"),
|
||||||
|
"title": fm.get("title"),
|
||||||
|
"path": note_pl.get("path") or path,
|
||||||
|
"note_id": note_pl.get("note_id"),
|
||||||
|
"tags": fm.get("tags"),
|
||||||
|
}
|
||||||
|
# make_chunk_payloads bestimmt Werte ebenfalls aus types.yaml (Frontmatter wird ignoriert)
|
||||||
|
chunk_pls = make_chunk_payloads(
|
||||||
|
chunk_note,
|
||||||
|
note_pl["path"],
|
||||||
|
chunks,
|
||||||
|
note_text=body_text,
|
||||||
|
types_cfg=None, # Loader aus Datei; kein Override von außen
|
||||||
|
file_path=path,
|
||||||
|
)
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"note_id": note_pl.get("note_id") or fm.get("id"),
|
||||||
|
"title": fm.get("title"),
|
||||||
|
"type": fm.get("type"),
|
||||||
|
"note_payload": {
|
||||||
|
"retriever_weight": note_pl.get("retriever_weight"),
|
||||||
|
"chunk_profile": note_pl.get("chunk_profile")
|
||||||
|
},
|
||||||
|
"chunks_summary": {
|
||||||
|
"count": len(chunk_pls),
|
||||||
|
"first": [
|
||||||
|
{k: chunk_pls[i].get(k) for k in ("chunk_id","index","ord","retriever_weight","chunk_profile","neighbors_prev","neighbors_next")}
|
||||||
|
for i in range(min(3, len(chunk_pls)))
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"path": note_pl.get("path")
|
||||||
|
}
|
||||||
|
|
||||||
|
if args.with_edges:
|
||||||
|
edges = build_edges_for_note(
|
||||||
|
note_id=note_pl.get("note_id") or fm.get("id"),
|
||||||
|
chunk_payloads=chunk_pls,
|
||||||
|
note_level_refs=note_pl.get("references") or [],
|
||||||
|
include_note_scope_refs=False,
|
||||||
|
)
|
||||||
|
kinds = {}
|
||||||
|
for e in edges:
|
||||||
|
k = (e.get("relation") or e.get("kind") or "edge")
|
||||||
|
kinds[k] = kinds.get(k, 0) + 1
|
||||||
|
out["edges_summary"] = {"total": len(edges), "by_kind": kinds}
|
||||||
|
|
||||||
|
print(json.dumps(out, ensure_ascii=False))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Script: scripts/report_hashes.py — Übersicht & Lücken bei Mehrfach-Hashes
|
||||||
|
Version: 1.0.0
|
||||||
|
Datum: 2025-09-10
|
||||||
|
|
||||||
|
Funktion
|
||||||
|
--------
|
||||||
|
Listet je Note die vorhandenen Einträge im Feld `hashes` (Signaturen: <mode>:<source>:<normalize>)
|
||||||
|
und meldet fehlende Soll-Keys. Eignet sich als CI-Check.
|
||||||
|
|
||||||
|
Optionen
|
||||||
|
--------
|
||||||
|
--prefix TEXT Collection-Prefix (CLI überschreibt ENV)
|
||||||
|
--require K [K ...] Zusätzliche Soll-Keys (Default: body|frontmatter|full:parsed:canonical)
|
||||||
|
--fail-on-missing Exitcode 2, wenn fehlende Keys gefunden werden
|
||||||
|
|
||||||
|
Beispiele
|
||||||
|
---------
|
||||||
|
python3 -m scripts.report_hashes --prefix mindnet
|
||||||
|
python3 -m scripts.report_hashes --require frontmatter:raw:none --fail-on-missing
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from typing import List, Dict, Any
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def collections(prefix: str):
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def _scroll_all(client, collection: str):
|
||||||
|
out = []
|
||||||
|
nextp = None
|
||||||
|
while True:
|
||||||
|
pts, nextp = client.scroll(collection_name=collection, with_payload=True, with_vectors=False, limit=256, offset=nextp)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
out.extend(pts)
|
||||||
|
if nextp is None:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||||
|
ap.add_argument("--require", nargs="+", help="Zusätzliche Soll-Keys <mode>:<source>:<normalize>")
|
||||||
|
ap.add_argument("--fail-on-missing", action="store_true", help="Exitcode 2 bei fehlenden Keys")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
if args.prefix:
|
||||||
|
cfg.prefix = args.prefix.strip()
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
notes_col, _, _ = collections(cfg.prefix)
|
||||||
|
pts = _scroll_all(client, notes_col)
|
||||||
|
|
||||||
|
required = set(args.require or [])
|
||||||
|
required |= {
|
||||||
|
"body:parsed:canonical",
|
||||||
|
"frontmatter:parsed:canonical",
|
||||||
|
"full:parsed:canonical",
|
||||||
|
}
|
||||||
|
|
||||||
|
missing_total = 0
|
||||||
|
for p in pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
hashes = pl.get("hashes") or {}
|
||||||
|
present = set(hashes.keys())
|
||||||
|
missing = sorted(list(required - present))
|
||||||
|
obj = {
|
||||||
|
"note_id": nid,
|
||||||
|
"present_count": len(present),
|
||||||
|
"missing": missing,
|
||||||
|
}
|
||||||
|
print(json.dumps(obj, ensure_ascii=False))
|
||||||
|
missing_total += len(missing)
|
||||||
|
|
||||||
|
if args.fail_on_missing and missing_total > 0:
|
||||||
|
raise SystemExit(2)
|
||||||
|
|
||||||
|
print(json.dumps({"summary_missing_total": missing_total}))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+33
-10
@@ -2,28 +2,33 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Name: scripts/reset_qdrant.py
|
Name: scripts/reset_qdrant.py
|
||||||
Version: v1.1.1 (2025-09-05)
|
Version: v1.2.0 (2025-11-11)
|
||||||
Kurzbeschreibung:
|
Kurzbeschreibung:
|
||||||
Sicheres Zurücksetzen der Qdrant-Collections für EIN Projektpräfix. Das Skript
|
Sicheres Zurücksetzen der Qdrant-Collections für EIN Projektpräfix. Das Skript
|
||||||
ermittelt zunächst die tatsächlich betroffenen Collections und zeigt eine
|
ermittelt zunächst die tatsächlich betroffenen Collections und zeigt eine
|
||||||
Übersicht an. Anschließend wird der User um Bestätigung gebeten (interaktive
|
Übersicht an. Anschließend wird der User um Bestätigung gebeten (interaktive
|
||||||
Abfrage), bevor eine der beiden Aktionen ausgeführt wird:
|
Abfrage), bevor eine der Aktionen ausgeführt wird:
|
||||||
|
|
||||||
- wipe: Löscht die Collections komplett und legt sie gemäß Projekt-Defaults neu an
|
- wipe: Löscht die Collections komplett und legt sie gemäß Projekt-Defaults neu an.
|
||||||
- truncate: Löscht nur alle Points (Inhalte), behält die Collection-Settings
|
**Neu**: Danach werden *immer* die Payload-Indizes & Schema-Ergänzungen
|
||||||
|
über ensure_payload_indexes() idempotent eingerichtet (falls nicht mit --no-indexes deaktiviert).
|
||||||
|
- truncate: Löscht nur alle Points (Inhalte), behält die Collection-Settings.
|
||||||
|
**Neu**: Optional (Default) werden im Anschluss die Payload-Indizes geprüft/ergänzt.
|
||||||
|
|
||||||
Aufruf (aus Projekt-Root, im venv):
|
Aufruf (aus Projekt-Root, im venv):
|
||||||
python3 -m scripts.reset_qdrant --mode wipe [--prefix PREFIX] [--yes] [--dry-run]
|
python3 -m scripts.reset_qdrant --mode wipe [--prefix PREFIX] [--yes] [--dry-run]
|
||||||
python3 -m scripts.reset_qdrant --mode truncate [--prefix PREFIX] [--yes] [--dry-run]
|
python3 -m scripts.reset_qdrant --mode truncate [--prefix PREFIX] [--yes] [--dry-run]
|
||||||
|
|
||||||
Parameter:
|
Parameter:
|
||||||
--mode wipe | truncate (Pflicht)
|
--mode wipe | truncate (Pflicht)
|
||||||
--prefix Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')
|
--prefix Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')
|
||||||
--yes Keine Rückfrage, direkt ausführen (CI/CD geeignet)
|
--yes Keine Rückfrage, direkt ausführen (CI/CD geeignet)
|
||||||
--dry-run Nur anzeigen, was passieren würde; nichts ändern
|
--dry-run Nur anzeigen, was passieren würde; nichts ändern
|
||||||
|
--no-indexes Überspringt den Schritt ensure_payload_indexes() (Standard: Indizes/Schema werden geprüft/ergänzt)
|
||||||
|
|
||||||
Umgebungsvariablen (optional):
|
Umgebungsvariablen (optional):
|
||||||
QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)
|
QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)
|
||||||
|
(Zusätzliche ENV für Named Vectors etc. werden innerhalb ensure_collections/ensure_payload_indexes berücksichtigt.)
|
||||||
|
|
||||||
Sicherheitsmerkmale:
|
Sicherheitsmerkmale:
|
||||||
- Betrachtet ausschließlich Collections des angegebenen Präfixes.
|
- Betrachtet ausschließlich Collections des angegebenen Präfixes.
|
||||||
@@ -34,6 +39,8 @@ Exitcodes:
|
|||||||
0 = OK, 1 = abgebrochen/keine Aktion, 2 = Verbindungs-/Konfigurationsfehler
|
0 = OK, 1 = abgebrochen/keine Aktion, 2 = Verbindungs-/Konfigurationsfehler
|
||||||
|
|
||||||
Changelog:
|
Changelog:
|
||||||
|
v1.2.0: ensure_payload_indexes() nach wipe/truncate standardmäßig ausführen (idempotent); --no-indexes Flag ergänzt.
|
||||||
|
v1.1.1: Stabilisierung & Preview (2025-09-05).
|
||||||
v1.1.0: Interaktive Bestätigung, --yes/--dry-run hinzugefügt, Preview der betroffenen Collections.
|
v1.1.0: Interaktive Bestätigung, --yes/--dry-run hinzugefügt, Preview der betroffenen Collections.
|
||||||
v1.0.0: Wipe/Truncate ohne Bestätigungsabfrage.
|
v1.0.0: Wipe/Truncate ohne Bestätigungsabfrage.
|
||||||
"""
|
"""
|
||||||
@@ -46,7 +53,13 @@ from typing import List
|
|||||||
from qdrant_client import QdrantClient
|
from qdrant_client import QdrantClient
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names
|
from app.core.qdrant import (
|
||||||
|
QdrantConfig,
|
||||||
|
get_client,
|
||||||
|
ensure_collections,
|
||||||
|
ensure_payload_indexes,
|
||||||
|
collection_names,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def resolve_existing_collections(client: QdrantClient, prefix: str) -> List[str]:
|
def resolve_existing_collections(client: QdrantClient, prefix: str) -> List[str]:
|
||||||
@@ -111,12 +124,13 @@ def wipe_collections(client: QdrantClient, all_col_names: List[str], existing: L
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
ap = argparse.ArgumentParser(description="Wipe oder truncate mindnet-Collections in Qdrant (mit Bestätigung).")
|
ap = argparse.ArgumentParser(description="Wipe oder truncate mindnet-Collections in Qdrant (mit Bestätigung & Index-Setup).")
|
||||||
ap.add_argument("--mode", choices=["wipe", "truncate"], required=True,
|
ap.add_argument("--mode", choices=["wipe", "truncate"], required=True,
|
||||||
help="wipe = Collections löschen & neu anlegen; truncate = nur Inhalte löschen")
|
help="wipe = Collections löschen & neu anlegen; truncate = nur Inhalte löschen")
|
||||||
ap.add_argument("--prefix", help="Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')")
|
ap.add_argument("--prefix", help="Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')")
|
||||||
ap.add_argument("--yes", action="store_true", help="Ohne Rückfrage ausführen (CI/CD)")
|
ap.add_argument("--yes", action="store_true", help="Ohne Rückfrage ausführen (CI/CD)")
|
||||||
ap.add_argument("--dry-run", action="store_true", help="Nur anzeigen, was passieren würde; nichts ändern")
|
ap.add_argument("--dry-run", action="store_true", help="Nur anzeigen, was passieren würde; nichts ändern")
|
||||||
|
ap.add_argument("--no-indexes", action="store_true", help="Überspringt ensure_payload_indexes()")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
# Qdrant-Konfiguration
|
# Qdrant-Konfiguration
|
||||||
@@ -155,13 +169,22 @@ def main():
|
|||||||
if args.mode == "wipe":
|
if args.mode == "wipe":
|
||||||
wipe_collections(client, all_col_names, existing)
|
wipe_collections(client, all_col_names, existing)
|
||||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||||
|
if not args.no_indexes:
|
||||||
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
print(f"Wiped & recreated (Prefix={cfg.prefix}): {all_col_names}")
|
print(f"Wiped & recreated (Prefix={cfg.prefix}): {all_col_names}")
|
||||||
|
if not args.no_indexes:
|
||||||
|
print("Payload-Indizes & Schema-Ergänzungen geprüft/ergänzt (ensure_payload_indexes).")
|
||||||
else:
|
else:
|
||||||
if not existing:
|
if not existing:
|
||||||
print("Keine existierenden Collections zum Truncaten gefunden. Beende ohne Aktion.")
|
print("Keine existierenden Collections zum Truncaten gefunden. Beende ohne Aktion.")
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
delete_all_points(client, existing)
|
delete_all_points(client, existing)
|
||||||
|
if not args.no_indexes:
|
||||||
|
# Auch nach truncate sicherstellen, dass neue/fehlende Indizes ergänzt werden
|
||||||
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
print(f"Truncated (deleted all points in): {existing}")
|
print(f"Truncated (deleted all points in): {existing}")
|
||||||
|
if not args.no_indexes:
|
||||||
|
print("Payload-Indizes & Schema-Ergänzungen geprüft/ergänzt (ensure_payload_indexes).")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/wp04_smoketest.py — E2E-Schnelltest der WP-04 Endpunkte
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
- Holt exemplarisch einen Vektor aus {prefix}_chunks (falls verfügbar),
|
||||||
|
- ruft POST /query mit diesem Vektor auf,
|
||||||
|
- ruft GET /graph/<note_id> für die Top-Note auf,
|
||||||
|
- druckt Kurzresultate auf STDOUT.
|
||||||
|
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, requests, qdrant-client
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
WP-04 /query & /graph
|
||||||
|
Nutzung:
|
||||||
|
export QDRANT_URL="http://localhost:6333"
|
||||||
|
export MINDNET_PREFIX="mindnet"
|
||||||
|
python3 scripts/wp04_smoketest.py --api http://localhost:8000
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import argparse
|
||||||
|
import requests
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
|
||||||
|
|
||||||
|
def get_any_chunk_vector(client: QdrantClient, prefix: str):
|
||||||
|
"""Holt eine beliebige Chunk-Vector/Note-ID-Kombi aus Qdrant (falls vorhanden)."""
|
||||||
|
col = f"{prefix}_chunks"
|
||||||
|
pts, _ = client.scroll(collection_name=col, limit=1, with_vectors=True, with_payload=True)
|
||||||
|
if not pts:
|
||||||
|
return None, None, None
|
||||||
|
p = pts[0]
|
||||||
|
vec = p.vector
|
||||||
|
payload = p.payload or {}
|
||||||
|
node_id = str(p.id)
|
||||||
|
note_id = payload.get("note_id")
|
||||||
|
return vec, node_id, note_id
|
||||||
|
|
||||||
|
|
||||||
|
def run(api_base: str, qdrant_url: str, prefix: str) -> int:
|
||||||
|
print(f"[i] API: {api_base}")
|
||||||
|
print(f"[i] Qdrant: {qdrant_url}")
|
||||||
|
client = QdrantClient(url=qdrant_url)
|
||||||
|
|
||||||
|
vec, node_id, note_id = get_any_chunk_vector(client, prefix)
|
||||||
|
if vec is None:
|
||||||
|
print("[!] Keine Chunks mit Vektoren gefunden. Bitte Import (WP-03) prüfen.")
|
||||||
|
return 2
|
||||||
|
|
||||||
|
print(f"[i] Beispiel-Chunk-ID: {node_id}, Note-ID: {note_id}")
|
||||||
|
print("[i] Rufe /query (hybrid) mit Beispielvektor auf ...")
|
||||||
|
|
||||||
|
q_payload = {
|
||||||
|
"mode": "hybrid",
|
||||||
|
"query_vector": vec, # 384d
|
||||||
|
"top_k": 5,
|
||||||
|
"expand": {"depth": 1, "edge_types": ["references", "belongs_to", "prev", "next"]},
|
||||||
|
}
|
||||||
|
r = requests.post(f"{api_base.rstrip('/')}/query", json=q_payload, timeout=60)
|
||||||
|
print(f"[i] /query Status: {r.status_code}")
|
||||||
|
if r.status_code != 200:
|
||||||
|
print(r.text)
|
||||||
|
return 3
|
||||||
|
|
||||||
|
data = r.json()
|
||||||
|
hits = data.get("results", [])
|
||||||
|
print(f"[i] Treffer: {len(hits)}")
|
||||||
|
for i, h in enumerate(hits, 1):
|
||||||
|
print(f" {i:02d}. node_id={h['node_id']} note_id={h.get('note_id')} total={h['total_score']:.3f}")
|
||||||
|
|
||||||
|
# Graph-Test (wenn Note-ID vorhanden)
|
||||||
|
gid = note_id or (hits[0].get("note_id") if hits else None)
|
||||||
|
if gid:
|
||||||
|
print(f"[i] Rufe /graph/{gid} ...")
|
||||||
|
r2 = requests.get(f"{api_base.rstrip('/')}/graph/{gid}?depth=1", timeout=60)
|
||||||
|
print(f"[i] /graph Status: {r2.status_code}")
|
||||||
|
if r2.status_code == 200:
|
||||||
|
g = r2.json()
|
||||||
|
print(f" Nodes: {len(g.get('nodes', []))}, Edges: {len(g.get('edges', []))}")
|
||||||
|
else:
|
||||||
|
print(r2.text)
|
||||||
|
|
||||||
|
print("[✔] Smoke-Test abgeschlossen.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--api", default="http://localhost:8000", help="Basis-URL der FastAPI.")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
qdrant_url = os.getenv("QDRANT_URL", "http://127.0.0.1:6333")
|
||||||
|
prefix = os.getenv("MINDNET_PREFIX", "mindnet")
|
||||||
|
sys.exit(run(args.api, qdrant_url, prefix))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/assert_payload_schema.py (compat v1.1)
|
||||||
|
|
||||||
|
Prüft, ob die erwarteten Payload-Indizes (payload_schema) auf den drei Collections vorhanden sind.
|
||||||
|
Kompatibel mit älteren qdrant-client Versionen (ohne with_payload_schema im Wrapper).
|
||||||
|
|
||||||
|
Collections & Pflichtfelder:
|
||||||
|
- mindnet_notes : note_id, type, title, updated, tags
|
||||||
|
- mindnet_chunks : note_id, chunk_id, index, type, tags
|
||||||
|
- mindnet_edges : note_id, kind, scope, source_id, target_id, chunk_id
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 -m tests.assert_payload_schema
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
REQUIRED = {
|
||||||
|
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||||
|
"chunks": ["note_id", "chunk_id", "index", "type", "tags"],
|
||||||
|
"edges": ["note_id", "kind", "scope", "source_id", "target_id", "chunk_id"],
|
||||||
|
}
|
||||||
|
|
||||||
|
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||||
|
if hasattr(obj, "model_dump"):
|
||||||
|
return obj.model_dump()
|
||||||
|
if hasattr(obj, "dict"):
|
||||||
|
return obj.dict()
|
||||||
|
return obj if isinstance(obj, dict) else {}
|
||||||
|
|
||||||
|
def _cfg_base_url(cfg) -> str:
|
||||||
|
if getattr(cfg, "url", None):
|
||||||
|
return cfg.url.rstrip('/')
|
||||||
|
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||||
|
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||||
|
return f"http://{host}:{port}"
|
||||||
|
|
||||||
|
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||||
|
req = urllib.request.Request(url, method="GET")
|
||||||
|
if api_key:
|
||||||
|
req.add_header("api-key", api_key)
|
||||||
|
req.add_header("Authorization", f"Bearer {api_key}")
|
||||||
|
req.add_header("Accept", "application/json")
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = resp.read()
|
||||||
|
return json.loads(data.decode("utf-8"))
|
||||||
|
|
||||||
|
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||||
|
# 1) OpenAPI client mit Flag
|
||||||
|
try:
|
||||||
|
oc = getattr(client, "openapi_client", None)
|
||||||
|
if oc is not None and hasattr(oc, "collections_api"):
|
||||||
|
api = oc.collections_api
|
||||||
|
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
return (d.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# 2) Wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
ps = (d.get("result") or {}).get("payload_schema")
|
||||||
|
if ps is not None:
|
||||||
|
return ps
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# 3) Direkter HTTP-Call
|
||||||
|
try:
|
||||||
|
base = _cfg_base_url(cfg)
|
||||||
|
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||||
|
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||||
|
return (raw.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def ensure_fields(schema: Dict[str, dict] | None, required: List[str]) -> Dict[str, bool]:
|
||||||
|
ks = set((schema or {}).keys())
|
||||||
|
return {k: (k in ks) for k in required}
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
names = {"notes": notes, "chunks": chunks, "edges": edges}
|
||||||
|
|
||||||
|
report = {}
|
||||||
|
ok_all = True
|
||||||
|
for kind, col in names.items():
|
||||||
|
sch = get_collection_payload_schema(client, cfg, col)
|
||||||
|
checks = ensure_fields(sch, REQUIRED[kind])
|
||||||
|
ok = all(checks.values())
|
||||||
|
ok_all = ok_all and ok
|
||||||
|
report[kind] = {
|
||||||
|
"collection": col,
|
||||||
|
"ok": ok,
|
||||||
|
"missing": [k for k,v in checks.items() if not v],
|
||||||
|
"present": [k for k,v in checks.items() if v],
|
||||||
|
}
|
||||||
|
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "ok": ok_all, "report": report}, ensure_ascii=False, indent=2))
|
||||||
|
if not ok_all:
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/audit_edges_now.py — Zählt Edges nach kind & scope und zeigt Chunk- und Note-Anzahl.
|
||||||
|
"""
|
||||||
|
import os, sys, json
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def scroll_all(c, col, flt=None):
|
||||||
|
out, nextp = [], None
|
||||||
|
while True:
|
||||||
|
pts, nextp = c.scroll(collection_name=col, with_payload=True, with_vectors=False, limit=256, scroll_filter=flt, offset=nextp)
|
||||||
|
if not pts: break
|
||||||
|
out.extend(pts)
|
||||||
|
if nextp is None: break
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
c = get_client(cfg)
|
||||||
|
notes = f"{cfg.prefix}_notes"
|
||||||
|
chunks = f"{cfg.prefix}_chunks"
|
||||||
|
edges = f"{cfg.prefix}_edges"
|
||||||
|
|
||||||
|
es = scroll_all(c, edges)
|
||||||
|
counts = {}
|
||||||
|
by_scope = {}
|
||||||
|
for p in es:
|
||||||
|
pl = p.payload or {}
|
||||||
|
k = pl.get("kind") or "?"
|
||||||
|
s = pl.get("scope") or "?"
|
||||||
|
counts[k] = counts.get(k, 0) + 1
|
||||||
|
key = f"{k}:{s}"
|
||||||
|
by_scope[key] = by_scope.get(key, 0) + 1
|
||||||
|
|
||||||
|
print(json.dumps({
|
||||||
|
"collections": {"notes": notes, "chunks": chunks, "edges": edges},
|
||||||
|
"total_edges": len(es),
|
||||||
|
"counts": counts,
|
||||||
|
"counts_by_scope": by_scope
|
||||||
|
}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/check_chunks_window_vs_text.py
|
||||||
|
Prüft pro Note und global:
|
||||||
|
- ob window == text (sollte bei Overlap normalerweise NEIN sein)
|
||||||
|
- ungefähre linke Overlap-Länge (Suffix von prev.text vs. Prefix von cur.window)
|
||||||
|
- Größen & einfache Statistiken
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 tests/check_chunks_window_vs_text.py --prefix mindnet [--vault ./test_vault]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os, sys, json, argparse
|
||||||
|
from typing import List, Dict, Any, Tuple, Optional
|
||||||
|
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def scroll_chunks_by_note(client, chunks_col: str, note_id: str) -> List[Dict[str, Any]]:
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
out = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=chunks_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
scroll_filter=f,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
out.extend([p.payload or {} for p in pts])
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
# sortieren: seq -> chunk_index -> chunk_id
|
||||||
|
def keyer(pl):
|
||||||
|
return (pl.get("seq", 1<<30), pl.get("chunk_index", 1<<30), str(pl.get("chunk_id", "")))
|
||||||
|
return sorted(out, key=keyer)
|
||||||
|
|
||||||
|
def list_note_ids(client, notes_col: str) -> List[str]:
|
||||||
|
out = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=notes_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
for p in pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if isinstance(nid, str):
|
||||||
|
out.append(nid)
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
return sorted(set(out))
|
||||||
|
|
||||||
|
def common_overlap_len(a_suffix: str, b_prefix: str, max_probe: int = 256) -> int:
|
||||||
|
"""
|
||||||
|
Länge des längsten Suffix von a_suffix, das Prefix von b_prefix ist (bruteforce, begrenzt).
|
||||||
|
"""
|
||||||
|
n = min(len(a_suffix), len(b_prefix), max_probe)
|
||||||
|
if n <= 0:
|
||||||
|
return 0
|
||||||
|
a = a_suffix[-n:]
|
||||||
|
b = b_prefix[:n]
|
||||||
|
# steigere den Match von 0..n
|
||||||
|
for k in range(n, 0, -1):
|
||||||
|
if a[-k:] == b[:k]:
|
||||||
|
return k
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", required=True)
|
||||||
|
ap.add_argument("--vault", help="optional, nur für Vergleichsausgaben")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
cfg.prefix = args.prefix
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
notes_col, chunks_col, _ = collections(cfg.prefix)
|
||||||
|
note_ids = list_note_ids(client, notes_col)
|
||||||
|
if not note_ids:
|
||||||
|
print(json.dumps({"error": "no notes found in qdrant for this prefix"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
global_total = 0
|
||||||
|
global_ident = 0
|
||||||
|
global_left_ov_sum = 0
|
||||||
|
detail = []
|
||||||
|
|
||||||
|
for nid in note_ids:
|
||||||
|
chunks = scroll_chunks_by_note(client, chunks_col, nid)
|
||||||
|
if not chunks:
|
||||||
|
detail.append({"note_id": nid, "chunks": 0})
|
||||||
|
continue
|
||||||
|
ident = 0
|
||||||
|
left_ov_sum = 0
|
||||||
|
for i, ch in enumerate(chunks):
|
||||||
|
txt = (ch.get("text") or "").replace("\r\n", "\n")
|
||||||
|
win = (ch.get("window") or "").replace("\r\n", "\n")
|
||||||
|
if txt == win:
|
||||||
|
ident += 1
|
||||||
|
if i > 0:
|
||||||
|
prev_txt = (chunks[i-1].get("text") or "").replace("\r\n", "\n")
|
||||||
|
left_ov = common_overlap_len(prev_txt, win, max_probe=1024)
|
||||||
|
left_ov_sum += left_ov
|
||||||
|
global_total += len(chunks)
|
||||||
|
global_ident += ident
|
||||||
|
global_left_ov_sum += left_ov_sum
|
||||||
|
detail.append({
|
||||||
|
"note_id": nid,
|
||||||
|
"chunks": len(chunks),
|
||||||
|
"identical_text_window": ident,
|
||||||
|
"identical_share": round(ident / max(1, len(chunks)), 3),
|
||||||
|
"avg_left_overlap_est": round(left_ov_sum / max(1, len(chunks)-1), 2) if len(chunks) > 1 else 0.0
|
||||||
|
})
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"notes": len(note_ids),
|
||||||
|
"chunks_total": global_total,
|
||||||
|
"identical_total": global_ident,
|
||||||
|
"identical_share_global": round(global_ident / max(1, global_total), 3),
|
||||||
|
"avg_left_overlap_est_global": round(global_left_ov_sum / max(1, global_total - len(note_ids)), 2) if global_total > len(note_ids) else 0.0,
|
||||||
|
"details": detail[:100] # capped
|
||||||
|
}
|
||||||
|
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/check_chunks_window_vs_text.py
|
||||||
|
Prüft pro Note und global:
|
||||||
|
- ob window == text (sollte bei Overlap normalerweise NEIN sein)
|
||||||
|
- ungefähre linke Overlap-Länge (Suffix von prev.text vs. Prefix von cur.window)
|
||||||
|
- Größen & einfache Statistiken
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 tests/check_chunks_window_vs_text.py --prefix mindnet [--vault ./test_vault]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os, sys, json, argparse
|
||||||
|
from typing import List, Dict, Any, Tuple, Optional
|
||||||
|
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def scroll_chunks_by_note(client, chunks_col: str, note_id: str) -> List[Dict[str, Any]]:
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
out = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=chunks_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
scroll_filter=f,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
out.extend([p.payload or {} for p in pts])
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
# sortieren: seq -> chunk_index -> chunk_id
|
||||||
|
def keyer(pl):
|
||||||
|
return (pl.get("seq", 1<<30), pl.get("chunk_index", 1<<30), str(pl.get("chunk_id", "")))
|
||||||
|
return sorted(out, key=keyer)
|
||||||
|
|
||||||
|
def list_note_ids(client, notes_col: str) -> List[str]:
|
||||||
|
out = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
pts, next_page = client.scroll(
|
||||||
|
collection_name=notes_col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=256,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
for p in pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if isinstance(nid, str):
|
||||||
|
out.append(nid)
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
return sorted(set(out))
|
||||||
|
|
||||||
|
def common_overlap_len(a_suffix: str, b_prefix: str, max_probe: int = 256) -> int:
|
||||||
|
"""
|
||||||
|
Länge des längsten Suffix von a_suffix, das Prefix von b_prefix ist (bruteforce, begrenzt).
|
||||||
|
"""
|
||||||
|
n = min(len(a_suffix), len(b_prefix), max_probe)
|
||||||
|
if n <= 0:
|
||||||
|
return 0
|
||||||
|
a = a_suffix[-n:]
|
||||||
|
b = b_prefix[:n]
|
||||||
|
# steigere den Match von 0..n
|
||||||
|
for k in range(n, 0, -1):
|
||||||
|
if a[-k:] == b[:k]:
|
||||||
|
return k
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", required=True)
|
||||||
|
ap.add_argument("--vault", help="optional, nur für Vergleichsausgaben")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
cfg.prefix = args.prefix
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
notes_col, chunks_col, _ = collections(cfg.prefix)
|
||||||
|
note_ids = list_note_ids(client, notes_col)
|
||||||
|
if not note_ids:
|
||||||
|
print(json.dumps({"error": "no notes found in qdrant for this prefix"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
global_total = 0
|
||||||
|
global_ident = 0
|
||||||
|
global_left_ov_sum = 0
|
||||||
|
detail = []
|
||||||
|
|
||||||
|
for nid in note_ids:
|
||||||
|
chunks = scroll_chunks_by_note(client, chunks_col, nid)
|
||||||
|
if not chunks:
|
||||||
|
detail.append({"note_id": nid, "chunks": 0})
|
||||||
|
continue
|
||||||
|
ident = 0
|
||||||
|
left_ov_sum = 0
|
||||||
|
for i, ch in enumerate(chunks):
|
||||||
|
txt = (ch.get("text") or "").replace("\r\n", "\n")
|
||||||
|
win = (ch.get("window") or "").replace("\r\n", "\n")
|
||||||
|
if txt == win:
|
||||||
|
ident += 1
|
||||||
|
if i > 0:
|
||||||
|
prev_txt = (chunks[i-1].get("text") or "").replace("\r\n", "\n")
|
||||||
|
left_ov = common_overlap_len(prev_txt, win, max_probe=1024)
|
||||||
|
left_ov_sum += left_ov
|
||||||
|
global_total += len(chunks)
|
||||||
|
global_ident += ident
|
||||||
|
global_left_ov_sum += left_ov_sum
|
||||||
|
detail.append({
|
||||||
|
"note_id": nid,
|
||||||
|
"chunks": len(chunks),
|
||||||
|
"identical_text_window": ident,
|
||||||
|
"identical_share": round(ident / max(1, len(chunks)), 3),
|
||||||
|
"avg_left_overlap_est": round(left_ov_sum / max(1, len(chunks)-1), 2) if len(chunks) > 1 else 0.0
|
||||||
|
})
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"notes": len(note_ids),
|
||||||
|
"chunks_total": global_total,
|
||||||
|
"identical_total": global_ident,
|
||||||
|
"identical_share_global": round(global_ident / max(1, global_total), 3),
|
||||||
|
"avg_left_overlap_est_global": round(global_left_ov_sum / max(1, global_total - len(note_ids)), 2) if global_total > len(note_ids) else 0.0,
|
||||||
|
"details": detail[:100] # capped
|
||||||
|
}
|
||||||
|
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
check_types_registry_edges.py
|
||||||
|
--------------------------------------------------
|
||||||
|
Prüft, ob die in config/types.yaml hinterlegten Default-Kanten
|
||||||
|
pro Note-Typ tatsächlich in der Qdrant *edges*-Collection auftauchen.
|
||||||
|
"""
|
||||||
|
import os, sys, json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, List, Optional, Tuple, Set
|
||||||
|
from dataclasses import dataclass, asdict
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # PyYAML
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"PyYAML not installed: {e}"})); sys.exit(2)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from qdrant_client.http import models as rest # noqa: F401
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"qdrant-client not installed: {e}"})); sys.exit(2)
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Cfg:
|
||||||
|
url: str
|
||||||
|
api_key: Optional[str]
|
||||||
|
prefix: str
|
||||||
|
notes: str
|
||||||
|
edges: str
|
||||||
|
types_file: Path
|
||||||
|
|
||||||
|
def _env_prefix() -> Optional[str]:
|
||||||
|
for k in ("COLLECTION_PREFIX", "MINDNET_PREFIX"):
|
||||||
|
v = os.environ.get(k, "").strip()
|
||||||
|
if v:
|
||||||
|
return v
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _load_types_yaml(path: Path) -> Dict:
|
||||||
|
if not path.exists():
|
||||||
|
print(json.dumps({"warn": f"types.yaml fehlt: {path}. Fallback: keine Vorgaben."}))
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
with path.open("r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
return data if isinstance(data, dict) else {}
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"warn": f"types.yaml defekt ({path}): {e}. Fallback: keine Vorgaben."}))
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def _cfg_from_env(argv_prefix: Optional[str]) -> Cfg:
|
||||||
|
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
|
||||||
|
api_key = os.environ.get("QDRANT_API_KEY", "").strip() or None
|
||||||
|
prefix = (argv_prefix or _env_prefix() or "mindnet").strip()
|
||||||
|
notes = f"{prefix}_notes"
|
||||||
|
edges = f"{prefix}_edges"
|
||||||
|
types_path = Path(os.environ.get("TYPES_FILE", "config/types.yaml")).resolve()
|
||||||
|
return Cfg(url=url, api_key=api_key, prefix=prefix, notes=notes, edges=edges, types_file=types_path)
|
||||||
|
|
||||||
|
def _mk_client(cfg: Cfg) -> QdrantClient:
|
||||||
|
return QdrantClient(url=cfg.url, api_key=cfg.api_key, timeout=30.0)
|
||||||
|
|
||||||
|
def _scroll_all_notes(client: QdrantClient, notes_col: str) -> Dict[str, Dict]:
|
||||||
|
out = {}
|
||||||
|
offset = None
|
||||||
|
while True:
|
||||||
|
points, offset = client.scroll(
|
||||||
|
collection_name=notes_col,
|
||||||
|
scroll_filter=None, limit=256, offset=offset,
|
||||||
|
with_payload=True, with_vectors=False,
|
||||||
|
)
|
||||||
|
if not points:
|
||||||
|
break
|
||||||
|
for p in points:
|
||||||
|
payload = p.payload or {}
|
||||||
|
nid = payload.get("note_id") or payload.get("id") or payload.get("uid") or payload.get("slug")
|
||||||
|
if not nid:
|
||||||
|
nid = str(p.id)
|
||||||
|
out[str(nid)] = payload
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _scroll_all_edges(client: QdrantClient, edges_col: str) -> List[Tuple[str, dict]]:
|
||||||
|
out = []
|
||||||
|
offset = None
|
||||||
|
while True:
|
||||||
|
points, offset = client.scroll(
|
||||||
|
collection_name=edges_col,
|
||||||
|
scroll_filter=None, limit=256, offset=offset,
|
||||||
|
with_payload=True, with_vectors=False,
|
||||||
|
)
|
||||||
|
if not points:
|
||||||
|
break
|
||||||
|
for p in points:
|
||||||
|
out.append((str(p.id), p.payload or {}))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _guess_src_note_id(ed_pl: dict) -> Optional[str]:
|
||||||
|
for k in ("src_note_id", "note_id", "src_id", "src"):
|
||||||
|
nid = ed_pl.get(k)
|
||||||
|
if nid:
|
||||||
|
return str(nid)
|
||||||
|
for k in ("src_ref", "from"):
|
||||||
|
nid = ed_pl.get(k)
|
||||||
|
if isinstance(nid, dict):
|
||||||
|
for kk in ("note_id", "id"):
|
||||||
|
if nid.get(kk):
|
||||||
|
return str(nid[kk])
|
||||||
|
elif isinstance(nid, str):
|
||||||
|
return nid
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _edge_type(ed_pl: dict) -> Optional[str]:
|
||||||
|
for k in ("edge_type", "type", "rel", "relation"):
|
||||||
|
if ed_pl.get(k):
|
||||||
|
return str(ed_pl[k])
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _cfg_to_jsonable(cfg: Cfg) -> dict:
|
||||||
|
d = asdict(cfg)
|
||||||
|
d["types_file"] = str(d.get("types_file"))
|
||||||
|
return d
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
import argparse
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", help="Collection-Prefix (mindnet, etc.)")
|
||||||
|
args = ap.parse_args(argv)
|
||||||
|
|
||||||
|
cfg = _cfg_from_env(args.prefix)
|
||||||
|
print(json.dumps({"cfg": _cfg_to_jsonable(cfg)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
tr = _load_types_yaml(cfg.types_file)
|
||||||
|
types_def = (tr.get("types") if isinstance(tr, dict) else {}) or {}
|
||||||
|
print(json.dumps({"types_defined": list(types_def.keys())}, ensure_ascii=False))
|
||||||
|
|
||||||
|
client = _mk_client(cfg)
|
||||||
|
|
||||||
|
notes = _scroll_all_notes(client, cfg.notes)
|
||||||
|
print(json.dumps({"notes_count": len(notes)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
edges = _scroll_all_edges(client, cfg.edges)
|
||||||
|
print(json.dumps({"edges_count": len(edges)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
note_type = {}
|
||||||
|
for nid, pl in notes.items():
|
||||||
|
t = pl.get("type") or "concept"
|
||||||
|
note_type[str(nid)] = str(t)
|
||||||
|
|
||||||
|
seen = {}
|
||||||
|
counts = {}
|
||||||
|
|
||||||
|
for edge_pid, ed_pl in edges:
|
||||||
|
et = _edge_type(ed_pl)
|
||||||
|
if not et:
|
||||||
|
continue
|
||||||
|
src_nid = _guess_src_note_id(ed_pl)
|
||||||
|
if not src_nid:
|
||||||
|
continue
|
||||||
|
t = note_type.get(src_nid)
|
||||||
|
if not t:
|
||||||
|
continue
|
||||||
|
seen.setdefault(t, set()).add(et)
|
||||||
|
counts.setdefault(t, {}).setdefault(et, 0)
|
||||||
|
counts[t][et] += 1
|
||||||
|
|
||||||
|
expected = {}
|
||||||
|
for tname, tdef in types_def.items():
|
||||||
|
eddefs = (tdef or {}).get("edge_defaults") or []
|
||||||
|
expected[tname] = set([str(x) for x in eddefs if x])
|
||||||
|
|
||||||
|
for tname, exp in expected.items():
|
||||||
|
obs = seen.get(tname, set())
|
||||||
|
missing = sorted(list(exp - obs))
|
||||||
|
extra = sorted(list(obs - exp))
|
||||||
|
print(json.dumps({
|
||||||
|
"type": tname,
|
||||||
|
"expected_defaults": sorted(list(exp)),
|
||||||
|
"observed": sorted(list(obs)),
|
||||||
|
"missing": missing,
|
||||||
|
"extra": extra,
|
||||||
|
"counts": counts.get(tname, {}),
|
||||||
|
}, ensure_ascii=False))
|
||||||
|
|
||||||
|
if not expected:
|
||||||
|
print(json.dumps({"warn": "Keine Typ-Defaults in types.yaml gefunden (edge_defaults leer?)."}))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import os, json, argparse, random
|
||||||
|
from typing import Any, Dict, List, Tuple, Optional
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
|
||||||
|
def _get_prefix(explicit: Optional[str]) -> str:
|
||||||
|
return (explicit or os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet").strip()
|
||||||
|
|
||||||
|
def _names(prefix: str) -> Dict[str,str]:
|
||||||
|
return {
|
||||||
|
"notes": f"{prefix}_notes",
|
||||||
|
"chunks": f"{prefix}_chunks",
|
||||||
|
"edges": f"{prefix}_edges",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _scroll(client: QdrantClient, collection: str, *, limit=5, with_payload=True):
|
||||||
|
# Support both modern and older SDKs (filter vs scroll_filter)
|
||||||
|
try:
|
||||||
|
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
|
||||||
|
except TypeError as e:
|
||||||
|
# older signatures might accept scroll_filter kw, but if we passed none it's fine
|
||||||
|
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--host", default=os.environ.get("QDRANT_HOST", "127.0.0.1"))
|
||||||
|
ap.add_argument("--port", default=int(os.environ.get("QDRANT_PORT", "6333")))
|
||||||
|
ap.add_argument("--prefix", default=None)
|
||||||
|
ap.add_argument("--limit", type=int, default=5)
|
||||||
|
ap.add_argument("--note-id", default=None)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
prefix = _get_prefix(args.prefix)
|
||||||
|
names = _names(prefix)
|
||||||
|
client = QdrantClient(host=args.host, port=args.port, prefer_grpc=False)
|
||||||
|
|
||||||
|
# Notes sample
|
||||||
|
try:
|
||||||
|
pts, _ = _scroll(client, names["notes"], limit=args.limit, with_payload=True)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"notes scroll failed: {type(e).__name__}: {e}", "collection": names["notes"]}))
|
||||||
|
return
|
||||||
|
|
||||||
|
sample = []
|
||||||
|
for p in pts:
|
||||||
|
pl = (p.payload or {})
|
||||||
|
sample.append({
|
||||||
|
"point_id": str(p.id),
|
||||||
|
"note_id": pl.get("note_id"),
|
||||||
|
"id": pl.get("id"),
|
||||||
|
"title": pl.get("title"),
|
||||||
|
"type": pl.get("type"),
|
||||||
|
"retriever_weight": pl.get("retriever_weight"),
|
||||||
|
"chunk_profile": pl.get("chunk_profile"),
|
||||||
|
})
|
||||||
|
print(json.dumps({"notes_sample": sample}, ensure_ascii=False))
|
||||||
|
|
||||||
|
# Pick a note to fetch chunks
|
||||||
|
target_note_id = args.note_id or None
|
||||||
|
if not target_note_id:
|
||||||
|
for s in sample:
|
||||||
|
if s.get("note_id"):
|
||||||
|
target_note_id = s["note_id"]
|
||||||
|
break
|
||||||
|
|
||||||
|
if not target_note_id:
|
||||||
|
print(json.dumps({"warn": "no note_id available to fetch chunks"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
# Chunks for target note
|
||||||
|
try:
|
||||||
|
# pull a few chunks and filter client-side by note_id
|
||||||
|
pts, _ = _scroll(client, names["chunks"], limit=50, with_payload=True)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"chunks scroll failed: {type(e).__name__}: {e}", "collection": names["chunks"]}))
|
||||||
|
return
|
||||||
|
|
||||||
|
chunks = [{
|
||||||
|
"id": str(p.id),
|
||||||
|
"chunk_id": (p.payload or {}).get("chunk_id"),
|
||||||
|
"note_id": (p.payload or {}).get("note_id"),
|
||||||
|
"type": (p.payload or {}).get("type"),
|
||||||
|
"retriever_weight": (p.payload or {}).get("retriever_weight"),
|
||||||
|
"window_len": len((p.payload or {}).get("window") or ""),
|
||||||
|
"text_len": len((p.payload or {}).get("text") or ""),
|
||||||
|
} for p in pts if (p.payload or {}).get("note_id") == target_note_id][:5]
|
||||||
|
|
||||||
|
print(json.dumps({"target_note_id": target_note_id, "chunk_samples": chunks}, ensure_ascii=False))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/compare_vaults.py
|
||||||
|
|
||||||
|
Vergleicht zwei Markdown-Vaults rekursiv:
|
||||||
|
- Schlüsselauswahl: id | title | filename | auto (default: auto = id>title>filename)
|
||||||
|
- Fokus: body | frontmatter | all (default: body)
|
||||||
|
- Unicode: NFC-Normalisierung für IDs/Titel/Dateinamen und Text
|
||||||
|
|
||||||
|
Beispiele:
|
||||||
|
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --focus body
|
||||||
|
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --key id --focus all
|
||||||
|
|
||||||
|
Ausgabe:
|
||||||
|
- Einträge mit 'missing' (nur in src oder nur in dst)
|
||||||
|
- Einträge mit 'diff' (vorhanden in beiden, aber Abweichungen gem. Fokus)
|
||||||
|
- Summary-Objekt am Ende
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import argparse
|
||||||
|
import unicodedata
|
||||||
|
from typing import Dict, Tuple, Optional, Any
|
||||||
|
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
# Wir nutzen denselben Parser wie im Projekt (fehlertolerant)
|
||||||
|
from app.core.parser import read_markdown # erwartet: (frontmatter, body)
|
||||||
|
|
||||||
|
MD_EXTS = {".md", ".markdown"}
|
||||||
|
|
||||||
|
def nfc(s: Optional[str]) -> str:
|
||||||
|
return unicodedata.normalize("NFC", s or "")
|
||||||
|
|
||||||
|
def norm_text(txt: str) -> str:
|
||||||
|
# normalize line endings + strip trailing spaces per line + NFC
|
||||||
|
t = txt.replace("\r\n", "\n").replace("\r", "\n")
|
||||||
|
t = "\n".join(line.rstrip() for line in t.split("\n")).strip()
|
||||||
|
return nfc(t)
|
||||||
|
|
||||||
|
def iter_md_files(root: str):
|
||||||
|
for base, _dirs, files in os.walk(root):
|
||||||
|
for fn in files:
|
||||||
|
ext = os.path.splitext(fn)[1].lower()
|
||||||
|
if ext in MD_EXTS:
|
||||||
|
yield os.path.join(base, fn)
|
||||||
|
|
||||||
|
def key_from(front: Dict[str, Any], path: str, pref: str) -> str:
|
||||||
|
"""
|
||||||
|
Liefert den Vergleichsschlüssel.
|
||||||
|
pref in {"auto","id","title","filename"}
|
||||||
|
"""
|
||||||
|
if pref == "id":
|
||||||
|
return nfc(str(front.get("id", "")).strip())
|
||||||
|
if pref == "title":
|
||||||
|
return nfc(str(front.get("title", "")).strip())
|
||||||
|
if pref == "filename":
|
||||||
|
name = os.path.splitext(os.path.basename(path))[0]
|
||||||
|
return nfc(name.strip())
|
||||||
|
|
||||||
|
# auto: id > title > filename
|
||||||
|
if "id" in front and str(front["id"]).strip():
|
||||||
|
return nfc(str(front["id"]).strip())
|
||||||
|
if "title" in front and str(front["title"]).strip():
|
||||||
|
return nfc(str(front["title"]).strip())
|
||||||
|
name = os.path.splitext(os.path.basename(path))[0]
|
||||||
|
return nfc(name.strip())
|
||||||
|
|
||||||
|
def load_index(root: str, key_pref: str) -> Dict[str, Tuple[str, Dict[str, Any], str]]:
|
||||||
|
"""
|
||||||
|
Baut ein Index-Dict: key -> (path, frontmatter, body)
|
||||||
|
"""
|
||||||
|
idx: Dict[str, Tuple[str, Dict[str, Any], str]] = {}
|
||||||
|
for p in iter_md_files(root):
|
||||||
|
try:
|
||||||
|
fm, body = read_markdown(p)
|
||||||
|
k = key_from(fm or {}, p, key_pref)
|
||||||
|
if not k:
|
||||||
|
# Fallback: filename
|
||||||
|
k = key_from({}, p, "filename")
|
||||||
|
idx[k] = (p, fm or {}, body or "")
|
||||||
|
except Exception as e:
|
||||||
|
# Datei nicht parsebar -> trotzdem indexieren nach Dateiname (leer FM/Body)
|
||||||
|
k = key_from({}, p, key_pref if key_pref != "auto" else "filename")
|
||||||
|
idx[k] = (p, {}, "")
|
||||||
|
return idx
|
||||||
|
|
||||||
|
def compare_entries(src: Tuple[str, Dict[str, Any], str],
|
||||||
|
dst: Tuple[str, Dict[str, Any], str],
|
||||||
|
focus: str) -> Dict[str, Any]:
|
||||||
|
sp, sfm, sbody = src
|
||||||
|
dp, dfm, dbody = dst
|
||||||
|
|
||||||
|
res = {"status": "ok"} # wird 'diff' falls Unterschiede
|
||||||
|
diffs = {}
|
||||||
|
|
||||||
|
if focus in ("frontmatter", "all"):
|
||||||
|
# Vergleiche FM minimal: id und title
|
||||||
|
s_id = nfc(str(sfm.get("id", "")).strip())
|
||||||
|
d_id = nfc(str(dfm.get("id", "")).strip())
|
||||||
|
s_title = nfc(str(sfm.get("title", "")).strip())
|
||||||
|
d_title = nfc(str(dfm.get("title", "")).strip())
|
||||||
|
if s_id != d_id:
|
||||||
|
diffs["frontmatter.id"] = {"src": s_id, "dst": d_id}
|
||||||
|
if s_title != d_title:
|
||||||
|
diffs["frontmatter.title"] = {"src": s_title, "dst": d_title}
|
||||||
|
|
||||||
|
if focus in ("body", "all"):
|
||||||
|
if norm_text(sbody) != norm_text(dbody):
|
||||||
|
diffs["body"] = {"src_len": len(sbody or ""), "dst_len": len(dbody or "")}
|
||||||
|
|
||||||
|
if diffs:
|
||||||
|
res["status"] = "diff"
|
||||||
|
res["diffs"] = diffs
|
||||||
|
res["src_path"] = sp
|
||||||
|
res["dst_path"] = dp
|
||||||
|
return res
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--src", required=True, help="Quell-Vault-Ordner")
|
||||||
|
ap.add_argument("--dst", required=True, help="Export-Vault-Ordner")
|
||||||
|
ap.add_argument("--focus", choices=["body", "frontmatter", "all"], default="body")
|
||||||
|
ap.add_argument("--key", choices=["auto", "id", "title", "filename"], default="auto",
|
||||||
|
help="Vergleichsschlüssel (default: auto=id>title>filename)")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
src_idx = load_index(args.src, args.key)
|
||||||
|
dst_idx = load_index(args.dst, args.key)
|
||||||
|
|
||||||
|
src_keys = set(src_idx.keys())
|
||||||
|
dst_keys = set(dst_idx.keys())
|
||||||
|
|
||||||
|
# Fehlende
|
||||||
|
only_src = sorted(src_keys - dst_keys)
|
||||||
|
only_dst = sorted(dst_keys - src_keys)
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
for k in only_src:
|
||||||
|
print(json.dumps({"note_id": k, "status": "missing", "src": True, "dst": False}, ensure_ascii=False))
|
||||||
|
count += 1
|
||||||
|
for k in only_dst:
|
||||||
|
print(json.dumps({"note_id": k, "status": "missing", "src": False, "dst": True}, ensure_ascii=False))
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
# Vergleiche, wo vorhanden
|
||||||
|
diff_count = 0
|
||||||
|
common = sorted(src_keys & dst_keys)
|
||||||
|
for k in common:
|
||||||
|
res = compare_entries(src_idx[k], dst_idx[k], args.focus)
|
||||||
|
if res["status"] == "diff":
|
||||||
|
res["note_id"] = k
|
||||||
|
print(json.dumps(res, ensure_ascii=False))
|
||||||
|
diff_count += 1
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"summary": "OK" if (count == 0 and diff_count == 0) else "DIFFS",
|
||||||
|
"missing_count": count,
|
||||||
|
"diff_count": diff_count,
|
||||||
|
"total_src": len(src_keys),
|
||||||
|
"total_dst": len(dst_keys),
|
||||||
|
"focus": args.focus,
|
||||||
|
"key": args.key,
|
||||||
|
}
|
||||||
|
print(json.dumps(summary, ensure_ascii=False))
|
||||||
|
# Exit-Code für CI
|
||||||
|
if summary["summary"] != "OK":
|
||||||
|
# non-zero exit optional; hier 0 lassen, damit CLI-Ausgaben sichtbar bleiben
|
||||||
|
pass
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Sorgt dafür, dass 'app' als Paket importierbar ist,
|
||||||
|
# wenn pytest aus dem Repo-Root oder aus 'tests/' gestartet wird.
|
||||||
|
import os, sys
|
||||||
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, ROOT)
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Kleine Diagnosehilfe für mindnet_edges.
|
||||||
|
|
||||||
|
Nutzung (aus Projektwurzel):
|
||||||
|
|
||||||
|
(.venv) python tests/diag_edges_for_note.py \
|
||||||
|
--note-id 20251130-relshow-embeddings-101
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
|
||||||
|
--prefix mindnet
|
||||||
|
|
||||||
|
Ziel:
|
||||||
|
- Zeigt alle Edges aus der Collection <prefix>_edges,
|
||||||
|
die sich auf die angegebene note_id beziehen:
|
||||||
|
* note_id == NOTE
|
||||||
|
* oder source_id == NOTE
|
||||||
|
* oder target_id == NOTE
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
# Projekt-Root (Ordner mit "app/") zum sys.path hinzufügen,
|
||||||
|
# damit "import app.core.qdrant" funktioniert, auch wenn das Script
|
||||||
|
# direkt aus tests/ gestartet wird.
|
||||||
|
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
PROJECT_ROOT = os.path.dirname(THIS_DIR)
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest # type: ignore
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_edges_for_note(note_id: str, prefix: str | None, limit: int = 512) -> List[dict]:
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
if prefix:
|
||||||
|
cfg.prefix = prefix
|
||||||
|
|
||||||
|
client = get_client(cfg)
|
||||||
|
_, _, edges_col = collection_names(cfg.prefix)
|
||||||
|
|
||||||
|
conditions = []
|
||||||
|
for field in ("note_id", "source_id", "target_id"):
|
||||||
|
conditions.append(
|
||||||
|
rest.FieldCondition(
|
||||||
|
key=field,
|
||||||
|
match=rest.MatchValue(value=str(note_id)),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
flt = rest.Filter(
|
||||||
|
should=conditions,
|
||||||
|
)
|
||||||
|
|
||||||
|
points, _ = client.scroll(
|
||||||
|
collection_name=edges_col,
|
||||||
|
scroll_filter=flt,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
out: List[dict] = []
|
||||||
|
for p in points or []:
|
||||||
|
pl = dict(p.payload or {})
|
||||||
|
if pl:
|
||||||
|
out.append(pl)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--note-id", required=True, help="Note-ID, z. B. 20251130-relshow-embeddings-101")
|
||||||
|
parser.add_argument("--prefix", default=None, help="Qdrant-Prefix (Default: aus ENV/QdrantConfig)")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
note_id = args.note_id
|
||||||
|
prefix = args.prefix
|
||||||
|
|
||||||
|
edges = fetch_edges_for_note(note_id, prefix)
|
||||||
|
print(f"Prefix: {prefix or '(aus ENV)'}")
|
||||||
|
print(f"Note-ID: {note_id}")
|
||||||
|
print(f"Gefundene Edges: {len(edges)}")
|
||||||
|
print("-" * 60)
|
||||||
|
|
||||||
|
for i, e in enumerate(edges, start=1):
|
||||||
|
print(f"[{i}] kind={e.get('kind')} confidence={e.get('confidence')}")
|
||||||
|
print(f" note_id = {e.get('note_id')}")
|
||||||
|
print(f" source_id= {e.get('source_id')}")
|
||||||
|
print(f" target_id= {e.get('target_id')}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/diag_payload_indexes.py (v1.1)
|
||||||
|
|
||||||
|
Zeigt payload_schema (Indizes) je Collection.
|
||||||
|
WICHTIG: Einige Qdrant-Versionen liefern payload_schema nur, wenn
|
||||||
|
`with_payload_schema=true` gesetzt wird. Daher setzen wir das Flag explizit.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, json
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
def compact_schema(ps: dict | None) -> dict:
|
||||||
|
if not isinstance(ps, dict):
|
||||||
|
return {}
|
||||||
|
out = {}
|
||||||
|
for k, v in ps.items():
|
||||||
|
if isinstance(v, dict) and "type" in v:
|
||||||
|
out[k] = v["type"]
|
||||||
|
else:
|
||||||
|
out[k] = v
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--raw", action="store_true")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
|
||||||
|
cols = [notes, chunks, edges]
|
||||||
|
result = []
|
||||||
|
for name in cols:
|
||||||
|
info = client.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = info.model_dump() if hasattr(info, "model_dump") else (info.dict() if hasattr(info, "dict") else info)
|
||||||
|
payload_schema = (d.get("result") or {}).get("payload_schema")
|
||||||
|
vectors = (d.get("result") or {}).get("vectors")
|
||||||
|
if isinstance(vectors, dict) and "config" in vectors:
|
||||||
|
vectors = vectors.get("config")
|
||||||
|
if args.raw:
|
||||||
|
result.append({"collection": name, "raw": d})
|
||||||
|
else:
|
||||||
|
result.append({
|
||||||
|
"name": name,
|
||||||
|
"vectors": vectors,
|
||||||
|
"payload_schema": compact_schema(payload_schema),
|
||||||
|
"segments_count": (d.get("result") or {}).get("segments_count"),
|
||||||
|
})
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "collections": result}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Diagnose für den Graph-Adapter (Subgraph) in mindnet.
|
||||||
|
|
||||||
|
Nutzung (aus Projektwurzel):
|
||||||
|
|
||||||
|
(.venv) python tests/diag_subgraph_for_note.py \
|
||||||
|
--note-id 20251130-relshow-embeddings-101 \
|
||||||
|
--depth 1
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
|
||||||
|
--prefix mindnet
|
||||||
|
--edge-type references --edge-type related_to ...
|
||||||
|
|
||||||
|
Ziel:
|
||||||
|
- Ruft app.core.graph_adapter.expand(...) für eine gegebene note_id auf.
|
||||||
|
- Gibt edge_bonus und centrality_bonus für diese note_id aus.
|
||||||
|
- Zeigt zusätzlich einige Nachbarn und deren Boni.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import List, Optional, Tuple
|
||||||
|
|
||||||
|
# sys.path anpassen, damit "import app...." aus tests/ funktioniert
|
||||||
|
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
PROJECT_ROOT = os.path.dirname(THIS_DIR)
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client # type: ignore
|
||||||
|
import app.core.graph_adapter as ga # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def build_subgraph_for_note(
|
||||||
|
note_id: str,
|
||||||
|
prefix: Optional[str],
|
||||||
|
depth: int,
|
||||||
|
edge_types: Optional[List[str]] = None,
|
||||||
|
) -> Tuple[ga.Subgraph, str]:
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
if prefix:
|
||||||
|
cfg.prefix = prefix
|
||||||
|
|
||||||
|
client = get_client(cfg)
|
||||||
|
sg = ga.expand(client, cfg.prefix, [note_id], depth=depth, edge_types=edge_types)
|
||||||
|
return sg, cfg.prefix
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
"--note-id",
|
||||||
|
required=True,
|
||||||
|
help="Note-ID, z. B. 20251130-relshow-embeddings-101",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--depth",
|
||||||
|
type=int,
|
||||||
|
default=1,
|
||||||
|
help="Expansionstiefe (Default: 1)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--prefix",
|
||||||
|
default=None,
|
||||||
|
help="Qdrant-Prefix (Default: aus ENV/QdrantConfig)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--edge-type",
|
||||||
|
action="append",
|
||||||
|
default=None,
|
||||||
|
help="Edge-Typ-Filter (kann mehrfach angegeben werden), z. B. --edge-type references",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
note_id = args.note_id
|
||||||
|
depth = max(0, int(args.depth))
|
||||||
|
prefix = args.prefix
|
||||||
|
edge_types = args.edge_type
|
||||||
|
|
||||||
|
sg, eff_prefix = build_subgraph_for_note(note_id, prefix, depth, edge_types=edge_types)
|
||||||
|
|
||||||
|
print(f"Prefix: {eff_prefix}")
|
||||||
|
print(f"Note-ID (Seed): {note_id}")
|
||||||
|
print(f"Depth: {depth}")
|
||||||
|
print(f"Edge-Types-Filter: {edge_types or '(keiner)'}")
|
||||||
|
print("-" * 60)
|
||||||
|
|
||||||
|
# Boni für die Seed-Note
|
||||||
|
eb = sg.edge_bonus(note_id)
|
||||||
|
cb = sg.centrality_bonus(note_id)
|
||||||
|
print(f"edge_bonus({note_id}) = {eb:.4f}")
|
||||||
|
print(f"centrality_bonus({note_id}) = {cb:.4f}")
|
||||||
|
print("-" * 60)
|
||||||
|
|
||||||
|
# Ein paar weitere Knoten anzeigen (falls vorhanden)
|
||||||
|
print("Beispiele weiterer Knoten mit Boni (max. 10):")
|
||||||
|
printed = 0
|
||||||
|
for node, edges in sg.adj.items():
|
||||||
|
if node == note_id:
|
||||||
|
continue
|
||||||
|
if printed >= 10:
|
||||||
|
break
|
||||||
|
eb2 = sg.edge_bonus(node)
|
||||||
|
cb2 = sg.centrality_bonus(node)
|
||||||
|
print(f"- node={node}")
|
||||||
|
print(f" edge_bonus={eb2:.4f} centrality={cb2:.4f}")
|
||||||
|
printed += 1
|
||||||
|
|
||||||
|
if printed == 0:
|
||||||
|
print("(keine weiteren Knoten im Subgraph registriert)")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
dump_edges_payload_sample.py
|
||||||
|
-----------------------------------
|
||||||
|
Zeigt einige Beispiel-Payloads aus der {prefix}_edges Collection,
|
||||||
|
um Feldnamen (edge_type, src_note_id, src_id, etc.) zu verifizieren.
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 tests/dump_edges_payload_sample.py --prefix mindnet --n 10
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
try:
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"qdrant-client not installed: {e}"}))
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
import argparse
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet")
|
||||||
|
ap.add_argument("--n", type=int, default=10)
|
||||||
|
args = ap.parse_args(argv)
|
||||||
|
|
||||||
|
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
|
||||||
|
api_key = os.environ.get("QDRANT_API_KEY") or None
|
||||||
|
edges = f"{args.prefix}_edges"
|
||||||
|
|
||||||
|
client = QdrantClient(url=url, api_key=api_key, timeout=30)
|
||||||
|
|
||||||
|
left = args.n
|
||||||
|
offset = None
|
||||||
|
while left > 0:
|
||||||
|
limit = min(left, 256)
|
||||||
|
pts, offset = client.scroll(
|
||||||
|
collection_name=edges,
|
||||||
|
scroll_filter=None,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
for p in pts:
|
||||||
|
print(json.dumps({"point_id": str(p.id), "payload_keys": list((p.payload or {}).keys()), "payload": p.payload}, ensure_ascii=False))
|
||||||
|
left -= len(pts)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/ensure_indexes_and_show.py (compat v1.0.1)
|
||||||
|
|
||||||
|
Erzwingt ensure_payload_indexes(...) und zeigt danach payload_schema je Collection.
|
||||||
|
Kompatibel mit qdrant-client Versionen ohne 'with_payload_schema' im Wrapper.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, ensure_payload_indexes, collection_names
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||||
|
if hasattr(obj, "model_dump"):
|
||||||
|
return obj.model_dump()
|
||||||
|
if hasattr(obj, "dict"):
|
||||||
|
return obj.dict()
|
||||||
|
return obj if isinstance(obj, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _cfg_base_url(cfg) -> str:
|
||||||
|
if getattr(cfg, "url", None):
|
||||||
|
return cfg.url.rstrip("/")
|
||||||
|
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||||
|
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||||
|
return f"http://{host}:{port}"
|
||||||
|
|
||||||
|
|
||||||
|
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||||
|
req = urllib.request.Request(url, method="GET")
|
||||||
|
if api_key:
|
||||||
|
req.add_header("api-key", api_key)
|
||||||
|
req.add_header("Authorization", f"Bearer {api_key}")
|
||||||
|
req.add_header("Accept", "application/json")
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = resp.read()
|
||||||
|
return json.loads(data.decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||||
|
# Strategy 1: OpenAPI client with flag
|
||||||
|
try:
|
||||||
|
oc = getattr(client, "openapi_client", None)
|
||||||
|
if oc is not None and hasattr(oc, "collections_api"):
|
||||||
|
api = oc.collections_api
|
||||||
|
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
return (d.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Strategy 2: wrapper
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
ps = (d.get("result") or {}).get("payload_schema")
|
||||||
|
if ps is not None:
|
||||||
|
return ps
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Strategy 3: direct HTTP
|
||||||
|
try:
|
||||||
|
base = _cfg_base_url(cfg)
|
||||||
|
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||||
|
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||||
|
return (raw.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
res = {}
|
||||||
|
for name in (notes, chunks, edges):
|
||||||
|
res[name] = get_collection_payload_schema(client, cfg, name)
|
||||||
|
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/inspect_one_note.py
|
||||||
|
Diagnose einer einzelnen Markdown-Datei (kein Upsert).
|
||||||
|
Usage:
|
||||||
|
python3 tests/inspect_one_note.py --path ./test_vault/30_projects/project-demo.md
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import os, sys, json, argparse
|
||||||
|
|
||||||
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||||
|
from app.core.chunker import assemble_chunks
|
||||||
|
from app.core.chunk_payload import make_chunk_payloads
|
||||||
|
try:
|
||||||
|
from app.core.derive_edges import build_edges_for_note
|
||||||
|
except Exception:
|
||||||
|
from app.core.edges import build_edges_for_note # type: ignore
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--path", required=True)
|
||||||
|
args = ap.parse_args()
|
||||||
|
p = os.path.abspath(args.path)
|
||||||
|
|
||||||
|
try:
|
||||||
|
parsed = read_markdown(p)
|
||||||
|
if parsed is None:
|
||||||
|
print(json.dumps({"path": p, "error": "read_markdown returned None"}))
|
||||||
|
return
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"path": p, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
|
validate_required_frontmatter(fm)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"path": p, "error": f"frontmatter invalid: {type(e).__name__}: {e}"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
body = getattr(parsed, "body", "") or ""
|
||||||
|
chunks = assemble_chunks(fm["id"], body, fm.get("type", "concept"))
|
||||||
|
rel_for_payload = os.path.relpath(p, os.path.join(p, "..", ".."))
|
||||||
|
chunk_pls = make_chunk_payloads(fm, rel_for_payload, chunks, note_text=body)
|
||||||
|
|
||||||
|
edge_error = None
|
||||||
|
edges_count = 0
|
||||||
|
try:
|
||||||
|
edges = build_edges_for_note(fm["id"], chunk_pls, include_note_scope_refs=True)
|
||||||
|
edges_count = len(edges)
|
||||||
|
except Exception as e:
|
||||||
|
edge_error = f"{type(e).__name__}: {e}"
|
||||||
|
|
||||||
|
print(json.dumps({
|
||||||
|
"path": p,
|
||||||
|
"id": fm.get("id"),
|
||||||
|
"title": fm.get("title"),
|
||||||
|
"body_len": len(body),
|
||||||
|
"chunks": len(chunk_pls),
|
||||||
|
"edges": edges_count,
|
||||||
|
"edge_error": edge_error
|
||||||
|
}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/list_md_seen_by_importer.py
|
||||||
|
Zeigt, welche Markdown-Dateien der Importer sehen würde (gleiche Filter & Sortierung).
|
||||||
|
Usage:
|
||||||
|
python3 tests/list_md_seen_by_importer.py --vault ./test_vault
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, os, sys, json
|
||||||
|
|
||||||
|
def iter_md(root: str):
|
||||||
|
out = []
|
||||||
|
for dirpath, _, filenames in os.walk(root):
|
||||||
|
for fn in filenames:
|
||||||
|
p = os.path.join(dirpath, fn)
|
||||||
|
pn = p.replace("\\", "/")
|
||||||
|
item = {
|
||||||
|
"path": os.path.abspath(p),
|
||||||
|
"rel": os.path.relpath(p, root).replace("\\", "/"),
|
||||||
|
"included": False,
|
||||||
|
"reason": "",
|
||||||
|
}
|
||||||
|
if not fn.lower().endswith(".md"):
|
||||||
|
item["reason"] = "skip_ext"
|
||||||
|
elif "/.obsidian/" in pn or "/_backup_frontmatter/" in pn or "/_imported/" in pn:
|
||||||
|
item["reason"] = "skip_special_dir"
|
||||||
|
else:
|
||||||
|
item["included"] = True
|
||||||
|
item["reason"] = "ok"
|
||||||
|
out.append(item)
|
||||||
|
# auch die ausgeschlossenen zeigen:
|
||||||
|
if not item["included"]:
|
||||||
|
print(json.dumps(item, ensure_ascii=False))
|
||||||
|
out_sorted = sorted(out, key=lambda x: x["rel"])
|
||||||
|
for i, it in enumerate(out_sorted):
|
||||||
|
it["order"] = i
|
||||||
|
print(json.dumps(it, ensure_ascii=False))
|
||||||
|
print(json.dumps({"summary":"done","included":len(out_sorted)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--vault", required=True)
|
||||||
|
args = ap.parse_args()
|
||||||
|
root = os.path.abspath(args.vault)
|
||||||
|
iter_md(root)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# tests/run_e2e_roundtrip.sh
|
||||||
|
# End-to-End Testpipeline für mindnet WP-03
|
||||||
|
#
|
||||||
|
# Schritte:
|
||||||
|
# 1) Qdrant truncate (Prefix!)
|
||||||
|
# 2) Import (Apply + Purge) => Create-Fall
|
||||||
|
# 3) Import (Apply - ohne Änderungen) => Idempotenz
|
||||||
|
# 4) Hash-Reporter (Fehlende Hash-Varianten)
|
||||||
|
# 5) Export nach _exportVault
|
||||||
|
# 6) Vergleich Vault vs. Export (Body-Fokus)
|
||||||
|
# 7) Sync-Deletes: Eine Datei temporär „entfernen“, Dry-Run + Apply, dann zurückrollen
|
||||||
|
#
|
||||||
|
# Aufrufe:
|
||||||
|
# ./tests/run_e2e_roundtrip.sh [--vault ./test_vault] [--prefix mindnet]
|
||||||
|
#
|
||||||
|
# ENV:
|
||||||
|
# COLLECTION_PREFIX (Default mindnet)
|
||||||
|
|
||||||
|
VAULT="./test_vault"
|
||||||
|
PREFIX="${COLLECTION_PREFIX:-mindnet}"
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--vault) VAULT="$2"; shift 2;;
|
||||||
|
--prefix) PREFIX="$2"; shift 2;;
|
||||||
|
*) echo "Unbekannte Option: $1" >&2; exit 2;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "== E2E Roundtrip =="
|
||||||
|
echo "Vault: $VAULT"
|
||||||
|
echo "Prefix: $PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
export COLLECTION_PREFIX="$PREFIX"
|
||||||
|
|
||||||
|
echo "-- 1) Qdrant truncate"
|
||||||
|
python3 -m scripts.reset_qdrant --mode truncate --prefix "$PREFIX"
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 2) Import (Create-Fall)"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --apply --purge-before-upsert --prefix "$PREFIX"
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 3) Import (Idempotenz: keine Änderungen erwartet)"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --apply --prefix "$PREFIX"
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 4) Hash-Reporter"
|
||||||
|
python3 -m scripts.report_hashes --prefix "$PREFIX" || true
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 5) Export nach _exportVault"
|
||||||
|
OUT_DIR="_exportVault"
|
||||||
|
rm -rf "$OUT_DIR"
|
||||||
|
python3 -m scripts.export_markdown --out "$OUT_DIR" --prefix "$PREFIX"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 6) Vergleich Vault vs. Export (Body-Fokus)"
|
||||||
|
python3 tests/compare_vaults.py --src "$VAULT" --dst "$OUT_DIR" --focus body
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "-- 7) Sync-Deletes (simuliert)"
|
||||||
|
# wähle eine Datei und verschiebe sie temporär weg
|
||||||
|
TMP_DIR="$(mktemp -d)"
|
||||||
|
SAMPLE_MD="$(find "$VAULT" -type f -name '*.md' | head -n 1 || true)"
|
||||||
|
if [[ -n "${SAMPLE_MD:-}" ]]; then
|
||||||
|
mv "$SAMPLE_MD" "$TMP_DIR/"
|
||||||
|
echo "Temporär entfernt: $(basename "$SAMPLE_MD")"
|
||||||
|
|
||||||
|
echo " 7a) Dry-Run"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --sync-deletes --prefix "$PREFIX"
|
||||||
|
echo " 7b) Apply"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --sync-deletes --apply --prefix "$PREFIX"
|
||||||
|
|
||||||
|
# zurückrollen
|
||||||
|
mv "$TMP_DIR/$(basename "$SAMPLE_MD")" "$SAMPLE_MD"
|
||||||
|
rm -rf "$TMP_DIR"
|
||||||
|
|
||||||
|
echo " 7c) Re-Import (wieder anlegen)"
|
||||||
|
python3 -m scripts.import_markdown --vault "$VAULT" --apply --prefix "$PREFIX"
|
||||||
|
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
|
||||||
|
else
|
||||||
|
echo "Hinweis: Keine MD-Datei im Vault gefunden – Sync-Deletes übersprungen."
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "== E2E Roundtrip abgeschlossen =="
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/test_retriever_smoke.py - einfacher Smoke-Test für den mindnet-Retriever
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
- Startet eine Query gegen die laufende FastAPI-Instanz
|
||||||
|
- Prüft, ob der /query-Endpunkt antwortet und sinnvolle Treffer liefert
|
||||||
|
- Gibt Top-K-Ergebnisse inkl. Scores auf stdout aus
|
||||||
|
|
||||||
|
Verwendung:
|
||||||
|
1) Stelle sicher, dass der mindnet-Service läuft, z. B.:
|
||||||
|
uvicorn app.main:create_app --factory --reload
|
||||||
|
|
||||||
|
2) Führe dieses Skript aus:
|
||||||
|
python scripts/test_retriever_smoke.py --query "karate" --mode hybrid
|
||||||
|
|
||||||
|
3) Optional:
|
||||||
|
python scripts/test_retriever_smoke.py --query "embeddings" --mode semantic --top-k 5
|
||||||
|
|
||||||
|
Hinweis:
|
||||||
|
- Standard-URL ist http://127.0.0.1:8000
|
||||||
|
- Endpoint ist /query (bestehender Router), der QueryRequest erwartet
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
def _build_payload(args: argparse.Namespace) -> Dict[str, Any]:
|
||||||
|
payload: Dict[str, Any] = {
|
||||||
|
"mode": args.mode,
|
||||||
|
"query": args.query,
|
||||||
|
"top_k": args.top_k,
|
||||||
|
}
|
||||||
|
if args.expand_depth is not None and args.expand_depth > 0:
|
||||||
|
expand: Dict[str, Any] = {"depth": args.expand_depth}
|
||||||
|
if args.expand_edge_types:
|
||||||
|
expand["edge_types"] = args.expand_edge_types
|
||||||
|
payload["expand"] = expand
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Smoke-Test für mindnet-Retriever (/query)")
|
||||||
|
parser.add_argument("--url", default="http://127.0.0.1:8000/query", help="Basis-URL des Query-Endpunkts")
|
||||||
|
parser.add_argument("--query", required=True, help="Text-Query für den Retriever")
|
||||||
|
parser.add_argument("--mode", choices=["semantic", "hybrid"], default="hybrid", help="Retrieval-Modus")
|
||||||
|
parser.add_argument("--top-k", type=int, default=5, help="Anzahl der gewünschten Treffer")
|
||||||
|
parser.add_argument("--expand-depth", type=int, default=0, help="Edge-Expansionstiefe (nur hybrid)")
|
||||||
|
parser.add_argument(
|
||||||
|
"--expand-edge-types",
|
||||||
|
nargs="*",
|
||||||
|
default=None,
|
||||||
|
help="Liste von Edge-Typen für die Expansion (z. B. references depends_on)",
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
payload = _build_payload(args)
|
||||||
|
|
||||||
|
try:
|
||||||
|
resp = requests.post(args.url, json=payload, timeout=10)
|
||||||
|
except Exception as exc: # pragma: no cover - Netzwerkfehler
|
||||||
|
print(f"[ERROR] HTTP-Request fehlgeschlagen: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
print(f"[ERROR] Unerwarteter Statuscode: {resp.status_code}", file=sys.stderr)
|
||||||
|
print(resp.text, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as exc: # pragma: no cover - JSON-Fehler
|
||||||
|
print(f"[ERROR] Antwort ist kein gültiges JSON: {exc}", file=sys.stderr)
|
||||||
|
print(resp.text, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print("=== mindnet Retriever Smoke-Test ===")
|
||||||
|
print(f"URL: {args.url}")
|
||||||
|
print(f"Mode: {args.mode}")
|
||||||
|
print(f"Query: {args.query!r}")
|
||||||
|
print(f"Top-K: {args.top_k}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
results = data.get("results") or []
|
||||||
|
used_mode = data.get("used_mode")
|
||||||
|
latency_ms = data.get("latency_ms")
|
||||||
|
|
||||||
|
print(f"used_mode = {used_mode}, latency = {latency_ms} ms")
|
||||||
|
print(f"Treffer insgesamt: {len(results)}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
for i, hit in enumerate(results, start=1):
|
||||||
|
note_id = hit.get("note_id")
|
||||||
|
node_id = hit.get("node_id")
|
||||||
|
total = hit.get("total_score")
|
||||||
|
sem = hit.get("semantic_score")
|
||||||
|
edge_bonus = hit.get("edge_bonus")
|
||||||
|
cent_bonus = hit.get("centrality_bonus")
|
||||||
|
src = hit.get("source") or {}
|
||||||
|
path = src.get("path")
|
||||||
|
section = src.get("section")
|
||||||
|
|
||||||
|
print(f"[{i}] note_id={note_id} node_id={node_id}")
|
||||||
|
print(f" total={total:.4f} (semantic={sem:.4f} edge={edge_bonus:.4f} centrality={cent_bonus:.4f})")
|
||||||
|
if path or section:
|
||||||
|
print(f" source: {path or '-'} :: {section or '-'}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/show_edges_for_note.py
|
||||||
|
|
||||||
|
Zeigt Kanten einer Note (gefiltert nach kind/scope) in einer kompakten Form.
|
||||||
|
Aufrufbeispiele:
|
||||||
|
python3 -m scripts.show_edges_for_note --note-id 20251110-ollama-llm-9f0a12 --kinds references,next,prev --limit 10
|
||||||
|
python3 -m scripts.show_edges_for_note --title "Qdrant Vektordatenbank" --scope note
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, json, os, sys
|
||||||
|
from typing import Dict, Any, List, Tuple
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def find_note_by_title(client, prefix: str, title: str) -> str | None:
|
||||||
|
notes_col, _, _ = collections(prefix)
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="title", match=rest.MatchText(text=title))])
|
||||||
|
pts, _ = client.scroll(collection_name=notes_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=1)
|
||||||
|
if not pts:
|
||||||
|
return None
|
||||||
|
return pts[0].payload.get("note_id")
|
||||||
|
|
||||||
|
def fetch_edges_for_note(client, prefix: str, note_id: str, kinds: List[str] | None, scope: str | None, limit: int) -> List[Dict[str, Any]]:
|
||||||
|
_, _, edges_col = collections(prefix)
|
||||||
|
must = [rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
|
||||||
|
if scope:
|
||||||
|
must.append(rest.FieldCondition(key="scope", match=rest.MatchValue(value=scope)))
|
||||||
|
if kinds:
|
||||||
|
must.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
|
||||||
|
f = rest.Filter(must=must)
|
||||||
|
pts, _ = client.scroll(collection_name=edges_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=limit)
|
||||||
|
return [p.payload for p in pts]
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--note-id")
|
||||||
|
ap.add_argument("--title")
|
||||||
|
ap.add_argument("--kinds", help="CSV: references,next,prev,belongs_to,backlink")
|
||||||
|
ap.add_argument("--scope", choices=["note","chunk"])
|
||||||
|
ap.add_argument("--limit", type=int, default=25)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
nid = args.note_id
|
||||||
|
if not nid and args.title:
|
||||||
|
nid = find_note_by_title(client, cfg.prefix, args.title)
|
||||||
|
if not nid:
|
||||||
|
print(json.dumps({"error": f"note with title '{args.title}' not found"}))
|
||||||
|
sys.exit(2)
|
||||||
|
if not nid:
|
||||||
|
print(json.dumps({"error": "please provide --note-id or --title"}))
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
kinds = None
|
||||||
|
if args.kinds:
|
||||||
|
kinds = [s.strip() for s in args.kinds.split(",") if s.strip()]
|
||||||
|
|
||||||
|
edges = fetch_edges_for_note(client, cfg.prefix, nid, kinds, args.scope, args.limit)
|
||||||
|
print(json.dumps({"note_id": nid, "count": len(edges), "edges": edges}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# Konfiguration
|
||||||
|
API_URL = "http://localhost:8002/chat/" # Port ggf. anpassen
|
||||||
|
QUESTION = "Warum nutzen wir Qdrant für mindnet?" # Eine Frage, zu der du Notizen hast
|
||||||
|
|
||||||
|
def test_chat():
|
||||||
|
payload = {
|
||||||
|
"message": QUESTION,
|
||||||
|
"top_k": 3,
|
||||||
|
"explain": True
|
||||||
|
}
|
||||||
|
|
||||||
|
print(f"Sending Question: '{QUESTION}'...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = requests.post(API_URL, json=payload)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
print("\n=== RESPONSE ===")
|
||||||
|
print(data["answer"])
|
||||||
|
print("================\n")
|
||||||
|
|
||||||
|
print(f"Query ID: {data['query_id']}")
|
||||||
|
print(f"Latency: {data['latency_ms']}ms")
|
||||||
|
print("\nUsed Sources:")
|
||||||
|
for source in data["sources"]:
|
||||||
|
score = source.get("total_score", 0)
|
||||||
|
note = source.get("note_id", "unknown")
|
||||||
|
print(f"- {note} (Score: {score:.3f})")
|
||||||
|
|
||||||
|
except requests.exceptions.ConnectionError:
|
||||||
|
print("Error: Could not connect to API. Is it running on port 8002?")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error: {e}")
|
||||||
|
if 'response' in locals():
|
||||||
|
print(response.text)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
test_chat()
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/test_edges_all.py
|
||||||
|
Ein knapper Integrationscheck:
|
||||||
|
- Es existieren Notes/Chunks/Edges
|
||||||
|
- Inline-Edges (rule_id startswith "inline:") werden erkannt
|
||||||
|
- Callout-Edges (rule_id == "callout:edge") werden erkannt
|
||||||
|
- Defaults (rule_id startswith "edge_defaults:") werden erkannt
|
||||||
|
- Strukturkanten stimmen (belongs_to == chunks; next == prev == chunks-1)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from typing import Dict, Any, List, Tuple
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
|
||||||
|
def _scroll_all(client, collection: str):
|
||||||
|
pts_all = []
|
||||||
|
offset = None
|
||||||
|
while True:
|
||||||
|
pts, offset = client.scroll(
|
||||||
|
collection_name=collection,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=2048,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
pts_all.extend(pts or [])
|
||||||
|
if offset is None:
|
||||||
|
break
|
||||||
|
return pts_all
|
||||||
|
|
||||||
|
|
||||||
|
def _rule_group(rule_id: str) -> str:
|
||||||
|
if not rule_id:
|
||||||
|
return "unknown"
|
||||||
|
if rule_id == "callout:edge":
|
||||||
|
return "callout"
|
||||||
|
if rule_id.startswith("inline:"): # <—— wichtig für inline:rel
|
||||||
|
return "inline"
|
||||||
|
if rule_id.startswith("edge_defaults:"):
|
||||||
|
return "defaults"
|
||||||
|
if rule_id.startswith("explicit:"):
|
||||||
|
return "explicit"
|
||||||
|
if rule_id in ("structure:belongs_to", "structure:order"):
|
||||||
|
return "structure"
|
||||||
|
return "other"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
col_notes = f"{cfg.prefix}_notes"
|
||||||
|
col_chunks = f"{cfg.prefix}_chunks"
|
||||||
|
col_edges = f"{cfg.prefix}_edges"
|
||||||
|
|
||||||
|
notes_n = client.count(collection_name=col_notes, exact=True).count
|
||||||
|
chunks_pts = _scroll_all(client, col_chunks)
|
||||||
|
edges_pts = _scroll_all(client, col_edges)
|
||||||
|
|
||||||
|
ok = True
|
||||||
|
|
||||||
|
# Basisbedingungen
|
||||||
|
if notes_n == 0 or len(chunks_pts) == 0 or len(edges_pts) == 0:
|
||||||
|
ok = False
|
||||||
|
|
||||||
|
# Gruppen zählen
|
||||||
|
g = Counter(_rule_group((p.payload or {}).get("rule_id", "")) for p in edges_pts)
|
||||||
|
structure = g.get("structure", 0)
|
||||||
|
explicit = g.get("explicit", 0)
|
||||||
|
inline = g.get("inline", 0)
|
||||||
|
callout = g.get("callout", 0)
|
||||||
|
defaults = g.get("defaults", 0)
|
||||||
|
|
||||||
|
if structure == 0:
|
||||||
|
ok = False
|
||||||
|
# mindestens eine der expliziten Varianten vorhanden
|
||||||
|
if (explicit + inline + callout) == 0:
|
||||||
|
ok = False
|
||||||
|
# defaults dürfen 0 sein, wenn types.yaml keine edge_defaults liefert – daher nur Info
|
||||||
|
|
||||||
|
# per-note checks
|
||||||
|
chunks_by_note = Counter([p.payload.get("note_id") for p in chunks_pts if p.payload])
|
||||||
|
belongs = Counter(
|
||||||
|
(p.payload or {}).get("note_id")
|
||||||
|
for p in edges_pts
|
||||||
|
if (p.payload or {}).get("kind") == "belongs_to"
|
||||||
|
)
|
||||||
|
nxt = Counter(
|
||||||
|
(p.payload or {}).get("note_id")
|
||||||
|
for p in edges_pts
|
||||||
|
if (p.payload or {}).get("kind") == "next"
|
||||||
|
)
|
||||||
|
prv = Counter(
|
||||||
|
(p.payload or {}).get("note_id")
|
||||||
|
for p in edges_pts
|
||||||
|
if (p.payload or {}).get("kind") == "prev"
|
||||||
|
)
|
||||||
|
|
||||||
|
for n_id, c in chunks_by_note.items():
|
||||||
|
if belongs.get(n_id, 0) != c:
|
||||||
|
ok = False
|
||||||
|
if (nxt.get(n_id, 0) != max(c - 1, 0)) or (prv.get(n_id, 0) != max(c - 1, 0)):
|
||||||
|
ok = False
|
||||||
|
|
||||||
|
print(json.dumps({"ok": ok, "notes_checked": len(chunks_by_note)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/test_edges_defaults_smoke.py
|
||||||
|
Aggregiert Edges je Relation (kind) in Qdrant zur schnellen Sichtprüfung.
|
||||||
|
Zeigt, ob typenbasierte edge_defaults-Relationen (z. B. depends_on/related_to) auftauchen.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
from collections import Counter
|
||||||
|
import json
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
cl = get_client(cfg)
|
||||||
|
col = f"{cfg.prefix}_edges"
|
||||||
|
pts, _ = cl.scroll(collection_name=col, with_payload=True, with_vectors=False, limit=10000)
|
||||||
|
by_rel = Counter()
|
||||||
|
for p in pts:
|
||||||
|
rel = p.payload.get("kind") or p.payload.get("relation") or "edge"
|
||||||
|
by_rel[rel] += 1
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "relations": dict(by_rel)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
|
||||||
|
def _rel(pl: dict) -> str:
|
||||||
|
return pl.get("relation") or pl.get("kind") or "edge"
|
||||||
|
|
||||||
|
|
||||||
|
def _scroll(client, col):
|
||||||
|
pts = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
res, next_page = client.scroll(
|
||||||
|
collection_name=col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=1024,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
pts.extend(res)
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
return pts
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
prefix = os.environ.get("COLLECTION_PREFIX", cfg.prefix)
|
||||||
|
|
||||||
|
cols = {
|
||||||
|
"notes": f"{prefix}_notes",
|
||||||
|
"chunks": f"{prefix}_chunks",
|
||||||
|
"edges": f"{prefix}_edges",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Index: notes -> title/type
|
||||||
|
notes_meta = {}
|
||||||
|
for p in _scroll(client, cols["notes"]):
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if nid:
|
||||||
|
notes_meta[nid] = {
|
||||||
|
"title": pl.get("title", ""),
|
||||||
|
"type": pl.get("type", ""),
|
||||||
|
}
|
||||||
|
|
||||||
|
# chunks je note
|
||||||
|
chunks_by_note = defaultdict(int)
|
||||||
|
for p in _scroll(client, cols["chunks"]):
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if nid:
|
||||||
|
chunks_by_note[nid] += 1
|
||||||
|
|
||||||
|
# edges je note
|
||||||
|
edges_by_note = defaultdict(list)
|
||||||
|
edges_all = _scroll(client, cols["edges"])
|
||||||
|
for p in edges_all:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if nid:
|
||||||
|
edges_by_note[nid].append(pl)
|
||||||
|
|
||||||
|
# pro note ausgeben
|
||||||
|
summary_edges = Counter()
|
||||||
|
total_chunks = 0
|
||||||
|
for nid in sorted(notes_meta.keys()):
|
||||||
|
meta = notes_meta[nid]
|
||||||
|
chunks = chunks_by_note.get(nid, 0)
|
||||||
|
total_chunks += chunks
|
||||||
|
|
||||||
|
kinds = Counter(_rel(pl) for pl in edges_by_note[nid])
|
||||||
|
summary_edges.update(kinds)
|
||||||
|
|
||||||
|
row = {
|
||||||
|
"note_id": nid,
|
||||||
|
"title": meta["title"],
|
||||||
|
"type": meta["type"],
|
||||||
|
"chunks": chunks,
|
||||||
|
"edges_by_kind": dict(kinds),
|
||||||
|
"checks": {
|
||||||
|
"belongs_to_equals_chunks": (kinds.get("belongs_to", 0) == chunks),
|
||||||
|
"next_prev_match": (kinds.get("next", 0) == kinds.get("prev", 0) == max(0, chunks - 1)),
|
||||||
|
"no_duplicate_edges": _no_dupes(edges_by_note[nid]),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
print(json.dumps(row, ensure_ascii=False))
|
||||||
|
|
||||||
|
# Gesamtsummary
|
||||||
|
total_notes = len(notes_meta)
|
||||||
|
out = {
|
||||||
|
"prefix": prefix,
|
||||||
|
"summary": {
|
||||||
|
"notes": total_notes,
|
||||||
|
"chunks": total_chunks,
|
||||||
|
"belongs_to": summary_edges.get("belongs_to", 0),
|
||||||
|
"next": summary_edges.get("next", 0),
|
||||||
|
"prev": summary_edges.get("prev", 0),
|
||||||
|
"refs_chunk": summary_edges.get("references", 0),
|
||||||
|
"refs_note": summary_edges.get("references_note", 0),
|
||||||
|
"backlink": summary_edges.get("backlink", 0),
|
||||||
|
"dup_edges": 0, # per-Note geprüft
|
||||||
|
},
|
||||||
|
}
|
||||||
|
print(json.dumps(out, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
def _no_dupes(pls):
|
||||||
|
seen = set()
|
||||||
|
for pl in pls:
|
||||||
|
key = (
|
||||||
|
str(pl.get("source_id") or ""),
|
||||||
|
str(pl.get("target_id") or ""),
|
||||||
|
str(pl.get("relation") or pl.get("kind") or ""),
|
||||||
|
str(pl.get("rule_id") or ""),
|
||||||
|
)
|
||||||
|
if key in seen:
|
||||||
|
return False
|
||||||
|
seen.add(key)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/test_edges_smoke_fast.py — Zählung über /count (sehr schnell)
|
||||||
|
|
||||||
|
Verwendet Qdrant 'count' API je Note/Kind (anstatt scroll), dadurch sehr schnelle Ausführung.
|
||||||
|
Optionen:
|
||||||
|
--max-notes N : prüft nur die ersten N Notizen
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, json
|
||||||
|
from typing import Dict, Any, List, Tuple
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
KINDS = ["belongs_to", "next", "prev", "references", "backlink"]
|
||||||
|
|
||||||
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def list_note_ids(client, notes_col: str, max_notes: int | None) -> List[Dict[str, Any]]:
|
||||||
|
pts, _ = client.scroll(collection_name=notes_col, with_payload=True, with_vectors=False, limit=max_notes or 1024)
|
||||||
|
out = []
|
||||||
|
for p in pts or []:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id") or pl.get("id")
|
||||||
|
if nid:
|
||||||
|
out.append({"note_id": nid, "title": pl.get("title"), "type": pl.get("type")})
|
||||||
|
return out
|
||||||
|
|
||||||
|
def count_points(client, col: str, filt: rest.Filter) -> int:
|
||||||
|
res = client.count(collection_name=col, count_filter=filt, exact=True)
|
||||||
|
return int(getattr(res, "count", 0))
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--max-notes", type=int)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes_col, chunks_col, edges_col = collections(cfg.prefix)
|
||||||
|
notes = list_note_ids(client, notes_col, args.max_notes)
|
||||||
|
|
||||||
|
summary = {"notes": 0, "chunks": 0, "edges": 0}
|
||||||
|
for n in notes:
|
||||||
|
nid = n["note_id"]
|
||||||
|
summary["notes"] += 1
|
||||||
|
|
||||||
|
filt_note = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||||
|
chunk_cnt = count_points(client, chunks_col, filt_note)
|
||||||
|
summary["chunks"] += chunk_cnt
|
||||||
|
|
||||||
|
# counts per kind (edges)
|
||||||
|
by_kind: Dict[str, int] = {}
|
||||||
|
for k in KINDS:
|
||||||
|
f = rest.Filter(must=[
|
||||||
|
rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid)),
|
||||||
|
rest.FieldCondition(key="kind", match=rest.MatchValue(value=k)),
|
||||||
|
])
|
||||||
|
c = count_points(client, edges_col, f)
|
||||||
|
if c:
|
||||||
|
by_kind[k] = c
|
||||||
|
|
||||||
|
summary["edges"] += sum(by_kind.values())
|
||||||
|
line = {"note_id": nid, "title": n.get("title"), "type": n.get("type"), "chunks": chunk_cnt, "edges_by_kind": by_kind}
|
||||||
|
print(json.dumps(line, ensure_ascii=False))
|
||||||
|
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "summary": summary}, ensure_ascii=False))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
import argparse
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Testet die Explanation-Funktion des Retrievers.")
|
||||||
|
parser.add_argument("--url", default="http://localhost:8002/query", help="URL zum Query Endpoint")
|
||||||
|
parser.add_argument("--query", default="mindnet", help="Suchbegriff")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"mode": "hybrid",
|
||||||
|
"query": args.query,
|
||||||
|
"top_k": 1,
|
||||||
|
"explain": True, # Das neue Flag aus WP-04b
|
||||||
|
"expand": {
|
||||||
|
"depth": 1,
|
||||||
|
"edge_types": ["references", "belongs_to", "depends_on", "related_to"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
print(f"--- Sende Request an {args.url} ---")
|
||||||
|
try:
|
||||||
|
r = requests.post(args.url, json=payload)
|
||||||
|
r.raise_for_status()
|
||||||
|
data = r.json()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"FEHLER: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
hits = data.get("results", [])
|
||||||
|
if not hits:
|
||||||
|
print("Keine Treffer gefunden.")
|
||||||
|
return
|
||||||
|
|
||||||
|
top_hit = hits[0]
|
||||||
|
print(f"\nTop Treffer: {top_hit.get('node_id')}")
|
||||||
|
print(f"Total Score: {top_hit.get('total_score'):.4f}")
|
||||||
|
|
||||||
|
# Explanation prüfen
|
||||||
|
expl = top_hit.get("explanation")
|
||||||
|
if not expl:
|
||||||
|
print("\n[FAIL] Keine 'explanation' im Response erhalten!")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print("\n--- EXPLANATION LAYER ---")
|
||||||
|
|
||||||
|
bd = expl.get("breakdown", {})
|
||||||
|
print(f"Breakdown:")
|
||||||
|
print(f" Semantic Contribution: {bd.get('semantic_contribution', 0):.4f}")
|
||||||
|
print(f" Edge Contribution: {bd.get('edge_contribution', 0):.4f}")
|
||||||
|
print(f" Centrality Contribution: {bd.get('centrality_contribution', 0):.4f}")
|
||||||
|
print(f" Typ-Gewicht: {bd.get('node_weight', 0):.2f}")
|
||||||
|
|
||||||
|
print("\nReasons:")
|
||||||
|
for reason in expl.get("reasons", []):
|
||||||
|
kind = reason.get("kind", "").upper()
|
||||||
|
msg = reason.get("message", "")
|
||||||
|
impact = reason.get("score_impact")
|
||||||
|
impact_str = f" ({impact:+.4f})" if impact is not None else ""
|
||||||
|
print(f" [{kind}] {msg}{impact_str}")
|
||||||
|
|
||||||
|
print("\n[SUCCESS] Explanation Layer antwortet korrekt.")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Loaded 100 of 164 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user