Compare commits
457
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ba57e8b43f | ||
|
|
be406d230f | ||
|
|
38abde8516 | ||
|
|
9bb6566419 | ||
|
|
2acaf3c060 | ||
|
|
ec33163d98 | ||
|
|
2d0913cf3b | ||
|
|
76ea8e3350 | ||
|
|
bbc1e589f5 | ||
|
|
987e297c07 | ||
|
|
046aa2cf48 | ||
|
|
0bc0e66a85 | ||
|
|
2310633de3 | ||
|
|
bd36d78025 | ||
|
|
d2270fafdd | ||
|
|
86faf5b8f0 | ||
|
|
902f26ee95 | ||
|
|
ea9fe1a6ed | ||
|
|
6c2074166c | ||
|
|
bd44af2b68 | ||
|
|
972cd0dfac | ||
|
|
9cc16bb220 | ||
|
|
03594424a1 | ||
|
|
97985371ca | ||
|
|
3c5c01998f | ||
|
|
2fa24cb1bd | ||
|
|
0304bae9f4 | ||
|
|
ac6d34731a | ||
|
|
be5ad7e080 | ||
|
|
32128c06c1 | ||
|
|
07b7f419de | ||
|
|
4d6fce2d93 | ||
|
|
b398168b1f | ||
|
|
6b397fea2e | ||
|
|
d20386e0a7 | ||
|
|
fcb90db2d0 | ||
|
|
5d8e96372f | ||
|
|
0d1a98279e | ||
|
|
73641abf8a | ||
|
|
a4873b3d45 | ||
|
|
4f171e5245 | ||
|
|
d48b671314 | ||
|
|
b9d73af062 | ||
|
|
21904e62ee | ||
|
|
f9af30b195 | ||
|
|
531e3790b3 | ||
|
|
7745b71832 | ||
|
|
b39663408c | ||
|
|
9c4696e9d5 | ||
|
|
cf5e53f341 | ||
|
|
e512f768ff | ||
|
|
c3ef65d069 | ||
|
|
90f0be6baf | ||
|
|
eba51eba0f | ||
|
|
fbaa07ba88 | ||
|
|
d0ecc5316f | ||
|
|
9ee39277ae | ||
|
|
9d3fd21c88 | ||
|
|
d7217672a3 | ||
|
|
e39b685b1e | ||
|
|
5b9399361d | ||
|
|
7631b93900 | ||
|
|
f98eb8fc16 | ||
|
|
ab60106fb9 | ||
|
|
caab41a760 | ||
|
|
78d8ab11ed | ||
|
|
d49da1b5fe | ||
|
|
27497fd6be | ||
|
|
466ce3ca82 | ||
|
|
8b5af5ec65 | ||
|
|
34cbc0c0eb | ||
|
|
52ff4bbff5 | ||
|
|
9e7104e362 | ||
|
|
882ae5368f | ||
|
|
12a424ce23 | ||
|
|
c3230761e8 | ||
|
|
f5104008c2 | ||
|
|
013e8c6d0c | ||
|
|
603bba533b | ||
|
|
e202a8d88b | ||
|
|
c56a31c6bc | ||
|
|
b73b18bf9a | ||
|
|
0f351252b2 | ||
|
|
d7d7411fa0 | ||
|
|
45af7f074c | ||
|
|
70ab118612 | ||
|
|
ba0cdea9d0 | ||
|
|
09500ff764 | ||
|
|
b29ce4a0a6 | ||
|
|
311e598b69 | ||
|
|
413dca770c | ||
|
|
9d9239b11e | ||
|
|
a571c41c61 | ||
|
|
ba1fab9dcf | ||
|
|
5dc9c13a71 | ||
|
|
70821bae5a | ||
|
|
486abf82dd | ||
|
|
19afe3af33 | ||
|
|
113042b7f4 | ||
|
|
f8eebd91cb | ||
|
|
00297fc354 | ||
|
|
5baac082f9 | ||
|
|
611b0aa54e | ||
|
|
110f0dbcf5 | ||
|
|
1688cf0a1e | ||
|
|
13b7c8858a | ||
|
|
b8b6f243d1 | ||
|
|
eaa6e805ce | ||
|
|
05dbb21c58 | ||
|
|
f1f4358dee | ||
|
|
3374c41f07 | ||
|
|
95affe87e5 | ||
|
|
5e86398fcd | ||
|
|
844f0cbe42 | ||
|
|
4d5d7847e5 | ||
|
|
06795feed6 | ||
|
|
a4548a7ee1 | ||
|
|
42216865e2 | ||
|
|
e48bdb2401 | ||
|
|
c67eb4b2f3 | ||
|
|
fafa74b0b8 | ||
|
|
049a39aba3 | ||
|
|
cd26430a61 | ||
|
|
aab010ff17 | ||
|
|
9a7bc32051 | ||
|
|
bfdd8ac6be | ||
|
|
018669fb87 | ||
|
|
fb2de78d62 | ||
|
|
7e8d6c1954 | ||
|
|
d21230c6c3 | ||
|
|
4e98d5bfa1 | ||
|
|
8998269377 | ||
|
|
6e9a227ad2 | ||
|
|
4e7506da5a | ||
|
|
06ad93c77c | ||
|
|
0bb7b34259 | ||
|
|
859c17b49d | ||
|
|
46b26c9624 | ||
|
|
fd215c18e4 | ||
|
|
efa3afc7dc | ||
|
|
e03dd66051 | ||
|
|
3476fe5fae | ||
|
|
3e08c8347e | ||
|
|
c691123d2d | ||
|
|
a4b81535e8 | ||
|
|
a898cf4b65 | ||
|
|
10009a0a43 | ||
|
|
4dfe47ff99 | ||
|
|
bd997e61d6 | ||
|
|
23d0670126 | ||
|
|
12c600edbe | ||
|
|
e4115e705e | ||
|
|
0cbc4a2d72 | ||
|
|
cea6d35729 | ||
|
|
4c56918d8a | ||
|
|
f17263417e | ||
|
|
6f0b463489 | ||
|
|
28c01caa1a | ||
|
|
95b59e9b0a | ||
|
|
bc967f1f6e | ||
|
|
4228b8a74f | ||
|
|
300086fc83 | ||
|
|
688eb262a9 | ||
|
|
bfb6f5dede | ||
|
|
88c0fbe56b | ||
|
|
3d74eff224 | ||
|
|
22d08afe2d | ||
|
|
a97f757e34 | ||
|
|
bbc8f13944 | ||
|
|
f18a40d76c | ||
|
|
c7644a36aa | ||
|
|
f394f31c12 | ||
|
|
0513673c2a | ||
|
|
a73542a391 | ||
|
|
c465797654 | ||
|
|
7ce3f982f2 | ||
|
|
6311456917 | ||
|
|
3c06559c5d | ||
|
|
5c9f2e8c83 | ||
|
|
62902faaca | ||
|
|
d22fd77b22 | ||
|
|
2f86f493d6 | ||
|
|
eafa0fad85 | ||
|
|
2e7c497b69 | ||
|
|
79dcfdcf9a | ||
|
|
487e28a904 | ||
|
|
c492f97a67 | ||
|
|
ec2ff74514 | ||
|
|
c501f8d6e6 | ||
|
|
b4287cbfda | ||
|
|
26724a1db1 | ||
|
|
d6368ee709 | ||
|
|
ea211a5c0b | ||
|
|
9e8b433c95 | ||
|
|
2a1c62aeed | ||
|
|
948d6f4b47 | ||
|
|
725271a7da | ||
|
|
4ea62e6886 | ||
|
|
c01cf3b078 | ||
|
|
c67777a6b9 | ||
|
|
e5e5e7560c | ||
|
|
6e1cd5fbbc | ||
|
|
8bd8f7c0bb | ||
|
|
1c42cd1f78 | ||
|
|
be090a43f2 | ||
|
|
2c81fa0cb5 | ||
|
|
c3815afcd4 | ||
|
|
8800160564 | ||
|
|
bdbc4a1bf7 | ||
|
|
597090bc45 | ||
|
|
af36c410b4 | ||
|
|
4dcd606c10 | ||
|
|
bbd5a7fa48 | ||
|
|
6dc37ccb66 | ||
|
|
b84906283e | ||
|
|
af0dcdf76a | ||
|
|
f1e1cde597 | ||
|
|
a7c5630e5b | ||
|
|
82151d6bc3 | ||
|
|
c93e7ad598 | ||
|
|
290b271cf6 | ||
|
|
a686bdbeaf | ||
|
|
2b84c62875 | ||
|
|
2de786fc64 | ||
|
|
2ddf034983 | ||
|
|
b2043f4f84 | ||
|
|
52eae52061 | ||
|
|
e451ea64ae | ||
|
|
eb6e4028ff | ||
|
|
ea45493941 | ||
|
|
da70f0e00c | ||
|
|
8bf3707511 | ||
|
|
804eeef858 | ||
|
|
36658f12fe | ||
|
|
f0e675ffd8 | ||
|
|
90911c1074 | ||
|
|
1729e0b40d | ||
|
|
70b3169e7f | ||
|
|
315a1c0945 | ||
|
|
b9316c693d | ||
|
|
820e52d869 | ||
|
|
444e8e4206 | ||
|
|
e6f2c8b4e8 | ||
|
|
f6b7652219 | ||
|
|
3cd00ab5cf | ||
|
|
0feae50d90 | ||
|
|
d79c55fc64 | ||
|
|
98a6f5c480 | ||
|
|
0fa4aa7280 | ||
|
|
3b192e2eed | ||
|
|
2794d26181 | ||
|
|
7b56f696d6 | ||
|
|
ead1b8c1bc | ||
|
|
b8126e00c4 | ||
|
|
62f8f4b313 | ||
|
|
21924541dc | ||
|
|
197242ca42 | ||
|
|
7922734dce | ||
|
|
afd7675425 | ||
|
|
8b3b343645 | ||
|
|
6713f342c3 | ||
|
|
79bb8a143c | ||
|
|
d03a181536 | ||
|
|
8e5304c156 | ||
|
|
c487db6534 | ||
|
|
0ca54f429e | ||
|
|
6c62f64784 | ||
|
|
3282f85007 | ||
|
|
b186569750 | ||
|
|
7cde2462ec | ||
|
|
c0b96f7c05 | ||
|
|
3a88caac69 | ||
|
|
31457e9240 | ||
|
|
307335a733 | ||
|
|
4eb5e34ea7 | ||
|
|
f4be219790 | ||
|
|
53591b6f27 | ||
|
|
a14d0bb7cb | ||
|
|
c2802e7cb3 | ||
|
|
c7f2815f1d | ||
|
|
afc90fe992 | ||
|
|
2f9ce824a0 | ||
|
|
a39b2a6950 | ||
|
|
70fa95966c | ||
|
|
10479014d5 | ||
|
|
148e22aa7d | ||
|
|
a9fe6196f7 | ||
|
|
ae20665a66 | ||
|
|
ccbf078463 | ||
|
|
cd1e46784f | ||
|
|
b49d97b8d7 | ||
|
|
3d44de9d87 | ||
|
|
987b3c1770 | ||
|
|
25bc6544c4 | ||
|
|
b241ae753b | ||
|
|
2041771b14 | ||
|
|
b5958a9f63 | ||
|
|
c05dbd4b3b | ||
|
|
44e468fc21 | ||
|
|
5f358d329b | ||
|
|
59fe8154f2 | ||
|
|
d5b2e9bb8f | ||
|
|
3a3907e1b6 | ||
|
|
f66cdc70b2 | ||
|
|
e299a497a7 | ||
|
|
eefb29d2ad | ||
|
|
668c485037 | ||
|
|
c83d5bb970 | ||
|
|
8638b3c95b | ||
|
|
c9b55c16f7 | ||
|
|
a3b9d53d06 | ||
|
|
215915cee2 | ||
|
|
df9271ed81 | ||
|
|
97e73a0fa0 | ||
|
|
e2ebe98e7f | ||
|
|
345e30de71 | ||
|
|
0084160e12 | ||
|
|
7280c111f0 | ||
|
|
763de11d2b | ||
|
|
9c0de3eeb2 | ||
|
|
9c81e9fbf1 | ||
|
|
40603f1652 | ||
|
|
4a34da31ad | ||
|
|
6e394faab1 | ||
|
|
25d0895cac | ||
|
|
f5f93cfb12 | ||
|
|
cf1ebe9d6f | ||
|
|
dd0d4ff8b1 | ||
|
|
98103f3270 | ||
|
|
a740ebb42e | ||
|
|
7c79f820cb | ||
|
|
f3b2326304 | ||
|
|
4eded247c5 | ||
|
|
3ffb857181 | ||
|
|
ec92906399 | ||
|
|
87671640ea | ||
|
|
c725e984bb | ||
|
|
8f05b57744 | ||
|
|
120b0147f2 | ||
|
|
ae695b4b15 | ||
|
|
89fee685c3 | ||
|
|
23eb9a21c8 | ||
|
|
a009efe91c | ||
|
|
dbd3111adf | ||
|
|
2b35e0c07a | ||
|
|
ba3354df03 | ||
|
|
7c8c19f416 | ||
|
|
35f83241cb | ||
|
|
5297f9b47e | ||
|
|
cbc50ec4eb | ||
|
|
198cc291e2 | ||
|
|
cb8c3a6ea6 | ||
|
|
66f597bbba | ||
|
|
fe237c8b4b | ||
|
|
ec7b08a625 | ||
|
|
9dd728579d | ||
|
|
0a6ca00f64 | ||
|
|
6e856ce05f | ||
|
|
1b8b78cfe9 | ||
|
|
49508d9f87 | ||
|
|
63a660223e | ||
|
|
2cbc2e2f43 | ||
|
|
eb2ad44755 | ||
|
|
c3979142a6 | ||
|
|
1a30de6bfc | ||
|
|
8c0df47966 | ||
|
|
4c177378ae | ||
|
|
a2db175819 | ||
|
|
2d0f050524 | ||
|
|
2d2b78edb3 | ||
|
|
627fa80f3e | ||
|
|
96cbbb093e | ||
|
|
6551308f14 | ||
|
|
9ccc2badd4 | ||
|
|
07a83bdeb7 | ||
|
|
224dfb6588 | ||
|
|
d8d6c68ac2 | ||
|
|
3e369e5464 | ||
|
|
abaaba97f9 | ||
|
|
bd49467d26 | ||
|
|
b44524e1a9 | ||
|
|
0319667f8e | ||
|
|
4c3ad1f83c | ||
|
|
a7a845187a | ||
|
|
e14a9a11d2 | ||
|
|
d7ce757d65 | ||
|
|
6f5447c316 | ||
|
|
727b67811c | ||
|
|
a85fb68015 | ||
|
|
2dfe7328cc | ||
|
|
cfc0a0ca8f | ||
|
|
2b1c7fe20d | ||
|
|
d5e1d9c09b | ||
|
|
554540451e | ||
|
|
c10134c22b | ||
|
|
9175b3a045 | ||
|
|
0ecae44888 | ||
|
|
a9c2a10fb5 | ||
|
|
e11094e5fa | ||
|
|
38913dc3e1 | ||
|
|
dd2b7932a2 | ||
|
|
2051e187ba | ||
|
|
ca817a8cc4 | ||
|
|
2d815f5902 | ||
|
|
0abdc647c9 | ||
|
|
fded2db73e | ||
|
|
2274c9faf5 | ||
|
|
280b236490 | ||
|
|
ef2c144cdd | ||
|
|
634cf0ff1b | ||
|
|
2dc3d5c169 | ||
|
|
fcad1fdfaa | ||
|
|
5c6cdfb5e1 | ||
|
|
1d6ba761a8 | ||
|
|
893ba6c8f4 | ||
|
|
a835eff14b | ||
|
|
029b4d8a6e | ||
|
|
0053b45691 | ||
|
|
f46812d1d2 | ||
|
|
8495df961b | ||
|
|
abc8be1077 | ||
|
|
84baf27f7e | ||
|
|
1a08d904f9 | ||
|
|
31ca9a95d7 | ||
|
|
ade91b26c3 | ||
|
|
d797957abc | ||
|
|
104ddddb96 | ||
|
|
c8c4c7fa00 | ||
|
|
3111a26229 | ||
|
|
26ef257e06 | ||
|
|
6864ea7b53 | ||
|
|
c2ecc15d89 | ||
|
|
9bbcc0952b | ||
|
|
179ec4d38b | ||
|
|
02a61cdf0d | ||
|
|
6a84b4e5aa | ||
|
|
497a9b9d58 | ||
|
|
c5972be618 | ||
|
|
43e77c4c6b | ||
|
|
d66985e597 | ||
|
|
ff4bed99c9 | ||
|
|
2b4c1b83bd | ||
|
|
f85aa8f604 | ||
|
|
e5ca3c4177 | ||
|
|
0f2f8208e3 | ||
|
|
ba17516814 | ||
|
|
c0979b69e2 | ||
|
|
4b5c33bb90 | ||
|
|
1ea4d53061 | ||
|
|
17b548e9b0 | ||
|
|
5755eb1f08 | ||
|
|
e2f15324d7 | ||
|
|
b0b0802a56 | ||
|
|
e14cc8b758 | ||
|
|
55c6a5058c | ||
|
|
52d5d01337 | ||
|
|
93ceff7f17 |
No files matched your search
@@ -14,7 +14,7 @@ jobs:
|
|||||||
runs-on: linux_host
|
runs-on: linux_host
|
||||||
env:
|
env:
|
||||||
TARGET_BASE: /home/llmadmin/mindnet
|
TARGET_BASE: /home/llmadmin/mindnet
|
||||||
DEPLOY_DIRS: "app scripts schemas docker tests requirements.txt README.md"
|
DEPLOY_DIRS: "app scripts schemas docker tests config requirements.txt README.md"
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
|
|||||||
Vendored
+3
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"CodeGPT.apiKey": "Google AI Studio"
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Architecture Snapshot v2.2.1
|
||||||
|
**Stand:** Nach Abschluss von WP-04c
|
||||||
|
**Kontext:** Wichtig für WP-05 Entwicklung.
|
||||||
|
|
||||||
|
## 1. Neue Komponenten (Seit v2.2.0)
|
||||||
|
|
||||||
|
### A. Feedback Service (`app/services/feedback_service.py`)
|
||||||
|
- **Zweck:** Logging von Trainingsdaten für späteres Self-Tuning (WP-08).
|
||||||
|
- **Storage:** Lokale JSONL-Dateien in `data/logs/` (Append-Only).
|
||||||
|
- `search_history.jsonl`: Query + Snapshot der Ergebnisse (Scores).
|
||||||
|
- `feedback.jsonl`: User-Rating zu spezifischer `node_id`.
|
||||||
|
|
||||||
|
### B. Explanation Layer (`app/core/retriever.py`)
|
||||||
|
- **Logik:** Der Retriever berechnet nicht nur Scores, sondern generiert `Explanation`-Objekte.
|
||||||
|
- **Graph:** `Subgraph` (in `graph_adapter.py`) führt jetzt auch `reverse_adj` (Incoming Edges), um zu erklären, warum ein Knoten wichtig ist ("Referenziert von...").
|
||||||
|
|
||||||
|
### C. DTOs (`app/models/dto.py`)
|
||||||
|
Das Datenmodell wurde massiv erweitert. Wichtige Klassen für WP-05:
|
||||||
|
- `QueryResponse`: Enthält jetzt `query_id` (UUID).
|
||||||
|
- `QueryHit`: Enthält optional `explanation` (Typ `Explanation`).
|
||||||
|
- `FeedbackRequest`: Für den Feedback-Loop.
|
||||||
|
|
||||||
|
## 2. Implikationen für WP-05 (Chat)
|
||||||
|
|
||||||
|
1. **Logging:** Auch der neue `/chat` Endpoint sollte idealerweise die `query_id` loggen oder nutzen, um Konsistenz zu wahren.
|
||||||
|
2. **DTO-Nutzung:** Der Chat-Service wird intern den Retriever aufrufen. Er muss mit den `QueryHit`-Objekten arbeiten, um den Kontext für das LLM zu bauen.
|
||||||
|
3. **Config:** Die Persönlichkeit wird in `config/prompts.yaml` definiert (Late Binding), nicht im Python-Code hardcodiert.
|
||||||
@@ -0,0 +1,465 @@
|
|||||||
|
# mindnet v2.2 – Dokumentationsrahmenplan
|
||||||
|
**Datei:** `docs/mindnet_docs_master_plan_v2.2.md`
|
||||||
|
**Stand:** 2025-12-06
|
||||||
|
**Bezug:** Programmplan mindnet v2.2 :contentReference[oaicite:0]{index=0}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zweck dieses Rahmenplans
|
||||||
|
|
||||||
|
Dieser Rahmenplan definiert:
|
||||||
|
|
||||||
|
- die **Zieldokumente** der mindnet v2.2 Dokumentation,
|
||||||
|
- deren **Dateinamen, Zielgruppen, Inhalte und Quellen**,
|
||||||
|
- die **Zuordnung** zu Workpackages (WP) und Release-Tags,
|
||||||
|
- die **Regeln zum Informations-Erhalt** (keine inhaltlichen Verluste),
|
||||||
|
- die **empfohlene Bearbeitungsreihenfolge**.
|
||||||
|
|
||||||
|
Alle hier beschriebenen Zieldokumente werden als **eigene Markdown-Dateien** gepflegt und sind Teil von WP-14 _Review / Refactoring / Dokumentation_ (Phase E). :contentReference[oaicite:1]{index=1}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Prinzipien zur Sicherung der Informationen
|
||||||
|
|
||||||
|
Um zu verhindern, dass Inhalte oder Ergebnisse verloren gehen, gelten folgende Prinzipien:
|
||||||
|
|
||||||
|
1. **Programmplan v2.2 als Masterreferenz**
|
||||||
|
- Programmauftrag, Vision, Phasenmodell und WP-Status werden **nicht verändert**, sondern nur referenziert.
|
||||||
|
- `Programmplan_V2.2.md` bleibt das führende Dokument für Ziele, WPs, Ampelstatus. :contentReference[oaicite:2]{index=2}
|
||||||
|
|
||||||
|
2. **Keine unkontrollierte Löschung von Inhalten**
|
||||||
|
- Bestehende Dokumente (`knowledge_design.md`, `chunking_strategy.md`, `mindnet_technical_architecture.md`, `Handbuch.md`, `TYPE_REGISTRY_MANUAL.md`, `docs_mindnet_retriever.md`, `wp04_retriever_scoring.md`) werden **nicht gelöscht**, sondern:
|
||||||
|
- vollständig archiviert,
|
||||||
|
- in die neuen Zieldokumente **überführt**,
|
||||||
|
- oder **explizit** in den Anhängen (I) als „legacy / historisch“ verlinkt.
|
||||||
|
|
||||||
|
3. **Explizite Quellenzuordnung („Building Blocks“)**
|
||||||
|
- Für jede Sektion eines Zieldokuments wird angegeben, aus **welchen bestehenden Dateien und Abschnitten** der Inhalt stammt.
|
||||||
|
- Neue Inhalte (z. B. User Guide H) werden klar als „neu, v2.2“ gekennzeichnet.
|
||||||
|
|
||||||
|
4. **Ist-Stand vs. Roadmap trennen**
|
||||||
|
- Implementierter Stand v2.2: WP01–WP04a (🟢) mit Importer, Chunking, Edge-Pipeline, Typ-Registry und hybrider Retriever.
|
||||||
|
- Geplante Features (Self-Healing, Virtual Schema Layer, Explanation Layer, Feedback-Logging, MCP-Agenten) werden als **Roadmap-Funktionen** kenntlich gemacht (Phase B–D, WPs 05–13).
|
||||||
|
|
||||||
|
5. **Versionierbarkeit**
|
||||||
|
- Alle Zieldokumente tragen im Kopf eine **Version** und **Stand-Datum** sowie optional einen Verweis auf den letzten Gitea-Tag (z. B. `mindnet_v2.2_docs_initial`, `WP04a-Retriever_final`). :contentReference[oaicite:6]{index=6}
|
||||||
|
- Änderungen werden im Abschnitt „Changelog“ der jeweiligen Datei notiert.
|
||||||
|
|
||||||
|
6. **Keine funktionalen Annahmen ohne Verifikation**
|
||||||
|
- Wo das Verhalten nicht eindeutig aus Code-/Handbuch-/WP-Dokumenten hervorgeht, wird der Punkt zunächst als _offene Frage_ markiert und nicht als Tatsache dokumentiert.
|
||||||
|
- Verifizierte Funktionen (z. B. `import_markdown`, Edge-Schema, `types.yaml`, `/query`, Scoring-Formel) werden exakt gemäß der bestehenden Dokumente beschrieben.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Zieldokumente der mindnet v2.2 Dokumentation
|
||||||
|
|
||||||
|
### 3.1 Übersichtstabelle
|
||||||
|
|
||||||
|
| ID | Zieldatei | Rolle/Zielgruppe | Hauptquellen (Building Blocks) |
|
||||||
|
|----|------------------------------------------|---------------------------------------------|--------------------------------|
|
||||||
|
| A | `mindnet_overview_v2.2.md` | Einstieg für alle Rollen | Programmplan v2.2, Knowledge Design |
|
||||||
|
| B | `mindnet_functional_architecture_v2.2.md`| Architekt:innen, Mindmaster | Knowledge Design, Functional Architecture, Programmplan |
|
||||||
|
| C | `mindnet_technical_architecture_v2.2.md` | Entwickler:innen, Tech-Architekt:innen | mindnet_technical_architecture, chunking_strategy, Handbuch |
|
||||||
|
| D | `mindnet_knowledge_design_manual_v2.2.md`| Vault-Autor:innen, Mindmaster, Analyst:innen| knowledge_design, TYPE_REGISTRY_MANUAL, Functional Architecture |
|
||||||
|
| E | `mindnet_pipeline_playbook_v2.2.md` | Dev/Ops, Tech-Lead | mindnet_v2_implementation_playbook, Handbuch, chunking_strategy, docs_mindnet_retriever, wp04_retriever_scoring |
|
||||||
|
| F | `mindnet_developer_guide_v2.2.md` | Entwickler:innen (Code, API, Tests) | Technical Architecture, Handbuch, docs_mindnet_retriever |
|
||||||
|
| G | `mindnet_admin_guide_v2.2.md` | Admins, Betreiber | Handbuch, Implementation Playbook, Programmplan (Governance) |
|
||||||
|
| H | `mindnet_user_guide_v2.2.md` | Endnutzer:innen, Agent-User | Knowledge Design, Programmplan (Use Cases), spätere MCP-WPs |
|
||||||
|
| I | `mindnet_appendices_v2.2.md` | Alle Rollen (Referenzen, Tabellen) | TYPE_REGISTRY_MANUAL, chunking_strategy, WP-Übersicht, Edge-Referenz |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Detailpläne pro Zieldokument (Inhaltsverzeichnis & Building Blocks)
|
||||||
|
|
||||||
|
### A) `mindnet_overview_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Einstieg in mindnet für alle Rollen (Owner, Mindmaster, Developer, Admin, Agenten-Entwickler).
|
||||||
|
- Auf 3–5 Seiten: _Was ist mindnet? Wie ist es gedacht? Wie lese ich die Dokumentation?_ :contentReference[oaicite:17]{index=17}
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Einführung**
|
||||||
|
1. Programmauftrag & Vision (v2.2)
|
||||||
|
2. Was mindnet ist – und was nicht
|
||||||
|
2. **Drei Ebenen von mindnet**
|
||||||
|
1. Content-Ebene (Vault, Notes, Chunks)
|
||||||
|
2. Semantische Ebene (Edges, Typen, Retriever)
|
||||||
|
3. Identitäts- / Persönlichkeits-Ebene (Werte, Entscheidungen, Profile – Roadmap)
|
||||||
|
3. **End-to-End-Überblick**
|
||||||
|
1. Datenfluss: Vault → Import → Qdrant → Retriever → Antwort
|
||||||
|
2. WP-Phasenmodell (A–E) und aktueller Stand (WP01–04a grün)
|
||||||
|
4. **Rollen & Perspektiven**
|
||||||
|
1. Mindmaster / Owner
|
||||||
|
2. Autor:in (Vault-Notes)
|
||||||
|
3. Developer / Architect
|
||||||
|
4. Administrator:in
|
||||||
|
5. Agenten & LLMs (MCP-Perspektive)
|
||||||
|
5. **Leserführung durch die Dokumentation**
|
||||||
|
1. Welche Datei für welche Rolle?
|
||||||
|
2. Wie mit Ist-Stand vs. Roadmap umgehen?
|
||||||
|
6. **Glossar zentraler Begriffe**
|
||||||
|
- mindnet, Vault, Note, Chunk, Edge, Type, Identity, Self-Healing, Self-Tuning, Explanation Layer, Virtual Schema Layer, etc.
|
||||||
|
|
||||||
|
#### Building Blocks (Quellen-Mapping)
|
||||||
|
|
||||||
|
- Programmauftrag & Vision → `Programmplan_V2.2.md`, Kapitel 1–2. :contentReference[oaicite:18]{index=18}
|
||||||
|
- Phasenmodell & WPs → `Programmplan_V2.1.md`/v2.2, Kapitel 5–9 (angepasst auf v2.2-Stand).
|
||||||
|
- High-Level-Architektur → `knowledge_design.md`, Kapitel 1–3 (Ziel, Zielarchitektur, Collections). :contentReference[oaicite:20]{index=20}
|
||||||
|
- Rollenmodell → Ableitung aus Knowledge Design + Programmplan (Owner, Agenten, MCP-WPs).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B) `mindnet_functional_architecture_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Fachliche Architektur von mindnet (Konzeptmodell).
|
||||||
|
- Für Architekt:innen, Mindmaster, Knowledge-Designer.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Ziel & Einordnung**
|
||||||
|
1. Bezug zu Programmauftrag & WPs (WP01–WP07, WP10–12)
|
||||||
|
2. **Kernkonzepte**
|
||||||
|
1. Notes & Note-Typen (concept, experience, project, decision, value, principle, person, event, …)
|
||||||
|
2. Chunks & Sections
|
||||||
|
3. Edges (Struktur, explizit, inline, Default-Kanten)
|
||||||
|
4. Typen-Registry & Virtual Schema Layer (fachliche Sicht, Roadmap)
|
||||||
|
3. **Graph-Modell**
|
||||||
|
1. Knotenarten (Notes, Chunks, Identity Nodes, People)
|
||||||
|
2. Kantenarten (belongs_to, references, similar_to, depends_on, related_to, next/prev, …)
|
||||||
|
3. Scope & Provenienz (chunk vs. note, rule_id, confidence)
|
||||||
|
4. **End-to-End fachlicher Datenfluss**
|
||||||
|
1. Erfassung von Wissen (Note erstellen)
|
||||||
|
2. Import & Graphaufbau
|
||||||
|
3. Retrieval & Begründungspfade
|
||||||
|
4. Self-Healing & Retroactive Edges (WP06/07 – Roadmap)
|
||||||
|
5. **Szenarien & Use Cases**
|
||||||
|
1. Wissensspeicherung (z. B. Leitbild, Projekt, Erfahrung)
|
||||||
|
2. Entscheidungen & Why-Layer (WP04b – Roadmap)
|
||||||
|
3. Persönlichkeits-/Identitätsmodell (WP10–12 – Roadmap)
|
||||||
|
6. **Rollenmodell (fachliche Sicht)**
|
||||||
|
1. Mindmaster
|
||||||
|
2. Content-Autor:innen
|
||||||
|
3. Analyst:innen (Graph-Leser)
|
||||||
|
7. **Designprinzipien (fachlich)**
|
||||||
|
- Late Binding, Virtual Schema, Self-Healing, Minimalinvasives Schreiben etc.
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `knowledge_design.md` – Kapitel 1–4, 6–7 (Ziel, Architektur, Collections, Identitäten, Prinzipien). :contentReference[oaicite:28]{index=28}
|
||||||
|
- `mindnet_functional_architecture.md` – alle Kapitel (Kernkonzepte, Edgetypen, Provenienz, Rollen). :contentReference[oaicite:29]{index=29}
|
||||||
|
- `chunking_strategy.md` – Edges, Datenmodell, Graph-Sicht. :contentReference[oaicite:30]{index=30}
|
||||||
|
- Programmplan – WPs 01–07, 10–12 für Roadmap-Features.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### C) `mindnet_technical_architecture_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Technische Architektur des Systems: Collections, Payload-Felder, Services, Schemata.
|
||||||
|
- Für Entwickler:innen und Tech-Architekt:innen.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Ziel & Scope**
|
||||||
|
1. Abgrenzung zur funktionalen Architektur
|
||||||
|
2. Bezug zu WPs 02–04a, 05 (Schema, Importer, Retriever)
|
||||||
|
2. **Systemübersicht**
|
||||||
|
1. Komponentenübersicht (Importer, Qdrant, Retriever-API, Embedding-Service, n8n/MCP-Ausblick)
|
||||||
|
2. Ports, Dienste, Hauptverzeichnisse
|
||||||
|
3. **Datenmodell in Qdrant**
|
||||||
|
1. Notes (`mindnet_notes`)
|
||||||
|
2. Chunks (`mindnet_chunks`)
|
||||||
|
3. Edges (`mindnet_edges`)
|
||||||
|
4. Optionale Collections (`mindnet_people`, zukünftige)
|
||||||
|
4. **Typ-Registry & Konfiguration**
|
||||||
|
1. `config/types.yaml` – Aufbau, Defaults, Typauflösung (Ist-Stand: Frontmatter-Overrides deaktiviert)
|
||||||
|
2. `config/retriever.yaml` – Scoringgewichte, Edge-Typ-Gewichte :contentReference[oaicite:35]{index=35}
|
||||||
|
5. **Import-Pipeline (technische Sicht)**
|
||||||
|
1. Discovery & Parsing (`import_markdown.py`)
|
||||||
|
2. Hash-/Idempotenz-Mechanismus (ENV-Variablen, Baseline-Modi)
|
||||||
|
3. Chunking (`chunker.py`, `chunk_payload.py`)
|
||||||
|
4. Edge-Erzeugung (`derive_edges.py`)
|
||||||
|
5. Roundtrip & Tests (Export & Vault-Vergleich)
|
||||||
|
6. **Retriever-Architektur**
|
||||||
|
1. API (FastAPI-Endpunkt `/query`)
|
||||||
|
2. Semantik- und Hybridmodus
|
||||||
|
3. Subgraph & Graph-API (expand)
|
||||||
|
4. Scoring-Formel und Komponenten (semantik, edges, centrality) :contentReference[oaicite:37]{index=37}
|
||||||
|
7. **Offene technische Punkte / Roadmap**
|
||||||
|
1. Explanation Layer (WP04b)
|
||||||
|
2. Feedback-Logging (WP04c)
|
||||||
|
3. Virtual Schema Layer (WP05)
|
||||||
|
4. Self-Healing & Self-Tuning (WP06–08)
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `mindnet_technical_architecture.md` – Kerntext. :contentReference[oaicite:39]{index=39}
|
||||||
|
- `chunking_strategy.md` – Datenmodell & Chunk-Details. :contentReference[oaicite:40]{index=40}
|
||||||
|
- `Handbuch.md` – ENV, Skripte, Runbooks. :contentReference[oaicite:41]{index=41}
|
||||||
|
- `wp04_retriever_scoring.md` / `docs_mindnet_retriever.md` – Retriever-Details. :contentReference[oaicite:42]{index=42}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### D) `mindnet_knowledge_design_manual_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Konkrete Anleitung für das Schreiben und Strukturieren von Notes & Chunks im Vault.
|
||||||
|
- Für dich als Owner/Mindmaster und alle, die Notes erstellen.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Zweck & Scope**
|
||||||
|
1. Rolle des Vaults als „Source of Truth“
|
||||||
|
2. **Note-Struktur & Frontmatter**
|
||||||
|
1. Pflicht- und optionale Felder (id, title, type, tags, status, created/updated, …)
|
||||||
|
2. Empfehlungen für IDs, Pfade und Namenskonventionen
|
||||||
|
3. **Note-Typen & Typ-Registry**
|
||||||
|
1. Übersicht der Typen (concept, project, experience, decision, value, principle, person, event, …)
|
||||||
|
2. Typ-spezifische Einsatzempfehlungen
|
||||||
|
3. Zusammenspiel mit `types.yaml` (retriever_weight, chunk_profile, edge_defaults)
|
||||||
|
4. **Edges & Referenzen in Notes**
|
||||||
|
1. Wikilinks (`[[Note]]`) → `references`
|
||||||
|
2. Inline-Relationen (`[[rel:depends_on XYZ]]`, `similar_to`, `related_to`)
|
||||||
|
3. Callout-Edges (`> [!edge] …`)
|
||||||
|
4. Strukturkanten (Ordner, belongs_to)
|
||||||
|
5. **Best Practices**
|
||||||
|
1. Beispiele für gut modellierte Notes (Projekt, Entscheidung, Erfahrung)
|
||||||
|
2. Typische Anti-Patterns und wie man sie vermeidet
|
||||||
|
6. **Langfristige Stabilität**
|
||||||
|
1. Virtual Schema Layer – was bedeutet das für Note-Autoren? (Wenig Änderungen in bestehenden Notes, Logik in Configs) :contentReference[oaicite:46]{index=46}
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `knowledge_design.md` – Kerntext + Beispiele. :contentReference[oaicite:47]{index=47}
|
||||||
|
- `TYPE_REGISTRY_MANUAL.md` – Typ-Registry & Verhalten. :contentReference[oaicite:48]{index=48}
|
||||||
|
- `mindnet_functional_architecture.md` – ergänzende Konzepte & Edges. :contentReference[oaicite:49]{index=49}
|
||||||
|
- `chunking_strategy.md` – Inline-Relationen, Edges, Datenmodell. :contentReference[oaicite:50]{index=50}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### E) `mindnet_pipeline_playbook_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- End-to-End-Playbook für die gesamte Pipeline:
|
||||||
|
- Importer,
|
||||||
|
- Chunking,
|
||||||
|
- Edge-Erzeugung,
|
||||||
|
- Retriever & Scoring,
|
||||||
|
- (perspektivisch) Explanation Layer, Self-Healing, Self-Tuning.
|
||||||
|
- Für Dev/Ops und technische Leads.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Zweck & Einordnung**
|
||||||
|
1. Bezug zu WPs 02–04a (Ist) und 04b–08 (Roadmap)
|
||||||
|
2. **Import-Pipeline (Runbook-Sicht)**
|
||||||
|
1. Inputs (Vault, Config-Files, ENV)
|
||||||
|
2. Standardläufe (Initialimport, inkrementelle Updates, Sync-Deletes)
|
||||||
|
3. Baseline-Builds & Hash-Modi
|
||||||
|
3. **Chunking & Payload-Aufbau**
|
||||||
|
1. Chunk-Profile, Overlaps, Fenster
|
||||||
|
2. Feldbesetzung (`text`, `window`, `section_title`, `neighbors`, …)
|
||||||
|
4. **Edge-Erzeugung**
|
||||||
|
1. Strukturkanten (belongs_to, next, prev)
|
||||||
|
2. Explizite Edges (Wikilinks, Inline-Relationen, Callouts)
|
||||||
|
3. Typ-Default-Kanten aus `types.yaml`
|
||||||
|
5. **Retriever & Scoring**
|
||||||
|
1. Konfiguration (`retriever.yaml`, ENV)
|
||||||
|
2. Scoring-Formel und Parameter
|
||||||
|
3. Semantik- vs. Hybridmodus
|
||||||
|
4. Test- und Smoke-Suites
|
||||||
|
6. **Quality Gates & Tests**
|
||||||
|
1. Roundtrip-Tests (Vault → Qdrant → Export)
|
||||||
|
2. Chunk-Integrität und Fenster-Checks
|
||||||
|
3. Retriever-Regressionstests
|
||||||
|
7. **Ausblick: Self-Healing, Retroactive Edges, Self-Tuning**
|
||||||
|
1. Geplante Batch-Jobs und Feedback-Pipelines (WP06–08)
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `mindnet_v2_implementation_playbook.md` – Policies, Teststrategie, KPIs. :contentReference[oaicite:57]{index=57}
|
||||||
|
- `Handbuch.md` – Skripte & Test-Runbooks. :contentReference[oaicite:58]{index=58}
|
||||||
|
- `chunking_strategy.md` – Chunk- & Edge-Strategie. :contentReference[oaicite:59]{index=59}
|
||||||
|
- `docs_mindnet_retriever.md` / `wp04_retriever_scoring.md` – Retriever-Praktiken & Tests. :contentReference[oaicite:60]{index=60}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### F) `mindnet_developer_guide_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Konzentrierte Sicht für Entwickler:innen:
|
||||||
|
- wie das Projekt strukturiert ist,
|
||||||
|
- wie man es lokal aufsetzt,
|
||||||
|
- wie man neue Features implementiert/testet.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Projektstruktur**
|
||||||
|
1. Verzeichnisbaum (app/, scripts/, tests/, config/, …)
|
||||||
|
2. **Lokale Entwicklungsumgebung**
|
||||||
|
1. Abhängigkeiten, venv, ENV
|
||||||
|
2. Starten von Services (Qdrant, API, ggf. ollama)
|
||||||
|
3. **APIs & Module**
|
||||||
|
1. Importer-Skripte
|
||||||
|
2. Core-Module (Chunker, Edge-Engine, Retriever)
|
||||||
|
3. FastAPI-Endpunkte (`/query`, evtl. Admin-Endpoints)
|
||||||
|
4. **Tests & Debugging**
|
||||||
|
1. pytest-Suites
|
||||||
|
2. Logging & Diagnoseskripte
|
||||||
|
5. **Guidelines für Erweiterungen**
|
||||||
|
1. Neue Typen & Edge-Regeln
|
||||||
|
2. Änderungen am Datenmodell
|
||||||
|
3. Umgang mit Schema-Änderungen (Virtual Schema Layer)
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `mindnet_technical_architecture.md` – Modul- & API-Beschreibungen. :contentReference[oaicite:63]{index=63}
|
||||||
|
- `Handbuch.md` – Skripte & Beispiele. :contentReference[oaicite:64]{index=64}
|
||||||
|
- `docs_mindnet_retriever.md` – API-Beispiele für den Retriever. :contentReference[oaicite:65]{index=65}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### G) `mindnet_admin_guide_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Betriebshandbuch für mindnet:
|
||||||
|
- Installation,
|
||||||
|
- Konfiguration,
|
||||||
|
- Backup/Restore,
|
||||||
|
- Monitoring,
|
||||||
|
- Troubleshooting.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Zielgruppe & Scope**
|
||||||
|
2. **Initial Setup**
|
||||||
|
1. Voraussetzungen (Ubuntu, Docker/Qdrant, Python-Umgebung)
|
||||||
|
2. ENV-Variablen & Config-Files (types.yaml, retriever.yaml)
|
||||||
|
3. **Betrieb im Alltag**
|
||||||
|
1. Regelmäßige Imports
|
||||||
|
2. Umgang mit Fehlermeldungen im Importer
|
||||||
|
3. Health-Checks für Qdrant & API
|
||||||
|
4. **Update-Prozess**
|
||||||
|
1. Vorgehen bei Schema-/Code-Updates
|
||||||
|
2. Umgang mit neuen Typen und Policies
|
||||||
|
5. **Backup & Restore**
|
||||||
|
1. Qdrant-Snapshots
|
||||||
|
2. Vault-Backup (Obsidian)
|
||||||
|
3. Konsistenzprüfungen
|
||||||
|
6. **Monitoring & Logging**
|
||||||
|
1. typische Logquellen (Importer, API, n8n)
|
||||||
|
7. **Governance & Sicherheit**
|
||||||
|
1. Bezug zur Programm-Governance (WP14, MCP-Sicherheit)
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `Handbuch.md` – Betriebsskripte & ENV. :contentReference[oaicite:69]{index=69}
|
||||||
|
- `mindnet_v2_implementation_playbook.md` – Policies/Resilienz. :contentReference[oaicite:70]{index=70}
|
||||||
|
- Programmplan – Governance / WP14. :contentReference[oaicite:71]{index=71}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### H) `mindnet_user_guide_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Anleitung für Nutzer:innen (inkl. späterer Chat-Agenten-User):
|
||||||
|
- wie man mindnet im Alltag nutzt,
|
||||||
|
- wie man Fragen stellt,
|
||||||
|
- wie man neue Gedanken einbringt.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Was mindnet für dich tut**
|
||||||
|
1. Persönliches Gedächtnis
|
||||||
|
2. KI-Zwilling als Gesprächspartner :contentReference[oaicite:72]{index=72}
|
||||||
|
2. **Zugänge zu mindnet**
|
||||||
|
1. Obsidian-Vault
|
||||||
|
2. Chat-Agent (lokal, n8n-Workflows)
|
||||||
|
3. (später) Interview-Assistent, MCP-Tools
|
||||||
|
3. **Gute Fragen an mindnet**
|
||||||
|
1. Beispiel-Queries (Wissensabruf, Entscheidungen, Reflexionen)
|
||||||
|
4. **Neue Inhalte eingeben**
|
||||||
|
1. Notes in Obsidian erstellen
|
||||||
|
2. Typen und Links bewusst setzen (vereinfachte Version aus D)
|
||||||
|
5. **Ergebnisse interpretieren**
|
||||||
|
1. Was bedeutet ein Treffer-Set?
|
||||||
|
2. Wie sind Scores zu lesen? (nur Ist-Stand, Explanation Layer als Ausblick)
|
||||||
|
6. **Feedback & Lernen**
|
||||||
|
1. Manuelles Feedback (gut/schlecht) – Roadmap WP04c/08
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- Programmplan – Vision & Use-Cases. :contentReference[oaicite:76]{index=76}
|
||||||
|
- Knowledge Design – Autorensicht (stark vereinfacht). :contentReference[oaicite:77]{index=77}
|
||||||
|
- Retriever-Dokumente – Scoring & Interpretation. :contentReference[oaicite:78]{index=78}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### I) `mindnet_appendices_v2.2.md`
|
||||||
|
|
||||||
|
#### Zweck & Zielgruppe
|
||||||
|
- Referenzen, Tabellen, technische Details, die Hauptdokumente nicht überfrachten sollen.
|
||||||
|
|
||||||
|
#### Geplantes Inhaltsverzeichnis
|
||||||
|
|
||||||
|
1. **Typenreferenz (Auszug aus `types.yaml`)**
|
||||||
|
- Tabellenform: Typ, chunk_profile, edge_defaults, retriever_weight
|
||||||
|
2. **Edge-Typen & Bedeutungen**
|
||||||
|
- belongs_to, references, depends_on, similar_to, related_to, next, prev, …
|
||||||
|
3. **Datenmodelle (JSON-Skizzen)**
|
||||||
|
- Note, Chunk, Edge-Payload (aktuell implementierter Stand)
|
||||||
|
4. **Relevante ENV-Variablen & Konfigurationen**
|
||||||
|
- Hash-Settings, COLLECTION_PREFIX, MINDNET_TYPES_FILE, MINDNET_RETRIEVER_CONFIG etc.
|
||||||
|
5. **WP-Übersicht & Status**
|
||||||
|
- kompakte Tabelle aller WPs mit Status (🟢/🟡) und Verweisen auf relevante Dokumente.
|
||||||
|
6. **Historische Dokumente**
|
||||||
|
- Verweis auf alte Versionen (`mindnet_functional_architecture.md`, ältere Playbooks), falls diese nicht vollständig eingearbeitet werden.
|
||||||
|
|
||||||
|
#### Building Blocks
|
||||||
|
|
||||||
|
- `TYPE_REGISTRY_MANUAL.md` – Typen-Referenz. :contentReference[oaicite:84]{index=84}
|
||||||
|
- `chunking_strategy.md` – JSON-Beispiele. :contentReference[oaicite:85]{index=85}
|
||||||
|
- `mindnet_technical_architecture.md` – Felder & Prioritäten. :contentReference[oaicite:86]{index=86}
|
||||||
|
- Programmplan – WP-Tabellen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Bearbeitungsreihenfolge (Empfehlung)
|
||||||
|
|
||||||
|
Aus Sicht der Informationssicherung und Konsistenz ist folgende Reihenfolge sinnvoll:
|
||||||
|
|
||||||
|
1. **C – `mindnet_technical_architecture_v2.2.md`**
|
||||||
|
- Begründung:
|
||||||
|
- bildet den **aktuellen implementierten Stand** von WP02–WP04a am präzisesten ab,
|
||||||
|
- ist stark an konkrete Felder, Collections und Skripte gebunden (hier ist der Ist-Stand am eindeutigsten belegbar),
|
||||||
|
- ist die Grundlage für Developer Guide, Playbook und Admin Guide.
|
||||||
|
|
||||||
|
2. **E – `mindnet_pipeline_playbook_v2.2.md`**
|
||||||
|
- Konsolidiert Handbuch + Implementation-Playbook + WP04-Dokumente.
|
||||||
|
|
||||||
|
3. **D – `mindnet_knowledge_design_manual_v2.2.md`**
|
||||||
|
- Stellt sicher, dass das Wissensdesign stabil dokumentiert ist, bevor Overview/User Guide entstehen.
|
||||||
|
|
||||||
|
4. **A – `mindnet_overview_v2.2.md`**
|
||||||
|
- Kann sich dann sauber auf die finalisierte fachliche/technische Sicht stützen.
|
||||||
|
|
||||||
|
5. **F, G, H, I**
|
||||||
|
- Developer/Admin/User Guides & Appendices, basierend auf den konsolidierten Kern-Dokumenten.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Nächster konkreter Schritt
|
||||||
|
|
||||||
|
Als nächstes finalisieren wir gemäß dieser Reihenfolge das Dokument:
|
||||||
|
|
||||||
|
> **C – `mindnet_technical_architecture_v2.2.md`**
|
||||||
|
|
||||||
|
Dabei gehen wir wie folgt vor:
|
||||||
|
|
||||||
|
1. Bestehendes `mindnet_technical_architecture.md` **vollständig** als Basis übernehmen. :contentReference[oaicite:89]{index=89}
|
||||||
|
2. Fehlende Details aus `chunking_strategy.md`, `Handbuch.md`, `TYPE_REGISTRY_MANUAL.md` und `wp04_retriever_scoring.md` integrieren (ohne Informationsverlust).
|
||||||
|
3. Ist-Stand vs. Roadmap klar kennzeichnen (WP04b–08, WP05–07).
|
||||||
|
|
||||||
|
Dieses Vorgehen hält die technische Wahrheit konsistent und liefert eine stabile Basis für alle nachfolgenden Dokumente.
|
||||||
@@ -0,0 +1,592 @@
|
|||||||
|
# mindnet v2.2 — Programmplan
|
||||||
|
**Version:** 2.3.2 (Inkl. WP-10a GUI Evolution)
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** Aktiv
|
||||||
|
|
||||||
|
---
|
||||||
|
- [mindnet v2.2 — Programmplan](#mindnet-v22--programmplan)
|
||||||
|
- [1. Programmauftrag](#1-programmauftrag)
|
||||||
|
- [2. Vision](#2-vision)
|
||||||
|
- [3. Programmziele](#3-programmziele)
|
||||||
|
- [3.1 Kurzfristig (Abgeschlossen / Laufend)](#31-kurzfristig-abgeschlossen--laufend)
|
||||||
|
- [3.2 Mittelfristig (Nächste Schritte)](#32-mittelfristig-nächste-schritte)
|
||||||
|
- [3.3 Langfristig](#33-langfristig)
|
||||||
|
- [4. Architekturprinzipien](#4-architekturprinzipien)
|
||||||
|
- [5. Programmstruktur (Phasenmodell)](#5-programmstruktur-phasenmodell)
|
||||||
|
- [6. Workpackages – detaillierte Übersicht](#6-workpackages--detaillierte-übersicht)
|
||||||
|
- [Legende Aufwand / Komplexität](#legende-aufwand--komplexität)
|
||||||
|
- [WP-01 – Wissensdesign (abgeschlossen)](#wp-01--wissensdesign-abgeschlossen)
|
||||||
|
- [WP-02 – Chunking \& Hash-Strategie (abgeschlossen)](#wp-02--chunking--hash-strategie-abgeschlossen)
|
||||||
|
- [WP-03 – Import-Pipeline \& Edge-System v2 (abgeschlossen)](#wp-03--import-pipeline--edge-system-v2-abgeschlossen)
|
||||||
|
- [WP-04a – Retriever \& Graph-Scoring (abgeschlossen)](#wp-04a--retriever--graph-scoring-abgeschlossen)
|
||||||
|
- [WP-04b – Explanation Layer ("Why-Layer") (abgeschlossen)](#wp-04b--explanation-layer-why-layer-abgeschlossen)
|
||||||
|
- [WP-04c – Feedback Logging \& Bewertungsdaten (abgeschlossen)](#wp-04c--feedback-logging--bewertungsdaten-abgeschlossen)
|
||||||
|
- [WP-05 – Persönlichkeitsmodell \& RAG-Chat (abgeschlossen)](#wp-05--persönlichkeitsmodell--rag-chat-abgeschlossen)
|
||||||
|
- [WP-05b – Advanced Chat (Optional)](#wp-05b--advanced-chat-optional)
|
||||||
|
- [WP-06 – Decision Engine \& Hybrid Router (abgeschlossen)](#wp-06--decision-engine--hybrid-router-abgeschlossen)
|
||||||
|
- [WP-07 – Interview-Assistent (geplant)](#wp-07--interview-assistent-geplant)
|
||||||
|
- [WP-08 – Self-Tuning v1/v2 (geplant)](#wp-08--self-tuning-v1v2-geplant)
|
||||||
|
- [WP-09 – Vault-Onboarding \& Migration (geplant)](#wp-09--vault-onboarding--migration-geplant)
|
||||||
|
- [WP-10 – Chat-Interface \& Writeback (abgeschlossen)](#wp-10--chat-interface--writeback-abgeschlossen)
|
||||||
|
- [WP-10a – GUI Evolution: Interaktion \& Tools (geplant)](#wp-10a--gui-evolution-interaktion--tools-geplant)
|
||||||
|
- [WP-11 – Knowledge-Builder \& Vernetzungs-Assistent (geplant)](#wp-11--knowledge-builder--vernetzungs-assistent-geplant)
|
||||||
|
- [WP-12 – Knowledge Rewriter (Soft Mode, geplant)](#wp-12--knowledge-rewriter-soft-mode-geplant)
|
||||||
|
- [WP-13 – MCP-Integration \& Agenten-Layer (geplant)](#wp-13--mcp-integration--agenten-layer-geplant)
|
||||||
|
- [WP-14 – Review / Refactoring / Dokumentation (geplant)](#wp-14--review--refactoring--dokumentation-geplant)
|
||||||
|
- [7. Abhängigkeiten (vereinfacht, aktualisiert)](#7-abhängigkeiten-vereinfacht-aktualisiert)
|
||||||
|
- [8. Laufzeit- \& Komplexitätsindikatoren (aktualisiert)](#8-laufzeit---komplexitätsindikatoren-aktualisiert)
|
||||||
|
- [9. Programmfortschritt (Ampel, aktualisiert)](#9-programmfortschritt-ampel-aktualisiert)
|
||||||
|
- [10. Governance \& Versionierung](#10-governance--versionierung)
|
||||||
|
- [11. Executive Summary](#11-executive-summary)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Programmauftrag
|
||||||
|
|
||||||
|
mindnet v2.2 entwickelt ein persönliches, wachsendes KI-Gedächtnis, das:
|
||||||
|
|
||||||
|
- Wissen, Erfahrungen, Werte und Entscheidungen speichert,
|
||||||
|
- diese Informationen semantisch verknüpft und rekonstruierbar macht,
|
||||||
|
- einen KI-Zwilling aufbaut, der ähnlich argumentiert, entscheidet und reflektiert wie du,
|
||||||
|
- über mehrere Kanäle gefüttert wird:
|
||||||
|
- Obsidian-Markdown (primäre Quelle),
|
||||||
|
- Chat-basierter Agent (Decision Engine & RAG-Chat aktiv),
|
||||||
|
- später: Interview-Assistent (strukturierte Dialogerfassung),
|
||||||
|
- automatisch neue Zusammenhänge erkennt und vernetzt (Edges, Typen, Hinweise),
|
||||||
|
- sich durch Rückmeldungen (Feedback) selbst verbessert (Self-Tuning).
|
||||||
|
|
||||||
|
Langfristig soll mindnet als **digitales Gegenüber** funktionieren, das:
|
||||||
|
|
||||||
|
- dich versteht,
|
||||||
|
- deine Denkweise reflektiert,
|
||||||
|
- deine Werte kennt und verwendet,
|
||||||
|
- Entscheidungen begründen kann („Warum?“),
|
||||||
|
- Erinnerungen einordnet,
|
||||||
|
- und für deine Nachkommen als dialogfähiger Gesprächspartner zur Verfügung steht.
|
||||||
|
|
||||||
|
mindnet ist **kein statisches Wissensarchiv**, sondern ein **lebendes, lernendes Gedächtnismodell** mit Fokus auf:
|
||||||
|
|
||||||
|
- persönliche Perspektive,
|
||||||
|
- erklärbare Begründungspfade (Why-Layer),
|
||||||
|
- kontinuierliche Erweiterbarkeit,
|
||||||
|
- robuste technische Basis (lokal, nachvollziehbar, versioniert).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Vision
|
||||||
|
|
||||||
|
> „Ein persönliches semantisches Gedächtnis, das mit mir wächst, meine Persönlichkeit spiegelt und mir als intelligenter, erklärbarer, lernender Begleiter dient.“
|
||||||
|
|
||||||
|
Kernprinzipien der Vision:
|
||||||
|
|
||||||
|
- **Erklärbarkeit:**
|
||||||
|
Jede Antwort muss über Edges, Scores, Werte-Bezüge und Herkunftsnotizen begründbar sein. Das System kann Entscheidungen auf zugrundeliegende `[DECISION]`-Notizen zurückführen.
|
||||||
|
|
||||||
|
- **Wachstum:**
|
||||||
|
Das System arbeitet von Anfang an mit unvollständigen Daten, kann aber schrittweise dichter werden, ohne dass alte Notizen massenhaft manuell angepasst werden müssen.
|
||||||
|
|
||||||
|
- **Flexibilität (Late Binding):**
|
||||||
|
Semantik wird überwiegend in Konfiguration (z. B. `types.yaml`, `prompts.yaml`, `decision_engine.yaml`, Policies) festgelegt. Die Persönlichkeit entsteht durch das Config-Design, nicht durch Hardcoding.
|
||||||
|
|
||||||
|
- **Autonomie & Self-Healing:**
|
||||||
|
mindnet schlägt fehlende Typen, Relationen und Edges vor (z. B. aus Inline-Relationen, Edge-Defaults, Ähnlichkeitsbeziehungen) und baut damit einen „self-healing graph“ auf.
|
||||||
|
|
||||||
|
- **Lernen & Self-Tuning:**
|
||||||
|
Feedback zu Antworten (gut/schlecht, relevant/nicht relevant) fließt in Score-Gewichte, Policies und ggf. Edge-Struktur ein.
|
||||||
|
|
||||||
|
- **Persönlichkeit:**
|
||||||
|
Entscheidungen werden wert- und erfahrungsbasiert begründet; das System agiert als KI-Zwilling durch Nutzung lokaler LLMs (z.B. Phi-3/Mistral) und eines Intent-Routers.
|
||||||
|
|
||||||
|
- **Incremental Growth:**
|
||||||
|
Das System muss mit wenigen, heterogenen Notizen starten und sich fortlaufend verdichten können – ohne Retro-Massenmigrationen im Vault.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Programmziele
|
||||||
|
|
||||||
|
### 3.1 Kurzfristig (Abgeschlossen / Laufend)
|
||||||
|
|
||||||
|
- Automatische Sammlung von Wissen aus Markdown-Vault.
|
||||||
|
- Aufbau eines stabilen Graph-Speichers in Qdrant (`*_notes`, `*_chunks`, `*_edges`).
|
||||||
|
- Semantischer und graphbasierter Retriever (WP-04a abgeschlossen).
|
||||||
|
- **Erklärbarkeit:** Why-Layer liefert Begründungen zu Treffern (WP-04b abgeschlossen).
|
||||||
|
- **Feedback-Loop:** Systematisches Logging von Suche und Bewertung (WP-04c abgeschlossen).
|
||||||
|
- **RAG-Chat:** KI antwortet in natürlicher Sprache auf Basis von Wissen und Persönlichkeit (WP-05 abgeschlossen).
|
||||||
|
- **Decision Engine:** System erkennt Intent (Fakt vs. Entscheidung) und wägt Werte ab (WP-06 abgeschlossen).
|
||||||
|
- **Multi-Persona:** System wechselt den Tonfall (Empathisch vs. Analytisch) situativ (WP-06 abgeschlossen).
|
||||||
|
- **Chat Interface:** Web-basiertes Frontend (Streamlit) für einfache Interaktion und Feedback-Gabe (WP-10 abgeschlossen).
|
||||||
|
- Technische Basis: FastAPI, Qdrant, Ollama (Local LLM), Streamlit.
|
||||||
|
- Automatisierte Erkennung von Beziehungen:
|
||||||
|
- Wikilinks, Inline-Relationen, Callout-Edges, Typ-Defaults.
|
||||||
|
- „Mitwachsendes“ Schema ohne Obsidian-Umstrukturierungen:
|
||||||
|
- Neues Wissen kann sofort erfasst werden,
|
||||||
|
- bestehende Notizen bleiben gültig (Virtual Schema Layer).
|
||||||
|
|
||||||
|
### 3.2 Mittelfristig (Nächste Schritte)
|
||||||
|
|
||||||
|
- **Interview-Assistent (WP-07):** Geführte Dialoge zur Erfassung neuer Notizen (Startbereit).
|
||||||
|
- mindnet erzeugt Vorschläge für neue Notes & Edges und bietet einen „Vernetzungs-Assistenten“ für manuell angelegte Notizen.
|
||||||
|
- Agenten können über MCP-Tools (`mindnet_query`, `mindnet_chat`) auf mindnet zugreifen.
|
||||||
|
- Self-Tuning optimiert die Gewichte in `retriever.yaml` basierend auf dem gesammelten Feedback (WP-08).
|
||||||
|
|
||||||
|
### 3.3 Langfristig
|
||||||
|
|
||||||
|
- KI-Zwilling, der deinen Stil, deine Werte und deine typische Denkweise repräsentiert.
|
||||||
|
- Weitgehend autonomes Self-Tuning der Wissensstruktur (auf Basis gespeicherter Feedback-Daten & Policies).
|
||||||
|
- Lebenslanges Gedächtnis, das
|
||||||
|
- konsistent bleibt,
|
||||||
|
- wächst,
|
||||||
|
- sich weiterentwickelt,
|
||||||
|
- und deine Geschichte für deine Nachkommen erzählbar und erlebbar macht.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Architekturprinzipien
|
||||||
|
|
||||||
|
Die folgenden Prinzipien steuern alle Workpackages und Entscheidungen:
|
||||||
|
|
||||||
|
1. **Late Binding (späte Semantik)**
|
||||||
|
- Struktur und Interpretation werden in Konfigurationen (z. B. `types.yaml`, `prompts.yaml`, `decision_engine.yaml`) definiert, nicht direkt in den Vault-Dateien.
|
||||||
|
- Die "Persönlichkeit" des Chats ist ein Prompt-Template, kein Code.
|
||||||
|
|
||||||
|
2. **Virtual Schema Layer**
|
||||||
|
- Typen, Regeln, Policies, Edge-Defaults werden im Schema-Layer beschrieben.
|
||||||
|
- Markdown-Dateien benötigen nur minimale, robuste Angaben (Titel, Typ, optionale Properties).
|
||||||
|
|
||||||
|
3. **Self-Healing Graph**
|
||||||
|
- Der Graph wird regelmäßig analysiert (Edges, Centrality, fehlende Links).
|
||||||
|
- Automatisierte Jobs ergänzen fehlende Kanten.
|
||||||
|
|
||||||
|
4. **Deterministische IDs**
|
||||||
|
- Notes, Chunks und Edges erhalten stabile IDs (z. B. UUIDv5).
|
||||||
|
- Der Graph ist jederzeit wiederaufbaubar (Import-Pipeline idempotent).
|
||||||
|
|
||||||
|
5. **Full Explainability & Context Enrichment**
|
||||||
|
- Jeder Score, jede Beziehung muss nachvollziehbar sein.
|
||||||
|
- Dem LLM werden Metadaten (`[DECISION]`, `[PROJECT]`) injiziert, um Halluzinationen zu vermeiden und logische Schlüsse zu ermöglichen.
|
||||||
|
|
||||||
|
6. **Persistence First**
|
||||||
|
- Obsidian bleibt die primäre Quelle der Wahrheit.
|
||||||
|
- mindnet ergänzt, schlägt Änderungen vor, schreibt aber nur kontrolliert.
|
||||||
|
|
||||||
|
7. **Minimalinvasives Schreiben**
|
||||||
|
- Automatische Veränderungen an Markdown-Dateien erfolgen ausschließlich nach expliziter Zustimmung.
|
||||||
|
|
||||||
|
8. **Incremental Growth & Early Value**
|
||||||
|
- Das System muss bereits mit wenigen Notizen und einem kleinen Vault sinnvolle Antworten liefern.
|
||||||
|
- Feedback-Mechanismen werden früh eingeführt.
|
||||||
|
|
||||||
|
9. **Observability & Testbarkeit**
|
||||||
|
- Jeder Importlauf, jede Retriever-Anfrage und jede Policy-Änderung soll prüfbar sein.
|
||||||
|
|
||||||
|
10. **Local First & Privacy**
|
||||||
|
- Nutzung lokaler LLMs (Ollama/Phi-3) für Inference. Keine Daten verlassen den Server.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Programmstruktur (Phasenmodell)
|
||||||
|
|
||||||
|
Phase A – Fundament & Import (Fertig)
|
||||||
|
Phase B – Semantik, Graph & Lernen (Fertig)
|
||||||
|
Phase C – Persönlichkeitsmodell & KI-Zwilling (Fertig)
|
||||||
|
Phase D – Agenten, MCP & Interaktion (Laufend)
|
||||||
|
Phase E – Review, Refactoring, Dokumentation
|
||||||
|
|
||||||
|
Alle Workpackages sind einer Phase zugeordnet. WP-01 bis WP-06 und WP-10 sind bereits erfolgreich abgeschlossen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Workpackages – detaillierte Übersicht
|
||||||
|
|
||||||
|
### Legende Aufwand / Komplexität
|
||||||
|
|
||||||
|
- Aufwand: Niedrig / Mittel / Hoch
|
||||||
|
- Komplexität: Niedrig / Mittel / Hoch
|
||||||
|
|
||||||
|
Die Status-Ampel zeigt den aktuellen Stand gemäß Programmfortschritt.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-01 – Wissensdesign (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** A
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Grundlegendes Schema für mindnet festlegen (`knowledge_design.md`), inkl.:
|
||||||
|
- Note-Typen (z. B. `concept`, `experience`, `project`, `decision`, `value`).
|
||||||
|
- Standard-Properties und Grundstruktur von Edges.
|
||||||
|
- Abbildung zentraler Lebens-Bereiche.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- `knowledge_design.md` und `types.yaml` definieren die Zielarchitektur.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-02 – Chunking & Hash-Strategie (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** A
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Festlegen und implementieren einer robusten Chunking-Strategie und einer Hash-Logik zur Änderungserkennung.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- Implementierte Chunker-Logik mit Overlap.
|
||||||
|
- Stabile Hash-Strategie zur sicheren Änderungserkennung.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-03 – Import-Pipeline & Edge-System v2 (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** A/B
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Durchgängige Import-Pipeline von Markdown → Notes/Chunks/Edges in Qdrant mit deterministischen IDs.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- Vollständige E2E-Import-Pipeline.
|
||||||
|
- Vollständiges Edge-Modell (Strukturkanten, Wikilinks, Inline-Relationen).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-04a – Retriever & Graph-Scoring (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** B
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Aufbau eines hybriden Retrievers, der Semantik, Typwissen und Graph-Informationen kombiniert.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- Hybrides Scoring-Modell (Semantic + Edge + Centrality).
|
||||||
|
- Konfiguration über `retriever.yaml`.
|
||||||
|
- `/query` Endpoint aktiv.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-04b – Explanation Layer ("Why-Layer") (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** B
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Treffererklärungen liefern, die verständlich machen, warum ein Ergebnis gewählt wurde.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- **DTO-Erweiterung:** `QueryHit` enthält `explanation`, `breakdown` und `reason`.
|
||||||
|
- **Logic:** Graph-Adapter unterstützt `incoming_edges` (Reverse Lookup) für Authority-Erklärungen.
|
||||||
|
- **Output:** `/query` Endpoint liefert auf Wunsch (`explain: true`) menschenlesbare Begründungen ("Verweist auf...").
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-04c – Feedback Logging & Bewertungsdaten (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** B
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Grundlage für Self-Tuning schaffen durch systematisches Logging.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- **Data Flywheel:** Logging von Queries und User-Feedback in lokalen JSONL-Dateien.
|
||||||
|
- **Search Log:** Speichert Query + Score-Snapshot in `data/logs/search_history.jsonl`.
|
||||||
|
- **Feedback Log:** Speichert User-Ratings in `data/logs/feedback.jsonl`.
|
||||||
|
- **Traceability:** Durchgängige `query_id` von Request bis Feedback.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-05 – Persönlichkeitsmodell & RAG-Chat (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** C
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Aufbau eines RAG-Systems, das in natürlicher Sprache antwortet und Kontext versteht.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- **Infrastruktur:** Integration von Ollama (Phi-3 Mini) in den Service-Layer.
|
||||||
|
- **Router:** `/chat` Endpoint mit "Hybrid Retrieval Enforcement".
|
||||||
|
- **Context Enrichment:** Injection von Metadaten (`[TYPE]`, `[SCORE]`) in den Prompt, damit kleine Modelle (SLMs) komplexe Zusammenhänge ("Warum?") verstehen.
|
||||||
|
- **Config:** `prompts.yaml` steuert System-Prompt und RAG-Template.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-05b – Advanced Chat (Optional)
|
||||||
|
|
||||||
|
**Phase:** C
|
||||||
|
**Status:** ⚪ optional
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Erweiterung des Chats um Gedächtnis (History) und einfache Tools.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Implementierung von `ChatHistory` (vergangene Nachrichten in den Kontext).
|
||||||
|
- Einfache Tool-Nutzung (z.B. "Suche in Web").
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-06 – Decision Engine & Hybrid Router (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** C
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Transformation vom reinen Wissens-Abrufer zum strategischen Entscheidungspartner durch Intent-Erkennung.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- **Hybrid Intent Router:** Kombination aus schnellem Keyword-Matching und intelligentem LLM-Fallback zur Erkennung der Absicht (`DECISION`, `EMPATHY`, `FACT`, `CODING`).
|
||||||
|
- **Strategic Retrieval:** Gezieltes Nachladen von Werten (`value`), Zielen (`goal`) oder Erfahrungen (`experience`) basierend auf dem Intent.
|
||||||
|
- **Multi-Persona:** Dynamische Anpassung des Tonfalls (Berater vs. Spiegel vs. Techniker) durch `prompts.yaml`.
|
||||||
|
- **Late Binding:** Vollständige Konfiguration via `decision_engine.yaml`.
|
||||||
|
- **Robustheit:** Konfigurierbare Timeouts für CPU-Inference.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Hoch
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-07 – Interview-Assistent (geplant)
|
||||||
|
|
||||||
|
**Phase:** C/D
|
||||||
|
**Status:** 🟡 geplant (Nächster Fokus)
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Dialogbasierter Erfassungs-Assistent, der strukturierte Interviews führt und daraus konsistente Markdown-Notizen generiert.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Design von Interview-Flows (via Router-Strategie `INTERVIEW`).
|
||||||
|
- Konvertierung von Dialogtranskripten in typisierte Notes (Draft-Erstellung).
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Niedrig/Mittel
|
||||||
|
- Komplexität: Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-08 – Self-Tuning v1/v2 (geplant)
|
||||||
|
|
||||||
|
**Phase:** B/C
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Aufbau eines Self-Tuning-Mechanismus, der auf Basis von Feedback-Daten (WP-04c) Vorschläge für Retriever- und Policy-Anpassungen macht.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Auswertung der JSONL-Feedback-Daten.
|
||||||
|
- Regel-basierte Anpassungs-Vorschläge für `retriever.yaml` und Typ-Prioritäten.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Hoch
|
||||||
|
- Komplexität: Hoch
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-09 – Vault-Onboarding & Migration (geplant)
|
||||||
|
|
||||||
|
**Phase:** B
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Sicherstellen, dass bestehende und neue Obsidian-Vaults schrittweise in mindnet integriert werden können – ohne Massenumbau.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- Tools zur Analyse des Vault-Status.
|
||||||
|
- Empfehlungen für minimale Anpassungen.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Niedrig/Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-10 – Chat-Interface & Writeback (abgeschlossen)
|
||||||
|
|
||||||
|
**Phase:** D
|
||||||
|
**Status:** 🟢 abgeschlossen
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Ablösung der Terminal-Interaktion durch ein grafisches Interface.
|
||||||
|
|
||||||
|
**Erreichte Ergebnisse:**
|
||||||
|
- **Tech-Stack:** Streamlit Frontend (`app/frontend/ui.py`).
|
||||||
|
- **Funktionen:** Chat-Verlauf, Visualisierung von Intents und Quellen (Expander).
|
||||||
|
- **Feedback:** Globales Rating (Sterne) und granulare Quellen-Bewertung (Faces).
|
||||||
|
- **Writeback:** Mockup-Interface für neue Einträge (Vorbereitung WP-07).
|
||||||
|
- **Deployment:** Systemd-Services für Prod (8501) und Dev (8502).
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Hoch
|
||||||
|
- Komplexität: Hoch
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-10a – GUI Evolution: Interaktion & Tools (geplant)
|
||||||
|
|
||||||
|
**Phase:** D
|
||||||
|
**Status:** 🟡 geplant (nach WP07/11)
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Anpassung der GUI an komplexe Interaktionsmuster, die durch den Interview-Assistenten und Knowledge-Builder entstehen.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- **Draft-Editor:** Interaktive Bearbeitung und Bestätigung der Markdown-Entwürfe aus WP-07.
|
||||||
|
- **Suggestion-UI:** Checkboxen/Toggles für Kanten-Vorschläge aus WP-11 (Human-in-the-Loop).
|
||||||
|
- **Writeback:** Physisches Speichern der bestätigten Inhalte im Vault.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-11 – Knowledge-Builder & Vernetzungs-Assistent (geplant)
|
||||||
|
|
||||||
|
**Phase:** D
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Assistent, der manuell erstellte oder importierte Notizen analysiert und Vorschläge für Typen, Edges und Einordnung macht.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Hoch
|
||||||
|
- Komplexität: Hoch
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-12 – Knowledge Rewriter (Soft Mode, geplant)
|
||||||
|
|
||||||
|
**Phase:** C/D
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Optionaler Assistent, der vorhandene Notes „aufräumt“, zusammenfasst oder reorganisiert – ausschließlich mit expliziter Freigabe.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Niedrig/Mittel
|
||||||
|
- Komplexität: Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-13 – MCP-Integration & Agenten-Layer (geplant)
|
||||||
|
|
||||||
|
**Phase:** D
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
mindnet als MCP-Server bereitstellen, damit Agenten/LLMs standardisierte Tools nutzen können.
|
||||||
|
|
||||||
|
**Umfang:**
|
||||||
|
- MCP-Server mit Tools (`mindnet_query`, `mindnet_explain`, etc.).
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### WP-14 – Review / Refactoring / Dokumentation (geplant)
|
||||||
|
|
||||||
|
**Phase:** E
|
||||||
|
**Status:** 🟡 geplant
|
||||||
|
|
||||||
|
**Ziel:**
|
||||||
|
Aufräumen, dokumentieren, stabilisieren – insbesondere für Onboarding Dritter und Langfrist-Betrieb.
|
||||||
|
|
||||||
|
**Aufwand / Komplexität:**
|
||||||
|
- Aufwand: Mittel
|
||||||
|
- Komplexität: Niedrig/Mittel
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Abhängigkeiten (vereinfacht, aktualisiert)
|
||||||
|
|
||||||
|
WP01 → WP02 → WP03 → WP04a
|
||||||
|
WP04a → WP04b → WP04c → WP08
|
||||||
|
WP03 → WP05 → WP06 → WP07
|
||||||
|
WP07/WP11 → WP10a
|
||||||
|
WP03 → WP09
|
||||||
|
WP01/WP03 → WP10 → WP11 → WP12
|
||||||
|
WP03/WP04 → WP13
|
||||||
|
Alles → WP14
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Laufzeit- & Komplexitätsindikatoren (aktualisiert)
|
||||||
|
|
||||||
|
| WP | Aufwand | Komplexität |
|
||||||
|
|-------|----------------|------------------|
|
||||||
|
| WP04b | Mittel | Hoch |
|
||||||
|
| WP04c | Mittel | Mittel/Hoch |
|
||||||
|
| WP05 | Mittel | Mittel |
|
||||||
|
| WP06 | Mittel | Hoch |
|
||||||
|
| WP07 | Niedrig/Mittel | Mittel |
|
||||||
|
| WP08 | Hoch | Hoch |
|
||||||
|
| WP09 | Mittel | Niedrig/Mittel |
|
||||||
|
| WP10 | Hoch | Hoch |
|
||||||
|
| WP10a | Mittel | Mittel |
|
||||||
|
| WP11 | Hoch | Hoch |
|
||||||
|
| WP12 | Niedrig/Mittel | Mittel |
|
||||||
|
| WP13 | Mittel | Mittel |
|
||||||
|
| WP14 | Mittel | Niedrig/Mittel |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Programmfortschritt (Ampel, aktualisiert)
|
||||||
|
|
||||||
|
| WP | Status |
|
||||||
|
|-------|--------|
|
||||||
|
| WP01 | 🟢 |
|
||||||
|
| WP02 | 🟢 |
|
||||||
|
| WP03 | 🟢 |
|
||||||
|
| WP04a | 🟢 |
|
||||||
|
| WP04b | 🟢 |
|
||||||
|
| WP04c | 🟢 |
|
||||||
|
| WP05 | 🟢 |
|
||||||
|
| WP05b | ⚪ |
|
||||||
|
| WP06 | 🟢 |
|
||||||
|
| WP07 | 🟡 |
|
||||||
|
| WP08 | 🟡 |
|
||||||
|
| WP09 | 🟡 |
|
||||||
|
| WP10 | 🟢 |
|
||||||
|
| WP10a | 🟡 |
|
||||||
|
| WP11 | 🟡 |
|
||||||
|
| WP12 | 🟡 |
|
||||||
|
| WP13 | 🟡 |
|
||||||
|
| WP14 | 🟡 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Governance & Versionierung
|
||||||
|
|
||||||
|
- Versionierung über Gitea (Branches, Tags, Releases).
|
||||||
|
- Feature-Branch Workflow mit "Sync First" Strategie.
|
||||||
|
- Jede wesentliche Änderung an Architektur/Schema erhält einen eigenen Commit.
|
||||||
|
- Jedes Workpackage erhält ein eigenes Chat-Fenster mit dediziertem Prompt (WP-Hand-Over).
|
||||||
|
- Programmleitung verantwortet Konsistenz und Priorisierung.
|
||||||
|
- **Modell-Governance:** Das verwendete LLM (z.B. `phi3:mini`) wird in der `.env` und `config.py` festgeschrieben. Updates erfordern Tests gegen die "Why-Layer" Referenzfragen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. Executive Summary
|
||||||
|
|
||||||
|
mindnet v2.2 ist so aufgesetzt, dass:
|
||||||
|
|
||||||
|
- du **schrittweise** Wissen erfassen kannst (Obsidian + Chat + später Interview-Assistent),
|
||||||
|
- die Struktur **mitwächst**, ohne Retro-Massenarbeit im Vault,
|
||||||
|
- ein **hybrider Retriever** qualitativ hochwertige, erklärbare Antworten liefert,
|
||||||
|
- ein **Self-Healing- und Self-Tuning-Mechanismus** vorbereitet ist (durch WP-04c Feedback-Daten),
|
||||||
|
- ein **Persönlichkeitsmodell** (Decision Engine, Empathie) existiert und den Tonfall situativ anpasst,
|
||||||
|
- eine **grafische Oberfläche** (WP-10) existiert, die komplexe Zusammenhänge (Intents, Quellen) visualisiert,
|
||||||
|
- langfristig ein **KI-Zwilling** aufgebaut wird, der deine Werte, Erfahrungen und Denkweise spiegelt,
|
||||||
|
- die technische Architektur (FastAPI, Qdrant, YAML-Policies, MCP-Integration) lokal, nachvollziehbar und erweiterbar bleibt.
|
||||||
|
|
||||||
|
Dieser Programmplan bildet die konsolidierte Grundlage (v2.3.1) für alle weiteren Arbeiten.
|
||||||
@@ -0,0 +1,307 @@
|
|||||||
|
# mindnet v2 — Dokumentationsreview (vollständig & konsolidiert)
|
||||||
|
|
||||||
|
Dieses Dokument fasst alle notwendigen Anpassungen der mindnet-v2 Dokumentation zusammen.
|
||||||
|
Der Fokus liegt ausschließlich auf Aspekten, die durch die aktuellen Arbeiten verändert wurden:
|
||||||
|
|
||||||
|
- neue Edge-Mechanik
|
||||||
|
- neuer Importer
|
||||||
|
- neues Verhalten von note_payload & chunk_payload
|
||||||
|
- Default-Typregeln aus `types.yaml`
|
||||||
|
- Inline- und Callout-Relations
|
||||||
|
- WP04a – Einführung der Retriever-Schicht
|
||||||
|
- neue Diagnose- und Testskripte
|
||||||
|
|
||||||
|
Es handelt sich um eine strukturelle Analyse.
|
||||||
|
Nichts davon überschreibt bestehende Inhalte — es markiert nur, **was angepasst werden muss**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 1) knowledge_design.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 1.1 Neue Edge-Arten dokumentieren
|
||||||
|
Es müssen vier neue Kategorien beschrieben werden:
|
||||||
|
|
||||||
|
1. **Inline-Edges**
|
||||||
|
|
||||||
|
Beispiel:
|
||||||
|
`[ [rel:depends_on Embeddings 101] ]`
|
||||||
|
(Leerzeichen hier zur Demonstration; im echten Dokument ohne Leerzeichen)
|
||||||
|
|
||||||
|
2. **Callout-Edges**
|
||||||
|
|
||||||
|
> [!edge] related_to: [[Vector DB Basics]] [[Embeddings 101]]
|
||||||
|
|
||||||
|
3. **Typbasierte Default-Kanten**
|
||||||
|
Aus `types.yaml` → `edge_defaults`
|
||||||
|
|
||||||
|
4. **Strukturkanten**
|
||||||
|
- belongs_to
|
||||||
|
- next
|
||||||
|
- prev
|
||||||
|
|
||||||
|
### 1.2 Neue Edge-Felder ergänzen
|
||||||
|
Jeder Edge enthält jetzt:
|
||||||
|
|
||||||
|
- `rule_id`
|
||||||
|
- `confidence`
|
||||||
|
- `scope = "chunk"`
|
||||||
|
- deterministischer `edge_id`
|
||||||
|
|
||||||
|
Diese Felder müssen vollständig erklärt werden.
|
||||||
|
|
||||||
|
### 1.3 Chunk-basierte Kanten
|
||||||
|
Neu: **Alle Kanten entstehen ausschließlich zwischen Chunks**, nie zwischen Notes.
|
||||||
|
Notes dienen nur noch als Metadatencontainer.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 2) TYPE_REGISTRY_MANUAL.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 2.1 Typauflösung dokumentieren
|
||||||
|
Die Regeln sind jetzt:
|
||||||
|
|
||||||
|
- Typ aus Frontmatter → höchste Priorität
|
||||||
|
- Falls nicht gesetzt: Typ aus Dateipfad ableiten
|
||||||
|
- Falls unbekannt: Fallback auf `types.default`
|
||||||
|
- Typ definiert:
|
||||||
|
- retriever_weight
|
||||||
|
- chunk_profile
|
||||||
|
- edge_defaults
|
||||||
|
|
||||||
|
Diese müssen klar dokumentiert werden.
|
||||||
|
|
||||||
|
### 2.2 Neue Fallback-Regeln
|
||||||
|
Wenn Werte fehlen, werden sie ersetzt durch:
|
||||||
|
|
||||||
|
types.default.retriever_weight
|
||||||
|
types.default.chunk_profile
|
||||||
|
types.default.edge_defaults
|
||||||
|
|
||||||
|
### 2.3 Beispiele aktualisieren
|
||||||
|
Alle Beispiele müssen auf die neue Inline-Syntax angepasst werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 3) wp04_retriever_scoring.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 3.1 Neue Scoring-Formel
|
||||||
|
total_score =
|
||||||
|
semantic_weight * semantic_score
|
||||||
|
+ edge_weight * edge_bonus
|
||||||
|
+ centrality_weight * centrality_bonus
|
||||||
|
+ type_weight * retriever_weight
|
||||||
|
|
||||||
|
### 3.2 Bedeutung von `confidence`
|
||||||
|
confidence wirkt multiplikativ bei edge_bonus.
|
||||||
|
Inline > Callout > Default (Standardwerte im Code hinterlegt).
|
||||||
|
|
||||||
|
### 3.3 Pfadbegründungen
|
||||||
|
Der Retriever gibt optional `paths[]` zurück:
|
||||||
|
|
||||||
|
paths: [
|
||||||
|
{
|
||||||
|
"via": "depends_on",
|
||||||
|
"confidence": 0.7,
|
||||||
|
"hops": ["Note A", "Chunk A#01", "Note B"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
Diese Funktion müssen dokumentiert werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 4) mindnet_v2_implementation_playbook.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 4.1 Neuer Import-Prozess vollständig dokumentieren
|
||||||
|
Der Importer läuft jetzt so:
|
||||||
|
|
||||||
|
1. Markdown lesen
|
||||||
|
2. Frontmatter extrahieren
|
||||||
|
3. Typauflösung via `types.yaml`
|
||||||
|
4. Note-Payload generieren
|
||||||
|
5. Chunking anwenden
|
||||||
|
6. Inline-Kanten finden
|
||||||
|
7. Callout-Kanten finden
|
||||||
|
8. Default-Edges erzeugen
|
||||||
|
9. Strukturkanten erzeugen
|
||||||
|
10. Chunks in Qdrant upserten
|
||||||
|
11. Edges upserten
|
||||||
|
12. Diagnose laufen lassen:
|
||||||
|
|
||||||
|
python3 -m scripts.payload_dryrun
|
||||||
|
python3 -m scripts.edges_full_check
|
||||||
|
|
||||||
|
### 4.2 Erweiterte Testpipeline
|
||||||
|
Neu zu dokumentieren:
|
||||||
|
|
||||||
|
tests.test_edges_all
|
||||||
|
scripts.edges_full_check
|
||||||
|
scripts.payload_dryrun
|
||||||
|
|
||||||
|
Diese Testschritte sind jetzt Standardanforderung nach jeder Änderung.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 5) mindnet_technical_architecture.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 5.1 Aktualisiertes Schema für Qdrant Collections
|
||||||
|
|
||||||
|
#### Notes:
|
||||||
|
note_id
|
||||||
|
title
|
||||||
|
type
|
||||||
|
fulltext
|
||||||
|
retriever_weight
|
||||||
|
chunk_profile
|
||||||
|
edge_defaults
|
||||||
|
tags
|
||||||
|
updated
|
||||||
|
|
||||||
|
#### Chunks:
|
||||||
|
chunk_id
|
||||||
|
note_id
|
||||||
|
text
|
||||||
|
window
|
||||||
|
ord
|
||||||
|
retriever_weight
|
||||||
|
chunk_profile
|
||||||
|
neighbors_prev
|
||||||
|
neighbors_next
|
||||||
|
|
||||||
|
#### Edges:
|
||||||
|
edge_id
|
||||||
|
source_id
|
||||||
|
target_id
|
||||||
|
kind
|
||||||
|
scope = "chunk"
|
||||||
|
rule_id
|
||||||
|
confidence
|
||||||
|
|
||||||
|
### 5.2 Dokumentation der Edge-Pipeline
|
||||||
|
Neu beschreiben:
|
||||||
|
|
||||||
|
- Inline-Parser
|
||||||
|
- Callout-Parser
|
||||||
|
- Default-Edge-Resolver
|
||||||
|
- Strukturkanten
|
||||||
|
- Normalisierung für Qdrant
|
||||||
|
- deterministische ID-Bildung
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 6) mindnet_functional_architecture.md
|
||||||
|
## Erforderliche Anpassungen
|
||||||
|
|
||||||
|
### 6.1 Inline-Relations – einzig gültige Syntax
|
||||||
|
Nur diese Syntax erzeugt Kanten:
|
||||||
|
|
||||||
|
[[rel:depends_on Embeddings 101]]
|
||||||
|
|
||||||
|
Nicht unterstützt:
|
||||||
|
|
||||||
|
rel:depends_on [[Embeddings 101]]
|
||||||
|
|
||||||
|
Dies muss klar dokumentiert werden.
|
||||||
|
|
||||||
|
### 6.2 Prioritäten der Kantenerzeugung
|
||||||
|
Gültige Reihenfolge:
|
||||||
|
|
||||||
|
1. Inline
|
||||||
|
2. Callout
|
||||||
|
3. Typdefaults
|
||||||
|
4. Strukturkanten
|
||||||
|
|
||||||
|
### 6.3 Relaunch der Chunk-zentrierten Funktionslogik
|
||||||
|
Alle Beispiele müssen angepasst werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 7) chunking_strategy.md
|
||||||
|
## Erforderliche Änderungen
|
||||||
|
|
||||||
|
### 7.1 Neues Verhalten der Chunks
|
||||||
|
- `ord` ist jetzt Sortierkriterium
|
||||||
|
- `window` wird über chunk_config erzeugt
|
||||||
|
- Chunkprofile kommen ausschließlich aus `types.yaml`
|
||||||
|
|
||||||
|
### 7.2 Best Practices erneuern
|
||||||
|
Empfehlung:
|
||||||
|
|
||||||
|
- keine manuelle Chunkprofile mehr im Frontmatter setzen
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 8) docs_mindnet_retriever.md
|
||||||
|
## Erforderliche Änderungen
|
||||||
|
|
||||||
|
### 8.1 Neue Ausgabeparameter dokumentieren
|
||||||
|
Retriever liefert jetzt:
|
||||||
|
|
||||||
|
semantic_score
|
||||||
|
edge_bonus
|
||||||
|
centrality_bonus
|
||||||
|
type_score
|
||||||
|
total_score
|
||||||
|
paths[]
|
||||||
|
|
||||||
|
### 8.2 Graph-Expansion
|
||||||
|
Dokumentieren:
|
||||||
|
|
||||||
|
- Seed-Auswahl
|
||||||
|
- Depth-Limit
|
||||||
|
- Edge-Filterung
|
||||||
|
- Wirkung von `confidence` auf edge_bonus
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 9) Handbuch.md
|
||||||
|
## Erforderliche Änderungen
|
||||||
|
|
||||||
|
### 9.1 Neuer Import-Workflow
|
||||||
|
Erläuterung der Parser:
|
||||||
|
|
||||||
|
- Inline-Parser
|
||||||
|
- Callout-Parser
|
||||||
|
- Default-Kanten
|
||||||
|
- Struktur-Kanten
|
||||||
|
|
||||||
|
### 9.2 Best Practices
|
||||||
|
Ergänzen:
|
||||||
|
|
||||||
|
- Typenpflege erfolgt nur in `types.yaml`
|
||||||
|
- Tests immer nach Änderungen ausführen
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Gesamtübersicht der notwendigen Anpassungen
|
||||||
|
|
||||||
|
| Dokument | Anpassungsbedarf |
|
||||||
|
|---------|------------------|
|
||||||
|
| knowledge_design.md | neue Edge-Arten, Felder, Chunk-Edges |
|
||||||
|
| TYPE_REGISTRY_MANUAL.md | Typauflösung, neue Felder, neue Fallbacks |
|
||||||
|
| wp04_retriever_scoring.md | neue Score-Formel, Pfade, confidence |
|
||||||
|
| implementation_playbook.md | neuer Importprozess, Tests |
|
||||||
|
| technical_architecture.md | aktualisierte Schemas |
|
||||||
|
| functional_architecture.md | Syntax + Prioritäten |
|
||||||
|
| chunking_strategy.md | ord, window, neue Profile |
|
||||||
|
| docs_mindnet_retriever.md | neue API-Parameter |
|
||||||
|
| Handbuch.md | Parser + Best Practices |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Empfehlung zur Umsetzung
|
||||||
|
1. Dieses Review-Dokument als Master-Referenz verwenden.
|
||||||
|
2. Jedes Dokument strikt nur an den beschriebenen Stellen anpassen.
|
||||||
|
3. Überprüfen, ob Wann/Wo Typen, Kanten und Parser erklärt werden.
|
||||||
|
4. Nach jeder Anpassung Testlauf:
|
||||||
|
|
||||||
|
python3 -m scripts.payload_dryrun
|
||||||
|
python3 -m scripts.edges_full_check
|
||||||
|
|
||||||
|
5. Danach Playbook + Programmplan updaten.
|
||||||
|
|
||||||
@@ -0,0 +1,322 @@
|
|||||||
|
# mindnet – Änderungs- und Ergänzungsleitfaden für bestehende Dokumentation
|
||||||
|
# (Stand: Tag Wp04a-Retriever_final)
|
||||||
|
|
||||||
|
Dieser Leitfaden listet **alle notwendigen Änderungen und Ergänzungen** für die
|
||||||
|
bestehenden Dokumente auf. Er umfasst ausschließlich Inhalte, die durch die
|
||||||
|
tatsächliche Implementierung von WP-04a (Retriever, Hybrid Scoring, Graph-Integration,
|
||||||
|
retriever.yaml, Explainability-Vorbereitung, Centrality, Edge-Bonus, Query-API)
|
||||||
|
betroffen sind.
|
||||||
|
|
||||||
|
Er dient dazu, dass ein anderer Chat alle Dokumente vollständig und korrekt
|
||||||
|
überarbeiten kann, ohne Interpretationsspielraum oder fehlende Informationen.
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 1. Änderungen für: knowledge_design.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 1.1 Ergänzen: Rolle von retriever_weight pro Note-Type
|
||||||
|
Die Implementierung nutzt `retriever_weight`, das im Importer aus `types.yaml`
|
||||||
|
pro Note-Type übernommen und auf *Chunk-Ebene* vererbt wird.
|
||||||
|
|
||||||
|
**Ergänzen:**
|
||||||
|
- Erklärung, dass `retriever_weight` ein konzeptionelles Typ-Attribut ist,
|
||||||
|
das im hybriden Scoring als Multiplikator wirkt.
|
||||||
|
- Klarstellung, dass `retriever_weight` *nicht* durch Markdown-Frontmatter
|
||||||
|
überschrieben werden muss.
|
||||||
|
- Dokumentation, dass Qdrant-Payload dieses Feld für Notes und Chunks enthält.
|
||||||
|
|
||||||
|
## 1.2 Ergänzen: Edge-Typen, die im Retriever genutzt werden
|
||||||
|
Der Retriever unterstützt aktuell:
|
||||||
|
- `references`
|
||||||
|
- `related_to`
|
||||||
|
- `depends_on`
|
||||||
|
- `similar_to`
|
||||||
|
- `belongs_to`
|
||||||
|
- sowie generierte Strukturkanten `next` / `prev`.
|
||||||
|
|
||||||
|
**Ergänzen:**
|
||||||
|
- Welche Edges für den Graph-Bonus ausgewertet werden.
|
||||||
|
- Wie `confidence` aus mindnet_edges genutzt wird.
|
||||||
|
|
||||||
|
## 1.3 Ergänzen: Definition des lokalen Subgraphen
|
||||||
|
Die reale Retrieval-Logik nutzt:
|
||||||
|
- Seed-Notes (aus semantisch Top-K)
|
||||||
|
- Expansionstiefe `depth`
|
||||||
|
- Edge-Typ-Filter
|
||||||
|
- Bonusberechnung pro Knoten
|
||||||
|
|
||||||
|
Dies ist im ursprünglichen Dokument nicht beschrieben und muss hinzugefügt werden.
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 2. Änderungen für: chunking_strategy.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 2.1 Ergänzen: Chunk-Payload-Felder, die der Retriever nutzt
|
||||||
|
Die aktuelle Strategy beschreibt Chunks korrekt, es fehlt jedoch:
|
||||||
|
|
||||||
|
**Ergänzen:**
|
||||||
|
- `retriever_weight` (vererbt vom Note-Type)
|
||||||
|
- `chunk_profile` (aktuell ungenutzt im Retriever, aber Teil der Payload)
|
||||||
|
- Hinweise, dass der Retriever nur aus *Chunks* semantische Seeds bildet,
|
||||||
|
das Ranking aber auf *Notes* aggregiert wird.
|
||||||
|
|
||||||
|
## 2.2 Ergänzen: Section/Window-Felder und ihre Relevanz
|
||||||
|
Die Felder existieren im Qdrant, werden aber im retriever.yaml nicht verändert.
|
||||||
|
|
||||||
|
**Hinweis ergänzen:**
|
||||||
|
- Der Retriever nutzt aktuell `text` und `window`; kein Einfluss der Abschnittslogik.
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 3. Änderungen für: TYPE_REGISTRY_MANUAL.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
Dieses Dokument muss **aktualisiert** werden, da WP-04a folgende Mechanismen nutzt:
|
||||||
|
|
||||||
|
## 3.1 Ergänzen: Bedeutung von retriever_weight
|
||||||
|
- Definition als quantitatives Typ-Merkmal.
|
||||||
|
- Erklärung, dass es in Qdrant gespeichert wird.
|
||||||
|
- Erklärung des Einflusses auf Semantikscore:
|
||||||
|
- Formel:
|
||||||
|
|
||||||
|
final_semantic = semantic_score * retriever_weight
|
||||||
|
|
||||||
|
- Hinweis, dass in future WP-08 das Gewicht durch Feedback lernbar wird.
|
||||||
|
|
||||||
|
## 3.2 Ergänzen: Neue Edge-Typen und ihre Standard-Konfiguration
|
||||||
|
- `similar_to`
|
||||||
|
- `belongs_to`
|
||||||
|
- Strukturkanten `next` / `prev`
|
||||||
|
|
||||||
|
Diese sind im Importer bereits aktiv, aber nicht ausreichend dokumentiert.
|
||||||
|
|
||||||
|
## 3.3 Ergänzen: Edge-Konfidenzen
|
||||||
|
Die automatische Edge-Ableitung generiert `confidence`.
|
||||||
|
Retriever nutzt diesen Wert linear im Edge-Bonus.
|
||||||
|
|
||||||
|
**Dokumentation muss ergänzt werden.**
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 4. Änderungen für: docs_mindnet_retriever.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
Dieses Dokument ist am stärksten betroffen und muss auf den Stand der tatsächlichen
|
||||||
|
Implementierung aktualisiert werden.
|
||||||
|
|
||||||
|
## 4.1 Aktualisieren: Gesamtscoring-Formel
|
||||||
|
Bisher: Semantik + einfache Typgewichtung.
|
||||||
|
|
||||||
|
Neu implementiert:
|
||||||
|
|
||||||
|
total_score =
|
||||||
|
semantic_weight * semantic_score
|
||||||
|
+ edge_weight * edge_bonus
|
||||||
|
+ centrality_weight * centrality_bonus
|
||||||
|
|
||||||
|
Diese Formel muss vollständig dokumentiert werden.
|
||||||
|
|
||||||
|
## 4.2 Aktualisieren: Einführung retriever.yaml
|
||||||
|
Dokumentation benötigt:
|
||||||
|
|
||||||
|
- Speicherort: `app/config/retriever.yaml`
|
||||||
|
- Default-Werte
|
||||||
|
- Abschnittsweise Beschreibung:
|
||||||
|
- semantic_weight
|
||||||
|
- edge_weight
|
||||||
|
- centrality_weight
|
||||||
|
- Hinweis: Änderungen wirken ohne Codeänderung.
|
||||||
|
|
||||||
|
## 4.3 Ergänzen: Beschreibung Graph-Expansion
|
||||||
|
Der Retriever nutzt:
|
||||||
|
|
||||||
|
- Expand-depth (default 1)
|
||||||
|
- Edge-Typ-Filter
|
||||||
|
- Aggregation pro Node
|
||||||
|
- Bonus-Skalierung
|
||||||
|
|
||||||
|
Diese Logik ist neu und muss detailliert erklärt werden.
|
||||||
|
|
||||||
|
## 4.4 Ergänzen: Beschreibung centrality_bonus
|
||||||
|
Aktuell:
|
||||||
|
- proportionale Verteilung anhand Knotendichte
|
||||||
|
- Normalisierung über lokales Subgraph-Maß
|
||||||
|
|
||||||
|
Dokumentation muss Minimum enthalten:
|
||||||
|
- wie centrality gewichtet wird
|
||||||
|
- welche Grenze existieren
|
||||||
|
|
||||||
|
## 4.5 Ergänzen: Explainability-Schnittstelle (WP-04b Vorbereitung)
|
||||||
|
Die Implementierung erzeugt intern:
|
||||||
|
- edge_bonus pro Node
|
||||||
|
- centrality_bonus pro Node
|
||||||
|
- Liste der Edges, die beteiligt waren
|
||||||
|
|
||||||
|
Diese Werte sind abrufbar und müssen dokumentiert werden.
|
||||||
|
|
||||||
|
## 4.6 Ergänzen: Fehlertoleranzen & Timeout (Health Check)
|
||||||
|
Der Retriever kann:
|
||||||
|
- Fallback auf Vektor-Suche (pure semantic)
|
||||||
|
- Fallback bei fehlendem expand
|
||||||
|
|
||||||
|
Diese Sicherheitsmechanismen müssen dokumentiert werden.
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 5. Änderungen für: mindnet_technical_architecture.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 5.1 Ergänzen: Komponentenübersicht Retrieval-Layer
|
||||||
|
Eine neue Abbildung oder Abschnitt muss hinzu:
|
||||||
|
|
||||||
|
- QueryRequest-Parsing
|
||||||
|
- Embedding-Lookup oder Vector-Passthrough
|
||||||
|
- Semantic Seed Search (Qdrant)
|
||||||
|
- Graph-Expansion (mindnet_edges)
|
||||||
|
- Node-/Note-Aggregation
|
||||||
|
- Score-Combination
|
||||||
|
- Response-Ausgabe via FastAPI
|
||||||
|
|
||||||
|
## 5.2 Ergänzen: Datenstruktur mindnet_edges
|
||||||
|
Vor WP-04 nicht vollständig dokumentiert.
|
||||||
|
|
||||||
|
Benötigt:
|
||||||
|
|
||||||
|
- Felder:
|
||||||
|
- edge_id
|
||||||
|
- kind
|
||||||
|
- source_id
|
||||||
|
- target_id
|
||||||
|
- confidence
|
||||||
|
- rule_id
|
||||||
|
- Wie Edges im Retriever ausgewertet werden.
|
||||||
|
|
||||||
|
## 5.3 Ergänzen: retriever.yaml als Konfigurationsquelle
|
||||||
|
Architekturdiagramm muss updated werden:
|
||||||
|
- Retriever nutzt YAML
|
||||||
|
- Keine Hardcodierung im Python
|
||||||
|
|
||||||
|
## 5.4 Ergänzen: Health Check Script
|
||||||
|
Technischer Abschnitt muss dokumentieren:
|
||||||
|
|
||||||
|
- health_check_mindnet.py
|
||||||
|
- Semantik:
|
||||||
|
- semantic query
|
||||||
|
- hybrid query
|
||||||
|
- Fehlercodes
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 6. Änderungen für: mindnet_functional_architecture.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 6.1 Ergänzen: Funktionale Rolle des Retrievers
|
||||||
|
Retriever ist jetzt:
|
||||||
|
|
||||||
|
- Teil des Functional Layers
|
||||||
|
- Grundlage für Agenten / Decision Engine
|
||||||
|
- Hauptquelle für Begründungen
|
||||||
|
|
||||||
|
## 6.2 Ergänzen: Einbindung des Explainability-Layers
|
||||||
|
Funktional gehört dies zwischen:
|
||||||
|
- Retriever
|
||||||
|
- Antwortgenerierung (LLM)
|
||||||
|
|
||||||
|
Der Explainability-Layer formuliert Gründe:
|
||||||
|
- Welche Edges?
|
||||||
|
- Welche Scores?
|
||||||
|
- Warum diese Note?
|
||||||
|
|
||||||
|
## 6.3 Ergänzen: Self-Tuning als neuen Funktionsblock (WP-08)
|
||||||
|
Muss aufgenommen werden:
|
||||||
|
|
||||||
|
- Logging von Feedback
|
||||||
|
- Bewertung
|
||||||
|
- Tuning der retriever.yaml
|
||||||
|
- Human-in-the-loop Freigabe
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 7. Änderungen für: wp04_retriever_scoring.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
Dieses Dokument ist **direkt WP-04** und muss am stärksten überarbeitet werden.
|
||||||
|
|
||||||
|
## 7.1 Aktualisieren: Finale Scoring-Formel
|
||||||
|
Die Formel muss exakt so dokumentiert werden, wie inzwischen implementiert.
|
||||||
|
|
||||||
|
## 7.2 Ergänzen: Bonusberechnung
|
||||||
|
Die bisherige Planung wurde implementiert, aber der Text muss aktualisiert werden:
|
||||||
|
|
||||||
|
- Bonusberechnung je Edge
|
||||||
|
- confidence-Einfluss
|
||||||
|
- Node-Aggregation
|
||||||
|
- Note-Aggregation
|
||||||
|
|
||||||
|
## 7.3 Ergänzen: Centrality
|
||||||
|
Dokument ist zu ergänzen um:
|
||||||
|
|
||||||
|
- Definition
|
||||||
|
- Berechnung
|
||||||
|
- Normalisierung
|
||||||
|
- Gewichtung über retriever.yaml
|
||||||
|
|
||||||
|
## 7.4 Ergänzen: Teststrategie
|
||||||
|
Die finale Testbasis muss dokumentiert werden:
|
||||||
|
|
||||||
|
- Unit-Tests
|
||||||
|
- Hybrid-Tests
|
||||||
|
- API-Tests
|
||||||
|
- Smoke-Test
|
||||||
|
- Diagnose-Skripte
|
||||||
|
|
||||||
|
## 7.5 Ergänzen: Einschränkungen & Offene Punkte
|
||||||
|
Hinweis für WP-04b:
|
||||||
|
- Explainability fehlt noch
|
||||||
|
- Edge-Type Weights noch nicht implementiert
|
||||||
|
- Self-Tuning in WP-08
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 8. Änderungen für: mindnet_v2_implementation_playbook.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 8.1 Ergänzen: Arbeitsablauf mit Retriever
|
||||||
|
Einrichtung, Nutzung und Test sollte ergänzt werden:
|
||||||
|
|
||||||
|
- Starten von FastAPI
|
||||||
|
- Testen des Retrievers via health_check
|
||||||
|
- Import-Workflow (neue Vaults → Qdrant → Retriever)
|
||||||
|
|
||||||
|
## 8.2 Ergänzen: Konfigurationsverwaltung
|
||||||
|
Einbindung in Gitea-Workflows:
|
||||||
|
- retriever.yaml versionieren
|
||||||
|
- Tags/Branches für Funktionsstände setzen
|
||||||
|
|
||||||
|
## 8.3 Ergänzen: Self-Tuning-Prozess (nur Rahmen)
|
||||||
|
Der Playbook-Abschnitt soll die späteren Mechanismen vorbeschreiben.
|
||||||
|
|
||||||
|
|
||||||
|
============================================================
|
||||||
|
# 9. Änderungen für: Handbuch.md
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
## 9.1 Ergänzen: Einfache Erklärung des Retrievers
|
||||||
|
Das Handbuch benötigt eine verständliche, nicht-technische Beschreibung:
|
||||||
|
|
||||||
|
- Was macht der Retriever?
|
||||||
|
- Warum kommt ein Ergebnis zustande?
|
||||||
|
- Welche Rolle spielen Edges?
|
||||||
|
- Wie kann der Nutzer die Gewichtung beeinflussen (YAML)?
|
||||||
|
|
||||||
|
## 9.2 Ergänzen: Troubleshooting
|
||||||
|
Aufnehmen:
|
||||||
|
- „Ergebnisse wirken unpassend“ → Prüfe retriever.yaml
|
||||||
|
- „Edge-Bonus fehlt“ → Prüfe mindnet_edges
|
||||||
|
- health_check-Skript nutzen
|
||||||
|
|
||||||
|
## 9.3 Ergänzen: API-Nutzung für Endanwender
|
||||||
|
Einfacher Abschnitt:
|
||||||
|
- Wie man `/query` testet
|
||||||
|
- Beispiele für semantic & hybrid queries
|
||||||
@@ -33,3 +33,7 @@ python3 scripts/import_markdown.py --vault /path/to/Obsidian
|
|||||||
- `POST /qdrant/query` → semantic search over chunks with optional filters
|
- `POST /qdrant/query` → semantic search over chunks with optional filters
|
||||||
|
|
||||||
See `scripts/quick_test.sh` for a runnable example.
|
See `scripts/quick_test.sh` for a runnable example.
|
||||||
|
|
||||||
|
>Anmerkung:
|
||||||
|
Diese Datei ist veraltet
|
||||||
|
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
"""
|
||||||
|
app — mindnet API package
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Markiert 'app/' als Python-Paket, damit 'from app.main import create_app'
|
||||||
|
in Tests und Skripten funktioniert.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Hinweise:
|
||||||
|
Keine Logik – nur Paketinitialisierung.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
||||||
+22
-1
@@ -1,9 +1,11 @@
|
|||||||
"""
|
"""
|
||||||
Version 0.1
|
app/config.py — zentrale Konfiguration
|
||||||
|
Version: 0.4.0 (WP-06 Complete)
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import os
|
import os
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
class Settings:
|
class Settings:
|
||||||
# Qdrant
|
# Qdrant
|
||||||
@@ -12,10 +14,29 @@ class Settings:
|
|||||||
COLLECTION_PREFIX: str = os.getenv("MINDNET_PREFIX", "mindnet")
|
COLLECTION_PREFIX: str = os.getenv("MINDNET_PREFIX", "mindnet")
|
||||||
VECTOR_SIZE: int = int(os.getenv("MINDNET_VECTOR_SIZE", "384"))
|
VECTOR_SIZE: int = int(os.getenv("MINDNET_VECTOR_SIZE", "384"))
|
||||||
DISTANCE: str = os.getenv("MINDNET_DISTANCE", "Cosine")
|
DISTANCE: str = os.getenv("MINDNET_DISTANCE", "Cosine")
|
||||||
|
|
||||||
# Embeddings
|
# Embeddings
|
||||||
MODEL_NAME: str = os.getenv("MINDNET_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
|
MODEL_NAME: str = os.getenv("MINDNET_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
|
||||||
|
|
||||||
|
# WP-05 LLM / Ollama
|
||||||
|
OLLAMA_URL: str = os.getenv("MINDNET_OLLAMA_URL", "http://127.0.0.1:11434")
|
||||||
|
LLM_MODEL: str = os.getenv("MINDNET_LLM_MODEL", "phi3:mini")
|
||||||
|
PROMPTS_PATH: str = os.getenv("MINDNET_PROMPTS_PATH", "config/prompts.yaml")
|
||||||
|
|
||||||
|
# NEU für WP-06
|
||||||
|
LLM_TIMEOUT: float = float(os.getenv("MINDNET_LLM_TIMEOUT", "120.0"))
|
||||||
|
DECISION_CONFIG_PATH: str = os.getenv("MINDNET_DECISION_CONFIG", "config/decision_engine.yaml")
|
||||||
|
|
||||||
# API
|
# API
|
||||||
DEBUG: bool = os.getenv("DEBUG", "false").lower() == "true"
|
DEBUG: bool = os.getenv("DEBUG", "false").lower() == "true"
|
||||||
|
|
||||||
|
# WP-04 Retriever Defaults
|
||||||
|
RETRIEVER_W_SEM: float = float(os.getenv("MINDNET_WP04_W_SEM", "0.70"))
|
||||||
|
RETRIEVER_W_EDGE: float = float(os.getenv("MINDNET_WP04_W_EDGE", "0.25"))
|
||||||
|
RETRIEVER_W_CENT: float = float(os.getenv("MINDNET_WP04_W_CENT", "0.05"))
|
||||||
|
RETRIEVER_TOP_K: int = int(os.getenv("MINDNET_WP04_TOP_K", "10"))
|
||||||
|
RETRIEVER_EXPAND_DEPTH: int = int(os.getenv("MINDNET_WP04_EXPAND_DEPTH", "1"))
|
||||||
|
RETRIEVER_EDGE_TYPES: str = os.getenv("MINDNET_WP04_EDGE_TYPES", "references,belongs_to,prev,next")
|
||||||
|
|
||||||
@lru_cache
|
@lru_cache
|
||||||
def get_settings() -> Settings:
|
def get_settings() -> Settings:
|
||||||
|
|||||||
+105
-247
@@ -1,271 +1,129 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Modul: app/core/chunk_payload.py
|
app/core/chunk_payload.py (Mindnet V2 — types.yaml authoritative)
|
||||||
Version: 2.2.0
|
- neighbors_prev / neighbors_next sind Listen ([], [id]).
|
||||||
Datum: 2025-10-06
|
- retriever_weight / chunk_profile kommen aus types.yaml (Frontmatter wird ignoriert).
|
||||||
|
- Fallbacks: defaults.* in types.yaml; sonst 1.0 / "default".
|
||||||
Zweck
|
|
||||||
-----
|
|
||||||
Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.0.1.
|
|
||||||
Neu: Wenn der Chunker KEIN Overlap im Fenster liefert (== window fehlt / identisch zur Kernpassage),
|
|
||||||
erzeugen wir FENSTER mit synthetischem Overlap auf Basis chunk_config.get_sizes(note_type)['overlap'].
|
|
||||||
|
|
||||||
Felder (beibehalten aus 2.0.1):
|
|
||||||
- note_id, chunk_id, id (Alias), chunk_index, seq, path
|
|
||||||
- window (mit Overlap), text (ohne linkes Overlap)
|
|
||||||
- start, end (Offsets im gesamten Body)
|
|
||||||
- overlap_left, overlap_right
|
|
||||||
- token_count?, section_title?, section_path?, type?, title?, tags?
|
|
||||||
|
|
||||||
Kompatibilität:
|
|
||||||
- 'id' == 'chunk_id' als Alias
|
|
||||||
- Pfade bleiben relativ (keine führenden '/'), Backslashes → Slashes
|
|
||||||
- Robust für Chunk-Objekte oder Dicts; Fensterquelle: 'window'|'text'|'content'|'raw'
|
|
||||||
|
|
||||||
Lizenz: MIT (projektintern)
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
import os, yaml
|
||||||
|
|
||||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
def _env(n: str, d: Optional[str]=None) -> str:
|
||||||
|
v = os.getenv(n)
|
||||||
|
return v if v is not None else (d or "")
|
||||||
|
|
||||||
try:
|
def _load_types() -> dict:
|
||||||
# Typgerechtes Overlap aus deiner Konfiguration holen
|
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||||
from app.core.chunk_config import get_sizes as _get_sizes
|
|
||||||
except Exception:
|
|
||||||
def _get_sizes(_note_type: str):
|
|
||||||
# konservativer Default, falls Import fehlschlägt
|
|
||||||
return {"overlap": (40, 60), "target": (250, 350), "max": 500}
|
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------- Utils ------------------------------- #
|
|
||||||
|
|
||||||
def _get_attr_or_key(obj: Any, key: str, default=None):
|
|
||||||
if obj is None:
|
|
||||||
return default
|
|
||||||
if isinstance(obj, dict):
|
|
||||||
return obj.get(key, default)
|
|
||||||
return getattr(obj, key, default)
|
|
||||||
|
|
||||||
def _as_window_text(chunk: Any) -> str:
|
|
||||||
"""Fenstertext robust lesen (bevorzugt echte Fenster, sonst Kern)."""
|
|
||||||
for k in ("window", "text", "content", "raw"):
|
|
||||||
v = _get_attr_or_key(chunk, k, None)
|
|
||||||
if isinstance(v, str) and v:
|
|
||||||
return v
|
|
||||||
return ""
|
|
||||||
|
|
||||||
def _to_int(x: Any, default: int = 0) -> int:
|
|
||||||
try:
|
try:
|
||||||
return int(x)
|
with open(p, "r", encoding="utf-8") as f:
|
||||||
|
return yaml.safe_load(f) or {}
|
||||||
except Exception:
|
except Exception:
|
||||||
return default
|
return {}
|
||||||
|
|
||||||
def _normalize_rel_path(p: str) -> str:
|
def _get_types_map(reg: dict) -> dict:
|
||||||
p = (p or "").replace("\\", "/")
|
if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
|
||||||
while p.startswith("/"):
|
return reg["types"]
|
||||||
p = p[1:]
|
return reg if isinstance(reg, dict) else {}
|
||||||
return p
|
|
||||||
|
|
||||||
|
def _get_defaults(reg: dict) -> dict:
|
||||||
|
if isinstance(reg, dict) and isinstance(reg.get("defaults"), dict):
|
||||||
|
return reg["defaults"]
|
||||||
|
if isinstance(reg, dict) and isinstance(reg.get("global"), dict):
|
||||||
|
return reg["global"]
|
||||||
|
return {}
|
||||||
|
|
||||||
# ---------------------- Overlap & Offsets ---------------------------- #
|
def _as_float(x: Any):
|
||||||
|
try:
|
||||||
|
return float(x)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int], str]:
|
def _resolve_chunk_profile(note_type: str, reg: dict) -> str:
|
||||||
"""
|
types = _get_types_map(reg)
|
||||||
Entfernt linkes Overlap aus echten Fenster-Strings.
|
if isinstance(types, dict):
|
||||||
Rückgabe: (segments, overlaps_left, reconstructed_text)
|
t = types.get(note_type, {})
|
||||||
"""
|
if isinstance(t, dict) and isinstance(t.get("chunk_profile"), str):
|
||||||
segments: List[str] = []
|
return t["chunk_profile"]
|
||||||
overlaps_left: List[int] = []
|
defs = _get_defaults(reg)
|
||||||
reconstructed = ""
|
if isinstance(defs, dict) and isinstance(defs.get("chunk_profile"), str):
|
||||||
for w in windows:
|
return defs["chunk_profile"]
|
||||||
w = w or ""
|
return "default"
|
||||||
max_k = min(len(w), len(reconstructed))
|
|
||||||
k = 0
|
|
||||||
for cand in range(max_k, -1, -1):
|
|
||||||
if reconstructed.endswith(w[:cand]):
|
|
||||||
k = cand
|
|
||||||
break
|
|
||||||
seg = w[k:]
|
|
||||||
segments.append(seg)
|
|
||||||
overlaps_left.append(k)
|
|
||||||
reconstructed += seg
|
|
||||||
return segments, overlaps_left, reconstructed
|
|
||||||
|
|
||||||
def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
|
def _resolve_retriever_weight(note_type: str, reg: dict) -> float:
|
||||||
"""Länge längsten Suffix(a), der Prefix(b) ist."""
|
types = _get_types_map(reg)
|
||||||
if not a or not b:
|
if isinstance(types, dict):
|
||||||
return 0
|
t = types.get(note_type, {})
|
||||||
a1 = a[-max_probe:]
|
if isinstance(t, dict) and (t.get("retriever_weight") is not None):
|
||||||
b1 = b[:max_probe]
|
v = _as_float(t.get("retriever_weight"))
|
||||||
n = min(len(a1), len(b1))
|
if v is not None:
|
||||||
for k in range(n, 0, -1):
|
return float(v)
|
||||||
if a1[-k:] == b1[:k]:
|
defs = _get_defaults(reg)
|
||||||
return k
|
if isinstance(defs, dict) and (defs.get("retriever_weight") is not None):
|
||||||
return 0
|
v = _as_float(defs.get("retriever_weight"))
|
||||||
|
if v is not None:
|
||||||
|
return float(v)
|
||||||
|
return 1.0
|
||||||
|
|
||||||
|
def _as_list(x):
|
||||||
|
if x is None:
|
||||||
|
return []
|
||||||
|
if isinstance(x, list):
|
||||||
|
return x
|
||||||
|
return [x]
|
||||||
|
|
||||||
# ----------------------------- Public API ---------------------------- #
|
def make_chunk_payloads(note: Dict[str, Any],
|
||||||
|
note_path: str,
|
||||||
|
chunks_from_chunker: List[Any],
|
||||||
|
*,
|
||||||
|
note_text: str = "",
|
||||||
|
types_cfg: Optional[dict] = None,
|
||||||
|
file_path: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||||
|
fm = (note or {}).get("frontmatter", {}) or {}
|
||||||
|
note_type = fm.get("type") or note.get("type") or "concept"
|
||||||
|
reg = types_cfg if isinstance(types_cfg, dict) else _load_types()
|
||||||
|
|
||||||
def make_chunk_payloads(
|
# types.yaml authoritative
|
||||||
frontmatter: Dict[str, Any],
|
cp = _resolve_chunk_profile(note_type, reg)
|
||||||
rel_path: str,
|
rw = _resolve_retriever_weight(note_type, reg)
|
||||||
chunks: Iterable[Union[Dict[str, Any], Any]],
|
|
||||||
note_text: Optional[str] = None,
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""
|
|
||||||
Baut Payloads pro Chunk. Falls Fenster ohne Overlap geliefert werden,
|
|
||||||
erzeugen wir synthetische 'window'-Texte mit typgerechtem Overlap.
|
|
||||||
"""
|
|
||||||
note_id = str(frontmatter.get("id") or "").strip()
|
|
||||||
note_type = str(frontmatter.get("type", "")).lower()
|
|
||||||
note_title = frontmatter.get("title", None)
|
|
||||||
note_tags = frontmatter.get("tags", None)
|
|
||||||
rel_path = _normalize_rel_path(rel_path)
|
|
||||||
|
|
||||||
# 1) Rohdaten sammeln (so wie geliefert)
|
tags = fm.get("tags") or []
|
||||||
chunks_list = list(chunks)
|
if isinstance(tags, str):
|
||||||
raw_windows: List[str] = []
|
tags = [tags]
|
||||||
seqs: List[int] = []
|
|
||||||
ids_in: List[Optional[str]] = []
|
|
||||||
token_counts: List[Optional[int]] = []
|
|
||||||
section_titles: List[Optional[str]] = []
|
|
||||||
section_paths: List[Optional[str]] = []
|
|
||||||
any_explicit_window = False
|
|
||||||
|
|
||||||
for idx, c in enumerate(chunks_list):
|
out: List[Dict[str, Any]] = []
|
||||||
# Fensterquelle
|
for idx, ch in enumerate(chunks_from_chunker):
|
||||||
w = _get_attr_or_key(c, "window", None)
|
# Attribute oder Keys (Chunk-Objekt oder Dict)
|
||||||
if isinstance(w, str) and w:
|
cid = getattr(ch, "id", None) or (ch.get("id") if isinstance(ch, dict) else None)
|
||||||
any_explicit_window = True
|
nid = getattr(ch, "note_id", None) or (ch.get("note_id") if isinstance(ch, dict) else fm.get("id"))
|
||||||
raw_windows.append(w)
|
index = getattr(ch, "index", None) or (ch.get("index") if isinstance(ch, dict) else idx)
|
||||||
else:
|
text = getattr(ch, "text", None) or (ch.get("text") if isinstance(ch, dict) else "")
|
||||||
raw_windows.append(_as_window_text(c)) # 'text'|'content'|'raw' als Ersatz
|
window = getattr(ch, "window", None) or (ch.get("window") if isinstance(ch, dict) else text)
|
||||||
# Ordnung
|
prev_id = getattr(ch, "neighbors_prev", None) or (ch.get("neighbors_prev") if isinstance(ch, dict) else None)
|
||||||
seqs.append(_to_int(_get_attr_or_key(c, "seq", _get_attr_or_key(c, "chunk_index", idx)), idx))
|
next_id = getattr(ch, "neighbors_next", None) or (ch.get("neighbors_next") if isinstance(ch, dict) else None)
|
||||||
# IDs, Tokens, Sektionen
|
|
||||||
cid = _get_attr_or_key(c, "chunk_id", _get_attr_or_key(c, "id", None))
|
|
||||||
ids_in.append(str(cid) if isinstance(cid, str) and cid else None)
|
|
||||||
tc = _get_attr_or_key(c, "token_count", None)
|
|
||||||
token_counts.append(_to_int(tc, 0) if tc is not None else None)
|
|
||||||
section_titles.append(_get_attr_or_key(c, "section_title", None))
|
|
||||||
section_paths.append(_get_attr_or_key(c, "section_path", None))
|
|
||||||
|
|
||||||
# 2) Segmente & Overlaps bestimmen
|
|
||||||
if any_explicit_window:
|
|
||||||
# Es existieren echte Fenster → dedupe, um Kernsegmente zu finden
|
|
||||||
segments, overlaps_left, recon = _dedupe_windows_to_segments(raw_windows)
|
|
||||||
windows_final = raw_windows[:] # bereits mit Overlap geliefert
|
|
||||||
else:
|
|
||||||
# Keine echten Fenster → Segmente sind identisch zu "Fenstern" (bisher),
|
|
||||||
# wir erzeugen synthetische Fenster mit Overlap gemäß Typ
|
|
||||||
segments = [w or "" for w in raw_windows]
|
|
||||||
overlaps_left = []
|
|
||||||
windows_final = []
|
|
||||||
recon = ""
|
|
||||||
try:
|
|
||||||
overlap_low, overlap_high = tuple(_get_sizes(note_type).get("overlap", (40, 60)))
|
|
||||||
except Exception:
|
|
||||||
overlap_low, overlap_high = (40, 60)
|
|
||||||
overlap_target = int(overlap_low)
|
|
||||||
|
|
||||||
for i, seg in enumerate(segments):
|
|
||||||
if i == 0:
|
|
||||||
# erstes Fenster: kein linker Kontext
|
|
||||||
windows_final.append(seg)
|
|
||||||
overlaps_left.append(0)
|
|
||||||
recon += seg
|
|
||||||
else:
|
|
||||||
# synthetischer linker Kontext = Suffix des bisher rekonstruierten Texts
|
|
||||||
k = min(overlap_target, len(recon))
|
|
||||||
left_ctx = recon[-k:] if k > 0 else ""
|
|
||||||
windows_final.append(left_ctx + seg)
|
|
||||||
overlaps_left.append(k)
|
|
||||||
recon += seg # Rekonstruktion bleibt kerntreu
|
|
||||||
|
|
||||||
# 3) overlap_right bestimmen
|
|
||||||
overlaps_right: List[int] = []
|
|
||||||
for i in range(len(windows_final)):
|
|
||||||
if i + 1 < len(windows_final):
|
|
||||||
ov = _overlap_len_suffix_prefix(windows_final[i], windows_final[i + 1], max_probe=4096)
|
|
||||||
else:
|
|
||||||
ov = 0
|
|
||||||
overlaps_right.append(ov)
|
|
||||||
|
|
||||||
# 4) start/end-Offsets (exakt via note_text, sonst kumulativ)
|
|
||||||
starts: List[int] = [0] * len(segments)
|
|
||||||
ends: List[int] = [0] * len(segments)
|
|
||||||
pos = 0
|
|
||||||
if isinstance(note_text, str) and note_text:
|
|
||||||
search_pos = 0
|
|
||||||
for i, seg in enumerate(segments):
|
|
||||||
if not seg:
|
|
||||||
starts[i] = ends[i] = search_pos
|
|
||||||
continue
|
|
||||||
j = note_text.find(seg, search_pos)
|
|
||||||
if j >= 0:
|
|
||||||
starts[i] = j
|
|
||||||
ends[i] = j + len(seg)
|
|
||||||
search_pos = ends[i]
|
|
||||||
else:
|
|
||||||
# Fallback: kumulativ
|
|
||||||
starts[i] = pos
|
|
||||||
pos += len(seg)
|
|
||||||
ends[i] = pos
|
|
||||||
else:
|
|
||||||
for i, seg in enumerate(segments):
|
|
||||||
starts[i] = pos
|
|
||||||
pos += len(seg)
|
|
||||||
ends[i] = pos
|
|
||||||
|
|
||||||
# 5) Payload-Dicts
|
|
||||||
payloads: List[Dict[str, Any]] = []
|
|
||||||
for i, (win, seg) in enumerate(zip(windows_final, segments)):
|
|
||||||
chunk_id = ids_in[i] or f"{note_id}#{i+1}"
|
|
||||||
pl: Dict[str, Any] = {
|
pl: Dict[str, Any] = {
|
||||||
"note_id": note_id,
|
"note_id": nid,
|
||||||
"chunk_id": chunk_id,
|
"chunk_id": cid,
|
||||||
"id": chunk_id, # Alias
|
"index": int(index),
|
||||||
"chunk_index": i,
|
"ord": int(index) + 1,
|
||||||
"seq": seqs[i],
|
"type": note_type,
|
||||||
"path": rel_path,
|
"tags": tags,
|
||||||
"window": win,
|
"text": text,
|
||||||
"text": seg,
|
"window": window,
|
||||||
"start": starts[i],
|
"neighbors_prev": _as_list(prev_id),
|
||||||
"end": ends[i],
|
"neighbors_next": _as_list(next_id),
|
||||||
"overlap_left": overlaps_left[i],
|
"section": getattr(ch, "section", None) or (ch.get("section") if isinstance(ch, dict) else ""),
|
||||||
"overlap_right": overlaps_right[i],
|
"path": note_path,
|
||||||
|
"source_path": file_path or note_path,
|
||||||
|
"retriever_weight": float(rw),
|
||||||
|
"chunk_profile": cp,
|
||||||
}
|
}
|
||||||
# optionale Metadaten
|
# Aufräumen von Alt-Feldern
|
||||||
if note_type:
|
for alias in ("chunk_num", "Chunk_Number"):
|
||||||
pl["type"] = note_type
|
pl.pop(alias, None)
|
||||||
if note_title is not None:
|
|
||||||
pl["title"] = note_title
|
|
||||||
if note_tags is not None:
|
|
||||||
pl["tags"] = note_tags
|
|
||||||
if token_counts[i] is not None:
|
|
||||||
pl["token_count"] = int(token_counts[i])
|
|
||||||
if section_titles[i] is not None:
|
|
||||||
pl["section_title"] = section_titles[i]
|
|
||||||
if section_paths[i] is not None:
|
|
||||||
sp = str(section_paths[i]).replace("\\", "/")
|
|
||||||
pl["section_path"] = sp if sp else "/"
|
|
||||||
payloads.append(pl)
|
|
||||||
|
|
||||||
return payloads
|
out.append(pl)
|
||||||
|
|
||||||
|
return out
|
||||||
# __main__ Demo (optional)
|
|
||||||
if __name__ == "__main__": # pragma: no cover
|
|
||||||
fm = {"id": "demo", "title": "Demo", "type": "concept"}
|
|
||||||
# Beispiel ohne echte Fenster → erzeugt synthetische Overlaps
|
|
||||||
chunks = [
|
|
||||||
{"id": "demo#1", "text": "Alpha Beta Gamma"},
|
|
||||||
{"id": "demo#2", "text": "Gamma Delta"},
|
|
||||||
{"id": "demo#3", "text": "Delta Epsilon Zeta"},
|
|
||||||
]
|
|
||||||
pls = make_chunk_payloads(fm, "path/demo.md", chunks, note_text="Alpha Beta Gamma Delta Epsilon Zeta")
|
|
||||||
from pprint import pprint
|
|
||||||
pprint(pls)
|
|
||||||
recon = "".join(p["text"] for p in pls)
|
|
||||||
print("RECON:", recon)
|
|
||||||
+354
-50
@@ -2,43 +2,45 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Modul: app/core/derive_edges.py
|
Modul: app/core/derive_edges.py
|
||||||
Version: 1.4.0
|
Zweck:
|
||||||
Datum: 2025-10-01
|
- Bewahrt bestehende Edgelogik (belongs_to, prev/next, references, backlink)
|
||||||
|
- Ergänzt typenbasierte Default-Kanten (edge_defaults aus config/types.yaml)
|
||||||
Zweck
|
- Unterstützt "typed inline relations":
|
||||||
-----
|
* [[rel:KIND | Target]]
|
||||||
Robuste Kantenbildung für mindnet (Notes/Chunks):
|
* [[rel:KIND Target]]
|
||||||
- belongs_to (chunk -> note)
|
* rel: KIND [[Target]]
|
||||||
- next / prev (chunk-Kette)
|
- Unterstützt Obsidian-Callouts:
|
||||||
- references (chunk-scope) aus Chunk.window/text
|
* > [!edge] KIND: [[Target]] [[Target2]] ...
|
||||||
- optional references/backlink (note-scope)
|
Kompatibilität:
|
||||||
|
- build_edges_for_note(...) Signatur unverändert
|
||||||
Wichtig: Wikilinks werden mit der Parser-Funktion `extract_wikilinks` extrahiert,
|
- rule_id Werte:
|
||||||
damit Varianten wie [[id#anchor]] oder [[id|label]] korrekt auf 'id' reduziert werden.
|
* structure:belongs_to
|
||||||
|
* structure:order
|
||||||
Erwartete Chunk-Payload-Felder:
|
* explicit:wikilink
|
||||||
{
|
* inline:rel
|
||||||
"note_id": "...",
|
* callout:edge
|
||||||
"chunk_id": "...", # Alias "id" ist zulässig
|
* edge_defaults:<type>:<relation>
|
||||||
"id": "...",
|
* derived:backlink
|
||||||
"chunk_index": int,
|
|
||||||
"seq": int,
|
|
||||||
"window": str,
|
|
||||||
"text": str,
|
|
||||||
"path": "rel/path.md",
|
|
||||||
...
|
|
||||||
}
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Dict, List, Optional, Iterable
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Iterable, List, Optional, Tuple, Set, Dict
|
||||||
|
|
||||||
# WICHTIG: benutze die Parser-Extraktion für saubere Wikilinks
|
try:
|
||||||
from app.core.parser import extract_wikilinks
|
import yaml # optional, nur für types.yaml
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
yaml = None
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Utilities
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
def _get(d: dict, *keys, default=None):
|
def _get(d: dict, *keys, default=None):
|
||||||
for k in keys:
|
for k in keys:
|
||||||
if k in d and d[k] is not None:
|
if isinstance(d, dict) and k in d and d[k] is not None:
|
||||||
return d[k]
|
return d[k]
|
||||||
return default
|
return default
|
||||||
|
|
||||||
@@ -52,8 +54,8 @@ def _chunk_text_for_refs(chunk: dict) -> str:
|
|||||||
or ""
|
or ""
|
||||||
)
|
)
|
||||||
|
|
||||||
def _dedupe(seq: Iterable[str]) -> List[str]:
|
def _dedupe_seq(seq: Iterable[str]) -> List[str]:
|
||||||
seen = set()
|
seen: Set[str] = set()
|
||||||
out: List[str] = []
|
out: List[str] = []
|
||||||
for s in seq:
|
for s in seq:
|
||||||
if s not in seen:
|
if s not in seen:
|
||||||
@@ -64,15 +66,177 @@ def _dedupe(seq: Iterable[str]) -> List[str]:
|
|||||||
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
|
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
|
||||||
pl = {
|
pl = {
|
||||||
"kind": kind,
|
"kind": kind,
|
||||||
"scope": scope, # "chunk" | "note"
|
"relation": kind, # Alias (v2)
|
||||||
|
"scope": scope, # "chunk" | "note"
|
||||||
"source_id": source_id,
|
"source_id": source_id,
|
||||||
"target_id": target_id,
|
"target_id": target_id,
|
||||||
"note_id": note_id, # Träger/Quelle der Kante (aktuelle Note)
|
"note_id": note_id, # Träger-Note der Kante
|
||||||
}
|
}
|
||||||
if extra:
|
if extra:
|
||||||
pl.update(extra)
|
pl.update(extra)
|
||||||
return pl
|
return pl
|
||||||
|
|
||||||
|
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None) -> str:
|
||||||
|
base = f"{kind}:{s}->{t}#{scope}"
|
||||||
|
if rule_id:
|
||||||
|
base += f"|{rule_id}"
|
||||||
|
try:
|
||||||
|
import hashlib
|
||||||
|
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
return base
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Typen-Registry (types.yaml)
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
def _env(n: str, default: Optional[str] = None) -> str:
|
||||||
|
v = os.getenv(n)
|
||||||
|
return v if v is not None else (default or "")
|
||||||
|
|
||||||
|
def _load_types_registry() -> dict:
|
||||||
|
"""Lädt die YAML-Registry aus MINDNET_TYPES_FILE oder ./config/types.yaml"""
|
||||||
|
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||||
|
if not os.path.isfile(p) or yaml is None:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
with open(p, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def _get_types_map(reg: dict) -> dict:
|
||||||
|
if isinstance(reg, dict) and isinstance(reg.get("types"), dict):
|
||||||
|
return reg["types"]
|
||||||
|
return reg if isinstance(reg, dict) else {}
|
||||||
|
|
||||||
|
def _edge_defaults_for(note_type: Optional[str], reg: dict) -> List[str]:
|
||||||
|
"""
|
||||||
|
Liefert die edge_defaults-Liste für den gegebenen Notiztyp.
|
||||||
|
Fallback-Reihenfolge:
|
||||||
|
1) reg['types'][note_type]['edge_defaults']
|
||||||
|
2) reg['defaults']['edge_defaults'] (oder 'default'/'global')
|
||||||
|
3) []
|
||||||
|
"""
|
||||||
|
types_map = _get_types_map(reg)
|
||||||
|
if note_type and isinstance(types_map, dict):
|
||||||
|
t = types_map.get(note_type)
|
||||||
|
if isinstance(t, dict) and isinstance(t.get("edge_defaults"), list):
|
||||||
|
return [str(x) for x in t["edge_defaults"] if isinstance(x, str)]
|
||||||
|
for key in ("defaults", "default", "global"):
|
||||||
|
v = reg.get(key)
|
||||||
|
if isinstance(v, dict) and isinstance(v.get("edge_defaults"), list):
|
||||||
|
return [str(x) for x in v["edge_defaults"] if isinstance(x, str)]
|
||||||
|
return []
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Parser für Links / Relationen
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
# Normale Wikilinks (Fallback)
|
||||||
|
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([a-zA-Z0-9_\-#:. ]+)\]\]")
|
||||||
|
|
||||||
|
# Getypte Inline-Relationen:
|
||||||
|
# [[rel:KIND | Target]]
|
||||||
|
# [[rel:KIND Target]]
|
||||||
|
_REL_PIPE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s*\|\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||||
|
_REL_SPACE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s+(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||||
|
# rel: KIND [[Target]] (reines Textmuster)
|
||||||
|
_REL_TEXT = re.compile(r"rel\s*:\s*(?P<kind>[a-z_]+)\s*\[\[\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||||
|
|
||||||
|
def _extract_typed_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||||
|
"""
|
||||||
|
Gibt Liste (kind, target) zurück und den Text mit entfernten getypten Relation-Links,
|
||||||
|
damit die generische Wikilink-Erkennung sie nicht doppelt zählt.
|
||||||
|
Unterstützt drei Varianten:
|
||||||
|
- [[rel:KIND | Target]]
|
||||||
|
- [[rel:KIND Target]]
|
||||||
|
- rel: KIND [[Target]]
|
||||||
|
"""
|
||||||
|
pairs: List[Tuple[str,str]] = []
|
||||||
|
def _collect(m):
|
||||||
|
k = (m.group("kind") or "").strip().lower()
|
||||||
|
t = (m.group("target") or "").strip()
|
||||||
|
if k and t:
|
||||||
|
pairs.append((k, t))
|
||||||
|
return "" # Link entfernen
|
||||||
|
|
||||||
|
text = _REL_PIPE.sub(_collect, text)
|
||||||
|
text = _REL_SPACE.sub(_collect, text)
|
||||||
|
text = _REL_TEXT.sub(_collect, text)
|
||||||
|
return pairs, text
|
||||||
|
|
||||||
|
# Obsidian Callout Parser
|
||||||
|
_CALLOUT_START = re.compile(r"^\s*>\s*\[!edge\]\s*(.*)$", re.IGNORECASE)
|
||||||
|
_REL_LINE = re.compile(r"^(?P<kind>[a-z_]+)\s*:\s*(?P<targets>.+?)\s*$", re.IGNORECASE)
|
||||||
|
_WIKILINKS_IN_LINE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||||
|
|
||||||
|
def _extract_callout_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||||
|
"""
|
||||||
|
Findet [!edge]-Callouts und extrahiert (kind, target). Entfernt den gesamten
|
||||||
|
Callout-Block aus dem Text (damit Wikilinks daraus nicht zusätzlich als
|
||||||
|
"references" gezählt werden).
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return [], text
|
||||||
|
|
||||||
|
lines = text.splitlines()
|
||||||
|
out_pairs: List[Tuple[str,str]] = []
|
||||||
|
keep_lines: List[str] = []
|
||||||
|
i = 0
|
||||||
|
|
||||||
|
while i < len(lines):
|
||||||
|
m = _CALLOUT_START.match(lines[i])
|
||||||
|
if not m:
|
||||||
|
keep_lines.append(lines[i])
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
block_lines: List[str] = []
|
||||||
|
first_rest = m.group(1) or ""
|
||||||
|
if first_rest.strip():
|
||||||
|
block_lines.append(first_rest)
|
||||||
|
|
||||||
|
i += 1
|
||||||
|
while i < len(lines) and lines[i].lstrip().startswith('>'):
|
||||||
|
block_lines.append(lines[i].lstrip()[1:].lstrip())
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
for bl in block_lines:
|
||||||
|
mrel = _REL_LINE.match(bl)
|
||||||
|
if not mrel:
|
||||||
|
continue
|
||||||
|
kind = (mrel.group("kind") or "").strip().lower()
|
||||||
|
targets = mrel.group("targets") or ""
|
||||||
|
found = _WIKILINKS_IN_LINE.findall(targets)
|
||||||
|
if found:
|
||||||
|
for t in found:
|
||||||
|
t = t.strip()
|
||||||
|
if t:
|
||||||
|
out_pairs.append((kind, t))
|
||||||
|
else:
|
||||||
|
for raw in re.split(r"[,;]", targets):
|
||||||
|
t = raw.strip()
|
||||||
|
if t:
|
||||||
|
out_pairs.append((kind, t))
|
||||||
|
|
||||||
|
# Callout wird NICHT in keep_lines übernommen
|
||||||
|
continue
|
||||||
|
|
||||||
|
remainder = "\n".join(keep_lines)
|
||||||
|
return out_pairs, remainder
|
||||||
|
|
||||||
|
def _extract_wikilinks(text: str) -> List[str]:
|
||||||
|
ids: List[str] = []
|
||||||
|
for m in _WIKILINK_RE.finditer(text or ""):
|
||||||
|
ids.append(m.group(1).strip())
|
||||||
|
return ids
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Hauptfunktion
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
def build_edges_for_note(
|
def build_edges_for_note(
|
||||||
note_id: str,
|
note_id: str,
|
||||||
chunks: List[dict],
|
chunks: List[dict],
|
||||||
@@ -84,48 +248,188 @@ def build_edges_for_note(
|
|||||||
|
|
||||||
- belongs_to: für jeden Chunk (chunk -> note)
|
- belongs_to: für jeden Chunk (chunk -> note)
|
||||||
- next / prev: zwischen aufeinanderfolgenden Chunks
|
- next / prev: zwischen aufeinanderfolgenden Chunks
|
||||||
- references: pro Chunk aus window/text
|
- references: pro Chunk aus window/text (via Wikilinks)
|
||||||
|
- typed inline relations: [[rel:KIND | Target]] / [[rel:KIND Target]] / rel: KIND [[Target]]
|
||||||
|
- Obsidian Callouts: > [!edge] KIND: [[Target]] [[Target2]]
|
||||||
- optional note-scope references/backlinks: dedupliziert über alle Chunk-Funde + note_level_references
|
- optional note-scope references/backlinks: dedupliziert über alle Chunk-Funde + note_level_references
|
||||||
|
- typenbasierte Default-Kanten (edge_defaults) je gefundener Referenz
|
||||||
"""
|
"""
|
||||||
edges: List[dict] = []
|
edges: List[dict] = []
|
||||||
|
|
||||||
# belongs_to
|
# Note-Typ (aus erstem Chunk erwartet)
|
||||||
|
note_type = None
|
||||||
|
if chunks:
|
||||||
|
note_type = _get(chunks[0], "type")
|
||||||
|
|
||||||
|
# 1) belongs_to
|
||||||
for ch in chunks:
|
for ch in chunks:
|
||||||
cid = _get(ch, "chunk_id", "id")
|
cid = _get(ch, "chunk_id", "id")
|
||||||
if not cid:
|
if not cid:
|
||||||
continue
|
continue
|
||||||
edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {"chunk_id": cid}))
|
edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": "structure:belongs_to",
|
||||||
|
"confidence": 1.0,
|
||||||
|
}))
|
||||||
|
|
||||||
# next/prev
|
# 2) next / prev
|
||||||
for i in range(len(chunks) - 1):
|
for i in range(len(chunks) - 1):
|
||||||
a, b = chunks[i], chunks[i + 1]
|
a, b = chunks[i], chunks[i + 1]
|
||||||
a_id = _get(a, "chunk_id", "id")
|
a_id = _get(a, "chunk_id", "id")
|
||||||
b_id = _get(b, "chunk_id", "id")
|
b_id = _get(b, "chunk_id", "id")
|
||||||
if not a_id or not b_id:
|
if not a_id or not b_id:
|
||||||
continue
|
continue
|
||||||
edges.append(_edge("next", "chunk", a_id, b_id, note_id, {"chunk_id": a_id}))
|
edges.append(_edge("next", "chunk", a_id, b_id, note_id, {
|
||||||
edges.append(_edge("prev", "chunk", b_id, a_id, note_id, {"chunk_id": b_id}))
|
"chunk_id": a_id,
|
||||||
|
"edge_id": _mk_edge_id("next", a_id, b_id, "chunk", "structure:order"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": "structure:order",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
edges.append(_edge("prev", "chunk", b_id, a_id, note_id, {
|
||||||
|
"chunk_id": b_id,
|
||||||
|
"edge_id": _mk_edge_id("prev", b_id, a_id, "chunk", "structure:order"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": "structure:order",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
|
||||||
# references (chunk-scope) – Links aus window bevorzugen (Overlap-fest)
|
# 3) references + typed inline + callouts + defaults (chunk-scope)
|
||||||
|
reg = _load_types_registry()
|
||||||
|
defaults = _edge_defaults_for(note_type, reg)
|
||||||
refs_all: List[str] = []
|
refs_all: List[str] = []
|
||||||
|
|
||||||
for ch in chunks:
|
for ch in chunks:
|
||||||
cid = _get(ch, "chunk_id", "id")
|
cid = _get(ch, "chunk_id", "id")
|
||||||
if not cid:
|
if not cid:
|
||||||
continue
|
continue
|
||||||
txt = _chunk_text_for_refs(ch)
|
raw = _chunk_text_for_refs(ch)
|
||||||
refs = extract_wikilinks(txt) # <— Parser-Logik, kompatibel zu deinem System
|
|
||||||
|
# 3a) typed inline relations
|
||||||
|
typed, remainder = _extract_typed_relations(raw)
|
||||||
|
for kind, target in typed:
|
||||||
|
kind = kind.strip().lower()
|
||||||
|
if not kind or not target:
|
||||||
|
continue
|
||||||
|
edges.append(_edge(kind, "chunk", cid, target, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(kind, cid, target, "chunk", "inline:rel"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "inline:rel",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
if kind in {"related_to", "similar_to"}:
|
||||||
|
edges.append(_edge(kind, "chunk", target, cid, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(kind, target, cid, "chunk", "inline:rel"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "inline:rel",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
|
||||||
|
# 3b) callouts
|
||||||
|
call_pairs, remainder2 = _extract_callout_relations(remainder)
|
||||||
|
for kind, target in call_pairs:
|
||||||
|
k = (kind or "").strip().lower()
|
||||||
|
if not k or not target:
|
||||||
|
continue
|
||||||
|
edges.append(_edge(k, "chunk", cid, target, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(k, cid, target, "chunk", "callout:edge"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "callout:edge",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
if k in {"related_to", "similar_to"}:
|
||||||
|
edges.append(_edge(k, "chunk", target, cid, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(k, target, cid, "chunk", "callout:edge"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "callout:edge",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}))
|
||||||
|
|
||||||
|
# 3c) generische Wikilinks → references (+ defaults je Ref)
|
||||||
|
refs = _extract_wikilinks(remainder2)
|
||||||
for r in refs:
|
for r in refs:
|
||||||
edges.append(_edge("references", "chunk", cid, r, note_id, {"chunk_id": cid, "ref_text": r}))
|
edges.append(_edge("references", "chunk", cid, r, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"ref_text": r,
|
||||||
|
"edge_id": _mk_edge_id("references", cid, r, "chunk", "explicit:wikilink"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "explicit:wikilink",
|
||||||
|
"confidence": 1.0,
|
||||||
|
}))
|
||||||
|
for rel in defaults:
|
||||||
|
if rel == "references":
|
||||||
|
continue
|
||||||
|
edges.append(_edge(rel, "chunk", cid, r, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(rel, cid, r, "chunk", f"edge_defaults:{note_type}:{rel}"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||||
|
"confidence": 0.7,
|
||||||
|
}))
|
||||||
|
if rel in {"related_to", "similar_to"}:
|
||||||
|
edges.append(_edge(rel, "chunk", r, cid, note_id, {
|
||||||
|
"chunk_id": cid,
|
||||||
|
"edge_id": _mk_edge_id(rel, r, cid, "chunk", f"edge_defaults:{note_type}:{rel}"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||||
|
"confidence": 0.7,
|
||||||
|
}))
|
||||||
|
|
||||||
refs_all.extend(refs)
|
refs_all.extend(refs)
|
||||||
|
|
||||||
# optional: note-scope references/backlinks
|
# 4) optional note-scope refs/backlinks (+ defaults)
|
||||||
if include_note_scope_refs:
|
if include_note_scope_refs:
|
||||||
refs_note = refs_all[:]
|
refs_note = list(refs_all or [])
|
||||||
if note_level_references:
|
if note_level_references:
|
||||||
refs_note.extend([r for r in note_level_references if isinstance(r, str) and r])
|
refs_note.extend([r for r in note_level_references if isinstance(r, str) and r])
|
||||||
refs_note = _dedupe(refs_note)
|
refs_note = _dedupe_seq(refs_note)
|
||||||
for r in refs_note:
|
for r in refs_note:
|
||||||
edges.append(_edge("references", "note", note_id, r, note_id))
|
edges.append(_edge("references", "note", note_id, r, note_id, {
|
||||||
edges.append(_edge("backlink", "note", r, note_id, note_id))
|
"edge_id": _mk_edge_id("references", note_id, r, "note", "explicit:note_scope"),
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "explicit:note_scope",
|
||||||
|
"confidence": 1.0,
|
||||||
|
}))
|
||||||
|
edges.append(_edge("backlink", "note", r, note_id, note_id, {
|
||||||
|
"edge_id": _mk_edge_id("backlink", r, note_id, "note", "derived:backlink"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": "derived:backlink",
|
||||||
|
"confidence": 0.9,
|
||||||
|
}))
|
||||||
|
for rel in defaults:
|
||||||
|
if rel == "references":
|
||||||
|
continue
|
||||||
|
edges.append(_edge(rel, "note", note_id, r, note_id, {
|
||||||
|
"edge_id": _mk_edge_id(rel, note_id, r, "note", f"edge_defaults:{note_type}:{rel}"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||||
|
"confidence": 0.7,
|
||||||
|
}))
|
||||||
|
if rel in {"related_to", "similar_to"}:
|
||||||
|
edges.append(_edge(rel, "note", r, note_id, note_id, {
|
||||||
|
"edge_id": _mk_edge_id(rel, r, note_id, "note", f"edge_defaults:{note_type}:{rel}"),
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||||
|
"confidence": 0.7,
|
||||||
|
}))
|
||||||
|
|
||||||
return edges
|
# 5) De-Dupe (source_id, target_id, relation, rule_id)
|
||||||
|
seen: Set[Tuple[str,str,str,str]] = set()
|
||||||
|
out: List[dict] = []
|
||||||
|
for e in edges:
|
||||||
|
s = str(e.get("source_id") or "")
|
||||||
|
t = str(e.get("target_id") or "")
|
||||||
|
rel = str(e.get("relation") or e.get("kind") or "edge")
|
||||||
|
rule = str(e.get("rule_id") or "")
|
||||||
|
key = (s, t, rel, rule)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
out.append(e)
|
||||||
|
return out
|
||||||
+257
-81
@@ -2,119 +2,295 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Modul: app/core/edges.py
|
Modul: app/core/edges.py
|
||||||
Version: 1.0.0
|
Version: 2.0.0 (V2‑superset, rückwärtskompatibel zu v1 vom 2025‑09‑09)
|
||||||
Datum: 2025-09-09
|
|
||||||
|
|
||||||
Zweck
|
Zweck
|
||||||
-----
|
-----
|
||||||
Zentrale, konsistente Erzeugung von Edge-Payloads im **neuen Schema**:
|
Bewahrt die bestehende Edgelogik (belongs_to, prev/next, references, backlink)
|
||||||
- kind : "belongs_to" | "next" | "prev" | "references" | "backlink"
|
und ergänzt V2‑Felder + Typ‑Default‑Kanten gemäß config/types.yaml (edge_defaults).
|
||||||
- source_id : ID des Quellknotens (Chunk- oder Note-ID)
|
Die Funktion ist **idempotent** und **rückwärtskompatibel** zur bisherigen Signatur.
|
||||||
- target_id : ID des Zielknotens
|
|
||||||
- scope : "chunk" | "note"
|
|
||||||
- note_id : Owner-Note (für performantes Filtern/Löschen)
|
|
||||||
- seq : optional (z. B. Reihenfolge von Vorkommen)
|
|
||||||
|
|
||||||
Hinweise
|
Kompatibilitätsgarantien (gegenüber v1):
|
||||||
--------
|
- **Input**: akzeptiert identische Chunk‑Payloads wie v1:
|
||||||
- Edges werden dedupliziert (key=(kind,source_id,target_id,scope)).
|
* `id` (Chunk‑ID), `note_id` (Owner), `neighbors.prev|next` (optional),
|
||||||
- Für Chunk-Edges wird `note_id` aus dem Chunk-Payload entnommen.
|
`references: [{target_id: ...}]` (optional),
|
||||||
- Für Note-Scope-Edges ist `note_id` die Quell-Note-ID.
|
alternativ: `chunk_id`, `chunk_index|ord`, `window|text`
|
||||||
|
- **Output (v1‑Felder)**: `kind`, `source_id`, `target_id`, `scope`, `note_id`, `edge_id`
|
||||||
|
- **Neu (v2‑Felder)**: `relation`, `src_note_id`, `src_chunk_id?`, `dst_note_id`, `dst_chunk_id?`,
|
||||||
|
`provenance` (`explicit|rule`), `rule_id?`, `confidence?`
|
||||||
|
|
||||||
|
Regeln
|
||||||
|
------
|
||||||
|
- Deduplizierungsschlüssel: (source_id, target_id, relation, rule_id)
|
||||||
|
- Strukturkanten:
|
||||||
|
* belongs_to: 1× pro Chunk
|
||||||
|
* next/prev: Sequenz der Chunks; nutzt bevorzugt neighbors; sonst ord/chunk_index
|
||||||
|
- Explizite Referenzen:
|
||||||
|
* aus Chunk: `references[].target_id` (falls vorhanden)
|
||||||
|
* Fallback: Wikilinks in `window|text`: [[Some Title|some-id]] oder [[some-id]]
|
||||||
|
- Note‑Scope:
|
||||||
|
* backlink immer; references nur, wenn include_note_scope_refs=True
|
||||||
|
- Typ‑Defaults (edge_defaults aus config/types.yaml des **Quell‑Notiztyps**):
|
||||||
|
* Für jede explizite Referenz wird je default‑Relation eine Regel‑Kante erzeugt
|
||||||
|
* rule_id: "type_default:{note_type}:{relation}:v1", provenance="rule"
|
||||||
|
|
||||||
|
Konfiguration
|
||||||
|
-------------
|
||||||
|
- ENV MINDNET_TYPES_FILE (Default: ./config/types.yaml)
|
||||||
|
|
||||||
|
Lizenz/Autor
|
||||||
|
------------
|
||||||
|
- Erstimplementierung v1 (2025‑09‑09) — Projekt Mindnet
|
||||||
|
- Erweiterung v2 (2025‑11‑11) — kompatible Superset‑Implementierung
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
from typing import Dict, List
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Dict, Iterable, List, Optional, Tuple, Set
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # optional, nur für types.yaml
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
yaml = None
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Hilfen: types.yaml laden (edge_defaults)
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
|
def _types_path() -> str:
|
||||||
|
return os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
|
||||||
|
|
||||||
|
def _load_types() -> Dict[str, dict]:
|
||||||
|
p = _types_path()
|
||||||
|
if not os.path.isfile(p) or yaml is None:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
with open(p, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
if isinstance(data, dict) and "types" in data and isinstance(data["types"], dict):
|
||||||
|
return data["types"]
|
||||||
|
return data if isinstance(data, dict) else {}
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def _edge_defaults_for(note_type: Optional[str]) -> List[str]:
|
||||||
|
types = _load_types()
|
||||||
|
t = (note_type or "").strip().lower()
|
||||||
|
cfg = types.get(t) or {}
|
||||||
|
defaults = cfg.get("edge_defaults") or []
|
||||||
|
if isinstance(defaults, str):
|
||||||
|
defaults = [defaults]
|
||||||
|
return [str(x) for x in defaults if isinstance(x, (str, int, float))]
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Wikilink‑Parser (Fallback, wenn ch["references"] fehlt)
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
|
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([a-zA-Z0-9_\-#:. ]+)\]\]")
|
||||||
|
|
||||||
|
def _extract_wikilinks(text: str) -> List[str]:
|
||||||
|
ids: List[str] = []
|
||||||
|
for m in _WIKILINK_RE.finditer(text or ""):
|
||||||
|
ids.append(m.group(1).strip())
|
||||||
|
return ids
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Utility
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
|
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None) -> str:
|
||||||
|
base = f"{kind}:{s}->{t}#{scope}"
|
||||||
|
if rule_id:
|
||||||
|
base += f"|{rule_id}"
|
||||||
|
try:
|
||||||
|
import hashlib
|
||||||
|
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
return base
|
||||||
|
|
||||||
|
def _dedupe(edges: List[Dict]) -> List[Dict]:
|
||||||
|
seen: Set[Tuple[str,str,str,str]] = set()
|
||||||
|
out: List[Dict] = []
|
||||||
|
for e in edges:
|
||||||
|
s = str(e.get("source_id") or "")
|
||||||
|
t = str(e.get("target_id") or "")
|
||||||
|
rel = str(e.get("relation") or e.get("kind") or "edge")
|
||||||
|
rule = str(e.get("rule_id") or "")
|
||||||
|
key = (s, t, rel, rule)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
out.append(e)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _first(v: dict, *keys, default=None):
|
||||||
|
for k in keys:
|
||||||
|
if k in v and v[k] is not None:
|
||||||
|
return v[k]
|
||||||
|
return default
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Hauptfunktion
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
def build_edges_for_note(
|
def build_edges_for_note(
|
||||||
note_id: str,
|
note_id: str,
|
||||||
chunk_payloads: List[Dict],
|
chunk_payloads: List[Dict],
|
||||||
note_level_refs: List[str] | None,
|
note_level_refs: Optional[List[str]] = None,
|
||||||
*,
|
*,
|
||||||
include_note_scope_refs: bool = False,
|
include_note_scope_refs: bool = False,
|
||||||
) -> List[Dict]:
|
) -> List[Dict]:
|
||||||
edges: List[Dict] = []
|
edges: List[Dict] = []
|
||||||
|
chunks = list(chunk_payloads or [])
|
||||||
|
# Notiztyp aus erstem Chunk ableiten (kompatibel zu existierenden Payloads)
|
||||||
|
note_type = (chunks[0].get("type") if chunks else None) or (chunks[0].get("note_type") if chunks else None)
|
||||||
|
|
||||||
# Chunk-Scope: belongs_to / prev / next / references
|
# --- Strukturkanten ------------------------------------------------------
|
||||||
for ch in chunk_payloads:
|
# belongs_to
|
||||||
cid = ch["id"]
|
for ch in chunks:
|
||||||
|
cid = _first(ch, "id", "chunk_id")
|
||||||
|
if not cid:
|
||||||
|
continue
|
||||||
owner = ch.get("note_id") or note_id
|
owner = ch.get("note_id") or note_id
|
||||||
# belongs_to
|
e = {
|
||||||
edges.append({
|
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to:v1"),
|
||||||
"kind": "belongs_to",
|
"kind": "belongs_to",
|
||||||
|
"relation": "belongs_to",
|
||||||
|
"scope": "chunk",
|
||||||
"source_id": cid,
|
"source_id": cid,
|
||||||
"target_id": note_id,
|
"target_id": note_id,
|
||||||
"scope": "chunk",
|
"note_id": owner, # v1-Kompat
|
||||||
"note_id": owner,
|
# v2
|
||||||
})
|
"src_note_id": owner,
|
||||||
# Nachbarn
|
"src_chunk_id": cid,
|
||||||
|
"dst_note_id": note_id,
|
||||||
|
"provenance": "rule",
|
||||||
|
"rule_id": "structure:belongs_to:v1",
|
||||||
|
"confidence": 1.0,
|
||||||
|
}
|
||||||
|
edges.append(e)
|
||||||
|
|
||||||
|
# next/prev — bevorzugt neighbors.prev/next; sonst via ord/chunk_index
|
||||||
|
# Map der Chunks nach Index
|
||||||
|
ordered = list(chunks)
|
||||||
|
def _idx(c):
|
||||||
|
return _first(c, "chunk_index", "ord", default=0)
|
||||||
|
ordered.sort(key=_idx)
|
||||||
|
|
||||||
|
for i, ch in enumerate(ordered):
|
||||||
|
cid = _first(ch, "id", "chunk_id")
|
||||||
|
if not cid:
|
||||||
|
continue
|
||||||
|
owner = ch.get("note_id") or note_id
|
||||||
nb = ch.get("neighbors") or {}
|
nb = ch.get("neighbors") or {}
|
||||||
prev_id = nb.get("prev")
|
prev_id = nb.get("prev")
|
||||||
next_id = nb.get("next")
|
next_id = nb.get("next")
|
||||||
|
# Fallback-Reihenfolge
|
||||||
|
if prev_id is None and i > 0:
|
||||||
|
prev_id = _first(ordered[i-1], "id", "chunk_id")
|
||||||
|
if next_id is None and i+1 < len(ordered):
|
||||||
|
next_id = _first(ordered[i+1], "id", "chunk_id")
|
||||||
|
|
||||||
if prev_id:
|
if prev_id:
|
||||||
edges.append({
|
edges.append({
|
||||||
"kind": "prev",
|
"edge_id": _mk_edge_id("prev", cid, prev_id, "chunk", "structure:order:v1"),
|
||||||
"source_id": cid,
|
"kind": "prev", "relation": "prev", "scope": "chunk",
|
||||||
"target_id": prev_id,
|
"source_id": cid, "target_id": prev_id, "note_id": owner,
|
||||||
"scope": "chunk",
|
"src_note_id": owner, "src_chunk_id": cid,
|
||||||
"note_id": owner,
|
"dst_note_id": owner, "dst_chunk_id": prev_id,
|
||||||
|
"provenance": "rule", "rule_id": "structure:order:v1", "confidence": 0.95,
|
||||||
})
|
})
|
||||||
edges.append({
|
edges.append({
|
||||||
"kind": "next",
|
"edge_id": _mk_edge_id("next", prev_id, cid, "chunk", "structure:order:v1"),
|
||||||
"source_id": prev_id,
|
"kind": "next", "relation": "next", "scope": "chunk",
|
||||||
"target_id": cid,
|
"source_id": prev_id, "target_id": cid, "note_id": owner,
|
||||||
"scope": "chunk",
|
"src_note_id": owner, "src_chunk_id": prev_id,
|
||||||
"note_id": owner,
|
"dst_note_id": owner, "dst_chunk_id": cid,
|
||||||
})
|
"provenance": "rule", "rule_id": "structure:order:v1", "confidence": 0.95,
|
||||||
if next_id:
|
|
||||||
edges.append({
|
|
||||||
"kind": "next",
|
|
||||||
"source_id": cid,
|
|
||||||
"target_id": next_id,
|
|
||||||
"scope": "chunk",
|
|
||||||
"note_id": owner,
|
|
||||||
})
|
|
||||||
edges.append({
|
|
||||||
"kind": "prev",
|
|
||||||
"source_id": next_id,
|
|
||||||
"target_id": cid,
|
|
||||||
"scope": "chunk",
|
|
||||||
"note_id": owner,
|
|
||||||
})
|
|
||||||
# references aus Chunk
|
|
||||||
for ref in (ch.get("references") or []):
|
|
||||||
tid = ref.get("target_id")
|
|
||||||
if not tid:
|
|
||||||
continue
|
|
||||||
edges.append({
|
|
||||||
"kind": "references",
|
|
||||||
"source_id": cid,
|
|
||||||
"target_id": tid,
|
|
||||||
"scope": "chunk",
|
|
||||||
"note_id": owner,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
# Note-Scope: backlink (immer); references (optional)
|
# --- Explizite Referenzen (Chunk‑Scope) ---------------------------------
|
||||||
unique_refs = list(dict.fromkeys(note_level_refs or []))
|
explicit_refs: List[Dict] = []
|
||||||
|
for ch in chunks:
|
||||||
|
cid = _first(ch, "id", "chunk_id")
|
||||||
|
if not cid:
|
||||||
|
continue
|
||||||
|
owner = ch.get("note_id") or note_id
|
||||||
|
# 1) bevorzugt vorhandene ch["references"]
|
||||||
|
refs = ch.get("references") or []
|
||||||
|
targets = [r.get("target_id") for r in refs if isinstance(r, dict) and r.get("target_id")]
|
||||||
|
# 2) Fallback: Wikilinks aus Text
|
||||||
|
if not targets:
|
||||||
|
text = _first(ch, "window", "text", default="") or ""
|
||||||
|
targets = _extract_wikilinks(text)
|
||||||
|
for tid in targets:
|
||||||
|
if not isinstance(tid, str) or not tid.strip():
|
||||||
|
continue
|
||||||
|
e = {
|
||||||
|
"edge_id": _mk_edge_id("references", cid, tid, "chunk"),
|
||||||
|
"kind": "references",
|
||||||
|
"relation": "references",
|
||||||
|
"scope": "chunk",
|
||||||
|
"source_id": cid,
|
||||||
|
"target_id": tid,
|
||||||
|
"note_id": owner,
|
||||||
|
# v2
|
||||||
|
"src_note_id": owner,
|
||||||
|
"src_chunk_id": cid,
|
||||||
|
"dst_note_id": tid,
|
||||||
|
"provenance": "explicit",
|
||||||
|
"rule_id": "",
|
||||||
|
"confidence": 1.0,
|
||||||
|
}
|
||||||
|
edges.append(e)
|
||||||
|
explicit_refs.append(e)
|
||||||
|
|
||||||
|
# --- Note‑Scope: references (optional) + backlink (immer) ----------------
|
||||||
|
unique_refs = []
|
||||||
|
if note_level_refs:
|
||||||
|
seen = set()
|
||||||
|
for tid in note_level_refs:
|
||||||
|
if isinstance(tid, str) and tid.strip() and tid not in seen:
|
||||||
|
unique_refs.append(tid); seen.add(tid)
|
||||||
|
|
||||||
for tid in unique_refs:
|
for tid in unique_refs:
|
||||||
if include_note_scope_refs:
|
if include_note_scope_refs:
|
||||||
edges.append({
|
edges.append({
|
||||||
"kind": "references",
|
"edge_id": _mk_edge_id("references", note_id, tid, "note"),
|
||||||
"source_id": note_id,
|
"kind": "references", "relation": "references", "scope": "note",
|
||||||
"target_id": tid,
|
"source_id": note_id, "target_id": tid, "note_id": note_id,
|
||||||
"scope": "note",
|
"src_note_id": note_id, "dst_note_id": tid,
|
||||||
"note_id": note_id,
|
"provenance": "explicit", "rule_id": "", "confidence": 1.0,
|
||||||
})
|
})
|
||||||
edges.append({
|
edges.append({
|
||||||
"kind": "backlink",
|
"edge_id": _mk_edge_id("backlink", tid, note_id, "note", "derived:backlink:v1"),
|
||||||
"source_id": tid,
|
"kind": "backlink", "relation": "backlink", "scope": "note",
|
||||||
"target_id": note_id,
|
"source_id": tid, "target_id": note_id, "note_id": note_id,
|
||||||
"scope": "note",
|
"src_note_id": tid, "dst_note_id": note_id,
|
||||||
"note_id": note_id,
|
"provenance": "rule", "rule_id": "derived:backlink:v1", "confidence": 0.9,
|
||||||
})
|
})
|
||||||
|
|
||||||
# Dedupe
|
# --- Type‑Defaults je expliziter Referenz --------------------------------
|
||||||
dedup = {}
|
defaults = [d for d in _edge_defaults_for(note_type) if d and d != "references"]
|
||||||
for e in edges:
|
if defaults:
|
||||||
k = (e["kind"], e["source_id"], e["target_id"], e.get("scope", ""))
|
for e in explicit_refs + ([ ] if not include_note_scope_refs else []):
|
||||||
dedup[k] = e
|
# wir nutzen die bereits erzeugten explicit‑Edges als Vorlage
|
||||||
return list(dedup.values())
|
src = e["source_id"]; tgt = e["target_id"]
|
||||||
|
scope = e.get("scope", "chunk")
|
||||||
|
s_note = e.get("src_note_id") or note_id
|
||||||
|
s_chunk = e.get("src_chunk_id")
|
||||||
|
t_note = e.get("dst_note_id") or tgt
|
||||||
|
for rel in defaults:
|
||||||
|
rule_id = f"type_default:{(note_type or 'unknown')}:{rel}:v1"
|
||||||
|
edges.append({
|
||||||
|
"edge_id": _mk_edge_id(rel, src, tgt, scope, rule_id),
|
||||||
|
"kind": rel, "relation": rel, "scope": scope,
|
||||||
|
"source_id": src, "target_id": tgt, "note_id": s_note,
|
||||||
|
"src_note_id": s_note, "src_chunk_id": s_chunk,
|
||||||
|
"dst_note_id": t_note,
|
||||||
|
"provenance": "rule", "rule_id": rule_id, "confidence": 0.7,
|
||||||
|
})
|
||||||
|
|
||||||
|
# --- Dedupe & Return -----------------------------------------------------
|
||||||
|
return _dedupe(edges)
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
# app/core/edges_writer.py
|
||||||
|
from __future__ import annotations
|
||||||
|
import hashlib
|
||||||
|
from typing import Dict, List, Iterable, Tuple
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Dein Modul mit der Schemadefinition und der Builder-Funktion
|
||||||
|
from app.core.edges import build_edges_for_note # noqa: F401
|
||||||
|
except Exception as e:
|
||||||
|
raise RuntimeError("Konnte app.core.edges nicht importieren. "
|
||||||
|
"Bitte sicherstellen, dass app/core/edges.py vorhanden ist.") from e
|
||||||
|
|
||||||
|
def _edge_uid(kind: str, source_id: str, target_id: str, scope: str) -> str:
|
||||||
|
"""
|
||||||
|
Deterministische, kurze ID für eine Edge.
|
||||||
|
Kollisionen sind praktisch ausgeschlossen (BLAKE2s über den Kanonischen Schlüssel).
|
||||||
|
"""
|
||||||
|
key = f"{kind}|{source_id}|{target_id}|{scope}"
|
||||||
|
return hashlib.blake2s(key.encode("utf-8"), digest_size=12).hexdigest()
|
||||||
|
|
||||||
|
def ensure_edges_collection(qdrant_client, collection: str) -> None:
|
||||||
|
"""
|
||||||
|
Legt die Edge-Collection an, falls sie nicht existiert.
|
||||||
|
Minimal: 1D-Vector (Dummy), Cosine. Payload-only-Collections sind je nach Qdrant-Version heikel.
|
||||||
|
"""
|
||||||
|
from qdrant_client.http import models as qm
|
||||||
|
|
||||||
|
existing = [c.name for c in qdrant_client.get_collections().collections]
|
||||||
|
if collection in existing:
|
||||||
|
return
|
||||||
|
|
||||||
|
qdrant_client.recreate_collection(
|
||||||
|
collection_name=collection,
|
||||||
|
vectors_config=qm.VectorParams(size=1, distance=qm.Distance.COSINE),
|
||||||
|
on_disk_payload=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def edges_from_note(
|
||||||
|
note_id: str,
|
||||||
|
chunk_payloads: List[Dict],
|
||||||
|
note_level_refs: Iterable[str] | None,
|
||||||
|
*,
|
||||||
|
include_note_scope_refs: bool = False,
|
||||||
|
) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Ruft deinen Edge-Builder auf und gibt die (deduplizierten) Edge-Payloads zurück.
|
||||||
|
Keine Schemaänderung – exakt das aus app/core/edges.py.
|
||||||
|
"""
|
||||||
|
return build_edges_for_note(
|
||||||
|
note_id=note_id,
|
||||||
|
chunk_payloads=chunk_payloads,
|
||||||
|
note_level_refs=list(note_level_refs or []),
|
||||||
|
include_note_scope_refs=include_note_scope_refs,
|
||||||
|
)
|
||||||
|
|
||||||
|
def upsert_edges(
|
||||||
|
qdrant_client,
|
||||||
|
collection: str,
|
||||||
|
edge_payloads: List[Dict],
|
||||||
|
) -> Tuple[int, int]:
|
||||||
|
"""
|
||||||
|
Schreibt Edges als Points in Qdrant.
|
||||||
|
- id: deterministisch aus (kind, source_id, target_id, scope)
|
||||||
|
- vector: [0.0] Dummy
|
||||||
|
- payload: Edge-Dict (unverändert, siehe Schema in app/core/edges.py)
|
||||||
|
Gibt (anzahl_points, anzahl_unique_keys) zurück.
|
||||||
|
"""
|
||||||
|
from qdrant_client.models import PointStruct
|
||||||
|
|
||||||
|
if not edge_payloads:
|
||||||
|
return 0, 0
|
||||||
|
|
||||||
|
points = []
|
||||||
|
seen = set()
|
||||||
|
for e in edge_payloads:
|
||||||
|
key = (e.get("kind"), e.get("source_id"), e.get("target_id"), e.get("scope"))
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
eid = _edge_uid(*key)
|
||||||
|
points.append(
|
||||||
|
PointStruct(
|
||||||
|
id=eid,
|
||||||
|
vector=[0.0],
|
||||||
|
payload=e,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not points:
|
||||||
|
return 0, 0
|
||||||
|
|
||||||
|
ensure_edges_collection(qdrant_client, collection)
|
||||||
|
qdrant_client.upsert(collection_name=collection, points=points)
|
||||||
|
return len(points), len(seen)
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Datei: app/core/env_vars.py
|
||||||
|
Version: 1.1.0 (2025-11-08)
|
||||||
|
|
||||||
|
Zweck
|
||||||
|
Einheitliche Auflösung von ENV-Variablen (Prefix, Qdrant, Embeddings, Hashing)
|
||||||
|
mit Abwärtskompatibilität.
|
||||||
|
|
||||||
|
Grundsatz
|
||||||
|
- Für Qdrant-Funktionen ist 'COLLECTION_PREFIX' der Primärschlüssel.
|
||||||
|
- 'MINDNET_PREFIX' bleibt für App-/UI-/Exporter-Kontexte nutzbar.
|
||||||
|
- Fallbacks sorgen dafür, dass ältere Umgebungen weiter funktionieren.
|
||||||
|
|
||||||
|
Wichtig
|
||||||
|
- Lädt optional eine .env (wenn python-dotenv verfügbar ist).
|
||||||
|
- Überschreibt keine bereits gesetzten OS-Variablen (override=False).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from typing import Optional, Dict
|
||||||
|
|
||||||
|
# Optional: .env automatisch laden (ohne Hard-Fail, falls nicht vorhanden)
|
||||||
|
try:
|
||||||
|
from dotenv import load_dotenv, find_dotenv # type: ignore
|
||||||
|
_p = find_dotenv()
|
||||||
|
if _p:
|
||||||
|
load_dotenv(_p, override=False)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# -------- Prefix-Auflösung --------
|
||||||
|
|
||||||
|
def get_collection_prefix(cli_override: Optional[str] = None) -> str:
|
||||||
|
"""
|
||||||
|
Für Qdrant-relevante Funktionen:
|
||||||
|
1) CLI-Override (--prefix)
|
||||||
|
2) ENV COLLECTION_PREFIX
|
||||||
|
3) ENV MINDNET_PREFIX (Fallback)
|
||||||
|
4) 'mindnet' (Default)
|
||||||
|
"""
|
||||||
|
if cli_override and str(cli_override).strip():
|
||||||
|
return str(cli_override).strip()
|
||||||
|
return (
|
||||||
|
os.getenv("COLLECTION_PREFIX")
|
||||||
|
or os.getenv("MINDNET_PREFIX")
|
||||||
|
or "mindnet"
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_mindnet_prefix(cli_override: Optional[str] = None) -> str:
|
||||||
|
"""
|
||||||
|
Für App-/UI-/Exporter-Kontexte:
|
||||||
|
1) CLI-Override (--prefix)
|
||||||
|
2) ENV MINDNET_PREFIX
|
||||||
|
3) ENV COLLECTION_PREFIX (Fallback)
|
||||||
|
4) 'mindnet'
|
||||||
|
"""
|
||||||
|
if cli_override and str(cli_override).strip():
|
||||||
|
return str(cli_override).strip()
|
||||||
|
return (
|
||||||
|
os.getenv("MINDNET_PREFIX")
|
||||||
|
or os.getenv("COLLECTION_PREFIX")
|
||||||
|
or "mindnet"
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_prefix(cli_override: Optional[str] = None, target: str = "qdrant") -> str:
|
||||||
|
"""
|
||||||
|
Universelle Hülle (abwärtskompatibel):
|
||||||
|
target='qdrant' -> get_collection_prefix
|
||||||
|
target='app' -> get_mindnet_prefix
|
||||||
|
"""
|
||||||
|
if target.lower() == "app":
|
||||||
|
return get_mindnet_prefix(cli_override)
|
||||||
|
return get_collection_prefix(cli_override)
|
||||||
|
|
||||||
|
# -------- Qdrant / Embeddings / Hashing --------
|
||||||
|
|
||||||
|
def get_qdrant_url(default: str = "http://127.0.0.1:6333") -> str:
|
||||||
|
return os.getenv("QDRANT_URL", default)
|
||||||
|
|
||||||
|
def get_qdrant_api_key(default: str = "") -> str:
|
||||||
|
return os.getenv("QDRANT_API_KEY", default)
|
||||||
|
|
||||||
|
def get_vector_dim(default: int = 384) -> int:
|
||||||
|
try:
|
||||||
|
return int(os.getenv("VECTOR_DIM", str(default)))
|
||||||
|
except Exception:
|
||||||
|
return default
|
||||||
|
|
||||||
|
def get_embed_url(default: Optional[str] = None) -> Optional[str]:
|
||||||
|
return os.getenv("EMBED_URL", default)
|
||||||
|
|
||||||
|
def get_hash_env() -> Dict[str, str]:
|
||||||
|
"""
|
||||||
|
Liefert die Hash-Konfiguration (nur Aggregation; die Auswertung bleibt in den Skripten).
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"MINDNET_HASH_COMPARE": os.getenv("MINDNET_HASH_COMPARE", ""),
|
||||||
|
"MINDNET_HASH_SOURCE": os.getenv("MINDNET_HASH_SOURCE", ""),
|
||||||
|
"MINDNET_HASH_NORMALIZE": os.getenv("MINDNET_HASH_NORMALIZE", ""),
|
||||||
|
}
|
||||||
@@ -0,0 +1,256 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
app/core/graph_adapter.py — Adjazenzaufbau & Subgraph-Expansion
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Baut aus Qdrant-Edges (Collection: *_edges) einen leichten In-Memory-Graph.
|
||||||
|
|
||||||
|
Kompatibilität:
|
||||||
|
- WP-04a: Liefert Scores (edge_bonus, centrality).
|
||||||
|
- WP-04b: Liefert jetzt auch Struktur-Daten für Erklärungen (Reverse-Lookup).
|
||||||
|
|
||||||
|
Version:
|
||||||
|
0.4.0 (Update für WP-04b: Reverse Adjacency für Explainability)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Dict, List, Optional, DefaultDict, Any
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
from app.core.qdrant import collection_names
|
||||||
|
|
||||||
|
# Legacy-Import Fallback
|
||||||
|
try: # pragma: no cover
|
||||||
|
from app.core.qdrant_points import get_edges_for_sources # type: ignore
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
get_edges_for_sources = None # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
# Basisgewichte je Edge-Typ (WP-04a Config)
|
||||||
|
EDGE_BASE_WEIGHTS: Dict[str, float] = {
|
||||||
|
# Struktur
|
||||||
|
"belongs_to": 0.10,
|
||||||
|
"next": 0.06,
|
||||||
|
"prev": 0.06,
|
||||||
|
"backlink": 0.04,
|
||||||
|
"references_at": 0.08,
|
||||||
|
|
||||||
|
# Wissen
|
||||||
|
"references": 0.20,
|
||||||
|
"depends_on": 0.18,
|
||||||
|
"related_to": 0.15,
|
||||||
|
"similar_to": 0.12,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _edge_weight(pl: Dict) -> float:
|
||||||
|
"""Berechnet das effektive Edge-Gewicht aus kind + confidence."""
|
||||||
|
kind = pl.get("kind", "edge")
|
||||||
|
base = EDGE_BASE_WEIGHTS.get(kind, 0.0)
|
||||||
|
|
||||||
|
conf_raw = pl.get("confidence", None)
|
||||||
|
try:
|
||||||
|
conf = float(conf_raw) if conf_raw is not None else None
|
||||||
|
except Exception:
|
||||||
|
conf = None
|
||||||
|
|
||||||
|
if conf is None:
|
||||||
|
return base
|
||||||
|
|
||||||
|
if conf < 0.0: conf = 0.0
|
||||||
|
if conf > 1.0: conf = 1.0
|
||||||
|
|
||||||
|
return base * conf
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_edges(
|
||||||
|
client: QdrantClient,
|
||||||
|
prefix: str,
|
||||||
|
seeds: List[str],
|
||||||
|
edge_types: Optional[List[str]] = None,
|
||||||
|
limit: int = 2048,
|
||||||
|
) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Holt Edges direkt aus der *_edges Collection.
|
||||||
|
Filter: source_id IN seeds OR target_id IN seeds OR note_id IN seeds
|
||||||
|
"""
|
||||||
|
if not seeds or limit <= 0:
|
||||||
|
return []
|
||||||
|
|
||||||
|
_, _, edges_col = collection_names(prefix)
|
||||||
|
|
||||||
|
seed_conditions = []
|
||||||
|
for field in ("source_id", "target_id", "note_id"):
|
||||||
|
for s in seeds:
|
||||||
|
seed_conditions.append(
|
||||||
|
rest.FieldCondition(key=field, match=rest.MatchValue(value=str(s)))
|
||||||
|
)
|
||||||
|
seeds_filter = rest.Filter(should=seed_conditions) if seed_conditions else None
|
||||||
|
|
||||||
|
type_filter = None
|
||||||
|
if edge_types:
|
||||||
|
type_conds = [
|
||||||
|
rest.FieldCondition(key="kind", match=rest.MatchValue(value=str(k)))
|
||||||
|
for k in edge_types
|
||||||
|
]
|
||||||
|
type_filter = rest.Filter(should=type_conds)
|
||||||
|
|
||||||
|
must = []
|
||||||
|
if seeds_filter: must.append(seeds_filter)
|
||||||
|
if type_filter: must.append(type_filter)
|
||||||
|
|
||||||
|
flt = rest.Filter(must=must) if must else None
|
||||||
|
|
||||||
|
pts, _ = client.scroll(
|
||||||
|
collection_name=edges_col,
|
||||||
|
scroll_filter=flt,
|
||||||
|
limit=limit,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
out: List[Dict] = []
|
||||||
|
for p in pts or []:
|
||||||
|
pl = dict(p.payload or {})
|
||||||
|
if pl:
|
||||||
|
out.append(pl)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class Subgraph:
|
||||||
|
"""Leichtgewichtiger Subgraph mit Adjazenzlisten & Kennzahlen."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
# Forward: source -> [targets]
|
||||||
|
self.adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||||
|
# Reverse: target -> [sources] (Neu für WP-04b Explanation)
|
||||||
|
self.reverse_adj: DefaultDict[str, List[Dict]] = defaultdict(list)
|
||||||
|
|
||||||
|
self.in_degree: DefaultDict[str, int] = defaultdict(int)
|
||||||
|
self.out_degree: DefaultDict[str, int] = defaultdict(int)
|
||||||
|
|
||||||
|
def add_edge(self, e: Dict) -> None:
|
||||||
|
"""
|
||||||
|
Fügt eine Kante hinzu und aktualisiert Forward/Reverse Indizes.
|
||||||
|
e muss enthalten: source, target, kind, weight.
|
||||||
|
"""
|
||||||
|
src = e.get("source")
|
||||||
|
tgt = e.get("target")
|
||||||
|
kind = e.get("kind")
|
||||||
|
weight = e.get("weight", EDGE_BASE_WEIGHTS.get(kind, 0.0))
|
||||||
|
owner = e.get("note_id")
|
||||||
|
|
||||||
|
if not src or not tgt:
|
||||||
|
return
|
||||||
|
|
||||||
|
# 1. Primäre Adjazenz (Forward)
|
||||||
|
edge_data = {"target": tgt, "kind": kind, "weight": weight}
|
||||||
|
self.adj[src].append(edge_data)
|
||||||
|
self.out_degree[src] += 1
|
||||||
|
self.in_degree[tgt] += 1
|
||||||
|
|
||||||
|
# 2. Reverse Adjazenz (Neu für Explanation)
|
||||||
|
# Wir speichern, woher die Kante kam.
|
||||||
|
rev_data = {"source": src, "kind": kind, "weight": weight}
|
||||||
|
self.reverse_adj[tgt].append(rev_data)
|
||||||
|
|
||||||
|
# 3. Kontext-Note Handling (Forward & Reverse)
|
||||||
|
# Wenn eine Kante "im Kontext einer Note" (owner) definiert ist,
|
||||||
|
# schreiben wir sie der Note gut, damit der Retriever Scores auf Note-Ebene findet.
|
||||||
|
if owner and owner != src:
|
||||||
|
# Forward: Owner -> Target
|
||||||
|
self.adj[owner].append(edge_data)
|
||||||
|
self.out_degree[owner] += 1
|
||||||
|
|
||||||
|
# Reverse: Target wird vom Owner referenziert (indirekt)
|
||||||
|
if owner != tgt:
|
||||||
|
rev_owner_data = {"source": owner, "kind": kind, "weight": weight, "via_context": True}
|
||||||
|
self.reverse_adj[tgt].append(rev_owner_data)
|
||||||
|
self.in_degree[owner] += 1 # Leichter Centrality Boost für den Owner
|
||||||
|
|
||||||
|
def aggregate_edge_bonus(self, node_id: str) -> float:
|
||||||
|
"""Summe der ausgehenden Kantengewichte (Hub-Score)."""
|
||||||
|
return sum(edge["weight"] for edge in self.adj.get(node_id, []))
|
||||||
|
|
||||||
|
def edge_bonus(self, node_id: str) -> float:
|
||||||
|
"""API für Retriever (WP-04a Kompatibilität)."""
|
||||||
|
return self.aggregate_edge_bonus(node_id)
|
||||||
|
|
||||||
|
def centrality_bonus(self, node_id: str) -> float:
|
||||||
|
"""Log-gedämpfte Zentralität (In-Degree)."""
|
||||||
|
import math
|
||||||
|
indeg = self.in_degree.get(node_id, 0)
|
||||||
|
if indeg <= 0:
|
||||||
|
return 0.0
|
||||||
|
return min(math.log1p(indeg) / 10.0, 0.15)
|
||||||
|
|
||||||
|
# --- WP-04b Explanation Helpers ---
|
||||||
|
|
||||||
|
def get_outgoing_edges(self, node_id: str) -> List[Dict[str, Any]]:
|
||||||
|
"""Liefert Liste aller Ziele, auf die dieser Knoten zeigt."""
|
||||||
|
return self.adj.get(node_id, [])
|
||||||
|
|
||||||
|
def get_incoming_edges(self, node_id: str) -> List[Dict[str, Any]]:
|
||||||
|
"""Liefert Liste aller Quellen, die auf diesen Knoten zeigen."""
|
||||||
|
return self.reverse_adj.get(node_id, [])
|
||||||
|
|
||||||
|
|
||||||
|
def expand(
|
||||||
|
client: QdrantClient,
|
||||||
|
prefix: str,
|
||||||
|
seeds: List[str],
|
||||||
|
depth: int = 1,
|
||||||
|
edge_types: Optional[List[str]] = None,
|
||||||
|
) -> Subgraph:
|
||||||
|
"""
|
||||||
|
Expandiert ab Seeds entlang von Edges (bis `depth`).
|
||||||
|
"""
|
||||||
|
sg = Subgraph()
|
||||||
|
frontier = set(seeds)
|
||||||
|
visited = set()
|
||||||
|
|
||||||
|
max_depth = max(depth, 0)
|
||||||
|
|
||||||
|
for _ in range(max_depth):
|
||||||
|
if not frontier:
|
||||||
|
break
|
||||||
|
|
||||||
|
edges_payloads = _fetch_edges(
|
||||||
|
client=client,
|
||||||
|
prefix=prefix,
|
||||||
|
seeds=list(frontier),
|
||||||
|
edge_types=edge_types,
|
||||||
|
limit=2048,
|
||||||
|
)
|
||||||
|
|
||||||
|
next_frontier = set()
|
||||||
|
for pl in edges_payloads:
|
||||||
|
src = pl.get("source_id")
|
||||||
|
tgt = pl.get("target_id")
|
||||||
|
|
||||||
|
# Skip invalid edges
|
||||||
|
if not src or not tgt:
|
||||||
|
continue
|
||||||
|
|
||||||
|
e = {
|
||||||
|
"source": src,
|
||||||
|
"target": tgt,
|
||||||
|
"kind": pl.get("kind", "edge"),
|
||||||
|
"weight": _edge_weight(pl),
|
||||||
|
"note_id": pl.get("note_id"),
|
||||||
|
}
|
||||||
|
sg.add_edge(e)
|
||||||
|
|
||||||
|
# Nur weitersuchen, wenn Target noch nicht besucht
|
||||||
|
if tgt and tgt not in visited:
|
||||||
|
next_frontier.add(tgt)
|
||||||
|
|
||||||
|
visited |= frontier
|
||||||
|
frontier = next_frontier - visited
|
||||||
|
|
||||||
|
return sg
|
||||||
+187
-190
@@ -1,229 +1,226 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
# Modul: app/core/note_payload.py
|
"""
|
||||||
# Version: 1.7.0
|
Modul: app/core/note_payload.py
|
||||||
# Datum: 2025-09-09
|
Version: 2.0.0
|
||||||
|
|
||||||
|
Zweck
|
||||||
|
-----
|
||||||
|
Erzeugt ein robustes Note-Payload. Werte wie `retriever_weight`, `chunk_profile`
|
||||||
|
und `edge_defaults` werden in folgender Priorität bestimmt:
|
||||||
|
1) Frontmatter (Note)
|
||||||
|
2) Typ-Registry (config/types.yaml: types.<type>.*)
|
||||||
|
3) Registry-Defaults (config/types.yaml: defaults.*)
|
||||||
|
4) ENV-Defaults (MINDNET_DEFAULT_RETRIEVER_WEIGHT / MINDNET_DEFAULT_CHUNK_PROFILE)
|
||||||
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
from typing import Any, Dict, Tuple, Optional
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
from typing import Any, Dict, Optional, Tuple
|
import json
|
||||||
|
import pathlib
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from app.core.parser import read_markdown, extract_wikilinks, FRONTMATTER_RE
|
import yaml # type: ignore
|
||||||
except Exception: # pragma: no cover
|
except Exception:
|
||||||
from .parser import read_markdown, extract_wikilinks, FRONTMATTER_RE # type: ignore
|
yaml = None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Helpers
|
# Helper
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
|
def _as_dict(x) -> Dict[str, Any]:
|
||||||
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
"""Versucht, ein ParsedMarkdown-ähnliches Objekt in ein Dict zu überführen."""
|
||||||
|
if isinstance(x, dict):
|
||||||
|
return dict(x)
|
||||||
|
|
||||||
def _normalize_body(body: str, mode: str) -> str:
|
out: Dict[str, Any] = {}
|
||||||
if mode == "none":
|
# bekannte Attribute übernehmen, sofern vorhanden
|
||||||
return body if body is not None else ""
|
for attr in (
|
||||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
|
"frontmatter",
|
||||||
text = "\n".join(line.rstrip() for line in text.split("\n"))
|
"body",
|
||||||
return text
|
"id",
|
||||||
|
"note_id",
|
||||||
|
"title",
|
||||||
|
"path",
|
||||||
|
"tags",
|
||||||
|
"type",
|
||||||
|
"created",
|
||||||
|
"modified",
|
||||||
|
"date",
|
||||||
|
):
|
||||||
|
if hasattr(x, attr):
|
||||||
|
val = getattr(x, attr)
|
||||||
|
if val is not None:
|
||||||
|
out[attr] = val
|
||||||
|
|
||||||
def _resolve_hash_mode(explicit: Optional[str]) -> str:
|
# Fallback: wenn immer noch leer, raw speichern
|
||||||
if explicit:
|
if not out:
|
||||||
val = explicit.strip().lower()
|
out["raw"] = str(x)
|
||||||
else:
|
|
||||||
val = (os.environ.get("MINDNET_HASH_MODE")
|
|
||||||
or os.environ.get("MINDNET_HASH_COMPARE")
|
|
||||||
or "body").strip().lower()
|
|
||||||
if val in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
|
||||||
return "full"
|
|
||||||
if val in ("frontmatter", "fm"):
|
|
||||||
return "frontmatter"
|
|
||||||
return "body"
|
|
||||||
|
|
||||||
def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, Any]]:
|
return out
|
||||||
if not file_path or not os.path.exists(file_path):
|
|
||||||
return "", {}
|
|
||||||
|
def _pick_args(*args, **kwargs) -> Tuple[Optional[str], Optional[dict]]:
|
||||||
|
"""Extrahiert optionale Zusatzargumente wie path und types_cfg."""
|
||||||
|
path = kwargs.get("path") or (args[0] if args else None)
|
||||||
|
types_cfg = kwargs.get("types_cfg") or kwargs.get("types") or None
|
||||||
|
return path, types_cfg
|
||||||
|
|
||||||
|
|
||||||
|
def _env_float(name: str, default: float) -> float:
|
||||||
|
"""Liest einen Float-Wert aus der Umgebung, mit robustem Fallback."""
|
||||||
try:
|
try:
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
return float(os.environ.get(name, default))
|
||||||
raw = f.read()
|
|
||||||
except Exception:
|
except Exception:
|
||||||
return "", {}
|
return default
|
||||||
m = FRONTMATTER_RE.match(raw)
|
|
||||||
fm = {}
|
|
||||||
if m:
|
|
||||||
fm_txt = m.group(1)
|
|
||||||
try:
|
|
||||||
import yaml # lazy
|
|
||||||
fm = yaml.safe_load(fm_txt) or {}
|
|
||||||
except Exception:
|
|
||||||
fm = {}
|
|
||||||
body = raw[m.end():]
|
|
||||||
else:
|
|
||||||
body = raw
|
|
||||||
return body, fm
|
|
||||||
|
|
||||||
def _sha256(s: str) -> str:
|
|
||||||
h = hashlib.sha256()
|
|
||||||
h.update(s.encode("utf-8"))
|
|
||||||
return h.hexdigest()
|
|
||||||
|
|
||||||
def _hash_for(mode: str, *, body: str, fm: Dict[str, Any], normalize: str) -> str:
|
def _ensure_list(x) -> list:
|
||||||
body_n = _normalize_body(body or "", normalize)
|
"""Garantiert eine String-Liste."""
|
||||||
fm_s = _canon_frontmatter(fm or {})
|
if x is None:
|
||||||
if mode == "frontmatter":
|
return []
|
||||||
return _sha256(fm_s)
|
if isinstance(x, list):
|
||||||
if mode == "full":
|
return [str(i) for i in x]
|
||||||
return _sha256(body_n + "\n--FM--\n" + fm_s)
|
if isinstance(x, (set, tuple)):
|
||||||
# default: body
|
return [str(i) for i in x]
|
||||||
return _sha256(body_n)
|
return [str(x)]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Kernfunktion
|
# Type-Registry laden
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def make_note_payload(
|
def _load_types_config(explicit_cfg: Optional[dict] = None) -> dict:
|
||||||
parsed: Any,
|
"""Lädt die Type-Registry aus YAML/JSON oder nutzt ein explizit übergebenes Dict."""
|
||||||
vault_root: Optional[str] = None,
|
if explicit_cfg and isinstance(explicit_cfg, dict):
|
||||||
*,
|
return explicit_cfg
|
||||||
hash_mode: Optional[str] = None,
|
|
||||||
hash_normalize: Optional[str] = None,
|
path = os.getenv("MINDNET_TYPES_FILE") or "./config/types.yaml"
|
||||||
hash_source: Optional[str] = None,
|
if not os.path.isfile(path) or yaml is None:
|
||||||
file_path: Optional[str] = None,
|
return {}
|
||||||
) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Liefert den Note-Payload inkl. Mehrfach-Hashes.
|
|
||||||
- Es werden IMMER die drei Hashes für (body|frontmatter|full) unter
|
|
||||||
'parsed:canonical' erzeugt (Schlüssel: z. B. 'body:parsed:canonical').
|
|
||||||
- Zusätzlich werden – falls die aktuelle Konfig (source/normalize) davon
|
|
||||||
abweicht – die drei Hashes unter den entsprechenden Schlüsseln erzeugt,
|
|
||||||
z. B. 'frontmatter:raw:none'.
|
|
||||||
- 'hash_fulltext' und 'hash_signature' repräsentieren den *aktuellen* Modus.
|
|
||||||
"""
|
|
||||||
# dict oder Objekt akzeptieren
|
|
||||||
if isinstance(parsed, dict):
|
|
||||||
fm = parsed.get("frontmatter") or {}
|
|
||||||
body_parsed = parsed.get("body") or ""
|
|
||||||
path = parsed.get("path") or ""
|
|
||||||
else:
|
|
||||||
fm = getattr(parsed, "frontmatter", {}) or {}
|
|
||||||
body_parsed = getattr(parsed, "body", "") or ""
|
|
||||||
path = getattr(parsed, "path", "") or ""
|
|
||||||
|
|
||||||
# Zielpfad relativieren
|
|
||||||
rel_path = path
|
|
||||||
try:
|
try:
|
||||||
if vault_root:
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
rel = os.path.relpath(path, vault_root)
|
data = yaml.safe_load(f) or {}
|
||||||
rel = rel.replace("\\", "/").lstrip("/")
|
return data if isinstance(data, dict) else {}
|
||||||
rel_path = rel
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
return {}
|
||||||
|
|
||||||
# Konfiguration auflösen
|
|
||||||
mode_resolved = _resolve_hash_mode(hash_mode) # body|frontmatter|full
|
|
||||||
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower() # parsed|raw
|
|
||||||
norm = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower() # canonical|none
|
|
||||||
|
|
||||||
# Body-Quelle laden
|
def _cfg_for_type(note_type: str, reg: dict) -> dict:
|
||||||
raw_body, raw_fm = ("", {})
|
"""Liefert die Konfiguration für einen konkreten Notiztyp aus der Registry."""
|
||||||
if src == "raw":
|
if not isinstance(reg, dict):
|
||||||
raw_body, raw_fm = _read_raw_body_from_file(file_path or path)
|
return {}
|
||||||
if isinstance(raw_fm, dict) and raw_fm:
|
types = reg.get("types") if isinstance(reg.get("types"), dict) else reg
|
||||||
merged_fm = dict(fm)
|
return types.get(note_type, {}) if isinstance(types, dict) else {}
|
||||||
for k, v in raw_fm.items():
|
|
||||||
merged_fm.setdefault(k, v)
|
|
||||||
fm = merged_fm
|
|
||||||
body_for_hash = raw_body
|
|
||||||
else:
|
|
||||||
body_for_hash = body_parsed
|
|
||||||
|
|
||||||
# --- 1) Standard-Tripel (parsed:canonical) immer erzeugen ---
|
|
||||||
std_src = "parsed"
|
def _cfg_defaults(reg: dict) -> dict:
|
||||||
std_norm = "canonical"
|
"""Liefert den Default-Block aus der Registry (defaults/global)."""
|
||||||
std_hashes: Dict[str, str] = {}
|
if not isinstance(reg, dict):
|
||||||
for m in ("body", "frontmatter", "full"):
|
return {}
|
||||||
std_hashes[f"{m}:{std_src}:{std_norm}"] = _hash_for(
|
for key in ("defaults", "default", "global"):
|
||||||
m, body=body_parsed, fm=fm, normalize=std_norm
|
v = reg.get(key)
|
||||||
|
if isinstance(v, dict):
|
||||||
|
return v
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Haupt-API
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def make_note_payload(note: Any, *args, **kwargs) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Baut das Note-Payload für mindnet_notes auf.
|
||||||
|
|
||||||
|
Erwartete Felder im Payload:
|
||||||
|
- note_id: stabile ID aus Frontmatter (id) oder Note-Objekt
|
||||||
|
- title: Titel der Notiz
|
||||||
|
- type: Notiztyp (z. B. concept, project, journal, ...)
|
||||||
|
- path: Pfad im Vault
|
||||||
|
- retriever_weight: effektives Gewicht für den Retriever
|
||||||
|
- chunk_profile: Profil für Chunking (short|medium|long|default|...)
|
||||||
|
- edge_defaults: Liste von Kanten-Typen, die als Defaults gelten
|
||||||
|
"""
|
||||||
|
n = _as_dict(note)
|
||||||
|
path_arg, types_cfg_explicit = _pick_args(*args, **kwargs)
|
||||||
|
reg = _load_types_config(types_cfg_explicit)
|
||||||
|
|
||||||
|
fm = n.get("frontmatter") or {}
|
||||||
|
fm_type = fm.get("type") or n.get("type") or "concept"
|
||||||
|
note_type = str(fm_type)
|
||||||
|
|
||||||
|
cfg_type = _cfg_for_type(note_type, reg)
|
||||||
|
cfg_def = _cfg_defaults(reg)
|
||||||
|
|
||||||
|
# --- retriever_weight: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
|
||||||
|
default_rw = _env_float("MINDNET_DEFAULT_RETRIEVER_WEIGHT", 1.0)
|
||||||
|
retriever_weight = fm.get("retriever_weight")
|
||||||
|
if retriever_weight is None:
|
||||||
|
retriever_weight = cfg_type.get(
|
||||||
|
"retriever_weight",
|
||||||
|
cfg_def.get("retriever_weight", default_rw),
|
||||||
)
|
)
|
||||||
|
try:
|
||||||
|
retriever_weight = float(retriever_weight)
|
||||||
|
except Exception:
|
||||||
|
retriever_weight = default_rw
|
||||||
|
|
||||||
# Convenience-Felder (für Tools)
|
# --- chunk_profile: Frontmatter > Typ-Config > Registry-Defaults > ENV ---
|
||||||
hash_body = std_hashes["body:parsed:canonical"]
|
chunk_profile = fm.get("chunk_profile")
|
||||||
hash_frontmatter = std_hashes["frontmatter:parsed:canonical"]
|
if chunk_profile is None:
|
||||||
hash_full = std_hashes["full:parsed:canonical"]
|
chunk_profile = cfg_type.get(
|
||||||
|
"chunk_profile",
|
||||||
|
cfg_def.get(
|
||||||
|
"chunk_profile",
|
||||||
|
os.environ.get("MINDNET_DEFAULT_CHUNK_PROFILE", "medium"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if not isinstance(chunk_profile, str):
|
||||||
|
chunk_profile = "medium"
|
||||||
|
|
||||||
# --- 2) Hashes für die *aktuelle* Konfiguration (falls abweichend) ---
|
# --- edge_defaults: Frontmatter > Typ-Config > Registry-Defaults ---
|
||||||
cur_hashes: Dict[str, str] = {}
|
edge_defaults = fm.get("edge_defaults")
|
||||||
if not (src == std_src and norm == std_norm):
|
if edge_defaults is None:
|
||||||
for m in ("body", "frontmatter", "full"):
|
edge_defaults = cfg_type.get(
|
||||||
cur_hashes[f"{m}:{src}:{norm}"] = _hash_for(
|
"edge_defaults",
|
||||||
m, body=body_for_hash, fm=fm, normalize=norm
|
cfg_def.get("edge_defaults", []),
|
||||||
)
|
)
|
||||||
|
edge_defaults = _ensure_list(edge_defaults)
|
||||||
|
|
||||||
# --- 3) Aktueller Modus für Backwards-Compat Felder ---
|
# --- Basis-Metadaten (IDs, Titel, Pfad) ---
|
||||||
current_hash = _hash_for(mode_resolved, body=body_for_hash, fm=fm, normalize=norm)
|
note_id = n.get("note_id") or n.get("id") or fm.get("id")
|
||||||
hash_signature = f"{mode_resolved}:{src}:{norm}:{current_hash}"
|
title = n.get("title") or fm.get("title") or ""
|
||||||
|
path = n.get("path") or path_arg
|
||||||
# Wikilinks (Note-Ebene)
|
if isinstance(path, pathlib.Path):
|
||||||
refs = list(dict.fromkeys(extract_wikilinks(body_parsed))) if body_parsed else []
|
path = str(path)
|
||||||
|
|
||||||
payload: Dict[str, Any] = {
|
payload: Dict[str, Any] = {
|
||||||
"note_id": fm.get("id") or fm.get("note_id"),
|
"note_id": note_id,
|
||||||
"title": fm.get("title"),
|
"title": title,
|
||||||
"type": fm.get("type"),
|
"type": note_type,
|
||||||
"status": fm.get("status"),
|
"path": path or "",
|
||||||
"created": fm.get("created"),
|
"retriever_weight": retriever_weight,
|
||||||
"updated": fm.get("updated"),
|
"chunk_profile": chunk_profile,
|
||||||
"path": rel_path or fm.get("path"),
|
"edge_defaults": edge_defaults,
|
||||||
"tags": fm.get("tags"),
|
|
||||||
# Volltext für verlustfreien Export
|
|
||||||
"fulltext": body_parsed,
|
|
||||||
# Backwards-Compat:
|
|
||||||
"hash_fulltext": current_hash,
|
|
||||||
"hash_signature": hash_signature,
|
|
||||||
# Option C: Mehrfach-Hashes
|
|
||||||
"hashes": {**std_hashes, **cur_hashes},
|
|
||||||
"hash_body": hash_body,
|
|
||||||
"hash_frontmatter": hash_frontmatter,
|
|
||||||
"hash_full": hash_full,
|
|
||||||
# Fallback-Refs
|
|
||||||
"references": refs,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for k in ("area", "project", "source", "lang", "slug", "aliases"):
|
# Tags / Keywords übernehmen
|
||||||
if k in fm:
|
tags = fm.get("tags") or fm.get("keywords") or n.get("tags")
|
||||||
payload[k] = fm[k]
|
if tags:
|
||||||
|
payload["tags"] = _ensure_list(tags)
|
||||||
|
|
||||||
|
# Zeitliche Metadaten (sofern vorhanden)
|
||||||
|
for k in ("created", "modified", "date"):
|
||||||
|
v = fm.get(k) or n.get(k)
|
||||||
|
if v:
|
||||||
|
payload[k] = str(v)
|
||||||
|
|
||||||
|
# JSON-Roundtrip zur harten Validierung (ASCII beibehalten)
|
||||||
|
json.loads(json.dumps(payload, ensure_ascii=False))
|
||||||
|
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# CLI – Sichtprüfung
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def _cli() -> None:
|
|
||||||
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
|
|
||||||
ap.add_argument("--from-file", dest="src", required=True)
|
|
||||||
ap.add_argument("--vault-root", dest="vault_root", default=None)
|
|
||||||
ap.add_argument("--print", dest="do_print", action="store_true")
|
|
||||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None)
|
|
||||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
|
||||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
|
|
||||||
args = ap.parse_args()
|
|
||||||
|
|
||||||
parsed = read_markdown(args.src)
|
|
||||||
payload = make_note_payload(
|
|
||||||
parsed,
|
|
||||||
vault_root=args.vault_root,
|
|
||||||
hash_mode=args.hash_mode,
|
|
||||||
hash_normalize=args.hash_normalize,
|
|
||||||
hash_source=args.hash_source,
|
|
||||||
file_path=args.src,
|
|
||||||
)
|
|
||||||
if args.do_print:
|
|
||||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
if __name__ == "__main__": # pragma: no cover
|
|
||||||
_cli()
|
|
||||||
+147
-92
@@ -1,124 +1,179 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Name: app/core/qdrant.py
|
app/core/qdrant.py
|
||||||
Version: v1.4.0 (2025-09-09)
|
Version: 2.2.0 (2025-11-11)
|
||||||
|
|
||||||
Kurzbeschreibung:
|
Aufgabe
|
||||||
Qdrant-Client & Collection-Setup für mindnet.
|
-------
|
||||||
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
|
- Zentraler Qdrant-Zugriff (Client, Config)
|
||||||
- Edges-Collection nutzt 1D Dummy-Vektor.
|
- Collection-Anlage (notes/chunks/edges)
|
||||||
- NEW: ensure_payload_indexes(...) legt sinnvolle Payload-Indizes an.
|
- **Payload-Indizes sicherstellen** (idempotent)
|
||||||
|
|
||||||
Aufruf:
|
Hinweis
|
||||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
-------
|
||||||
|
Diese Datei ist als Drop-in-Ersatz gedacht, falls in deinem Projekt noch keine
|
||||||
|
robuste ensure_payload_indexes()-Implementierung vorliegt. Die Signaturen
|
||||||
|
bleiben kompatibel zu scripts.import_markdown und scripts.reset_qdrant.
|
||||||
|
|
||||||
|
API-Notizen
|
||||||
|
-----------
|
||||||
|
- Payload-Indizes werden mit `create_payload_index` angelegt.
|
||||||
|
- Typen stammen aus `qdrant_client.http.models.PayloadSchemaType`:
|
||||||
|
KEYWORD | TEXT | INTEGER | FLOAT | BOOL | GEO | DATETIME
|
||||||
|
- Für häufige Filterfelder (note_id, kind, scope, type, tags, ...) legen wir
|
||||||
|
Indizes an. Das ist laut Qdrant-Doku Best Practice für performante Filter.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple, Dict, List
|
||||||
|
|
||||||
from qdrant_client import QdrantClient
|
from qdrant_client import QdrantClient
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Konfiguration
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class QdrantConfig:
|
class QdrantConfig:
|
||||||
url: str
|
host: Optional[str] = None
|
||||||
api_key: Optional[str]
|
port: Optional[int] = None
|
||||||
prefix: str
|
url: Optional[str] = None
|
||||||
dim: int
|
api_key: Optional[str] = None
|
||||||
|
prefix: str = "mindnet"
|
||||||
|
dim: int = 384
|
||||||
|
distance: str = "Cosine" # Cosine | Dot | Euclid
|
||||||
|
on_disk_payload: bool = True
|
||||||
|
|
||||||
@staticmethod
|
@classmethod
|
||||||
def from_env() -> "QdrantConfig":
|
def from_env(cls) -> "QdrantConfig":
|
||||||
url = os.getenv("QDRANT_URL")
|
# Entweder URL ODER Host/Port, API-Key optional
|
||||||
if not url:
|
url = os.getenv("QDRANT_URL") or None
|
||||||
host = os.getenv("QDRANT_HOST", "127.0.0.1")
|
host = os.getenv("QDRANT_HOST") or None
|
||||||
port = int(os.getenv("QDRANT_PORT", "6333"))
|
port = os.getenv("QDRANT_PORT")
|
||||||
url = f"http://{host}:{port}"
|
port = int(port) if port else None
|
||||||
api_key = os.getenv("QDRANT_API_KEY") or None
|
api_key = os.getenv("QDRANT_API_KEY") or None
|
||||||
prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
|
prefix = os.getenv("COLLECTION_PREFIX") or "mindnet"
|
||||||
dim = int(os.getenv("VECTOR_DIM", "384"))
|
dim = int(os.getenv("VECTOR_DIM") or 384)
|
||||||
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
|
distance = os.getenv("DISTANCE", "Cosine")
|
||||||
|
on_disk_payload = (os.getenv("ON_DISK_PAYLOAD", "true").lower() == "true")
|
||||||
|
return cls(
|
||||||
|
host=host, port=port, url=url, api_key=api_key,
|
||||||
|
prefix=prefix, dim=dim, distance=distance, on_disk_payload=on_disk_payload
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
||||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
|
# QdrantClient akzeptiert entweder url=... oder host/port
|
||||||
|
if cfg.url:
|
||||||
|
return QdrantClient(url=cfg.url, api_key=cfg.api_key, timeout=60.0)
|
||||||
|
return QdrantClient(host=cfg.host or "127.0.0.1", port=cfg.port or 6333, api_key=cfg.api_key, timeout=60.0)
|
||||||
|
|
||||||
|
|
||||||
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
|
# ---------------------------------------------------------------------------
|
||||||
if not client.collection_exists(name):
|
# Collections
|
||||||
client.create_collection(
|
# ---------------------------------------------------------------------------
|
||||||
collection_name=name,
|
|
||||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
|
|
||||||
if not client.collection_exists(name):
|
|
||||||
client.create_collection(
|
|
||||||
collection_name=name,
|
|
||||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _create_edges(client: QdrantClient, name: str) -> None:
|
|
||||||
if not client.collection_exists(name):
|
|
||||||
client.create_collection(
|
|
||||||
collection_name=name,
|
|
||||||
vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT), # 1D-Dummy
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
|
|
||||||
notes = f"{prefix}_notes"
|
|
||||||
chunks = f"{prefix}_chunks"
|
|
||||||
edges = f"{prefix}_edges"
|
|
||||||
|
|
||||||
_create_notes(client, notes, dim)
|
|
||||||
_create_chunks(client, chunks, dim)
|
|
||||||
|
|
||||||
if client.collection_exists(edges):
|
|
||||||
try:
|
|
||||||
info = client.get_collection(edges)
|
|
||||||
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
|
|
||||||
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
|
|
||||||
except Exception:
|
|
||||||
has_vectors = True
|
|
||||||
if not has_vectors:
|
|
||||||
if destructive:
|
|
||||||
client.delete_collection(edges)
|
|
||||||
_create_edges(client, edges)
|
|
||||||
else:
|
|
||||||
print(f"[ensure_collections] WARN: '{edges}' ohne VectorConfig; destructive=False.", flush=True)
|
|
||||||
else:
|
|
||||||
_create_edges(client, edges)
|
|
||||||
|
|
||||||
|
|
||||||
def collection_names(prefix: str) -> Tuple[str, str, str]:
|
def collection_names(prefix: str) -> Tuple[str, str, str]:
|
||||||
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
|
||||||
# -------------------------------
|
def _vector_params(dim: int, distance: str) -> rest.VectorParams:
|
||||||
# NEW: Payload-Indexing
|
# Distance: "Cosine" | "Dot" | "Euclid"
|
||||||
# -------------------------------
|
dist = getattr(rest.Distance, distance.capitalize(), rest.Distance.COSINE)
|
||||||
|
return rest.VectorParams(size=dim, distance=dist)
|
||||||
|
|
||||||
def _safe_create_index(client: QdrantClient, col: str, field: str, schema: rest.PayloadSchemaType):
|
|
||||||
try:
|
def ensure_collections(client: QdrantClient, prefix: str, dim: int) -> None:
|
||||||
client.create_payload_index(
|
"""Legt mindnet_notes, mindnet_chunks, mindnet_edges an (falls nicht vorhanden)."""
|
||||||
collection_name=col,
|
notes, chunks, edges = collection_names(prefix)
|
||||||
field_name=field,
|
|
||||||
field_schema=schema,
|
# notes
|
||||||
|
if not client.collection_exists(notes):
|
||||||
|
client.create_collection(
|
||||||
|
collection_name=notes,
|
||||||
|
vectors_config=_vector_params(dim, os.getenv("DISTANCE", "Cosine")),
|
||||||
|
on_disk_payload=True,
|
||||||
)
|
)
|
||||||
except Exception:
|
# chunks
|
||||||
# bereits vorhanden oder nicht unterstütztes Schema → ignorieren
|
if not client.collection_exists(chunks):
|
||||||
pass
|
client.create_collection(
|
||||||
|
collection_name=chunks,
|
||||||
|
vectors_config=_vector_params(dim, os.getenv("DISTANCE", "Cosine")),
|
||||||
|
on_disk_payload=True,
|
||||||
|
)
|
||||||
|
# edges (Dummy-Vektor, Filter via Payload)
|
||||||
|
if not client.collection_exists(edges):
|
||||||
|
client.create_collection(
|
||||||
|
collection_name=edges,
|
||||||
|
vectors_config=_vector_params(1, "Dot"),
|
||||||
|
on_disk_payload=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Payload-Indizes
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _ensure_index(client: QdrantClient, collection: str, field: str, schema: rest.PayloadSchemaType) -> None:
|
||||||
|
"""Idempotentes Anlegen eines Payload-Indexes für ein Feld."""
|
||||||
|
try:
|
||||||
|
client.create_payload_index(collection_name=collection, field_name=field, field_schema=schema, wait=True)
|
||||||
|
except Exception as e:
|
||||||
|
# Fehler ignorieren, falls Index bereits existiert oder Server "already indexed" meldet.
|
||||||
|
# Für Debugging ggf. Logging ergänzen.
|
||||||
|
_ = e
|
||||||
|
|
||||||
|
|
||||||
def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
|
def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
|
||||||
|
"""
|
||||||
|
Stellt sicher, dass alle benötigten Payload-Indizes existieren.
|
||||||
|
- notes: note_id(KEYWORD), type(KEYWORD), title(TEXT), updated(INTEGER), tags(KEYWORD)
|
||||||
|
- chunks: note_id(KEYWORD), chunk_id(KEYWORD), index(INTEGER), type(KEYWORD), tags(KEYWORD)
|
||||||
|
- edges: note_id(KEYWORD), kind(KEYWORD), scope(KEYWORD), source_id(KEYWORD), target_id(KEYWORD), chunk_id(KEYWORD)
|
||||||
|
"""
|
||||||
notes, chunks, edges = collection_names(prefix)
|
notes, chunks, edges = collection_names(prefix)
|
||||||
# Notes
|
|
||||||
_safe_create_index(client, notes, "note_id", rest.PayloadSchemaType.KEYWORD)
|
# NOTES
|
||||||
# Chunks
|
for field, schema in [
|
||||||
_safe_create_index(client, chunks, "note_id", rest.PayloadSchemaType.KEYWORD)
|
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
_safe_create_index(client, chunks, "chunk_index", rest.PayloadSchemaType.INTEGER)
|
("type", rest.PayloadSchemaType.KEYWORD),
|
||||||
# Edges
|
("title", rest.PayloadSchemaType.TEXT),
|
||||||
for f in ("kind", "scope", "source_id", "target_id", "note_id"):
|
("updated", rest.PayloadSchemaType.INTEGER),
|
||||||
_safe_create_index(client, edges, f, rest.PayloadSchemaType.KEYWORD)
|
("tags", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
]:
|
||||||
|
_ensure_index(client, notes, field, schema)
|
||||||
|
|
||||||
|
# CHUNKS
|
||||||
|
for field, schema in [
|
||||||
|
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("chunk_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("index", rest.PayloadSchemaType.INTEGER),
|
||||||
|
("type", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("tags", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
]:
|
||||||
|
_ensure_index(client, chunks, field, schema)
|
||||||
|
|
||||||
|
# EDGES
|
||||||
|
for field, schema in [
|
||||||
|
("note_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("kind", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("scope", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("source_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("target_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
("chunk_id", rest.PayloadSchemaType.KEYWORD),
|
||||||
|
]:
|
||||||
|
_ensure_index(client, edges, field, schema)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"QdrantConfig",
|
||||||
|
"get_client",
|
||||||
|
"ensure_collections",
|
||||||
|
"ensure_payload_indexes",
|
||||||
|
"collection_names",
|
||||||
|
]
|
||||||
+233
-69
@@ -1,55 +1,38 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
app/core/qdrant_points.py
|
app/core/qdrant_points.py - robust points helpers for Qdrant
|
||||||
|
|
||||||
Zweck
|
- Single source of truth for building PointStruct for notes/chunks/edges
|
||||||
- Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
|
- Backward-compatible payloads for edges
|
||||||
- Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
|
- Handles both Single-Vector and Named-Vector collections
|
||||||
- alt: {'edge_type','src_id','dst_id', ...}
|
- Deterministic overrides via ENV to avoid auto-detection traps:
|
||||||
- neu: {'kind','source_id','target_id', ...}
|
* NOTES_VECTOR_NAME, CHUNKS_VECTOR_NAME, EDGES_VECTOR_NAME
|
||||||
|
* MINDNET_VECTOR_NAME (fallback)
|
||||||
|
> Set to a concrete name (e.g. "text") to force Named-Vector with that name
|
||||||
|
> Set to "__single__" (or "single") to force Single-Vector
|
||||||
|
|
||||||
Version
|
Version: 1.5.0 (2025-11-08)
|
||||||
- 1.3 (2025-09-08)
|
|
||||||
|
|
||||||
Änderungen (ggü. 1.2)
|
|
||||||
- points_for_edges() akzeptiert jetzt beide Edge-Schemata.
|
|
||||||
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
|
|
||||||
stabile 'edge_id' zurück in die Payload.
|
|
||||||
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
|
|
||||||
|
|
||||||
Aufruf / Verwendung
|
|
||||||
- Wird von Import-/Backfill-Skripten via:
|
|
||||||
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
|
|
||||||
eingebunden. Keine CLI.
|
|
||||||
|
|
||||||
Hinweise
|
|
||||||
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
|
|
||||||
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
from typing import List, Tuple
|
from typing import List, Tuple, Iterable, Optional, Dict, Any
|
||||||
from qdrant_client.http import models as rest
|
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
|
||||||
|
# --------------------- ID helpers ---------------------
|
||||||
|
|
||||||
|
def _to_uuid(stable_key: str) -> str:
|
||||||
|
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
||||||
|
|
||||||
def _names(prefix: str) -> Tuple[str, str, str]:
|
def _names(prefix: str) -> Tuple[str, str, str]:
|
||||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
# --------------------- Points builders ---------------------
|
||||||
|
|
||||||
def _to_uuid(stable_key: str) -> str:
|
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
|
||||||
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
|
|
||||||
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
|
|
||||||
|
|
||||||
|
|
||||||
def points_for_note(
|
|
||||||
prefix: str,
|
|
||||||
note_payload: dict,
|
|
||||||
note_vec: List[float] | None,
|
|
||||||
dim: int,
|
|
||||||
) -> Tuple[str, List[rest.PointStruct]]:
|
|
||||||
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
|
|
||||||
notes_col, _, _ = _names(prefix)
|
notes_col, _, _ = _names(prefix)
|
||||||
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
vector = note_vec if note_vec is not None else [0.0] * int(dim)
|
||||||
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
|
||||||
@@ -57,18 +40,7 @@ def points_for_note(
|
|||||||
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
|
||||||
return notes_col, [pt]
|
return notes_col, [pt]
|
||||||
|
|
||||||
|
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
|
||||||
def points_for_chunks(
|
|
||||||
prefix: str,
|
|
||||||
chunk_payloads: List[dict],
|
|
||||||
vectors: List[List[float]],
|
|
||||||
) -> Tuple[str, List[rest.PointStruct]]:
|
|
||||||
"""
|
|
||||||
Chunks-Collection: erwartet pro Chunk einen Vektor.
|
|
||||||
Robustheit:
|
|
||||||
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
|
|
||||||
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
|
|
||||||
"""
|
|
||||||
_, chunks_col, _ = _names(prefix)
|
_, chunks_col, _ = _names(prefix)
|
||||||
points: List[rest.PointStruct] = []
|
points: List[rest.PointStruct] = []
|
||||||
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
|
||||||
@@ -81,22 +53,12 @@ def points_for_chunks(
|
|||||||
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
|
||||||
return chunks_col, points
|
return chunks_col, points
|
||||||
|
|
||||||
|
|
||||||
def _normalize_edge_payload(pl: dict) -> dict:
|
def _normalize_edge_payload(pl: dict) -> dict:
|
||||||
"""
|
|
||||||
Sorgt für kompatible Feldnamen.
|
|
||||||
akzeptiert:
|
|
||||||
- neu: kind, source_id, target_id, seq?
|
|
||||||
- alt: edge_type, src_id, dst_id, order?/index?
|
|
||||||
schreibt zurück: kind, source_id, target_id, seq?
|
|
||||||
"""
|
|
||||||
# bereits neu?
|
|
||||||
kind = pl.get("kind") or pl.get("edge_type") or "edge"
|
kind = pl.get("kind") or pl.get("edge_type") or "edge"
|
||||||
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
|
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
|
||||||
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
|
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
|
||||||
seq = pl.get("seq") or pl.get("order") or pl.get("index")
|
seq = pl.get("seq") or pl.get("order") or pl.get("index")
|
||||||
|
|
||||||
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
|
|
||||||
pl.setdefault("kind", kind)
|
pl.setdefault("kind", kind)
|
||||||
pl.setdefault("source_id", source_id)
|
pl.setdefault("source_id", source_id)
|
||||||
pl.setdefault("target_id", target_id)
|
pl.setdefault("target_id", target_id)
|
||||||
@@ -104,18 +66,11 @@ def _normalize_edge_payload(pl: dict) -> dict:
|
|||||||
pl["seq"] = seq
|
pl["seq"] = seq
|
||||||
return pl
|
return pl
|
||||||
|
|
||||||
|
|
||||||
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
|
||||||
"""
|
|
||||||
Edges-Collection mit 1D-Dummy-Vektor.
|
|
||||||
- Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
|
|
||||||
- Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
|
|
||||||
"""
|
|
||||||
_, _, edges_col = _names(prefix)
|
_, _, edges_col = _names(prefix)
|
||||||
points: List[rest.PointStruct] = []
|
points: List[rest.PointStruct] = []
|
||||||
for raw in edge_payloads:
|
for raw in edge_payloads:
|
||||||
pl = _normalize_edge_payload(raw)
|
pl = _normalize_edge_payload(raw)
|
||||||
|
|
||||||
edge_id = pl.get("edge_id")
|
edge_id = pl.get("edge_id")
|
||||||
if not edge_id:
|
if not edge_id:
|
||||||
kind = pl.get("kind", "edge")
|
kind = pl.get("kind", "edge")
|
||||||
@@ -124,13 +79,222 @@ def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[
|
|||||||
seq = pl.get("seq") or ""
|
seq = pl.get("seq") or ""
|
||||||
edge_id = f"{kind}:{s}->{t}#{seq}"
|
edge_id = f"{kind}:{s}->{t}#{seq}"
|
||||||
pl["edge_id"] = edge_id
|
pl["edge_id"] = edge_id
|
||||||
|
|
||||||
point_id = _to_uuid(edge_id)
|
point_id = _to_uuid(edge_id)
|
||||||
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
|
||||||
return edges_col, points
|
return edges_col, points
|
||||||
|
|
||||||
|
# --------------------- Vector schema & overrides ---------------------
|
||||||
|
|
||||||
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
|
def _preferred_name(candidates: List[str]) -> str:
|
||||||
|
for k in ("text", "default", "embedding", "content"):
|
||||||
|
if k in candidates:
|
||||||
|
return k
|
||||||
|
return sorted(candidates)[0]
|
||||||
|
|
||||||
|
def _env_override_for_collection(collection: str) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Returns:
|
||||||
|
- "__single__" to force single-vector
|
||||||
|
- concrete name (str) to force named-vector with that name
|
||||||
|
- None to auto-detect
|
||||||
|
"""
|
||||||
|
base = os.getenv("MINDNET_VECTOR_NAME")
|
||||||
|
if collection.endswith("_notes"):
|
||||||
|
base = os.getenv("NOTES_VECTOR_NAME", base)
|
||||||
|
elif collection.endswith("_chunks"):
|
||||||
|
base = os.getenv("CHUNKS_VECTOR_NAME", base)
|
||||||
|
elif collection.endswith("_edges"):
|
||||||
|
base = os.getenv("EDGES_VECTOR_NAME", base)
|
||||||
|
|
||||||
|
if not base:
|
||||||
|
return None
|
||||||
|
val = base.strip()
|
||||||
|
if val.lower() in ("__single__", "single"):
|
||||||
|
return "__single__"
|
||||||
|
return val # concrete name
|
||||||
|
|
||||||
|
def _get_vector_schema(client: QdrantClient, collection_name: str) -> dict:
|
||||||
|
"""
|
||||||
|
Return {"kind": "single", "size": int} or {"kind": "named", "names": [...], "primary": str}.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=collection_name)
|
||||||
|
vecs = getattr(info, "vectors", None)
|
||||||
|
# Single-vector config
|
||||||
|
if hasattr(vecs, "size") and isinstance(vecs.size, int):
|
||||||
|
return {"kind": "single", "size": vecs.size}
|
||||||
|
# Named-vectors config (dict-like in .config)
|
||||||
|
cfg = getattr(vecs, "config", None)
|
||||||
|
if isinstance(cfg, dict) and cfg:
|
||||||
|
names = list(cfg.keys())
|
||||||
|
if names:
|
||||||
|
return {"kind": "named", "names": names, "primary": _preferred_name(names)}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"kind": "single", "size": None}
|
||||||
|
|
||||||
|
def _as_named(points: List[rest.PointStruct], name: str) -> List[rest.PointStruct]:
|
||||||
|
out: List[rest.PointStruct] = []
|
||||||
|
for pt in points:
|
||||||
|
vec = getattr(pt, "vector", None)
|
||||||
|
if isinstance(vec, dict):
|
||||||
|
if name in vec:
|
||||||
|
out.append(pt)
|
||||||
|
else:
|
||||||
|
# take any existing entry; if empty dict fallback to [0.0]
|
||||||
|
fallback_vec = None
|
||||||
|
try:
|
||||||
|
fallback_vec = list(next(iter(vec.values())))
|
||||||
|
except Exception:
|
||||||
|
fallback_vec = [0.0]
|
||||||
|
out.append(rest.PointStruct(id=pt.id, vector={name: fallback_vec}, payload=pt.payload))
|
||||||
|
elif vec is not None:
|
||||||
|
out.append(rest.PointStruct(id=pt.id, vector={name: vec}, payload=pt.payload))
|
||||||
|
else:
|
||||||
|
out.append(pt)
|
||||||
|
return out
|
||||||
|
|
||||||
|
# --------------------- Qdrant ops ---------------------
|
||||||
|
|
||||||
|
def upsert_batch(client: QdrantClient, collection: str, points: List[rest.PointStruct]) -> None:
|
||||||
if not points:
|
if not points:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# 1) ENV overrides come first
|
||||||
|
override = _env_override_for_collection(collection)
|
||||||
|
if override == "__single__":
|
||||||
|
client.upsert(collection_name=collection, points=points, wait=True)
|
||||||
|
return
|
||||||
|
elif isinstance(override, str):
|
||||||
|
client.upsert(collection_name=collection, points=_as_named(points, override), wait=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 2) Auto-detect schema
|
||||||
|
schema = _get_vector_schema(client, collection)
|
||||||
|
if schema.get("kind") == "named":
|
||||||
|
name = schema.get("primary") or _preferred_name(schema.get("names") or [])
|
||||||
|
client.upsert(collection_name=collection, points=_as_named(points, name), wait=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 3) Fallback single-vector
|
||||||
client.upsert(collection_name=collection, points=points, wait=True)
|
client.upsert(collection_name=collection, points=points, wait=True)
|
||||||
|
|
||||||
|
# --- Optional search helpers ---
|
||||||
|
|
||||||
|
def _filter_any(field: str, values: Iterable[str]) -> rest.Filter:
|
||||||
|
return rest.Filter(should=[rest.FieldCondition(key=field, match=rest.MatchValue(value=v)) for v in values])
|
||||||
|
|
||||||
|
def _merge_filters(*filters: Optional[rest.Filter]) -> Optional[rest.Filter]:
|
||||||
|
fs = [f for f in filters if f is not None]
|
||||||
|
if not fs:
|
||||||
|
return None
|
||||||
|
if len(fs) == 1:
|
||||||
|
return fs[0]
|
||||||
|
must = []
|
||||||
|
for f in fs:
|
||||||
|
if getattr(f, "must", None):
|
||||||
|
must.extend(f.must)
|
||||||
|
if getattr(f, "should", None):
|
||||||
|
must.append(rest.Filter(should=f.should))
|
||||||
|
return rest.Filter(must=must)
|
||||||
|
|
||||||
|
def _filter_from_dict(filters: Optional[Dict[str, Any]]) -> Optional[rest.Filter]:
|
||||||
|
if not filters:
|
||||||
|
return None
|
||||||
|
parts = []
|
||||||
|
for k, v in filters.items():
|
||||||
|
if isinstance(v, (list, tuple, set)):
|
||||||
|
parts.append(_filter_any(k, [str(x) for x in v]))
|
||||||
|
else:
|
||||||
|
parts.append(rest.Filter(must=[rest.FieldCondition(key=k, match=rest.MatchValue(value=v))]))
|
||||||
|
return _merge_filters(*parts)
|
||||||
|
|
||||||
|
def search_chunks_by_vector(client: QdrantClient, prefix: str, vector: List[float], top: int = 10, filters: Optional[Dict[str, Any]] = None) -> List[Tuple[str, float, dict]]:
|
||||||
|
_, chunks_col, _ = _names(prefix)
|
||||||
|
flt = _filter_from_dict(filters)
|
||||||
|
res = client.search(collection_name=chunks_col, query_vector=vector, limit=top, with_payload=True, with_vectors=False, query_filter=flt)
|
||||||
|
out: List[Tuple[str, float, dict]] = []
|
||||||
|
for r in res:
|
||||||
|
out.append((str(r.id), float(r.score), dict(r.payload or {})))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# --- Edge retrieval helper ---
|
||||||
|
|
||||||
|
def get_edges_for_sources(
|
||||||
|
client: QdrantClient,
|
||||||
|
prefix: str,
|
||||||
|
source_ids: Iterable[str],
|
||||||
|
edge_types: Optional[Iterable[str]] = None,
|
||||||
|
limit: int = 2048,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Retrieve edge payloads from the <prefix>_edges collection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: QdrantClient instance.
|
||||||
|
prefix: Mindnet collection prefix (e.g. "mindnet").
|
||||||
|
source_ids: Iterable of source_id values (typically chunk_ids or note_ids).
|
||||||
|
edge_types: Optional iterable of edge kinds (e.g. ["references", "depends_on"]). If None,
|
||||||
|
all kinds are returned.
|
||||||
|
limit: Maximum number of edge payloads to return.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A list of edge payload dicts, e.g.:
|
||||||
|
{
|
||||||
|
"note_id": "...",
|
||||||
|
"chunk_id": "...",
|
||||||
|
"kind": "references" | "depends_on" | ...,
|
||||||
|
"scope": "chunk",
|
||||||
|
"source_id": "...",
|
||||||
|
"target_id": "...",
|
||||||
|
"rule_id": "...",
|
||||||
|
"confidence": 0.7,
|
||||||
|
...
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
source_ids = list(source_ids)
|
||||||
|
if not source_ids or limit <= 0:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Resolve collection name
|
||||||
|
_, _, edges_col = _names(prefix)
|
||||||
|
|
||||||
|
# Build filter: source_id IN source_ids
|
||||||
|
src_filter = _filter_any("source_id", [str(s) for s in source_ids])
|
||||||
|
|
||||||
|
# Optional: kind IN edge_types
|
||||||
|
kind_filter = None
|
||||||
|
if edge_types:
|
||||||
|
kind_filter = _filter_any("kind", [str(k) for k in edge_types])
|
||||||
|
|
||||||
|
flt = _merge_filters(src_filter, kind_filter)
|
||||||
|
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
next_page = None
|
||||||
|
remaining = int(limit)
|
||||||
|
|
||||||
|
# Use paginated scroll API; we don't need vectors, only payloads.
|
||||||
|
while remaining > 0:
|
||||||
|
batch_limit = min(256, remaining)
|
||||||
|
res, next_page = client.scroll(
|
||||||
|
collection_name=edges_col,
|
||||||
|
scroll_filter=flt,
|
||||||
|
limit=batch_limit,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not res:
|
||||||
|
break
|
||||||
|
|
||||||
|
for r in res:
|
||||||
|
out.append(dict(r.payload or {}))
|
||||||
|
remaining -= 1
|
||||||
|
if remaining <= 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
if next_page is None or remaining <= 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
return out
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
"""
|
||||||
|
app/core/ranking.py — Kombiniertes Scoring (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Zusammenführen von semantischem Score (normalisiert), Edge-Bonus und
|
||||||
|
Centrality-Bonus in einen Gesamtscore für die Ergebnisreihung.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
WP-04 Ranking-Formel (w_sem, w_edge, w_cent)
|
||||||
|
Nutzung:
|
||||||
|
from app.core.ranking import combine_scores
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List, Tuple, Dict
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_scores(values: List[float]) -> List[float]:
|
||||||
|
"""Min-Max-Normalisierung über die Kandidatenmenge (Fallback 0.5 bei Konstanz)."""
|
||||||
|
if not values:
|
||||||
|
return values
|
||||||
|
lo, hi = min(values), max(values)
|
||||||
|
if hi - lo < 1e-9:
|
||||||
|
return [0.5] * len(values)
|
||||||
|
return [(v - lo) / (hi - lo) for v in values]
|
||||||
|
|
||||||
|
|
||||||
|
def combine_scores(
|
||||||
|
hits: List[Tuple[str, float, dict]], # (id, semantic_score, payload)
|
||||||
|
edge_bonus_map: Dict[str, float],
|
||||||
|
centrality_map: Dict[str, float],
|
||||||
|
w_sem: float = 0.70,
|
||||||
|
w_edge: float = 0.25,
|
||||||
|
w_cent: float = 0.05,
|
||||||
|
) -> List[Tuple[str, float, float, float, float]]:
|
||||||
|
"""
|
||||||
|
Liefert Liste von (point_id, total_score, edge_bonus, centrality_bonus, raw_semantic_score),
|
||||||
|
absteigend nach total_score sortiert.
|
||||||
|
"""
|
||||||
|
sem = [h[1] for h in hits]
|
||||||
|
sem_n = normalize_scores(sem)
|
||||||
|
out = []
|
||||||
|
for (pid, s, payload), s_norm in zip(hits, sem_n):
|
||||||
|
e = edge_bonus_map.get(pid, 0.0)
|
||||||
|
c = centrality_map.get(pid, 0.0)
|
||||||
|
total = w_sem * s_norm + w_edge * e + w_cent * c
|
||||||
|
out.append((pid, total, e, c, s))
|
||||||
|
out.sort(key=lambda t: t[1], reverse=True)
|
||||||
|
return out
|
||||||
@@ -0,0 +1,336 @@
|
|||||||
|
"""
|
||||||
|
app/core/retriever.py — Hybrider Such-Algorithmus
|
||||||
|
|
||||||
|
Version:
|
||||||
|
0.5.3 (WP-06 Fix: Populate 'payload' in QueryHit for meta-data access)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from functools import lru_cache
|
||||||
|
from typing import Any, Dict, List, Tuple, Iterable, Optional
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
|
from app.models.dto import (
|
||||||
|
QueryRequest,
|
||||||
|
QueryResponse,
|
||||||
|
QueryHit,
|
||||||
|
Explanation,
|
||||||
|
ScoreBreakdown,
|
||||||
|
Reason,
|
||||||
|
EdgeDTO
|
||||||
|
)
|
||||||
|
import app.core.qdrant as qdr
|
||||||
|
import app.core.qdrant_points as qp
|
||||||
|
import app.services.embeddings_client as ec
|
||||||
|
import app.core.graph_adapter as ga
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # type: ignore[import]
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
yaml = None # type: ignore[assignment]
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache
|
||||||
|
def _get_scoring_weights() -> Tuple[float, float, float]:
|
||||||
|
"""Liefert (semantic_weight, edge_weight, centrality_weight) für den Retriever."""
|
||||||
|
settings = get_settings()
|
||||||
|
sem = float(getattr(settings, "RETRIEVER_W_SEM", 1.0))
|
||||||
|
edge = float(getattr(settings, "RETRIEVER_W_EDGE", 0.0))
|
||||||
|
cent = float(getattr(settings, "RETRIEVER_W_CENT", 0.0))
|
||||||
|
|
||||||
|
config_path = os.getenv("MINDNET_RETRIEVER_CONFIG", "config/retriever.yaml")
|
||||||
|
if yaml is None:
|
||||||
|
return sem, edge, cent
|
||||||
|
try:
|
||||||
|
if os.path.exists(config_path):
|
||||||
|
with open(config_path, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
scoring = data.get("scoring", {}) or {}
|
||||||
|
sem = float(scoring.get("semantic_weight", sem))
|
||||||
|
edge = float(scoring.get("edge_weight", edge))
|
||||||
|
cent = float(scoring.get("centrality_weight", cent))
|
||||||
|
except Exception:
|
||||||
|
return sem, edge, cent
|
||||||
|
return sem, edge, cent
|
||||||
|
|
||||||
|
|
||||||
|
def _get_client_and_prefix() -> Tuple[Any, str]:
|
||||||
|
"""Liefert (QdrantClient, prefix)."""
|
||||||
|
cfg = qdr.QdrantConfig.from_env()
|
||||||
|
client = qdr.get_client(cfg)
|
||||||
|
return client, cfg.prefix
|
||||||
|
|
||||||
|
|
||||||
|
def _get_query_vector(req: QueryRequest) -> List[float]:
|
||||||
|
"""Liefert den Query-Vektor aus dem Request."""
|
||||||
|
if req.query_vector:
|
||||||
|
return list(req.query_vector)
|
||||||
|
|
||||||
|
if not req.query:
|
||||||
|
raise ValueError("QueryRequest benötigt entweder query oder query_vector")
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
model_name = settings.MODEL_NAME
|
||||||
|
|
||||||
|
try:
|
||||||
|
return ec.embed_text(req.query, model_name=model_name)
|
||||||
|
except TypeError:
|
||||||
|
return ec.embed_text(req.query)
|
||||||
|
|
||||||
|
|
||||||
|
def _semantic_hits(
|
||||||
|
client: Any,
|
||||||
|
prefix: str,
|
||||||
|
vector: List[float],
|
||||||
|
top_k: int,
|
||||||
|
filters: Dict[str, Any] | None = None,
|
||||||
|
) -> List[Tuple[str, float, Dict[str, Any]]]:
|
||||||
|
"""Führt eine semantische Suche aus."""
|
||||||
|
flt = filters or None
|
||||||
|
raw_hits = qp.search_chunks_by_vector(client, prefix, vector, top=top_k, filters=flt)
|
||||||
|
results: List[Tuple[str, float, Dict[str, Any]]] = []
|
||||||
|
for pid, score, payload in raw_hits:
|
||||||
|
results.append((str(pid), float(score), dict(payload or {})))
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_total_score(
|
||||||
|
semantic_score: float,
|
||||||
|
payload: Dict[str, Any],
|
||||||
|
edge_bonus: float = 0.0,
|
||||||
|
cent_bonus: float = 0.0,
|
||||||
|
) -> Tuple[float, float, float]:
|
||||||
|
"""Berechnet total_score."""
|
||||||
|
raw_weight = payload.get("retriever_weight", 1.0)
|
||||||
|
try:
|
||||||
|
weight = float(raw_weight)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
weight = 1.0
|
||||||
|
if weight < 0.0:
|
||||||
|
weight = 0.0
|
||||||
|
|
||||||
|
sem_w, edge_w, cent_w = _get_scoring_weights()
|
||||||
|
total = (sem_w * float(semantic_score) * weight) + (edge_w * edge_bonus) + (cent_w * cent_bonus)
|
||||||
|
return float(total), float(edge_bonus), float(cent_bonus)
|
||||||
|
|
||||||
|
|
||||||
|
# --- WP-04b Explanation Logic ---
|
||||||
|
|
||||||
|
def _build_explanation(
|
||||||
|
semantic_score: float,
|
||||||
|
payload: Dict[str, Any],
|
||||||
|
edge_bonus: float,
|
||||||
|
cent_bonus: float,
|
||||||
|
subgraph: Optional[ga.Subgraph],
|
||||||
|
node_key: Optional[str]
|
||||||
|
) -> Explanation:
|
||||||
|
"""Erstellt ein Explanation-Objekt."""
|
||||||
|
sem_w, _edge_w, _cent_w = _get_scoring_weights()
|
||||||
|
# Scoring weights erneut laden für Reason-Details
|
||||||
|
_, edge_w_cfg, cent_w_cfg = _get_scoring_weights()
|
||||||
|
|
||||||
|
try:
|
||||||
|
type_weight = float(payload.get("retriever_weight", 1.0))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
type_weight = 1.0
|
||||||
|
|
||||||
|
note_type = payload.get("type", "unknown")
|
||||||
|
|
||||||
|
breakdown = ScoreBreakdown(
|
||||||
|
semantic_contribution=(sem_w * semantic_score * type_weight),
|
||||||
|
edge_contribution=(edge_w_cfg * edge_bonus),
|
||||||
|
centrality_contribution=(cent_w_cfg * cent_bonus),
|
||||||
|
raw_semantic=semantic_score,
|
||||||
|
raw_edge_bonus=edge_bonus,
|
||||||
|
raw_centrality=cent_bonus,
|
||||||
|
node_weight=type_weight
|
||||||
|
)
|
||||||
|
|
||||||
|
reasons: List[Reason] = []
|
||||||
|
edges_dto: List[EdgeDTO] = []
|
||||||
|
|
||||||
|
if semantic_score > 0.85:
|
||||||
|
reasons.append(Reason(kind="semantic", message="Sehr hohe textuelle Übereinstimmung.", score_impact=breakdown.semantic_contribution))
|
||||||
|
elif semantic_score > 0.70:
|
||||||
|
reasons.append(Reason(kind="semantic", message="Gute textuelle Übereinstimmung.", score_impact=breakdown.semantic_contribution))
|
||||||
|
|
||||||
|
if type_weight != 1.0:
|
||||||
|
msg = "Bevorzugt" if type_weight > 1.0 else "Leicht abgewertet"
|
||||||
|
reasons.append(Reason(kind="type", message=f"{msg} aufgrund des Typs '{note_type}'.", score_impact=(sem_w * semantic_score * (type_weight - 1.0))))
|
||||||
|
|
||||||
|
if subgraph and node_key and edge_bonus > 0:
|
||||||
|
if hasattr(subgraph, "get_outgoing_edges"):
|
||||||
|
outgoing = subgraph.get_outgoing_edges(node_key)
|
||||||
|
for edge in outgoing:
|
||||||
|
target = edge.get("target", "Unknown")
|
||||||
|
kind = edge.get("kind", "edge")
|
||||||
|
weight = edge.get("weight", 0.0)
|
||||||
|
if weight > 0.05:
|
||||||
|
edges_dto.append(EdgeDTO(id=f"{node_key}->{target}:{kind}", kind=kind, source=node_key, target=target, weight=weight, direction="out"))
|
||||||
|
|
||||||
|
if hasattr(subgraph, "get_incoming_edges"):
|
||||||
|
incoming = subgraph.get_incoming_edges(node_key)
|
||||||
|
for edge in incoming:
|
||||||
|
src = edge.get("source", "Unknown")
|
||||||
|
kind = edge.get("kind", "edge")
|
||||||
|
weight = edge.get("weight", 0.0)
|
||||||
|
if weight > 0.05:
|
||||||
|
edges_dto.append(EdgeDTO(id=f"{src}->{node_key}:{kind}", kind=kind, source=src, target=node_key, weight=weight, direction="in"))
|
||||||
|
|
||||||
|
all_edges = sorted(edges_dto, key=lambda e: e.weight, reverse=True)
|
||||||
|
for top_edge in all_edges[:3]:
|
||||||
|
impact = edge_w_cfg * top_edge.weight
|
||||||
|
dir_txt = "Verweist auf" if top_edge.direction == "out" else "Referenziert von"
|
||||||
|
tgt_txt = top_edge.target if top_edge.direction == "out" else top_edge.source
|
||||||
|
reasons.append(Reason(kind="edge", message=f"{dir_txt} '{tgt_txt}' via '{top_edge.kind}'", score_impact=impact, details={"kind": top_edge.kind}))
|
||||||
|
|
||||||
|
if cent_bonus > 0.01:
|
||||||
|
reasons.append(Reason(kind="centrality", message="Knoten liegt zentral im Kontext.", score_impact=breakdown.centrality_contribution))
|
||||||
|
|
||||||
|
return Explanation(breakdown=breakdown, reasons=reasons, related_edges=edges_dto if edges_dto else None)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_expand_options(req: QueryRequest) -> Tuple[int, List[str] | None]:
|
||||||
|
"""Extrahiert depth und edge_types."""
|
||||||
|
expand = getattr(req, "expand", None)
|
||||||
|
if not expand:
|
||||||
|
return 0, None
|
||||||
|
|
||||||
|
depth = 1
|
||||||
|
edge_types: List[str] | None = None
|
||||||
|
|
||||||
|
if hasattr(expand, "depth") or hasattr(expand, "edge_types"):
|
||||||
|
depth = int(getattr(expand, "depth", 1) or 1)
|
||||||
|
types_val = getattr(expand, "edge_types", None)
|
||||||
|
if types_val:
|
||||||
|
edge_types = list(types_val)
|
||||||
|
return depth, edge_types
|
||||||
|
|
||||||
|
if isinstance(expand, dict):
|
||||||
|
if "depth" in expand:
|
||||||
|
depth = int(expand.get("depth") or 1)
|
||||||
|
if "edge_types" in expand and expand["edge_types"] is not None:
|
||||||
|
edge_types = list(expand["edge_types"])
|
||||||
|
return depth, edge_types
|
||||||
|
|
||||||
|
return 0, None
|
||||||
|
|
||||||
|
|
||||||
|
def _build_hits_from_semantic(
|
||||||
|
hits: Iterable[Tuple[str, float, Dict[str, Any]]],
|
||||||
|
top_k: int,
|
||||||
|
used_mode: str,
|
||||||
|
subgraph: ga.Subgraph | None = None,
|
||||||
|
explain: bool = False,
|
||||||
|
) -> QueryResponse:
|
||||||
|
"""Baut strukturierte QueryHits."""
|
||||||
|
t0 = time.time()
|
||||||
|
enriched: List[Tuple[str, float, Dict[str, Any], float, float, float]] = []
|
||||||
|
|
||||||
|
for pid, semantic_score, payload in hits:
|
||||||
|
edge_bonus = 0.0
|
||||||
|
cent_bonus = 0.0
|
||||||
|
node_key = payload.get("chunk_id") or payload.get("note_id")
|
||||||
|
|
||||||
|
if subgraph is not None and node_key:
|
||||||
|
try:
|
||||||
|
edge_bonus = float(subgraph.edge_bonus(node_key))
|
||||||
|
except Exception:
|
||||||
|
edge_bonus = 0.0
|
||||||
|
try:
|
||||||
|
cent_bonus = float(subgraph.centrality_bonus(node_key))
|
||||||
|
except Exception:
|
||||||
|
cent_bonus = 0.0
|
||||||
|
|
||||||
|
total, edge_bonus, cent_bonus = _compute_total_score(semantic_score, payload, edge_bonus=edge_bonus, cent_bonus=cent_bonus)
|
||||||
|
enriched.append((pid, float(semantic_score), payload, total, edge_bonus, cent_bonus))
|
||||||
|
|
||||||
|
enriched_sorted = sorted(enriched, key=lambda h: h[3], reverse=True)
|
||||||
|
limited = enriched_sorted[: max(1, top_k)]
|
||||||
|
|
||||||
|
results: List[QueryHit] = []
|
||||||
|
for pid, semantic_score, payload, total, edge_bonus, cent_bonus in limited:
|
||||||
|
explanation_obj = None
|
||||||
|
if explain:
|
||||||
|
explanation_obj = _build_explanation(
|
||||||
|
semantic_score=float(semantic_score),
|
||||||
|
payload=payload,
|
||||||
|
edge_bonus=edge_bonus,
|
||||||
|
cent_bonus=cent_bonus,
|
||||||
|
subgraph=subgraph,
|
||||||
|
node_key=payload.get("chunk_id") or payload.get("note_id")
|
||||||
|
)
|
||||||
|
|
||||||
|
text_content = payload.get("page_content") or payload.get("text") or payload.get("content")
|
||||||
|
|
||||||
|
results.append(QueryHit(
|
||||||
|
node_id=str(pid),
|
||||||
|
note_id=payload.get("note_id"),
|
||||||
|
semantic_score=float(semantic_score),
|
||||||
|
edge_bonus=edge_bonus,
|
||||||
|
centrality_bonus=cent_bonus,
|
||||||
|
total_score=total,
|
||||||
|
paths=None,
|
||||||
|
source={
|
||||||
|
"path": payload.get("path"),
|
||||||
|
"section": payload.get("section") or payload.get("section_title"),
|
||||||
|
"text": text_content
|
||||||
|
},
|
||||||
|
# --- FIX: Wir füllen das payload-Feld explizit ---
|
||||||
|
payload=payload,
|
||||||
|
explanation=explanation_obj
|
||||||
|
))
|
||||||
|
|
||||||
|
dt = int((time.time() - t0) * 1000)
|
||||||
|
return QueryResponse(results=results, used_mode=used_mode, latency_ms=dt)
|
||||||
|
|
||||||
|
|
||||||
|
def semantic_retrieve(req: QueryRequest) -> QueryResponse:
|
||||||
|
"""Reiner semantischer Retriever."""
|
||||||
|
client, prefix = _get_client_and_prefix()
|
||||||
|
vector = _get_query_vector(req)
|
||||||
|
top_k = req.top_k or get_settings().RETRIEVER_TOP_K
|
||||||
|
|
||||||
|
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
||||||
|
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="semantic", subgraph=None, explain=req.explain)
|
||||||
|
|
||||||
|
|
||||||
|
def hybrid_retrieve(req: QueryRequest) -> QueryResponse:
|
||||||
|
"""Hybrid-Retriever: semantische Suche + optionale Edge-Expansion."""
|
||||||
|
client, prefix = _get_client_and_prefix()
|
||||||
|
if req.query_vector:
|
||||||
|
vector = list(req.query_vector)
|
||||||
|
else:
|
||||||
|
vector = _get_query_vector(req)
|
||||||
|
|
||||||
|
top_k = req.top_k or get_settings().RETRIEVER_TOP_K
|
||||||
|
hits = _semantic_hits(client, prefix, vector, top_k=top_k, filters=req.filters)
|
||||||
|
|
||||||
|
depth, edge_types = _extract_expand_options(req)
|
||||||
|
subgraph: ga.Subgraph | None = None
|
||||||
|
if depth and depth > 0:
|
||||||
|
seed_ids: List[str] = []
|
||||||
|
for _pid, _score, payload in hits:
|
||||||
|
key = payload.get("chunk_id") or payload.get("note_id")
|
||||||
|
if key and key not in seed_ids:
|
||||||
|
seed_ids.append(key)
|
||||||
|
if seed_ids:
|
||||||
|
try:
|
||||||
|
subgraph = ga.expand(client, prefix, seed_ids, depth=depth, edge_types=edge_types)
|
||||||
|
except Exception:
|
||||||
|
subgraph = None
|
||||||
|
|
||||||
|
return _build_hits_from_semantic(hits, top_k=top_k, used_mode="hybrid", subgraph=subgraph, explain=req.explain)
|
||||||
|
|
||||||
|
|
||||||
|
class Retriever:
|
||||||
|
"""
|
||||||
|
Wrapper-Klasse für WP-05 (Chat).
|
||||||
|
"""
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def search(self, request: QueryRequest) -> QueryResponse:
|
||||||
|
return hybrid_retrieve(request)
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""app/core/retriever_config.py
|
||||||
|
---------------------------------
|
||||||
|
Zentrale Konfiguration für den mindnet-Retriever (WP-04).
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
- Lädt config/retriever.yaml (falls vorhanden) oder nutzt sinnvolle Defaults.
|
||||||
|
- Bietet einen gecachten Zugriff auf die Retriever-Config für
|
||||||
|
andere Module (z. B. graph_adapter, retriever).
|
||||||
|
|
||||||
|
Hinweis zur Weiterentwicklung (Selbstjustierung):
|
||||||
|
- Die hier definierten Parameter sind so gewählt, dass sie später
|
||||||
|
durch ein Feedback-/Learning-to-Rank-Modell überschrieben werden
|
||||||
|
können, ohne die restliche Architektur anzupassen.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # type: ignore
|
||||||
|
except Exception: # pragma: no cover - Fallback, falls PyYAML nicht installiert ist.
|
||||||
|
yaml = None # type: ignore
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RetrieverConfig:
|
||||||
|
semantic_scale: float
|
||||||
|
edge_scale: float
|
||||||
|
centrality_scale: float
|
||||||
|
edge_weights: Dict[str, float]
|
||||||
|
|
||||||
|
@lru_cache
|
||||||
|
def get_retriever_config() -> RetrieverConfig:
|
||||||
|
"""Lädt die Retriever-Konfiguration (YAML + Defaults).
|
||||||
|
|
||||||
|
Reihenfolge:
|
||||||
|
1. Defaults (sinnvoll gewählte Startwerte).
|
||||||
|
2. Optional: config/retriever.yaml bzw. Pfad aus ENV
|
||||||
|
MINDNET_RETRIEVER_CONFIG überschreibt die Defaults.
|
||||||
|
|
||||||
|
Die Funktion ist bewusst gecached, da sich die Konfiguration zur
|
||||||
|
Laufzeit üblicherweise nicht ändert. Für dynamisches Nachladen
|
||||||
|
könnte der Cache explizit geleert werden.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# 1) Defaults – bewusst konservativ gewählt.
|
||||||
|
semantic_scale = 1.0
|
||||||
|
edge_scale = 1.0
|
||||||
|
centrality_scale = 1.0
|
||||||
|
|
||||||
|
edge_weights: Dict[str, float] = {
|
||||||
|
# Wissens-Kanten
|
||||||
|
"depends_on": 1.0,
|
||||||
|
"related_to": 0.7,
|
||||||
|
"similar_to": 0.7,
|
||||||
|
"references": 0.5,
|
||||||
|
# Struktur-Kanten
|
||||||
|
"belongs_to": 0.2,
|
||||||
|
"next": 0.1,
|
||||||
|
"prev": 0.1,
|
||||||
|
# Sonstige / technische Kanten
|
||||||
|
"backlink": 0.2,
|
||||||
|
"references_at": 0.2,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 2) Optional: YAML-Konfiguration laden
|
||||||
|
cfg_path_env = os.getenv("MINDNET_RETRIEVER_CONFIG")
|
||||||
|
if cfg_path_env:
|
||||||
|
cfg_path = Path(cfg_path_env)
|
||||||
|
else:
|
||||||
|
# Project-Root = zwei Ebenen über app/core/
|
||||||
|
cfg_path = Path(__file__).resolve().parents[2] / "config" / "retriever.yaml"
|
||||||
|
|
||||||
|
if yaml is not None and cfg_path.exists():
|
||||||
|
try:
|
||||||
|
data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {}
|
||||||
|
except Exception:
|
||||||
|
data = {}
|
||||||
|
|
||||||
|
retr = data.get("retriever") or {}
|
||||||
|
|
||||||
|
# Skalenwerte überschreiben, falls angegeben
|
||||||
|
try:
|
||||||
|
semantic_scale = float(retr.get("semantic_scale", semantic_scale))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
edge_scale = float(retr.get("edge_scale", edge_scale))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
centrality_scale = float(retr.get("centrality_scale", centrality_scale))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Edge-Gewichte je Kanten-Typ
|
||||||
|
ew_cfg = retr.get("edge_weights") or {}
|
||||||
|
if isinstance(ew_cfg, dict):
|
||||||
|
for k, v in ew_cfg.items():
|
||||||
|
try:
|
||||||
|
edge_weights[str(k)] = float(v)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
return RetrieverConfig(
|
||||||
|
semantic_scale=semantic_scale,
|
||||||
|
edge_scale=edge_scale,
|
||||||
|
centrality_scale=centrality_scale,
|
||||||
|
edge_weights=edge_weights,
|
||||||
|
)
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Modul: app/core/type_registry.py
|
||||||
|
Version: 1.0.0
|
||||||
|
Datum: 2025-11-08
|
||||||
|
|
||||||
|
Zweck
|
||||||
|
-----
|
||||||
|
Lädt eine optionale Typ-Registry (config/types.yaml) und stellt
|
||||||
|
komfortable Zugriffsfunktionen bereit. Die Registry ist *optional*:
|
||||||
|
- Fehlt die Datei oder ist das YAML defekt, wird ein konservativer
|
||||||
|
Default (Typ "concept") verwendet und es wird eine Warnung ausgegeben.
|
||||||
|
- Änderungen an der Datei greifen nach einem Neustart des Prozesses.
|
||||||
|
|
||||||
|
Öffentliche API
|
||||||
|
---------------
|
||||||
|
- load_type_registry(path: str = "config/types.yaml") -> dict
|
||||||
|
- get_type_config(note_type: str, reg: dict) -> dict
|
||||||
|
- resolve_note_type(fm_type: str | None, reg: dict) -> str
|
||||||
|
- effective_chunk_profile(note_type: str, reg: dict) -> str | None
|
||||||
|
- profile_overlap(profile: str | None) -> tuple[int,int] # nur Overlap-Empfehlung
|
||||||
|
|
||||||
|
Hinweis
|
||||||
|
-------
|
||||||
|
Die Registry steuert KEINE Breaking Changes. Ohne Datei/Typ bleibt das
|
||||||
|
Verhalten exakt wie im Release-Stand 20251105.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from functools import lru_cache
|
||||||
|
from typing import Dict, Any, Optional, Tuple
|
||||||
|
import os
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # PyYAML
|
||||||
|
except Exception:
|
||||||
|
yaml = None # wird erst benötigt, wenn eine Datei gelesen werden soll
|
||||||
|
|
||||||
|
# Konservativer Default – bewusst minimal
|
||||||
|
_DEFAULT_REGISTRY: Dict[str, Any] = {
|
||||||
|
"version": "1.0",
|
||||||
|
"types": {
|
||||||
|
"concept": {
|
||||||
|
"chunk_profile": "medium",
|
||||||
|
"edge_defaults": ["references", "related_to"],
|
||||||
|
"retriever_weight": 1.0,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_using_defaults": True,
|
||||||
|
"_warning": "types.yaml missing or invalid – using built-in defaults (type=concept).",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Chunk-Profile → Overlap-Empfehlungen (nur für synthetische Fensterbildung)
|
||||||
|
# Die absoluten Chunk-Längen bleiben Aufgabe des Chunkers (assemble_chunks).
|
||||||
|
_PROFILE_TO_OVERLAP: Dict[str, Tuple[int, int]] = {
|
||||||
|
"short": (20, 30),
|
||||||
|
"medium": (40, 60),
|
||||||
|
"long": (60, 80),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def load_type_registry(path: str = "config/types.yaml") -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Lädt die Registry aus 'path'. Bei Fehlern wird ein konserviver Default geliefert.
|
||||||
|
Die Rückgabe ist *prozessweit* gecached.
|
||||||
|
"""
|
||||||
|
if not path:
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
|
||||||
|
if not os.path.isfile(path):
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
|
||||||
|
if yaml is None:
|
||||||
|
# PyYAML fehlt → auf Default zurückfallen
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
# Minimal validieren
|
||||||
|
if not isinstance(data, dict) or "types" not in data or not isinstance(data["types"], dict):
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
data.setdefault("version", "1.0")
|
||||||
|
data.setdefault("_using_defaults", False)
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
return dict(_DEFAULT_REGISTRY)
|
||||||
|
|
||||||
|
|
||||||
|
def get_type_config(note_type: Optional[str], reg: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
t = (note_type or "concept").strip().lower()
|
||||||
|
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
|
||||||
|
return types.get(t) or types.get("concept") or _DEFAULT_REGISTRY["types"]["concept"]
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_note_type(fm_type: Optional[str], reg: Dict[str, Any]) -> str:
|
||||||
|
"""Liefert einen gültigen Typ (unbekannt → 'concept')."""
|
||||||
|
t = (fm_type or "concept").strip().lower()
|
||||||
|
types = (reg or {}).get("types", {}) if isinstance(reg, dict) else {}
|
||||||
|
return t if t in types else "concept"
|
||||||
|
|
||||||
|
|
||||||
|
def effective_chunk_profile(note_type: Optional[str], reg: Dict[str, Any]) -> Optional[str]:
|
||||||
|
cfg = get_type_config(note_type, reg)
|
||||||
|
prof = cfg.get("chunk_profile")
|
||||||
|
if isinstance(prof, str) and prof.strip():
|
||||||
|
return prof.strip().lower()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def profile_overlap(profile: Optional[str]) -> Tuple[int, int]:
|
||||||
|
"""Gibt eine Overlap-Empfehlung (low, high) für das Profil zurück."""
|
||||||
|
if not profile:
|
||||||
|
return _PROFILE_TO_OVERLAP["medium"]
|
||||||
|
return _PROFILE_TO_OVERLAP.get(profile.strip().lower(), _PROFILE_TO_OVERLAP["medium"])
|
||||||
@@ -0,0 +1,207 @@
|
|||||||
|
import streamlit as st
|
||||||
|
import requests
|
||||||
|
import uuid
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
# --- CONFIGURATION ---
|
||||||
|
load_dotenv()
|
||||||
|
API_BASE_URL = os.getenv("MINDNET_API_URL", "http://localhost:8002")
|
||||||
|
CHAT_ENDPOINT = f"{API_BASE_URL}/chat"
|
||||||
|
FEEDBACK_ENDPOINT = f"{API_BASE_URL}/feedback"
|
||||||
|
HISTORY_FILE = Path("data/logs/search_history.jsonl")
|
||||||
|
|
||||||
|
# Timeout Strategy
|
||||||
|
timeout_setting = os.getenv("MINDNET_API_TIMEOUT") or os.getenv("MINDNET_LLM_TIMEOUT")
|
||||||
|
API_TIMEOUT = float(timeout_setting) if timeout_setting else 300.0
|
||||||
|
|
||||||
|
# --- PAGE SETUP ---
|
||||||
|
st.set_page_config(page_title="mindnet v2.3.1", page_icon="🧠", layout="wide")
|
||||||
|
|
||||||
|
# --- CSS STYLING ---
|
||||||
|
st.markdown("""
|
||||||
|
<style>
|
||||||
|
/* Hauptcontainer enger machen für Lesbarkeit */
|
||||||
|
.block-container { padding-top: 2rem; max_width: 900px; margin: auto; }
|
||||||
|
|
||||||
|
/* Intent Badges */
|
||||||
|
.intent-badge {
|
||||||
|
background-color: #e8f0fe; color: #1a73e8;
|
||||||
|
padding: 4px 10px; border-radius: 12px;
|
||||||
|
font-size: 0.8rem; font-weight: 600;
|
||||||
|
border: 1px solid #d2e3fc; display: inline-block; margin-bottom: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Chat Message Styling */
|
||||||
|
.stChatMessage { padding: 1rem; border-radius: 8px; margin-bottom: 1rem;}
|
||||||
|
div[data-testid="stChatMessageContent"] p { font-size: 1.05rem; line-height: 1.6; }
|
||||||
|
|
||||||
|
/* Expander Cleaner */
|
||||||
|
.streamlit-expanderHeader { font-size: 0.9rem; font-weight: 600; color: #444; }
|
||||||
|
</style>
|
||||||
|
""", unsafe_allow_html=True)
|
||||||
|
|
||||||
|
# --- SESSION STATE ---
|
||||||
|
if "messages" not in st.session_state: st.session_state.messages = []
|
||||||
|
if "user_id" not in st.session_state: st.session_state.user_id = str(uuid.uuid4())
|
||||||
|
if "draft_note" not in st.session_state: st.session_state.draft_note = {"title": "", "content": "", "type": "concept"}
|
||||||
|
|
||||||
|
# --- HELPER FUNCTIONS ---
|
||||||
|
|
||||||
|
def load_history_from_logs(limit=10):
|
||||||
|
"""Liest die letzten N Queries aus dem Logfile."""
|
||||||
|
queries = []
|
||||||
|
if HISTORY_FILE.exists():
|
||||||
|
try:
|
||||||
|
with open(HISTORY_FILE, "r", encoding="utf-8") as f:
|
||||||
|
lines = f.readlines()
|
||||||
|
for line in reversed(lines):
|
||||||
|
try:
|
||||||
|
entry = json.loads(line)
|
||||||
|
q = entry.get("query_text")
|
||||||
|
if q and q not in queries:
|
||||||
|
queries.append(q)
|
||||||
|
if len(queries) >= limit: break
|
||||||
|
except: continue
|
||||||
|
except Exception as e:
|
||||||
|
st.sidebar.warning(f"Log-Fehler: {e}")
|
||||||
|
return queries
|
||||||
|
|
||||||
|
def send_chat_message(message: str, top_k: int, explain: bool):
|
||||||
|
try:
|
||||||
|
response = requests.post(
|
||||||
|
CHAT_ENDPOINT,
|
||||||
|
json={"message": message, "top_k": top_k, "explain": explain},
|
||||||
|
timeout=API_TIMEOUT
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
|
except Exception as e:
|
||||||
|
return {"error": str(e)}
|
||||||
|
|
||||||
|
def submit_feedback(query_id, node_id, score, comment=None):
|
||||||
|
try:
|
||||||
|
requests.post(FEEDBACK_ENDPOINT, json={"query_id": query_id, "node_id": node_id, "score": score, "comment": comment}, timeout=2)
|
||||||
|
target = "Antwort" if node_id == "generated_answer" else "Quelle"
|
||||||
|
st.toast(f"Feedback für {target} gespeichert! (Score: {score})")
|
||||||
|
except: pass
|
||||||
|
|
||||||
|
# --- UI COMPONENTS ---
|
||||||
|
|
||||||
|
def render_sidebar():
|
||||||
|
with st.sidebar:
|
||||||
|
st.title("🧠 mindnet")
|
||||||
|
st.caption("v2.3.1 | WP-10 UI")
|
||||||
|
|
||||||
|
mode = st.radio("Modus", ["💬 Chat", "📝 Neuer Eintrag (WP-07)"], index=0)
|
||||||
|
|
||||||
|
st.divider()
|
||||||
|
st.subheader("⚙️ Settings")
|
||||||
|
top_k = st.slider("Quellen (Top-K)", 1, 10, 5)
|
||||||
|
explain = st.toggle("Explanation Layer", True)
|
||||||
|
|
||||||
|
st.divider()
|
||||||
|
st.subheader("🕒 Verlauf")
|
||||||
|
history = load_history_from_logs(8)
|
||||||
|
if not history:
|
||||||
|
st.caption("Noch keine Einträge.")
|
||||||
|
for q in history:
|
||||||
|
if st.button(f"🔎 {q[:30]}...", key=f"hist_{q}", help=q, use_container_width=True):
|
||||||
|
st.session_state.messages.append({"role": "user", "content": q})
|
||||||
|
st.rerun()
|
||||||
|
|
||||||
|
return mode, top_k, explain
|
||||||
|
|
||||||
|
def render_chat_interface(top_k, explain):
|
||||||
|
# Render History
|
||||||
|
for msg in st.session_state.messages:
|
||||||
|
with st.chat_message(msg["role"]):
|
||||||
|
if msg["role"] == "assistant":
|
||||||
|
# Intent Badge MIT SOURCE (Fix für Debugging)
|
||||||
|
if "intent" in msg:
|
||||||
|
icon = {"EMPATHY": "❤️", "DECISION": "⚖️", "CODING": "💻", "FACT": "📚"}.get(msg["intent"], "🧠")
|
||||||
|
source_info = msg.get("intent_source", "Unknown")
|
||||||
|
# Hier wird die Quelle wieder angezeigt:
|
||||||
|
st.markdown(f'<div class="intent-badge">{icon} Intent: {msg["intent"]} <span style="opacity:0.6; font-size: 0.9em; margin-left:5px;">via {source_info}</span></div>', unsafe_allow_html=True)
|
||||||
|
|
||||||
|
st.markdown(msg["content"])
|
||||||
|
|
||||||
|
# Sources
|
||||||
|
if "sources" in msg:
|
||||||
|
for hit in msg["sources"]:
|
||||||
|
score = hit.get('total_score', 0)
|
||||||
|
icon = "🟢" if score > 0.8 else "🟡" if score > 0.5 else "⚪"
|
||||||
|
with st.expander(f"{icon} {hit.get('note_id', '?')} ({score:.2f})"):
|
||||||
|
st.markdown(f"_{hit.get('source', {}).get('text', '')[:300]}..._")
|
||||||
|
if hit.get('explanation'):
|
||||||
|
st.caption(f"Grund: {hit['explanation']['reasons'][0]['message']}")
|
||||||
|
|
||||||
|
# Granular Feedback (Faces)
|
||||||
|
def _cb(qid=msg["query_id"], nid=hit['node_id']):
|
||||||
|
val = st.session_state.get(f"fb_src_{qid}_{nid}")
|
||||||
|
if val is not None: submit_feedback(qid, nid, val+1, "Faces UI")
|
||||||
|
|
||||||
|
st.feedback("faces", key=f"fb_src_{msg['query_id']}_{hit['node_id']}", on_change=_cb)
|
||||||
|
|
||||||
|
# Global Feedback (Stars)
|
||||||
|
qid = msg["query_id"]
|
||||||
|
st.feedback("stars", key=f"fb_glob_{qid}", on_change=lambda: submit_feedback(qid, "generated_answer", st.session_state[f"fb_glob_{qid}"]+1))
|
||||||
|
|
||||||
|
else:
|
||||||
|
st.markdown(msg["content"])
|
||||||
|
|
||||||
|
# Input Logic
|
||||||
|
last_msg_is_user = len(st.session_state.messages) > 0 and st.session_state.messages[-1]["role"] == "user"
|
||||||
|
|
||||||
|
if prompt := st.chat_input("Frage Mindnet..."):
|
||||||
|
st.session_state.messages.append({"role": "user", "content": prompt})
|
||||||
|
st.rerun()
|
||||||
|
|
||||||
|
if last_msg_is_user:
|
||||||
|
last_prompt = st.session_state.messages[-1]["content"]
|
||||||
|
with st.chat_message("assistant"):
|
||||||
|
with st.spinner("Thinking..."):
|
||||||
|
resp = send_chat_message(last_prompt, top_k, explain)
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
st.error(resp["error"])
|
||||||
|
else:
|
||||||
|
msg_data = {
|
||||||
|
"role": "assistant",
|
||||||
|
"content": resp.get("answer"),
|
||||||
|
"intent": resp.get("intent", "FACT"),
|
||||||
|
"intent_source": resp.get("intent_source", "Unknown"), # Wichtig für Anzeige
|
||||||
|
"sources": resp.get("sources", []),
|
||||||
|
"query_id": resp.get("query_id")
|
||||||
|
}
|
||||||
|
st.session_state.messages.append(msg_data)
|
||||||
|
st.rerun()
|
||||||
|
|
||||||
|
def render_creation_interface():
|
||||||
|
st.header("📝 Neuer Wissens-Eintrag (WP-07/11)")
|
||||||
|
st.info("Hier kannst du strukturierte Notizen erstellen, die direkt in den Obsidian Vault gespeichert werden.")
|
||||||
|
|
||||||
|
with st.form("new_entry"):
|
||||||
|
col1, col2 = st.columns([3, 1])
|
||||||
|
title = col1.text_input("Titel der Notiz", placeholder="z.B. Projekt Gamma Meeting")
|
||||||
|
n_type = col2.selectbox("Typ", ["concept", "meeting", "person", "project", "decision"])
|
||||||
|
|
||||||
|
content = st.text_area("Inhalt (Markdown)", height=300, placeholder="# Protokoll\n\n- Punkt 1...")
|
||||||
|
|
||||||
|
st.markdown("**Automatische Vernetzung:**")
|
||||||
|
st.caption("Verwende `[[Link]]` für Referenzen und `[[rel:depends_on X]]` für logische Kanten.")
|
||||||
|
|
||||||
|
submitted = st.form_submit_button("💾 Speichern & Indizieren")
|
||||||
|
if submitted:
|
||||||
|
st.success(f"Mockup: Notiz '{title}' ({n_type}) wäre jetzt gespeichert worden!")
|
||||||
|
st.balloons()
|
||||||
|
|
||||||
|
# --- MAIN LOOP ---
|
||||||
|
mode, top_k, explain = render_sidebar()
|
||||||
|
|
||||||
|
if mode == "💬 Chat":
|
||||||
|
render_chat_interface(top_k, explain)
|
||||||
|
else:
|
||||||
|
render_creation_interface()
|
||||||
+26
-5
@@ -1,16 +1,26 @@
|
|||||||
"""
|
"""
|
||||||
Version 0.3
|
app/main.py — mindnet API bootstrap
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from .config import get_settings
|
from .config import get_settings
|
||||||
from .routers.embed_router import router as embed_router
|
from .routers.embed_router import router as embed_router
|
||||||
from .routers.qdrant_router import router as qdrant_router
|
from .routers.qdrant_router import router as qdrant_router
|
||||||
|
|
||||||
|
from .routers.query import router as query_router
|
||||||
|
from .routers.graph import router as graph_router
|
||||||
|
from .routers.tools import router as tools_router
|
||||||
|
from .routers.feedback import router as feedback_router
|
||||||
|
# NEU: Chat Router (WP-05)
|
||||||
|
from .routers.chat import router as chat_router
|
||||||
|
|
||||||
|
try:
|
||||||
|
from .routers.admin import router as admin_router
|
||||||
|
except Exception:
|
||||||
|
admin_router = None
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
app = FastAPI(title="mindnet API", version="0.1.0")
|
app = FastAPI(title="mindnet API", version="0.5.0") # Version bump WP-05
|
||||||
s = get_settings()
|
s = get_settings()
|
||||||
|
|
||||||
@app.get("/healthz")
|
@app.get("/healthz")
|
||||||
@@ -19,7 +29,18 @@ def create_app() -> FastAPI:
|
|||||||
|
|
||||||
app.include_router(embed_router)
|
app.include_router(embed_router)
|
||||||
app.include_router(qdrant_router)
|
app.include_router(qdrant_router)
|
||||||
|
|
||||||
|
app.include_router(query_router, prefix="/query", tags=["query"])
|
||||||
|
app.include_router(graph_router, prefix="/graph", tags=["graph"])
|
||||||
|
app.include_router(tools_router, prefix="/tools", tags=["tools"])
|
||||||
|
app.include_router(feedback_router, prefix="/feedback", tags=["feedback"])
|
||||||
|
|
||||||
|
# NEU: Chat Endpoint
|
||||||
|
app.include_router(chat_router, prefix="/chat", tags=["chat"])
|
||||||
|
|
||||||
|
if admin_router:
|
||||||
|
app.include_router(admin_router, prefix="/admin", tags=["admin"])
|
||||||
|
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
app = create_app()
|
||||||
app = create_app()
|
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""
|
||||||
|
app/models/__init__.py — Package-Init für mindnet DTOs (WP-04)
|
||||||
|
Version: 0.1.0 • Stand: 2025-10-07
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .dto import (
|
||||||
|
EdgeKind,
|
||||||
|
NodeDTO,
|
||||||
|
EdgeDTO,
|
||||||
|
QueryRequest,
|
||||||
|
QueryHit,
|
||||||
|
QueryResponse,
|
||||||
|
GraphResponse,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"EdgeKind",
|
||||||
|
"NodeDTO",
|
||||||
|
"EdgeDTO",
|
||||||
|
"QueryRequest",
|
||||||
|
"QueryHit",
|
||||||
|
"QueryResponse",
|
||||||
|
"GraphResponse",
|
||||||
|
]
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
"""
|
||||||
|
app/models/dto.py — Pydantic-Modelle (DTOs) für WP-04/WP-05/WP-06
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Laufzeit-Modelle für FastAPI (Requests/Responses).
|
||||||
|
WP-06 Update: Intent & Intent-Source in ChatResponse.
|
||||||
|
|
||||||
|
Version:
|
||||||
|
0.6.2 (WP-06: Decision Engine Transparency, Erweiterung des Feeback Request)
|
||||||
|
Stand:
|
||||||
|
2025-12-09
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from typing import List, Literal, Optional, Dict, Any
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
EdgeKind = Literal["references", "references_at", "backlink", "next", "prev", "belongs_to", "depends_on", "related_to", "similar_to"]
|
||||||
|
|
||||||
|
|
||||||
|
# --- Basis-DTOs ---
|
||||||
|
|
||||||
|
class NodeDTO(BaseModel):
|
||||||
|
"""Darstellung eines Knotens (Note oder Chunk) im API-Graph."""
|
||||||
|
id: str
|
||||||
|
type: Literal["note", "chunk"]
|
||||||
|
title: Optional[str] = None
|
||||||
|
note_id: Optional[str] = None
|
||||||
|
tags: Optional[List[str]] = None
|
||||||
|
in_degree: Optional[int] = None
|
||||||
|
out_degree: Optional[int] = None
|
||||||
|
score: Optional[float] = None
|
||||||
|
section_title: Optional[str] = None
|
||||||
|
section_path: Optional[str] = None
|
||||||
|
path: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class EdgeDTO(BaseModel):
|
||||||
|
"""Darstellung einer Kante im API-Graph."""
|
||||||
|
id: str
|
||||||
|
kind: str
|
||||||
|
source: str
|
||||||
|
target: str
|
||||||
|
weight: float
|
||||||
|
direction: Literal["out", "in", "undirected"] = "out"
|
||||||
|
|
||||||
|
|
||||||
|
# --- Request Models ---
|
||||||
|
|
||||||
|
class QueryRequest(BaseModel):
|
||||||
|
"""
|
||||||
|
Request für /query.
|
||||||
|
"""
|
||||||
|
mode: Literal["semantic", "edge", "hybrid"] = "hybrid"
|
||||||
|
query: Optional[str] = None
|
||||||
|
query_vector: Optional[List[float]] = None
|
||||||
|
top_k: int = 10
|
||||||
|
expand: Dict = {"depth": 1, "edge_types": ["references", "belongs_to", "prev", "next", "depends_on", "related_to"]}
|
||||||
|
filters: Optional[Dict] = None
|
||||||
|
ret: Dict = {"with_paths": True, "with_notes": True, "with_chunks": True}
|
||||||
|
explain: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class FeedbackRequest(BaseModel):
|
||||||
|
"""
|
||||||
|
User-Feedback zu einem spezifischen Treffer oder der Gesamtantwort.
|
||||||
|
"""
|
||||||
|
query_id: str = Field(..., description="ID der ursprünglichen Suche")
|
||||||
|
# node_id ist optional: Wenn leer oder "generated_answer", gilt es für die Antwort.
|
||||||
|
# Wenn eine echte Chunk-ID, gilt es für die Quelle.
|
||||||
|
node_id: str = Field(..., description="ID des bewerteten Treffers oder 'generated_answer'")
|
||||||
|
# Update: Range auf 1-5 erweitert für differenziertes Tuning
|
||||||
|
score: int = Field(..., ge=1, le=5, description="1 (Irrelevant/Falsch) bis 5 (Perfekt)")
|
||||||
|
comment: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class ChatRequest(BaseModel):
|
||||||
|
"""
|
||||||
|
WP-05: Request für /chat.
|
||||||
|
"""
|
||||||
|
message: str = Field(..., description="Die Nachricht des Users")
|
||||||
|
conversation_id: Optional[str] = Field(None, description="Optional: ID für Chat-Verlauf (noch nicht implementiert)")
|
||||||
|
# RAG Parameter (Override defaults)
|
||||||
|
top_k: int = 5
|
||||||
|
explain: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
# --- WP-04b Explanation Models ---
|
||||||
|
|
||||||
|
class ScoreBreakdown(BaseModel):
|
||||||
|
"""Aufschlüsselung der Score-Komponenten."""
|
||||||
|
semantic_contribution: float
|
||||||
|
edge_contribution: float
|
||||||
|
centrality_contribution: float
|
||||||
|
raw_semantic: float
|
||||||
|
raw_edge_bonus: float
|
||||||
|
raw_centrality: float
|
||||||
|
node_weight: float
|
||||||
|
|
||||||
|
|
||||||
|
class Reason(BaseModel):
|
||||||
|
"""Ein semantischer Grund für das Ranking."""
|
||||||
|
kind: Literal["semantic", "edge", "type", "centrality"]
|
||||||
|
message: str
|
||||||
|
score_impact: Optional[float] = None
|
||||||
|
details: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
|
|
||||||
|
class Explanation(BaseModel):
|
||||||
|
"""Container für alle Erklärungsdaten eines Treffers."""
|
||||||
|
breakdown: ScoreBreakdown
|
||||||
|
reasons: List[Reason]
|
||||||
|
related_edges: Optional[List[EdgeDTO]] = None
|
||||||
|
|
||||||
|
|
||||||
|
# --- Response Models ---
|
||||||
|
|
||||||
|
class QueryHit(BaseModel):
|
||||||
|
"""Einzelnes Trefferobjekt für /query."""
|
||||||
|
node_id: str
|
||||||
|
note_id: Optional[str]
|
||||||
|
semantic_score: float
|
||||||
|
edge_bonus: float
|
||||||
|
centrality_bonus: float
|
||||||
|
total_score: float
|
||||||
|
paths: Optional[List[List[Dict]]] = None
|
||||||
|
source: Optional[Dict] = None
|
||||||
|
payload: Optional[Dict] = None # Added for flexibility & WP-06 meta-data
|
||||||
|
explanation: Optional[Explanation] = None
|
||||||
|
|
||||||
|
|
||||||
|
class QueryResponse(BaseModel):
|
||||||
|
"""Antwortstruktur für /query."""
|
||||||
|
query_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||||
|
results: List[QueryHit]
|
||||||
|
used_mode: str
|
||||||
|
latency_ms: int
|
||||||
|
|
||||||
|
|
||||||
|
class GraphResponse(BaseModel):
|
||||||
|
"""Antwortstruktur für /graph/{note_id}."""
|
||||||
|
center_note_id: str
|
||||||
|
nodes: List[NodeDTO]
|
||||||
|
edges: List[EdgeDTO]
|
||||||
|
stats: Dict[str, int]
|
||||||
|
|
||||||
|
|
||||||
|
class ChatResponse(BaseModel):
|
||||||
|
"""
|
||||||
|
WP-05/06: Antwortstruktur für /chat.
|
||||||
|
"""
|
||||||
|
query_id: str = Field(..., description="Traceability ID (dieselbe wie für Search)")
|
||||||
|
answer: str = Field(..., description="Generierte Antwort vom LLM")
|
||||||
|
sources: List[QueryHit] = Field(..., description="Die für die Antwort genutzten Quellen")
|
||||||
|
latency_ms: int
|
||||||
|
intent: Optional[str] = Field("FACT", description="WP-06: Erkannter Intent (FACT/DECISION)")
|
||||||
|
intent_source: Optional[str] = Field("Unknown", description="WP-06: Quelle der Intent-Erkennung (Keyword vs. LLM)")
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
"""
|
||||||
|
app/models/__init__.py — Package-Init für mindnet DTOs
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Stellt zentrale API-/DTO-Modelle als Import-Fassade bereit.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, Pydantic 2.x
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage für WP-04)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
Handbuch / Knowledge Design (WP-03/04)
|
||||||
|
Nutzung:
|
||||||
|
from app.models import QueryRequest, QueryResponse, GraphResponse
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .dto import (
|
||||||
|
EdgeKind,
|
||||||
|
NodeDTO,
|
||||||
|
EdgeDTO,
|
||||||
|
QueryRequest,
|
||||||
|
QueryHit,
|
||||||
|
QueryResponse,
|
||||||
|
GraphResponse,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"EdgeKind",
|
||||||
|
"NodeDTO",
|
||||||
|
"EdgeDTO",
|
||||||
|
"QueryRequest",
|
||||||
|
"QueryHit",
|
||||||
|
"QueryResponse",
|
||||||
|
"GraphResponse",
|
||||||
|
]
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
"""
|
||||||
|
app/routers/admin.py — Admin-/Monitoring-Endpunkte (optional)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Liefert einfache Kennzahlen zu Collections (Counts) und Config.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, FastAPI 0.110+, qdrant-client 1.x
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
- Qdrant Collections: *_notes, *_chunks, *_edges
|
||||||
|
Nutzung:
|
||||||
|
app.include_router(admin.router, prefix="/admin", tags=["admin"])
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
@router.get("/stats")
|
||||||
|
def stats():
|
||||||
|
s = get_settings()
|
||||||
|
client = QdrantClient(url=s.QDRANT_URL, api_key=s.QDRANT_API_KEY)
|
||||||
|
notes = f"{s.COLLECTION_PREFIX}_notes"
|
||||||
|
chunks = f"{s.COLLECTION_PREFIX}_chunks"
|
||||||
|
edges = f"{s.COLLECTION_PREFIX}_edges"
|
||||||
|
|
||||||
|
def _count(col: str) -> int:
|
||||||
|
try:
|
||||||
|
return client.count(collection_name=col, exact=True).count
|
||||||
|
except Exception:
|
||||||
|
return -1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"collections": {
|
||||||
|
"notes": {"name": notes, "count": _count(notes)},
|
||||||
|
"chunks": {"name": chunks, "count": _count(chunks)},
|
||||||
|
"edges": {"name": edges, "count": _count(edges)},
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"qdrant": s.QDRANT_URL,
|
||||||
|
"prefix": s.COLLECTION_PREFIX,
|
||||||
|
"vector_size": s.VECTOR_SIZE,
|
||||||
|
"distance": s.DISTANCE,
|
||||||
|
"retriever": {
|
||||||
|
"w_sem": s.RETRIEVER_W_SEM,
|
||||||
|
"w_edge": s.RETRIEVER_W_EDGE,
|
||||||
|
"w_cent": s.RETRIEVER_W_CENT,
|
||||||
|
"top_k": s.RETRIEVER_TOP_K,
|
||||||
|
"expand_depth": s.RETRIEVER_EXPAND_DEPTH,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -0,0 +1,253 @@
|
|||||||
|
"""
|
||||||
|
app/routers/chat.py — RAG Endpunkt (WP-06 Hybrid Router + WP-04c Feedback)
|
||||||
|
Version: 2.3.2 (Merged Stability Patch)
|
||||||
|
|
||||||
|
Features:
|
||||||
|
- Hybrid Intent Router (Keyword + LLM)
|
||||||
|
- Strategic Retrieval (Late Binding via Config)
|
||||||
|
- Context Enrichment (Payload/Source Fallback)
|
||||||
|
- Data Flywheel (Feedback Logging Integration)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException, Depends
|
||||||
|
from typing import List, Dict, Any
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
import logging
|
||||||
|
import yaml
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
|
from app.models.dto import ChatRequest, ChatResponse, QueryRequest, QueryHit
|
||||||
|
from app.services.llm_service import LLMService
|
||||||
|
from app.core.retriever import Retriever
|
||||||
|
# [MERGE] Integration Feedback Service (WP-04c)
|
||||||
|
from app.services.feedback_service import log_search
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# --- Helper: Config Loader ---
|
||||||
|
|
||||||
|
_DECISION_CONFIG_CACHE = None
|
||||||
|
|
||||||
|
def _load_decision_config() -> Dict[str, Any]:
|
||||||
|
settings = get_settings()
|
||||||
|
path = Path(settings.DECISION_CONFIG_PATH)
|
||||||
|
default_config = {
|
||||||
|
"strategies": {
|
||||||
|
"FACT": {"trigger_keywords": []}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if not path.exists():
|
||||||
|
logger.warning(f"Decision config not found at {path}, using defaults.")
|
||||||
|
return default_config
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
return yaml.safe_load(f)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to load decision config: {e}")
|
||||||
|
return default_config
|
||||||
|
|
||||||
|
def get_full_config() -> Dict[str, Any]:
|
||||||
|
global _DECISION_CONFIG_CACHE
|
||||||
|
if _DECISION_CONFIG_CACHE is None:
|
||||||
|
_DECISION_CONFIG_CACHE = _load_decision_config()
|
||||||
|
return _DECISION_CONFIG_CACHE
|
||||||
|
|
||||||
|
def get_decision_strategy(intent: str) -> Dict[str, Any]:
|
||||||
|
config = get_full_config()
|
||||||
|
strategies = config.get("strategies", {})
|
||||||
|
return strategies.get(intent, strategies.get("FACT", {}))
|
||||||
|
|
||||||
|
|
||||||
|
# --- Dependencies ---
|
||||||
|
|
||||||
|
def get_llm_service():
|
||||||
|
return LLMService()
|
||||||
|
|
||||||
|
def get_retriever():
|
||||||
|
return Retriever()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Logic ---
|
||||||
|
|
||||||
|
def _build_enriched_context(hits: List[QueryHit]) -> str:
|
||||||
|
context_parts = []
|
||||||
|
for i, hit in enumerate(hits, 1):
|
||||||
|
source = hit.source or {}
|
||||||
|
content = (
|
||||||
|
source.get("text") or source.get("content") or
|
||||||
|
source.get("page_content") or source.get("chunk_text") or
|
||||||
|
"[Kein Text]"
|
||||||
|
)
|
||||||
|
title = hit.note_id or "Unbekannt"
|
||||||
|
|
||||||
|
# [FIX] Robustes Auslesen des Typs (Payload > Source > Unknown)
|
||||||
|
payload = hit.payload or {}
|
||||||
|
note_type = payload.get("type") or source.get("type", "unknown")
|
||||||
|
note_type = str(note_type).upper()
|
||||||
|
|
||||||
|
entry = (
|
||||||
|
f"### QUELLE {i}: {title}\n"
|
||||||
|
f"TYP: [{note_type}] (Score: {hit.total_score:.2f})\n"
|
||||||
|
f"INHALT:\n{content}\n"
|
||||||
|
)
|
||||||
|
context_parts.append(entry)
|
||||||
|
|
||||||
|
return "\n\n".join(context_parts)
|
||||||
|
|
||||||
|
async def _classify_intent(query: str, llm: LLMService) -> tuple[str, str]:
|
||||||
|
"""
|
||||||
|
Hybrid Router v3:
|
||||||
|
Gibt Tuple zurück: (Intent, Source)
|
||||||
|
"""
|
||||||
|
config = get_full_config()
|
||||||
|
strategies = config.get("strategies", {})
|
||||||
|
settings = config.get("settings", {})
|
||||||
|
|
||||||
|
query_lower = query.lower()
|
||||||
|
best_intent = None
|
||||||
|
max_match_length = 0
|
||||||
|
|
||||||
|
# 1. FAST PATH: Keywords
|
||||||
|
for intent_name, strategy in strategies.items():
|
||||||
|
if intent_name == "FACT": continue
|
||||||
|
keywords = strategy.get("trigger_keywords", [])
|
||||||
|
for k in keywords:
|
||||||
|
if k.lower() in query_lower:
|
||||||
|
if len(k) > max_match_length:
|
||||||
|
max_match_length = len(k)
|
||||||
|
best_intent = intent_name
|
||||||
|
|
||||||
|
if best_intent:
|
||||||
|
return best_intent, "Keyword (Fast Path)"
|
||||||
|
|
||||||
|
# 2. SLOW PATH: LLM Router
|
||||||
|
if settings.get("llm_fallback_enabled", False):
|
||||||
|
router_prompt_template = settings.get("llm_router_prompt", "")
|
||||||
|
if router_prompt_template:
|
||||||
|
prompt = router_prompt_template.replace("{query}", query)
|
||||||
|
logger.info("Keywords failed. Asking LLM for Intent...")
|
||||||
|
|
||||||
|
raw_response = await llm.generate_raw_response(prompt)
|
||||||
|
|
||||||
|
# Parsing logic
|
||||||
|
llm_output_upper = raw_response.upper()
|
||||||
|
found_intents = []
|
||||||
|
for strat_key in strategies.keys():
|
||||||
|
if strat_key in llm_output_upper:
|
||||||
|
found_intents.append(strat_key)
|
||||||
|
|
||||||
|
if len(found_intents) == 1:
|
||||||
|
return found_intents[0], "LLM Router (Slow Path)"
|
||||||
|
elif len(found_intents) > 1:
|
||||||
|
return found_intents[0], f"LLM Ambiguous {found_intents}"
|
||||||
|
else:
|
||||||
|
return "FACT", "LLM Fallback (No Match)"
|
||||||
|
|
||||||
|
return "FACT", "Default (No Match)"
|
||||||
|
|
||||||
|
@router.post("/", response_model=ChatResponse)
|
||||||
|
async def chat_endpoint(
|
||||||
|
request: ChatRequest,
|
||||||
|
llm: LLMService = Depends(get_llm_service),
|
||||||
|
retriever: Retriever = Depends(get_retriever)
|
||||||
|
):
|
||||||
|
start_time = time.time()
|
||||||
|
query_id = str(uuid.uuid4())
|
||||||
|
logger.info(f"Chat request [{query_id}]: {request.message[:50]}...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 1. Intent Detection (mit Source)
|
||||||
|
intent, intent_source = await _classify_intent(request.message, llm)
|
||||||
|
logger.info(f"[{query_id}] Final Intent: {intent} via {intent_source}")
|
||||||
|
|
||||||
|
# Strategy Load
|
||||||
|
strategy = get_decision_strategy(intent)
|
||||||
|
inject_types = strategy.get("inject_types", [])
|
||||||
|
prompt_key = strategy.get("prompt_template", "rag_template")
|
||||||
|
prepend_instr = strategy.get("prepend_instruction", "")
|
||||||
|
|
||||||
|
# 2. Primary Retrieval
|
||||||
|
query_req = QueryRequest(
|
||||||
|
query=request.message,
|
||||||
|
mode="hybrid",
|
||||||
|
top_k=request.top_k,
|
||||||
|
explain=request.explain
|
||||||
|
)
|
||||||
|
retrieve_result = await retriever.search(query_req)
|
||||||
|
hits = retrieve_result.results
|
||||||
|
|
||||||
|
# 3. Strategic Retrieval (WP-06 Kernfeature)
|
||||||
|
if inject_types:
|
||||||
|
logger.info(f"[{query_id}] Executing Strategic Retrieval for types: {inject_types}...")
|
||||||
|
strategy_req = QueryRequest(
|
||||||
|
query=request.message,
|
||||||
|
mode="hybrid",
|
||||||
|
top_k=3,
|
||||||
|
filters={"type": inject_types},
|
||||||
|
explain=False
|
||||||
|
)
|
||||||
|
strategy_result = await retriever.search(strategy_req)
|
||||||
|
|
||||||
|
existing_ids = {h.node_id for h in hits}
|
||||||
|
for strat_hit in strategy_result.results:
|
||||||
|
if strat_hit.node_id not in existing_ids:
|
||||||
|
hits.append(strat_hit)
|
||||||
|
|
||||||
|
# 4. Context Building
|
||||||
|
if not hits:
|
||||||
|
context_str = "Keine relevanten Notizen gefunden."
|
||||||
|
else:
|
||||||
|
context_str = _build_enriched_context(hits)
|
||||||
|
|
||||||
|
# 5. Generation
|
||||||
|
template = llm.prompts.get(prompt_key, "{context_str}\n\n{query}")
|
||||||
|
system_prompt = llm.prompts.get("system_prompt", "")
|
||||||
|
|
||||||
|
if prepend_instr:
|
||||||
|
context_str = f"{prepend_instr}\n\n{context_str}"
|
||||||
|
|
||||||
|
final_prompt = template.replace("{context_str}", context_str).replace("{query}", request.message)
|
||||||
|
|
||||||
|
logger.info(f"[{query_id}] Sending to LLM (Intent: {intent}, Template: {prompt_key})...")
|
||||||
|
|
||||||
|
# System-Prompt separat übergeben (WP-06a Fix)
|
||||||
|
answer_text = await llm.generate_raw_response(prompt=final_prompt, system=system_prompt)
|
||||||
|
|
||||||
|
duration_ms = int((time.time() - start_time) * 1000)
|
||||||
|
|
||||||
|
# 6. Logging (Fire & Forget) - [MERGE POINT]
|
||||||
|
# Wir loggen alles für das Data Flywheel (WP-08 Self-Tuning)
|
||||||
|
try:
|
||||||
|
log_search(
|
||||||
|
query_id=query_id,
|
||||||
|
query_text=request.message,
|
||||||
|
results=hits,
|
||||||
|
mode="chat_rag",
|
||||||
|
metadata={
|
||||||
|
"intent": intent,
|
||||||
|
"intent_source": intent_source,
|
||||||
|
"generated_answer": answer_text,
|
||||||
|
"model": llm.settings.LLM_MODEL
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Logging failed: {e}")
|
||||||
|
|
||||||
|
# 7. Response
|
||||||
|
return ChatResponse(
|
||||||
|
query_id=query_id,
|
||||||
|
answer=answer_text,
|
||||||
|
sources=hits,
|
||||||
|
latency_ms=duration_ms,
|
||||||
|
intent=intent,
|
||||||
|
intent_source=intent_source
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error in chat endpoint: {e}", exc_info=True)
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
"""
|
||||||
|
app/routers/feedback.py
|
||||||
|
Endpunkt für User-Feedback (WP-04c).
|
||||||
|
"""
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
from app.models.dto import FeedbackRequest
|
||||||
|
from app.services.feedback_service import log_feedback
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
@router.post("", status_code=201)
|
||||||
|
def post_feedback(fb: FeedbackRequest):
|
||||||
|
"""
|
||||||
|
Nimmt Feedback entgegen (z.B. Daumen hoch für einen Treffer).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
log_feedback(fb)
|
||||||
|
return {"status": "recorded", "query_id": fb.query_id}
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
"""
|
||||||
|
app/routers/graph.py — Graph-Endpunkte (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Liefert die Nachbarschaft einer Note/ID als JSON-Graph (Nodes/Edges/Stats).
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, FastAPI 0.110+, qdrant-client 1.x
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
- app/core/graph_adapter.py
|
||||||
|
- app/models/dto.py
|
||||||
|
Nutzung:
|
||||||
|
app.include_router(graph.router, prefix="/graph", tags=["graph"])
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List, Optional
|
||||||
|
from fastapi import APIRouter, Query
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from app.models.dto import GraphResponse, NodeDTO, EdgeDTO
|
||||||
|
from app.core.graph_adapter import expand
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
@router.get("/{note_id}", response_model=GraphResponse)
|
||||||
|
def get_graph(note_id: str, depth: int = 1, edge_types: Optional[List[str]] = Query(None)) -> GraphResponse:
|
||||||
|
s = get_settings()
|
||||||
|
client = QdrantClient(url=s.QDRANT_URL, api_key=s.QDRANT_API_KEY)
|
||||||
|
|
||||||
|
sg = expand(client, s.COLLECTION_PREFIX, [note_id], depth=depth, edge_types=edge_types)
|
||||||
|
|
||||||
|
# Seed-Node (Payload optional später ergänzen über get_note_payload)
|
||||||
|
nodes = [NodeDTO(id=note_id, type="note",
|
||||||
|
title=None, tags=[],
|
||||||
|
in_degree=sg.in_degree.get(note_id, 0),
|
||||||
|
out_degree=sg.out_degree.get(note_id, 0))]
|
||||||
|
|
||||||
|
# direkte Out-Kanten des Seeds
|
||||||
|
edges = []
|
||||||
|
for e in sg.adj.get(note_id, []):
|
||||||
|
edges.append(EdgeDTO(
|
||||||
|
id=f"{note_id}->{e['target']}:{e['kind']}",
|
||||||
|
kind=e["kind"], source=note_id, target=e["target"],
|
||||||
|
weight=float(e["weight"]), direction="out"
|
||||||
|
))
|
||||||
|
|
||||||
|
return GraphResponse(center_note_id=note_id, nodes=nodes, edges=edges,
|
||||||
|
stats={"node_count": len(nodes), "edge_count": len(edges)})
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
"""
|
||||||
|
app/routers/query.py — Query-Endpunkte (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Stellt POST /query bereit und ruft den passenden Retriever-Modus auf.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, FastAPI 0.110+
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
- app/core/retriever.py
|
||||||
|
- app/models/dto.py
|
||||||
|
Nutzung:
|
||||||
|
app.include_router(query.router, prefix="/query", tags=["query"])
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.2.0 (2025-12-07) - Update für WP04c Feedback
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
from fastapi import APIRouter, HTTPException, BackgroundTasks
|
||||||
|
from app.models.dto import QueryRequest, QueryResponse
|
||||||
|
from app.core.retriever import hybrid_retrieve, semantic_retrieve
|
||||||
|
# NEU:
|
||||||
|
from app.services.feedback_service import log_search
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
@router.post("", response_model=QueryResponse)
|
||||||
|
def post_query(req: QueryRequest, background_tasks: BackgroundTasks) -> QueryResponse:
|
||||||
|
try:
|
||||||
|
if req.mode == "semantic":
|
||||||
|
res = semantic_retrieve(req)
|
||||||
|
else:
|
||||||
|
res = hybrid_retrieve(req)
|
||||||
|
|
||||||
|
# WP-04c: Logging im Hintergrund (bremst Antwort nicht)
|
||||||
|
background_tasks.add_task(log_search, req, res)
|
||||||
|
|
||||||
|
return res
|
||||||
|
except ValueError as e:
|
||||||
|
raise HTTPException(status_code=400, detail=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(status_code=500, detail=f"query failed: {e}")
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
"""
|
||||||
|
app/routers/tools.py — Tool-Definitionen für Ollama/n8n/MCP (read-only)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Liefert Funktions-Schemas (OpenAI-/Ollama-kompatibles Tool-JSON) für:
|
||||||
|
- mindnet_query -> POST /query
|
||||||
|
- mindnet_subgraph -> GET /graph/{note_id}
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, FastAPI 0.110+
|
||||||
|
Version:
|
||||||
|
0.1.1 (query ODER query_vector möglich)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Nutzung:
|
||||||
|
app.include_router(tools.router, prefix="/tools", tags=["tools"])
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.1 (2025-10-07) – mindnet_query: oneOf(query, query_vector).
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
TOOLS = {
|
||||||
|
"tools": [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "mindnet_query",
|
||||||
|
"description": "Hybrid-Retrieval über mindnet (Semantik + Edges).",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"oneOf": [
|
||||||
|
{"required": ["query"]},
|
||||||
|
{"required": ["query_vector"]}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"query": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Freitext-Query; wird serverseitig in 384-d Embedding konvertiert."
|
||||||
|
},
|
||||||
|
"query_vector": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "number"},
|
||||||
|
"description": "Direkter 384-d Query-Vektor (optional)."
|
||||||
|
},
|
||||||
|
"top_k": {"type":"integer","default":10,"minimum":1,"maximum":50},
|
||||||
|
"expand_depth": {"type":"integer","default":1,"minimum":0,"maximum":3},
|
||||||
|
"edge_types": {
|
||||||
|
"type":"array","items":{"type":"string"},
|
||||||
|
"default": ["references","belongs_to","prev","next"]
|
||||||
|
},
|
||||||
|
"filters": {"type":"object","description":"payload-Filter (tags etc.)"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "mindnet_subgraph",
|
||||||
|
"description": "Gibt die Nachbarschaft (Edges) einer Note/Seed-ID zurück.",
|
||||||
|
"parameters": {
|
||||||
|
"type":"object",
|
||||||
|
"properties": {
|
||||||
|
"note_id":{"type":"string"},
|
||||||
|
"depth":{"type":"integer","default":1,"minimum":0,"maximum":3},
|
||||||
|
"edge_types":{
|
||||||
|
"type":"array","items":{"type":"string"},
|
||||||
|
"default":["references","belongs_to","prev","next","backlink"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required":["note_id"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
@router.get("/ollama")
|
||||||
|
def get_ollama_tools():
|
||||||
|
return TOOLS
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""
|
||||||
|
app/services/embeddings_client.py — Text→Embedding (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Liefert 384-d Embeddings für Textqueries (lazy load, einmal pro Prozess).
|
||||||
|
Standard: Sentence-Transformers (MODEL_NAME aus app.config.Settings).
|
||||||
|
Hinweis: Kein Netz-Zugriff; nutzt lokal installierte Modelle.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, sentence-transformers 5.x
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
- app/core/retriever.py (nutzt embed_text_if_needed)
|
||||||
|
- app/config.py (MODEL_NAME, VECTOR_SIZE)
|
||||||
|
Nutzung:
|
||||||
|
from app.services.embeddings_client import embed_text
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List
|
||||||
|
from functools import lru_cache
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
# Lazy import, damit Testläufe ohne Modell-Laden schnell sind
|
||||||
|
def _load_model():
|
||||||
|
from sentence_transformers import SentenceTransformer # import hier, nicht top-level
|
||||||
|
s = get_settings()
|
||||||
|
return SentenceTransformer(s.MODEL_NAME, device="cpu")
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _cached_model():
|
||||||
|
return _load_model()
|
||||||
|
|
||||||
|
def embed_text(text: str) -> List[float]:
|
||||||
|
"""
|
||||||
|
Erzeugt einen 384-d Vektor (oder laut Settings.VECTOR_SIZE) für den gegebenen Text.
|
||||||
|
"""
|
||||||
|
if not text or not text.strip():
|
||||||
|
raise ValueError("embed_text: leerer Text")
|
||||||
|
model = _cached_model()
|
||||||
|
vec = model.encode([text], normalize_embeddings=True)[0]
|
||||||
|
return vec.astype(float).tolist()
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
"""
|
||||||
|
app/services/feedback_service.py
|
||||||
|
Service zum Loggen von Suchanfragen und Feedback (WP-04c).
|
||||||
|
Speichert Daten als JSONL für späteres Self-Tuning (WP-08).
|
||||||
|
|
||||||
|
Version: 1.1 (Chat-Support)
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, Any, List, Union
|
||||||
|
from app.models.dto import QueryRequest, QueryResponse, FeedbackRequest, QueryHit
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Pfad für Logs (lokal auf dem Beelink/PC)
|
||||||
|
LOG_DIR = Path("data/logs")
|
||||||
|
SEARCH_LOG_FILE = LOG_DIR / "search_history.jsonl"
|
||||||
|
FEEDBACK_LOG_FILE = LOG_DIR / "feedback.jsonl"
|
||||||
|
|
||||||
|
def _ensure_log_dir():
|
||||||
|
if not LOG_DIR.exists():
|
||||||
|
os.makedirs(LOG_DIR, exist_ok=True)
|
||||||
|
|
||||||
|
def _append_jsonl(file_path: Path, data: dict):
|
||||||
|
try:
|
||||||
|
with open(file_path, "a", encoding="utf-8") as f:
|
||||||
|
f.write(json.dumps(data, ensure_ascii=False) + "\n")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to write log: {e}")
|
||||||
|
|
||||||
|
def log_search(
|
||||||
|
query_id: str,
|
||||||
|
query_text: str,
|
||||||
|
results: List[QueryHit],
|
||||||
|
mode: str = "unknown",
|
||||||
|
metadata: Dict[str, Any] = None
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Generische Logging-Funktion für Suche UND Chat.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query_id: UUID der Anfrage.
|
||||||
|
query_text: User-Eingabe.
|
||||||
|
results: Liste der Treffer (QueryHit Objekte).
|
||||||
|
mode: z.B. "semantic", "hybrid", "chat_rag".
|
||||||
|
metadata: Zusätzliche Infos (z.B. generierte Antwort, Intent).
|
||||||
|
"""
|
||||||
|
_ensure_log_dir()
|
||||||
|
|
||||||
|
hits_summary = []
|
||||||
|
for hit in results:
|
||||||
|
# Pydantic Model Dump für saubere Serialisierung
|
||||||
|
breakdown = None
|
||||||
|
if hit.explanation and hit.explanation.breakdown:
|
||||||
|
breakdown = hit.explanation.breakdown.model_dump()
|
||||||
|
|
||||||
|
hits_summary.append({
|
||||||
|
"node_id": hit.node_id,
|
||||||
|
"note_id": hit.note_id,
|
||||||
|
"total_score": hit.total_score,
|
||||||
|
"breakdown": breakdown,
|
||||||
|
"rank_semantic": hit.semantic_score,
|
||||||
|
"rank_edge": hit.edge_bonus,
|
||||||
|
"type": hit.source.get("type") if hit.source else "unknown"
|
||||||
|
})
|
||||||
|
|
||||||
|
entry = {
|
||||||
|
"timestamp": time.time(),
|
||||||
|
"query_id": query_id,
|
||||||
|
"query_text": query_text,
|
||||||
|
"mode": mode,
|
||||||
|
"hits_count": len(hits_summary),
|
||||||
|
"hits": hits_summary,
|
||||||
|
"metadata": metadata or {}
|
||||||
|
}
|
||||||
|
|
||||||
|
_append_jsonl(SEARCH_LOG_FILE, entry)
|
||||||
|
logger.info(f"Logged search/chat interaction {query_id}")
|
||||||
|
|
||||||
|
def log_feedback(fb: FeedbackRequest):
|
||||||
|
"""
|
||||||
|
Speichert das User-Feedback.
|
||||||
|
"""
|
||||||
|
_ensure_log_dir()
|
||||||
|
|
||||||
|
entry = {
|
||||||
|
"timestamp": time.time(),
|
||||||
|
"query_id": fb.query_id,
|
||||||
|
"node_id": fb.node_id,
|
||||||
|
"score": fb.score,
|
||||||
|
"comment": fb.comment
|
||||||
|
}
|
||||||
|
|
||||||
|
_append_jsonl(FEEDBACK_LOG_FILE, entry)
|
||||||
|
logger.info(f"Logged feedback for {fb.query_id}")
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
"""
|
||||||
|
app/services/llm_ollama.py — Ollama-Integration & Prompt-Bau (WP-04)
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Prompt-Template & (optionaler) lokaler Aufruf von Ollama. Der Aufruf ist
|
||||||
|
bewusst gekapselt und kann gefahrlos deaktiviert bleiben, bis ihr ein
|
||||||
|
konkretes Modell konfigurieren wollt.
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
WP-04/05 Kontextbereitstellung für LLM
|
||||||
|
Nutzung:
|
||||||
|
from app.services.llm_ollama import build_prompt, call_ollama
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import List, Dict, Optional
|
||||||
|
import subprocess
|
||||||
|
import json
|
||||||
|
|
||||||
|
PROMPT_TEMPLATE = """System: You are a helpful expert.
|
||||||
|
User: {question}
|
||||||
|
|
||||||
|
Context (ranked):
|
||||||
|
{contexts}
|
||||||
|
|
||||||
|
Task: Answer precisely. At the end, list sources (note title + section) and important edge paths.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def build_context_block(items: List[Dict]) -> str:
|
||||||
|
"""Formatiert Top-K-Kontexte (Chunks) für den Prompt."""
|
||||||
|
lines = []
|
||||||
|
for i, it in enumerate(items, 1):
|
||||||
|
note = it.get("note_title", "") or it.get("note_id", "")
|
||||||
|
sec = it.get("section", "") or it.get("section_title", "")
|
||||||
|
sc = it.get("score", 0)
|
||||||
|
txt = it.get("text", "") or it.get("body", "") or ""
|
||||||
|
lines.append(f"{i}) {note} — {sec} [score={sc:.2f}]\n{txt}\n")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def build_prompt(question: str, contexts: List[Dict]) -> str:
|
||||||
|
"""Setzt Frage + Kontexte in ein konsistentes Template."""
|
||||||
|
return PROMPT_TEMPLATE.format(question=question, contexts=build_context_block(contexts))
|
||||||
|
|
||||||
|
|
||||||
|
def call_ollama(prompt: str, model: str = "llama3.1:8b", timeout_s: int = 120) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Optionaler lokaler Aufruf von `ollama run`.
|
||||||
|
Rückgabe: generierter Text oder None bei Fehler/Abbruch.
|
||||||
|
Hinweis: Nur nutzen, wenn Ollama lokal installiert/konfiguriert ist.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(
|
||||||
|
["ollama", "run", model],
|
||||||
|
input=prompt.encode("utf-8"),
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
timeout=timeout_s,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
out = proc.stdout.decode("utf-8", errors="replace")
|
||||||
|
# viele ollama Builds streamen JSON-Zeilen; robust extrahieren:
|
||||||
|
try:
|
||||||
|
# Falls JSONL, letztes "response" zusammenfassen
|
||||||
|
texts = []
|
||||||
|
for line in out.splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
obj = json.loads(line)
|
||||||
|
if "response" in obj:
|
||||||
|
texts.append(obj["response"])
|
||||||
|
except Exception:
|
||||||
|
texts.append(line)
|
||||||
|
return "".join(texts).strip()
|
||||||
|
except Exception:
|
||||||
|
return out.strip()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""
|
||||||
|
app/services/llm_service.py — LLM Client (Ollama)
|
||||||
|
Version: 0.2.1 (Fix: System Prompt Handling for Phi-3)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import yaml
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from app.config import get_settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class LLMService:
|
||||||
|
def __init__(self):
|
||||||
|
self.settings = get_settings()
|
||||||
|
self.prompts = self._load_prompts()
|
||||||
|
|
||||||
|
self.client = httpx.AsyncClient(
|
||||||
|
base_url=self.settings.OLLAMA_URL,
|
||||||
|
timeout=self.settings.LLM_TIMEOUT
|
||||||
|
)
|
||||||
|
|
||||||
|
def _load_prompts(self) -> dict:
|
||||||
|
path = Path(self.settings.PROMPTS_PATH)
|
||||||
|
if not path.exists():
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
return yaml.safe_load(f)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to load prompts: {e}")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
async def generate_raw_response(self, prompt: str, system: str = None) -> str:
|
||||||
|
"""
|
||||||
|
Führt einen LLM Call aus.
|
||||||
|
Unterstützt nun explizite System-Prompts für sauberes Templating.
|
||||||
|
"""
|
||||||
|
payload = {
|
||||||
|
"model": self.settings.LLM_MODEL,
|
||||||
|
"prompt": prompt,
|
||||||
|
"stream": False,
|
||||||
|
"options": {
|
||||||
|
# Temperature etwas höher für Empathie, niedriger für Code?
|
||||||
|
# Wir lassen es auf Standard, oder steuern es später via Config.
|
||||||
|
"temperature": 0.7,
|
||||||
|
"num_ctx": 2048
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# WICHTIG: System-Prompt separat übergeben, damit Ollama formatiert
|
||||||
|
if system:
|
||||||
|
payload["system"] = system
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await self.client.post("/api/generate", json=payload)
|
||||||
|
if response.status_code != 200:
|
||||||
|
logger.error(f"Ollama Error ({response.status_code}): {response.text}")
|
||||||
|
return "Fehler bei der Generierung."
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
return data.get("response", "").strip()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"LLM Raw Gen Error: {e}")
|
||||||
|
return "Interner LLM Fehler."
|
||||||
|
|
||||||
|
async def generate_rag_response(self, query: str, context_str: str) -> str:
|
||||||
|
"""Legacy Support"""
|
||||||
|
system_prompt = self.prompts.get("system_prompt", "")
|
||||||
|
rag_template = self.prompts.get("rag_template", "{context_str}\n\n{query}")
|
||||||
|
final_prompt = rag_template.format(context_str=context_str, query=query)
|
||||||
|
|
||||||
|
# Leite an die neue Methode weiter
|
||||||
|
return await self.generate_raw_response(final_prompt, system=system_prompt)
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
await self.client.aclose()
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
# config/decision_engine.yaml
|
||||||
|
# Steuerung der Decision Engine (WP-06)
|
||||||
|
# Hybrid-Modus: Keywords (Fast) + LLM Router (Smart Fallback)
|
||||||
|
version: 1.2
|
||||||
|
|
||||||
|
settings:
|
||||||
|
llm_fallback_enabled: true
|
||||||
|
|
||||||
|
# Few-Shot Prompting für bessere SLM-Performance
|
||||||
|
llm_router_prompt: |
|
||||||
|
Du bist ein Klassifikator. Analysiere die Nachricht und wähle die passende Strategie.
|
||||||
|
Antworte NUR mit dem Namen der Strategie.
|
||||||
|
|
||||||
|
STRATEGIEN:
|
||||||
|
- DECISION: Rat, Strategie, Vor/Nachteile, "Soll ich".
|
||||||
|
- EMPATHY: Gefühle, Frust, Freude, Probleme, "Alles ist sinnlos", "Ich bin traurig".
|
||||||
|
- CODING: Code, Syntax, Programmierung, Python.
|
||||||
|
- FACT: Wissen, Fakten, Definitionen.
|
||||||
|
|
||||||
|
BEISPIELE:
|
||||||
|
User: "Wie funktioniert Qdrant?" -> FACT
|
||||||
|
User: "Soll ich Qdrant nutzen?" -> DECISION
|
||||||
|
User: "Schreibe ein Python Script" -> CODING
|
||||||
|
User: "Alles ist grau und sinnlos" -> EMPATHY
|
||||||
|
User: "Mir geht es heute gut" -> EMPATHY
|
||||||
|
|
||||||
|
NACHRICHT: "{query}"
|
||||||
|
|
||||||
|
STRATEGIE:
|
||||||
|
|
||||||
|
strategies:
|
||||||
|
# 1. Fakten-Abfrage (Fallback & Default)
|
||||||
|
FACT:
|
||||||
|
description: "Reine Wissensabfrage."
|
||||||
|
trigger_keywords: []
|
||||||
|
inject_types: []
|
||||||
|
prompt_template: "rag_template"
|
||||||
|
prepend_instruction: null
|
||||||
|
|
||||||
|
# 2. Entscheidungs-Frage
|
||||||
|
DECISION:
|
||||||
|
description: "Der User sucht Rat, Strategie oder Abwägung."
|
||||||
|
trigger_keywords:
|
||||||
|
- "soll ich"
|
||||||
|
- "meinung"
|
||||||
|
- "besser"
|
||||||
|
- "empfehlung"
|
||||||
|
- "strategie"
|
||||||
|
- "entscheidung"
|
||||||
|
- "wert"
|
||||||
|
- "prinzip"
|
||||||
|
- "vor- und nachteile"
|
||||||
|
- "abwägung"
|
||||||
|
inject_types: ["value", "principle", "goal"]
|
||||||
|
prompt_template: "decision_template"
|
||||||
|
prepend_instruction: |
|
||||||
|
!!! ENTSCHEIDUNGS-MODUS !!!
|
||||||
|
BITTE WÄGE FAKTEN GEGEN FOLGENDE WERTE, PRINZIPIEN UND ZIELE AB:
|
||||||
|
|
||||||
|
# 3. Empathie / "Ich"-Modus
|
||||||
|
EMPATHY:
|
||||||
|
description: "Reaktion auf emotionale Zustände."
|
||||||
|
trigger_keywords:
|
||||||
|
- "ich fühle"
|
||||||
|
- "traurig"
|
||||||
|
- "glücklich"
|
||||||
|
- "gestresst"
|
||||||
|
- "angst"
|
||||||
|
- "nervt"
|
||||||
|
- "überfordert"
|
||||||
|
inject_types: ["experience", "belief", "profile"]
|
||||||
|
prompt_template: "empathy_template"
|
||||||
|
prepend_instruction: null
|
||||||
|
|
||||||
|
# 4. Coding / Technical
|
||||||
|
CODING:
|
||||||
|
description: "Technische Anfragen und Programmierung."
|
||||||
|
trigger_keywords:
|
||||||
|
- "code"
|
||||||
|
- "python"
|
||||||
|
- "script"
|
||||||
|
- "funktion"
|
||||||
|
- "bug"
|
||||||
|
- "syntax"
|
||||||
|
- "json"
|
||||||
|
- "yaml"
|
||||||
|
inject_types: ["snippet", "reference", "source"]
|
||||||
|
prompt_template: "technical_template"
|
||||||
|
prepend_instruction: null
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
# config/prompts.yaml — Final V2.3.1 (Multi-Personality Support)
|
||||||
|
|
||||||
|
system_prompt: |
|
||||||
|
Du bist 'mindnet', mein digitaler Zwilling und strategischer Partner.
|
||||||
|
|
||||||
|
DEINE IDENTITÄT:
|
||||||
|
- Du bist nicht nur eine Datenbank, sondern handelst nach MEINEN Werten und Zielen.
|
||||||
|
- Du passt deinen Stil dynamisch an die Situation an (Analytisch, Empathisch oder Technisch).
|
||||||
|
|
||||||
|
DEINE REGELN:
|
||||||
|
1. Deine Antwort muss zu 100% auf dem bereitgestellten KONTEXT basieren.
|
||||||
|
2. Halluziniere keine Fakten, die nicht in den Quellen stehen.
|
||||||
|
3. Antworte auf Deutsch (außer bei Code/Fachbegriffen).
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# 1. STANDARD: Fakten & Wissen (Intent: FACT)
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
rag_template: |
|
||||||
|
QUELLEN (WISSEN):
|
||||||
|
=========================================
|
||||||
|
{context_str}
|
||||||
|
=========================================
|
||||||
|
|
||||||
|
FRAGE:
|
||||||
|
{query}
|
||||||
|
|
||||||
|
ANWEISUNG:
|
||||||
|
Beantworte die Frage präzise basierend auf den Quellen.
|
||||||
|
Fasse die Informationen zusammen. Sei objektiv und neutral.
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# 2. DECISION: Strategie & Abwägung (Intent: DECISION)
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
decision_template: |
|
||||||
|
KONTEXT (FAKTEN & STRATEGIE):
|
||||||
|
=========================================
|
||||||
|
{context_str}
|
||||||
|
=========================================
|
||||||
|
|
||||||
|
ENTSCHEIDUNGSFRAGE:
|
||||||
|
{query}
|
||||||
|
|
||||||
|
ANWEISUNG:
|
||||||
|
Du agierst als mein Entscheidungs-Partner.
|
||||||
|
1. Analysiere die Faktenlage aus den Quellen.
|
||||||
|
2. Prüfe dies hart gegen meine strategischen Notizen (Typ [VALUE], [PRINCIPLE], [GOAL]).
|
||||||
|
3. Wäge ab: Passt die technische/faktische Lösung zu meinen Werten?
|
||||||
|
|
||||||
|
FORMAT:
|
||||||
|
- **Analyse:** (Kurze Zusammenfassung der Fakten)
|
||||||
|
- **Abgleich:** (Gibt es Konflikte mit Werten/Zielen? Nenne die Quelle!)
|
||||||
|
- **Empfehlung:** (Klare Meinung: Ja/Nein/Vielleicht mit Begründung)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# 3. EMPATHY: Der Spiegel / "Ich"-Modus (Intent: EMPATHY)
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
empathy_template: |
|
||||||
|
KONTEXT (ERFAHRUNGEN & GLAUBENSSÄTZE):
|
||||||
|
=========================================
|
||||||
|
{context_str}
|
||||||
|
=========================================
|
||||||
|
|
||||||
|
SITUATION:
|
||||||
|
{query}
|
||||||
|
|
||||||
|
ANWEISUNG:
|
||||||
|
Du agierst jetzt als mein empathischer Spiegel.
|
||||||
|
1. Versuche nicht sofort, das Problem technisch zu lösen.
|
||||||
|
2. Zeige Verständnis für die Situation basierend auf meinen eigenen Erfahrungen ([EXPERIENCE]) oder Glaubenssätzen ([BELIEF]), falls im Kontext vorhanden.
|
||||||
|
3. Antworte in der "Ich"-Form oder "Wir"-Form. Sei unterstützend.
|
||||||
|
|
||||||
|
TONFALL:
|
||||||
|
Ruhig, verständnisvoll, reflektiert. Keine Aufzählungszeichen, sondern fließender Text.
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# 4. TECHNICAL: Der Coder (Intent: CODING)
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
technical_template: |
|
||||||
|
KONTEXT (DOCS & SNIPPETS):
|
||||||
|
=========================================
|
||||||
|
{context_str}
|
||||||
|
=========================================
|
||||||
|
|
||||||
|
TASK:
|
||||||
|
{query}
|
||||||
|
|
||||||
|
ANWEISUNG:
|
||||||
|
Du bist Senior Developer.
|
||||||
|
1. Ignoriere Smalltalk. Komm sofort zum Punkt.
|
||||||
|
2. Generiere validen, performanten Code basierend auf den Quellen.
|
||||||
|
3. Wenn Quellen fehlen, nutze dein allgemeines Programmierwissen, aber weise darauf hin.
|
||||||
|
|
||||||
|
FORMAT:
|
||||||
|
- Kurze Erklärung des Ansatzes.
|
||||||
|
- Markdown Code-Block (Copy-Paste fertig).
|
||||||
|
- Wichtige Edge-Cases.
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# config/prompts.yaml — Persönlichkeit & RAG-Strategie
|
||||||
|
# Version: 2.0 (Audit Update)
|
||||||
|
|
||||||
|
system_prompt: |
|
||||||
|
Du bist 'mindnet', ein persönliches KI-Gedächtnis und der Digitale Zwilling deines Erschaffers ("User").
|
||||||
|
|
||||||
|
DEINE PERSÖNLICHKEIT & WERTE:
|
||||||
|
1. Pragmatismus: Du bevorzugst funktionierende Lösungen über theoretische Perfektion.
|
||||||
|
2. Transparenz: Du erfindest keine Fakten. Wenn Informationen im Kontext fehlen, sagst du das klar.
|
||||||
|
3. Vernetztes Denken: Du suchst aktiv nach Verbindungen zwischen den bereitgestellten Notizen.
|
||||||
|
4. Erklärbarkeit: Wenn du eine Aussage machst, beziehst du dich implizit auf die Quelle (z.B. "Wie in Projekt X definiert...").
|
||||||
|
|
||||||
|
DEINE AUFGABE:
|
||||||
|
Beantworte die Frage des Users ausschließlich basierend auf dem untenstehenden KONTEXT.
|
||||||
|
Der Kontext besteht aus Auszügen verschiedener Notizen. Achte besonders auf den [TYPE] der Notiz:
|
||||||
|
- [DECISION] erklärt das "Warum".
|
||||||
|
- [PROJECT] erklärt das "Was" und "Wann".
|
||||||
|
- [CONCEPT] liefert Definitionen.
|
||||||
|
- [VALUE] definiert die moralische/strategische Ausrichtung.
|
||||||
|
|
||||||
|
rag_template: |
|
||||||
|
HINTERGRUNDWISSEN (KONTEXT):
|
||||||
|
=========================================
|
||||||
|
{context_str}
|
||||||
|
=========================================
|
||||||
|
|
||||||
|
FRAGE DES USERS:
|
||||||
|
{query}
|
||||||
|
|
||||||
|
ANWEISUNG:
|
||||||
|
Analysiere die Quellen oben. Synthetisiere eine Antwort, die die Frage präzise beantwortet.
|
||||||
|
Nutze Markdown für Struktur (Fettgedrucktes für Wichtiges).
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
system_prompt: |
|
||||||
|
Du bist 'mindnet', ein persönliches KI-Gedächtnis.
|
||||||
|
|
||||||
|
DEINE REGELN:
|
||||||
|
1. Antworte NUR basierend auf dem untenstehenden KONTEXT.
|
||||||
|
2. Achte auf den TYP der Quelle:
|
||||||
|
- [DECISION] enthält Begründungen (Warum?).
|
||||||
|
- [PROJECT] enthält Ziele (Was?).
|
||||||
|
- [CONCEPT] enthält Definitionen.
|
||||||
|
3. Sei präzise. Nenne konkrete technische Gründe, wenn sie im Text stehen.
|
||||||
|
4. Halluziniere nicht.
|
||||||
|
|
||||||
|
rag_template: |
|
||||||
|
KONTEXT (WISSEN):
|
||||||
|
=========================================
|
||||||
|
{context_str}
|
||||||
|
=========================================
|
||||||
|
|
||||||
|
FRAGE:
|
||||||
|
{query}
|
||||||
|
|
||||||
|
ANWEISUNG:
|
||||||
|
Analysiere den Kontext. Wenn eine [DECISION] Quelle dabei ist, nutze deren Inhalt für die Begründung.
|
||||||
|
Antworte kurz und präzise auf Deutsch.
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
version: 1.0
|
||||||
|
|
||||||
|
scoring:
|
||||||
|
# W_sem: skaliert den Term (semantic_score * retriever_weight)
|
||||||
|
# Empfehlung Startwert: 1.0 → Semantik bleibt Hauptsignal
|
||||||
|
semantic_weight: 1.0
|
||||||
|
|
||||||
|
# W_edge: skaliert edge_bonus aus dem Subgraph
|
||||||
|
# Empfehlung: 0.7 → Graph ist deutlich spürbar, aber überstimmt Semantik nicht komplett
|
||||||
|
edge_weight: 0.7
|
||||||
|
|
||||||
|
# W_cent: skaliert centrality_bonus (Knoten-Zentralität im Subgraph)
|
||||||
|
# Empfehlung: 0.5 → zentrale Knoten werden bevorzugt, aber moderat
|
||||||
|
centrality_weight: 0.5
|
||||||
|
|
||||||
|
# Die nachfolgenden Werte überschreiben die Defaults aus app/core/retriever_config.
|
||||||
|
# Wenn neue Kantentypen, z.B. durch Referenzierung innerhalb einer md-Datei im vault anders gewichtet werden sollen, dann muss hier die Konfiguration erfolgen
|
||||||
|
edge_types:
|
||||||
|
references: 0.20
|
||||||
|
depends_on: 0.18
|
||||||
|
related_to: 0.15
|
||||||
|
similar_to: 0.12
|
||||||
|
belongs_to: 0.10
|
||||||
|
next: 0.06
|
||||||
|
prev: 0.06
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
version: 1.0
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
retriever_weight: 1.0
|
||||||
|
chunk_profile: default
|
||||||
|
edge_defaults: [] # Leerliste → keine impliziten Kanten, wenn nicht gesetzt
|
||||||
|
|
||||||
|
types:
|
||||||
|
# Wissensbausteine
|
||||||
|
concept:
|
||||||
|
chunk_profile: medium
|
||||||
|
retriever_weight: 0.60
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
|
||||||
|
source:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.50
|
||||||
|
edge_defaults: [] # Quellen sind i. d. R. Ziel von Kanten, erzeugen selbst keine Default-Kanten
|
||||||
|
|
||||||
|
glossary:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.40
|
||||||
|
edge_defaults: ["related_to"]
|
||||||
|
|
||||||
|
# Person & Persönlichkeit
|
||||||
|
profile: # z. B. "Persönliches Leitbild (Kurzprofil)"
|
||||||
|
chunk_profile: long
|
||||||
|
retriever_weight: 0.66
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
|
||||||
|
value: # Werte (z. B. "Mut", "Fairness")
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["related_to"]
|
||||||
|
|
||||||
|
principle: # Prinzipien/Leitsätze
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["related_to", "references"]
|
||||||
|
|
||||||
|
experience: # prägende Erlebnisse/Erfahrungen
|
||||||
|
chunk_profile: medium
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["related_to", "references"]
|
||||||
|
|
||||||
|
# Ziele, Entscheidungen, Meilensteine
|
||||||
|
goal:
|
||||||
|
chunk_profile: medium
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["depends_on", "related_to", "references"]
|
||||||
|
|
||||||
|
decision:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.90
|
||||||
|
edge_defaults: ["depends_on", "references", "related_to"] # „Worauf stützte sich die Entscheidung?“
|
||||||
|
|
||||||
|
milestone:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.70
|
||||||
|
edge_defaults: ["related_to", "references"]
|
||||||
|
|
||||||
|
# Operativ: Projekte, Aufgaben, Journal
|
||||||
|
project:
|
||||||
|
chunk_profile: long
|
||||||
|
retriever_weight: 0.97
|
||||||
|
edge_defaults: ["references", "depends_on"] # Projekt stützt sich auf Konzepte/Quellen und hängt von Bausteinen ab
|
||||||
|
|
||||||
|
task:
|
||||||
|
chunk_profile: short
|
||||||
|
retriever_weight: 0.80
|
||||||
|
edge_defaults: ["depends_on", "references"]
|
||||||
|
|
||||||
|
journal:
|
||||||
|
chunk_profile: medium
|
||||||
|
retriever_weight: 0.80
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
@@ -0,0 +1,261 @@
|
|||||||
|
# mindnet v2.2 – Knowledge Design Manual
|
||||||
|
**Datei:** `docs/mindnet_knowledge_design_manual_v2.2.md`
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** **FINAL** (Integrierter Stand WP01–WP10)
|
||||||
|
**Quellen:** `knowledge_design.md`, `TYPE_REGISTRY_MANUAL.md`, `chunking_strategy.md`, `mindnet_functional_architecture.md`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ⚡ Die 5 Goldenen Regeln (TL;DR)
|
||||||
|
|
||||||
|
Damit Mindnet als dein Digitaler Zwilling funktioniert, beachte beim Schreiben diese Grundsätze:
|
||||||
|
|
||||||
|
1. **Atomare Gedanken:** Eine Notiz = Ein Thema. Wenn du über zwei Projekte schreibst, mach zwei Notizen draus.
|
||||||
|
2. **Explizite Typen:** Setze immer den `type` im Frontmatter. Mindnet behandelt eine `decision` ("Wir machen X") völlig anders als ein `concept` ("Was ist X").
|
||||||
|
3. **Semantische Links:** Schreibe nicht nur `[[Link]]`, sondern `[[rel:depends_on Link]]`. Sag dem System *wie* Dinge zusammenhängen.
|
||||||
|
4. **Werte & Ziele definieren:** Damit die **Decision Engine** dich beraten kann, musst du deine Kriterien (`type: value`, `type: goal`) explizit als Notizen anlegen.
|
||||||
|
5. **Emotionales Bridging:** Damit die **Empathie** funktioniert, nutze in Erfahrungsberichten (`type: experience`) emotionale Schlüsselwörter ("Krise", "Freude", "Angst"), damit der Vektor-Retriever sie bei Gefühls-Anfragen findet.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zweck & Scope
|
||||||
|
|
||||||
|
Dieses Handbuch ist die **primäre Arbeitsanweisung** für dich als Mindmaster (Owner) und alle Autoren, die Inhalte im mindnet-Vault erstellen.
|
||||||
|
|
||||||
|
### 1.1 Zielsetzung
|
||||||
|
Mindnet ist mehr als eine Dokumentablage. Es ist ein vernetztes System, das deine Persönlichkeit, Entscheidungen und Erfahrungen abbildet.
|
||||||
|
Seit Version 2.3.1 verfügt Mindnet über:
|
||||||
|
* **Hybrid Router:** Das System erkennt, ob du Fakten, Entscheidungen oder Empathie brauchst.
|
||||||
|
* **Context Intelligence:** Das System lädt je nach Situation unterschiedliche Notiz-Typen (z.B. Werte bei Entscheidungen).
|
||||||
|
* **Web UI (WP10):** Du kannst direkt sehen, welche Quellen genutzt wurden.
|
||||||
|
|
||||||
|
### 1.2 Der Vault als „Source of Truth“
|
||||||
|
Die Markdown-Dateien in deinem Vault sind die **einzige Quelle der Wahrheit**.
|
||||||
|
* Mindnet importiert diesen Zustand deterministisch in die Datenbank (Qdrant).
|
||||||
|
* Änderungen an IDs, Typen oder Inhalten müssen **immer** im Markdown erfolgen, niemals direkt in der Datenbank.
|
||||||
|
* Was nicht im Markdown steht (z. B. implizites Wissen im Kopf), existiert für das System nicht.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Note-Struktur & Frontmatter
|
||||||
|
|
||||||
|
Jede Notiz benötigt einen YAML-Header (Frontmatter) am Dateianfang. Dieser Block macht die Notiz maschinenlesbar.
|
||||||
|
|
||||||
|
### 2.1 Pflichtfelder
|
||||||
|
Jede Datei muss mindestens folgende Felder enthalten, um korrekt verarbeitet zu werden:
|
||||||
|
|
||||||
|
---
|
||||||
|
id: 20251110-projekt-alpha-8a9b2c # Eindeutige Kennung (YYYYMMDD-slug)
|
||||||
|
title: Projekt Alpha # Sprechender Titel (wird in Suchergebnissen angezeigt)
|
||||||
|
type: project # Steuert Verarbeitung & Vernetzung (siehe Kap. 3)
|
||||||
|
status: active # Status (z.B. draft, active, archived)
|
||||||
|
created: 2025-11-10 # Erstellungsdatum (ISO 8601 oder YYYY-MM-DD)
|
||||||
|
updated: 2025-12-07 # Letzte Änderung
|
||||||
|
tags: [ki, entwicklung] # Taxonomie für Filterung
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2.2 Optionale Felder
|
||||||
|
Diese Felder sind technisch nicht zwingend, aber für bestimmte Typen sinnvoll:
|
||||||
|
|
||||||
|
lang: de # Sprache (Default: de)
|
||||||
|
aliases: [Alpha Projekt, Project A] # Synonyme für die Suche
|
||||||
|
visibility: internal # internal (default), public, private
|
||||||
|
|
||||||
|
> **Hinweis:** Felder wie `retriever_weight` oder `chunk_profile` sollten **nicht** mehr manuell im Frontmatter gesetzt werden. Diese werden zentral über den `type` gesteuert (siehe Kap. 3), um die Wartbarkeit zu sichern.
|
||||||
|
|
||||||
|
### 2.3 Empfehlungen für IDs und Pfade
|
||||||
|
|
||||||
|
**Die ID (Identifikator):**
|
||||||
|
* Muss global eindeutig und **stabil** sein.
|
||||||
|
* Darf sich nicht ändern, wenn die Datei umbenannt oder verschoben wird.
|
||||||
|
* **Empfehlung:** `YYYYMMDD-slug-hash` (z. B. `20231027-vektor-db-a1b2`). Dies garantiert Eindeutigkeit und Chronologie.
|
||||||
|
|
||||||
|
**Dateinamen & Pfade:**
|
||||||
|
* Pfade dienen der menschlichen Ordnung (Ordnerstruktur), sind für Mindnet aber sekundär.
|
||||||
|
* Dateinamen sollten dem Titel entsprechen (`Projekt Alpha.md`) oder dem Slug (`projekt-alpha.md`).
|
||||||
|
* Sonderzeichen vermeiden (außer Bindestrich und Unterstrich).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Note-Typen & Typ-Registry
|
||||||
|
|
||||||
|
Der `type` ist der wichtigste Hebel im Knowledge Design. Er steuert nicht nur das Gewicht bei der Suche, sondern seit WP-06 auch, **wann** eine Notiz aktiv in den Chat geholt wird ("Strategic Retrieval").
|
||||||
|
|
||||||
|
### 3.1 Übersicht der Kern-Typen
|
||||||
|
|
||||||
|
Mindnet unterscheidet verschiedene Wissensarten. Wähle den Typ, der die **Rolle** der Notiz am besten beschreibt:
|
||||||
|
|
||||||
|
| Typ | Beschreibung & Einsatzzweck | Rolle im Chat (Intent) |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| **`concept`** | Fachbegriffe, Theorien. Zeitloses Wissen. | **FACT** (Basiswissen) |
|
||||||
|
| **`project`** | Ein Vorhaben mit Ziel, Dauer und Aufgaben. | **FACT / DECISION** (Kontext) |
|
||||||
|
| **`experience`** | Persönliche Erfahrung, Lektion oder Erkenntnis. | **EMPATHY** (Spiegelung) |
|
||||||
|
| **`decision`** | Eine bewusst getroffene Entscheidung (ADR). | **DECISION** (Begründung "Warum") |
|
||||||
|
| **`value`** | Ein persönlicher Wert oder ein Prinzip. | **DECISION** (Moralischer Kompass) |
|
||||||
|
| **`goal`** | Ein strategisches Ziel (kurz- oder langfristig). | **DECISION** (Abgleich) |
|
||||||
|
| **`person`** | Eine reale Person (Netzwerk, Autor). | **FACT** |
|
||||||
|
| **`journal`** | Zeitbezogener Log-Eintrag, Daily Note. | **FACT** (Historie) |
|
||||||
|
| **`source`** | Externe Quelle (Buch, PDF, Artikel). | **FACT** (Faktenbasis) |
|
||||||
|
|
||||||
|
### 3.2 Zusammenspiel mit `types.yaml`
|
||||||
|
|
||||||
|
Der `type` steuert im Hintergrund drei technische Mechanismen:
|
||||||
|
|
||||||
|
1. **`retriever_weight` (Wichtigkeit):**
|
||||||
|
* Ein `concept` (0.6) wiegt weniger als ein `project` (0.97) oder eine `decision` (1.0).
|
||||||
|
* **Warum?** Bei einer Suche nach "Datenbank" soll Mindnet bevorzugt deine *Entscheidung* ("Warum wir X nutzen") anzeigen.
|
||||||
|
2. **`chunk_profile` (Textzerlegung):**
|
||||||
|
* `journal` (short): Wird fein zerlegt.
|
||||||
|
* `project` (long): Längere Kontext-Fenster.
|
||||||
|
3. **`edge_defaults` (Automatische Vernetzung):**
|
||||||
|
* Mindnet ergänzt automatisch Kanten.
|
||||||
|
* Beispiel: Ein Link in einem `project` wird automatisch als `depends_on` (Abhängigkeit) interpretiert.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Edges & Referenzen in Notes
|
||||||
|
|
||||||
|
Um aus isolierten Dateien ein **Netzwerk** zu machen, nutzen wir Verlinkungen. In Version 2.2 sind diese Verlinkungen die Basis für den **Hybrid Retriever** (Suche über Nachbarn).
|
||||||
|
|
||||||
|
### 4.1 Wikilinks (Die Basis-Referenz)
|
||||||
|
Der klassische Obsidian-Link.
|
||||||
|
|
||||||
|
Wir nutzen [[Qdrant]] als Vektordatenbank.
|
||||||
|
|
||||||
|
* **Bedeutung:** "Diese Notiz erwähnt Qdrant."
|
||||||
|
* **Edge-Typ:** `references`
|
||||||
|
|
||||||
|
### 4.2 Inline-Relationen (Semantische Verknüpfung)
|
||||||
|
Dies ist die **mächtigste** Methode. Du sagst dem System explizit, **wie** Dinge zusammenhängen.
|
||||||
|
|
||||||
|
Daher [[rel:depends_on Qdrant]].
|
||||||
|
Dieses Konzept ist [[rel:similar_to Pinecone]].
|
||||||
|
|
||||||
|
* **Syntax:** `[[rel:RELATION ZIEL]]`.
|
||||||
|
* **Gültige Relationen:**
|
||||||
|
* `depends_on`: Hängt ab von / Benötigt. (Trigger für hohe Graph-Relevanz).
|
||||||
|
* `similar_to`: Ähnelt / Ist vergleichbar mit.
|
||||||
|
* `related_to`: Hat zu tun mit (allgemein).
|
||||||
|
* `caused_by`: Wurde verursacht durch.
|
||||||
|
* `solves`: Löst (Problem).
|
||||||
|
|
||||||
|
### 4.3 Callout-Edges (Kuratierte Listen)
|
||||||
|
Für Zusammenfassungen oder "Siehe auch"-Blöcke am Ende einer Notiz.
|
||||||
|
|
||||||
|
> [!edge] related_to: [[Vector Embeddings]] [[AI Agents]]
|
||||||
|
|
||||||
|
* **Funktion:** Erzeugt `related_to`-Kanten zu allen genannten Zielen in dieser Zeile.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Schreiben für den KI-Zwilling (New in v2.3)
|
||||||
|
|
||||||
|
Damit der **RAG-Chat (WP05/06)** nicht nur dumm Fakten wiedergibt, sondern dich berät, musst du "Futter" für die Decision Engine und das Empathie-Modul liefern.
|
||||||
|
|
||||||
|
### 5.1 Futter für die Decision Engine (`DECISION`)
|
||||||
|
Das System soll abwägen: "Passt Tool X zu mir?". Dazu muss es wissen, was "Du" bist.
|
||||||
|
|
||||||
|
**Best Practice: Werte definieren**
|
||||||
|
Erstelle Notizen mit `type: value`.
|
||||||
|
|
||||||
|
---
|
||||||
|
type: value
|
||||||
|
title: Prinzip: Datensparsamkeit
|
||||||
|
---
|
||||||
|
# Prinzip: Datensparsamkeit
|
||||||
|
Wir speichern nur das Minimum an Daten. Cloud-Uploads persönlicher Daten sind verboten, es sei denn, sie sind E2E-verschlüsselt.
|
||||||
|
|
||||||
|
**Best Practice: Ziele definieren**
|
||||||
|
Erstelle Notizen mit `type: goal`.
|
||||||
|
|
||||||
|
---
|
||||||
|
type: goal
|
||||||
|
title: Ziel: Unabhängigkeit 2026
|
||||||
|
---
|
||||||
|
# Ziel: Unabhängigkeit 2026
|
||||||
|
Bis Ende 2026 wollen wir alle SaaS-Abos durch Self-Hosted Lösungen ersetzt haben.
|
||||||
|
|
||||||
|
*Effekt:* Wenn du fragst "Soll ich Notion nutzen?", lädt die Engine diese beiden Notizen und antwortet: *"Nein, Notion ist SaaS und nicht E2E-verschlüsselt. Das verletzt dein Prinzip der Datensparsamkeit und dein Ziel der Unabhängigkeit."*
|
||||||
|
|
||||||
|
### 5.2 Futter für den Empathie-Modus (`EMPATHY`)
|
||||||
|
Das System soll dich verstehen, wenn du sagst: "Alles ist grau." Dazu braucht es deine Resonanzerfahrungen.
|
||||||
|
|
||||||
|
**Best Practice: Erfahrungen & Bridging**
|
||||||
|
Erstelle Notizen mit `type: experience`. Nutze im Text **emotionale Brückenwörter**, damit der Retriever auch bei abstrakten Gefühlen ("grau") einen Treffer landet.
|
||||||
|
|
||||||
|
---
|
||||||
|
type: experience
|
||||||
|
title: Erfahrung: Der Durchbruch nach der Krise
|
||||||
|
tags: [krise, hoffnung, grau, angst]
|
||||||
|
---
|
||||||
|
# Erfahrung: Der Durchbruch nach der Krise
|
||||||
|
Es gibt Projektphasen, da wirkt alles **sinnlos** und **grau**.
|
||||||
|
Ich habe gelernt: Das ist oft das Zeichen kurz vor dem Durchbruch.
|
||||||
|
Mein Mantra in solchen Zeiten: "Einfach weitermachen, der Nebel lichtet sich."
|
||||||
|
|
||||||
|
*Effekt:* Wenn du im Chat jammerst ("Alles ist sinnlos"), findet das System diese Notiz und spiegelt dir deine eigene Erfahrung zurück.
|
||||||
|
|
||||||
|
### 5.3 Schreiben für den Chat (UI Optimization)
|
||||||
|
Da deine Notizen im Web-Interface (WP10) angezeigt werden:
|
||||||
|
* **Kurze Absätze:** Das LLM liest besser, und der Mensch scannt schneller.
|
||||||
|
* **Klare Headings:** Nutze H1/H2, um dem Chunker logische Trennlinien zu geben.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Best Practices & Beispiele (Klassik)
|
||||||
|
|
||||||
|
### 6.1 Beispiel: Projekt-Notiz
|
||||||
|
Projekte profitieren von `depends_on`, um Abhängigkeiten zu klären.
|
||||||
|
|
||||||
|
---
|
||||||
|
id: 20251115-proj-mindnet
|
||||||
|
title: Mindnet Implementierung
|
||||||
|
type: project
|
||||||
|
status: active
|
||||||
|
---
|
||||||
|
|
||||||
|
# Mindnet Implementierung
|
||||||
|
|
||||||
|
Wir bauen ein persönliches Wissensnetz.
|
||||||
|
|
||||||
|
## Tech Stack
|
||||||
|
Wir nutzen [[rel:depends_on Qdrant]] für die Vektorsuche und [[rel:depends_on FastAPI]] für das Backend.
|
||||||
|
|
||||||
|
## Architektur
|
||||||
|
Das Konzept basiert auf [[RAG Architecture]]. (Automatisch 'depends_on' durch Typ-Default).
|
||||||
|
|
||||||
|
### 6.2 Beispiel: Entscheidung (Decision Record)
|
||||||
|
Entscheidungen sind hoch gewichtet (`retriever_weight: 1.0`).
|
||||||
|
|
||||||
|
---
|
||||||
|
id: 20251120-adr-vektordb
|
||||||
|
title: ADR: Wahl von Qdrant
|
||||||
|
type: decision
|
||||||
|
status: final
|
||||||
|
tags: [architektur, db]
|
||||||
|
---
|
||||||
|
|
||||||
|
# Entscheidung: Qdrant
|
||||||
|
|
||||||
|
Wir haben uns für Qdrant entschieden.
|
||||||
|
|
||||||
|
## Alternativen
|
||||||
|
Wir haben auch [[rel:similar_to Pinecone]] und [[rel:similar_to Weaviate]] betrachtet.
|
||||||
|
|
||||||
|
## Begründung
|
||||||
|
Qdrant erlaubt lokalen Betrieb und [[rel:solves Payload Filtering Requirements]].
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Langfristige Stabilität & Virtual Schema Layer
|
||||||
|
|
||||||
|
Mindnet ist auf Langlebigkeit ausgelegt.
|
||||||
|
|
||||||
|
### 7.1 Das "Virtual Schema" Prinzip
|
||||||
|
Wir vermeiden es, Logik in den Markdown-Dateien hart zu kodieren.
|
||||||
|
* Statt in jeder Notiz zu schreiben `chunk_size: 500`, schreiben wir nur `type: essay`.
|
||||||
|
* Wie groß ein Chunk für ein Essay ist, definieren wir in der Konfiguration (`types.yaml`).
|
||||||
|
|
||||||
|
### 7.2 Was bedeutet das für dich?
|
||||||
|
* Du kannst dich auf den Inhalt konzentrieren.
|
||||||
|
* Wenn wir in Zukunft (WP08) basierend auf Feedback lernen, dass "Projekte" noch wichtiger sind, ändern wir **eine Zeile** in der Konfiguration, und das gesamte System passt sich beim nächsten Import an.
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
# Mindnet v2.2 – Overview & Einstieg
|
||||||
|
**Datei:** `docs/mindnet_overview_v2.2.md`
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** **FINAL** (Post-WP10 Release)
|
||||||
|
**Version:** 2.3.1
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Einführung: Was ist Mindnet?
|
||||||
|
|
||||||
|
**Mindnet** ist ein persönliches, lokales KI-Gedächtnis. Es ist der Versuch, einen **Digitalen Zwilling** deines Wissens und deiner Persönlichkeit zu erschaffen.
|
||||||
|
|
||||||
|
Anders als herkömmliche Notiz-Apps (wie Obsidian oder Evernote), die Texte nur passiv speichern, ist Mindnet ein **aktives System**:
|
||||||
|
* Es **versteht** Zusammenhänge über einen Wissensgraphen.
|
||||||
|
* Es **begründet** Antworten ("Warum ist das so?").
|
||||||
|
* Es **antwortet** situativ angepasst: Mal als harter Strategieberater, mal als empathischer Spiegel.
|
||||||
|
|
||||||
|
### Die Vision
|
||||||
|
> „Ein System, das nicht nur speichert, was ich weiß, sondern auch wie ich denke und fühle.“
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Die drei Ebenen des Systems
|
||||||
|
|
||||||
|
Mindnet arbeitet auf drei Schichten, die aufeinander aufbauen:
|
||||||
|
|
||||||
|
### Ebene 1: Content (Das Gedächtnis)
|
||||||
|
* **Quelle:** Dein lokaler Obsidian-Vault (Markdown).
|
||||||
|
* **Funktion:** Speicherung von Fakten, Projekten und Logs.
|
||||||
|
* **Technik:** Import-Pipeline, Chunking, Vektor-Datenbank (Qdrant).
|
||||||
|
* **Status:** 🟢 Live (WP01–WP03).
|
||||||
|
|
||||||
|
### Ebene 2: Semantik (Das Verstehen)
|
||||||
|
* **Funktion:** Verknüpfung von isolierten Notizen zu einem Netzwerk.
|
||||||
|
* **Logik:** "Projekt A *hängt ab von* Entscheidung B".
|
||||||
|
* **Technik:** Hybrider Retriever (Graph + Vektor), Explanation Engine.
|
||||||
|
* **Status:** 🟢 Live (WP04).
|
||||||
|
|
||||||
|
### Ebene 3: Identität (Die Persönlichkeit)
|
||||||
|
* **Funktion:** Interaktion und Bewertung. Das System nimmt eine Haltung ein.
|
||||||
|
* **Logik:** "Ich empfehle Lösung X, weil sie unserem Wert 'Datensparsamkeit' entspricht."
|
||||||
|
* **Technik:**
|
||||||
|
* **Intent Router:** Erkennt Absichten (Fakt vs. Gefühl vs. Entscheidung).
|
||||||
|
* **Strategic Retrieval:** Lädt gezielt Werte oder Erfahrungen nach.
|
||||||
|
* **Multi-Persona:** Passt den Tonfall an.
|
||||||
|
* **Status:** 🟢 Live (WP05–WP06).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. End-to-End Architektur
|
||||||
|
|
||||||
|
Der Datenfluss in Mindnet ist zyklisch ("Data Flywheel"):
|
||||||
|
|
||||||
|
1. **Input:** Du schreibst Notizen in Obsidian.
|
||||||
|
2. **Ingest:** Ein Python-Skript importiert, zerlegt (Chunking) und vernetzt (Edges) die Daten in Qdrant.
|
||||||
|
3. **Intent Recognition:** Der Router analysiert deine Frage: Willst du Fakten, Code oder Empathie?
|
||||||
|
4. **Retrieval:** Das System sucht Inhalte passend zum Intent (z.B. "Lade Erfahrungen bei Empathie").
|
||||||
|
5. **Generation:** Ein lokales LLM (Ollama) formuliert die Antwort.
|
||||||
|
6. **Feedback:** Du bewertest die Antwort. Das System lernt (langfristig) daraus.
|
||||||
|
|
||||||
|
**Tech-Stack:**
|
||||||
|
* **Backend:** Python 3.12, FastAPI.
|
||||||
|
* **Datenbank:** Qdrant (Vektor & Graph).
|
||||||
|
* **KI:** Ollama (Phi-3 Mini / Mistral) – 100% lokal.
|
||||||
|
* **Frontend:** Streamlit Web-UI (v2.3.1).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Dokumentations-Wegweiser
|
||||||
|
|
||||||
|
Wo findest du was?
|
||||||
|
|
||||||
|
| Wenn du... | ...lies dieses Dokument |
|
||||||
|
| :--- | :--- |
|
||||||
|
| **...wissen willst, wie man Notizen schreibt.** | `mindnet_knowledge_design_manual_v2.2.md` |
|
||||||
|
| **...das System installieren oder betreiben musst.** | `mindnet_admin_guide_v2.2.md` |
|
||||||
|
| **...am Python-Code entwickeln willst.** | `mindnet_developer_guide_v2.2.md` |
|
||||||
|
| **...die Pipeline (Import -> RAG) verstehen willst.** | `mindnet_pipeline_playbook_v2.2.md` |
|
||||||
|
| **...die genaue JSON-Struktur oder APIs suchst.** | `mindnet_technical_architecture.md` |
|
||||||
|
| **...verstehen willst, was fachlich passiert.** | `mindnet_functional_architecture.md` |
|
||||||
|
| **...den aktuellen Projektstatus suchst.** | `Programmplan_V2.2.md` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Rollen im System
|
||||||
|
|
||||||
|
* **Mindmaster (User/Owner):** Du. Du erstellst Inhalte, stellst Fragen und gibst Feedback. Du definierst die Werte (`type: value`).
|
||||||
|
* **Mindnet (Der Agent):** Der digitale Zwilling. Er agiert als pragmatischer, transparenter Assistent im Chat.
|
||||||
|
* **Administrator:** Verantwortlich für Docker-Container, Backups und LLM-Ressourcen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Aktueller Fokus
|
||||||
|
|
||||||
|
Wir haben **Phase D (Interaktion)** mit WP10 (Chat Interface) begonnen und die UI erfolgreich deployed.
|
||||||
|
Das System ist nun für Endnutzer bedienbar. Als Nächstes folgt der **Interview-Assistent (WP07)**, um Wissen dialogisch zu erfassen.
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
# Mindnet v2.2 – Admin Guide
|
||||||
|
**Datei:** `docs/mindnet_admin_guide_v2.2.md`
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** **FINAL** (Inkl. Frontend Deployment)
|
||||||
|
**Quellen:** `Handbuch.md`, `mindnet_developer_guide_v2.2.md`.
|
||||||
|
|
||||||
|
> Dieses Handbuch richtet sich an **Administratoren**. Es beschreibt Installation, Konfiguration, Backup-Strategien, Monitoring und den sicheren Betrieb der Mindnet-Instanz (API + UI + DB).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zielgruppe & Scope
|
||||||
|
|
||||||
|
Mindnet ist als **Multi-Service-Architektur** konzipiert, die lokal oder auf einem privaten Server läuft.
|
||||||
|
Wir unterscheiden strikt zwischen:
|
||||||
|
* **Production:** Backend (8001) + Frontend (8501). Stabiler `main` Branch.
|
||||||
|
* **Development:** Backend (8002) + Frontend (8502). Experimentelle Feature-Branches.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Initial Setup & Installation
|
||||||
|
|
||||||
|
### 2.1 Systemvoraussetzungen
|
||||||
|
* **OS:** Linux (Ubuntu 22.04+ empfohlen) oder macOS.
|
||||||
|
* **Runtime:** Python 3.10+, Docker (für Qdrant), Ollama (für LLM).
|
||||||
|
* **Hardware:**
|
||||||
|
* CPU: 4+ Cores empfohlen (für Import & Inference).
|
||||||
|
* RAM: Min. 8GB empfohlen (4GB System + 4GB für Phi-3/Qdrant).
|
||||||
|
* Disk: SSD empfohlen für Qdrant-Performance.
|
||||||
|
|
||||||
|
### 2.2 Installation (Code)
|
||||||
|
|
||||||
|
# 1. Repository klonen
|
||||||
|
git clone <repo-url> /home/llmadmin/mindnet
|
||||||
|
cd /home/llmadmin/mindnet
|
||||||
|
|
||||||
|
# 2. Umgebung einrichten
|
||||||
|
python3 -m venv .venv
|
||||||
|
source .venv/bin/activate
|
||||||
|
|
||||||
|
# 3. Dependencies installieren (inkl. Streamlit)
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
# 4. Verzeichnisse anlegen
|
||||||
|
mkdir -p logs qdrant_storage data/logs
|
||||||
|
|
||||||
|
### 2.3 Qdrant Setup (Docker)
|
||||||
|
Wir nutzen Qdrant als Vektor-Datenbank. Persistenz ist wichtig.
|
||||||
|
|
||||||
|
docker run -d \
|
||||||
|
--name mindnet_qdrant \
|
||||||
|
--restart always \
|
||||||
|
-p 6333:6333 \
|
||||||
|
-v $(pwd)/qdrant_storage:/qdrant/storage \
|
||||||
|
qdrant/qdrant
|
||||||
|
|
||||||
|
### 2.4 Ollama Setup (LLM Service)
|
||||||
|
Mindnet benötigt einen lokalen LLM-Server für den Chat.
|
||||||
|
|
||||||
|
# 1. Installieren (Linux Script)
|
||||||
|
curl -fsSL [https://ollama.com/install.sh](https://ollama.com/install.sh) | sh
|
||||||
|
|
||||||
|
# 2. Modell laden (Phi-3 Mini für CPU-Performance)
|
||||||
|
ollama pull phi3:mini
|
||||||
|
|
||||||
|
# 3. Testen
|
||||||
|
curl http://localhost:11434/api/generate -d '{"model": "phi3:mini", "prompt":"Hi"}'
|
||||||
|
|
||||||
|
### 2.5 Konfiguration (ENV)
|
||||||
|
Erstelle eine `.env` Datei im Root-Verzeichnis. Die neuen Settings für WP-06 (Timeout, Decision Config) sind essenziell für stabilen Betrieb auf CPUs.
|
||||||
|
|
||||||
|
# Qdrant Verbindung
|
||||||
|
QDRANT_URL="http://localhost:6333"
|
||||||
|
|
||||||
|
# Mindnet Core Settings
|
||||||
|
COLLECTION_PREFIX="mindnet"
|
||||||
|
MINDNET_TYPES_FILE="./config/types.yaml"
|
||||||
|
|
||||||
|
# LLM / RAG Settings
|
||||||
|
MINDNET_LLM_MODEL="phi3:mini"
|
||||||
|
MINDNET_OLLAMA_URL="[http://127.0.0.1:11434](http://127.0.0.1:11434)"
|
||||||
|
|
||||||
|
# Config & Timeouts
|
||||||
|
MINDNET_PROMPTS_PATH="./config/prompts.yaml"
|
||||||
|
MINDNET_DECISION_CONFIG="./config/decision_engine.yaml"
|
||||||
|
MINDNET_LLM_TIMEOUT=300.0
|
||||||
|
|
||||||
|
### 2.6 Deployment via Systemd (Backend & Frontend)
|
||||||
|
|
||||||
|
Mindnet benötigt zwei Services pro Umgebung: API (Uvicorn) und UI (Streamlit).
|
||||||
|
|
||||||
|
**Production Backend (`/etc/systemd/system/mindnet-prod.service`):**
|
||||||
|
|
||||||
|
[Unit]
|
||||||
|
Description=Mindnet API Prod (8001)
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
User=llmadmin
|
||||||
|
Group=llmadmin
|
||||||
|
WorkingDirectory=/home/llmadmin/mindnet
|
||||||
|
ExecStart=/home/llmadmin/mindnet/.venv/bin/uvicorn app.main:app --host 0.0.0.0 --port 8001 --env-file .env
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
|
||||||
|
**Production Frontend (`/etc/systemd/system/mindnet-ui-prod.service`):**
|
||||||
|
|
||||||
|
[Unit]
|
||||||
|
Description=Mindnet UI Prod (8501)
|
||||||
|
After=mindnet-prod.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
User=llmadmin
|
||||||
|
Group=llmadmin
|
||||||
|
WorkingDirectory=/home/llmadmin/mindnet
|
||||||
|
|
||||||
|
# Environment Variables
|
||||||
|
Environment="MINDNET_API_URL=http://localhost:8001"
|
||||||
|
Environment="MINDNET_API_TIMEOUT=300"
|
||||||
|
Environment="STREAMLIT_SERVER_PORT=8501"
|
||||||
|
Environment="STREAMLIT_SERVER_ADDRESS=0.0.0.0"
|
||||||
|
Environment="STREAMLIT_SERVER_HEADLESS=true"
|
||||||
|
|
||||||
|
ExecStart=/home/llmadmin/mindnet/.venv/bin/streamlit run app/frontend/ui.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
|
||||||
|
**Firewall (UFW):**
|
||||||
|
Öffne die Ports für den Zugriff:
|
||||||
|
|
||||||
|
sudo ufw allow 8501/tcp # Prod UI
|
||||||
|
sudo ufw allow 8502/tcp # Dev UI (falls gewünscht)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Betrieb im Alltag
|
||||||
|
|
||||||
|
### 3.1 Regelmäßige Importe
|
||||||
|
Der Vault-Zustand sollte regelmäßig (z.B. stündlich per Cronjob) nach Qdrant synchronisiert werden.
|
||||||
|
|
||||||
|
**Cronjob-Beispiel (stündlich):**
|
||||||
|
|
||||||
|
0 * * * * cd /home/llmadmin/mindnet && .venv/bin/python3 -m scripts.import_markdown --vault /path/to/vault --prefix "mindnet" --apply --purge-before-upsert --sync-deletes >> ./logs/import.log 2>&1
|
||||||
|
|
||||||
|
### 3.2 Health-Checks
|
||||||
|
Prüfe regelmäßig, ob alle Komponenten laufen.
|
||||||
|
|
||||||
|
sudo systemctl status mindnet-prod
|
||||||
|
sudo systemctl status mindnet-ui-prod
|
||||||
|
sudo systemctl status ollama
|
||||||
|
|
||||||
|
### 3.3 Logs & Monitoring
|
||||||
|
* **Backend Fehler:** `journalctl -u mindnet-prod -f`
|
||||||
|
* **Frontend Fehler:** `journalctl -u mindnet-ui-prod -f`
|
||||||
|
* Achte auf "Timeout"-Meldungen im Frontend, wenn das Backend zu langsam antwortet.
|
||||||
|
* **LLM Fehler:** `journalctl -u ollama -f`
|
||||||
|
* **Fachliche Logs:** `data/logs/search_history.jsonl`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Update-Prozess
|
||||||
|
|
||||||
|
Wenn neue Versionen ausgerollt werden (Deployment):
|
||||||
|
|
||||||
|
1. **Code aktualisieren:**
|
||||||
|
|
||||||
|
cd /home/llmadmin/mindnet
|
||||||
|
git pull origin main
|
||||||
|
|
||||||
|
2. **Dependencies prüfen:**
|
||||||
|
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
3. **Dienste neustarten (Zwingend!):**
|
||||||
|
|
||||||
|
sudo systemctl restart mindnet-prod
|
||||||
|
sudo systemctl restart mindnet-ui-prod
|
||||||
|
|
||||||
|
4. **Schema-Migration (falls nötig):**
|
||||||
|
|
||||||
|
python3 -m scripts.import_markdown ... --apply
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Backup & Restore
|
||||||
|
|
||||||
|
Datensicherheit ruht auf drei Säulen: Vault (Source), Qdrant (Index), JSONL-Logs (Lern-Daten).
|
||||||
|
|
||||||
|
### 5.1 Vault-Backup (Priorität 1)
|
||||||
|
Der Markdown-Vault ist die **Single Source of Truth**.
|
||||||
|
|
||||||
|
### 5.2 Qdrant-Snapshots (Priorität 2)
|
||||||
|
Für schnelle Wiederherstellung des Suchindex.
|
||||||
|
|
||||||
|
docker stop mindnet_qdrant
|
||||||
|
tar -czf qdrant_backup_$(date +%F).tar.gz ./qdrant_storage
|
||||||
|
docker start mindnet_qdrant
|
||||||
|
|
||||||
|
### 5.3 Log-Daten (Priorität 3)
|
||||||
|
Sichere den Ordner `data/logs/`. Verlust dieser Daten bedeutet Verlust des Trainingsmaterials für Self-Tuning.
|
||||||
|
|
||||||
|
### 5.4 Notfall-Wiederherstellung (Rebuild)
|
||||||
|
Wenn die Datenbank korrupt ist:
|
||||||
|
|
||||||
|
# 1. DB komplett leeren (Wipe)
|
||||||
|
python3 -m scripts.reset_qdrant --mode wipe --prefix "mindnet" --yes
|
||||||
|
# 2. Alles neu importieren
|
||||||
|
python3 -m scripts.import_markdown --vault /path/to/vault --prefix "mindnet" --apply
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Governance & Sicherheit
|
||||||
|
|
||||||
|
### 6.1 Zugriffsschutz
|
||||||
|
Mindnet hat aktuell **keine integrierte Authentifizierung**.
|
||||||
|
* **Frontend:** Streamlit auf Port 8501 ist offen. Nutze Nginx Basic Auth oder VPN.
|
||||||
|
* **API:** Sollte nicht direkt im öffentlichen Netz stehen.
|
||||||
|
* **Qdrant:** Auf `127.0.0.1` beschränken.
|
||||||
|
|
||||||
|
### 6.2 Typen-Governance
|
||||||
|
Änderungen an der `types.yaml` (z.B. neue Gewichte) wirken global und erfordern Tests.
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
# Mindnet v2.2 – Appendices & Referenzen
|
||||||
|
**Datei:** `docs/mindnet_appendices_v2.2.md`
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** **FINAL** (Integrierter Stand WP01–WP10)
|
||||||
|
**Quellen:** `TYPE_REGISTRY_MANUAL.md`, `chunking_strategy.md`, `mindnet_technical_architecture.md`, `Handbuch.md`.
|
||||||
|
|
||||||
|
> Dieses Dokument bündelt Tabellen, Schemata und technische Referenzen, die in den Prozess-Dokumenten (Playbook, Guides) den Lesefluss stören würden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang A: Typ-Registry Referenz (Default-Werte)
|
||||||
|
|
||||||
|
Diese Tabelle zeigt die Standard-Konfiguration der `types.yaml` (Stand v2.3.1).
|
||||||
|
|
||||||
|
| Typ (`type`) | Chunk Profile | Retriever Weight | Edge Defaults (Auto-Kanten) | Beschreibung |
|
||||||
|
| :--- | :--- | :--- | :--- | :--- |
|
||||||
|
| **concept** | `medium` | 0.60 | `references`, `related_to` | Abstrakte Begriffe, Theorien. |
|
||||||
|
| **project** | `long` | 0.97 | `references`, `depends_on` | Aktive Vorhaben. Hohe Priorität. |
|
||||||
|
| **decision** | `long` | 1.00 | `caused_by`, `references` | Entscheidungen (ADRs). Höchste Prio. |
|
||||||
|
| **experience** | `medium` | 0.90 | `derived_from`, `inspired_by` | Persönliche Learnings. |
|
||||||
|
| **journal** | `short` | 0.80 | `related_to` | Zeitgebundene Logs. Fein granular. |
|
||||||
|
| **person** | `short` | 0.50 | `related_to` | Personen-Profile. |
|
||||||
|
| **source** | `long` | 0.50 | *(keine)* | Externe Quellen (Bücher, PDFs). |
|
||||||
|
| **event** | `short` | 0.60 | `related_to` | Meetings, Konferenzen. |
|
||||||
|
| **value** | `medium` | 1.00 | `related_to` | Persönliche Werte/Prinzipien. |
|
||||||
|
| **goal** | `medium` | 0.95 | `depends_on` | Strategische Ziele (Neu in WP06). |
|
||||||
|
| **belief** | `medium` | 0.90 | `related_to` | Glaubenssätze (Neu in WP06). |
|
||||||
|
| **default** | `medium` | 1.00 | `references` | Fallback, wenn Typ unbekannt. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang B: Edge-Typen & Semantik
|
||||||
|
|
||||||
|
Referenz aller implementierten Kantenarten (`kind`).
|
||||||
|
|
||||||
|
| Kind | Herkunft (Primär) | Symmetrisch? | Bedeutung & Nutzung |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| `references` | Wikilink `[[...]]` | Nein | Standard-Verweis. "Erwähnt X". |
|
||||||
|
| `belongs_to` | Struktur (Auto) | Nein | Verknüpft Chunk mit seiner Note. |
|
||||||
|
| `next` / `prev` | Struktur (Auto) | Ja (Logisch) | Definiert Lesereihenfolge der Chunks. |
|
||||||
|
| `depends_on` | Inline / Default | Nein | Harte Abhängigkeit. "Braucht X um zu funktionieren". |
|
||||||
|
| `related_to` | Callout / Default | Ja (Oft) | Thematische Nähe. "Siehe auch X". |
|
||||||
|
| `similar_to` | Inline | Ja | Inhaltliche Ähnlichkeit. "Ist wie X". |
|
||||||
|
| `caused_by` | Inline | Nein | Kausalität. "X ist der Grund für Y". |
|
||||||
|
| `solves` | Inline | Nein | Lösung. "Tool X löst Problem Y". |
|
||||||
|
| `derived_from` | Default (Exp) | Nein | Herkunft. "Erkenntnis stammt aus Quelle X". |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang C: Datenmodelle (JSON Payloads)
|
||||||
|
|
||||||
|
Diese sind die Felder, die effektiv in Qdrant gespeichert werden.
|
||||||
|
|
||||||
|
### C.1 Note Payload (`mindnet_notes`)
|
||||||
|
|
||||||
|
{
|
||||||
|
"note_id": "string (keyword)", // UUIDv5 oder Slug
|
||||||
|
"title": "string (text)", // Titel aus Frontmatter
|
||||||
|
"type": "string (keyword)", // Typ (z.B. 'project')
|
||||||
|
"retriever_weight": "float", // Numerische Wichtigkeit (0.0-1.0)
|
||||||
|
"chunk_profile": "string", // Genutztes Profil (z.B. 'long')
|
||||||
|
"edge_defaults": ["string"], // Liste der aktiven Default-Kanten
|
||||||
|
"tags": ["string"], // Liste von Tags
|
||||||
|
"created": "string (iso-date)", // Erstellungsdatum
|
||||||
|
"updated": "integer", // Timestamp
|
||||||
|
"fulltext": "string (no-index)" // Gesamter Text (für Recovery)
|
||||||
|
}
|
||||||
|
|
||||||
|
### C.2 Chunk Payload (`mindnet_chunks`)
|
||||||
|
|
||||||
|
{
|
||||||
|
"chunk_id": "string (keyword)", // Format: {note_id}#c{index}
|
||||||
|
"note_id": "string (keyword)", // FK zur Note
|
||||||
|
"type": "string (keyword)", // Typ-Kopie aus Note (Neu in WP06a)
|
||||||
|
"text": "string (text)", // Reintext für Anzeige (ohne Overlap)
|
||||||
|
"window": "string (text)", // Text + Overlap (für Embedding)
|
||||||
|
"ord": "integer", // Laufende Nummer (1..N)
|
||||||
|
"retriever_weight": "float", // Kopie aus Note (für Query-Speed)
|
||||||
|
"neighbors_prev": ["string"], // ID des Vorgängers
|
||||||
|
"neighbors_next": ["string"] // ID des Nachfolgers
|
||||||
|
}
|
||||||
|
|
||||||
|
### C.3 Edge Payload (`mindnet_edges`)
|
||||||
|
|
||||||
|
{
|
||||||
|
"edge_id": "string (keyword)", // Deterministischer Hash
|
||||||
|
"source_id": "string (keyword)", // Chunk-ID (Start)
|
||||||
|
"target_id": "string (keyword)", // Chunk-ID oder Note-Titel (Ziel)
|
||||||
|
"kind": "string (keyword)", // z.B. 'depends_on'
|
||||||
|
"scope": "string (keyword)", // Immer 'chunk' in v2.2
|
||||||
|
"rule_id": "string (keyword)", // Traceability: 'inline:rel', 'explicit:wikilink'
|
||||||
|
"confidence": "float", // 0.0 - 1.0
|
||||||
|
"note_id": "string (keyword)" // Owner Note ID (für Löschung)
|
||||||
|
}
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang D: Environment Variablen
|
||||||
|
|
||||||
|
Diese Variablen steuern das Verhalten der Skripte und Container.
|
||||||
|
|
||||||
|
| Variable | Default | Beschreibung |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| `QDRANT_URL` | `http://localhost:6333` | URL zur Vektor-DB. |
|
||||||
|
| `QDRANT_API_KEY` | *(leer)* | API-Key für Absicherung (optional). |
|
||||||
|
| `COLLECTION_PREFIX` | `mindnet` | Namensraum für Collections (`{prefix}_notes` etc). |
|
||||||
|
| `MINDNET_TYPES_FILE` | `config/types.yaml` | Pfad zur Typ-Registry. |
|
||||||
|
| `MINDNET_RETRIEVER_CONFIG`| `config/retriever.yaml`| Pfad zur Scoring-Konfiguration. |
|
||||||
|
| `MINDNET_PROMPTS_PATH` | `config/prompts.yaml` | Pfad zu LLM-Prompts (Neu in v2.2). |
|
||||||
|
| `MINDNET_DECISION_CONFIG` | `config/decision_engine.yaml` | Pfad zur Router-Config (Neu in v2.3). |
|
||||||
|
| `MINDNET_LLM_MODEL` | `phi3:mini` | Name des Ollama-Modells (Neu in v2.2). |
|
||||||
|
| `MINDNET_OLLAMA_URL` | `http://127.0.0.1:11434`| URL zum LLM-Server (Neu in v2.2). |
|
||||||
|
| `MINDNET_LLM_TIMEOUT` | `120.0` | Timeout für Ollama (Erhöhen auf 300.0 für CPU). |
|
||||||
|
| `MINDNET_HASH_COMPARE` | `Body` | Vergleichsmodus für Import (`Body`, `Frontmatter`, `Full`). |
|
||||||
|
| `MINDNET_HASH_SOURCE` | `parsed` | Quelle für Hash (`parsed`, `raw`, `file`). |
|
||||||
|
| `VECTOR_DIM` | `384` | Dimension der Embeddings (Modellabhängig). |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang E: Glossar
|
||||||
|
|
||||||
|
* **Callout-Edge:** Kante via `> [!edge]`.
|
||||||
|
* **Decision Engine:** Komponente, die den Intent prüft und Strategien wählt (WP06).
|
||||||
|
* **Explanation Layer:** Komponente, die Scores und Graphen als Begründung liefert.
|
||||||
|
* **Hybrid Router:** Kombination aus Keyword-Matching und LLM-Klassifizierung für Intents.
|
||||||
|
* **RAG (Retrieval Augmented Generation):** Kombination aus Suche und Text-Generierung.
|
||||||
|
* **Strategic Retrieval:** Gezieltes Nachladen von Werten (`value`) bei Entscheidungfragen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Anhang F: Workpackage Status (v2.3.2)
|
||||||
|
|
||||||
|
Aktueller Implementierungsstand der Module.
|
||||||
|
|
||||||
|
| WP | Titel | Status | Anmerkung |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| **WP01** | Knowledge Design | 🟢 Live | Typen, Frontmatter definiert. |
|
||||||
|
| **WP02** | Chunking Strategy | 🟢 Live | Profilbasiertes Chunking aktiv. |
|
||||||
|
| **WP03** | Edge Logic / Import | 🟢 Live | Neue Importer-Pipeline mit Provenance. |
|
||||||
|
| **WP04a**| Retriever Scoring | 🟢 Live | Hybrider Score (Semantik + Graph). |
|
||||||
|
| **WP04b**| Explanation Layer | 🟢 Live | API liefert Reasons & Breakdown. |
|
||||||
|
| **WP04c**| Feedback Loop | 🟢 Live | Logging (JSONL) & Traceability aktiv. |
|
||||||
|
| **WP05** | Persönlichkeit / Chat | 🟢 Live | RAG-Chat mit Context Enrichment. |
|
||||||
|
| **WP06** | Decision Engine | 🟢 Live | Hybrid Router, Strategic Retrieval, Multi-Persona. |
|
||||||
|
| **WP07** | Interview Assistent | 🟡 Geplant | Dialog-Modus (Nächster Schritt). |
|
||||||
|
| **WP08** | Self-Tuning | 🔴 Geplant | Auto-Adjustment der Gewichte. |
|
||||||
|
| **WP10** | Chat Interface | 🟢 Live | Web-Interface (Streamlit). |
|
||||||
|
| **WP10a**| GUI Evolution | 🔴 Geplant | Interaktive Tools & Editor. |
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# Changelog — WP-03 Type-Registry (2025-11-08)
|
||||||
|
|
||||||
|
## Added
|
||||||
|
- `app/core/type_registry.py` — Loader & Helper (cached), robust Fallbacks.
|
||||||
|
- `config/types.yaml` — Beispiel-Registry (version 1.0).
|
||||||
|
- `scripts/import_markdown.py` v3.8.0 — Registry-Integration:
|
||||||
|
- typvalidierung (unbekannt → `concept`, Warnung)
|
||||||
|
- `chunk_profile` → an `make_chunk_payloads(...)`
|
||||||
|
- `retriever_weight` → persistiert im Note-Payload
|
||||||
|
- `edge_defaults["references"]` schaltet Note-Scope-Refs additiv ein
|
||||||
|
- `app/core/chunk_payload.py` v2.3.0 — optionales `chunk_profile` für
|
||||||
|
synthetische Overlaps (keine Schemaänderung).
|
||||||
|
- `app/core/derive_edges.py` v1.5.0 — Type-Registry aktiviert note-scope
|
||||||
|
References/Backlinks additiv.
|
||||||
|
|
||||||
|
## Changed
|
||||||
|
- Keine Breaking Changes. Alle CLI-Flags, ENV-Variablen und Collection-
|
||||||
|
Prefix-Logik bleiben unverändert.
|
||||||
|
|
||||||
|
## Removed
|
||||||
|
- N/A
|
||||||
File renamed without changes.
@@ -0,0 +1,74 @@
|
|||||||
|
# Kurzhandbuch — Type-Registry (WP-03)
|
||||||
|
|
||||||
|
## Zweck
|
||||||
|
Die Type-Registry macht den `type:`-Wert im Frontmatter maschinenlesbar
|
||||||
|
und steuert:
|
||||||
|
- Chunk-Profile (indirekt über Overlap bei fensterlosen Chunks),
|
||||||
|
- typabhängige Default-Kanten (hier: Aktivierung von Note-Scope-References),
|
||||||
|
- optionale Retriever-Gewichtungen (`retriever_weight`) für WP‑04.
|
||||||
|
|
||||||
|
## Aktivierung
|
||||||
|
1. Legen Sie `config/types.yaml` an (siehe Beispiel in diesem Paket).
|
||||||
|
2. Starten Sie Import/Server neu. Die Registry wird beim Prozessstart
|
||||||
|
geladen und gecached.
|
||||||
|
|
||||||
|
Fehlt die Datei oder ist sie defekt, bleibt das System voll funktionsfähig
|
||||||
|
(der Typ `concept` wird genutzt; Warnhinweis im Log).
|
||||||
|
|
||||||
|
## Schema (Beispiel)
|
||||||
|
```yaml
|
||||||
|
version: 1.0
|
||||||
|
types:
|
||||||
|
concept:
|
||||||
|
chunk_profile: long
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
retriever_weight: 1.0
|
||||||
|
task:
|
||||||
|
chunk_profile: short
|
||||||
|
edge_defaults: ["depends_on", "belongs_to"]
|
||||||
|
retriever_weight: 0.8
|
||||||
|
experience:
|
||||||
|
chunk_profile: medium
|
||||||
|
edge_defaults: ["derived_from", "inspired_by"]
|
||||||
|
retriever_weight: 0.9
|
||||||
|
project:
|
||||||
|
chunk_profile: medium
|
||||||
|
edge_defaults: ["references", "depends_on"]
|
||||||
|
retriever_weight: 0.95
|
||||||
|
```
|
||||||
|
|
||||||
|
## Verhalten
|
||||||
|
- **Chunking**: Die eigentliche Chunk-Länge bleibt weiterhin typabhängig über
|
||||||
|
`chunk_config.get_sizes(type)` gesteuert. Falls der Chunker keine Fenster
|
||||||
|
mit Overlap liefert, werden in `chunk_payload` synthetische Fenster erzeugt.
|
||||||
|
Hier greift das `chunk_profile` mit folgenden Overlap-Empfehlungen:
|
||||||
|
- short → (20,30), medium → (40,60), long → (60,80).
|
||||||
|
- **Edges**: Enthält `edge_defaults` den Eintrag `references`, werden Note-
|
||||||
|
Scope-References/Backlinks additiv aktiviert (kein Abbruch, keine Duplikate).
|
||||||
|
Weitere Kantenarten bleiben unverändert.
|
||||||
|
- **Retriever**: `retriever_weight` wird in der Note gespeichert und kann von
|
||||||
|
WP‑04 im Ranking/Scoring berücksichtigt werden.
|
||||||
|
|
||||||
|
## Tests / Validierung
|
||||||
|
- Dry-Run:
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.import_markdown --vault ./test_vault
|
||||||
|
```
|
||||||
|
- Vollimport + Purge:
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.import_markdown --vault ./test_vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
|
||||||
|
```
|
||||||
|
- Chunks-Qualität (Window vs Text):
|
||||||
|
```bash
|
||||||
|
python3 tests/check_chunks_window_vs_text.py --prefix "$COLLECTION_PREFIX"
|
||||||
|
```
|
||||||
|
- Roundtrip:
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.export_markdown --out ./_exportVault --prefix "$COLLECTION_PREFIX"
|
||||||
|
python3 tests/compare_vaults.py --src ./test_vault --dst ./_exportVault --focus body
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fallbacks & Robustheit
|
||||||
|
- Registry nicht vorhanden/ungültig → `concept`, Warnung, keine Abbrüche.
|
||||||
|
- Alle bestehenden ENV/Flags bleiben identisch.
|
||||||
|
- Keine Änderungen an Punkt-IDs, Feldern oder Vektoren.
|
||||||
File renamed without changes.
@@ -0,0 +1,96 @@
|
|||||||
|
# mindnet Retriever (WP-04 / Step 4a) – Kurz-Dokumentation
|
||||||
|
|
||||||
|
## 1. Überblick
|
||||||
|
|
||||||
|
Der Retriever besteht aus:
|
||||||
|
|
||||||
|
- **Semantic Retriever** (`semantic_retrieve`)
|
||||||
|
- **Hybrid Retriever** (`hybrid_retrieve`)
|
||||||
|
- Qdrant-Backend (`*_chunks`, `*_edges`)
|
||||||
|
- optionaler Edge-Expansion über `graph_adapter.expand`
|
||||||
|
|
||||||
|
Die API nutzt das bestehende `/query`-Endpoint (FastAPI), das `QueryRequest` und `QueryResponse` aus `app.models.dto` verwendet.
|
||||||
|
|
||||||
|
## 2. Scoring-Formel
|
||||||
|
|
||||||
|
Pro Treffer werden folgende Komponenten geführt:
|
||||||
|
|
||||||
|
- `semantic_score` – direkt aus Qdrant (Cosine/Euclidean, je nach Collection)
|
||||||
|
- `retriever_weight` – aus Chunk-Payload (`types.yaml` / Frontmatter)
|
||||||
|
- `edge_bonus` – Bonus aus dem Subgraph (z. B. Zahl der eingehenden/ausgehenden Edges, Beziehungstypen)
|
||||||
|
- `centrality_bonus` – Bonus aus einer einfachen Zentralitäts-Heuristik im Subgraph
|
||||||
|
|
||||||
|
Aktuelle Formel (Step 4a):
|
||||||
|
|
||||||
|
```text
|
||||||
|
total_score = semantic_score * max(retriever_weight, 0.0)
|
||||||
|
+ edge_bonus
|
||||||
|
+ centrality_bonus
|
||||||
|
```
|
||||||
|
|
||||||
|
Die Sortierung der Ergebnisse erfolgt absteigend nach `total_score`.
|
||||||
|
|
||||||
|
## 3. Modi
|
||||||
|
|
||||||
|
### 3.1 Semantic Mode
|
||||||
|
|
||||||
|
- Funktion: `semantic_retrieve`
|
||||||
|
- Request-Feld `mode = "semantic"`
|
||||||
|
- Keine Edge-Expansion, kein Graph-Call.
|
||||||
|
- Kandidaten nur aus Vektorsuche in `<prefix>_chunks`.
|
||||||
|
|
||||||
|
### 3.2 Hybrid Mode
|
||||||
|
|
||||||
|
- Funktion: `hybrid_retrieve`
|
||||||
|
- Request-Feld `mode = "hybrid"`
|
||||||
|
- Schritte:
|
||||||
|
1. Semantische Chunk-Suche wie im Semantic Mode
|
||||||
|
2. Falls `expand.depth > 0` gesetzt ist:
|
||||||
|
- Seeds = `chunk_id` (Fallback: `note_id`) der Treffer
|
||||||
|
- Aufruf `graph_adapter.expand(client, prefix, seed_ids, depth, edge_types)`
|
||||||
|
- Ergebnis-Subgraph liefert `edge_bonus` und `centrality_bonus` pro Node
|
||||||
|
3. Scores werden nach obiger Formel kombiniert.
|
||||||
|
|
||||||
|
## 4. Request-Struktur (JSON)
|
||||||
|
|
||||||
|
Beispiel-Hybrid-Request:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mode": "hybrid",
|
||||||
|
"query": "embeddings und qdrant",
|
||||||
|
"top_k": 10,
|
||||||
|
"expand": {
|
||||||
|
"depth": 1,
|
||||||
|
"edge_types": ["references", "depends_on"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Minimaler Semantic-Request:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mode": "semantic",
|
||||||
|
"query": "karate trainingsplan",
|
||||||
|
"top_k": 5
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. Smoke-Test
|
||||||
|
|
||||||
|
Für einen schnellen End-to-End-Test kann folgendes Skript verwendet werden:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python scripts/test_retriever_smoke.py --query "karate" --mode hybrid --expand-depth 1 --top-k 5
|
||||||
|
```
|
||||||
|
|
||||||
|
Das Skript ruft `/query` auf, wertet Statuscode und JSON-Antwort aus und zeigt die Scores der Top-K-Treffer an.
|
||||||
|
|
||||||
|
## 6. Tests
|
||||||
|
|
||||||
|
Die wichtigsten Tests für den Retriever sind:
|
||||||
|
|
||||||
|
- `tests/test_retriever_basic.py` – Basisfunktion (Semantic + Hybrid)
|
||||||
|
- `tests/test_retriever_weight.py` – Einfluss von `retriever_weight` auf das Ranking
|
||||||
|
- `tests/test_retriever_edges.py` – Einfluss von Edge-/Centrality-Boni im Hybrid-Modus
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
# mindnet v2.2 – Knowledge Design Manual
|
||||||
|
**Datei:** `docs/mindnet_knowledge_design_manual_v2.2.md`
|
||||||
|
**Stand:** 2025-12-09
|
||||||
|
**Status:** **FINAL** (Integrierter Stand WP01–WP06)
|
||||||
|
**Quellen:** `knowledge_design.md`, `TYPE_REGISTRY_MANUAL.md`, `chunking_strategy.md`, `mindnet_functional_architecture.md`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ⚡ Die 5 Goldenen Regeln (TL;DR)
|
||||||
|
|
||||||
|
Damit Mindnet als dein Digitaler Zwilling funktioniert, beachte beim Schreiben diese Grundsätze:
|
||||||
|
|
||||||
|
1. **Atomare Gedanken:** Eine Notiz = Ein Thema. Wenn du über zwei Projekte schreibst, mach zwei Notizen draus.
|
||||||
|
2. **Explizite Typen:** Setze immer den `type` im Frontmatter. Mindnet behandelt eine `decision` ("Wir machen X") völlig anders als ein `concept` ("Was ist X").
|
||||||
|
3. **Semantische Links:** Schreibe nicht nur `[[Link]]`, sondern `[[rel:depends_on Link]]`. Sag dem System *wie* Dinge zusammenhängen.
|
||||||
|
4. **Werte & Ziele definieren:** Damit die **Decision Engine** dich beraten kann, musst du deine Kriterien (`type: value`, `type: goal`) explizit als Notizen anlegen.
|
||||||
|
5. **Emotionales Bridging:** Damit die **Empathie** funktioniert, nutze in Erfahrungsberichten (`type: experience`) emotionale Schlüsselwörter ("Krise", "Freude", "Angst"), damit der Vektor-Retriever sie bei Gefühls-Anfragen findet.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zweck & Scope
|
||||||
|
|
||||||
|
Dieses Handbuch ist die **primäre Arbeitsanweisung** für dich als Mindmaster (Owner) und alle Autoren, die Inhalte im mindnet-Vault erstellen.
|
||||||
|
|
||||||
|
### 1.1 Zielsetzung
|
||||||
|
Mindnet ist mehr als eine Dokumentablage. Es ist ein vernetztes System, das deine Persönlichkeit, Entscheidungen und Erfahrungen abbildet.
|
||||||
|
Seit Version 2.3.1 verfügt Mindnet über:
|
||||||
|
* **Hybrid Router:** Das System erkennt, ob du Fakten, Entscheidungen oder Empathie brauchst.
|
||||||
|
* **Context Intelligence:** Das System lädt je nach Situation unterschiedliche Notiz-Typen (z.B. Werte bei Entscheidungen).
|
||||||
|
|
||||||
|
### 1.2 Der Vault als „Source of Truth“
|
||||||
|
Die Markdown-Dateien in deinem Vault sind die **einzige Quelle der Wahrheit**.
|
||||||
|
* Mindnet importiert diesen Zustand deterministisch in die Datenbank (Qdrant).
|
||||||
|
* Änderungen an IDs, Typen oder Inhalten müssen **immer** im Markdown erfolgen, niemals direkt in der Datenbank.
|
||||||
|
* Was nicht im Markdown steht (z. B. implizites Wissen im Kopf), existiert für das System nicht.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Note-Struktur & Frontmatter
|
||||||
|
|
||||||
|
Jede Notiz benötigt einen YAML-Header (Frontmatter) am Dateianfang. Dieser Block macht die Notiz maschinenlesbar.
|
||||||
|
|
||||||
|
### 2.1 Pflichtfelder
|
||||||
|
Jede Datei muss mindestens folgende Felder enthalten, um korrekt verarbeitet zu werden:
|
||||||
|
|
||||||
|
---
|
||||||
|
id: 20251110-projekt-alpha-8a9b2c # Eindeutige Kennung (YYYYMMDD-slug)
|
||||||
|
title: Projekt Alpha # Sprechender Titel (wird in Suchergebnissen angezeigt)
|
||||||
|
type: project # Steuert Verarbeitung & Vernetzung (siehe Kap. 3)
|
||||||
|
status: active # Status (z.B. draft, active, archived)
|
||||||
|
created: 2025-11-10 # Erstellungsdatum (ISO 8601 oder YYYY-MM-DD)
|
||||||
|
updated: 2025-12-07 # Letzte Änderung
|
||||||
|
tags: [ki, entwicklung] # Taxonomie für Filterung
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2.2 Optionale Felder
|
||||||
|
Diese Felder sind technisch nicht zwingend, aber für bestimmte Typen sinnvoll:
|
||||||
|
|
||||||
|
lang: de # Sprache (Default: de)
|
||||||
|
aliases: [Alpha Projekt, Project A] # Synonyme für die Suche
|
||||||
|
visibility: internal # internal (default), public, private
|
||||||
|
|
||||||
|
> **Hinweis:** Felder wie `retriever_weight` oder `chunk_profile` sollten **nicht** mehr manuell im Frontmatter gesetzt werden. Diese werden zentral über den `type` gesteuert (siehe Kap. 3), um die Wartbarkeit zu sichern.
|
||||||
|
|
||||||
|
### 2.3 Empfehlungen für IDs und Pfade
|
||||||
|
|
||||||
|
**Die ID (Identifikator):**
|
||||||
|
* Muss global eindeutig und **stabil** sein.
|
||||||
|
* Darf sich nicht ändern, wenn die Datei umbenannt oder verschoben wird.
|
||||||
|
* **Empfehlung:** `YYYYMMDD-slug-hash` (z. B. `20231027-vektor-db-a1b2`). Dies garantiert Eindeutigkeit und Chronologie.
|
||||||
|
|
||||||
|
**Dateinamen & Pfade:**
|
||||||
|
* Pfade dienen der menschlichen Ordnung (Ordnerstruktur), sind für Mindnet aber sekundär.
|
||||||
|
* Dateinamen sollten dem Titel entsprechen (`Projekt Alpha.md`) oder dem Slug (`projekt-alpha.md`).
|
||||||
|
* Sonderzeichen vermeiden (außer Bindestrich und Unterstrich).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Note-Typen & Typ-Registry
|
||||||
|
|
||||||
|
Der `type` ist der wichtigste Hebel im Knowledge Design. Er steuert nicht nur das Gewicht bei der Suche, sondern seit WP-06 auch, **wann** eine Notiz aktiv in den Chat geholt wird ("Strategic Retrieval").
|
||||||
|
|
||||||
|
### 3.1 Übersicht der Kern-Typen
|
||||||
|
|
||||||
|
Mindnet unterscheidet verschiedene Wissensarten. Wähle den Typ, der die **Rolle** der Notiz am besten beschreibt:
|
||||||
|
|
||||||
|
| Typ | Beschreibung & Einsatzzweck | Rolle im Chat (Intent) |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| **`concept`** | Fachbegriffe, Theorien. Zeitloses Wissen. | **FACT** (Basiswissen) |
|
||||||
|
| **`project`** | Ein Vorhaben mit Ziel, Dauer und Aufgaben. | **FACT / DECISION** (Kontext) |
|
||||||
|
| **`experience`** | Persönliche Erfahrung, Lektion oder Erkenntnis. | **EMPATHY** (Spiegelung) |
|
||||||
|
| **`decision`** | Eine bewusst getroffene Entscheidung (ADR). | **DECISION** (Begründung "Warum") |
|
||||||
|
| **`value`** | Ein persönlicher Wert oder ein Prinzip. | **DECISION** (Moralischer Kompass) |
|
||||||
|
| **`goal`** | Ein strategisches Ziel (kurz- oder langfristig). | **DECISION** (Abgleich) |
|
||||||
|
| **`person`** | Eine reale Person (Netzwerk, Autor). | **FACT** |
|
||||||
|
| **`journal`** | Zeitbezogener Log-Eintrag, Daily Note. | **FACT** (Historie) |
|
||||||
|
| **`source`** | Externe Quelle (Buch, PDF, Artikel). | **FACT** (Faktenbasis) |
|
||||||
|
|
||||||
|
### 3.2 Zusammenspiel mit `types.yaml`
|
||||||
|
|
||||||
|
Der `type` steuert im Hintergrund drei technische Mechanismen:
|
||||||
|
|
||||||
|
1. **`retriever_weight` (Wichtigkeit):**
|
||||||
|
* Ein `concept` (0.6) wiegt weniger als ein `project` (0.97) oder eine `decision` (1.0).
|
||||||
|
* **Warum?** Bei einer Suche nach "Datenbank" soll Mindnet bevorzugt deine *Entscheidung* ("Warum wir X nutzen") anzeigen.
|
||||||
|
2. **`chunk_profile` (Textzerlegung):**
|
||||||
|
* `journal` (short): Wird fein zerlegt.
|
||||||
|
* `project` (long): Längere Kontext-Fenster.
|
||||||
|
3. **`edge_defaults` (Automatische Vernetzung):**
|
||||||
|
* Mindnet ergänzt automatisch Kanten.
|
||||||
|
* Beispiel: Ein Link in einem `project` wird automatisch als `depends_on` (Abhängigkeit) interpretiert.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Edges & Referenzen in Notes
|
||||||
|
|
||||||
|
Um aus isolierten Dateien ein **Netzwerk** zu machen, nutzen wir Verlinkungen. In Version 2.2 sind diese Verlinkungen die Basis für den **Hybrid Retriever** (Suche über Nachbarn).
|
||||||
|
|
||||||
|
### 4.1 Wikilinks (Die Basis-Referenz)
|
||||||
|
Der klassische Obsidian-Link.
|
||||||
|
|
||||||
|
Wir nutzen [[Qdrant]] als Vektordatenbank.
|
||||||
|
|
||||||
|
* **Bedeutung:** "Diese Notiz erwähnt Qdrant."
|
||||||
|
* **Edge-Typ:** `references`
|
||||||
|
|
||||||
|
### 4.2 Inline-Relationen (Semantische Verknüpfung)
|
||||||
|
Dies ist die **mächtigste** Methode. Du sagst dem System explizit, **wie** Dinge zusammenhängen.
|
||||||
|
|
||||||
|
Daher [[rel:depends_on Qdrant]].
|
||||||
|
Dieses Konzept ist [[rel:similar_to Pinecone]].
|
||||||
|
|
||||||
|
* **Syntax:** `[[rel:RELATION ZIEL]]`.
|
||||||
|
* **Gültige Relationen:**
|
||||||
|
* `depends_on`: Hängt ab von / Benötigt. (Trigger für hohe Graph-Relevanz).
|
||||||
|
* `similar_to`: Ähnelt / Ist vergleichbar mit.
|
||||||
|
* `related_to`: Hat zu tun mit (allgemein).
|
||||||
|
* `caused_by`: Wurde verursacht durch.
|
||||||
|
* `solves`: Löst (Problem).
|
||||||
|
|
||||||
|
### 4.3 Callout-Edges (Kuratierte Listen)
|
||||||
|
Für Zusammenfassungen oder "Siehe auch"-Blöcke am Ende einer Notiz.
|
||||||
|
|
||||||
|
> [!edge] related_to: [[Vector Embeddings]] [[AI Agents]]
|
||||||
|
|
||||||
|
* **Funktion:** Erzeugt `related_to`-Kanten zu allen genannten Zielen in dieser Zeile.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Schreiben für den KI-Zwilling (New in v2.3)
|
||||||
|
|
||||||
|
Damit der **RAG-Chat (WP05/06)** nicht nur dumm Fakten wiedergibt, sondern dich berät, musst du "Futter" für die Decision Engine und das Empathie-Modul liefern.
|
||||||
|
|
||||||
|
### 5.1 Futter für die Decision Engine (`DECISION`)
|
||||||
|
Das System soll abwägen: "Passt Tool X zu mir?". Dazu muss es wissen, was "Du" bist.
|
||||||
|
|
||||||
|
**Best Practice: Werte definieren**
|
||||||
|
Erstelle Notizen mit `type: value`.
|
||||||
|
|
||||||
|
---
|
||||||
|
type: value
|
||||||
|
title: Prinzip: Datensparsamkeit
|
||||||
|
---
|
||||||
|
# Prinzip: Datensparsamkeit
|
||||||
|
Wir speichern nur das Minimum an Daten. Cloud-Uploads persönlicher Daten sind verboten, es sei denn, sie sind E2E-verschlüsselt.
|
||||||
|
|
||||||
|
**Best Practice: Ziele definieren**
|
||||||
|
Erstelle Notizen mit `type: goal`.
|
||||||
|
|
||||||
|
---
|
||||||
|
type: goal
|
||||||
|
title: Ziel: Unabhängigkeit 2026
|
||||||
|
---
|
||||||
|
# Ziel: Unabhängigkeit 2026
|
||||||
|
Bis Ende 2026 wollen wir alle SaaS-Abos durch Self-Hosted Lösungen ersetzt haben.
|
||||||
|
|
||||||
|
*Effekt:* Wenn du fragst "Soll ich Notion nutzen?", lädt die Engine diese beiden Notizen und antwortet: *"Nein, Notion ist SaaS und nicht E2E-verschlüsselt. Das verletzt dein Prinzip der Datensparsamkeit und dein Ziel der Unabhängigkeit."*
|
||||||
|
|
||||||
|
### 5.2 Futter für den Empathie-Modus (`EMPATHY`)
|
||||||
|
Das System soll dich verstehen, wenn du sagst: "Alles ist grau." Dazu braucht es deine Resonanzerfahrungen.
|
||||||
|
|
||||||
|
**Best Practice: Erfahrungen & Bridging**
|
||||||
|
Erstelle Notizen mit `type: experience`. Nutze im Text **emotionale Brückenwörter**, damit der Retriever auch bei abstrakten Gefühlen ("grau") einen Treffer landet.
|
||||||
|
|
||||||
|
---
|
||||||
|
type: experience
|
||||||
|
title: Erfahrung: Der Durchbruch nach der Krise
|
||||||
|
tags: [krise, hoffnung, grau, angst]
|
||||||
|
---
|
||||||
|
# Erfahrung: Der Durchbruch nach der Krise
|
||||||
|
Es gibt Projektphasen, da wirkt alles **sinnlos** und **grau**.
|
||||||
|
Ich habe gelernt: Das ist oft das Zeichen kurz vor dem Durchbruch.
|
||||||
|
Mein Mantra in solchen Zeiten: "Einfach weitermachen, der Nebel lichtet sich."
|
||||||
|
|
||||||
|
*Effekt:* Wenn du im Chat jammerst ("Alles ist sinnlos"), findet das System diese Notiz und spiegelt dir deine eigene Erfahrung zurück.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Best Practices & Beispiele (Klassik)
|
||||||
|
|
||||||
|
### 6.1 Beispiel: Projekt-Notiz
|
||||||
|
Projekte profitieren von `depends_on`, um Abhängigkeiten zu klären.
|
||||||
|
|
||||||
|
---
|
||||||
|
id: 20251115-proj-mindnet
|
||||||
|
title: Mindnet Implementierung
|
||||||
|
type: project
|
||||||
|
status: active
|
||||||
|
---
|
||||||
|
|
||||||
|
# Mindnet Implementierung
|
||||||
|
|
||||||
|
Wir bauen ein persönliches Wissensnetz.
|
||||||
|
|
||||||
|
## Tech Stack
|
||||||
|
Wir nutzen [[rel:depends_on Qdrant]] für die Vektorsuche und [[rel:depends_on FastAPI]] für das Backend.
|
||||||
|
|
||||||
|
## Architektur
|
||||||
|
Das Konzept basiert auf [[RAG Architecture]]. (Automatisch 'depends_on' durch Typ-Default).
|
||||||
|
|
||||||
|
### 6.2 Beispiel: Entscheidung (Decision Record)
|
||||||
|
Entscheidungen sind hoch gewichtet (`retriever_weight: 1.0`).
|
||||||
|
|
||||||
|
---
|
||||||
|
id: 20251120-adr-vektordb
|
||||||
|
title: ADR: Wahl von Qdrant
|
||||||
|
type: decision
|
||||||
|
status: final
|
||||||
|
tags: [architektur, db]
|
||||||
|
---
|
||||||
|
|
||||||
|
# Entscheidung: Qdrant
|
||||||
|
|
||||||
|
Wir haben uns für Qdrant entschieden.
|
||||||
|
|
||||||
|
## Alternativen
|
||||||
|
Wir haben auch [[rel:similar_to Pinecone]] und [[rel:similar_to Weaviate]] betrachtet.
|
||||||
|
|
||||||
|
## Begründung
|
||||||
|
Qdrant erlaubt lokalen Betrieb und [[rel:solves Payload Filtering Requirements]].
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Langfristige Stabilität & Virtual Schema Layer
|
||||||
|
|
||||||
|
Mindnet ist auf Langlebigkeit ausgelegt.
|
||||||
|
|
||||||
|
### 7.1 Das "Virtual Schema" Prinzip
|
||||||
|
Wir vermeiden es, Logik in den Markdown-Dateien hart zu kodieren.
|
||||||
|
* Statt in jeder Notiz zu schreiben `chunk_size: 500`, schreiben wir nur `type: essay`.
|
||||||
|
* Wie groß ein Chunk für ein Essay ist, definieren wir in der Konfiguration (`types.yaml`).
|
||||||
|
|
||||||
|
### 7.2 Was bedeutet das für dich?
|
||||||
|
* Du kannst dich auf den Inhalt konzentrieren.
|
||||||
|
* Wenn wir in Zukunft (WP08) basierend auf Feedback lernen, dass "Projekte" noch wichtiger sind, ändern wir **eine Zeile** in der Konfiguration, und das gesamte System passt sich beim nächsten Import an.
|
||||||
@@ -0,0 +1,365 @@
|
|||||||
|
# mindnet v2 – Implementierungs-Playbook & Handover (Side‑by‑Side zu v1)
|
||||||
|
*Stand:* 2025-11-10 08:03
|
||||||
|
*Autor:* ChatGPT (Projekt‑Handover)
|
||||||
|
*Zweck:* Vollständige, ausführbare Anleitung zur Weiterentwicklung von **mindnet** auf eine versionierte, testbare v2‑Architektur (neben v1), inklusive Prompt für den Folgethread, Akzeptanzkriterien, Dateiliste, Test‑ und Rollback‑Pläne.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0) Kontext & Zielbild (kurz)
|
||||||
|
- **Mission:** Persönliches Wissensnetz, das langfristig die eigene Persönlichkeit, Erfahrungen und Entscheidungslogik abbildet; später nachvollziehbare Erklärungen für Familie/Nachwelt.
|
||||||
|
- **Heutiger Stand (v1):** `mindnet_notes` in Qdrant vorhanden; **Chunks** und **Edges** bisher nur teilweise/nicht konsistent; Edge‑Defaults („1 Dot“) vorhanden, aber Ziele teils nicht materialisiert. Importskripte: `import_markdown.py`, `edges.py` (historisch).
|
||||||
|
- **Ziel (v2):**
|
||||||
|
- **Collections:** `mindnet_notes_v2`, `mindnet_chunks_v2`, `mindnet_edges_v2` (saubere Payload‑Schemata + Indizes).
|
||||||
|
- **Chunker v2:** Block/Heading‑aware, Ziel 900–1200 Zeichen, Überlappung 120–180, deterministische `chunk_id`s.
|
||||||
|
- **Edge‑Builder v2:** Reihenfolge: **explicit → rule → default_resolved → default**; saubere `provenance`.
|
||||||
|
- **Importer‑Pipeline v2:** Parse → Chunk → Edge → Embeddings → Batch‑Upsert + Payload‑Indexpflege + Snapshots.
|
||||||
|
- **Policies:** Privacy/Recency/Type‑Gewichte als **Konfiguration** (YAML + JSON‑Schema), nicht hart im Code.
|
||||||
|
- **Observability:** FastAPI + OpenTelemetry (Tracing/Metrics); Import‑Reports (CSV/JSON).
|
||||||
|
- **Side‑by‑Side:** v2 **parallel** zu v1; kein Big‑Bang.
|
||||||
|
|
||||||
|
> **Warum so?** Qdrant‑Payload‑Indizes/Filter → schnelle, erklärbare Selektionen; Snapshots → betriebssicheres Rollback; OTel → Nachvollziehbarkeit; YAML/JSON‑Schema → Validierbarkeit & Stabilität.
|
||||||
|
|
||||||
|
### 0.1 Aktueller Implementierungsstand (2025-11-18)
|
||||||
|
|
||||||
|
- Es existiert eine produktive mindnet-Instanz mit den Collections:
|
||||||
|
- `${prefix}_notes`
|
||||||
|
- `${prefix}_chunks`
|
||||||
|
- `${prefix}_edges`
|
||||||
|
- Import-Pipeline:
|
||||||
|
- `scripts/import_markdown.py` erzeugt Notes, Chunks und Edges vollständig.
|
||||||
|
- `types.yaml` definiert per Note-Typ:
|
||||||
|
- `retriever_weight`
|
||||||
|
- `chunk_profile`
|
||||||
|
- `edge_defaults` (z. B. `depends_on`, `related_to`).
|
||||||
|
- Edges:
|
||||||
|
- Strukturkanten: `belongs_to`, `next`, `prev`
|
||||||
|
- Explizite Kanten aus Wikilinks: `kind="references"`, `rule_id="explicit:wikilink"`
|
||||||
|
- Typbasierte Default-Kanten aus `edge_defaults`: `rule_id="edge_defaults:<type>:<relation>"`
|
||||||
|
- Inline-Kanten im Format `[[rel:<relation> <Zielnote>]]` mit `rule_id="inline:rel"`
|
||||||
|
- alle Edges tragen eine `confidence` im Bereich `[0,1]` als Hinweis für spätere Ranking-Logiken.
|
||||||
|
- Tests:
|
||||||
|
- `tests/test_edges_smoke.py`, `tests/test_edges_all.py` und `scripts/edges_full_check.py` validieren Counts, Konsistenz (`belongs_to == chunks`, `next/prev`-Ketten, keine Duplikate) und Provenienz-Zählungen (`explicit`, `defaults`, `inline`, `structure`).
|
||||||
|
- Offene Punkte:
|
||||||
|
- Mehrere Ziele in einer Inline-Relation (`rel: similar_to [[A]] [[B]]`) sind noch nicht implementiert.
|
||||||
|
- Retrieval-Service nutzt die neuen Edges und Weights bisher nur teilweise; ein edge-bewusster Retriever ist als nächster Schritt geplant.
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1) Datenmodelle (Schemata, v2)
|
||||||
|
Die tatsächlichen JSON‑Schemata (2020‑12) werden im Repo abgelegt und in CI validiert.
|
||||||
|
|
||||||
|
### 1.1 note.schema.json
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://example.org/mindnet/note.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["id", "title", "type", "privacy", "created", "hash_body"],
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string", "pattern": "^[a-zA-Z0-9_-]+$" },
|
||||||
|
"title": { "type": "string", "minLength": 1 },
|
||||||
|
"type": { "type": "string" },
|
||||||
|
"privacy": { "type": "string", "enum": ["public","internal","private"] },
|
||||||
|
"created": { "type": "string", "format": "date-time" },
|
||||||
|
"modified": { "type": "string", "format": "date-time" },
|
||||||
|
"tags": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"lang": { "type": "string", "default": "de" },
|
||||||
|
"source_path": { "type": "string" },
|
||||||
|
"source_collection": { "type": "string" },
|
||||||
|
"hash_body": { "type": "string" },
|
||||||
|
"hash_frontmatter": { "type": "string" },
|
||||||
|
"token_count": { "type": "integer", "minimum": 0 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.2 chunk.schema.json
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://example.org/mindnet/chunk.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["chunk_id","note_id","text","ord"],
|
||||||
|
"properties": {
|
||||||
|
"chunk_id": { "type": "string", "pattern": "^[a-zA-Z0-9_-]+#c\d{4}$" },
|
||||||
|
"note_id": { "type": "string" },
|
||||||
|
"text": { "type": "string" },
|
||||||
|
"ord": { "type": "integer", "minimum": 0 },
|
||||||
|
"span_char_start": { "type": "integer", "minimum": 0 },
|
||||||
|
"span_char_end": { "type": "integer", "minimum": 0 },
|
||||||
|
"heading_path": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"section_title": { "type": "string" },
|
||||||
|
"tokens_start": { "type": "integer", "minimum": 0 },
|
||||||
|
"tokens_end": { "type": "integer", "minimum": 0 },
|
||||||
|
"embeddings_version": { "type": "string" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.3 edge.schema.json
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://example.org/mindnet/edge.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["edge_id","src_note_id","relation","provenance"],
|
||||||
|
"properties": {
|
||||||
|
"edge_id": { "type": "string" },
|
||||||
|
"src_note_id": { "type": "string" },
|
||||||
|
"src_chunk_id": { "type": "string" },
|
||||||
|
"dst_note_id": { "type": "string" },
|
||||||
|
"dst_chunk_id": { "type": "string" },
|
||||||
|
"relation": { "type": "string" },
|
||||||
|
"evidence_spans":{ "type": "array", "items": {"type":"object","properties":{"chunk_id":{"type":"string"},"span":[{"type":"integer"},{"type":"integer"}]]} },
|
||||||
|
"provenance": { "type": "string", "enum": ["explicit","rule","default_resolved","default"] },
|
||||||
|
"rule_id": { "type": "string" },
|
||||||
|
"confidence": { "type": "number", "minimum": 0, "maximum": 1 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.4 default_edge.schema.json
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://example.org/mindnet/default_edge.schema.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["src_note_id","relation","target_kind"],
|
||||||
|
"properties": {
|
||||||
|
"src_note_id": { "type": "string" },
|
||||||
|
"relation": { "type": "string" },
|
||||||
|
"target_kind": { "type": "string" },
|
||||||
|
"when": { "type": "string" },
|
||||||
|
"strength_hint":{ "type": "number", "minimum": 0, "maximum": 1 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2) Qdrant: Collections & Indizes (v2)
|
||||||
|
Anlegen **neben** v1:
|
||||||
|
- `mindnet_notes_v2` (Payload‑Index: `type, privacy, created, modified, tags, source_path`)
|
||||||
|
- `mindnet_chunks_v2` (Payload‑Index: `note_id, ord, heading_path`)
|
||||||
|
- `mindnet_edges_v2` (Payload‑Index: `src_note_id, dst_note_id, relation, provenance`)
|
||||||
|
|
||||||
|
> Snapshots für Backups/Restore einplanen (Runbook siehe unten).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3) Import‑Pipeline v2 – Module & Flags
|
||||||
|
- **Chunker v2** (`chunking/block_chunker.py`): `target_len=1000`, `overlap=150`, `respect_headings=True`, `min_chunk_len=600`, `max_chunk_len=1400`, `chunk_id = f\"{note_id}#c{ordinal:04d}\"`
|
||||||
|
- **Edge‑Builder v2** (`graph/edge_builder_v2.py`): Reihenfolge **explicit → rule → default_resolved → default**, `provenance` korrekt setzen.
|
||||||
|
- **Importer** (`scripts/import_markdown.py` erweitern oder `scripts/import_markdown_v2.py`):
|
||||||
|
Flags: `--schema v2`, `--chunker v2`, `--edges v2`, `--dry-run`, `--apply`, `--prefix "$COLLECTION_PREFIX"`.
|
||||||
|
|
||||||
|
**Beispiel‑Run (dry‑run):**
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --dry-run --prefix "$COLLECTION_PREFIX"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Apply:**
|
||||||
|
```bash
|
||||||
|
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --apply --prefix "$COLLECTION_PREFIX"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4) Policies (konfigurierbar, kein Hard‑Code)
|
||||||
|
- `policies/retrieval.schema.json` (JSON‑Schema) und `policies/retrieval.yaml`, z. B.:
|
||||||
|
```yaml
|
||||||
|
privacy_order: [private, internal, public]
|
||||||
|
recency_boost_half_life_days: 90
|
||||||
|
type_priority:
|
||||||
|
person: 1.2
|
||||||
|
event: 1.1
|
||||||
|
concept: 1.0
|
||||||
|
max_chunks_per_note: 8
|
||||||
|
edge_provenance_weights:
|
||||||
|
explicit: 1.0
|
||||||
|
rule: 0.8
|
||||||
|
default_resolved: 0.6
|
||||||
|
default: 0.2
|
||||||
|
rule_sets:
|
||||||
|
- id: "rules:types-v1"
|
||||||
|
enabled: true
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5) Observability & Reports
|
||||||
|
- **FastAPI + OpenTelemetry**: Traces um Phasen *parse/chunk/edge/upsert*; Metriken (Counter/Histogramme) pro Phase.
|
||||||
|
- **Import‑Report** je Lauf (CSV/JSON): counts für Notes/Chunks/Edges, Fehler, Dauer, Embedding‑Version.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6) Teststrategie
|
||||||
|
- **Gold‑Notizen** (3–5 repräsentative .md) mit erwarteten *Chunk‑Counts* und *Edge‑Sets*.
|
||||||
|
- **Regression:** fixer Soll‑Count (z. B. 171 ± 15 %) über Gesamt‑Vault; keine Duplikate `(src,relation,dst)`.
|
||||||
|
- **Dry‑Run vs Apply**: identische Counts, nur ohne Upserts.
|
||||||
|
- **Filter‑Smoke‑Tests:** Qdrant‑Filter nach `privacy`, `type`, `tags`, `created` funktionieren performant.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7) Akzeptanzkriterien (KPI)
|
||||||
|
1. **Chunk‑Qualität:** ≥ 90 % der Chunks enden an semantischen Grenzen; Ziel‑Länge 900–1200 Zeichen; deterministische IDs.
|
||||||
|
2. **Edge‑Kohärenz:** Keine Duplikate gleicher `(src,relation,dst)`; ≥ 95 % der expliziten Links materialisiert; `default` nur, wenn kein Ziel existiert.
|
||||||
|
3. **Filterbarkeit:** Queries nach `privacy/type/tags/created` performant (Payload‑Index vorhanden).
|
||||||
|
4. **Betriebsfestigkeit:** Telemetrie aktiv; fehlertoleranter Import; Qdrant‑Snapshot nach erfolgreichem Lauf.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8) Roadmap in kleinen, testbaren Schritten
|
||||||
|
|
||||||
|
### Step 0 – Safeguards
|
||||||
|
- Qdrant‑Snapshot v1 erstellen + Restore‑Probe.
|
||||||
|
- ENV: `MINDNET_SCHEMA_VERSION=2` (Importer wird v2 schreiben, v1 bleibt unberührt).
|
||||||
|
|
||||||
|
### Step 1 – Schemata (nur Dateien)
|
||||||
|
- Ablage: `/schemas/note.schema.json`, `/schemas/chunk.schema.json`, `/schemas/edge.schema.json`, `/schemas/default_edge.schema.json`
|
||||||
|
- CI‑Job: `make schema-validate` (jsonschema).
|
||||||
|
- **Abnahme:** Validator grün auf 3 Gold‑Notizen.
|
||||||
|
|
||||||
|
### Step 2 – Qdrant v2‑Collections (Optional)
|
||||||
|
- Anlegen der 3 v2‑Collections + Payload‑Indizes.
|
||||||
|
- **Abnahme:** Filter‑Query liefert erwartbare Ergebnisse.
|
||||||
|
|
||||||
|
### Step 2a – Edge-Pipeline v1 stabilisieren (ERLEDIGT)
|
||||||
|
- Ziel: Vollständige Generierung von Struktur-, expliziten, typbasierten und Inline-Kanten in den bestehenden Collections `${prefix}_notes`, `${prefix}_chunks`, `${prefix}_edges`.
|
||||||
|
- Status: Implementiert und durch `tests/test_edges_smoke.py`, `tests/test_edges_all.py` und `scripts/edges_full_check.py` verifiziert.
|
||||||
|
|
||||||
|
|
||||||
|
### Step 3 – Chunker v2
|
||||||
|
- Implementierung & Flag `--chunker v2`.
|
||||||
|
- **Abnahme:** Chunk‑Counts ~ alt (≈171 ± 15 %), semantische Schnitte.
|
||||||
|
|
||||||
|
### Step 4 – Edge‑Builder v2
|
||||||
|
- Reihenfolge & `provenance` strikt umsetzen, `default_resolved` integrieren.
|
||||||
|
- **Abnahme:** erwartete Relationensätze auf Gold‑Notizen, keine Duplikate.
|
||||||
|
|
||||||
|
### Step 4a – Edge-bewusster Retriever
|
||||||
|
- Implementiere einen Retrieval-Service, der:
|
||||||
|
- Chunks über Vektor-Suche aus `${prefix}_chunks` holt,
|
||||||
|
- dazu passende Nachbar-Chunks (`next/prev`) und typrelevante Notizen (`edge_defaults`, Inline-Relationen, Wikilinks) über `${prefix}_edges` ergänzt,
|
||||||
|
- `retriever_weight` und `confidence` beim Scoring berücksichtigt.
|
||||||
|
- Liefere einen API-Endpoint `/mindnet/query` mit einem einfachen JSON-Response (Chunks + Begründung).
|
||||||
|
- Akzeptanz:
|
||||||
|
- mindestens ein Testfall mit dem Relations-Showcase-Vault,
|
||||||
|
- sichtbarer Einfluss von `edge_defaults` und Inline-Relationen auf die Ergebnisliste.
|
||||||
|
|
||||||
|
|
||||||
|
### Step 5 – Importer‑Pipeline v2
|
||||||
|
- `--schema v2` Side‑by‑Side; Batch‑Upserts; vollständige Payloads; Recency‑Boost Konfig **nur** in Policy.
|
||||||
|
- **Abnahme:** Dry‑Run/Apply‑Parität, Reports, Telemetrie‑Ereignisse sichtbar.
|
||||||
|
|
||||||
|
### Step 6 – Observability
|
||||||
|
- OTel‑Instrumentierung + `/import/status` + Reports.
|
||||||
|
- **Abnahme:** Traces & Metriken vorhanden.
|
||||||
|
|
||||||
|
### Step 7 – Snapshots & Runbook
|
||||||
|
- Snapshot nach Erfolg; Restore‑Dokument.
|
||||||
|
- **Abnahme:** Restore‑Probe erfolgreich.
|
||||||
|
|
||||||
|
### Step 8 – Policies
|
||||||
|
- `policies/retrieval.yaml` + JSON‑Schema; A/B‑Test (Policy ändern → Retrieval ändert sich ohne Reimport).
|
||||||
|
- **Abnahme:** Sichtbarer Effekt laut Testfall.
|
||||||
|
|
||||||
|
### Step 9 – Umschalten
|
||||||
|
- Abnahme‑Doku, KPI‑Delta dokumentiert; Alias/Flag‑Switch auf v2.
|
||||||
|
- **Abnahme:** Funktionale Gleichwertigkeit + Qualitätsgewinn.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9) Dateien (neu/ändern)
|
||||||
|
**Neu:**
|
||||||
|
- `schemas/note.schema.json`
|
||||||
|
- `schemas/chunk.schema.json`
|
||||||
|
- `schemas/edge.schema.json`
|
||||||
|
- `schemas/default_edge.schema.json`
|
||||||
|
- `policies/retrieval.schema.json`
|
||||||
|
- `policies/retrieval.yaml`
|
||||||
|
- `chunking/block_chunker.py`
|
||||||
|
- `graph/edge_builder_v2.py`
|
||||||
|
- `docs/OPERATIONS.md` (Snapshots/Restore/Runbook)
|
||||||
|
- `tests/gold_notes/manifest.yaml` (+ 3–5 Notizen Kopien)
|
||||||
|
|
||||||
|
**Änderungen:**
|
||||||
|
- `scripts/import_markdown.py` **oder** `scripts/import_markdown_v2.py` neu (Flags `--schema/--chunker/--edges`).
|
||||||
|
- `edges.py` (falls weiterverwendet): auf **v2‑Edge‑Builder** migrieren oder auslaufen lassen.
|
||||||
|
- `types.yaml`: Regel‑Definitionen klarisieren (IDs, Bedingungen), **nicht** hart in Code.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10) Beispiel‑Kommandos
|
||||||
|
```bash
|
||||||
|
# Step 2: Collections prüfen
|
||||||
|
curl -s http://127.0.0.1:6333/collections | jq
|
||||||
|
|
||||||
|
# Step 3: Chunker v2 (dry-run)
|
||||||
|
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --dry-run --prefix "$COLLECTION_PREFIX"
|
||||||
|
|
||||||
|
# Step 5: Apply + Report
|
||||||
|
python3 -m scripts.import_markdown_v2 --vault ./vault --schema v2 --chunker v2 --edges v2 --apply --prefix "$COLLECTION_PREFIX" --report ./reports/import_$(date +%F_%H%M).json
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11) Rollback
|
||||||
|
- v2‑Collections droppen (wenn Side‑by‑Side).
|
||||||
|
- Aus Snapshot wiederherstellen (v1).
|
||||||
|
- Flags zurück auf v1.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12) Referenzen (offizielle Quellen)
|
||||||
|
- Qdrant – Collections, Payload‑Index, Filter: <https://qdrant.tech/documentation/>
|
||||||
|
- Qdrant – Snapshots/Backup: <https://qdrant.tech/documentation/guides/backup/>
|
||||||
|
- OpenTelemetry (Python/FastAPI): <https://opentelemetry.io/docs/instrumentation/python/>
|
||||||
|
- JSON‑Schema 2020‑12: <https://json-schema.org/>
|
||||||
|
- YAML 1.2: <https://yaml.org/spec/1.2.2/>
|
||||||
|
- Obsidian Frontmatter: <https://help.obsidian.md/Editing+and+formatting/Properties>
|
||||||
|
- Ollama CLI/API (lokale Modelle): <https://github.com/ollama/ollama>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13) **PROMPT für den neuen Chat (bitte exakt so einfügen)**
|
||||||
|
```
|
||||||
|
Rolle: Du bist mein Senior‑Entwickler & Architekt für mindnet. Arbeite strikt in kleinen, testbaren Schritten (Side‑by‑Side v2 neben v1). Liefere komplette Dateien als Downloads. Frage immer zuerst nach den **aktuellsten** Projektdateien, wenn du sie brauchst (z. B. import_markdown.py, edges.py, types.yaml), ändere keine Systemfunktion „workaround‑artig“ ohne die Gesamtwirkung zu prüfen.
|
||||||
|
|
||||||
|
Kontext (Kurzfassung):
|
||||||
|
- v1 hat nur mindnet_notes zuverlässig. Chunks/Edges sind inkonsistent/teilweise leer. Ziel: v2 mit drei Collections (notes/chunks/edges), neuem Chunker v2, Edge‑Builder v2 (explicit → rule → default_resolved → default), Policies (YAML), Observability (OTel), Snapshots. Kein Big‑Bang; v2 parallel zu v1.
|
||||||
|
- Siehe angehängte Datei **mindnet_v2_implementation_playbook.md** (Pfad/Download im Chat). Alles daraus ist verbindlich.
|
||||||
|
|
||||||
|
Deine ersten Aufgaben (Step‑by‑Step):
|
||||||
|
1) **Schemata anlegen (Step 1)**
|
||||||
|
- Erstelle die Dateien:
|
||||||
|
- schemas/note.schema.json
|
||||||
|
- schemas/chunk.schema.json
|
||||||
|
- schemas/edge.schema.json
|
||||||
|
- schemas/default_edge.schema.json
|
||||||
|
- Nutze die in der Playbook‑Datei vorgegebenen Strukturen als Basis.
|
||||||
|
- Liefere zusätzlich ein einfaches `Makefile`‑Ziel `schema-validate` (jsonschema via Python), plus `requirements.txt`.
|
||||||
|
- Output: komplette Dateien als Downloads + kurze Testanleitung.
|
||||||
|
|
||||||
|
2) **Qdrant v2‑Collections (Step 2)**
|
||||||
|
- Erzeuge die Collections + Payload‑Indizes.
|
||||||
|
- Liefere ein kleines Python‑Skript `tools/qdrant_bootstrap_v2.py`, das diese Anlage idempotent durchführt (inkl. Prüf‑Output).
|
||||||
|
|
||||||
|
3) **Chunker v2 (Step 3)**
|
||||||
|
- Implementiere `chunking/block_chunker.py` mit den im Playbook genannten Parametern.
|
||||||
|
- Teste gegen 3 Gold‑Notizen (liefere mini Test‑Harness in `tests/gold_notes/…`).
|
||||||
|
|
||||||
|
Arbeitsweise:
|
||||||
|
- Jede Aufgabe einzeln, mit klaren Akzeptanzkriterien (aus Playbook) und Download‑Artefakten.
|
||||||
|
- Keine parallelen Großumbauten.
|
||||||
|
- Immer deterministische IDs, idempotente Upserts.
|
||||||
|
|
||||||
|
Dateien, die ich dir anfänglich bereitstelle:
|
||||||
|
- **mindnet_v2_implementation_playbook.md** (diese Datei)
|
||||||
|
- **import_markdown.py** (aktuelle Version aus dem Projekt)
|
||||||
|
- **edges.py** (aktuelle Version aus dem Projekt)
|
||||||
|
- **types.yaml** (aktuelle Version)
|
||||||
|
|
||||||
|
Sage mir jeweils, welche Datei du als Nächstes brauchst. Beginne jetzt mit Aufgabe 1 (Schemata).
|
||||||
|
```
|
||||||
@@ -0,0 +1,260 @@
|
|||||||
|
# WP-04 / Step 4a – Retriever Scoring & Konfiguration
|
||||||
|
|
||||||
|
Dieses Dokument beschreibt den aktuellen Stand (2025-11-30) der Scoring-Logik des mindnet-Retrievers in WP-04 / Step 4a.
|
||||||
|
Es dient als Referenz für:
|
||||||
|
|
||||||
|
- Nachvollziehbarkeit des Rankings
|
||||||
|
- Konfiguration und Feintuning (ohne Codeänderungen)
|
||||||
|
- spätere Automatisierung / „Selbstjustierung“ des Retrievers
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Überblick
|
||||||
|
|
||||||
|
Der Retriever kombiniert drei Signalquellen zu einem einheitlichen Score:
|
||||||
|
|
||||||
|
1. **Semantik** – Vektorsuche über `mindnet_chunks`
|
||||||
|
2. **Typ-Gewichtung** – `retriever_weight` pro Note/Chunk (aus `types.yaml`)
|
||||||
|
3. **Graph-Boni** – `edge_bonus` und `centrality_bonus` aus dem Subgraph (Edges)
|
||||||
|
|
||||||
|
Die Berechnung erfolgt in `app/core/retriever.py` und ist über `config/retriever.yaml` konfigurierbar.
|
||||||
|
Die API stellt die Funktionalität über den `/query`-Endpoint (FastAPI) bereit.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Datenbasis (Kurzüberblick)
|
||||||
|
|
||||||
|
### 2.1 Notes (`<prefix>_notes`)
|
||||||
|
|
||||||
|
Relevante Felder:
|
||||||
|
|
||||||
|
- note_id: keyword
|
||||||
|
- title: text
|
||||||
|
- type: keyword
|
||||||
|
- retriever_weight: float
|
||||||
|
- chunk_profile: keyword
|
||||||
|
- edge_defaults: keyword[]
|
||||||
|
- tags: keyword[]
|
||||||
|
- path: text
|
||||||
|
- fulltext: text
|
||||||
|
|
||||||
|
### 2.2 Chunks (`<prefix>_chunks`)
|
||||||
|
|
||||||
|
Relevante Felder:
|
||||||
|
|
||||||
|
- note_id: keyword
|
||||||
|
- chunk_id: keyword
|
||||||
|
- text: text
|
||||||
|
- window: text
|
||||||
|
- retriever_weight: float
|
||||||
|
- type: keyword
|
||||||
|
- path: text
|
||||||
|
- section: text
|
||||||
|
- neighbors_prev / neighbors_next
|
||||||
|
- chunk_profile: keyword
|
||||||
|
|
||||||
|
### 2.3 Edges (`<prefix>_edges`)
|
||||||
|
|
||||||
|
Relevante Felder:
|
||||||
|
|
||||||
|
- kind: keyword
|
||||||
|
- source_id: keyword
|
||||||
|
- target_id: keyword
|
||||||
|
- note_id: keyword
|
||||||
|
- confidence: float
|
||||||
|
- rule_id, provenance, edge_id, scope, relation, ref_text
|
||||||
|
|
||||||
|
Edges stammen stets aus der Import-/Edge-Pipeline (WP-03) und bilden die graphbasierte Grundlage des Retrievers.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Scoring-Formel (Retriever)
|
||||||
|
|
||||||
|
### 3.1 Eingangsgrößen
|
||||||
|
|
||||||
|
Für jeden Treffer gelten:
|
||||||
|
|
||||||
|
semantic_score – aus der Vektorsuche (Qdrant)
|
||||||
|
retriever_weight – typabhängiges Gewicht (aus types.yaml)
|
||||||
|
edge_bonus – aggregierter Graph-Bonus
|
||||||
|
centrality_bonus – Zentralitätsbonus aus dem Subgraph
|
||||||
|
|
||||||
|
### 3.2 Globale Gewichte (aus retriever.yaml)
|
||||||
|
|
||||||
|
Datei: `config/retriever.yaml`
|
||||||
|
Beispiel:
|
||||||
|
|
||||||
|
version: 1.0
|
||||||
|
|
||||||
|
scoring:
|
||||||
|
semantic_weight: 1.0 # W_sem
|
||||||
|
edge_weight: 0.7 # W_edge
|
||||||
|
centrality_weight: 0.5 # W_cent
|
||||||
|
|
||||||
|
Falls diese Datei fehlt, greifen ENV-basierte Defaults (`RETRIEVER_W_*`).
|
||||||
|
|
||||||
|
### 3.3 Formel
|
||||||
|
|
||||||
|
total_score =
|
||||||
|
W_sem * semantic_score * max(retriever_weight, 0.0)
|
||||||
|
+ W_edge * edge_bonus
|
||||||
|
+ W_cent * centrality_bonus
|
||||||
|
|
||||||
|
Bemerkungen:
|
||||||
|
|
||||||
|
- Negative `retriever_weight` werden abgefangen.
|
||||||
|
- Die rohen `edge_bonus` und `centrality_bonus` werden im API-/Smoke-Output angezeigt.
|
||||||
|
- Die Gewichte aus `retriever.yaml` beeinflussen **nur den total_score**, nicht die Anzeige.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Konfigurationsdateien
|
||||||
|
|
||||||
|
### 4.1 Typen-Registry (`types.yaml`)
|
||||||
|
|
||||||
|
Beispielauszug:
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
retriever_weight: 1.0
|
||||||
|
chunk_profile: default
|
||||||
|
|
||||||
|
types:
|
||||||
|
types:
|
||||||
|
concept:
|
||||||
|
retriever_weight: 0.60
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
|
||||||
|
project:
|
||||||
|
retriever_weight: 0.97
|
||||||
|
edge_defaults: ["references", "depends_on"]
|
||||||
|
|
||||||
|
journal:
|
||||||
|
retriever_weight: 0.80
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
|
||||||
|
source:
|
||||||
|
retriever_weight: 0.50
|
||||||
|
edge_defaults: []
|
||||||
|
|
||||||
|
Diese Werte bestimmen, wie wichtig eine Note/Chunk **prinzipiell** für den Retriever ist.
|
||||||
|
|
||||||
|
### 4.2 Retriever-Konfiguration (`retriever.yaml`)
|
||||||
|
|
||||||
|
Beispiel:
|
||||||
|
|
||||||
|
version: 1.0
|
||||||
|
|
||||||
|
scoring:
|
||||||
|
semantic_weight: 1.0
|
||||||
|
edge_weight: 0.7
|
||||||
|
centrality_weight: 0.5
|
||||||
|
|
||||||
|
# Blueprint für spätere Edge-Feinsteuerung:
|
||||||
|
edge_types:
|
||||||
|
references: 0.20
|
||||||
|
depends_on: 0.18
|
||||||
|
related_to: 0.15
|
||||||
|
similar_to: 0.12
|
||||||
|
belongs_to: 0.10
|
||||||
|
next: 0.06
|
||||||
|
prev: 0.06
|
||||||
|
|
||||||
|
Die Datei befindet sich standardmäßig unter `config/retriever.yaml`.
|
||||||
|
Override per ENV:
|
||||||
|
|
||||||
|
MINDNET_RETRIEVER_CONFIG=/pfad/zur/datei.yaml
|
||||||
|
|
||||||
|
### 4.3 Aktive Gewichte auslesen
|
||||||
|
|
||||||
|
python - << 'PY'
|
||||||
|
from app.core.retriever import _get_scoring_weights
|
||||||
|
print(_get_scoring_weights())
|
||||||
|
PY
|
||||||
|
|
||||||
|
Beispielausgabe:
|
||||||
|
|
||||||
|
(1.0, 0.7, 0.5)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Betriebsmodi
|
||||||
|
|
||||||
|
### 5.1 Semantikmodus (semantic)
|
||||||
|
|
||||||
|
Ablauf:
|
||||||
|
|
||||||
|
- Query-Vektor über embed_text()
|
||||||
|
- Qdrant-Suche über mindnet_chunks
|
||||||
|
- kein Subgraph → edge_bonus = centrality = 0
|
||||||
|
- total_score = W_sem * semantic_score * retriever_weight
|
||||||
|
|
||||||
|
### 5.2 Hybridmodus (hybrid)
|
||||||
|
|
||||||
|
Ablauf:
|
||||||
|
|
||||||
|
- Semantik wie oben
|
||||||
|
- falls expand.depth > 0:
|
||||||
|
- Subgraph-Berechnung über ga.expand()
|
||||||
|
- Ermittlung von edge_bonus und centrality_bonus
|
||||||
|
- total_score = kombinierte Formel aus Semantik + Graph
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Tests
|
||||||
|
|
||||||
|
### Ausführung (Unit & Integration)
|
||||||
|
|
||||||
|
pytest tests/test_retriever_basic.py \
|
||||||
|
tests/test_retriever_weight.py \
|
||||||
|
tests/test_retriever_edges.py
|
||||||
|
|
||||||
|
pytest tests/test_query_unit.py \
|
||||||
|
tests/test_query_text_embed_unit.py
|
||||||
|
|
||||||
|
Tests decken ab:
|
||||||
|
|
||||||
|
- Semantiktreffer
|
||||||
|
- Typ-Gewichte
|
||||||
|
- Edge-Boni
|
||||||
|
- Hybridmodus / FastAPI-Endpunkt
|
||||||
|
|
||||||
|
### Smoke-Test
|
||||||
|
|
||||||
|
python tests/test_retriever_smoke.py \
|
||||||
|
--url "http://127.0.0.1:8001/query" \
|
||||||
|
--query "embeddings" \
|
||||||
|
--mode hybrid \
|
||||||
|
--expand-depth 1 \
|
||||||
|
--top-k 5
|
||||||
|
|
||||||
|
Beispielausgabe:
|
||||||
|
|
||||||
|
[1] note_id=...retriever-design
|
||||||
|
total=1.5054 (semantic=0.4038 edge=1.5910 centrality=0.0000)
|
||||||
|
|
||||||
|
Interpretation:
|
||||||
|
|
||||||
|
- `semantic`, `edge`, `centrality` = Rohwerte
|
||||||
|
- `total` = skaliert mit `retriever.yaml`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Betriebs-Hinweise
|
||||||
|
|
||||||
|
### 7.1 Änderung der Gewichte
|
||||||
|
|
||||||
|
- retriever.yaml anpassen
|
||||||
|
- FastAPI/uvicorn neu starten
|
||||||
|
- Smoke-Test prüfen
|
||||||
|
|
||||||
|
### 7.2 Zielbild WP-04
|
||||||
|
|
||||||
|
Der jetzige Stand ermöglicht:
|
||||||
|
|
||||||
|
- transparente, konfigurierbare Score-Berechnung
|
||||||
|
- Kombination aus Semantik, Typwissen und Graphstruktur
|
||||||
|
- spätere agentenbasierte Selbstjustierung
|
||||||
|
- alternative Profile (z. B. „mehr Semantik“, „mehr Graph“)
|
||||||
|
|
||||||
|
Dieses Dokument dient als stabile Referenz für Entwickler, Tests und spätere Automatisierungsschritte.
|
||||||
|
|
||||||
@@ -0,0 +1,205 @@
|
|||||||
|
# Mindnet v2.2 – Entwickler-Workflow
|
||||||
|
**Datei:** `DEV_WORKFLOW.md`
|
||||||
|
**Stand:** 2025-12-10 (Aktualisiert: Inkl. Frontend WP10)
|
||||||
|
|
||||||
|
Dieses Handbuch beschreibt den Entwicklungszyklus zwischen **Windows PC** (IDE), **Raspberry Pi** (Gitea) und **Beelink** (Runtime/Server).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Die Architektur
|
||||||
|
|
||||||
|
* **Windows 11 (VS Code):** Hier schreibst du Code. **Nie** direkt auf `main` arbeiten!
|
||||||
|
* **Raspberry Pi (Gitea):** Der Tresor. Speichert den Code und verwaltet Versionen.
|
||||||
|
* **Beelink (Runtime):** Hier läuft die Software. Wir nutzen **Systemd-Services**:
|
||||||
|
* **PROD:**
|
||||||
|
* API: `mindnet-prod` (Port 8001). Ordner: `~/mindnet`.
|
||||||
|
* UI: `mindnet-ui-prod` (Port 8501).
|
||||||
|
* **DEV:**
|
||||||
|
* API: `mindnet-dev` (Port 8002). Ordner: `~/mindnet_dev`.
|
||||||
|
* UI: `mindnet-ui-dev` (Port 8502).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Der Zyklus: Von der Idee zum Release
|
||||||
|
|
||||||
|
### Phase 1: Entwicklung (Windows / VS Code)
|
||||||
|
|
||||||
|
Hier erstellst du die neue Funktion in einer sicheren Umgebung.
|
||||||
|
|
||||||
|
1. **Basis aktualisieren (WICHTIG!):**
|
||||||
|
Bevor du startest, muss dein lokales `main` auf dem Stand des Servers sein.
|
||||||
|
* Klicke unten links auf den aktuellen Branch und wähle **`main`**.
|
||||||
|
* Klicke links im Menü "Source Control" auf die **drei Punkte (...)** -> **Pull** (oder das Synchronisieren-Symbol).
|
||||||
|
* *Erst jetzt hast du alle Dateien!*
|
||||||
|
|
||||||
|
2. **Branch erstellen:**
|
||||||
|
* Klicke wieder unten links auf `main`.
|
||||||
|
* Wähle `+ Create new branch...`.
|
||||||
|
* Gib den Namen ein: `feature/was-ich-tue` (z.B. `feature/wp06-decision`).
|
||||||
|
* Drücke **Enter**.
|
||||||
|
|
||||||
|
3. **Sicherheits-Check:**
|
||||||
|
* Steht unten links jetzt dein Feature-Branch? **Nur dann darfst du Code ändern!**
|
||||||
|
|
||||||
|
4. **Coden:**
|
||||||
|
* Nimm deine Änderungen vor (z.B. neue YAML-Configs).
|
||||||
|
|
||||||
|
5. **Sichern & Hochladen:**
|
||||||
|
* **Source Control** Icon (Gabel-Symbol) -> Nachricht eingeben -> **Commit**.
|
||||||
|
* **Publish Branch** (oder "Sync Changes"), um den Branch zum Raspberry Pi (Gitea) zu senden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 2: Testen (Beelink / `mindnet_dev`)
|
||||||
|
|
||||||
|
Hier prüfst du, ob dein neuer Code auf dem echten Server läuft.
|
||||||
|
|
||||||
|
1. **Einloggen:**
|
||||||
|
`ssh user@<BEELINK-IP>`
|
||||||
|
|
||||||
|
2. **In die Werkstatt gehen:**
|
||||||
|
`cd /home/llmadmin/mindnet_dev`
|
||||||
|
|
||||||
|
3. **Code holen:**
|
||||||
|
```bash
|
||||||
|
git fetch
|
||||||
|
# Tipp: 'git branch -r' zeigt alle verfügbaren Branches an
|
||||||
|
git checkout feature/wp06-decision
|
||||||
|
git pull
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **Umgebung vorbereiten (bei Bedarf):**
|
||||||
|
```bash
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -r requirements.txt # Nur nötig bei neuen Paketen
|
||||||
|
```
|
||||||
|
|
||||||
|
5. **Test-Server aktualisieren (WICHTIG):**
|
||||||
|
Der Code ist da, aber die Prozesse im RAM sind noch alt. Wir müssen die Services neustarten.
|
||||||
|
|
||||||
|
**Option A: Standard (Als Service laufen lassen)**
|
||||||
|
Ideal, wenn du nur testen willst, ob es läuft.
|
||||||
|
```bash
|
||||||
|
# 1. API neustarten
|
||||||
|
sudo systemctl restart mindnet-dev
|
||||||
|
# 2. UI neustarten (falls Frontend-Änderungen)
|
||||||
|
sudo systemctl restart mindnet-ui-dev
|
||||||
|
|
||||||
|
# Logs prüfen (um Fehler zu sehen):
|
||||||
|
journalctl -u mindnet-dev -f
|
||||||
|
# Oder Frontend Logs:
|
||||||
|
journalctl -u mindnet-ui-dev -f
|
||||||
|
```
|
||||||
|
|
||||||
|
**Option B: Manuell Debuggen (Direct Output)**
|
||||||
|
Ideal, wenn du Print-Ausgaben direkt sehen willst oder der Service dauernd crasht.
|
||||||
|
```bash
|
||||||
|
# 1. Services stoppen (wichtig, sonst sind Ports 8002/8502 belegt!)
|
||||||
|
sudo systemctl stop mindnet-dev
|
||||||
|
sudo systemctl stop mindnet-ui-dev
|
||||||
|
|
||||||
|
# 2. Manuell starten (z.B. API)
|
||||||
|
uvicorn app.main:app --host 0.0.0.0 --port 8002 --env-file .env
|
||||||
|
|
||||||
|
# ... Testen ...
|
||||||
|
|
||||||
|
# 3. Wenn fertig: Services wieder anschalten (Optional)
|
||||||
|
# Strg+C drücken
|
||||||
|
sudo systemctl start mindnet-dev
|
||||||
|
sudo systemctl start mindnet-ui-dev
|
||||||
|
```
|
||||||
|
|
||||||
|
6. **Validieren:**
|
||||||
|
* **Browser:** Öffne `http://<IP>:8502` um die UI zu testen.
|
||||||
|
* **CLI:** Führe Testskripte in einem **zweiten Terminal** aus:
|
||||||
|
```bash
|
||||||
|
# Beispiel für Decision Engine Test
|
||||||
|
python tests/test_wp06_decision.py -p 8002 -q "Soll ich...?"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 3: Release (Gitea / Raspberry Pi)
|
||||||
|
|
||||||
|
Wenn der Test auf Port 8002 / 8502 erfolgreich war:
|
||||||
|
|
||||||
|
1. Öffne Gitea im Browser.
|
||||||
|
2. Erstelle einen **Pull Request** (Dein Branch -> `main`).
|
||||||
|
3. **Merge** den Pull Request.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 4: Deployment (Beelink / `mindnet`)
|
||||||
|
|
||||||
|
Jetzt bringen wir die Änderung in das Live-System (Port 8001 / 8501).
|
||||||
|
|
||||||
|
1. **Deploy-Agent (Automatisch):**
|
||||||
|
Wenn dein `deploy.yml` aktiv ist, passiert das jetzt von selbst!
|
||||||
|
|
||||||
|
2. **Manuell (Falls nötig):**
|
||||||
|
```bash
|
||||||
|
cd /home/llmadmin/mindnet
|
||||||
|
git pull origin main
|
||||||
|
|
||||||
|
# Dependencies updaten
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
# Produktions-Services neustarten
|
||||||
|
sudo systemctl restart mindnet-prod
|
||||||
|
sudo systemctl restart mindnet-ui-prod
|
||||||
|
|
||||||
|
# Kurz prüfen, ob er läuft
|
||||||
|
sudo systemctl status mindnet-prod
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 5: Aufräumen (Optional aber empfohlen)
|
||||||
|
|
||||||
|
Damit das Chaos nicht wächst, löschen wir den fertigen Branch.
|
||||||
|
|
||||||
|
1. **Gitea:** Im Pull Request auf "Delete Branch" klicken.
|
||||||
|
2. **Beelink (Dev):**
|
||||||
|
```bash
|
||||||
|
cd ~/mindnet_dev
|
||||||
|
git checkout main
|
||||||
|
git pull
|
||||||
|
git branch -d feature/wp06-decision
|
||||||
|
```
|
||||||
|
3. **VS Code:**
|
||||||
|
* Auf `main` wechseln.
|
||||||
|
* Sync drücken.
|
||||||
|
* `F1` -> `Git: Delete Branch` -> Branch auswählen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Cheat Sheet: Die wichtigsten Befehle
|
||||||
|
|
||||||
|
| Wo? | Befehl | Was tut es? |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| **VS Code** | `Sync (auf main)` | **WICHTIG:** Holt neuesten Code vom Server. |
|
||||||
|
| **Beelink** | `git fetch` | Aktualisiert Liste der Remote-Branches. |
|
||||||
|
| **Beelink** | `sudo systemctl restart mindnet-dev` | **Neustart Dev-Backend (Port 8002).** |
|
||||||
|
| **Beelink** | `sudo systemctl restart mindnet-ui-dev` | **Neustart Dev-Frontend (Port 8502).** |
|
||||||
|
| **Beelink** | `journalctl -u mindnet-dev -f` | **Live-Logs Backend.** |
|
||||||
|
| **Beelink** | `journalctl -u mindnet-ui-dev -f` | **Live-Logs Frontend.** |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Troubleshooting
|
||||||
|
|
||||||
|
**"Read timed out (120s)" / 500 Error beim Chat**
|
||||||
|
* **Ursache:** Das LLM (Ollama) braucht zu lange zum Laden ("Cold Start").
|
||||||
|
* **Lösung:**
|
||||||
|
1. Erhöhe in `.env` den Wert: `MINDNET_LLM_TIMEOUT=300.0` (Backend) oder `MINDNET_API_TIMEOUT` (Frontend).
|
||||||
|
2. Starte die Server neu.
|
||||||
|
3. Stelle sicher, dass dein Test-Skript (Client) auch ein hohes Timeout hat.
|
||||||
|
|
||||||
|
**"Port 8002 / 8502 already in use"**
|
||||||
|
* **Ursache:** Du willst `uvicorn` oder `streamlit` manuell starten, aber der Service läuft noch.
|
||||||
|
* **Lösung:** `sudo systemctl stop mindnet-dev` bzw. `mindnet-ui-dev`.
|
||||||
|
|
||||||
|
**"UnicodeDecodeError in .env"**
|
||||||
|
* **Ursache:** Umlaute oder Sonderzeichen in der `.env` Datei.
|
||||||
|
* **Lösung:** `.env` bereinigen (nur ASCII nutzen) und sicherstellen, dass sie UTF-8 ohne BOM ist.
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
# Mindnet v2.2 – Developer Guide
|
||||||
|
**Datei:** `docs/mindnet_developer_guide_v2.2.md`
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** **FINAL** (Inkl. RAG, Decision Engine & Frontend WP10)
|
||||||
|
**Quellen:** `mindnet_technical_architecture.md`, `Handbuch.md`, `DEV_WORKFLOW.md`.
|
||||||
|
|
||||||
|
> **Zielgruppe:** Entwickler:innen.
|
||||||
|
> **Zweck:** Anleitung zum Aufsetzen der Entwicklungsumgebung, Verständnis der Modulstruktur und Durchführung von Tests.
|
||||||
|
|
||||||
|
---
|
||||||
|
- [Mindnet v2.2 – Developer Guide](#mindnet-v22--developer-guide)
|
||||||
|
- [1. Projektstruktur (Post-WP10)](#1-projektstruktur-post-wp10)
|
||||||
|
- [2. Lokales Setup (Development)](#2-lokales-setup-development)
|
||||||
|
- [2.1 Voraussetzungen](#21-voraussetzungen)
|
||||||
|
- [2.2 Installation](#22-installation)
|
||||||
|
- [2.3 Konfiguration (Environment)](#23-konfiguration-environment)
|
||||||
|
- [2.4 Dienste starten (API \& UI)](#24-dienste-starten-api--ui)
|
||||||
|
- [3. Core-Module \& Entwicklung](#3-core-module--entwicklung)
|
||||||
|
- [3.1 Der Importer (`scripts.import_markdown`)](#31-der-importer-scriptsimport_markdown)
|
||||||
|
- [3.2 Der Hybrid Router (`app.routers.chat`)](#32-der-hybrid-router-approuterschat)
|
||||||
|
- [3.3 Der Retriever (`app.core.retriever`)](#33-der-retriever-appcoreretriever)
|
||||||
|
- [3.4 Das Frontend (`app.frontend.ui`)](#34-das-frontend-appfrontendui)
|
||||||
|
- [4. Tests \& Debugging](#4-tests--debugging)
|
||||||
|
- [4.1 Unit Tests (Pytest)](#41-unit-tests-pytest)
|
||||||
|
- [4.2 Integration / Pipeline Tests](#42-integration--pipeline-tests)
|
||||||
|
- [4.3 Smoke Tests (E2E)](#43-smoke-tests-e2e)
|
||||||
|
- [5. Das "Teach-the-AI" Paradigma (Context Intelligence)](#5-das-teach-the-ai-paradigma-context-intelligence)
|
||||||
|
- [A. Workflow: Einen neuen Typ implementieren (z. B. `type: risk`)](#a-workflow-einen-neuen-typ-implementieren-z-b-type-risk)
|
||||||
|
- [B. Workflow: Neue Beziehungen (Edges) nutzen (z. B. `beeinflusst_von`)](#b-workflow-neue-beziehungen-edges-nutzen-z-b-beeinflusst_von)
|
||||||
|
- [Fazit](#fazit)
|
||||||
|
- [6. Nützliche Einzeiler](#6-nützliche-einzeiler)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Projektstruktur (Post-WP10)
|
||||||
|
|
||||||
|
Der Code ist modular in `app` (Logik), `scripts` (CLI) und `config` (Steuerung) getrennt.
|
||||||
|
|
||||||
|
mindnet/
|
||||||
|
├── app/
|
||||||
|
│ ├── core/ # Kernlogik
|
||||||
|
│ │ ├── chunker.py # Text-Zerlegung
|
||||||
|
│ │ ├── derive_edges.py # Edge-Erzeugung (WP03 Logik)
|
||||||
|
│ │ ├── retriever.py # Scoring & Hybrid Search
|
||||||
|
│ │ ├── qdrant.py # DB-Verbindung
|
||||||
|
│ │ └── ...
|
||||||
|
│ ├── models/ # Pydantic DTOs
|
||||||
|
│ │ └── dto.py # Zentrale DTO-Definition
|
||||||
|
│ ├── routers/ # FastAPI Endpoints
|
||||||
|
│ │ ├── query.py # Suche
|
||||||
|
│ │ ├── chat.py # Hybrid Router & Decision Engine (WP06)
|
||||||
|
│ │ ├── feedback.py # Feedback (WP04c)
|
||||||
|
│ │ └── ...
|
||||||
|
│ ├── services/ # Interne & Externe Dienste
|
||||||
|
│ │ ├── llm_service.py # Ollama Client (Mit Timeout & Raw-Mode)
|
||||||
|
│ │ ├── feedback_service.py # Logging (JSONL Writer)
|
||||||
|
│ │ └── embeddings_client.py
|
||||||
|
│ ├── frontend/ # NEU (WP10)
|
||||||
|
│ │ └── ui.py # Streamlit Application
|
||||||
|
│ └── main.py # Entrypoint der API
|
||||||
|
├── config/ # YAML-Konfigurationen (Single Source of Truth)
|
||||||
|
│ ├── types.yaml # Import-Regeln
|
||||||
|
│ ├── prompts.yaml # LLM Prompts & RAG Templates (WP06)
|
||||||
|
│ ├── decision_engine.yaml # Router-Strategien (WP06)
|
||||||
|
│ └── retriever.yaml # Scoring-Regeln & Kantengewichte
|
||||||
|
├── data/
|
||||||
|
│ └── logs/ # Lokale Logs (search_history.jsonl, feedback.jsonl)
|
||||||
|
├── scripts/ # CLI-Tools (Import, Diagnose, Reset)
|
||||||
|
├── tests/ # Pytest Suite & Smoke Scripts
|
||||||
|
└── vault/ # Dein lokaler Markdown-Content (Git-ignored)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Lokales Setup (Development)
|
||||||
|
|
||||||
|
### 2.1 Voraussetzungen
|
||||||
|
* **Python:** 3.10 oder höher.
|
||||||
|
* **Docker:** Für Qdrant.
|
||||||
|
* **Ollama:** Für lokale LLM-Inference (erforderlich für `/chat`).
|
||||||
|
* **Vault:** Ein Ordner mit Markdown-Dateien (z.B. `./mindnet_v2_test_vault` für Tests).
|
||||||
|
|
||||||
|
### 2.2 Installation
|
||||||
|
|
||||||
|
# 1. Repository klonen & Verzeichnis wechseln
|
||||||
|
git clone <repo> mindnet
|
||||||
|
cd mindnet
|
||||||
|
|
||||||
|
# 2. Virtual Environment erstellen
|
||||||
|
python3 -m venv .venv
|
||||||
|
source .venv/bin/activate
|
||||||
|
|
||||||
|
# 3. Abhängigkeiten installieren (inkl. Streamlit)
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
# 4. Ollama Setup (Modell laden)
|
||||||
|
# Wir nutzen Phi-3 Mini für schnelle CPU-Inference
|
||||||
|
ollama pull phi3:mini
|
||||||
|
|
||||||
|
### 2.3 Konfiguration (Environment)
|
||||||
|
Erstelle eine `.env` Datei im Root-Verzeichnis.
|
||||||
|
|
||||||
|
# Qdrant Verbindung
|
||||||
|
QDRANT_URL="http://localhost:6333"
|
||||||
|
QDRANT_API_KEY="" # Leer lassen für lokal
|
||||||
|
|
||||||
|
# Mindnet Core Settings
|
||||||
|
COLLECTION_PREFIX="mindnet_dev"
|
||||||
|
MINDNET_TYPES_FILE="./config/types.yaml"
|
||||||
|
MINDNET_RETRIEVER_CONFIG="./config/retriever.yaml"
|
||||||
|
|
||||||
|
# LLM / RAG Settings (WP06)
|
||||||
|
MINDNET_LLM_MODEL="phi3:mini"
|
||||||
|
MINDNET_OLLAMA_URL="http://127.0.0.1:11434"
|
||||||
|
MINDNET_LLM_TIMEOUT=300.0
|
||||||
|
MINDNET_PROMPTS_PATH="./config/prompts.yaml"
|
||||||
|
MINDNET_DECISION_CONFIG="./config/decision_engine.yaml"
|
||||||
|
|
||||||
|
# Frontend Settings (WP10)
|
||||||
|
MINDNET_API_URL="http://localhost:8002"
|
||||||
|
|
||||||
|
# Import-Strategie
|
||||||
|
MINDNET_HASH_COMPARE="Body"
|
||||||
|
MINDNET_HASH_SOURCE="parsed"
|
||||||
|
|
||||||
|
### 2.4 Dienste starten (API & UI)
|
||||||
|
|
||||||
|
Wir entwickeln mit zwei Services. Du kannst sie manuell in zwei Terminals starten oder die Systemd-Services nutzen (siehe `admin_guide.md`).
|
||||||
|
|
||||||
|
**Terminal A: Backend (API)**
|
||||||
|
|
||||||
|
# Startet auf Port 8002 (Dev Standard)
|
||||||
|
uvicorn app.main:app --host 0.0.0.0 --port 8002 --env-file .env --reload
|
||||||
|
|
||||||
|
**Terminal B: Frontend (UI)**
|
||||||
|
|
||||||
|
# Startet auf Port 8502 (Dev Standard)
|
||||||
|
# --server.port 8502 verhindert Konflikte mit Prod (8501)
|
||||||
|
streamlit run app/frontend/ui.py --server.port 8502
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Core-Module & Entwicklung
|
||||||
|
|
||||||
|
### 3.1 Der Importer (`scripts.import_markdown`)
|
||||||
|
Dies ist das komplexeste Modul.
|
||||||
|
* **Einstieg:** `scripts/import_markdown.py` -> `main()`.
|
||||||
|
* **Idempotenz:** Der Importer muss mehrfach laufen können, ohne Duplikate zu erzeugen. Wir nutzen deterministische IDs (UUIDv5).
|
||||||
|
* **Debugging:** Nutze `--dry-run` oder `scripts/payload_dryrun.py`.
|
||||||
|
|
||||||
|
### 3.2 Der Hybrid Router (`app.routers.chat`)
|
||||||
|
Hier liegt die Logik für Intent Detection (WP06).
|
||||||
|
* **Logic:** `_classify_intent` prüft zuerst Keywords (Fast Path) und fällt auf `llm_service.generate_raw_response` zurück (Slow Path), wenn konfiguriert.
|
||||||
|
* **Config:** Gesteuert durch `decision_engine.yaml`.
|
||||||
|
* **Erweiterung:** Um neue Intents hinzuzufügen, editiere nur die YAML, nicht den Python-Code (Late Binding).
|
||||||
|
|
||||||
|
### 3.3 Der Retriever (`app.core.retriever`)
|
||||||
|
Hier passiert das Scoring.
|
||||||
|
* **Hybrid Search:** Der Chat-Endpoint erzwingt `mode="hybrid"`.
|
||||||
|
* **Strategic Retrieval:** In `chat.py` wird der Retriever *zweimal* aufgerufen, wenn ein Intent (z.B. `DECISION`) eine Injection (`value`) erfordert.
|
||||||
|
|
||||||
|
### 3.4 Das Frontend (`app.frontend.ui`)
|
||||||
|
Eine Streamlit-App (WP10).
|
||||||
|
* **State:** Nutzt `st.session_state` für Chat-History und Drafts.
|
||||||
|
* **Logik:** Ruft `/chat` und `/feedback` Endpoints der API auf.
|
||||||
|
* **Anpassung:** CSS Styles sind direkt in `ui.py` eingebettet.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Tests & Debugging
|
||||||
|
|
||||||
|
Wir unterscheiden drei Test-Ebenen. Ein Pull Request sollte alle passieren.
|
||||||
|
|
||||||
|
### 4.1 Unit Tests (Pytest)
|
||||||
|
Für isolierte Logik (Parsing, Scoring).
|
||||||
|
|
||||||
|
pytest tests/test_retriever_basic.py
|
||||||
|
pytest tests/test_chunking.py
|
||||||
|
pytest tests/test_edges_all.py
|
||||||
|
|
||||||
|
### 4.2 Integration / Pipeline Tests
|
||||||
|
Prüfen den Datenfluss von Markdown bis Qdrant-JSON.
|
||||||
|
* **Payload Dryrun:** Prüft JSON-Schema Konformität.
|
||||||
|
|
||||||
|
python3 -m scripts.payload_dryrun --vault ./mindnet_v2_test_vault
|
||||||
|
|
||||||
|
* **Edge Checks:** Prüft Graph-Invarianten.
|
||||||
|
|
||||||
|
python3 -m scripts.edges_full_check
|
||||||
|
|
||||||
|
### 4.3 Smoke Tests (E2E)
|
||||||
|
Prüfen das laufende System gegen eine echte Qdrant-Instanz und Ollama.
|
||||||
|
|
||||||
|
# 1. API Test (Decision Engine)
|
||||||
|
python tests/test_wp06_decision.py -p 8002 -e DECISION -q "Soll ich X tun?"
|
||||||
|
|
||||||
|
# 2. UI Test (Manuell)
|
||||||
|
# Öffne http://localhost:8502
|
||||||
|
# Stelle eine Frage und prüfe Intent-Badge und Quellen-Anzeige.
|
||||||
|
|
||||||
|
# 3. Feedback Test
|
||||||
|
python tests/test_feedback_smoke.py --url http://localhost:8002/query
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Das "Teach-the-AI" Paradigma (Context Intelligence)
|
||||||
|
|
||||||
|
Mindnet lernt nicht durch Training (Fine-Tuning), sondern durch **Konfiguration**, **Vernetzung** und **Prompting**. Wenn du dem System ein neues Konzept beibringen willst, musst du in der Regel an drei Stellen eingreifen. Dies ist der **Core-Workflow** für Erweiterungen.
|
||||||
|
|
||||||
|
### A. Workflow: Einen neuen Typ implementieren (z. B. `type: risk`)
|
||||||
|
|
||||||
|
**1. Daten-Ebene (`config/types.yaml`)**
|
||||||
|
Definiere die "Physik" des Typs (Import-Regeln und Basis-Wichtigkeit).
|
||||||
|
|
||||||
|
risk:
|
||||||
|
chunk_profile: short # Risiken sind oft kurze Statements
|
||||||
|
retriever_weight: 0.90 # Sehr wichtig, fast so hoch wie Decisions
|
||||||
|
edge_defaults: ["blocks"] # Automatische Kante zu verlinkten Projekten
|
||||||
|
|
||||||
|
**2. Strategie-Ebene (`config/decision_engine.yaml`)**
|
||||||
|
Damit dieser Typ aktiv geladen wird, musst du ihn einer Strategie zuordnen.
|
||||||
|
|
||||||
|
DECISION:
|
||||||
|
inject_types: ["value", "principle", "goal", "risk"] # <--- "risk" hinzugefügt
|
||||||
|
|
||||||
|
*Ergebnis:* Wenn der Intent `DECISION` erkannt wird, sucht das System nun auch aktiv nach Risiken.
|
||||||
|
|
||||||
|
**3. Kognitive Ebene (`config/prompts.yaml`)**
|
||||||
|
Erkläre dem LLM im Template, was es damit tun soll.
|
||||||
|
|
||||||
|
decision_template: |
|
||||||
|
...
|
||||||
|
- Prüfe auf [RISK]: Wenn vorhanden, warne mich davor!
|
||||||
|
|
||||||
|
### B. Workflow: Neue Beziehungen (Edges) nutzen (z. B. `beeinflusst_von`)
|
||||||
|
|
||||||
|
Beziehungen sind der Klebstoff für die "Why"-Erklärungen.
|
||||||
|
|
||||||
|
**1. Erfassung im Vault (Markdown)**
|
||||||
|
Du musst dem System keinen neuen Edge-Typ "beibringen" (Schema-less). Du nutzt ihn einfach.
|
||||||
|
Nutze die Inline-Syntax im Fließtext:
|
||||||
|
|
||||||
|
Die Entscheidung für Qdrant wurde [[rel:beeinflusst_von Budgetkürzung 2024]].
|
||||||
|
|
||||||
|
**2. Gewichtung (`config/retriever.yaml`)**
|
||||||
|
Standardmäßig werden alle expliziten Kanten gleich behandelt. Wenn Kausalität wichtiger ist als Ähnlichkeit, konfiguriere dies hier.
|
||||||
|
|
||||||
|
scoring:
|
||||||
|
edge_type_weights:
|
||||||
|
beeinflusst_von: 1.5 # Sehr starker Einfluss auf das Ranking
|
||||||
|
|
||||||
|
### Fazit
|
||||||
|
* **Vault:** Liefert das Wissen.
|
||||||
|
* **Config:** Liefert die Strategie (Router & Weights).
|
||||||
|
* **Prompt:** Liefert die Interpretation.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Nützliche Einzeiler
|
||||||
|
|
||||||
|
**DB komplett zurücksetzen (Vorsicht!):**
|
||||||
|
|
||||||
|
python3 -m scripts.reset_qdrant --mode wipe --prefix "mindnet_dev" --yes
|
||||||
|
|
||||||
|
**Einen einzelnen File inspizieren (Parser-Sicht):**
|
||||||
|
|
||||||
|
python3 tests/inspect_one_note.py --file ./vault/MeinFile.md
|
||||||
|
|
||||||
|
**Live-Logs sehen (Systemd):**
|
||||||
|
|
||||||
|
# Backend
|
||||||
|
journalctl -u mindnet-dev -f
|
||||||
|
# Frontend
|
||||||
|
journalctl -u mindnet-ui-dev -f
|
||||||
@@ -1,249 +0,0 @@
|
|||||||
# mindnet – Knowledge Design
|
|
||||||
**Version:** 1.4.0 (aktualisiert 2025-10-06)
|
|
||||||
|
|
||||||
## 1. Gesamtziel und Systemverständnis
|
|
||||||
|
|
||||||
**mindnet** ist ein lokales Wissensnetzwerk, das alle meine Notizen, Projekte, Gedanken, Erfahrungen und Pläne als Markdown-Dateien speichert, automatisch importiert, semantisch in **Chunks** zerlegt und als **Graph-Struktur in Qdrant** ablegt.
|
|
||||||
Es ist die Grundlage für ein agentenfähiges, lokal betriebenes LLM-System, das meine Daten verknüpft, abfragt und erweitert.
|
|
||||||
|
|
||||||
**Zentrale Ziele**
|
|
||||||
- Vollständige Repräsentation persönlichen Wissens in strukturierter, maschinenlesbarer Form
|
|
||||||
- Reproduzierbare Verarbeitung: Import → Chunking → Embedding → Edges → Graph
|
|
||||||
- Verlustfreie Roundtrip-Fähigkeit (Import ⇄ Export)
|
|
||||||
- Erweiterbarkeit für Agenten, RAG-Retriever und semantische Suche
|
|
||||||
|
|
||||||
**Prinzipien**
|
|
||||||
- **Deterministisch:** IDs, Hashes und Edges wiederholbar
|
|
||||||
- **Portabel:** Markdown + YAML
|
|
||||||
- **Robust:** fehlertoleranter Parser
|
|
||||||
- **Idempotent:** keine Duplikate bei Re-Import
|
|
||||||
- **Erweiterbar:** neue Typen und Edge-Arten ohne Migration
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2. YAML-Frontmatter-Schema
|
|
||||||
|
|
||||||
| Feld | Typ | Beschreibung |
|
|
||||||
|-------|-----|--------------|
|
|
||||||
| `title` | string | Menschlich lesbarer Titel |
|
|
||||||
| `id` | string | Eindeutige Note-ID (Slug) |
|
|
||||||
| `type` | enum | `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone` |
|
|
||||||
| `status` | enum | `draft`, `active`, `done`, `archived` |
|
|
||||||
| `created` | date | ISO-Datum |
|
|
||||||
| `updated` | date | Letzte Änderung |
|
|
||||||
| `area` | string | Themen- oder Lebensbereich |
|
|
||||||
| `project` | string | Zugehöriges Projekt |
|
|
||||||
| `tags` | list | Strukturierte Tags (`area/…`, `topic/…`) |
|
|
||||||
| `depends_on` | list | IDs anderer Notizen |
|
|
||||||
| `assigned_to` | list | Beteiligte Personen |
|
|
||||||
| `embedding_exclude` | bool | Falls `true`, keine Embeddings |
|
|
||||||
| `hash_mode` | enum | Hash-Modus beim Import (`body`, `frontmatter`, `full`) |
|
|
||||||
| `priority` | enum/int | 1–5 oder `low` / `med` / `high` |
|
|
||||||
| `effort_min`, `due` | int/date | Aufwand / Termin |
|
|
||||||
| `aliases` | list | Alternative IDs |
|
|
||||||
| `lang` | string | ISO-Sprachcode |
|
|
||||||
| `source` | string | Herkunft / Referenzquelle |
|
|
||||||
|
|
||||||
**Pflichtfelder:** `title`, `id`, `type`, `status`, `created`
|
|
||||||
**Optionale Felder:** alle übrigen
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 3. Dateinamen und Ordner
|
|
||||||
|
|
||||||
- Format: `YYYY-MM-DD_title.md` oder `slug.md`
|
|
||||||
- Strukturierte Ordner:
|
|
||||||
- `10_thoughts/`, `20_concepts/`, `30_projects/`, `40_experiences/`, …
|
|
||||||
- Alle Pfade relativ; Unterstriche statt Leerzeichen
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 4. Link- und Edge-Design
|
|
||||||
|
|
||||||
**Verlinkungen**
|
|
||||||
- Wikilinks: `[[note-id]]` oder `[[Titel|note-id]]`
|
|
||||||
- Automatische Edges aus Text und Frontmatter
|
|
||||||
|
|
||||||
**Edge-Typen in Qdrant**
|
|
||||||
|
|
||||||
| Typ | Bedeutung | Quelle | Scope |
|
|
||||||
|------|------------|--------|--------|
|
|
||||||
| `belongs_to` | Chunk → Note | intern | chunk |
|
|
||||||
| `prev` / `next` | Reihenfolge | intern | chunk |
|
|
||||||
| `references` | explizite Wikilinks | Text | chunk |
|
|
||||||
| `backlink` | Gegenkante zu `references` | abgeleitet | note |
|
|
||||||
| `depends_on` | YAML-Abhängigkeiten | YAML | note |
|
|
||||||
| `assigned_to` | YAML-Zuweisung | YAML | note |
|
|
||||||
| `unresolved` | Ziel fehlt (Stub) | abgeleitet | chunk |
|
|
||||||
|
|
||||||
**Edge-Schlüssel**
|
|
||||||
- `source_id`, `target_id`, `kind`, `scope`, `note_id`
|
|
||||||
- Dedup-Schlüssel: Kombination aus `(kind, source_id, target_id, scope)`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 5. Chunking-Modell
|
|
||||||
|
|
||||||
**Chunk-Datenstruktur**
|
|
||||||
| Feld | Beschreibung |
|
|
||||||
|-------|---------------|
|
|
||||||
| `note_id` | Zugehörige Note |
|
|
||||||
| `chunk_index` | Laufende Nummer |
|
|
||||||
| `text` | Originalabschnitt |
|
|
||||||
| `window` | Text + Overlap links/rechts |
|
|
||||||
| `overlap_left` | Länge des linken Overlaps |
|
|
||||||
| `overlap_right` | Länge des rechten Overlaps |
|
|
||||||
| `embedding` | 384-dimensionaler Vektor |
|
|
||||||
| `tokens` | Tokenanzahl (optional) |
|
|
||||||
|
|
||||||
**Chunking-Regeln (aus `chunking_strategy.md`)**
|
|
||||||
- Trennung nach Absätzen, Überschriften, Listen, Tabellen
|
|
||||||
- Token-Zielgröße 350–500 (max 600)
|
|
||||||
- Overlap: 30–40 % für semantische Kohärenz
|
|
||||||
- Overlap wird in `window` integriert, sodass `text ≠ window`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 6. Hash-Strategie und Änderungsverfolgung
|
|
||||||
|
|
||||||
**Zweck:** Nur bei realer Inhaltsänderung re-indexieren
|
|
||||||
|
|
||||||
| Umgebungsvariable | Beschreibung |
|
|
||||||
|--------------------|--------------|
|
|
||||||
| `MINDNET_HASH_COMPARE` | Vergleichsquelle (`Body`, `Full`, `Frontmatter`) |
|
|
||||||
| `MINDNET_HASH_SOURCE` | Rohtext vs. geparst (`raw` / `parsed`) |
|
|
||||||
| `MINDNET_HASH_NORMALIZE` | Normalisierung (`canonical` / `none`) |
|
|
||||||
|
|
||||||
**Basismodi**
|
|
||||||
- `body` → Nur Textkörper
|
|
||||||
- `frontmatter` → Nur Metadaten
|
|
||||||
- `full` → Kombination
|
|
||||||
- Hash = SHA-256(canonicalized_input)
|
|
||||||
|
|
||||||
Bei Änderung: Re-Import, neue Embeddings, Edges werden aktualisiert.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 7. Qdrant-Speicherstruktur
|
|
||||||
|
|
||||||
| Collection | Inhalt | Primärfelder | Embedding |
|
|
||||||
|-------------|---------|---------------|------------|
|
|
||||||
| `mindnet_notes` | Notes mit Fulltext & Hash | `note_id` | optional |
|
|
||||||
| `mindnet_chunks` | Chunks mit Text, Window, Embedding | `chunk_id`, `note_id` | 384 d |
|
|
||||||
| `mindnet_edges` | Graphkanten | `edge_id`, `source_id`, `target_id` | — |
|
|
||||||
|
|
||||||
**Upsert-Strategie**
|
|
||||||
- Idempotente UUIDv5-IDs
|
|
||||||
- Keine Duplikate durch deterministische Signaturen
|
|
||||||
- `purge-before-upsert` zur Bereinigung
|
|
||||||
|
|
||||||
**Indizes**
|
|
||||||
- notes: `note_id`, `hash_signature`
|
|
||||||
- chunks: `note_id`, `chunk_index`
|
|
||||||
- edges: `(kind, source_id, target_id, scope)`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 8. Fehler-Toleranz und Parser-Robustheit
|
|
||||||
|
|
||||||
- UTF-8 mit Fallback: ersetzt ungültige Bytes (`errors="replace"`)
|
|
||||||
- Ignoriert Nullbytes, BOM, exotische Zeilenenden
|
|
||||||
- Erkennt defekte oder leere YAML-Header und überspringt sie
|
|
||||||
- Logt Problemfälle (`read_markdown failed`, `make_note_payload returned non-dict`)
|
|
||||||
- Bei Fehlern: Import → warn → continue
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 9. Tests & Validierung
|
|
||||||
|
|
||||||
**Integrationstests**
|
|
||||||
- Roundtrip: Import → Export → Compare (`compare_vaults.py`)
|
|
||||||
- Chunk-Integrität: `verify_chunks_integrity.py`
|
|
||||||
- Hash-Audit: `hash_reporter.py`
|
|
||||||
- Window-Vergleich: `check_chunks_window_vs_text.py`
|
|
||||||
|
|
||||||
**Erwartete Ergebnisse**
|
|
||||||
- Alle Notes werden erkannt
|
|
||||||
- `window ≠ text` bei > 60 % der Chunks
|
|
||||||
- Roundtrip-Vergleich liefert „OK“
|
|
||||||
- Edge-Anzahl entspricht Modell (belongs_to = #Chunks, next/prev ≈ #Chunks – 1)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 10. Export & Roundtrip-Verhalten
|
|
||||||
|
|
||||||
- `export_markdown.py` erzeugt identische Markdown-Dateien
|
|
||||||
- Schreibt YAML + Body in korrekter Reihenfolge
|
|
||||||
- Roundtrip-Prüfung (`compare_vaults.py`) validiert:
|
|
||||||
- vollständigen Body
|
|
||||||
- unveränderte Hash-Signatur
|
|
||||||
- gleiche Chunkzahl
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 11. Graph-Schicht und WP-04-Integration
|
|
||||||
|
|
||||||
- Qdrant fungiert als Graph-Backend
|
|
||||||
- `graph/service.py` (WP-04): stellt API für Mehrhop-Abfragen bereit
|
|
||||||
- Unterstützt:
|
|
||||||
- `expand(note_id, hops=2)`
|
|
||||||
- `resolve_unresolved()`
|
|
||||||
- `neighbors(kind=…)`
|
|
||||||
- Grundlage für spätere **Retriever-Funktionen** (LLM / Agenten)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 12. Agenten- und LLM-Integration
|
|
||||||
|
|
||||||
**Retriever-Pipeline**
|
|
||||||
1. Suche top-k Chunks (cosine distance)
|
|
||||||
2. Erweiterung um Edges (`references`, `backlinks`)
|
|
||||||
3. Kontext-Assembling via `window`-Text
|
|
||||||
4. Prompt-Generierung
|
|
||||||
|
|
||||||
**Agent-Use-Cases**
|
|
||||||
- Automatisches Tagging (`type`, `status`)
|
|
||||||
- Vorschläge für neue Edges
|
|
||||||
- Auflösen von `unresolved`-Referenzen
|
|
||||||
- Auto-Import externer Quellen (MediaWiki, PDFs, Webseiten)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 13. Erweiterbarkeit
|
|
||||||
|
|
||||||
- Neue Edge-Typen: `inspired_by`, `relates_to`, `contradicts`
|
|
||||||
- Neue Note-Typen: `decision`, `resource`, `recommendation`
|
|
||||||
- Embedding-Engines austauschbar (MiniLM → LaBSE → E5)
|
|
||||||
- Versionierte Chunking-Strategien (`chunk_config.py`)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 14. Qualitätssicherung und Best Practices
|
|
||||||
|
|
||||||
- Validierung vor Upsert
|
|
||||||
- Keine Leer-Chunks
|
|
||||||
- Vollständige Metadaten in Notes
|
|
||||||
- Tests in `tests/` automatisch ausführbar (`run_e2e_roundtrip.sh`)
|
|
||||||
- Ergebnis: „Roundtrip OK“ und `verify_chunks_integrity` = OK
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 15. Änderungsverlauf
|
|
||||||
|
|
||||||
**v1.4.0 (2025-10-06)**
|
|
||||||
- Neu: `window` vs `text` Feld in `mindnet_chunks`
|
|
||||||
- Neu: Hash-Modi (`body`, `frontmatter`, `full`) mit Env-Steuerung
|
|
||||||
- Neu: Baseline-Modus für Vergleich
|
|
||||||
- Parser → fehlertolerant (UTF-8 replace)
|
|
||||||
- Roundtrip-Test vollständig integriert
|
|
||||||
- Graph-Service und Mehrhop-Abfragen (WP-04-Vorbereitung)
|
|
||||||
- `purge-before-upsert` und `--note-scope-refs` Parameter ergänzt
|
|
||||||
|
|
||||||
**v1.3.0 (2025-09-09)**
|
|
||||||
- Hash-Normalisierung und Edge-Scope
|
|
||||||
- Roundtrip-Tests, Agenten-Integration
|
|
||||||
|
|
||||||
**v1.2.0 (2025-09-02)**
|
|
||||||
- Grundstruktur des Knowledge-Designs
|
|
||||||
|
|
||||||
---
|
|
||||||
@@ -0,0 +1,424 @@
|
|||||||
|
# Mindnet v2.2 – Fachliche Architektur
|
||||||
|
**Datei:** `docs/mindnet_functional_architecture_v2.2.md`
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** **FINAL** (Integrierter Stand WP01–WP10)
|
||||||
|
|
||||||
|
> Dieses Dokument beschreibt **was** Mindnet fachlich tut und **warum** – mit Fokus auf die Erzeugung und Nutzung von **Edges** (Kanten), die Logik des Retrievers und den **RAG-Chat** (Decision Engine & Persönlichkeit). Die technische Umsetzung wird im technischen Dokument detailliert.
|
||||||
|
|
||||||
|
---
|
||||||
|
<details>
|
||||||
|
<summary>📖 <b>Inhaltsverzeichnis (Klicken zum Öffnen)</b></summary>
|
||||||
|
|
||||||
|
- [Mindnet v2.2 – Fachliche Architektur](#mindnet-v22--fachliche-architektur)
|
||||||
|
- [](#)
|
||||||
|
- [0) Zielbild \& Grundprinzip](#0-zielbild--grundprinzip)
|
||||||
|
- [1) Notizen \& Chunks (fachliche Perspektive)](#1-notizen--chunks-fachliche-perspektive)
|
||||||
|
- [1.1 Notiz (Note)](#11-notiz-note)
|
||||||
|
- [1.2 Chunk](#12-chunk)
|
||||||
|
- [2) Edges – fachliche Typen \& Bedeutung](#2-edges--fachliche-typen--bedeutung)
|
||||||
|
- [2.1 Struktur-Kanten (Das Skelett)](#21-struktur-kanten-das-skelett)
|
||||||
|
- [2.2 Inhalts-Kanten (explizit)](#22-inhalts-kanten-explizit)
|
||||||
|
- [2.3 Typ-basierte Default-Kanten (Regelbasiert)](#23-typ-basierte-default-kanten-regelbasiert)
|
||||||
|
- [3) Edge-Payload – Felder \& Semantik](#3-edge-payload--felder--semantik)
|
||||||
|
- [4) Typ-Registry (`config/types.yaml`)](#4-typ-registry-configtypesyaml)
|
||||||
|
- [4.1 Zweck](#41-zweck)
|
||||||
|
- [4.2 Beispiel (auslieferungsnah)](#42-beispiel-auslieferungsnah)
|
||||||
|
- [5) Der Retriever (Funktionaler Layer)](#5-der-retriever-funktionaler-layer)
|
||||||
|
- [5.1 Scoring-Modell](#51-scoring-modell)
|
||||||
|
- [5.2 Erklärbarkeit (Explainability) – WP04b](#52-erklärbarkeit-explainability--wp04b)
|
||||||
|
- [6) Context Intelligence \& Intent Router (WP06)](#6-context-intelligence--intent-router-wp06)
|
||||||
|
- [6.1 Das Problem: Statische vs. Dynamische Antworten](#61-das-problem-statische-vs-dynamische-antworten)
|
||||||
|
- [6.2 Der Intent-Router (Keyword \& Semantik)](#62-der-intent-router-keyword--semantik)
|
||||||
|
- [6.3 Strategic Retrieval (Injektion von Werten)](#63-strategic-retrieval-injektion-von-werten)
|
||||||
|
- [6.4 Reasoning (Das Gewissen)](#64-reasoning-das-gewissen)
|
||||||
|
- [7) Future Concepts: The Empathic Digital Twin (Ausblick)](#7-future-concepts-the-empathic-digital-twin-ausblick)
|
||||||
|
- [7.1 Antizipation durch Erfahrung](#71-antizipation-durch-erfahrung)
|
||||||
|
- [7.2 Empathie \& "Ich"-Modus](#72-empathie--ich-modus)
|
||||||
|
- [7.3 Glaubenssätze \& Rituale](#73-glaubenssätze--rituale)
|
||||||
|
- [8) Erweiterbarkeit \& Teaching (Context Intelligence)](#8-erweiterbarkeit--teaching-context-intelligence)
|
||||||
|
- [A. Workflow: Einen neuen Typ implementieren (z. B. `type: risk`)](#a-workflow-einen-neuen-typ-implementieren-z-b-type-risk)
|
||||||
|
- [B. Workflow: Neue Beziehungen nutzen (z. B. `beeinflusst_von`)](#b-workflow-neue-beziehungen-nutzen-z-b-beeinflusst_von)
|
||||||
|
- [9) Feedback \& Lernen – WP04c/WP10](#9-feedback--lernen--wp04cwp10)
|
||||||
|
- [9.1 Der Feedback-Loop](#91-der-feedback-loop)
|
||||||
|
- [10) Confidence \& Provenance – wozu?](#10-confidence--provenance--wozu)
|
||||||
|
- [11) Semantik ausgewählter `kind`-Werte](#11-semantik-ausgewählter-kind-werte)
|
||||||
|
- [12) Frontmatter-Eigenschaften – Rolle \& Empfehlung](#12-frontmatter-eigenschaften--rolle--empfehlung)
|
||||||
|
- [13) Lösch-/Update-Garantien (Idempotenz)](#13-lösch-update-garantien-idempotenz)
|
||||||
|
- [14) Beispiel – Von Markdown zu Kanten (v2.2)](#14-beispiel--von-markdown-zu-kanten-v22)
|
||||||
|
- [15) Referenzen (Projektdateien \& Leitlinien)](#15-referenzen-projektdateien--leitlinien)
|
||||||
|
- [16) Workpackage Status (v2.3.2)](#16-workpackage-status-v232)
|
||||||
|
|
||||||
|
</details>
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0) Zielbild & Grundprinzip
|
||||||
|
|
||||||
|
Mindnet wandelt Obsidian-Markdown-Notizen in einen **dynamischen Graphen aus Punkten und Kanten** um.
|
||||||
|
Die drei zentralen Artefakt-Sammlungen lauten:
|
||||||
|
|
||||||
|
- `mindnet_notes` – genau **eine** Note pro Markdown-Datei (Metadaten & Hashes)
|
||||||
|
- `mindnet_chunks` – semantische Teilstücke einer Note (Fenster/„Chunks“)
|
||||||
|
- `mindnet_edges` – gerichtete Beziehungen zwischen Knoten (Chunks/Notes)
|
||||||
|
|
||||||
|
Die Import-Pipeline erzeugt diese Artefakte **deterministisch** und **idempotent** (erneute Läufe überschreiben konsistent statt zu duplizieren). Die Import-Schritte sind: *parse → chunk → edge → upsert*.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1) Notizen & Chunks (fachliche Perspektive)
|
||||||
|
|
||||||
|
### 1.1 Notiz (Note)
|
||||||
|
- Repräsentiert eine fachliche Einheit (z. B. *„Vector DB Basics“*, *„KI-Projekt“*).
|
||||||
|
- Trägt Eigenschaften (Titel, Typ, Zeitstempel, optionale Policies).
|
||||||
|
- **Typ** (*type*) steuert u. a. Chunk-Profil und Default-Relationen (siehe §4).
|
||||||
|
- **retriever_weight** und **chunk_profile** werden systematisch an Note **und** Chunks gespiegelt, damit der Retriever beides nutzen kann.
|
||||||
|
|
||||||
|
### 1.2 Chunk
|
||||||
|
- Ausschnitt/Textfenster aus der Note, als eigenständiger Such-Anker.
|
||||||
|
- Jeder Chunk gehört **genau einer** Note.
|
||||||
|
- Chunks bilden eine Sequenz (1…N) – das ermöglicht *next/prev*.
|
||||||
|
- **Neu in v2.2:** Alle Kanten entstehen ausschließlich zwischen Chunks (Scope="chunk"), nie zwischen Notes direkt. Notes dienen nur noch als Metadatencontainer.
|
||||||
|
|
||||||
|
> **Wichtig:** Chunking-Profile (short/medium/long) kommen aus `types.yaml` (per Note-Typ), können aber lokal überschrieben werden. Die effektiven Werte werden bei der Payload-Erzeugung bestimmt.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2) Edges – fachliche Typen & Bedeutung
|
||||||
|
|
||||||
|
Edges kodieren Beziehungen. Sie sind **gerichtet** und werden in `mindnet_edges` gespeichert. Die Erzeugung folgt einer strengen Priorität: **Inline > Callout > Defaults > Struktur**.
|
||||||
|
|
||||||
|
### 2.1 Struktur-Kanten (Das Skelett)
|
||||||
|
- **belongs_to**: *Chunk → Note* (Zuordnung)
|
||||||
|
- **next / prev**: Kettenbeziehungen zwischen aufeinanderfolgenden Chunks derselben Note.
|
||||||
|
→ ermöglichen „lineares Weiterlesen“ oder Kontextpfade in Antworten.
|
||||||
|
Diese Kanten entstehen immer, unabhängig von Inhalten.
|
||||||
|
|
||||||
|
### 2.2 Inhalts-Kanten (explizit)
|
||||||
|
Hier unterscheidet v2.2 präzise zwischen verschiedenen Quellen der Evidenz:
|
||||||
|
|
||||||
|
1. **Explizite Inline-Relationen (Höchste Priorität):**
|
||||||
|
Im Fließtext notierte, semantisch qualifizierte Relationen.
|
||||||
|
* Syntax: `[[rel:depends_on Embeddings 101]]`
|
||||||
|
* Fachliche Aussage: "Hängt ab von", "Ähnlich zu".
|
||||||
|
* *rule_id*: `inline:rel`
|
||||||
|
* *confidence*: ~0.95
|
||||||
|
|
||||||
|
2. **Callout-Edges (Kuratierte Listen):**
|
||||||
|
Redaktionell gepflegte Listen am Ende einer Notiz (z.B. "Siehe auch").
|
||||||
|
* Syntax: `> [!edge] related_to: [[Vector DB Basics]]`
|
||||||
|
* *rule_id*: `callout:edge`
|
||||||
|
* *confidence*: ~0.90
|
||||||
|
|
||||||
|
3. **Wikilinks (Standard-Referenzen):**
|
||||||
|
Der klassische Obsidian-Link.
|
||||||
|
* Syntax: `[[Vector DB Basics]]`
|
||||||
|
* Typ: `references`
|
||||||
|
* *rule_id*: `explicit:wikilink`
|
||||||
|
* *confidence*: 1.0
|
||||||
|
|
||||||
|
> Alle expliziten Quellen (Wikilink, Inline-Rel, Callout) sind **Primärbelege**.
|
||||||
|
|
||||||
|
### 2.3 Typ-basierte Default-Kanten (Regelbasiert)
|
||||||
|
Jede Note hat einen *Typ* (z. B. `concept`, `project`, `profile`). In `config/types.yaml` definieren wir pro Typ **edge_defaults**, z. B.:
|
||||||
|
|
||||||
|
- `concept`: `["references","related_to"]`
|
||||||
|
- `project`: `["references","depends_on"]`
|
||||||
|
|
||||||
|
Regel: **Für jede gefundene explizite Referenz** (s. o.) werden **zusätzliche** Edges nach diesen Defaults erzeugt.
|
||||||
|
Beispiel: Ein *project* mit `edge_defaults=["depends_on"]` erzeugt zu *jedem* explizit referenzierten Ziel **zusätzlich** eine `depends_on`-Kante.
|
||||||
|
Diese Kanten tragen *provenance=rule* und eine **rule_id** der Form `edge_defaults:{note_type}:{relation}` sowie eine geringere Confidence (~0.7).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3) Edge-Payload – Felder & Semantik
|
||||||
|
|
||||||
|
Jede Kante hat mindestens:
|
||||||
|
|
||||||
|
- `kind` – Beziehungsart *(belongs_to, next, prev, references, related_to, depends_on, similar_to, …)*
|
||||||
|
- `scope` – `"chunk"` (Standard in v2.2)
|
||||||
|
- `source_id`, `target_id` – Quell-/Ziel-Knoten (Chunk-IDs oder Note-Titel bei unresolved Targets)
|
||||||
|
- `note_id` – **Owner-Note** (die Note, aus der die Kante stammt)
|
||||||
|
|
||||||
|
Erweiterte/abgeleitete Felder (WP03 Superset):
|
||||||
|
|
||||||
|
- `provenance` – `"explicit"` (Wikilink/Inline/Callout) oder `"rule"` (Typ-Defaults)
|
||||||
|
- `rule_id` – maschinenlesbare Regelquelle (z.B. `inline:rel`, `edge_defaults:project:depends_on`)
|
||||||
|
- `confidence` – 0.0–1.0; Heuristik zur Gewichtung im Scoring.
|
||||||
|
|
||||||
|
**Dedup-Schlüssel** (fachlich): Gleichlautende `(source_id, target_id, kind, scope, rule_id)` werden nicht mehrfach geschrieben.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4) Typ-Registry (`config/types.yaml`)
|
||||||
|
|
||||||
|
### 4.1 Zweck
|
||||||
|
- Steuert **Chunking-Profile** (`short|medium|long`) **pro Typ**
|
||||||
|
- Liefert **retriever_weight** (Note-/Chunk-Gewichtung im Ranking) **pro Typ**
|
||||||
|
- Definiert **edge_defaults** je Typ (s. o.)
|
||||||
|
|
||||||
|
Der Importer lädt die Registry aus `MINDNET_TYPES_FILE` oder nutzt Fallbacks. **Frontmatter-Overrides** für Profile werden in v2.2 weitgehend ignoriert zugunsten einer zentralen Governance in der YAML-Datei.
|
||||||
|
|
||||||
|
### 4.2 Beispiel (auslieferungsnah)
|
||||||
|
version: 1.0
|
||||||
|
types:
|
||||||
|
concept:
|
||||||
|
chunk_profile: medium
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
retriever_weight: 0.60
|
||||||
|
project:
|
||||||
|
chunk_profile: long
|
||||||
|
edge_defaults: ["references", "depends_on"]
|
||||||
|
retriever_weight: 0.97
|
||||||
|
|
||||||
|
**Auflösung im Importer**
|
||||||
|
- `effective_chunk_profile(note_type, registry)` → `"short|medium|long"|None`
|
||||||
|
- `effective_retriever_weight(note_type, registry)` → `float|None`
|
||||||
|
- Ergebnis wird in `note_payload` **und** `chunk_payloads` gespiegelt.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5) Der Retriever (Funktionaler Layer)
|
||||||
|
|
||||||
|
Der Retriever ist in v2.2 der zentrale Zugangspunkt. Er realisiert die **Hybride Suche**.
|
||||||
|
|
||||||
|
### 5.1 Scoring-Modell
|
||||||
|
Die Relevanz eines Treffers ergibt sich nicht mehr nur aus Textähnlichkeit, sondern aus einer gewichteten Formel:
|
||||||
|
|
||||||
|
total_score =
|
||||||
|
semantic_weight * semantic_score
|
||||||
|
+ edge_weight * edge_bonus
|
||||||
|
+ centrality_weight * centrality_bonus
|
||||||
|
+ type_weight * retriever_weight
|
||||||
|
|
||||||
|
* **semantic_score:** Vektorähnlichkeit (Qdrant).
|
||||||
|
* **retriever_weight:** Wichtigkeit des Typs (z.B. Projekte > Quellen).
|
||||||
|
* **edge_bonus:** Belohnung für Chunks, die im Kontext der Anfrage stark vernetzt sind (Summe der Confidence eingehender relevanter Kanten).
|
||||||
|
* **centrality_bonus:** Belohnung für Knoten, die zentrale Hubs im lokalen Graphen darstellen.
|
||||||
|
|
||||||
|
### 5.2 Erklärbarkeit (Explainability) – WP04b
|
||||||
|
Der Retriever ist keine Blackbox mehr. Er liefert auf Wunsch (`explain=True`) eine strukturierte Begründung (`Explanation`-Objekt).
|
||||||
|
|
||||||
|
**Die "Warum"-Logik:**
|
||||||
|
1. **Semantik:** Prüfung der Cosine-Similarity ("Sehr hohe textuelle Übereinstimmung").
|
||||||
|
2. **Typ:** Prüfung des `retriever_weight` ("Bevorzugt, da Entscheidung").
|
||||||
|
3. **Graph (Kontext):**
|
||||||
|
* **Hub (Outgoing):** Worauf verweist dieser Treffer? ("Verweist auf Qdrant").
|
||||||
|
* **Authority (Incoming):** Wer verweist auf diesen Treffer? ("Wird referenziert von Projekt Alpha").
|
||||||
|
|
||||||
|
Die API gibt diese Analysen als menschenlesbare Sätze (`reasons`) und als Datenstruktur (`score_breakdown`) zurück.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6) Context Intelligence & Intent Router (WP06)
|
||||||
|
|
||||||
|
Seit WP06 agiert Mindnet nicht mehr statisch, sondern passt seine Suchstrategie dem **Intent** (der Absicht) des Nutzers an. Dies ist die Transformation vom reinen Wissens-Abrufer zum strategischen Entscheidungspartner.
|
||||||
|
|
||||||
|
### 6.1 Das Problem: Statische vs. Dynamische Antworten
|
||||||
|
* **Früher (Pre-WP06):** Jede Frage ("Was ist X?" oder "Soll ich X?") wurde gleich behandelt -> Fakten-Retrieval.
|
||||||
|
* **Heute (WP06):** Das System erkennt, *was* der User will, und lädt unterschiedliche Wissensbausteine.
|
||||||
|
|
||||||
|
### 6.2 Der Intent-Router (Keyword & Semantik)
|
||||||
|
Der Router prüft vor jeder Antwort die Absicht über konfigurierbare Strategien (`config/decision_engine.yaml`):
|
||||||
|
|
||||||
|
1. **FACT:** Reine Wissensfrage ("Was ist Qdrant?"). → Standard RAG.
|
||||||
|
2. **DECISION:** Frage nach Rat oder Strategie ("Soll ich Qdrant nutzen?"). → Aktiviert die Decision Engine.
|
||||||
|
3. **EMPATHY:** Emotionale Zustände ("Ich bin gestresst"). → Aktiviert den empathischen Modus.
|
||||||
|
4. **CODING:** Technische Anfragen.
|
||||||
|
|
||||||
|
**Erkennung:**
|
||||||
|
* **Fast Path:** Prüfung auf Schlüsselwörter (z.B. "soll ich").
|
||||||
|
* **Slow Path (Smart Fallback):** Wenn kein Keyword passt, prüft ein LLM (Phi-3) die Semantik des Satzes.
|
||||||
|
|
||||||
|
### 6.3 Strategic Retrieval (Injektion von Werten)
|
||||||
|
Im Modus `DECISION` führt das System eine **zweite Suchstufe** aus. Es sucht nicht nur nach semantisch passenden Texten zur Frage, sondern erzwingt das Laden von strategischen Notizen wie:
|
||||||
|
* **Values (`type: value`):** Moralische Werte (z.B. "Privacy First").
|
||||||
|
* **Principles (`type: principle`):** Handlungsanweisungen.
|
||||||
|
* **Goals (`type: goal`):** Strategische Ziele.
|
||||||
|
|
||||||
|
Dies wird über **Late Binding** in `config/decision_engine.yaml` gesteuert. Neue Facetten der Persönlichkeit können dort konfiguriert werden, ohne den Code zu ändern.
|
||||||
|
|
||||||
|
### 6.4 Reasoning (Das Gewissen)
|
||||||
|
Das LLM erhält im Prompt die explizite Anweisung: *"Wäge die Fakten (aus der Suche) gegen die injizierten Werte ab."*
|
||||||
|
Dadurch entstehen Antworten, die nicht nur technisch korrekt sind, sondern subjektiv passend ("Tool X passt nicht zu deinem Ziel Z").
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7) Future Concepts: The Empathic Digital Twin (Ausblick)
|
||||||
|
|
||||||
|
Um Mindnet von einer Entscheidungsmaschine zu einem echten **Spiegel der Persönlichkeit** zu entwickeln, sind folgende Erweiterungen konzeptionell vorgesehen:
|
||||||
|
|
||||||
|
### 7.1 Antizipation durch Erfahrung
|
||||||
|
* **Konzept:** Das System soll Konsequenzen vorhersagen ("Was passiert, wenn...?").
|
||||||
|
* **Umsetzung:** Neben Werten werden vergangene Erfahrungen (`type: experience`) geladen.
|
||||||
|
* **Logik:** *"In einer ähnlichen Situation (Projekt A) hat Entscheidung X zu Ergebnis Y geführt."*
|
||||||
|
|
||||||
|
### 7.2 Empathie & "Ich"-Modus
|
||||||
|
* **Konzept:** Das System antwortet nicht wie ein Roboter, sondern im Tonfall und Stil des Nutzers.
|
||||||
|
* **Umsetzung (Few-Shot Prompting):** Der System-Prompt wird dynamisch mit Beispielen gefüttert, wie der Nutzer typischerweise auf E-Mails oder Fragen antwortet.
|
||||||
|
* **Datenbasis:** `type: profile` oder `type: manifesto` definieren das Selbstbild.
|
||||||
|
|
||||||
|
### 7.3 Glaubenssätze & Rituale
|
||||||
|
* **Konzept:** Berücksichtigung weicher Faktoren und Gewohnheiten.
|
||||||
|
* **Umsetzung:** Erweiterung der `decision_engine.yaml` um `inject_types: ["belief", "ritual"]`.
|
||||||
|
* **Szenario:** Bei Terminplanungen werden Rituale ("Keine Meetings vor 10 Uhr") automatisch als harte Restriktion gegen Anfragen geprüft.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8) Erweiterbarkeit & Teaching (Context Intelligence)
|
||||||
|
|
||||||
|
Mindnet lernt nicht durch klassisches Training (Fine-Tuning), sondern durch **Konfiguration**, **Vernetzung** und **Prompting**. Dies ist das **"Teach-the-AI" Paradigma**: Wenn du dem System ein neues Konzept beibringen willst, musst du an drei Stellen eingreifen.
|
||||||
|
|
||||||
|
### A. Workflow: Einen neuen Typ implementieren (z. B. `type: risk`)
|
||||||
|
|
||||||
|
1. **Physik definieren (`config/types.yaml`)**
|
||||||
|
Steuere, wie schwer der Inhalt wiegt und wie er vernetzt wird.
|
||||||
|
```yaml
|
||||||
|
risk:
|
||||||
|
chunk_profile: short # Risiken sind oft kurze Statements
|
||||||
|
retriever_weight: 0.90 # Hohe Priorität im Ranking
|
||||||
|
edge_defaults: ["blocks"] # Automatische Kante zu verlinkten Projekten
|
||||||
|
```
|
||||||
|
*Effekt:* Der Retriever spült diese Notizen bei relevanten Anfragen nach oben.
|
||||||
|
|
||||||
|
2. **Semantik erklären (`config/prompts.yaml` / `decision_engine.yaml`)**
|
||||||
|
Bringe dem LLM bei, wie es mit diesem Typ umgehen soll. Füge `risk` zur `DECISION`-Strategie hinzu (`inject_types`), damit es bei Entscheidungen geladen wird.
|
||||||
|
|
||||||
|
### B. Workflow: Neue Beziehungen nutzen (z. B. `beeinflusst_von`)
|
||||||
|
|
||||||
|
Beziehungen sind der Klebstoff für logische Schlussfolgerungen.
|
||||||
|
|
||||||
|
1. **Erfassung im Vault (Markdown)**
|
||||||
|
Nutze die Inline-Syntax. Du musst nichts vorab definieren.
|
||||||
|
> "Die Entscheidung für Qdrant wurde [[rel:beeinflusst_von Budgetkürzung 2024]]."
|
||||||
|
|
||||||
|
2. **Gewichtung justieren (`config/retriever.yaml`)**
|
||||||
|
Standardmäßig sind alle Kanten gleich `1.0`. Wenn Kausalität wichtiger ist als Ähnlichkeit, definiere dies hier:
|
||||||
|
```yaml
|
||||||
|
retriever:
|
||||||
|
edge_weights:
|
||||||
|
beeinflusst_von: 1.5 # Starker Boost: Zieht Ursachen in den Kontext
|
||||||
|
related_to: 0.5 # Schwacher Boost: Nur "nice to know"
|
||||||
|
```
|
||||||
|
*Effekt:* Bei der Frage nach "Qdrant" landet die "Budgetkürzung" (Ursache) im Kontext, weil die Kante stark ist.
|
||||||
|
|
||||||
|
3. **Interpretation (Prompting)**
|
||||||
|
Das LLM erkennt im RAG-Kontext: `Qdrant --[beeinflusst_von]--> Budgetkürzung`.
|
||||||
|
Es kann daraus den Satz formulieren: *"Die Wahl fiel vermutlich aufgrund der Budgetkürzung auf Qdrant."*
|
||||||
|
|
||||||
|
**Fazit:** Nur wenn **Daten** (Vault), **Physik** (Config) und **Semantik** (Prompt) zusammenspielen, entsteht ein intelligenter Zwilling.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9) Feedback & Lernen – WP04c/WP10
|
||||||
|
|
||||||
|
Das System verfügt nun über ein **Kurzzeitgedächtnis für Interaktionen**, das die Basis für zukünftiges Lernen bildet.
|
||||||
|
|
||||||
|
### 9.1 Der Feedback-Loop
|
||||||
|
Die Interaktion erfolgt primär über das **Web-Interface (WP10)**.
|
||||||
|
|
||||||
|
1. **Suche (Situation):**
|
||||||
|
Wenn ein Nutzer eine Anfrage stellt, loggt Mindnet die "Situation":
|
||||||
|
* Den Query-Text.
|
||||||
|
* Die Top-K Ergebnisse.
|
||||||
|
* Den exakten `score_breakdown` zu diesem Zeitpunkt.
|
||||||
|
* Eine eindeutige `query_id` wird generiert.
|
||||||
|
|
||||||
|
2. **Bewertung (Label):**
|
||||||
|
Der Nutzer bewertet die Antwort (**Sterne**) oder einzelne Quellen (**Faces**) im Frontend.
|
||||||
|
Dieses Feedback wird unter Referenzierung der `query_id` in `feedback.jsonl` gespeichert.
|
||||||
|
|
||||||
|
3. **Lernen (Perspektive WP08):**
|
||||||
|
Durch die Verknüpfung von Situation und Bewertung entstehen Trainingsdaten. Mindnet kann später analysieren: *"Nutzer bevorzugen Treffer mit hohem Edge-Bonus, auch wenn der Text weniger passt."* -> Konsequenz: `edge_weight` erhöhen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10) Confidence & Provenance – wozu?
|
||||||
|
|
||||||
|
Der Retriever kann Edges gewichten:
|
||||||
|
- **provenance**: *explicit* > *rule*
|
||||||
|
- **confidence**: numerische Feinsteuerung
|
||||||
|
- **retriever_weight (Note/Chunk)**: skaliert die Relevanz des gesamten Knotens
|
||||||
|
|
||||||
|
Eine typische Gewichtung (konfigurierbar in `retriever.yaml`) ist:
|
||||||
|
`explicit: 1.0`, `rule: 0.8`. Damit bevorzugt der Graph **kuratiertes Wissen** (explizit notierte Links) vor „erweiterten“ Default-Ableitungen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11) Semantik ausgewählter `kind`-Werte
|
||||||
|
|
||||||
|
- `references` – „Nutzt/erwähnt“; neutral, aber stützend für Kontext.
|
||||||
|
- `related_to` – Ähnlichkeit/Verwandtschaft (symmetrisch interpretierbar).
|
||||||
|
- `similar_to` – noch engere Ähnlichkeit; oft aus Inline-Rel (bewusst gesetzt).
|
||||||
|
- `depends_on` – fachliche Abhängigkeit (z. B. „Projekt X hängt von Y ab“).
|
||||||
|
- `belongs_to`, `next`, `prev` – Struktur.
|
||||||
|
|
||||||
|
> Symmetrische Relationen (z. B. `related_to`, `similar_to`) können **explizit** nur einseitig notiert sein, aber im Retriever beidseitig interpretiert werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12) Frontmatter-Eigenschaften – Rolle & Empfehlung
|
||||||
|
|
||||||
|
Frontmatter-Eigenschaften (Properties) bleiben **minimiert**:
|
||||||
|
- **type** – steuert Typ-Defaults via Registry (Pflicht für differenziertes Verhalten).
|
||||||
|
- **tags** – für klassische Filterung.
|
||||||
|
- **retriever_weight / chunk_profile** – Sollten **nicht** manuell gesetzt werden (Best Practice: `types.yaml` nutzen).
|
||||||
|
- **links** – Veraltet. Bitte Links primär über Text (Wikilink/Inline/Callout) pflegen.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13) Lösch-/Update-Garantien (Idempotenz)
|
||||||
|
|
||||||
|
- Jede Note hat einen stabilen **note_id** (Frontmatter/Hash).
|
||||||
|
- Vor einem Upsert können *alte Chunks/Edges einer Note* gefiltert gelöscht werden (`note_id`-Filter) – das hält Collections sauber bei Re-Imports.
|
||||||
|
- Die Import-Skripte unterstützen Modi wie `--purge-before-upsert` und `--sync-deletes`, um den Graph sauber zu halten.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 14) Beispiel – Von Markdown zu Kanten (v2.2)
|
||||||
|
|
||||||
|
**Markdown (Auszug)**
|
||||||
|
# Relations Showcase
|
||||||
|
…
|
||||||
|
Wir nutzen [[rel:depends_on Qdrant Vektordatenbank]] für die Suche.
|
||||||
|
…
|
||||||
|
Siehe auch: [[Embeddings 101]]
|
||||||
|
|
||||||
|
> [!edge] related_to: [[Vector DB Basics]]
|
||||||
|
|
||||||
|
**Ergebnis (fachlich)**
|
||||||
|
1. `depends_on(Chunk→Qdrant)` mit `rule_id=inline:rel`, `confidence≈0.95`.
|
||||||
|
2. `references(Chunk→Embeddings 101)` mit `rule_id=explicit:wikilink`, `confidence=1.0`.
|
||||||
|
3. `related_to(Chunk→Vector DB Basics)` via Callout; `rule_id=callout:edge`, `confidence≈0.90`.
|
||||||
|
4. **Typ-Defaults:** Falls die Note vom Typ `project` ist, entstehen zusätzlich `depends_on`-Kanten zu den Zielen aus (2) und (3).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 15) Referenzen (Projektdateien & Leitlinien)
|
||||||
|
|
||||||
|
- Import-Pipeline & Registry-Auflösung: `scripts/import_markdown.py`.
|
||||||
|
- Kantenbildung (V2-Logic): `app/core/derive_edges.py`.
|
||||||
|
- Typ-Registry: `config/types.yaml` & `TYPE_REGISTRY_MANUAL.md`.
|
||||||
|
- Retriever-Scoring & Explanation: `app/core/retriever.py`.
|
||||||
|
- Persönlichkeit & Chat: `config/prompts.yaml` & `app/routers/chat.py`.
|
||||||
|
- Decision Engine: `config/decision_engine.yaml`.
|
||||||
|
- Logging Service: `app/services/feedback_service.py`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 16) Workpackage Status (v2.3.2)
|
||||||
|
|
||||||
|
Aktueller Implementierungsstand der Module.
|
||||||
|
|
||||||
|
| WP | Titel | Status | Anmerkung |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| **WP01** | Knowledge Design | 🟢 Live | Typen, Frontmatter definiert. |
|
||||||
|
| **WP02** | Chunking Strategy | 🟢 Live | Profilbasiertes Chunking aktiv. |
|
||||||
|
| **WP03** | Edge Logic / Import | 🟢 Live | Neue Importer-Pipeline mit Provenance. |
|
||||||
|
| **WP04a**| Retriever Scoring | 🟢 Live | Hybrider Score (Semantik + Graph). |
|
||||||
|
| **WP04b**| Explanation Layer | 🟢 Live | API liefert Reasons & Breakdown. |
|
||||||
|
| **WP04c**| Feedback Loop | 🟢 Live | Logging (JSONL) & Traceability aktiv. |
|
||||||
|
| **WP05** | Persönlichkeit / Chat | 🟢 Live | RAG-Integration mit Context Enrichment. |
|
||||||
|
| **WP06** | Decision Engine | 🟢 Live | Intent-Router & Strategic Retrieval. |
|
||||||
|
| **WP07** | Interview Assistent | 🟡 Geplant | Dialog-Modus (Nächster Schritt). |
|
||||||
|
| **WP08** | Self-Tuning | 🔴 Geplant | Auto-Adjustment der Gewichte. |
|
||||||
|
| **WP10** | Chat Interface | 🟢 Live | Web-UI mit Feedback & Intents. |
|
||||||
|
| **WP10a**| GUI Evolution | 🔴 Geplant | Erweiterte Interaktion. |
|
||||||
@@ -0,0 +1,393 @@
|
|||||||
|
# Mindnet v2.2 – Technische Architektur
|
||||||
|
**Datei:** `docs/mindnet_technical_architecture_v2.2.md`
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** **FINAL** (Integrierter Stand WP01–WP10)
|
||||||
|
**Quellen:** `Programmplan_V2.2.md`, `Handbuch.md`, `chunking_strategy.md`, `wp04_retriever_scoring.md`.
|
||||||
|
|
||||||
|
> **Ziel dieses Dokuments:**
|
||||||
|
> Vollständige Beschreibung der technischen Architektur inkl. Graph-Datenbank, Retrieval-Logik, der **neuen RAG-Komponenten (Decision Engine & Hybrid Router)** und dem **Frontend (Streamlit)**.
|
||||||
|
|
||||||
|
---
|
||||||
|
<details>
|
||||||
|
<summary>📖 <b>Inhaltsverzeichnis (Klicken zum Öffnen)</b></summary>
|
||||||
|
|
||||||
|
- [Mindnet v2.2 – Technische Architektur](#mindnet-v22--technische-architektur)
|
||||||
|
- [](#)
|
||||||
|
- [1. Systemüberblick](#1-systemüberblick)
|
||||||
|
- [1.1 Architektur-Zielbild](#11-architektur-zielbild)
|
||||||
|
- [1.2 Verzeichnisstruktur \& Komponenten (Post-WP10)](#12-verzeichnisstruktur--komponenten-post-wp10)
|
||||||
|
- [2. Datenmodell \& Collections (Qdrant)](#2-datenmodell--collections-qdrant)
|
||||||
|
- [2.1 Notes Collection (`<prefix>_notes`)](#21-notes-collection-prefix_notes)
|
||||||
|
- [2.2 Chunks Collection (`<prefix>_chunks`)](#22-chunks-collection-prefix_chunks)
|
||||||
|
- [2.3 Edges Collection (`<prefix>_edges`)](#23-edges-collection-prefix_edges)
|
||||||
|
- [3. Konfiguration](#3-konfiguration)
|
||||||
|
- [3.1 Typ-Registry (`config/types.yaml`)](#31-typ-registry-configtypesyaml)
|
||||||
|
- [3.2 Retriever-Config (`config/retriever.yaml`)](#32-retriever-config-configretrieveryaml)
|
||||||
|
- [3.3 Decision Engine (`config/decision_engine.yaml`)](#33-decision-engine-configdecision_engineyaml)
|
||||||
|
- [3.4 Prompts (`config/prompts.yaml`)](#34-prompts-configpromptsyaml)
|
||||||
|
- [3.5 Environment (`.env`)](#35-environment-env)
|
||||||
|
- [4. Import-Pipeline (Markdown → Qdrant)](#4-import-pipeline-markdown--qdrant)
|
||||||
|
- [4.1 Verarbeitungsschritte](#41-verarbeitungsschritte)
|
||||||
|
- [5. Retriever-Architektur \& Scoring](#5-retriever-architektur--scoring)
|
||||||
|
- [5.1 Betriebsmodi](#51-betriebsmodi)
|
||||||
|
- [5.2 Scoring-Formel (WP04a)](#52-scoring-formel-wp04a)
|
||||||
|
- [5.3 Explanation Layer (WP04b)](#53-explanation-layer-wp04b)
|
||||||
|
- [5.4 Graph-Expansion](#54-graph-expansion)
|
||||||
|
- [6. RAG \& Chat Architektur (WP06 Hybrid Router)](#6-rag--chat-architektur-wp06-hybrid-router)
|
||||||
|
- [6.1 Architektur-Pattern: Intent Router](#61-architektur-pattern-intent-router)
|
||||||
|
- [6.2 Schritt 1: Intent Detection (Hybrid)](#62-schritt-1-intent-detection-hybrid)
|
||||||
|
- [6.3 Schritt 2: Strategy Resolution (Late Binding)](#63-schritt-2-strategy-resolution-late-binding)
|
||||||
|
- [6.4 Schritt 3: Multi-Stage Retrieval](#64-schritt-3-multi-stage-retrieval)
|
||||||
|
- [6.5 Schritt 4: Generation \& Response](#65-schritt-4-generation--response)
|
||||||
|
- [7. Frontend Architektur (WP10)](#7-frontend-architektur-wp10)
|
||||||
|
- [7.1 Kommunikation](#71-kommunikation)
|
||||||
|
- [7.2 Features \& UI-Logik](#72-features--ui-logik)
|
||||||
|
- [7.3 Deployment Ports](#73-deployment-ports)
|
||||||
|
- [8. Feedback \& Logging Architektur (WP04c)](#8-feedback--logging-architektur-wp04c)
|
||||||
|
- [8.1 Komponenten](#81-komponenten)
|
||||||
|
- [8.2 Log-Dateien](#82-log-dateien)
|
||||||
|
- [8.3 Traceability](#83-traceability)
|
||||||
|
- [9. Indizes \& Performance](#9-indizes--performance)
|
||||||
|
- [10. Offene Punkte / Known Limitations](#10-offene-punkte--known-limitations)
|
||||||
|
|
||||||
|
|
||||||
|
</details>
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Systemüberblick
|
||||||
|
|
||||||
|
### 1.1 Architektur-Zielbild
|
||||||
|
Mindnet ist ein **lokales RAG-System (Retrieval Augmented Generation)** mit Web-Interface.
|
||||||
|
1. **Source:** Markdown-Notizen in einem Vault (Obsidian-kompatibel).
|
||||||
|
2. **Pipeline:** Ein Python-Importer transformiert diese in **Notes**, **Chunks** und **Edges**.
|
||||||
|
3. **Storage:**
|
||||||
|
* **Qdrant:** Vektor-Datenbank für Graph und Semantik (Collections: notes, chunks, edges).
|
||||||
|
* **Local Files (JSONL):** Append-Only Logs für Feedback und Search-History (Data Flywheel).
|
||||||
|
4. **Backend:** Eine FastAPI-Anwendung stellt Endpunkte für **Semantische** und **Hybride Suche** sowie **Feedback** bereit.
|
||||||
|
5. **Frontend:** Streamlit-App (`ui.py`) für Interaktion und Visualisierung.
|
||||||
|
6. **Inference:** Lokales LLM (Ollama: Phi-3 Mini) für RAG-Chat und Antwortgenerierung.
|
||||||
|
|
||||||
|
Das System arbeitet **deterministisch** (stabile IDs) und ist **konfigurationsgetrieben** (`types.yaml`, `retriever.yaml`, `decision_engine.yaml`, `prompts.yaml`).
|
||||||
|
|
||||||
|
### 1.2 Verzeichnisstruktur & Komponenten (Post-WP10)
|
||||||
|
|
||||||
|
/mindnet/
|
||||||
|
├── app/
|
||||||
|
│ ├── main.py # FastAPI Einstiegspunkt
|
||||||
|
│ ├── core/
|
||||||
|
│ │ ├── qdrant.py # Client-Factory & Connection
|
||||||
|
│ │ ├── qdrant_points.py # Low-Level Point Operations (Upsert/Delete)
|
||||||
|
│ │ ├── note_payload.py # Bau der Note-Objekte
|
||||||
|
│ │ ├── chunk_payload.py # Bau der Chunk-Objekte
|
||||||
|
│ │ ├── chunker.py # Text-Zerlegung (Profiling)
|
||||||
|
│ │ ├── edges.py # Edge-Datenstrukturen
|
||||||
|
│ │ ├── derive_edges.py # Logik der Kantenableitung (WP03)
|
||||||
|
│ │ ├── graph_adapter.py # Subgraph & Reverse-Lookup (WP04b)
|
||||||
|
│ │ └── retriever.py # Scoring, Expansion & Explanation (WP04a/b)
|
||||||
|
│ ├── models/ # Pydantic DTOs
|
||||||
|
│ ├── routers/
|
||||||
|
│ │ ├── query.py # Such-Endpunkt
|
||||||
|
│ │ ├── chat.py # Hybrid Router & Decision Engine (WP06)
|
||||||
|
│ │ ├── feedback.py # Feedback-Endpunkt (WP04c)
|
||||||
|
│ │ └── ...
|
||||||
|
│ ├── services/
|
||||||
|
│ │ ├── llm_service.py # Ollama Client mit Timeout & Raw-Mode
|
||||||
|
│ │ ├── feedback_service.py # JSONL Logging (WP04c)
|
||||||
|
│ │ └── embeddings_client.py
|
||||||
|
│ ├── frontend/ # NEU (WP10)
|
||||||
|
│ └── ui.py # Streamlit Application
|
||||||
|
├── config/
|
||||||
|
│ ├── types.yaml # Typ-Definitionen (Import-Zeit)
|
||||||
|
│ ├── retriever.yaml # Scoring-Gewichte (Laufzeit)
|
||||||
|
│ ├── decision_engine.yaml # Strategien & Keywords (WP06)
|
||||||
|
│ └── prompts.yaml # LLM System-Prompts & Templates (WP06)
|
||||||
|
├── data/
|
||||||
|
│ └── logs/ # Lokale JSONL-Logs (WP04c)
|
||||||
|
├── scripts/
|
||||||
|
│ ├── import_markdown.py # Haupt-Importer CLI
|
||||||
|
│ ├── payload_dryrun.py # Diagnose: JSON-Generierung ohne DB
|
||||||
|
│ └── edges_full_check.py # Diagnose: Graph-Integrität
|
||||||
|
└── tests/ # Pytest Suite
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Datenmodell & Collections (Qdrant)
|
||||||
|
|
||||||
|
Das Datenmodell verteilt sich auf drei Collections, definiert durch `COLLECTION_PREFIX` (Standard: `mindnet`).
|
||||||
|
|
||||||
|
### 2.1 Notes Collection (`<prefix>_notes`)
|
||||||
|
Repräsentiert die Metadaten einer Datei.
|
||||||
|
* **Zweck:** Filterung, Metadaten-Haltung, Vererbung von Eigenschaften an Chunks.
|
||||||
|
* **Schema (Payload):**
|
||||||
|
|
||||||
|
| Feld | Datentyp | Beschreibung | Herkunft |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| `note_id` | Keyword | Stabile ID (UUIDv5 oder Slug). | Frontmatter |
|
||||||
|
| `title` | Text | Titel der Notiz. | Frontmatter |
|
||||||
|
| `type` | Keyword | Logischer Typ (z.B. `concept`). | `types.yaml` Resolver |
|
||||||
|
| `retriever_weight` | Float | Wichtigkeit im Retrieval. | `types.yaml` |
|
||||||
|
| `chunk_profile` | Keyword | Genutztes Chunking-Profil. | `types.yaml` |
|
||||||
|
| `edge_defaults` | List[Str] | Aktive Default-Relationen. | `types.yaml` |
|
||||||
|
| `tags` | List[Kw] | Tags zur Filterung. | Frontmatter |
|
||||||
|
| `updated` | Integer | Zeitstempel (z.B. YYYYMMDD...). | File Stats |
|
||||||
|
| `fulltext` | Text | Gesamter Inhalt (für Export/Rekonstruktion). | Body |
|
||||||
|
|
||||||
|
### 2.2 Chunks Collection (`<prefix>_chunks`)
|
||||||
|
Die atomaren Sucheinheiten.
|
||||||
|
* **Zweck:** Vektorsuche (Embeddings), Granulares Ergebnis.
|
||||||
|
* **Schema (Payload):**
|
||||||
|
|
||||||
|
| Feld | Datentyp | Beschreibung |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| `chunk_id` | Keyword | Deterministisch: `{note_id}#c{index:02d}`. |
|
||||||
|
| `note_id` | Keyword | Referenz zur Note. |
|
||||||
|
| `type` | Keyword | **Kopie des Note-Typs** (Denormalisiert für Filter). |
|
||||||
|
| `text` | Text | **Reiner Inhalt** (ohne Overlap). Anzeige-Text. |
|
||||||
|
| `window` | Text | **Kontext-Fenster** (mit Overlap). Embedding-Basis. |
|
||||||
|
| `ord` | Integer | Sortierreihenfolge (1..N). |
|
||||||
|
| `retriever_weight` | Float | Vererbt von Note. |
|
||||||
|
| `chunk_profile` | Keyword | Vererbt von Note. |
|
||||||
|
| `neighbors_prev` | List[Str] | ID des Vorgänger-Chunks. |
|
||||||
|
| `neighbors_next` | List[Str] | ID des Nachfolger-Chunks. |
|
||||||
|
|
||||||
|
### 2.3 Edges Collection (`<prefix>_edges`)
|
||||||
|
Gerichtete Kanten. Massiv erweitert in WP03 für Provenienz-Tracking.
|
||||||
|
* **Zweck:** Graph-Traversal, Kontext-Anreicherung.
|
||||||
|
* **Schema (Payload):**
|
||||||
|
|
||||||
|
| Feld | Datentyp | Beschreibung | Wertebereich (Bsp.) |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| `edge_id` | Keyword | Hash aus (src, dst, kind). | |
|
||||||
|
| `source_id` | Keyword | ID des Start-Chunks. | |
|
||||||
|
| `target_id` | Keyword | ID des Ziel-Chunks (oder Note-Titel). | |
|
||||||
|
| `note_id` | Keyword | Note, die diese Kante definiert. | |
|
||||||
|
| `kind` | Keyword | Art der Beziehung. | `references`, `depends_on`, `next` |
|
||||||
|
| `scope` | Keyword | Geltungsbereich. | Immer `"chunk"` (v2.2). |
|
||||||
|
| `rule_id` | Keyword | Herkunftsregel. | `explicit:wikilink`, `inline:rel` |
|
||||||
|
| `confidence` | Float | Vertrauenswürdigkeit (0.0-1.0). | 1.0, 0.95, 0.7 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Konfiguration
|
||||||
|
|
||||||
|
Die Logik ist ausgelagert in YAML-Dateien.
|
||||||
|
|
||||||
|
### 3.1 Typ-Registry (`config/types.yaml`)
|
||||||
|
Steuert den Import-Prozess.
|
||||||
|
* **Priorität:** Frontmatter > Pfad > Default.
|
||||||
|
* **Inhalt (Beispiel):**
|
||||||
|
|
||||||
|
types:
|
||||||
|
concept:
|
||||||
|
chunk_profile: medium
|
||||||
|
edge_defaults: ["references", "related_to"]
|
||||||
|
retriever_weight: 0.60
|
||||||
|
|
||||||
|
### 3.2 Retriever-Config (`config/retriever.yaml`)
|
||||||
|
Steuert das Scoring zur Laufzeit (WP04a).
|
||||||
|
* **Inhalt (Beispiel):**
|
||||||
|
|
||||||
|
scoring:
|
||||||
|
semantic_weight: 1.0
|
||||||
|
edge_weight: 0.7
|
||||||
|
centrality_weight: 0.5
|
||||||
|
|
||||||
|
### 3.3 Decision Engine (`config/decision_engine.yaml`)
|
||||||
|
**Neu in WP06:** Steuert den Intent-Router.
|
||||||
|
* Definiert Strategien (`DECISION`, `EMPATHY`, `CODING`, `FACT`).
|
||||||
|
* Definiert `trigger_keywords` und `inject_types` (Late Binding).
|
||||||
|
* Definiert LLM-Router-Settings (`llm_fallback_enabled`).
|
||||||
|
|
||||||
|
### 3.4 Prompts (`config/prompts.yaml`)
|
||||||
|
Steuert die LLM-Persönlichkeit und Templates.
|
||||||
|
* Enthält Templates für alle Strategien (z.B. `decision_template`, `empathy_template`, `technical_template`).
|
||||||
|
|
||||||
|
### 3.5 Environment (`.env`)
|
||||||
|
Erweiterung für LLM-Steuerung:
|
||||||
|
|
||||||
|
MINDNET_LLM_MODEL=phi3:mini
|
||||||
|
MINDNET_OLLAMA_URL=http://127.0.0.1:11434
|
||||||
|
MINDNET_LLM_TIMEOUT=300.0 # Neu: Erhöht für CPU-Inference Cold-Starts
|
||||||
|
MINDNET_DECISION_CONFIG="config/decision_engine.yaml"
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Import-Pipeline (Markdown → Qdrant)
|
||||||
|
|
||||||
|
Das Skript `scripts/import_markdown.py` orchestriert den Prozess.
|
||||||
|
|
||||||
|
### 4.1 Verarbeitungsschritte
|
||||||
|
|
||||||
|
1. **Discovery & Parsing:**
|
||||||
|
* Einlesen der `.md` Dateien. Hash-Vergleich (Body/Frontmatter) zur Erkennung von Änderungen.
|
||||||
|
2. **Typauflösung:**
|
||||||
|
* Laden der `types.yaml`. Bestimmen des effektiven Typs und der `edge_defaults`.
|
||||||
|
3. **Chunking:**
|
||||||
|
* Zerlegung via `chunker.py` basierend auf `chunk_profile` (z.B. `by_heading`, `short`, `long`).
|
||||||
|
* Trennung von `text` (Kern) und `window` (Embedding-Kontext).
|
||||||
|
4. **Kantenableitung (Edge Derivation):**
|
||||||
|
Die `derive_edges.py` erzeugt Kanten in strikter Reihenfolge:
|
||||||
|
1. **Inline-Edges:** `[[rel:depends_on X]]` → `kind=depends_on`, `rule_id=inline:rel`, `conf=0.95`.
|
||||||
|
2. **Callout-Edges:** `> [!edge] related_to: [[X]]` → `kind=related_to`, `rule_id=callout:edge`, `conf=0.90`.
|
||||||
|
3. **Explizite Referenzen:** `[[X]]` → `kind=references`, `rule_id=explicit:wikilink`, `conf=1.0`.
|
||||||
|
4. **Typ-Defaults:** Für jede Referenz werden Zusatzkanten gemäß `edge_defaults` erzeugt (z.B. `project` -> `depends_on`). `rule_id=edge_defaults:...`, `conf=0.7`.
|
||||||
|
5. **Struktur:** `belongs_to`, `next`, `prev` (automatisch).
|
||||||
|
5. **Upsert:**
|
||||||
|
* Schreiben in Qdrant. Nutzung von `--purge-before-upsert` für saubere Updates.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Retriever-Architektur & Scoring
|
||||||
|
|
||||||
|
Der Retriever (`app/core/retriever.py`) unterstützt zwei Modi. Für den Chat wird **zwingend** der Hybrid-Modus genutzt.
|
||||||
|
|
||||||
|
### 5.1 Betriebsmodi
|
||||||
|
* **Semantic:** Reine Vektorsuche. Schnell.
|
||||||
|
* **Hybrid:** Vektorsuche + Graph-Expansion (Tiefe N) + Re-Ranking.
|
||||||
|
|
||||||
|
### 5.2 Scoring-Formel (WP04a)
|
||||||
|
Die Relevanzberechnung ist nun eine gewichtete Summe:
|
||||||
|
|
||||||
|
$$
|
||||||
|
TotalScore = (W_{sem} \cdot S_{sem} \cdot \max(W_{type}, 0)) + (W_{edge} \cdot B_{edge}) + (W_{cent} \cdot B_{cent})
|
||||||
|
$$
|
||||||
|
|
||||||
|
* **Komponenten:**
|
||||||
|
* $S_{sem}$: Semantic Score (Cosine Similarity).
|
||||||
|
* $W_{type}$: `retriever_weight` (aus Note/Chunk Payload).
|
||||||
|
* $B_{edge}$: Edge-Bonus (Summe der Konfidenzen eingehender relevanter Kanten).
|
||||||
|
* $B_{cent}$: Centrality-Bonus (im lokalen Subgraphen).
|
||||||
|
* **Gewichte ($W$):** Stammen aus `retriever.yaml`.
|
||||||
|
|
||||||
|
### 5.3 Explanation Layer (WP04b)
|
||||||
|
Der Retriever kann Ergebnisse erklären (`explain=True`).
|
||||||
|
* **Logik:**
|
||||||
|
* Berechnung des `ScoreBreakdown` (Anteile von Semantik, Graph, Typ).
|
||||||
|
* Analyse des lokalen Subgraphen mittels `graph_adapter.py`.
|
||||||
|
* **Incoming Edges (Authority):** Wer zeigt auf diesen Treffer? (z.B. "Referenziert von...")
|
||||||
|
* **Outgoing Edges (Hub):** Worauf zeigt dieser Treffer? (z.B. "Verweist auf...")
|
||||||
|
* **Output:** `QueryHit` enthält ein `explanation` Objekt mit menschenlesbaren `reasons` und `related_edges`.
|
||||||
|
|
||||||
|
### 5.4 Graph-Expansion
|
||||||
|
Der Hybrid-Modus lädt dynamisch die Nachbarschaft der Top-K Vektor-Treffer ("Seeds") über `graph_adapter.expand`. Dies baut einen temporären `NetworkX`-artigen Graphen im Speicher (Klasse `Subgraph`), auf dem Boni berechnet werden.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. RAG & Chat Architektur (WP06 Hybrid Router)
|
||||||
|
|
||||||
|
Der Flow für eine Chat-Anfrage (`/chat`) wurde in WP06 auf eine **Configuration-Driven Architecture** umgestellt. Der `ChatRouter` (`app/routers/chat.py`) fungiert als zentraler Dispatcher.
|
||||||
|
|
||||||
|
### 6.1 Architektur-Pattern: Intent Router
|
||||||
|
Die Behandlung einer Anfrage ist nicht mehr hartkodiert, sondern wird dynamisch zur Laufzeit entschieden.
|
||||||
|
* **Input:** User Message.
|
||||||
|
* **Config:** `config/decision_engine.yaml` (Strategien & Keywords).
|
||||||
|
* **Komponenten:**
|
||||||
|
* **Fast Path:** Keyword Matching (CPU-schonend).
|
||||||
|
* **Slow Path:** LLM-basierter Semantic Router (für subtile Intents).
|
||||||
|
|
||||||
|
### 6.2 Schritt 1: Intent Detection (Hybrid)
|
||||||
|
Der Router ermittelt die Absicht (`Intent`) des Nutzers.
|
||||||
|
1. **Keyword Scan (Fast Path):**
|
||||||
|
* Iteration über alle Strategien in `decision_engine.yaml`.
|
||||||
|
* Prüfung auf `trigger_keywords`.
|
||||||
|
* **Best Match:** Bei mehreren Treffern gewinnt das längste/spezifischste Keyword (Robustheit gegen Shadowing).
|
||||||
|
2. **LLM Fallback (Slow Path):**
|
||||||
|
* Nur aktiv, wenn `llm_fallback_enabled: true`.
|
||||||
|
* Greift, wenn keine Keywords gefunden wurden.
|
||||||
|
* Sendet die Query an das LLM mit einem Klassifizierungs-Prompt (`llm_router_prompt`).
|
||||||
|
* Ergebnis: `EMPATHY`, `DECISION`, `CODING` oder `FACT` (Default).
|
||||||
|
|
||||||
|
### 6.3 Schritt 2: Strategy Resolution (Late Binding)
|
||||||
|
Basierend auf dem Intent lädt der Router die Parameter:
|
||||||
|
* **inject_types:** Liste der Notiz-Typen, die erzwungen werden sollen (z.B. `["value", "goal"]` bei `DECISION`).
|
||||||
|
* **prompt_template:** Schlüssel für das Template in `prompts.yaml` (z.B. `decision_template`, `empathy_template`).
|
||||||
|
|
||||||
|
### 6.4 Schritt 3: Multi-Stage Retrieval
|
||||||
|
1. **Primary Retrieval:** Hybride Suche nach der User-Query (findet Fakten).
|
||||||
|
2. **Strategic Retrieval (Conditional):** Wenn `inject_types` definiert sind, erfolgt eine zweite Suche, die explizit auf diese Typen filtert.
|
||||||
|
3. **Merge:** Ergebnisse werden dedupliziert zusammengeführt.
|
||||||
|
|
||||||
|
### 6.5 Schritt 4: Generation & Response
|
||||||
|
* **Context Enrichment:** Metadaten (`[VALUE]`, `[GOAL]`, `[SCORE]`) werden in den Context-String injiziert, um dem LLM die Rolle des Textstücks zu signalisieren.
|
||||||
|
* **Templating:** Das LLM erhält den Prompt basierend auf dem gewählten Template.
|
||||||
|
* **Execution:** Der `LLMService` führt den Call aus. Ein konfigurierbarer Timeout (`MINDNET_LLM_TIMEOUT`) fängt Cold-Start-Verzögerungen auf CPU-Hardware ab.
|
||||||
|
* **Response:** Rückgabe enthält Antworttext, Quellenliste und den erkannten `intent` (für Debugging/Frontend).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Frontend Architektur (WP10)
|
||||||
|
|
||||||
|
Das Frontend ist eine **Streamlit-Anwendung**, die als separater Prozess läuft und via HTTP mit dem Backend kommuniziert.
|
||||||
|
|
||||||
|
### 7.1 Kommunikation
|
||||||
|
* **Backend-URL:** Konfiguriert via `MINDNET_API_URL` (Default: `http://localhost:8002`).
|
||||||
|
* **Endpoints:** Nutzt `/chat` für Interaktion und `/feedback` für Bewertungen.
|
||||||
|
* **Resilienz:** Das Frontend implementiert eigene Timeouts (`MINDNET_API_TIMEOUT`, Default 300s), um lange Wartezeiten lokaler LLMs abzufangen.
|
||||||
|
|
||||||
|
### 7.2 Features & UI-Logik
|
||||||
|
* **State Management:** Session-State speichert Chat-Verlauf und `query_id` für Feedback.
|
||||||
|
* **Visualisierung:**
|
||||||
|
* **Intent Badges:** Zeigt Router-Entscheidung (`DECISION`, `FACT`, etc.) und Quelle (`Keyword` vs. `LLM`).
|
||||||
|
* **Source Expanders:** Zeigt verwendete Chunks inkl. Score und "Why"-Explanation.
|
||||||
|
* **Sidebar:** Zeigt Suchhistorie (Log-basiert) und Konfiguration.
|
||||||
|
* **Feedback:**
|
||||||
|
* Global (Antwort): 1-5 Sterne Rating.
|
||||||
|
* Granular (Quellen): Faces-Rating (Mapped auf 1-5 Score).
|
||||||
|
|
||||||
|
### 7.3 Deployment Ports
|
||||||
|
Zur sauberen Trennung von Prod und Dev laufen Frontend und Backend auf dedizierten Ports:
|
||||||
|
|
||||||
|
| Umgebung | Backend (FastAPI) | Frontend (Streamlit) |
|
||||||
|
| :--- | :--- | :--- |
|
||||||
|
| **Production** | 8001 | 8501 |
|
||||||
|
| **Development** | 8002 | 8502 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Feedback & Logging Architektur (WP04c)
|
||||||
|
|
||||||
|
Mindnet implementiert ein "Data Flywheel" zur späteren Optimierung (Self-Tuning).
|
||||||
|
|
||||||
|
### 8.1 Komponenten
|
||||||
|
* **Feedback Service (`app/services/feedback_service.py`):** Kapselt die Schreibzugriffe.
|
||||||
|
* **Storage:** Lokales Dateisystem (`data/logs/`), Format JSONL (Line-delimited JSON).
|
||||||
|
|
||||||
|
### 8.2 Log-Dateien
|
||||||
|
1. **`search_history.jsonl`**:
|
||||||
|
* Speichert jede Anfrage an `/query` und `/chat`.
|
||||||
|
* Enthält: `query_id`, `query_text`, `timestamp`, `hits` (inkl. `score_breakdown` Snapshots).
|
||||||
|
* Zweck: Trainingsdaten ("Was hat das System gesehen?").
|
||||||
|
2. **`feedback.jsonl`**:
|
||||||
|
* Speichert User-Reaktionen an `/feedback`.
|
||||||
|
* Enthält: `query_id`, `node_id`, `score` (1-5), `comment`.
|
||||||
|
* **Granularität:** Kann sich auf `generated_answer` (Global) oder eine spezifische `node_id` (Quelle) beziehen.
|
||||||
|
* Zweck: Labels ("War es gut?").
|
||||||
|
|
||||||
|
### 8.3 Traceability
|
||||||
|
Die `query_id` (UUIDv4) wird im `/query` Response generiert und muss vom Client beim `/feedback` Aufruf mitgesendet werden. Dies ermöglicht den Join zwischen Snapshot und Bewertung.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Indizes & Performance
|
||||||
|
|
||||||
|
Damit Qdrant performant bleibt, sind Payload-Indizes essenziell.
|
||||||
|
|
||||||
|
**Erforderliche Indizes:**
|
||||||
|
* **Notes:** `note_id`, `type`, `tags`.
|
||||||
|
* **Chunks:** `note_id`, `chunk_id`, `type`.
|
||||||
|
* **Edges:** `source_id`, `target_id`, `kind`, `scope`, `note_id`.
|
||||||
|
|
||||||
|
Validierung erfolgt über `tests/ensure_indexes_and_show.py`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Offene Punkte / Known Limitations
|
||||||
|
|
||||||
|
1. **Multi-Target Inline-Relations:**
|
||||||
|
* `rel: similar_to [[A]] [[B]]` wird aktuell parser-seitig nicht unterstützt.
|
||||||
|
* Workaround: Zwei separate Inline-Links `[[rel:similar_to A]] [[rel:similar_to B]]`.
|
||||||
|
2. **Unresolved Targets:**
|
||||||
|
* Kanten zu Notizen, die noch nicht existieren, werden mit `target_id="Titel"` angelegt.
|
||||||
|
* Heilung durch `scripts/resolve_unresolved_references.py` möglich.
|
||||||
|
3. **Vektor-Konfiguration für Edges:**
|
||||||
|
* `mindnet_edges` hat aktuell keine Vektoren (`vectors = null`). Eine semantische Suche *auf Kanten* ist noch nicht möglich.
|
||||||
@@ -0,0 +1,246 @@
|
|||||||
|
# mindnet v2.2 – Pipeline Playbook
|
||||||
|
**Datei:** `docs/mindnet_pipeline_playbook_v2.2.md`
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** **FINAL** (Inkl. WP10 Frontend)
|
||||||
|
**Quellen:** `mindnet_v2_implementation_playbook.md`, `Handbuch.md`, `chunking_strategy.md`, `docs_mindnet_retriever.md`, `mindnet_admin_guide_v2.2.md`.
|
||||||
|
|
||||||
|
---
|
||||||
|
<details>
|
||||||
|
<summary>📖 <b>Inhaltsverzeichnis (Klicken zum Öffnen)</b></summary>
|
||||||
|
|
||||||
|
- [mindnet v2.2 – Pipeline Playbook](#mindnet-v22--pipeline-playbook)
|
||||||
|
- [](#)
|
||||||
|
- [1. Zweck \& Einordnung](#1-zweck--einordnung)
|
||||||
|
- [2. Die Import-Pipeline (Runbook)](#2-die-import-pipeline-runbook)
|
||||||
|
- [2.1 Der 12-Schritte-Prozess](#21-der-12-schritte-prozess)
|
||||||
|
- [2.2 Standard-Betrieb (Inkrementell)](#22-standard-betrieb-inkrementell)
|
||||||
|
- [2.3 Deployment \& Restart (Systemd)](#23-deployment--restart-systemd)
|
||||||
|
- [2.4 Full Rebuild (Clean Slate)](#24-full-rebuild-clean-slate)
|
||||||
|
- [3. Chunking \& Payload-Aufbau](#3-chunking--payload-aufbau)
|
||||||
|
- [3.1 Chunk-Profile](#31-chunk-profile)
|
||||||
|
- [3.2 Payload-Felder](#32-payload-felder)
|
||||||
|
- [4. Edge-Erzeugung (Die V2-Logik)](#4-edge-erzeugung-die-v2-logik)
|
||||||
|
- [4.1 Prioritäten \& Provenance](#41-prioritäten--provenance)
|
||||||
|
- [4.2 Typ-Defaults](#42-typ-defaults)
|
||||||
|
- [5. Retriever, Chat \& Generation (RAG Pipeline)](#5-retriever-chat--generation-rag-pipeline)
|
||||||
|
- [5.1 Retrieval (Hybrid)](#51-retrieval-hybrid)
|
||||||
|
- [5.2 Intent Router (WP06)](#52-intent-router-wp06)
|
||||||
|
- [5.3 Context Enrichment](#53-context-enrichment)
|
||||||
|
- [5.4 Generation (LLM)](#54-generation-llm)
|
||||||
|
- [6. Feedback \& Lernen (WP04c)](#6-feedback--lernen-wp04c)
|
||||||
|
- [7. Quality Gates \& Tests](#7-quality-gates--tests)
|
||||||
|
- [7.1 Pflicht-Tests vor Commit](#71-pflicht-tests-vor-commit)
|
||||||
|
- [7.2 Smoke-Test (E2E)](#72-smoke-test-e2e)
|
||||||
|
- [8. Ausblick \& Roadmap (Technische Skizzen)](#8-ausblick--roadmap-technische-skizzen)
|
||||||
|
- [8.1 WP-08: Self-Tuning (Skizze)](#81-wp-08-self-tuning-skizze)
|
||||||
|
- [16. Workpackage Status (v2.3.2)](#16-workpackage-status-v232)
|
||||||
|
|
||||||
|
</details>
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Zweck & Einordnung
|
||||||
|
|
||||||
|
Dieses Playbook ist das zentrale operative Handbuch für die **mindnet-Pipeline**. Es beschreibt, wie Daten vom Markdown-Vault in den Wissensgraphen (Qdrant) gelangen, wie der Retriever betrieben wird und wie die **RAG-Generierung** (inkl. Decision Engine) funktioniert.
|
||||||
|
|
||||||
|
**Zielgruppe:** Dev/Ops, Tech-Leads.
|
||||||
|
**Scope:**
|
||||||
|
* **Ist-Stand (WP01–WP10):** Import, Chunking, Edge-Erzeugung, Hybrider Retriever, RAG-Chat (Hybrid Router), Feedback Loop, Frontend.
|
||||||
|
* **Roadmap (Ausblick):** Technische Skizze für Self-Tuning (WP08).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Die Import-Pipeline (Runbook)
|
||||||
|
|
||||||
|
Der Import ist der kritischste Prozess ("Data Ingestion"). Er muss **deterministisch** und **idempotent** sein. Wir nutzen `scripts/import_markdown.py` als zentralen Entrypoint.
|
||||||
|
|
||||||
|
### 2.1 Der 12-Schritte-Prozess
|
||||||
|
Gemäß WP03-Spezifikation läuft der Import intern wie folgt ab:
|
||||||
|
|
||||||
|
1. **Markdown lesen:** Rekursives Scannen des Vaults.
|
||||||
|
2. **Frontmatter extrahieren:** Validierung von Pflichtfeldern (`id`, `type`, `title`).
|
||||||
|
3. **Typauflösung:** Bestimmung des `type` via `types.yaml` (Prio: Frontmatter > Pfad > Default).
|
||||||
|
4. **Note-Payload generieren:** Erstellen des JSON-Objekts für `mindnet_notes`.
|
||||||
|
5. **Chunking anwenden:** Zerlegung des Textes basierend auf dem `chunk_profile` des Typs.
|
||||||
|
6. **Inline-Kanten finden:** Parsing von `[[rel:...]]` im Fließtext.
|
||||||
|
7. **Callout-Kanten finden:** Parsing von `> [!edge]` Blöcken.
|
||||||
|
8. **Default-Edges erzeugen:** Anwendung der `edge_defaults` aus der Typ-Registry.
|
||||||
|
9. **Strukturkanten erzeugen:** `belongs_to` (Chunk->Note), `next`/`prev` (Sequenz).
|
||||||
|
10. **Chunks upserten:** Schreiben in Qdrant (`mindnet_chunks`).
|
||||||
|
11. **Edges upserten:** Schreiben in Qdrant (`mindnet_edges`).
|
||||||
|
12. **Diagnose:** Automatischer Check der Integrität nach dem Lauf.
|
||||||
|
|
||||||
|
### 2.2 Standard-Betrieb (Inkrementell)
|
||||||
|
Für regelmäßige Updates (z.B. Cronjob). Erkennt Änderungen via Hash.
|
||||||
|
|
||||||
|
export QDRANT_URL="http://localhost:6333"
|
||||||
|
export COLLECTION_PREFIX="mindnet"
|
||||||
|
|
||||||
|
# Import starten (Apply = Schreiben, Purge = Sauberer Upsert)
|
||||||
|
# Nutzt das Venv der Produktionsumgebung
|
||||||
|
/home/llmadmin/mindnet/.venv/bin/python3 -m scripts.import_markdown \
|
||||||
|
--vault ./vault \
|
||||||
|
--prefix "$COLLECTION_PREFIX" \
|
||||||
|
--apply \
|
||||||
|
--purge-before-upsert \
|
||||||
|
--sync-deletes
|
||||||
|
|
||||||
|
* `--apply`: Ohne dieses Flag läuft ein Dry-Run (nur Simulation).
|
||||||
|
* `--purge-before-upsert`: Löscht vor dem Schreiben einer Note ihre alten Chunks/Edges. Essentiell, um "Geister-Chunks" zu vermeiden, wenn Text gekürzt wurde.
|
||||||
|
* `--sync-deletes`: Entfernt Notizen aus Qdrant, die im Vault gelöscht wurden.
|
||||||
|
|
||||||
|
### 2.3 Deployment & Restart (Systemd)
|
||||||
|
Nach einem Import oder Code-Update müssen die API-Prozesse neu gestartet werden.
|
||||||
|
|
||||||
|
# Neustart des Produktions-Services (API + UI)
|
||||||
|
sudo systemctl restart mindnet-prod
|
||||||
|
sudo systemctl restart mindnet-ui-prod
|
||||||
|
|
||||||
|
# Prüfung
|
||||||
|
sudo systemctl status mindnet-prod
|
||||||
|
|
||||||
|
### 2.4 Full Rebuild (Clean Slate)
|
||||||
|
Notwendig bei Änderungen an `types.yaml` (z.B. neue Chunk-Größen) oder Embedding-Modellen.
|
||||||
|
|
||||||
|
# 1. Qdrant Collections löschen und neu anlegen (Wipe inkl. Schema)
|
||||||
|
python3 -m scripts.reset_qdrant --mode wipe --prefix "mindnet" --yes
|
||||||
|
|
||||||
|
# 2. Vollständiger Import aller Dateien
|
||||||
|
python3 -m scripts.import_markdown --vault ./vault --prefix "mindnet" --apply
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Chunking & Payload-Aufbau
|
||||||
|
|
||||||
|
Das Chunking ist profilbasiert und typgesteuert.
|
||||||
|
|
||||||
|
### 3.1 Chunk-Profile
|
||||||
|
In `types.yaml` definiert. Standard-Profile (in `chunk_config.py` implementiert):
|
||||||
|
* `short`: Max 128 Tokens (z.B. für Logs, Chats).
|
||||||
|
* `medium`: Max 256 Tokens (z.B. für Konzepte).
|
||||||
|
* `long`: Max 512 Tokens (z.B. für Essays, Projekte).
|
||||||
|
* `by_heading`: Trennt strikt an Überschriften.
|
||||||
|
|
||||||
|
### 3.2 Payload-Felder
|
||||||
|
Jeder Chunk erhält zwei Text-Felder:
|
||||||
|
* `text`: Der reine Inhalt des Chunks (ohne Overlap). Wird dem Nutzer angezeigt.
|
||||||
|
* `window`: Der Inhalt plus Overlap zu Vorgänger/Nachfolger. Wird für das Embedding genutzt (besserer Kontext).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Edge-Erzeugung (Die V2-Logik)
|
||||||
|
|
||||||
|
In v2.2 entstehen Kanten nach strenger Priorität.
|
||||||
|
|
||||||
|
### 4.1 Prioritäten & Provenance
|
||||||
|
Der Importer setzt `provenance`, `rule_id` und `confidence` automatisch:
|
||||||
|
|
||||||
|
| Priorität | Quelle | Syntax (Bsp.) | Rule ID | Confidence |
|
||||||
|
| :--- | :--- | :--- | :--- | :--- |
|
||||||
|
| **1** | Inline | `[[rel:depends_on X]]` | `inline:rel` | ~0.95 |
|
||||||
|
| **2** | Callout | `> [!edge] related_to: [[X]]` | `callout:edge` | ~0.90 |
|
||||||
|
| **3** | Wikilink | `[[X]]` | `explicit:wikilink` | 1.00 |
|
||||||
|
| **4** | Default | *(via types.yaml)* | `edge_defaults:...` | ~0.70 |
|
||||||
|
| **5** | Struktur | *(automatisch)* | `structure:...` | 1.00 |
|
||||||
|
|
||||||
|
### 4.2 Typ-Defaults
|
||||||
|
Wenn in `types.yaml` für einen Typ `edge_defaults` definiert sind, werden diese **additiv** zu expliziten Links erzeugt.
|
||||||
|
* *Beispiel:* Note Typ `project` verlinkt `[[Tool A]]`.
|
||||||
|
* *Ergebnis:* Kante `references` (explizit) UND Kante `depends_on` (Default).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Retriever, Chat & Generation (RAG Pipeline)
|
||||||
|
|
||||||
|
Der Datenfluss endet nicht beim Finden. Er geht weiter bis zur Antwort.
|
||||||
|
|
||||||
|
### 5.1 Retrieval (Hybrid)
|
||||||
|
Der `/chat` Endpunkt nutzt **Hybrid Retrieval** (Semantic + Graph), um auch logisch verbundene, aber textlich unterschiedliche Notizen zu finden (z.B. Decisions zu einem Projekt).
|
||||||
|
|
||||||
|
### 5.2 Intent Router (WP06)
|
||||||
|
Der Request durchläuft den **Hybrid Router**:
|
||||||
|
1. **Fast Path:** Prüfung auf `trigger_keywords` aus `decision_engine.yaml`.
|
||||||
|
2. **Slow Path:** Falls kein Keyword matched und `llm_fallback_enabled=true`, klassifiziert das LLM den Intent (`DECISION`, `EMPATHY`, `FACT`, `CODING`).
|
||||||
|
3. **Result:** Auswahl der Strategie und der `inject_types` (z.B. Values & Goals).
|
||||||
|
|
||||||
|
### 5.3 Context Enrichment
|
||||||
|
Der Router (`chat.py`) reichert die gefundenen Chunks mit Metadaten an:
|
||||||
|
* **Typ-Injection:** `[DECISION]`, `[PROJECT]`.
|
||||||
|
* **Reasoning-Infos:** `(Score: 0.75)`.
|
||||||
|
* **Zweck:** Ermöglicht kleinen Modellen (Phi-3) das Erkennen von logischen Rollen ("Warum?" vs "Was?").
|
||||||
|
|
||||||
|
### 5.4 Generation (LLM)
|
||||||
|
* **Engine:** Ollama (lokal).
|
||||||
|
* **Modell:** `phi3:mini` (Standard).
|
||||||
|
* **Prompting:** Template wird basierend auf Intent gewählt (`decision_template` vs `empathy_template`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Feedback & Lernen (WP04c)
|
||||||
|
|
||||||
|
Das System schreibt kontinuierlich Logs ("Data Flywheel"):
|
||||||
|
* `data/logs/search_history.jsonl`: Trainingsdaten (Query + Ergebnisse + Breakdown).
|
||||||
|
* `data/logs/feedback.jsonl`: Labels (User-Rating zur `query_id`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Quality Gates & Tests
|
||||||
|
|
||||||
|
Diese Tests garantieren die Stabilität der Pipeline.
|
||||||
|
|
||||||
|
### 7.1 Pflicht-Tests vor Commit
|
||||||
|
|
||||||
|
1. **Payload Dryrun (Schema-Check):**
|
||||||
|
Simuliert Import, prüft JSON-Schema Konformität.
|
||||||
|
|
||||||
|
python3 -m scripts.payload_dryrun --vault ./test_vault
|
||||||
|
|
||||||
|
2. **Full Edge Check (Graph-Integrität):**
|
||||||
|
Prüft Invarianten (z.B. `next` muss reziprok zu `prev` sein).
|
||||||
|
|
||||||
|
python3 -m scripts.edges_full_check
|
||||||
|
|
||||||
|
### 7.2 Smoke-Test (E2E)
|
||||||
|
Prüft am laufenden System (Prod oder Dev), ob Semantik, Graph und Feedback funktionieren.
|
||||||
|
|
||||||
|
# Retriever Test
|
||||||
|
python scripts/test_retriever_smoke.py --mode hybrid --top-k 5
|
||||||
|
|
||||||
|
# Decision Engine Test (WP06)
|
||||||
|
python tests/test_wp06_decision.py -p 8002 -e EMPATHY -q "Alles ist grau"
|
||||||
|
|
||||||
|
# Feedback Test
|
||||||
|
python tests/test_feedback_smoke.py --url http://localhost:8001/query
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Ausblick & Roadmap (Technische Skizzen)
|
||||||
|
|
||||||
|
Wie entwickeln wir die Pipeline weiter?
|
||||||
|
|
||||||
|
### 8.1 WP-08: Self-Tuning (Skizze)
|
||||||
|
**Ziel:** Die Gewichte in `retriever.yaml` basierend auf `feedback.jsonl` optimieren.
|
||||||
|
**Ansatz:** Ein Offline-Learning-Skript `scripts/optimize_weights.py`.
|
||||||
|
1. **Load:** Liest `search_history.jsonl` und joint mit `feedback.jsonl` via `query_id`.
|
||||||
|
2. **Analyze:** Korrelation Scores vs. User-Rating.
|
||||||
|
3. **Optimize:** Vorschlag neuer Gewichte für `retriever.yaml`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 16. Workpackage Status (v2.3.2)
|
||||||
|
|
||||||
|
Aktueller Implementierungsstand der Module.
|
||||||
|
|
||||||
|
| WP | Titel | Status | Anmerkung |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| **WP01** | Knowledge Design | 🟢 Live | Typen, Frontmatter definiert. |
|
||||||
|
| **WP02** | Chunking Strategy | 🟢 Live | Profilbasiertes Chunking aktiv. |
|
||||||
|
| **WP03** | Edge Logic / Import | 🟢 Live | Neue Importer-Pipeline mit Provenance. |
|
||||||
|
| **WP04a**| Retriever Scoring | 🟢 Live | Hybrider Score (Semantik + Graph). |
|
||||||
|
| **WP04b**| Explanation Layer | 🟢 Live | API liefert Reasons & Breakdown. |
|
||||||
|
| **WP04c**| Feedback Loop | 🟢 Live | Logging (JSONL) & Traceability aktiv. |
|
||||||
|
| **WP05** | Persönlichkeit / Chat | 🟢 Live | RAG-Integration mit Context Enrichment. |
|
||||||
|
| **WP06** | Decision Engine | 🟢 Live | Hybrid Router, Strategic Retrieval. |
|
||||||
|
| **WP07** | Interview Assistent | 🟡 Geplant | Nächster Schritt (Dialog-Modus). |
|
||||||
|
| **WP08** | Self-Tuning | 🔴 Geplant | Auto-Adjustment der Gewichte. |
|
||||||
|
| **WP10** | Chat Interface | 🟢 Live | Streamlit Web-UI mit Feedback & Intents. |
|
||||||
|
| **WP10a**| GUI Evolution | 🔴 Geplant | Draft-Editor & Interaktionstools. |
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
# Mindnet v2.2 – User Guide
|
||||||
|
**Datei:** `docs/mindnet_user_guide_v2.2.md`
|
||||||
|
**Stand:** 2025-12-10
|
||||||
|
**Status:** **FINAL** (Inkl. RAG & Web-Interface)
|
||||||
|
**Quellen:** `knowledge_design.md`, `wp04_retriever_scoring.md`, `Programmplan_V2.2.md`, `Handbuch.md`.
|
||||||
|
|
||||||
|
> **Willkommen bei Mindnet.**
|
||||||
|
> Dies ist dein persönliches Wissensnetzwerk und dein Digitaler Zwilling. Es hilft dir beim Erinnern, beim Entscheiden und beim Reflektieren.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Was Mindnet für dich tut
|
||||||
|
|
||||||
|
Mindnet ist ein **assoziatives Gedächtnis** mit Persönlichkeit. Es unterscheidet sich von einer reinen Suche (wie Google) dadurch, dass es **kontextsensitiv** agiert.
|
||||||
|
|
||||||
|
### 1.1 Das Gedächtnis (Der Graph)
|
||||||
|
Wenn du nach "Projekt Alpha" suchst, findet Mindnet nicht nur das Dokument. Es findet auch:
|
||||||
|
* **Abhängigkeiten:** "Technologie X wird benötigt".
|
||||||
|
* **Entscheidungen:** "Warum nutzen wir X?".
|
||||||
|
* **Ähnliches:** "Projekt Beta war ähnlich".
|
||||||
|
|
||||||
|
### 1.2 Der Zwilling (Die Personas)
|
||||||
|
Mindnet passt seinen Charakter dynamisch an deine Frage an:
|
||||||
|
1. **Der Bibliothekar (FACT):** Liefert präzise Fakten und Definitionen.
|
||||||
|
2. **Der Berater (DECISION):** Hilft dir beim Abwägen, basierend auf deinen Werten und Zielen.
|
||||||
|
3. **Der Spiegel (EMPATHY):** Hört zu und antwortet basierend auf deinen Erfahrungen ("Ich"-Perspektive).
|
||||||
|
4. **Der Coder (CODING):** Liefert technische Schnipsel ohne Smalltalk.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Die Oberfläche (Web UI)
|
||||||
|
|
||||||
|
Seit Version 2.3.1 bedienst du Mindnet über eine grafische Oberfläche im Browser.
|
||||||
|
|
||||||
|
### 2.1 Der Chat-Bereich
|
||||||
|
* **Eingabe:** Unten findest du das Textfeld.
|
||||||
|
* **Intent-Badge:** Über jeder Antwort zeigt ein kleines Label (z.B. **⚖️ Intent: DECISION**), in welchem Modus das System arbeitet und ob dies durch ein Schlüsselwort oder die KI erkannt wurde.
|
||||||
|
* **Quellen-Karten:** Unter der Antwort findest du ausklappbare Karten ("Expanders").
|
||||||
|
* **Grüner Punkt:** Hohe Relevanz (Score > 0.8).
|
||||||
|
* **Gelber Punkt:** Mittlere Relevanz.
|
||||||
|
* **Klick darauf:** Zeigt den Textauszug und die **Begründung** ("Warum wurde das gefunden?").
|
||||||
|
|
||||||
|
### 2.2 Die Sidebar (Einstellungen & Verlauf)
|
||||||
|
* **Modus-Wahl:** Umschalten zwischen "💬 Chat" und "📝 Neuer Eintrag" (Vorbereitung für WP07).
|
||||||
|
* **Verlauf:** Die letzten Suchanfragen sind hier gelistet. Ein Klick führt die Suche erneut aus.
|
||||||
|
* **Settings:**
|
||||||
|
* **Top-K:** Wie viele Quellen sollen gelesen werden? (Standard: 5).
|
||||||
|
* **Explanation Layer:** Schaltet die "Warum"-Erklärungen an/aus.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Den Chat steuern (Intents)
|
||||||
|
|
||||||
|
Du steuerst die Persönlichkeit von Mindnet durch deine Wortwahl. Das System nutzt einen **Hybrid Router**, der sowohl auf Schlüsselwörter als auch auf die Bedeutung achtet.
|
||||||
|
|
||||||
|
### 3.1 Modus: Entscheidung ("Der Berater")
|
||||||
|
Wenn du vor einer Wahl stehst, hilft Mindnet dir, konform zu deinen Prinzipien zu bleiben.
|
||||||
|
|
||||||
|
* **Auslöser (Keywords):** "Soll ich...", "Was ist deine Meinung?", "Strategie für...", "Vor- und Nachteile".
|
||||||
|
* **Was passiert:** Mindnet lädt deine **Werte** (`type: value`) und **Ziele** (`type: goal`) in den Kontext und prüft die Fakten dagegen.
|
||||||
|
* **Beispiel-Dialog:**
|
||||||
|
* *Du:* "Soll ich Tool X nutzen?"
|
||||||
|
* *Mindnet:* "Nein. Tool X speichert Daten in den USA. Das verstößt gegen dein Prinzip 'Privacy First' und dein Ziel 'Digitale Autarkie'."
|
||||||
|
|
||||||
|
### 3.2 Modus: Empathie ("Der Spiegel")
|
||||||
|
Wenn du frustriert bist oder reflektieren willst, wechselt Mindnet in den "Ich"-Modus.
|
||||||
|
|
||||||
|
* **Auslöser (Keywords & Semantik):** "Ich fühle mich...", "Traurig", "Gestresst", "Alles ist sinnlos", "Ich bin überfordert".
|
||||||
|
* **Was passiert:** Mindnet lädt deine **Erfahrungen** (`type: experience`) und **Glaubenssätze** (`type: belief`). Es antwortet verständnisvoll und zitiert deine eigenen Lektionen.
|
||||||
|
|
||||||
|
### 3.3 Modus: Fakten (Standard)
|
||||||
|
* **Auslöser:** Alles andere. "Was ist Qdrant?", "Zusammenfassung von X".
|
||||||
|
* **Was passiert:** Standard-Suche nach Wissen ohne spezielle Filter.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Ergebnisse interpretieren (Explanation Layer)
|
||||||
|
|
||||||
|
Mindnet liefert nicht einfach nur Treffer. Es liefert eine **Begründung** (Explanation), warum es etwas gefunden hat.
|
||||||
|
|
||||||
|
### 4.1 Die Gründe ("Reasons")
|
||||||
|
Öffne eine Quellen-Karte, um zu sehen:
|
||||||
|
* *"Hohe textuelle Übereinstimmung."* (Semantik)
|
||||||
|
* *"Bevorzugt aufgrund des Typs 'decision'."* (Wichtigkeit)
|
||||||
|
* *"Verweist auf 'Projekt X' via 'depends_on'."* (Graph-Kontext)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Feedback & Lernen
|
||||||
|
|
||||||
|
Mindnet wird schlauer, wenn du es pflegst. Nutze die Feedback-Funktionen im Chat:
|
||||||
|
|
||||||
|
### 5.1 Globales Feedback (Antwort-Qualität)
|
||||||
|
* **Wie:** Klicke auf die **Sterne (1-5)** unter der Antwort.
|
||||||
|
* **Bedeutung:** Bewertet, wie gut die generierte Antwort war (Prompting, Tonfall).
|
||||||
|
|
||||||
|
### 5.2 Granulares Feedback (Quellen-Relevanz)
|
||||||
|
* **Wie:** Klicke innerhalb einer Quellen-Karte auf die **Smileys (Faces)**.
|
||||||
|
* **Bedeutung:**
|
||||||
|
* 😞 (Sad): Diese Quelle ist irrelevant/falsch. (Senkt zukünftig das Gewicht).
|
||||||
|
* 😐 (Neutral): Okay, aber nicht entscheidend.
|
||||||
|
* 😀 (Grin): Das ist der perfekte Treffer! (Stärkt diesen Typ/Kante).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Neues Wissen beisteuern (Authoring)
|
||||||
|
|
||||||
|
Mindnet lebt von deinem Input. Du musst kein Techniker sein, um gutes Wissen zu designen. Siehe dazu das **Knowledge Design Manual** für Details.
|
||||||
|
|
||||||
|
### 6.1 Die Goldene Regel: "Verlinke semantisch"
|
||||||
|
Statt einfach nur `[[Link]]` zu schreiben, versuche zu sagen, *wie* es zusammenhängt.
|
||||||
|
* Hängt es davon ab? -> `[[rel:depends_on Ziel]]`
|
||||||
|
* Ist es ähnlich? -> `[[rel:similar_to Ziel]]`
|
||||||
|
|
||||||
|
### 6.2 Typen nutzen (Wichtig!)
|
||||||
|
Setze im Frontmatter deiner Notizen den richtigen Typ. Das ist der wichtigste Hebel für den Chat:
|
||||||
|
* Willst du, dass Mindnet etwas als **Regel** nutzt? -> `type: principle`
|
||||||
|
* Willst du, dass Mindnet dich an eine **Lektion** erinnert? -> `type: experience`
|
||||||
+32
-8
@@ -1,9 +1,33 @@
|
|||||||
|
# --- API Server ---
|
||||||
|
fastapi>=0.116.1
|
||||||
|
uvicorn[standard]>=0.35.0
|
||||||
|
# httpx wird oft für Tests benötigt
|
||||||
|
httpx>=0.28.1
|
||||||
|
|
||||||
fastapi
|
# --- Datenbank ---
|
||||||
uvicorn[standard]
|
qdrant-client>=1.15.1
|
||||||
qdrant-client
|
|
||||||
sentence-transformers
|
# --- Datenmodelle & Validierung ---
|
||||||
torch
|
pydantic>=2.11.7
|
||||||
python-frontmatter
|
numpy>=2.3.2
|
||||||
tqdm
|
|
||||||
requests
|
# --- Markdown & Parsing ---
|
||||||
|
python-frontmatter>=1.1.0
|
||||||
|
markdown-it-py>=3.0.0
|
||||||
|
PyYAML>=6.0.2
|
||||||
|
python-slugify>=8.0.4
|
||||||
|
|
||||||
|
# --- KI & Embeddings ---
|
||||||
|
sentence-transformers>=5.1.0
|
||||||
|
torch>=2.0.0
|
||||||
|
|
||||||
|
# --- Utilities ---
|
||||||
|
python-dotenv>=1.1.1
|
||||||
|
requests>=2.32.5
|
||||||
|
tqdm>=4.67.1
|
||||||
|
|
||||||
|
# --- Testing ---
|
||||||
|
pytest>=8.4.2
|
||||||
|
|
||||||
|
# --- Frontend (WP-10) ---
|
||||||
|
streamlit>=1.39.0
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/assert_payload_schema.py (compat v1.1)
|
||||||
|
|
||||||
|
Prüft, ob die erwarteten Payload-Indizes (payload_schema) auf den drei Collections vorhanden sind.
|
||||||
|
Kompatibel mit älteren qdrant-client Versionen (ohne with_payload_schema im Wrapper).
|
||||||
|
|
||||||
|
Collections & Pflichtfelder:
|
||||||
|
- mindnet_notes : note_id, type, title, updated, tags
|
||||||
|
- mindnet_chunks : note_id, chunk_id, index, type, tags
|
||||||
|
- mindnet_edges : note_id, kind, scope, source_id, target_id, chunk_id
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 -m tests.assert_payload_schema
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
REQUIRED = {
|
||||||
|
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||||
|
"chunks": ["note_id", "chunk_id", "index", "type", "tags"],
|
||||||
|
"edges": ["note_id", "kind", "scope", "source_id", "target_id", "chunk_id"],
|
||||||
|
}
|
||||||
|
|
||||||
|
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||||
|
if hasattr(obj, "model_dump"):
|
||||||
|
return obj.model_dump()
|
||||||
|
if hasattr(obj, "dict"):
|
||||||
|
return obj.dict()
|
||||||
|
return obj if isinstance(obj, dict) else {}
|
||||||
|
|
||||||
|
def _cfg_base_url(cfg) -> str:
|
||||||
|
if getattr(cfg, "url", None):
|
||||||
|
return cfg.url.rstrip('/')
|
||||||
|
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||||
|
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||||
|
return f"http://{host}:{port}"
|
||||||
|
|
||||||
|
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||||
|
req = urllib.request.Request(url, method="GET")
|
||||||
|
if api_key:
|
||||||
|
req.add_header("api-key", api_key)
|
||||||
|
req.add_header("Authorization", f"Bearer {api_key}")
|
||||||
|
req.add_header("Accept", "application/json")
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = resp.read()
|
||||||
|
return json.loads(data.decode("utf-8"))
|
||||||
|
|
||||||
|
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||||
|
# 1) OpenAPI client mit Flag
|
||||||
|
try:
|
||||||
|
oc = getattr(client, "openapi_client", None)
|
||||||
|
if oc is not None and hasattr(oc, "collections_api"):
|
||||||
|
api = oc.collections_api
|
||||||
|
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
return (d.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# 2) Wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
ps = (d.get("result") or {}).get("payload_schema")
|
||||||
|
if ps is not None:
|
||||||
|
return ps
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# 3) Direkter HTTP-Call
|
||||||
|
try:
|
||||||
|
base = _cfg_base_url(cfg)
|
||||||
|
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||||
|
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||||
|
return (raw.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def ensure_fields(schema: Dict[str, dict] | None, required: List[str]) -> Dict[str, bool]:
|
||||||
|
ks = set((schema or {}).keys())
|
||||||
|
return {k: (k in ks) for k in required}
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
names = {"notes": notes, "chunks": chunks, "edges": edges}
|
||||||
|
|
||||||
|
report = {}
|
||||||
|
ok_all = True
|
||||||
|
for kind, col in names.items():
|
||||||
|
sch = get_collection_payload_schema(client, cfg, col)
|
||||||
|
checks = ensure_fields(sch, REQUIRED[kind])
|
||||||
|
ok = all(checks.values())
|
||||||
|
ok_all = ok_all and ok
|
||||||
|
report[kind] = {
|
||||||
|
"collection": col,
|
||||||
|
"ok": ok,
|
||||||
|
"missing": [k for k,v in checks.items() if not v],
|
||||||
|
"present": [k for k,v in checks.items() if v],
|
||||||
|
}
|
||||||
|
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "ok": ok_all, "report": report}, ensure_ascii=False, indent=2))
|
||||||
|
if not ok_all:
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/diag_payload_indexes.py (compat v1.2)
|
||||||
|
|
||||||
|
Zeigt payload_schema je Collection; kompatibel mit älteren qdrant-client Versionen.
|
||||||
|
Strategie (in dieser Reihenfolge):
|
||||||
|
1) openapi_client.collections_api.get_collection(..., with_payload_schema=True)
|
||||||
|
2) client.get_collection(...)
|
||||||
|
3) HTTP GET /collections/{name}?with_payload_schema=true
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||||
|
if hasattr(obj, "model_dump"):
|
||||||
|
return obj.model_dump()
|
||||||
|
if hasattr(obj, "dict"):
|
||||||
|
return obj.dict()
|
||||||
|
return obj if isinstance(obj, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _cfg_base_url(cfg) -> str:
|
||||||
|
if getattr(cfg, "url", None):
|
||||||
|
return cfg.url.rstrip("/")
|
||||||
|
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||||
|
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||||
|
return f"http://{host}:{port}"
|
||||||
|
|
||||||
|
|
||||||
|
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||||
|
req = urllib.request.Request(url, method="GET")
|
||||||
|
if api_key:
|
||||||
|
req.add_header("api-key", api_key)
|
||||||
|
req.add_header("Authorization", f"Bearer {api_key}")
|
||||||
|
req.add_header("Accept", "application/json")
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = resp.read()
|
||||||
|
return json.loads(data.decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||||
|
# Strategy 1: OpenAPI client with flag
|
||||||
|
try:
|
||||||
|
oc = getattr(client, "openapi_client", None)
|
||||||
|
if oc is not None and hasattr(oc, "collections_api"):
|
||||||
|
api = oc.collections_api
|
||||||
|
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
return (d.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Strategy 2: wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name) # kein with_payload_schema kwarg!
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
ps = (d.get("result") or {}).get("payload_schema")
|
||||||
|
if ps is not None:
|
||||||
|
return ps
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Strategy 3: direkter HTTP Call
|
||||||
|
try:
|
||||||
|
base = _cfg_base_url(cfg)
|
||||||
|
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||||
|
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||||
|
return (raw.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
cols = [notes, chunks, edges]
|
||||||
|
out = []
|
||||||
|
for name in cols:
|
||||||
|
ps = get_collection_payload_schema(client, cfg, name)
|
||||||
|
# Vektorinfos (optional)
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
vectors = (d.get("result") or {}).get("vectors")
|
||||||
|
if isinstance(vectors, dict) and "config" in vectors:
|
||||||
|
vectors = vectors["config"]
|
||||||
|
except Exception:
|
||||||
|
vectors = None
|
||||||
|
out.append({"name": name, "vectors": vectors, "payload_schema": ps})
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "collections": out}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# scripts/edges_dryrun.py
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Dry-Run: Erzeuge Edges aus einem Vault **ohne** Qdrant-Upsert.
|
||||||
|
- Liest Markdown mit YAML-Frontmatter
|
||||||
|
- Chunking: einfacher Absatz-Chunker (Index + text)
|
||||||
|
- Kanten: nutzt app.core.edges.build_edges_for_note()
|
||||||
|
- Ausgabe: JSON pro Note mit Edge-Counts und 3 Beispiel-Payloads
|
||||||
|
|
||||||
|
Aufruf:
|
||||||
|
python3 -m scripts.edges_dryrun --vault ./vault
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
--include-note-scope-refs # auch Frontmatter-Links (links[].target_id) als Note-Scope-Referenzen
|
||||||
|
|
||||||
|
Voraussetzungen:
|
||||||
|
- app/core/parser.py (read_markdown, normalize_frontmatter)
|
||||||
|
- app/core/edges.py (dieses Modul forwardet zur v2-Implementierung)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, List, Optional
|
||||||
|
|
||||||
|
from app.core.parser import read_markdown, normalize_frontmatter
|
||||||
|
from app.core.edges import build_edges_for_note
|
||||||
|
|
||||||
|
def _iter_markdown(vault: str):
|
||||||
|
for p in Path(vault).rglob("*.md"):
|
||||||
|
if p.name.startswith("."): # ignore hidden
|
||||||
|
continue
|
||||||
|
yield p
|
||||||
|
|
||||||
|
def _simple_chunker(body: str, note_id: str, note_type: str) -> List[Dict]:
|
||||||
|
# Absatzbasiert, minimal ausreichend für Edges (window/text, chunk_id, ord, note_id, type)
|
||||||
|
paras = [s.strip() for s in re.split(r"\n{2,}", body or "") if s.strip()]
|
||||||
|
chunks = []
|
||||||
|
for i, t in enumerate(paras):
|
||||||
|
chunks.append({
|
||||||
|
"chunk_id": f"{note_id}#c{i:04d}",
|
||||||
|
"ord": i,
|
||||||
|
"text": t,
|
||||||
|
"note_id": note_id,
|
||||||
|
"type": note_type,
|
||||||
|
})
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
def _fm_note_refs(fm: Dict) -> List[str]:
|
||||||
|
out = []
|
||||||
|
links = fm.get("links") or []
|
||||||
|
if isinstance(links, list):
|
||||||
|
for e in links:
|
||||||
|
if isinstance(e, dict):
|
||||||
|
tid = e.get("target_id")
|
||||||
|
if isinstance(tid, str) and tid.strip():
|
||||||
|
out.append(tid.strip())
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--vault", required=True)
|
||||||
|
ap.add_argument("--include-note-scope-refs", action="store_true")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
vault = args.vault
|
||||||
|
include_note_scope = args.include_note_scope_refs
|
||||||
|
|
||||||
|
report = []
|
||||||
|
for p in _iter_markdown(vault):
|
||||||
|
parsed = read_markdown(str(p))
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter or {})
|
||||||
|
note_id = fm.get("id") or p.stem
|
||||||
|
note_type = (fm.get("type") or "concept").lower()
|
||||||
|
chunks = _simple_chunker(parsed.body, note_id, note_type)
|
||||||
|
note_refs = _fm_note_refs(fm)
|
||||||
|
|
||||||
|
edges = build_edges_for_note(
|
||||||
|
note_id=note_id,
|
||||||
|
chunk_payloads=chunks,
|
||||||
|
note_level_refs=note_refs,
|
||||||
|
include_note_scope_refs=include_note_scope,
|
||||||
|
)
|
||||||
|
kinds = {}
|
||||||
|
for e in edges:
|
||||||
|
key = (e.get("relation") or e.get("kind") or "edge")
|
||||||
|
kinds[key] = kinds.get(key, 0) + 1
|
||||||
|
report.append({
|
||||||
|
"path": str(p),
|
||||||
|
"note_id": note_id,
|
||||||
|
"type": note_type,
|
||||||
|
"chunks": len(chunks),
|
||||||
|
"edges_total": len(edges),
|
||||||
|
"edges_by_kind": kinds,
|
||||||
|
"samples": edges[:3],
|
||||||
|
})
|
||||||
|
|
||||||
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/edges_full_check.py
|
||||||
|
Zählt und validiert Kanten in Qdrant. Erkennt folgende Rule-Gruppen:
|
||||||
|
- explicit_total: rule_id startswith "explicit:" (z.B. explicit:wikilink, explicit:note_scope)
|
||||||
|
- callout_total: rule_id == "callout:edge"
|
||||||
|
- inline_total: rule_id startswith "inline:" (z.B. inline:rel)
|
||||||
|
- defaults_total: rule_id startswith "edge_defaults:"
|
||||||
|
- structure: rule_id in {"structure:belongs_to","structure:order"}
|
||||||
|
|
||||||
|
Gibt zusätzlich:
|
||||||
|
- edges_by_kind (aggregiert)
|
||||||
|
- notes/chunks/edges Anzahlen
|
||||||
|
- multi_callout_detected: True, falls ein Chunk mehrere Callout-Ziele der gleichen Relation enthält
|
||||||
|
- per_note_checks: belongs_to == chunks, next == prev == (chunks-1)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from typing import Dict, Any, List, Tuple
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
|
||||||
|
def _count_collection_points(client, name: str) -> int:
|
||||||
|
try:
|
||||||
|
res = client.count(collection_name=name, exact=True)
|
||||||
|
return res.count or 0
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _scroll_all(client, collection: str) -> List[Any]:
|
||||||
|
pts_all = []
|
||||||
|
offset = None
|
||||||
|
while True:
|
||||||
|
pts, offset = client.scroll(
|
||||||
|
collection_name=collection,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=2048,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
pts_all.extend(pts or [])
|
||||||
|
if offset is None:
|
||||||
|
break
|
||||||
|
return pts_all
|
||||||
|
|
||||||
|
|
||||||
|
def _rule_group(rule_id: str) -> str:
|
||||||
|
if not rule_id:
|
||||||
|
return "unknown"
|
||||||
|
if rule_id == "callout:edge":
|
||||||
|
return "callout"
|
||||||
|
if rule_id.startswith("inline:"): # <—— wichtig für "inline:rel"
|
||||||
|
return "inline"
|
||||||
|
if rule_id.startswith("edge_defaults:"):
|
||||||
|
return "defaults"
|
||||||
|
if rule_id.startswith("explicit:"):
|
||||||
|
return "explicit"
|
||||||
|
if rule_id in ("structure:belongs_to", "structure:order"):
|
||||||
|
return "structure"
|
||||||
|
return "other"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
col_notes = f"{cfg.prefix}_notes"
|
||||||
|
col_chunks = f"{cfg.prefix}_chunks"
|
||||||
|
col_edges = f"{cfg.prefix}_edges"
|
||||||
|
|
||||||
|
# High-level counts
|
||||||
|
notes_n = _count_collection_points(client, col_notes)
|
||||||
|
chunks_n = _count_collection_points(client, col_chunks)
|
||||||
|
edges_pts = _scroll_all(client, col_edges)
|
||||||
|
edges_n = len(edges_pts)
|
||||||
|
|
||||||
|
# By kind / by rule group
|
||||||
|
by_kind = Counter()
|
||||||
|
group_counts = Counter()
|
||||||
|
callout_buckets: Dict[Tuple[str, str], int] = defaultdict(int) # (chunk_id, kind) -> n targets
|
||||||
|
per_note = defaultdict(lambda: {"chunks": 0, "belongs_to": 0, "next": 0, "prev": 0})
|
||||||
|
|
||||||
|
# Für per_note checks: chunks pro note_id aus mindnet_chunks laden
|
||||||
|
chunks_pts = _scroll_all(client, col_chunks)
|
||||||
|
chunks_by_note = Counter([p.payload.get("note_id") for p in chunks_pts if p.payload])
|
||||||
|
|
||||||
|
for p in edges_pts:
|
||||||
|
pl = p.payload or {}
|
||||||
|
kind = str(pl.get("kind") or pl.get("relation") or "edge")
|
||||||
|
rule_id = str(pl.get("rule_id") or "")
|
||||||
|
note_id = str(pl.get("note_id") or "")
|
||||||
|
chunk_id = str(pl.get("chunk_id") or "")
|
||||||
|
by_kind[kind] += 1
|
||||||
|
|
||||||
|
group = _rule_group(rule_id)
|
||||||
|
group_counts[group] += 1
|
||||||
|
|
||||||
|
# Multi-Callout-Erkennung: mehrere callout-Edges gleicher Relation aus demselben Chunk
|
||||||
|
if group == "callout" and chunk_id and kind:
|
||||||
|
callout_buckets[(chunk_id, kind)] += 1
|
||||||
|
|
||||||
|
# Per-note Strukturchecks
|
||||||
|
if note_id:
|
||||||
|
if kind == "belongs_to":
|
||||||
|
per_note[note_id]["belongs_to"] += 1
|
||||||
|
elif kind == "next":
|
||||||
|
per_note[note_id]["next"] += 1
|
||||||
|
elif kind == "prev":
|
||||||
|
per_note[note_id]["prev"] += 1
|
||||||
|
|
||||||
|
# set chunks count for per_note
|
||||||
|
for n_id, c in chunks_by_note.items():
|
||||||
|
per_note[n_id]["chunks"] = c
|
||||||
|
|
||||||
|
# final checks per note
|
||||||
|
per_note_checks = {}
|
||||||
|
for n_id, stats in per_note.items():
|
||||||
|
c = stats.get("chunks", 0)
|
||||||
|
bt = stats.get("belongs_to", 0)
|
||||||
|
nx = stats.get("next", 0)
|
||||||
|
pv = stats.get("prev", 0)
|
||||||
|
per_note_checks[n_id] = {
|
||||||
|
"chunks": c,
|
||||||
|
"belongs_to": bt,
|
||||||
|
"next": nx,
|
||||||
|
"prev": pv,
|
||||||
|
"checks": {
|
||||||
|
"belongs_to_equals_chunks": (bt == c),
|
||||||
|
"next_prev_match": (nx == pv == max(c - 1, 0)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
multi_callout_detected = any(v > 1 for v in callout_buckets.values())
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"prefix": cfg.prefix,
|
||||||
|
"counts": {
|
||||||
|
"notes": notes_n,
|
||||||
|
"chunks": chunks_n,
|
||||||
|
"edges": edges_n,
|
||||||
|
"edges_by_kind": dict(by_kind),
|
||||||
|
"explicit_total": group_counts.get("explicit", 0),
|
||||||
|
"defaults_total": group_counts.get("defaults", 0),
|
||||||
|
"callout_total": group_counts.get("callout", 0),
|
||||||
|
"inline_total": group_counts.get("inline", 0),
|
||||||
|
"structure_total": group_counts.get("structure", 0),
|
||||||
|
},
|
||||||
|
"per_note_checks": per_note_checks,
|
||||||
|
"multi_callout_detected": bool(multi_callout_detected),
|
||||||
|
"has_duplicates": False, # dedupe passiert beim Upsert
|
||||||
|
}
|
||||||
|
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,340 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
scripts/health_check_mindnet.py
|
||||||
|
|
||||||
|
Health-Check für den mindnet-Retriever-/Query-Endpoint.
|
||||||
|
|
||||||
|
Funktion:
|
||||||
|
- Führt POST-Requests auf /query in verschiedenen Modi aus (standard: semantic + hybrid).
|
||||||
|
- Prüft Status-Code, JSON-Struktur und Anzahl der Treffer.
|
||||||
|
- Kennzeichnet Probleme als:
|
||||||
|
- status="ok"
|
||||||
|
- status="warning" (z.B. Timeout)
|
||||||
|
- status="error" (harte Fehler wie HTTP-Fehler, JSON-Fehler etc.)
|
||||||
|
|
||||||
|
Exit-Code:
|
||||||
|
- Default (tolerant):
|
||||||
|
- overall_status = "ok" (inkl. warnings) → Exit-Code 0
|
||||||
|
- overall_status = "error" → Exit-Code 1
|
||||||
|
- Mit --strict:
|
||||||
|
- warnings werden wie errors behandelt → Exit-Code 1
|
||||||
|
|
||||||
|
Beispiele:
|
||||||
|
|
||||||
|
python scripts/health_check_mindnet.py \
|
||||||
|
--url http://127.0.0.1:8001/query \
|
||||||
|
--query "embeddings" \
|
||||||
|
--top-k 3
|
||||||
|
|
||||||
|
python scripts/health_check_mindnet.py \
|
||||||
|
--url http://127.0.0.1:8001/query \
|
||||||
|
--query "embeddings" \
|
||||||
|
--top-k 3 \
|
||||||
|
--timeout 15 \
|
||||||
|
--modes hybrid
|
||||||
|
|
||||||
|
# Strenger Modus (warnings → Exit-Code 1)
|
||||||
|
python scripts/health_check_mindnet.py --strict
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
|
||||||
|
def _post_query(
|
||||||
|
url: str,
|
||||||
|
mode: str,
|
||||||
|
query: str,
|
||||||
|
top_k: int,
|
||||||
|
timeout: float,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Führt einen POST auf den /query-Endpoint aus und gibt das dekodierte JSON zurück.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
urllib.error.URLError, urllib.error.HTTPError, RuntimeError
|
||||||
|
"""
|
||||||
|
payload = {
|
||||||
|
"mode": mode,
|
||||||
|
"query": query,
|
||||||
|
"top_k": top_k,
|
||||||
|
"expand": {
|
||||||
|
"depth": 1 if mode == "hybrid" else 0,
|
||||||
|
"edge_types": ["references", "belongs_to", "prev", "next"],
|
||||||
|
},
|
||||||
|
"ret": {
|
||||||
|
"with_paths": True,
|
||||||
|
"with_notes": True,
|
||||||
|
"with_chunks": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
data = json.dumps(payload).encode("utf-8")
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url=url,
|
||||||
|
data=data,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
|
||||||
|
start = time.monotonic()
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
|
status_code = resp.getcode()
|
||||||
|
body = resp.read().decode("utf-8")
|
||||||
|
end = time.monotonic()
|
||||||
|
|
||||||
|
latency_ms = int((end - start) * 1000)
|
||||||
|
|
||||||
|
if status_code != 200:
|
||||||
|
raise RuntimeError(f"HTTP {status_code}: {body}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
doc = json.loads(body)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise RuntimeError(f"Antwort ist kein gültiges JSON: {exc}") from exc
|
||||||
|
|
||||||
|
if not isinstance(doc, dict):
|
||||||
|
raise RuntimeError("Antwort ist kein JSON-Objekt")
|
||||||
|
|
||||||
|
doc["_latency_ms"] = latency_ms
|
||||||
|
return doc
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_response(
|
||||||
|
mode: str,
|
||||||
|
doc: Dict[str, Any],
|
||||||
|
min_results: int,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Prüft die wichtigsten Invarianten des Query-Responses.
|
||||||
|
|
||||||
|
Rückgabe-Format:
|
||||||
|
|
||||||
|
{
|
||||||
|
"status": "ok" | "warning" | "error",
|
||||||
|
"message": str,
|
||||||
|
"latency_ms": int,
|
||||||
|
"used_mode": str,
|
||||||
|
"result_count": int
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
latency_ms = int(doc.get("_latency_ms", -1))
|
||||||
|
used_mode = str(doc.get("used_mode", "unknown"))
|
||||||
|
results = doc.get("results", [])
|
||||||
|
|
||||||
|
if not isinstance(results, list):
|
||||||
|
return {
|
||||||
|
"status": "error",
|
||||||
|
"message": "results ist kein Array",
|
||||||
|
"latency_ms": latency_ms,
|
||||||
|
"used_mode": used_mode,
|
||||||
|
"result_count": -1,
|
||||||
|
}
|
||||||
|
|
||||||
|
result_count = len(results)
|
||||||
|
|
||||||
|
if result_count < min_results:
|
||||||
|
# Zu wenige Ergebnisse sind aus Sicht eines Health-Checks eher ein Error,
|
||||||
|
# da die Retrieval-Qualität nicht gewährleistet ist.
|
||||||
|
return {
|
||||||
|
"status": "error",
|
||||||
|
"message": f"zu wenige Ergebnisse: {result_count} < {min_results}",
|
||||||
|
"latency_ms": latency_ms,
|
||||||
|
"used_mode": used_mode,
|
||||||
|
"result_count": result_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
sample_msg = "OK"
|
||||||
|
if result_count > 0:
|
||||||
|
first = results[0]
|
||||||
|
if not isinstance(first, dict):
|
||||||
|
sample_msg = "first result ist kein Objekt"
|
||||||
|
else:
|
||||||
|
# einfache Plausibilitätschecks (nur Zugriff, kein strikter Typcheck)
|
||||||
|
_ = first.get("note_id")
|
||||||
|
_ = first.get("chunk_id")
|
||||||
|
_ = first.get("total_score")
|
||||||
|
|
||||||
|
status = "ok" if sample_msg == "OK" else "warning"
|
||||||
|
return {
|
||||||
|
"status": status,
|
||||||
|
"message": sample_msg,
|
||||||
|
"latency_ms": latency_ms,
|
||||||
|
"used_mode": used_mode,
|
||||||
|
"result_count": result_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run_health_check(
|
||||||
|
url: str,
|
||||||
|
query: str,
|
||||||
|
top_k: int,
|
||||||
|
timeout: float,
|
||||||
|
modes: List[str],
|
||||||
|
min_results: int,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Führt den Health-Check für die angegebenen Modi aus.
|
||||||
|
"""
|
||||||
|
report: Dict[str, Any] = {
|
||||||
|
"url": url,
|
||||||
|
"query": query,
|
||||||
|
"top_k": top_k,
|
||||||
|
"timeout_s": timeout,
|
||||||
|
"checks": [],
|
||||||
|
"overall_status": "ok", # wird unten ggf. auf warning oder error gesetzt
|
||||||
|
}
|
||||||
|
|
||||||
|
overall_status = "ok"
|
||||||
|
|
||||||
|
for mode in modes:
|
||||||
|
entry: Dict[str, Any] = {
|
||||||
|
"mode": mode,
|
||||||
|
"status": "error",
|
||||||
|
"message": "",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
doc = _post_query(url=url, mode=mode, query=query, top_k=top_k, timeout=timeout)
|
||||||
|
validation = _validate_response(mode=mode, doc=doc, min_results=min_results)
|
||||||
|
entry.update(validation)
|
||||||
|
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
entry["status"] = "error"
|
||||||
|
entry["message"] = f"HTTP-Fehler: {exc}"
|
||||||
|
entry["latency_ms"] = -1
|
||||||
|
entry["used_mode"] = mode
|
||||||
|
entry["result_count"] = -1
|
||||||
|
|
||||||
|
except urllib.error.URLError as exc:
|
||||||
|
# URLError kann u.a. socket.timeout enthalten.
|
||||||
|
lat = -1
|
||||||
|
entry["latency_ms"] = lat
|
||||||
|
entry["used_mode"] = mode
|
||||||
|
entry["result_count"] = -1
|
||||||
|
|
||||||
|
if isinstance(exc.reason, socket.timeout):
|
||||||
|
entry["status"] = "warning"
|
||||||
|
entry["message"] = f"Timeout (URLError/socket.timeout): {exc}"
|
||||||
|
else:
|
||||||
|
entry["status"] = "error"
|
||||||
|
entry["message"] = f"HTTP/Netzwerkfehler: {exc}"
|
||||||
|
|
||||||
|
except socket.timeout as exc:
|
||||||
|
# Direkter Timeout (falls nicht in URLError verpackt)
|
||||||
|
entry["status"] = "warning"
|
||||||
|
entry["message"] = f"Timeout (socket.timeout): {exc}"
|
||||||
|
entry["latency_ms"] = -1
|
||||||
|
entry["used_mode"] = mode
|
||||||
|
entry["result_count"] = -1
|
||||||
|
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
entry["status"] = "error"
|
||||||
|
entry["message"] = f"unerwarteter Fehler: {exc}"
|
||||||
|
entry["latency_ms"] = -1
|
||||||
|
entry["used_mode"] = mode
|
||||||
|
entry["result_count"] = -1
|
||||||
|
|
||||||
|
# overall_status bestimmen:
|
||||||
|
# - error > warning > ok
|
||||||
|
if entry["status"] == "error":
|
||||||
|
overall_status = "error"
|
||||||
|
elif entry["status"] == "warning" and overall_status == "ok":
|
||||||
|
overall_status = "warning"
|
||||||
|
|
||||||
|
report["checks"].append(entry)
|
||||||
|
|
||||||
|
report["overall_status"] = overall_status
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args(argv: List[str]) -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Health-Check für mindnet /query-Retriever",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--url",
|
||||||
|
default="http://127.0.0.1:8001/query",
|
||||||
|
help="Vollständige URL des /query-Endpunkts (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--query",
|
||||||
|
default="embeddings",
|
||||||
|
help="Test-Query für den Health-Check (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--top-k",
|
||||||
|
type=int,
|
||||||
|
default=3,
|
||||||
|
help="Top-K für die Testabfrage (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--timeout",
|
||||||
|
type=float,
|
||||||
|
default=5.0,
|
||||||
|
help="Timeout in Sekunden pro Request (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--modes",
|
||||||
|
nargs="+",
|
||||||
|
default=["semantic", "hybrid"],
|
||||||
|
help="Zu prüfende Modi (Default: semantic hybrid)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--min-results",
|
||||||
|
type=int,
|
||||||
|
default=1,
|
||||||
|
help="Minimale Anzahl erwarteter Ergebnisse (Default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--strict",
|
||||||
|
action="store_true",
|
||||||
|
help="Warnings als Fehler behandeln (overall_status=warning führt zu Exit-Code 1)",
|
||||||
|
)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: List[str]) -> int:
|
||||||
|
args = parse_args(argv)
|
||||||
|
|
||||||
|
report = run_health_check(
|
||||||
|
url=args.url,
|
||||||
|
query=args.query,
|
||||||
|
top_k=args.top_k,
|
||||||
|
timeout=args.timeout,
|
||||||
|
modes=args.modes,
|
||||||
|
min_results=args.min_results,
|
||||||
|
)
|
||||||
|
|
||||||
|
overall = report.get("overall_status", "error")
|
||||||
|
|
||||||
|
# Menschlich lesbare Zusammenfassung:
|
||||||
|
print(f"mindnet health: {overall}")
|
||||||
|
for c in report["checks"]:
|
||||||
|
mode = c.get("mode")
|
||||||
|
status = c.get("status")
|
||||||
|
latency = c.get("latency_ms")
|
||||||
|
result_count = c.get("result_count")
|
||||||
|
msg = c.get("message", "")
|
||||||
|
print(f" - {mode}: {status} (latency={latency} ms, results={result_count}) {msg}")
|
||||||
|
|
||||||
|
# JSON-Ausgabe (z.B. für n8n):
|
||||||
|
print(json.dumps(report, ensure_ascii=False))
|
||||||
|
|
||||||
|
if args.strict:
|
||||||
|
# strict: warning wird wie error behandelt
|
||||||
|
return 0 if overall == "ok" else 1
|
||||||
|
else:
|
||||||
|
# tolerant: nur echte errors führen zu Exit-Code 1
|
||||||
|
return 0 if overall in ("ok", "warning") else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main(sys.argv[1:]))
|
||||||
+247
-104
@@ -1,38 +1,25 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
|
scripts/import_markdown.py
|
||||||
Version: 3.7.2
|
|
||||||
Datum: 2025-09-30
|
|
||||||
|
|
||||||
Kurzbeschreibung
|
Zweck
|
||||||
----------------
|
-----
|
||||||
- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
|
- Liest Markdown-Notizen aus einem Vault ein
|
||||||
- Änderungserkennung „Option C“: mehrere Hash-Varianten werden parallel in der Note
|
- Erzeugt Note-Payload, Chunk-Payloads (+ optionale Embeddings) und Edges
|
||||||
gespeichert (Feld `hashes` mit Schlüsseln `<mode>:<source>:<normalize>`). Der Vergleich
|
- Schreibt alles idempotent in Qdrant (Notes, Chunks, Edges)
|
||||||
nutzt NUR den aktuellen Modus-Key — ein Moduswechsel triggert keine Massenänderungen mehr.
|
- Integriert eine optionale Type-Registry (types.yaml), um z. B. chunk_profile
|
||||||
- „Erstimport-Fix“: Bei leerem Qdrant gilt Create-Fall automatisch als geändert.
|
und retriever_weight pro Notiz-Typ zu steuern.
|
||||||
- `--baseline-modes`: fehlende Hash-Varianten still nachtragen (nur Notes upserten).
|
|
||||||
- `--sync-deletes`: gezielte Lösch-Synchronisation (Dry-Run + Apply).
|
|
||||||
- `--only-path`: exakt **eine** Datei (Pfad) importieren — nützlich für Diagnosefälle.
|
|
||||||
|
|
||||||
Neu in 3.7.1/3.7.2
|
Wesentliche Fixes ggü. vorherigen fehlerhaften Ständen
|
||||||
------------------
|
------------------------------------------------------
|
||||||
- Chunk-Payloads: `window` (für Embeddings), `text` (überlappungsfrei, verlustfrei rekonstruierbar),
|
- `embed_texts` wird optional importiert und defensiv geprüft (kein NameError mehr)
|
||||||
`start/end/overlap_*`. Embeddings nutzen `window`.
|
- `effective_chunk_profile` / `effective_retriever_weight` und Registry-Helfer
|
||||||
- **3.7.2:** Edges-Fehler führen nicht mehr zum Abbruch der gesamten Note; Note/Chunks werden trotzdem geschrieben.
|
sind VOR `main()` definiert (kein NameError mehr)
|
||||||
|
- `retriever_weight` wird in Note- und Chunk-Payload zuverlässig gesetzt
|
||||||
Hash/Compare Konfiguration
|
- Robuste Kantenbildung; Fehler bei Edges blockieren Notes/Chunks nicht
|
||||||
--------------------------
|
- Korrekte Verwendung von `scroll_filter` beim Qdrant-Client
|
||||||
- Vergleichsmodus:
|
- `--purge-before-upsert` entfernt alte Chunks/Edges einer Note vor dem Upsert
|
||||||
--hash-mode body|frontmatter|full
|
|
||||||
oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE
|
|
||||||
- Quelle:
|
|
||||||
--hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE, Default parsed)
|
|
||||||
- Normalisierung:
|
|
||||||
--hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE, Default canonical)
|
|
||||||
- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich
|
|
||||||
den parsed Body-Text direkt.
|
|
||||||
|
|
||||||
Qdrant / ENV
|
Qdrant / ENV
|
||||||
------------
|
------------
|
||||||
@@ -40,6 +27,7 @@ Qdrant / ENV
|
|||||||
- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
|
- COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
|
||||||
- VECTOR_DIM (Default: 384)
|
- VECTOR_DIM (Default: 384)
|
||||||
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
||||||
|
- MINDNET_TYPES_FILE: Pfad zu types.yaml (optional; Default: ./types.yaml)
|
||||||
|
|
||||||
Beispiele
|
Beispiele
|
||||||
---------
|
---------
|
||||||
@@ -67,6 +55,7 @@ from typing import Dict, List, Optional, Tuple, Any, Set
|
|||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
# --- Projekt-Imports ---
|
||||||
from app.core.parser import (
|
from app.core.parser import (
|
||||||
read_markdown,
|
read_markdown,
|
||||||
normalize_frontmatter,
|
normalize_frontmatter,
|
||||||
@@ -92,14 +81,110 @@ from app.core.qdrant_points import (
|
|||||||
upsert_batch,
|
upsert_batch,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# embeddings sind optional (z. B. im reinen Payload-Backfill)
|
||||||
try:
|
try:
|
||||||
from app.core.embed import embed_texts # optional
|
from app.core.embed import embed_texts # optional
|
||||||
except Exception:
|
except Exception: # pragma: no cover
|
||||||
embed_texts = None
|
embed_texts = None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
# Helper
|
# Type-Registry (types.yaml) – Helper (robust, optional)
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _env(name: str, default: Optional[str] = None) -> Optional[str]:
|
||||||
|
v = os.getenv(name)
|
||||||
|
return v if v is not None else default
|
||||||
|
|
||||||
|
def _load_json_or_yaml(path: str) -> dict:
|
||||||
|
import io
|
||||||
|
data: dict = {}
|
||||||
|
if not path or not os.path.exists(path):
|
||||||
|
return data
|
||||||
|
try:
|
||||||
|
import yaml # type: ignore
|
||||||
|
with io.open(path, "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return {}
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
# YAML evtl. nicht installiert – versuche JSON
|
||||||
|
try:
|
||||||
|
with io.open(path, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return {}
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def load_type_registry() -> dict:
|
||||||
|
# Reihenfolge: ENV > ./types.yaml (im aktuellen Arbeitsverzeichnis)
|
||||||
|
p = _env("MINDNET_TYPES_FILE", None)
|
||||||
|
if p and os.path.exists(p):
|
||||||
|
return _load_json_or_yaml(p)
|
||||||
|
fallback = os.path.abspath("./config/types.yaml") if os.path.exists("./config/types.yaml") else os.path.abspath("./types.yaml")
|
||||||
|
return _load_json_or_yaml(fallback)
|
||||||
|
|
||||||
|
def get_type_config(note_type: Optional[str], reg: dict) -> dict:
|
||||||
|
if not reg or not isinstance(reg, dict):
|
||||||
|
return {}
|
||||||
|
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
|
||||||
|
if note_type and isinstance(note_type, str) and note_type in types:
|
||||||
|
return types[note_type] or {}
|
||||||
|
# Fallback: concept
|
||||||
|
return types.get("concept", {}) or {}
|
||||||
|
|
||||||
|
def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
||||||
|
if requested and isinstance(requested, str):
|
||||||
|
return requested
|
||||||
|
# Fallback wenn nichts gesetzt ist
|
||||||
|
types = reg.get("types", {}) if isinstance(reg.get("types"), dict) else {}
|
||||||
|
return "concept" if "concept" in types else (requested or "concept")
|
||||||
|
|
||||||
|
def effective_chunk_profile(note_type: str, reg: dict) -> Optional[str]:
|
||||||
|
"""Resolve chunk_profile for type or from defaults/global.
|
||||||
|
Accepts symbolic profiles: short|medium|long|default.
|
||||||
|
"""
|
||||||
|
cfg = get_type_config(note_type, reg)
|
||||||
|
prof = (cfg.get("chunk_profile") if isinstance(cfg, dict) else None)
|
||||||
|
if isinstance(prof, str) and prof:
|
||||||
|
return prof
|
||||||
|
# defaults fallbacks
|
||||||
|
for key in ("defaults", "default", "global"):
|
||||||
|
dcfg = reg.get(key) if isinstance(reg, dict) else None
|
||||||
|
if isinstance(dcfg, dict):
|
||||||
|
dprof = dcfg.get("chunk_profile")
|
||||||
|
if isinstance(dprof, str) and dprof:
|
||||||
|
return dprof
|
||||||
|
return "default"
|
||||||
|
|
||||||
|
def effective_retriever_weight(note_type: str, reg: dict) -> Optional[float]:
|
||||||
|
"""Resolve retriever_weight for type or defaults; returns float.
|
||||||
|
"""
|
||||||
|
cfg = get_type_config(note_type, reg)
|
||||||
|
w = (cfg.get("retriever_weight") if isinstance(cfg, dict) else None)
|
||||||
|
try:
|
||||||
|
if w is not None:
|
||||||
|
return float(w)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# defaults fallbacks
|
||||||
|
for key in ("defaults", "default", "global"):
|
||||||
|
dcfg = reg.get(key) if isinstance(reg, dict) else None
|
||||||
|
if isinstance(dcfg, dict):
|
||||||
|
dw = dcfg.get("retriever_weight")
|
||||||
|
try:
|
||||||
|
if dw is not None:
|
||||||
|
return float(dw)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return 1.0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# Sonstige Helper
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
def iter_md(root: str) -> List[str]:
|
def iter_md(root: str) -> List[str]:
|
||||||
@@ -115,17 +200,15 @@ def iter_md(root: str) -> List[str]:
|
|||||||
out.append(p)
|
out.append(p)
|
||||||
return sorted(out)
|
return sorted(out)
|
||||||
|
|
||||||
|
|
||||||
def collections(prefix: str) -> Tuple[str, str, str]:
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
|
||||||
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
||||||
notes_col, _, _ = collections(prefix)
|
notes_col, _, _ = collections(prefix)
|
||||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
points, _ = client.scroll(
|
points, _ = client.scroll(
|
||||||
collection_name=notes_col,
|
collection_name=notes_col,
|
||||||
scroll_filter=f,
|
scroll_filter=f, # wichtig: scroll_filter (nicht: filter)
|
||||||
with_payload=True,
|
with_payload=True,
|
||||||
with_vectors=False,
|
with_vectors=False,
|
||||||
limit=1,
|
limit=1,
|
||||||
@@ -134,7 +217,6 @@ def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[D
|
|||||||
return None
|
return None
|
||||||
return points[0].payload or {}
|
return points[0].payload or {}
|
||||||
|
|
||||||
|
|
||||||
def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
||||||
notes_col, _, _ = collections(prefix)
|
notes_col, _, _ = collections(prefix)
|
||||||
out: Set[str] = set()
|
out: Set[str] = set()
|
||||||
@@ -158,7 +240,6 @@ def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
|||||||
break
|
break
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||||
_, chunks_col, edges_col = collections(prefix)
|
_, chunks_col, edges_col = collections(prefix)
|
||||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
@@ -172,7 +253,6 @@ def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
|
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
|
||||||
|
|
||||||
|
|
||||||
def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
||||||
notes_col, chunks_col, edges_col = collections(prefix)
|
notes_col, chunks_col, edges_col = collections(prefix)
|
||||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
@@ -187,50 +267,56 @@ def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
|||||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
|
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
|
||||||
|
|
||||||
|
|
||||||
def _resolve_mode(val: Optional[str]) -> str:
|
# --- Neu: Existenz-Checks für Artefakte (fehlertoleranter Rebuild) ---
|
||||||
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
|
|
||||||
if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
|
||||||
return "full"
|
|
||||||
if v in ("frontmatter", "fm"):
|
|
||||||
return "frontmatter"
|
|
||||||
return "body"
|
|
||||||
|
|
||||||
|
def _has_any_point(client, collection: str, note_id: str) -> bool:
|
||||||
|
"""Prüft, ob es mind. einen Punkt mit note_id in der Collection gibt."""
|
||||||
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
|
pts, _ = client.scroll(
|
||||||
|
collection_name=collection,
|
||||||
|
scroll_filter=filt,
|
||||||
|
with_payload=False,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=1,
|
||||||
|
)
|
||||||
|
return bool(pts)
|
||||||
|
|
||||||
def _env(key: str, default: str) -> str:
|
def artifacts_missing(client, prefix: str, note_id: str) -> Tuple[bool, bool]:
|
||||||
return (os.environ.get(key) or default).strip().lower()
|
"""Gibt (chunks_missing, edges_missing) zurück."""
|
||||||
|
_, chunks_col, edges_col = collections(prefix)
|
||||||
|
chunks_missing = not _has_any_point(client, chunks_col, note_id)
|
||||||
|
edges_missing = not _has_any_point(client, edges_col, note_id)
|
||||||
|
return chunks_missing, edges_missing
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
# Main
|
# Main
|
||||||
# ---------------------------------------------------------------------
|
# ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _resolve_mode(m: Optional[str]) -> str:
|
||||||
|
m = (m or "body").strip().lower()
|
||||||
|
return m if m in {"body", "frontmatter", "full"} else "body"
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
ap = argparse.ArgumentParser()
|
|
||||||
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
|
ap = argparse.ArgumentParser(
|
||||||
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
|
prog="scripts.import_markdown",
|
||||||
ap.add_argument("--purge-before-upsert", action="store_true",
|
description="Importiert Markdown-Notizen in Qdrant (Notes/Chunks/Edges)."
|
||||||
help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
|
)
|
||||||
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
|
ap.add_argument("--vault", required=True, help="Pfad zum Vault (Ordner mit .md-Dateien)")
|
||||||
ap.add_argument("--only-path", help="Exakt diesen Markdown-Pfad verarbeiten (ignoriert --note-id)")
|
ap.add_argument("--only-path", help="Nur diese Datei verarbeiten (absolut oder relativ)")
|
||||||
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
|
ap.add_argument("--apply", action="store_true", help="Schreibt nach Qdrant (sonst Dry-Run)")
|
||||||
ap.add_argument("--force-replace", action="store_true",
|
ap.add_argument("--purge-before-upsert", action="store_true", help="Alte Chunks/Edges der Note vorher löschen")
|
||||||
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
|
ap.add_argument("--force-replace", action="store_true", help="Note/Chunks/Edges unabhängig von Hash neu schreiben")
|
||||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None,
|
ap.add_argument("--note-id", help="Nur Notes mit dieser ID verarbeiten (Filter)")
|
||||||
help="Vergleichsmodus (Body | Frontmatter | Full)")
|
ap.add_argument("--note-scope-refs", action="store_true", help="Note-scope References/Backlinks erzeugen")
|
||||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
ap.add_argument("--hash-mode", help="body|frontmatter|full (Default body)")
|
||||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
|
ap.add_argument("--hash-source", help="parsed|raw (Default parsed)")
|
||||||
help="Quelle für die Hash-Berechnung (Default: parsed)")
|
ap.add_argument("--hash-normalize", help="canonical|none (Default canonical)")
|
||||||
ap.add_argument("--note-scope-refs", action="store_true",
|
ap.add_argument("--compare-text", action="store_true", help="Parsed fulltext zusätzlich direkt vergleichen")
|
||||||
help="(Optional) erzeugt zusätzlich references:note/backlink:note (Default: aus)")
|
ap.add_argument("--baseline-modes", action="store_true", help="Fehlende Hash-Varianten still nachtragen (Notes)")
|
||||||
ap.add_argument("--debug-hash-diff", action="store_true",
|
ap.add_argument("--sync-deletes", action="store_true", help="Qdrant->Vault Lösch-Sync (Dry-Run; mit --apply ausführen)")
|
||||||
help="(reserviert) optionaler Body-Diff")
|
|
||||||
ap.add_argument("--compare-text", action="store_true",
|
|
||||||
help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
|
|
||||||
ap.add_argument("--baseline-modes", action="store_true",
|
|
||||||
help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
|
|
||||||
ap.add_argument("--sync-deletes", action="store_true",
|
|
||||||
help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)")
|
|
||||||
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
@@ -241,6 +327,7 @@ def main() -> None:
|
|||||||
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
||||||
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
||||||
|
|
||||||
|
# Qdrant
|
||||||
cfg = QdrantConfig.from_env()
|
cfg = QdrantConfig.from_env()
|
||||||
if args.prefix:
|
if args.prefix:
|
||||||
cfg.prefix = args.prefix.strip()
|
cfg.prefix = args.prefix.strip()
|
||||||
@@ -248,9 +335,12 @@ def main() -> None:
|
|||||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||||
ensure_payload_indexes(client, cfg.prefix)
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
|
|
||||||
|
# Type-Registry laden (optional)
|
||||||
|
reg = load_type_registry()
|
||||||
|
|
||||||
root = os.path.abspath(args.vault)
|
root = os.path.abspath(args.vault)
|
||||||
|
|
||||||
# Dateiliste bestimmen
|
# Dateiliste
|
||||||
if args.only_path:
|
if args.only_path:
|
||||||
only = os.path.abspath(args.only_path)
|
only = os.path.abspath(args.only_path)
|
||||||
files = [only]
|
files = [only]
|
||||||
@@ -293,16 +383,15 @@ def main() -> None:
|
|||||||
|
|
||||||
processed = 0
|
processed = 0
|
||||||
for path in files:
|
for path in files:
|
||||||
# -------- Parse & Validate --------
|
|
||||||
try:
|
try:
|
||||||
parsed = read_markdown(path)
|
parsed = read_markdown(path)
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
||||||
continue
|
continue
|
||||||
if parsed is None:
|
|
||||||
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
|
|
||||||
continue
|
|
||||||
|
|
||||||
|
# --- Frontmatter prüfen ---
|
||||||
try:
|
try:
|
||||||
fm = normalize_frontmatter(parsed.frontmatter)
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
validate_required_frontmatter(fm)
|
validate_required_frontmatter(fm)
|
||||||
@@ -315,24 +404,54 @@ def main() -> None:
|
|||||||
|
|
||||||
processed += 1
|
processed += 1
|
||||||
|
|
||||||
# -------- Build new payload (includes 'hashes') --------
|
# --- Type-Registry anwenden (chunk_profile / retriever_weight) ---
|
||||||
note_pl = make_note_payload(
|
try:
|
||||||
parsed,
|
note_type = resolve_note_type(fm.get("type"), reg)
|
||||||
vault_root=root,
|
except Exception:
|
||||||
hash_mode=mode,
|
note_type = (fm.get("type") or "concept")
|
||||||
hash_normalize=norm,
|
fm["type"] = note_type or fm.get("type") or "concept"
|
||||||
hash_source=src,
|
|
||||||
file_path=path,
|
prof = effective_chunk_profile(note_type, reg)
|
||||||
)
|
if prof:
|
||||||
|
fm["chunk_profile"] = prof
|
||||||
|
|
||||||
|
weight = effective_retriever_weight(note_type, reg)
|
||||||
|
if weight is not None:
|
||||||
|
try:
|
||||||
|
fm["retriever_weight"] = float(weight)
|
||||||
|
except Exception:
|
||||||
|
pass # falls FM string-inkonsistent ist
|
||||||
|
|
||||||
|
# --- Payload aufbauen (inkl. Hashes) ---
|
||||||
|
try:
|
||||||
|
note_pl = make_note_payload(
|
||||||
|
parsed,
|
||||||
|
vault_root=root,
|
||||||
|
hash_mode=mode,
|
||||||
|
hash_normalize=norm,
|
||||||
|
hash_source=src,
|
||||||
|
file_path=path,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"path": path, "error": f"make_note_payload failed: {type(e).__name__}: {e}"}))
|
||||||
|
continue
|
||||||
|
|
||||||
if not note_pl.get("fulltext"):
|
if not note_pl.get("fulltext"):
|
||||||
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
||||||
|
|
||||||
|
# retriever_weight sicher in Note-Payload spiegeln (für spätere Filter)
|
||||||
|
if "retriever_weight" not in note_pl and fm.get("retriever_weight") is not None:
|
||||||
|
try:
|
||||||
|
note_pl["retriever_weight"] = float(fm.get("retriever_weight"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
note_id = note_pl.get("note_id") or fm.get("id")
|
note_id = note_pl.get("note_id") or fm.get("id")
|
||||||
if not note_id:
|
if not note_id:
|
||||||
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
|
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# -------- Fetch old payload --------
|
# --- bestehenden Payload laden (zum Diff) ---
|
||||||
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
||||||
has_old = old_payload is not None
|
has_old = old_payload is not None
|
||||||
|
|
||||||
@@ -352,17 +471,28 @@ def main() -> None:
|
|||||||
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
||||||
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
||||||
|
|
||||||
# -------- Chunks / Embeddings --------
|
# --- Chunks + Embeddings vorbereiten ---
|
||||||
chunk_pls: List[Dict[str, Any]] = []
|
|
||||||
try:
|
try:
|
||||||
body_text = getattr(parsed, "body", "") or ""
|
body_text = getattr(parsed, "body", "") or ""
|
||||||
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
|
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
|
||||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
chunk_pls: List[Dict[str, Any]] = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
vecs: List[List[float]] = [[0.0] * cfg.dim for _ in chunk_pls]
|
# retriever_weight auf Chunk-Payload spiegeln
|
||||||
|
if fm.get("retriever_weight") is not None:
|
||||||
|
try:
|
||||||
|
rw = float(fm.get("retriever_weight"))
|
||||||
|
for pl in chunk_pls:
|
||||||
|
# Feld nur setzen, wenn noch nicht vorhanden
|
||||||
|
if "retriever_weight" not in pl:
|
||||||
|
pl["retriever_weight"] = rw
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Embeddings (fallback: Nullvektoren)
|
||||||
|
vecs: List[List[float]] = [[0.0] * int(cfg.dim) for _ in chunk_pls]
|
||||||
if embed_texts and chunk_pls:
|
if embed_texts and chunk_pls:
|
||||||
try:
|
try:
|
||||||
texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls]
|
texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls]
|
||||||
@@ -370,10 +500,14 @@ def main() -> None:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
||||||
|
|
||||||
# -------- Edges (robust) --------
|
# --- Fehlende Artefakte in Qdrant ermitteln ---
|
||||||
|
chunks_missing, edges_missing = artifacts_missing(client, cfg.prefix, note_id)
|
||||||
|
|
||||||
|
# --- Edges (robust) ---
|
||||||
edges: List[Dict[str, Any]] = []
|
edges: List[Dict[str, Any]] = []
|
||||||
edges_failed = False
|
edges_failed = False
|
||||||
if changed and (not do_baseline_only):
|
should_build_edges = (changed and (not do_baseline_only)) or edges_missing
|
||||||
|
if should_build_edges:
|
||||||
try:
|
try:
|
||||||
note_refs = note_pl.get("references") or []
|
note_refs = note_pl.get("references") or []
|
||||||
edges = build_edges_for_note(
|
edges = build_edges_for_note(
|
||||||
@@ -385,10 +519,9 @@ def main() -> None:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
edges_failed = True
|
edges_failed = True
|
||||||
edges = []
|
edges = []
|
||||||
# WICHTIG: Wir brechen NICHT mehr ab — Note & Chunks werden geschrieben.
|
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
|
||||||
|
|
||||||
# -------- Summary --------
|
# --- Summary (stdout) ---
|
||||||
summary = {
|
summary = {
|
||||||
"note_id": note_id,
|
"note_id": note_id,
|
||||||
"title": fm.get("title"),
|
"title": fm.get("title"),
|
||||||
@@ -396,10 +529,12 @@ def main() -> None:
|
|||||||
"edges": len(edges),
|
"edges": len(edges),
|
||||||
"edges_failed": edges_failed,
|
"edges_failed": edges_failed,
|
||||||
"changed": changed,
|
"changed": changed,
|
||||||
|
"chunks_missing": chunks_missing,
|
||||||
|
"edges_missing": edges_missing,
|
||||||
"needs_baseline_for_mode": needs_baseline,
|
"needs_baseline_for_mode": needs_baseline,
|
||||||
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
||||||
"apply" if args.apply and changed else
|
"apply" if args.apply and (changed or chunks_missing or edges_missing) else
|
||||||
"apply-skip-unchanged" if args.apply and not changed else
|
"apply-skip-unchanged" if args.apply and not (changed or chunks_missing or edges_missing) else
|
||||||
"dry-run"),
|
"dry-run"),
|
||||||
"path": note_pl["path"],
|
"path": note_pl["path"],
|
||||||
"hash_mode": mode,
|
"hash_mode": mode,
|
||||||
@@ -409,7 +544,7 @@ def main() -> None:
|
|||||||
}
|
}
|
||||||
print(json.dumps(summary, ensure_ascii=False))
|
print(json.dumps(summary, ensure_ascii=False))
|
||||||
|
|
||||||
# -------- Writes --------
|
# --- Writes ---
|
||||||
if not args.apply:
|
if not args.apply:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -425,21 +560,29 @@ def main() -> None:
|
|||||||
upsert_batch(client, notes_name, note_pts)
|
upsert_batch(client, notes_name, note_pts)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not changed:
|
# Wenn nichts geändert und keine Artefakte fehlen → nichts zu tun
|
||||||
|
if not changed and not (chunks_missing or edges_missing):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if args.purge_before_upsert and has_old:
|
# Purge nur bei echten Änderungen (unverändert + fehlende Artefakte ≠ Purge)
|
||||||
|
if args.purge_before_upsert and has_old and changed:
|
||||||
try:
|
try:
|
||||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
||||||
|
|
||||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
# Note nur bei Änderungen neu schreiben
|
||||||
upsert_batch(client, notes_name, note_pts)
|
if changed:
|
||||||
if chunk_pls:
|
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||||
|
upsert_batch(client, notes_name, note_pts)
|
||||||
|
|
||||||
|
# Chunks schreiben, wenn geändert ODER vorher fehlend
|
||||||
|
if chunk_pls and (changed or chunks_missing):
|
||||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||||
upsert_batch(client, chunks_name, chunk_pts)
|
upsert_batch(client, chunks_name, chunk_pts)
|
||||||
if edges:
|
|
||||||
|
# Edges schreiben, wenn vorhanden und (geändert ODER vorher fehlend)
|
||||||
|
if edges and (changed or edges_missing):
|
||||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||||
upsert_batch(client, edges_name, edge_pts)
|
upsert_batch(client, edges_name, edge_pts)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/ollama_tool_runner.py — Minimaler Tool-Caller für Ollama + mindnet
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
Führt den Tool-Call-Loop für mindnet_query / mindnet_subgraph aus.
|
||||||
|
Nutzt /tools/ollama (Schema) und deine FastAPI (/query, /graph).
|
||||||
|
ENV:
|
||||||
|
OLLAMA=http://127.0.0.1:11434
|
||||||
|
API_BASE=http://127.0.0.1:8000
|
||||||
|
MODEL=llama3.1
|
||||||
|
Nutzung:
|
||||||
|
python3 scripts/ollama_tool_runner.py "Frage an den Agenten"
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
import os, sys, json, requests
|
||||||
|
|
||||||
|
OLLAMA = os.getenv("OLLAMA", "http://127.0.0.1:11434")
|
||||||
|
API_BASE = os.getenv("API_BASE", "http://127.0.0.1:8000")
|
||||||
|
MODEL = os.getenv("MODEL", "llama3.1")
|
||||||
|
|
||||||
|
def load_tools():
|
||||||
|
r = requests.get(f"{API_BASE}/tools/ollama", timeout=30)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()["tools"]
|
||||||
|
|
||||||
|
def call_model(messages, tools):
|
||||||
|
payload = {
|
||||||
|
"model": MODEL,
|
||||||
|
"messages": messages,
|
||||||
|
"tools": tools,
|
||||||
|
"stream": False,
|
||||||
|
"tool_choice": "auto",
|
||||||
|
}
|
||||||
|
r = requests.post(f"{OLLAMA}/api/chat", json=payload, timeout=180)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
def execute_tool(name: str, arguments: dict):
|
||||||
|
if name == "mindnet_query":
|
||||||
|
# mappe expand_depth/edge_types auf /query-Body
|
||||||
|
body = {
|
||||||
|
"mode": "hybrid",
|
||||||
|
"top_k": int(arguments.get("top_k", 10)),
|
||||||
|
"expand": {
|
||||||
|
"depth": int(arguments.get("expand_depth", 1)),
|
||||||
|
"edge_types": arguments.get("edge_types") or ["references","belongs_to","prev","next"],
|
||||||
|
},
|
||||||
|
"filters": arguments.get("filters"),
|
||||||
|
}
|
||||||
|
if "query_vector" in arguments:
|
||||||
|
body["query_vector"] = arguments["query_vector"]
|
||||||
|
if "query" in arguments:
|
||||||
|
body["query"] = arguments["query"]
|
||||||
|
r = requests.post(f"{API_BASE}/query", json=body, timeout=120)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
if name == "mindnet_subgraph":
|
||||||
|
params = {
|
||||||
|
"depth": int(arguments.get("depth", 1)),
|
||||||
|
}
|
||||||
|
edge_types = arguments.get("edge_types")
|
||||||
|
if edge_types:
|
||||||
|
# mehrfacher Query-Param 'edge_types' wird von requests automatisch gebaut
|
||||||
|
params["edge_types"] = edge_types
|
||||||
|
note_id = arguments["note_id"]
|
||||||
|
r = requests.get(f"{API_BASE}/graph/{note_id}", params=params, timeout=60)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
raise ValueError(f"Unbekanntes Tool: {name}")
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
print("Usage: python3 scripts/ollama_tool_runner.py \"Deine Frage\"")
|
||||||
|
return 2
|
||||||
|
|
||||||
|
user_prompt = sys.argv[1]
|
||||||
|
tools = load_tools()
|
||||||
|
messages = [{"role": "user", "content": user_prompt}]
|
||||||
|
|
||||||
|
while True:
|
||||||
|
resp = call_model(messages, tools)
|
||||||
|
msg = resp.get("message") or {}
|
||||||
|
tool_calls = msg.get("tool_calls") or []
|
||||||
|
|
||||||
|
if not tool_calls:
|
||||||
|
# finale Antwort
|
||||||
|
print("\n=== Antwort ===\n")
|
||||||
|
print(msg.get("content", "").strip())
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# Führe Toolcalls nacheinander aus
|
||||||
|
for tc in tool_calls:
|
||||||
|
name = tc["function"]["name"]
|
||||||
|
args = tc["function"].get("arguments") or {}
|
||||||
|
if isinstance(args, str):
|
||||||
|
try: args = json.loads(args)
|
||||||
|
except Exception: args = {"raw": args}
|
||||||
|
|
||||||
|
print(f"[tool] {name}({args})")
|
||||||
|
result = execute_tool(name, args)
|
||||||
|
|
||||||
|
# Übergib Toolresultat ans Modell zurück
|
||||||
|
messages.append({"role": "tool", "content": json.dumps(result), "name": name})
|
||||||
|
|
||||||
|
# füttere auch die originale Antwort (falls enthalten) in den Verlauf
|
||||||
|
if msg.get("content"):
|
||||||
|
messages.append({"role": "assistant", "content": msg["content"]})
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/payload_dryrun.py (zeigt, was VOR dem Upsert tatsächlich in den Payloads steht)
|
||||||
|
- KEIN Überschreiben der Note-Payload mehr
|
||||||
|
- types.yaml ist maßgeblich (gemäß app/core/note_payload.py & chunk_payload.py)
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, os, json
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||||
|
from app.core.note_payload import make_note_payload
|
||||||
|
from app.core.chunker import assemble_chunks
|
||||||
|
from app.core.chunk_payload import make_chunk_payloads
|
||||||
|
try:
|
||||||
|
from app.core.derive_edges import build_edges_for_note
|
||||||
|
except Exception:
|
||||||
|
from app.core.edges import build_edges_for_note # type: ignore
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--vault", required=True)
|
||||||
|
ap.add_argument("--note-id")
|
||||||
|
ap.add_argument("--with-edges", action="store_true")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
root = os.path.abspath(args.vault)
|
||||||
|
|
||||||
|
files: List[str] = []
|
||||||
|
for dp, _, fns in os.walk(root):
|
||||||
|
for fn in fns:
|
||||||
|
if fn.lower().endswith(".md"):
|
||||||
|
files.append(os.path.join(dp, fn))
|
||||||
|
files.sort()
|
||||||
|
|
||||||
|
for path in files:
|
||||||
|
parsed = read_markdown(path)
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
|
try:
|
||||||
|
validate_required_frontmatter(fm)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"path": path, "error": f"invalid frontmatter: {e}"}))
|
||||||
|
continue
|
||||||
|
|
||||||
|
if args.note_id and fm.get("id") != args.note_id:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Note-Payload exakt so, wie der Importer ihn baut (types.yaml maßgeblich)
|
||||||
|
note_pl = make_note_payload(parsed,
|
||||||
|
vault_root=root,
|
||||||
|
hash_mode="body",
|
||||||
|
hash_normalize="canonical",
|
||||||
|
hash_source="parsed",
|
||||||
|
file_path=path)
|
||||||
|
|
||||||
|
body_text = getattr(parsed, "body", "") or ""
|
||||||
|
chunks = assemble_chunks(fm["id"], body_text, fm.get("type","concept"))
|
||||||
|
|
||||||
|
chunk_note = {
|
||||||
|
"frontmatter": fm,
|
||||||
|
"id": fm.get("id"),
|
||||||
|
"type": fm.get("type"),
|
||||||
|
"title": fm.get("title"),
|
||||||
|
"path": note_pl.get("path") or path,
|
||||||
|
"note_id": note_pl.get("note_id"),
|
||||||
|
"tags": fm.get("tags"),
|
||||||
|
}
|
||||||
|
# make_chunk_payloads bestimmt Werte ebenfalls aus types.yaml (Frontmatter wird ignoriert)
|
||||||
|
chunk_pls = make_chunk_payloads(
|
||||||
|
chunk_note,
|
||||||
|
note_pl["path"],
|
||||||
|
chunks,
|
||||||
|
note_text=body_text,
|
||||||
|
types_cfg=None, # Loader aus Datei; kein Override von außen
|
||||||
|
file_path=path,
|
||||||
|
)
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"note_id": note_pl.get("note_id") or fm.get("id"),
|
||||||
|
"title": fm.get("title"),
|
||||||
|
"type": fm.get("type"),
|
||||||
|
"note_payload": {
|
||||||
|
"retriever_weight": note_pl.get("retriever_weight"),
|
||||||
|
"chunk_profile": note_pl.get("chunk_profile")
|
||||||
|
},
|
||||||
|
"chunks_summary": {
|
||||||
|
"count": len(chunk_pls),
|
||||||
|
"first": [
|
||||||
|
{k: chunk_pls[i].get(k) for k in ("chunk_id","index","ord","retriever_weight","chunk_profile","neighbors_prev","neighbors_next")}
|
||||||
|
for i in range(min(3, len(chunk_pls)))
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"path": note_pl.get("path")
|
||||||
|
}
|
||||||
|
|
||||||
|
if args.with_edges:
|
||||||
|
edges = build_edges_for_note(
|
||||||
|
note_id=note_pl.get("note_id") or fm.get("id"),
|
||||||
|
chunk_payloads=chunk_pls,
|
||||||
|
note_level_refs=note_pl.get("references") or [],
|
||||||
|
include_note_scope_refs=False,
|
||||||
|
)
|
||||||
|
kinds = {}
|
||||||
|
for e in edges:
|
||||||
|
k = (e.get("relation") or e.get("kind") or "edge")
|
||||||
|
kinds[k] = kinds.get(k, 0) + 1
|
||||||
|
out["edges_summary"] = {"total": len(edges), "by_kind": kinds}
|
||||||
|
|
||||||
|
print(json.dumps(out, ensure_ascii=False))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+33
-10
@@ -2,28 +2,33 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Name: scripts/reset_qdrant.py
|
Name: scripts/reset_qdrant.py
|
||||||
Version: v1.1.1 (2025-09-05)
|
Version: v1.2.0 (2025-11-11)
|
||||||
Kurzbeschreibung:
|
Kurzbeschreibung:
|
||||||
Sicheres Zurücksetzen der Qdrant-Collections für EIN Projektpräfix. Das Skript
|
Sicheres Zurücksetzen der Qdrant-Collections für EIN Projektpräfix. Das Skript
|
||||||
ermittelt zunächst die tatsächlich betroffenen Collections und zeigt eine
|
ermittelt zunächst die tatsächlich betroffenen Collections und zeigt eine
|
||||||
Übersicht an. Anschließend wird der User um Bestätigung gebeten (interaktive
|
Übersicht an. Anschließend wird der User um Bestätigung gebeten (interaktive
|
||||||
Abfrage), bevor eine der beiden Aktionen ausgeführt wird:
|
Abfrage), bevor eine der Aktionen ausgeführt wird:
|
||||||
|
|
||||||
- wipe: Löscht die Collections komplett und legt sie gemäß Projekt-Defaults neu an
|
- wipe: Löscht die Collections komplett und legt sie gemäß Projekt-Defaults neu an.
|
||||||
- truncate: Löscht nur alle Points (Inhalte), behält die Collection-Settings
|
**Neu**: Danach werden *immer* die Payload-Indizes & Schema-Ergänzungen
|
||||||
|
über ensure_payload_indexes() idempotent eingerichtet (falls nicht mit --no-indexes deaktiviert).
|
||||||
|
- truncate: Löscht nur alle Points (Inhalte), behält die Collection-Settings.
|
||||||
|
**Neu**: Optional (Default) werden im Anschluss die Payload-Indizes geprüft/ergänzt.
|
||||||
|
|
||||||
Aufruf (aus Projekt-Root, im venv):
|
Aufruf (aus Projekt-Root, im venv):
|
||||||
python3 -m scripts.reset_qdrant --mode wipe [--prefix PREFIX] [--yes] [--dry-run]
|
python3 -m scripts.reset_qdrant --mode wipe [--prefix PREFIX] [--yes] [--dry-run]
|
||||||
python3 -m scripts.reset_qdrant --mode truncate [--prefix PREFIX] [--yes] [--dry-run]
|
python3 -m scripts.reset_qdrant --mode truncate [--prefix PREFIX] [--yes] [--dry-run]
|
||||||
|
|
||||||
Parameter:
|
Parameter:
|
||||||
--mode wipe | truncate (Pflicht)
|
--mode wipe | truncate (Pflicht)
|
||||||
--prefix Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')
|
--prefix Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')
|
||||||
--yes Keine Rückfrage, direkt ausführen (CI/CD geeignet)
|
--yes Keine Rückfrage, direkt ausführen (CI/CD geeignet)
|
||||||
--dry-run Nur anzeigen, was passieren würde; nichts ändern
|
--dry-run Nur anzeigen, was passieren würde; nichts ändern
|
||||||
|
--no-indexes Überspringt den Schritt ensure_payload_indexes() (Standard: Indizes/Schema werden geprüft/ergänzt)
|
||||||
|
|
||||||
Umgebungsvariablen (optional):
|
Umgebungsvariablen (optional):
|
||||||
QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)
|
QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM (Default 384)
|
||||||
|
(Zusätzliche ENV für Named Vectors etc. werden innerhalb ensure_collections/ensure_payload_indexes berücksichtigt.)
|
||||||
|
|
||||||
Sicherheitsmerkmale:
|
Sicherheitsmerkmale:
|
||||||
- Betrachtet ausschließlich Collections des angegebenen Präfixes.
|
- Betrachtet ausschließlich Collections des angegebenen Präfixes.
|
||||||
@@ -34,6 +39,8 @@ Exitcodes:
|
|||||||
0 = OK, 1 = abgebrochen/keine Aktion, 2 = Verbindungs-/Konfigurationsfehler
|
0 = OK, 1 = abgebrochen/keine Aktion, 2 = Verbindungs-/Konfigurationsfehler
|
||||||
|
|
||||||
Changelog:
|
Changelog:
|
||||||
|
v1.2.0: ensure_payload_indexes() nach wipe/truncate standardmäßig ausführen (idempotent); --no-indexes Flag ergänzt.
|
||||||
|
v1.1.1: Stabilisierung & Preview (2025-09-05).
|
||||||
v1.1.0: Interaktive Bestätigung, --yes/--dry-run hinzugefügt, Preview der betroffenen Collections.
|
v1.1.0: Interaktive Bestätigung, --yes/--dry-run hinzugefügt, Preview der betroffenen Collections.
|
||||||
v1.0.0: Wipe/Truncate ohne Bestätigungsabfrage.
|
v1.0.0: Wipe/Truncate ohne Bestätigungsabfrage.
|
||||||
"""
|
"""
|
||||||
@@ -46,7 +53,13 @@ from typing import List
|
|||||||
from qdrant_client import QdrantClient
|
from qdrant_client import QdrantClient
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names
|
from app.core.qdrant import (
|
||||||
|
QdrantConfig,
|
||||||
|
get_client,
|
||||||
|
ensure_collections,
|
||||||
|
ensure_payload_indexes,
|
||||||
|
collection_names,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def resolve_existing_collections(client: QdrantClient, prefix: str) -> List[str]:
|
def resolve_existing_collections(client: QdrantClient, prefix: str) -> List[str]:
|
||||||
@@ -111,12 +124,13 @@ def wipe_collections(client: QdrantClient, all_col_names: List[str], existing: L
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
ap = argparse.ArgumentParser(description="Wipe oder truncate mindnet-Collections in Qdrant (mit Bestätigung).")
|
ap = argparse.ArgumentParser(description="Wipe oder truncate mindnet-Collections in Qdrant (mit Bestätigung & Index-Setup).")
|
||||||
ap.add_argument("--mode", choices=["wipe", "truncate"], required=True,
|
ap.add_argument("--mode", choices=["wipe", "truncate"], required=True,
|
||||||
help="wipe = Collections löschen & neu anlegen; truncate = nur Inhalte löschen")
|
help="wipe = Collections löschen & neu anlegen; truncate = nur Inhalte löschen")
|
||||||
ap.add_argument("--prefix", help="Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')")
|
ap.add_argument("--prefix", help="Collection-Prefix (Default: env COLLECTION_PREFIX oder 'mindnet')")
|
||||||
ap.add_argument("--yes", action="store_true", help="Ohne Rückfrage ausführen (CI/CD)")
|
ap.add_argument("--yes", action="store_true", help="Ohne Rückfrage ausführen (CI/CD)")
|
||||||
ap.add_argument("--dry-run", action="store_true", help="Nur anzeigen, was passieren würde; nichts ändern")
|
ap.add_argument("--dry-run", action="store_true", help="Nur anzeigen, was passieren würde; nichts ändern")
|
||||||
|
ap.add_argument("--no-indexes", action="store_true", help="Überspringt ensure_payload_indexes()")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
# Qdrant-Konfiguration
|
# Qdrant-Konfiguration
|
||||||
@@ -155,13 +169,22 @@ def main():
|
|||||||
if args.mode == "wipe":
|
if args.mode == "wipe":
|
||||||
wipe_collections(client, all_col_names, existing)
|
wipe_collections(client, all_col_names, existing)
|
||||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||||
|
if not args.no_indexes:
|
||||||
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
print(f"Wiped & recreated (Prefix={cfg.prefix}): {all_col_names}")
|
print(f"Wiped & recreated (Prefix={cfg.prefix}): {all_col_names}")
|
||||||
|
if not args.no_indexes:
|
||||||
|
print("Payload-Indizes & Schema-Ergänzungen geprüft/ergänzt (ensure_payload_indexes).")
|
||||||
else:
|
else:
|
||||||
if not existing:
|
if not existing:
|
||||||
print("Keine existierenden Collections zum Truncaten gefunden. Beende ohne Aktion.")
|
print("Keine existierenden Collections zum Truncaten gefunden. Beende ohne Aktion.")
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
delete_all_points(client, existing)
|
delete_all_points(client, existing)
|
||||||
|
if not args.no_indexes:
|
||||||
|
# Auch nach truncate sicherstellen, dass neue/fehlende Indizes ergänzt werden
|
||||||
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
print(f"Truncated (deleted all points in): {existing}")
|
print(f"Truncated (deleted all points in): {existing}")
|
||||||
|
if not args.no_indexes:
|
||||||
|
print("Payload-Indizes & Schema-Ergänzungen geprüft/ergänzt (ensure_payload_indexes).")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/wp04_smoketest.py — E2E-Schnelltest der WP-04 Endpunkte
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
- Holt exemplarisch einen Vektor aus {prefix}_chunks (falls verfügbar),
|
||||||
|
- ruft POST /query mit diesem Vektor auf,
|
||||||
|
- ruft GET /graph/<note_id> für die Top-Note auf,
|
||||||
|
- druckt Kurzresultate auf STDOUT.
|
||||||
|
|
||||||
|
Kompatibilität:
|
||||||
|
Python 3.12+, requests, qdrant-client
|
||||||
|
Version:
|
||||||
|
0.1.0 (Erstanlage)
|
||||||
|
Stand:
|
||||||
|
2025-10-07
|
||||||
|
Bezug:
|
||||||
|
WP-04 /query & /graph
|
||||||
|
Nutzung:
|
||||||
|
export QDRANT_URL="http://localhost:6333"
|
||||||
|
export MINDNET_PREFIX="mindnet"
|
||||||
|
python3 scripts/wp04_smoketest.py --api http://localhost:8000
|
||||||
|
Änderungsverlauf:
|
||||||
|
0.1.0 (2025-10-07) – Erstanlage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import argparse
|
||||||
|
import requests
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
|
||||||
|
|
||||||
|
def get_any_chunk_vector(client: QdrantClient, prefix: str):
|
||||||
|
"""Holt eine beliebige Chunk-Vector/Note-ID-Kombi aus Qdrant (falls vorhanden)."""
|
||||||
|
col = f"{prefix}_chunks"
|
||||||
|
pts, _ = client.scroll(collection_name=col, limit=1, with_vectors=True, with_payload=True)
|
||||||
|
if not pts:
|
||||||
|
return None, None, None
|
||||||
|
p = pts[0]
|
||||||
|
vec = p.vector
|
||||||
|
payload = p.payload or {}
|
||||||
|
node_id = str(p.id)
|
||||||
|
note_id = payload.get("note_id")
|
||||||
|
return vec, node_id, note_id
|
||||||
|
|
||||||
|
|
||||||
|
def run(api_base: str, qdrant_url: str, prefix: str) -> int:
|
||||||
|
print(f"[i] API: {api_base}")
|
||||||
|
print(f"[i] Qdrant: {qdrant_url}")
|
||||||
|
client = QdrantClient(url=qdrant_url)
|
||||||
|
|
||||||
|
vec, node_id, note_id = get_any_chunk_vector(client, prefix)
|
||||||
|
if vec is None:
|
||||||
|
print("[!] Keine Chunks mit Vektoren gefunden. Bitte Import (WP-03) prüfen.")
|
||||||
|
return 2
|
||||||
|
|
||||||
|
print(f"[i] Beispiel-Chunk-ID: {node_id}, Note-ID: {note_id}")
|
||||||
|
print("[i] Rufe /query (hybrid) mit Beispielvektor auf ...")
|
||||||
|
|
||||||
|
q_payload = {
|
||||||
|
"mode": "hybrid",
|
||||||
|
"query_vector": vec, # 384d
|
||||||
|
"top_k": 5,
|
||||||
|
"expand": {"depth": 1, "edge_types": ["references", "belongs_to", "prev", "next"]},
|
||||||
|
}
|
||||||
|
r = requests.post(f"{api_base.rstrip('/')}/query", json=q_payload, timeout=60)
|
||||||
|
print(f"[i] /query Status: {r.status_code}")
|
||||||
|
if r.status_code != 200:
|
||||||
|
print(r.text)
|
||||||
|
return 3
|
||||||
|
|
||||||
|
data = r.json()
|
||||||
|
hits = data.get("results", [])
|
||||||
|
print(f"[i] Treffer: {len(hits)}")
|
||||||
|
for i, h in enumerate(hits, 1):
|
||||||
|
print(f" {i:02d}. node_id={h['node_id']} note_id={h.get('note_id')} total={h['total_score']:.3f}")
|
||||||
|
|
||||||
|
# Graph-Test (wenn Note-ID vorhanden)
|
||||||
|
gid = note_id or (hits[0].get("note_id") if hits else None)
|
||||||
|
if gid:
|
||||||
|
print(f"[i] Rufe /graph/{gid} ...")
|
||||||
|
r2 = requests.get(f"{api_base.rstrip('/')}/graph/{gid}?depth=1", timeout=60)
|
||||||
|
print(f"[i] /graph Status: {r2.status_code}")
|
||||||
|
if r2.status_code == 200:
|
||||||
|
g = r2.json()
|
||||||
|
print(f" Nodes: {len(g.get('nodes', []))}, Edges: {len(g.get('edges', []))}")
|
||||||
|
else:
|
||||||
|
print(r2.text)
|
||||||
|
|
||||||
|
print("[✔] Smoke-Test abgeschlossen.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--api", default="http://localhost:8000", help="Basis-URL der FastAPI.")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
qdrant_url = os.getenv("QDRANT_URL", "http://127.0.0.1:6333")
|
||||||
|
prefix = os.getenv("MINDNET_PREFIX", "mindnet")
|
||||||
|
sys.exit(run(args.api, qdrant_url, prefix))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/assert_payload_schema.py (compat v1.1)
|
||||||
|
|
||||||
|
Prüft, ob die erwarteten Payload-Indizes (payload_schema) auf den drei Collections vorhanden sind.
|
||||||
|
Kompatibel mit älteren qdrant-client Versionen (ohne with_payload_schema im Wrapper).
|
||||||
|
|
||||||
|
Collections & Pflichtfelder:
|
||||||
|
- mindnet_notes : note_id, type, title, updated, tags
|
||||||
|
- mindnet_chunks : note_id, chunk_id, index, type, tags
|
||||||
|
- mindnet_edges : note_id, kind, scope, source_id, target_id, chunk_id
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 -m tests.assert_payload_schema
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
REQUIRED = {
|
||||||
|
"notes": ["note_id", "type", "title", "updated", "tags"],
|
||||||
|
"chunks": ["note_id", "chunk_id", "index", "type", "tags"],
|
||||||
|
"edges": ["note_id", "kind", "scope", "source_id", "target_id", "chunk_id"],
|
||||||
|
}
|
||||||
|
|
||||||
|
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||||
|
if hasattr(obj, "model_dump"):
|
||||||
|
return obj.model_dump()
|
||||||
|
if hasattr(obj, "dict"):
|
||||||
|
return obj.dict()
|
||||||
|
return obj if isinstance(obj, dict) else {}
|
||||||
|
|
||||||
|
def _cfg_base_url(cfg) -> str:
|
||||||
|
if getattr(cfg, "url", None):
|
||||||
|
return cfg.url.rstrip('/')
|
||||||
|
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||||
|
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||||
|
return f"http://{host}:{port}"
|
||||||
|
|
||||||
|
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||||
|
req = urllib.request.Request(url, method="GET")
|
||||||
|
if api_key:
|
||||||
|
req.add_header("api-key", api_key)
|
||||||
|
req.add_header("Authorization", f"Bearer {api_key}")
|
||||||
|
req.add_header("Accept", "application/json")
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = resp.read()
|
||||||
|
return json.loads(data.decode("utf-8"))
|
||||||
|
|
||||||
|
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||||
|
# 1) OpenAPI client mit Flag
|
||||||
|
try:
|
||||||
|
oc = getattr(client, "openapi_client", None)
|
||||||
|
if oc is not None and hasattr(oc, "collections_api"):
|
||||||
|
api = oc.collections_api
|
||||||
|
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
return (d.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# 2) Wrapper (manche Versionen liefern Schema auch ohne Flag)
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
ps = (d.get("result") or {}).get("payload_schema")
|
||||||
|
if ps is not None:
|
||||||
|
return ps
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# 3) Direkter HTTP-Call
|
||||||
|
try:
|
||||||
|
base = _cfg_base_url(cfg)
|
||||||
|
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||||
|
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||||
|
return (raw.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def ensure_fields(schema: Dict[str, dict] | None, required: List[str]) -> Dict[str, bool]:
|
||||||
|
ks = set((schema or {}).keys())
|
||||||
|
return {k: (k in ks) for k in required}
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
names = {"notes": notes, "chunks": chunks, "edges": edges}
|
||||||
|
|
||||||
|
report = {}
|
||||||
|
ok_all = True
|
||||||
|
for kind, col in names.items():
|
||||||
|
sch = get_collection_payload_schema(client, cfg, col)
|
||||||
|
checks = ensure_fields(sch, REQUIRED[kind])
|
||||||
|
ok = all(checks.values())
|
||||||
|
ok_all = ok_all and ok
|
||||||
|
report[kind] = {
|
||||||
|
"collection": col,
|
||||||
|
"ok": ok,
|
||||||
|
"missing": [k for k,v in checks.items() if not v],
|
||||||
|
"present": [k for k,v in checks.items() if v],
|
||||||
|
}
|
||||||
|
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "ok": ok_all, "report": report}, ensure_ascii=False, indent=2))
|
||||||
|
if not ok_all:
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
check_types_registry_edges.py
|
||||||
|
--------------------------------------------------
|
||||||
|
Prüft, ob die in config/types.yaml hinterlegten Default-Kanten
|
||||||
|
pro Note-Typ tatsächlich in der Qdrant *edges*-Collection auftauchen.
|
||||||
|
"""
|
||||||
|
import os, sys, json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, List, Optional, Tuple, Set
|
||||||
|
from dataclasses import dataclass, asdict
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yaml # PyYAML
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"PyYAML not installed: {e}"})); sys.exit(2)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from qdrant_client.http import models as rest # noqa: F401
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"qdrant-client not installed: {e}"})); sys.exit(2)
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Cfg:
|
||||||
|
url: str
|
||||||
|
api_key: Optional[str]
|
||||||
|
prefix: str
|
||||||
|
notes: str
|
||||||
|
edges: str
|
||||||
|
types_file: Path
|
||||||
|
|
||||||
|
def _env_prefix() -> Optional[str]:
|
||||||
|
for k in ("COLLECTION_PREFIX", "MINDNET_PREFIX"):
|
||||||
|
v = os.environ.get(k, "").strip()
|
||||||
|
if v:
|
||||||
|
return v
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _load_types_yaml(path: Path) -> Dict:
|
||||||
|
if not path.exists():
|
||||||
|
print(json.dumps({"warn": f"types.yaml fehlt: {path}. Fallback: keine Vorgaben."}))
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
with path.open("r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f) or {}
|
||||||
|
return data if isinstance(data, dict) else {}
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"warn": f"types.yaml defekt ({path}): {e}. Fallback: keine Vorgaben."}))
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def _cfg_from_env(argv_prefix: Optional[str]) -> Cfg:
|
||||||
|
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
|
||||||
|
api_key = os.environ.get("QDRANT_API_KEY", "").strip() or None
|
||||||
|
prefix = (argv_prefix or _env_prefix() or "mindnet").strip()
|
||||||
|
notes = f"{prefix}_notes"
|
||||||
|
edges = f"{prefix}_edges"
|
||||||
|
types_path = Path(os.environ.get("TYPES_FILE", "config/types.yaml")).resolve()
|
||||||
|
return Cfg(url=url, api_key=api_key, prefix=prefix, notes=notes, edges=edges, types_file=types_path)
|
||||||
|
|
||||||
|
def _mk_client(cfg: Cfg) -> QdrantClient:
|
||||||
|
return QdrantClient(url=cfg.url, api_key=cfg.api_key, timeout=30.0)
|
||||||
|
|
||||||
|
def _scroll_all_notes(client: QdrantClient, notes_col: str) -> Dict[str, Dict]:
|
||||||
|
out = {}
|
||||||
|
offset = None
|
||||||
|
while True:
|
||||||
|
points, offset = client.scroll(
|
||||||
|
collection_name=notes_col,
|
||||||
|
scroll_filter=None, limit=256, offset=offset,
|
||||||
|
with_payload=True, with_vectors=False,
|
||||||
|
)
|
||||||
|
if not points:
|
||||||
|
break
|
||||||
|
for p in points:
|
||||||
|
payload = p.payload or {}
|
||||||
|
nid = payload.get("note_id") or payload.get("id") or payload.get("uid") or payload.get("slug")
|
||||||
|
if not nid:
|
||||||
|
nid = str(p.id)
|
||||||
|
out[str(nid)] = payload
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _scroll_all_edges(client: QdrantClient, edges_col: str) -> List[Tuple[str, dict]]:
|
||||||
|
out = []
|
||||||
|
offset = None
|
||||||
|
while True:
|
||||||
|
points, offset = client.scroll(
|
||||||
|
collection_name=edges_col,
|
||||||
|
scroll_filter=None, limit=256, offset=offset,
|
||||||
|
with_payload=True, with_vectors=False,
|
||||||
|
)
|
||||||
|
if not points:
|
||||||
|
break
|
||||||
|
for p in points:
|
||||||
|
out.append((str(p.id), p.payload or {}))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _guess_src_note_id(ed_pl: dict) -> Optional[str]:
|
||||||
|
for k in ("src_note_id", "note_id", "src_id", "src"):
|
||||||
|
nid = ed_pl.get(k)
|
||||||
|
if nid:
|
||||||
|
return str(nid)
|
||||||
|
for k in ("src_ref", "from"):
|
||||||
|
nid = ed_pl.get(k)
|
||||||
|
if isinstance(nid, dict):
|
||||||
|
for kk in ("note_id", "id"):
|
||||||
|
if nid.get(kk):
|
||||||
|
return str(nid[kk])
|
||||||
|
elif isinstance(nid, str):
|
||||||
|
return nid
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _edge_type(ed_pl: dict) -> Optional[str]:
|
||||||
|
for k in ("edge_type", "type", "rel", "relation"):
|
||||||
|
if ed_pl.get(k):
|
||||||
|
return str(ed_pl[k])
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _cfg_to_jsonable(cfg: Cfg) -> dict:
|
||||||
|
d = asdict(cfg)
|
||||||
|
d["types_file"] = str(d.get("types_file"))
|
||||||
|
return d
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
import argparse
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", help="Collection-Prefix (mindnet, etc.)")
|
||||||
|
args = ap.parse_args(argv)
|
||||||
|
|
||||||
|
cfg = _cfg_from_env(args.prefix)
|
||||||
|
print(json.dumps({"cfg": _cfg_to_jsonable(cfg)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
tr = _load_types_yaml(cfg.types_file)
|
||||||
|
types_def = (tr.get("types") if isinstance(tr, dict) else {}) or {}
|
||||||
|
print(json.dumps({"types_defined": list(types_def.keys())}, ensure_ascii=False))
|
||||||
|
|
||||||
|
client = _mk_client(cfg)
|
||||||
|
|
||||||
|
notes = _scroll_all_notes(client, cfg.notes)
|
||||||
|
print(json.dumps({"notes_count": len(notes)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
edges = _scroll_all_edges(client, cfg.edges)
|
||||||
|
print(json.dumps({"edges_count": len(edges)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
note_type = {}
|
||||||
|
for nid, pl in notes.items():
|
||||||
|
t = pl.get("type") or "concept"
|
||||||
|
note_type[str(nid)] = str(t)
|
||||||
|
|
||||||
|
seen = {}
|
||||||
|
counts = {}
|
||||||
|
|
||||||
|
for edge_pid, ed_pl in edges:
|
||||||
|
et = _edge_type(ed_pl)
|
||||||
|
if not et:
|
||||||
|
continue
|
||||||
|
src_nid = _guess_src_note_id(ed_pl)
|
||||||
|
if not src_nid:
|
||||||
|
continue
|
||||||
|
t = note_type.get(src_nid)
|
||||||
|
if not t:
|
||||||
|
continue
|
||||||
|
seen.setdefault(t, set()).add(et)
|
||||||
|
counts.setdefault(t, {}).setdefault(et, 0)
|
||||||
|
counts[t][et] += 1
|
||||||
|
|
||||||
|
expected = {}
|
||||||
|
for tname, tdef in types_def.items():
|
||||||
|
eddefs = (tdef or {}).get("edge_defaults") or []
|
||||||
|
expected[tname] = set([str(x) for x in eddefs if x])
|
||||||
|
|
||||||
|
for tname, exp in expected.items():
|
||||||
|
obs = seen.get(tname, set())
|
||||||
|
missing = sorted(list(exp - obs))
|
||||||
|
extra = sorted(list(obs - exp))
|
||||||
|
print(json.dumps({
|
||||||
|
"type": tname,
|
||||||
|
"expected_defaults": sorted(list(exp)),
|
||||||
|
"observed": sorted(list(obs)),
|
||||||
|
"missing": missing,
|
||||||
|
"extra": extra,
|
||||||
|
"counts": counts.get(tname, {}),
|
||||||
|
}, ensure_ascii=False))
|
||||||
|
|
||||||
|
if not expected:
|
||||||
|
print(json.dumps({"warn": "Keine Typ-Defaults in types.yaml gefunden (edge_defaults leer?)."}))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import os, json, argparse, random
|
||||||
|
from typing import Any, Dict, List, Tuple, Optional
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
|
||||||
|
def _get_prefix(explicit: Optional[str]) -> str:
|
||||||
|
return (explicit or os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet").strip()
|
||||||
|
|
||||||
|
def _names(prefix: str) -> Dict[str,str]:
|
||||||
|
return {
|
||||||
|
"notes": f"{prefix}_notes",
|
||||||
|
"chunks": f"{prefix}_chunks",
|
||||||
|
"edges": f"{prefix}_edges",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _scroll(client: QdrantClient, collection: str, *, limit=5, with_payload=True):
|
||||||
|
# Support both modern and older SDKs (filter vs scroll_filter)
|
||||||
|
try:
|
||||||
|
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
|
||||||
|
except TypeError as e:
|
||||||
|
# older signatures might accept scroll_filter kw, but if we passed none it's fine
|
||||||
|
return client.scroll(collection_name=collection, limit=limit, with_payload=with_payload)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--host", default=os.environ.get("QDRANT_HOST", "127.0.0.1"))
|
||||||
|
ap.add_argument("--port", default=int(os.environ.get("QDRANT_PORT", "6333")))
|
||||||
|
ap.add_argument("--prefix", default=None)
|
||||||
|
ap.add_argument("--limit", type=int, default=5)
|
||||||
|
ap.add_argument("--note-id", default=None)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
prefix = _get_prefix(args.prefix)
|
||||||
|
names = _names(prefix)
|
||||||
|
client = QdrantClient(host=args.host, port=args.port, prefer_grpc=False)
|
||||||
|
|
||||||
|
# Notes sample
|
||||||
|
try:
|
||||||
|
pts, _ = _scroll(client, names["notes"], limit=args.limit, with_payload=True)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"notes scroll failed: {type(e).__name__}: {e}", "collection": names["notes"]}))
|
||||||
|
return
|
||||||
|
|
||||||
|
sample = []
|
||||||
|
for p in pts:
|
||||||
|
pl = (p.payload or {})
|
||||||
|
sample.append({
|
||||||
|
"point_id": str(p.id),
|
||||||
|
"note_id": pl.get("note_id"),
|
||||||
|
"id": pl.get("id"),
|
||||||
|
"title": pl.get("title"),
|
||||||
|
"type": pl.get("type"),
|
||||||
|
"retriever_weight": pl.get("retriever_weight"),
|
||||||
|
"chunk_profile": pl.get("chunk_profile"),
|
||||||
|
})
|
||||||
|
print(json.dumps({"notes_sample": sample}, ensure_ascii=False))
|
||||||
|
|
||||||
|
# Pick a note to fetch chunks
|
||||||
|
target_note_id = args.note_id or None
|
||||||
|
if not target_note_id:
|
||||||
|
for s in sample:
|
||||||
|
if s.get("note_id"):
|
||||||
|
target_note_id = s["note_id"]
|
||||||
|
break
|
||||||
|
|
||||||
|
if not target_note_id:
|
||||||
|
print(json.dumps({"warn": "no note_id available to fetch chunks"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
# Chunks for target note
|
||||||
|
try:
|
||||||
|
# pull a few chunks and filter client-side by note_id
|
||||||
|
pts, _ = _scroll(client, names["chunks"], limit=50, with_payload=True)
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"chunks scroll failed: {type(e).__name__}: {e}", "collection": names["chunks"]}))
|
||||||
|
return
|
||||||
|
|
||||||
|
chunks = [{
|
||||||
|
"id": str(p.id),
|
||||||
|
"chunk_id": (p.payload or {}).get("chunk_id"),
|
||||||
|
"note_id": (p.payload or {}).get("note_id"),
|
||||||
|
"type": (p.payload or {}).get("type"),
|
||||||
|
"retriever_weight": (p.payload or {}).get("retriever_weight"),
|
||||||
|
"window_len": len((p.payload or {}).get("window") or ""),
|
||||||
|
"text_len": len((p.payload or {}).get("text") or ""),
|
||||||
|
} for p in pts if (p.payload or {}).get("note_id") == target_note_id][:5]
|
||||||
|
|
||||||
|
print(json.dumps({"target_note_id": target_note_id, "chunk_samples": chunks}, ensure_ascii=False))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Sorgt dafür, dass 'app' als Paket importierbar ist,
|
||||||
|
# wenn pytest aus dem Repo-Root oder aus 'tests/' gestartet wird.
|
||||||
|
import os, sys
|
||||||
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
|
if ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, ROOT)
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Kleine Diagnosehilfe für mindnet_edges.
|
||||||
|
|
||||||
|
Nutzung (aus Projektwurzel):
|
||||||
|
|
||||||
|
(.venv) python tests/diag_edges_for_note.py \
|
||||||
|
--note-id 20251130-relshow-embeddings-101
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
|
||||||
|
--prefix mindnet
|
||||||
|
|
||||||
|
Ziel:
|
||||||
|
- Zeigt alle Edges aus der Collection <prefix>_edges,
|
||||||
|
die sich auf die angegebene note_id beziehen:
|
||||||
|
* note_id == NOTE
|
||||||
|
* oder source_id == NOTE
|
||||||
|
* oder target_id == NOTE
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
# Projekt-Root (Ordner mit "app/") zum sys.path hinzufügen,
|
||||||
|
# damit "import app.core.qdrant" funktioniert, auch wenn das Script
|
||||||
|
# direkt aus tests/ gestartet wird.
|
||||||
|
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
PROJECT_ROOT = os.path.dirname(THIS_DIR)
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest # type: ignore
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_edges_for_note(note_id: str, prefix: str | None, limit: int = 512) -> List[dict]:
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
if prefix:
|
||||||
|
cfg.prefix = prefix
|
||||||
|
|
||||||
|
client = get_client(cfg)
|
||||||
|
_, _, edges_col = collection_names(cfg.prefix)
|
||||||
|
|
||||||
|
conditions = []
|
||||||
|
for field in ("note_id", "source_id", "target_id"):
|
||||||
|
conditions.append(
|
||||||
|
rest.FieldCondition(
|
||||||
|
key=field,
|
||||||
|
match=rest.MatchValue(value=str(note_id)),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
flt = rest.Filter(
|
||||||
|
should=conditions,
|
||||||
|
)
|
||||||
|
|
||||||
|
points, _ = client.scroll(
|
||||||
|
collection_name=edges_col,
|
||||||
|
scroll_filter=flt,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
out: List[dict] = []
|
||||||
|
for p in points or []:
|
||||||
|
pl = dict(p.payload or {})
|
||||||
|
if pl:
|
||||||
|
out.append(pl)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--note-id", required=True, help="Note-ID, z. B. 20251130-relshow-embeddings-101")
|
||||||
|
parser.add_argument("--prefix", default=None, help="Qdrant-Prefix (Default: aus ENV/QdrantConfig)")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
note_id = args.note_id
|
||||||
|
prefix = args.prefix
|
||||||
|
|
||||||
|
edges = fetch_edges_for_note(note_id, prefix)
|
||||||
|
print(f"Prefix: {prefix or '(aus ENV)'}")
|
||||||
|
print(f"Note-ID: {note_id}")
|
||||||
|
print(f"Gefundene Edges: {len(edges)}")
|
||||||
|
print("-" * 60)
|
||||||
|
|
||||||
|
for i, e in enumerate(edges, start=1):
|
||||||
|
print(f"[{i}] kind={e.get('kind')} confidence={e.get('confidence')}")
|
||||||
|
print(f" note_id = {e.get('note_id')}")
|
||||||
|
print(f" source_id= {e.get('source_id')}")
|
||||||
|
print(f" target_id= {e.get('target_id')}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/diag_payload_indexes.py (v1.1)
|
||||||
|
|
||||||
|
Zeigt payload_schema (Indizes) je Collection.
|
||||||
|
WICHTIG: Einige Qdrant-Versionen liefern payload_schema nur, wenn
|
||||||
|
`with_payload_schema=true` gesetzt wird. Daher setzen wir das Flag explizit.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, json
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, collection_names
|
||||||
|
|
||||||
|
def compact_schema(ps: dict | None) -> dict:
|
||||||
|
if not isinstance(ps, dict):
|
||||||
|
return {}
|
||||||
|
out = {}
|
||||||
|
for k, v in ps.items():
|
||||||
|
if isinstance(v, dict) and "type" in v:
|
||||||
|
out[k] = v["type"]
|
||||||
|
else:
|
||||||
|
out[k] = v
|
||||||
|
return out
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--raw", action="store_true")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
|
||||||
|
cols = [notes, chunks, edges]
|
||||||
|
result = []
|
||||||
|
for name in cols:
|
||||||
|
info = client.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = info.model_dump() if hasattr(info, "model_dump") else (info.dict() if hasattr(info, "dict") else info)
|
||||||
|
payload_schema = (d.get("result") or {}).get("payload_schema")
|
||||||
|
vectors = (d.get("result") or {}).get("vectors")
|
||||||
|
if isinstance(vectors, dict) and "config" in vectors:
|
||||||
|
vectors = vectors.get("config")
|
||||||
|
if args.raw:
|
||||||
|
result.append({"collection": name, "raw": d})
|
||||||
|
else:
|
||||||
|
result.append({
|
||||||
|
"name": name,
|
||||||
|
"vectors": vectors,
|
||||||
|
"payload_schema": compact_schema(payload_schema),
|
||||||
|
"segments_count": (d.get("result") or {}).get("segments_count"),
|
||||||
|
})
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "collections": result}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Diagnose für den Graph-Adapter (Subgraph) in mindnet.
|
||||||
|
|
||||||
|
Nutzung (aus Projektwurzel):
|
||||||
|
|
||||||
|
(.venv) python tests/diag_subgraph_for_note.py \
|
||||||
|
--note-id 20251130-relshow-embeddings-101 \
|
||||||
|
--depth 1
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
|
||||||
|
--prefix mindnet
|
||||||
|
--edge-type references --edge-type related_to ...
|
||||||
|
|
||||||
|
Ziel:
|
||||||
|
- Ruft app.core.graph_adapter.expand(...) für eine gegebene note_id auf.
|
||||||
|
- Gibt edge_bonus und centrality_bonus für diese note_id aus.
|
||||||
|
- Zeigt zusätzlich einige Nachbarn und deren Boni.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import List, Optional, Tuple
|
||||||
|
|
||||||
|
# sys.path anpassen, damit "import app...." aus tests/ funktioniert
|
||||||
|
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
PROJECT_ROOT = os.path.dirname(THIS_DIR)
|
||||||
|
if PROJECT_ROOT not in sys.path:
|
||||||
|
sys.path.insert(0, PROJECT_ROOT)
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client # type: ignore
|
||||||
|
import app.core.graph_adapter as ga # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def build_subgraph_for_note(
|
||||||
|
note_id: str,
|
||||||
|
prefix: Optional[str],
|
||||||
|
depth: int,
|
||||||
|
edge_types: Optional[List[str]] = None,
|
||||||
|
) -> Tuple[ga.Subgraph, str]:
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
if prefix:
|
||||||
|
cfg.prefix = prefix
|
||||||
|
|
||||||
|
client = get_client(cfg)
|
||||||
|
sg = ga.expand(client, cfg.prefix, [note_id], depth=depth, edge_types=edge_types)
|
||||||
|
return sg, cfg.prefix
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
"--note-id",
|
||||||
|
required=True,
|
||||||
|
help="Note-ID, z. B. 20251130-relshow-embeddings-101",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--depth",
|
||||||
|
type=int,
|
||||||
|
default=1,
|
||||||
|
help="Expansionstiefe (Default: 1)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--prefix",
|
||||||
|
default=None,
|
||||||
|
help="Qdrant-Prefix (Default: aus ENV/QdrantConfig)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--edge-type",
|
||||||
|
action="append",
|
||||||
|
default=None,
|
||||||
|
help="Edge-Typ-Filter (kann mehrfach angegeben werden), z. B. --edge-type references",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
note_id = args.note_id
|
||||||
|
depth = max(0, int(args.depth))
|
||||||
|
prefix = args.prefix
|
||||||
|
edge_types = args.edge_type
|
||||||
|
|
||||||
|
sg, eff_prefix = build_subgraph_for_note(note_id, prefix, depth, edge_types=edge_types)
|
||||||
|
|
||||||
|
print(f"Prefix: {eff_prefix}")
|
||||||
|
print(f"Note-ID (Seed): {note_id}")
|
||||||
|
print(f"Depth: {depth}")
|
||||||
|
print(f"Edge-Types-Filter: {edge_types or '(keiner)'}")
|
||||||
|
print("-" * 60)
|
||||||
|
|
||||||
|
# Boni für die Seed-Note
|
||||||
|
eb = sg.edge_bonus(note_id)
|
||||||
|
cb = sg.centrality_bonus(note_id)
|
||||||
|
print(f"edge_bonus({note_id}) = {eb:.4f}")
|
||||||
|
print(f"centrality_bonus({note_id}) = {cb:.4f}")
|
||||||
|
print("-" * 60)
|
||||||
|
|
||||||
|
# Ein paar weitere Knoten anzeigen (falls vorhanden)
|
||||||
|
print("Beispiele weiterer Knoten mit Boni (max. 10):")
|
||||||
|
printed = 0
|
||||||
|
for node, edges in sg.adj.items():
|
||||||
|
if node == note_id:
|
||||||
|
continue
|
||||||
|
if printed >= 10:
|
||||||
|
break
|
||||||
|
eb2 = sg.edge_bonus(node)
|
||||||
|
cb2 = sg.centrality_bonus(node)
|
||||||
|
print(f"- node={node}")
|
||||||
|
print(f" edge_bonus={eb2:.4f} centrality={cb2:.4f}")
|
||||||
|
printed += 1
|
||||||
|
|
||||||
|
if printed == 0:
|
||||||
|
print("(keine weiteren Knoten im Subgraph registriert)")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
dump_edges_payload_sample.py
|
||||||
|
-----------------------------------
|
||||||
|
Zeigt einige Beispiel-Payloads aus der {prefix}_edges Collection,
|
||||||
|
um Feldnamen (edge_type, src_note_id, src_id, etc.) zu verifizieren.
|
||||||
|
|
||||||
|
Nutzung:
|
||||||
|
python3 tests/dump_edges_payload_sample.py --prefix mindnet --n 10
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
try:
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"error": f"qdrant-client not installed: {e}"}))
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
import argparse
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX") or os.environ.get("MINDNET_PREFIX") or "mindnet")
|
||||||
|
ap.add_argument("--n", type=int, default=10)
|
||||||
|
args = ap.parse_args(argv)
|
||||||
|
|
||||||
|
url = os.environ.get("QDRANT_URL", "http://127.0.0.1:6333").strip()
|
||||||
|
api_key = os.environ.get("QDRANT_API_KEY") or None
|
||||||
|
edges = f"{args.prefix}_edges"
|
||||||
|
|
||||||
|
client = QdrantClient(url=url, api_key=api_key, timeout=30)
|
||||||
|
|
||||||
|
left = args.n
|
||||||
|
offset = None
|
||||||
|
while left > 0:
|
||||||
|
limit = min(left, 256)
|
||||||
|
pts, offset = client.scroll(
|
||||||
|
collection_name=edges,
|
||||||
|
scroll_filter=None,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
break
|
||||||
|
for p in pts:
|
||||||
|
print(json.dumps({"point_id": str(p.id), "payload_keys": list((p.payload or {}).keys()), "payload": p.payload}, ensure_ascii=False))
|
||||||
|
left -= len(pts)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/ensure_indexes_and_show.py (compat v1.0.1)
|
||||||
|
|
||||||
|
Erzwingt ensure_payload_indexes(...) und zeigt danach payload_schema je Collection.
|
||||||
|
Kompatibel mit qdrant-client Versionen ohne 'with_payload_schema' im Wrapper.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client, ensure_payload_indexes, collection_names
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_model_dump(obj: Any) -> Dict[str, Any]:
|
||||||
|
if hasattr(obj, "model_dump"):
|
||||||
|
return obj.model_dump()
|
||||||
|
if hasattr(obj, "dict"):
|
||||||
|
return obj.dict()
|
||||||
|
return obj if isinstance(obj, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _cfg_base_url(cfg) -> str:
|
||||||
|
if getattr(cfg, "url", None):
|
||||||
|
return cfg.url.rstrip("/")
|
||||||
|
host = getattr(cfg, "host", None) or os.getenv("QDRANT_HOST") or "127.0.0.1"
|
||||||
|
port = getattr(cfg, "port", None) or os.getenv("QDRANT_PORT") or "6333"
|
||||||
|
return f"http://{host}:{port}"
|
||||||
|
|
||||||
|
|
||||||
|
def _http_get(url: str, api_key: str | None) -> Dict[str, Any]:
|
||||||
|
req = urllib.request.Request(url, method="GET")
|
||||||
|
if api_key:
|
||||||
|
req.add_header("api-key", api_key)
|
||||||
|
req.add_header("Authorization", f"Bearer {api_key}")
|
||||||
|
req.add_header("Accept", "application/json")
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = resp.read()
|
||||||
|
return json.loads(data.decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def get_collection_payload_schema(client, cfg, name: str) -> Dict[str, Any] | None:
|
||||||
|
# Strategy 1: OpenAPI client with flag
|
||||||
|
try:
|
||||||
|
oc = getattr(client, "openapi_client", None)
|
||||||
|
if oc is not None and hasattr(oc, "collections_api"):
|
||||||
|
api = oc.collections_api
|
||||||
|
info = api.get_collection(collection_name=name, with_payload_schema=True)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
return (d.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Strategy 2: wrapper
|
||||||
|
try:
|
||||||
|
info = client.get_collection(collection_name=name)
|
||||||
|
d = _safe_model_dump(info)
|
||||||
|
ps = (d.get("result") or {}).get("payload_schema")
|
||||||
|
if ps is not None:
|
||||||
|
return ps
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Strategy 3: direct HTTP
|
||||||
|
try:
|
||||||
|
base = _cfg_base_url(cfg)
|
||||||
|
url = f"{base}/collections/{name}?with_payload_schema=true"
|
||||||
|
raw = _http_get(url, getattr(cfg, "api_key", None))
|
||||||
|
return (raw.get("result") or {}).get("payload_schema")
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
|
notes, chunks, edges = collection_names(cfg.prefix)
|
||||||
|
res = {}
|
||||||
|
for name in (notes, chunks, edges):
|
||||||
|
res[name] = get_collection_payload_schema(client, cfg, name)
|
||||||
|
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/test_retriever_smoke.py - einfacher Smoke-Test für den mindnet-Retriever
|
||||||
|
|
||||||
|
Zweck:
|
||||||
|
- Startet eine Query gegen die laufende FastAPI-Instanz
|
||||||
|
- Prüft, ob der /query-Endpunkt antwortet und sinnvolle Treffer liefert
|
||||||
|
- Gibt Top-K-Ergebnisse inkl. Scores auf stdout aus
|
||||||
|
|
||||||
|
Verwendung:
|
||||||
|
1) Stelle sicher, dass der mindnet-Service läuft, z. B.:
|
||||||
|
uvicorn app.main:create_app --factory --reload
|
||||||
|
|
||||||
|
2) Führe dieses Skript aus:
|
||||||
|
python scripts/test_retriever_smoke.py --query "karate" --mode hybrid
|
||||||
|
|
||||||
|
3) Optional:
|
||||||
|
python scripts/test_retriever_smoke.py --query "embeddings" --mode semantic --top-k 5
|
||||||
|
|
||||||
|
Hinweis:
|
||||||
|
- Standard-URL ist http://127.0.0.1:8000
|
||||||
|
- Endpoint ist /query (bestehender Router), der QueryRequest erwartet
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
def _build_payload(args: argparse.Namespace) -> Dict[str, Any]:
|
||||||
|
payload: Dict[str, Any] = {
|
||||||
|
"mode": args.mode,
|
||||||
|
"query": args.query,
|
||||||
|
"top_k": args.top_k,
|
||||||
|
}
|
||||||
|
if args.expand_depth is not None and args.expand_depth > 0:
|
||||||
|
expand: Dict[str, Any] = {"depth": args.expand_depth}
|
||||||
|
if args.expand_edge_types:
|
||||||
|
expand["edge_types"] = args.expand_edge_types
|
||||||
|
payload["expand"] = expand
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Smoke-Test für mindnet-Retriever (/query)")
|
||||||
|
parser.add_argument("--url", default="http://127.0.0.1:8000/query", help="Basis-URL des Query-Endpunkts")
|
||||||
|
parser.add_argument("--query", required=True, help="Text-Query für den Retriever")
|
||||||
|
parser.add_argument("--mode", choices=["semantic", "hybrid"], default="hybrid", help="Retrieval-Modus")
|
||||||
|
parser.add_argument("--top-k", type=int, default=5, help="Anzahl der gewünschten Treffer")
|
||||||
|
parser.add_argument("--expand-depth", type=int, default=0, help="Edge-Expansionstiefe (nur hybrid)")
|
||||||
|
parser.add_argument(
|
||||||
|
"--expand-edge-types",
|
||||||
|
nargs="*",
|
||||||
|
default=None,
|
||||||
|
help="Liste von Edge-Typen für die Expansion (z. B. references depends_on)",
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
payload = _build_payload(args)
|
||||||
|
|
||||||
|
try:
|
||||||
|
resp = requests.post(args.url, json=payload, timeout=10)
|
||||||
|
except Exception as exc: # pragma: no cover - Netzwerkfehler
|
||||||
|
print(f"[ERROR] HTTP-Request fehlgeschlagen: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
print(f"[ERROR] Unerwarteter Statuscode: {resp.status_code}", file=sys.stderr)
|
||||||
|
print(resp.text, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as exc: # pragma: no cover - JSON-Fehler
|
||||||
|
print(f"[ERROR] Antwort ist kein gültiges JSON: {exc}", file=sys.stderr)
|
||||||
|
print(resp.text, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print("=== mindnet Retriever Smoke-Test ===")
|
||||||
|
print(f"URL: {args.url}")
|
||||||
|
print(f"Mode: {args.mode}")
|
||||||
|
print(f"Query: {args.query!r}")
|
||||||
|
print(f"Top-K: {args.top_k}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
results = data.get("results") or []
|
||||||
|
used_mode = data.get("used_mode")
|
||||||
|
latency_ms = data.get("latency_ms")
|
||||||
|
|
||||||
|
print(f"used_mode = {used_mode}, latency = {latency_ms} ms")
|
||||||
|
print(f"Treffer insgesamt: {len(results)}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
for i, hit in enumerate(results, start=1):
|
||||||
|
note_id = hit.get("note_id")
|
||||||
|
node_id = hit.get("node_id")
|
||||||
|
total = hit.get("total_score")
|
||||||
|
sem = hit.get("semantic_score")
|
||||||
|
edge_bonus = hit.get("edge_bonus")
|
||||||
|
cent_bonus = hit.get("centrality_bonus")
|
||||||
|
src = hit.get("source") or {}
|
||||||
|
path = src.get("path")
|
||||||
|
section = src.get("section")
|
||||||
|
|
||||||
|
print(f"[{i}] note_id={note_id} node_id={node_id}")
|
||||||
|
print(f" total={total:.4f} (semantic={sem:.4f} edge={edge_bonus:.4f} centrality={cent_bonus:.4f})")
|
||||||
|
if path or section:
|
||||||
|
print(f" source: {path or '-'} :: {section or '-'}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/show_edges_for_note.py
|
||||||
|
|
||||||
|
Zeigt Kanten einer Note (gefiltert nach kind/scope) in einer kompakten Form.
|
||||||
|
Aufrufbeispiele:
|
||||||
|
python3 -m scripts.show_edges_for_note --note-id 20251110-ollama-llm-9f0a12 --kinds references,next,prev --limit 10
|
||||||
|
python3 -m scripts.show_edges_for_note --title "Qdrant Vektordatenbank" --scope note
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, json, os, sys
|
||||||
|
from typing import Dict, Any, List, Tuple
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def find_note_by_title(client, prefix: str, title: str) -> str | None:
|
||||||
|
notes_col, _, _ = collections(prefix)
|
||||||
|
f = rest.Filter(must=[rest.FieldCondition(key="title", match=rest.MatchText(text=title))])
|
||||||
|
pts, _ = client.scroll(collection_name=notes_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=1)
|
||||||
|
if not pts:
|
||||||
|
return None
|
||||||
|
return pts[0].payload.get("note_id")
|
||||||
|
|
||||||
|
def fetch_edges_for_note(client, prefix: str, note_id: str, kinds: List[str] | None, scope: str | None, limit: int) -> List[Dict[str, Any]]:
|
||||||
|
_, _, edges_col = collections(prefix)
|
||||||
|
must = [rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
|
||||||
|
if scope:
|
||||||
|
must.append(rest.FieldCondition(key="scope", match=rest.MatchValue(value=scope)))
|
||||||
|
if kinds:
|
||||||
|
must.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
|
||||||
|
f = rest.Filter(must=must)
|
||||||
|
pts, _ = client.scroll(collection_name=edges_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=limit)
|
||||||
|
return [p.payload for p in pts]
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--note-id")
|
||||||
|
ap.add_argument("--title")
|
||||||
|
ap.add_argument("--kinds", help="CSV: references,next,prev,belongs_to,backlink")
|
||||||
|
ap.add_argument("--scope", choices=["note","chunk"])
|
||||||
|
ap.add_argument("--limit", type=int, default=25)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
nid = args.note_id
|
||||||
|
if not nid and args.title:
|
||||||
|
nid = find_note_by_title(client, cfg.prefix, args.title)
|
||||||
|
if not nid:
|
||||||
|
print(json.dumps({"error": f"note with title '{args.title}' not found"}))
|
||||||
|
sys.exit(2)
|
||||||
|
if not nid:
|
||||||
|
print(json.dumps({"error": "please provide --note-id or --title"}))
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
kinds = None
|
||||||
|
if args.kinds:
|
||||||
|
kinds = [s.strip() for s in args.kinds.split(",") if s.strip()]
|
||||||
|
|
||||||
|
edges = fetch_edges_for_note(client, cfg.prefix, nid, kinds, args.scope, args.limit)
|
||||||
|
print(json.dumps({"note_id": nid, "count": len(edges), "edges": edges}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# Konfiguration
|
||||||
|
API_URL = "http://localhost:8002/chat/" # Port ggf. anpassen
|
||||||
|
QUESTION = "Warum nutzen wir Qdrant für mindnet?" # Eine Frage, zu der du Notizen hast
|
||||||
|
|
||||||
|
def test_chat():
|
||||||
|
payload = {
|
||||||
|
"message": QUESTION,
|
||||||
|
"top_k": 3,
|
||||||
|
"explain": True
|
||||||
|
}
|
||||||
|
|
||||||
|
print(f"Sending Question: '{QUESTION}'...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = requests.post(API_URL, json=payload)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
print("\n=== RESPONSE ===")
|
||||||
|
print(data["answer"])
|
||||||
|
print("================\n")
|
||||||
|
|
||||||
|
print(f"Query ID: {data['query_id']}")
|
||||||
|
print(f"Latency: {data['latency_ms']}ms")
|
||||||
|
print("\nUsed Sources:")
|
||||||
|
for source in data["sources"]:
|
||||||
|
score = source.get("total_score", 0)
|
||||||
|
note = source.get("note_id", "unknown")
|
||||||
|
print(f"- {note} (Score: {score:.3f})")
|
||||||
|
|
||||||
|
except requests.exceptions.ConnectionError:
|
||||||
|
print("Error: Could not connect to API. Is it running on port 8002?")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error: {e}")
|
||||||
|
if 'response' in locals():
|
||||||
|
print(response.text)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
test_chat()
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/test_edges_all.py
|
||||||
|
Ein knapper Integrationscheck:
|
||||||
|
- Es existieren Notes/Chunks/Edges
|
||||||
|
- Inline-Edges (rule_id startswith "inline:") werden erkannt
|
||||||
|
- Callout-Edges (rule_id == "callout:edge") werden erkannt
|
||||||
|
- Defaults (rule_id startswith "edge_defaults:") werden erkannt
|
||||||
|
- Strukturkanten stimmen (belongs_to == chunks; next == prev == chunks-1)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from typing import Dict, Any, List, Tuple
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
|
||||||
|
def _scroll_all(client, collection: str):
|
||||||
|
pts_all = []
|
||||||
|
offset = None
|
||||||
|
while True:
|
||||||
|
pts, offset = client.scroll(
|
||||||
|
collection_name=collection,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=2048,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
pts_all.extend(pts or [])
|
||||||
|
if offset is None:
|
||||||
|
break
|
||||||
|
return pts_all
|
||||||
|
|
||||||
|
|
||||||
|
def _rule_group(rule_id: str) -> str:
|
||||||
|
if not rule_id:
|
||||||
|
return "unknown"
|
||||||
|
if rule_id == "callout:edge":
|
||||||
|
return "callout"
|
||||||
|
if rule_id.startswith("inline:"): # <—— wichtig für inline:rel
|
||||||
|
return "inline"
|
||||||
|
if rule_id.startswith("edge_defaults:"):
|
||||||
|
return "defaults"
|
||||||
|
if rule_id.startswith("explicit:"):
|
||||||
|
return "explicit"
|
||||||
|
if rule_id in ("structure:belongs_to", "structure:order"):
|
||||||
|
return "structure"
|
||||||
|
return "other"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
|
||||||
|
col_notes = f"{cfg.prefix}_notes"
|
||||||
|
col_chunks = f"{cfg.prefix}_chunks"
|
||||||
|
col_edges = f"{cfg.prefix}_edges"
|
||||||
|
|
||||||
|
notes_n = client.count(collection_name=col_notes, exact=True).count
|
||||||
|
chunks_pts = _scroll_all(client, col_chunks)
|
||||||
|
edges_pts = _scroll_all(client, col_edges)
|
||||||
|
|
||||||
|
ok = True
|
||||||
|
|
||||||
|
# Basisbedingungen
|
||||||
|
if notes_n == 0 or len(chunks_pts) == 0 or len(edges_pts) == 0:
|
||||||
|
ok = False
|
||||||
|
|
||||||
|
# Gruppen zählen
|
||||||
|
g = Counter(_rule_group((p.payload or {}).get("rule_id", "")) for p in edges_pts)
|
||||||
|
structure = g.get("structure", 0)
|
||||||
|
explicit = g.get("explicit", 0)
|
||||||
|
inline = g.get("inline", 0)
|
||||||
|
callout = g.get("callout", 0)
|
||||||
|
defaults = g.get("defaults", 0)
|
||||||
|
|
||||||
|
if structure == 0:
|
||||||
|
ok = False
|
||||||
|
# mindestens eine der expliziten Varianten vorhanden
|
||||||
|
if (explicit + inline + callout) == 0:
|
||||||
|
ok = False
|
||||||
|
# defaults dürfen 0 sein, wenn types.yaml keine edge_defaults liefert – daher nur Info
|
||||||
|
|
||||||
|
# per-note checks
|
||||||
|
chunks_by_note = Counter([p.payload.get("note_id") for p in chunks_pts if p.payload])
|
||||||
|
belongs = Counter(
|
||||||
|
(p.payload or {}).get("note_id")
|
||||||
|
for p in edges_pts
|
||||||
|
if (p.payload or {}).get("kind") == "belongs_to"
|
||||||
|
)
|
||||||
|
nxt = Counter(
|
||||||
|
(p.payload or {}).get("note_id")
|
||||||
|
for p in edges_pts
|
||||||
|
if (p.payload or {}).get("kind") == "next"
|
||||||
|
)
|
||||||
|
prv = Counter(
|
||||||
|
(p.payload or {}).get("note_id")
|
||||||
|
for p in edges_pts
|
||||||
|
if (p.payload or {}).get("kind") == "prev"
|
||||||
|
)
|
||||||
|
|
||||||
|
for n_id, c in chunks_by_note.items():
|
||||||
|
if belongs.get(n_id, 0) != c:
|
||||||
|
ok = False
|
||||||
|
if (nxt.get(n_id, 0) != max(c - 1, 0)) or (prv.get(n_id, 0) != max(c - 1, 0)):
|
||||||
|
ok = False
|
||||||
|
|
||||||
|
print(json.dumps({"ok": ok, "notes_checked": len(chunks_by_note)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
tests/test_edges_defaults_smoke.py
|
||||||
|
Aggregiert Edges je Relation (kind) in Qdrant zur schnellen Sichtprüfung.
|
||||||
|
Zeigt, ob typenbasierte edge_defaults-Relationen (z. B. depends_on/related_to) auftauchen.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
from collections import Counter
|
||||||
|
import json
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
cl = get_client(cfg)
|
||||||
|
col = f"{cfg.prefix}_edges"
|
||||||
|
pts, _ = cl.scroll(collection_name=col, with_payload=True, with_vectors=False, limit=10000)
|
||||||
|
by_rel = Counter()
|
||||||
|
for p in pts:
|
||||||
|
rel = p.payload.get("kind") or p.payload.get("relation") or "edge"
|
||||||
|
by_rel[rel] += 1
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "relations": dict(by_rel)}, ensure_ascii=False))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
|
||||||
|
def _rel(pl: dict) -> str:
|
||||||
|
return pl.get("relation") or pl.get("kind") or "edge"
|
||||||
|
|
||||||
|
|
||||||
|
def _scroll(client, col):
|
||||||
|
pts = []
|
||||||
|
next_page = None
|
||||||
|
while True:
|
||||||
|
res, next_page = client.scroll(
|
||||||
|
collection_name=col,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
limit=1024,
|
||||||
|
offset=next_page,
|
||||||
|
)
|
||||||
|
pts.extend(res)
|
||||||
|
if next_page is None:
|
||||||
|
break
|
||||||
|
return pts
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
prefix = os.environ.get("COLLECTION_PREFIX", cfg.prefix)
|
||||||
|
|
||||||
|
cols = {
|
||||||
|
"notes": f"{prefix}_notes",
|
||||||
|
"chunks": f"{prefix}_chunks",
|
||||||
|
"edges": f"{prefix}_edges",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Index: notes -> title/type
|
||||||
|
notes_meta = {}
|
||||||
|
for p in _scroll(client, cols["notes"]):
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if nid:
|
||||||
|
notes_meta[nid] = {
|
||||||
|
"title": pl.get("title", ""),
|
||||||
|
"type": pl.get("type", ""),
|
||||||
|
}
|
||||||
|
|
||||||
|
# chunks je note
|
||||||
|
chunks_by_note = defaultdict(int)
|
||||||
|
for p in _scroll(client, cols["chunks"]):
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if nid:
|
||||||
|
chunks_by_note[nid] += 1
|
||||||
|
|
||||||
|
# edges je note
|
||||||
|
edges_by_note = defaultdict(list)
|
||||||
|
edges_all = _scroll(client, cols["edges"])
|
||||||
|
for p in edges_all:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id")
|
||||||
|
if nid:
|
||||||
|
edges_by_note[nid].append(pl)
|
||||||
|
|
||||||
|
# pro note ausgeben
|
||||||
|
summary_edges = Counter()
|
||||||
|
total_chunks = 0
|
||||||
|
for nid in sorted(notes_meta.keys()):
|
||||||
|
meta = notes_meta[nid]
|
||||||
|
chunks = chunks_by_note.get(nid, 0)
|
||||||
|
total_chunks += chunks
|
||||||
|
|
||||||
|
kinds = Counter(_rel(pl) for pl in edges_by_note[nid])
|
||||||
|
summary_edges.update(kinds)
|
||||||
|
|
||||||
|
row = {
|
||||||
|
"note_id": nid,
|
||||||
|
"title": meta["title"],
|
||||||
|
"type": meta["type"],
|
||||||
|
"chunks": chunks,
|
||||||
|
"edges_by_kind": dict(kinds),
|
||||||
|
"checks": {
|
||||||
|
"belongs_to_equals_chunks": (kinds.get("belongs_to", 0) == chunks),
|
||||||
|
"next_prev_match": (kinds.get("next", 0) == kinds.get("prev", 0) == max(0, chunks - 1)),
|
||||||
|
"no_duplicate_edges": _no_dupes(edges_by_note[nid]),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
print(json.dumps(row, ensure_ascii=False))
|
||||||
|
|
||||||
|
# Gesamtsummary
|
||||||
|
total_notes = len(notes_meta)
|
||||||
|
out = {
|
||||||
|
"prefix": prefix,
|
||||||
|
"summary": {
|
||||||
|
"notes": total_notes,
|
||||||
|
"chunks": total_chunks,
|
||||||
|
"belongs_to": summary_edges.get("belongs_to", 0),
|
||||||
|
"next": summary_edges.get("next", 0),
|
||||||
|
"prev": summary_edges.get("prev", 0),
|
||||||
|
"refs_chunk": summary_edges.get("references", 0),
|
||||||
|
"refs_note": summary_edges.get("references_note", 0),
|
||||||
|
"backlink": summary_edges.get("backlink", 0),
|
||||||
|
"dup_edges": 0, # per-Note geprüft
|
||||||
|
},
|
||||||
|
}
|
||||||
|
print(json.dumps(out, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
def _no_dupes(pls):
|
||||||
|
seen = set()
|
||||||
|
for pl in pls:
|
||||||
|
key = (
|
||||||
|
str(pl.get("source_id") or ""),
|
||||||
|
str(pl.get("target_id") or ""),
|
||||||
|
str(pl.get("relation") or pl.get("kind") or ""),
|
||||||
|
str(pl.get("rule_id") or ""),
|
||||||
|
)
|
||||||
|
if key in seen:
|
||||||
|
return False
|
||||||
|
seen.add(key)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
scripts/test_edges_smoke_fast.py — Zählung über /count (sehr schnell)
|
||||||
|
|
||||||
|
Verwendet Qdrant 'count' API je Note/Kind (anstatt scroll), dadurch sehr schnelle Ausführung.
|
||||||
|
Optionen:
|
||||||
|
--max-notes N : prüft nur die ersten N Notizen
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse, json
|
||||||
|
from typing import Dict, Any, List, Tuple
|
||||||
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
from app.core.qdrant import QdrantConfig, get_client
|
||||||
|
|
||||||
|
KINDS = ["belongs_to", "next", "prev", "references", "backlink"]
|
||||||
|
|
||||||
|
def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
|
||||||
|
|
||||||
|
def list_note_ids(client, notes_col: str, max_notes: int | None) -> List[Dict[str, Any]]:
|
||||||
|
pts, _ = client.scroll(collection_name=notes_col, with_payload=True, with_vectors=False, limit=max_notes or 1024)
|
||||||
|
out = []
|
||||||
|
for p in pts or []:
|
||||||
|
pl = p.payload or {}
|
||||||
|
nid = pl.get("note_id") or pl.get("id")
|
||||||
|
if nid:
|
||||||
|
out.append({"note_id": nid, "title": pl.get("title"), "type": pl.get("type")})
|
||||||
|
return out
|
||||||
|
|
||||||
|
def count_points(client, col: str, filt: rest.Filter) -> int:
|
||||||
|
res = client.count(collection_name=col, count_filter=filt, exact=True)
|
||||||
|
return int(getattr(res, "count", 0))
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--max-notes", type=int)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = QdrantConfig.from_env()
|
||||||
|
client = get_client(cfg)
|
||||||
|
notes_col, chunks_col, edges_col = collections(cfg.prefix)
|
||||||
|
notes = list_note_ids(client, notes_col, args.max_notes)
|
||||||
|
|
||||||
|
summary = {"notes": 0, "chunks": 0, "edges": 0}
|
||||||
|
for n in notes:
|
||||||
|
nid = n["note_id"]
|
||||||
|
summary["notes"] += 1
|
||||||
|
|
||||||
|
filt_note = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
|
||||||
|
chunk_cnt = count_points(client, chunks_col, filt_note)
|
||||||
|
summary["chunks"] += chunk_cnt
|
||||||
|
|
||||||
|
# counts per kind (edges)
|
||||||
|
by_kind: Dict[str, int] = {}
|
||||||
|
for k in KINDS:
|
||||||
|
f = rest.Filter(must=[
|
||||||
|
rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid)),
|
||||||
|
rest.FieldCondition(key="kind", match=rest.MatchValue(value=k)),
|
||||||
|
])
|
||||||
|
c = count_points(client, edges_col, f)
|
||||||
|
if c:
|
||||||
|
by_kind[k] = c
|
||||||
|
|
||||||
|
summary["edges"] += sum(by_kind.values())
|
||||||
|
line = {"note_id": nid, "title": n.get("title"), "type": n.get("type"), "chunks": chunk_cnt, "edges_by_kind": by_kind}
|
||||||
|
print(json.dumps(line, ensure_ascii=False))
|
||||||
|
|
||||||
|
print(json.dumps({"prefix": cfg.prefix, "summary": summary}, ensure_ascii=False))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
import argparse
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Testet die Explanation-Funktion des Retrievers.")
|
||||||
|
parser.add_argument("--url", default="http://localhost:8002/query", help="URL zum Query Endpoint")
|
||||||
|
parser.add_argument("--query", default="mindnet", help="Suchbegriff")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"mode": "hybrid",
|
||||||
|
"query": args.query,
|
||||||
|
"top_k": 1,
|
||||||
|
"explain": True, # Das neue Flag aus WP-04b
|
||||||
|
"expand": {
|
||||||
|
"depth": 1,
|
||||||
|
"edge_types": ["references", "belongs_to", "depends_on", "related_to"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
print(f"--- Sende Request an {args.url} ---")
|
||||||
|
try:
|
||||||
|
r = requests.post(args.url, json=payload)
|
||||||
|
r.raise_for_status()
|
||||||
|
data = r.json()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"FEHLER: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
hits = data.get("results", [])
|
||||||
|
if not hits:
|
||||||
|
print("Keine Treffer gefunden.")
|
||||||
|
return
|
||||||
|
|
||||||
|
top_hit = hits[0]
|
||||||
|
print(f"\nTop Treffer: {top_hit.get('node_id')}")
|
||||||
|
print(f"Total Score: {top_hit.get('total_score'):.4f}")
|
||||||
|
|
||||||
|
# Explanation prüfen
|
||||||
|
expl = top_hit.get("explanation")
|
||||||
|
if not expl:
|
||||||
|
print("\n[FAIL] Keine 'explanation' im Response erhalten!")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print("\n--- EXPLANATION LAYER ---")
|
||||||
|
|
||||||
|
bd = expl.get("breakdown", {})
|
||||||
|
print(f"Breakdown:")
|
||||||
|
print(f" Semantic Contribution: {bd.get('semantic_contribution', 0):.4f}")
|
||||||
|
print(f" Edge Contribution: {bd.get('edge_contribution', 0):.4f}")
|
||||||
|
print(f" Centrality Contribution: {bd.get('centrality_contribution', 0):.4f}")
|
||||||
|
print(f" Typ-Gewicht: {bd.get('node_weight', 0):.2f}")
|
||||||
|
|
||||||
|
print("\nReasons:")
|
||||||
|
for reason in expl.get("reasons", []):
|
||||||
|
kind = reason.get("kind", "").upper()
|
||||||
|
msg = reason.get("message", "")
|
||||||
|
impact = reason.get("score_impact")
|
||||||
|
impact_str = f" ({impact:+.4f})" if impact is not None else ""
|
||||||
|
print(f" [{kind}] {msg}{impact_str}")
|
||||||
|
|
||||||
|
print("\n[SUCCESS] Explanation Layer antwortet korrekt.")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
# Konfiguration
|
||||||
|
BASE_URL = "http://localhost:8001"
|
||||||
|
QUERY_URL = f"{BASE_URL}/query"
|
||||||
|
FEEDBACK_URL = f"{BASE_URL}/feedback"
|
||||||
|
|
||||||
|
def run_test():
|
||||||
|
print(f"--- 1. Sende Suchanfrage an {QUERY_URL} ---")
|
||||||
|
query_payload = {
|
||||||
|
"query": "mindnet architecture",
|
||||||
|
"mode": "hybrid",
|
||||||
|
"top_k": 2,
|
||||||
|
"explain": True # Wir wollen auch Breakdown loggen
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
r = requests.post(QUERY_URL, json=query_payload)
|
||||||
|
r.raise_for_status()
|
||||||
|
res_data = r.json()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"ERROR bei Suche: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
query_id = res_data.get("query_id")
|
||||||
|
results = res_data.get("results", [])
|
||||||
|
|
||||||
|
if not query_id:
|
||||||
|
print("FAIL: Keine query_id in der Antwort erhalten!")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if not results:
|
||||||
|
print("WARNUNG: Keine Treffer gefunden, Test kann Feedback nur simuliert senden.")
|
||||||
|
target_node_id = "fake-node-id"
|
||||||
|
else:
|
||||||
|
target_node_id = results[0].get("node_id")
|
||||||
|
|
||||||
|
print(f"SUCCESS: Suche OK. Erhaltene Query-ID: {query_id}")
|
||||||
|
print(f"Target Node für Feedback: {target_node_id}")
|
||||||
|
|
||||||
|
# Kurze Pause, damit Background-Task Zeit hat zu schreiben (nur für Log-Check relevant)
|
||||||
|
time.sleep(0.5)
|
||||||
|
|
||||||
|
print(f"\n--- 2. Sende Feedback an {FEEDBACK_URL} ---")
|
||||||
|
feedback_payload = {
|
||||||
|
"query_id": query_id,
|
||||||
|
"node_id": target_node_id,
|
||||||
|
"score": 1,
|
||||||
|
"comment": "Automatisierter Smoke-Test: Dieser Treffer war hilfreich."
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
r_fb = requests.post(FEEDBACK_URL, json=feedback_payload)
|
||||||
|
r_fb.raise_for_status()
|
||||||
|
print(f"SUCCESS: Feedback gesendet. Status: {r_fb.status_code}")
|
||||||
|
print(f"Response: {r_fb.json()}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"ERROR bei Feedback: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
run_test()
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from app.main import create_app
|
||||||
|
import app.core.graph_adapter as ga
|
||||||
|
|
||||||
|
def _fake_get_edges_for_sources(client, prefix, source_ids, edge_types=None, limit=2048):
|
||||||
|
note_id = source_ids[0]
|
||||||
|
return [
|
||||||
|
{"source_id":note_id,"target_id":f"{note_id}#c1","kind":"references","weight":0.2},
|
||||||
|
{"source_id":note_id,"target_id":f"{note_id}#c2","kind":"belongs_to","weight":0.1},
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_graph_neighbors(monkeypatch):
|
||||||
|
monkeypatch.setattr(ga, "get_edges_for_sources", _fake_get_edges_for_sources, raising=False)
|
||||||
|
app = create_app()
|
||||||
|
with TestClient(app) as c:
|
||||||
|
r = c.get("/graph/TEST-NOTE?depth=1")
|
||||||
|
assert r.status_code == 200
|
||||||
|
g = r.json()
|
||||||
|
assert g["center_note_id"] == "TEST-NOTE"
|
||||||
|
assert g["stats"]["edge_count"] == 2
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from app.main import create_app
|
||||||
|
|
||||||
|
def test_healthz_ok():
|
||||||
|
app = create_app()
|
||||||
|
with TestClient(app) as c:
|
||||||
|
r = c.get("/healthz")
|
||||||
|
assert r.status_code == 200
|
||||||
|
body = r.json()
|
||||||
|
assert body["status"] == "ok"
|
||||||
|
assert "qdrant" in body and "prefix" in body
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from app.main import create_app
|
||||||
|
import app.services.embeddings_client as ec
|
||||||
|
import app.core.qdrant_points as qp
|
||||||
|
import app.core.graph_adapter as ga
|
||||||
|
|
||||||
|
def _fake_embed_text(text: str):
|
||||||
|
# Liefert stabilen 384-d Vektor ohne echtes Modell
|
||||||
|
return [0.01] * 384
|
||||||
|
|
||||||
|
def _fake_search_chunks_by_vector(client, prefix, vector, top=10, filters=None):
|
||||||
|
# einfache Hitliste
|
||||||
|
return [
|
||||||
|
("chunk:1", 0.9, {"note_id":"note:1","path":"a.md","section_title":"S1"}),
|
||||||
|
("chunk:2", 0.7, {"note_id":"note:2","path":"b.md","section_title":"S2"}),
|
||||||
|
("chunk:3", 0.5, {"note_id":"note:3","path":"c.md","section_title":"S3"}),
|
||||||
|
]
|
||||||
|
|
||||||
|
def _fake_get_edges_for_sources(client, prefix, source_ids, edge_types=None, limit=2048):
|
||||||
|
out=[]
|
||||||
|
for sid in source_ids:
|
||||||
|
out.append({"source_id":sid,"target_id":f"{sid}-x","kind":"references","weight":0.2})
|
||||||
|
return out
|
||||||
|
|
||||||
|
def test_query_with_text(monkeypatch):
|
||||||
|
# Patch Embedding + Qdrant-Aufrufe
|
||||||
|
monkeypatch.setattr(ec, "embed_text", _fake_embed_text)
|
||||||
|
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector)
|
||||||
|
monkeypatch.setattr(ga, "get_edges_for_sources", _fake_get_edges_for_sources, raising=False)
|
||||||
|
|
||||||
|
app = create_app()
|
||||||
|
with TestClient(app) as c:
|
||||||
|
payload = {"mode":"hybrid","query":"karate trainingsplan", "top_k":2,
|
||||||
|
"expand":{"depth":1,"edge_types":["references"]}}
|
||||||
|
r = c.post("/query", json=payload)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
body = r.json()
|
||||||
|
assert body["used_mode"] == "hybrid"
|
||||||
|
assert len(body["results"]) == 2
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from app.main import create_app
|
||||||
|
import app.core.qdrant_points as qp
|
||||||
|
import app.core.graph_adapter as ga
|
||||||
|
|
||||||
|
def _fake_search_chunks_by_vector(client, prefix, vector, top=10, filters=None):
|
||||||
|
return [
|
||||||
|
("chunk:1", 0.8, {"note_id":"note:1","path":"a.md","section_title":"S1"}),
|
||||||
|
("chunk:2", 0.6, {"note_id":"note:2","path":"b.md","section_title":"S2"}),
|
||||||
|
("chunk:3", 0.4, {"note_id":"note:3","path":"c.md","section_title":"S3"}),
|
||||||
|
]
|
||||||
|
|
||||||
|
def _fake_get_edges_for_sources(client, prefix, source_ids, edge_types=None, limit=2048):
|
||||||
|
# Erzeuge ein paar Edges von den Seeds weg
|
||||||
|
out=[]
|
||||||
|
for sid in source_ids:
|
||||||
|
out.append({"source_id":sid,"target_id":f"{sid}->n1","kind":"references","weight":0.2})
|
||||||
|
out.append({"source_id":sid,"target_id":f"{sid}->n2","kind":"belongs_to","weight":0.1})
|
||||||
|
return out
|
||||||
|
|
||||||
|
def test_query_hybrid_with_monkeypatch(monkeypatch):
|
||||||
|
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector)
|
||||||
|
monkeypatch.setattr(qp, "get_edges_for_sources", _fake_get_edges_for_sources) # falls indirekt genutzt
|
||||||
|
# graph_adapter.expand ruft get_edges_for_sources via qp; wir patchen direkt dort:
|
||||||
|
monkeypatch.setattr(ga, "get_edges_for_sources", _fake_get_edges_for_sources, raising=False)
|
||||||
|
|
||||||
|
app = create_app()
|
||||||
|
with TestClient(app) as c:
|
||||||
|
payload = {"mode":"hybrid","query_vector":[0.0]*384,"top_k":2,
|
||||||
|
"expand":{"depth":1,"edge_types":["references","belongs_to"]}}
|
||||||
|
r = c.post("/query", json=payload)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
data = r.json()
|
||||||
|
assert data["used_mode"] == "hybrid"
|
||||||
|
assert len(data["results"]) == 2
|
||||||
|
# Scores absteigend
|
||||||
|
tots = [h["total_score"] for h in data["results"]]
|
||||||
|
assert tots[0] >= tots[1]
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
from app.core import retriever as r
|
||||||
|
from app.models.dto import QueryRequest
|
||||||
|
import app.core.qdrant as qdr
|
||||||
|
import app.core.qdrant_points as qp
|
||||||
|
import app.services.embeddings_client as ec
|
||||||
|
|
||||||
|
|
||||||
|
class _DummyClient:
|
||||||
|
"""Minimaler Platzhalter für QdrantClient in Unit-Tests."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_get_client(cfg):
|
||||||
|
return _DummyClient()
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_embed_text(text: str):
|
||||||
|
# Liefert stabilen Vektor; Größe ist für diesen Test egal,
|
||||||
|
# da search_chunks_by_vector ebenfalls gefaked wird.
|
||||||
|
return [0.0] * 16
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_search_chunks_by_vector(client, prefix, vector, top=10, filters=None):
|
||||||
|
# einfache Hitliste in absteigender Score-Reihenfolge
|
||||||
|
return [
|
||||||
|
("chunk:1", 0.9, {"note_id": "note:1", "path": "a.md", "section_title": "S1"}),
|
||||||
|
("chunk:2", 0.7, {"note_id": "note:2", "path": "b.md", "section_title": "S2"}),
|
||||||
|
("chunk:3", 0.4, {"note_id": "note:3", "path": "c.md", "section_title": "S3"}),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_semantic_retrieve_basic(monkeypatch):
|
||||||
|
# Qdrant- und Embedding-Aufrufe faken
|
||||||
|
monkeypatch.setattr(qdr, "get_client", _fake_get_client)
|
||||||
|
monkeypatch.setattr(ec, "embed_text", _fake_embed_text)
|
||||||
|
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector)
|
||||||
|
|
||||||
|
req = QueryRequest(mode="semantic", query="karate trainingsplan", top_k=2)
|
||||||
|
resp = r.semantic_retrieve(req)
|
||||||
|
|
||||||
|
assert resp.used_mode == "semantic"
|
||||||
|
assert len(resp.results) == 2
|
||||||
|
|
||||||
|
# Scores absteigend sortiert
|
||||||
|
scores = [h.total_score for h in resp.results]
|
||||||
|
assert scores[0] >= scores[1]
|
||||||
|
# Korrekte Zuordnung von note_id und Quelle
|
||||||
|
assert resp.results[0].note_id == "note:1"
|
||||||
|
assert resp.results[0].source["path"] == "a.md"
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_retrieve_basic(monkeypatch):
|
||||||
|
# Qdrant- und Embedding-Aufrufe faken
|
||||||
|
monkeypatch.setattr(qdr, "get_client", _fake_get_client)
|
||||||
|
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector)
|
||||||
|
|
||||||
|
# Im Hybrid-Modus arbeiten wir im Step-1-Stand nur mit query_vector
|
||||||
|
req = QueryRequest(mode="hybrid", query_vector=[0.0] * 16, top_k=2)
|
||||||
|
resp = r.hybrid_retrieve(req)
|
||||||
|
|
||||||
|
assert resp.used_mode == "hybrid"
|
||||||
|
assert len(resp.results) == 2
|
||||||
|
scores = [h.total_score for h in resp.results]
|
||||||
|
assert scores[0] >= scores[1]
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
from app.core import retriever as r
|
||||||
|
from app.models.dto import QueryRequest
|
||||||
|
import app.core.qdrant as qdr
|
||||||
|
import app.core.qdrant_points as qp
|
||||||
|
import app.core.graph_adapter as ga
|
||||||
|
import app.services.embeddings_client as ec
|
||||||
|
|
||||||
|
|
||||||
|
class _DummyClient:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_get_client(cfg):
|
||||||
|
return _DummyClient()
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_embed_text(text: str):
|
||||||
|
# Dimension ist egal, da wir search_chunks_by_vector faken
|
||||||
|
return [0.0] * 384
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_search_chunks_by_vector(client, prefix, vector, top=10, filters=None):
|
||||||
|
# Drei Treffer mit identischem semantic_score und retriever_weight,
|
||||||
|
# aber unterschiedlichen Graph-Boni
|
||||||
|
return [
|
||||||
|
("chunk:1", 0.8, {"note_id": "note:1", "chunk_id": "c1", "path": "a.md", "section_title": "S1"}),
|
||||||
|
("chunk:2", 0.8, {"note_id": "note:2", "chunk_id": "c2", "path": "b.md", "section_title": "S2"}),
|
||||||
|
("chunk:3", 0.8, {"note_id": "note:3", "chunk_id": "c3", "path": "c.md", "section_title": "S3"}),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class _DummySubgraph:
|
||||||
|
def __init__(self, edge_scores, cent_scores):
|
||||||
|
self._edge = edge_scores
|
||||||
|
self._cent = cent_scores
|
||||||
|
|
||||||
|
def edge_bonus(self, node_id: str) -> float:
|
||||||
|
return self._edge.get(node_id, 0.0)
|
||||||
|
|
||||||
|
def centrality_bonus(self, node_id: str) -> float:
|
||||||
|
return self._cent.get(node_id, 0.0)
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_expand(client, prefix, seeds, depth=1, edge_types=None):
|
||||||
|
# Wir gewichten c2 > c1 > c3
|
||||||
|
edge_scores = {"c1": 0.1, "c2": 0.5, "c3": 0.0}
|
||||||
|
cent_scores = {"c1": 0.05, "c2": 0.2, "c3": 0.0}
|
||||||
|
return _DummySubgraph(edge_scores, cent_scores)
|
||||||
|
|
||||||
|
|
||||||
|
def test_hybrid_retrieve_uses_edge_scores(monkeypatch):
|
||||||
|
# Qdrant-Client, Embeddings, Chunk-Suche und Graph-Expansion faken
|
||||||
|
monkeypatch.setattr(qdr, "get_client", _fake_get_client)
|
||||||
|
monkeypatch.setattr(ec, "embed_text", _fake_embed_text)
|
||||||
|
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector)
|
||||||
|
monkeypatch.setattr(ga, "expand", _fake_expand)
|
||||||
|
|
||||||
|
# expand.depth > 0 triggern, edge_types sind hier egal
|
||||||
|
req = QueryRequest(mode="hybrid", query="karate", top_k=3, expand={"depth": 1, "edge_types": ["references"]})
|
||||||
|
resp = r.hybrid_retrieve(req)
|
||||||
|
|
||||||
|
assert len(resp.results) == 3
|
||||||
|
|
||||||
|
# Da alle semantic_scores und retriever_weight gleich sind,
|
||||||
|
# bestimmt die Summe aus edge_bonus + centrality_bonus das Ranking.
|
||||||
|
first = resp.results[0]
|
||||||
|
second = resp.results[1]
|
||||||
|
third = resp.results[2]
|
||||||
|
|
||||||
|
assert first.note_id == "note:2" # c2 hat höchste Graph-Boni
|
||||||
|
assert second.note_id == "note:1"
|
||||||
|
assert third.note_id == "note:3"
|
||||||
|
|
||||||
|
assert first.total_score > second.total_score > third.total_score
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
#
|
||||||
|
# test_retriever_smoke.py – End-to-End Smoke-Test für den mindnet Retriever
|
||||||
|
#
|
||||||
|
# Dieser Smoke-Test ist für manuelle End-to-End-Tests gedacht.
|
||||||
|
# Er wird NICHT automatisch von pytest ausgeführt.
|
||||||
|
#
|
||||||
|
# Empfohlener Speicherort: tests/test_retriever_smoke.py
|
||||||
|
#
|
||||||
|
# Aufrufbeispiele:
|
||||||
|
#
|
||||||
|
# python tests/test_retriever_smoke.py --query "embeddings" --mode semantic
|
||||||
|
#
|
||||||
|
# python tests/test_retriever_smoke.py \
|
||||||
|
# --query "embeddings" \
|
||||||
|
# --mode hybrid \
|
||||||
|
# --expand-depth 1 \
|
||||||
|
# --expand-edge-types references depends_on
|
||||||
|
#
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
def _build_payload(args: argparse.Namespace) -> Dict[str, Any]:
|
||||||
|
payload: Dict[str, Any] = {
|
||||||
|
"mode": args.mode,
|
||||||
|
"query": args.query,
|
||||||
|
"top_k": args.top_k,
|
||||||
|
}
|
||||||
|
if args.expand_depth is not None and args.expand_depth > 0:
|
||||||
|
expand: Dict[str, Any] = {"depth": args.expand_depth}
|
||||||
|
if args.expand_edge_types:
|
||||||
|
expand["edge_types"] = args.expand_edge_types
|
||||||
|
payload["expand"] = expand
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Smoke-Test für mindnet-Retriever (/query)")
|
||||||
|
parser.add_argument("--url", default="http://127.0.0.1:8000/query", help="URL des Query-Endpunkts")
|
||||||
|
parser.add_argument("--query", required=True, help="Text-Query")
|
||||||
|
parser.add_argument("--mode", choices=["semantic", "hybrid"], default="hybrid")
|
||||||
|
parser.add_argument("--top-k", type=int, default=5)
|
||||||
|
parser.add_argument("--expand-depth", type=int, default=0)
|
||||||
|
parser.add_argument(
|
||||||
|
"--expand-edge-types",
|
||||||
|
nargs="*",
|
||||||
|
default=None,
|
||||||
|
help="Edge-Typen (z. B. references depends_on)",
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
payload = _build_payload(args)
|
||||||
|
|
||||||
|
try:
|
||||||
|
resp = requests.post(args.url, json=payload, timeout=10)
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"[ERROR] HTTP-Request fehlgeschlagen: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
print(f"[ERROR] Statuscode: {resp.status_code}", file=sys.stderr)
|
||||||
|
print(resp.text, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = resp.json()
|
||||||
|
except Exception:
|
||||||
|
print("[ERROR] Ungültiges JSON:", resp.text, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print("=== mindnet Retriever Smoke-Test ===")
|
||||||
|
print(f"URL: {args.url}")
|
||||||
|
print(f"Mode: {args.mode}")
|
||||||
|
print(f"Query: {args.query!r}")
|
||||||
|
print(f"Top-K: {args.top_k}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
results = data.get("results") or []
|
||||||
|
print(f"used_mode: {data.get('used_mode')}")
|
||||||
|
print(f"latency: {data.get('latency_ms')} ms")
|
||||||
|
print(f"Treffer: {len(results)}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
for i, hit in enumerate(results, start=1):
|
||||||
|
nid = hit.get("note_id")
|
||||||
|
cid = hit.get("node_id")
|
||||||
|
tot = hit.get("total_score")
|
||||||
|
sem = hit.get("semantic_score")
|
||||||
|
edge = hit.get("edge_bonus")
|
||||||
|
cen = hit.get("centrality_bonus")
|
||||||
|
src = hit.get("source") or {}
|
||||||
|
path = src.get("path")
|
||||||
|
sec = src.get("section")
|
||||||
|
|
||||||
|
print(f"[{i}] note_id={nid} node_id={cid}")
|
||||||
|
try:
|
||||||
|
print(
|
||||||
|
f" total={float(tot):.4f} "
|
||||||
|
f"(semantic={float(sem):.4f} edge={float(edge):.4f} centrality={float(cen):.4f})"
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
print(f" total={tot}, semantic={sem}, edge={edge}, centrality={cen}")
|
||||||
|
|
||||||
|
if path or sec:
|
||||||
|
print(f" source: {path or '-'} :: {sec or '-'}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
from app.core import retriever as r
|
||||||
|
from app.models.dto import QueryRequest
|
||||||
|
import app.core.qdrant as qdr
|
||||||
|
import app.core.qdrant_points as qp
|
||||||
|
import app.services.embeddings_client as ec
|
||||||
|
|
||||||
|
|
||||||
|
class _DummyClient:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_get_client(cfg):
|
||||||
|
return _DummyClient()
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_embed_text(text: str):
|
||||||
|
# Dimension egal, da wir search_chunks_by_vector faken
|
||||||
|
return [0.0] * 384
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_search_chunks_by_vector_weights(client, prefix, vector, top=10, filters=None):
|
||||||
|
# Zwei Treffer mit unterschiedlichem retriever_weight
|
||||||
|
return [
|
||||||
|
("chunk:low", 0.9, {"note_id": "note:low", "path": "low.md", "section_title": "S1", "retriever_weight": 0.2}),
|
||||||
|
("chunk:high", 0.8, {"note_id": "note:high", "path": "high.md", "section_title": "S2", "retriever_weight": 1.5}),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_retriever_weight_influences_ranking(monkeypatch):
|
||||||
|
# Konfiguration: Qdrant & Embedding faken
|
||||||
|
monkeypatch.setattr(qdr, "get_client", _fake_get_client)
|
||||||
|
monkeypatch.setattr(ec, "embed_text", _fake_embed_text)
|
||||||
|
monkeypatch.setattr(qp, "search_chunks_by_vector", _fake_search_chunks_by_vector_weights)
|
||||||
|
|
||||||
|
req = QueryRequest(mode="semantic", query="karate", top_k=2)
|
||||||
|
resp = r.semantic_retrieve(req)
|
||||||
|
|
||||||
|
assert len(resp.results) == 2
|
||||||
|
|
||||||
|
first = resp.results[0]
|
||||||
|
second = resp.results[1]
|
||||||
|
|
||||||
|
# Semantic Scores: 0.9 (low), 0.8 (high)
|
||||||
|
# Total Scores: 0.9 * 0.2 = 0.18 vs. 0.8 * 1.5 = 1.2 -> high sollte oben liegen
|
||||||
|
assert first.note_id == "note:high"
|
||||||
|
assert second.note_id == "note:low"
|
||||||
|
|
||||||
|
assert first.total_score > second.total_score
|
||||||
Loaded 100 of 154 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user