Compare commits

..
Author SHA1 Message Date
Lars 0a891cfd95 docs/knowledge_design.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-10-06 18:33:42 +02:00
Lars 66adc6a354 docs/Handbuch.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-10-06 14:22:47 +02:00
Lars e99f1c2ba0 tests/compare_vaults.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 7s
2025-10-06 13:29:00 +02:00
Lars f52a9face8 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-10-06 12:35:54 +02:00
Lars 4fcf22131c tests/check_chunks_window_vs_text.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-10-01 15:25:17 +02:00
Lars d27324f8dd tests/check_chunks_vs_text.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-10-01 15:23:08 +02:00
Lars dd86a15555 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-10-01 15:20:10 +02:00
Lars 28d173b8fd app/core/parser.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-10-01 12:08:50 +02:00
Lars b4b7ea76ab app/core/parser.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-10-01 10:58:23 +02:00
Lars 6ea452cc3f app/core/derive_edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-10-01 10:54:46 +02:00
Lars 6e0e965883 tests/inspect_one_note.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-10-01 10:27:15 +02:00
Lars 6b4b3ce2ba tests/list_md_seen_by_importer.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-10-01 10:24:25 +02:00
Lars 18cdbab885 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 15:50:38 +02:00
Lars ebca9b4caf scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 4s
2025-09-30 15:45:31 +02:00
Lars 6945ef8ddc scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 15:39:55 +02:00
Lars f20bf03ee1 tests/audit_edges_now.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 12:37:56 +02:00
Lars 7cda15553d app/core/derive_edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 12:36:19 +02:00
Lars 3c67fd5f9b app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 12:26:33 +02:00
Lars 0150931df4 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 12:22:07 +02:00
Lars 5ff1b06009 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 12:19:12 +02:00
Lars 5279bcae18 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 12:18:16 +02:00
Lars daa58e6b27 tests/verify_chunks_integrity.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 12:09:18 +02:00
Lars 589d021744 tests/verify_chunks_integrity.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 12:05:13 +02:00
Lars e58b9c8ada scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 3s
2025-09-30 10:54:36 +02:00
Lars 507c28e006 tests/compare_vaults.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 2s
2025-09-29 16:07:22 +02:00
Lars 2d74fe4057 .gitea/workflows/deploy.yml aktualisiert
Deploy mindnet to llm-node / deploy (push) Successful in 9s
2025-09-29 15:41:20 +02:00
Lars e8312268d2 .gitea/workflows/deploy.yml aktualisiert 2025-09-29 15:35:27 +02:00
Lars f4cd1e51c0 tests/compare_vaults.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-24 12:35:53 +02:00
Lars 60e707e7df .gitea/workflows/deploy.yml aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-24 12:35:20 +02:00
Lars 0d6eb4dac4 tests/compare_vaults.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-24 12:34:53 +02:00
Lars 80d8f986bd tests/test_importer_detect_changes.sh hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-24 12:34:22 +02:00
Lars 9ee3bb9007 tests/run_e2e_roundtrip.sh hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-24 12:33:55 +02:00
Lars 8a07010704 app/graph/service.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-24 12:17:02 +02:00
Lars 314fc8ac94 scripts/debug_qdrant_state.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-24 12:16:35 +02:00
Lars b46e5461a7 scripts/report_hashes.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-24 12:16:05 +02:00
Lars ada7c1797d scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-24 12:15:34 +02:00
Lars 54197995f5 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-24 12:14:56 +02:00
Lars 13e510dd9c scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-24 12:14:30 +02:00
Lars 5c937fbffe scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-24 12:13:04 +02:00
Lars 58bce0ea19 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 19:57:32 +02:00
Lars 3e888cc669 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 19:47:16 +02:00
Lars 78446c74d3 schemas/note.schema.json aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 19:43:12 +02:00
Lars 51f08f193c scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 19:42:35 +02:00
Lars 4872374a6e app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 19:42:09 +02:00
Lars 81c1400ef4 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:55:36 +02:00
Lars 298c1fb180 scripts/debug_note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:55:12 +02:00
Lars f8108bd724 scripts/debug_note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:52:25 +02:00
Lars 65c40b287f scripts/debug_note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:48:41 +02:00
Lars 13e50d4695 scripts/debug_note_payload.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:47:13 +02:00
Lars 791cda23af scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:44:59 +02:00
Lars 466ea64e63 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:39:48 +02:00
Lars f5e6fcc097 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:36:27 +02:00
Lars 9ef2e8d397 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:33:38 +02:00
Lars 4637091125 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:31:25 +02:00
Lars 43a8c80508 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:31:00 +02:00
Lars 9362064ad1 scripts/make_test_vault.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:17:27 +02:00
Lars 23952aa3c5 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:16:34 +02:00
Lars 79e19c85eb schemas/note.schema.json aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:15:53 +02:00
Lars 62d8299480 schemas/edge.schema.json aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:15:22 +02:00
Lars 8f342ea2d3 schemas/chunk.schema.json aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 16:15:05 +02:00
Lars 5d5de06290 scripts/verify_chunk_texts.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 15:58:12 +02:00
Lars d7e5f398f0 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 12:40:54 +02:00
Lars 5bf5316af5 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 12:40:23 +02:00
Lars d9fd6d8a8f docs/knowledge_design.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 12:29:55 +02:00
Lars e672eade8b docs/knowledge_design.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 12:22:26 +02:00
Lars ec346905ed docs/chunking_strategy.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 12:22:04 +02:00
Lars c9cbbbfcb9 docs/knowledge_design.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:57:34 +02:00
Lars 93739935b7 docs/knowledge_design.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:56:54 +02:00
Lars 6a293e842e docs/chunking_strategy.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:56:26 +02:00
Lars f4abb1d873 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:52:16 +02:00
Lars 8f48701ea0 app/core/qdrant.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:51:52 +02:00
Lars 6f32c60c47 app/core/edges.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:51:28 +02:00
Lars e040fcc0dd scripts/audit_edges_vs_expectations.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:34:08 +02:00
Lars 305089fcf6 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:27:18 +02:00
Lars f9c0b5df19 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:26:57 +02:00
Lars 77fa57a2a6 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:19:06 +02:00
Lars dcb02dd194 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:17:30 +02:00
Lars 35338dea64 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:17:06 +02:00
Lars c34df96839 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:15:19 +02:00
Lars 0617163c42 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 2s
2025-09-09 11:15:01 +02:00
Lars 47797ecd29 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:12:52 +02:00
Lars 897d0c9e6d scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:12:33 +02:00
Lars 6c25d76135 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:05:32 +02:00
Lars 22885a4857 app/core/chunk_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 11:04:50 +02:00
Lars a807cc8bd1 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 10:13:57 +02:00
Lars c2954b5663 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 09:33:51 +02:00
Lars 8f97f42618 app/core/qdrant_points.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-09 09:12:50 +02:00
Lars 0a116ea3f4 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 18:36:59 +02:00
Lars aa7d0190c8 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 18:04:11 +02:00
Lars af5d0a0c91 scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:43:44 +02:00
Lars fb016c6c6a scripts/export_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:42:57 +02:00
Lars 77732445ea scripts/prune_qdrant_vs_vault.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:37:08 +02:00
Lars 1b833f76ce scripts/export_markdown.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 17:36:40 +02:00
Lars b394cadf73 app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 12:42:51 +02:00
Lars f3b6166daa app/core/note_payload.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-08 12:22:05 +02:00
Lars ce720090ec docs/Handbuch.md aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-07 07:48:59 +02:00
Lars bf2149bbeb docs/Handbuch.md hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-07 07:46:39 +02:00
Lars 80a7db0e54 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-06 15:42:10 +02:00
Lars 47e6d56b21 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-06 14:04:46 +02:00
Lars df33293621 scripts/resolve_unresolved_references.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 19:16:27 +02:00
Lars a202c53594 scripts/make_test_vault.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 15:48:06 +02:00
Lars 4f54e750b2 scripts/make_test_vault.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 15:35:41 +02:00
Lars 5478be60c1 scripts/resolve_unresolved_references.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 12:37:59 +02:00
Lars 3deb99f053 scripts/gc_qdrant_after_vault_scan.py hinzugefügt
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 12:18:23 +02:00
Lars 8df2069c6f scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 12:03:33 +02:00
Lars d790c0ef75 scripts/import_markdown.py aktualisiert
Deploy mindnet to llm-node / deploy (push) Failing after 1s
2025-09-05 11:53:59 +02:00
35 changed files with 5248 additions and 1054 deletions

No files matched your search

+31 -12
View File
@@ -1,22 +1,36 @@
# .gitea/workflows/deploy.yml # .gitea/workflows/deploy.yml
name: Deploy mindnet to llm-node name: Deploy mindnet to llm-node
on: { push: { branches: [ "main" ] } }
# on: {}
concurrency: { group: deploy-mindnet, cancel-in-progress: false } on:
push:
branches: ["main"]
concurrency:
group: deploy-mindnet
cancel-in-progress: false
jobs: jobs:
deploy: deploy:
runs-on: linux_host runs-on: linux_host
env: env:
DEPLOY_DIRS: "app scripts schemas docker requirements.txt README.md" TARGET_BASE: /home/llmadmin/mindnet
TARGET_BASE: "/home/llmadmin/mindnet" DEPLOY_DIRS: "app scripts schemas docker tests requirements.txt README.md"
steps: steps:
- name: Checkout - name: Checkout
uses: https://github.com/actions/checkout@v4 uses: actions/checkout@v4
- name: Ensure target base
- name: Ensure target base exists
run: install -d "$TARGET_BASE" run: install -d "$TARGET_BASE"
- name: Deploy whitelisted
- name: Stop API (graceful)
continue-on-error: true
run: |
if systemctl --user list-unit-files | grep -q '^mindnet-api.service'; then
systemctl --user stop mindnet-api.service || true
fi
- name: Deploy whitelisted (rsync)
run: | run: |
set -euo pipefail set -euo pipefail
IFS=' ' read -r -a DIRS <<< "$DEPLOY_DIRS" IFS=' ' read -r -a DIRS <<< "$DEPLOY_DIRS"
@@ -24,22 +38,27 @@ jobs:
if [ -e "$d" ]; then if [ -e "$d" ]; then
if [ -d "$d" ]; then if [ -d "$d" ]; then
install -d "$TARGET_BASE/$d" install -d "$TARGET_BASE/$d"
rsync -a --delete --exclude='.git' \ rsync -a --delete \
--exclude='.env' --exclude='.env.*' --exclude='**/.env*' \ --exclude='.git' \
--exclude='.env*' \
--exclude='.venv' \
--exclude='hf_cache' --exclude='hf_cache/**' \
"$d"/ "$TARGET_BASE/$d"/ "$d"/ "$TARGET_BASE/$d"/
else else
rsync -a "$d" "$TARGET_BASE/$d" rsync -a "$d" "$TARGET_BASE/$d"
fi fi
fi fi
done done
- name: Python venv (idempotent)
- name: Python venv & requirements
run: | run: |
cd "$TARGET_BASE" cd "$TARGET_BASE"
[ -d .venv ] || python3 -m venv .venv [ -d .venv ] || python3 -m venv .venv
source .venv/bin/activate source .venv/bin/activate
pip install --upgrade pip pip install --upgrade pip
[ -f requirements.txt ] && pip install -r requirements.txt || true [ -f requirements.txt ] && pip install -r requirements.txt || true
- name: Optional — restart mindnet-api
- name: Start/Restart mindnet-api
continue-on-error: true continue-on-error: true
run: | run: |
if systemctl --user list-unit-files | grep -q '^mindnet-api.service'; then if systemctl --user list-unit-files | grep -q '^mindnet-api.service'; then
+269 -35
View File
@@ -1,37 +1,271 @@
from __future__ import annotations #!/usr/bin/env python3
from typing import Dict, List # -*- coding: utf-8 -*-
from .chunker import Chunk """
from .parser import extract_wikilinks Modul: app/core/chunk_payload.py
Version: 2.2.0
Datum: 2025-10-06
def make_chunk_payloads(note_meta: Dict, path: str, chunks: List[Chunk]) -> List[Dict]: Zweck
res = [] -----
for ch in chunks: Erzeugt Qdrant-Payloads für Chunks. Voll abwärtskompatibel zu v2.0.1.
wikilinks = extract_wikilinks(ch.text) Neu: Wenn der Chunker KEIN Overlap im Fenster liefert (== window fehlt / identisch zur Kernpassage),
payload = { erzeugen wir FENSTER mit synthetischem Overlap auf Basis chunk_config.get_sizes(note_type)['overlap'].
"id": ch.id,
"note_id": note_meta["id"], Felder (beibehalten aus 2.0.1):
"note_title": note_meta["title"], - note_id, chunk_id, id (Alias), chunk_index, seq, path
"chunk_index": ch.index, - window (mit Overlap), text (ohne linkes Overlap)
"char_start": ch.char_start, - start, end (Offsets im gesamten Body)
"char_end": ch.char_end, - overlap_left, overlap_right
"token_count": ch.token_count, - token_count?, section_title?, section_path?, type?, title?, tags?
"type": note_meta.get("type"),
"area": note_meta.get("area"), Kompatibilität:
"project": note_meta.get("project"), - 'id' == 'chunk_id' als Alias
"tags": note_meta.get("tags", []), - Pfade bleiben relativ (keine führenden '/'), Backslashes → Slashes
"section_title": ch.section_title, - Robust für Chunk-Objekte oder Dicts; Fensterquelle: 'window'|'text'|'content'|'raw'
"section_path": ch.section_path,
"lang": note_meta.get("lang"), Lizenz: MIT (projektintern)
"wikilinks": wikilinks, """
"external_links": [], # (optional später ergänzen) from __future__ import annotations
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
"neighbors": { from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
"prev": ch.neighbors_prev,
"next": ch.neighbors_next try:
}, # Typgerechtes Overlap aus deiner Konfiguration holen
"path": path, from app.core.chunk_config import get_sizes as _get_sizes
except Exception:
def _get_sizes(_note_type: str):
# konservativer Default, falls Import fehlschlägt
return {"overlap": (40, 60), "target": (250, 350), "max": 500}
# ------------------------------- Utils ------------------------------- #
def _get_attr_or_key(obj: Any, key: str, default=None):
if obj is None:
return default
if isinstance(obj, dict):
return obj.get(key, default)
return getattr(obj, key, default)
def _as_window_text(chunk: Any) -> str:
"""Fenstertext robust lesen (bevorzugt echte Fenster, sonst Kern)."""
for k in ("window", "text", "content", "raw"):
v = _get_attr_or_key(chunk, k, None)
if isinstance(v, str) and v:
return v
return ""
def _to_int(x: Any, default: int = 0) -> int:
try:
return int(x)
except Exception:
return default
def _normalize_rel_path(p: str) -> str:
p = (p or "").replace("\\", "/")
while p.startswith("/"):
p = p[1:]
return p
# ---------------------- Overlap & Offsets ---------------------------- #
def _dedupe_windows_to_segments(windows: List[str]) -> Tuple[List[str], List[int], str]:
"""
Entfernt linkes Overlap aus echten Fenster-Strings.
Rückgabe: (segments, overlaps_left, reconstructed_text)
"""
segments: List[str] = []
overlaps_left: List[int] = []
reconstructed = ""
for w in windows:
w = w or ""
max_k = min(len(w), len(reconstructed))
k = 0
for cand in range(max_k, -1, -1):
if reconstructed.endswith(w[:cand]):
k = cand
break
seg = w[k:]
segments.append(seg)
overlaps_left.append(k)
reconstructed += seg
return segments, overlaps_left, reconstructed
def _overlap_len_suffix_prefix(a: str, b: str, max_probe: int = 4096) -> int:
"""Länge längsten Suffix(a), der Prefix(b) ist."""
if not a or not b:
return 0
a1 = a[-max_probe:]
b1 = b[:max_probe]
n = min(len(a1), len(b1))
for k in range(n, 0, -1):
if a1[-k:] == b1[:k]:
return k
return 0
# ----------------------------- Public API ---------------------------- #
def make_chunk_payloads(
frontmatter: Dict[str, Any],
rel_path: str,
chunks: Iterable[Union[Dict[str, Any], Any]],
note_text: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""
Baut Payloads pro Chunk. Falls Fenster ohne Overlap geliefert werden,
erzeugen wir synthetische 'window'-Texte mit typgerechtem Overlap.
"""
note_id = str(frontmatter.get("id") or "").strip()
note_type = str(frontmatter.get("type", "")).lower()
note_title = frontmatter.get("title", None)
note_tags = frontmatter.get("tags", None)
rel_path = _normalize_rel_path(rel_path)
# 1) Rohdaten sammeln (so wie geliefert)
chunks_list = list(chunks)
raw_windows: List[str] = []
seqs: List[int] = []
ids_in: List[Optional[str]] = []
token_counts: List[Optional[int]] = []
section_titles: List[Optional[str]] = []
section_paths: List[Optional[str]] = []
any_explicit_window = False
for idx, c in enumerate(chunks_list):
# Fensterquelle
w = _get_attr_or_key(c, "window", None)
if isinstance(w, str) and w:
any_explicit_window = True
raw_windows.append(w)
else:
raw_windows.append(_as_window_text(c)) # 'text'|'content'|'raw' als Ersatz
# Ordnung
seqs.append(_to_int(_get_attr_or_key(c, "seq", _get_attr_or_key(c, "chunk_index", idx)), idx))
# IDs, Tokens, Sektionen
cid = _get_attr_or_key(c, "chunk_id", _get_attr_or_key(c, "id", None))
ids_in.append(str(cid) if isinstance(cid, str) and cid else None)
tc = _get_attr_or_key(c, "token_count", None)
token_counts.append(_to_int(tc, 0) if tc is not None else None)
section_titles.append(_get_attr_or_key(c, "section_title", None))
section_paths.append(_get_attr_or_key(c, "section_path", None))
# 2) Segmente & Overlaps bestimmen
if any_explicit_window:
# Es existieren echte Fenster → dedupe, um Kernsegmente zu finden
segments, overlaps_left, recon = _dedupe_windows_to_segments(raw_windows)
windows_final = raw_windows[:] # bereits mit Overlap geliefert
else:
# Keine echten Fenster → Segmente sind identisch zu "Fenstern" (bisher),
# wir erzeugen synthetische Fenster mit Overlap gemäß Typ
segments = [w or "" for w in raw_windows]
overlaps_left = []
windows_final = []
recon = ""
try:
overlap_low, overlap_high = tuple(_get_sizes(note_type).get("overlap", (40, 60)))
except Exception:
overlap_low, overlap_high = (40, 60)
overlap_target = int(overlap_low)
for i, seg in enumerate(segments):
if i == 0:
# erstes Fenster: kein linker Kontext
windows_final.append(seg)
overlaps_left.append(0)
recon += seg
else:
# synthetischer linker Kontext = Suffix des bisher rekonstruierten Texts
k = min(overlap_target, len(recon))
left_ctx = recon[-k:] if k > 0 else ""
windows_final.append(left_ctx + seg)
overlaps_left.append(k)
recon += seg # Rekonstruktion bleibt kerntreu
# 3) overlap_right bestimmen
overlaps_right: List[int] = []
for i in range(len(windows_final)):
if i + 1 < len(windows_final):
ov = _overlap_len_suffix_prefix(windows_final[i], windows_final[i + 1], max_probe=4096)
else:
ov = 0
overlaps_right.append(ov)
# 4) start/end-Offsets (exakt via note_text, sonst kumulativ)
starts: List[int] = [0] * len(segments)
ends: List[int] = [0] * len(segments)
pos = 0
if isinstance(note_text, str) and note_text:
search_pos = 0
for i, seg in enumerate(segments):
if not seg:
starts[i] = ends[i] = search_pos
continue
j = note_text.find(seg, search_pos)
if j >= 0:
starts[i] = j
ends[i] = j + len(seg)
search_pos = ends[i]
else:
# Fallback: kumulativ
starts[i] = pos
pos += len(seg)
ends[i] = pos
else:
for i, seg in enumerate(segments):
starts[i] = pos
pos += len(seg)
ends[i] = pos
# 5) Payload-Dicts
payloads: List[Dict[str, Any]] = []
for i, (win, seg) in enumerate(zip(windows_final, segments)):
chunk_id = ids_in[i] or f"{note_id}#{i+1}"
pl: Dict[str, Any] = {
"note_id": note_id,
"chunk_id": chunk_id,
"id": chunk_id, # Alias
"chunk_index": i,
"seq": seqs[i],
"path": rel_path,
"window": win,
"text": seg,
"start": starts[i],
"end": ends[i],
"overlap_left": overlaps_left[i],
"overlap_right": overlaps_right[i],
} }
# None/Leere bereinigen # optionale Metadaten
payload = {k: v for k, v in payload.items() if v not in (None, [], {})} if note_type:
res.append(payload) pl["type"] = note_type
return res if note_title is not None:
pl["title"] = note_title
if note_tags is not None:
pl["tags"] = note_tags
if token_counts[i] is not None:
pl["token_count"] = int(token_counts[i])
if section_titles[i] is not None:
pl["section_title"] = section_titles[i]
if section_paths[i] is not None:
sp = str(section_paths[i]).replace("\\", "/")
pl["section_path"] = sp if sp else "/"
payloads.append(pl)
return payloads
# __main__ Demo (optional)
if __name__ == "__main__": # pragma: no cover
fm = {"id": "demo", "title": "Demo", "type": "concept"}
# Beispiel ohne echte Fenster → erzeugt synthetische Overlaps
chunks = [
{"id": "demo#1", "text": "Alpha Beta Gamma"},
{"id": "demo#2", "text": "Gamma Delta"},
{"id": "demo#3", "text": "Delta Epsilon Zeta"},
]
pls = make_chunk_payloads(fm, "path/demo.md", chunks, note_text="Alpha Beta Gamma Delta Epsilon Zeta")
from pprint import pprint
pprint(pls)
recon = "".join(p["text"] for p in pls)
print("RECON:", recon)
+114 -109
View File
@@ -1,126 +1,131 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
Name: app/core/derive_edges.py Modul: app/core/derive_edges.py
Version: v1.1.0 (2025-09-05) Version: 1.4.0
Datum: 2025-10-01
Kurzbeschreibung Zweck
Leitet Edges aus Wikilinks ([[…]]) ab und löst Zielnoten robust auf. -----
Erzeugt: Robuste Kantenbildung für mindnet (Notes/Chunks):
- "references" (Note -> Note) mit seq="body", pro Match eine eigene Occurrence 'occ' - belongs_to (chunk -> note)
- "backlink" (inverse zu "references", gleiche seq/occ) - next / prev (chunk-Kette)
- "references_at" (Chunk -> Note) mit seq=<chunk_index> und eigener 'occ' je Match - references (chunk-scope) aus Chunk.window/text
- optional references/backlink (note-scope)
Aufruf Wichtig: Wikilinks werden mit der Parser-Funktion `extract_wikilinks` extrahiert,
from app.core.derive_edges import build_note_index, derive_wikilink_edges damit Varianten wie [[id#anchor]] oder [[id|label]] korrekt auf 'id' reduziert werden.
Parameter / Felder Erwartete Chunk-Payload-Felder:
- note_payload: {"note_id","title","path","fulltext": <body> , …} {
- chunks_payloads: [{"chunk_id","text",…}, …] "note_id": "...",
- note_index: build_note_index([...]) -> (by_id, by_slug, by_file_slug) "chunk_id": "...", # Alias "id" ist zulässig
"id": "...",
Kompatibilität "chunk_index": int,
- Rückwärtskompatible Payload-Felder, nur erweitert um 'seq' und 'occ'. "seq": int,
"window": str,
Changelog "text": str,
v1.1.0: Occurrence-Zählung ('occ') je Match; 'seq="body"' für references. "path": "rel/path.md",
...
}
""" """
from __future__ import annotations from __future__ import annotations
import re
import unicodedata
from typing import Dict, List, Tuple
# [[Ziel]], [[Ziel|Alias]], [[Ziel#Heading]], [[Ziel#Heading|Alias]] from typing import Dict, List, Optional, Iterable
WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#([^\]|]+))?(?:\|([^\]]+))?\]\]")
def _slug(s: str) -> str: # WICHTIG: benutze die Parser-Extraktion für saubere Wikilinks
s = s.strip() from app.core.parser import extract_wikilinks
if s.endswith(".md"):
s = s[:-3]
s = unicodedata.normalize("NFKD", s)
s = "".join(ch for ch in s if not unicodedata.combining(ch))
s = s.replace("\\", "/")
s = s.split("/")[-1]
s = s.lower().replace(" ", "-")
s = re.sub(r"[^a-z0-9\-]+", "", s)
s = re.sub(r"-{2,}", "-", s).strip("-")
return s
def build_note_index(notes_payloads: List[dict]) -> Tuple[Dict[str, dict], Dict[str, dict], Dict[str, dict]]: def _get(d: dict, *keys, default=None):
by_id: Dict[str, dict] = {} for k in keys:
by_slug: Dict[str, dict] = {} if k in d and d[k] is not None:
by_file_slug: Dict[str, dict] = {} return d[k]
for n in notes_payloads: return default
nid = n.get("note_id") or n.get("id")
if not nid:
continue
by_id[nid] = n
title = n.get("title", "")
path = n.get("path", "")
file_slug = _slug(path.split("/")[-1]) if path else ""
if title:
by_slug[_slug(title)] = n
if file_slug:
by_file_slug[file_slug] = n
return by_id, by_slug, by_file_slug
def resolve_target(note_like: str, idx: Tuple[Dict[str,dict],Dict[str,dict],Dict[str,dict]]): def _chunk_text_for_refs(chunk: dict) -> str:
by_id, by_slug, by_file_slug = idx # bevorzugt 'window' → dann 'text' → 'content' → 'raw'
key = note_like.strip() return (
if key in by_id: _get(chunk, "window")
return by_id[key]["note_id"], "by_id" or _get(chunk, "text")
s = _slug(key) or _get(chunk, "content")
if s in by_slug: or _get(chunk, "raw")
return by_slug[s]["note_id"], "by_slug" or ""
if s in by_file_slug: )
return by_file_slug[s]["note_id"], "by_file_slug"
return None, "unresolved"
def derive_wikilink_edges(note_payload: dict, chunks_payloads: List[dict], note_index) -> List[dict]: def _dedupe(seq: Iterable[str]) -> List[str]:
seen = set()
out: List[str] = []
for s in seq:
if s not in seen:
seen.add(s)
out.append(s)
return out
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
pl = {
"kind": kind,
"scope": scope, # "chunk" | "note"
"source_id": source_id,
"target_id": target_id,
"note_id": note_id, # Träger/Quelle der Kante (aktuelle Note)
}
if extra:
pl.update(extra)
return pl
def build_edges_for_note(
note_id: str,
chunks: List[dict],
note_level_references: Optional[List[str]] = None,
include_note_scope_refs: bool = False,
) -> List[dict]:
"""
Erzeugt Kanten für eine Note.
- belongs_to: für jeden Chunk (chunk -> note)
- next / prev: zwischen aufeinanderfolgenden Chunks
- references: pro Chunk aus window/text
- optional note-scope references/backlinks: dedupliziert über alle Chunk-Funde + note_level_references
"""
edges: List[dict] = [] edges: List[dict] = []
source_note_id = note_payload["note_id"]
def _make_edge(kind: str, src: str, tgt: str, seq=None, occ=None, extra: dict|None=None): # belongs_to
e = {"edge_id": None, "kind": kind, "source_id": src, "target_id": tgt} for ch in chunks:
if seq is not None: cid = _get(ch, "chunk_id", "id")
e["seq"] = seq if not cid:
if occ is not None:
e["occ"] = occ
if extra:
e.update(extra)
return e
# Volltext (Note-Ebene)
fulltext = note_payload.get("fulltext") or note_payload.get("body") or ""
if fulltext:
for k, m in enumerate(WIKILINK_RE.finditer(fulltext), start=1):
raw_target, heading, alias = m.groups()
target_id, how = resolve_target(raw_target, note_index)
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how}
if target_id:
edges.append(_make_edge("references", source_note_id, target_id, seq="body", occ=k, extra=extra))
edges.append(_make_edge("backlink", target_id, source_note_id, seq="body", occ=k, extra=extra))
else:
extra["status"] = "unresolved"
extra["target_label"] = raw_target
edges.append(_make_edge("references", source_note_id, raw_target, seq="body", occ=k, extra=extra))
# Chunks (Chunk-Ebene)
for i, ch in enumerate(chunks_payloads, start=1):
txt = ch.get("text") or ch.get("content") or ""
if not txt:
continue continue
occ = 0 edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {"chunk_id": cid}))
for m in WIKILINK_RE.finditer(txt):
occ += 1 # next/prev
raw_target, heading, alias = m.groups() for i in range(len(chunks) - 1):
target_id, how = resolve_target(raw_target, note_index) a, b = chunks[i], chunks[i + 1]
extra = {"raw": raw_target, "alias": alias, "heading": heading, "resolution": how} a_id = _get(a, "chunk_id", "id")
if target_id: b_id = _get(b, "chunk_id", "id")
edges.append(_make_edge("references_at", ch["chunk_id"], target_id, seq=i, occ=occ, extra=extra)) if not a_id or not b_id:
else: continue
extra["status"] = "unresolved" edges.append(_edge("next", "chunk", a_id, b_id, note_id, {"chunk_id": a_id}))
extra["target_label"] = raw_target edges.append(_edge("prev", "chunk", b_id, a_id, note_id, {"chunk_id": b_id}))
edges.append(_make_edge("references_at", ch["chunk_id"], raw_target, seq=i, occ=occ, extra=extra))
# references (chunk-scope) – Links aus window bevorzugen (Overlap-fest)
refs_all: List[str] = []
for ch in chunks:
cid = _get(ch, "chunk_id", "id")
if not cid:
continue
txt = _chunk_text_for_refs(ch)
refs = extract_wikilinks(txt) # <— Parser-Logik, kompatibel zu deinem System
for r in refs:
edges.append(_edge("references", "chunk", cid, r, note_id, {"chunk_id": cid, "ref_text": r}))
refs_all.extend(refs)
# optional: note-scope references/backlinks
if include_note_scope_refs:
refs_note = refs_all[:]
if note_level_references:
refs_note.extend([r for r in note_level_references if isinstance(r, str) and r])
refs_note = _dedupe(refs_note)
for r in refs_note:
edges.append(_edge("references", "note", note_id, r, note_id))
edges.append(_edge("backlink", "note", r, note_id, note_id))
return edges return edges
+108 -33
View File
@@ -1,45 +1,120 @@
from __future__ import annotations #!/usr/bin/env python3
from typing import List, Dict # -*- coding: utf-8 -*-
"""
Modul: app/core/edges.py
Version: 1.0.0
Datum: 2025-09-09
def deriv_edges_for_note(note_meta: Dict, chunk_payloads: List[Dict]) -> List[Dict]: Zweck
-----
Zentrale, konsistente Erzeugung von Edge-Payloads im **neuen Schema**:
- kind : "belongs_to" | "next" | "prev" | "references" | "backlink"
- source_id : ID des Quellknotens (Chunk- oder Note-ID)
- target_id : ID des Zielknotens
- scope : "chunk" | "note"
- note_id : Owner-Note (für performantes Filtern/Löschen)
- seq : optional (z. B. Reihenfolge von Vorkommen)
Hinweise
--------
- Edges werden dedupliziert (key=(kind,source_id,target_id,scope)).
- Für Chunk-Edges wird `note_id` aus dem Chunk-Payload entnommen.
- Für Note-Scope-Edges ist `note_id` die Quell-Note-ID.
"""
from __future__ import annotations
from typing import Dict, List
def build_edges_for_note(
note_id: str,
chunk_payloads: List[Dict],
note_level_refs: List[str] | None,
*,
include_note_scope_refs: bool = False,
) -> List[Dict]:
edges: List[Dict] = [] edges: List[Dict] = []
# Chunk → Note (belongs_to) + prev/next # Chunk-Scope: belongs_to / prev / next / references
for idx, ch in enumerate(chunk_payloads): for ch in chunk_payloads:
cid = ch["id"]
owner = ch.get("note_id") or note_id
# belongs_to
edges.append({ edges.append({
"src_id": ch["id"], "dst_id": note_meta["id"], "kind": "belongs_to",
"edge_type": "belongs_to", "scope": "chunk" "source_id": cid,
"target_id": note_id,
"scope": "chunk",
"note_id": owner,
}) })
prev_id = ch.get("neighbors",{}).get("prev") # Nachbarn
next_id = ch.get("neighbors",{}).get("next") nb = ch.get("neighbors") or {}
prev_id = nb.get("prev")
next_id = nb.get("next")
if prev_id: if prev_id:
edges.append({"src_id": ch["id"], "dst_id": prev_id, "edge_type": "next", "scope": "chunk"}) edges.append({
edges.append({"src_id": prev_id, "dst_id": ch["id"], "edge_type": "prev", "scope": "chunk"}) "kind": "prev",
"source_id": cid,
"target_id": prev_id,
"scope": "chunk",
"note_id": owner,
})
edges.append({
"kind": "next",
"source_id": prev_id,
"target_id": cid,
"scope": "chunk",
"note_id": owner,
})
if next_id: if next_id:
edges.append({"src_id": ch["id"], "dst_id": next_id, "edge_type": "next", "scope": "chunk"}) edges.append({
edges.append({"src_id": next_id, "dst_id": ch["id"], "edge_type": "prev", "scope": "chunk"}) "kind": "next",
"source_id": cid,
# references aus wikilinks (Chunk-Scope) "target_id": next_id,
for ref in ch.get("references", []): "scope": "chunk",
"note_id": owner,
})
edges.append({
"kind": "prev",
"source_id": next_id,
"target_id": cid,
"scope": "chunk",
"note_id": owner,
})
# references aus Chunk
for ref in (ch.get("references") or []):
tid = ref.get("target_id") tid = ref.get("target_id")
if not tid: continue if not tid:
edges.append({"src_id": ch["id"], "dst_id": tid, "edge_type": "references", "scope": "chunk"}) continue
edges.append({"src_id": tid, "dst_id": ch["id"], "edge_type": "backlink", "scope": "chunk"}) edges.append({
"kind": "references",
"source_id": cid,
"target_id": tid,
"scope": "chunk",
"note_id": owner,
})
# depends_on / assigned_to (Note-Scope, aus Frontmatter) # Note-Scope: backlink (immer); references (optional)
for dep in note_meta.get("depends_on", []) or []: unique_refs = list(dict.fromkeys(note_level_refs or []))
edges.append({"src_id": note_meta["id"], "dst_id": dep, "edge_type": "depends_on", "scope": "note"}) for tid in unique_refs:
for ass in note_meta.get("assigned_to", []) or []: if include_note_scope_refs:
edges.append({"src_id": note_meta["id"], "dst_id": ass, "edge_type": "assigned_to", "scope": "note"}) edges.append({
"kind": "references",
# Note-Level references (optional: falls du im Note-Payload `references` sammelst) "source_id": note_id,
for tid in note_meta.get("references", []) or []: "target_id": tid,
edges.append({"src_id": note_meta["id"], "dst_id": tid, "edge_type": "references", "scope": "note"}) "scope": "note",
edges.append({"src_id": tid, "dst_id": note_meta["id"], "edge_type": "backlink", "scope": "note"}) "note_id": note_id,
})
edges.append({
"kind": "backlink",
"source_id": tid,
"target_id": note_id,
"scope": "note",
"note_id": note_id,
})
# Dedupe # Dedupe
uniq = {} dedup = {}
for e in edges: for e in edges:
k = (e["src_id"], e["edge_type"], e["dst_id"], e.get("scope","note")) k = (e["kind"], e["source_id"], e["target_id"], e.get("scope", ""))
uniq[k] = e dedup[k] = e
return list(uniq.values()) return list(dedup.values())
+218 -28
View File
@@ -1,39 +1,229 @@
from __future__ import annotations #!/usr/bin/env python3
import os # -*- coding: utf-8 -*-
import hashlib # Modul: app/core/note_payload.py
from typing import Dict # Version: 1.7.0
from .parser import ParsedNote # Datum: 2025-09-09
from .parser import extract_wikilinks
def sha256_text(text: str) -> str: from __future__ import annotations
import argparse
import hashlib
import json
import os
from typing import Any, Dict, Optional, Tuple
try:
from app.core.parser import read_markdown, extract_wikilinks, FRONTMATTER_RE
except Exception: # pragma: no cover
from .parser import read_markdown, extract_wikilinks, FRONTMATTER_RE # type: ignore
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def _normalize_body(body: str, mode: str) -> str:
if mode == "none":
return body if body is not None else ""
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
text = "\n".join(line.rstrip() for line in text.split("\n"))
return text
def _resolve_hash_mode(explicit: Optional[str]) -> str:
if explicit:
val = explicit.strip().lower()
else:
val = (os.environ.get("MINDNET_HASH_MODE")
or os.environ.get("MINDNET_HASH_COMPARE")
or "body").strip().lower()
if val in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
return "full"
if val in ("frontmatter", "fm"):
return "frontmatter"
return "body"
def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, Any]]:
if not file_path or not os.path.exists(file_path):
return "", {}
try:
with open(file_path, "r", encoding="utf-8") as f:
raw = f.read()
except Exception:
return "", {}
m = FRONTMATTER_RE.match(raw)
fm = {}
if m:
fm_txt = m.group(1)
try:
import yaml # lazy
fm = yaml.safe_load(fm_txt) or {}
except Exception:
fm = {}
body = raw[m.end():]
else:
body = raw
return body, fm
def _sha256(s: str) -> str:
h = hashlib.sha256() h = hashlib.sha256()
h.update(text.encode("utf-8")) h.update(s.encode("utf-8"))
return h.hexdigest() return h.hexdigest()
def make_note_payload(parsed: ParsedNote, vault_root: str) -> Dict: def _hash_for(mode: str, *, body: str, fm: Dict[str, Any], normalize: str) -> str:
fm = parsed.frontmatter body_n = _normalize_body(body or "", normalize)
body = parsed.body fm_s = _canon_frontmatter(fm or {})
rel_path = os.path.relpath(parsed.path, vault_root) if vault_root else parsed.path if mode == "frontmatter":
return _sha256(fm_s)
if mode == "full":
return _sha256(body_n + "\n--FM--\n" + fm_s)
# default: body
return _sha256(body_n)
payload = { # ---------------------------------------------------------------------------
"note_id": fm.get("id"), # Kernfunktion
# ---------------------------------------------------------------------------
def make_note_payload(
parsed: Any,
vault_root: Optional[str] = None,
*,
hash_mode: Optional[str] = None,
hash_normalize: Optional[str] = None,
hash_source: Optional[str] = None,
file_path: Optional[str] = None,
) -> Dict[str, Any]:
"""
Liefert den Note-Payload inkl. Mehrfach-Hashes.
- Es werden IMMER die drei Hashes für (body|frontmatter|full) unter
'parsed:canonical' erzeugt (Schlüssel: z. B. 'body:parsed:canonical').
- Zusätzlich werden – falls die aktuelle Konfig (source/normalize) davon
abweicht – die drei Hashes unter den entsprechenden Schlüsseln erzeugt,
z. B. 'frontmatter:raw:none'.
- 'hash_fulltext' und 'hash_signature' repräsentieren den *aktuellen* Modus.
"""
# dict oder Objekt akzeptieren
if isinstance(parsed, dict):
fm = parsed.get("frontmatter") or {}
body_parsed = parsed.get("body") or ""
path = parsed.get("path") or ""
else:
fm = getattr(parsed, "frontmatter", {}) or {}
body_parsed = getattr(parsed, "body", "") or ""
path = getattr(parsed, "path", "") or ""
# Zielpfad relativieren
rel_path = path
try:
if vault_root:
rel = os.path.relpath(path, vault_root)
rel = rel.replace("\\", "/").lstrip("/")
rel_path = rel
except Exception:
pass
# Konfiguration auflösen
mode_resolved = _resolve_hash_mode(hash_mode) # body|frontmatter|full
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower() # parsed|raw
norm = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower() # canonical|none
# Body-Quelle laden
raw_body, raw_fm = ("", {})
if src == "raw":
raw_body, raw_fm = _read_raw_body_from_file(file_path or path)
if isinstance(raw_fm, dict) and raw_fm:
merged_fm = dict(fm)
for k, v in raw_fm.items():
merged_fm.setdefault(k, v)
fm = merged_fm
body_for_hash = raw_body
else:
body_for_hash = body_parsed
# --- 1) Standard-Tripel (parsed:canonical) immer erzeugen ---
std_src = "parsed"
std_norm = "canonical"
std_hashes: Dict[str, str] = {}
for m in ("body", "frontmatter", "full"):
std_hashes[f"{m}:{std_src}:{std_norm}"] = _hash_for(
m, body=body_parsed, fm=fm, normalize=std_norm
)
# Convenience-Felder (für Tools)
hash_body = std_hashes["body:parsed:canonical"]
hash_frontmatter = std_hashes["frontmatter:parsed:canonical"]
hash_full = std_hashes["full:parsed:canonical"]
# --- 2) Hashes für die *aktuelle* Konfiguration (falls abweichend) ---
cur_hashes: Dict[str, str] = {}
if not (src == std_src and norm == std_norm):
for m in ("body", "frontmatter", "full"):
cur_hashes[f"{m}:{src}:{norm}"] = _hash_for(
m, body=body_for_hash, fm=fm, normalize=norm
)
# --- 3) Aktueller Modus für Backwards-Compat Felder ---
current_hash = _hash_for(mode_resolved, body=body_for_hash, fm=fm, normalize=norm)
hash_signature = f"{mode_resolved}:{src}:{norm}:{current_hash}"
# Wikilinks (Note-Ebene)
refs = list(dict.fromkeys(extract_wikilinks(body_parsed))) if body_parsed else []
payload: Dict[str, Any] = {
"note_id": fm.get("id") or fm.get("note_id"),
"title": fm.get("title"), "title": fm.get("title"),
"type": fm.get("type"), "type": fm.get("type"),
"status": fm.get("status"), "status": fm.get("status"),
"created": fm.get("created"), "created": fm.get("created"),
"updated": fm.get("updated"), "updated": fm.get("updated"),
"tags": fm.get("tags") or [], "path": rel_path or fm.get("path"),
"priority": fm.get("priority"), "tags": fm.get("tags"),
"effort_min": fm.get("effort_min"), # Volltext für verlustfreien Export
"due": fm.get("due"), "fulltext": body_parsed,
"people": fm.get("people") or [], # Backwards-Compat:
"aliases": fm.get("aliases") or [], "hash_fulltext": current_hash,
"depends_on": fm.get("depends_on") or [], "hash_signature": hash_signature,
"assigned_to": fm.get("assigned_to") or [], # Option C: Mehrfach-Hashes
"references": list(sorted(set(extract_wikilinks(body)))), "hashes": {**std_hashes, **cur_hashes},
"lang": fm.get("lang") or None, "hash_body": hash_body,
"path": rel_path.replace("\\", "/"), "hash_frontmatter": hash_frontmatter,
"hash_fulltext": sha256_text(body.strip()), "hash_full": hash_full,
# Fallback-Refs
"references": refs,
} }
# Entferne None-Werte für saubere Payloads
return {k: v for k, v in payload.items() if v not in (None, [])} for k in ("area", "project", "source", "lang", "slug", "aliases"):
if k in fm:
payload[k] = fm[k]
return payload
# ---------------------------------------------------------------------------
# CLI – Sichtprüfung
# ---------------------------------------------------------------------------
def _cli() -> None:
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
ap.add_argument("--from-file", dest="src", required=True)
ap.add_argument("--vault-root", dest="vault_root", default=None)
ap.add_argument("--print", dest="do_print", action="store_true")
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None)
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
args = ap.parse_args()
parsed = read_markdown(args.src)
payload = make_note_payload(
parsed,
vault_root=args.vault_root,
hash_mode=args.hash_mode,
hash_normalize=args.hash_normalize,
hash_source=args.hash_source,
file_path=args.src,
)
if args.do_print:
print(json.dumps(payload, ensure_ascii=False, indent=2))
if __name__ == "__main__": # pragma: no cover
_cli()
+244 -45
View File
@@ -1,67 +1,266 @@
from __future__ import annotations #!/usr/bin/env python3
from dataclasses import dataclass # -*- coding: utf-8 -*-
import re """
import os Modul: app/core/parser.py
import unicodedata Version: 1.7.1 (fault-tolerant, API-kompatibel)
import yaml Datum: 2025-10-01
from typing import Tuple, Dict
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?", re.DOTALL) # YAML-Frontmatter am Anfang Zweck
-----
Fehlertolerantes Einlesen von Markdown-Dateien mit YAML-Frontmatter.
Kompatibel zur bisherigen Parser-API, aber robust gegenüber Nicht-UTF-8-Dateien:
- Versucht nacheinander: utf-8 → utf-8-sig → cp1252 → latin-1.
- Bei Fallback wird ein JSON-Warnhinweis auf stdout ausgegeben; der Import bricht NICHT ab.
- YAML-Frontmatter wird mit '---' am Anfang und '---' als Abschluss erkannt.
- extract_wikilinks() normalisiert [[id#anchor|label]] → 'id'.
Öffentliche API (kompatibel):
- class ParsedNote(frontmatter: dict, body: str, path: str)
- read_markdown(path) -> ParsedNote | None
- normalize_frontmatter(fm) -> dict
- validate_required_frontmatter(fm, required: tuple[str,...]=("id","title")) -> None
- extract_wikilinks(text) -> list[str]
- FRONTMATTER_RE (Kompatibilitäts-Konstante; Regex für '---'-Zeilen)
Beispiele
---------
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
parsed = read_markdown("./vault/30_projects/project-demo.md")
fm = normalize_frontmatter(parsed.frontmatter)
validate_required_frontmatter(fm)
body = parsed.body
from app.core.parser import extract_wikilinks
links = extract_wikilinks(body)
Abhängigkeiten
--------------
- PyYAML (yaml)
Lizenz: MIT (projektintern)
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Dict, Optional, Tuple, Iterable, List
import io
import json
import os
import re
try:
import yaml # PyYAML
except Exception as e: # pragma: no cover
yaml = None # Fehler wird zur Laufzeit geworfen, falls wirklich benötigt
# ---------------------------------------------------------------------
# Datamodell
# ---------------------------------------------------------------------
@dataclass @dataclass
class ParsedNote: class ParsedNote:
frontmatter: Dict frontmatter: Dict[str, Any]
body: str body: str
path: str path: str
def _strip_bom(text: str) -> str:
return text.lstrip("\ufeff")
def _normalize_text(t: str) -> str: # ---------------------------------------------------------------------
# Unicode-NFKC + vereinheitlichte Zeilenenden + Trim # Frontmatter-Erkennung
t = unicodedata.normalize("NFKC", t) # ---------------------------------------------------------------------
t = t.replace("\r\n", "\n").replace("\r", "\n")
return t
def read_markdown(path: str) -> ParsedNote: # Öffentliche Kompatibilitäts-Konstante: frühere Skripte importieren FRONTMATTER_RE
with open(path, "r", encoding="utf-8") as f: FRONTMATTER_RE = re.compile(r"^\s*---\s*$") # <— public
raw = _strip_bom(f.read()) # Zusätzlich interner Alias (falls jemand ihn referenziert)
raw = _normalize_text(raw) FRONTMATTER_END = FRONTMATTER_RE # <— public alias
m = FRONTMATTER_RE.match(raw) # interne Namen bleiben bestehen
front, body = {}, raw _FRONTMATTER_HEAD = FRONTMATTER_RE
if m: _FRONTMATTER_END = FRONTMATTER_RE
yaml_block = m.group(1)
body = raw[m.end():]
def _split_frontmatter(text: str) -> Tuple[Dict[str, Any], str]:
"""
Zerlegt Text in (frontmatter: dict, body: str).
Erkennt Frontmatter nur, wenn die erste Zeile '---' ist und später ein zweites '---' folgt.
YAML-Fehler im Frontmatter führen NICHT zum Abbruch: es wird dann ein leeres dict benutzt.
"""
lines = text.splitlines(True) # keep line endings
if not lines:
return {}, ""
if not _FRONTMATTER_HEAD.match(lines[0]):
# kein Frontmatter-Header → gesamter Text ist Body
return {}, text
end_idx = None
# Suche nach nächstem '---' (max. 2000 Zeilen als Sicherheitslimit)
for i in range(1, min(len(lines), 2000)):
if _FRONTMATTER_END.match(lines[i]):
end_idx = i
break
if end_idx is None:
# unvollständiger Frontmatter-Block → behandle alles als Body
return {}, text
fm_raw = "".join(lines[1:end_idx])
body = "".join(lines[end_idx + 1:])
data: Dict[str, Any] = {}
if yaml is None:
raise RuntimeError("PyYAML ist nicht installiert (pip install pyyaml).")
try:
loaded = yaml.safe_load(fm_raw) or {}
if isinstance(loaded, dict):
data = loaded
else:
data = {}
except Exception as e:
# YAML-Fehler nicht fatal machen
print(json.dumps({"warn": "frontmatter_yaml_parse_failed", "error": str(e)}))
data = {}
# optionales kosmetisches Trim: eine führende Leerzeile im Body entfernen
if body.startswith("\n"):
body = body[1:]
return data, body
# ---------------------------------------------------------------------
# Robustes Lesen mit Encoding-Fallback
# ---------------------------------------------------------------------
_FALLBACK_ENCODINGS: Tuple[str, ...] = ("utf-8", "utf-8-sig", "cp1252", "latin-1")
def _read_text_with_fallback(path: str) -> Tuple[str, str, bool]:
"""
Liest Datei mit mehreren Decodierungsversuchen.
Rückgabe: (text, used_encoding, had_fallback)
- had_fallback=True, falls NICHT 'utf-8' verwendet wurde (oder 'utf-8-sig').
"""
last_err: Optional[str] = None
for enc in _FALLBACK_ENCODINGS:
try: try:
front = yaml.safe_load(yaml_block) or {} with io.open(path, "r", encoding=enc, errors="strict") as f:
if not isinstance(front, dict): text = f.read()
raise ValueError("Frontmatter must be a mapping") # 'utf-8-sig' zählt hier als Fallback (weil BOM), aber ist unproblematisch
except yaml.YAMLError as e: return text, enc, (enc != "utf-8")
raise ValueError(f"Invalid YAML frontmatter in {path}: {e}") from e except UnicodeDecodeError as e:
last_err = f"{type(e).__name__}: {e}"
continue
return ParsedNote(frontmatter=front, body=body, path=path) # Letzter, extrem defensiver Fallback: Bytes → UTF-8 mit REPLACE (keine Exception)
with open(path, "rb") as fb:
raw = fb.read()
text = raw.decode("utf-8", errors="replace")
print(json.dumps({
"path": path,
"warn": "encoding_fallback_exhausted",
"info": last_err or "unknown"
}, ensure_ascii=False))
return text, "utf-8(replace)", True
RE_WIKILINK = re.compile(r"\[\[([^\]\|#]+)(?:#[^\]]+)?(?:\|[^\]]+)?\]\]") # [[id]] | [[id#anchor]] | [[id|label]]
def extract_wikilinks(text: str) -> list[str]: # ---------------------------------------------------------------------
return list({m.group(1).strip() for m in RE_WIKILINK.finditer(text)}) # Öffentliche API
# ---------------------------------------------------------------------
def validate_required_frontmatter(fm: dict, required=("title","id","type","status","created")): def read_markdown(path: str) -> Optional[ParsedNote]:
missing = [k for k in required if k not in fm or fm[k] in (None, "")] """
Liest eine Markdown-Datei fehlertolerant:
- Erlaubt verschiedene Encodings (UTF-8 bevorzugt, cp1252/latin-1 als Fallback).
- Schlägt NICHT mit UnicodeDecodeError fehl.
- Gibt ParsedNote(frontmatter, body, path) zurück oder None, falls die Datei nicht existiert.
Bei Decoding-Fallback wird ein JSON-Warnhinweis geloggt:
{"path": "...", "warn": "encoding_fallback_used", "used": "cp1252"}
"""
if not os.path.exists(path):
return None
text, enc, had_fb = _read_text_with_fallback(path)
if had_fb:
print(json.dumps({"path": path, "warn": "encoding_fallback_used", "used": enc}, ensure_ascii=False))
fm, body = _split_frontmatter(text)
return ParsedNote(frontmatter=fm or {}, body=body or "", path=path)
def validate_required_frontmatter(fm: Dict[str, Any],
required: Tuple[str, ...] = ("id", "title")) -> None:
"""
Prüft, ob alle Pflichtfelder vorhanden sind.
Default-kompatibel: ('id', 'title'), kann aber vom Aufrufer erweitert werden, z. B.:
validate_required_frontmatter(fm, required=("id","title","type","status","created"))
Hebt ValueError, falls Felder fehlen oder leer sind.
"""
if fm is None:
fm = {}
missing = []
for k in required:
v = fm.get(k)
if v is None:
missing.append(k)
elif isinstance(v, str) and not v.strip():
missing.append(k)
if missing: if missing:
raise ValueError(f"Missing required frontmatter fields: {', '.join(missing)}") raise ValueError(f"Missing required frontmatter fields: {', '.join(missing)}")
# einfache Plausibilitäten # Plausibilitäten: 'tags' sollte eine Liste sein, wenn vorhanden
if not isinstance(fm.get("tags", []), (list, tuple)): if "tags" in fm and fm["tags"] not in (None, "") and not isinstance(fm["tags"], (list, tuple)):
if "tags" in fm and fm["tags"] not in (None, ""): raise ValueError("frontmatter 'tags' must be a list of strings")
raise ValueError("tags must be a list of strings")
def normalize_frontmatter(fm: dict) -> dict:
# kleinere Normalisierungen ohne die Semantik zu verändern def normalize_frontmatter(fm: Dict[str, Any]) -> Dict[str, Any]:
out = dict(fm) """
if "tags" in out and isinstance(out["tags"], list): Sanfte Normalisierung ohne Semantikänderung:
out["tags"] = [str(t).strip() for t in out["tags"]] - 'tags' → Liste von Strings (Trim)
- 'embedding_exclude' → bool
- andere Felder unverändert
"""
out = dict(fm or {})
if "tags" in out:
if isinstance(out["tags"], str):
out["tags"] = [out["tags"].strip()] if out["tags"].strip() else []
elif isinstance(out["tags"], list):
out["tags"] = [str(t).strip() for t in out["tags"] if t is not None]
else:
out["tags"] = [str(out["tags"]).strip()] if out["tags"] not in (None, "") else []
if "embedding_exclude" in out: if "embedding_exclude" in out:
out["embedding_exclude"] = bool(out["embedding_exclude"]) out["embedding_exclude"] = bool(out["embedding_exclude"])
return out return out
# ------------------------------ Wikilinks ---------------------------- #
# Basismuster für [[...]]; die Normalisierung (id vor '#', vor '|') macht extract_wikilinks
_WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
def extract_wikilinks(text: str) -> List[str]:
"""
Extrahiert Wikilinks wie [[id]], [[id#anchor]], [[id|label]], [[id#anchor|label]].
Rückgabe sind NUR die Ziel-IDs (ohne Anchor/Label), führend/folgend getrimmt.
Keine aggressive Slug-Normalisierung (die kann später im Resolver erfolgen).
"""
if not text:
return []
out: List[str] = []
for m in _WIKILINK_RE.finditer(text):
raw = (m.group(1) or "").strip()
if not raw:
continue
# Split an Pipe (Label) → links vor '|'
if "|" in raw:
raw = raw.split("|", 1)[0].strip()
# Split an Anchor
if "#" in raw:
raw = raw.split("#", 1)[0].strip()
if raw:
out.append(raw)
return out
+31 -59
View File
@@ -2,30 +2,16 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
Name: app/core/qdrant.py Name: app/core/qdrant.py
Version: v1.3.1 (2025-09-05) Version: v1.4.0 (2025-09-09)
Kurzbeschreibung: Kurzbeschreibung:
Qdrant-Client & Collection-Setup für mindnet. Qdrant-Client & Collection-Setup für mindnet.
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren. - Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
- Edges-Collection nutzt 1D Dummy-Vektor (Workaround für Python-Client). - Edges-Collection nutzt 1D Dummy-Vektor.
- **Nicht-destruktiv per Default**: ensure_collections(..., destructive=False). - NEW: ensure_payload_indexes(...) legt sinnvolle Payload-Indizes an.
- **Abwärtskompatibel**: collection_names(prefix) wieder verfügbar.
Aufruf/Verwendung: Aufruf:
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
Umgebungsvariablen (optional):
QDRANT_URL | QDRANT_HOST/QDRANT_PORT, QDRANT_API_KEY,
COLLECTION_PREFIX (Default "mindnet"), VECTOR_DIM (Default 384)
Änderungen:
v1.3.1: Helper collection_names(prefix) wiederhergestellt (für reset_qdrant usw.).
v1.3.0: ensure_collections(..., destructive=False) – keine stillen Drops im Dry-Run.
Edges-Collection nur bei explicit destructive=True neu anlegen.
≤v1.2.x: Konnte {prefix}_edges bei fehlender VectorConfig automatisch neu erstellen (riskant).
Bezug/Quelle:
Alte Core-Variante enthielt collection_names(prefix); diverse Scripts nutzen das weiterhin. :contentReference[oaicite:1]{index=1}
""" """
from __future__ import annotations from __future__ import annotations
import os import os
@@ -36,10 +22,6 @@ from qdrant_client import QdrantClient
from qdrant_client.http import models as rest from qdrant_client.http import models as rest
# -------------------------------
# Konfiguration
# -------------------------------
@dataclass @dataclass
class QdrantConfig: class QdrantConfig:
url: str url: str
@@ -60,18 +42,10 @@ class QdrantConfig:
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim) return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
# -------------------------------
# Client
# -------------------------------
def get_client(cfg: QdrantConfig) -> QdrantClient: def get_client(cfg: QdrantConfig) -> QdrantClient:
return QdrantClient(url=cfg.url, api_key=cfg.api_key) return QdrantClient(url=cfg.url, api_key=cfg.api_key)
# -------------------------------
# Collection-Erzeuger (Hilfsfunktionen)
# -------------------------------
def _create_notes(client: QdrantClient, name: str, dim: int) -> None: def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
if not client.collection_exists(name): if not client.collection_exists(name):
client.create_collection( client.create_collection(
@@ -94,16 +68,7 @@ def _create_edges(client: QdrantClient, name: str) -> None:
) )
# -------------------------------
# Public API
# -------------------------------
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None: def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
"""
Stellt sicher, dass die drei Collections existieren.
- Default **nicht destruktiv**: vorhandene Collections bleiben unangetastet.
- Nur wenn 'destructive=True', wird eine ungeeignete Edges-Collection gelöscht und neu angelegt.
"""
notes = f"{prefix}_notes" notes = f"{prefix}_notes"
chunks = f"{prefix}_chunks" chunks = f"{prefix}_chunks"
edges = f"{prefix}_edges" edges = f"{prefix}_edges"
@@ -112,41 +77,48 @@ def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive:
_create_chunks(client, chunks, dim) _create_chunks(client, chunks, dim)
if client.collection_exists(edges): if client.collection_exists(edges):
# Prüfen, ob die Edges-Collection bereits eine Vektorkonfig hat
try: try:
info = client.get_collection(edges) info = client.get_collection(edges)
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None) vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
has_vectors = getattr(vectors_cfg, "vectors", None) is not None has_vectors = getattr(vectors_cfg, "vectors", None) is not None
except Exception: except Exception:
# konservativ: nichts anfassen, um Datenverlust zu vermeiden
has_vectors = True has_vectors = True
if not has_vectors: if not has_vectors:
if destructive: if destructive:
client.delete_collection(edges) client.delete_collection(edges)
_create_edges(client, edges) _create_edges(client, edges)
else: else:
print( print(f"[ensure_collections] WARN: '{edges}' ohne VectorConfig; destructive=False.", flush=True)
f"[ensure_collections] WARN: '{edges}' ohne VectorConfig gefunden; "
f"keine destruktive Änderung (destructive=False).",
flush=True,
)
else: else:
_create_edges(client, edges) _create_edges(client, edges)
def collection_names(prefix: str) -> Tuple[str, str, str]: def collection_names(prefix: str) -> Tuple[str, str, str]:
"""
Abwärtskompatibler Helper für Scripts:
returns (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
"""
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges") return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
def wipe_collections(client: QdrantClient, prefix: str) -> None: # -------------------------------
""" # NEW: Payload-Indexing
Löscht alle drei Collections – nur verwenden, wenn bewusst ein Clean-Rebuild gewünscht ist. # -------------------------------
"""
for name in collection_names(prefix): def _safe_create_index(client: QdrantClient, col: str, field: str, schema: rest.PayloadSchemaType):
if client.collection_exists(name): try:
client.delete_collection(name) client.create_payload_index(
collection_name=col,
field_name=field,
field_schema=schema,
)
except Exception:
# bereits vorhanden oder nicht unterstütztes Schema → ignorieren
pass
def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
notes, chunks, edges = collection_names(prefix)
# Notes
_safe_create_index(client, notes, "note_id", rest.PayloadSchemaType.KEYWORD)
# Chunks
_safe_create_index(client, chunks, "note_id", rest.PayloadSchemaType.KEYWORD)
_safe_create_index(client, chunks, "chunk_index", rest.PayloadSchemaType.INTEGER)
# Edges
for f in ("kind", "scope", "source_id", "target_id", "note_id"):
_safe_create_index(client, edges, f, rest.PayloadSchemaType.KEYWORD)
+79 -21
View File
@@ -1,19 +1,31 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
Name: app/core/qdrant_points.py app/core/qdrant_points.py
Version: v1.2.0 (2025-09-05)
Kurzbeschreibung Zweck
Qdrant-Points für Notes/Chunks/Edges. - Gemeinsame Helfer zum Erzeugen von Qdrant-Points für Notes, Chunks und Edges.
NEU: Edge-ID berücksichtigt optional 'occ' (Occurrence je Match): - Abwärtskompatibel zu altem Edge-Payload-Schema aus edges.py:
"{kind}:{src}->{tgt}#{seq}@{occ}" - alt: {'edge_type','src_id','dst_id', ...}
- neu: {'kind','source_id','target_id', ...}
Aufruf Version
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch - 1.3 (2025-09-08)
Changelog Änderungen (ggü. 1.2)
v1.2.0: Edge-ID mit '@occ' falls vorhanden. - points_for_edges() akzeptiert jetzt beide Edge-Schemata.
- Normalisiert alte Felder auf 'kind' / 'source_id' / 'target_id' und schreibt eine
stabile 'edge_id' zurück in die Payload.
- Verhindert, dass mehrere Edges dieselbe Point-ID erhalten (Root Cause deiner 1-Edge-Sammlung).
Aufruf / Verwendung
- Wird von Import-/Backfill-Skripten via:
from app.core.qdrant_points import points_for_note, points_for_chunks, points_for_edges, upsert_batch
eingebunden. Keine CLI.
Hinweise
- Edges bekommen absichtlich einen 1D-Dummy-Vektor [0.0], damit Qdrant das Objekt akzeptiert.
- Die Point-IDs werden deterministisch aus stabilen Strings (UUIDv5) abgeleitet.
""" """
from __future__ import annotations from __future__ import annotations
@@ -21,13 +33,23 @@ import uuid
from typing import List, Tuple from typing import List, Tuple
from qdrant_client.http import models as rest from qdrant_client.http import models as rest
def _names(prefix: str) -> Tuple[str, str, str]: def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges" return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _to_uuid(stable_key: str) -> str: def _to_uuid(stable_key: str) -> str:
"""Stabile UUIDv5 aus einem String-Key (deterministisch)."""
return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key)) return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_key))
def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | None, dim: int) -> Tuple[str, List[rest.PointStruct]]:
def points_for_note(
prefix: str,
note_payload: dict,
note_vec: List[float] | None,
dim: int,
) -> Tuple[str, List[rest.PointStruct]]:
"""Notes-Collection: falls kein Note-Embedding -> Nullvektor der Länge dim."""
notes_col, _, _ = _names(prefix) notes_col, _, _ = _names(prefix)
vector = note_vec if note_vec is not None else [0.0] * int(dim) vector = note_vec if note_vec is not None else [0.0] * int(dim)
raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id" raw_note_id = note_payload.get("note_id") or note_payload.get("id") or "missing-note-id"
@@ -35,7 +57,18 @@ def points_for_note(prefix: str, note_payload: dict, note_vec: List[float] | Non
pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload) pt = rest.PointStruct(id=point_id, vector=vector, payload=note_payload)
return notes_col, [pt] return notes_col, [pt]
def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[List[float]]) -> Tuple[str, List[rest.PointStruct]]:
def points_for_chunks(
prefix: str,
chunk_payloads: List[dict],
vectors: List[List[float]],
) -> Tuple[str, List[rest.PointStruct]]:
"""
Chunks-Collection: erwartet pro Chunk einen Vektor.
Robustheit:
- Fehlt 'chunk_id', nutze 'id', sonst baue '${note_id}#${i}' (1-basiert).
- Schreibe die abgeleitete ID zurück in die Payload (pl['chunk_id']).
"""
_, chunks_col, _ = _names(prefix) _, chunks_col, _ = _names(prefix)
points: List[rest.PointStruct] = [] points: List[rest.PointStruct] = []
for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1): for i, (pl, vec) in enumerate(zip(chunk_payloads, vectors), start=1):
@@ -48,30 +81,55 @@ def points_for_chunks(prefix: str, chunk_payloads: List[dict], vectors: List[Lis
points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl)) points.append(rest.PointStruct(id=point_id, vector=vec, payload=pl))
return chunks_col, points return chunks_col, points
def _normalize_edge_payload(pl: dict) -> dict:
"""
Sorgt für kompatible Feldnamen.
akzeptiert:
- neu: kind, source_id, target_id, seq?
- alt: edge_type, src_id, dst_id, order?/index?
schreibt zurück: kind, source_id, target_id, seq?
"""
# bereits neu?
kind = pl.get("kind") or pl.get("edge_type") or "edge"
source_id = pl.get("source_id") or pl.get("src_id") or "unknown-src"
target_id = pl.get("target_id") or pl.get("dst_id") or "unknown-tgt"
seq = pl.get("seq") or pl.get("order") or pl.get("index")
# in Payload zurückschreiben (ohne alte Felder zu entfernen → maximal kompatibel)
pl.setdefault("kind", kind)
pl.setdefault("source_id", source_id)
pl.setdefault("target_id", target_id)
if seq is not None and "seq" not in pl:
pl["seq"] = seq
return pl
def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]: def points_for_edges(prefix: str, edge_payloads: List[dict]) -> Tuple[str, List[rest.PointStruct]]:
""" """
Edges-Collection mit 1D-Dummy-Vektor (keine Vektorsuche auf Edges). Edges-Collection mit 1D-Dummy-Vektor.
Stable-ID: - Akzeptiert sowohl neues als auch altes Edge-Schema (siehe _normalize_edge_payload).
Basis: "{kind}:{src}->{tgt}#{seq}" - Fehlt 'edge_id', wird sie stabil aus (kind, source_id, target_id, seq) konstruiert.
Neu: Falls 'occ' vorhanden -> "{…}@{occ}"
""" """
_, _, edges_col = _names(prefix) _, _, edges_col = _names(prefix)
points: List[rest.PointStruct] = [] points: List[rest.PointStruct] = []
for pl in edge_payloads: for raw in edge_payloads:
pl = _normalize_edge_payload(raw)
edge_id = pl.get("edge_id") edge_id = pl.get("edge_id")
if not edge_id: if not edge_id:
kind = pl.get("kind", "edge") kind = pl.get("kind", "edge")
s = pl.get("source_id", "unknown-src") s = pl.get("source_id", "unknown-src")
t = pl.get("target_id", "unknown-tgt") t = pl.get("target_id", "unknown-tgt")
seq = pl.get("seq") or pl.get("order") or "" seq = pl.get("seq") or ""
occ = pl.get("occ") edge_id = f"{kind}:{s}->{t}#{seq}"
base = f"{kind}:{s}->{t}#{seq}"
edge_id = f"{base}@{occ}" if occ is not None else base
pl["edge_id"] = edge_id pl["edge_id"] = edge_id
point_id = _to_uuid(edge_id) point_id = _to_uuid(edge_id)
points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl)) points.append(rest.PointStruct(id=point_id, vector=[0.0], payload=pl))
return edges_col, points return edges_col, points
def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None: def upsert_batch(client, collection: str, points: List[rest.PointStruct]) -> None:
if not points: if not points:
return return
+172
View File
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Modul: app/graph/service.py
Version: 0.1.0
Datum: 2025-09-10
Zweck
-----
Leichtgewichtiger Graph-Layer über Qdrant:
- get_note(note_id)
- get_chunks(note_id)
- neighbors(source_id, kinds=[...], scope=['note','chunk'], depth=1)
- walk_bfs(source_id, kinds, max_depth)
- context_for_note(note_id, max_neighbors): heuristische Kontextsammlung
Hinweise
--------
- Nutzt die bestehenden Collections <prefix>_notes/_chunks/_edges.
- Edges werden über Payload-Felder (`kind`, `source_id`, `target_id`) abgefragt.
"""
from __future__ import annotations
from typing import List, Dict, Any, Optional, Iterable, Set, Tuple
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client
def _cols(prefix: str):
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
class GraphService:
def __init__(self, cfg: Optional[QdrantConfig] = None, prefix: Optional[str] = None):
self.cfg = cfg or QdrantConfig.from_env()
if prefix:
self.cfg.prefix = prefix
self.client = get_client(self.cfg)
self.notes_col, self.chunks_col, self.edges_col = _cols(self.cfg.prefix)
# ------------------------ fetch helpers ------------------------
def _scroll(self, col: str, flt: Optional[rest.Filter] = None, limit: int = 256):
out = []
nextp = None
while True:
pts, nextp = self.client.scroll(
collection_name=col,
with_payload=True,
with_vectors=False,
limit=limit,
offset=nextp,
scroll_filter=flt,
)
if not pts:
break
out.extend(pts)
if nextp is None:
break
return out
# ------------------------ public API ---------------------------
def get_note(self, note_id: str) -> Optional[Dict[str, Any]]:
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
pts, _ = self.client.scroll(self.notes_col, with_payload=True, with_vectors=False, limit=1, scroll_filter=f)
return (pts[0].payload or None) if pts else None
def get_chunks(self, note_id: str) -> List[Dict[str, Any]]:
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
pts = self._scroll(self.chunks_col, f)
# Sortierung analog Export
def key(pl):
p = pl.payload or {}
s = p.get("seq") or 0
ci = p.get("chunk_index") or 0
n = 0
cid = p.get("chunk_id") or ""
if isinstance(cid, str) and "#" in cid:
try:
n = int(cid.rsplit("#", 1)[-1])
except Exception:
n = 0
return (int(s), int(ci), n)
pts_sorted = sorted(pts, key=key)
return [p.payload or {} for p in pts_sorted]
def neighbors(self, source_id: str, kinds: Optional[Iterable[str]] = None,
scope: Optional[Iterable[str]] = None, depth: int = 1) -> Dict[str, List[Dict[str, Any]]]:
"""
Liefert eingehende & ausgehende Nachbarn (nur nach kind gefiltert).
depth==1: direkte Kanten.
"""
kinds = list(kinds) if kinds else None
must = [rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_id))]
if kinds:
must.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
f = rest.Filter(must=must)
edges = self._scroll(self.edges_col, f)
out = {"out": [], "in": []}
for e in edges:
out["out"].append(e.payload or {})
# Inverse Richtung (eingehend)
must_in = [rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_id))]
if kinds:
must_in.append(rest.FieldCondition(key="kind", match=rest.MatchAny(any=kinds)))
f_in = rest.Filter(must=must_in)
edges_in = self._scroll(self.edges_col, f_in)
for e in edges_in:
out["in"].append(e.payload or {})
return out
def walk_bfs(self, source_id: str, kinds: Iterable[str], max_depth: int = 2) -> Set[str]:
visited: Set[str] = {source_id}
frontier: Set[str] = {source_id}
kinds = list(kinds)
for _ in range(max_depth):
nxt: Set[str] = set()
for s in frontier:
neigh = self.neighbors(s, kinds=kinds)
for e in neigh["out"]:
t = e.get("target_id")
if isinstance(t, str) and t not in visited:
visited.add(t)
nxt.add(t)
frontier = nxt
if not frontier:
break
return visited
def context_for_note(self, note_id: str, kinds: Iterable[str] = ("references","backlink"), max_neighbors: int = 12) -> Dict[str, Any]:
"""
Heuristischer Kontext: eigene Chunks + Nachbarn nach Kantenarten, dedupliziert.
"""
note = self.get_note(note_id) or {}
chunks = self.get_chunks(note_id)
neigh = self.neighbors(note_id, kinds=list(kinds))
targets = []
for e in neigh["out"]:
t = e.get("target_id")
if isinstance(t, str):
targets.append(t)
for e in neigh["in"]:
s = e.get("source_id")
if isinstance(s, str):
targets.append(s)
# de-dupe
seen = set()
uniq = []
for t in targets:
if t not in seen:
seen.add(t)
uniq.append(t)
uniq = uniq[:max_neighbors]
neighbor_notes = [self.get_note(t) for t in uniq]
return {
"note": note,
"chunks": chunks,
"neighbors": [n for n in neighbor_notes if n],
"edges_out": neigh["out"],
"edges_in": neigh["in"],
}
# Optional: Mini-CLI
if __name__ == "__main__": # pragma: no cover
import argparse, json
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV)")
ap.add_argument("--note-id", required=True)
ap.add_argument("--neighbors", action="store_true", help="Nur Nachbarn anzeigen")
args = ap.parse_args()
svc = GraphService(prefix=args.prefix)
if args.neighbors:
out = svc.neighbors(args.note_id, kinds=["references","backlink","prev","next","belongs_to"])
else:
out = svc.context_for_note(args.note_id)
print(json.dumps(out, ensure_ascii=False, indent=2))
+349
View File
@@ -0,0 +1,349 @@
# 📘 Mindnet Skript-Handbuch (aktualisiert)
Dieses Handbuch dokumentiert die Skripte und Tests für das Projekt **mindnet Wissensnetzwerk**.
Es beschreibt Zweck, Parameter, typische Aufrufe sowie aktuelle Verhaltensweisen (Hash-Modi, Parser-Robustheit, `window`/`text`-Trennung in Chunks, Export-Fixes).
Alle Beispiele sind so formatiert, dass sie direkt kopierbar sind. **Code/CLI-Beispiele sind eingerückt**, damit die Markdown-Darstellung nicht abbricht.
---
## 0) Umgebung & Konfiguration
**Wichtige ENV-Variablen**
- `QDRANT_URL` – z. B. `http://localhost:6333`
- `QDRANT_API_KEY` – API-Key (falls aktiviert)
- `COLLECTION_PREFIX` – Prefix der drei Collections, Standard: `mindnet` → `mindnet_notes`, `mindnet_chunks`, `mindnet_edges`
- `VECTOR_DIM` – Embedding-Dimension (Chunks), Standard: `384`
**Hash-Vergleich & Änderungs­erkennung**
- `MINDNET_HASH_COMPARE` – steuert, **welcher Anteil** für „changed“ verglichen wird
Werte: `Body` (Standard), `Frontmatter`, `Full`
- `MINDNET_HASH_SOURCE` – Quelle: `parsed` (Standard), `raw`, `file`
- `MINDNET_HASH_NORMALIZE` – Normalisierung: `canonical` (Standard) oder `none`
**Beispiel (nur Body berücksichtigen, kanonisch, geparst):**
export MINDNET_HASH_COMPARE=Body
export MINDNET_HASH_SOURCE=parsed
export MINDNET_HASH_NORMALIZE=canonical
source .venv/bin/activate
**Baseline-Signaturen (mehrere Modi):**
Der Importer kann **Signaturen aller Modi** (Body/Frontmatter/Full) vorhalten, damit spätere Moduswechsel deterministisch sind (keine „Mass-Changes“). Siehe `--baseline-modes` unter **Importer**.
---
## 1) Vault anlegen (für Tests)
### `scripts/make_test_vault.py`
**Zweck:**
Erstellt einen kleinen, nachvollziehbaren Test-Vault mit Markdown-Dateien.
Szenarien: offene Links, nachträgliche Anlage fehlender Notes, geänderte Chunk-Grenzen.
**Parameter:**
- `--out PATH` : Zielverzeichnis (Default: `./test_vault`)
- `--force` : Existierenden Ordner löschen und neu anlegen
- `--with-missing` : Optional – auch die `missing-note` gleich anlegen
**Hinweise für Tests**
1. Erster Import (dry-run oder `--apply`): Es gibt einen „leeren“ Link `[[missing-note]]`.
2. Lege danach eine Datei für „missing-note“ an (mit gleicher `id` im YAML) und importiere erneut:
→ Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (`references` + `backlink`).
3. Ändere den Body von `exp-one.md` so, dass andere Chunk-Grenzen entstehen und importiere erneut:
→ Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
**Kompatibilität:**
- `note.schema.json`: `created`/`updated` sind Strings.
**Beispiele:**
python3 -m scripts.make_test_vault --out ./test_vault --force
python3 -m scripts.make_test_vault --out ./test_vault --force --with-missing
---
## 2) Import in Qdrant
### `scripts/import_markdown.py`
**Zweck:**
Importiert Notes/Chunks/Edges aus einem Vault nach Qdrant.
- validiert Frontmatter
- chunked Inhalte + Embeddings
- erzeugt Edges (`references`, `references_at`, `backlink`, `next`, `prev`, `belongs_to`)
- erkennt geänderte Dateien **modusgesteuert** (Body/Frontmatter/Full)
- optional: Purge der alten Chunks/Edges pro Note
**Parameter:**
- `--vault PATH` : (Pflicht) Vault-Verzeichnis
- `--apply` : Änderungen wirklich durchführen (ohne: Dry-Run)
- `--purge-before-upsert` : Vor Upsert alte Chunks/Edges **nur der geänderten Notes** löschen
- `--prefix STR` : überschreibt `COLLECTION_PREFIX`
- `--note-id ID` : nur eine bestimmte Note importieren
- `--note-scope-refs` : zusätzlich **note-weite** `references` erzeugen (nicht nur chunk-weise)
- `--baseline-modes` : berechne & speichere Hash-Signaturen für **alle** Modi (`Body`, `Frontmatter`, `Full`) als Baseline
**Erwartetes Verhalten:**
- Unveränderte Notes → `"changed": false`, Entscheidung: `apply-skip-unchanged` (bei `--apply`)
- Geänderte Notes → `"changed": true` (abhängig von `MINDNET_HASH_COMPARE`)
- Idempotenz: mehrfacher Lauf ohne Änderungen erzeugt **keine** neuen Punkte
- Baseline: Bei `--baseline-modes` erscheinen Felder wie `hash_signature`, `hash_fulltext`, ggf. `hash_frontmatter` und `needs_baseline_for_mode: false` bei Folgeläufen.
**Beispiele:**
# Dry-Run (zeigt Änderungen, schreibt NICHT)
python3 -m scripts.import_markdown --vault ./vault --prefix mindnet
# Apply + gezielte Vorreinigung pro geänderter Note
python3 -m scripts.import_markdown --vault ./vault --prefix mindnet --apply --purge-before-upsert
# Baseline-Signaturen aller Modi vorhalten
python3 -m scripts.import_markdown --vault ./vault --apply --baseline-modes
**Hash-Modi in der Praxis:**
- `MINDNET_HASH_COMPARE=Body` → Änderungen im Body werden erkannt; reine Frontmatter-Änderungen bleiben **ohne Effekt**.
- `MINDNET_HASH_COMPARE=Frontmatter` → nur Frontmatter maßgeblich.
- `MINDNET_HASH_COMPARE=Full` → beide Anteile zählen.
---
## 3) Konsistenzprüfungen
### `scripts/audit_vault_vs_qdrant.py`
**Zweck:**
Vergleicht Vault-Inhalte mit Qdrant.
- Zählt Notes/Chunks/Edges
- Verifiziert Anzahl Wikilinks vs. Edges
- Meldet Deltas (z. B. `references` im Vault vs. in Qdrant)
**Parameter:**
- `--vault PATH` : Vault-Verzeichnis
- `--prefix STR` : Prefix für Qdrant-Collections (Default: `mindnet`)
**Beispiel:**
python3 -m scripts.audit_vault_vs_qdrant --vault ./vault --prefix mindnet
### `scripts/validate_edges.py`
**Zweck:**
Prüft die Edges-Collection in Qdrant.
- Zählt Edges pro Typ (`references`, `references_at`, `backlink`, `next`, `prev`, `belongs_to`)
- Kontrolliert Invarianten (z. B. jede `references` hat einen `backlink`)
- Listet `unresolved` Edges auf
**Parameter:**
- `--prefix STR` : Prefix für Qdrant-Collections
- `--details` : Detailausgabe mit Beispielen
**Beispiel:**
python3 -m scripts.validate_edges --prefix mindnet --details
---
## 4) Export Qdrant → Markdown (Vault)
### `scripts/export_markdown.py`
**Zweck:**
Exportiert Notes + rekonstruierten Body in einen Zielordner.
- Pfade werden **relativ** aus `note.payload.path` abgeleitet (führende `/` entfernt, Backslashes → Slashes; Unicode-NFC empfohlen)
- **Body-Rekonstruktion** priorisiert:
1. `fulltext` (falls in Note-Payload vorhanden)
2. sonst **Chunks**: Sortierung `seq` → `chunk_index` → Nummer in `chunk_id`; **nur `text`** (Overlap bereits entfernt)
- `--prefix` (CLI) wird unterstützt (oder ENV `COLLECTION_PREFIX`)
**Parameter:**
- `--out PATH` : Ziel-Root (muss existieren)
- `--overwrite` : vorhandene Dateien überschreiben
- `--note-id ID` : nur eine bestimmte Note exportieren
- `--prefix STR` : überschreibt `COLLECTION_PREFIX`
- `--include-edges {none,footer,yaml}` *(optional, falls aktiviert)*: Edges als YAML-Liste im Frontmatter oder im Footer beilegen
**Beispiele:**
export COLLECTION_PREFIX="mindnet"
python3 -m scripts.export_markdown --out ./_exportVault
# Nur eine bestimmte Note:
python3 -m scripts.export_markdown --out ./_exportVault --note-id 20250821-architektur-ki-trainerassistent-761cfe
# Existierende Dateien überschreiben:
python3 -m scripts.export_markdown --out ./_exportVault --overwrite
---
## 5) Qdrant zurücksetzen
### `scripts/reset_qdrant.py`
**Zweck:**
Setzt die Qdrant-Collections zurück.
- `wipe`: löscht Collections komplett (inkl. Schema)
- `truncate`: löscht nur Inhalte, behält Collections
- `truncate-edges`: löscht nur Punkte in `*_edges`
**Parameter:**
- `--mode {wipe,truncate,truncate-edges}`
- `--prefix STR` : Prefix der Collections (Default: `mindnet`)
- `--yes` : Automatische Bestätigung
- `--dry-run` : Nur anzeigen, nicht ausführen
**Beispiele:**
python3 -m scripts.reset_qdrant --mode truncate-edges --prefix mindnet
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
python3 -m scripts.reset_qdrant --mode wipe --prefix mindnet --yes
---
## 6) Diagnose & Tools
### `scripts/diagnose_changed.py`
**Zweck:**
Prüft, ob Datei-Hashes (`sha256`) mit gespeicherten `hash_*` in Qdrant übereinstimmen.
Hilft, wenn der Importer zu viele Dateien als „changed“ erkennt oder Moduswechsel geplant sind.
**Parameter:**
- `--vault PATH` : Vault-Verzeichnis
- `--prefix STR` : Prefix (Default: `mindnet`)
- `--note-id ID` : Nur bestimmte Note prüfen
- `--all` : Alle Notes prüfen
**Beispiel:**
python3 -m scripts.diagnose_changed --vault ./vault --prefix mindnet --all
### `scripts/resolve_unresolved_references.py`
**Zweck:**
Prüft offene Wikilinks (`status=unresolved`) und fügt Edges nach, wenn inzwischen passende Notizen existieren.
Ersetzt `unresolved` durch gültige `references`/`backlink`.
**Parameter:**
- `--prefix STR` : Prefix für Qdrant-Collections
- `--apply` : Änderungen wirklich ausführen (sonst nur Vorschau)
**Beispiele:**
python3 -m scripts.resolve_unresolved_references --prefix mindnet
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
---
## 7) Parser-Verhalten (robust & abwärtskompatibel)
- Der Parser liest Markdown **fehlertolerant** (UTF-8 mit Fallbacks), extrahiert YAML-Frontmatter und Body.
- Keine harten Abhängigkeiten von alten Regex-Symbolen (`FRONTMATTER_RE` entfällt zugunsten interner Marker).
- `read_markdown(path)` gibt `(frontmatter_dict, body_str)` zurück.
- **Sonderfälle aus Fremd-Tools** (z. B. Silverbullet-Dateien wie `CONFIG.md`, `index.md`) werden **nicht importiert**, sofern sie nicht zum Vault gehören.
---
## 8) Chunk-Payload (Fenster vs. Kern)
- **`window`** = kontextualisierter Fenster-Text (mit linkem Overlap)
- **`text`** = **Kerntext** ohne das vom vorherigen Chunk übernommene Overlap
- Felder: `start`, `end`, `overlap_left`, `overlap_right`, `chunk_index`, `seq`, `path`, `section_title`, `section_path`, `token_count` (falls vorhanden)
- **Fallback (synthetisch):** Falls der Chunker **keinen** Overlap-Fenstertext liefert, erzeugt die Payload-Schicht Fenster mit typgerechtem Overlap aus `chunk_config.get_sizes(type)['overlap']` (unterer Wert).
- **Edges** werden aus `window` abgeleitet (um Links an Chunk-Grenzen nicht zu verlieren); der **Export** rekonstruiert den Body aus `text` (Overlap-frei).
---
## 9) Edge-Modell & IDs
- Collections:
`*_notes` (optional Vektor / Nullvektor), `*_chunks` (384-d Embeddings), `*_edges` (1-d Dummy-Vektor)
- **Deterministische IDs** (UUIDv5 / stabile Keys):
- `note_id` stabil aus Frontmatter/Slug
- `chunk_id` i. d. R. `note_id#<n>`
- `edge_id` stabil aus `(kind, source_id, target_id, seq)`
- Kantenarten: `references`, `references_at`, `backlink`, `next`, `prev`, `belongs_to`
- `unresolved` wird gekennzeichnet und kann per Resolver später aufgelöst werden.
---
## 10) Tests & Qualitätssicherung
**Tests unter `tests/`**
- `check_chunks_window_vs_text.py` – vergleicht je Chunk `window` vs. `text`, schätzt Overlap
- `compare_vaults.py` – rekursiver Vault-Vergleich (Body/Frontmatter/All, Key: auto/id/title/filename), Unicode-NFC-robust
- `verify_chunks_integrity.py` – prüft, ob Rekonstruktion aus Chunks (`text`) zu `fulltext` bzw. zum Vault-Body passt
- `list_md_seen_by_importer.py` – zeigt, welche `.md` der Importer sieht (inkl. Gründe/Filter)
- `inspect_one_note.py` – Parser-Diagnose für eine konkrete Datei
- `run_e2e_roundtrip.sh` – End-to-End: Reset → Import → Audit → (optional) Hash-Reporter → Export → Vergleich
**Typische Aufrufe:**
# Overlap-Qualität (Fenster vs. Kern)
python3 tests/check_chunks_window_vs_text.py --prefix "$COLLECTION_PREFIX"
# Roundtrip-Vergleich (rekursiv)
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --focus body
# Chunks-Rekonstruktion (gegen Vault-Body)
python3 tests/verify_chunks_integrity.py --prefix "$COLLECTION_PREFIX" --vault ./vault
---
## 11) Typische Workflows
**A) Clean-Start mit Test-Vault**
python3 -m scripts.make_test_vault --out ./test_vault --force
python3 -m scripts.reset_qdrant --mode truncate --prefix mindnet --yes
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet --apply --purge-before-upsert
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
python3 -m scripts.validate_edges --prefix mindnet --details
export COLLECTION_PREFIX="mindnet"
python3 -m scripts.export_markdown --out ./_exportVault
**B) Änderungs-Propagation & Idempotenz**
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet --apply --purge-before-upsert
python3 -m scripts.audit_vault_vs_qdrant --vault ./test_vault --prefix mindnet
python3 -m scripts.validate_edges --prefix mindnet --details
python3 -m scripts.import_markdown --vault ./test_vault --prefix mindnet
**C) Unresolved Links lösen**
python3 -m scripts.import_markdown --vault ./test_vault --apply --prefix mindnet
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
python3 -m scripts.validate_edges --prefix mindnet --details
**D) Export & Roundtrip (Produkt-Vault)**
export COLLECTION_PREFIX="mindnet"
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
python3 -m scripts.export_markdown --out ./_exportVault --prefix "$COLLECTION_PREFIX"
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --focus body
---
## 12) Troubleshooting (kurz)
- **Importer markiert alle als changed (Moduswechsel):**
→ `--baseline-modes` einmalig laufen lassen; sicherstellen: `MINDNET_HASH_COMPARE` passt zum Ziel.
- **Edges verschwinden / nur 1 Edge sichtbar:**
→ `app/core/qdrant_points.py` muss Edge-Payload normalisieren (Schema Alt/Neu) und `edge_id` eindeutig bilden.
- **Export ohne Body / falsches Zielverzeichnis:**
→ Exporter nutzt relativierte Pfade und rekonstruiert Body aus `fulltext` oder Chunks (`text`). ENV/CLI `--prefix` beachten.
- **Parser-Fehler (UTF-8):**
→ Neuer Parser arbeitet mit Fallbacks; mit `tests/inspect_one_note.py` prüfen, was ankommt.
- **„Fehlende“ Dateien im Vergleich:**
→ Prüfe, ob Dateien **wirklich zum Vault gehören** (z. B. Silverbullet-Artefakte `CONFIG.md`, `index.md` ausblenden).
---
## 13) Hinweise zu Silverbullet-Dateien
- Dateien wie `CONFIG.md` oder `index.md`, die **nicht** zum inhaltlichen Vault gehören, werden **nicht importiert** und erscheinen daher nicht im Export-Roundtrip.
- Diese Dateien bitte vom produktiven Vault trennen oder im Vergleich (`tests/compare_vaults.py
+106 -116
View File
@@ -13,74 +13,67 @@ Entwickle eine robuste, parser-freundliche **Chunking-Strategie** für Markdown-
--- ---
## 2) Annahmen aus `knowledge_design.md` ## 2) Annahmen aus `knowledge_design.md`
- **Stabile IDs** & Dateinamen-Schema (z. B. `YYYYMMDD-HHMM-type-slug` bzw. `type-slug`). - **IDs**: deterministische, stabile `id` pro Note (Slug / semantischer Identifier), ggf. `aliases`.
- **Pflichtfelder** in YAML: `title`, `id`, `type`, `status`, `created`, `tags` (empfohlene Felder ggf. mehr). - **Dateinamen**: entweder `YYYY-MM-DD_title.md` oder `slug.md`; Pfade sind **relativ** zum Vault.
- **Abschnitts-Konventionen** je `type` (z. B. `## Zusammenfassung`, `## Kontext` …) → ideale **Chunk-Grenzen**. - **Frontmatter-Pflichtfelder**: `id`, `title`, `type`, `created`, `updated` (Details in `knowledge_design.md`).
- **Edge-Typen**: `belongs_to`, `references`, `backlink`, `depends_on`, `assigned_to`, `discussed_in`, `authored_by`, `related`. Für Chunking primär `belongs_to` & `references` (+ optionale `prev`/`next`). - **Backlink-/Link-Regeln**: Wikilinks `[[id]]` im Body; am Ende optional redaktionelle Sektion „Mögliche Verbindungen“ (erzeugt **keine** Kanten).
--- ---
## 3) Chunking-Prinzipien ## 3) Chunking-Prinzipien
1. **Frontmatter extrahieren** → als strukturiertes `metadata` (nicht in `text` des Chunks). - **Semantische Schnitte**: Chunks enden an Absätzen, Überschriften, Listenpunkten, Codeblöcken, Zitaten.
2. **Markdown AST** (oder strukturierter Parser) nutzen: - **Frontmatter**: YAML **nicht** embedden; stattdessen komplett im Note-Payload speichern.
- Primäre Grenzen: **Überschriftenebene H2/H3** (H1 = Dokumenttitel). - **Verlustfreiheit**:
- Sekundär: **Absätze**, **nummerierte/unnummerierte Listen** als atomare Einheiten, **Codeblöcke** als zusammenhängender Block (nicht splitten). - Note-Payload: `fulltext` (gesamter Body).
3. **Längensteuerung**: Semantik vor harter Länge. Wenn Abschnitt > Zielgröße, **weich segmentieren** entlang von Absätzen/Listenpunkten. - Chunk-Payload: `text` (Originalchunk).
4. **Referenz-Extraktion**: `[[Wikilinks]]` + Markdown-Links → `references`. Backlinks werden downstream generiert. Damit kann der Export den Body **verlustfrei** rekonstruieren (erst `fulltext`, sonst Chunks).
5. **Kontext-Overlap**: Chunks erhalten **Überlappung** über Satz-/Absatzgrenzen, um Kohärenz beim Retrieval zu erhöhen.
--- ---
## 4) Optimale Chunkgrößen & Overlaps ## 4) Optimale Chunkgrößen & Overlaps
> Ziel: gutes Recall@k, wenig Fragmentierung, stabile Antwortqualität. - **Zielgröße**: 250–400 Tokens (Richtwert, abhängig vom `type`).
- **Maxgröße**: 600–800 Tokens (nur in Ausnahmen).
- **Mingröße**: 80–120 Tokens (sonst mit Nachbar verschmelzen).
- **Overlap**: 40–60 Tokens, um Kontextverlust zu vermeiden.
- **Kontextabhängig**:
- `concept`, `thought`, `experience`: 250–350 Tokens
- `task`, `project`: eher 200–300 Tokens (mehr Bullet-Struktur)
- `journal`: 300–400 Tokens (tagesabschnittsweise)
| Notiz-Typ (`type`) | Inhalt | Zielgröße (Tokens) | Max. Größe (Tokens) | Overlap (Tokens) | Hinweise | ---
|---|---|---:|---:|---:|---|
| `thought` | kurze Thesen/Ideen | 150–250 | 300 | 30–40 | meist 1–2 Chunks |
| `experience` | Kontext/Beobachtung/Interpretation | 250–350 | 450 | 40–60 | Trenne entlang der Template-Sektionen |
| `journal` | Tagebuch, episodisch | 200–300 | 400 | 30–50 | Tagesabschnitte bündeln |
| `task` | knapp, Checklisten | 120–200 | 250 | 20–30 | Listen nicht splitten |
| `project` | Scope/WPs/Risiken/Status | 300–450 | 600 | 50–70 | Pro Hauptsektion ein Chunk |
| `concept` | definierte Begriffe, Erklärungen | 250–400 | 550 | 40–60 | Definition separat halten |
| `source` | Metadaten + Auszüge/Notizen | 200–350 | 500 | 30–50 | Zitate als eigene Chunks (Urheberrecht) |
## 4.1 Overlap-Regeln ## 4.1 Overlap-Regeln
- Overlap **nie mitten im Satz** beenden → mindestens **ein kompletter Satz** wandert in den nächsten Chunk. - Overlap betrifft **nur den Body**; die YAML-Frontmatter bleibt **außerhalb**.
- An **Absatzgrenzen** ausrichten, wenn möglich. - Keine Overlaps, die Codeblöcke oder Tabellen „aufreißen“. Falls nötig: Chunk-Grenze an Blockgrenze ziehen.
- Bei **Listen** ggf. den vorherigen Punkt als Overlap mitführen (falls < 50 Tokens). - Bei reinen Listen: zusammenhängende Bullets gruppieren, ggf. kleinere Overlaps.
- Codeblöcke und Tabellen **nicht splitten**; wenn sie größer als `Max-Tokens` sind, als **eigener Chunk** belassen (Ausnahmefall zulassen).
**Daumenregel (zeichenzentriert):** ~4 Zeichen ≈ 1 Token → 800–1600 Zeichen Ziel; Overlap ~120–240 Zeichen (≈ 30–60 Tokens).
**Mindestanforderung:** Keine Trennung innerhalb von Sätzen. Sicherstellung, dass immer an Satzenden gechunct wird
**Default für mindnet:** **~300 Tokens** + **~50 Tokens Overlap**; per `type` feinjustieren (siehe Tabelle).
--- ---
## 5) Semantische Regeln je Strukturelement ## 5) Semantische Regeln je Strukturelement
- **Überschriften (H2/H3)**: new chunk start; speichere `section_path` (z. B. `/Kontext/Beobachtung`). - **Überschriften**: vorzugsweise an H2/H3 trennen; H1 meist Dokumenttitel → Teil der Frontmatter.
- **Absätze**: Primäre Untereinheiten; wenn Zusammenfassung + Beispiel direkt folgen, zusammen lassen. - **Absätze**: primäre Trennkandidaten.
- **Listen**: Jeder **Listeneintrag** bleibt zusammen mit seinem einleitenden Satz (falls vorhanden). Große Listen → in **logische Blöcke** (5–8 Items) splitten. - **Listen**: zusammenhängende Gruppe als ein Chunk, wenn sie einen semantischen Block bilden.
- **Codeblöcke**: Nicht splitten; ggf. separater Chunk, **aber** im Retrieval **downweighten** (optional). - **Codeblöcke**: niemals splitten; Code bleibt in einem Chunk.
- **Zitate/Blockquotes**: zusammenhalten; Quelle im `references_meta` anreichern (falls Link vorhanden). - **Zitate**: ab 2+ Zeilen als eigener Chunk.
- **Tabellen**: als ein Chunk; zusätzlich **Plain-Text-Extrakt** (Header + 1–2 Zeilen) in `summary` für Retriever, falls Modelle Tabellen schlechter verarbeiten. - **Bilder/Anhänge**: Referenzen im Text belassen; Pfade relativ halten.
--- ---
## 6) Normalisierung & Preprocessing ## 6) Normalisierung & Preprocessing
- **Nicht embedden:** YAML-Frontmatter (aber spezifische Felder als `metadata.*` mitschreiben: `type`, `tags`, `area`, `project`, `priority`, `people`, `aliases`). - **Zeilenenden**: `\r\n` → `\n`, trailing spaces trimmen (nur für Hashing, nicht für Anzeige).
- **Bereinigen:** führende/trailing Spaces, Mehrfach-Leerzeilen → 1; Unicode-NFKC; Zeilenenden normalisieren. - **Whitespace**: mehrfach-Blankzeilen reduzieren (optional, Hash-Modus beachten).
- **Bewahren:** Markdown-Semantik (Überschriften-Hashes, Listenpräfixe, Codefences) **im Text beibehalten**, damit RAG-Antworten sauber zitieren können. - **Links**: `[[Titel|id]]` zu `(display="Titel", target_id="id")` normalisieren.
- **Sprachhinweis:** `lang` heuristisch bestimmen (DE/EN/…); in `metadata.lang` ablegen → später für sprachspezifische Embeddings nützlich. - **Pfade**: `path` im Payload **zwingend relativ** (Backslashes → Slashes).
- **Stop-Abschnitte:** „Mögliche Verbindungen“ am Ende **separat** chunken (nur Links), um Link-Graph sauber zu extrahieren.
--- ---
## 7) Edge-Modell (Graph) ## 7) Edge-Modell (Graph)
- **Chunk → Note:** `belongs_to(note_id)` (obligatorisch). - **Chunk → Note:** `belongs_to(note_id)` (obligatorisch; Edge-Payload enthält zusätzlich `note_id` als Owner-Note).
- **Chunk → Chunk (gleiche Note):** `prev`, `next` (lineare Leserichtung). - **Chunk → Chunk (gleiche Note):** `prev`, `next` (symmetrisch für lineare Navigation; beide Richtungen werden erzeugt).
- **Note/Chunk → Note:** `references(target_id)` aus `[[Wikilinks]]` & Markdown-Links; optional `related` (schwach). - **Chunk → Note:** `references(target_id)` aus `[[Wikilinks]]` oder Markdown-Links im Chunk-Text (Standard; *Chunk-Scope*).
- **Backlink:** systemisch generiert: inverse Kante `backlink`. - **Note → Note:** `backlink(target_id)` automatisch generiert, dedupliziert pro Note (*Note-Scope*).
- **Sektionen:** `has_section(note_id, section_path)` (optional, für Navigations-UI). - **Optional:** `references:note` (Note-Scope-References) per Flag/ENV aktivierbar; **Default = aus**, um Doppelzählungen zu vermeiden.
- **Task-Spezialfälle:** `depends_on`, `assigned_to(person-id)` bleiben **Note-Level** (nicht Chunk-Level). - **Unresolved:** Wenn `target_id` nicht existiert, wird eine Kante mit `status: "unresolved"` angelegt (später durch Resolver/Agent heilbar).
--- ---
@@ -100,26 +93,49 @@ Entwickle eine robuste, parser-freundliche **Chunking-Strategie** für Markdown-
"project": "project-mindnet", "project": "project-mindnet",
"tags": ["area/mindnet","type/thought","topic/yaml"], "tags": ["area/mindnet","type/thought","topic/yaml"],
"chunk_index": 2, "chunk_index": 2,
"char_start": 1240, "char_start": 1024,
"char_end": 2012, "char_end": 1789,
"token_count": 305,
"section_title": "Begründung / Details", "section_title": "Begründung / Details",
"section_path": "/Begründung / Details", "section_path": "Begründung / Details",
"lang": "de", "neighbors": {"prev": "…#c01", "next": "…#c03"},
"wikilinks": ["concept-vektorsuche-qdrant","source-obsidian-properties"], "wikilinks": ["concept-yaml-styleguide"],
"external_links": [], "references": [{"kind":"wikilink", "target_id":"concept-yaml-styleguide"}],
"references": [ "text": "… ursprünglicher Chunk-Text …"
{"target_id": "concept-vektorsuche-qdrant", "kind": "wikilink"}
],
"neighbors": {"prev": "#c01", "next": "#c03"},
"created_at": "2025-09-02T18:35:00+02:00"
} }
} }
**Collection-Einstellungen (Empfehlung)** **Point (Note)**
- `distance`: `Cosine` (gemischte Textlängen)
- `vectors`: 384–1024 Dim (abh. Embedding-Modell) {
- `hnsw_ef_construct`: 128–256; `m`: 16–32 (Workload-abhängig) "id": "20250902-1830-thought-yaml-recall",
"vector": [/* optional Nullvektor oder Zentroid */],
"payload": {
"note_id": "20250902-1830-thought-yaml-recall",
"title": "Gedanke: Einheitliche YAML-Standards steigern Recall",
"type": "thought",
"status": "draft",
"created": "2025-09-02",
"updated": "2025-09-02",
"path": "10_thoughts/20250902-1830-thought-yaml-recall.md",
"tags": ["area/mindnet","type/thought","topic/yaml"],
"hash_fulltext": "sha256:…",
"fulltext": "… gesamter Body für verlustfreie Rekonstruktion …",
"references": ["concept-yaml-styleguide"] // Fallback für Note-Level-Backlinks
}
}
**Edges (Payload; neues Schema)**
{
"kind": "references" | "backlink" | "belongs_to" | "prev" | "next",
"source_id": "<chunk_id|note_id>",
"target_id": "<note_id|chunk_id>",
"scope": "chunk" | "note",
"note_id": "<owner-note-id>", // neu: für schnelles Filtern/Purge
"status": "ok" | "unresolved" // optional
}
**Performance-Hinweis (Qdrant):** Für schnelle Filter/Deletes Payload-Indizes anlegen: **edges** → `kind`, `scope`, `source_id`, `target_id`, `note_id`; **chunks** → `note_id`, `chunk_index`; **notes** → `note_id`.
--- ---
@@ -136,79 +152,53 @@ Entwickle eine robuste, parser-freundliche **Chunking-Strategie** für Markdown-
blocks = group_blocks(sec, keep_code=True, keep_lists=True) blocks = group_blocks(sec, keep_code=True, keep_lists=True)
for group in soft_wrap(blocks, target_tokens=target_size(meta["type"]), for group in soft_wrap(blocks, target_tokens=target_size(meta["type"]),
max_tokens=max_size(meta["type"])): max_tokens=max_size(meta["type"])):
raw_chunks.append(render_markdown(group)) raw_chunks.append(render_md(group))
chunks = apply_overlap(raw_chunks, overlap_tokens=overlap_size(meta["type"])) chunks = apply_overlap(raw_chunks, overlap_tokens=overlap(meta["type"]))
chunks = trim_whitespace(chunks) chunks = normalize_chunks(chunks) # trim, fix code fences, etc.
# link extraction # IDs & Nachbarn
for i, ch in enumerate(chunks): for i, ch in enumerate(chunks, start=1):
ch.payload.update({ ch.id = f"{meta['id']}#c{i:02d}"
"wikilinks": extract_wikilinks(ch.text), ch.index = i
"external_links": extract_md_links(ch.text), ch.neighbors_prev = chunks[i-2].id if i > 1 else None
"references": [{"target_id": t, "kind": "wikilink"} for t in extract_wikilinks(ch.text)], ch.neighbors_next = chunks[i].id if i < len(chunks) else None
"neighbors": {"prev": f"#c{i-1}" if i>0 else None,
"next": f"#c{i+1}" if i<len(chunks)-1 else None}
})
# attach metadata # Wikilinks je Chunk
for idx, ch in enumerate(chunks): for ch in chunks:
ch.id = f'{meta["id"]}#c{idx:02d}' ch.wikilinks = extract_wikilinks(ch.text)
ch.payload |= minimal_payload_from_meta(meta, file_path, idx, ch)
return chunks return chunks
**Hinweise zur Implementierung**
- **Tokenizer**: gleiche Tokenizer-Funktion für **Längensteuerung** und **Embedding** nutzen (Leak vermeiden).
- **Overlap** implementieren auf **Satz-/Absatzebene**; notfalls anhand Token-Fenster (kein Wort-Cut).
- **Heuristik** für `target_size`/`overlap_size` per `type` (siehe Tabelle).
- **Später**: Re-Chunking ermöglichen (idempotente Re-Runs), Chunk-IDs stabil halten (Index + Hash über `section_path + first_64_tokens`).
--- ---
## 10) Einbettung (Embeddings) ## 10) Einbettung (Embeddings)
- **Standard:** nur `chunk.text` embedden (ohne YAML), **aber**: - Modell: MiniLM/all-MiniLM-L6-v2 (384d) oder E5 (besseres Retrieval, ggf. multilingual).
- `aliases`, `title` können als **synthetischer Kontext** *vorangestellt* werden (konkateniert, durch Trennzeichen), falls Recall für Synonyme wichtig ist. - **Note-Embeddings** optional (Zentroid der Chunks).
- **Mehrsprachigkeit:** pro Chunk die Sprache detektieren; ggf. **mehrsprachige Modelle**/Pipelines verwenden. - **Hashing für Change-Detection** (Importer):
- **Code-Gewichtung:** optional zweite Vektor-Spur für Code (separate Collection) falls viel Technik. - `MINDNET_HASH_MODE = body|frontmatter|body+frontmatter` (Default: `body`)
- `MINDNET_HASH_NORMALIZE = canonical|none` (Default: `canonical`)
--- ---
## 11) Qualitätssicherung ## 11) Qualitätssicherung
- **Unit-Tests**: Gold-Notes mit erwarteten Section-Splits und Link-Graph. - **Zähler**: `notes/chunks/edges` je Lauf prüfen.
- **Metrics**: Retrieval-Eval (nDCG@k, Recall@k) auf internen QA-Fragen. - **Edge-Invarianten**:
- **Linting**: Pre-commit prüft Pflichtfelder & Link-Integrität (IDs, Dateinamen, Wikilinks). - `belongs_to == #Chunks`
- `next == prev == Σ(max(chunks_in_note-1, 0))`
- `references (chunk) == Σ(Chunk-Wikilinks)`
- `backlink == Σ(eindeutige Wikilinks je Note)`
- **Unresolved**: sollten durch Resolver-Jobs über Zeit abnehmen.
- **Roundtrip-Test**: Import → Export → Diff; exakt rekonstruierbare Bodies.
--- ---
## 12) Deliverables ## 12) Deliverables
1. **Python-Modul** `mindnet_chunker/` (Parser, Chunker, Link-Extractor, Qdrant-Client). - Code: Parser, Chunker, Importer, Exporter, Edges.
2. **CLI** `mindnet-chunk` (stdin→stdout, oder Datei→NDJSON). - Schemas: `note.schema.json`, Payload-Layouts.
3. **Docs**: README mit Schema & Beispielen; Tests + Fixtures. - Tests: Mini-Vault (`scripts/make_test_vault.py`), Audit/Validate.
--- ---
## 13) Prompt für neues Chatfenster (kopieren & einfügen) ## 13) Prompt für neues Chatfenster (kopieren & einfügen)
*(Unverändert)*
**Projekt:** mindnet Wissensnetzwerk
**Rolle:** Du bist mein Assistent für NLP-Strategien.
**Ziel:** Hilf mir, eine robuste Chunking-Strategie für Markdown-Notizen zu entwickeln, die später in Qdrant gespeichert werden.
**Anforderungen:**
- Nutze das YAML-Frontmatter-Schema und die Strukturregeln aus `knowledge_design.md`. (Pflichtfelder, Abschnitts-Konventionen, Edge-Typen)
- Definiere optimale Chunkgrößen (Tokens) nach Notiz-Typ, inkl. **Overlap-Empfehlungen**.
- Beschreibe, wie **logische Abschnitte** (Überschriften/Absätze/Listen/Code) als Grenzen dienen.
- Erkläre das **Edge-Modell** zwischen **Chunks** und **Notizen** (`belongs_to`, `references`, `backlink`, `prev/next`).
- Ergebnis: ein **konkretes Konzept + Python-Outline**, das ich direkt implementieren kann (inkl. Datenmodell für Qdrant).
**Kontext zum System:**
- IDs/Dateinamen/Tags gemäß `knowledge_design.md`.
- Zielgröße pro Chunk ~**300 Tokens**, **Overlap ~50 Tokens** als Default; per `type` feinjustiert (Tabelle).
- YAML wird **nicht** in die Embeddings gemischt; ausgewählte Felder als `payload`.
Bitte liefere:
1) Zusammenfassung der Strategie,
2) Tabelle mit Größen/Overlaps,
3) Python-Pseudocode für Chunker,
4) Qdrant-Payload-Beispiel,
5) Test-Plan kurz.
+186 -373
View File
@@ -1,436 +1,249 @@
--- # mindnet – Knowledge Design
title: "mindnet – Knowledge Design" **Version:** 1.4.0 (aktualisiert 2025-10-06)
id: "design-knowledge-architecture"
type: "guide"
status: "active"
version: "1.2.0"
created: 2025-09-02
updated: 2025-09-02
tags: ["type/guide","area/mindnet","topic/obsidian","topic/knowledge-architecture"]
owner: "Ich"
---
# mindnet – Knowledge Design ## 1. Gesamtziel und Systemverständnis
## 1. Gesamtziel, Anspruch & Nutzen **mindnet** ist ein lokales Wissensnetzwerk, das alle meine Notizen, Projekte, Gedanken, Erfahrungen und Pläne als Markdown-Dateien speichert, automatisch importiert, semantisch in **Chunks** zerlegt und als **Graph-Struktur in Qdrant** ablegt.
Es ist die Grundlage für ein agentenfähiges, lokal betriebenes LLM-System, das meine Daten verknüpft, abfragt und erweitert.
**Ziel:** **Zentrale Ziele**
*mindnet* ist ein persönliches Wissensnetzwerk, das **alle** relevanten Inhalte – Gedanken, Erfahrungen, Aufgaben, Projekte, Quellen, Personen, Meetings, Tagebuch/Journal, Wendepunkte und Leitbilder – **einheitlich** in Markdown + YAML ablegt. Es bildet einen **Spiegel der Persönlichkeit** (Werte, Überzeugungen, Erlebnisse) und ist zugleich so strukturiert, dass Tools (Dataview, Embeddings, Vektorsuche/Qdrant, Agenten) es **maschinenlesbar** nutzen können. - Vollständige Repräsentation persönlichen Wissens in strukturierter, maschinenlesbarer Form
- Reproduzierbare Verarbeitung: Import → Chunking → Embedding → Edges → Graph
- Verlustfreie Roundtrip-Fähigkeit (Import ⇄ Export)
- Erweiterbarkeit für Agenten, RAG-Retriever und semantische Suche
**Anspruch an die Struktur:** **Prinzipien**
- **Stabil**: robuste IDs, deterministische Dateinamen, konsistente Feldnamen - **Deterministisch:** IDs, Hashes und Edges wiederholbar
- **Atomar**: ein Kerninhalt pro Datei (keine Sammelcontainer) - **Portabel:** Markdown + YAML
- **Erweiterbar**: neue Typen (z. B. `milestone`, `manifesto`) jederzeit möglich - **Robust:** fehlertoleranter Parser
- **Portabel**: reines Markdown + YAML → funktioniert in Obsidian, Git und Parsern - **Idempotent:** keine Duplikate bei Re-Import
- **Vernetzbar**: systematische Links/Backlinks erzeugen ein sinnvoll navigierbares Netz - **Erweiterbar:** neue Typen und Edge-Arten ohne Migration
- **Parser-freundlich**: wiederkehrende Abschnittsüberschriften; Felder/Enums klar definiert
**Späterer Nutzen / Einsatzzwecke:**
- **Schnelles Wiederfinden**: Dataview-Abfragen, Filter, Dashboards
- **Selbstreflexion & Entwicklung**: persönliche Werte, Wendepunkte, Kinderentwicklung, Lernpfade
- **Automatisierung**: Vektor-Index (Qdrant), Embeddings, LLM-Q&A, Agenten-Workflows
- **Langfristiges Gedächtnis**: eigene Lebens-/Projektchronik reproduzierbar pflegen
- **Aufgaben-/Projektsteuerung**: im selben System, vernetzt mit Kontextwissen
--- ---
## 2. YAML-Frontmatter-Schema ## 2. YAML-Frontmatter-Schema
> **Pflichtfelder:** `title`, `id`, `type`, `status`, `created`, `tags` | Feld | Typ | Beschreibung |
> **Empfohlen:** `updated`, `area`, `project`, `priority`, `aliases` (u. a.) |-------|-----|--------------|
| `title` | string | Menschlich lesbarer Titel |
| `id` | string | Eindeutige Note-ID (Slug) |
| `type` | enum | `concept`, `thought`, `experience`, `task`, `project`, `journal`, `person`, `meeting`, `milestone` |
| `status` | enum | `draft`, `active`, `done`, `archived` |
| `created` | date | ISO-Datum |
| `updated` | date | Letzte Änderung |
| `area` | string | Themen- oder Lebensbereich |
| `project` | string | Zugehöriges Projekt |
| `tags` | list | Strukturierte Tags (`area/…`, `topic/…`) |
| `depends_on` | list | IDs anderer Notizen |
| `assigned_to` | list | Beteiligte Personen |
| `embedding_exclude` | bool | Falls `true`, keine Embeddings |
| `hash_mode` | enum | Hash-Modus beim Import (`body`, `frontmatter`, `full`) |
| `priority` | enum/int | 1–5 oder `low` / `med` / `high` |
| `effort_min`, `due` | int/date | Aufwand / Termin |
| `aliases` | list | Alternative IDs |
| `lang` | string | ISO-Sprachcode |
| `source` | string | Herkunft / Referenzquelle |
### 2.1 Feldübersicht **Pflichtfelder:** `title`, `id`, `type`, `status`, `created`
**Optionale Felder:** alle übrigen
| Feld | Typ | Bedeutung | Beispiel(e) |
|---------------------|-------------------------------|---------------------------------------------------------------------------|------------------------------------------------------------------------------|
| `title` | string | Menschlich lesbarer Titel | `"Vektorsuche in Qdrant"` |
| `id` | string (slug/identifier) | **Stabile** technische ID (nie umbenennen) | `"20250902-1710-thought-obsidian-struktur"`, `"concept-persoenliches-leitbild"` |
| `type` | enum | Notiz-Typ | `thought`, `experience`, `task`, `project`, `source`, `concept`, `person`, `meeting`, `journal`, `milestone` |
| `status` | enum | Lebenszyklus | `idea`, `draft`, `active`, `paused`, `blocked`, `done`, `archived` |
| `created` | date/datetime | Erstellzeitpunkt | `2025-09-02` oder `2025-09-02T17:10:00+02:00` |
| `updated` | date/datetime | Letzte inhaltliche Änderung | `2025-09-02T18:05:00+02:00` |
| `tags` | string[] | Schlagwörter gemäß Taxonomie | `["area/mindnet","topic/obsidian","type/thought"]` |
| `area` | enum/string | Lebens-/Arbeitsbereich | `mindnet`, `family`, `karate`, `travel`, `health`, `home-automation`, … |
| `role` | string | Rolle/Hut (Filter) | `"father"`, `"researcher"`, `"project-owner"` |
| `project` | string | Zugehöriges Projekt (ID oder sprechend) | `"project-mindnet"` |
| `priority` | int (1–5) | 1 = hoch, 5 = niedrig | `2` |
| `effort_min` | int | Aufwand in Minuten (für `task`) | `90` |
| `due` | date | Fälligkeit (Task/Projekt) | `2025-09-10` |
| `start` | date | geplanter Start (Task/Projekt) | `2025-09-05` |
| `people` | string[] (Wikilinks erlaubt) | Beteiligte Personen | `["Ich","[[person-kathie]]"]` |
| `location` | string | Ort/Platz | `"Home Office"`, `"München"` |
| `source_url` | url | Kanonische URL (für `source`) | `"https://qdrant.tech/docs/"` |
| `authors` | string[] | Autoren (für `source`) | `["Jane Doe"]` |
| `published` | date | Veröffentlichungsdatum (für `source`) | `2024-11-20` |
| `accessed` | date | Zugriffsdatum (für `source`) | `2025-09-02` |
| `aliases` | string[] | Alternative Namen/Titel | `["Manifest","Leitbild"]` |
| `related_ids` | string[] | ID-basierte Verknüpfungen | `["concept-persoenliches-leitbild"]` |
| `embedding_exclude` | boolean | Von Embeddings/Index ausschließen | `false` |
### 2.2 Konventionen
- **IDs:** entweder Zeitstempel-basiert (`YYYYMMDD-HHMM-type-slug`) **oder** stabile, sprechende (`concept-*`, `project-*`, `person-*`).
- **Datums-/Zeitformat:** ISO 8601, Zeiten möglichst mit TZ (z. B. `+02:00`).
- **Felder:** klein geschrieben; zusammengesetzte Felder in `snake_case`.
### 2.3 Minimalbeispiel Frontmatter
---
title: "Kurzer, klarer Gedanke"
id: "20250902-1715-thought-klarer-gedanke"
type: "thought"
status: "idea"
created: 2025-09-02T17:15:00+02:00
updated: 2025-09-02T17:15:00+02:00
tags: ["area/mindnet","type/thought","topic/obsidian"]
---
## 2.4 Enum-Werte
Zur Sicherstellung der Konsistenz werden für bestimmte Felder feste Wertemengen (Enums) verwendet:
- **type**
- `idea` – spontane Gedanken, Inspirationen
- `experience` – persönliche Erfahrungen, Reflexionen
- `task` – konkrete Aufgaben
- `project` – Projekte, bestehend aus mehreren Tasks
- `concept` – abstrakte Konzepte, Modelle, Methoden
- `milestone` – wichtige Ereignisse oder Wendepunkte
- `person` – Personen (z. B. Kontakte, Vorbilder)
- `meeting` – Besprechungen, Treffen, Gespräche
- `resource` – externe Quellen (Artikel, Bücher, Links)
- `journal` – Tagebuchnotizen
- **status**
- `open`
- `in-progress`
- `done`
- `blocked`
- `canceled`
- **priority**
- `low`
- `medium`
- `high`
- **area** (Beispiele – erweiterbar)
- `personal`
- `work`
- `family`
- `karate`
- `health`
- `finance`
--- ---
## 3. Dateinamen-Konventionen ## 3. Dateinamen und Ordner
> Keine Sonderzeichen wie `: / \` im Dateinamen. - Format: `YYYY-MM-DD_title.md` oder `slug.md`
- Strukturierte Ordner:
- **Zeitbasierte Notizen (Gedanken, Journal, tagesgebundene Einträge):** - `10_thoughts/`, `20_concepts/`, `30_projects/`, `40_experiences/`, …
`YYYYMMDD-HHMM-<type>-<slug>.md` - Alle Pfade relativ; Unterstriche statt Leerzeichen
Beispiel: `20250902-1710-thought-obsidian-struktur.md`
- **Zeitunabhängige Wissensknoten (Konzepte, Personen, Projekte, Leitbild):**
`<type>-<slug>.md`
Beispiele: `concept-vektorsuche-qdrant.md`, `project-mindnet.md`, `person-kathie.md`, `concept-persoenliches-leitbild.md`
- **Ordnerstruktur (Empfehlung):**
00_inbox/
10_thoughts/
20_experiences/
30_projects/
31_tasks/
40_concepts/
50_sources/
60_people/
70_meetings/
80_journal/
90_templates/
_attachments/
_meta/
--- ---
## 4. Tagging-Regeln & Taxonomie ## 4. Link- und Edge-Design
- **Hierarchische Tags:** **Verlinkungen**
- `area/*` – Lebens-/Arbeitsbereiche: `area/mindnet`, `area/family`, `area/karate`, `area/health`, … - Wikilinks: `[[note-id]]` oder `[[Titel|note-id]]`
- `type/*` – optionaler Spiegel von `type`: `type/thought`, `type/experience`, … - Automatische Edges aus Text und Frontmatter
- `topic/*` – thematische Schwerpunkte: `topic/obsidian`, `topic/yaml`, `topic/qdrant`, `topic/werte`, …
- optional: `place/*`, `person/*`, `level/*` (z. B. `person/rouven`, `person/rohan`) **Edge-Typen in Qdrant**
- **Regeln:**
- Jede Notiz hat **mindestens ein** `area/*`. | Typ | Bedeutung | Quelle | Scope |
- `topic/*` sparsam, aber konsistent; ein Mapping in `_meta/tags.md` pflegen. |------|------------|--------|--------|
- Möglichst **keine** losen Ein-Wort-Tags ohne Präfix (verhindert Wildwuchs). | `belongs_to` | Chunk → Note | intern | chunk |
| `prev` / `next` | Reihenfolge | intern | chunk |
| `references` | explizite Wikilinks | Text | chunk |
| `backlink` | Gegenkante zu `references` | abgeleitet | note |
| `depends_on` | YAML-Abhängigkeiten | YAML | note |
| `assigned_to` | YAML-Zuweisung | YAML | note |
| `unresolved` | Ziel fehlt (Stub) | abgeleitet | chunk |
**Edge-Schlüssel**
- `source_id`, `target_id`, `kind`, `scope`, `note_id`
- Dedup-Schlüssel: Kombination aus `(kind, source_id, target_id, scope)`
--- ---
## 5. Backlink- & Link-Regeln ## 5. Chunking-Modell
- **Wikilinks** verwenden: `[[stabile-id-oder-dateiname]]` **Chunk-Datenstruktur**
- Im **Fließtext** bei erster Nennung verlinken; am **Ende** die Sektion **„Mögliche Verbindungen“** pflegen: | Feld | Beschreibung |
|-------|---------------|
| `note_id` | Zugehörige Note |
| `chunk_index` | Laufende Nummer |
| `text` | Originalabschnitt |
| `window` | Text + Overlap links/rechts |
| `overlap_left` | Länge des linken Overlaps |
| `overlap_right` | Länge des rechten Overlaps |
| `embedding` | 384-dimensionaler Vektor |
| `tokens` | Tokenanzahl (optional) |
## Mögliche Verbindungen **Chunking-Regeln (aus `chunking_strategy.md`)**
- [[concept-persoenliches-leitbild]] - Trennung nach Absätzen, Überschriften, Listen, Tabellen
- [[project-mindnet]] - Token-Zielgröße 350–500 (max 600)
- Overlap: 30–40 % für semantische Kohärenz
- **Linkrichtung:** „aufwärts“ (Detail → Konzept/Leitbild) und „seitwärts“ (verwandte Knoten). - Overlap wird in `window` integriert, sodass `text ≠ window`
- Optional zusätzlich **ID-Relationen** im YAML (`related_ids`) für Parser.
## 5.1 Edge-Typen
Beziehungen zwischen Notizen, Chunks und Objekten werden als **Edges** gespeichert. Jeder Edge erhält einen `type` und optionale `meta`-Informationen (z. B. Gewichtung, Richtung).
- **belongs_to**
Chunk → Note (ein Chunk gehört zu genau einer Note)
- **references**
Note → Note (eine Notiz verweist auf eine andere)
- **backlink**
Automatisch generiert: Gegenkante von `references`
- **depends_on**
Task → Task oder Task → Project (Aufgabe hängt von anderer ab)
- **assigned_to**
Task → Person (eine Aufgabe ist einer Person zugeordnet)
- **discussed_in**
Note/Concept → Meeting (ein Thema wurde in einem Meeting behandelt)
- **authored_by**
Note → Person (z. B. externer Autor oder eigene Urheberschaft)
- **related**
Generische Relation für schwache Assoziationen
--- ---
## 6. Abschnitts-Konventionen im Body ## 6. Hash-Strategie und Änderungsverfolgung
Einheitliche Überschriften erleichtern spätere Parser und Abfragen. **Zweck:** Nur bei realer Inhaltsänderung re-indexieren
- **Gedanke (`thought`)** | Umgebungsvariable | Beschreibung |
`## Zusammenfassung` · `## Begründung / Details` · `## Belege / Quellen` · `## Nächste Schritte` · `## Mögliche Verbindungen` |--------------------|--------------|
- **Erfahrung (`experience`)** | `MINDNET_HASH_COMPARE` | Vergleichsquelle (`Body`, `Full`, `Frontmatter`) |
`## Kontext` · `## Beobachtung` · `## Interpretation` · `## Implikationen` · `## Mögliche Verbindungen` | `MINDNET_HASH_SOURCE` | Rohtext vs. geparst (`raw` / `parsed`) |
- **Aufgabe (`task`)** | `MINDNET_HASH_NORMALIZE` | Normalisierung (`canonical` / `none`) |
`## Checkliste` · `## Notizen` · `## Mögliche Verbindungen`
- **Projekt (`project`)** **Basismodi**
`## Scope` · `## Arbeitspakete` · `## Risiken & Gegenmaßnahmen` · `## Status / Fortschritt` · `## Mögliche Verbindungen` - `body` → Nur Textkörper
- **Journal (`journal`)** - `frontmatter` → Nur Metadaten
`## Tageszusammenfassung` · `## Highlights` · `## Gelernt / Notizen` · `## Mögliche Verbindungen` - `full` → Kombination
- Hash = SHA-256(canonicalized_input)
Bei Änderung: Re-Import, neue Embeddings, Edges werden aktualisiert.
--- ---
## 7. Abbildung persönlicher Inhalte auf Typen ## 7. Qdrant-Speicherstruktur
- **Persönliches Leitbild** → `type: "concept"` (oder optional neuer Typ `manifesto`) | Collection | Inhalt | Primärfelder | Embedding |
Datei: `concept-persoenliches-leitbild.md` (zeitunabhängig) |-------------|---------|---------------|------------|
Verlinkt zu prägenden Erfahrungen und Wendepunkten. | `mindnet_notes` | Notes mit Fulltext & Hash | `note_id` | optional |
- **Erfahrungen, die dich geprägt haben** → `type: "experience"` | `mindnet_chunks` | Chunks mit Text, Window, Embedding | `chunk_id`, `note_id` | 384 d |
Je **eine** prägende Erfahrung pro Datei; klarer Kontext + Interpretation. | `mindnet_edges` | Graphkanten | `edge_id`, `source_id`, `target_id` | — |
- **Wendepunkte im Leben** → `type: "milestone"`
Je Wendepunkt ein Knoten; verlinkt zu den auslösenden Erfahrungen.
- **Aktuelle Erfahrungen** → `type: "experience"`, `status: "active"`
Bei Bedarf später in „prägende“ Erfahrungen überführen.
- **Entwicklungsschritte der Kinder** → `type: "journal"` (chronologisch) **oder** `experience` (wenn es prägend ist); Tags `person/<name>`.
- **Tagebucheinträge** → `type: "journal"`; ein Eintrag pro Tag; tagesbasierte Dateinamen.
Damit entsteht ein **netzartiges Persönlichkeitsprofil** mit klaren Pfaden zwischen Werten (Leitbild), Ereignissen (Erfahrungen), Wendepunkten und fortlaufenden Beobachtungen (Journal). **Upsert-Strategie**
- Idempotente UUIDv5-IDs
- Keine Duplikate durch deterministische Signaturen
- `purge-before-upsert` zur Bereinigung
**Indizes**
- notes: `note_id`, `hash_signature`
- chunks: `note_id`, `chunk_index`
- edges: `(kind, source_id, target_id, scope)`
--- ---
## 8. Beispiel-Notizen ## 8. Fehler-Toleranz und Parser-Robustheit
### 8.1 Gedanke (`thought`) - UTF-8 mit Fallback: ersetzt ungültige Bytes (`errors="replace"`)
--- - Ignoriert Nullbytes, BOM, exotische Zeilenenden
title: "Gedanke: Einheitliche YAML-Standards steigern Recall" - Erkennt defekte oder leere YAML-Header und überspringt sie
id: "20250902-1830-thought-yaml-recall" - Logt Problemfälle (`read_markdown failed`, `make_note_payload returned non-dict`)
type: "thought" - Bei Fehlern: Import → warn → continue
status: "idea"
created: 2025-09-02T18:30:00+02:00
updated: 2025-09-02T18:30:00+02:00
tags: ["area/mindnet","type/thought","topic/yaml","topic/recall"]
area: "mindnet"
project: "project-mindnet"
aliases: []
---
## Zusammenfassung
Konsistente YAML-Felder verbessern Filter, Dataview-Queries und Vektor-Retrieval.
## Begründung / Details
- Parser benötigen stabile Feldnamen (z. B. `type`, `status`).
- Einheitliche Werte erleichtern Dashboards und Automationen.
## Belege / Quellen
- [[source-obsidian-properties]]
- [[concept-vektorsuche-qdrant]]
## Nächste Schritte
- [ ] `tags`-Mapping in `_meta/tags.md` erstellen
- [ ] Beispiel-Queries für Dataview hinterlegen
## Mögliche Verbindungen
- [[project-mindnet]]
- [[20250902-1730-task-yaml-standard-migration]]
### 8.2 Erfahrung (`experience`)
---
title: "Erfahrung: Journal → Erfahrung extrahieren erhöht Klarheit"
id: "20250902-1840-experience-journal-extraktion"
type: "experience"
status: "active"
created: 2025-09-02T18:40:00+02:00
updated: 2025-09-02T18:40:00+02:00
tags: ["area/personal","type/experience","topic/journal","topic/reflexion"]
area: "personal"
project: "project-mindnet"
---
## Kontext
Wiederkehrende Journaleinträge enthalten Rohmaterial für prägende Einsichten.
## Beobachtung
Wird Material regelmäßig zu `experience`-Knoten verdichtet, sind spätere Bezüge einfacher.
## Interpretation
Abstraktion + Verlinkung (z. B. auf [[concept-persoenliches-leitbild]]) erhöhen Nutzen und Wiederfindbarkeit.
## Implikationen
Pro Woche 15–30 Minuten „Verdichtungs-Review“ einplanen.
## Mögliche Verbindungen
- [[journal-20250902]]
- [[concept-persoenliches-leitbild]]
### 8.3 Aufgabe (`task`)
---
title: "Task: YAML-Standard in Altbeständen ergänzen"
id: "20250902-1850-task-yaml-migration"
type: "task"
status: "active"
created: 2025-09-02
updated: 2025-09-02
due: 2025-09-10
start: 2025-09-03
priority: 2
effort_min: 120
area: "mindnet"
project: "project-mindnet"
tags: ["area/mindnet","type/task","topic/migration"]
depends_on: []
---
## Checkliste
- [ ] Triage: Notiz-Typ bestimmen
- [ ] Minimal-YAML ergänzen (Pflichtfelder)
- [ ] Dateinamen vereinheitlichen
- [ ] „Mögliche Verbindungen“ pflegen
## Notizen
Beginne mit zuletzt genutzten Notizen (Pareto 20/80).
## Mögliche Verbindungen
- [[project-mindnet]]
- [[20250902-1830-thought-yaml-recall]]
--- ---
## 9. Schrittweise Migration bestehender freier Notizen ## 9. Tests & Validierung
**Phase A – Triage (schnell)** **Integrationstests**
1. Sichtung & **Typ** bestimmen (`type`). - Roundtrip: Import → Export → Compare (`compare_vaults.py`)
2. **Minimal-YAML** einfügen (`title`, `id`, `type`, `status=draft`, `created≈Dateidatum`, `tags`). - Chunk-Integrität: `verify_chunks_integrity.py`
3. **Dateiname** grob anpassen (Schema + Slug). - Hash-Audit: `hash_reporter.py`
- Window-Vergleich: `check_chunks_window_vs_text.py`
**Phase B – Strukturieren (fokussiert)** **Erwartete Ergebnisse**
1. **Abschnitte** nach Typ-Template ergänzen. - Alle Notes werden erkannt
2. **Zerlegen**: mehrere Gedanken/Ereignisse → je **eigene Datei**, ursprüngliche Datei wird zum **Index** mit Links. - `window ≠ text` bei > 60 % der Chunks
3. **Verlinken**: relevante `[[Knoten]]` + **„Mögliche Verbindungen“** pflegen. - Roundtrip-Vergleich liefert „OK“
- Edge-Anzahl entspricht Modell (belongs_to = #Chunks, next/prev ≈ #Chunks – 1)
**Phase C – Verfeinern (bei Bedarf)**
1. **Status & Felder** pflegen (`status`, `priority`, `due`, `effort_min`).
2. **Tag-Harmonisierung**: Mapping-Tabelle pflegen (`_meta/tags.md`).
3. **Aliases** ergänzen (Such-/Linkvarianten).
**Priorisierung:** Starte mit **aktuellen & häufig genutzten** Notizen (Pareto-Prinzip), Altbestände nachziehen, wenn sie wieder gebraucht werden.
--- ---
## 10. Dataview-Beispiele (direkt nutzbar) ## 10. Export & Roundtrip-Verhalten
**Offene Aufgaben dieser Woche (nach Priorität):** - `export_markdown.py` erzeugt identische Markdown-Dateien
- Schreibt YAML + Body in korrekter Reihenfolge
TABLE due, priority, effort_min - Roundtrip-Prüfung (`compare_vaults.py`) validiert:
FROM "31_tasks" - vollständigen Body
WHERE type = "task" AND status != "done" - unveränderte Hash-Signatur
AND due >= date(today) - dur(1 week) - gleiche Chunkzahl
AND due <= date(today) + dur(1 week)
SORT priority ASC, due ASC
**Aktive Projekte mit Meilensteinen:**
LIST
FROM "30_projects"
WHERE type = "project" AND status = "active"
SORT priority ASC, file.ctime DESC
**Gedanken im Bereich mindnet, jüngste zuerst:**
TABLE created, status
FROM "10_thoughts"
WHERE type = "thought" AND area = "mindnet"
SORT created DESC
LIMIT 50
**Quellen ohne `source_url` (Pflegebedarf):**
TABLE title, created
FROM "50_sources"
WHERE type = "source" AND (source_url = "" OR !source_url)
--- ---
## 11. Validierung & Pflege ## 11. Graph-Schicht und WP-04-Integration
- **Wöchentlicher Review-Slot (15–30 Min.):** - Qdrant fungiert als Graph-Backend
- Neue Notizen triagieren (`type`, `status`, `area`) - `graph/service.py` (WP-04): stellt API für Mehrhop-Abfragen bereit
- 5 Alt-Notizen migrieren (YAML + Dateiname + Verlinkung) - Unterstützt:
- **Dataview-Checks:** fehlende `source_url`, offene Tasks, aktive Projekte, veraltete `status`. - `expand(note_id, hops=2)`
- **Optional (technisch):** - `resolve_unresolved()`
- **Pre-commit Hook**: prüft Pflichtfelder (`title`, `id`, `type`, `status`, `created`, `tags`) - `neighbors(kind=…)`
- **ID-Generator**: `YYYYMMDD-HHMM-type-slug` bei Neuanlage - Grundlage für spätere **Retriever-Funktionen** (LLM / Agenten)
- **Link-Linter**: prüft tote `[[Links]]`
--- ---
## 12. Versions- & Änderungsrichtlinien ## 12. Agenten- und LLM-Integration
- Dieses Dokument pflegt `version` & `updated`. **Retriever-Pipeline**
- **Breaking Changes** (z. B. Feldumbenennungen) in `_meta/changelog.md` dokumentieren. 1. Suche top-k Chunks (cosine distance)
- **Migrationen** (Tag-Refactorings, Typänderungen) als Skript oder Batch-Checkliste planen. 2. Erweiterung um Edges (`references`, `backlinks`)
3. Kontext-Assembling via `window`-Text
4. Prompt-Generierung
**Agent-Use-Cases**
- Automatisches Tagging (`type`, `status`)
- Vorschläge für neue Edges
- Auflösen von `unresolved`-Referenzen
- Auto-Import externer Quellen (MediaWiki, PDFs, Webseiten)
--- ---
## 13. Typen-Kurzreferenz ## 13. Erweiterbarkeit
- `thought` – Gedanke/These - Neue Edge-Typen: `inspired_by`, `relates_to`, `contradicts`
- `experience` – Beobachtung/Erfahrung mit Interpretation & Implikationen - Neue Note-Typen: `decision`, `resource`, `recommendation`
- `task` – umsetzbare Arbeitseinheit (mit `due`, `effort_min`, `priority`) - Embedding-Engines austauschbar (MiniLM → LaBSE → E5)
- `project` – Container für Ziele/Tasks/Meilensteine - Versionierte Chunking-Strategien (`chunk_config.py`)
- `source` – externe Quelle (Artikel, Buch, Video)
- `concept` – definierter Begriff / Prinzip / Leitbild
- `person` – Bezugsperson/Kontakt
- `meeting` – Termin mit Agenda, Notizen, erzeugten Aufgaben
- `journal` – Tagebuch-/Reiseeintrag (tagesbasiert)
- `milestone` – Wendepunkt / Meilenstein im Lebens-/Projektverlauf
--- ---
## 14 Erweiterungen ## 14. Qualitätssicherung und Best Practices
Zukünftige Typen möglich (z. B. decision, recommendation). - Validierung vor Upsert
- Keine Leer-Chunks
- Vollständige Metadaten in Notes
- Tests in `tests/` automatisch ausführbar (`run_e2e_roundtrip.sh`)
- Ergebnis: „Roundtrip OK“ und `verify_chunks_integrity` = OK
Personalisierung durch zusätzliche Properties (values, goals) ---
## 15. Änderungsverlauf
**v1.4.0 (2025-10-06)**
- Neu: `window` vs `text` Feld in `mindnet_chunks`
- Neu: Hash-Modi (`body`, `frontmatter`, `full`) mit Env-Steuerung
- Neu: Baseline-Modus für Vergleich
- Parser → fehlertolerant (UTF-8 replace)
- Roundtrip-Test vollständig integriert
- Graph-Service und Mehrhop-Abfragen (WP-04-Vorbereitung)
- `purge-before-upsert` und `--note-scope-refs` Parameter ergänzt
**v1.3.0 (2025-09-09)**
- Hash-Normalisierung und Edge-Scope
- Roundtrip-Tests, Agenten-Integration
**v1.2.0 (2025-09-02)**
- Grundstruktur des Knowledge-Designs
---
+79 -27
View File
@@ -2,37 +2,89 @@
"$schema": "https://json-schema.org/draft/2020-12/schema", "$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "mindnet_chunk", "title": "mindnet_chunk",
"type": "object", "type": "object",
"required": ["id", "note_id", "chunk_index", "type", "path", "token_count", "lang"], "description": "Chunk-Payload (Qdrant). Kompatibel mit Alt-Feldern und neuen Feldern für Export/Roundtrip.",
"required": ["id", "note_id", "chunk_index", "path"],
"properties": { "properties": {
"id": {"type": "string"}, "id": { "type": "string" },
"note_id": {"type": "string"}, "scope": { "type": "string", "enum": ["chunk"] },
"note_title": {"type": "string"}, "note_id": { "type": "string" },
"chunk_index": {"type": "integer"}, "note_title": { "type": "string" },
"char_start": {"type": "integer"}, "note_type": { "type": "string" },
"char_end": {"type": "integer"}, "note_status": { "type": "string" },
"token_count": {"type": "integer"}, "type": { "type": "string", "description": "Legacy: früherer Chunk-Typ; kann dem Note-Typ entsprechen" },
"type": {"type": "string"}, "area": { "type": "string" },
"area": {"type": "string"}, "project": { "type": "string" },
"project": {"type": "string"}, "tags": { "type": "array", "items": { "type": "string" } },
"tags": {"type": "array", "items": {"type": "string"}},
"section_title": {"type": "string"}, "note_path": { "type": "string" },
"section_path": {"type": "string"}, "path": { "type": "string" },
"lang": {"type": "string"},
"wikilinks": {"type": "array", "items": {"type": "string"}}, "chunk_index": { "type": "integer" },
"external_links": {"type": "array", "items": {"type": "string"}}, "section_title":{ "type": ["string","null"] },
"section_path": { "type": ["string","null"] },
"char_start": { "type": ["integer","null"] },
"char_end": { "type": ["integer","null"] },
"char_len": { "type": "integer" },
"token_count": { "type": "integer", "description": "Legacy: frühere Token-Zahl" },
"token_est": { "type": "integer", "description": "Neue grobe Token-Schätzung (≈ len(text)/4)" },
"neighbors": {
"type": "object",
"properties": {
"prev": { "type": ["string","null"] },
"next": { "type": ["string","null"] }
},
"additionalProperties": false
},
"text": { "type": "string" },
"text_sha256": { "type": "string", "pattern": "^sha256:[0-9a-fA-F]{64}$" },
"lang": { "type": "string" },
"wikilinks": { "type": "array", "items": { "type": "string" } },
"external_links": {
"type": "array",
"items": {
"anyOf": [
{ "type": "string" },
{
"type": "object",
"properties": {
"href": { "type": "string" },
"label": { "type": ["string","null"] }
},
"required": ["href"],
"additionalProperties": false
}
]
}
},
"references": { "references": {
"type": "array", "type": "array",
"items": {"type": "object", "properties": { "items": {
"target_id": {"type": "string"}, "type": "object",
"kind": {"type": "string"} "properties": {
}, "required": ["target_id", "kind"]} "target_id": { "type": "string" },
"kind": { "type": "string" }
},
"required": ["target_id","kind"],
"additionalProperties": true
}
}, },
"neighbors": {"type": "object", "properties": {
"prev": {"type": ["string","null"]}, "embed_model": { "type": "string" },
"next": {"type": ["string","null"]} "embed_dim": { "type": "integer" },
}}, "embed_version": { "type": "integer" },
"path": {"type": "string"},
"created_at": {"type": "string"} "created_at": { "type": "string" }
}, },
"allOf": [
{ "anyOf": [ { "required": ["token_count"] }, { "required": ["token_est"] } ] },
{ "anyOf": [ { "required": ["type"] }, { "required": ["note_type"] } ] }
],
"additionalProperties": true "additionalProperties": true
} }
+23 -8
View File
@@ -2,15 +2,30 @@
"$schema": "https://json-schema.org/draft/2020-12/schema", "$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "mindnet_edge", "title": "mindnet_edge",
"type": "object", "type": "object",
"required": ["src_id", "dst_id", "edge_type", "scope"], "description": "Edge-Payload (Qdrant). Unterstützt Legacy (edge_type/src_id/dst_id) und neues Schema (kind/source_id/target_id/note_id/status).",
"properties": { "properties": {
"src_id": {"type": "string"}, "scope": { "type": "string", "enum": ["note","chunk"] },
"dst_id": {"type": "string"},
"edge_type": {"type": "string"}, "edge_type": { "type": "string", "description": "Legacy: z. B. references/backlink/belongs_to/prev/next" },
"scope": {"type": "string", "enum": ["note","chunk"]}, "src_id": { "type": "string", "description": "Legacy: source_id" },
"weight": {"type": "number"}, "dst_id": { "type": "string", "description": "Legacy: target_id" },
"meta": {"type": "object"},
"created_at": {"type": "string"} "kind": { "type": "string", "description": "Neu: z. B. references/backlink/belongs_to/prev/next" },
"source_id": { "type": "string" },
"target_id": { "type": "string" },
"note_id": { "type": "string", "description": "Owner-Note für diesen Edge (Filter/Purge)" },
"status": { "type": "string", "description": "optional, z. B. 'unresolved'" },
"weight": { "type": "number" },
"meta": { "type": "object" },
"created_at":{ "type": "string" }
}, },
"anyOf": [
{ "required": ["src_id", "dst_id", "edge_type", "scope"] },
{ "required": ["source_id", "target_id", "kind", "scope"] }
],
"additionalProperties": true "additionalProperties": true
} }
+38 -40
View File
@@ -1,47 +1,45 @@
{ {
"$schema": "https://json-schema.org/draft/2020-12/schema", "$schema": "http://json-schema.org/draft-07/schema#",
"title": "mindnet_note", "title": "mindnet note payload",
"type": "object", "type": "object",
"required": ["note_id", "title", "type", "status", "created", "path"],
"properties": { "properties": {
"note_id": { "type": "string" }, "note_id": { "type": "string" },
"title": { "type": "string" }, "title": { "type": ["string","null"] },
"type": { "type": "string" }, "type": { "type": ["string","null"] },
"status": { "type": "string" }, "status": { "type": ["string","null"] },
"created": { "type": "string" }, "created": { "type": ["string","null"] },
"updated": { "type": "string" }, "updated": { "type": ["string","null"] },
"tags": { "path": { "type": ["string","null"] },
"type": "array", "tags": { "type": ["array","null"], "items": { "type": "string" } },
"items": { "type": "string" } "area": { "type": ["string","null"] },
}, "project": { "type": ["string","null"] },
"area": { "type": "string" }, "source": { "type": ["string","null"] },
"project": { "type": "string" }, "lang": { "type": ["string","null"] },
"priority": { "type": "number" }, "slug": { "type": ["string","null"] },
"effort_min": { "type": "number" }, "aliases": { "type": ["array","null"], "items": { "type": "string" } },
"due": { "type": "string" },
"people": { "fulltext": { "type": ["string","null"] },
"type": "array", "references": { "type": ["array","null"], "items": { "type": "string" } },
"items": { "type": "string" }
}, "hash_fulltext": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
"aliases": { "hash_signature": { "type": ["string","null"] },
"type": "array",
"items": { "type": "string" } "hash_body": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
}, "hash_frontmatter": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
"references": { "hash_full": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
"type": "array",
"items": { "type": "string" } "hashes": {
}, "type": ["object","null"],
"depends_on": { "description": "Mapping: <mode>:<source>:<normalize> -> sha256 hex",
"type": "array", "patternProperties": {
"items": { "type": "string" } "^(body|frontmatter|full):(parsed|raw):(canonical|none)$": {
}, "type": "string",
"assigned_to": { "pattern": "^[a-f0-9]{64}$"
"type": "array", }
"items": { "type": "string" } },
}, "additionalProperties": false
"lang": { "type": "string" }, }
"path": { "type": "string" },
"hash_fulltext": { "type": "string" }
}, },
"required": ["note_id"],
"additionalProperties": true "additionalProperties": true
} }
+208
View File
@@ -0,0 +1,208 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: audit_edges_vs_expectations.py — Prüfe Kanten in Qdrant gegen Vault-Erwartungen
Version: 1.0.0
Datum: 2025-09-09
Zweck
-----
- Liest Edges/Chunks/Notes aus Qdrant.
- Ermittelt erwartete Kanten-Anzahlen aus dem Vault:
* belongs_to : sollte == #Chunks
* next / prev : je Note (#Chunks_in_Note - 1)
* references : Summe aller Chunk-Wikilinks
* backlink : Summe einzigartiger Wikilinks pro Note (Note-Level)
- Vergleicht IST vs. SOLL und meldet Abweichungen.
ENV/Qdrant
----------
QDRANT_URL, QDRANT_API_KEY (optional), COLLECTION_PREFIX (Default: mindnet)
Aufrufe
-------
# Gesamtaudit
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault
# Mit anderem Prefix
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --prefix mindnet_dev
# Details anzeigen
python3 -m scripts.audit_edges_vs_expectations --vault ./test_vault --details
"""
from __future__ import annotations
import argparse
import json
import os
import re
from collections import defaultdict, Counter
from typing import Dict, List, Tuple
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from dotenv import load_dotenv
# Projektmodule – nur leichtgewichtige Funktionen
try:
from app.core.parser import read_markdown
except Exception:
# sehr einfacher Fallback für Wikilinks
read_markdown = None
WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
# ------------------------------
# Qdrant Helpers
# ------------------------------
def _names(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _scroll_all(client: QdrantClient, col: str, flt=None, with_payload=True, with_vectors=False, limit=256):
out, next_page = [], None
while True:
pts, next_page = client.scroll(
collection_name=col,
scroll_filter=flt,
with_payload=with_payload,
with_vectors=with_vectors,
limit=limit,
offset=next_page,
)
if not pts:
break
out.extend(pts)
if not next_page:
break
return out
# ------------------------------
# Vault scan
# ------------------------------
def _iter_md(root: str) -> List[str]:
out: List[str] = []
for dp, _, fns in os.walk(root):
for fn in fns:
if fn.lower().endswith(".md"):
p = os.path.join(dp, fn)
if "/.obsidian/" in p.replace("\\", "/"):
continue
out.append(p)
return sorted(out)
def _wikilinks_in_text(text: str) -> List[str]:
return WIKILINK_RE.findall(text or "")
def _wikilinks_per_note(vault_root: str) -> Dict[str, List[str]]:
res: Dict[str, List[str]] = {}
for p in _iter_md(vault_root):
body = ""
try:
if read_markdown:
parsed = read_markdown(p)
body = parsed.body or ""
fm = parsed.frontmatter or {}
nid = fm.get("id") or fm.get("note_id") or os.path.splitext(os.path.basename(p))[0]
else:
with open(p, "r", encoding="utf-8") as f:
txt = f.read()
# sehr einfacher Split: YAML-Frontmatter rausnehmen
if txt.lstrip().startswith("---"):
parts = txt.split("\n---", 1)
body = parts[1] if len(parts) > 1 else txt
else:
body = txt
nid = os.path.splitext(os.path.basename(p))[0]
res[nid] = _wikilinks_in_text(body)
except Exception:
continue
return res
# ------------------------------
# Main Audit
# ------------------------------
def main():
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Vault (für Erwartungswerte)")
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX", "mindnet"), help="Collection-Prefix")
ap.add_argument("--details", action="store_true", help="Detail-Listen ausgeben")
args = ap.parse_args()
client = QdrantClient(url=os.environ.get("QDRANT_URL", "http://127.0.0.1:6333"),
api_key=os.environ.get("QDRANT_API_KEY") or None)
notes_col, chunks_col, edges_col = _names(args.prefix)
# Qdrant laden
notes = _scroll_all(client, notes_col, with_payload=True, with_vectors=False)
chunks = _scroll_all(client, chunks_col, with_payload=True, with_vectors=False)
edges = _scroll_all(client, edges_col, with_payload=True, with_vectors=False)
# --- Ist-Zähler
cnt_kind = Counter()
cnt_scope = Counter()
by_note_chunks: Dict[str, int] = defaultdict(int)
chunk_wikilinks_total = 0
for p in chunks:
pl = p.payload or {}
by_note_chunks[pl.get("note_id")] += 1
wl = pl.get("wikilinks") or []
if isinstance(wl, list):
chunk_wikilinks_total += len(wl)
for p in edges:
pl = p.payload or {}
kind = pl.get("kind") or pl.get("edge_type") or "?"
scope = pl.get("scope") or "?"
cnt_kind[kind] += 1
cnt_scope[f"{kind}:{scope}"] += 1
total_chunks = sum(by_note_chunks.values())
# --- Soll-Zähler aus Vault
wl_per_note = _wikilinks_per_note(args.vault)
backlink_expected = sum(len(set(v)) for v in wl_per_note.values())
next_expected = sum(max(c - 1, 0) for c in by_note_chunks.values())
prev_expected = next_expected # symmetrische Kanten
belongs_to_expected = total_chunks
references_expected = chunk_wikilinks_total # aus Chunk-Payloads
# --- Ergebnis
result = {
"qdrant_counts": dict(cnt_kind),
"qdrant_counts_by_scope": dict(cnt_scope),
"chunks_total": total_chunks,
"by_note_chunks": dict(by_note_chunks),
"vault_expected": {
"belongs_to": belongs_to_expected,
"next": next_expected,
"prev": prev_expected,
"references": references_expected,
"backlink": backlink_expected,
},
"deltas": {
"belongs_to": cnt_kind.get("belongs_to", 0) - belongs_to_expected,
"next": cnt_kind.get("next", 0) - next_expected,
"prev": cnt_kind.get("prev", 0) - prev_expected,
"references": cnt_kind.get("references", 0) - references_expected,
"backlink": cnt_kind.get("backlink", 0) - backlink_expected,
},
"collections": {
"notes": notes_col, "chunks": chunks_col, "edges": edges_col
}
}
print(json.dumps(result, ensure_ascii=False, indent=2))
if args.details:
# optionale Stichproben (z. B. fehlerhafte Kantenarten)
pass
if __name__ == "__main__":
main()
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
scripts/debug_note_payload.py
Zeigt, welche app.core.note_payload geladen wird, ruft make_note_payload auf
und druckt Typ/Preview der Rückgabe.
Aufruf:
python3 -m scripts.debug_note_payload --file ./test_vault/40_concepts/concept-alpha.md --vault-root ./test_vault
"""
import argparse, json, os, pprint
from app.core import note_payload as np
from app.core.parser import read_markdown
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--file", required=True)
ap.add_argument("--vault-root", default=None)
ap.add_argument("--hash-mode", default=None)
ap.add_argument("--hash-normalize", default=None)
ap.add_argument("--hash-source", default=None)
args = ap.parse_args()
print("module_path:", getattr(np, "__file__", "<unknown>"))
parsed = read_markdown(args.file)
res = np.make_note_payload(parsed, vault_root=args.vault_root,
hash_mode=args.hash_mode,
hash_normalize=args.hash_normalize,
hash_source=args.hash_source,
file_path=args.file)
print("return_type:", type(res).__name__)
if isinstance(res, dict):
print("keys:", list(res.keys()))
print(json.dumps(res, ensure_ascii=False, indent=2))
else:
preview = repr(res)
if len(preview) > 600: preview = preview[:600] + "…"
print("preview:", preview)
if __name__ == "__main__":
main()
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
scripts/debug_qdrant_state.py
Zeigt Prefix, Collections und einfache Counts (notes/chunks/edges) + sample note_ids.
Aufruf:
export COLLECTION_PREFIX="mindnet"
python3 -m scripts.debug_qdrant_state
# oder:
python3 -m scripts.debug_qdrant_state --prefix mindnet
"""
from __future__ import annotations
import argparse, os, json
from app.core.qdrant import QdrantConfig, get_client
def count_points(client, collection: str) -> int:
try:
res = client.count(collection, exact=True)
return int(getattr(res, "count", 0))
except Exception:
pts, _ = client.scroll(collection, limit=1)
return 1 if pts else 0
def sample_ids(client, collection: str, id_key: str, limit: int = 5):
pts, _ = client.scroll(collection, with_payload=True, with_vectors=False, limit=limit)
out = []
for p in pts or []:
pl = p.payload or {}
if id_key in pl:
out.append(pl[id_key])
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", default=os.environ.get("COLLECTION_PREFIX", "mindnet"))
args = ap.parse_args()
cfg = QdrantConfig.from_env()
cfg.prefix = args.prefix
client = get_client(cfg)
notes = f"{cfg.prefix}_notes"
chunks = f"{cfg.prefix}_chunks"
edges = f"{cfg.prefix}_edges"
print(json.dumps({
"prefix": cfg.prefix,
"collections": {"notes": notes, "chunks": chunks, "edges": edges},
"counts": {
"notes": count_points(client, notes),
"chunks": count_points(client, chunks),
"edges": count_points(client, edges),
},
"samples": {
"notes.note_id": sample_ids(client, notes, "note_id"),
"chunks.note_id": sample_ids(client, chunks, "note_id"),
"edges.note_id": sample_ids(client, edges, "note_id"),
}
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+227
View File
@@ -0,0 +1,227 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: scripts/export_markdown.py — Qdrant → Markdown (Vault)
Version: 1.4.1
Datum: 2025-09-10
Funktion
--------
Exportiert Notes (Frontmatter + Body) aus Qdrant in einen Zielordner. Der Body wird
bevorzugt aus dem Feld `fulltext` rekonstruiert; falls leer/nicht vorhanden, aus Chunks
(Sortierung: seq → chunk_index → Nummer in chunk_id). Pfade werden **relativ** geschrieben.
Optionen
--------
--out PATH Zielordner (erforderlich)
--prefix TEXT Collection-Prefix (CLI überschreibt ENV COLLECTION_PREFIX)
--note-id ID Nur eine Note exportieren
--overwrite Existierende Dateien überschreiben
--include-edges MODE none|yaml|footer (Default: none)
--flatten-paths Alle Dateien flach schreiben; Originalpfad in FM: orig_path
ENV
---
COLLECTION_PREFIX, QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
Beispiele
---------
export COLLECTION_PREFIX="mindnet"
python3 -m scripts.export_markdown --out ./_exportVault
# Nur eine Note, Edges als YAML-Feld 'references'
python3 -m scripts.export_markdown --out ./_exportVault --note-id concept-alpha --include-edges yaml
# Flach schreiben mit Überschreiben
python3 -m scripts.export_markdown --out ./_exportVault --flatten-paths --overwrite
"""
from __future__ import annotations
import argparse
import os
import json
from typing import Dict, List, Optional, Tuple, Any
import yaml
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client
from app.core.qdrant import ensure_collections # safety
# ---------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------
def collections(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _norm_rel_path(path: str) -> str:
p = (path or "").replace("\\", "/").lstrip("/")
return p if p else ""
def _ensure_dir(path: str) -> None:
d = os.path.dirname(path)
if d and not os.path.exists(d):
os.makedirs(d, exist_ok=True)
def _yaml_dump(data: Dict[str, Any]) -> str:
return yaml.safe_dump(data, allow_unicode=True, sort_keys=False).strip()
def _frontmatter_block(fm: Dict[str, Any]) -> str:
y = _yaml_dump(fm)
return f"---\n{y}\n---\n"
def _scroll_all(client, collection: str, flt: Optional[rest.Filter] = None) -> List[Any]:
out = []
nextp = None
while True:
pts, nextp = client.scroll(
collection_name=collection,
with_payload=True,
with_vectors=False,
limit=256,
scroll_filter=flt,
offset=nextp,
)
if not pts:
break
out.extend(pts)
if nextp is None:
break
return out
def _reconstruct_body_from_chunks(chunks: List[Any]) -> str:
def seq_key(pl: Dict[str, Any]) -> Tuple[int, int, int]:
s = pl.get("seq")
ci = pl.get("chunk_index")
cid = pl.get("chunk_id") or ""
n = 0
if isinstance(cid, str) and "#" in cid:
try:
n = int(cid.rsplit("#", 1)[-1])
except Exception:
n = 0
return (int(s) if isinstance(s, int) else 0,
int(ci) if isinstance(ci, int) else 0,
n)
chunks_sorted = sorted(chunks, key=lambda p: seq_key(p.payload or {}))
texts: List[str] = []
for p in chunks_sorted:
pl = p.payload or {}
t = pl.get("text") or pl.get("content") or pl.get("raw") or ""
if isinstance(t, str) and t:
texts.append(t)
return "\n".join(texts).strip()
def _collect_forward_refs_from_edges(edges: List[Any]) -> List[str]:
refs = []
for p in edges:
pl = p.payload or {}
if pl.get("kind") == "references" and isinstance(pl.get("target_id"), str):
refs.append(pl["target_id"])
# de-dupe, preserve order
seen = set()
out = []
for r in refs:
if r not in seen:
seen.add(r)
out.append(r)
return out
# ---------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--out", required=True, help="Zielordner für exportierte Markdown-Dateien")
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID exportieren")
ap.add_argument("--overwrite", action="store_true", help="Existierende Dateien überschreiben")
ap.add_argument("--include-edges", choices=["none", "yaml", "footer"], default="none",
help="Forward-Links mit exportieren (aus Edges oder Note-Payload)")
ap.add_argument("--flatten-paths", action="store_true", help="Alle Dateien flach schreiben (orig_path in Frontmatter)")
args = ap.parse_args()
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix.strip()
client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim)
out_root = os.path.abspath(args.out)
os.makedirs(out_root, exist_ok=True)
notes_col, chunks_col, edges_col = collections(cfg.prefix)
# Filter nach note-id (optional)
flt = None
if args.note_id:
flt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=args.note_id))])
notes = _scroll_all(client, notes_col, flt)
total = 0
for n in notes:
pl = n.payload or {}
nid = pl.get("note_id")
rel_path = _norm_rel_path(pl.get("path") or "")
if args.flatten_paths or not rel_path:
fname = f"{(nid or 'note')}.md"
out_path = os.path.join(out_root, fname)
else:
out_path = os.path.join(out_root, rel_path)
out_path = out_path.replace("\\", "/")
_ensure_dir(out_path)
# Frontmatter aufbauen (nur sinnvolle Felder)
fm_fields = ["id","title","type","status","created","updated","tags","area","project","source","lang","slug","aliases"]
fm: Dict[str, Any] = {}
fm["id"] = nid
for k in fm_fields:
if k == "id":
continue
if k in pl and pl[k] is not None:
fm[k] = pl[k]
if args.flatten_paths and rel_path:
fm["orig_path"] = rel_path
# Body ermitteln (fulltext oder Chunks)
body = (pl.get("fulltext") or "").strip()
if not body:
flt_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
chunks = _scroll_all(client, chunks_col, flt_chunks)
body = _reconstruct_body_from_chunks(chunks)
# Edges (optional)
refs: List[str] = []
if args.include_edges != "none":
# aus Note-Payload, falls vorhanden
if isinstance(pl.get("references"), list) and pl["references"]:
refs = [r for r in pl["references"] if isinstance(r, str)]
else:
flt_edges = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
edges = _scroll_all(client, edges_col, flt_edges)
refs = _collect_forward_refs_from_edges(edges)
if args.include_edges == "yaml" and refs:
fm["references"] = refs
# Datei schreiben
if (not args.overwrite) and os.path.exists(out_path):
print(json.dumps({"note_id": nid, "path": out_path, "decision": "skip-exists"}))
continue
content = _frontmatter_block(fm) + (body + "\n" if body else "")
if args.include_edges == "footer" and refs:
content += "\n---\nLinks:\n" + "\n".join(f"- [[{r}]]" for r in refs) + "\n"
with open(out_path, "w", encoding="utf-8") as f:
f.write(content)
print(json.dumps({"note_id": nid, "path": out_path, "decision": "write"}))
total += 1
print(f"Done. Exported notes: {total}")
if __name__ == "__main__":
main()
+242
View File
@@ -0,0 +1,242 @@
#!/usr/bin/env python3
# scripts/gc_qdrant_after_vault_scan.py
# Version: 1.0.0 (2025-09-05)
#
# Zweck
# -----
# Garbage-Collector für Qdrant: löscht Einträge (Edges, Chunks, Notes), deren Note-ID
# nicht mehr im Obsidian-Vault vorhanden ist. So bleiben Vektorindex/Graph mit dem Vault konsistent.
#
# Aufrufparameter
# ---------------
# --vault PATH Pfad zum Obsidian-Vault (z. B. ./vault) [erforderlich]
# --mode {edges,content,all}
# edges -> nur Kanten mit source_id/target_id ∉ Vault-Notes löschen
# content -> Chunks & Notes löschen, deren note_id ∉ Vault-Notes (keine Edges)
# all -> zuerst edges, dann content
# --prefix PREFIX Collection-Präfix (Default: aus ENV COLLECTION_PREFIX oder "mindnet")
# --apply Ohne diesen Schalter: Dry-Run (nur Vorschau). Mit Schalter: ausführen.
# --yes Sicherheitsabfrage überspringen (non-interaktiv)
# --batch-size N Größe für In-Filter (Default: 1000)
#
# Hinweise
# --------
# - Benötigt funktionierendes Python-venv mit allen Abhängigkeiten von mindnet (qdrant-client usw.).
# - Läuft gegen die drei Collections {prefix}_notes, {prefix}_chunks, {prefix}_edges (keine anderen Projekte).
# - Nutzt OR-Filter (Filter.should) korrekt, keine minimum_should (Pydantic v2 lässt die Option nicht zu).
#
# Änderungen ggü. vorher
# ----------------------
# - Neu: eigenes GC-Tool; berührt ausschließlich das übergebene Projekt-Präfix.
# - Sichere Dry-Run-Vorschau mit Zählung pro Collection; interaktive Bestätigung.
# - Löscht Edges mittels Filter auf (source_id ∈ MISSING) ODER (target_id ∈ MISSING).
# - Löscht Chunks/Notes mittels Filter auf note_id ∈ MISSING.
#
# Beispiele
# ---------
# DRY RUN: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all
# APPLY: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode all --apply
# CI-Modus: python3 -m scripts.gc_qdrant_after_vault_scan --vault ./vault --mode edges --apply --yes
#
# Kompatibilität / Kontext
# ------------------------
# - Frontmatter-Validierung & Parsing wie im Projekt (parser.read_markdown, validate_required_frontmatter). :contentReference[oaicite:0]{index=0}
# - Note-Payload-Schema (u. a. note_id, hash_fulltext) wie spezifiziert. :contentReference[oaicite:1]{index=1}
# - Collection-Namen & Anlage-Logik über app.core.qdrant (notes/chunks = dim, edges = 1D). :contentReference[oaicite:2]{index=2}
# - Edges nutzen Felder source_id/target_id; Chunks/Notes tragen note_id in payload. :contentReference[oaicite:3]{index=3} :contentReference[oaicite:4]{index=4}
#
from __future__ import annotations
import argparse, os, glob, json, sys
from typing import Iterable, Set, Tuple, List
from dotenv import load_dotenv
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter # :contentReference[oaicite:5]{index=5}
from app.core.qdrant import QdrantConfig, get_client, collection_names # :contentReference[oaicite:6]{index=6}
def iter_note_ids_from_vault(vault_root: str) -> Set[str]:
"""Liest alle Markdown-Dateien im Vault und sammelt valide Frontmatter-IDs."""
paths = [p for p in glob.glob(os.path.join(vault_root, "**", "*.md"), recursive=True)]
out: Set[str] = set()
for p in paths:
try:
parsed = read_markdown(p)
fm = normalize_frontmatter(parsed.frontmatter)
validate_required_frontmatter(fm)
nid = fm.get("id") or fm.get("note_id")
if nid:
out.add(nid)
except Exception:
# ungültige oder Template-Dateien ohne Pflichtfelder ignorieren
continue
return out
def scroll_payload_values(client: QdrantClient, collection: str, field: str, limit: int = 2000) -> Set[str]:
"""Scrollt eine Collection und sammelt alle Payload-Werte eines Feldes (falls vorhanden)."""
out: Set[str] = set()
offset = None
while True:
res = client.scroll(
collection_name=collection,
scroll_filter=None,
with_payload=True,
with_vectors=False,
limit=limit,
offset=offset
)
# qdrant-client (neuere Versionen) geben (points, next_offset)
if isinstance(res, tuple) and len(res) == 2:
points, offset = res
else:
# Fallback, falls sich API signifikant ändert
points, offset = res[0], res[1] if len(res) > 1 else None
for pt in points or []:
pl = getattr(pt, "payload", None) or {}
v = pl.get(field)
if isinstance(v, str):
out.add(v)
if not offset:
break
return out
def chunked(iterable: Iterable[str], n: int) -> Iterable[List[str]]:
buf: List[str] = []
for x in iterable:
buf.append(x)
if len(buf) >= n:
yield buf
buf = []
if buf:
yield buf
def count_matches(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
"""Grober Zähler über scroll (Qdrant Count-API ist optional je Version)."""
total = 0
offset = None
while True:
points, offset = client.scroll(
collection_name=collection,
scroll_filter=filter_,
with_payload=False,
with_vectors=False,
limit=2000,
offset=offset
)
total += len(points or [])
if not offset:
break
return total
def build_filters_for_missing(prefix: str, missing_note_ids: Set[str]) -> Tuple[rest.Filter, rest.Filter, rest.Filter]:
"""
Erzeugt je Collection passende Filter:
- Notes/Chunks: note_id ∈ missing
- Edges: (source_id ∈ missing) ODER (target_id ∈ missing)
"""
notes_col, chunks_col, edges_col = collection_names(prefix) # :contentReference[oaicite:7]{index=7}
# MatchAny statt MatchValue für große Mengen
any_missing = rest.MatchAny(any=list(missing_note_ids))
f_notes = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=any_missing)])
f_edges = rest.Filter(should=[
rest.FieldCondition(key="source_id", match=any_missing),
rest.FieldCondition(key="target_id", match=any_missing),
])
return f_notes, f_chunks, f_edges
def preview(client: QdrantClient, prefix: str, missing: Set[str]) -> dict:
"""Erstellt eine Dry-Run-Vorschau (Anzahl Treffer pro Collection)."""
notes_col, chunks_col, edges_col = collection_names(prefix)
f_notes, f_chunks, f_edges = build_filters_for_missing(prefix, missing)
return {
"prefix": prefix,
"collections": {"notes": notes_col, "chunks": chunks_col, "edges": edges_col},
"missing_note_ids_count": len(missing),
"to_delete_counts": {
"edges": count_matches(client, edges_col, f_edges),
"chunks": count_matches(client, chunks_col, f_chunks),
"notes": count_matches(client, notes_col, f_notes),
}
}
def delete_with_filter(client: QdrantClient, collection: str, filter_: rest.Filter) -> int:
"""Löscht alle Punkte, die den Filter matchen. Gibt grobe Trefferzahl zurück."""
# erst zählen (für Feedback)
to_del = count_matches(client, collection, filter_)
if to_del == 0:
return 0
client.delete(
collection_name=collection,
points_selector=filter_,
wait=True
)
return to_del
def main():
load_dotenv()
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (z. B. ./vault)")
ap.add_argument("--mode", choices=["edges", "content", "all"], default="all",
help="Löschmodus (edges=nur Kanten, content=Chunks+Notes, all=beides)")
ap.add_argument("--prefix", help="Collection-Präfix (Default aus ENV COLLECTION_PREFIX)")
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter: Dry-Run")
ap.add_argument("--yes", action="store_true", help="Rückfrage unterdrücken")
ap.add_argument("--batch-size", type=int, default=1000, help="Batch-Größe für MatchAny")
args = ap.parse_args()
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix
client = get_client(cfg)
notes_col, chunks_col, edges_col = collection_names(cfg.prefix)
# 1) Vault lesen → Menge gültiger Note-IDs
vault_root = os.path.abspath(args.vault)
vault_ids = iter_note_ids_from_vault(vault_root)
# 2) Qdrant vorhandene Note-IDs einsammeln
existing_note_ids = scroll_payload_values(client, notes_col, "note_id")
# 3) Differenz ermitteln
missing = existing_note_ids - vault_ids
# 4) Vorschau
summary = preview(client, cfg.prefix, missing)
print(json.dumps({"mode": args.mode, "apply": args.apply, "summary": summary}, ensure_ascii=False, indent=2))
if not missing:
# nichts zu tun
return
if not args.apply:
# Dry-Run Ende
return
if not args.yes:
resp = input("Fortfahren und die oben gezeigten Objekte löschen? (yes/no): ").strip().lower()
if resp not in ("y", "yes"):
print("Abgebrochen.")
return
# 5) Löschen je Modus
f_notes, f_chunks, f_edges = build_filters_for_missing(cfg.prefix, missing)
report = {"deleted": {"edges": 0, "chunks": 0, "notes": 0}}
try:
if args.mode in ("edges", "all"):
report["deleted"]["edges"] = delete_with_filter(client, edges_col, f_edges)
if args.mode in ("content", "all"):
# Reihenfolge: erst chunks, dann notes
report["deleted"]["chunks"] = delete_with_filter(client, chunks_col, f_chunks)
report["deleted"]["notes"] = delete_with_filter(client, notes_col, f_notes)
finally:
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+413 -148
View File
@@ -1,185 +1,450 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
Script: scripts/import_markdown.py Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
Version: v2.3.1 (2025-09-05) Version: 3.7.2
Datum: 2025-09-30
Beschreibung Kurzbeschreibung
Importiert Markdown-Notizen in Qdrant (Notes, Chunks, Edges). ----------------
- Chunking + Embedding (MiniLM 384d, externer Embed-Server) - Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
- Edges direkt beim Import aus Wikilinks ([[…]]) ableiten - Änderungserkennung „Option C“: mehrere Hash-Varianten werden parallel in der Note
- Idempotente UUIDv5-IDs; Collections werden bereitgestellt gespeichert (Feld `hashes` mit Schlüsseln `<mode>:<source>:<normalize>`). Der Vergleich
nutzt NUR den aktuellen Modus-Key — ein Moduswechsel triggert keine Massenänderungen mehr.
- „Erstimport-Fix“: Bei leerem Qdrant gilt Create-Fall automatisch als geändert.
- `--baseline-modes`: fehlende Hash-Varianten still nachtragen (nur Notes upserten).
- `--sync-deletes`: gezielte Lösch-Synchronisation (Dry-Run + Apply).
- `--only-path`: exakt **eine** Datei (Pfad) importieren — nützlich für Diagnosefälle.
Aufruf Neu in 3.7.1/3.7.2
python3 -m scripts.import_markdown --vault ./vault [--apply] [--note-id ID] [--embed-note] [--force-replace] ------------------
- Chunk-Payloads: `window` (für Embeddings), `text` (überlappungsfrei, verlustfrei rekonstruierbar),
`start/end/overlap_*`. Embeddings nutzen `window`.
- **3.7.2:** Edges-Fehler führen nicht mehr zum Abbruch der gesamten Note; Note/Chunks werden trotzdem geschrieben.
Parameter Hash/Compare Konfiguration
--vault Pfad zum Obsidian-Vault (erforderlich) --------------------------
--apply Ohne Flag: Dry-Run (nur JSON-Zeilen). Mit Flag: schreibt in Qdrant. - Vergleichsmodus:
--note-id Nur eine spezifische Note-ID verarbeiten --hash-mode body|frontmatter|full
--embed-note Optional: Note-Volltext zusätzlich einbetten oder ENV: MINDNET_HASH_MODE | MINDNET_HASH_COMPARE
--force-replace Vor Upsert: zugehörige Edges der Quell-Note in Qdrant löschen (harte Ersetzung) - Quelle:
--hash-source parsed|raw (ENV: MINDNET_HASH_SOURCE, Default parsed)
- Normalisierung:
--hash-normalize canonical|none (ENV: MINDNET_HASH_NORMALIZE, Default canonical)
- Optional: --compare-text (oder ENV MINDNET_COMPARE_TEXT=true) vergleicht zusätzlich
den parsed Body-Text direkt.
Hinweise Qdrant / ENV
- Im venv laufen: `source .venv/bin/activate` ------------
- Erwartet laufenden Embed-Server (http://127.0.0.1:8990) - QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
- Qdrant via env: QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM - COLLECTION_PREFIX (Default: mindnet), via --prefix überschreibbar
- VECTOR_DIM (Default: 384)
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
Changelog Beispiele
v2.3.1: FIX – Für derive_wikilink_edges werden jetzt echte Chunk-Texte übergeben ---------
(chunks_for_links mit {"chunk_id","text"}), damit `references_at` erzeugt werden. # Standard (Body, parsed, canonical)
v2.3.0: Umstellung auf app.core.derive_edges; Edge-IDs unterstützen Occurrence. python3 -m scripts.import_markdown --vault ./vault
v2.2.x: Fix Filter-API (pydantic v2) bei Purge; Kleinkosmetik.
# Erstimport nach truncate (Create-Fall)
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert
# Nur eine Datei (Diagnose)
python3 -m scripts.import_markdown --vault ./vault --only-path ./vault/30_projects/project-demo.md --apply
# Sync-Deletes (Dry-Run → Apply)
python3 -m scripts.import_markdown --vault ./vault --sync-deletes
python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply
""" """
from __future__ import annotations from __future__ import annotations
import argparse, os, glob, json, sys
from dotenv import load_dotenv
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter import argparse
from app.core.note_payload import make_note_payload import json
from app.core.validate_note import validate_note_payload import os
from app.core.chunker import assemble_chunks import sys
from app.core.chunk_payload import make_chunk_payloads from typing import Dict, List, Optional, Tuple, Any, Set
from app.core.embed import embed_texts, embed_one
from app.core.qdrant import QdrantConfig, ensure_collections, get_client from dotenv import load_dotenv
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
from app.core.derive_edges import build_note_index, derive_wikilink_edges # nutzt 'text' je Chunk :contentReference[oaicite:1]{index=1}
from qdrant_client.http import models as rest from qdrant_client.http import models as rest
def iter_md(root: str, exclude_dirs=("/.obsidian/", "/_backup_frontmatter/", "/_imported/")): from app.core.parser import (
files = [p for p in glob.glob(os.path.join(root, "**", "*.md"), recursive=True)] read_markdown,
out = [] normalize_frontmatter,
for p in files: validate_required_frontmatter,
pn = p.replace("\\","/") )
if any(ex in pn for ex in exclude_dirs): from app.core.note_payload import make_note_payload
continue from app.core.chunker import assemble_chunks
out.append(p) from app.core.chunk_payload import make_chunk_payloads
try:
from app.core.derive_edges import build_edges_for_note
except Exception: # pragma: no cover
from app.core.edges import build_edges_for_note # type: ignore
from app.core.qdrant import (
QdrantConfig,
get_client,
ensure_collections,
ensure_payload_indexes,
)
from app.core.qdrant_points import (
points_for_chunks,
points_for_note,
points_for_edges,
upsert_batch,
)
try:
from app.core.embed import embed_texts # optional
except Exception:
embed_texts = None
# ---------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------
def iter_md(root: str) -> List[str]:
out: List[str] = []
for dirpath, _, filenames in os.walk(root):
for fn in filenames:
if not fn.lower().endswith(".md"):
continue
p = os.path.join(dirpath, fn)
pn = p.replace("\\", "/")
if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
continue
out.append(p)
return sorted(out)
def collections(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
notes_col, _, _ = collections(prefix)
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
points, _ = client.scroll(
collection_name=notes_col,
scroll_filter=f,
with_payload=True,
with_vectors=False,
limit=1,
)
if not points:
return None
return points[0].payload or {}
def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
notes_col, _, _ = collections(prefix)
out: Set[str] = set()
next_page = None
while True:
pts, next_page = client.scroll(
collection_name=notes_col,
with_payload=True,
with_vectors=False,
limit=256,
offset=next_page,
)
if not pts:
break
for p in pts:
pl = p.payload or {}
nid = pl.get("note_id")
if isinstance(nid, str):
out.add(nid)
if next_page is None:
break
return out return out
def purge_note_edges(client, prefix: str, source_note_id: str):
# Löscht Edges, deren source_id == source_note_id ODER (kind==backlink && target_id==source_note_id)
edges_col = f"{prefix}_edges"
f = rest.Filter(
should=[
rest.FieldCondition(key="source_id", match=rest.MatchValue(value=source_note_id)),
rest.Filter(
must=[
rest.FieldCondition(key="kind", match=rest.MatchValue(value="backlink")),
rest.FieldCondition(key="target_id", match=rest.MatchValue(value=source_note_id)),
]
),
]
)
# pydantic v2: client.delete(...) statt delete_points(...)
client.delete(collection_name=edges_col, points_selector=f, wait=True)
def main(): def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
_, chunks_col, edges_col = collections(prefix)
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
for col in (chunks_col, edges_col):
try:
client.delete(
collection_name=col,
points_selector=rest.FilterSelector(filter=filt),
wait=True
)
except Exception as e:
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
notes_col, chunks_col, edges_col = collections(prefix)
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
for col in (edges_col, chunks_col, notes_col):
try:
client.delete(
collection_name=col,
points_selector=rest.FilterSelector(filter=filt),
wait=True
)
except Exception as e:
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
def _resolve_mode(val: Optional[str]) -> str:
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
return "full"
if v in ("frontmatter", "fm"):
return "frontmatter"
return "body"
def _env(key: str, default: str) -> str:
return (os.environ.get(key) or default).strip().lower()
# ---------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------
def main() -> None:
load_dotenv() load_dotenv()
ap = argparse.ArgumentParser() ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True, help="Pfad zum Obsidian Vault (z.B. ./vault)") ap.add_argument("--vault", required=True, help="Pfad zum Obsidian-Vault (Root-Ordner)")
ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant (sonst Dry-Run)") ap.add_argument("--apply", action="store_true", help="Schreibt in Qdrant; ohne Flag nur Dry-Run")
ap.add_argument("--note-id", help="Nur eine Note-ID verarbeiten") ap.add_argument("--purge-before-upsert", action="store_true",
ap.add_argument("--embed-note", action="store_true", help="Auch Note-Volltext einbetten (optional)") help="Vor Upsert Chunks & Edges der GEÄNDERTEN Note löschen")
ap.add_argument("--force-replace", action="store_true", help="Vor Upsert alte Edges der Note löschen") ap.add_argument("--note-id", help="Nur eine bestimmte Note-ID verarbeiten")
ap.add_argument("--only-path", help="Exakt diesen Markdown-Pfad verarbeiten (ignoriert --note-id)")
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
ap.add_argument("--force-replace", action="store_true",
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None,
help="Vergleichsmodus (Body | Frontmatter | Full)")
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
help="Quelle für die Hash-Berechnung (Default: parsed)")
ap.add_argument("--note-scope-refs", action="store_true",
help="(Optional) erzeugt zusätzlich references:note/backlink:note (Default: aus)")
ap.add_argument("--debug-hash-diff", action="store_true",
help="(reserviert) optionaler Body-Diff")
ap.add_argument("--compare-text", action="store_true",
help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
ap.add_argument("--baseline-modes", action="store_true",
help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
ap.add_argument("--sync-deletes", action="store_true",
help="Notes/Chunks/Edges löschen, die in Qdrant existieren aber im Vault fehlen (Dry-Run; mit --apply ausführen)")
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
args = ap.parse_args() args = ap.parse_args()
# Qdrant mode = _resolve_mode(args.hash_mode) # body|frontmatter|full
cfg = QdrantConfig( src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed") # parsed|raw
url=os.getenv("QDRANT_URL", "http://127.0.0.1:6333"), norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical") # canonical|none
api_key=os.getenv("QDRANT_API_KEY") or None, note_scope_refs_env = (_env("MINDNET_NOTE_SCOPE_REFS", "false") == "true")
prefix=os.getenv("COLLECTION_PREFIX", "mindnet"), note_scope_refs = args.note_scope_refs or note_scope_refs_env
dim=int(os.getenv("VECTOR_DIM","384")), compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
)
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix.strip()
client = get_client(cfg) client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim) ensure_collections(client, cfg.prefix, cfg.dim)
ensure_payload_indexes(client, cfg.prefix)
root = os.path.abspath(args.vault) root = os.path.abspath(args.vault)
files = iter_md(root)
# Dateiliste bestimmen
if args.only_path:
only = os.path.abspath(args.only_path)
files = [only]
else:
files = iter_md(root)
if not files: if not files:
print("Keine Markdown-Dateien gefunden.", file=sys.stderr); sys.exit(2) print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
sys.exit(2)
# --- Note-Index (für robuste Zielauflösung) --- # Optional: Sync-Deletes vorab
note_stubs = [] if args.sync_deletes:
for path in files: vault_note_ids: Set[str] = set()
parsed = read_markdown(path) for path in files:
fm = normalize_frontmatter(parsed.frontmatter) try:
try: parsed = read_markdown(path)
validate_required_frontmatter(fm) if not parsed:
except Exception: continue
continue fm = normalize_frontmatter(parsed.frontmatter)
if args.note_id and fm.get("id") != args.note_id: nid = fm.get("id")
continue if isinstance(nid, str):
rel = os.path.relpath(parsed.path, root).replace("\\","/") vault_note_ids.add(nid)
note_stubs.append({"note_id": fm["id"], "title": fm.get("title",""), "path": rel}) except Exception:
note_index = build_note_index(note_stubs) continue
qdrant_note_ids = list_qdrant_note_ids(client, cfg.prefix)
total_notes = 0 to_delete = sorted(qdrant_note_ids - vault_note_ids)
for path in files:
parsed = read_markdown(path)
fm = normalize_frontmatter(parsed.frontmatter)
try:
validate_required_frontmatter(fm)
except Exception:
continue
if args.note_id and fm.get("id") != args.note_id:
continue
total_notes += 1
# Note-Payload
note_pl = make_note_payload(parsed, vault_root=root)
validate_note_payload(note_pl)
note_pl["fulltext"] = parsed.body # für derive_edges
# Chunks
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
# Embeddings (Chunks)
texts = [ch.text for ch in chunks]
vectors = embed_texts(texts)
# Optional: Note-Vektor
note_vec = embed_one(parsed.body) if args.embed_note else None
# --- WICHTIG: Chunk-Texte für derive_wikilink_edges ---
chunks_for_links = [
{"chunk_id": (pl.get("chunk_id") or pl.get("id") or f"{fm['id']}#{i+1}"),
"text": chunks[i].text}
for i, pl in enumerate(chunk_pls)
if i < len(chunks)
]
# Kanten (Note- & Chunk-Ebene)
edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index)
# Dry-Run-Ausgabe
print(json.dumps({ print(json.dumps({
"note_id": fm["id"], "action": "sync-deletes",
"title": fm["title"], "prefix": cfg.prefix,
"qdrant_total": len(qdrant_note_ids),
"vault_total": len(vault_note_ids),
"to_delete_count": len(to_delete),
"to_delete": to_delete[:50] + (["…"] if len(to_delete) > 50 else [])
}, ensure_ascii=False))
if args.apply and to_delete:
for nid in to_delete:
print(json.dumps({"action": "delete", "note_id": nid, "decision": "apply"}))
delete_note_everywhere(client, cfg.prefix, nid)
key_current = f"{mode}:{src}:{norm}"
processed = 0
for path in files:
# -------- Parse & Validate --------
try:
parsed = read_markdown(path)
except Exception as e:
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
continue
if parsed is None:
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
continue
try:
fm = normalize_frontmatter(parsed.frontmatter)
validate_required_frontmatter(fm)
except Exception as e:
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {type(e).__name__}: {e}"}))
continue
if args.note_id and not args.only_path and fm.get("id") != args.note_id:
continue
processed += 1
# -------- Build new payload (includes 'hashes') --------
note_pl = make_note_payload(
parsed,
vault_root=root,
hash_mode=mode,
hash_normalize=norm,
hash_source=src,
file_path=path,
)
if not note_pl.get("fulltext"):
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
note_id = note_pl.get("note_id") or fm.get("id")
if not note_id:
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
continue
# -------- Fetch old payload --------
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
has_old = old_payload is not None
old_hashes = (old_payload or {}).get("hashes") or {}
old_hash_exact = old_hashes.get(key_current)
new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
needs_baseline = (old_hash_exact is None)
hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact)
text_changed = False
if compare_text:
old_text = (old_payload or {}).get("fulltext") or ""
new_text = note_pl.get("fulltext") or ""
text_changed = (old_text != new_text)
changed = args.force_replace or (not has_old) or hash_changed or text_changed
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
# -------- Chunks / Embeddings --------
chunk_pls: List[Dict[str, Any]] = []
try:
body_text = getattr(parsed, "body", "") or ""
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
except Exception as e:
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
continue
vecs: List[List[float]] = [[0.0] * cfg.dim for _ in chunk_pls]
if embed_texts and chunk_pls:
try:
texts_for_embed = [(pl.get("window") or pl.get("text") or "") for pl in chunk_pls]
vecs = embed_texts(texts_for_embed)
except Exception as e:
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
# -------- Edges (robust) --------
edges: List[Dict[str, Any]] = []
edges_failed = False
if changed and (not do_baseline_only):
try:
note_refs = note_pl.get("references") or []
edges = build_edges_for_note(
note_id,
chunk_pls,
note_level_references=note_refs,
include_note_scope_refs=note_scope_refs,
)
except Exception as e:
edges_failed = True
edges = []
# WICHTIG: Wir brechen NICHT mehr ab — Note & Chunks werden geschrieben.
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
# -------- Summary --------
summary = {
"note_id": note_id,
"title": fm.get("title"),
"chunks": len(chunk_pls), "chunks": len(chunk_pls),
"edges": len(edges), "edges": len(edges),
"path": note_pl["path"] "edges_failed": edges_failed,
}, ensure_ascii=False)) "changed": changed,
"needs_baseline_for_mode": needs_baseline,
"decision": ("baseline-only" if args.apply and do_baseline_only else
"apply" if args.apply and changed else
"apply-skip-unchanged" if args.apply and not changed else
"dry-run"),
"path": note_pl["path"],
"hash_mode": mode,
"hash_normalize": norm,
"hash_source": src,
"prefix": cfg.prefix,
}
print(json.dumps(summary, ensure_ascii=False))
if args.apply: # -------- Writes --------
if args.force_replace: if not args.apply:
purge_note_edges(client, cfg.prefix, fm["id"]) continue
# Notes upsert if do_baseline_only:
notes_col, note_pts = points_for_note(cfg.prefix, note_pl, note_vec, cfg.dim) merged_hashes = {}
upsert_batch(client, notes_col, note_pts) merged_hashes.update(old_hashes)
merged_hashes.update(note_pl.get("hashes") or {})
if old_payload:
note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext"))
note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature"))
note_pl["hashes"] = merged_hashes
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
upsert_batch(client, notes_name, note_pts)
continue
# Chunks upsert if not changed:
chunks_col, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vectors) continue
upsert_batch(client, chunks_col, chunk_pts)
# Edges upsert if args.purge_before_upsert and has_old:
edges_col, edge_pts = points_for_edges(cfg.prefix, edges) try:
upsert_batch(client, edges_col, edge_pts) purge_note_artifacts(client, cfg.prefix, note_id)
except Exception as e:
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
upsert_batch(client, notes_name, note_pts)
if chunk_pls:
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
upsert_batch(client, chunks_name, chunk_pts)
if edges:
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
upsert_batch(client, edges_name, edge_pts)
print(f"Done. Processed notes: {processed}")
print(f"Done. Processed notes: {total_notes}")
if __name__ == "__main__": if __name__ == "__main__":
main() main()
+227
View File
@@ -0,0 +1,227 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
make_test_vault.py — erzeugt einen minimalen, nachvollziehbaren Test-Vault
Version: 1.2 (2025-09-09)
Änderungen ggü. 1.1:
- Fügt **externe Links** (http/mailto) in `exp-two.md` hinzu, damit der Chunk-Payload-Exporter
`external_links` realistisch testen kann.
- Kleine Textanpassungen zur stabilen Chunk-Bildung (Absatzstruktur).
Zweck
- Kleiner Obsidian-ähnlicher Vault zum Durchspielen des Importers (Chunks/Edges).
- Szenarien: leere Links, spätere Anlage fehlender Noten, Chunk-Neuaufteilung, externe Links.
Struktur (Default: ./test_vault)
- 40_concepts/concept-alpha.md
- 20_experiences/exp-one.md (verlinkt auf [[concept-alpha]] und [[missing-note]])
- 20_experiences/exp-two.md (verlinkt auf [[concept-alpha]] + externe Links)
- 30_projects/project-demo.md (verlinkt auf [[concept-alpha]] und [[exp-one]])
Voraussetzungen
- Keine. Rein Dateigenerierung.
Aufruf
python3 -m scripts.make_test_vault [--out ./test_vault] [--force] [--with-missing]
Parameter
--out Zielverzeichnis (Standard: ./test_vault)
--force Bestehenden Ordner löschen und neu anlegen.
--with-missing Auch die 'missing-note' direkt anlegen (Standard: erst im 2. Lauf)
Hinweise für Tests
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
2) Lege danach eine Datei für „missing-note“ an (mit gleicher ID im YAML) und importiere erneut:
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
4) Prüfe externe Links (verify_chunk_texts.py betrifft nur Text; externe Links siehst du in chunk_payloads).
Kompatibel mit:
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
"""
from __future__ import annotations
import argparse
import os
import shutil
from datetime import datetime, timezone
def iso_now() -> str:
# ISO 8601 mit Offset; als String zurückgeben
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
def q(s: str) -> str:
# Sicheres Quoten für YAML-Frontmatter
return f"\"{s}\""
TEMPLATE_CONCEPT = """---
title: "Concept Alpha"
id: "concept-alpha"
type: "concept"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/concept","topic/test"]
---
## Beschreibung
Dies ist ein einfaches Konzept für Testzwecke. Es wird von anderen Notizen referenziert.
## Mögliche Verbindungen
- [[exp-one]]
- [[project-demo]]
"""
TEMPLATE_EXP_ONE = """---
title: "Experience One"
id: "exp-one"
type: "experience"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/experience","topic/test"]
---
## Kontext
Diese Notiz verlinkt auf ein existierendes Konzept und auf eine noch **fehlende** Note.
## Beobachtung
- Bezug zu [[concept-alpha]]
- Offener Verweis auf [[missing-note]]
## Interpretation
Wenn später eine Datei für 'missing-note' angelegt wird, sollte der Importer die Kanten nachziehen.
## Mögliche Verbindungen
- [[concept-alpha]]
- [[missing-note]]
"""
TEMPLATE_EXP_TWO = """---
title: "Experience Two"
id: "exp-two"
type: "experience"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/experience","topic/test"]
---
## Kontext
Zweite Experience, die auf dasselbe Konzept verweist, ergänzt um **externe Links**.
## Beobachtung
Verweis auf [[concept-alpha]]. Zusätzliche Ressourcen:
- Link: [Beispielseite](https://example.com "Referenz")
- Kontakt: [E-Mail](mailto:info@example.com)
## Mögliche Verbindungen
- [[concept-alpha]]
"""
TEMPLATE_PROJECT = """---
title: "Project Demo"
id: "project-demo"
type: "project"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/project","topic/test"]
---
## Scope
Ein kleines Demo-Projekt, das andere Notizen referenziert.
## Arbeitspakete
- AP-1: Zusammenhang erklären (siehe [[concept-alpha]])
- AP-2: Ereignis berücksichtigen (siehe [[exp-one]])
## Mögliche Verbindungen
- [[concept-alpha]]
- [[exp-one]]
"""
TEMPLATE_MISSING = """---
title: "Missing Note (jetzt angelegt)"
id: "missing-note"
type: "concept"
status: "active"
created: {created}
updated: {updated}
tags: ["area/mindnet","type/concept","topic/test"]
---
## Kontext
Diese Notiz wurde **nachträglich** erstellt, um einen zuvor offenen Link zu schließen.
## Mögliche Verbindungen
- [[exp-one]]
"""
def write_file(path: str, content: str) -> None:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(content)
def build_vault(base: str, include_missing_note: bool) -> None:
created = q(iso_now())
updated = q(iso_now())
# 40_concepts
write_file(
os.path.join(base, "40_concepts", "concept-alpha.md"),
TEMPLATE_CONCEPT.format(created=created, updated=updated),
)
# 20_experiences
write_file(
os.path.join(base, "20_experiences", "exp-one.md"),
TEMPLATE_EXP_ONE.format(created=created, updated=updated),
)
write_file(
os.path.join(base, "20_experiences", "exp-two.md"),
TEMPLATE_EXP_TWO.format(created=created, updated=updated),
)
# 30_projects
write_file(
os.path.join(base, "30_projects", "project-demo.md"),
TEMPLATE_PROJECT.format(created=created, updated=updated),
)
# Optional: zunächst NICHT anlegen (Szenario „leerer Link“),
# später für den 2. Testlauf anlegen.
if include_missing_note:
write_file(
os.path.join(base, "40_concepts", "missing-note.md"),
TEMPLATE_MISSING.format(created=created, updated=updated),
)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="./test_vault", help="Zielverzeichnis für den Test-Vault")
ap.add_argument("--force", action="store_true", help="Falls vorhanden: Verzeichnis löschen und neu anlegen")
ap.add_argument("--with-missing", action="store_true",
help="Auch die 'missing-note' direkt anlegen (Standard: erst im 2. Lauf)")
args = ap.parse_args()
base = os.path.abspath(args.out)
if os.path.exists(base):
if args.force:
shutil.rmtree(base)
else:
print(f"Ziel existiert bereits: {base}\nNutze --force zum Überschreiben.")
return
build_vault(base, include_missing_note=args.with_missing)
print(f"Test-Vault erstellt unter: {base}")
print("Hinweis:")
print(" 1) Zuerst ohne missing-note anlegen (Standard), importieren, Edges prüfen.")
print(" 2) Dann erneut ausführen mit --force --with-missing (missing-note wird erzeugt),")
print(" und Import wiederholen → Edges/Backlinks sollten nachgezogen werden.")
print(" 3) In exp-two.md sind **externe Links** enthalten → prüfe `external_links` in Chunk-Payloads.")
if __name__ == "__main__":
main()
+232
View File
@@ -0,0 +1,232 @@
# scripts/prune_qdrant_vs_vault.py
# -----------------------------------------------------------------------------
# Name: prune_qdrant_vs_vault.py
# Version: 1.0.0 (2025-09-08)
# Zweck: Entfernt verwaiste Qdrant-Einträge (notes/chunks/edges), wenn
# die zugehörigen Markdown-Dateien im Vault nicht mehr existieren.
#
# Was es macht:
# - Liest alle note_id aus dem Vault (Frontmatter: id / note_id).
# - Liest alle note_id aus Qdrant (mindnet_notes).
# - Bildet die Differenz (nur in Qdrant vorhandene, im Vault fehlende).
# - Löscht für jede verwaiste note_id:
# * Notes-Point(s)
# * Alle Chunks der Note
# * Alle Edges, die auf diese Note referenzieren
# (source_id == note_id ODER target_id == note_id ODER
# source_note_id == note_id ODER target_note_id == note_id)
#
# Hinweise:
# - Kein globaler Delete. Nur betroffene note_id.
# - Dry-Run standardmäßig; tatsächliches Löschen erst mit --apply.
# - Interaktive Bestätigung (abschaltbar mit --yes).
#
# Aufruf:
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply
# python3 -m scripts.prune_qdrant_vs_vault --vault ./vault --prefix mindnet --apply --yes
#
# Voraussetzungen:
# - Ausführung im aktivierten venv empfohlen: source .venv/bin/activate
# - Qdrant läuft lokal (oder URL/API-Key in ENV), siehe app/core/qdrant.py
#
# Änderungen:
# - 1.0.0: Erster Release.
# -----------------------------------------------------------------------------
import argparse
import json
import os
import sys
from pathlib import Path
from typing import Dict, List, Set, Tuple
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client, collection_names
from app.core.parser import parse_markdown_file # nutzt euer bestehendes Parsing/Schema
def read_vault_note_ids(vault_dir: Path) -> Set[str]:
note_ids: Set[str] = set()
for p in vault_dir.rglob("*.md"):
try:
parsed = parse_markdown_file(str(p))
fm = parsed.frontmatter if hasattr(parsed, "frontmatter") else (parsed.get("frontmatter") or {})
nid = fm.get("id") or fm.get("note_id")
if nid:
note_ids.add(str(nid))
except Exception:
# still und leise weiter – wir wollen robust sein
continue
return note_ids
def qdrant_note_ids(client: QdrantClient, notes_col: str) -> Set[str]:
ids: Set[str] = set()
scroll_filter = None
offset = None
while True:
res = client.scroll(
collection_name=notes_col,
scroll_filter=scroll_filter,
offset=offset,
limit=256,
with_payload=True,
with_vectors=False,
)
points, next_offset = res
for pt in points:
pl = pt.payload or {}
nid = pl.get("note_id")
if nid:
ids.add(str(nid))
if next_offset is None:
break
offset = next_offset
return ids
def delete_by_filter(client: QdrantClient, collection: str, flt: rest.Filter) -> int:
# Sammel erst IDs via scroll (robuster Überblick), lösche dann über point_ids (batch)
to_delete: List[int] = []
offset = None
while True:
pts, next_offset = client.scroll(
collection_name=collection,
scroll_filter=flt,
offset=offset,
limit=256,
with_payload=False,
with_vectors=False,
)
for pt in pts:
to_delete.append(pt.id)
if next_offset is None:
break
offset = next_offset
if not to_delete:
return 0
client.delete(collection_name=collection, points_selector=rest.PointIdsList(points=to_delete), wait=True)
return len(to_delete)
def prune_for_note(
client: QdrantClient,
cols: Dict[str, str],
note_id: str,
dry_run: bool = True,
) -> Dict[str, int]:
stats = {"notes": 0, "chunks": 0, "edges": 0}
# Notes löschen (Filter auf payload.note_id)
f_notes = rest.Filter(
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
)
# Chunks löschen (Filter auf payload.note_id)
f_chunks = rest.Filter(
must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]
)
# Edges löschen: mehrere Teilmengen erfassen
edge_filters = [
rest.Filter(must=[rest.FieldCondition(key="source_id", match=rest.MatchValue(value=note_id))]),
rest.Filter(must=[rest.FieldCondition(key="target_id", match=rest.MatchValue(value=note_id))]),
rest.Filter(must=[rest.FieldCondition(key="source_note_id", match=rest.MatchValue(value=note_id))]),
rest.Filter(must=[rest.FieldCondition(key="target_note_id", match=rest.MatchValue(value=note_id))]),
]
if dry_run:
# Zähle nur, was *wäre* gelöscht worden
def count(flt: rest.Filter, col: str) -> int:
total = 0
offset = None
while True:
pts, next_offset = client.scroll(
collection_name=col,
scroll_filter=flt,
offset=offset,
limit=256,
with_payload=False,
with_vectors=False,
)
total += len(pts)
if next_offset is None:
break
offset = next_offset
return total
stats["notes"] = count(f_notes, cols["notes"])
stats["chunks"] = count(f_chunks, cols["chunks"])
stats["edges"] = sum(count(f, cols["edges"]) for f in edge_filters)
return stats
# tatsächliches Löschen
stats["notes"] = delete_by_filter(client, cols["notes"], f_notes)
stats["chunks"] = delete_by_filter(client, cols["chunks"], f_chunks)
e_deleted = 0
for f in edge_filters:
e_deleted += delete_by_filter(client, cols["edges"], f)
stats["edges"] = e_deleted
return stats
def main():
ap = argparse.ArgumentParser(description="Prune Qdrant data for notes missing in the Vault.")
ap.add_argument("--vault", required=True, help="Pfad zum Vault-Verzeichnis (Root).")
ap.add_argument("--prefix", default="mindnet", help="Collections-Präfix (Default: mindnet).")
ap.add_argument("--apply", action="store_true", help="Ohne diesen Schalter wird nur ein Dry-Run gemacht.")
ap.add_argument("--yes", action="store_true", help="Ohne Nachfrage ausführen.")
args = ap.parse_args()
vault_root = Path(args.vault).resolve()
if not vault_root.exists():
print(f"Vault-Verzeichnis nicht gefunden: {vault_root}", file=sys.stderr)
sys.exit(1)
cfg = QdrantConfig()
client = get_client(cfg)
cols = collection_names(args.prefix)
vault_ids = read_vault_note_ids(vault_root)
qdrant_ids = qdrant_note_ids(client, cols["notes"])
orphans = sorted(qdrant_ids - vault_ids)
preview = {
"mode": "APPLY" if args.apply else "DRY-RUN",
"prefix": args.prefix,
"vault_root": str(vault_root),
"collections": cols,
"counts": {
"vault_note_ids": len(vault_ids),
"qdrant_note_ids": len(qdrant_ids),
"orphans": len(orphans),
},
"orphans_sample": orphans[:20],
}
print(json.dumps(preview, ensure_ascii=False, indent=2))
if not orphans:
print("Keine verwaisten Einträge gefunden. Nichts zu tun.")
return
if not args.yes:
resp = input("\nFortfahren mit dem oben angezeigten Modus? (yes/no): ").strip().lower()
if resp not in ("y", "yes", "j", "ja"):
print("Abgebrochen.")
return
total_stats = {"notes": 0, "chunks": 0, "edges": 0}
for nid in orphans:
s = prune_for_note(client, cols, nid, dry_run=(not args.apply))
total_stats["notes"] += s["notes"]
total_stats["chunks"] += s["chunks"]
total_stats["edges"] += s["edges"]
print(json.dumps({"deleted": total_stats}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: scripts/report_hashes.py — Übersicht & Lücken bei Mehrfach-Hashes
Version: 1.0.0
Datum: 2025-09-10
Funktion
--------
Listet je Note die vorhandenen Einträge im Feld `hashes` (Signaturen: <mode>:<source>:<normalize>)
und meldet fehlende Soll-Keys. Eignet sich als CI-Check.
Optionen
--------
--prefix TEXT Collection-Prefix (CLI überschreibt ENV)
--require K [K ...] Zusätzliche Soll-Keys (Default: body|frontmatter|full:parsed:canonical)
--fail-on-missing Exitcode 2, wenn fehlende Keys gefunden werden
Beispiele
---------
python3 -m scripts.report_hashes --prefix mindnet
python3 -m scripts.report_hashes --require frontmatter:raw:none --fail-on-missing
"""
from __future__ import annotations
import argparse
import json
import os
from typing import List, Dict, Any
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client
def collections(prefix: str):
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _scroll_all(client, collection: str):
out = []
nextp = None
while True:
pts, nextp = client.scroll(collection_name=collection, with_payload=True, with_vectors=False, limit=256, offset=nextp)
if not pts:
break
out.extend(pts)
if nextp is None:
break
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
ap.add_argument("--require", nargs="+", help="Zusätzliche Soll-Keys <mode>:<source>:<normalize>")
ap.add_argument("--fail-on-missing", action="store_true", help="Exitcode 2 bei fehlenden Keys")
args = ap.parse_args()
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix.strip()
client = get_client(cfg)
notes_col, _, _ = collections(cfg.prefix)
pts = _scroll_all(client, notes_col)
required = set(args.require or [])
required |= {
"body:parsed:canonical",
"frontmatter:parsed:canonical",
"full:parsed:canonical",
}
missing_total = 0
for p in pts:
pl = p.payload or {}
nid = pl.get("note_id")
hashes = pl.get("hashes") or {}
present = set(hashes.keys())
missing = sorted(list(required - present))
obj = {
"note_id": nid,
"present_count": len(present),
"missing": missing,
}
print(json.dumps(obj, ensure_ascii=False))
missing_total += len(missing)
if args.fail_on_missing and missing_total > 0:
raise SystemExit(2)
print(json.dumps({"summary_missing_total": missing_total}))
if __name__ == "__main__":
main()
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python3
# scripts/resolve_unresolved_references.py
"""
resolve_unresolved_references.py — Unaufgelöste Wikilinks in Qdrant nachträglich auflösen
Version: 1.0.0 (2025-09-05)
Zweck
------
- Findet Edges in {prefix}_edges mit payload.status=="unresolved" und versucht, den Zielknoten
anhand bereits vorhandener Notes in {prefix}_notes aufzulösen.
- Aktualisiert die Edges (setzt target_id, entfernt status, setzt resolution), und erzeugt
– NUR für Note-Level 'references' – die symmetrische 'backlink'-Kante.
Warum?
------
- Beim ersten Import können Links auf (noch) nicht existierende Notizen zeigen.
- Sobald die Zielnotiz später existiert, kann dieses Skript die Kanten reparieren.
Aufruf
------
# Dry-Run (Standard):
python3 -m scripts.resolve_unresolved_references --prefix mindnet
# Anwenden:
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply
# Optional: nur X Edges anfassen
python3 -m scripts.resolve_unresolved_references --prefix mindnet --apply --limit 500
Parameter
---------
--prefix : Collection-Prefix (Default: aus Env COLLECION_PREFIX oder "mindnet")
--apply : Änderungen tatsächlich schreiben (ohne --apply = Dry-Run)
--limit : Max. Anzahl unaufgelöster Edges, die in diesem Lauf bearbeitet werden (Default: keine Begrenzung)
--batch : Upsert-Batchgröße (Default: 512)
Voraussetzungen / Hinweise
--------------------------
- Bitte im aktivierten venv laufen lassen (deine Umgebung: `.venv`).
- Qdrant-URL/Key/Prefix/Vektor-Dim werden wie üblich aus ENV gelesen (sieh app/core/qdrant.py). # noqa
- Nutzt die vorhandenen Utilities:
- app/core/qdrant.py (Client/Collections)
- app/core/qdrant_points.py (points_for_edges/upsert_batch)
- app/core/derive_edges.py (build_note_index/resolve_target)
Änderungshistorie
-----------------
1.0.0 Erstveröffentlichung.
"""
from __future__ import annotations
import argparse
import json
from typing import Any, Dict, List, Tuple, Iterable
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names # :contentReference[oaicite:3]{index=3}
from app.core.qdrant_points import points_for_edges, upsert_batch # :contentReference[oaicite:4]{index=4}
from app.core.derive_edges import build_note_index, resolve_target # :contentReference[oaicite:5]{index=5}
def _scroll(client: QdrantClient, **kwargs):
"""
Wrapper um qdrant_client.scroll() für unterschiedliche Client-Versionen:
neuere: (points, next_offset)
ältere: (points, next_page_offset, _)
"""
res = client.scroll(**kwargs)
if isinstance(res, tuple):
if len(res) == 2:
points, next_off = res
else:
# ältere Signatur: (points, next_off, _)
points, next_off, _ = res[0], res[1], res[2]
else:
# sehr alte Clients -> konservativ behandeln
points, next_off = res, None
return points, next_off
def _load_all_notes(client: QdrantClient, notes_col: str) -> List[Dict[str, Any]]:
notes: List[Dict[str, Any]] = []
next_off = None
while True:
pts, next_off = _scroll(
client,
collection_name=notes_col,
with_payload=True,
with_vectors=False,
limit=1024,
offset=next_off,
)
for p in pts or []:
pl = getattr(p, "payload", {}) or {}
# Erwartet Felder: note_id, title, path etc. (gemäß Schema) # :contentReference[oaicite:6]{index=6}
if pl.get("note_id"):
notes.append(pl)
if not next_off:
break
return notes
def _iter_unresolved_edges(client: QdrantClient, edges_col: str) -> Iterable[rest.Record]:
"""
Liefert alle Edge-Records mit payload.status == 'unresolved' und 'target_label' (string).
"""
f = rest.Filter(
must=[
rest.FieldCondition(key="status", match=rest.MatchValue(value="unresolved")),
]
)
next_off = None
while True:
pts, next_off = _scroll(
client,
collection_name=edges_col,
scroll_filter=f,
with_payload=True,
with_vectors=False,
limit=1024,
offset=next_off,
)
for p in pts or []:
pl = getattr(p, "payload", {}) or {}
if isinstance(pl.get("target_label"), str):
yield p
if not next_off:
break
def _make_backlink(source_note_id: str, target_note_id: str, extra: Dict[str, Any]) -> Dict[str, Any]:
"""
Baue eine 'backlink'-Edge-Payload source <- target (note-level).
"""
e = {
"kind": "backlink",
"source_id": target_note_id,
"target_id": source_note_id,
}
# Metafelder aus dem Original übernehmen (ohne status)
copy_keys = ["raw", "alias", "heading", "resolution"]
for k in copy_keys:
if k in extra:
e[k] = extra[k]
return e
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", help="Collection-Prefix (Default: Env/COLLECTION_PREFIX oder 'mindnet')")
ap.add_argument("--apply", action="store_true", help="Änderungen schreiben (ohne Flag = Dry-Run)")
ap.add_argument("--limit", type=int, default=0, help="Max. Anzahl unaufgelöster Edges bearbeiten (0 = alle)")
ap.add_argument("--batch", type=int, default=512, help="Upsert-Batchgröße")
args = ap.parse_args()
# Qdrant-Setup
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix
client = get_client(cfg)
ensure_collections(client, cfg.prefix, cfg.dim) # sorgt u. a. für 1D-Vektor-Collection bei Edges :contentReference[oaicite:7]{index=7}
notes_col, _, edges_col = collection_names(cfg.prefix) # :contentReference[oaicite:8]{index=8}
# Notes laden & Index bauen
notes = _load_all_notes(client, notes_col)
idx = build_note_index(notes) # (by_id, by_slug, by_file_slug) :contentReference[oaicite:9]{index=9}
# Unresolved-Edges scannen
to_fix: List[dict] = []
backlinks: List[dict] = []
processed = 0
resolved = 0
for rec in _iter_unresolved_edges(client, edges_col):
if args.limit and processed >= args.limit:
break
processed += 1
pl = dict(rec.payload or {})
kind = pl.get("kind") or "references"
src = pl.get("source_id")
tgt_label = pl.get("target_label") or pl.get("target_id") # Fallback
# Zielauflösung
resolved_id, how = resolve_target(str(tgt_label), idx) # :contentReference[oaicite:10]{index=10}
if not resolved_id:
continue # weiterhin unresolved
# Edge-Update
new_pl = dict(pl)
new_pl["target_id"] = resolved_id
new_pl["resolution"] = how
if "status" in new_pl:
del new_pl["status"]
# ID stabil lassen -> points_for_edges erzeugt UUID aus edge_id/Fallback :contentReference[oaicite:11]{index=11}
if "edge_id" not in new_pl:
# stabiler Key aus (kind, src, tgt, evtl. seq)
seq = new_pl.get("seq") or new_pl.get("order") or ""
new_pl["edge_id"] = f"{kind}:{src}->{resolved_id}#{seq}"
to_fix.append(new_pl)
resolved += 1
# Nur bei Note-Level references (nicht references_at) -> Backlink erzeugen
if kind == "references":
extra = {k: new_pl.get(k) for k in ("raw", "alias", "heading")}
extra["resolution"] = how
backlinks.append(_make_backlink(source_note_id=src, target_note_id=resolved_id, extra=extra))
# Ergebnis ausgeben
summary = {
"prefix": cfg.prefix,
"scanned_unresolved": processed,
"resolved": resolved,
"backlinks_to_create": len(backlinks),
"apply": bool(args.apply),
}
print(json.dumps(summary, ensure_ascii=False))
if not args.apply:
return
# Upserts (in Batches)
def _batched(items: List[dict], n: int) -> Iterable[List[dict]]:
for i in range(0, len(items), n):
yield items[i : i + n]
# 1) Updates für reparierte Edges
for chunk in _batched(to_fix, args.batch):
col, pts = points_for_edges(cfg.prefix, chunk) # sorgt für Edge-UUID & Dummy-Vector :contentReference[oaicite:12]{index=12}
upsert_batch(client, col, pts)
# 2) Backlinks (nur references)
for chunk in _batched(backlinks, args.batch):
col, pts = points_for_edges(cfg.prefix, chunk)
upsert_batch(client, col, pts)
if __name__ == "__main__":
main()
+185
View File
@@ -0,0 +1,185 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: scripts/verify_chunk_texts.py
Version: 1.0.0
Datum: 2025-09-09
Kurzbeschreibung
---------------
Verifiziert, dass in Qdrant für jede Note die zugehörigen Chunks ein Textfeld
enthalten und der Body (notes.payload.fulltext) aus den Chunk-Texten sinnvoll
rekonstruiert werden kann.
Prüfungen pro Note:
- Alle Chunks vorhanden (>=1).
- Jedes Chunk-Payload hat einen nutzbaren Textschlüssel: "text" (bevorzugt), sonst "content", sonst "raw".
- Reihenfolge der Chunks wird stabil bestimmt (payload.chunk_index -> Nummer aus chunk_id).
- Coverage: Summe der im Fulltext gefundenen Chunk-Textsegmente / len(Fulltext) (Toleranz für Overlaps).
-> OK wenn coverage >= 0.90 (konfigurierbar via --min-coverage)
Ausgabe:
- JSON mit Gesamtsummen und Details je Note.
ENV:
- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
- COLLECTION_PREFIX (Fallback, wenn --prefix fehlt)
Beispiele:
# Alle Notes prüfen (Prefix aus ENV)
python3 -m scripts.verify_chunk_texts
# Nur eine Note prüfen
python3 -m scripts.verify_chunk_texts --note-id concept-alpha
# Prefix explizit setzen und strengere Coverage verlangen
python3 -m scripts.verify_chunk_texts --prefix mindnet --min-coverage 0.95
"""
from __future__ import annotations
import argparse, json, os, re, sys
from typing import Dict, List, Tuple, Optional
from qdrant_client import QdrantClient
from qdrant_client.http import models as rest
def _names(prefix: str) -> Tuple[str,str,str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def _client() -> QdrantClient:
url = os.getenv("QDRANT_URL")
if not url:
host = os.getenv("QDRANT_HOST", "127.0.0.1")
port = int(os.getenv("QDRANT_PORT", "6333"))
url = f"http://{host}:{port}"
api_key = os.getenv("QDRANT_API_KEY") or None
return QdrantClient(url=url, api_key=api_key)
def _chunk_sort_key(p: Dict, pid: str) -> Tuple[int,int,str]:
# Primär: payload.chunk_index, sekundär: Nummer am Ende der ID (#cNN oder #NN), sonst 0
ci = p.get("chunk_index")
n = 0
m = re.search(r'#c?(\d+)$', pid or "")
if m:
try:
n = int(m.group(1))
except Exception:
n = 0
return (ci if isinstance(ci, int) else 1_000_000 + n, n, pid)
def _choose_text(payload: Dict) -> Optional[str]:
for k in ("text", "content", "raw"):
v = payload.get(k)
if isinstance(v, str) and v.strip():
return v
return None
def _coverage(fulltext: str, pieces: List[str]) -> float:
"""Berechnet die Abdeckungsquote der Stücke im Fulltext (sequenzielles Matching)."""
if not fulltext:
return 0.0 if pieces else 1.0
cursor = 0
covered = 0
ft = fulltext
for piece in pieces:
if not piece:
continue
# Tolerant gegen Whitespace-Unterschiede: normalisieren nur \r\n→\n
p = piece.replace("\r\n", "\n").replace("\r", "\n")
idx = ft.find(p, cursor)
if idx == -1:
# Versuche ein paar Heuristiken: trimmen
p2 = p.strip()
if p2 and len(p2) > 8:
idx = ft.find(p2, cursor)
if idx != -1:
covered += len(p)
cursor = idx + len(p)
# sonst: nicht abgedeckt
return covered / max(1, len(ft))
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", help="Collection-Prefix (Default: ENV COLLECTION_PREFIX oder 'mindnet')")
ap.add_argument("--note-id", help="Nur eine bestimmte Note prüfen")
ap.add_argument("--min-coverage", type=float, default=0.90, help="Mindestabdeckung durch Chunks (Default: 0.90)")
args = ap.parse_args()
prefix = args.prefix or os.getenv("COLLECTION_PREFIX", "mindnet")
notes_col, chunks_col, _ = _names(prefix)
cli = _client()
# Notes abrufen (optional filter by note_id)
notes_filter = None
if args.note_id:
notes_filter = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=args.note_id))])
notes: List[Dict] = []
off = None
while True:
pts, off = cli.scroll(collection_name=notes_col, scroll_filter=notes_filter,
with_payload=True, with_vectors=False, limit=256, offset=off)
if not pts:
break
for p in pts:
notes.append({"id": p.id, "payload": p.payload or {}})
if off is None:
break
results = []
total_missing_text = 0
total_notes_ok = 0
for n in notes:
pl = n["payload"]
nid = pl.get("note_id") or pl.get("id") or n.get("id")
fulltext = pl.get("fulltext") or ""
# Chunks der Note holen
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
chunks = []
off = None
while True:
pts, off = cli.scroll(collection_name=chunks_col, scroll_filter=f,
with_payload=True, with_vectors=False, limit=256, offset=off)
if not pts:
break
for p in pts:
chunks.append({"id": p.id, "payload": p.payload or {}})
if off is None:
break
# sortieren
chunks.sort(key=lambda c: _chunk_sort_key(c["payload"], c["id"]))
texts = []
missing_text = 0
for c in chunks:
t = _choose_text(c["payload"])
if t is None:
missing_text += 1
texts.append("")
else:
texts.append(t)
cov = _coverage(fulltext, texts)
ok = (missing_text == 0) and (cov >= args.min_coverage or not fulltext)
if ok:
total_notes_ok += 1
total_missing_text += missing_text
results.append({
"note_id": nid,
"title": pl.get("title"),
"chunks": len(chunks),
"missing_chunk_texts": missing_text,
"coverage": round(cov, 4),
"has_fulltext": bool(fulltext),
"ok": ok
})
out = {
"collections": {"notes": notes_col, "chunks": chunks_col},
"notes_checked": len(notes),
"notes_ok": total_notes_ok,
"total_missing_chunk_texts": total_missing_text,
"min_coverage": args.min_coverage,
"details": results
}
print(json.dumps(out, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
tests/audit_edges_now.py — Zählt Edges nach kind & scope und zeigt Chunk- und Note-Anzahl.
"""
import os, sys, json
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client
def scroll_all(c, col, flt=None):
out, nextp = [], None
while True:
pts, nextp = c.scroll(collection_name=col, with_payload=True, with_vectors=False, limit=256, scroll_filter=flt, offset=nextp)
if not pts: break
out.extend(pts)
if nextp is None: break
return out
def main():
cfg = QdrantConfig.from_env()
c = get_client(cfg)
notes = f"{cfg.prefix}_notes"
chunks = f"{cfg.prefix}_chunks"
edges = f"{cfg.prefix}_edges"
es = scroll_all(c, edges)
counts = {}
by_scope = {}
for p in es:
pl = p.payload or {}
k = pl.get("kind") or "?"
s = pl.get("scope") or "?"
counts[k] = counts.get(k, 0) + 1
key = f"{k}:{s}"
by_scope[key] = by_scope.get(key, 0) + 1
print(json.dumps({
"collections": {"notes": notes, "chunks": chunks, "edges": edges},
"total_edges": len(es),
"counts": counts,
"counts_by_scope": by_scope
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+146
View File
@@ -0,0 +1,146 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
tests/check_chunks_window_vs_text.py
Prüft pro Note und global:
- ob window == text (sollte bei Overlap normalerweise NEIN sein)
- ungefähre linke Overlap-Länge (Suffix von prev.text vs. Prefix von cur.window)
- Größen & einfache Statistiken
Nutzung:
python3 tests/check_chunks_window_vs_text.py --prefix mindnet [--vault ./test_vault]
"""
from __future__ import annotations
import os, sys, json, argparse
from typing import List, Dict, Any, Tuple, Optional
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from app.core.qdrant import QdrantConfig, get_client
from qdrant_client.http import models as rest
def collections(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def scroll_chunks_by_note(client, chunks_col: str, note_id: str) -> List[Dict[str, Any]]:
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
out = []
next_page = None
while True:
pts, next_page = client.scroll(
collection_name=chunks_col,
with_payload=True,
with_vectors=False,
limit=256,
offset=next_page,
scroll_filter=f,
)
if not pts:
break
out.extend([p.payload or {} for p in pts])
if next_page is None:
break
# sortieren: seq -> chunk_index -> chunk_id
def keyer(pl):
return (pl.get("seq", 1<<30), pl.get("chunk_index", 1<<30), str(pl.get("chunk_id", "")))
return sorted(out, key=keyer)
def list_note_ids(client, notes_col: str) -> List[str]:
out = []
next_page = None
while True:
pts, next_page = client.scroll(
collection_name=notes_col,
with_payload=True,
with_vectors=False,
limit=256,
offset=next_page,
)
if not pts:
break
for p in pts:
pl = p.payload or {}
nid = pl.get("note_id")
if isinstance(nid, str):
out.append(nid)
if next_page is None:
break
return sorted(set(out))
def common_overlap_len(a_suffix: str, b_prefix: str, max_probe: int = 256) -> int:
"""
Länge des längsten Suffix von a_suffix, das Prefix von b_prefix ist (bruteforce, begrenzt).
"""
n = min(len(a_suffix), len(b_prefix), max_probe)
if n <= 0:
return 0
a = a_suffix[-n:]
b = b_prefix[:n]
# steigere den Match von 0..n
for k in range(n, 0, -1):
if a[-k:] == b[:k]:
return k
return 0
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", required=True)
ap.add_argument("--vault", help="optional, nur für Vergleichsausgaben")
args = ap.parse_args()
cfg = QdrantConfig.from_env()
cfg.prefix = args.prefix
client = get_client(cfg)
notes_col, chunks_col, _ = collections(cfg.prefix)
note_ids = list_note_ids(client, notes_col)
if not note_ids:
print(json.dumps({"error": "no notes found in qdrant for this prefix"}))
return
global_total = 0
global_ident = 0
global_left_ov_sum = 0
detail = []
for nid in note_ids:
chunks = scroll_chunks_by_note(client, chunks_col, nid)
if not chunks:
detail.append({"note_id": nid, "chunks": 0})
continue
ident = 0
left_ov_sum = 0
for i, ch in enumerate(chunks):
txt = (ch.get("text") or "").replace("\r\n", "\n")
win = (ch.get("window") or "").replace("\r\n", "\n")
if txt == win:
ident += 1
if i > 0:
prev_txt = (chunks[i-1].get("text") or "").replace("\r\n", "\n")
left_ov = common_overlap_len(prev_txt, win, max_probe=1024)
left_ov_sum += left_ov
global_total += len(chunks)
global_ident += ident
global_left_ov_sum += left_ov_sum
detail.append({
"note_id": nid,
"chunks": len(chunks),
"identical_text_window": ident,
"identical_share": round(ident / max(1, len(chunks)), 3),
"avg_left_overlap_est": round(left_ov_sum / max(1, len(chunks)-1), 2) if len(chunks) > 1 else 0.0
})
out = {
"notes": len(note_ids),
"chunks_total": global_total,
"identical_total": global_ident,
"identical_share_global": round(global_ident / max(1, global_total), 3),
"avg_left_overlap_est_global": round(global_left_ov_sum / max(1, global_total - len(note_ids)), 2) if global_total > len(note_ids) else 0.0,
"details": detail[:100] # capped
}
print(json.dumps(out, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+146
View File
@@ -0,0 +1,146 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
tests/check_chunks_window_vs_text.py
Prüft pro Note und global:
- ob window == text (sollte bei Overlap normalerweise NEIN sein)
- ungefähre linke Overlap-Länge (Suffix von prev.text vs. Prefix von cur.window)
- Größen & einfache Statistiken
Nutzung:
python3 tests/check_chunks_window_vs_text.py --prefix mindnet [--vault ./test_vault]
"""
from __future__ import annotations
import os, sys, json, argparse
from typing import List, Dict, Any, Tuple, Optional
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from app.core.qdrant import QdrantConfig, get_client
from qdrant_client.http import models as rest
def collections(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def scroll_chunks_by_note(client, chunks_col: str, note_id: str) -> List[Dict[str, Any]]:
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
out = []
next_page = None
while True:
pts, next_page = client.scroll(
collection_name=chunks_col,
with_payload=True,
with_vectors=False,
limit=256,
offset=next_page,
scroll_filter=f,
)
if not pts:
break
out.extend([p.payload or {} for p in pts])
if next_page is None:
break
# sortieren: seq -> chunk_index -> chunk_id
def keyer(pl):
return (pl.get("seq", 1<<30), pl.get("chunk_index", 1<<30), str(pl.get("chunk_id", "")))
return sorted(out, key=keyer)
def list_note_ids(client, notes_col: str) -> List[str]:
out = []
next_page = None
while True:
pts, next_page = client.scroll(
collection_name=notes_col,
with_payload=True,
with_vectors=False,
limit=256,
offset=next_page,
)
if not pts:
break
for p in pts:
pl = p.payload or {}
nid = pl.get("note_id")
if isinstance(nid, str):
out.append(nid)
if next_page is None:
break
return sorted(set(out))
def common_overlap_len(a_suffix: str, b_prefix: str, max_probe: int = 256) -> int:
"""
Länge des längsten Suffix von a_suffix, das Prefix von b_prefix ist (bruteforce, begrenzt).
"""
n = min(len(a_suffix), len(b_prefix), max_probe)
if n <= 0:
return 0
a = a_suffix[-n:]
b = b_prefix[:n]
# steigere den Match von 0..n
for k in range(n, 0, -1):
if a[-k:] == b[:k]:
return k
return 0
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", required=True)
ap.add_argument("--vault", help="optional, nur für Vergleichsausgaben")
args = ap.parse_args()
cfg = QdrantConfig.from_env()
cfg.prefix = args.prefix
client = get_client(cfg)
notes_col, chunks_col, _ = collections(cfg.prefix)
note_ids = list_note_ids(client, notes_col)
if not note_ids:
print(json.dumps({"error": "no notes found in qdrant for this prefix"}))
return
global_total = 0
global_ident = 0
global_left_ov_sum = 0
detail = []
for nid in note_ids:
chunks = scroll_chunks_by_note(client, chunks_col, nid)
if not chunks:
detail.append({"note_id": nid, "chunks": 0})
continue
ident = 0
left_ov_sum = 0
for i, ch in enumerate(chunks):
txt = (ch.get("text") or "").replace("\r\n", "\n")
win = (ch.get("window") or "").replace("\r\n", "\n")
if txt == win:
ident += 1
if i > 0:
prev_txt = (chunks[i-1].get("text") or "").replace("\r\n", "\n")
left_ov = common_overlap_len(prev_txt, win, max_probe=1024)
left_ov_sum += left_ov
global_total += len(chunks)
global_ident += ident
global_left_ov_sum += left_ov_sum
detail.append({
"note_id": nid,
"chunks": len(chunks),
"identical_text_window": ident,
"identical_share": round(ident / max(1, len(chunks)), 3),
"avg_left_overlap_est": round(left_ov_sum / max(1, len(chunks)-1), 2) if len(chunks) > 1 else 0.0
})
out = {
"notes": len(note_ids),
"chunks_total": global_total,
"identical_total": global_ident,
"identical_share_global": round(global_ident / max(1, global_total), 3),
"avg_left_overlap_est_global": round(global_left_ov_sum / max(1, global_total - len(note_ids)), 2) if global_total > len(note_ids) else 0.0,
"details": detail[:100] # capped
}
print(json.dumps(out, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+177
View File
@@ -0,0 +1,177 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
tests/compare_vaults.py
Vergleicht zwei Markdown-Vaults rekursiv:
- Schlüsselauswahl: id | title | filename | auto (default: auto = id>title>filename)
- Fokus: body | frontmatter | all (default: body)
- Unicode: NFC-Normalisierung für IDs/Titel/Dateinamen und Text
Beispiele:
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --focus body
python3 tests/compare_vaults.py --src ./vault --dst ./_exportVault --key id --focus all
Ausgabe:
- Einträge mit 'missing' (nur in src oder nur in dst)
- Einträge mit 'diff' (vorhanden in beiden, aber Abweichungen gem. Fokus)
- Summary-Objekt am Ende
"""
from __future__ import annotations
import os
import sys
import json
import argparse
import unicodedata
from typing import Dict, Tuple, Optional, Any
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
# Wir nutzen denselben Parser wie im Projekt (fehlertolerant)
from app.core.parser import read_markdown # erwartet: (frontmatter, body)
MD_EXTS = {".md", ".markdown"}
def nfc(s: Optional[str]) -> str:
return unicodedata.normalize("NFC", s or "")
def norm_text(txt: str) -> str:
# normalize line endings + strip trailing spaces per line + NFC
t = txt.replace("\r\n", "\n").replace("\r", "\n")
t = "\n".join(line.rstrip() for line in t.split("\n")).strip()
return nfc(t)
def iter_md_files(root: str):
for base, _dirs, files in os.walk(root):
for fn in files:
ext = os.path.splitext(fn)[1].lower()
if ext in MD_EXTS:
yield os.path.join(base, fn)
def key_from(front: Dict[str, Any], path: str, pref: str) -> str:
"""
Liefert den Vergleichsschlüssel.
pref in {"auto","id","title","filename"}
"""
if pref == "id":
return nfc(str(front.get("id", "")).strip())
if pref == "title":
return nfc(str(front.get("title", "")).strip())
if pref == "filename":
name = os.path.splitext(os.path.basename(path))[0]
return nfc(name.strip())
# auto: id > title > filename
if "id" in front and str(front["id"]).strip():
return nfc(str(front["id"]).strip())
if "title" in front and str(front["title"]).strip():
return nfc(str(front["title"]).strip())
name = os.path.splitext(os.path.basename(path))[0]
return nfc(name.strip())
def load_index(root: str, key_pref: str) -> Dict[str, Tuple[str, Dict[str, Any], str]]:
"""
Baut ein Index-Dict: key -> (path, frontmatter, body)
"""
idx: Dict[str, Tuple[str, Dict[str, Any], str]] = {}
for p in iter_md_files(root):
try:
fm, body = read_markdown(p)
k = key_from(fm or {}, p, key_pref)
if not k:
# Fallback: filename
k = key_from({}, p, "filename")
idx[k] = (p, fm or {}, body or "")
except Exception as e:
# Datei nicht parsebar -> trotzdem indexieren nach Dateiname (leer FM/Body)
k = key_from({}, p, key_pref if key_pref != "auto" else "filename")
idx[k] = (p, {}, "")
return idx
def compare_entries(src: Tuple[str, Dict[str, Any], str],
dst: Tuple[str, Dict[str, Any], str],
focus: str) -> Dict[str, Any]:
sp, sfm, sbody = src
dp, dfm, dbody = dst
res = {"status": "ok"} # wird 'diff' falls Unterschiede
diffs = {}
if focus in ("frontmatter", "all"):
# Vergleiche FM minimal: id und title
s_id = nfc(str(sfm.get("id", "")).strip())
d_id = nfc(str(dfm.get("id", "")).strip())
s_title = nfc(str(sfm.get("title", "")).strip())
d_title = nfc(str(dfm.get("title", "")).strip())
if s_id != d_id:
diffs["frontmatter.id"] = {"src": s_id, "dst": d_id}
if s_title != d_title:
diffs["frontmatter.title"] = {"src": s_title, "dst": d_title}
if focus in ("body", "all"):
if norm_text(sbody) != norm_text(dbody):
diffs["body"] = {"src_len": len(sbody or ""), "dst_len": len(dbody or "")}
if diffs:
res["status"] = "diff"
res["diffs"] = diffs
res["src_path"] = sp
res["dst_path"] = dp
return res
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--src", required=True, help="Quell-Vault-Ordner")
ap.add_argument("--dst", required=True, help="Export-Vault-Ordner")
ap.add_argument("--focus", choices=["body", "frontmatter", "all"], default="body")
ap.add_argument("--key", choices=["auto", "id", "title", "filename"], default="auto",
help="Vergleichsschlüssel (default: auto=id>title>filename)")
args = ap.parse_args()
src_idx = load_index(args.src, args.key)
dst_idx = load_index(args.dst, args.key)
src_keys = set(src_idx.keys())
dst_keys = set(dst_idx.keys())
# Fehlende
only_src = sorted(src_keys - dst_keys)
only_dst = sorted(dst_keys - src_keys)
count = 0
for k in only_src:
print(json.dumps({"note_id": k, "status": "missing", "src": True, "dst": False}, ensure_ascii=False))
count += 1
for k in only_dst:
print(json.dumps({"note_id": k, "status": "missing", "src": False, "dst": True}, ensure_ascii=False))
count += 1
# Vergleiche, wo vorhanden
diff_count = 0
common = sorted(src_keys & dst_keys)
for k in common:
res = compare_entries(src_idx[k], dst_idx[k], args.focus)
if res["status"] == "diff":
res["note_id"] = k
print(json.dumps(res, ensure_ascii=False))
diff_count += 1
summary = {
"summary": "OK" if (count == 0 and diff_count == 0) else "DIFFS",
"missing_count": count,
"diff_count": diff_count,
"total_src": len(src_keys),
"total_dst": len(dst_keys),
"focus": args.focus,
"key": args.key,
}
print(json.dumps(summary, ensure_ascii=False))
# Exit-Code für CI
if summary["summary"] != "OK":
# non-zero exit optional; hier 0 lassen, damit CLI-Ausgaben sichtbar bleiben
pass
if __name__ == "__main__":
main()
+70
View File
@@ -0,0 +1,70 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
tests/inspect_one_note.py
Diagnose einer einzelnen Markdown-Datei (kein Upsert).
Usage:
python3 tests/inspect_one_note.py --path ./test_vault/30_projects/project-demo.md
"""
from __future__ import annotations
import os, sys, json, argparse
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter
from app.core.chunker import assemble_chunks
from app.core.chunk_payload import make_chunk_payloads
try:
from app.core.derive_edges import build_edges_for_note
except Exception:
from app.core.edges import build_edges_for_note # type: ignore
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--path", required=True)
args = ap.parse_args()
p = os.path.abspath(args.path)
try:
parsed = read_markdown(p)
if parsed is None:
print(json.dumps({"path": p, "error": "read_markdown returned None"}))
return
except Exception as e:
print(json.dumps({"path": p, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
return
try:
fm = normalize_frontmatter(parsed.frontmatter)
validate_required_frontmatter(fm)
except Exception as e:
print(json.dumps({"path": p, "error": f"frontmatter invalid: {type(e).__name__}: {e}"}))
return
body = getattr(parsed, "body", "") or ""
chunks = assemble_chunks(fm["id"], body, fm.get("type", "concept"))
rel_for_payload = os.path.relpath(p, os.path.join(p, "..", ".."))
chunk_pls = make_chunk_payloads(fm, rel_for_payload, chunks, note_text=body)
edge_error = None
edges_count = 0
try:
edges = build_edges_for_note(fm["id"], chunk_pls, include_note_scope_refs=True)
edges_count = len(edges)
except Exception as e:
edge_error = f"{type(e).__name__}: {e}"
print(json.dumps({
"path": p,
"id": fm.get("id"),
"title": fm.get("title"),
"body_len": len(body),
"chunks": len(chunk_pls),
"edges": edges_count,
"edge_error": edge_error
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
tests/list_md_seen_by_importer.py
Zeigt, welche Markdown-Dateien der Importer sehen würde (gleiche Filter & Sortierung).
Usage:
python3 tests/list_md_seen_by_importer.py --vault ./test_vault
"""
from __future__ import annotations
import argparse, os, sys, json
def iter_md(root: str):
out = []
for dirpath, _, filenames in os.walk(root):
for fn in filenames:
p = os.path.join(dirpath, fn)
pn = p.replace("\\", "/")
item = {
"path": os.path.abspath(p),
"rel": os.path.relpath(p, root).replace("\\", "/"),
"included": False,
"reason": "",
}
if not fn.lower().endswith(".md"):
item["reason"] = "skip_ext"
elif "/.obsidian/" in pn or "/_backup_frontmatter/" in pn or "/_imported/" in pn:
item["reason"] = "skip_special_dir"
else:
item["included"] = True
item["reason"] = "ok"
out.append(item)
# auch die ausgeschlossenen zeigen:
if not item["included"]:
print(json.dumps(item, ensure_ascii=False))
out_sorted = sorted(out, key=lambda x: x["rel"])
for i, it in enumerate(out_sorted):
it["order"] = i
print(json.dumps(it, ensure_ascii=False))
print(json.dumps({"summary":"done","included":len(out_sorted)}, ensure_ascii=False))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--vault", required=True)
args = ap.parse_args()
root = os.path.abspath(args.vault)
iter_md(root)
if __name__ == "__main__":
main()
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env bash
set -euo pipefail
# tests/run_e2e_roundtrip.sh
# End-to-End Testpipeline für mindnet WP-03
#
# Schritte:
# 1) Qdrant truncate (Prefix!)
# 2) Import (Apply + Purge) => Create-Fall
# 3) Import (Apply - ohne Änderungen) => Idempotenz
# 4) Hash-Reporter (Fehlende Hash-Varianten)
# 5) Export nach _exportVault
# 6) Vergleich Vault vs. Export (Body-Fokus)
# 7) Sync-Deletes: Eine Datei temporär „entfernen“, Dry-Run + Apply, dann zurückrollen
#
# Aufrufe:
# ./tests/run_e2e_roundtrip.sh [--vault ./test_vault] [--prefix mindnet]
#
# ENV:
# COLLECTION_PREFIX (Default mindnet)
VAULT="./test_vault"
PREFIX="${COLLECTION_PREFIX:-mindnet}"
while [[ $# -gt 0 ]]; do
case "$1" in
--vault) VAULT="$2"; shift 2;;
--prefix) PREFIX="$2"; shift 2;;
*) echo "Unbekannte Option: $1" >&2; exit 2;;
esac
done
echo "== E2E Roundtrip =="
echo "Vault: $VAULT"
echo "Prefix: $PREFIX"
echo
export COLLECTION_PREFIX="$PREFIX"
echo "-- 1) Qdrant truncate"
python3 -m scripts.reset_qdrant --mode truncate --prefix "$PREFIX"
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
echo
echo "-- 2) Import (Create-Fall)"
python3 -m scripts.import_markdown --vault "$VAULT" --apply --purge-before-upsert --prefix "$PREFIX"
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
echo
echo "-- 3) Import (Idempotenz: keine Änderungen erwartet)"
python3 -m scripts.import_markdown --vault "$VAULT" --apply --prefix "$PREFIX"
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
echo
echo "-- 4) Hash-Reporter"
python3 -m scripts.report_hashes --prefix "$PREFIX" || true
echo
echo "-- 5) Export nach _exportVault"
OUT_DIR="_exportVault"
rm -rf "$OUT_DIR"
python3 -m scripts.export_markdown --out "$OUT_DIR" --prefix "$PREFIX"
echo
echo "-- 6) Vergleich Vault vs. Export (Body-Fokus)"
python3 tests/compare_vaults.py --src "$VAULT" --dst "$OUT_DIR" --focus body
echo
echo "-- 7) Sync-Deletes (simuliert)"
# wähle eine Datei und verschiebe sie temporär weg
TMP_DIR="$(mktemp -d)"
SAMPLE_MD="$(find "$VAULT" -type f -name '*.md' | head -n 1 || true)"
if [[ -n "${SAMPLE_MD:-}" ]]; then
mv "$SAMPLE_MD" "$TMP_DIR/"
echo "Temporär entfernt: $(basename "$SAMPLE_MD")"
echo " 7a) Dry-Run"
python3 -m scripts.import_markdown --vault "$VAULT" --sync-deletes --prefix "$PREFIX"
echo " 7b) Apply"
python3 -m scripts.import_markdown --vault "$VAULT" --sync-deletes --apply --prefix "$PREFIX"
# zurückrollen
mv "$TMP_DIR/$(basename "$SAMPLE_MD")" "$SAMPLE_MD"
rm -rf "$TMP_DIR"
echo " 7c) Re-Import (wieder anlegen)"
python3 -m scripts.import_markdown --vault "$VAULT" --apply --prefix "$PREFIX"
python3 -m scripts.debug_qdrant_state --prefix "$PREFIX"
else
echo "Hinweis: Keine MD-Datei im Vault gefunden – Sync-Deletes übersprungen."
fi
echo
echo "== E2E Roundtrip abgeschlossen =="
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env bash
set -euo pipefail
# tests/test_importer_detect_changes.sh
# Prüft Änderungserkennung:
# - Body-Change -> wird erkannt bei hash-mode=body
# - Frontmatter-Change -> wird NICHT erkannt bei hash-mode=body
# - Frontmatter-Change -> wird erkannt bei hash-mode=frontmatter (bzw. MINDNET_HASH_COMPARE=Frontmatter)
#
# Aufruf:
# ./tests/test_importer_detect_changes.sh [--vault ./test_vault] [--prefix mindnet] [--note-id concept-alpha]
VAULT="./test_vault"
PREFIX="${COLLECTION_PREFIX:-mindnet}"
NOTE_ID="concept-alpha"
while [[ $# -gt 0 ]]; do
case "$1" in
--vault) VAULT="$2"; shift 2;;
--prefix) PREFIX="$2"; shift 2;;
--note-id) NOTE_ID="$2"; shift 2;;
*) echo "Unbekannte Option: $1" >&2; exit 2;;
esac
done
export COLLECTION_PREFIX="$PREFIX"
# Hilfsfunktion: lese Pfad zur Note aus Frontmatter id:
PYFIND=$(cat <<'PY'
import sys, os, glob, yaml
vault = sys.argv[1]
nid = sys.argv[2]
for p in glob.glob(os.path.join(vault, "**", "*.md"), recursive=True):
with open(p, "r", encoding="utf-8") as f:
s = f.read()
if s.startswith("---"):
try:
fm_txt, body = s.split("\n---\n", 1)
fm = yaml.safe_load(fm_txt.strip("- \n")) or {}
except Exception:
continue
if fm.get("id") == nid:
print(p)
break
PY
)
NOTE_PATH=$(python3 -c "$PYFIND" "$VAULT" "$NOTE_ID")
if [[ -z "${NOTE_PATH:-}" ]]; then
echo "Note mit id=$NOTE_ID nicht gefunden im Vault=$VAULT" >&2
exit 3
fi
echo "Note: $NOTE_PATH"
# Backup
ORIG="$(mktemp)"
cp "$NOTE_PATH" "$ORIG"
cleanup() {
cp "$ORIG" "$NOTE_PATH" || true
rm -f "$ORIG"
}
trap cleanup EXIT
# Ausgangszustand importieren (Baseline)
python3 -m scripts.import_markdown --vault "$VAULT" --apply --prefix "$PREFIX" >/dev/null
echo "-- Test A: Body-Änderung soll erkannt werden (hash-mode=body)"
# Body um eine Zeile erweitern
python3 - "$NOTE_PATH" <<'PY'
import sys
p=sys.argv[1]
with open(p,"r+",encoding="utf-8") as f:
s=f.read()
if s.startswith("---"):
fm, body = s.split("\n---\n",1)
body = body + "\n\nBodyTest: Änderung A"
f.seek(0); f.truncate(); f.write(fm+"\n---\n"+body)
PY
OUT=$(python3 -m scripts.import_markdown --vault "$VAULT" --prefix "$PREFIX" | grep "\"note_id\": \"$NOTE_ID\"" || true)
echo "$OUT"
echo "$OUT" | grep -q '"changed": true' || { echo "ERWARTET: changed=true (Body) — FEHLER"; exit 4; }
echo "-- Test B: Frontmatter-Änderung soll NICHT erkannt werden bei mode=body"
# Frontmatter-Feld hinzufügen
python3 - "$NOTE_PATH" <<'PY'
import sys, yaml, io
p=sys.argv[1]
with open(p,"r+",encoding="utf-8") as f:
s=f.read()
if s.startswith("---"):
fm_txt, body = s.split("\n---\n",1)
fm = yaml.safe_load(fm_txt.strip("- \n")) or {}
fm["test_flag"] = True
buf = io.StringIO()
yaml.safe_dump(fm, buf, allow_unicode=True, sort_keys=False)
f.seek(0); f.truncate(); f.write("---\n"+buf.getvalue()+"---\n"+body)
PY
OUT=$(python3 -m scripts.import_markdown --vault "$VAULT" --prefix "$PREFIX" | grep "\"note_id\": \"$NOTE_ID\"" || true)
echo "$OUT"
echo "$OUT" | grep -q '"changed": false' || { echo "ERWARTET: changed=false (FM bei mode=body) — FEHLER"; exit 5; }
echo "-- Test C: Frontmatter-Änderung soll erkannt werden bei mode=frontmatter"
OUT=$(MINDNET_HASH_COMPARE=Frontmatter python3 -m scripts.import_markdown --vault "$VAULT" --prefix "$PREFIX" | grep "\"note_id\": \"$NOTE_ID\"" || true)
echo "$OUT"
echo "$OUT" | grep -q '"changed": true' || { echo "ERWARTET: changed=true (FM bei mode=frontmatter) — FEHLER"; exit 6; }
echo "✔ Änderungserkennung: OK"
+193
View File
@@ -0,0 +1,193 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script: tests/verify_chunks_integrity.py
Version: 1.0.1
Datum: 2025-09-10
Zweck
-----
Verifiziert die Text-Integrität der gespeicherten Chunks:
1) Rekonstruiert den Body aus den Chunks (Sortierung: seq → chunk_index → # in chunk_id).
2) Vergleicht mit dem in Qdrant gespeicherten Note-`fulltext` (falls vorhanden).
3) Optional: Vergleicht zusätzlich mit dem Body der zugehörigen Markdown-Datei im Vault.
Aufrufe
-------
# Nur gegen Qdrant (fulltext vs. Chunks)
python3 tests/verify_chunks_integrity.py --prefix mindnet
# Zusätzlich gegen den Vault abgleichen (Body der .md-Datei)
python3 tests/verify_chunks_integrity.py --prefix mindnet --vault ./test_vault
# Streng + CI-geeignet (Fehlercode bei Abweichungen):
python3 tests/verify_chunks_integrity.py --prefix mindnet --vault ./test_vault --strict --fail-on-mismatch
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from typing import Any, Dict, List, Optional, Tuple
# --- FIX: Projekt-Root in sys.path aufnehmen, damit 'app.*' importierbar ist ---
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
if PROJECT_ROOT not in sys.path:
sys.path.insert(0, PROJECT_ROOT)
import yaml
from qdrant_client.http import models as rest
from app.core.qdrant import QdrantConfig, get_client
# --------------------------- Helpers ---------------------------
def collections(prefix: str) -> Tuple[str, str, str]:
return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges"
def scroll_all(client, collection: str, flt: Optional[rest.Filter] = None, limit: int = 256) -> List[Any]:
out: List[Any] = []
nextp = None
while True:
pts, nextp = client.scroll(
collection_name=collection,
with_payload=True,
with_vectors=False,
limit=limit,
offset=nextp,
scroll_filter=flt,
)
if not pts:
break
out.extend(pts)
if nextp is None:
break
return out
def sort_key_for_chunk_payload(pl: Dict[str, Any]) -> Tuple[int, int, int]:
s = pl.get("seq") or 0
ci = pl.get("chunk_index") or 0
n = 0
cid = pl.get("chunk_id") or ""
if isinstance(cid, str) and "#" in cid:
try:
n = int(cid.rsplit("#", 1)[-1])
except Exception:
n = 0
return (int(s), int(ci), int(n))
def reconstruct_from_chunks(chunks_points: List[Any]) -> Tuple[str, int, int]:
"""Gibt (text, total_chunks, chunks_with_text) zurück."""
chunks_sorted = sorted(chunks_points, key=lambda p: sort_key_for_chunk_payload(p.payload or {}))
texts: List[str] = []
have = 0
for p in chunks_sorted:
pl = p.payload or {}
t = pl.get("text") or pl.get("content") or pl.get("raw") or ""
if isinstance(t, str) and t:
have += 1
texts.append(t)
return ("\n".join(texts).strip(), len(chunks_sorted), have)
def normalize(s: str, strict: bool = False) -> str:
if s is None:
return ""
s = s.replace("\r\n", "\n").replace("\r", "\n")
if strict:
return s
lines = [ln.rstrip() for ln in s.strip().split("\n")]
return "\n".join(lines).strip()
def read_vault_body(vault_root: str, rel_path: str) -> Optional[str]:
if not rel_path:
return None
p = os.path.join(vault_root, rel_path.replace("\\", "/").lstrip("/"))
if not os.path.exists(p):
return None
with open(p, "r", encoding="utf-8") as f:
s = f.read()
if s.startswith("---"):
try:
fm_txt, body = s.split("\n---\n", 1)
except Exception:
return s
else:
body = s
return body
# --------------------------- Main ------------------------------
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
ap.add_argument("--vault", help="Optional: Vault-Wurzelordner für Abgleich gegen .md")
ap.add_argument("--strict", action="store_true", help="Strikter Vergleich (kein Trimmen/Normalisieren)")
ap.add_argument("--fail-on-mismatch", action="store_true", help="Exit 1, wenn ein Mismatch gefunden wurde")
args = ap.parse_args()
cfg = QdrantConfig.from_env()
if args.prefix:
cfg.prefix = args.prefix.strip()
client = get_client(cfg)
notes_col, chunks_col, _ = collections(cfg.prefix)
notes = scroll_all(client, notes_col)
mismatches = 0
total = 0
for n in notes:
total += 1
pl = n.payload or {}
nid = pl.get("note_id")
rel_path = (pl.get("path") or "").replace("\\", "/").lstrip("/")
fulltext = (pl.get("fulltext") or "").strip()
flt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=nid))])
chunks = scroll_all(client, chunks_col, flt)
recon, cnt, have = reconstruct_from_chunks(chunks)
norm_recon = normalize(recon, strict=args.strict)
norm_full = normalize(fulltext, strict=args.strict)
match_full = (bool(norm_full) and (norm_recon == norm_full))
match_vault = None
issues: List[str] = []
if cnt == 0:
issues.append("no_chunks")
if have < cnt:
issues.append(f"missing_text_in_chunks:{cnt-have}/{cnt}")
if norm_full == "":
issues.append("note_fulltext_empty")
if not match_full:
issues.append("reconstructed_vs_fulltext_mismatch")
if args.vault:
body = read_vault_body(args.vault, rel_path) or ""
norm_body = normalize(body, strict=args.strict)
match_vault = (norm_body == norm_recon)
if not match_vault:
issues.append("reconstructed_vs_vault_body_mismatch")
obj = {
"note_id": nid,
"path": rel_path,
"chunks_count": cnt,
"chunks_with_text": have,
"match_fulltext": match_full,
"match_vault": match_vault,
"issues": issues,
}
print(json.dumps(obj, ensure_ascii=False))
if ("reconstructed_vs_fulltext_mismatch" in issues) or ("reconstructed_vs_vault_body_mismatch" in issues):
mismatches += 1
summary = {"summary": "OK" if mismatches == 0 else "DIFFS", "mismatch_count": mismatches, "notes": total}
print(json.dumps(summary, ensure_ascii=False))
if args.fail_on_mismatch and mismatches:
raise SystemExit(1)
if __name__ == "__main__":
main()