Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
58bce0ea19 | ||
|
|
3e888cc669 | ||
|
|
78446c74d3 | ||
|
|
51f08f193c | ||
|
|
4872374a6e | ||
|
|
81c1400ef4 | ||
|
|
298c1fb180 | ||
|
|
f8108bd724 | ||
|
|
65c40b287f | ||
|
|
13e50d4695 | ||
|
|
791cda23af | ||
|
|
466ea64e63 | ||
|
|
f5e6fcc097 | ||
|
|
9ef2e8d397 | ||
|
|
4637091125 | ||
|
|
43a8c80508 | ||
|
|
9362064ad1 | ||
|
|
23952aa3c5 | ||
|
|
79e19c85eb | ||
|
|
62d8299480 | ||
|
|
8f342ea2d3 |
No files matched your search
+185
-91
@@ -2,157 +2,251 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/chunk_payload.py
|
||||
Version: 1.1.3
|
||||
Version: 2.0.0
|
||||
Datum: 2025-09-09
|
||||
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
Erzeugt Qdrant-Payloads für Text-Chunks einer Note. Jeder Chunk enthält
|
||||
den tatsächlichen Text unter dem Schlüssel ``text``. Dadurch kann der
|
||||
Exporter den vollständigen Body verlässlich aus Chunks rekonstruieren,
|
||||
falls ``notes.payload.fulltext`` fehlt.
|
||||
Erzeugt **Chunk-Payloads** für die Qdrant-Collection `<prefix>_chunks` auf Basis der
|
||||
vom Chunker gelieferten `Chunk`-Objekte. Ziel:
|
||||
- *Verlustfreie Rekonstruktion*: Jeder Chunk enthält seinen **Text** (payload["text"]).
|
||||
- *Schnelle Abfragen*: Wichtige Note-Metadaten werden gespiegelt, um Filter ohne Join zu erlauben.
|
||||
- *Graph-Kompatibilität*: Wikilinks und externe Links werden extrahiert; Nachbarschaften werden übernommen.
|
||||
- *Monitoring*: Token- und Längenmetriken sowie Text-Hash erleichtern Audits und Re-Embeddings.
|
||||
|
||||
Wesentliche Features
|
||||
--------------------
|
||||
- Stabile, idempotente Payload-Struktur für Chunks
|
||||
- Persistenter Chunk-Text (``text``)
|
||||
- Extraktion von Wikilinks pro Chunk (``wikilinks`` & ``references``)
|
||||
- Pfadübernahme (relativ zum Vault, wird vom Aufrufer geliefert)
|
||||
- Bereinigung leerer Felder (keine ``None``/leere Collections im Payload)
|
||||
|
||||
Abhängigkeiten
|
||||
Kompatibilität
|
||||
--------------
|
||||
- ``app.core.chunker.Chunk`` und ``assemble_chunks`` (nur für CLI-Demo)
|
||||
- ``app.core.parser.extract_wikilinks`` und ``read_markdown`` (nur CLI-Demo)
|
||||
- **Abwärtskompatibel** zur bisherigen `make_chunk_payloads`-Signatur.
|
||||
- Zusätzliche Felder stören bestehende Upserts nicht (Payload ist schema-flexibel).
|
||||
- Erwartet, dass `Chunk` u. a. die Attribute `id`, `index`, `text`, `char_start`, `char_end`,
|
||||
`section_title`, `section_path`, `neighbors_prev`, `neighbors_next` liefert.
|
||||
|
||||
Beispiele (CLI – Sichtprüfung)
|
||||
------------------------------
|
||||
python3 -m app.core.chunk_payload --from-file ./vault/demo.md --print
|
||||
CLI (Schnelltest)
|
||||
-----------------
|
||||
# Preview aus einer Markdown-Datei
|
||||
python3 -m app.core.chunk_payload --from-file ./test_vault/20_experiences/exp-two.md --vault-root ./test_vault
|
||||
|
||||
# Nur IDs & Tokenmengen
|
||||
python3 -m app.core.chunk_payload --from-file ./test_vault/20_experiences/exp-two.md --vault-root ./test_vault --summary
|
||||
|
||||
Felder (Auszug)
|
||||
---------------
|
||||
id : "<note_id>#cNN"
|
||||
scope : "chunk"
|
||||
note_id : "<note_id>"
|
||||
note_title : str
|
||||
note_type : str
|
||||
note_status : str
|
||||
area, project : optional
|
||||
tags : list[str]
|
||||
note_path : str (relativ, Slashes normalisiert)
|
||||
chunk_index : int
|
||||
section_title : str | None
|
||||
section_path : str | None
|
||||
char_start : int | None
|
||||
char_end : int | None
|
||||
char_len : int
|
||||
token_est : int (≈ len(text)/4)
|
||||
neighbors : {"prev": str|None, "next": str|None}
|
||||
text : str (Chunk-Text, **Pflicht**)
|
||||
text_sha256 : str "sha256:<hex>"
|
||||
lang : optional
|
||||
wikilinks : list[str]
|
||||
external_links : list[{"href": str, "label": str|None}]
|
||||
references : list[{"target_id": str, "kind": "wikilink"}]
|
||||
embed_model : optional (durchreichbar)
|
||||
embed_dim : optional
|
||||
embed_version : optional
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Dict, List, Any
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
# WICHTIG: Keine direkten Selbst- oder Kreuz-Imports, die Zyklen erzeugen!
|
||||
# Für die Kernfunktion reicht extract_wikilinks als Typ-unabhängige Hilfsfunktion.
|
||||
try:
|
||||
from app.core.parser import extract_wikilinks # nur Funktion, erzeugt keine Zyklen
|
||||
# Paket-Import (normaler Betrieb)
|
||||
from app.core.chunker import Chunk
|
||||
from app.core.parser import extract_wikilinks, read_markdown, normalize_frontmatter, validate_required_frontmatter
|
||||
except Exception: # pragma: no cover
|
||||
from .parser import extract_wikilinks # type: ignore
|
||||
|
||||
# Typ-Hinweis für Chunk (zur Laufzeit nicht notwendig)
|
||||
try:
|
||||
from app.core.chunker import Chunk # nur Typ-Annotation / Nutzung im CLI
|
||||
except Exception: # pragma: no cover
|
||||
class Chunk: # type: ignore
|
||||
pass
|
||||
# Relativ (lokale Tests)
|
||||
from .chunker import Chunk # type: ignore
|
||||
from .parser import extract_wikilinks, read_markdown, normalize_frontmatter, validate_required_frontmatter # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utils
|
||||
# Utilities
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _drop_empty(d: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Entfernt leere/None-Felder aus einem Dict (für saubere Payloads)."""
|
||||
return {k: v for k, v in d.items() if v not in (None, [], {}, "")}
|
||||
RE_MD_LINK = re.compile(r"\[([^\]]*)\]\(([^)\s]+)(?:\s+\"([^\"]+)\")?\)")
|
||||
RE_HTTP_SCHEMES = ("http://", "https://", "mailto:", "obsidian://", "tel:")
|
||||
|
||||
def _estimate_tokens(text: str) -> int:
|
||||
"""Grobe Token-Schätzung (≈ 1 Token pro 4 Zeichen)."""
|
||||
return max(0, int(round(len((text or '').strip()) / 4)))
|
||||
|
||||
def _sha256_text(text: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
h.update((text or "").encode("utf-8"))
|
||||
return "sha256:" + h.hexdigest()
|
||||
|
||||
def _normalize_rel_path(path: Optional[str], vault_root: Optional[str]) -> Optional[str]:
|
||||
if not path:
|
||||
return None
|
||||
p = str(path)
|
||||
p = p.replace("\\", "/")
|
||||
if vault_root and os.path.isabs(p):
|
||||
try:
|
||||
p = os.path.relpath(p, vault_root)
|
||||
except Exception:
|
||||
pass
|
||||
p = p.replace("\\", "/").lstrip("/")
|
||||
return p
|
||||
|
||||
def _extract_external_links(text: str) -> List[Dict[str, Optional[str]]]:
|
||||
"""Findet Markdown-Links [label](href "title") mit erlaubten Schemes."""
|
||||
out: List[Dict[str, Optional[str]]] = []
|
||||
if not text:
|
||||
return out
|
||||
for m in RE_MD_LINK.finditer(text):
|
||||
label = (m.group(1) or "").strip() or None
|
||||
href = (m.group(2) or "").strip()
|
||||
title = (m.group(3) or "").strip() or None
|
||||
if any(href.startswith(s) for s in RE_HTTP_SCHEMES):
|
||||
out.append({"href": href, "label": label or title})
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kernfunktion
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_chunk_payloads(note_meta: Dict[str, Any], path: str, chunks: List[Chunk]) -> List[Dict[str, Any]]:
|
||||
def make_chunk_payloads(note_meta: Dict, path: str, chunks: List[Chunk]) -> List[Dict]:
|
||||
"""
|
||||
Baut Payloads für alle ``chunks`` der gegebenen Note.
|
||||
Erzeugt Payload-Dicts für alle Chunks einer Note.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
note_meta : Dict[str, Any]
|
||||
Minimale Metadaten der Note (mind. ``id``, ``title``; optional ``type``,
|
||||
``area``, ``project``, ``tags``, ``lang``).
|
||||
note_meta : Dict
|
||||
Normalisierte Frontmatter der Note (mind.: id, title, type, status, tags, [area, project, lang]).
|
||||
path : str
|
||||
Relativer Pfad der Note innerhalb des Vaults (z. B. "area/topic/file.md").
|
||||
Pfad zur Note (sollte relativ zum Vault sein; wird hier zur Sicherheit normalisiert).
|
||||
chunks : List[Chunk]
|
||||
Liste vorsegmentierter Chunks (vgl. app.core.chunker.Chunk).
|
||||
Vom Chunker erzeugte Chunks.
|
||||
|
||||
Returns
|
||||
-------
|
||||
List[Dict[str, Any]]
|
||||
Payload-Objekte, bereit für Qdrant-Upserts.
|
||||
List[Dict]
|
||||
Liste von Payloads (ein Eintrag pro Chunk).
|
||||
"""
|
||||
res: List[Dict[str, Any]] = []
|
||||
res: List[Dict] = []
|
||||
rel_path = _normalize_rel_path(path, vault_root=None)
|
||||
|
||||
for ch in chunks:
|
||||
# 'text' kann bei leerem Abschnitt None/'' sein → robust behandeln
|
||||
text_val = getattr(ch, "text", "") or ""
|
||||
wikilinks = extract_wikilinks(text_val)
|
||||
payload = {
|
||||
text: str = getattr(ch, "text", "") or ""
|
||||
wikilinks = extract_wikilinks(text)
|
||||
ext_links = _extract_external_links(text)
|
||||
|
||||
payload: Dict = {
|
||||
"id": getattr(ch, "id", None),
|
||||
"scope": "chunk",
|
||||
"note_id": note_meta.get("id"),
|
||||
"note_title": note_meta.get("title"),
|
||||
# gespiegelt für schnelle Filter:
|
||||
"note_type": note_meta.get("type"),
|
||||
"note_status": note_meta.get("status"),
|
||||
"area": note_meta.get("area"),
|
||||
"project": note_meta.get("project"),
|
||||
"tags": note_meta.get("tags"),
|
||||
# Pfad
|
||||
"note_path": rel_path,
|
||||
"path": rel_path, # Back-compat
|
||||
# Reihenfolge & Section
|
||||
"chunk_index": getattr(ch, "index", None),
|
||||
"char_start": getattr(ch, "char_start", None),
|
||||
"char_end": getattr(ch, "char_end", None),
|
||||
"token_count": getattr(ch, "token_count", None),
|
||||
"section_title": getattr(ch, "section_title", None),
|
||||
"section_path": getattr(ch, "section_path", None),
|
||||
"lang": note_meta.get("lang"),
|
||||
"wikilinks": wikilinks,
|
||||
"external_links": [],
|
||||
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
|
||||
# Position
|
||||
"char_start": getattr(ch, "char_start", None),
|
||||
"char_end": getattr(ch, "char_end", None),
|
||||
"char_len": max(0, int(getattr(ch, "char_end", 0) or 0) - int(getattr(ch, "char_start", 0) or 0)) or len(text),
|
||||
# Nachbarn
|
||||
"neighbors": {
|
||||
"prev": getattr(ch, "neighbors_prev", None),
|
||||
"next": getattr(ch, "neighbors_next", None),
|
||||
},
|
||||
"path": path, # vom Aufrufer relativ geliefert
|
||||
"text": text_val, # WICHTIG für Export/Rekonstruktion
|
||||
# Inhalt & Metrik
|
||||
"text": text,
|
||||
"text_sha256": _sha256_text(text),
|
||||
"token_est": _estimate_tokens(text),
|
||||
# Sprache
|
||||
"lang": note_meta.get("lang"),
|
||||
# Links
|
||||
"wikilinks": wikilinks,
|
||||
"external_links": ext_links,
|
||||
"references": [{"target_id": w, "kind": "wikilink"} for w in wikilinks],
|
||||
}
|
||||
res.append(_drop_empty(payload))
|
||||
|
||||
# Entferne Nones/Leeres, aber **text** bleibt (darf leer sein z. B. bei Bild-Only-Chunks)
|
||||
cleaned = {}
|
||||
for k, v in payload.items():
|
||||
if v in (None, [], {}):
|
||||
# immer behalten:
|
||||
if k in ("text", "neighbors"):
|
||||
cleaned[k] = v
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
cleaned[k] = v
|
||||
|
||||
res.append(cleaned)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI – nur zur Sichtprüfung / Debug
|
||||
# CLI zum schnellen Testen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _cli() -> None:
|
||||
# CLI lädt hier *lazy* die schweren Abhängigkeiten,
|
||||
# damit die Modul-Initialisierung zirkularfrei bleibt.
|
||||
try:
|
||||
from app.core.parser import read_markdown
|
||||
from app.core.chunker import assemble_chunks
|
||||
except Exception: # pragma: no cover
|
||||
from .parser import read_markdown # type: ignore
|
||||
from .chunker import assemble_chunks # type: ignore
|
||||
|
||||
ap = argparse.ArgumentParser(description="Vorschau: Chunk-Payloads erzeugen und anzeigen")
|
||||
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zu einer Markdown-Datei")
|
||||
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
|
||||
ap = argparse.ArgumentParser(description="Chunk-Payloads aus einer einzelnen Markdown-Datei erzeugen")
|
||||
ap.add_argument("--from-file", required=True, help="Pfad zur Markdown-Datei")
|
||||
ap.add_argument("--vault-root", default=None, help="Vault-Wurzel (zur Pfad-Relativierung)")
|
||||
ap.add_argument("--summary", action="store_true", help="Nur kurze Übersicht je Chunk ausgeben")
|
||||
args = ap.parse_args()
|
||||
|
||||
note = read_markdown(args.src)
|
||||
note_meta = {
|
||||
"id": note.frontmatter.get("id") or note.frontmatter.get("note_id"),
|
||||
"title": note.frontmatter.get("title"),
|
||||
"type": note.frontmatter.get("type"),
|
||||
"area": note.frontmatter.get("area"),
|
||||
"project": note.frontmatter.get("project"),
|
||||
"tags": note.frontmatter.get("tags", []),
|
||||
"lang": note.frontmatter.get("lang"),
|
||||
}
|
||||
parsed = read_markdown(args.from_file)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
|
||||
chunks = assemble_chunks(note.frontmatter.get("id"), note.body or "", note.frontmatter.get("type", "concept"))
|
||||
# lazy import, um Zyklen zu vermeiden
|
||||
try:
|
||||
from app.core.chunker import assemble_chunks
|
||||
except Exception:
|
||||
from .chunker import assemble_chunks # type: ignore
|
||||
|
||||
# Vault-Root heuristisch relativieren (nur für Demo)
|
||||
vault_root = os.path.dirname(os.path.dirname(args.src)) # heuristisch
|
||||
rel_path = os.path.relpath(args.src, vault_root).replace("\\", "/").lstrip("/")
|
||||
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
|
||||
rel = _normalize_rel_path(parsed.path, args.vault_root)
|
||||
|
||||
payloads = make_chunk_payloads(note_meta, rel_path, chunks)
|
||||
pls = make_chunk_payloads(fm, rel or parsed.path, chunks)
|
||||
|
||||
if args.do_print:
|
||||
print(json.dumps(payloads, ensure_ascii=False, indent=2))
|
||||
if args.summary:
|
||||
out = []
|
||||
for p in pls:
|
||||
out.append({
|
||||
"id": p.get("id"),
|
||||
"chunk_index": p.get("chunk_index"),
|
||||
"token_est": p.get("token_est"),
|
||||
"wikilinks": p.get("wikilinks"),
|
||||
"ext_links": [e.get("href") for e in p.get("external_links", [])],
|
||||
"prev": (p.get("neighbors") or {}).get("prev"),
|
||||
"next": (p.get("neighbors") or {}).get("next"),
|
||||
})
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(json.dumps(pls, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
|
||||
+98
-162
@@ -1,50 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Modul: app/core/note_payload.py
|
||||
Version: 1.5.1
|
||||
Datum: 2025-09-09
|
||||
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
Erzeugt den Qdrant-Payload für **Notes**. Neben stabiler Hash-Bildung zur
|
||||
Idempotenz wird der vollständige Body unter ``fulltext`` persistiert und
|
||||
der Dateipfad relativ zum Vault gespeichert. Das erlaubt eine verlustfreie
|
||||
Rekonstruktion im Export (erst ``fulltext``, sonst Chunks).
|
||||
|
||||
Wichtig
|
||||
-------
|
||||
- **Nur Inhalte** gehen in den Hash: Weder Dateisystem-Zeitstempel (mtime/ctime)
|
||||
noch sonstige FS-Metadaten werden berücksichtigt.
|
||||
- Hash-Quelle: Parser-Body (Default) oder Rohdatei-Body (Frontmatter via Regex entfernt).
|
||||
|
||||
Änderungen in v1.5.1
|
||||
--------------------
|
||||
- Neue Env-Var **MINDNET_HASH_COMPARE** als Synonym zu MINDNET_HASH_MODE.
|
||||
- Akzeptiert komfortable Werte (case-insensitive): **Body**, **Frontmatter**, **Full**.
|
||||
*Full* wird intern zu ``body+frontmatter`` normalisiert.
|
||||
- CLI akzeptiert weiterhin ``--hash-mode``; Importer reicht diese Einstellung durch.
|
||||
|
||||
Hash-Steuerung
|
||||
--------------
|
||||
- Modus (welche Teile in den Hash einfließen):
|
||||
* ``body`` (Default)
|
||||
* ``frontmatter``
|
||||
* ``body+frontmatter`` (Synonym CLI/ENV: ``full``)
|
||||
Quelle:
|
||||
* Funktionsparameter ``hash_mode`` (höchste Priorität)
|
||||
* Env ``MINDNET_HASH_MODE`` oder **``MINDNET_HASH_COMPARE``** (Fallback)
|
||||
Normalisierung:
|
||||
* ``canonical`` (Default) | ``none`` — via Param ``hash_normalize`` oder Env ``MINDNET_HASH_NORMALIZE``
|
||||
Quelle des Body-Textes:
|
||||
* ``parsed`` (Default) | ``raw`` — via Param ``hash_source`` oder Env ``MINDNET_HASH_SOURCE``
|
||||
|
||||
CLI (Sichtprüfung)
|
||||
------------------
|
||||
python3 -m app.core.note_payload --from-file ./vault/demo.md --vault-root ./vault --print --hash-source raw
|
||||
"""
|
||||
# Modul: app/core/note_payload.py
|
||||
# Version: 1.7.0
|
||||
# Datum: 2025-09-09
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
@@ -56,48 +17,34 @@ try:
|
||||
except Exception: # pragma: no cover
|
||||
from .parser import read_markdown, extract_wikilinks, FRONTMATTER_RE # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _canon_frontmatter(fm: Dict[str, Any]) -> str:
|
||||
"""Kanonische, stabile JSON-Serialisierung der Frontmatter für Hashbildung."""
|
||||
return json.dumps(fm or {}, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
def _normalize_body(body: str, mode: str) -> str:
|
||||
"""Normalisiert den Body für reproduzierbare Hashes (oder nicht)."""
|
||||
if mode == "none":
|
||||
return body if body is not None else ""
|
||||
# canonical: \r\n→\n, trailing spaces entfernen
|
||||
text = (body or "").replace("\r\n", "\n").replace("\r", "\n")
|
||||
text = "\n".join(line.rstrip() for line in text.split("\n"))
|
||||
return text
|
||||
|
||||
def _resolve_hash_mode(explicit: Optional[str]) -> str:
|
||||
"""
|
||||
Normalisiert den Hash-Modus auf einen der Werte:
|
||||
'body' | 'frontmatter' | 'body+frontmatter'
|
||||
Akzeptiert auch 'full' als Alias für 'body+frontmatter'.
|
||||
Beachtet zusätzlich die Env-Variablen MINDNET_HASH_MODE und MINDNET_HASH_COMPARE.
|
||||
"""
|
||||
if explicit:
|
||||
val = explicit.strip().lower()
|
||||
else:
|
||||
val = (os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
|
||||
val = (os.environ.get("MINDNET_HASH_MODE")
|
||||
or os.environ.get("MINDNET_HASH_COMPARE")
|
||||
or "body").strip().lower()
|
||||
if val in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
||||
return "body+frontmatter"
|
||||
return "full"
|
||||
if val in ("frontmatter", "fm"):
|
||||
return "frontmatter"
|
||||
# default & fallbacks
|
||||
return "body"
|
||||
|
||||
def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, Any]]:
|
||||
"""Liest die Rohdatei und extrahiert Body & Frontmatter ohne Parser-Logik.
|
||||
|
||||
Rückgabe:
|
||||
(body_text, frontmatter_dict)
|
||||
"""
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return "", {}
|
||||
try:
|
||||
@@ -105,7 +52,6 @@ def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, A
|
||||
raw = f.read()
|
||||
except Exception:
|
||||
return "", {}
|
||||
# Frontmatter per Regex entfernen
|
||||
m = FRONTMATTER_RE.match(raw)
|
||||
fm = {}
|
||||
if m:
|
||||
@@ -120,46 +66,20 @@ def _read_raw_body_from_file(file_path: Optional[str]) -> Tuple[str, Dict[str, A
|
||||
body = raw
|
||||
return body, fm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hashing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compute_hash(
|
||||
*,
|
||||
body: Optional[str],
|
||||
frontmatter: Optional[Dict[str, Any]],
|
||||
mode: Optional[str] = None,
|
||||
normalize: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Berechnet einen Hex-Hash gemäß ``mode`` und ``normalize``.
|
||||
|
||||
mode:
|
||||
- "body" (Default)
|
||||
- "frontmatter"
|
||||
- "body+frontmatter"
|
||||
normalize:
|
||||
- "canonical" (Default)
|
||||
- "none"
|
||||
"""
|
||||
mode = _resolve_hash_mode(mode)
|
||||
normalize = (normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower()
|
||||
|
||||
body = _normalize_body(body or "", normalize)
|
||||
fm_s = _canon_frontmatter(frontmatter or {})
|
||||
|
||||
def _sha256(s: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
if mode == "frontmatter":
|
||||
h.update(fm_s.encode("utf-8"))
|
||||
elif mode == "body+frontmatter":
|
||||
h.update(body.encode("utf-8"))
|
||||
h.update(b"\n--FM--\n")
|
||||
h.update(fm_s.encode("utf-8"))
|
||||
else: # body
|
||||
h.update(body.encode("utf-8"))
|
||||
h.update(s.encode("utf-8"))
|
||||
return h.hexdigest()
|
||||
|
||||
def _hash_for(mode: str, *, body: str, fm: Dict[str, Any], normalize: str) -> str:
|
||||
body_n = _normalize_body(body or "", normalize)
|
||||
fm_s = _canon_frontmatter(fm or {})
|
||||
if mode == "frontmatter":
|
||||
return _sha256(fm_s)
|
||||
if mode == "full":
|
||||
return _sha256(body_n + "\n--FM--\n" + fm_s)
|
||||
# default: body
|
||||
return _sha256(body_n)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kernfunktion
|
||||
@@ -175,66 +95,80 @@ def make_note_payload(
|
||||
file_path: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Erzeugt den Payload für eine geparste Note.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
parsed : Any
|
||||
Objekt mit Attributen/Keys ``frontmatter``, ``body``, ``path``.
|
||||
vault_root : Optional[str]
|
||||
Vault-Wurzel (für Pfad-Relativierung). Wenn ``None``, wird ``path`` unverändert übernommen.
|
||||
hash_mode : Optional[str]
|
||||
"body" | "frontmatter" | "body+frontmatter" | "full" (Alias; überschreibt ENV).
|
||||
hash_normalize : Optional[str]
|
||||
"canonical" | "none" (überschreibt ENV).
|
||||
hash_source : Optional[str]
|
||||
"parsed" (Default) oder "raw". Wenn "raw", wird der Body aus der Rohdatei gelesen.
|
||||
file_path : Optional[str]
|
||||
Pfad zur Markdown-Datei, erforderlich für ``hash_source=raw``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Dict[str, Any]
|
||||
Qdrant-Payload für die Notes-Collection.
|
||||
Liefert den Note-Payload inkl. Mehrfach-Hashes.
|
||||
- Es werden IMMER die drei Hashes für (body|frontmatter|full) unter
|
||||
'parsed:canonical' erzeugt (Schlüssel: z. B. 'body:parsed:canonical').
|
||||
- Zusätzlich werden – falls die aktuelle Konfig (source/normalize) davon
|
||||
abweicht – die drei Hashes unter den entsprechenden Schlüsseln erzeugt,
|
||||
z. B. 'frontmatter:raw:none'.
|
||||
- 'hash_fulltext' und 'hash_signature' repräsentieren den *aktuellen* Modus.
|
||||
"""
|
||||
# "Duck typing": dict oder Objekt akzeptieren
|
||||
# dict oder Objekt akzeptieren
|
||||
if isinstance(parsed, dict):
|
||||
fm = parsed.get("frontmatter") or {}
|
||||
body = parsed.get("body") or ""
|
||||
body_parsed = parsed.get("body") or ""
|
||||
path = parsed.get("path") or ""
|
||||
else:
|
||||
fm = getattr(parsed, "frontmatter", {}) or {}
|
||||
body = getattr(parsed, "body", "") or ""
|
||||
body_parsed = getattr(parsed, "body", "") or ""
|
||||
path = getattr(parsed, "path", "") or ""
|
||||
|
||||
# Hash-Quelle bestimmen
|
||||
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower()
|
||||
# Zielpfad relativieren
|
||||
rel_path = path
|
||||
try:
|
||||
if vault_root:
|
||||
rel = os.path.relpath(path, vault_root)
|
||||
rel = rel.replace("\\", "/").lstrip("/")
|
||||
rel_path = rel
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Konfiguration auflösen
|
||||
mode_resolved = _resolve_hash_mode(hash_mode) # body|frontmatter|full
|
||||
src = (hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed")).strip().lower() # parsed|raw
|
||||
norm = (hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical")).strip().lower() # canonical|none
|
||||
|
||||
# Body-Quelle laden
|
||||
raw_body, raw_fm = ("", {})
|
||||
if src == "raw":
|
||||
raw_body, raw_fm = _read_raw_body_from_file(file_path or path)
|
||||
# Roh-FM ergänzen (nicht überschreiben)
|
||||
if isinstance(raw_fm, dict) and raw_fm:
|
||||
merged_fm = dict(fm)
|
||||
for k, v in raw_fm.items():
|
||||
merged_fm.setdefault(k, v)
|
||||
fm = merged_fm
|
||||
body_for_hash = raw_body
|
||||
else:
|
||||
body_for_hash = body_parsed
|
||||
|
||||
# Hash gemäß Modus/Quelle bilden
|
||||
body_for_hash = raw_body if src == "raw" else body
|
||||
hash_fulltext = compute_hash(body=body_for_hash, frontmatter=fm, mode=hash_mode, normalize=hash_normalize)
|
||||
# --- 1) Standard-Tripel (parsed:canonical) immer erzeugen ---
|
||||
std_src = "parsed"
|
||||
std_norm = "canonical"
|
||||
std_hashes: Dict[str, str] = {}
|
||||
for m in ("body", "frontmatter", "full"):
|
||||
std_hashes[f"{m}:{std_src}:{std_norm}"] = _hash_for(
|
||||
m, body=body_parsed, fm=fm, normalize=std_norm
|
||||
)
|
||||
|
||||
# Pfad relativieren
|
||||
rel_path = path
|
||||
try:
|
||||
if vault_root:
|
||||
rel = os.path.relpath(path, vault_root)
|
||||
rel = rel.replace("\\", "/").lstrip("/") # normalisieren
|
||||
rel_path = rel
|
||||
except Exception:
|
||||
pass # Pfad ist nicht kritisch für Hash/ID
|
||||
# Convenience-Felder (für Tools)
|
||||
hash_body = std_hashes["body:parsed:canonical"]
|
||||
hash_frontmatter = std_hashes["frontmatter:parsed:canonical"]
|
||||
hash_full = std_hashes["full:parsed:canonical"]
|
||||
|
||||
# Note-Level-Wikilinks (Fallback, wenn Chunks nicht geliefert werden)
|
||||
note_level_refs = list(dict.fromkeys(extract_wikilinks(body))) if body else []
|
||||
# --- 2) Hashes für die *aktuelle* Konfiguration (falls abweichend) ---
|
||||
cur_hashes: Dict[str, str] = {}
|
||||
if not (src == std_src and norm == std_norm):
|
||||
for m in ("body", "frontmatter", "full"):
|
||||
cur_hashes[f"{m}:{src}:{norm}"] = _hash_for(
|
||||
m, body=body_for_hash, fm=fm, normalize=norm
|
||||
)
|
||||
|
||||
# --- 3) Aktueller Modus für Backwards-Compat Felder ---
|
||||
current_hash = _hash_for(mode_resolved, body=body_for_hash, fm=fm, normalize=norm)
|
||||
hash_signature = f"{mode_resolved}:{src}:{norm}:{current_hash}"
|
||||
|
||||
# Wikilinks (Note-Ebene)
|
||||
refs = list(dict.fromkeys(extract_wikilinks(body_parsed))) if body_parsed else []
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"note_id": fm.get("id") or fm.get("note_id"),
|
||||
@@ -245,49 +179,51 @@ def make_note_payload(
|
||||
"updated": fm.get("updated"),
|
||||
"path": rel_path or fm.get("path"),
|
||||
"tags": fm.get("tags"),
|
||||
"hash_fulltext": hash_fulltext,
|
||||
# Volltext persistieren (verlustfreie Rekonstruktion) – aus PARSED-Body
|
||||
"fulltext": body,
|
||||
# Fallback-Refs auf Note-Ebene
|
||||
"references": note_level_refs,
|
||||
# Volltext für verlustfreien Export
|
||||
"fulltext": body_parsed,
|
||||
# Backwards-Compat:
|
||||
"hash_fulltext": current_hash,
|
||||
"hash_signature": hash_signature,
|
||||
# Option C: Mehrfach-Hashes
|
||||
"hashes": {**std_hashes, **cur_hashes},
|
||||
"hash_body": hash_body,
|
||||
"hash_frontmatter": hash_frontmatter,
|
||||
"hash_full": hash_full,
|
||||
# Fallback-Refs
|
||||
"references": refs,
|
||||
}
|
||||
|
||||
for k in ("area", "project", "source", "lang", "slug", "aliases"):
|
||||
if k in fm:
|
||||
payload[k] = fm[k]
|
||||
|
||||
# Optional: Roh-Body-Hash zusätzlich persistieren
|
||||
if os.environ.get("MINDNET_HASH_STORE_RAW", "false").strip().lower() == "true" and src == "raw":
|
||||
try:
|
||||
payload["hash_raw_body"] = compute_hash(body=raw_body, frontmatter=fm, mode="body", normalize="none")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI – Sichtprüfung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _cli() -> None:
|
||||
ap = argparse.ArgumentParser(description="Note-Payload aus Markdown erzeugen und anzeigen")
|
||||
ap.add_argument("--from-file", dest="src", required=True, help="Pfad zur Markdown-Datei")
|
||||
ap.add_argument("--vault-root", dest="vault_root", default=None, help="Vault-Wurzel zur Pfad-Relativierung")
|
||||
ap.add_argument("--print", dest="do_print", action="store_true", help="Payload auf stdout ausgeben")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter", "full"], default=None)
|
||||
ap.add_argument("--from-file", dest="src", required=True)
|
||||
ap.add_argument("--vault-root", dest="vault_root", default=None)
|
||||
ap.add_argument("--print", dest="do_print", action="store_true")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None)
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
parsed = read_markdown(args.src)
|
||||
payload = make_note_payload(parsed, vault_root=args.vault_root,
|
||||
hash_mode=args.hash_mode, hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source, file_path=args.src)
|
||||
|
||||
payload = make_note_payload(
|
||||
parsed,
|
||||
vault_root=args.vault_root,
|
||||
hash_mode=args.hash_mode,
|
||||
hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source,
|
||||
file_path=args.src,
|
||||
)
|
||||
if args.do_print:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
_cli()
|
||||
+79
-27
@@ -2,37 +2,89 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "mindnet_chunk",
|
||||
"type": "object",
|
||||
"required": ["id", "note_id", "chunk_index", "type", "path", "token_count", "lang"],
|
||||
"description": "Chunk-Payload (Qdrant). Kompatibel mit Alt-Feldern und neuen Feldern für Export/Roundtrip.",
|
||||
"required": ["id", "note_id", "chunk_index", "path"],
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"note_id": {"type": "string"},
|
||||
"note_title": {"type": "string"},
|
||||
"chunk_index": {"type": "integer"},
|
||||
"char_start": {"type": "integer"},
|
||||
"char_end": {"type": "integer"},
|
||||
"token_count": {"type": "integer"},
|
||||
"type": {"type": "string"},
|
||||
"area": {"type": "string"},
|
||||
"project": {"type": "string"},
|
||||
"tags": {"type": "array", "items": {"type": "string"}},
|
||||
"section_title": {"type": "string"},
|
||||
"section_path": {"type": "string"},
|
||||
"lang": {"type": "string"},
|
||||
"wikilinks": {"type": "array", "items": {"type": "string"}},
|
||||
"external_links": {"type": "array", "items": {"type": "string"}},
|
||||
"id": { "type": "string" },
|
||||
"scope": { "type": "string", "enum": ["chunk"] },
|
||||
"note_id": { "type": "string" },
|
||||
"note_title": { "type": "string" },
|
||||
"note_type": { "type": "string" },
|
||||
"note_status": { "type": "string" },
|
||||
"type": { "type": "string", "description": "Legacy: früherer Chunk-Typ; kann dem Note-Typ entsprechen" },
|
||||
"area": { "type": "string" },
|
||||
"project": { "type": "string" },
|
||||
"tags": { "type": "array", "items": { "type": "string" } },
|
||||
|
||||
"note_path": { "type": "string" },
|
||||
"path": { "type": "string" },
|
||||
|
||||
"chunk_index": { "type": "integer" },
|
||||
"section_title":{ "type": ["string","null"] },
|
||||
"section_path": { "type": ["string","null"] },
|
||||
|
||||
"char_start": { "type": ["integer","null"] },
|
||||
"char_end": { "type": ["integer","null"] },
|
||||
"char_len": { "type": "integer" },
|
||||
|
||||
"token_count": { "type": "integer", "description": "Legacy: frühere Token-Zahl" },
|
||||
"token_est": { "type": "integer", "description": "Neue grobe Token-Schätzung (≈ len(text)/4)" },
|
||||
|
||||
"neighbors": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prev": { "type": ["string","null"] },
|
||||
"next": { "type": ["string","null"] }
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
|
||||
"text": { "type": "string" },
|
||||
"text_sha256": { "type": "string", "pattern": "^sha256:[0-9a-fA-F]{64}$" },
|
||||
"lang": { "type": "string" },
|
||||
|
||||
"wikilinks": { "type": "array", "items": { "type": "string" } },
|
||||
"external_links": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{ "type": "string" },
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"href": { "type": "string" },
|
||||
"label": { "type": ["string","null"] }
|
||||
},
|
||||
"required": ["href"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"references": {
|
||||
"type": "array",
|
||||
"items": {"type": "object", "properties": {
|
||||
"target_id": {"type": "string"},
|
||||
"kind": {"type": "string"}
|
||||
}, "required": ["target_id", "kind"]}
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"target_id": { "type": "string" },
|
||||
"kind": { "type": "string" }
|
||||
},
|
||||
"required": ["target_id","kind"],
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"neighbors": {"type": "object", "properties": {
|
||||
"prev": {"type": ["string","null"]},
|
||||
"next": {"type": ["string","null"]}
|
||||
}},
|
||||
"path": {"type": "string"},
|
||||
"created_at": {"type": "string"}
|
||||
|
||||
"embed_model": { "type": "string" },
|
||||
"embed_dim": { "type": "integer" },
|
||||
"embed_version": { "type": "integer" },
|
||||
|
||||
"created_at": { "type": "string" }
|
||||
},
|
||||
|
||||
"allOf": [
|
||||
{ "anyOf": [ { "required": ["token_count"] }, { "required": ["token_est"] } ] },
|
||||
{ "anyOf": [ { "required": ["type"] }, { "required": ["note_type"] } ] }
|
||||
],
|
||||
|
||||
"additionalProperties": true
|
||||
}
|
||||
@@ -2,15 +2,30 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "mindnet_edge",
|
||||
"type": "object",
|
||||
"required": ["src_id", "dst_id", "edge_type", "scope"],
|
||||
"description": "Edge-Payload (Qdrant). Unterstützt Legacy (edge_type/src_id/dst_id) und neues Schema (kind/source_id/target_id/note_id/status).",
|
||||
|
||||
"properties": {
|
||||
"src_id": {"type": "string"},
|
||||
"dst_id": {"type": "string"},
|
||||
"edge_type": {"type": "string"},
|
||||
"scope": {"type": "string", "enum": ["note","chunk"]},
|
||||
"weight": {"type": "number"},
|
||||
"meta": {"type": "object"},
|
||||
"created_at": {"type": "string"}
|
||||
"scope": { "type": "string", "enum": ["note","chunk"] },
|
||||
|
||||
"edge_type": { "type": "string", "description": "Legacy: z. B. references/backlink/belongs_to/prev/next" },
|
||||
"src_id": { "type": "string", "description": "Legacy: source_id" },
|
||||
"dst_id": { "type": "string", "description": "Legacy: target_id" },
|
||||
|
||||
"kind": { "type": "string", "description": "Neu: z. B. references/backlink/belongs_to/prev/next" },
|
||||
"source_id": { "type": "string" },
|
||||
"target_id": { "type": "string" },
|
||||
"note_id": { "type": "string", "description": "Owner-Note für diesen Edge (Filter/Purge)" },
|
||||
"status": { "type": "string", "description": "optional, z. B. 'unresolved'" },
|
||||
|
||||
"weight": { "type": "number" },
|
||||
"meta": { "type": "object" },
|
||||
"created_at":{ "type": "string" }
|
||||
},
|
||||
|
||||
"anyOf": [
|
||||
{ "required": ["src_id", "dst_id", "edge_type", "scope"] },
|
||||
{ "required": ["source_id", "target_id", "kind", "scope"] }
|
||||
],
|
||||
|
||||
"additionalProperties": true
|
||||
}
|
||||
+38
-40
@@ -1,47 +1,45 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "mindnet_note",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "mindnet note payload",
|
||||
"type": "object",
|
||||
"required": ["note_id", "title", "type", "status", "created", "path"],
|
||||
"properties": {
|
||||
"note_id": { "type": "string" },
|
||||
"title": { "type": "string" },
|
||||
"type": { "type": "string" },
|
||||
"status": { "type": "string" },
|
||||
"created": { "type": "string" },
|
||||
"updated": { "type": "string" },
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"area": { "type": "string" },
|
||||
"project": { "type": "string" },
|
||||
"priority": { "type": "number" },
|
||||
"effort_min": { "type": "number" },
|
||||
"due": { "type": "string" },
|
||||
"people": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"aliases": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"references": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"depends_on": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"assigned_to": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"lang": { "type": "string" },
|
||||
"path": { "type": "string" },
|
||||
"hash_fulltext": { "type": "string" }
|
||||
"title": { "type": ["string","null"] },
|
||||
"type": { "type": ["string","null"] },
|
||||
"status": { "type": ["string","null"] },
|
||||
"created": { "type": ["string","null"] },
|
||||
"updated": { "type": ["string","null"] },
|
||||
"path": { "type": ["string","null"] },
|
||||
"tags": { "type": ["array","null"], "items": { "type": "string" } },
|
||||
"area": { "type": ["string","null"] },
|
||||
"project": { "type": ["string","null"] },
|
||||
"source": { "type": ["string","null"] },
|
||||
"lang": { "type": ["string","null"] },
|
||||
"slug": { "type": ["string","null"] },
|
||||
"aliases": { "type": ["array","null"], "items": { "type": "string" } },
|
||||
|
||||
"fulltext": { "type": ["string","null"] },
|
||||
"references": { "type": ["array","null"], "items": { "type": "string" } },
|
||||
|
||||
"hash_fulltext": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
|
||||
"hash_signature": { "type": ["string","null"] },
|
||||
|
||||
"hash_body": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
|
||||
"hash_frontmatter": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
|
||||
"hash_full": { "type": ["string","null"], "pattern": "^[a-f0-9]{64}$" },
|
||||
|
||||
"hashes": {
|
||||
"type": ["object","null"],
|
||||
"description": "Mapping: <mode>:<source>:<normalize> -> sha256 hex",
|
||||
"patternProperties": {
|
||||
"^(body|frontmatter|full):(parsed|raw):(canonical|none)$": {
|
||||
"type": "string",
|
||||
"pattern": "^[a-f0-9]{64}$"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": ["note_id"],
|
||||
"additionalProperties": true
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/debug_note_payload.py
|
||||
Zeigt, welche app.core.note_payload geladen wird, ruft make_note_payload auf
|
||||
und druckt Typ/Preview der Rückgabe.
|
||||
|
||||
Aufruf:
|
||||
python3 -m scripts.debug_note_payload --file ./test_vault/40_concepts/concept-alpha.md --vault-root ./test_vault
|
||||
"""
|
||||
import argparse, json, os, pprint
|
||||
from app.core import note_payload as np
|
||||
from app.core.parser import read_markdown
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--file", required=True)
|
||||
ap.add_argument("--vault-root", default=None)
|
||||
ap.add_argument("--hash-mode", default=None)
|
||||
ap.add_argument("--hash-normalize", default=None)
|
||||
ap.add_argument("--hash-source", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
print("module_path:", getattr(np, "__file__", "<unknown>"))
|
||||
parsed = read_markdown(args.file)
|
||||
res = np.make_note_payload(parsed, vault_root=args.vault_root,
|
||||
hash_mode=args.hash_mode,
|
||||
hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source,
|
||||
file_path=args.file)
|
||||
print("return_type:", type(res).__name__)
|
||||
if isinstance(res, dict):
|
||||
print("keys:", list(res.keys()))
|
||||
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
preview = repr(res)
|
||||
if len(preview) > 600: preview = preview[:600] + "…"
|
||||
print("preview:", preview)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+192
-93
@@ -2,42 +2,60 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Script: scripts/import_markdown.py — Markdown → Qdrant (Notes, Chunks, Edges)
|
||||
Version: 3.4.1
|
||||
Version: 3.6.2
|
||||
Datum: 2025-09-09
|
||||
|
||||
Kurzbeschreibung
|
||||
----------------
|
||||
- Liest Markdown-Dateien ein, erzeugt Notes/Chunks/Edges **idempotent**.
|
||||
- Change-Detection (nur **Inhalte**, keine FS-Zeitstempel) konfigurierbar:
|
||||
* ``--hash-mode``: body | frontmatter | body+frontmatter | full (Alias)
|
||||
- Env: ``MINDNET_HASH_MODE`` **oder** ``MINDNET_HASH_COMPARE`` (Body/Frontmatter/Full)
|
||||
* ``--hash-normalize``: canonical | none (Default: canonical)
|
||||
* ``--hash-source``: parsed (Default) | raw
|
||||
- "raw" hasht den **ungeparsten** Body aus der Datei (Frontmatter vorher entfernt).
|
||||
- Optionales Diff-Logging: ``--debug-hash-diff`` zeigt bei Bedarf einen kompakten Diff.
|
||||
- Liest Markdown-Dateien ein und erzeugt Notes/Chunks/Edges **idempotent**.
|
||||
- Robuste Änderungserkennung: Mehrfach-Hashes werden parallel in der Note gespeichert
|
||||
(Option C). Vergleich erfolgt **modusgenau** anhand von `hashes[<mode>:<source>:<normalize>]`.
|
||||
Ein Wechsel des Vergleichsmodus führt so **nicht** zu Massenänderungen.
|
||||
- **Fix (v3.6.2):** Bei **erstem Import** (kein Alt-Payload) wird die Note als **geändert**
|
||||
behandelt → Create (Notes/Chunks/Edges) findet zuverlässig statt.
|
||||
- Baseline-Modus: Mit `--baseline-modes` werden **fehlende** Hash-Varianten
|
||||
im Feld `hashes` „still“ nachgetragen (Upsert NUR Notes; Legacy-Hashfelder bleiben unangetastet).
|
||||
|
||||
ENV / Qdrant
|
||||
------------
|
||||
Konfiguration Hash/Compare
|
||||
--------------------------
|
||||
- Vergleichsmodus: `--hash-mode body|frontmatter|full`
|
||||
- oder ENV: `MINDNET_HASH_MODE` / `MINDNET_HASH_COMPARE` (Body|Frontmatter|Full)
|
||||
- Quelle: `--hash-source parsed|raw` (ENV: `MINDNET_HASH_SOURCE`, Default parsed)
|
||||
- Normalisierung: `--hash-normalize canonical|none` (ENV: `MINDNET_HASH_NORMALIZE`, Default canonical)
|
||||
- Optional: `--compare-text` (oder `MINDNET_COMPARE_TEXT=true`) vergleicht zusätzlich den parsed Body-Text direkt.
|
||||
|
||||
Weitere ENV / Qdrant
|
||||
--------------------
|
||||
- QDRANT_URL | QDRANT_HOST/QDRANT_PORT | QDRANT_API_KEY
|
||||
- COLLECTION_PREFIX (Default: mindnet)
|
||||
- VECTOR_DIM (Default: 384)
|
||||
- MINDNET_NOTE_SCOPE_REFS: true|false (Default: false)
|
||||
|
||||
Aufruf
|
||||
------
|
||||
Aufruf-Beispiele
|
||||
----------------
|
||||
# Standard (Body, parsed, canonical)
|
||||
python3 -m scripts.import_markdown --vault ./vault
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --hash-source raw --hash-normalize none
|
||||
MINDNET_HASH_COMPARE=Full python3 -m scripts.import_markdown --vault ./vault --apply
|
||||
"""
|
||||
|
||||
# Baseline (füllt hashes für parsed:canonical Tripel „still“ auf)
|
||||
MINDNET_HASH_SOURCE=parsed MINDNET_HASH_NORMALIZE=canonical \
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --baseline-modes
|
||||
|
||||
# Frontmatter-Vergleich ohne Massenänderungen (nach Baseline)
|
||||
MINDNET_HASH_COMPARE=Frontmatter \
|
||||
python3 -m scripts.import_markdown --vault ./vault
|
||||
|
||||
# Sehr sensibel (raw + none) und direkten Textvergleich
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --hash-source raw --hash-normalize none --compare-text
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from collections.abc import Mapping
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from qdrant_client.http import models as rest
|
||||
@@ -105,11 +123,29 @@ def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[D
|
||||
return points[0].payload or {}
|
||||
|
||||
def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||
"""
|
||||
Löscht alle Chunks & Edges zu einer Note mittels Filter-Selector (kompatibel mit aktuellen Qdrant-Clients).
|
||||
"""
|
||||
_, chunks_col, edges_col = collections(prefix)
|
||||
f_chunks = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
client.delete(collection_name=chunks_col, points_selector=f_chunks, wait=True)
|
||||
f_edges = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
client.delete(collection_name=edges_col, points_selector=f_edges, wait=True)
|
||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
|
||||
try:
|
||||
client.delete(
|
||||
collection_name=chunks_col,
|
||||
points_selector=rest.FilterSelector(filter=filt),
|
||||
wait=True
|
||||
)
|
||||
except Exception as e:
|
||||
print(json.dumps({"note_id": note_id, "warn": f"delete chunks via filter failed: {e}"}))
|
||||
|
||||
try:
|
||||
client.delete(
|
||||
collection_name=edges_col,
|
||||
points_selector=rest.FilterSelector(filter=filt),
|
||||
wait=True
|
||||
)
|
||||
except Exception as e:
|
||||
print(json.dumps({"note_id": note_id, "warn": f"delete edges via filter failed: {e}"}))
|
||||
|
||||
def _normalize_rel_path(abs_path: str, vault_root: str) -> str:
|
||||
try:
|
||||
@@ -118,6 +154,17 @@ def _normalize_rel_path(abs_path: str, vault_root: str) -> str:
|
||||
rel = abs_path
|
||||
return rel.replace("\\", "/").lstrip("/")
|
||||
|
||||
def _resolve_mode(val: Optional[str]) -> str:
|
||||
v = (val or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE") or "body").strip().lower()
|
||||
if v in ("full", "fulltext", "body+frontmatter", "bodyplusfrontmatter"):
|
||||
return "full"
|
||||
if v in ("frontmatter", "fm"):
|
||||
return "frontmatter"
|
||||
return "body"
|
||||
|
||||
def _env(key: str, default: str) -> str:
|
||||
return (os.environ.get(key) or default).strip().lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Main
|
||||
@@ -134,20 +181,27 @@ def main() -> None:
|
||||
ap.add_argument("--embed-note", action="store_true", help="Optional: Note-Volltext einbetten")
|
||||
ap.add_argument("--force-replace", action="store_true",
|
||||
help="Änderungserkennung ignorieren und immer upserten (+ optional Purge)")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "body+frontmatter", "full"], default=None,
|
||||
help="Vergleichsmodus: Body | Frontmatter | body+frontmatter (Alias: full)")
|
||||
ap.add_argument("--hash-mode", choices=["body", "frontmatter", "full"], default=None,
|
||||
help="Vergleichsmodus (Body | Frontmatter | Full)")
|
||||
ap.add_argument("--hash-normalize", choices=["canonical", "none"], default=None)
|
||||
ap.add_argument("--hash-source", choices=["parsed", "raw"], default=None,
|
||||
help="Quelle für die Hash-Berechnung (Default: parsed)")
|
||||
ap.add_argument("--note-scope-refs", action="store_true",
|
||||
help="(Optional) erzeugt zusätzlich references:note (Default: aus)")
|
||||
ap.add_argument("--debug-hash-diff", action="store_true",
|
||||
help="Zeigt bei Bedarf einen kurzen Diff zwischen altem und neuem Body")
|
||||
help="Zeigt einen kurzen Diff zwischen altem und neuem Body")
|
||||
ap.add_argument("--compare-text", action="store_true",
|
||||
help="Parsed fulltext zusätzlich direkt vergleichen (über Hash hinaus)")
|
||||
ap.add_argument("--baseline-modes", action="store_true",
|
||||
help="Fehlende Hash-Varianten im Feld 'hashes' still nachtragen (Upsert NUR Notes)")
|
||||
args = ap.parse_args()
|
||||
|
||||
note_scope_refs_env = (os.environ.get("MINDNET_NOTE_SCOPE_REFS", "false").strip().lower() == "true")
|
||||
note_scope_refs = args.note_sCope_refs if hasattr(args, "note_sCope_refs") else args.note_scope_refs # defensive
|
||||
note_scope_refs = note_scope_refs or note_scope_refs_env
|
||||
mode = _resolve_mode(args.hash_mode) # body|frontmatter|full
|
||||
src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed") # parsed|raw
|
||||
norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical") # canonical|none
|
||||
note_scope_refs_env = (_env("MINDNET_NOTE_SCOPE_REFS", "false") == "true")
|
||||
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
||||
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
||||
|
||||
cfg = QdrantConfig.from_env()
|
||||
client = get_client(cfg)
|
||||
@@ -160,12 +214,22 @@ def main() -> None:
|
||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
key_current = f"{mode}:{src}:{norm}"
|
||||
|
||||
processed = 0
|
||||
for path in files:
|
||||
parsed = read_markdown(path)
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
# -------- Parse & Validate --------
|
||||
try:
|
||||
parsed = read_markdown(path)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"read_markdown failed: {e}"}))
|
||||
continue
|
||||
if parsed is None:
|
||||
print(json.dumps({"path": path, "error": "read_markdown returned None"}))
|
||||
continue
|
||||
|
||||
try:
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"Frontmatter invalid: {e}"}))
|
||||
@@ -176,103 +240,138 @@ def main() -> None:
|
||||
|
||||
processed += 1
|
||||
|
||||
# Note-Payload (inkl. Hash-Steuerung & Quelle)
|
||||
# -------- Build new payload (includes 'hashes') --------
|
||||
note_pl = make_note_payload(
|
||||
parsed,
|
||||
vault_root=root,
|
||||
hash_mode=args.hash_mode,
|
||||
hash_normalize=args.hash_normalize,
|
||||
hash_source=args.hash_source,
|
||||
hash_mode=mode,
|
||||
hash_normalize=norm,
|
||||
hash_source=src,
|
||||
file_path=path,
|
||||
)
|
||||
if not note_pl.get("fulltext"):
|
||||
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
||||
|
||||
if "fulltext" not in (note_pl or {}):
|
||||
note_pl["fulltext"] = parsed.body or ""
|
||||
if note_pl.get("path"):
|
||||
note_pl["path"] = _normalize_rel_path(
|
||||
os.path.join(root, note_pl["path"]) if not os.path.isabs(note_pl["path"]) else note_pl["path"], root
|
||||
)
|
||||
else:
|
||||
note_pl["path"] = _normalize_rel_path(parsed.path, root)
|
||||
note_id = note_pl.get("note_id") or fm.get("id")
|
||||
if not note_id:
|
||||
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
|
||||
continue
|
||||
|
||||
note_id = note_pl["note_id"]
|
||||
|
||||
# Change-Detection (nur Inhalte, keine FS-Timestamps)
|
||||
# -------- Fetch old payload --------
|
||||
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
||||
old_hash = None if not old_payload else old_payload.get("hash_fulltext")
|
||||
new_hash = note_pl.get("hash_fulltext")
|
||||
changed = args.force_replace or (old_hash != new_hash)
|
||||
has_old = old_payload is not None
|
||||
|
||||
# Optionales Debugging: kompakten Diff anzeigen
|
||||
if args.debug_hash_diff:
|
||||
old_hashes = (old_payload or {}).get("hashes") or {}
|
||||
old_hash_exact = old_hashes.get(key_current)
|
||||
|
||||
# Neu-Hash (aktueller Modus) aus neuem Payload
|
||||
new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
|
||||
|
||||
needs_baseline = (old_hash_exact is None)
|
||||
|
||||
# Change-Detection:
|
||||
# - CREATE: wenn es KEIN Alt-Payload gibt -> changed=True
|
||||
# - UPDATE: baseline existiert UND Hash differiert
|
||||
# - force/text_changed wie gehabt
|
||||
hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact)
|
||||
|
||||
text_changed = False
|
||||
if compare_text:
|
||||
old_text = (old_payload or {}).get("fulltext") or ""
|
||||
new_text = note_pl.get("fulltext") or ""
|
||||
# Wenn Hash gleich, aber Text verschieden → Hinweis auf Normalisierung/Quelle
|
||||
if (old_hash == new_hash) and old_text != new_text:
|
||||
print(json.dumps({
|
||||
"debug": "hash_equal_but_text_differs",
|
||||
"note_id": note_id,
|
||||
"hash_mode": args.hash_mode or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE", "body"),
|
||||
"hash_normalize": args.hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical"),
|
||||
"hash_source": args.hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed"),
|
||||
}, ensure_ascii=False))
|
||||
if old_text and new_text:
|
||||
ud = list(difflib.unified_diff(
|
||||
old_text.splitlines(), new_text.splitlines(),
|
||||
fromfile="qdrant_fulltext(old)", tofile="vault_body(new)",
|
||||
n=3
|
||||
))
|
||||
if ud:
|
||||
preview = "\n".join(ud[:50])
|
||||
print(json.dumps({"note_id": note_id, "diff_preview": preview}, ensure_ascii=False))
|
||||
text_changed = (old_text != new_text)
|
||||
|
||||
# Chunks + Embeddings (Nullvektor-Fallback)
|
||||
chunks = assemble_chunks(fm["id"], parsed.body, fm.get("type", "concept"))
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
|
||||
if embed_texts:
|
||||
vecs = embed_texts([getattr(c, "text", "") for c in chunks]) # type: ignore[attr-defined]
|
||||
else:
|
||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
||||
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
||||
|
||||
# Edges (neues Schema, mit note_id als Owner)
|
||||
note_refs = note_pl.get("references") or []
|
||||
edges = build_edges_for_note(
|
||||
note_id,
|
||||
chunk_pls,
|
||||
note_refs,
|
||||
include_note_scope_refs=note_scope_refs,
|
||||
)
|
||||
# Baseline-only nur, wenn Alt-Payload existiert UND Key fehlt UND keine sonstige Änderung
|
||||
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
||||
|
||||
# Zusammenfassung pro Datei
|
||||
# -------- Optional: Chunks / Embeddings / Edges vorbereiten --------
|
||||
# CREATE/UPDATE: wir brauchen Chunks/Edges; Baseline-only: nein
|
||||
chunks = []
|
||||
chunk_pls = []
|
||||
edges = []
|
||||
vecs = []
|
||||
if (args.apply and (changed and (not do_baseline_only))):
|
||||
try:
|
||||
chunks = assemble_chunks(fm["id"], getattr(parsed, "body", "") or "", fm.get("type", "concept"))
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {e}"}))
|
||||
continue
|
||||
|
||||
if embed_texts:
|
||||
try:
|
||||
vecs = embed_texts([getattr(c, "text", "") for c in chunks]) # type: ignore[attr-defined]
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
||||
else:
|
||||
vecs = [[0.0] * cfg.dim for _ in chunks]
|
||||
|
||||
try:
|
||||
note_refs = note_pl.get("references") or []
|
||||
edges = build_edges_for_note(
|
||||
note_id,
|
||||
chunk_pls,
|
||||
note_refs,
|
||||
include_note_scope_refs=note_scope_refs,
|
||||
)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"build_edges_for_note failed: {e}"}))
|
||||
continue
|
||||
|
||||
# -------- Summary --------
|
||||
summary = {
|
||||
"note_id": note_id,
|
||||
"title": fm.get("title"),
|
||||
"chunks": len(chunk_pls),
|
||||
"edges": len(edges),
|
||||
"changed": changed,
|
||||
"decision": ("apply" if args.apply and changed else
|
||||
"needs_baseline_for_mode": needs_baseline,
|
||||
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
||||
"apply" if args.apply and changed else
|
||||
"apply-skip-unchanged" if args.apply and not changed else
|
||||
"dry-run"),
|
||||
"path": note_pl["path"],
|
||||
"hash_mode": args.hash_mode or os.environ.get("MINDNET_HASH_MODE") or os.environ.get("MINDNET_HASH_COMPARE", "body"),
|
||||
"hash_normalize": args.hash_normalize or os.environ.get("MINDNET_HASH_NORMALIZE", "canonical"),
|
||||
"hash_source": args.hash_source or os.environ.get("MINDNET_HASH_SOURCE", "parsed"),
|
||||
"hash_mode": mode,
|
||||
"hash_normalize": norm,
|
||||
"hash_source": src,
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
|
||||
# -------- Writes --------
|
||||
if not args.apply:
|
||||
continue
|
||||
|
||||
if changed and args.purge_before_upsert:
|
||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||
# BASELINE-ONLY: fehlenden Key nachtragen, ohne legacy Felder anzutasten
|
||||
if do_baseline_only:
|
||||
merged_hashes = {}
|
||||
merged_hashes.update(old_hashes)
|
||||
merged_hashes.update(note_pl.get("hashes") or {})
|
||||
# Legacy-Hashfelder unverändert lassen, falls vorhanden
|
||||
if old_payload:
|
||||
note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext"))
|
||||
note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature"))
|
||||
note_pl["hashes"] = merged_hashes
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
continue
|
||||
|
||||
# Normale CREATE/UPDATE
|
||||
if not changed:
|
||||
continue
|
||||
|
||||
if args.purge_before_upsert and has_old:
|
||||
try:
|
||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
||||
|
||||
# Upserts
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
|
||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||
upsert_batch(client, chunks_name, chunk_pts)
|
||||
|
||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||
upsert_batch(client, edges_name, edge_pts)
|
||||
|
||||
|
||||
+18
-14
@@ -3,31 +3,32 @@
|
||||
"""
|
||||
make_test_vault.py — erzeugt einen minimalen, nachvollziehbaren Test-Vault
|
||||
|
||||
Version: 1.1 (2025-09-05)
|
||||
Änderungen ggü. 1.0:
|
||||
- created/updated werden als QUOTED ISO-Strings geschrieben, damit der YAML-Parser
|
||||
sie nicht in datetime-Objekte konvertiert (Schema verlangt strings).
|
||||
- Kommentare/Output gestrafft.
|
||||
Version: 1.2 (2025-09-09)
|
||||
Änderungen ggü. 1.1:
|
||||
- Fügt **externe Links** (http/mailto) in `exp-two.md` hinzu, damit der Chunk-Payload-Exporter
|
||||
`external_links` realistisch testen kann.
|
||||
- Kleine Textanpassungen zur stabilen Chunk-Bildung (Absatzstruktur).
|
||||
|
||||
Zweck
|
||||
- Kleiner Obsidian-ähnlicher Vault zum Durchspielen des Importers (Chunks/Edges).
|
||||
- Szenarien: leere Links, spätere Anlage fehlender Noten, Chunk-Neuaufteilung.
|
||||
- Szenarien: leere Links, spätere Anlage fehlender Noten, Chunk-Neuaufteilung, externe Links.
|
||||
|
||||
Struktur (Default: ./test_vault)
|
||||
- 40_concepts/concept-alpha.md
|
||||
- 20_experiences/exp-one.md (verlinkt auf [[concept-alpha]] und [[missing-note]])
|
||||
- 20_experiences/exp-two.md (verlinkt auf [[concept-alpha]])
|
||||
- 20_experiences/exp-two.md (verlinkt auf [[concept-alpha]] + externe Links)
|
||||
- 30_projects/project-demo.md (verlinkt auf [[concept-alpha]] und [[exp-one]])
|
||||
|
||||
Voraussetzungen
|
||||
- Keine. Rein Dateigenerierung.
|
||||
|
||||
Aufruf
|
||||
python3 -m scripts.make_test_vault [--out ./test_vault] [--force]
|
||||
python3 -m scripts.make_test_vault [--out ./test_vault] [--force] [--with-missing]
|
||||
|
||||
Parameter
|
||||
--out Zielverzeichnis (Standard: ./test_vault)
|
||||
--force Bestehenden Ordner löschen und neu anlegen.
|
||||
--out Zielverzeichnis (Standard: ./test_vault)
|
||||
--force Bestehenden Ordner löschen und neu anlegen.
|
||||
--with-missing Auch die 'missing-note' direkt anlegen (Standard: erst im 2. Lauf)
|
||||
|
||||
Hinweise für Tests
|
||||
1) Erster Import (dry-run oder --apply): Es gibt einen „leeren“ Link [[missing-note]].
|
||||
@@ -35,6 +36,7 @@ Hinweise für Tests
|
||||
-> Erwartung: Edges für ehemals leeren Link werden korrekt nachgezogen (references + backlink).
|
||||
3) Ändere den Body von exp-one.md so, dass andere Chunk-Grenzen entstehen und importiere erneut:
|
||||
-> Erwartung: Chunks/Edges/Note werden für betroffene Noten aktualisiert.
|
||||
4) Prüfe externe Links (verify_chunk_texts.py betrifft nur Text; externe Links siehst du in chunk_payloads).
|
||||
|
||||
Kompatibel mit:
|
||||
- note.schema.json: 'created'/'updated' müssen strings sein. (Wichtig!)
|
||||
@@ -52,7 +54,6 @@ def iso_now() -> str:
|
||||
|
||||
def q(s: str) -> str:
|
||||
# Sicheres Quoten für YAML-Frontmatter
|
||||
# (einfaches doppelt-Quoten reicht hier, da wir keine eingebetteten """ verwenden)
|
||||
return f"\"{s}\""
|
||||
|
||||
TEMPLATE_CONCEPT = """---
|
||||
@@ -66,7 +67,7 @@ tags: ["area/mindnet","type/concept","topic/test"]
|
||||
---
|
||||
|
||||
## Beschreibung
|
||||
Dies ist ein einfaches Konzept für Testzwecke.
|
||||
Dies ist ein einfaches Konzept für Testzwecke. Es wird von anderen Notizen referenziert.
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[exp-one]]
|
||||
@@ -109,10 +110,12 @@ tags: ["area/mindnet","type/experience","topic/test"]
|
||||
---
|
||||
|
||||
## Kontext
|
||||
Zweite Experience, die auf dasselbe Konzept verweist.
|
||||
Zweite Experience, die auf dasselbe Konzept verweist, ergänzt um **externe Links**.
|
||||
|
||||
## Beobachtung
|
||||
Verweis auf [[concept-alpha]].
|
||||
Verweis auf [[concept-alpha]]. Zusätzliche Ressourcen:
|
||||
- Link: [Beispielseite](https://example.com "Referenz")
|
||||
- Kontakt: [E-Mail](mailto:info@example.com)
|
||||
|
||||
## Mögliche Verbindungen
|
||||
- [[concept-alpha]]
|
||||
@@ -218,6 +221,7 @@ def main():
|
||||
print(" 1) Zuerst ohne missing-note anlegen (Standard), importieren, Edges prüfen.")
|
||||
print(" 2) Dann erneut ausführen mit --force --with-missing (missing-note wird erzeugt),")
|
||||
print(" und Import wiederholen → Edges/Backlinks sollten nachgezogen werden.")
|
||||
print(" 3) In exp-two.md sind **externe Links** enthalten → prüfe `external_links` in Chunk-Payloads.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user