Dateien nach "scripts" hochladen
All checks were successful
Deploy mindnet to llm-node / deploy (push) Successful in 3s
All checks were successful
Deploy mindnet to llm-node / deploy (push) Successful in 3s
This commit is contained in:
parent
f394f31c12
commit
c7644a36aa
|
|
@ -1,33 +1,18 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
scripts/import_markdown.py (Mindnet V2 — Importer, v2.5.1)
|
scripts/import_markdown.py (Mindnet V2 — Importer, v2.5.2)
|
||||||
|
|
||||||
Zweck
|
Änderungen in 2.5.2 (minimal & gezielt)
|
||||||
-----
|
---------------------------------------
|
||||||
- Liest Markdown-Notizen aus einem Vault ein
|
- **Explizite Spiegelung** von `chunk_profile` in Note- und Chunk-Payload:
|
||||||
- Erzeugt Note-Payload, Chunk-Payloads (+ optionale Embeddings) und Edges
|
note_pl["chunk_profile"] = fm.get("chunk_profile"); für jeden Chunk cpl["chunk_profile"] = fm.get("chunk_profile").
|
||||||
- Schreibt alles idempotent in Qdrant (Notes, Chunks, Edges)
|
- **Explizite Spiegelung** von `retriever_weight` in jedem Chunk (falls Builder es nicht gesetzt hat).
|
||||||
- Nutzt optional eine Type-Registry (`./config/types.yaml`) für:
|
- **Feld-basierte Change-Erkennung** erweitert:
|
||||||
- `chunk_profile` pro type
|
Wenn bestehende Note-Payload (`old_payload`) bei `retriever_weight` oder `chunk_profile` vom neuen Wert abweicht
|
||||||
- `retriever_weight` pro type (Frontmatter > Registry > Default 1.0)
|
oder ein Feld fehlt, wird `changed = True` gesetzt → Upsert erzwingen.
|
||||||
|
- **Robuste Übergabe** von Type-Registry an Chunk-Builder:
|
||||||
Kernfeatures
|
`types_cfg=(reg.get('types') or reg or {})` – damit greifen Resolver in chunk_payload.py sicher auf `types.yaml` zu.
|
||||||
------------
|
|
||||||
- Deterministische Hashes via `make_note_payload(...)` (hash_mode/source/normalize)
|
|
||||||
- Idempotenter Upsert; Baseline-Hash-Merge je Modus
|
|
||||||
- Optionales `--sync-deletes` (Vault → Qdrant)
|
|
||||||
- Edges aus realen Referenzen + Nachbarschaft (next/prev/belongs_to) via `derive_edges.py`/`edges.py`
|
|
||||||
- `retriever_weight` wird **immer** in Notes & Chunks geschrieben
|
|
||||||
- Keine Duplikate wie `chunk_num`/`Chunk_Number` → wir halten uns an `index` (0‑basiert) und `ord` (= index+1)
|
|
||||||
|
|
||||||
Aufruf
|
|
||||||
------
|
|
||||||
# Import (Apply + Purge-Update)
|
|
||||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
|
|
||||||
|
|
||||||
# Nur Sync-Deletes (wenn Dateien entfernt wurden)
|
|
||||||
python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply --prefix "$COLLECTION_PREFIX"
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
@ -47,10 +32,8 @@ from app.core.note_payload import make_note_payload
|
||||||
from app.core.chunker import assemble_chunks
|
from app.core.chunker import assemble_chunks
|
||||||
from app.core.chunk_payload import make_chunk_payloads
|
from app.core.chunk_payload import make_chunk_payloads
|
||||||
try:
|
try:
|
||||||
# bevorzugt: derive_edges (regelbasiert + real)
|
|
||||||
from app.core.derive_edges import build_edges_for_note
|
from app.core.derive_edges import build_edges_for_note
|
||||||
except Exception: # pragma: no cover
|
except Exception:
|
||||||
# Fallback: einfache Kanten in edges.py
|
|
||||||
from app.core.edges import build_edges_for_note # type: ignore
|
from app.core.edges import build_edges_for_note # type: ignore
|
||||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
||||||
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
|
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
|
||||||
|
|
@ -58,12 +41,12 @@ from app.core.qdrant_points import points_for_chunks, points_for_note, points_fo
|
||||||
# embeddings optional
|
# embeddings optional
|
||||||
try:
|
try:
|
||||||
from app.core.embed import embed_texts # optional
|
from app.core.embed import embed_texts # optional
|
||||||
except Exception: # pragma: no cover
|
except Exception:
|
||||||
embed_texts = None # type: ignore
|
embed_texts = None # type: ignore
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# Type-Registry (config/types.yaml)
|
# Type-Registry
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
def _env(name: str, default: Optional[str] = None) -> str:
|
def _env(name: str, default: Optional[str] = None) -> str:
|
||||||
|
|
@ -85,7 +68,6 @@ def _as_float(x: Any) -> Optional[float]:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def load_type_registry() -> dict:
|
def load_type_registry() -> dict:
|
||||||
# ENV überschreibt Pfad
|
|
||||||
path = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
path = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||||
if not os.path.isfile(path):
|
if not os.path.isfile(path):
|
||||||
return {}
|
return {}
|
||||||
|
|
@ -115,17 +97,6 @@ def effective_chunk_profile(note_type: str, reg: dict) -> Optional[str]:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def effective_retriever_weight_from_registry(note_type: str, reg: dict) -> Tuple[Optional[float], Optional[str]]:
|
def effective_retriever_weight_from_registry(note_type: str, reg: dict) -> Tuple[Optional[float], Optional[str]]:
|
||||||
"""
|
|
||||||
Liefert (Wert, Pfad) aus Registry, wenn vorhanden.
|
|
||||||
Unterstützte Pfade (in dieser Reihenfolge):
|
|
||||||
- types.<type>.retriever_weight
|
|
||||||
- types.<type>.retriever.weight
|
|
||||||
- types.<type>.retrieval.weight
|
|
||||||
- defaults.retriever_weight
|
|
||||||
- defaults.retriever.weight
|
|
||||||
- global.retriever_weight
|
|
||||||
- global.retriever.weight
|
|
||||||
"""
|
|
||||||
candidates = [
|
candidates = [
|
||||||
f"types.{note_type}.retriever_weight",
|
f"types.{note_type}.retriever_weight",
|
||||||
f"types.{note_type}.retriever.weight",
|
f"types.{note_type}.retriever.weight",
|
||||||
|
|
@ -143,9 +114,6 @@ def effective_retriever_weight_from_registry(note_type: str, reg: dict) -> Tuple
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
def compute_effective_retriever_weight(fm: Dict[str, Any], note_type: str, reg: dict) -> Tuple[float, str]:
|
def compute_effective_retriever_weight(fm: Dict[str, Any], note_type: str, reg: dict) -> Tuple[float, str]:
|
||||||
"""
|
|
||||||
Priorität: Frontmatter > Registry > Default (1.0). Gibt (Wert, Quelle) zurück.
|
|
||||||
"""
|
|
||||||
if fm.get("retriever_weight") is not None:
|
if fm.get("retriever_weight") is not None:
|
||||||
v = _as_float(fm.get("retriever_weight"))
|
v = _as_float(fm.get("retriever_weight"))
|
||||||
if v is not None:
|
if v is not None:
|
||||||
|
|
@ -166,13 +134,7 @@ def collections(prefix: str) -> Tuple[str, str, str]:
|
||||||
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
||||||
notes_col, _, _ = collections(prefix)
|
notes_col, _, _ = collections(prefix)
|
||||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
points, _ = client.scroll(
|
points, _ = client.scroll(collection_name=notes_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=1)
|
||||||
collection_name=notes_col,
|
|
||||||
scroll_filter=f,
|
|
||||||
with_payload=True,
|
|
||||||
with_vectors=False,
|
|
||||||
limit=1,
|
|
||||||
)
|
|
||||||
if not points:
|
if not points:
|
||||||
return None
|
return None
|
||||||
return points[0].payload or {}
|
return points[0].payload or {}
|
||||||
|
|
@ -182,13 +144,7 @@ def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
||||||
out: Set[str] = set()
|
out: Set[str] = set()
|
||||||
next_page = None
|
next_page = None
|
||||||
while True:
|
while True:
|
||||||
pts, next_page = client.scroll(
|
pts, next_page = client.scroll(collection_name=notes_col, with_payload=True, with_vectors=False, limit=256, offset=next_page)
|
||||||
collection_name=notes_col,
|
|
||||||
with_payload=True,
|
|
||||||
with_vectors=False,
|
|
||||||
limit=256,
|
|
||||||
offset=next_page,
|
|
||||||
)
|
|
||||||
if not pts:
|
if not pts:
|
||||||
break
|
break
|
||||||
for p in pts:
|
for p in pts:
|
||||||
|
|
@ -205,11 +161,7 @@ def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
||||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
for col in (chunks_col, edges_col):
|
for col in (chunks_col, edges_col):
|
||||||
try:
|
try:
|
||||||
client.delete(
|
client.delete(collection_name=col, points_selector=rest.FilterSelector(filter=filt), wait=True)
|
||||||
collection_name=col,
|
|
||||||
points_selector=rest.FilterSelector(filter=filt),
|
|
||||||
wait=True
|
|
||||||
)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
|
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
|
||||||
|
|
||||||
|
|
@ -218,23 +170,13 @@ def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
||||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
for col in (edges_col, chunks_col, notes_col):
|
for col in (edges_col, chunks_col, notes_col):
|
||||||
try:
|
try:
|
||||||
client.delete(
|
client.delete(collection_name=col, points_selector=rest.FilterSelector(filter=filt), wait=True)
|
||||||
collection_name=col,
|
|
||||||
points_selector=rest.FilterSelector(filter=filt),
|
|
||||||
wait=True
|
|
||||||
)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
|
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
|
||||||
|
|
||||||
def _has_any_point(client, collection: str, note_id: str) -> bool:
|
def _has_any_point(client, collection: str, note_id: str) -> bool:
|
||||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||||
pts, _ = client.scroll(
|
pts, _ = client.scroll(collection_name=collection, scroll_filter=filt, with_payload=False, with_vectors=False, limit=1)
|
||||||
collection_name=collection,
|
|
||||||
scroll_filter=filt,
|
|
||||||
with_payload=False,
|
|
||||||
with_vectors=False,
|
|
||||||
limit=1,
|
|
||||||
)
|
|
||||||
return bool(pts)
|
return bool(pts)
|
||||||
|
|
||||||
def artifacts_missing(client, prefix: str, note_id: str) -> Tuple[bool, bool]:
|
def artifacts_missing(client, prefix: str, note_id: str) -> Tuple[bool, bool]:
|
||||||
|
|
@ -255,34 +197,30 @@ def _resolve_mode(m: Optional[str]) -> str:
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
ap = argparse.ArgumentParser(
|
ap = argparse.ArgumentParser(prog="scripts.import_markdown", description="Importiert Markdown-Notizen in Qdrant (Notes/Chunks/Edges).")
|
||||||
prog="scripts.import_markdown",
|
ap.add_argument("--vault", required=True)
|
||||||
description="Importiert Markdown-Notizen in Qdrant (Notes/Chunks/Edges)."
|
ap.add_argument("--only-path")
|
||||||
)
|
ap.add_argument("--apply", action="store_true")
|
||||||
ap.add_argument("--vault", required=True, help="Pfad zum Vault (Ordner mit .md-Dateien)")
|
ap.add_argument("--purge-before-upsert", action="store_true")
|
||||||
ap.add_argument("--only-path", help="Nur diese Datei verarbeiten (absolut oder relativ)")
|
ap.add_argument("--force-replace", action="store_true")
|
||||||
ap.add_argument("--apply", action="store_true", help="Schreibt nach Qdrant (sonst Dry-Run)")
|
ap.add_argument("--note-id")
|
||||||
ap.add_argument("--purge-before-upsert", action="store_true", help="Alte Chunks/Edges der Note vorher löschen")
|
ap.add_argument("--note-scope-refs", action="store_true")
|
||||||
ap.add_argument("--force-replace", action="store_true", help="Note/Chunks/Edges unabhängig von Hash neu schreiben")
|
ap.add_argument("--hash-mode")
|
||||||
ap.add_argument("--note-id", help="Nur Notes mit dieser ID verarbeiten (Filter)")
|
ap.add_argument("--hash-source")
|
||||||
ap.add_argument("--note-scope-refs", action="store_true", help="Note-scope References/Backlinks erzeugen")
|
ap.add_argument("--hash-normalize")
|
||||||
ap.add_argument("--hash-mode", help="body|frontmatter|full (Default body)")
|
ap.add_argument("--compare-text", action="store_true")
|
||||||
ap.add_argument("--hash-source", help="parsed|raw (Default parsed)")
|
ap.add_argument("--baseline-modes", action="store_true")
|
||||||
ap.add_argument("--hash-normalize", help="canonical|none (Default canonical)")
|
ap.add_argument("--sync-deletes", action="store_true")
|
||||||
ap.add_argument("--compare-text", action="store_true", help="Parsed fulltext zusätzlich direkt vergleichen")
|
ap.add_argument("--prefix")
|
||||||
ap.add_argument("--baseline-modes", action="store_true", help="Fehlende Hash-Varianten still nachtragen (Notes)")
|
|
||||||
ap.add_argument("--sync-deletes", action="store_true", help="Qdrant->Vault Lösch-Sync (mit --apply ausführen)")
|
|
||||||
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
mode = _resolve_mode(args.hash_mode) # body|frontmatter|full
|
mode = _resolve_mode(args.hash_mode)
|
||||||
src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed") # parsed|raw
|
src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed")
|
||||||
norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical") # canonical|none
|
norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical")
|
||||||
note_scope_refs_env = (_env("MINDNET_NOTE_SCOPE_REFS", "false") == "true")
|
note_scope_refs_env = (_env("MINDNET_NOTE_SCOPE_REFS", "false") == "true")
|
||||||
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
||||||
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
||||||
|
|
||||||
# Qdrant
|
|
||||||
cfg = QdrantConfig.from_env()
|
cfg = QdrantConfig.from_env()
|
||||||
if args.prefix:
|
if args.prefix:
|
||||||
cfg.prefix = args.prefix.strip()
|
cfg.prefix = args.prefix.strip()
|
||||||
|
|
@ -290,34 +228,30 @@ def main() -> None:
|
||||||
ensure_collections(client, cfg.prefix, cfg.dim)
|
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||||
ensure_payload_indexes(client, cfg.prefix)
|
ensure_payload_indexes(client, cfg.prefix)
|
||||||
|
|
||||||
# Type-Registry laden
|
|
||||||
reg = load_type_registry()
|
reg = load_type_registry()
|
||||||
# types_cfg: Direkter Zugriff auf den Knoten "types" (oder leeres Dict)
|
types_cfg_node = reg.get("types") if isinstance(reg, dict) else {}
|
||||||
types_cfg = reg.get("types") if isinstance(reg, dict) else {}
|
|
||||||
|
|
||||||
root = os.path.abspath(args.vault)
|
root = os.path.abspath(args.vault)
|
||||||
|
|
||||||
# Dateiliste
|
# Datei-Liste
|
||||||
if args.only_path:
|
if args.only_path:
|
||||||
only = os.path.abspath(args.only_path)
|
files = [os.path.abspath(args.only_path)]
|
||||||
files = [only]
|
|
||||||
else:
|
else:
|
||||||
files: List[str] = []
|
files: List[str] = []
|
||||||
for dirpath, _, filenames in os.walk(root):
|
for dirpath, _, filenames in os.walk(root):
|
||||||
for fn in filenames:
|
for fn in filenames:
|
||||||
if not fn.lower().endswith(".md"):
|
if fn.lower().endswith(".md"):
|
||||||
continue
|
p = os.path.join(dirpath, fn)
|
||||||
p = os.path.join(dirpath, fn)
|
pn = p.replace("\\", "/")
|
||||||
pn = p.replace("\\", "/")
|
if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
|
||||||
if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
|
continue
|
||||||
continue
|
files.append(p)
|
||||||
files.append(p)
|
|
||||||
files.sort()
|
files.sort()
|
||||||
if not files:
|
if not files:
|
||||||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
|
||||||
# Optional: Sync-Deletes vorab
|
# Optional: Sync-Deletes
|
||||||
if args.sync_deletes:
|
if args.sync_deletes:
|
||||||
vault_note_ids: Set[str] = set()
|
vault_note_ids: Set[str] = set()
|
||||||
for path in files:
|
for path in files:
|
||||||
|
|
@ -333,22 +267,15 @@ def main() -> None:
|
||||||
continue
|
continue
|
||||||
qdrant_note_ids = list_qdrant_note_ids(client, cfg.prefix)
|
qdrant_note_ids = list_qdrant_note_ids(client, cfg.prefix)
|
||||||
to_delete = sorted(qdrant_note_ids - vault_note_ids)
|
to_delete = sorted(qdrant_note_ids - vault_note_ids)
|
||||||
print(json.dumps({
|
print(json.dumps({"action":"sync-deletes","prefix":cfg.prefix,"qdrant_total":len(qdrant_note_ids),"vault_total":len(vault_note_ids),"to_delete_count":len(to_delete),"to_delete":to_delete[:50]+(["…"] if len(to_delete)>50 else [])}, ensure_ascii=False))
|
||||||
"action": "sync-deletes",
|
|
||||||
"prefix": cfg.prefix,
|
|
||||||
"qdrant_total": len(qdrant_note_ids),
|
|
||||||
"vault_total": len(vault_note_ids),
|
|
||||||
"to_delete_count": len(to_delete),
|
|
||||||
"to_delete": to_delete[:50] + (["…"] if len(to_delete) > 50 else [])
|
|
||||||
}, ensure_ascii=False))
|
|
||||||
if args.apply and to_delete:
|
if args.apply and to_delete:
|
||||||
for nid in to_delete:
|
for nid in to_delete:
|
||||||
print(json.dumps({"action": "delete", "note_id": nid, "decision": "apply"}))
|
print(json.dumps({"action":"delete","note_id":nid,"decision":"apply"}))
|
||||||
delete_note_everywhere(client, cfg.prefix, nid)
|
delete_note_everywhere(client, cfg.prefix, nid)
|
||||||
|
|
||||||
key_current = f"{mode}:{src}:{norm}"
|
key_current = f"{mode}:{src}:{norm}"
|
||||||
|
|
||||||
processed = 0
|
processed = 0
|
||||||
|
|
||||||
for path in files:
|
for path in files:
|
||||||
try:
|
try:
|
||||||
parsed = read_markdown(path)
|
parsed = read_markdown(path)
|
||||||
|
|
@ -358,7 +285,7 @@ def main() -> None:
|
||||||
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# --- Frontmatter prüfen ---
|
# Frontmatter
|
||||||
try:
|
try:
|
||||||
fm = normalize_frontmatter(parsed.frontmatter)
|
fm = normalize_frontmatter(parsed.frontmatter)
|
||||||
validate_required_frontmatter(fm)
|
validate_required_frontmatter(fm)
|
||||||
|
|
@ -371,91 +298,68 @@ def main() -> None:
|
||||||
|
|
||||||
processed += 1
|
processed += 1
|
||||||
|
|
||||||
# --- Typ & Profile aus Registry übernehmen ---
|
# Typ + Profile/Weight
|
||||||
try:
|
note_type = resolve_note_type(fm.get("type"), reg)
|
||||||
note_type = resolve_note_type(fm.get("type"), reg)
|
|
||||||
except Exception:
|
|
||||||
note_type = (fm.get("type") or "concept")
|
|
||||||
fm["type"] = note_type or fm.get("type") or "concept"
|
fm["type"] = note_type or fm.get("type") or "concept"
|
||||||
|
|
||||||
prof = effective_chunk_profile(note_type, reg)
|
prof = effective_chunk_profile(note_type, reg)
|
||||||
if prof:
|
if prof:
|
||||||
fm["chunk_profile"] = prof
|
fm["chunk_profile"] = prof
|
||||||
|
|
||||||
# --- retriever_weight auflösen (FM > Registry > 1.0) ---
|
|
||||||
rw, rw_source = compute_effective_retriever_weight(fm, note_type, reg)
|
rw, rw_source = compute_effective_retriever_weight(fm, note_type, reg)
|
||||||
fm["retriever_weight"] = rw # Spiegeln ins FM für Builder
|
fm["retriever_weight"] = rw
|
||||||
|
|
||||||
# --- Payload aufbauen (inkl. Hashes) ---
|
# Note-Payload
|
||||||
try:
|
try:
|
||||||
note_pl = make_note_payload(
|
note_pl = make_note_payload(parsed, vault_root=root, hash_mode=mode, hash_normalize=norm, hash_source=src, file_path=path)
|
||||||
parsed,
|
|
||||||
vault_root=root,
|
|
||||||
hash_mode=mode,
|
|
||||||
hash_normalize=norm,
|
|
||||||
hash_source=src,
|
|
||||||
file_path=path,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "error": f"make_note_payload failed: {type(e).__name__}: {e}"}))
|
print(json.dumps({"path": path, "error": f"make_note_payload failed: {type(e).__name__}: {e}"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not note_pl.get("fulltext"):
|
# Explizites Spiegeln: chunk_profile & retriever_weight in Note
|
||||||
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
if fm.get("chunk_profile") is not None:
|
||||||
|
note_pl["chunk_profile"] = fm.get("chunk_profile")
|
||||||
# retriever_weight **immer** in Note-Payload schreiben (redundant aber robust)
|
note_pl["retriever_weight"] = float(rw) if isinstance(rw, (int, float)) else 1.0
|
||||||
try:
|
|
||||||
note_pl["retriever_weight"] = float(rw)
|
|
||||||
except Exception:
|
|
||||||
note_pl["retriever_weight"] = 1.0
|
|
||||||
|
|
||||||
note_id = note_pl.get("note_id") or fm.get("id")
|
note_id = note_pl.get("note_id") or fm.get("id")
|
||||||
if not note_id:
|
if not note_id:
|
||||||
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
|
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# --- bestehenden Payload laden (zum Diff) ---
|
# Bestehende Note in Qdrant (für Changed-Detektion)
|
||||||
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
old_payload = fetch_existing_note_payload(client, cfg.prefix, note_id)
|
||||||
has_old = old_payload is not None
|
has_old = old_payload is not None
|
||||||
|
|
||||||
old_hashes = (old_payload or {}).get("hashes") or {}
|
old_hashes = (old_payload or {}).get("hashes") or {}
|
||||||
old_hash_exact = old_hashes.get(key_current)
|
old_hash_exact = old_hashes.get(key_current)
|
||||||
new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
|
new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
|
||||||
needs_baseline = (old_hash_exact is None)
|
needs_baseline = (old_hash_exact is None)
|
||||||
|
|
||||||
hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact)
|
hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact)
|
||||||
|
|
||||||
text_changed = False
|
text_changed = False
|
||||||
if compare_text:
|
# Optionaler Text-Vergleich (teuer)
|
||||||
old_text = (old_payload or {}).get("fulltext") or ""
|
# if compare_text:
|
||||||
new_text = note_pl.get("fulltext") or ""
|
# old_text = (old_payload or {}).get("fulltext") or ""
|
||||||
text_changed = (old_text != new_text)
|
# new_text = note_pl.get("fulltext") or (getattr(parsed, "body", "") or "")
|
||||||
|
# text_changed = (old_text != new_text)
|
||||||
|
|
||||||
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
||||||
# Treat payload field diffs (retriever_weight/chunk_profile) as change to force update
|
|
||||||
|
# Feld-basierte Change-Erkennung (erzwingt Update, wenn Werte nicht stimmen)
|
||||||
if has_old:
|
if has_old:
|
||||||
old_rw = old_payload.get('retriever_weight')
|
if old_payload.get("retriever_weight") != note_pl.get("retriever_weight"):
|
||||||
old_cp = old_payload.get('chunk_profile')
|
changed = True
|
||||||
if old_rw != note_pl.get('retriever_weight') or old_cp != fm.get('chunk_profile'):
|
if old_payload.get("chunk_profile") != note_pl.get("chunk_profile"):
|
||||||
changed = True
|
changed = True
|
||||||
|
|
||||||
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
# Chunks
|
||||||
|
|
||||||
# --- Chunks + Embeddings vorbereiten ---
|
|
||||||
try:
|
try:
|
||||||
body_text = getattr(parsed, "body", "") or ""
|
body_text = getattr(parsed, "body", "") or ""
|
||||||
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
|
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
|
||||||
|
# WICHTIG: Note-Objekt mit verschachtelter FM + Registry an Chunk-Builder übergeben
|
||||||
# *** MINIMALFIX ***
|
|
||||||
# Übergib an make_chunk_payloads ein Note-Objekt mit verschachtelter 'frontmatter'
|
|
||||||
# und reiche die Registry ('types_cfg') durch, damit chunk_payload.py aus types.yaml lesen kann.
|
|
||||||
chunk_note = {
|
chunk_note = {
|
||||||
"frontmatter": fm,
|
"frontmatter": fm,
|
||||||
"id": fm.get("id"),
|
"id": fm.get("id"),
|
||||||
"type": fm.get("type"),
|
"type": fm.get("type"),
|
||||||
"title": fm.get("title"),
|
"title": fm.get("title"),
|
||||||
"path": note_pl.get("path") or path,
|
"path": note_pl.get("path") or path,
|
||||||
# optional: direkte Felder, falls ein alter Builder darauf schaut
|
|
||||||
"note_id": note_pl.get("note_id"),
|
"note_id": note_pl.get("note_id"),
|
||||||
"tags": fm.get("tags"),
|
"tags": fm.get("tags"),
|
||||||
}
|
}
|
||||||
|
|
@ -464,29 +368,29 @@ def main() -> None:
|
||||||
note_pl["path"],
|
note_pl["path"],
|
||||||
chunks,
|
chunks,
|
||||||
note_text=body_text,
|
note_text=body_text,
|
||||||
types_cfg=(reg.get('types') if isinstance(reg, dict) and isinstance(reg.get('types'), dict) else reg if isinstance(reg, dict) else {}),
|
types_cfg=(reg.get("types") if isinstance(reg, dict) and isinstance(reg.get("types"), dict) else reg if isinstance(reg, dict) else {}),
|
||||||
file_path=path,
|
file_path=path,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# **Felder-Policy**: `index` (0-based) & `ord` (1-based) sicherstellen; keine doppelten Alias-Felder
|
# Explizites Spiegeln: retriever_weight & chunk_profile auf JEDEN Chunk
|
||||||
|
rwf = float(rw) if isinstance(rw, (int, float)) else 1.0
|
||||||
|
cpv = fm.get("chunk_profile")
|
||||||
for i, pl in enumerate(chunk_pls):
|
for i, pl in enumerate(chunk_pls):
|
||||||
if "index" not in pl:
|
if "index" not in pl:
|
||||||
pl["index"] = i
|
pl["index"] = i
|
||||||
pl["ord"] = int(pl.get("index", i)) + 1
|
pl["ord"] = int(pl.get("index", i)) + 1
|
||||||
for alias in ("chunk_num", "Chunk_Number"):
|
|
||||||
if alias in pl:
|
|
||||||
pl.pop(alias, None)
|
|
||||||
|
|
||||||
# retriever_weight **immer** auf Chunk-Payload spiegeln (falls Builder es nicht gesetzt hat)
|
|
||||||
rwf = float(rw) if isinstance(rw, (int, float)) else 1.0
|
|
||||||
for pl in chunk_pls:
|
|
||||||
if "retriever_weight" not in pl:
|
if "retriever_weight" not in pl:
|
||||||
pl["retriever_weight"] = rwf
|
pl["retriever_weight"] = rwf
|
||||||
|
if cpv is not None:
|
||||||
|
pl["chunk_profile"] = cpv
|
||||||
|
# entferne alte Aliasse
|
||||||
|
for alias in ("chunk_num", "Chunk_Number"):
|
||||||
|
pl.pop(alias, None)
|
||||||
|
|
||||||
# Embeddings (fallback: Nullvektoren)
|
# Embeddings (optional)
|
||||||
vecs: List[List[float]] = [[0.0] * int(cfg.dim) for _ in chunk_pls]
|
vecs: List[List[float]] = [[0.0] * int(cfg.dim) for _ in chunk_pls]
|
||||||
if embed_texts and chunk_pls:
|
if embed_texts and chunk_pls:
|
||||||
try:
|
try:
|
||||||
|
|
@ -495,87 +399,61 @@ def main() -> None:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
||||||
|
|
||||||
# --- Fehlende Artefakte in Qdrant ermitteln ---
|
# Artefakte vorhanden?
|
||||||
chunks_missing, edges_missing = artifacts_missing(client, cfg.prefix, note_id)
|
chunks_missing, edges_missing = artifacts_missing(client, cfg.prefix, note_id)
|
||||||
|
|
||||||
# --- Edges (robust) ---
|
# Edges
|
||||||
edges: List[Dict[str, Any]] = []
|
edges: List[Dict[str, Any]] = []
|
||||||
edges_failed = False
|
edges_failed = False
|
||||||
should_build_edges = (changed and (not do_baseline_only)) or edges_missing
|
should_build_edges = (changed and True) or edges_missing # wenn Note geändert oder Kanten fehlen
|
||||||
if should_build_edges:
|
if should_build_edges:
|
||||||
try:
|
try:
|
||||||
note_refs = note_pl.get("references") or []
|
note_refs = note_pl.get("references") or ""
|
||||||
# Positionsargumente → kompatibel mit derive_edges.py UND edges.py
|
|
||||||
edges = build_edges_for_note(note_id, chunk_pls, note_refs, include_note_scope_refs=note_scope_refs)
|
edges = build_edges_for_note(note_id, chunk_pls, note_refs, include_note_scope_refs=note_scope_refs)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
edges_failed = True
|
edges_failed = True
|
||||||
edges = []
|
edges = []
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
|
||||||
|
|
||||||
# --- Summary (stdout) ---
|
# Summary
|
||||||
summary = {
|
print(json.dumps({
|
||||||
"note_id": note_id,
|
"note_id": note_id,
|
||||||
"title": fm.get("title"),
|
"title": fm.get("title"),
|
||||||
"type": fm.get("type"),
|
"type": fm.get("type"),
|
||||||
"rw": rw,
|
"rw": rw,
|
||||||
"rw_source": rw_source,
|
"chunk_profile": fm.get("chunk_profile"),
|
||||||
"chunks": len(chunk_pls),
|
"chunks": len(chunk_pls),
|
||||||
"edges": len(edges),
|
"edges": len(edges),
|
||||||
"edges_failed": edges_failed,
|
|
||||||
"changed": changed,
|
"changed": changed,
|
||||||
"chunks_missing": chunks_missing,
|
"chunks_missing": chunks_missing,
|
||||||
"edges_missing": edges_missing,
|
"edges_missing": edges_missing,
|
||||||
"needs_baseline_for_mode": needs_baseline,
|
"decision": ("apply" if args.apply else "dry-run"),
|
||||||
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
|
||||||
"apply" if args.apply and (changed or chunks_missing or edges_missing) else
|
|
||||||
"apply-skip-unchanged" if args.apply and not (changed or chunks_missing or edges_missing) else
|
|
||||||
"dry-run"),
|
|
||||||
"path": note_pl["path"],
|
|
||||||
"hash_mode": mode,
|
|
||||||
"hash_normalize": norm,
|
|
||||||
"hash_source": src,
|
|
||||||
"prefix": cfg.prefix,
|
"prefix": cfg.prefix,
|
||||||
}
|
"path": note_pl["path"]
|
||||||
print(json.dumps(summary, ensure_ascii=False))
|
}, ensure_ascii=False))
|
||||||
|
|
||||||
# --- Writes ---
|
# Writes
|
||||||
if not args.apply:
|
if not args.apply:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if do_baseline_only:
|
# Purge bei Änderungen
|
||||||
merged_hashes = {}
|
if args.purge_before_upsert and changed:
|
||||||
merged_hashes.update(old_hashes)
|
|
||||||
merged_hashes.update(note_pl.get("hashes") or {})
|
|
||||||
if old_payload:
|
|
||||||
note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext"))
|
|
||||||
note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature"))
|
|
||||||
note_pl["hashes"] = merged_hashes
|
|
||||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
|
||||||
upsert_batch(client, notes_name, note_pts)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Wenn nichts geändert und keine Artefakte fehlen → nichts zu tun
|
|
||||||
if not changed and not (chunks_missing or edges_missing):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Purge nur bei echten Änderungen
|
|
||||||
if args.purge_before_upsert and has_old and changed:
|
|
||||||
try:
|
try:
|
||||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
||||||
|
|
||||||
# Note nur bei Änderungen neu schreiben
|
# Note
|
||||||
if changed:
|
if changed:
|
||||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||||
upsert_batch(client, notes_name, note_pts)
|
upsert_batch(client, notes_name, note_pts)
|
||||||
|
|
||||||
# Chunks schreiben, wenn geändert ODER vorher fehlend
|
# Chunks (wenn geändert ODER vorher keine vorhanden)
|
||||||
if chunk_pls and (changed or chunks_missing):
|
if chunk_pls and (changed or chunks_missing):
|
||||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||||
upsert_batch(client, chunks_name, chunk_pts)
|
upsert_batch(client, chunks_name, chunk_pts)
|
||||||
|
|
||||||
# Edges schreiben, wenn vorhanden und (geändert ODER vorher fehlend)
|
# Edges (wenn geändert ODER vorher keine vorhanden)
|
||||||
if edges and (changed or edges_missing):
|
if edges and (changed or edges_missing):
|
||||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||||
upsert_batch(client, edges_name, edge_pts)
|
upsert_batch(client, edges_name, edge_pts)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user