Dateien nach "scripts" hochladen
All checks were successful
Deploy mindnet to llm-node / deploy (push) Successful in 3s
All checks were successful
Deploy mindnet to llm-node / deploy (push) Successful in 3s
This commit is contained in:
parent
f394f31c12
commit
c7644a36aa
|
|
@ -1,33 +1,18 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
scripts/import_markdown.py (Mindnet V2 — Importer, v2.5.1)
|
||||
scripts/import_markdown.py (Mindnet V2 — Importer, v2.5.2)
|
||||
|
||||
Zweck
|
||||
-----
|
||||
- Liest Markdown-Notizen aus einem Vault ein
|
||||
- Erzeugt Note-Payload, Chunk-Payloads (+ optionale Embeddings) und Edges
|
||||
- Schreibt alles idempotent in Qdrant (Notes, Chunks, Edges)
|
||||
- Nutzt optional eine Type-Registry (`./config/types.yaml`) für:
|
||||
- `chunk_profile` pro type
|
||||
- `retriever_weight` pro type (Frontmatter > Registry > Default 1.0)
|
||||
|
||||
Kernfeatures
|
||||
------------
|
||||
- Deterministische Hashes via `make_note_payload(...)` (hash_mode/source/normalize)
|
||||
- Idempotenter Upsert; Baseline-Hash-Merge je Modus
|
||||
- Optionales `--sync-deletes` (Vault → Qdrant)
|
||||
- Edges aus realen Referenzen + Nachbarschaft (next/prev/belongs_to) via `derive_edges.py`/`edges.py`
|
||||
- `retriever_weight` wird **immer** in Notes & Chunks geschrieben
|
||||
- Keine Duplikate wie `chunk_num`/`Chunk_Number` → wir halten uns an `index` (0‑basiert) und `ord` (= index+1)
|
||||
|
||||
Aufruf
|
||||
------
|
||||
# Import (Apply + Purge-Update)
|
||||
python3 -m scripts.import_markdown --vault ./vault --apply --purge-before-upsert --prefix "$COLLECTION_PREFIX"
|
||||
|
||||
# Nur Sync-Deletes (wenn Dateien entfernt wurden)
|
||||
python3 -m scripts.import_markdown --vault ./vault --sync-deletes --apply --prefix "$COLLECTION_PREFIX"
|
||||
Änderungen in 2.5.2 (minimal & gezielt)
|
||||
---------------------------------------
|
||||
- **Explizite Spiegelung** von `chunk_profile` in Note- und Chunk-Payload:
|
||||
note_pl["chunk_profile"] = fm.get("chunk_profile"); für jeden Chunk cpl["chunk_profile"] = fm.get("chunk_profile").
|
||||
- **Explizite Spiegelung** von `retriever_weight` in jedem Chunk (falls Builder es nicht gesetzt hat).
|
||||
- **Feld-basierte Change-Erkennung** erweitert:
|
||||
Wenn bestehende Note-Payload (`old_payload`) bei `retriever_weight` oder `chunk_profile` vom neuen Wert abweicht
|
||||
oder ein Feld fehlt, wird `changed = True` gesetzt → Upsert erzwingen.
|
||||
- **Robuste Übergabe** von Type-Registry an Chunk-Builder:
|
||||
`types_cfg=(reg.get('types') or reg or {})` – damit greifen Resolver in chunk_payload.py sicher auf `types.yaml` zu.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
|
@ -47,10 +32,8 @@ from app.core.note_payload import make_note_payload
|
|||
from app.core.chunker import assemble_chunks
|
||||
from app.core.chunk_payload import make_chunk_payloads
|
||||
try:
|
||||
# bevorzugt: derive_edges (regelbasiert + real)
|
||||
from app.core.derive_edges import build_edges_for_note
|
||||
except Exception: # pragma: no cover
|
||||
# Fallback: einfache Kanten in edges.py
|
||||
except Exception:
|
||||
from app.core.edges import build_edges_for_note # type: ignore
|
||||
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
||||
from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch
|
||||
|
|
@ -58,12 +41,12 @@ from app.core.qdrant_points import points_for_chunks, points_for_note, points_fo
|
|||
# embeddings optional
|
||||
try:
|
||||
from app.core.embed import embed_texts # optional
|
||||
except Exception: # pragma: no cover
|
||||
except Exception:
|
||||
embed_texts = None # type: ignore
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Type-Registry (config/types.yaml)
|
||||
# Type-Registry
|
||||
# ============================================================
|
||||
|
||||
def _env(name: str, default: Optional[str] = None) -> str:
|
||||
|
|
@ -85,7 +68,6 @@ def _as_float(x: Any) -> Optional[float]:
|
|||
return None
|
||||
|
||||
def load_type_registry() -> dict:
|
||||
# ENV überschreibt Pfad
|
||||
path = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||
if not os.path.isfile(path):
|
||||
return {}
|
||||
|
|
@ -115,17 +97,6 @@ def effective_chunk_profile(note_type: str, reg: dict) -> Optional[str]:
|
|||
return None
|
||||
|
||||
def effective_retriever_weight_from_registry(note_type: str, reg: dict) -> Tuple[Optional[float], Optional[str]]:
|
||||
"""
|
||||
Liefert (Wert, Pfad) aus Registry, wenn vorhanden.
|
||||
Unterstützte Pfade (in dieser Reihenfolge):
|
||||
- types.<type>.retriever_weight
|
||||
- types.<type>.retriever.weight
|
||||
- types.<type>.retrieval.weight
|
||||
- defaults.retriever_weight
|
||||
- defaults.retriever.weight
|
||||
- global.retriever_weight
|
||||
- global.retriever.weight
|
||||
"""
|
||||
candidates = [
|
||||
f"types.{note_type}.retriever_weight",
|
||||
f"types.{note_type}.retriever.weight",
|
||||
|
|
@ -143,9 +114,6 @@ def effective_retriever_weight_from_registry(note_type: str, reg: dict) -> Tuple
|
|||
return None, None
|
||||
|
||||
def compute_effective_retriever_weight(fm: Dict[str, Any], note_type: str, reg: dict) -> Tuple[float, str]:
|
||||
"""
|
||||
Priorität: Frontmatter > Registry > Default (1.0). Gibt (Wert, Quelle) zurück.
|
||||
"""
|
||||
if fm.get("retriever_weight") is not None:
|
||||
v = _as_float(fm.get("retriever_weight"))
|
||||
if v is not None:
|
||||
|
|
@ -166,13 +134,7 @@ def collections(prefix: str) -> Tuple[str, str, str]:
|
|||
def fetch_existing_note_payload(client, prefix: str, note_id: str) -> Optional[Dict]:
|
||||
notes_col, _, _ = collections(prefix)
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
points, _ = client.scroll(
|
||||
collection_name=notes_col,
|
||||
scroll_filter=f,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=1,
|
||||
)
|
||||
points, _ = client.scroll(collection_name=notes_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=1)
|
||||
if not points:
|
||||
return None
|
||||
return points[0].payload or {}
|
||||
|
|
@ -182,13 +144,7 @@ def list_qdrant_note_ids(client, prefix: str) -> Set[str]:
|
|||
out: Set[str] = set()
|
||||
next_page = None
|
||||
while True:
|
||||
pts, next_page = client.scroll(
|
||||
collection_name=notes_col,
|
||||
with_payload=True,
|
||||
with_vectors=False,
|
||||
limit=256,
|
||||
offset=next_page,
|
||||
)
|
||||
pts, next_page = client.scroll(collection_name=notes_col, with_payload=True, with_vectors=False, limit=256, offset=next_page)
|
||||
if not pts:
|
||||
break
|
||||
for p in pts:
|
||||
|
|
@ -205,11 +161,7 @@ def purge_note_artifacts(client, prefix: str, note_id: str) -> None:
|
|||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
for col in (chunks_col, edges_col):
|
||||
try:
|
||||
client.delete(
|
||||
collection_name=col,
|
||||
points_selector=rest.FilterSelector(filter=filt),
|
||||
wait=True
|
||||
)
|
||||
client.delete(collection_name=col, points_selector=rest.FilterSelector(filter=filt), wait=True)
|
||||
except Exception as e:
|
||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} via filter failed: {e}"}))
|
||||
|
||||
|
|
@ -218,23 +170,13 @@ def delete_note_everywhere(client, prefix: str, note_id: str) -> None:
|
|||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
for col in (edges_col, chunks_col, notes_col):
|
||||
try:
|
||||
client.delete(
|
||||
collection_name=col,
|
||||
points_selector=rest.FilterSelector(filter=filt),
|
||||
wait=True
|
||||
)
|
||||
client.delete(collection_name=col, points_selector=rest.FilterSelector(filter=filt), wait=True)
|
||||
except Exception as e:
|
||||
print(json.dumps({"note_id": note_id, "warn": f"delete in {col} failed: {e}"}))
|
||||
|
||||
def _has_any_point(client, collection: str, note_id: str) -> bool:
|
||||
filt = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
pts, _ = client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=filt,
|
||||
with_payload=False,
|
||||
with_vectors=False,
|
||||
limit=1,
|
||||
)
|
||||
pts, _ = client.scroll(collection_name=collection, scroll_filter=filt, with_payload=False, with_vectors=False, limit=1)
|
||||
return bool(pts)
|
||||
|
||||
def artifacts_missing(client, prefix: str, note_id: str) -> Tuple[bool, bool]:
|
||||
|
|
@ -255,34 +197,30 @@ def _resolve_mode(m: Optional[str]) -> str:
|
|||
def main() -> None:
|
||||
load_dotenv()
|
||||
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="scripts.import_markdown",
|
||||
description="Importiert Markdown-Notizen in Qdrant (Notes/Chunks/Edges)."
|
||||
)
|
||||
ap.add_argument("--vault", required=True, help="Pfad zum Vault (Ordner mit .md-Dateien)")
|
||||
ap.add_argument("--only-path", help="Nur diese Datei verarbeiten (absolut oder relativ)")
|
||||
ap.add_argument("--apply", action="store_true", help="Schreibt nach Qdrant (sonst Dry-Run)")
|
||||
ap.add_argument("--purge-before-upsert", action="store_true", help="Alte Chunks/Edges der Note vorher löschen")
|
||||
ap.add_argument("--force-replace", action="store_true", help="Note/Chunks/Edges unabhängig von Hash neu schreiben")
|
||||
ap.add_argument("--note-id", help="Nur Notes mit dieser ID verarbeiten (Filter)")
|
||||
ap.add_argument("--note-scope-refs", action="store_true", help="Note-scope References/Backlinks erzeugen")
|
||||
ap.add_argument("--hash-mode", help="body|frontmatter|full (Default body)")
|
||||
ap.add_argument("--hash-source", help="parsed|raw (Default parsed)")
|
||||
ap.add_argument("--hash-normalize", help="canonical|none (Default canonical)")
|
||||
ap.add_argument("--compare-text", action="store_true", help="Parsed fulltext zusätzlich direkt vergleichen")
|
||||
ap.add_argument("--baseline-modes", action="store_true", help="Fehlende Hash-Varianten still nachtragen (Notes)")
|
||||
ap.add_argument("--sync-deletes", action="store_true", help="Qdrant->Vault Lösch-Sync (mit --apply ausführen)")
|
||||
ap.add_argument("--prefix", help="Collection-Prefix (überschreibt ENV COLLECTION_PREFIX)")
|
||||
ap = argparse.ArgumentParser(prog="scripts.import_markdown", description="Importiert Markdown-Notizen in Qdrant (Notes/Chunks/Edges).")
|
||||
ap.add_argument("--vault", required=True)
|
||||
ap.add_argument("--only-path")
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--purge-before-upsert", action="store_true")
|
||||
ap.add_argument("--force-replace", action="store_true")
|
||||
ap.add_argument("--note-id")
|
||||
ap.add_argument("--note-scope-refs", action="store_true")
|
||||
ap.add_argument("--hash-mode")
|
||||
ap.add_argument("--hash-source")
|
||||
ap.add_argument("--hash-normalize")
|
||||
ap.add_argument("--compare-text", action="store_true")
|
||||
ap.add_argument("--baseline-modes", action="store_true")
|
||||
ap.add_argument("--sync-deletes", action="store_true")
|
||||
ap.add_argument("--prefix")
|
||||
args = ap.parse_args()
|
||||
|
||||
mode = _resolve_mode(args.hash_mode) # body|frontmatter|full
|
||||
src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed") # parsed|raw
|
||||
norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical") # canonical|none
|
||||
mode = _resolve_mode(args.hash_mode)
|
||||
src = _env("MINDNET_HASH_SOURCE", args.hash_source or "parsed")
|
||||
norm = _env("MINDNET_HASH_NORMALIZE", args.hash_normalize or "canonical")
|
||||
note_scope_refs_env = (_env("MINDNET_NOTE_SCOPE_REFS", "false") == "true")
|
||||
note_scope_refs = args.note_scope_refs or note_scope_refs_env
|
||||
compare_text = args.compare_text or (_env("MINDNET_COMPARE_TEXT", "false") == "true")
|
||||
|
||||
# Qdrant
|
||||
cfg = QdrantConfig.from_env()
|
||||
if args.prefix:
|
||||
cfg.prefix = args.prefix.strip()
|
||||
|
|
@ -290,23 +228,19 @@ def main() -> None:
|
|||
ensure_collections(client, cfg.prefix, cfg.dim)
|
||||
ensure_payload_indexes(client, cfg.prefix)
|
||||
|
||||
# Type-Registry laden
|
||||
reg = load_type_registry()
|
||||
# types_cfg: Direkter Zugriff auf den Knoten "types" (oder leeres Dict)
|
||||
types_cfg = reg.get("types") if isinstance(reg, dict) else {}
|
||||
types_cfg_node = reg.get("types") if isinstance(reg, dict) else {}
|
||||
|
||||
root = os.path.abspath(args.vault)
|
||||
|
||||
# Dateiliste
|
||||
# Datei-Liste
|
||||
if args.only_path:
|
||||
only = os.path.abspath(args.only_path)
|
||||
files = [only]
|
||||
files = [os.path.abspath(args.only_path)]
|
||||
else:
|
||||
files: List[str] = []
|
||||
for dirpath, _, filenames in os.walk(root):
|
||||
for fn in filenames:
|
||||
if not fn.lower().endswith(".md"):
|
||||
continue
|
||||
if fn.lower().endswith(".md"):
|
||||
p = os.path.join(dirpath, fn)
|
||||
pn = p.replace("\\", "/")
|
||||
if any(ex in pn for ex in ["/.obsidian/", "/_backup_frontmatter/", "/_imported/"]):
|
||||
|
|
@ -317,7 +251,7 @@ def main() -> None:
|
|||
print("Keine Markdown-Dateien gefunden.", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# Optional: Sync-Deletes vorab
|
||||
# Optional: Sync-Deletes
|
||||
if args.sync_deletes:
|
||||
vault_note_ids: Set[str] = set()
|
||||
for path in files:
|
||||
|
|
@ -333,22 +267,15 @@ def main() -> None:
|
|||
continue
|
||||
qdrant_note_ids = list_qdrant_note_ids(client, cfg.prefix)
|
||||
to_delete = sorted(qdrant_note_ids - vault_note_ids)
|
||||
print(json.dumps({
|
||||
"action": "sync-deletes",
|
||||
"prefix": cfg.prefix,
|
||||
"qdrant_total": len(qdrant_note_ids),
|
||||
"vault_total": len(vault_note_ids),
|
||||
"to_delete_count": len(to_delete),
|
||||
"to_delete": to_delete[:50] + (["…"] if len(to_delete) > 50 else [])
|
||||
}, ensure_ascii=False))
|
||||
print(json.dumps({"action":"sync-deletes","prefix":cfg.prefix,"qdrant_total":len(qdrant_note_ids),"vault_total":len(vault_note_ids),"to_delete_count":len(to_delete),"to_delete":to_delete[:50]+(["…"] if len(to_delete)>50 else [])}, ensure_ascii=False))
|
||||
if args.apply and to_delete:
|
||||
for nid in to_delete:
|
||||
print(json.dumps({"action":"delete","note_id":nid,"decision":"apply"}))
|
||||
delete_note_everywhere(client, cfg.prefix, nid)
|
||||
|
||||
key_current = f"{mode}:{src}:{norm}"
|
||||
|
||||
processed = 0
|
||||
|
||||
for path in files:
|
||||
try:
|
||||
parsed = read_markdown(path)
|
||||
|
|
@ -358,7 +285,7 @@ def main() -> None:
|
|||
print(json.dumps({"path": path, "error": f"read_markdown failed: {type(e).__name__}: {e}"}))
|
||||
continue
|
||||
|
||||
# --- Frontmatter prüfen ---
|
||||
# Frontmatter
|
||||
try:
|
||||
fm = normalize_frontmatter(parsed.frontmatter)
|
||||
validate_required_frontmatter(fm)
|
||||
|
|
@ -371,91 +298,68 @@ def main() -> None:
|
|||
|
||||
processed += 1
|
||||
|
||||
# --- Typ & Profile aus Registry übernehmen ---
|
||||
try:
|
||||
# Typ + Profile/Weight
|
||||
note_type = resolve_note_type(fm.get("type"), reg)
|
||||
except Exception:
|
||||
note_type = (fm.get("type") or "concept")
|
||||
fm["type"] = note_type or fm.get("type") or "concept"
|
||||
|
||||
prof = effective_chunk_profile(note_type, reg)
|
||||
if prof:
|
||||
fm["chunk_profile"] = prof
|
||||
|
||||
# --- retriever_weight auflösen (FM > Registry > 1.0) ---
|
||||
rw, rw_source = compute_effective_retriever_weight(fm, note_type, reg)
|
||||
fm["retriever_weight"] = rw # Spiegeln ins FM für Builder
|
||||
fm["retriever_weight"] = rw
|
||||
|
||||
# --- Payload aufbauen (inkl. Hashes) ---
|
||||
# Note-Payload
|
||||
try:
|
||||
note_pl = make_note_payload(
|
||||
parsed,
|
||||
vault_root=root,
|
||||
hash_mode=mode,
|
||||
hash_normalize=norm,
|
||||
hash_source=src,
|
||||
file_path=path,
|
||||
)
|
||||
note_pl = make_note_payload(parsed, vault_root=root, hash_mode=mode, hash_normalize=norm, hash_source=src, file_path=path)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "error": f"make_note_payload failed: {type(e).__name__}: {e}"}))
|
||||
continue
|
||||
|
||||
if not note_pl.get("fulltext"):
|
||||
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
||||
|
||||
# retriever_weight **immer** in Note-Payload schreiben (redundant aber robust)
|
||||
try:
|
||||
note_pl["retriever_weight"] = float(rw)
|
||||
except Exception:
|
||||
note_pl["retriever_weight"] = 1.0
|
||||
# Explizites Spiegeln: chunk_profile & retriever_weight in Note
|
||||
if fm.get("chunk_profile") is not None:
|
||||
note_pl["chunk_profile"] = fm.get("chunk_profile")
|
||||
note_pl["retriever_weight"] = float(rw) if isinstance(rw, (int, float)) else 1.0
|
||||
|
||||
note_id = note_pl.get("note_id") or fm.get("id")
|
||||
if not note_id:
|
||||
print(json.dumps({"path": path, "error": "Missing note_id after payload build"}))
|
||||
continue
|
||||
|
||||
# --- bestehenden Payload laden (zum Diff) ---
|
||||
old_payload = None if args.force_replace else fetch_existing_note_payload(client, cfg.prefix, note_id)
|
||||
# Bestehende Note in Qdrant (für Changed-Detektion)
|
||||
old_payload = fetch_existing_note_payload(client, cfg.prefix, note_id)
|
||||
has_old = old_payload is not None
|
||||
|
||||
old_hashes = (old_payload or {}).get("hashes") or {}
|
||||
old_hash_exact = old_hashes.get(key_current)
|
||||
new_hash_exact = (note_pl.get("hashes") or {}).get(key_current)
|
||||
needs_baseline = (old_hash_exact is None)
|
||||
|
||||
hash_changed = (old_hash_exact is not None and new_hash_exact is not None and old_hash_exact != new_hash_exact)
|
||||
|
||||
text_changed = False
|
||||
if compare_text:
|
||||
old_text = (old_payload or {}).get("fulltext") or ""
|
||||
new_text = note_pl.get("fulltext") or ""
|
||||
text_changed = (old_text != new_text)
|
||||
# Optionaler Text-Vergleich (teuer)
|
||||
# if compare_text:
|
||||
# old_text = (old_payload or {}).get("fulltext") or ""
|
||||
# new_text = note_pl.get("fulltext") or (getattr(parsed, "body", "") or "")
|
||||
# text_changed = (old_text != new_text)
|
||||
|
||||
changed = args.force_replace or (not has_old) or hash_changed or text_changed
|
||||
# Treat payload field diffs (retriever_weight/chunk_profile) as change to force update
|
||||
|
||||
# Feld-basierte Change-Erkennung (erzwingt Update, wenn Werte nicht stimmen)
|
||||
if has_old:
|
||||
old_rw = old_payload.get('retriever_weight')
|
||||
old_cp = old_payload.get('chunk_profile')
|
||||
if old_rw != note_pl.get('retriever_weight') or old_cp != fm.get('chunk_profile'):
|
||||
if old_payload.get("retriever_weight") != note_pl.get("retriever_weight"):
|
||||
changed = True
|
||||
if old_payload.get("chunk_profile") != note_pl.get("chunk_profile"):
|
||||
changed = True
|
||||
|
||||
do_baseline_only = (args.baseline_modes and has_old and needs_baseline and not changed)
|
||||
|
||||
# --- Chunks + Embeddings vorbereiten ---
|
||||
# Chunks
|
||||
try:
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
chunks = assemble_chunks(fm["id"], body_text, fm.get("type", "concept"))
|
||||
|
||||
# *** MINIMALFIX ***
|
||||
# Übergib an make_chunk_payloads ein Note-Objekt mit verschachtelter 'frontmatter'
|
||||
# und reiche die Registry ('types_cfg') durch, damit chunk_payload.py aus types.yaml lesen kann.
|
||||
# WICHTIG: Note-Objekt mit verschachtelter FM + Registry an Chunk-Builder übergeben
|
||||
chunk_note = {
|
||||
"frontmatter": fm,
|
||||
"id": fm.get("id"),
|
||||
"type": fm.get("type"),
|
||||
"title": fm.get("title"),
|
||||
"path": note_pl.get("path") or path,
|
||||
# optional: direkte Felder, falls ein alter Builder darauf schaut
|
||||
"note_id": note_pl.get("note_id"),
|
||||
"tags": fm.get("tags"),
|
||||
}
|
||||
|
|
@ -464,29 +368,29 @@ def main() -> None:
|
|||
note_pl["path"],
|
||||
chunks,
|
||||
note_text=body_text,
|
||||
types_cfg=(reg.get('types') if isinstance(reg, dict) and isinstance(reg.get('types'), dict) else reg if isinstance(reg, dict) else {}),
|
||||
types_cfg=(reg.get("types") if isinstance(reg, dict) and isinstance(reg.get("types"), dict) else reg if isinstance(reg, dict) else {}),
|
||||
file_path=path,
|
||||
)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "error": f"chunk build failed: {type(e).__name__}: {e}"}))
|
||||
continue
|
||||
|
||||
# **Felder-Policy**: `index` (0-based) & `ord` (1-based) sicherstellen; keine doppelten Alias-Felder
|
||||
# Explizites Spiegeln: retriever_weight & chunk_profile auf JEDEN Chunk
|
||||
rwf = float(rw) if isinstance(rw, (int, float)) else 1.0
|
||||
cpv = fm.get("chunk_profile")
|
||||
for i, pl in enumerate(chunk_pls):
|
||||
if "index" not in pl:
|
||||
pl["index"] = i
|
||||
pl["ord"] = int(pl.get("index", i)) + 1
|
||||
for alias in ("chunk_num", "Chunk_Number"):
|
||||
if alias in pl:
|
||||
pl.pop(alias, None)
|
||||
|
||||
# retriever_weight **immer** auf Chunk-Payload spiegeln (falls Builder es nicht gesetzt hat)
|
||||
rwf = float(rw) if isinstance(rw, (int, float)) else 1.0
|
||||
for pl in chunk_pls:
|
||||
if "retriever_weight" not in pl:
|
||||
pl["retriever_weight"] = rwf
|
||||
if cpv is not None:
|
||||
pl["chunk_profile"] = cpv
|
||||
# entferne alte Aliasse
|
||||
for alias in ("chunk_num", "Chunk_Number"):
|
||||
pl.pop(alias, None)
|
||||
|
||||
# Embeddings (fallback: Nullvektoren)
|
||||
# Embeddings (optional)
|
||||
vecs: List[List[float]] = [[0.0] * int(cfg.dim) for _ in chunk_pls]
|
||||
if embed_texts and chunk_pls:
|
||||
try:
|
||||
|
|
@ -495,87 +399,61 @@ def main() -> None:
|
|||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"embed_texts failed, using zeros: {e}"}))
|
||||
|
||||
# --- Fehlende Artefakte in Qdrant ermitteln ---
|
||||
# Artefakte vorhanden?
|
||||
chunks_missing, edges_missing = artifacts_missing(client, cfg.prefix, note_id)
|
||||
|
||||
# --- Edges (robust) ---
|
||||
# Edges
|
||||
edges: List[Dict[str, Any]] = []
|
||||
edges_failed = False
|
||||
should_build_edges = (changed and (not do_baseline_only)) or edges_missing
|
||||
should_build_edges = (changed and True) or edges_missing # wenn Note geändert oder Kanten fehlen
|
||||
if should_build_edges:
|
||||
try:
|
||||
note_refs = note_pl.get("references") or []
|
||||
# Positionsargumente → kompatibel mit derive_edges.py UND edges.py
|
||||
note_refs = note_pl.get("references") or ""
|
||||
edges = build_edges_for_note(note_id, chunk_pls, note_refs, include_note_scope_refs=note_scope_refs)
|
||||
except Exception as e:
|
||||
edges_failed = True
|
||||
edges = []
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"build_edges_for_note failed, skipping edges: {type(e).__name__}: {e}"}))
|
||||
|
||||
# --- Summary (stdout) ---
|
||||
summary = {
|
||||
# Summary
|
||||
print(json.dumps({
|
||||
"note_id": note_id,
|
||||
"title": fm.get("title"),
|
||||
"type": fm.get("type"),
|
||||
"rw": rw,
|
||||
"rw_source": rw_source,
|
||||
"chunk_profile": fm.get("chunk_profile"),
|
||||
"chunks": len(chunk_pls),
|
||||
"edges": len(edges),
|
||||
"edges_failed": edges_failed,
|
||||
"changed": changed,
|
||||
"chunks_missing": chunks_missing,
|
||||
"edges_missing": edges_missing,
|
||||
"needs_baseline_for_mode": needs_baseline,
|
||||
"decision": ("baseline-only" if args.apply and do_baseline_only else
|
||||
"apply" if args.apply and (changed or chunks_missing or edges_missing) else
|
||||
"apply-skip-unchanged" if args.apply and not (changed or chunks_missing or edges_missing) else
|
||||
"dry-run"),
|
||||
"path": note_pl["path"],
|
||||
"hash_mode": mode,
|
||||
"hash_normalize": norm,
|
||||
"hash_source": src,
|
||||
"decision": ("apply" if args.apply else "dry-run"),
|
||||
"prefix": cfg.prefix,
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
"path": note_pl["path"]
|
||||
}, ensure_ascii=False))
|
||||
|
||||
# --- Writes ---
|
||||
# Writes
|
||||
if not args.apply:
|
||||
continue
|
||||
|
||||
if do_baseline_only:
|
||||
merged_hashes = {}
|
||||
merged_hashes.update(old_hashes)
|
||||
merged_hashes.update(note_pl.get("hashes") or {})
|
||||
if old_payload:
|
||||
note_pl["hash_fulltext"] = old_payload.get("hash_fulltext", note_pl.get("hash_fulltext"))
|
||||
note_pl["hash_signature"] = old_payload.get("hash_signature", note_pl.get("hash_signature"))
|
||||
note_pl["hashes"] = merged_hashes
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
continue
|
||||
|
||||
# Wenn nichts geändert und keine Artefakte fehlen → nichts zu tun
|
||||
if not changed and not (chunks_missing or edges_missing):
|
||||
continue
|
||||
|
||||
# Purge nur bei echten Änderungen
|
||||
if args.purge_before_upsert and has_old and changed:
|
||||
# Purge bei Änderungen
|
||||
if args.purge_before_upsert and changed:
|
||||
try:
|
||||
purge_note_artifacts(client, cfg.prefix, note_id)
|
||||
except Exception as e:
|
||||
print(json.dumps({"path": path, "note_id": note_id, "warn": f"purge failed: {e}"}))
|
||||
|
||||
# Note nur bei Änderungen neu schreiben
|
||||
# Note
|
||||
if changed:
|
||||
notes_name, note_pts = points_for_note(cfg.prefix, note_pl, None, cfg.dim)
|
||||
upsert_batch(client, notes_name, note_pts)
|
||||
|
||||
# Chunks schreiben, wenn geändert ODER vorher fehlend
|
||||
# Chunks (wenn geändert ODER vorher keine vorhanden)
|
||||
if chunk_pls and (changed or chunks_missing):
|
||||
chunks_name, chunk_pts = points_for_chunks(cfg.prefix, chunk_pls, vecs)
|
||||
upsert_batch(client, chunks_name, chunk_pts)
|
||||
|
||||
# Edges schreiben, wenn vorhanden und (geändert ODER vorher fehlend)
|
||||
# Edges (wenn geändert ODER vorher keine vorhanden)
|
||||
if edges and (changed or edges_missing):
|
||||
edges_name, edge_pts = points_for_edges(cfg.prefix, edges)
|
||||
upsert_batch(client, edges_name, edge_pts)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user