From 8df2069c6fe422b172e1e0008a46603b3661f932 Mon Sep 17 00:00:00 2001 From: Lars Date: Fri, 5 Sep 2025 12:03:33 +0200 Subject: [PATCH] scripts/import_markdown.py aktualisiert --- scripts/import_markdown.py | 51 ++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/scripts/import_markdown.py b/scripts/import_markdown.py index 75dc536..7aafd05 100644 --- a/scripts/import_markdown.py +++ b/scripts/import_markdown.py @@ -2,12 +2,12 @@ # -*- coding: utf-8 -*- """ Script: scripts/import_markdown.py -Version: v2.4.0 (2025-09-05) +Version: v2.4.1 (2025-09-05) Beschreibung Importiert Markdown-Notizen in Qdrant (Notes, Chunks, Edges) mit Delta-Detection. - Chunking + Embedding (MiniLM 384d, externer Embed-Server) - - Edges direkt beim Import aus Wikilinks ([[…]]) ableiten + - Edges direkt beim Import aus Wikilinks ([[…]]) ableiten (inkl. references_at) - Idempotenz via stabile UUIDv5-IDs und hash_fulltext - Create/Update/Skip pro Note: * Unverändert (hash_fulltext gleich) ⇒ Skip @@ -31,11 +31,13 @@ Hinweise - Qdrant via ENV: QDRANT_URL, QDRANT_API_KEY, COLLECTION_PREFIX, VECTOR_DIM Changelog + v2.4.1: FIX – Kompatibilität zu verschiedenen qdrant-client Versionen: + `scroll()`-Rückgabe kann 2- oder 3-teilig sein → robustes Unpacking. v2.4.0: NEU – Delta-Detection über hash_fulltext; Skip/Replace-Entscheidung. Purge bei Updates: löscht Chunks & Edges der Quelle, dann Upsert. Dry-Run garantiert ohne Mutationen. v2.3.1: FIX – Für derive_wikilink_edges werden echte Chunk-Texte übergeben - ({"chunk_id","text"}) → erzeugt `references_at`. :contentReference[oaicite:5]{index=5} + ({"chunk_id","text"}) → erzeugt `references_at`. v2.3.0: Umstellung auf app.core.derive_edges; Edge-IDs inkl. Occurrence. """ @@ -46,16 +48,15 @@ from dotenv import load_dotenv from qdrant_client.http import models as rest from qdrant_client import QdrantClient -from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter # FM-Pflichtfelder usw. :contentReference[oaicite:6]{index=6} +from app.core.parser import read_markdown, normalize_frontmatter, validate_required_frontmatter from app.core.note_payload import make_note_payload from app.core.validate_note import validate_note_payload from app.core.chunker import assemble_chunks from app.core.chunk_payload import make_chunk_payloads from app.core.embed import embed_texts, embed_one from app.core.qdrant import QdrantConfig, ensure_collections, get_client -from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch # stabile IDs, 1D-Dummy-Vektor für Edges :contentReference[oaicite:7]{index=7} -from app.core.derive_edges import build_note_index, derive_wikilink_edges # benötigt Chunk-texte für references_at :contentReference[oaicite:8]{index=8} - +from app.core.qdrant_points import points_for_chunks, points_for_note, points_for_edges, upsert_batch +from app.core.derive_edges import build_note_index, derive_wikilink_edges # ------------------------------- # Utility @@ -77,11 +78,30 @@ def sha256_hex(text: str) -> str: def collection_names(prefix: str) -> Tuple[str, str, str]: return f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges" +def _scroll(client: QdrantClient, **kwargs): + """ + Wrapper für client.scroll, der 2-teilige und 3-teilige Rückgaben unterstützt. + Neuere qdrant-client Versionen liefern (points, next_page), ältere evtl. (points, offset, next_page). + """ + res = client.scroll(**kwargs) + if isinstance(res, tuple): + if len(res) == 2: + points, _ = res + return points + elif len(res) == 3: + points, _, _ = res + return points + # Fallback: wenn sich API ändert, versuchen wir, wie eine Sequenz zuzugreifen + try: + return res[0] + except Exception: + return [] + def fetch_existing_note_hash(client: QdrantClient, prefix: str, note_id: str) -> Optional[str]: """Liest hash_fulltext der Note aus Qdrant (falls vorhanden).""" notes_col, _, _ = collection_names(prefix) f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) - pts, _, _ = client.scroll(collection_name=notes_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=1) + pts = _scroll(client, collection_name=notes_col, scroll_filter=f, with_payload=True, with_vectors=False, limit=1) if not pts: return None pl = (pts[0].payload or {}) @@ -106,7 +126,6 @@ def purge_note_chunks(client: QdrantClient, prefix: str, note_id: str) -> None: f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))]) client.delete(collection_name=chunks_col, points_selector=f, wait=True) - # ------------------------------- # Hauptlogik # ------------------------------- @@ -149,14 +168,14 @@ def main(): continue rel = os.path.relpath(parsed.path, root).replace("\\","/") note_stubs.append({"note_id": fm["id"], "title": fm.get("title",""), "path": rel}) - note_index = build_note_index(note_stubs) # benötigt für Wikilink-Auflösung :contentReference[oaicite:9]{index=9} + note_index = build_note_index(note_stubs) total_notes = 0 for path in files: parsed = read_markdown(path) fm = normalize_frontmatter(parsed.frontmatter) try: - validate_required_frontmatter(fm) # Pflichtfelder lt. Schema/Design + validate_required_frontmatter(fm) # Pflichtfelder lt. Schema/Design except Exception: continue if args.note_id and fm.get("id") != args.note_id: @@ -165,7 +184,7 @@ def main(): total_notes += 1 note_id = fm["id"] - # --- Delta-Detection: Hash berechnen + bestehenden Hash aus Qdrant lesen --- + # --- Delta-Detection --- fulltext = parsed.body new_hash = sha256_hex(fulltext) old_hash = fetch_existing_note_hash(client, cfg.prefix, note_id) @@ -173,11 +192,11 @@ def main(): # Note-Payload note_pl = make_note_payload(parsed, vault_root=root) - note_pl["fulltext"] = fulltext # für derive_edges (references) :contentReference[oaicite:11]{index=11} - note_pl["hash_fulltext"] = new_hash # Schema-Feld vorhanden :contentReference[oaicite:12]{index=12} + note_pl["fulltext"] = fulltext # für derive_edges (references) + note_pl["hash_fulltext"] = new_hash # Schema-Feld vorhanden validate_note_payload(note_pl) - # Früher Exit (Dry-Run/Skip): wenn unverändert und kein force-replace, nur melden + # Früher Exit (Dry-Run/Skip) if not changed: print(json.dumps({ "note_id": note_id, @@ -206,7 +225,7 @@ def main(): note_vec = embed_one(fulltext) if args.embed_note else None # Kanten (Note- & Chunk-Ebene) - edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index) # erzeugt references, backlink, references_at :contentReference[oaicite:13]{index=13} + edges = derive_wikilink_edges(note_pl, chunks_for_links, note_index) # Dry-Run-Ausgabe print(json.dumps({