app/core/qdrant.py aktualisiert
Some checks failed
Deploy mindnet to llm-node / deploy (push) Failing after 1s
Some checks failed
Deploy mindnet to llm-node / deploy (push) Failing after 1s
This commit is contained in:
parent
7f9981bf31
commit
fd30e2c026
|
|
@ -2,37 +2,44 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Name: app/core/qdrant.py
|
Name: app/core/qdrant.py
|
||||||
Version: v1.3.0 (2025-09-05)
|
Version: v1.3.1 (2025-09-05)
|
||||||
|
|
||||||
Kurzbeschreibung:
|
Kurzbeschreibung:
|
||||||
Qdrant-Client & Collection-Setup für mindnet.
|
Qdrant-Client & Collection-Setup für mindnet.
|
||||||
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges vorhanden sind.
|
- Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
|
||||||
- **NEU:** ensure_collections(..., destructive=False) → keine Datenverluste im Dry-Run.
|
|
||||||
- Edges-Collection nutzt 1D Dummy-Vektor (Workaround für Python-Client).
|
- Edges-Collection nutzt 1D Dummy-Vektor (Workaround für Python-Client).
|
||||||
|
- **Nicht-destruktiv per Default**: ensure_collections(..., destructive=False).
|
||||||
|
- **Abwärtskompatibel**: collection_names(prefix) wieder verfügbar.
|
||||||
|
|
||||||
API:
|
Aufruf/Verwendung:
|
||||||
- QdrantConfig.from_env()
|
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, collection_names
|
||||||
- get_client(cfg)
|
|
||||||
- ensure_collections(client, prefix, dim, destructive=False)
|
Umgebungsvariablen (optional):
|
||||||
|
QDRANT_URL | QDRANT_HOST/QDRANT_PORT, QDRANT_API_KEY,
|
||||||
|
COLLECTION_PREFIX (Default "mindnet"), VECTOR_DIM (Default 384)
|
||||||
|
|
||||||
Änderungen:
|
Änderungen:
|
||||||
v1.3.0: Destruktive Re-Creation von {prefix}_edges nur noch optional via destructive=True.
|
v1.3.1: Helper collection_names(prefix) wiederhergestellt (für reset_qdrant usw.).
|
||||||
Default ist sicher (keine Löschung vorhandener Collections).
|
v1.3.0: ensure_collections(..., destructive=False) – keine stillen Drops im Dry-Run.
|
||||||
v1.2.x und älter: konnten {prefix}_edges automatisch löschen/re-anlegen.
|
Edges-Collection nur bei explicit destructive=True neu anlegen.
|
||||||
|
≤v1.2.x: Konnte {prefix}_edges bei fehlender VectorConfig automatisch neu erstellen (riskant).
|
||||||
|
|
||||||
Quellen:
|
Bezug/Quelle:
|
||||||
- QdrantClient & REST-Modelle (qdrant-client)
|
Alte Core-Variante enthielt collection_names(prefix); diverse Scripts nutzen das weiterhin. :contentReference[oaicite:1]{index=1}
|
||||||
- Mindnet Edge-Workaround (1D-Vektor) wie zuvor. :contentReference[oaicite:1]{index=1}
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import os
|
import os
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Optional
|
from typing import Optional, Tuple
|
||||||
|
|
||||||
from qdrant_client import QdrantClient
|
from qdrant_client import QdrantClient
|
||||||
from qdrant_client.http import models as rest
|
from qdrant_client.http import models as rest
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------
|
||||||
|
# Konfiguration
|
||||||
|
# -------------------------------
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class QdrantConfig:
|
class QdrantConfig:
|
||||||
url: str
|
url: str
|
||||||
|
|
@ -53,10 +60,18 @@ class QdrantConfig:
|
||||||
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
|
return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------
|
||||||
|
# Client
|
||||||
|
# -------------------------------
|
||||||
|
|
||||||
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
def get_client(cfg: QdrantConfig) -> QdrantClient:
|
||||||
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
|
return QdrantClient(url=cfg.url, api_key=cfg.api_key)
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------
|
||||||
|
# Collection-Erzeuger (Hilfsfunktionen)
|
||||||
|
# -------------------------------
|
||||||
|
|
||||||
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
|
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
|
||||||
if not client.collection_exists(name):
|
if not client.collection_exists(name):
|
||||||
client.create_collection(
|
client.create_collection(
|
||||||
|
|
@ -64,7 +79,6 @@ def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
|
||||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
|
def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
|
||||||
if not client.collection_exists(name):
|
if not client.collection_exists(name):
|
||||||
client.create_collection(
|
client.create_collection(
|
||||||
|
|
@ -72,7 +86,6 @@ def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
|
||||||
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _create_edges(client: QdrantClient, name: str) -> None:
|
def _create_edges(client: QdrantClient, name: str) -> None:
|
||||||
if not client.collection_exists(name):
|
if not client.collection_exists(name):
|
||||||
client.create_collection(
|
client.create_collection(
|
||||||
|
|
@ -81,15 +94,15 @@ def _create_edges(client: QdrantClient, name: str) -> None:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------
|
||||||
|
# Public API
|
||||||
|
# -------------------------------
|
||||||
|
|
||||||
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
|
def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
|
||||||
"""
|
"""
|
||||||
Stellt sicher, dass die drei Collections existieren.
|
Stellt sicher, dass die drei Collections existieren.
|
||||||
- Default **nicht destruktiv**: vorhandene Collections bleiben unangetastet.
|
- Default **nicht destruktiv**: vorhandene Collections bleiben unangetastet.
|
||||||
- Nur wenn 'destructive=True', wird eine ungeeignete Edges-Collection gelöscht und neu angelegt.
|
- Nur wenn 'destructive=True', wird eine ungeeignete Edges-Collection gelöscht und neu angelegt.
|
||||||
|
|
||||||
Hinweis:
|
|
||||||
Frühere Versionen haben {prefix}_edges ggf. automatisch gelöscht (riskant in Dry-Runs).
|
|
||||||
Diese Version tut das **nur** auf ausdrücklichen Wunsch (destructive=True).
|
|
||||||
"""
|
"""
|
||||||
notes = f"{prefix}_notes"
|
notes = f"{prefix}_notes"
|
||||||
chunks = f"{prefix}_chunks"
|
chunks = f"{prefix}_chunks"
|
||||||
|
|
@ -105,16 +118,35 @@ def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive:
|
||||||
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
|
vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
|
||||||
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
|
has_vectors = getattr(vectors_cfg, "vectors", None) is not None
|
||||||
except Exception:
|
except Exception:
|
||||||
has_vectors = True # konservativ: nichts anfassen
|
# konservativ: nichts anfassen, um Datenverlust zu vermeiden
|
||||||
|
has_vectors = True
|
||||||
|
|
||||||
if not has_vectors:
|
if not has_vectors:
|
||||||
if destructive:
|
if destructive:
|
||||||
client.delete_collection(edges)
|
client.delete_collection(edges)
|
||||||
_create_edges(client, edges)
|
_create_edges(client, edges)
|
||||||
else:
|
else:
|
||||||
# Sicher: behalten und nur warnen – keine Datenverluste
|
print(
|
||||||
print(f"[ensure_collections] WARN: '{edges}' ohne VectorConfig gefunden; "
|
f"[ensure_collections] WARN: '{edges}' ohne VectorConfig gefunden; "
|
||||||
f"keine destruktive Änderung (destructive=False).", flush=True)
|
f"keine destruktive Änderung (destructive=False).",
|
||||||
# sonst: alles gut, nichts tun
|
flush=True,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
_create_edges(client, edges)
|
_create_edges(client, edges)
|
||||||
|
|
||||||
|
|
||||||
|
def collection_names(prefix: str) -> Tuple[str, str, str]:
|
||||||
|
"""
|
||||||
|
Abwärtskompatibler Helper für Scripts:
|
||||||
|
returns (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
||||||
|
"""
|
||||||
|
return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")
|
||||||
|
|
||||||
|
|
||||||
|
def wipe_collections(client: QdrantClient, prefix: str) -> None:
|
||||||
|
"""
|
||||||
|
Löscht alle drei Collections – nur verwenden, wenn bewusst ein Clean-Rebuild gewünscht ist.
|
||||||
|
"""
|
||||||
|
for name in collection_names(prefix):
|
||||||
|
if client.collection_exists(name):
|
||||||
|
client.delete_collection(name)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user