mindnet/app/core/qdrant.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Name:        app/core/qdrant.py
Version:     v1.6.0 (2025-11-08)

Kurzbeschreibung:
  Qdrant-Client & Collection-Setup für mindnet.
  - Stellt sicher, dass {prefix}_notes / {prefix}_chunks / {prefix}_edges existieren.
  - Edges-Collection nutzt 1D Dummy-Vektor.
  - ensure_payload_indexes(...) legt sinnvolle Payload-Indizes an.

NEU / Änderungen:
  v1.5.0:
    * ensure_collections_for_prefix(...)  → Wrapper für legacy-Importer
    * count_points(client, collection)    → stabile Zählfunktion (mit Fallback)
    * get_counts_for_prefix(...)          → Summary über alle drei Collections
    * truncate_collections(...)           → löscht *alle Punkte* in den Collections
  v1.6.0:
    * list_note_ids(client, notes_collection) → liefert alle payload.note_id-Werte
      (wird von import_markdown.py v3.9.0 erwartet)

Aufruf:
  from app.core.qdrant import (
      QdrantConfig, get_client,
      ensure_collections, ensure_payload_indexes,
      ensure_collections_for_prefix, count_points,
      collection_names, get_counts_for_prefix, truncate_collections,
      list_note_ids,
  )
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Optional, Tuple, Dict, List

from qdrant_client import QdrantClient
from qdrant_client.http import models as rest


@dataclass
class QdrantConfig:
    url: str
    api_key: Optional[str]
    prefix: str
    dim: int

    @staticmethod
    def from_env() -> "QdrantConfig":
        url = os.getenv("QDRANT_URL")
        if not url:
            host = os.getenv("QDRANT_HOST", "127.0.0.1")
            port = int(os.getenv("QDRANT_PORT", "6333"))
            url = f"http://{host}:{port}"
        api_key = os.getenv("QDRANT_API_KEY") or None
        prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
        dim = int(os.getenv("VECTOR_DIM", "384"))
        return QdrantConfig(url=url, api_key=api_key, prefix=prefix, dim=dim)


def get_client(cfg: QdrantConfig) -> QdrantClient:
    return QdrantClient(url=cfg.url, api_key=cfg.api_key)


# -------------------------------
# Collection-Erstellung
# -------------------------------
def _create_notes(client: QdrantClient, name: str, dim: int) -> None:
    if not client.collection_exists(name):
        client.create_collection(
            collection_name=name,
            vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
        )


def _create_chunks(client: QdrantClient, name: str, dim: int) -> None:
    if not client.collection_exists(name):
        client.create_collection(
            collection_name=name,
            vectors_config=rest.VectorParams(size=dim, distance=rest.Distance.COSINE),
        )


def _create_edges(client: QdrantClient, name: str) -> None:
    if not client.collection_exists(name):
        client.create_collection(
            collection_name=name,
            vectors_config=rest.VectorParams(size=1, distance=rest.Distance.DOT),  # 1D-Dummy
        )


def ensure_collections(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> None:
    notes = f"{prefix}_notes"
    chunks = f"{prefix}_chunks"
    edges = f"{prefix}_edges"

    _create_notes(client, notes, dim)
    _create_chunks(client, chunks, dim)

    if client.collection_exists(edges):
        try:
            info = client.get_collection(edges)
            vectors_cfg = getattr(getattr(info.result, "config", None), "params", None)
            has_vectors = getattr(vectors_cfg, "vectors", None) is not None
        except Exception:
            has_vectors = True
        if not has_vectors:
            if destructive:
                client.delete_collection(edges)
                _create_edges(client, edges)
            else:
                print(f"[ensure_collections] WARN: '{edges}' ohne VectorConfig; destructive=False.", flush=True)
    else:
        _create_edges(client, edges)


def collection_names(prefix: str) -> Tuple[str, str, str]:
    return (f"{prefix}_notes", f"{prefix}_chunks", f"{prefix}_edges")


# -------------------------------
# Payload-Indexing
# -------------------------------
def _safe_create_index(client: QdrantClient, col: str, field: str, schema: rest.PayloadSchemaType):
    try:
        client.create_payload_index(
            collection_name=col,
            field_name=field,
            field_schema=schema,
        )
    except Exception:
        # bereits vorhanden oder nicht unterstütztes Schema → ignorieren
        pass


def ensure_payload_indexes(client: QdrantClient, prefix: str) -> None:
    notes, chunks, edges = collection_names(prefix)
    # Notes
    _safe_create_index(client, notes, "note_id", rest.PayloadSchemaType.KEYWORD)
    # Chunks
    _safe_create_index(client, chunks, "note_id", rest.PayloadSchemaType.KEYWORD)
    _safe_create_index(client, chunks, "chunk_index", rest.PayloadSchemaType.INTEGER)
    _safe_create_index(client, chunks, "chunk_id", rest.PayloadSchemaType.KEYWORD)
    # Edges
    for f in ("kind", "scope", "source_id", "target_id", "note_id", "edge_id"):
        _safe_create_index(client, edges, f, rest.PayloadSchemaType.KEYWORD)


# -------------------------------
# NEU: Abwärtskompatible Helfer
# -------------------------------
def ensure_collections_for_prefix(client: QdrantClient, prefix: str, dim: int, destructive: bool = False) -> Tuple[str, str, str]:
    """
    Legacy-Wrapper, damit ältere Skripte (Importer bis v3.7.x) funktionieren.
    Gibt die Collection-Namen zurück.
    """
    ensure_collections(client, prefix, dim, destructive=destructive)
    ensure_payload_indexes(client, prefix)
    return collection_names(prefix)


def count_points(client: QdrantClient, collection: str) -> int:
    """
    Zähle Punkte in einer Collection robust:
      1) bevorzugt client.count(..., exact=True)
      2) Fallback: Scrollen ohne Filter und mitzählen
    """
    try:
        res = client.count(collection_name=collection, count_filter=None, exact=True)
        cnt = getattr(res, "count", None)
        if isinstance(cnt, int):
            return cnt
        if isinstance(res, dict) and "count" in res:
            return int(res["count"])
    except Exception:
        pass

    # Fallback via Scroll
    total = 0
    next_page = None
    while True:
        points, next_page = client.scroll(
            collection_name=collection,
            limit=2048,
            with_payload=False,
            with_vectors=False,
            offset=next_page,
        )
        total += len(points)
        if next_page is None or not points:
            break
    return total


def get_counts_for_prefix(client: QdrantClient, prefix: str) -> Dict[str, int]:
    notes, chunks, edges = collection_names(prefix)
    return {
        "notes": count_points(client, notes),
        "chunks": count_points(client, chunks),
        "edges": count_points(client, edges),
    }


def truncate_collections(client: QdrantClient, prefix: str) -> None:
    """
    Löscht *alle Punkte* (nicht die Collections selber) für {prefix}.
    Entspricht funktional einem "truncate" in deinem Reset-Skript.
    """
    for col in collection_names(prefix):
        try:
            client.delete(
                collection_name=col,
                points_selector=rest.FilterSelector(
                    filter=rest.Filter(must=[])  # leeres Filter => alle Punkte
                ),
                wait=True,
            )
        except Exception:
            # Fallback: Collection ggf. leer/nicht vorhanden → ignorieren
            pass


# -------------------------------
# NEU v1.6.0: list_note_ids
# -------------------------------
def list_note_ids(client: QdrantClient, notes_collection: str, limit: int = 100000) -> List[str]:
    """
    Liefert alle payload.note_id aus der angegebenen Notes-Collection.
    - Wird von import_markdown.py (>= v3.9.0) verwendet, z.B. für Baseline-/Idempotenz-Checks.
    - Robust gegen fehlende Felder: ignoriert Punkte ohne 'note_id'.

    Args:
        client: QdrantClient
        notes_collection: Name der Notes-Collection (z.B. 'mindnet_notes')
        limit: harte Obergrenze für die Anzahl der zurückzugebenden IDs

    Returns:
        Liste der note_id-Strings (ohne Duplikate, Reihenfolge nicht garantiert).
    """
    out: List[str] = []
    seen = set()
    next_page = None
    fetched = 0

    flt = None  # kein Filter → alle Punkte

    while True:
        # scroll_filter in neueren Clients; ältere akzeptieren 'scroll_filter' oder 'filter'
        points, next_page = client.scroll(
            collection_name=notes_collection,
            scroll_filter=flt,
            limit=min(512, max(1, limit - fetched)),
            with_payload=True,
            with_vectors=False,
            offset=next_page,
        )
        if not points:
            break

        for p in points:
            pl = p.payload or {}
            nid = pl.get("note_id")
            if isinstance(nid, str) and nid not in seen:
                seen.add(nid)
                out.append(nid)
                fetched += 1
                if fetched >= limit:
                    return out

        if next_page is None:
            break

    return out