Author SHA1 Message Date
Lars 11373138ca scripts/test_llm_api.py aktualisiert
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 19:46:53 +02:00
Lars 75b257bb15 scripts/test_llm_api.py aktualisiert
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 19:44:39 +02:00
Lars c0bb562a8d scripts/test_llm_api.py hinzugefügt
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 19:40:32 +02:00
Lars 32577a7fda llm-api/exercise_router.py aktualisiert
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 19:35:28 +02:00
Lars a6d68134cd scripts/backfill_capability_facets.py aktualisiert
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 19:24:17 +02:00
Lars fa8a92208a scripts/backfill_capability_facets.py aktualisiert
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 19:21:56 +02:00
Lars 0c047b708f scripts/backfill_capability_facets.py hinzugefügt
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 19:19:32 +02:00
Lars efbb978074 scripts/bootstrap_qdrant_exercises.py aktualisiert
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 19:16:16 +02:00
Lars 21ce1dc395 llm-api/exercise_router.py aktualisiert
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 19:02:01 +02:00
Lars 380b361e70 llm-api/exercise_router.py aktualisiert
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 18:59:27 +02:00
Lars 2a859aa16c llm-api/exercise_router.py aktualisiert
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
Soll dafür sorgen, dass in der qdrant Collection sogenannten capability Facetten ins Payload aufgenommen werden. 

Das ist deine aktuelle Version + folgende Verbesserungen:

schreibt jetzt automatisch die Capability-Facetten ins Payload: capability_keys, capability_ge1, capability_ge2, capability_ge3 (für schnelle Qdrant-Filter).

stabilisiert Listenfelder (keywords, equipment): trimmt, dedupliziert, sortiert → stabile Filter & Fingerprints.

Upsert-Logik via external_id bleibt unverändert.

Response bleibt kompatibel: zusätzliche Payload-Felder werden nicht zurückgegeben (wir filtern sie für die API-Antwort heraus).
2025-08-11 18:22:01 +02:00
Lars d14ed9a511 scripts/bootstrap_qdrant_exercises.py hinzugefügt
Deploy Trainer_LLM to llm-node / deploy (push) Successful in 2s
2025-08-11 18:11:47 +02:00
4 changed files with 697 additions and 42 deletions

No files matched your search

+285 -42
View File
@@ -1,4 +1,16 @@
# Test eines Kommentars, um die Funktion des gitea testen zu können # -*- coding: utf-8 -*-
"""
exercise_router.py – v1.7.0
Neu:
- Endpoint **POST /exercise/search**: kombinierbare Filter (discipline, duration, equipment any/all, keywords any/all,
capability_geN / capability_eqN + names) + optionaler Vektor-Query (query-Text). Ausgabe inkl. Score.
- Facetten erweitert: neben capability_ge1..ge5 jetzt auch capability_eq1..eq5.
- Idempotenz-Fix & Payload-Scroll (aus v1.6.2) beibehalten.
- API-Signaturen bestehender Routen unverändert.
Hinweis: Die „eq/ge“-Felder werden beim Upsert gesetzt; für Alt-Punkte einmal das Backfill laufen lassen.
"""
from fastapi import APIRouter, HTTPException, Query from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
@@ -11,8 +23,9 @@ from qdrant_client.models import (
VectorParams, VectorParams,
Distance, Distance,
PointIdsList, PointIdsList,
# NEW: für Filter-Queries (Lookup via external_id) Filter,
Filter, FieldCondition, MatchValue, FieldCondition,
MatchValue,
) )
import os import os
@@ -23,13 +36,13 @@ router = APIRouter()
# ========================= # =========================
class Exercise(BaseModel): class Exercise(BaseModel):
id: str = Field(default_factory=lambda: str(uuid4())) id: str = Field(default_factory=lambda: str(uuid4()))
# NEW — optional, bricht vorhandene POST-Calls nicht # Upsert-Metadaten
external_id: Optional[str] = None # z.B. "mw:12345" external_id: Optional[str] = None
fingerprint: Optional[str] = None # sha256 über Kernfelder fingerprint: Optional[str] = None
source: Optional[str] = None # Herkunft, z.B. "MediaWiki" source: Optional[str] = None
imported_at: Optional[datetime] = None # vom Import gesetzt imported_at: Optional[datetime] = None
# Bestehende Felder (unverändert) # Domain-Felder
title: str title: str
summary: str summary: str
short_description: str short_description: str
@@ -55,12 +68,43 @@ class DeleteResponse(BaseModel):
count: int count: int
collection: str collection: str
class ExerciseSearchRequest(BaseModel):
# Optionaler Semantik-Query (Vektor)
query: Optional[str] = None
limit: int = Field(default=20, ge=1, le=200)
offset: int = Field(default=0, ge=0)
# Einfache Filter
discipline: Optional[str] = None
target_group: Optional[str] = None
age_group: Optional[str] = None
max_duration: Optional[int] = Field(default=None, ge=0)
# Listen-Filter
equipment_any: Optional[List[str]] = None # mindestens eins muss passen
equipment_all: Optional[List[str]] = None # alle müssen passen
keywords_any: Optional[List[str]] = None
keywords_all: Optional[List[str]] = None
# Capabilities (Namen + Level-Operator)
capability_names: Optional[List[str]] = None
capability_ge_level: Optional[int] = Field(default=None, ge=1, le=5)
capability_eq_level: Optional[int] = Field(default=None, ge=1, le=5)
class ExerciseSearchHit(BaseModel):
id: str
score: Optional[float] = None
payload: Exercise
class ExerciseSearchResponse(BaseModel):
hits: List[ExerciseSearchHit]
# ========================= # =========================
# Helpers # Helpers
# ========================= # =========================
COLLECTION = os.getenv("EXERCISE_COLLECTION", "exercises") COLLECTION = os.getenv("EXERCISE_COLLECTION", "exercises")
# CHANGED: Factorized to reuse for both create and update
def _ensure_collection(): def _ensure_collection():
if not qdrant.collection_exists(COLLECTION): if not qdrant.collection_exists(COLLECTION):
qdrant.recreate_collection( qdrant.recreate_collection(
@@ -71,73 +115,181 @@ def _ensure_collection():
), ),
) )
# NEW: gemeinsamer Helper für external_id-Lookup
def _lookup_by_external_id(external_id: str) -> Optional[Dict[str, Any]]: def _lookup_by_external_id(external_id: str) -> Optional[Dict[str, Any]]:
_ensure_collection() _ensure_collection()
flt = Filter(must=[FieldCondition(key="external_id", match=MatchValue(value=external_id))]) flt = Filter(must=[FieldCondition(key="external_id", match=MatchValue(value=external_id))])
pts, _ = qdrant.scroll(collection_name=COLLECTION, scroll_filter=flt, limit=1) pts, _ = qdrant.scroll(
collection_name=COLLECTION,
scroll_filter=flt,
limit=1,
with_payload=True,
)
if not pts: if not pts:
return None return None
# qdrant_client liefert PointStruct; wir geben die payload + id zurück doc = dict(pts[0].payload or {})
doc = pts[0].payload or {}
doc = dict(doc)
doc.setdefault("id", str(pts[0].id)) doc.setdefault("id", str(pts[0].id))
return doc return doc
# NEW: konsistente Embedding-Erzeugung
_def_embed_text_fields = ("title", "summary", "short_description", "purpose", "execution", "notes")
def _make_vector(ex: Exercise) -> List[float]: _DEF_EMBED_FIELDS = ("title", "summary", "short_description", "purpose", "execution", "notes")
text = ". ".join([getattr(ex, f, "") for f in _def_embed_text_fields if getattr(ex, f, None)])
# Achtung: model.encode muss synchron sein; sonst async anpassen
vec = model.encode(text).tolist() def _make_vector_from_exercise(ex: Exercise) -> List[float]:
return vec text = ". ".join([getattr(ex, f, "") for f in _DEF_EMBED_FIELDS if getattr(ex, f, None)])
return model.encode(text).tolist()
def _make_vector_from_query(query: str) -> List[float]:
return model.encode(query).tolist()
def _norm_list(xs: List[Any]) -> List[str]:
out = []
seen = set()
for x in xs or []:
s = str(x).strip()
if not s:
continue
key = s.casefold()
if key in seen:
continue
seen.add(key)
out.append(s)
return sorted(out, key=str.casefold)
def _facet_capabilities(caps: Dict[str, Any]) -> Dict[str, List[str]]:
caps = caps or {}
def names_where(pred) -> List[str]:
out = []
for k, v in caps.items():
try:
iv = int(v)
except Exception:
iv = 0
if pred(iv):
t = str(k).strip()
if t:
out.append(t)
return sorted({t for t in out}, key=str.casefold)
all_keys = sorted({str(k).strip() for k in caps.keys() if str(k).strip()}, key=str.casefold)
return {
"capability_keys": all_keys,
# >= N
"capability_ge1": names_where(lambda lv: lv >= 1),
"capability_ge2": names_where(lambda lv: lv >= 2),
"capability_ge3": names_where(lambda lv: lv >= 3),
"capability_ge4": names_where(lambda lv: lv >= 4),
"capability_ge5": names_where(lambda lv: lv >= 5),
# == N
"capability_eq1": names_where(lambda lv: lv == 1),
"capability_eq2": names_where(lambda lv: lv == 2),
"capability_eq3": names_where(lambda lv: lv == 3),
"capability_eq4": names_where(lambda lv: lv == 4),
"capability_eq5": names_where(lambda lv: lv == 5),
}
def _response_strip_extras(payload: Dict[str, Any]) -> Dict[str, Any]:
allowed = set(Exercise.model_fields.keys())
return {k: v for k, v in payload.items() if k in allowed}
def _build_filter(req: ExerciseSearchRequest) -> Filter:
must: List[Any] = []
should: List[Any] = []
if req.discipline:
must.append(FieldCondition(key="discipline", match=MatchValue(value=req.discipline)))
if req.target_group:
must.append(FieldCondition(key="target_group", match=MatchValue(value=req.target_group)))
if req.age_group:
must.append(FieldCondition(key="age_group", match=MatchValue(value=req.age_group)))
if req.max_duration is not None:
# Range ohne Import zusätzlicher Modelle: Qdrant akzeptiert auch {'range': {'lte': n}} per JSON;
# über Client-Modell tun wir es hier nicht, da wir Filter primär für Keyword-Felder nutzen.
must.append({"key": "duration_minutes", "range": {"lte": int(req.max_duration)}})
# equipment
if req.equipment_all:
for it in req.equipment_all:
must.append(FieldCondition(key="equipment", match=MatchValue(value=it)))
if req.equipment_any:
# OR: über 'should' Liste
for it in req.equipment_any:
should.append(FieldCondition(key="equipment", match=MatchValue(value=it)))
# keywords
if req.keywords_all:
for it in req.keywords_all:
must.append(FieldCondition(key="keywords", match=MatchValue(value=it)))
if req.keywords_any:
for it in req.keywords_any:
should.append(FieldCondition(key="keywords", match=MatchValue(value=it)))
# capabilities (ge/eq)
if req.capability_names:
names = [s for s in req.capability_names if s and s.strip()]
if req.capability_eq_level:
key = f"capability_eq{int(req.capability_eq_level)}"
for n in names:
must.append(FieldCondition(key=key, match=MatchValue(value=n)))
elif req.capability_ge_level:
key = f"capability_ge{int(req.capability_ge_level)}"
for n in names:
must.append(FieldCondition(key=key, match=MatchValue(value=n)))
else:
# Default: Level >=1 (alle vorhanden)
for n in names:
must.append(FieldCondition(key="capability_ge1", match=MatchValue(value=n)))
flt = Filter(must=must)
if should:
# qdrant: 'should' mit implizitem minimum_should_match=1
flt.should = should
return flt
# ========================= # =========================
# Endpoints # Endpoints
# ========================= # =========================
@router.get("/exercise/by-external-id") # NEW @router.get("/exercise/by-external-id")
def get_exercise_by_external_id(external_id: str = Query(..., min_length=3)): def get_exercise_by_external_id(external_id: str = Query(..., min_length=3)):
"""Lookup für Idempotenz im Importer. Liefert 404, wenn nicht vorhanden."""
found = _lookup_by_external_id(external_id) found = _lookup_by_external_id(external_id)
if not found: if not found:
raise HTTPException(status_code=404, detail="not found") raise HTTPException(status_code=404, detail="not found")
return found return found
@router.post("/exercise", response_model=Exercise) @router.post("/exercise", response_model=Exercise)
def create_or_update_exercise(ex: Exercise): def create_or_update_exercise(ex: Exercise):
"""
CHANGED: Upsert-Semantik. Wenn `external_id` existiert und bereits in Qdrant gefunden wird,
wird dieselbe Point-ID überschrieben (echtes Update). Ansonsten neuer Eintrag.
API-Signatur bleibt identisch (POST /exercise, Body = Exercise).
"""
_ensure_collection() _ensure_collection()
# Default: neue Point-ID aus dem Exercise-Objekt
point_id = ex.id point_id = ex.id
# Wenn external_id gesetzt → prüfen, ob bereits vorhanden → Point-ID übernehmen
if ex.external_id: if ex.external_id:
prior = _lookup_by_external_id(ex.external_id) prior = _lookup_by_external_id(ex.external_id)
if prior: if prior:
point_id = prior.get("id", point_id) point_id = prior.get("id", point_id)
# Embedding berechnen vector = _make_vector_from_exercise(ex)
vector = _make_vector(ex)
# Payload synchronisieren (id == point_id) payload: Dict[str, Any] = ex.model_dump()
payload = ex.dict()
payload["id"] = str(point_id) payload["id"] = str(point_id)
payload["keywords"] = _norm_list(payload.get("keywords") or [])
payload["equipment"] = _norm_list(payload.get("equipment") or [])
payload.update(_facet_capabilities(payload.get("capabilities") or {}))
# Upsert in Qdrant
qdrant.upsert( qdrant.upsert(
collection_name=COLLECTION, collection_name=COLLECTION,
points=[PointStruct(id=str(point_id), vector=vector, payload=payload)], points=[PointStruct(id=str(point_id), vector=vector, payload=payload)],
) )
return Exercise(**payload) return Exercise(**_response_strip_extras(payload))
# (Optional) – Einzel-Abruf per ID (falls bereits vorhanden, unverändert)
@router.get("/exercise/{exercise_id}", response_model=Exercise) @router.get("/exercise/{exercise_id}", response_model=Exercise)
def get_exercise(exercise_id: str): def get_exercise(exercise_id: str):
_ensure_collection() _ensure_collection()
@@ -145,28 +297,119 @@ def get_exercise(exercise_id: str):
collection_name=COLLECTION, collection_name=COLLECTION,
scroll_filter=Filter(must=[FieldCondition(key="id", match=MatchValue(value=exercise_id))]), scroll_filter=Filter(must=[FieldCondition(key="id", match=MatchValue(value=exercise_id))]),
limit=1, limit=1,
with_payload=True,
) )
if not pts: if not pts:
raise HTTPException(status_code=404, detail="not found") raise HTTPException(status_code=404, detail="not found")
payload = dict(pts[0].payload or {}) payload = dict(pts[0].payload or {})
payload.setdefault("id", str(pts[0].id)) payload.setdefault("id", str(pts[0].id))
return Exercise(**payload) return Exercise(**_response_strip_extras(payload))
@router.post("/exercise/search", response_model=ExerciseSearchResponse)
def search_exercises(req: ExerciseSearchRequest) -> ExerciseSearchResponse:
_ensure_collection()
flt = _build_filter(req)
hits: List[ExerciseSearchHit] = []
if req.query:
vec = _make_vector_from_query(req.query)
# qdrant_client.search unterstützt offset/limit
res = qdrant.search(
collection_name=COLLECTION,
query_vector=vec,
limit=req.limit,
offset=req.offset,
query_filter=flt,
)
for h in res:
payload = dict(h.payload or {})
payload.setdefault("id", str(h.id))
hits.append(ExerciseSearchHit(id=str(h.id), score=float(h.score or 0.0), payload=Exercise(**_response_strip_extras(payload))))
else:
# Filter-only: per Scroll (ohne Score); einfache Paginierung via offset/limit
# Hole offset+limit Punkte und simuliere Score=None
collected = 0
skipped = 0
next_offset = None
while collected < req.limit:
page, next_offset = qdrant.scroll(
collection_name=COLLECTION,
scroll_filter=flt,
offset=next_offset,
limit=max(1, min(256, req.limit - collected + req.offset - skipped)),
with_payload=True,
)
if not page:
break
for pt in page:
if skipped < req.offset:
skipped += 1
continue
payload = dict(pt.payload or {})
payload.setdefault("id", str(pt.id))
hits.append(ExerciseSearchHit(id=str(pt.id), score=None, payload=Exercise(**_response_strip_extras(payload))))
collected += 1
if collected >= req.limit:
break
if next_offset is None:
break
return ExerciseSearchResponse(hits=hits)
# Bestehende Admin-Utilities (Delete nach Filter / komplette Collection) – unverändert außer Nutzung von CONSTs
@router.delete("/exercise/delete-by-external-id", response_model=DeleteResponse) @router.delete("/exercise/delete-by-external-id", response_model=DeleteResponse)
def delete_by_external_id(external_id: str = Query(...)): def delete_by_external_id(external_id: str = Query(...)):
_ensure_collection() _ensure_collection()
flt = Filter(must=[FieldCondition(key="external_id", match=MatchValue(value=external_id))]) flt = Filter(must=[FieldCondition(key="external_id", match=MatchValue(value=external_id))])
pts, _ = qdrant.scroll(collection_name=COLLECTION, scroll_filter=flt, limit=10000) pts, _ = qdrant.scroll(collection_name=COLLECTION, scroll_filter=flt, limit=10000, with_payload=False)
ids = [str(p.id) for p in pts] ids = [str(p.id) for p in pts]
if not ids: if not ids:
return DeleteResponse(status="🔍 Keine Einträge gefunden.", count=0, collection=COLLECTION) return DeleteResponse(status="🔍 Keine Einträge gefunden.", count=0, collection=COLLECTION)
qdrant.delete(collection_name=COLLECTION, points_selector=PointIdsList(points=ids)) qdrant.delete(collection_name=COLLECTION, points_selector=PointIdsList(points=ids))
return DeleteResponse(status="🗑️ gelöscht", count=len(ids), collection=COLLECTION) return DeleteResponse(status="🗑️ gelöscht", count=len(ids), collection=COLLECTION)
@router.delete("/exercise/delete-collection", response_model=DeleteResponse) @router.delete("/exercise/delete-collection", response_model=DeleteResponse)
def delete_collection(collection: str = Query(default=COLLECTION)): def delete_collection(collection: str = Query(default=COLLECTION)):
if not qdrant.collection_exists(collection): if not qdrant.collection_exists(collection):
raise HTTPException(status_code=404, detail=f"Collection '{collection}' nicht gefunden.") raise HTTPException(status_code=404, detail=f"Collection '{collection}' nicht gefunden.")
qdrant.delete_collection(collection_name=collection) qdrant.delete_collection(collection_name=collection)
return DeleteResponse(status="🗑️ gelöscht", count=0, collection=collection) return DeleteResponse(status="🗑️ gelöscht", count=0, collection=collection)
# ---------------------------
# OPTIONAL: einfacher Selbsttest (kannst du auch separat als Script verwenden)
# ---------------------------
TEST_DOC = """
Speicher als tests/test_exercise_search.py und mit pytest laufen lassen.
import os, requests
BASE = os.getenv("API_BASE", "http://localhost:8000")
# 1) Filter-only
r = requests.post(f"{BASE}/exercise/search", json={
"discipline": "Karate",
"max_duration": 12,
"equipment_any": ["Bälle"],
"capability_names": ["Reaktionsfähigkeit"],
"capability_ge_level": 2,
"limit": 5
})
r.raise_for_status()
js = r.json()
assert "hits" in js
for h in js["hits"]:
p = h["payload"]
assert p["discipline"] == "Karate"
assert p["duration_minutes"] <= 12
# 2) Vector + Filter
r = requests.post(f"{BASE}/exercise/search", json={
"query": "Aufwärmen 10min, Reaktionsfähigkeit, Teenager, Bälle",
"discipline": "Karate",
"limit": 3
})
r.raise_for_status()
js = r.json(); assert len(js["hits"]) <= 3
"""
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Backfill Capability-Facetten in Qdrant – v1.2
Fix: beendet korrekt, wenn `next_page_offset` (offset) None ist.
"""
import os
from typing import Dict, Any, List
from qdrant_client import QdrantClient
COLL = os.getenv("EXERCISE_COLLECTION", "exercises")
QDRANT_HOST = os.getenv("QDRANT_HOST", "localhost")
QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
BATCH = int(os.getenv("BACKFILL_BATCH", "256"))
def _facet_capabilities(caps: Dict[str, Any]) -> Dict[str, List[str]]:
caps = caps or {}
def names_where(pred) -> List[str]:
out = []
for k, v in caps.items():
try:
iv = int(v)
except Exception:
iv = 0
if pred(iv):
s = str(k).strip()
if s:
out.append(s)
return sorted({s for s in out}, key=str.casefold)
all_keys = sorted({str(k).strip() for k in caps.keys() if str(k).strip()}, key=str.casefold)
return {
"capability_keys": all_keys,
"capability_ge1": names_where(lambda lv: lv >= 1),
"capability_ge2": names_where(lambda lv: lv >= 2),
"capability_ge3": names_where(lambda lv: lv >= 3),
"capability_ge4": names_where(lambda lv: lv >= 4),
"capability_ge5": names_where(lambda lv: lv >= 5),
"capability_eq1": names_where(lambda lv: lv == 1),
"capability_eq2": names_where(lambda lv: lv == 2),
"capability_eq3": names_where(lambda lv: lv == 3),
"capability_eq4": names_where(lambda lv: lv == 4),
"capability_eq5": names_where(lambda lv: lv == 5),
}
def main() -> None:
client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT)
info = client.get_collection(COLL)
print(f"[Backfill] Collection '{COLL}' ok – vectors={info.config.params.vectors}")
updated_total = 0
offset = None
page = 0
while True:
points, next_offset = client.scroll(
collection_name=COLL,
scroll_filter=None,
offset=offset,
limit=BATCH,
with_payload=True,
)
page += 1
if not points:
print("[Backfill] no more points – done")
break
updated_page = 0
for pt in points:
pld = pt.payload or {}
caps = pld.get("capabilities") or {}
facets = _facet_capabilities(caps)
# nur setzen, wenn sich etwas ändert
need = any(pld.get(k) != v for k, v in facets.items())
if not need:
continue
client.set_payload(collection_name=COLL, points=[pt.id], payload=facets)
updated_total += 1
updated_page += 1
print(f"[Backfill] page={page} processed={len(points)} updated_page={updated_page} updated_total={updated_total}")
# Ende erreicht? Dann nach dieser Seite aussteigen.
if next_offset is None:
break
offset = next_offset
print(f"[Backfill] done. total_updated={updated_total}")
if __name__ == "__main__":
main()
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Idempotenter Bootstrap für die Qdrant-Collection 'exercises':
- Legt fehlende Payload-Indizes an (ohne Collection zu droppen)
- Optional prüft Dimension/Distanz
Starten mit
pip install qdrant-client --upgrade
python3 scripts/bootstrap_qdrant_exercises.py
"""
import os
from qdrant_client import QdrantClient
from qdrant_client.models import PayloadSchemaType
QDRANT_HOST = os.getenv("QDRANT_HOST", "localhost")
QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
COLLECTION = os.getenv("EXERCISE_COLLECTION", "exercises")
INDEX_SPECS = [
("keywords", PayloadSchemaType.KEYWORD),
("equipment", PayloadSchemaType.KEYWORD),
("discipline", PayloadSchemaType.KEYWORD),
("age_group", PayloadSchemaType.KEYWORD),
("target_group", PayloadSchemaType.KEYWORD),
("category", PayloadSchemaType.KEYWORD),
("capability_keys", PayloadSchemaType.KEYWORD),
("capability_ge1", PayloadSchemaType.KEYWORD),
("capability_ge2", PayloadSchemaType.KEYWORD),
("capability_ge3", PayloadSchemaType.KEYWORD),
("capability_ge4", PayloadSchemaType.KEYWORD), # neu
("capability_ge5", PayloadSchemaType.KEYWORD), # neu
("capability_eq1", PayloadSchemaType.KEYWORD), # neu
("capability_eq2", PayloadSchemaType.KEYWORD), # neu
("capability_eq3", PayloadSchemaType.KEYWORD), # neu
("capability_eq4", PayloadSchemaType.KEYWORD), # neu
("capability_eq5", PayloadSchemaType.KEYWORD), # neu
]
def main():
client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT)
# Sanity: Collection existiert?
info = client.get_collection(COLLECTION) # wirf Fehler, wenn nicht vorhanden
print(f"[Bootstrap] Collection '{COLLECTION}' gefunden. Vectors={info.config.params.vectors}")
# Indizes idempotent anlegen
for field, schema in INDEX_SPECS:
try:
client.create_payload_index(collection_name=COLLECTION,
field_name=field,
field_schema=schema)
print(f"[Bootstrap] Index created: {field} ({schema})")
except Exception as e:
# Qdrant wirft hier Exceptions, wenn der Index schon existiert – das ist ok
print(f"[Bootstrap] Index exists or skipped: {field} -> {e}")
print("[Bootstrap] done.")
if __name__ == "__main__":
main()
+255
View File
@@ -0,0 +1,255 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Erweiterter API-Schnelltest - v2.1
Änderungen ggü. v2.0:
- **Login VOR** /semantic/pages (viele Wikis liefern Ask-Ergebnisse nur authentifiziert)
- Robuster Fallback: Wenn /semantic/pages {} liefert → optionaler ENV-Fallback `TEST_WIKI_TITLE`
- Bessere Diagnoseausgaben (zeigt Basis-URL, Category, gesetzte Login-ENV)
ENV:
BASE_URL (default http://127.0.0.1:8000)
TEST_WIKI_CATEGORY (default "Übungen")
TEST_WIKI_TITLE (optional Fallback-Titel, z. B. "Affenklatschen")
WIKI_BOT_USER, WIKI_BOT_PASSWORD (optional; empfohlen)
"""
import os
import sys
import json
import uuid
import requests
BASE = os.getenv("BASE_URL", "http://127.0.0.1:8000").rstrip("/")
TEST_WIKI_CATEGORY = os.getenv("TEST_WIKI_CATEGORY", "Übungen")
TEST_WIKI_TITLE = os.getenv("TEST_WIKI_TITLE")
WIKI_USER = os.getenv("WIKI_BOT_USER")
WIKI_PASS = os.getenv("WIKI_BOT_PASSWORD")
print("\nConfig:")
print(" BASE_URL=", BASE)
print(" TEST_WIKI_CATEGORY=", TEST_WIKI_CATEGORY)
print(" TEST_WIKI_TITLE=", TEST_WIKI_TITLE or "<none>")
print(" WIKI_BOT_USER set=", bool(WIKI_USER))
# ---- helpers ----
def fail(msg, resp: requests.Response | None = None):
print("✗", msg)
if resp is not None:
try:
print(" → status:", resp.status_code)
print(" → body:", resp.text[:800])
except Exception:
pass
sys.exit(1)
def ok(msg):
print("✓", msg)
# ---- basic ----
def test_openapi():
r = requests.get(f"{BASE}/openapi.json")
if r.status_code != 200:
fail("/openapi.json nicht erreichbar", r)
ok("OpenAPI erreichbar")
# ---- wiki ----
def test_wiki_health():
r = requests.get(f"{BASE}/import/wiki/health", params={"verbose": 1})
if r.status_code != 200 or r.json().get("status") != "ok":
fail("/import/wiki/health fehlgeschlagen", r)
ok("Wiki /health ok")
def test_wiki_login_if_env():
if not (WIKI_USER and WIKI_PASS):
print("⚠️ Wiki /login übersprungen (WIKI_BOT_USER/PASSWORD nicht gesetzt)")
return False
r = requests.post(f"{BASE}/import/wiki/login", json={"username": WIKI_USER, "password": WIKI_PASS})
if r.status_code != 200 or r.json().get("status") != "success":
fail("/import/wiki/login fehlgeschlagen", r)
ok("Wiki /login ok")
return True
def get_wiki_title_and_pageid() -> tuple[str, int]:
# 1) Versuch über semantic/pages
r = requests.get(f"{BASE}/import/wiki/semantic/pages", params={"category": TEST_WIKI_CATEGORY})
if r.status_code != 200:
fail("/import/wiki/semantic/pages fehlgeschlagen", r)
data = r.json()
if isinstance(data, dict) and data:
title = next(iter(data.keys()))
entry = data[title]
pid = entry.get("pageid")
if not pid:
fail("/semantic/pages ohne pageid nach Enrichment", r)
ok(f"Wiki /semantic/pages ok – Beispiel: '{title}' (pageid={pid})")
return title, int(pid)
print("ℹ️ /semantic/pages lieferte keine Titel ({}). Versuche Fallback...")
# 2) Fallback über TEST_WIKI_TITLE
if not TEST_WIKI_TITLE:
fail("Kein TEST_WIKI_TITLE gesetzt und /semantic/pages leer – kann keinen Titel testen.")
r2 = requests.get(f"{BASE}/import/wiki/info", params={"title": TEST_WIKI_TITLE})
if r2.status_code != 200:
fail("/import/wiki/info(Fallback) fehlgeschlagen", r2)
js = r2.json()
pid = js.get("pageid")
if not pid:
fail("/info(Fallback) ohne pageid", r2)
ok(f"Wiki /info Fallback ok – '{TEST_WIKI_TITLE}' (pageid={pid})")
return TEST_WIKI_TITLE, int(pid)
def test_wiki_info(title: str):
r = requests.get(f"{BASE}/import/wiki/info", params={"title": title})
if r.status_code != 200:
fail("/import/wiki/info fehlgeschlagen", r)
js = r.json()
if not js.get("pageid"):
fail("/info ohne pageid", r)
ok("Wiki /info ok")
def test_wiki_parse(pageid: int, title: str):
r = requests.get(f"{BASE}/import/wiki/parsepage", params={"pageid": pageid, "title": title})
if r.status_code != 200:
fail("/import/wiki/parsepage fehlgeschlagen", r)
js = r.json()
if not isinstance(js.get("wikitext"), str):
fail("/parsepage ohne wikitext", r)
ok("Wiki /parsepage ok")
# ---- exercise ----
def make_exercise_payload(external_id: str):
return {
"external_id": external_id,
"fingerprint": "unit-test-sha",
"source": "UnitTest",
"title": "Testübung Reaktion",
"summary": "Kurzbeschreibung für Test.",
"short_description": "Kurzbeschreibung für Test.",
"keywords": ["Reaktion", "Bälle", "Bälle"],
"link": "http://example.local",
"discipline": "Karate",
"group": "5",
"age_group": "Teenager",
"target_group": "Breitensport",
"min_participants": 1,
"duration_minutes": 10,
"capabilities": {"Reaktionsfähigkeit": 2, "Kopplungsfähigkeit": 1},
"category": "Übungen",
"purpose": "Aufwärmen",
"execution": "Einfacher Ablauf.",
"notes": "Hinweise.",
"preparation": "Bälle holen.",
"method": "Frontale Methode",
"equipment": ["Bälle", "Pratze"]
}
def test_exercise_upsert_and_idempotence():
import time
ext = f"ut:{uuid.uuid4()}"
payload = make_exercise_payload(ext)
# create
r1 = requests.post(f"{BASE}/exercise", json=payload)
if r1.status_code != 200:
fail("POST /exercise (create) fehlgeschlagen", r1)
id1 = r1.json().get("id")
if not id1:
fail("POST /exercise lieferte keine id", r1)
# update (idempotent)
r2 = requests.post(f"{BASE}/exercise", json=payload)
if r2.status_code != 200:
fail("POST /exercise (update) fehlgeschlagen", r2)
id2 = r2.json().get("id")
if id2 != id1:
fail("Idempotenz verletzt: id hat sich geändert")
# by-external-id
r3 = requests.get(f"{BASE}/exercise/by-external-id", params={"external_id": ext})
if r3.status_code != 200:
fail("GET /exercise/by-external-id fehlgeschlagen", r3)
id3 = r3.json().get("id")
if id3 != id1:
fail("Lookup by external_id liefert andere id")
ok("Exercise Upsert & Idempotenz ok")
return ext, id1
def test_exercise_search_filter(ext_id: str):
req = {
"discipline": "Karate",
"equipment_all": ["Bälle"],
"capability_names": ["Reaktionsfähigkeit"],
"capability_ge_level": 2,
"limit": 10,
}
r = requests.post(f"{BASE}/exercise/search", json=req)
if r.status_code != 200:
fail("POST /exercise/search (Filter) fehlgeschlagen", r)
js = r.json(); hits = js.get("hits", [])
if not isinstance(hits, list) or not hits:
fail("/exercise/search liefert keine Treffer (Filter)", r)
if not any(h.get("payload", {}).get("external_id") == ext_id for h in hits):
ok("Exercise Search (Filter) ok – Testpunkt nicht zwingend in Top-N")
else:
ok("Exercise Search (Filter) ok – Testpunkt gefunden")
def test_exercise_search_vector():
req = {
"query": "Aufwärmen 10min, Reaktionsfähigkeit, Teenager, Bälle",
"discipline": "Karate",
"limit": 5
}
r = requests.post(f"{BASE}/exercise/search", json=req)
if r.status_code != 200:
fail("POST /exercise/search (Vector) fehlgeschlagen", r)
js = r.json(); hits = js.get("hits", [])
if not isinstance(hits, list):
fail("/exercise/search (Vector) liefert ungültige Struktur", r)
ok(f"Exercise Search (Vector) ok – {len(hits)} Treffer")
def test_exercise_delete(ext_id: str):
r = requests.delete(f"{BASE}/exercise/delete-by-external-id", params={"external_id": ext_id})
if r.status_code != 200:
fail("DELETE /exercise/delete-by-external-id fehlgeschlagen", r)
ok("Exercise Delete-by-external-id ok")
if __name__ == "__main__":
print("\nStarte API-Tests...\n")
test_openapi()
test_wiki_health()
# Login nun VOR Semantic Query
did_login = test_wiki_login_if_env()
# Titel ermitteln (semantic/pages oder Fallback)
title, pageid = get_wiki_title_and_pageid()
test_wiki_info(title)
test_wiki_parse(pageid, title)
# Exercise
ext, _id = test_exercise_upsert_and_idempotence()
test_exercise_search_filter(ext)
test_exercise_search_vector()
test_exercise_delete(ext)
print("\n🎉 Alle Tests erfolgreich durchlaufen!\n")