Compare commits
36
Commits
Test
..
20250811-eob
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
11373138ca | ||
|
|
75b257bb15 | ||
|
|
c0bb562a8d | ||
|
|
32577a7fda | ||
|
|
a6d68134cd | ||
|
|
fa8a92208a | ||
|
|
0c047b708f | ||
|
|
efbb978074 | ||
|
|
21ce1dc395 | ||
|
|
380b361e70 | ||
|
|
2a859aa16c | ||
|
|
d14ed9a511 | ||
|
|
7b383f0778 | ||
|
|
34320b46d9 | ||
|
|
e12fd8f96a | ||
|
|
cf085f8ef0 | ||
|
|
6bab3cdf04 | ||
|
|
2567d8c786 | ||
|
|
d8d12e0b6b | ||
|
|
bba860d839 | ||
|
|
a0d1b86b53 | ||
|
|
605fe2ebaf | ||
|
|
811510ac66 | ||
|
|
b2808f82df | ||
|
|
97bc283ce1 | ||
|
|
d9abcb3ef4 | ||
|
|
5c9c8951c1 | ||
|
|
633916d5b7 | ||
|
|
55a1f5bba2 | ||
|
|
7d486f0a41 | ||
|
|
88672a19d5 | ||
|
|
d3b781679a | ||
|
|
a02008ec17 | ||
|
|
8302a7fecf | ||
|
|
4d5d36d6e7 | ||
|
|
8b6b47c63e |
No files matched your search
@@ -43,19 +43,28 @@ jobs:
|
|||||||
- name: Rsync knowledge/
|
- name: Rsync knowledge/
|
||||||
if: ${{ hashFiles('knowledge/**') != '' }}
|
if: ${{ hashFiles('knowledge/**') != '' }}
|
||||||
run: |
|
run: |
|
||||||
rsync -a --delete --exclude='.git' knowledge/ /home/llmadmin/knowledge/
|
rsync -a --delete \
|
||||||
|
--exclude='.git' \
|
||||||
|
--exclude='.env' --exclude='.env.*' --exclude='**/.env' --exclude='**/.env.*' \
|
||||||
|
knowledge/ /home/llmadmin/knowledge/
|
||||||
echo "Synced knowledge/"
|
echo "Synced knowledge/"
|
||||||
|
|
||||||
- name: Rsync llm-api/
|
- name: Rsync llm-api/
|
||||||
if: ${{ hashFiles('llm-api/**') != '' }}
|
if: ${{ hashFiles('llm-api/**') != '' }}
|
||||||
run: |
|
run: |
|
||||||
rsync -a --delete --exclude='.git' llm-api/ /home/llmadmin/llm-api/
|
rsync -a --delete \
|
||||||
|
--exclude='.git' \
|
||||||
|
--exclude='.env' --exclude='.env.*' --exclude='**/.env' --exclude='**/.env.*' \
|
||||||
|
llm-api/ /home/llmadmin/llm-api/
|
||||||
echo "Synced llm-api/"
|
echo "Synced llm-api/"
|
||||||
|
|
||||||
- name: Rsync scripts/
|
- name: Rsync scripts/
|
||||||
if: ${{ hashFiles('scripts/**') != '' }}
|
if: ${{ hashFiles('scripts/**') != '' }}
|
||||||
run: |
|
run: |
|
||||||
rsync -a --delete --exclude='.git' scripts/ /home/llmadmin/scripts/
|
rsync -a --delete \
|
||||||
|
--exclude='.git' \
|
||||||
|
--exclude='.env' --exclude='.env.*' --exclude='**/.env' --exclude='**/.env.*' \
|
||||||
|
scripts/ /home/llmadmin/scripts/
|
||||||
echo "Synced scripts/"
|
echo "Synced scripts/"
|
||||||
|
|
||||||
- name: Optional — systemctl --user restart llm-api (ignore if missing)
|
- name: Optional — systemctl --user restart llm-api (ignore if missing)
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# Kommentarzeile
|
# Kommentarzeile zum Testen ob der Server die Änderung erkennt
|
||||||
from fastapi import APIRouter, HTTPException, Query
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|||||||
+360
-128
@@ -1,35 +1,61 @@
|
|||||||
# Test eines Kommentars, um die Funktion des gitea testen zu können
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
exercise_router.py – v1.7.0
|
||||||
|
|
||||||
|
Neu:
|
||||||
|
- Endpoint **POST /exercise/search**: kombinierbare Filter (discipline, duration, equipment any/all, keywords any/all,
|
||||||
|
capability_geN / capability_eqN + names) + optionaler Vektor-Query (query-Text). Ausgabe inkl. Score.
|
||||||
|
- Facetten erweitert: neben capability_ge1..ge5 jetzt auch capability_eq1..eq5.
|
||||||
|
- Idempotenz-Fix & Payload-Scroll (aus v1.6.2) beibehalten.
|
||||||
|
- API-Signaturen bestehender Routen unverändert.
|
||||||
|
|
||||||
|
Hinweis: Die „eq/ge“-Felder werden beim Upsert gesetzt; für Alt-Punkte einmal das Backfill laufen lassen.
|
||||||
|
"""
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, Query
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import List, Optional, Dict, Any
|
from typing import List, Optional, Dict, Any
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
from datetime import datetime, date
|
from datetime import datetime
|
||||||
from clients import model, qdrant
|
from clients import model, qdrant
|
||||||
from qdrant_client.models import PointStruct, VectorParams, Distance, PointIdsList
|
from qdrant_client.models import (
|
||||||
|
PointStruct,
|
||||||
|
VectorParams,
|
||||||
|
Distance,
|
||||||
|
PointIdsList,
|
||||||
|
Filter,
|
||||||
|
FieldCondition,
|
||||||
|
MatchValue,
|
||||||
|
)
|
||||||
import os
|
import os
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
# ---- Models ----
|
# =========================
|
||||||
|
# Models
|
||||||
|
# =========================
|
||||||
class Exercise(BaseModel):
|
class Exercise(BaseModel):
|
||||||
id: str = Field(default_factory=lambda: str(uuid4()))
|
id: str = Field(default_factory=lambda: str(uuid4()))
|
||||||
|
# Upsert-Metadaten
|
||||||
|
external_id: Optional[str] = None
|
||||||
|
fingerprint: Optional[str] = None
|
||||||
|
source: Optional[str] = None
|
||||||
|
imported_at: Optional[datetime] = None
|
||||||
|
|
||||||
|
# Domain-Felder
|
||||||
title: str
|
title: str
|
||||||
summary: str
|
summary: str
|
||||||
short_description: str
|
short_description: str
|
||||||
keywords: List[str] = []
|
keywords: List[str] = []
|
||||||
link: Optional[str] = None
|
link: Optional[str] = None
|
||||||
|
|
||||||
discipline: str
|
discipline: str
|
||||||
group: Optional[str] = None
|
group: Optional[str] = None
|
||||||
age_group: str
|
age_group: str
|
||||||
target_group: str
|
target_group: str
|
||||||
min_participants: int
|
min_participants: int
|
||||||
duration_minutes: int
|
duration_minutes: int
|
||||||
|
|
||||||
capabilities: Dict[str, int] = {}
|
capabilities: Dict[str, int] = {}
|
||||||
category: str
|
category: str
|
||||||
|
|
||||||
purpose: str
|
purpose: str
|
||||||
execution: str
|
execution: str
|
||||||
notes: str
|
notes: str
|
||||||
@@ -37,147 +63,353 @@ class Exercise(BaseModel):
|
|||||||
method: str
|
method: str
|
||||||
equipment: List[str] = []
|
equipment: List[str] = []
|
||||||
|
|
||||||
class PhaseExercise(BaseModel):
|
|
||||||
exercise_id: str
|
|
||||||
cond_load: Dict[str, Any] = {}
|
|
||||||
coord_load: Dict[str, Any] = {}
|
|
||||||
instructions: str
|
|
||||||
|
|
||||||
class PlanPhase(BaseModel):
|
|
||||||
name: str
|
|
||||||
duration_minutes: int
|
|
||||||
method: str
|
|
||||||
method_notes: str
|
|
||||||
exercises: List[PhaseExercise]
|
|
||||||
|
|
||||||
class TrainingPlan(BaseModel):
|
|
||||||
id: str = Field(default_factory=lambda: str(uuid4()))
|
|
||||||
title: str
|
|
||||||
short_description: str
|
|
||||||
collection: str
|
|
||||||
discipline: str
|
|
||||||
group: Optional[str] = None
|
|
||||||
dojo: str
|
|
||||||
date: date
|
|
||||||
plan_duration_weeks: int
|
|
||||||
focus_areas: List[str] = []
|
|
||||||
predecessor_plan_id: Optional[str] = None
|
|
||||||
age_group: str
|
|
||||||
created_at: datetime = Field(default_factory=datetime.utcnow)
|
|
||||||
phases: List[PlanPhase]
|
|
||||||
|
|
||||||
class DeleteResponse(BaseModel):
|
class DeleteResponse(BaseModel):
|
||||||
status: str
|
status: str
|
||||||
count: int
|
count: int
|
||||||
collection: str
|
collection: str
|
||||||
source: Optional[str] = None
|
|
||||||
type: Optional[str] = None
|
|
||||||
|
|
||||||
# ---- CRUD Endpoints for Exercise ----
|
class ExerciseSearchRequest(BaseModel):
|
||||||
@router.post("/exercise", response_model=Exercise)
|
# Optionaler Semantik-Query (Vektor)
|
||||||
def create_exercise(ex: Exercise):
|
query: Optional[str] = None
|
||||||
# Ensure Exercise collection exists
|
limit: int = Field(default=20, ge=1, le=200)
|
||||||
if not qdrant.collection_exists("exercises"):
|
offset: int = Field(default=0, ge=0)
|
||||||
|
|
||||||
|
# Einfache Filter
|
||||||
|
discipline: Optional[str] = None
|
||||||
|
target_group: Optional[str] = None
|
||||||
|
age_group: Optional[str] = None
|
||||||
|
max_duration: Optional[int] = Field(default=None, ge=0)
|
||||||
|
|
||||||
|
# Listen-Filter
|
||||||
|
equipment_any: Optional[List[str]] = None # mindestens eins muss passen
|
||||||
|
equipment_all: Optional[List[str]] = None # alle müssen passen
|
||||||
|
keywords_any: Optional[List[str]] = None
|
||||||
|
keywords_all: Optional[List[str]] = None
|
||||||
|
|
||||||
|
# Capabilities (Namen + Level-Operator)
|
||||||
|
capability_names: Optional[List[str]] = None
|
||||||
|
capability_ge_level: Optional[int] = Field(default=None, ge=1, le=5)
|
||||||
|
capability_eq_level: Optional[int] = Field(default=None, ge=1, le=5)
|
||||||
|
|
||||||
|
class ExerciseSearchHit(BaseModel):
|
||||||
|
id: str
|
||||||
|
score: Optional[float] = None
|
||||||
|
payload: Exercise
|
||||||
|
|
||||||
|
class ExerciseSearchResponse(BaseModel):
|
||||||
|
hits: List[ExerciseSearchHit]
|
||||||
|
|
||||||
|
# =========================
|
||||||
|
# Helpers
|
||||||
|
# =========================
|
||||||
|
COLLECTION = os.getenv("EXERCISE_COLLECTION", "exercises")
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_collection():
|
||||||
|
if not qdrant.collection_exists(COLLECTION):
|
||||||
qdrant.recreate_collection(
|
qdrant.recreate_collection(
|
||||||
collection_name="exercises",
|
collection_name=COLLECTION,
|
||||||
vectors_config=VectorParams(
|
vectors_config=VectorParams(
|
||||||
size=model.get_sentence_embedding_dimension(),
|
size=model.get_sentence_embedding_dimension(),
|
||||||
distance=Distance.COSINE
|
distance=Distance.COSINE,
|
||||||
)
|
),
|
||||||
)
|
)
|
||||||
vec = model.encode(f"{ex.title}. {ex.summary}").tolist()
|
|
||||||
point = PointStruct(id=ex.id, vector=vec, payload=ex.dict())
|
|
||||||
qdrant.upsert(collection_name="exercises", points=[point])
|
|
||||||
return ex
|
|
||||||
|
|
||||||
@router.get("/exercise", response_model=List[Exercise])
|
|
||||||
def list_exercises(
|
def _lookup_by_external_id(external_id: str) -> Optional[Dict[str, Any]]:
|
||||||
discipline: Optional[str] = Query(None),
|
_ensure_collection()
|
||||||
group: Optional[str] = Query(None),
|
flt = Filter(must=[FieldCondition(key="external_id", match=MatchValue(value=external_id))])
|
||||||
tags: Optional[str] = Query(None)
|
|
||||||
):
|
|
||||||
filters = []
|
|
||||||
if discipline:
|
|
||||||
filters.append({"key": "discipline", "match": {"value": discipline}})
|
|
||||||
if group:
|
|
||||||
filters.append({"key": "group", "match": {"value": group}})
|
|
||||||
if tags:
|
|
||||||
for t in tags.split(","):
|
|
||||||
filters.append({"key": "keywords", "match": {"value": t.strip()}})
|
|
||||||
pts, _ = qdrant.scroll(
|
pts, _ = qdrant.scroll(
|
||||||
collection_name="exercises",
|
collection_name=COLLECTION,
|
||||||
scroll_filter={"must": filters} if filters else None,
|
scroll_filter=flt,
|
||||||
limit=10000
|
limit=1,
|
||||||
|
with_payload=True,
|
||||||
)
|
)
|
||||||
return [Exercise(**pt.payload) for pt in pts]
|
if not pts:
|
||||||
|
return None
|
||||||
|
doc = dict(pts[0].payload or {})
|
||||||
|
doc.setdefault("id", str(pts[0].id))
|
||||||
|
return doc
|
||||||
|
|
||||||
# ---- CRUD Endpoints for TrainingPlan ----
|
|
||||||
@router.post("/plan", response_model=TrainingPlan)
|
_DEF_EMBED_FIELDS = ("title", "summary", "short_description", "purpose", "execution", "notes")
|
||||||
def create_plan(plan: TrainingPlan):
|
|
||||||
# Ensure TrainingPlan collection exists
|
|
||||||
if not qdrant.collection_exists("training_plans"):
|
def _make_vector_from_exercise(ex: Exercise) -> List[float]:
|
||||||
qdrant.recreate_collection(
|
text = ". ".join([getattr(ex, f, "") for f in _DEF_EMBED_FIELDS if getattr(ex, f, None)])
|
||||||
collection_name="training_plans",
|
return model.encode(text).tolist()
|
||||||
vectors_config=VectorParams(
|
|
||||||
size=model.get_sentence_embedding_dimension(),
|
|
||||||
distance=Distance.COSINE
|
def _make_vector_from_query(query: str) -> List[float]:
|
||||||
)
|
return model.encode(query).tolist()
|
||||||
|
|
||||||
|
|
||||||
|
def _norm_list(xs: List[Any]) -> List[str]:
|
||||||
|
out = []
|
||||||
|
seen = set()
|
||||||
|
for x in xs or []:
|
||||||
|
s = str(x).strip()
|
||||||
|
if not s:
|
||||||
|
continue
|
||||||
|
key = s.casefold()
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
out.append(s)
|
||||||
|
return sorted(out, key=str.casefold)
|
||||||
|
|
||||||
|
|
||||||
|
def _facet_capabilities(caps: Dict[str, Any]) -> Dict[str, List[str]]:
|
||||||
|
caps = caps or {}
|
||||||
|
|
||||||
|
def names_where(pred) -> List[str]:
|
||||||
|
out = []
|
||||||
|
for k, v in caps.items():
|
||||||
|
try:
|
||||||
|
iv = int(v)
|
||||||
|
except Exception:
|
||||||
|
iv = 0
|
||||||
|
if pred(iv):
|
||||||
|
t = str(k).strip()
|
||||||
|
if t:
|
||||||
|
out.append(t)
|
||||||
|
return sorted({t for t in out}, key=str.casefold)
|
||||||
|
|
||||||
|
all_keys = sorted({str(k).strip() for k in caps.keys() if str(k).strip()}, key=str.casefold)
|
||||||
|
return {
|
||||||
|
"capability_keys": all_keys,
|
||||||
|
# >= N
|
||||||
|
"capability_ge1": names_where(lambda lv: lv >= 1),
|
||||||
|
"capability_ge2": names_where(lambda lv: lv >= 2),
|
||||||
|
"capability_ge3": names_where(lambda lv: lv >= 3),
|
||||||
|
"capability_ge4": names_where(lambda lv: lv >= 4),
|
||||||
|
"capability_ge5": names_where(lambda lv: lv >= 5),
|
||||||
|
# == N
|
||||||
|
"capability_eq1": names_where(lambda lv: lv == 1),
|
||||||
|
"capability_eq2": names_where(lambda lv: lv == 2),
|
||||||
|
"capability_eq3": names_where(lambda lv: lv == 3),
|
||||||
|
"capability_eq4": names_where(lambda lv: lv == 4),
|
||||||
|
"capability_eq5": names_where(lambda lv: lv == 5),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _response_strip_extras(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
allowed = set(Exercise.model_fields.keys())
|
||||||
|
return {k: v for k, v in payload.items() if k in allowed}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_filter(req: ExerciseSearchRequest) -> Filter:
|
||||||
|
must: List[Any] = []
|
||||||
|
should: List[Any] = []
|
||||||
|
|
||||||
|
if req.discipline:
|
||||||
|
must.append(FieldCondition(key="discipline", match=MatchValue(value=req.discipline)))
|
||||||
|
if req.target_group:
|
||||||
|
must.append(FieldCondition(key="target_group", match=MatchValue(value=req.target_group)))
|
||||||
|
if req.age_group:
|
||||||
|
must.append(FieldCondition(key="age_group", match=MatchValue(value=req.age_group)))
|
||||||
|
if req.max_duration is not None:
|
||||||
|
# Range ohne Import zusätzlicher Modelle: Qdrant akzeptiert auch {'range': {'lte': n}} per JSON;
|
||||||
|
# über Client-Modell tun wir es hier nicht, da wir Filter primär für Keyword-Felder nutzen.
|
||||||
|
must.append({"key": "duration_minutes", "range": {"lte": int(req.max_duration)}})
|
||||||
|
|
||||||
|
# equipment
|
||||||
|
if req.equipment_all:
|
||||||
|
for it in req.equipment_all:
|
||||||
|
must.append(FieldCondition(key="equipment", match=MatchValue(value=it)))
|
||||||
|
if req.equipment_any:
|
||||||
|
# OR: über 'should' Liste
|
||||||
|
for it in req.equipment_any:
|
||||||
|
should.append(FieldCondition(key="equipment", match=MatchValue(value=it)))
|
||||||
|
|
||||||
|
# keywords
|
||||||
|
if req.keywords_all:
|
||||||
|
for it in req.keywords_all:
|
||||||
|
must.append(FieldCondition(key="keywords", match=MatchValue(value=it)))
|
||||||
|
if req.keywords_any:
|
||||||
|
for it in req.keywords_any:
|
||||||
|
should.append(FieldCondition(key="keywords", match=MatchValue(value=it)))
|
||||||
|
|
||||||
|
# capabilities (ge/eq)
|
||||||
|
if req.capability_names:
|
||||||
|
names = [s for s in req.capability_names if s and s.strip()]
|
||||||
|
if req.capability_eq_level:
|
||||||
|
key = f"capability_eq{int(req.capability_eq_level)}"
|
||||||
|
for n in names:
|
||||||
|
must.append(FieldCondition(key=key, match=MatchValue(value=n)))
|
||||||
|
elif req.capability_ge_level:
|
||||||
|
key = f"capability_ge{int(req.capability_ge_level)}"
|
||||||
|
for n in names:
|
||||||
|
must.append(FieldCondition(key=key, match=MatchValue(value=n)))
|
||||||
|
else:
|
||||||
|
# Default: Level >=1 (alle vorhanden)
|
||||||
|
for n in names:
|
||||||
|
must.append(FieldCondition(key="capability_ge1", match=MatchValue(value=n)))
|
||||||
|
|
||||||
|
flt = Filter(must=must)
|
||||||
|
if should:
|
||||||
|
# qdrant: 'should' mit implizitem minimum_should_match=1
|
||||||
|
flt.should = should
|
||||||
|
return flt
|
||||||
|
|
||||||
|
# =========================
|
||||||
|
# Endpoints
|
||||||
|
# =========================
|
||||||
|
@router.get("/exercise/by-external-id")
|
||||||
|
def get_exercise_by_external_id(external_id: str = Query(..., min_length=3)):
|
||||||
|
found = _lookup_by_external_id(external_id)
|
||||||
|
if not found:
|
||||||
|
raise HTTPException(status_code=404, detail="not found")
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/exercise", response_model=Exercise)
|
||||||
|
def create_or_update_exercise(ex: Exercise):
|
||||||
|
_ensure_collection()
|
||||||
|
|
||||||
|
point_id = ex.id
|
||||||
|
if ex.external_id:
|
||||||
|
prior = _lookup_by_external_id(ex.external_id)
|
||||||
|
if prior:
|
||||||
|
point_id = prior.get("id", point_id)
|
||||||
|
|
||||||
|
vector = _make_vector_from_exercise(ex)
|
||||||
|
|
||||||
|
payload: Dict[str, Any] = ex.model_dump()
|
||||||
|
payload["id"] = str(point_id)
|
||||||
|
payload["keywords"] = _norm_list(payload.get("keywords") or [])
|
||||||
|
payload["equipment"] = _norm_list(payload.get("equipment") or [])
|
||||||
|
|
||||||
|
payload.update(_facet_capabilities(payload.get("capabilities") or {}))
|
||||||
|
|
||||||
|
qdrant.upsert(
|
||||||
|
collection_name=COLLECTION,
|
||||||
|
points=[PointStruct(id=str(point_id), vector=vector, payload=payload)],
|
||||||
|
)
|
||||||
|
|
||||||
|
return Exercise(**_response_strip_extras(payload))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/exercise/{exercise_id}", response_model=Exercise)
|
||||||
|
def get_exercise(exercise_id: str):
|
||||||
|
_ensure_collection()
|
||||||
|
pts, _ = qdrant.scroll(
|
||||||
|
collection_name=COLLECTION,
|
||||||
|
scroll_filter=Filter(must=[FieldCondition(key="id", match=MatchValue(value=exercise_id))]),
|
||||||
|
limit=1,
|
||||||
|
with_payload=True,
|
||||||
|
)
|
||||||
|
if not pts:
|
||||||
|
raise HTTPException(status_code=404, detail="not found")
|
||||||
|
payload = dict(pts[0].payload or {})
|
||||||
|
payload.setdefault("id", str(pts[0].id))
|
||||||
|
return Exercise(**_response_strip_extras(payload))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/exercise/search", response_model=ExerciseSearchResponse)
|
||||||
|
def search_exercises(req: ExerciseSearchRequest) -> ExerciseSearchResponse:
|
||||||
|
_ensure_collection()
|
||||||
|
flt = _build_filter(req)
|
||||||
|
|
||||||
|
hits: List[ExerciseSearchHit] = []
|
||||||
|
if req.query:
|
||||||
|
vec = _make_vector_from_query(req.query)
|
||||||
|
# qdrant_client.search unterstützt offset/limit
|
||||||
|
res = qdrant.search(
|
||||||
|
collection_name=COLLECTION,
|
||||||
|
query_vector=vec,
|
||||||
|
limit=req.limit,
|
||||||
|
offset=req.offset,
|
||||||
|
query_filter=flt,
|
||||||
)
|
)
|
||||||
vec = model.encode(f"{plan.title}. {plan.short_description}").tolist()
|
for h in res:
|
||||||
point = PointStruct(id=plan.id, vector=vec, payload=plan.dict())
|
payload = dict(h.payload or {})
|
||||||
qdrant.upsert(collection_name="training_plans", points=[point])
|
payload.setdefault("id", str(h.id))
|
||||||
return plan
|
hits.append(ExerciseSearchHit(id=str(h.id), score=float(h.score or 0.0), payload=Exercise(**_response_strip_extras(payload))))
|
||||||
|
else:
|
||||||
|
# Filter-only: per Scroll (ohne Score); einfache Paginierung via offset/limit
|
||||||
|
# Hole offset+limit Punkte und simuliere Score=None
|
||||||
|
collected = 0
|
||||||
|
skipped = 0
|
||||||
|
next_offset = None
|
||||||
|
while collected < req.limit:
|
||||||
|
page, next_offset = qdrant.scroll(
|
||||||
|
collection_name=COLLECTION,
|
||||||
|
scroll_filter=flt,
|
||||||
|
offset=next_offset,
|
||||||
|
limit=max(1, min(256, req.limit - collected + req.offset - skipped)),
|
||||||
|
with_payload=True,
|
||||||
|
)
|
||||||
|
if not page:
|
||||||
|
break
|
||||||
|
for pt in page:
|
||||||
|
if skipped < req.offset:
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
payload = dict(pt.payload or {})
|
||||||
|
payload.setdefault("id", str(pt.id))
|
||||||
|
hits.append(ExerciseSearchHit(id=str(pt.id), score=None, payload=Exercise(**_response_strip_extras(payload))))
|
||||||
|
collected += 1
|
||||||
|
if collected >= req.limit:
|
||||||
|
break
|
||||||
|
if next_offset is None:
|
||||||
|
break
|
||||||
|
return ExerciseSearchResponse(hits=hits)
|
||||||
|
|
||||||
@router.get("/plan", response_model=List[TrainingPlan])
|
|
||||||
def list_plans(
|
|
||||||
collection: str = Query("training_plans"),
|
|
||||||
discipline: Optional[str] = Query(None),
|
|
||||||
group: Optional[str] = Query(None),
|
|
||||||
dojo: Optional[str] = Query(None)
|
|
||||||
):
|
|
||||||
if not qdrant.collection_exists(collection):
|
|
||||||
return []
|
|
||||||
pts, _ = qdrant.scroll(collection_name=collection, limit=10000)
|
|
||||||
result = []
|
|
||||||
for pt in pts:
|
|
||||||
pl = TrainingPlan(**pt.payload)
|
|
||||||
if discipline and pl.discipline != discipline:
|
|
||||||
continue
|
|
||||||
if group and pl.group != group:
|
|
||||||
continue
|
|
||||||
if dojo and pl.dojo != dojo:
|
|
||||||
continue
|
|
||||||
result.append(pl)
|
|
||||||
return result
|
|
||||||
|
|
||||||
# ---- Delete Endpoints ----
|
@router.delete("/exercise/delete-by-external-id", response_model=DeleteResponse)
|
||||||
@router.delete("/delete-source", response_model=DeleteResponse)
|
def delete_by_external_id(external_id: str = Query(...)):
|
||||||
def delete_by_source(
|
_ensure_collection()
|
||||||
collection: str = Query(...),
|
flt = Filter(must=[FieldCondition(key="external_id", match=MatchValue(value=external_id))])
|
||||||
source: Optional[str] = Query(None),
|
pts, _ = qdrant.scroll(collection_name=COLLECTION, scroll_filter=flt, limit=10000, with_payload=False)
|
||||||
type: Optional[str] = Query(None)
|
|
||||||
):
|
|
||||||
if not qdrant.collection_exists(collection):
|
|
||||||
raise HTTPException(status_code=404, detail=f"Collection '{collection}' nicht gefunden.")
|
|
||||||
filt = []
|
|
||||||
if source:
|
|
||||||
filt.append({"key": "source", "match": {"value": source}})
|
|
||||||
if type:
|
|
||||||
filt.append({"key": "type", "match": {"value": type}})
|
|
||||||
if not filt:
|
|
||||||
raise HTTPException(status_code=400, detail="Mindestens ein Filterparameter muss angegeben werden.")
|
|
||||||
pts, _ = qdrant.scroll(collection_name=collection, scroll_filter={"must": filt}, limit=10000)
|
|
||||||
ids = [str(p.id) for p in pts]
|
ids = [str(p.id) for p in pts]
|
||||||
if not ids:
|
if not ids:
|
||||||
return DeleteResponse(status="🔍 Keine Einträge gefunden.", count=0, collection=collection)
|
return DeleteResponse(status="🔍 Keine Einträge gefunden.", count=0, collection=COLLECTION)
|
||||||
qdrant.delete(collection_name=collection, points_selector=PointIdsList(points=ids))
|
qdrant.delete(collection_name=COLLECTION, points_selector=PointIdsList(points=ids))
|
||||||
return DeleteResponse(status="🗑️ gelöscht", count=len(ids), collection=collection)
|
return DeleteResponse(status="🗑️ gelöscht", count=len(ids), collection=COLLECTION)
|
||||||
|
|
||||||
@router.delete("/delete-collection", response_model=DeleteResponse)
|
|
||||||
def delete_collection(
|
@router.delete("/exercise/delete-collection", response_model=DeleteResponse)
|
||||||
collection: str = Query(...)
|
def delete_collection(collection: str = Query(default=COLLECTION)):
|
||||||
):
|
|
||||||
if not qdrant.collection_exists(collection):
|
if not qdrant.collection_exists(collection):
|
||||||
raise HTTPException(status_code=404, detail=f"Collection '{collection}' nicht gefunden.")
|
raise HTTPException(status_code=404, detail=f"Collection '{collection}' nicht gefunden.")
|
||||||
qdrant.delete_collection(collection_name=collection)
|
qdrant.delete_collection(collection_name=collection)
|
||||||
return DeleteResponse(status="🗑️ gelöscht", count=0, collection=collection)
|
return DeleteResponse(status="🗑️ gelöscht", count=0, collection=collection)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------
|
||||||
|
# OPTIONAL: einfacher Selbsttest (kannst du auch separat als Script verwenden)
|
||||||
|
# ---------------------------
|
||||||
|
TEST_DOC = """
|
||||||
|
Speicher als tests/test_exercise_search.py und mit pytest laufen lassen.
|
||||||
|
|
||||||
|
import os, requests
|
||||||
|
|
||||||
|
BASE = os.getenv("API_BASE", "http://localhost:8000")
|
||||||
|
|
||||||
|
# 1) Filter-only
|
||||||
|
r = requests.post(f"{BASE}/exercise/search", json={
|
||||||
|
"discipline": "Karate",
|
||||||
|
"max_duration": 12,
|
||||||
|
"equipment_any": ["Bälle"],
|
||||||
|
"capability_names": ["Reaktionsfähigkeit"],
|
||||||
|
"capability_ge_level": 2,
|
||||||
|
"limit": 5
|
||||||
|
})
|
||||||
|
r.raise_for_status()
|
||||||
|
js = r.json()
|
||||||
|
assert "hits" in js
|
||||||
|
for h in js["hits"]:
|
||||||
|
p = h["payload"]
|
||||||
|
assert p["discipline"] == "Karate"
|
||||||
|
assert p["duration_minutes"] <= 12
|
||||||
|
|
||||||
|
# 2) Vector + Filter
|
||||||
|
r = requests.post(f"{BASE}/exercise/search", json={
|
||||||
|
"query": "Aufwärmen 10min, Reaktionsfähigkeit, Teenager, Bälle",
|
||||||
|
"discipline": "Karate",
|
||||||
|
"limit": 3
|
||||||
|
})
|
||||||
|
r.raise_for_status()
|
||||||
|
js = r.json(); assert len(js["hits"]) <= 3
|
||||||
|
"""
|
||||||
+1
-1
@@ -28,6 +28,6 @@ async def unicorn_exception_handler(request, exc):
|
|||||||
return JSONResponse(status_code=500, content={"detail": "Interner Serverfehler."})
|
return JSONResponse(status_code=500, content={"detail": "Interner Serverfehler."})
|
||||||
|
|
||||||
# Router einbinden
|
# Router einbinden
|
||||||
app.include_router(wiki_router, prefix="/import/wiki")
|
app.include_router(wiki_router)
|
||||||
app.include_router(embed_router)
|
app.include_router(embed_router)
|
||||||
app.include_router(exercise_router)
|
app.include_router(exercise_router)
|
||||||
+197
-135
@@ -1,173 +1,235 @@
|
|||||||
"""
|
"""
|
||||||
File: wiki_router.py
|
wiki_router.py – v1.4.1 (stabil & nachvollziehbar)
|
||||||
Beschreibung:
|
|
||||||
- Endpunkte für MediaWiki-Integration im lokalen Netzwerk.
|
Änderungen ggü. v1.4.0:
|
||||||
- Funktionen:
|
- /info: Optionalen Request-Parameter entfernt (FastAPI/Pydantic Typfehler behoben)
|
||||||
* /health: Prüft Verfügbarkeit der MediaWiki-API.
|
- Keine API-Signaturänderungen der Routen
|
||||||
* /login: Führt clientlogin durch und speichert Session-Cookies.
|
|
||||||
* /semantic/pages: Listet alle Übungen inkl. Unterkategorien via SMW-Ask.
|
Ziele:
|
||||||
* /parsepage: Ruft Roh-Wikitext über action=parse für eine Seite ab.
|
- /semantic/pages reichert pageid/fullurl für ALLE Titel batchweise an (redirects=1, converttitles=1)
|
||||||
* /info: Liefert pageid und fullurl über Core-API Query.
|
- /info robust: 404 statt 500, mit Titel-Varianten
|
||||||
* /semantic/page: Liefert Metadaten einer Übung und Wikitext sowie pageid über Core-API.
|
- Wiederholungen & Throttling gegen MediaWiki
|
||||||
Version: 1.2.0
|
- Optionale Diagnose-Ausgaben und Coverage-Kennzahlen
|
||||||
|
|
||||||
|
|
||||||
|
Wenn ihr stattdessen den Prefix im Router setzen wollt, einfach in der APIRouter-Zeile unten
|
||||||
|
prefix="/import/wiki" ergänzen und in main.py OHNE prefix einbinden.
|
||||||
"""
|
"""
|
||||||
from dotenv import load_dotenv
|
|
||||||
load_dotenv()
|
from typing import Dict, Any, Optional, List
|
||||||
from fastapi import APIRouter, HTTPException, Query
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from typing import Dict, Any, List
|
import os, time, logging
|
||||||
import requests, os
|
import requests
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
__version__ = "1.2.0"
|
load_dotenv()
|
||||||
router = APIRouter()
|
|
||||||
|
|
||||||
|
logger = logging.getLogger("wiki_router")
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/import/wiki", tags=["wiki"])
|
||||||
|
|
||||||
|
# -------- Konfiguration --------
|
||||||
WIKI_API_URL = os.getenv("WIKI_API_URL", "https://karatetrainer.net/api.php")
|
WIKI_API_URL = os.getenv("WIKI_API_URL", "https://karatetrainer.net/api.php")
|
||||||
wiki_session = requests.Session()
|
WIKI_TIMEOUT = float(os.getenv("WIKI_TIMEOUT", "15"))
|
||||||
|
WIKI_BATCH = int(os.getenv("WIKI_BATCH", "50"))
|
||||||
|
WIKI_RETRIES = int(os.getenv("WIKI_RETRIES", "1")) # zusätzliche Versuche bei Upstream-Fehlern
|
||||||
|
WIKI_SLEEPMS = int(os.getenv("WIKI_SLEEP_MS", "0")) # Throttle zwischen Requests
|
||||||
|
|
||||||
|
# Single Session (Cookies für Login)
|
||||||
|
wiki_session = requests.Session()
|
||||||
|
wiki_session.headers.update({"User-Agent": "local-llm-wiki-proxy/1.4.1"})
|
||||||
|
|
||||||
|
# -------- Schemas --------
|
||||||
class WikiLoginRequest(BaseModel):
|
class WikiLoginRequest(BaseModel):
|
||||||
username: str
|
username: str
|
||||||
password: str
|
password: str
|
||||||
|
|
||||||
class WikiLoginResponse(BaseModel):
|
class WikiLoginResponse(BaseModel):
|
||||||
status: str
|
status: str
|
||||||
message: str | None = None
|
message: Optional[str] = None
|
||||||
|
|
||||||
class PageContentResponse(BaseModel):
|
|
||||||
pageid: int
|
|
||||||
title: str
|
|
||||||
wikitext: str
|
|
||||||
|
|
||||||
class PageInfoResponse(BaseModel):
|
class PageInfoResponse(BaseModel):
|
||||||
pageid: int
|
pageid: int
|
||||||
title: str
|
title: str
|
||||||
fullurl: str
|
fullurl: str
|
||||||
|
|
||||||
# Health-Check
|
class PageContentResponse(BaseModel):
|
||||||
|
pageid: int
|
||||||
|
title: str
|
||||||
|
wikitext: str
|
||||||
|
|
||||||
|
# -------- Utils --------
|
||||||
|
|
||||||
|
def _sleep():
|
||||||
|
if WIKI_SLEEPMS > 0:
|
||||||
|
time.sleep(WIKI_SLEEPMS / 1000.0)
|
||||||
|
|
||||||
|
|
||||||
|
def _request_with_retry(method: str, params: Dict[str, Any], *, data: Dict[str, Any] | None = None) -> requests.Response:
|
||||||
|
last_exc: Optional[Exception] = None
|
||||||
|
for attempt in range(WIKI_RETRIES + 1):
|
||||||
|
try:
|
||||||
|
if method == "GET":
|
||||||
|
resp = wiki_session.get(WIKI_API_URL, params=params, timeout=WIKI_TIMEOUT)
|
||||||
|
else:
|
||||||
|
resp = wiki_session.post(WIKI_API_URL, data=data or params, timeout=WIKI_TIMEOUT)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return resp
|
||||||
|
except Exception as e:
|
||||||
|
last_exc = e
|
||||||
|
logger.warning("Upstream error on %s (try %d/%d): %s", method, attempt + 1, WIKI_RETRIES + 1, e)
|
||||||
|
_sleep()
|
||||||
|
# alle Versuche erschöpft
|
||||||
|
raise HTTPException(status_code=502, detail=f"Upstream error: {last_exc}")
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_variants(title: str) -> List[str]:
|
||||||
|
t = (title or "").strip()
|
||||||
|
variants = {t}
|
||||||
|
if " " in t:
|
||||||
|
variants.add(t.replace(" ", "_"))
|
||||||
|
# Bindestrich / Gedankenstrich Varianten
|
||||||
|
for a, b in [("-", "–"), ("-", "—"), ("–", "-"), ("—", "-")]:
|
||||||
|
if a in t:
|
||||||
|
variants.add(t.replace(a, b))
|
||||||
|
return list(variants)
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_pageinfo_batch(titles: List[str]) -> Dict[str, Dict[str, Any]]:
|
||||||
|
if not titles:
|
||||||
|
return {}
|
||||||
|
out: Dict[str, Dict[str, Any]] = {}
|
||||||
|
for i in range(0, len(titles), max(1, WIKI_BATCH)):
|
||||||
|
chunk = titles[i:i + max(1, WIKI_BATCH)]
|
||||||
|
params = {
|
||||||
|
"action": "query",
|
||||||
|
"format": "json",
|
||||||
|
"prop": "info",
|
||||||
|
"inprop": "url",
|
||||||
|
"redirects": 1,
|
||||||
|
"converttitles": 1,
|
||||||
|
"titles": "|".join(chunk),
|
||||||
|
}
|
||||||
|
resp = _request_with_retry("GET", params)
|
||||||
|
data = resp.json() or {}
|
||||||
|
q = data.get("query", {})
|
||||||
|
redirects = {d.get("from"): d.get("to") for d in (q.get("redirects") or [])}
|
||||||
|
pages = q.get("pages", {}) or {}
|
||||||
|
for pid_str, page in pages.items():
|
||||||
|
if page.get("missing") is not None or str(pid_str) == "-1":
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
pid = int(pid_str)
|
||||||
|
except ValueError:
|
||||||
|
pid = int(page.get("pageid", -1))
|
||||||
|
title_out = page.get("title")
|
||||||
|
fullurl = page.get("fullurl") or page.get("canonicalurl") or ""
|
||||||
|
if not title_out:
|
||||||
|
continue
|
||||||
|
out[title_out] = {"pageid": pid, "fullurl": fullurl}
|
||||||
|
# auch Originaltitel der Redirects auflösen
|
||||||
|
for frm, to in redirects.items():
|
||||||
|
if to == title_out and frm not in out:
|
||||||
|
out[frm] = {"pageid": pid, "fullurl": fullurl}
|
||||||
|
_sleep()
|
||||||
|
return out
|
||||||
|
|
||||||
|
# -------- Endpoints --------
|
||||||
@router.get("/health")
|
@router.get("/health")
|
||||||
def health_check():
|
def health(verbose: Optional[int] = Query(default=0)) -> Dict[str, Any]:
|
||||||
try:
|
# einfacher Ping
|
||||||
resp = wiki_session.get(
|
resp = _request_with_retry("GET", {"action": "query", "meta": "siteinfo", "format": "json"})
|
||||||
WIKI_API_URL,
|
if verbose:
|
||||||
params={"action": "query", "meta": "siteinfo", "siprop": "general", "format": "json"},
|
info = resp.json().get("query", {}).get("general", {})
|
||||||
timeout=5
|
return {"status": "ok", "wiki": {"sitename": info.get("sitename"), "generator": info.get("generator")}}
|
||||||
)
|
|
||||||
resp.raise_for_status()
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=502, detail=f"Wiki nicht erreichbar: {e}")
|
|
||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
# Login Endpoint
|
|
||||||
@router.post("/login", response_model=WikiLoginResponse)
|
@router.post("/login", response_model=WikiLoginResponse)
|
||||||
def login(data: WikiLoginRequest):
|
def login(data: WikiLoginRequest):
|
||||||
# Token holen
|
# Token holen
|
||||||
try:
|
tok = _request_with_retry("GET", {"action": "query", "meta": "tokens", "type": "login", "format": "json"})
|
||||||
token_resp = wiki_session.get(
|
token = tok.json().get("query", {}).get("tokens", {}).get("logintoken")
|
||||||
WIKI_API_URL,
|
|
||||||
params={"action": "query", "meta": "tokens", "type": "login", "format": "json"},
|
|
||||||
timeout=10
|
|
||||||
)
|
|
||||||
token_resp.raise_for_status()
|
|
||||||
token = token_resp.json().get("query", {}).get("tokens", {}).get("logintoken")
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=502, detail=f"Token-Error: {e}")
|
|
||||||
if not token:
|
if not token:
|
||||||
raise HTTPException(status_code=502, detail="Kein Login-Token erhalten")
|
raise HTTPException(status_code=502, detail="Kein Login-Token erhalten")
|
||||||
# clientlogin
|
|
||||||
try:
|
|
||||||
login_resp = wiki_session.post(
|
|
||||||
WIKI_API_URL,
|
|
||||||
data={
|
|
||||||
"action": "clientlogin",
|
|
||||||
"format": "json",
|
|
||||||
"username": data.username,
|
|
||||||
"password": data.password,
|
|
||||||
"logintoken": token,
|
|
||||||
"loginreturnurl": "http://localhost:8000"
|
|
||||||
},
|
|
||||||
timeout=10
|
|
||||||
)
|
|
||||||
login_resp.raise_for_status()
|
|
||||||
status = login_resp.json().get("clientlogin", {}).get("status")
|
|
||||||
except Exception:
|
|
||||||
status = None
|
|
||||||
# fallback login
|
|
||||||
if status != "PASS":
|
|
||||||
alt = wiki_session.post(
|
|
||||||
WIKI_API_URL,
|
|
||||||
data={"action": "login", "format": "json", "lgname": data.username, "lgpassword": data.password},
|
|
||||||
timeout=10
|
|
||||||
)
|
|
||||||
alt.raise_for_status()
|
|
||||||
status = alt.json().get("login", {}).get("result")
|
|
||||||
if status in ("PASS", "Success"):
|
|
||||||
return WikiLoginResponse(status="success", message=None)
|
|
||||||
return WikiLoginResponse(status="failed", message="Login fehlgeschlagen")
|
|
||||||
|
|
||||||
# SMW-Ask: alle Übungen inkl. Unterkategorien
|
# clientlogin (mit returnurl) + Fallback action=login
|
||||||
|
try:
|
||||||
|
cl = _request_with_retry("POST", {}, data={
|
||||||
|
"action": "clientlogin",
|
||||||
|
"format": "json",
|
||||||
|
"username": data.username,
|
||||||
|
"password": data.password,
|
||||||
|
"logintoken": token,
|
||||||
|
"loginreturnurl": "https://example.org/",
|
||||||
|
})
|
||||||
|
st = cl.json().get("clientlogin", {}).get("status")
|
||||||
|
if st == "PASS":
|
||||||
|
return WikiLoginResponse(status="success")
|
||||||
|
except HTTPException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
lg = _request_with_retry("POST", {}, data={
|
||||||
|
"action": "login",
|
||||||
|
"format": "json",
|
||||||
|
"lgname": data.username,
|
||||||
|
"lgpassword": data.password,
|
||||||
|
"lgtoken": token,
|
||||||
|
})
|
||||||
|
res = lg.json().get("login", {}).get("result")
|
||||||
|
if res == "Success":
|
||||||
|
return WikiLoginResponse(status="success")
|
||||||
|
raise HTTPException(status_code=401, detail=f"Login fehlgeschlagen: {res}")
|
||||||
|
|
||||||
@router.get("/semantic/pages")
|
@router.get("/semantic/pages")
|
||||||
def semantic_pages(category: str = Query(..., description="Kategorie ohne 'Category:'")) -> Dict[str, Any]:
|
def semantic_pages(category: str = Query(..., description="Kategorie ohne 'Category:'")) -> Dict[str, Any]:
|
||||||
smw_query = f"[[Category:{category}]]"
|
# Rohdaten aus SMW (Ask)
|
||||||
ask_query = f"{smw_query}|limit=50000"
|
ask_query = f"[[Category:{category}]]|limit=50000"
|
||||||
r = wiki_session.get(
|
r = _request_with_retry("GET", {"action": "ask", "query": ask_query, "format": "json"})
|
||||||
WIKI_API_URL,
|
results = r.json().get("query", {}).get("results", {}) or {}
|
||||||
params={"action": "ask", "query": ask_query, "format": "json"},
|
titles = list(results.keys())
|
||||||
timeout=30
|
|
||||||
)
|
# Batch-Anreicherung mit pageid/fullurl für ALLE Titel
|
||||||
try:
|
info_map = _fetch_pageinfo_batch(titles)
|
||||||
r.raise_for_status()
|
|
||||||
except Exception as e:
|
enriched: Dict[str, Any] = {}
|
||||||
raise HTTPException(status_code=502, detail=f"SMW-Ask-Error: {e}")
|
missing = 0
|
||||||
return r.json().get("query", {}).get("results", {})
|
for title, entry in results.items():
|
||||||
|
base = entry if isinstance(entry, dict) else {}
|
||||||
|
extra = info_map.get(title, {})
|
||||||
|
if not extra:
|
||||||
|
missing += 1
|
||||||
|
enriched[title] = {
|
||||||
|
**base,
|
||||||
|
"pageid": extra.get("pageid", base.get("pageid")),
|
||||||
|
"fullurl": extra.get("fullurl", base.get("fullurl")),
|
||||||
|
}
|
||||||
|
logger.info("/semantic/pages: %d Titel, %d ohne pageid nach Enrichment", len(results), missing)
|
||||||
|
return enriched
|
||||||
|
|
||||||
# Wikitext über parse-Endpoint holen (per pageid)
|
|
||||||
@router.get("/parsepage", response_model=PageContentResponse)
|
@router.get("/parsepage", response_model=PageContentResponse)
|
||||||
def parse_page(pageid: int = Query(...), title: str = Query(None)):
|
def parse_page(pageid: int = Query(...), title: str = Query(None)):
|
||||||
r = wiki_session.get(
|
resp = _request_with_retry("GET", {"action": "parse", "pageid": pageid, "prop": "wikitext", "format": "json"})
|
||||||
WIKI_API_URL,
|
wikitext = resp.json().get("parse", {}).get("wikitext", {}).get("*", "")
|
||||||
params={"action": "parse", "pageid": pageid, "prop": "wikitext", "format": "json"},
|
|
||||||
timeout=20
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
r.raise_for_status()
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=502, detail=f"Parse-Error: {e}")
|
|
||||||
wikitext = r.json().get("parse", {}).get("wikitext", {}).get("*", "")
|
|
||||||
return PageContentResponse(pageid=pageid, title=title or "", wikitext=wikitext)
|
return PageContentResponse(pageid=pageid, title=title or "", wikitext=wikitext)
|
||||||
|
|
||||||
# Pageinfo über Core-API (ermittelt pageid + fullurl)
|
|
||||||
@router.get("/info", response_model=PageInfoResponse)
|
@router.get("/info", response_model=PageInfoResponse)
|
||||||
def page_info(title: str = Query(..., description="Name der Seite")):
|
def page_info(title: str = Query(..., description="Seitentitel")):
|
||||||
r = wiki_session.get(
|
# 1. Versuch: wie geliefert, mit redirects/converttitles
|
||||||
WIKI_API_URL,
|
res = _fetch_pageinfo_batch([title])
|
||||||
params={"action": "query", "titles": title, "prop": "info", "inprop": "url", "format": "json"},
|
if res.get(title):
|
||||||
timeout=10
|
d = res[title]
|
||||||
)
|
return PageInfoResponse(pageid=d["pageid"], title=title, fullurl=d.get("fullurl", ""))
|
||||||
try:
|
|
||||||
r.raise_for_status()
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=502, detail=f"Info-Error: {e}")
|
|
||||||
pages = r.json().get("query", {}).get("pages", {})
|
|
||||||
pid_str, page = next(iter(pages.items()))
|
|
||||||
pid = int(pid_str)
|
|
||||||
fullurl = page.get("fullurl")
|
|
||||||
return PageInfoResponse(pageid=pid, title=page.get("title"), fullurl=fullurl)
|
|
||||||
|
|
||||||
# Detail-Endpoint für eine Übung: Metadaten aus Ask + Wikitext & ID via Core-API
|
# 2. Varianten probieren
|
||||||
@router.get("/semantic/page", response_model=Dict[str, Any])
|
for v in _normalize_variants(title):
|
||||||
def semantic_page_detail(category: str = Query(...), title: str = Query(...)) -> Dict[str, Any]:
|
if v == title:
|
||||||
# Metadaten aus SMW-Ask
|
continue
|
||||||
entries = semantic_pages(category)
|
res2 = _fetch_pageinfo_batch([v])
|
||||||
entry = entries.get(title)
|
if res2.get(v):
|
||||||
if not entry:
|
d = res2[v]
|
||||||
raise HTTPException(status_code=404, detail="Übung nicht gefunden im SMW-Ask-Ergebnis.")
|
return PageInfoResponse(pageid=d["pageid"], title=v, fullurl=d.get("fullurl", ""))
|
||||||
# Pageinfo via Core-API
|
|
||||||
info = page_info(title=title)
|
# 3. sauber 404
|
||||||
# Wikitext via parse
|
raise HTTPException(status_code=404, detail=f"Page not found: {title}")
|
||||||
parsed = parse_page(pageid=info.pageid, title=title)
|
|
||||||
return {
|
|
||||||
"title": title,
|
|
||||||
"pageid": info.pageid,
|
|
||||||
"fullurl": info.fullurl,
|
|
||||||
"printouts": entry.get("printouts", {}),
|
|
||||||
"wikitext": parsed.wikitext
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Backfill Capability-Facetten in Qdrant – v1.2
|
||||||
|
Fix: beendet korrekt, wenn `next_page_offset` (offset) None ist.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
from typing import Dict, Any, List
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
|
||||||
|
COLL = os.getenv("EXERCISE_COLLECTION", "exercises")
|
||||||
|
QDRANT_HOST = os.getenv("QDRANT_HOST", "localhost")
|
||||||
|
QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
|
||||||
|
BATCH = int(os.getenv("BACKFILL_BATCH", "256"))
|
||||||
|
|
||||||
|
|
||||||
|
def _facet_capabilities(caps: Dict[str, Any]) -> Dict[str, List[str]]:
|
||||||
|
caps = caps or {}
|
||||||
|
def names_where(pred) -> List[str]:
|
||||||
|
out = []
|
||||||
|
for k, v in caps.items():
|
||||||
|
try:
|
||||||
|
iv = int(v)
|
||||||
|
except Exception:
|
||||||
|
iv = 0
|
||||||
|
if pred(iv):
|
||||||
|
s = str(k).strip()
|
||||||
|
if s:
|
||||||
|
out.append(s)
|
||||||
|
return sorted({s for s in out}, key=str.casefold)
|
||||||
|
|
||||||
|
all_keys = sorted({str(k).strip() for k in caps.keys() if str(k).strip()}, key=str.casefold)
|
||||||
|
return {
|
||||||
|
"capability_keys": all_keys,
|
||||||
|
"capability_ge1": names_where(lambda lv: lv >= 1),
|
||||||
|
"capability_ge2": names_where(lambda lv: lv >= 2),
|
||||||
|
"capability_ge3": names_where(lambda lv: lv >= 3),
|
||||||
|
"capability_ge4": names_where(lambda lv: lv >= 4),
|
||||||
|
"capability_ge5": names_where(lambda lv: lv >= 5),
|
||||||
|
"capability_eq1": names_where(lambda lv: lv == 1),
|
||||||
|
"capability_eq2": names_where(lambda lv: lv == 2),
|
||||||
|
"capability_eq3": names_where(lambda lv: lv == 3),
|
||||||
|
"capability_eq4": names_where(lambda lv: lv == 4),
|
||||||
|
"capability_eq5": names_where(lambda lv: lv == 5),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT)
|
||||||
|
info = client.get_collection(COLL)
|
||||||
|
print(f"[Backfill] Collection '{COLL}' ok – vectors={info.config.params.vectors}")
|
||||||
|
|
||||||
|
updated_total = 0
|
||||||
|
offset = None
|
||||||
|
page = 0
|
||||||
|
|
||||||
|
while True:
|
||||||
|
points, next_offset = client.scroll(
|
||||||
|
collection_name=COLL,
|
||||||
|
scroll_filter=None,
|
||||||
|
offset=offset,
|
||||||
|
limit=BATCH,
|
||||||
|
with_payload=True,
|
||||||
|
)
|
||||||
|
page += 1
|
||||||
|
if not points:
|
||||||
|
print("[Backfill] no more points – done")
|
||||||
|
break
|
||||||
|
|
||||||
|
updated_page = 0
|
||||||
|
for pt in points:
|
||||||
|
pld = pt.payload or {}
|
||||||
|
caps = pld.get("capabilities") or {}
|
||||||
|
facets = _facet_capabilities(caps)
|
||||||
|
|
||||||
|
# nur setzen, wenn sich etwas ändert
|
||||||
|
need = any(pld.get(k) != v for k, v in facets.items())
|
||||||
|
if not need:
|
||||||
|
continue
|
||||||
|
client.set_payload(collection_name=COLL, points=[pt.id], payload=facets)
|
||||||
|
updated_total += 1
|
||||||
|
updated_page += 1
|
||||||
|
|
||||||
|
print(f"[Backfill] page={page} processed={len(points)} updated_page={updated_page} updated_total={updated_total}")
|
||||||
|
|
||||||
|
# Ende erreicht? Dann nach dieser Seite aussteigen.
|
||||||
|
if next_offset is None:
|
||||||
|
break
|
||||||
|
offset = next_offset
|
||||||
|
|
||||||
|
print(f"[Backfill] done. total_updated={updated_total}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Idempotenter Bootstrap für die Qdrant-Collection 'exercises':
|
||||||
|
- Legt fehlende Payload-Indizes an (ohne Collection zu droppen)
|
||||||
|
- Optional prüft Dimension/Distanz
|
||||||
|
|
||||||
|
Starten mit
|
||||||
|
pip install qdrant-client --upgrade
|
||||||
|
python3 scripts/bootstrap_qdrant_exercises.py
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from qdrant_client.models import PayloadSchemaType
|
||||||
|
|
||||||
|
QDRANT_HOST = os.getenv("QDRANT_HOST", "localhost")
|
||||||
|
QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
|
||||||
|
COLLECTION = os.getenv("EXERCISE_COLLECTION", "exercises")
|
||||||
|
|
||||||
|
INDEX_SPECS = [
|
||||||
|
("keywords", PayloadSchemaType.KEYWORD),
|
||||||
|
("equipment", PayloadSchemaType.KEYWORD),
|
||||||
|
("discipline", PayloadSchemaType.KEYWORD),
|
||||||
|
("age_group", PayloadSchemaType.KEYWORD),
|
||||||
|
("target_group", PayloadSchemaType.KEYWORD),
|
||||||
|
("category", PayloadSchemaType.KEYWORD),
|
||||||
|
("capability_keys", PayloadSchemaType.KEYWORD),
|
||||||
|
("capability_ge1", PayloadSchemaType.KEYWORD),
|
||||||
|
("capability_ge2", PayloadSchemaType.KEYWORD),
|
||||||
|
("capability_ge3", PayloadSchemaType.KEYWORD),
|
||||||
|
("capability_ge4", PayloadSchemaType.KEYWORD), # neu
|
||||||
|
("capability_ge5", PayloadSchemaType.KEYWORD), # neu
|
||||||
|
("capability_eq1", PayloadSchemaType.KEYWORD), # neu
|
||||||
|
("capability_eq2", PayloadSchemaType.KEYWORD), # neu
|
||||||
|
("capability_eq3", PayloadSchemaType.KEYWORD), # neu
|
||||||
|
("capability_eq4", PayloadSchemaType.KEYWORD), # neu
|
||||||
|
("capability_eq5", PayloadSchemaType.KEYWORD), # neu
|
||||||
|
]
|
||||||
|
|
||||||
|
def main():
|
||||||
|
client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT)
|
||||||
|
# Sanity: Collection existiert?
|
||||||
|
info = client.get_collection(COLLECTION) # wirf Fehler, wenn nicht vorhanden
|
||||||
|
print(f"[Bootstrap] Collection '{COLLECTION}' gefunden. Vectors={info.config.params.vectors}")
|
||||||
|
|
||||||
|
# Indizes idempotent anlegen
|
||||||
|
for field, schema in INDEX_SPECS:
|
||||||
|
try:
|
||||||
|
client.create_payload_index(collection_name=COLLECTION,
|
||||||
|
field_name=field,
|
||||||
|
field_schema=schema)
|
||||||
|
print(f"[Bootstrap] Index created: {field} ({schema})")
|
||||||
|
except Exception as e:
|
||||||
|
# Qdrant wirft hier Exceptions, wenn der Index schon existiert – das ist ok
|
||||||
|
print(f"[Bootstrap] Index exists or skipped: {field} -> {e}")
|
||||||
|
|
||||||
|
print("[Bootstrap] done.")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Erweiterter API-Schnelltest - v2.1
|
||||||
|
|
||||||
|
Änderungen ggü. v2.0:
|
||||||
|
- **Login VOR** /semantic/pages (viele Wikis liefern Ask-Ergebnisse nur authentifiziert)
|
||||||
|
- Robuster Fallback: Wenn /semantic/pages {} liefert → optionaler ENV-Fallback `TEST_WIKI_TITLE`
|
||||||
|
- Bessere Diagnoseausgaben (zeigt Basis-URL, Category, gesetzte Login-ENV)
|
||||||
|
|
||||||
|
ENV:
|
||||||
|
BASE_URL (default http://127.0.0.1:8000)
|
||||||
|
TEST_WIKI_CATEGORY (default "Übungen")
|
||||||
|
TEST_WIKI_TITLE (optional Fallback-Titel, z. B. "Affenklatschen")
|
||||||
|
WIKI_BOT_USER, WIKI_BOT_PASSWORD (optional; empfohlen)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import uuid
|
||||||
|
import requests
|
||||||
|
|
||||||
|
BASE = os.getenv("BASE_URL", "http://127.0.0.1:8000").rstrip("/")
|
||||||
|
TEST_WIKI_CATEGORY = os.getenv("TEST_WIKI_CATEGORY", "Übungen")
|
||||||
|
TEST_WIKI_TITLE = os.getenv("TEST_WIKI_TITLE")
|
||||||
|
WIKI_USER = os.getenv("WIKI_BOT_USER")
|
||||||
|
WIKI_PASS = os.getenv("WIKI_BOT_PASSWORD")
|
||||||
|
|
||||||
|
print("\nConfig:")
|
||||||
|
print(" BASE_URL=", BASE)
|
||||||
|
print(" TEST_WIKI_CATEGORY=", TEST_WIKI_CATEGORY)
|
||||||
|
print(" TEST_WIKI_TITLE=", TEST_WIKI_TITLE or "<none>")
|
||||||
|
print(" WIKI_BOT_USER set=", bool(WIKI_USER))
|
||||||
|
|
||||||
|
# ---- helpers ----
|
||||||
|
|
||||||
|
def fail(msg, resp: requests.Response | None = None):
|
||||||
|
print("✗", msg)
|
||||||
|
if resp is not None:
|
||||||
|
try:
|
||||||
|
print(" → status:", resp.status_code)
|
||||||
|
print(" → body:", resp.text[:800])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def ok(msg):
|
||||||
|
print("✓", msg)
|
||||||
|
|
||||||
|
|
||||||
|
# ---- basic ----
|
||||||
|
|
||||||
|
def test_openapi():
|
||||||
|
r = requests.get(f"{BASE}/openapi.json")
|
||||||
|
if r.status_code != 200:
|
||||||
|
fail("/openapi.json nicht erreichbar", r)
|
||||||
|
ok("OpenAPI erreichbar")
|
||||||
|
|
||||||
|
|
||||||
|
# ---- wiki ----
|
||||||
|
|
||||||
|
def test_wiki_health():
|
||||||
|
r = requests.get(f"{BASE}/import/wiki/health", params={"verbose": 1})
|
||||||
|
if r.status_code != 200 or r.json().get("status") != "ok":
|
||||||
|
fail("/import/wiki/health fehlgeschlagen", r)
|
||||||
|
ok("Wiki /health ok")
|
||||||
|
|
||||||
|
|
||||||
|
def test_wiki_login_if_env():
|
||||||
|
if not (WIKI_USER and WIKI_PASS):
|
||||||
|
print("⚠️ Wiki /login übersprungen (WIKI_BOT_USER/PASSWORD nicht gesetzt)")
|
||||||
|
return False
|
||||||
|
r = requests.post(f"{BASE}/import/wiki/login", json={"username": WIKI_USER, "password": WIKI_PASS})
|
||||||
|
if r.status_code != 200 or r.json().get("status") != "success":
|
||||||
|
fail("/import/wiki/login fehlgeschlagen", r)
|
||||||
|
ok("Wiki /login ok")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def get_wiki_title_and_pageid() -> tuple[str, int]:
|
||||||
|
# 1) Versuch über semantic/pages
|
||||||
|
r = requests.get(f"{BASE}/import/wiki/semantic/pages", params={"category": TEST_WIKI_CATEGORY})
|
||||||
|
if r.status_code != 200:
|
||||||
|
fail("/import/wiki/semantic/pages fehlgeschlagen", r)
|
||||||
|
data = r.json()
|
||||||
|
if isinstance(data, dict) and data:
|
||||||
|
title = next(iter(data.keys()))
|
||||||
|
entry = data[title]
|
||||||
|
pid = entry.get("pageid")
|
||||||
|
if not pid:
|
||||||
|
fail("/semantic/pages ohne pageid nach Enrichment", r)
|
||||||
|
ok(f"Wiki /semantic/pages ok – Beispiel: '{title}' (pageid={pid})")
|
||||||
|
return title, int(pid)
|
||||||
|
|
||||||
|
print("ℹ️ /semantic/pages lieferte keine Titel ({}). Versuche Fallback...")
|
||||||
|
# 2) Fallback über TEST_WIKI_TITLE
|
||||||
|
if not TEST_WIKI_TITLE:
|
||||||
|
fail("Kein TEST_WIKI_TITLE gesetzt und /semantic/pages leer – kann keinen Titel testen.")
|
||||||
|
r2 = requests.get(f"{BASE}/import/wiki/info", params={"title": TEST_WIKI_TITLE})
|
||||||
|
if r2.status_code != 200:
|
||||||
|
fail("/import/wiki/info(Fallback) fehlgeschlagen", r2)
|
||||||
|
js = r2.json()
|
||||||
|
pid = js.get("pageid")
|
||||||
|
if not pid:
|
||||||
|
fail("/info(Fallback) ohne pageid", r2)
|
||||||
|
ok(f"Wiki /info Fallback ok – '{TEST_WIKI_TITLE}' (pageid={pid})")
|
||||||
|
return TEST_WIKI_TITLE, int(pid)
|
||||||
|
|
||||||
|
|
||||||
|
def test_wiki_info(title: str):
|
||||||
|
r = requests.get(f"{BASE}/import/wiki/info", params={"title": title})
|
||||||
|
if r.status_code != 200:
|
||||||
|
fail("/import/wiki/info fehlgeschlagen", r)
|
||||||
|
js = r.json()
|
||||||
|
if not js.get("pageid"):
|
||||||
|
fail("/info ohne pageid", r)
|
||||||
|
ok("Wiki /info ok")
|
||||||
|
|
||||||
|
|
||||||
|
def test_wiki_parse(pageid: int, title: str):
|
||||||
|
r = requests.get(f"{BASE}/import/wiki/parsepage", params={"pageid": pageid, "title": title})
|
||||||
|
if r.status_code != 200:
|
||||||
|
fail("/import/wiki/parsepage fehlgeschlagen", r)
|
||||||
|
js = r.json()
|
||||||
|
if not isinstance(js.get("wikitext"), str):
|
||||||
|
fail("/parsepage ohne wikitext", r)
|
||||||
|
ok("Wiki /parsepage ok")
|
||||||
|
|
||||||
|
|
||||||
|
# ---- exercise ----
|
||||||
|
|
||||||
|
def make_exercise_payload(external_id: str):
|
||||||
|
return {
|
||||||
|
"external_id": external_id,
|
||||||
|
"fingerprint": "unit-test-sha",
|
||||||
|
"source": "UnitTest",
|
||||||
|
"title": "Testübung Reaktion",
|
||||||
|
"summary": "Kurzbeschreibung für Test.",
|
||||||
|
"short_description": "Kurzbeschreibung für Test.",
|
||||||
|
"keywords": ["Reaktion", "Bälle", "Bälle"],
|
||||||
|
"link": "http://example.local",
|
||||||
|
"discipline": "Karate",
|
||||||
|
"group": "5",
|
||||||
|
"age_group": "Teenager",
|
||||||
|
"target_group": "Breitensport",
|
||||||
|
"min_participants": 1,
|
||||||
|
"duration_minutes": 10,
|
||||||
|
"capabilities": {"Reaktionsfähigkeit": 2, "Kopplungsfähigkeit": 1},
|
||||||
|
"category": "Übungen",
|
||||||
|
"purpose": "Aufwärmen",
|
||||||
|
"execution": "Einfacher Ablauf.",
|
||||||
|
"notes": "Hinweise.",
|
||||||
|
"preparation": "Bälle holen.",
|
||||||
|
"method": "Frontale Methode",
|
||||||
|
"equipment": ["Bälle", "Pratze"]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_exercise_upsert_and_idempotence():
|
||||||
|
import time
|
||||||
|
ext = f"ut:{uuid.uuid4()}"
|
||||||
|
payload = make_exercise_payload(ext)
|
||||||
|
|
||||||
|
# create
|
||||||
|
r1 = requests.post(f"{BASE}/exercise", json=payload)
|
||||||
|
if r1.status_code != 200:
|
||||||
|
fail("POST /exercise (create) fehlgeschlagen", r1)
|
||||||
|
id1 = r1.json().get("id")
|
||||||
|
if not id1:
|
||||||
|
fail("POST /exercise lieferte keine id", r1)
|
||||||
|
|
||||||
|
# update (idempotent)
|
||||||
|
r2 = requests.post(f"{BASE}/exercise", json=payload)
|
||||||
|
if r2.status_code != 200:
|
||||||
|
fail("POST /exercise (update) fehlgeschlagen", r2)
|
||||||
|
id2 = r2.json().get("id")
|
||||||
|
if id2 != id1:
|
||||||
|
fail("Idempotenz verletzt: id hat sich geändert")
|
||||||
|
|
||||||
|
# by-external-id
|
||||||
|
r3 = requests.get(f"{BASE}/exercise/by-external-id", params={"external_id": ext})
|
||||||
|
if r3.status_code != 200:
|
||||||
|
fail("GET /exercise/by-external-id fehlgeschlagen", r3)
|
||||||
|
id3 = r3.json().get("id")
|
||||||
|
if id3 != id1:
|
||||||
|
fail("Lookup by external_id liefert andere id")
|
||||||
|
|
||||||
|
ok("Exercise Upsert & Idempotenz ok")
|
||||||
|
return ext, id1
|
||||||
|
|
||||||
|
|
||||||
|
def test_exercise_search_filter(ext_id: str):
|
||||||
|
req = {
|
||||||
|
"discipline": "Karate",
|
||||||
|
"equipment_all": ["Bälle"],
|
||||||
|
"capability_names": ["Reaktionsfähigkeit"],
|
||||||
|
"capability_ge_level": 2,
|
||||||
|
"limit": 10,
|
||||||
|
}
|
||||||
|
r = requests.post(f"{BASE}/exercise/search", json=req)
|
||||||
|
if r.status_code != 200:
|
||||||
|
fail("POST /exercise/search (Filter) fehlgeschlagen", r)
|
||||||
|
js = r.json(); hits = js.get("hits", [])
|
||||||
|
if not isinstance(hits, list) or not hits:
|
||||||
|
fail("/exercise/search liefert keine Treffer (Filter)", r)
|
||||||
|
if not any(h.get("payload", {}).get("external_id") == ext_id for h in hits):
|
||||||
|
ok("Exercise Search (Filter) ok – Testpunkt nicht zwingend in Top-N")
|
||||||
|
else:
|
||||||
|
ok("Exercise Search (Filter) ok – Testpunkt gefunden")
|
||||||
|
|
||||||
|
|
||||||
|
def test_exercise_search_vector():
|
||||||
|
req = {
|
||||||
|
"query": "Aufwärmen 10min, Reaktionsfähigkeit, Teenager, Bälle",
|
||||||
|
"discipline": "Karate",
|
||||||
|
"limit": 5
|
||||||
|
}
|
||||||
|
r = requests.post(f"{BASE}/exercise/search", json=req)
|
||||||
|
if r.status_code != 200:
|
||||||
|
fail("POST /exercise/search (Vector) fehlgeschlagen", r)
|
||||||
|
js = r.json(); hits = js.get("hits", [])
|
||||||
|
if not isinstance(hits, list):
|
||||||
|
fail("/exercise/search (Vector) liefert ungültige Struktur", r)
|
||||||
|
ok(f"Exercise Search (Vector) ok – {len(hits)} Treffer")
|
||||||
|
|
||||||
|
|
||||||
|
def test_exercise_delete(ext_id: str):
|
||||||
|
r = requests.delete(f"{BASE}/exercise/delete-by-external-id", params={"external_id": ext_id})
|
||||||
|
if r.status_code != 200:
|
||||||
|
fail("DELETE /exercise/delete-by-external-id fehlgeschlagen", r)
|
||||||
|
ok("Exercise Delete-by-external-id ok")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
print("\nStarte API-Tests...\n")
|
||||||
|
test_openapi()
|
||||||
|
test_wiki_health()
|
||||||
|
|
||||||
|
# Login nun VOR Semantic Query
|
||||||
|
did_login = test_wiki_login_if_env()
|
||||||
|
|
||||||
|
# Titel ermitteln (semantic/pages oder Fallback)
|
||||||
|
title, pageid = get_wiki_title_and_pageid()
|
||||||
|
test_wiki_info(title)
|
||||||
|
test_wiki_parse(pageid, title)
|
||||||
|
|
||||||
|
# Exercise
|
||||||
|
ext, _id = test_exercise_upsert_and_idempotence()
|
||||||
|
test_exercise_search_filter(ext)
|
||||||
|
test_exercise_search_vector()
|
||||||
|
test_exercise_delete(ext)
|
||||||
|
|
||||||
|
print("\n🎉 Alle Tests erfolgreich durchlaufen!\n")
|
||||||
+356
-124
@@ -1,183 +1,436 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Module: wiki_importer.py
|
wiki_importer.py – v2.3.8
|
||||||
Beschreibung:
|
|
||||||
- Importiert Übungen aus dem MediaWiki via FastAPI wiki_router
|
|
||||||
- Führt vor dem Import einen Login gegen /import/wiki/login durch (falls nicht via --skip-login deaktiviert)
|
|
||||||
- Holt Liste aller Übungs-Titel (SMW-Ask) via `/semantic/pages`
|
|
||||||
- Für jede Übung:
|
|
||||||
* Fetch pageinfo (pageid, fullurl) via `/info`
|
|
||||||
* Parse Wikitext (Templates: ÜbungInfoBox, Übungsbeschreibung, SkillDevelopment) via `/parsepage`
|
|
||||||
* Baut Payload entsprechend Exercise-Datenmodell
|
|
||||||
* POST an `/exercise` Endpoint (exercise_router)
|
|
||||||
- Unterstützt Single-Import via `--title` (oder ENV `WIKI_EXERCISE_TITLE`) und Full-Import via `--all`
|
|
||||||
- Optional: Credentials via CLI (--username/--password) oder `.env` (WIKI_BOT_USER / WIKI_BOT_PASSWORD)
|
|
||||||
|
|
||||||
Version: 2.1.0
|
Ziel dieses Patches: Die Felder `discipline`, `execution`, `keywords`, `equipment`, `duration_minutes` usw.
|
||||||
|
kommen bei dir teilweise leer an. Ursache sind zu aggressive Normalisierungen/Matcher.
|
||||||
|
|
||||||
|
Fix (konservativ & robust):
|
||||||
|
- Parser liest jetzt **gezielt** die bekannten Templates **ohne** Over-Normalisierung:
|
||||||
|
• `{{ÜbungInfoBox}}` / `{{UebungInfoBox}}`
|
||||||
|
• `{{Übungsbeschreibung}}` / `{{Uebungsbeschreibung}}`
|
||||||
|
• `{{Hilfsmittel}}`
|
||||||
|
• `{{SkillDevelopment}}`
|
||||||
|
- Feld-Extraktion nutzt **zuerst die exakten Wiki-Parameternamen** (deutsch/mit Umlauten),
|
||||||
|
erst danach schmale Synonym-Fallbacks. Das stellt sicher, dass z. B. `Schlüsselworte=`
|
||||||
|
wirklich in `keywords` landet.
|
||||||
|
- `imported_at` wird bei **Create und Update** gesetzt.
|
||||||
|
- Optionales Debugging: `--debug-raw` druckt die gefundenen Raw-Keys (einfach, nachvollziehbar).
|
||||||
|
|
||||||
|
Bestehende API-Endpunkte bleiben unverändert.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import argparse
|
import argparse
|
||||||
from typing import Dict, Any
|
from typing import Dict, Any, Tuple, Optional, List
|
||||||
import requests
|
import requests
|
||||||
import mwparserfromhell
|
import mwparserfromhell
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
|
||||||
# ----- Konfiguration / Defaults -----
|
# ----- Konfiguration / Defaults -----
|
||||||
load_dotenv() # .env laden, falls vorhanden
|
load_dotenv()
|
||||||
|
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki")
|
||||||
|
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise")
|
||||||
|
DEFAULT_CAT = os.getenv("WIKI_CATEGORY", "Übungen")
|
||||||
|
DEFAULT_TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
|
||||||
|
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "60"))
|
||||||
|
|
||||||
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki") # FastAPI-Wiki-Proxy
|
# ----- Helpers für Wiki-Router -----
|
||||||
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise") # Exercise-Endpoint
|
|
||||||
DEFAULT_CAT = os.getenv("WIKI_CATEGORY", "Übungen")
|
|
||||||
DEFAULT_TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
|
|
||||||
|
|
||||||
# ---- Hilfsfunktionen für Wiki-Router ----
|
|
||||||
def wiki_health() -> None:
|
def wiki_health() -> None:
|
||||||
r = requests.get(f"{API_BASE_URL}/health", timeout=15)
|
r = requests.get(f"{API_BASE_URL}/health", timeout=15)
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
print("[Sanity] Wiki health OK")
|
print("[Sanity] Wiki health OK")
|
||||||
|
|
||||||
|
|
||||||
def wiki_login(username: str, password: str) -> None:
|
def wiki_login(username: str, password: str) -> None:
|
||||||
"""
|
|
||||||
Führt einen Login gegen den wiki_router durch.
|
|
||||||
Erwartet: {"status":"success"} bei Erfolg.
|
|
||||||
"""
|
|
||||||
payload = {"username": username, "password": password}
|
payload = {"username": username, "password": password}
|
||||||
r = requests.post(f"{API_BASE_URL}/login", json=payload, timeout=30)
|
r = requests.post(f"{API_BASE_URL}/login", json=payload, timeout=30)
|
||||||
# kein raise_for_status(), wir wollen die JSON-Fehler sauber ausgeben
|
|
||||||
try:
|
try:
|
||||||
data = r.json()
|
data = r.json()
|
||||||
except Exception:
|
except Exception:
|
||||||
print(f"[Login] HTTP {r.status_code}: {r.text}")
|
print(f"[Login] HTTP {r.status_code}: {r.text}")
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
|
|
||||||
status = (data or {}).get("status")
|
status = (data or {}).get("status")
|
||||||
if status != "success":
|
if status != "success":
|
||||||
msg = (data or {}).get("message", "Login fehlgeschlagen")
|
msg = (data or {}).get("message", "Login fehlgeschlagen")
|
||||||
raise RuntimeError(f"[Login] {msg}")
|
raise RuntimeError(f"[Login] {msg}")
|
||||||
print("[Login] success")
|
print("[Login] success")
|
||||||
|
|
||||||
|
|
||||||
def fetch_all_pages(category: str) -> Dict[str, Any]:
|
def fetch_all_pages(category: str) -> Dict[str, Any]:
|
||||||
resp = requests.get(f"{API_BASE_URL}/semantic/pages", params={"category": category}, timeout=60)
|
resp = requests.get(f"{API_BASE_URL}/semantic/pages", params={"category": category}, timeout=REQUEST_TIMEOUT)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
return resp.json()
|
return resp.json()
|
||||||
|
|
||||||
|
|
||||||
def fetch_page_info(title: str) -> Dict[str, Any]:
|
def fetch_page_info(title: str) -> Dict[str, Any]:
|
||||||
r = requests.get(f"{API_BASE_URL}/info", params={"title": title}, timeout=30)
|
r = requests.get(f"{API_BASE_URL}/info", params={"title": title}, timeout=30)
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
info = r.json()
|
info = r.json()
|
||||||
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
|
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
|
||||||
|
|
||||||
|
# ----- Parser (konservativ) -----
|
||||||
|
|
||||||
|
T_INFOS = {"ÜbungInfoBox", "UebungInfoBox"}
|
||||||
|
T_BESCHR = {"Übungsbeschreibung", "Uebungsbeschreibung"}
|
||||||
|
T_HILFS = {"Hilfsmittel"}
|
||||||
|
T_SKILL = {"SkillDevelopment"}
|
||||||
|
|
||||||
|
|
||||||
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
|
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
|
||||||
print(f"[Parse] Lade '{title}' (ID={pageid})")
|
print(f"[Parse] Lade '{title}' (ID={pageid})")
|
||||||
resp = requests.get(
|
resp = requests.get(
|
||||||
f"{API_BASE_URL}/parsepage",
|
f"{API_BASE_URL}/parsepage",
|
||||||
params={"pageid": pageid, "title": title},
|
params={"pageid": pageid, "title": title},
|
||||||
timeout=60
|
timeout=REQUEST_TIMEOUT,
|
||||||
)
|
)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
wikitext = resp.json().get("wikitext", "")
|
wikitext = resp.json().get("wikitext", "")
|
||||||
wikicode = mwparserfromhell.parse(wikitext)
|
wikicode = mwparserfromhell.parse(wikitext)
|
||||||
|
|
||||||
raw: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid}
|
raw: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid, "wikitext": wikitext}
|
||||||
|
|
||||||
for tpl in wikicode.filter_templates():
|
for tpl in wikicode.filter_templates():
|
||||||
name = str(tpl.name).strip()
|
name = str(tpl.name).strip()
|
||||||
if name == "ÜbungInfoBox":
|
if name in T_INFOS or name in T_BESCHR or name in T_HILFS:
|
||||||
for p in tpl.params:
|
for p in tpl.params:
|
||||||
raw[str(p.name).strip()] = str(p.value).strip()
|
key = str(p.name).strip()
|
||||||
elif name == "Übungsbeschreibung":
|
val = str(p.value).strip()
|
||||||
for p in tpl.params:
|
raw[key] = val
|
||||||
raw[str(p.name).strip()] = str(p.value).strip()
|
elif name in T_SKILL:
|
||||||
elif name == "SkillDevelopment":
|
|
||||||
raw.setdefault("capabilities", [])
|
raw.setdefault("capabilities", [])
|
||||||
|
def _getp(t, k):
|
||||||
|
try:
|
||||||
|
return str(t.get(k).value).strip()
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
cap = _getp(tpl, "PrimaryCapability")
|
||||||
|
lvl = _getp(tpl, "CapabilityLevel")
|
||||||
try:
|
try:
|
||||||
cap = str(tpl.get("PrimaryCapability").value).strip()
|
lvl_i = int(lvl)
|
||||||
except Exception:
|
except Exception:
|
||||||
cap = ""
|
lvl_i = 0
|
||||||
try:
|
|
||||||
lvl = int(str(tpl.get("CapabilityLevel").value).strip())
|
|
||||||
except Exception:
|
|
||||||
lvl = 0
|
|
||||||
if cap:
|
if cap:
|
||||||
raw["capabilities"].append({"capability": cap, "level": lvl})
|
raw["capabilities"].append({"capability": cap, "level": lvl_i})
|
||||||
raw["wikitext"] = wikitext
|
|
||||||
return raw
|
return raw
|
||||||
|
|
||||||
def build_payload(raw: Dict[str, Any], fullurl: str, category: str) -> Dict[str, Any]:
|
# ----- Fingerprint (stabil, wie zuvor) -----
|
||||||
# Exercise.capabilities erwartet Dict[str,int]
|
|
||||||
caps_list = raw.get("capabilities", [])
|
def _normalize(v: Any) -> str:
|
||||||
capabilities = {}
|
if v is None:
|
||||||
for c in caps_list:
|
return ""
|
||||||
cap = c.get("capability")
|
if isinstance(v, (list, tuple)):
|
||||||
lvl = c.get("level")
|
return ",".join(_normalize(x) for x in v)
|
||||||
|
if isinstance(v, dict):
|
||||||
|
return json.dumps(v, sort_keys=True, ensure_ascii=False)
|
||||||
|
return str(v).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _norm_text(s: str) -> str:
|
||||||
|
if s is None:
|
||||||
|
return ""
|
||||||
|
s = str(s).replace("\u00a0", " ")
|
||||||
|
s = " ".join(s.split())
|
||||||
|
return s.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _canon_title(t: str) -> str:
|
||||||
|
t = (t or "").strip().replace("_", " ")
|
||||||
|
return t.replace("–", "-").replace("—", "-")
|
||||||
|
|
||||||
|
|
||||||
|
def compute_fingerprint(payload: Dict[str, Any]) -> str:
|
||||||
|
kws = payload.get("keywords") or []
|
||||||
|
kws = [k.replace("\u2013", "-").replace("\u2014", "-") for k in kws]
|
||||||
|
kws = sorted({(k or "").strip() for k in kws if (k or "").strip()}, key=str.casefold)
|
||||||
|
dur = payload.get("duration_minutes") or 0
|
||||||
|
try:
|
||||||
|
dur = int(round(float(dur)))
|
||||||
|
except Exception:
|
||||||
|
dur = 0
|
||||||
|
fields = [
|
||||||
|
_canon_title(payload.get("title", "")),
|
||||||
|
_norm_text(payload.get("summary", "")),
|
||||||
|
_norm_text(payload.get("execution", "")),
|
||||||
|
_norm_text(payload.get("notes", "")),
|
||||||
|
dur,
|
||||||
|
payload.get("capabilities", {}),
|
||||||
|
kws,
|
||||||
|
]
|
||||||
|
base = "|".join(_normalize(f) for f in fields)
|
||||||
|
return hashlib.sha256(base.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
# ----- Payload (exakte DE-Keys zuerst, dann schmale Fallbacks) -----
|
||||||
|
|
||||||
|
EXACT_KEYS = {
|
||||||
|
"summary": ["Summary", "Kurzbeschreibung"],
|
||||||
|
"execution": ["Durchführung", "Durchfuehrung", "Ablauf"],
|
||||||
|
"duration": ["Dauer", "Zeit"],
|
||||||
|
"keywords": ["Schlüsselworte", "Schlüsselwörter", "Schluesselworte", "Schluesselwoerter", "Keywords", "Tags"],
|
||||||
|
"equipment_prim": ["Hilfsmittel"],
|
||||||
|
"equipment_alt": ["Geräte", "Geraete", "Gerät", "Geraet", "Material"],
|
||||||
|
"discipline": ["Übungstyp", "Uebungstyp", "Disziplin"],
|
||||||
|
"group": ["Gruppengröße", "Gruppengroesse", "Group"],
|
||||||
|
"age_group": ["Altersgruppe"],
|
||||||
|
"target_group": ["Zielgruppe"],
|
||||||
|
"purpose": ["Ziel", "Zweck"],
|
||||||
|
"notes": ["Hinweise", "Notes"],
|
||||||
|
"preparation": ["Vorbereitung", "RefMethode"],
|
||||||
|
"method": ["Methode", "Method"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _first_any(raw: Dict[str, Any], keys: List[str]) -> Optional[str]:
|
||||||
|
for k in keys:
|
||||||
|
v = raw.get(k)
|
||||||
|
if isinstance(v, str) and v.strip():
|
||||||
|
return v.strip()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def build_payload(raw: Dict[str, Any], fullurl: str, category: str, *, mutate: bool = False) -> Dict[str, Any]:
|
||||||
|
# Capabilities -> Dict[str,int]
|
||||||
|
capabilities: Dict[str, int] = {}
|
||||||
|
for c in raw.get("capabilities", []) or []:
|
||||||
|
cap = c.get("capability"); lvl = c.get("level")
|
||||||
if isinstance(cap, str) and cap:
|
if isinstance(cap, str) and cap:
|
||||||
try:
|
try:
|
||||||
capabilities[cap] = int(lvl)
|
capabilities[cap] = int(lvl)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Defaults/Fallbacks
|
# Exakte Schlüssel zuerst
|
||||||
duration = 0.0
|
summary = _first_any(raw, EXACT_KEYS["summary"]) or ""
|
||||||
|
execution = _first_any(raw, EXACT_KEYS["execution"]) or ""
|
||||||
|
duration = _first_any(raw, EXACT_KEYS["duration"]) or "0"
|
||||||
|
|
||||||
|
kw_raw = _first_any(raw, EXACT_KEYS["keywords"]) or ""
|
||||||
|
if kw_raw:
|
||||||
|
parts = [p.strip() for p in kw_raw.replace("\n", ",").split(",")]
|
||||||
|
keywords = [p for p in parts if p]
|
||||||
|
else:
|
||||||
|
keywords = []
|
||||||
|
|
||||||
|
eq_raw = _first_any(raw, EXACT_KEYS["equipment_prim"]) or _first_any(raw, EXACT_KEYS["equipment_alt"]) or ""
|
||||||
|
if eq_raw:
|
||||||
|
equipment = [e.strip() for e in eq_raw.replace("\n", ",").split(",") if e.strip()]
|
||||||
|
else:
|
||||||
|
equipment = []
|
||||||
|
|
||||||
|
notes = _first_any(raw, EXACT_KEYS["notes"]) or ""
|
||||||
|
discipline = _first_any(raw, EXACT_KEYS["discipline"]) or ""
|
||||||
|
group = _first_any(raw, EXACT_KEYS["group"]) or None
|
||||||
|
age_group = _first_any(raw, EXACT_KEYS["age_group"]) or ""
|
||||||
|
target_group = _first_any(raw, EXACT_KEYS["target_group"]) or ""
|
||||||
|
purpose = _first_any(raw, EXACT_KEYS["purpose"]) or ""
|
||||||
|
preparation = _first_any(raw, EXACT_KEYS["preparation"]) or ""
|
||||||
|
method = _first_any(raw, EXACT_KEYS["method"]) or ""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
duration = float(raw.get("Dauer", 0) or 0)
|
duration_f = float(duration or 0)
|
||||||
except Exception:
|
except Exception:
|
||||||
duration = 0.0
|
duration_f = 0.0
|
||||||
|
|
||||||
keywords = []
|
|
||||||
kw_raw = raw.get("Schlüsselworte", "")
|
|
||||||
if isinstance(kw_raw, str):
|
|
||||||
keywords = [k.strip() for k in kw_raw.split(",") if k.strip()]
|
|
||||||
|
|
||||||
equipment = []
|
|
||||||
eq_raw = raw.get("equipment", [])
|
|
||||||
if isinstance(eq_raw, str):
|
|
||||||
equipment = [e.strip() for e in eq_raw.split(",") if e.strip()]
|
|
||||||
elif isinstance(eq_raw, list):
|
|
||||||
equipment = [str(e).strip() for e in eq_raw if str(e).strip()]
|
|
||||||
|
|
||||||
payload: Dict[str, Any] = {
|
payload: Dict[str, Any] = {
|
||||||
"title": raw.get("title") or "",
|
"title": raw.get("title") or "",
|
||||||
"summary": raw.get("Summary", "") or "",
|
"summary": summary,
|
||||||
"short_description": raw.get("Summary", "") or "",
|
"short_description": summary,
|
||||||
"keywords": keywords,
|
"keywords": keywords,
|
||||||
"link": fullurl or "",
|
"link": fullurl or "",
|
||||||
"discipline": raw.get("Übungstyp", "") or "",
|
"discipline": discipline,
|
||||||
"group": raw.get("Gruppengröße", "") or None,
|
"group": group,
|
||||||
"age_group": raw.get("Altersgruppe", "") or "",
|
"age_group": age_group,
|
||||||
"target_group": raw.get("Zielgruppe", "") or "",
|
"target_group": target_group,
|
||||||
"min_participants": 1,
|
"min_participants": 1,
|
||||||
"duration_minutes": int(round(duration)), # Exercise erwartet int
|
"duration_minutes": int(round(duration_f)),
|
||||||
"capabilities": capabilities,
|
"capabilities": capabilities,
|
||||||
"category": category or "",
|
"category": category or "",
|
||||||
"purpose": raw.get("Ziel", "") or "",
|
"purpose": purpose,
|
||||||
"execution": raw.get("Durchführung", "") or "",
|
"execution": execution,
|
||||||
"notes": raw.get("Hinweise", "") or "",
|
"notes": (notes + (" [auto-update]" if mutate else "")).strip(),
|
||||||
"preparation": raw.get("RefMethode", "") or "",
|
"preparation": preparation,
|
||||||
"method": raw.get("method", "") or "", # falls im Wikitext vorhanden
|
"method": method,
|
||||||
"equipment": equipment,
|
"equipment": equipment,
|
||||||
"fullurl": fullurl or "", # optionales Feld
|
"fullurl": fullurl or "",
|
||||||
# Idempotenz (optional nutzbar in exercise_router):
|
"external_id": f"mw:{raw.get('pageid')}",
|
||||||
"external_id": f"wiki:{raw.get('pageid')}",
|
"source": "MediaWiki",
|
||||||
"source": "MediaWiki"
|
|
||||||
}
|
}
|
||||||
|
payload["fingerprint"] = compute_fingerprint(payload)
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
def ingest_exercise(payload: Dict[str, Any]) -> None:
|
# ----- Lookup/Upsert -----
|
||||||
|
|
||||||
|
def lookup_by_external_id(external_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[int]]:
|
||||||
|
url = f"{EXERCISE_API}/by-external-id"
|
||||||
|
try:
|
||||||
|
r = requests.get(url, params={"external_id": external_id}, timeout=REQUEST_TIMEOUT)
|
||||||
|
if r.status_code == 404:
|
||||||
|
return None, 404
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json(), r.status_code
|
||||||
|
except requests.HTTPError as e:
|
||||||
|
return {"error": str(e), "status_code": getattr(e.response, "status_code", None)}, getattr(e.response, "status_code", None)
|
||||||
|
except Exception as e:
|
||||||
|
return {"error": str(e)}, None
|
||||||
|
|
||||||
|
|
||||||
|
def _payload_subset_for_fp(p: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"title": p.get("title"),
|
||||||
|
"summary": p.get("summary"),
|
||||||
|
"execution": p.get("execution"),
|
||||||
|
"notes": p.get("notes"),
|
||||||
|
"duration_minutes": p.get("duration_minutes"),
|
||||||
|
"capabilities": p.get("capabilities") or {},
|
||||||
|
"keywords": p.get("keywords") or [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _print_diff(before: Dict[str, Any], after: Dict[str, Any]) -> None:
|
||||||
|
keys = ["title","summary","execution","notes","duration_minutes","capabilities","keywords"]
|
||||||
|
b = {k: before.get(k) for k in keys}
|
||||||
|
a = {k: after.get(k) for k in keys}
|
||||||
|
def _kws(x):
|
||||||
|
return sorted({(k or "").strip() for k in (x or [])}, key=str.casefold)
|
||||||
|
b_norm = {
|
||||||
|
"title": _canon_title(b.get("title")),
|
||||||
|
"summary": _norm_text(b.get("summary")),
|
||||||
|
"execution": _norm_text(b.get("execution")),
|
||||||
|
"notes": _norm_text(b.get("notes")),
|
||||||
|
"duration_minutes": b.get("duration_minutes"),
|
||||||
|
"capabilities": b.get("capabilities"),
|
||||||
|
"keywords": _kws(b.get("keywords")),
|
||||||
|
}
|
||||||
|
a_norm = {
|
||||||
|
"title": _canon_title(a.get("title")),
|
||||||
|
"summary": _norm_text(a.get("summary")),
|
||||||
|
"execution": _norm_text(a.get("execution")),
|
||||||
|
"notes": _norm_text(a.get("notes")),
|
||||||
|
"duration_minutes": a.get("duration_minutes"),
|
||||||
|
"capabilities": a.get("capabilities"),
|
||||||
|
"keywords": _kws(a.get("keywords")),
|
||||||
|
}
|
||||||
|
diff = {k: (b_norm[k], a_norm[k]) for k in keys if b_norm.get(k) != a_norm.get(k)}
|
||||||
|
if diff:
|
||||||
|
print("[Diff] changes:", json.dumps(diff, ensure_ascii=False))
|
||||||
|
else:
|
||||||
|
print("[Diff] (none in hash fields)")
|
||||||
|
|
||||||
|
|
||||||
|
def _now_iso() -> str:
|
||||||
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_exercise(payload: Dict[str, Any], *, dry_run: bool = False) -> str:
|
||||||
title = payload.get("title", "<ohne Titel>")
|
title = payload.get("title", "<ohne Titel>")
|
||||||
resp = requests.post(EXERCISE_API, json=payload, timeout=60)
|
ext_id = payload.get("external_id")
|
||||||
if resp.status_code == 422:
|
fp_new = payload.get("fingerprint")
|
||||||
print(f"[Ingest] '{title}' -> FAILED 422:\n{resp.text}")
|
|
||||||
|
found, status = lookup_by_external_id(ext_id)
|
||||||
|
|
||||||
|
action = "create"; reason = "not found (lookup 404)"; found_payload = {}
|
||||||
|
|
||||||
|
if not (status == 404 or found is None):
|
||||||
|
if isinstance(found, dict):
|
||||||
|
found_payload = found.get("payload", found)
|
||||||
|
fp_old_stored = found.get("fingerprint") or found_payload.get("fingerprint")
|
||||||
|
fp_old_recalc = compute_fingerprint(_payload_subset_for_fp(found_payload))
|
||||||
|
if fp_new == fp_old_stored or fp_new == fp_old_recalc:
|
||||||
|
action, reason = "skip", "fingerprint unchanged"
|
||||||
|
else:
|
||||||
|
action, reason = "update", "fingerprint changed"
|
||||||
|
else:
|
||||||
|
action, reason = "create", "unexpected lookup type"
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
print(f"[DryRun] {action.upper():6} '{title}' ({ext_id}) – {reason}")
|
||||||
|
if action == "update":
|
||||||
|
_print_diff(found_payload, payload)
|
||||||
|
return action
|
||||||
|
|
||||||
|
if action == "create":
|
||||||
|
body = dict(payload); body["imported_at"] = _now_iso()
|
||||||
|
resp = requests.post(EXERCISE_API, json=body, timeout=REQUEST_TIMEOUT)
|
||||||
|
if resp.status_code == 422:
|
||||||
|
print(f"[Create] '{title}' -> FAILED 422:\n{resp.text}")
|
||||||
|
try: resp.raise_for_status()
|
||||||
|
except Exception: pass
|
||||||
|
else:
|
||||||
|
resp.raise_for_status(); print(f"[Create] '{title}' – {reason} -> OK")
|
||||||
|
elif action == "update":
|
||||||
|
body = dict(payload); body["imported_at"] = _now_iso()
|
||||||
|
resp = requests.post(EXERCISE_API, json=body, timeout=REQUEST_TIMEOUT)
|
||||||
|
if resp.status_code == 422:
|
||||||
|
print(f"[Update] '{title}' -> FAILED 422:\n{resp.text}")
|
||||||
|
try: resp.raise_for_status()
|
||||||
|
except Exception: pass
|
||||||
|
else:
|
||||||
|
resp.raise_for_status(); print(f"[Update] '{title}' – {reason} -> OK"); _print_diff(found_payload, payload)
|
||||||
|
else:
|
||||||
|
print(f"[Skip] '{title}' – {reason}")
|
||||||
|
return action
|
||||||
|
|
||||||
|
# ----- Orchestrierung -----
|
||||||
|
|
||||||
|
def process_one(title: str, category: str, *, mutate: bool = False, dry_run: bool = False, debug_raw: bool = False) -> str:
|
||||||
|
info = fetch_page_info(title)
|
||||||
|
pid = info.get("pageid"); fullurl = info.get("fullurl") or ""
|
||||||
|
if not pid:
|
||||||
|
print(f"[Error] pageid für '{title}' nicht gefunden.", file=sys.stderr); return "failed"
|
||||||
|
raw = parse_exercise(title, pid)
|
||||||
|
if debug_raw:
|
||||||
|
print("[Debug] Raw-Keys:", sorted([k for k in raw.keys() if k not in {"wikitext"}]))
|
||||||
|
payload = build_payload(raw, fullurl, category, mutate=mutate)
|
||||||
|
return upsert_exercise(payload, dry_run=dry_run)
|
||||||
|
|
||||||
|
|
||||||
|
def process_all(category: str, *, dry_run: bool = False, debug_raw: bool = False) -> Dict[str, int]:
|
||||||
|
stats = {"created": 0, "updated": 0, "skipped": 0, "failed": 0}
|
||||||
|
print(f"[Main] Lade Liste der Übungen aus Kategorie '{category}'…")
|
||||||
|
pages = fetch_all_pages(category)
|
||||||
|
print(f"[Main] {len(pages)} Seiten gefunden.")
|
||||||
|
|
||||||
|
for idx, (title, entry) in enumerate(pages.items(), 1):
|
||||||
try:
|
try:
|
||||||
resp.raise_for_status()
|
getter = getattr(entry, "get", None)
|
||||||
except Exception:
|
pid = getter("pageid") if callable(getter) else None
|
||||||
pass
|
fullurl = getter("fullurl") if callable(getter) else None
|
||||||
return
|
if not pid:
|
||||||
resp.raise_for_status()
|
info = fetch_page_info(title); pid = info.get("pageid"); fullurl = fullurl or info.get("fullurl")
|
||||||
print(f"[Ingest] '{title}' -> OK")
|
if not pid:
|
||||||
|
print(f"[Skip] '{title}' hat keine pageid"); stats["failed"] += 1; continue
|
||||||
|
raw = parse_exercise(title, pid)
|
||||||
|
if debug_raw and idx <= 5:
|
||||||
|
print(f"[Debug] #{idx} '{title}' Raw-Keys:", sorted([k for k in raw.keys() if k not in {"wikitext"}]))
|
||||||
|
payload = build_payload(raw, fullurl or "", category)
|
||||||
|
act = upsert_exercise(payload, dry_run=dry_run)
|
||||||
|
stats["created" if act=="create" else "updated" if act=="update" else "skipped"] += 1
|
||||||
|
except requests.HTTPError as e:
|
||||||
|
code = getattr(e, "response", None).status_code if getattr(e, "response", None) else None
|
||||||
|
if code == 404:
|
||||||
|
print(f"[Skip] '{title}': page not found (404)"); stats["failed"] += 1
|
||||||
|
else:
|
||||||
|
print(f"[Error] '{title}': {e}"); stats["failed"] += 1
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[Error] '{title}': {e}"); stats["failed"] += 1
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def run_smoke_test(title: str, category: str, *, debug_raw: bool = False) -> None:
|
||||||
|
print("\n[SmokeTest] Lauf 1/3: CREATE (Erstimport)"); act1 = process_one(title, category, mutate=False, debug_raw=debug_raw); print("[SmokeTest] Aktion:", act1)
|
||||||
|
print("\n[SmokeTest] Lauf 2/3: SKIP (Wiederholung, unverändert)"); act2 = process_one(title, category, mutate=False, debug_raw=debug_raw); print("[SmokeTest] Aktion:", act2)
|
||||||
|
print("\n[SmokeTest] Lauf 3/3: UPDATE (simulierte Wiki-Änderung an 'notes')"); act3 = process_one(title, category, mutate=True, debug_raw=debug_raw); print("[SmokeTest] Aktion:", act3)
|
||||||
|
print("\n[SmokeTest] Zusammenfassung:"); print(json.dumps({"run1": act1, "run2": act2, "run3": act3}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
# ----- Main -----
|
# ----- Main -----
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
parser = argparse.ArgumentParser(description="Import exercises from Wiki to Qdrant (via FastAPI wiki_router)")
|
parser = argparse.ArgumentParser(description="Import exercises from Wiki to Qdrant (via FastAPI wiki_router)")
|
||||||
parser.add_argument("--all", action="store_true", help="Alle Übungen importieren (SMW-Ask)")
|
parser.add_argument("--all", action="store_true", help="Alle Übungen importieren (SMW-Ask)")
|
||||||
@@ -186,13 +439,13 @@ def main() -> None:
|
|||||||
parser.add_argument("--username", type=str, default=os.getenv("WIKI_BOT_USER"), help="Wiki-Login Benutzer (überschreibt .env)")
|
parser.add_argument("--username", type=str, default=os.getenv("WIKI_BOT_USER"), help="Wiki-Login Benutzer (überschreibt .env)")
|
||||||
parser.add_argument("--password", type=str, default=os.getenv("WIKI_BOT_PASSWORD"), help="Wiki-Login Passwort (überschreibt .env)")
|
parser.add_argument("--password", type=str, default=os.getenv("WIKI_BOT_PASSWORD"), help="Wiki-Login Passwort (überschreibt .env)")
|
||||||
parser.add_argument("--skip-login", action="store_true", help="Login-Schritt überspringen (falls Session schon aktiv)")
|
parser.add_argument("--skip-login", action="store_true", help="Login-Schritt überspringen (falls Session schon aktiv)")
|
||||||
|
parser.add_argument("--dry-run", action="store_true", help="Kein Schreiben; nur Entscheidungen (create/update/skip) + Gründe loggen")
|
||||||
|
parser.add_argument("--smoke-test", action="store_true", help="3 Durchläufe (create→skip→update) für --title")
|
||||||
|
parser.add_argument("--debug-raw", action="store_true", help="Zeigt die aus dem Wiki gelesenen Roh-Keys je Seite")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
# Sanity
|
|
||||||
wiki_health()
|
wiki_health()
|
||||||
|
|
||||||
# Login (sofern nicht explizit übersprungen)
|
|
||||||
if not args.skip_login:
|
if not args.skip_login:
|
||||||
if not args.username or not args.password:
|
if not args.username or not args.password:
|
||||||
print("[Login] Fehler: fehlende Credentials. Setze .env (WIKI_BOT_USER/WIKI_BOT_PASSWORD) oder CLI --username/--password.", file=sys.stderr)
|
print("[Login] Fehler: fehlende Credentials. Setze .env (WIKI_BOT_USER/WIKI_BOT_PASSWORD) oder CLI --username/--password.", file=sys.stderr)
|
||||||
@@ -200,39 +453,18 @@ def main() -> None:
|
|||||||
try:
|
try:
|
||||||
wiki_login(args.username, args.password)
|
wiki_login(args.username, args.password)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(str(e), file=sys.stderr)
|
print(str(e), file=sys.stderr); sys.exit(1)
|
||||||
sys.exit(1)
|
|
||||||
|
if args.smoke_test:
|
||||||
|
run_smoke_test(args.title, args.category, debug_raw=args.debug_raw); return
|
||||||
|
|
||||||
# Einzel- oder Vollimport
|
|
||||||
if args.all:
|
if args.all:
|
||||||
print(f"[Main] Lade Liste der Übungen aus Kategorie '{args.category}'…")
|
stats = process_all(args.category, dry_run=args.dry_run, debug_raw=args.debug_raw)
|
||||||
pages = fetch_all_pages(args.category)
|
print("\n[Stats] created={created} updated={updated} skipped={skipped} failed={failed}".format(**stats))
|
||||||
print(f"[Main] {len(pages)} Seiten gefunden.")
|
|
||||||
for title, entry in pages.items():
|
|
||||||
pid = entry.get("pageid")
|
|
||||||
fullurl = entry.get("fullurl")
|
|
||||||
if not pid:
|
|
||||||
# Core-Info nachschlagen
|
|
||||||
info = fetch_page_info(title)
|
|
||||||
pid = info.get("pageid")
|
|
||||||
fullurl = fullurl or info.get("fullurl")
|
|
||||||
if not pid:
|
|
||||||
print(f"[Skip] '{title}' hat keine pageid")
|
|
||||||
continue
|
|
||||||
raw = parse_exercise(title, pid)
|
|
||||||
payload = build_payload(raw, fullurl or "", args.category)
|
|
||||||
ingest_exercise(payload)
|
|
||||||
else:
|
else:
|
||||||
print(f"[Main] Import single exercise: {args.title}")
|
print(f"[Main] Import single exercise: {args.title}")
|
||||||
info = fetch_page_info(args.title)
|
result = process_one(args.title, args.category, mutate=False, dry_run=args.dry_run, debug_raw=args.debug_raw)
|
||||||
pid = info.get("pageid")
|
print(f"[Result] {result}")
|
||||||
fullurl = info.get("fullurl") or ""
|
|
||||||
if not pid:
|
|
||||||
print(f"[Error] pageid für '{args.title}' nicht gefunden.", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
raw = parse_exercise(args.title, pid)
|
|
||||||
payload = build_payload(raw, fullurl, args.category)
|
|
||||||
ingest_exercise(payload)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
Reference in new issue
Block a user