Compare commits
41
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7b383f0778 | ||
|
|
34320b46d9 | ||
|
|
e12fd8f96a | ||
|
|
cf085f8ef0 | ||
|
|
6bab3cdf04 | ||
|
|
2567d8c786 | ||
|
|
d8d12e0b6b | ||
|
|
bba860d839 | ||
|
|
a0d1b86b53 | ||
|
|
605fe2ebaf | ||
|
|
811510ac66 | ||
|
|
b2808f82df | ||
|
|
97bc283ce1 | ||
|
|
d9abcb3ef4 | ||
|
|
5c9c8951c1 | ||
|
|
633916d5b7 | ||
|
|
55a1f5bba2 | ||
|
|
7d486f0a41 | ||
|
|
88672a19d5 | ||
|
|
d3b781679a | ||
|
|
a02008ec17 | ||
|
|
8302a7fecf | ||
|
|
4d5d36d6e7 | ||
|
|
8b6b47c63e | ||
|
|
839220856b | ||
|
|
aa8f3f69a2 | ||
|
|
1ae474c48b | ||
|
|
edf997a4c1 | ||
|
|
508bd49512 | ||
|
|
37bd250eeb | ||
|
|
0da4076a32 | ||
|
|
37854a62db | ||
|
|
c5be891b6a | ||
|
|
9fb399b3d7 | ||
|
|
135f878203 | ||
|
|
efea0476d8 | ||
|
|
1fe7c0da5c | ||
|
|
3986722477 | ||
|
|
7228ea0a39 | ||
|
|
9a4c23af12 | ||
|
|
19e8bc923c |
No files matched your search
+11
-9
@@ -1,11 +1,13 @@
|
|||||||
* text=auto eol=lf
|
* text=auto eol=lf
|
||||||
*.sh text eol=lf
|
|
||||||
*.py text eol=lf
|
|
||||||
*.yml text eol=lf
|
|
||||||
*.json text eol=lf
|
|
||||||
*.md text eol=lf
|
|
||||||
*.bat text eol=crlf
|
|
||||||
|
|
||||||
*.png binary
|
# Typische Binärformate nicht normalisieren
|
||||||
*.jpg binary
|
*.png -text
|
||||||
*.pdf binary
|
*.jpg -text
|
||||||
|
*.jpeg -text
|
||||||
|
*.gif -text
|
||||||
|
*.pdf -text
|
||||||
|
*.zip -text
|
||||||
|
*.tar -text
|
||||||
|
*.tar.gz -text
|
||||||
|
*.tgz -text
|
||||||
|
*.mp4 -text
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
name: Deploy Trainer_LLM to llm-node
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ "main" ]
|
||||||
|
paths:
|
||||||
|
- "knowledge/**"
|
||||||
|
- "llm-api/**"
|
||||||
|
- "scripts/**"
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: deploy-trainer-llm
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
runs-on: linux_host # muss zum Runner-Label passen
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
# Absolute URL reduziert Abhängigkeit von DEFAULT_ACTIONS_URL
|
||||||
|
uses: https://github.com/actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Sanity — Runner & Commit
|
||||||
|
run: |
|
||||||
|
echo "Runner: $RUNNER_NAME Labels: $RUNNER_LABELS"
|
||||||
|
echo "Commit: $GITEA_SHA Ref: $GITEA_REF"
|
||||||
|
uname -a
|
||||||
|
|
||||||
|
- name: Debug whoami & write test
|
||||||
|
run: |
|
||||||
|
whoami
|
||||||
|
id
|
||||||
|
getent passwd $(whoami) || true
|
||||||
|
# Testet Schreibrecht unter /home/llmadmin
|
||||||
|
touch /home/llmadmin/.write_test && rm /home/llmadmin/.write_test || echo "no write"
|
||||||
|
|
||||||
|
- name: Ensure target directories exist
|
||||||
|
run: |
|
||||||
|
mkdir -p /home/llmadmin/knowledge
|
||||||
|
mkdir -p /home/llmadmin/llm-api
|
||||||
|
mkdir -p /home/llmadmin/scripts
|
||||||
|
|
||||||
|
- name: Rsync knowledge/
|
||||||
|
if: ${{ hashFiles('knowledge/**') != '' }}
|
||||||
|
run: |
|
||||||
|
rsync -a --delete \
|
||||||
|
--exclude='.git' \
|
||||||
|
--exclude='.env' --exclude='.env.*' --exclude='**/.env' --exclude='**/.env.*' \
|
||||||
|
knowledge/ /home/llmadmin/knowledge/
|
||||||
|
echo "Synced knowledge/"
|
||||||
|
|
||||||
|
- name: Rsync llm-api/
|
||||||
|
if: ${{ hashFiles('llm-api/**') != '' }}
|
||||||
|
run: |
|
||||||
|
rsync -a --delete \
|
||||||
|
--exclude='.git' \
|
||||||
|
--exclude='.env' --exclude='.env.*' --exclude='**/.env' --exclude='**/.env.*' \
|
||||||
|
llm-api/ /home/llmadmin/llm-api/
|
||||||
|
echo "Synced llm-api/"
|
||||||
|
|
||||||
|
- name: Rsync scripts/
|
||||||
|
if: ${{ hashFiles('scripts/**') != '' }}
|
||||||
|
run: |
|
||||||
|
rsync -a --delete \
|
||||||
|
--exclude='.git' \
|
||||||
|
--exclude='.env' --exclude='.env.*' --exclude='**/.env' --exclude='**/.env.*' \
|
||||||
|
scripts/ /home/llmadmin/scripts/
|
||||||
|
echo "Synced scripts/"
|
||||||
|
|
||||||
|
- name: Optional — systemctl --user restart llm-api (ignore if missing)
|
||||||
|
continue-on-error: true
|
||||||
|
env:
|
||||||
|
XDG_RUNTIME_DIR: /run/user/${{ inputs.uid || 1000 }}
|
||||||
|
run: |
|
||||||
|
# Versuche nur zu restarten, wenn der Service existiert
|
||||||
|
if systemctl --user list-unit-files | grep -q '^llm-api.service'; then
|
||||||
|
systemctl --user restart llm-api.service
|
||||||
|
systemctl --user --no-pager status llm-api.service --full -l || true
|
||||||
|
else
|
||||||
|
echo "llm-api.service nicht gefunden — Schritt wird übersprungen."
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Post-check — show latest changes
|
||||||
|
run: |
|
||||||
|
echo "Deploy complete. Listing targets:"
|
||||||
|
ls -la /home/llmadmin/knowledge | tail -n +1 || true
|
||||||
|
ls -la /home/llmadmin/llm-api | tail -n +1 || true
|
||||||
|
ls -la /home/llmadmin/scripts | tail -n +1 || true
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
# Kommentarzeile zum Testen ob der Server die Änderung erkennt
|
||||||
from fastapi import APIRouter, HTTPException, Query
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|||||||
+123
-132
@@ -1,33 +1,48 @@
|
|||||||
|
# Test eines Kommentars, um die Funktion des gitea testen zu können
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, Query
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import List, Optional, Dict, Any
|
from typing import List, Optional, Dict, Any
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
from datetime import datetime, date
|
from datetime import datetime
|
||||||
from clients import model, qdrant
|
from clients import model, qdrant
|
||||||
from qdrant_client.models import PointStruct, VectorParams, Distance, PointIdsList
|
from qdrant_client.models import (
|
||||||
|
PointStruct,
|
||||||
|
VectorParams,
|
||||||
|
Distance,
|
||||||
|
PointIdsList,
|
||||||
|
# NEW: für Filter-Queries (Lookup via external_id)
|
||||||
|
Filter, FieldCondition, MatchValue,
|
||||||
|
)
|
||||||
import os
|
import os
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
# ---- Models ----
|
# =========================
|
||||||
|
# Models
|
||||||
|
# =========================
|
||||||
class Exercise(BaseModel):
|
class Exercise(BaseModel):
|
||||||
id: str = Field(default_factory=lambda: str(uuid4()))
|
id: str = Field(default_factory=lambda: str(uuid4()))
|
||||||
|
# NEW — optional, bricht vorhandene POST-Calls nicht
|
||||||
|
external_id: Optional[str] = None # z.B. "mw:12345"
|
||||||
|
fingerprint: Optional[str] = None # sha256 über Kernfelder
|
||||||
|
source: Optional[str] = None # Herkunft, z.B. "MediaWiki"
|
||||||
|
imported_at: Optional[datetime] = None # vom Import gesetzt
|
||||||
|
|
||||||
|
# Bestehende Felder (unverändert)
|
||||||
title: str
|
title: str
|
||||||
summary: str
|
summary: str
|
||||||
short_description: str
|
short_description: str
|
||||||
keywords: List[str] = []
|
keywords: List[str] = []
|
||||||
link: Optional[str] = None
|
link: Optional[str] = None
|
||||||
|
|
||||||
discipline: str
|
discipline: str
|
||||||
group: Optional[str] = None
|
group: Optional[str] = None
|
||||||
age_group: str
|
age_group: str
|
||||||
target_group: str
|
target_group: str
|
||||||
min_participants: int
|
min_participants: int
|
||||||
duration_minutes: int
|
duration_minutes: int
|
||||||
|
|
||||||
capabilities: Dict[str, int] = {}
|
capabilities: Dict[str, int] = {}
|
||||||
category: str
|
category: str
|
||||||
|
|
||||||
purpose: str
|
purpose: str
|
||||||
execution: str
|
execution: str
|
||||||
notes: str
|
notes: str
|
||||||
@@ -35,146 +50,122 @@ class Exercise(BaseModel):
|
|||||||
method: str
|
method: str
|
||||||
equipment: List[str] = []
|
equipment: List[str] = []
|
||||||
|
|
||||||
class PhaseExercise(BaseModel):
|
|
||||||
exercise_id: str
|
|
||||||
cond_load: Dict[str, Any] = {}
|
|
||||||
coord_load: Dict[str, Any] = {}
|
|
||||||
instructions: str
|
|
||||||
|
|
||||||
class PlanPhase(BaseModel):
|
|
||||||
name: str
|
|
||||||
duration_minutes: int
|
|
||||||
method: str
|
|
||||||
method_notes: str
|
|
||||||
exercises: List[PhaseExercise]
|
|
||||||
|
|
||||||
class TrainingPlan(BaseModel):
|
|
||||||
id: str = Field(default_factory=lambda: str(uuid4()))
|
|
||||||
title: str
|
|
||||||
short_description: str
|
|
||||||
collection: str
|
|
||||||
discipline: str
|
|
||||||
group: Optional[str] = None
|
|
||||||
dojo: str
|
|
||||||
date: date
|
|
||||||
plan_duration_weeks: int
|
|
||||||
focus_areas: List[str] = []
|
|
||||||
predecessor_plan_id: Optional[str] = None
|
|
||||||
age_group: str
|
|
||||||
created_at: datetime = Field(default_factory=datetime.utcnow)
|
|
||||||
phases: List[PlanPhase]
|
|
||||||
|
|
||||||
class DeleteResponse(BaseModel):
|
class DeleteResponse(BaseModel):
|
||||||
status: str
|
status: str
|
||||||
count: int
|
count: int
|
||||||
collection: str
|
collection: str
|
||||||
source: Optional[str] = None
|
|
||||||
type: Optional[str] = None
|
|
||||||
|
|
||||||
# ---- CRUD Endpoints for Exercise ----
|
# =========================
|
||||||
|
# Helpers
|
||||||
|
# =========================
|
||||||
|
COLLECTION = os.getenv("EXERCISE_COLLECTION", "exercises")
|
||||||
|
|
||||||
|
# CHANGED: Factorized to reuse for both create and update
|
||||||
|
def _ensure_collection():
|
||||||
|
if not qdrant.collection_exists(COLLECTION):
|
||||||
|
qdrant.recreate_collection(
|
||||||
|
collection_name=COLLECTION,
|
||||||
|
vectors_config=VectorParams(
|
||||||
|
size=model.get_sentence_embedding_dimension(),
|
||||||
|
distance=Distance.COSINE,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# NEW: gemeinsamer Helper für external_id-Lookup
|
||||||
|
def _lookup_by_external_id(external_id: str) -> Optional[Dict[str, Any]]:
|
||||||
|
_ensure_collection()
|
||||||
|
flt = Filter(must=[FieldCondition(key="external_id", match=MatchValue(value=external_id))])
|
||||||
|
pts, _ = qdrant.scroll(collection_name=COLLECTION, scroll_filter=flt, limit=1)
|
||||||
|
if not pts:
|
||||||
|
return None
|
||||||
|
# qdrant_client liefert PointStruct; wir geben die payload + id zurück
|
||||||
|
doc = pts[0].payload or {}
|
||||||
|
doc = dict(doc)
|
||||||
|
doc.setdefault("id", str(pts[0].id))
|
||||||
|
return doc
|
||||||
|
|
||||||
|
# NEW: konsistente Embedding-Erzeugung
|
||||||
|
_def_embed_text_fields = ("title", "summary", "short_description", "purpose", "execution", "notes")
|
||||||
|
|
||||||
|
def _make_vector(ex: Exercise) -> List[float]:
|
||||||
|
text = ". ".join([getattr(ex, f, "") for f in _def_embed_text_fields if getattr(ex, f, None)])
|
||||||
|
# Achtung: model.encode muss synchron sein; sonst async anpassen
|
||||||
|
vec = model.encode(text).tolist()
|
||||||
|
return vec
|
||||||
|
|
||||||
|
# =========================
|
||||||
|
# Endpoints
|
||||||
|
# =========================
|
||||||
|
@router.get("/exercise/by-external-id") # NEW
|
||||||
|
def get_exercise_by_external_id(external_id: str = Query(..., min_length=3)):
|
||||||
|
"""Lookup für Idempotenz im Importer. Liefert 404, wenn nicht vorhanden."""
|
||||||
|
found = _lookup_by_external_id(external_id)
|
||||||
|
if not found:
|
||||||
|
raise HTTPException(status_code=404, detail="not found")
|
||||||
|
return found
|
||||||
|
|
||||||
@router.post("/exercise", response_model=Exercise)
|
@router.post("/exercise", response_model=Exercise)
|
||||||
def create_exercise(ex: Exercise):
|
def create_or_update_exercise(ex: Exercise):
|
||||||
# Ensure Exercise collection exists
|
"""
|
||||||
if not qdrant.collection_exists("exercises"):
|
CHANGED: Upsert-Semantik. Wenn `external_id` existiert und bereits in Qdrant gefunden wird,
|
||||||
qdrant.recreate_collection(
|
wird dieselbe Point-ID überschrieben (echtes Update). Ansonsten neuer Eintrag.
|
||||||
collection_name="exercises",
|
API-Signatur bleibt identisch (POST /exercise, Body = Exercise).
|
||||||
vectors_config=VectorParams(
|
"""
|
||||||
size=model.get_sentence_embedding_dimension(),
|
_ensure_collection()
|
||||||
distance=Distance.COSINE
|
|
||||||
)
|
|
||||||
)
|
|
||||||
vec = model.encode(f"{ex.title}. {ex.summary}").tolist()
|
|
||||||
point = PointStruct(id=ex.id, vector=vec, payload=ex.dict())
|
|
||||||
qdrant.upsert(collection_name="exercises", points=[point])
|
|
||||||
return ex
|
|
||||||
|
|
||||||
@router.get("/exercise", response_model=List[Exercise])
|
# Default: neue Point-ID aus dem Exercise-Objekt
|
||||||
def list_exercises(
|
point_id = ex.id
|
||||||
discipline: Optional[str] = Query(None),
|
|
||||||
group: Optional[str] = Query(None),
|
# Wenn external_id gesetzt → prüfen, ob bereits vorhanden → Point-ID übernehmen
|
||||||
tags: Optional[str] = Query(None)
|
if ex.external_id:
|
||||||
):
|
prior = _lookup_by_external_id(ex.external_id)
|
||||||
filters = []
|
if prior:
|
||||||
if discipline:
|
point_id = prior.get("id", point_id)
|
||||||
filters.append({"key": "discipline", "match": {"value": discipline}})
|
|
||||||
if group:
|
# Embedding berechnen
|
||||||
filters.append({"key": "group", "match": {"value": group}})
|
vector = _make_vector(ex)
|
||||||
if tags:
|
|
||||||
for t in tags.split(","):
|
# Payload synchronisieren (id == point_id)
|
||||||
filters.append({"key": "keywords", "match": {"value": t.strip()}})
|
payload = ex.dict()
|
||||||
pts, _ = qdrant.scroll(
|
payload["id"] = str(point_id)
|
||||||
collection_name="exercises",
|
|
||||||
scroll_filter={"must": filters} if filters else None,
|
# Upsert in Qdrant
|
||||||
limit=10000
|
qdrant.upsert(
|
||||||
|
collection_name=COLLECTION,
|
||||||
|
points=[PointStruct(id=str(point_id), vector=vector, payload=payload)],
|
||||||
)
|
)
|
||||||
return [Exercise(**pt.payload) for pt in pts]
|
|
||||||
|
|
||||||
# ---- CRUD Endpoints for TrainingPlan ----
|
return Exercise(**payload)
|
||||||
@router.post("/plan", response_model=TrainingPlan)
|
|
||||||
def create_plan(plan: TrainingPlan):
|
|
||||||
# Ensure TrainingPlan collection exists
|
|
||||||
if not qdrant.collection_exists("training_plans"):
|
|
||||||
qdrant.recreate_collection(
|
|
||||||
collection_name="training_plans",
|
|
||||||
vectors_config=VectorParams(
|
|
||||||
size=model.get_sentence_embedding_dimension(),
|
|
||||||
distance=Distance.COSINE
|
|
||||||
)
|
|
||||||
)
|
|
||||||
vec = model.encode(f"{plan.title}. {plan.short_description}").tolist()
|
|
||||||
point = PointStruct(id=plan.id, vector=vec, payload=plan.dict())
|
|
||||||
qdrant.upsert(collection_name="training_plans", points=[point])
|
|
||||||
return plan
|
|
||||||
|
|
||||||
@router.get("/plan", response_model=List[TrainingPlan])
|
# (Optional) – Einzel-Abruf per ID (falls bereits vorhanden, unverändert)
|
||||||
def list_plans(
|
@router.get("/exercise/{exercise_id}", response_model=Exercise)
|
||||||
collection: str = Query("training_plans"),
|
def get_exercise(exercise_id: str):
|
||||||
discipline: Optional[str] = Query(None),
|
_ensure_collection()
|
||||||
group: Optional[str] = Query(None),
|
pts, _ = qdrant.scroll(
|
||||||
dojo: Optional[str] = Query(None)
|
collection_name=COLLECTION,
|
||||||
):
|
scroll_filter=Filter(must=[FieldCondition(key="id", match=MatchValue(value=exercise_id))]),
|
||||||
if not qdrant.collection_exists(collection):
|
limit=1,
|
||||||
return []
|
)
|
||||||
pts, _ = qdrant.scroll(collection_name=collection, limit=10000)
|
if not pts:
|
||||||
result = []
|
raise HTTPException(status_code=404, detail="not found")
|
||||||
for pt in pts:
|
payload = dict(pts[0].payload or {})
|
||||||
pl = TrainingPlan(**pt.payload)
|
payload.setdefault("id", str(pts[0].id))
|
||||||
if discipline and pl.discipline != discipline:
|
return Exercise(**payload)
|
||||||
continue
|
|
||||||
if group and pl.group != group:
|
|
||||||
continue
|
|
||||||
if dojo and pl.dojo != dojo:
|
|
||||||
continue
|
|
||||||
result.append(pl)
|
|
||||||
return result
|
|
||||||
|
|
||||||
# ---- Delete Endpoints ----
|
# Bestehende Admin-Utilities (Delete nach Filter / komplette Collection) – unverändert außer Nutzung von CONSTs
|
||||||
@router.delete("/delete-source", response_model=DeleteResponse)
|
@router.delete("/exercise/delete-by-external-id", response_model=DeleteResponse)
|
||||||
def delete_by_source(
|
def delete_by_external_id(external_id: str = Query(...)):
|
||||||
collection: str = Query(...),
|
_ensure_collection()
|
||||||
source: Optional[str] = Query(None),
|
flt = Filter(must=[FieldCondition(key="external_id", match=MatchValue(value=external_id))])
|
||||||
type: Optional[str] = Query(None)
|
pts, _ = qdrant.scroll(collection_name=COLLECTION, scroll_filter=flt, limit=10000)
|
||||||
):
|
|
||||||
if not qdrant.collection_exists(collection):
|
|
||||||
raise HTTPException(status_code=404, detail=f"Collection '{collection}' nicht gefunden.")
|
|
||||||
filt = []
|
|
||||||
if source:
|
|
||||||
filt.append({"key": "source", "match": {"value": source}})
|
|
||||||
if type:
|
|
||||||
filt.append({"key": "type", "match": {"value": type}})
|
|
||||||
if not filt:
|
|
||||||
raise HTTPException(status_code=400, detail="Mindestens ein Filterparameter muss angegeben werden.")
|
|
||||||
pts, _ = qdrant.scroll(collection_name=collection, scroll_filter={"must": filt}, limit=10000)
|
|
||||||
ids = [str(p.id) for p in pts]
|
ids = [str(p.id) for p in pts]
|
||||||
if not ids:
|
if not ids:
|
||||||
return DeleteResponse(status="🔍 Keine Einträge gefunden.", count=0, collection=collection)
|
return DeleteResponse(status="🔍 Keine Einträge gefunden.", count=0, collection=COLLECTION)
|
||||||
qdrant.delete(collection_name=collection, points_selector=PointIdsList(points=ids))
|
qdrant.delete(collection_name=COLLECTION, points_selector=PointIdsList(points=ids))
|
||||||
return DeleteResponse(status="🗑️ gelöscht", count=len(ids), collection=collection)
|
return DeleteResponse(status="🗑️ gelöscht", count=len(ids), collection=COLLECTION)
|
||||||
|
|
||||||
@router.delete("/delete-collection", response_model=DeleteResponse)
|
@router.delete("/exercise/delete-collection", response_model=DeleteResponse)
|
||||||
def delete_collection(
|
def delete_collection(collection: str = Query(default=COLLECTION)):
|
||||||
collection: str = Query(...)
|
|
||||||
):
|
|
||||||
if not qdrant.collection_exists(collection):
|
if not qdrant.collection_exists(collection):
|
||||||
raise HTTPException(status_code=404, detail=f"Collection '{collection}' nicht gefunden.")
|
raise HTTPException(status_code=404, detail=f"Collection '{collection}' nicht gefunden.")
|
||||||
qdrant.delete_collection(collection_name=collection)
|
qdrant.delete_collection(collection_name=collection)
|
||||||
|
|||||||
+1
-1
@@ -28,6 +28,6 @@ async def unicorn_exception_handler(request, exc):
|
|||||||
return JSONResponse(status_code=500, content={"detail": "Interner Serverfehler."})
|
return JSONResponse(status_code=500, content={"detail": "Interner Serverfehler."})
|
||||||
|
|
||||||
# Router einbinden
|
# Router einbinden
|
||||||
app.include_router(wiki_router, prefix="/import/wiki")
|
app.include_router(wiki_router)
|
||||||
app.include_router(embed_router)
|
app.include_router(embed_router)
|
||||||
app.include_router(exercise_router)
|
app.include_router(exercise_router)
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
Ein kurzer Test
|
||||||
+197
-135
@@ -1,173 +1,235 @@
|
|||||||
"""
|
"""
|
||||||
File: wiki_router.py
|
wiki_router.py – v1.4.1 (stabil & nachvollziehbar)
|
||||||
Beschreibung:
|
|
||||||
- Endpunkte für MediaWiki-Integration im lokalen Netzwerk.
|
Änderungen ggü. v1.4.0:
|
||||||
- Funktionen:
|
- /info: Optionalen Request-Parameter entfernt (FastAPI/Pydantic Typfehler behoben)
|
||||||
* /health: Prüft Verfügbarkeit der MediaWiki-API.
|
- Keine API-Signaturänderungen der Routen
|
||||||
* /login: Führt clientlogin durch und speichert Session-Cookies.
|
|
||||||
* /semantic/pages: Listet alle Übungen inkl. Unterkategorien via SMW-Ask.
|
Ziele:
|
||||||
* /parsepage: Ruft Roh-Wikitext über action=parse für eine Seite ab.
|
- /semantic/pages reichert pageid/fullurl für ALLE Titel batchweise an (redirects=1, converttitles=1)
|
||||||
* /info: Liefert pageid und fullurl über Core-API Query.
|
- /info robust: 404 statt 500, mit Titel-Varianten
|
||||||
* /semantic/page: Liefert Metadaten einer Übung und Wikitext sowie pageid über Core-API.
|
- Wiederholungen & Throttling gegen MediaWiki
|
||||||
Version: 1.2.0
|
- Optionale Diagnose-Ausgaben und Coverage-Kennzahlen
|
||||||
|
|
||||||
|
|
||||||
|
Wenn ihr stattdessen den Prefix im Router setzen wollt, einfach in der APIRouter-Zeile unten
|
||||||
|
prefix="/import/wiki" ergänzen und in main.py OHNE prefix einbinden.
|
||||||
"""
|
"""
|
||||||
from dotenv import load_dotenv
|
|
||||||
load_dotenv()
|
from typing import Dict, Any, Optional, List
|
||||||
from fastapi import APIRouter, HTTPException, Query
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from typing import Dict, Any, List
|
import os, time, logging
|
||||||
import requests, os
|
import requests
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
__version__ = "1.2.0"
|
load_dotenv()
|
||||||
router = APIRouter()
|
|
||||||
|
|
||||||
|
logger = logging.getLogger("wiki_router")
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/import/wiki", tags=["wiki"])
|
||||||
|
|
||||||
|
# -------- Konfiguration --------
|
||||||
WIKI_API_URL = os.getenv("WIKI_API_URL", "https://karatetrainer.net/api.php")
|
WIKI_API_URL = os.getenv("WIKI_API_URL", "https://karatetrainer.net/api.php")
|
||||||
wiki_session = requests.Session()
|
WIKI_TIMEOUT = float(os.getenv("WIKI_TIMEOUT", "15"))
|
||||||
|
WIKI_BATCH = int(os.getenv("WIKI_BATCH", "50"))
|
||||||
|
WIKI_RETRIES = int(os.getenv("WIKI_RETRIES", "1")) # zusätzliche Versuche bei Upstream-Fehlern
|
||||||
|
WIKI_SLEEPMS = int(os.getenv("WIKI_SLEEP_MS", "0")) # Throttle zwischen Requests
|
||||||
|
|
||||||
|
# Single Session (Cookies für Login)
|
||||||
|
wiki_session = requests.Session()
|
||||||
|
wiki_session.headers.update({"User-Agent": "local-llm-wiki-proxy/1.4.1"})
|
||||||
|
|
||||||
|
# -------- Schemas --------
|
||||||
class WikiLoginRequest(BaseModel):
|
class WikiLoginRequest(BaseModel):
|
||||||
username: str
|
username: str
|
||||||
password: str
|
password: str
|
||||||
|
|
||||||
class WikiLoginResponse(BaseModel):
|
class WikiLoginResponse(BaseModel):
|
||||||
status: str
|
status: str
|
||||||
message: str | None = None
|
message: Optional[str] = None
|
||||||
|
|
||||||
class PageContentResponse(BaseModel):
|
|
||||||
pageid: int
|
|
||||||
title: str
|
|
||||||
wikitext: str
|
|
||||||
|
|
||||||
class PageInfoResponse(BaseModel):
|
class PageInfoResponse(BaseModel):
|
||||||
pageid: int
|
pageid: int
|
||||||
title: str
|
title: str
|
||||||
fullurl: str
|
fullurl: str
|
||||||
|
|
||||||
# Health-Check
|
class PageContentResponse(BaseModel):
|
||||||
|
pageid: int
|
||||||
|
title: str
|
||||||
|
wikitext: str
|
||||||
|
|
||||||
|
# -------- Utils --------
|
||||||
|
|
||||||
|
def _sleep():
|
||||||
|
if WIKI_SLEEPMS > 0:
|
||||||
|
time.sleep(WIKI_SLEEPMS / 1000.0)
|
||||||
|
|
||||||
|
|
||||||
|
def _request_with_retry(method: str, params: Dict[str, Any], *, data: Dict[str, Any] | None = None) -> requests.Response:
|
||||||
|
last_exc: Optional[Exception] = None
|
||||||
|
for attempt in range(WIKI_RETRIES + 1):
|
||||||
|
try:
|
||||||
|
if method == "GET":
|
||||||
|
resp = wiki_session.get(WIKI_API_URL, params=params, timeout=WIKI_TIMEOUT)
|
||||||
|
else:
|
||||||
|
resp = wiki_session.post(WIKI_API_URL, data=data or params, timeout=WIKI_TIMEOUT)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return resp
|
||||||
|
except Exception as e:
|
||||||
|
last_exc = e
|
||||||
|
logger.warning("Upstream error on %s (try %d/%d): %s", method, attempt + 1, WIKI_RETRIES + 1, e)
|
||||||
|
_sleep()
|
||||||
|
# alle Versuche erschöpft
|
||||||
|
raise HTTPException(status_code=502, detail=f"Upstream error: {last_exc}")
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_variants(title: str) -> List[str]:
|
||||||
|
t = (title or "").strip()
|
||||||
|
variants = {t}
|
||||||
|
if " " in t:
|
||||||
|
variants.add(t.replace(" ", "_"))
|
||||||
|
# Bindestrich / Gedankenstrich Varianten
|
||||||
|
for a, b in [("-", "–"), ("-", "—"), ("–", "-"), ("—", "-")]:
|
||||||
|
if a in t:
|
||||||
|
variants.add(t.replace(a, b))
|
||||||
|
return list(variants)
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_pageinfo_batch(titles: List[str]) -> Dict[str, Dict[str, Any]]:
|
||||||
|
if not titles:
|
||||||
|
return {}
|
||||||
|
out: Dict[str, Dict[str, Any]] = {}
|
||||||
|
for i in range(0, len(titles), max(1, WIKI_BATCH)):
|
||||||
|
chunk = titles[i:i + max(1, WIKI_BATCH)]
|
||||||
|
params = {
|
||||||
|
"action": "query",
|
||||||
|
"format": "json",
|
||||||
|
"prop": "info",
|
||||||
|
"inprop": "url",
|
||||||
|
"redirects": 1,
|
||||||
|
"converttitles": 1,
|
||||||
|
"titles": "|".join(chunk),
|
||||||
|
}
|
||||||
|
resp = _request_with_retry("GET", params)
|
||||||
|
data = resp.json() or {}
|
||||||
|
q = data.get("query", {})
|
||||||
|
redirects = {d.get("from"): d.get("to") for d in (q.get("redirects") or [])}
|
||||||
|
pages = q.get("pages", {}) or {}
|
||||||
|
for pid_str, page in pages.items():
|
||||||
|
if page.get("missing") is not None or str(pid_str) == "-1":
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
pid = int(pid_str)
|
||||||
|
except ValueError:
|
||||||
|
pid = int(page.get("pageid", -1))
|
||||||
|
title_out = page.get("title")
|
||||||
|
fullurl = page.get("fullurl") or page.get("canonicalurl") or ""
|
||||||
|
if not title_out:
|
||||||
|
continue
|
||||||
|
out[title_out] = {"pageid": pid, "fullurl": fullurl}
|
||||||
|
# auch Originaltitel der Redirects auflösen
|
||||||
|
for frm, to in redirects.items():
|
||||||
|
if to == title_out and frm not in out:
|
||||||
|
out[frm] = {"pageid": pid, "fullurl": fullurl}
|
||||||
|
_sleep()
|
||||||
|
return out
|
||||||
|
|
||||||
|
# -------- Endpoints --------
|
||||||
@router.get("/health")
|
@router.get("/health")
|
||||||
def health_check():
|
def health(verbose: Optional[int] = Query(default=0)) -> Dict[str, Any]:
|
||||||
try:
|
# einfacher Ping
|
||||||
resp = wiki_session.get(
|
resp = _request_with_retry("GET", {"action": "query", "meta": "siteinfo", "format": "json"})
|
||||||
WIKI_API_URL,
|
if verbose:
|
||||||
params={"action": "query", "meta": "siteinfo", "siprop": "general", "format": "json"},
|
info = resp.json().get("query", {}).get("general", {})
|
||||||
timeout=5
|
return {"status": "ok", "wiki": {"sitename": info.get("sitename"), "generator": info.get("generator")}}
|
||||||
)
|
|
||||||
resp.raise_for_status()
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=502, detail=f"Wiki nicht erreichbar: {e}")
|
|
||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
# Login Endpoint
|
|
||||||
@router.post("/login", response_model=WikiLoginResponse)
|
@router.post("/login", response_model=WikiLoginResponse)
|
||||||
def login(data: WikiLoginRequest):
|
def login(data: WikiLoginRequest):
|
||||||
# Token holen
|
# Token holen
|
||||||
try:
|
tok = _request_with_retry("GET", {"action": "query", "meta": "tokens", "type": "login", "format": "json"})
|
||||||
token_resp = wiki_session.get(
|
token = tok.json().get("query", {}).get("tokens", {}).get("logintoken")
|
||||||
WIKI_API_URL,
|
|
||||||
params={"action": "query", "meta": "tokens", "type": "login", "format": "json"},
|
|
||||||
timeout=10
|
|
||||||
)
|
|
||||||
token_resp.raise_for_status()
|
|
||||||
token = token_resp.json().get("query", {}).get("tokens", {}).get("logintoken")
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=502, detail=f"Token-Error: {e}")
|
|
||||||
if not token:
|
if not token:
|
||||||
raise HTTPException(status_code=502, detail="Kein Login-Token erhalten")
|
raise HTTPException(status_code=502, detail="Kein Login-Token erhalten")
|
||||||
# clientlogin
|
|
||||||
try:
|
|
||||||
login_resp = wiki_session.post(
|
|
||||||
WIKI_API_URL,
|
|
||||||
data={
|
|
||||||
"action": "clientlogin",
|
|
||||||
"format": "json",
|
|
||||||
"username": data.username,
|
|
||||||
"password": data.password,
|
|
||||||
"logintoken": token,
|
|
||||||
"loginreturnurl": "http://localhost:8000"
|
|
||||||
},
|
|
||||||
timeout=10
|
|
||||||
)
|
|
||||||
login_resp.raise_for_status()
|
|
||||||
status = login_resp.json().get("clientlogin", {}).get("status")
|
|
||||||
except Exception:
|
|
||||||
status = None
|
|
||||||
# fallback login
|
|
||||||
if status != "PASS":
|
|
||||||
alt = wiki_session.post(
|
|
||||||
WIKI_API_URL,
|
|
||||||
data={"action": "login", "format": "json", "lgname": data.username, "lgpassword": data.password},
|
|
||||||
timeout=10
|
|
||||||
)
|
|
||||||
alt.raise_for_status()
|
|
||||||
status = alt.json().get("login", {}).get("result")
|
|
||||||
if status in ("PASS", "Success"):
|
|
||||||
return WikiLoginResponse(status="success", message=None)
|
|
||||||
return WikiLoginResponse(status="failed", message="Login fehlgeschlagen")
|
|
||||||
|
|
||||||
# SMW-Ask: alle Übungen inkl. Unterkategorien
|
# clientlogin (mit returnurl) + Fallback action=login
|
||||||
|
try:
|
||||||
|
cl = _request_with_retry("POST", {}, data={
|
||||||
|
"action": "clientlogin",
|
||||||
|
"format": "json",
|
||||||
|
"username": data.username,
|
||||||
|
"password": data.password,
|
||||||
|
"logintoken": token,
|
||||||
|
"loginreturnurl": "https://example.org/",
|
||||||
|
})
|
||||||
|
st = cl.json().get("clientlogin", {}).get("status")
|
||||||
|
if st == "PASS":
|
||||||
|
return WikiLoginResponse(status="success")
|
||||||
|
except HTTPException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
lg = _request_with_retry("POST", {}, data={
|
||||||
|
"action": "login",
|
||||||
|
"format": "json",
|
||||||
|
"lgname": data.username,
|
||||||
|
"lgpassword": data.password,
|
||||||
|
"lgtoken": token,
|
||||||
|
})
|
||||||
|
res = lg.json().get("login", {}).get("result")
|
||||||
|
if res == "Success":
|
||||||
|
return WikiLoginResponse(status="success")
|
||||||
|
raise HTTPException(status_code=401, detail=f"Login fehlgeschlagen: {res}")
|
||||||
|
|
||||||
@router.get("/semantic/pages")
|
@router.get("/semantic/pages")
|
||||||
def semantic_pages(category: str = Query(..., description="Kategorie ohne 'Category:'")) -> Dict[str, Any]:
|
def semantic_pages(category: str = Query(..., description="Kategorie ohne 'Category:'")) -> Dict[str, Any]:
|
||||||
smw_query = f"[[Category:{category}]]"
|
# Rohdaten aus SMW (Ask)
|
||||||
ask_query = f"{smw_query}|limit=50000"
|
ask_query = f"[[Category:{category}]]|limit=50000"
|
||||||
r = wiki_session.get(
|
r = _request_with_retry("GET", {"action": "ask", "query": ask_query, "format": "json"})
|
||||||
WIKI_API_URL,
|
results = r.json().get("query", {}).get("results", {}) or {}
|
||||||
params={"action": "ask", "query": ask_query, "format": "json"},
|
titles = list(results.keys())
|
||||||
timeout=30
|
|
||||||
)
|
# Batch-Anreicherung mit pageid/fullurl für ALLE Titel
|
||||||
try:
|
info_map = _fetch_pageinfo_batch(titles)
|
||||||
r.raise_for_status()
|
|
||||||
except Exception as e:
|
enriched: Dict[str, Any] = {}
|
||||||
raise HTTPException(status_code=502, detail=f"SMW-Ask-Error: {e}")
|
missing = 0
|
||||||
return r.json().get("query", {}).get("results", {})
|
for title, entry in results.items():
|
||||||
|
base = entry if isinstance(entry, dict) else {}
|
||||||
|
extra = info_map.get(title, {})
|
||||||
|
if not extra:
|
||||||
|
missing += 1
|
||||||
|
enriched[title] = {
|
||||||
|
**base,
|
||||||
|
"pageid": extra.get("pageid", base.get("pageid")),
|
||||||
|
"fullurl": extra.get("fullurl", base.get("fullurl")),
|
||||||
|
}
|
||||||
|
logger.info("/semantic/pages: %d Titel, %d ohne pageid nach Enrichment", len(results), missing)
|
||||||
|
return enriched
|
||||||
|
|
||||||
# Wikitext über parse-Endpoint holen (per pageid)
|
|
||||||
@router.get("/parsepage", response_model=PageContentResponse)
|
@router.get("/parsepage", response_model=PageContentResponse)
|
||||||
def parse_page(pageid: int = Query(...), title: str = Query(None)):
|
def parse_page(pageid: int = Query(...), title: str = Query(None)):
|
||||||
r = wiki_session.get(
|
resp = _request_with_retry("GET", {"action": "parse", "pageid": pageid, "prop": "wikitext", "format": "json"})
|
||||||
WIKI_API_URL,
|
wikitext = resp.json().get("parse", {}).get("wikitext", {}).get("*", "")
|
||||||
params={"action": "parse", "pageid": pageid, "prop": "wikitext", "format": "json"},
|
|
||||||
timeout=20
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
r.raise_for_status()
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=502, detail=f"Parse-Error: {e}")
|
|
||||||
wikitext = r.json().get("parse", {}).get("wikitext", {}).get("*", "")
|
|
||||||
return PageContentResponse(pageid=pageid, title=title or "", wikitext=wikitext)
|
return PageContentResponse(pageid=pageid, title=title or "", wikitext=wikitext)
|
||||||
|
|
||||||
# Pageinfo über Core-API (ermittelt pageid + fullurl)
|
|
||||||
@router.get("/info", response_model=PageInfoResponse)
|
@router.get("/info", response_model=PageInfoResponse)
|
||||||
def page_info(title: str = Query(..., description="Name der Seite")):
|
def page_info(title: str = Query(..., description="Seitentitel")):
|
||||||
r = wiki_session.get(
|
# 1. Versuch: wie geliefert, mit redirects/converttitles
|
||||||
WIKI_API_URL,
|
res = _fetch_pageinfo_batch([title])
|
||||||
params={"action": "query", "titles": title, "prop": "info", "inprop": "url", "format": "json"},
|
if res.get(title):
|
||||||
timeout=10
|
d = res[title]
|
||||||
)
|
return PageInfoResponse(pageid=d["pageid"], title=title, fullurl=d.get("fullurl", ""))
|
||||||
try:
|
|
||||||
r.raise_for_status()
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=502, detail=f"Info-Error: {e}")
|
|
||||||
pages = r.json().get("query", {}).get("pages", {})
|
|
||||||
pid_str, page = next(iter(pages.items()))
|
|
||||||
pid = int(pid_str)
|
|
||||||
fullurl = page.get("fullurl")
|
|
||||||
return PageInfoResponse(pageid=pid, title=page.get("title"), fullurl=fullurl)
|
|
||||||
|
|
||||||
# Detail-Endpoint für eine Übung: Metadaten aus Ask + Wikitext & ID via Core-API
|
# 2. Varianten probieren
|
||||||
@router.get("/semantic/page", response_model=Dict[str, Any])
|
for v in _normalize_variants(title):
|
||||||
def semantic_page_detail(category: str = Query(...), title: str = Query(...)) -> Dict[str, Any]:
|
if v == title:
|
||||||
# Metadaten aus SMW-Ask
|
continue
|
||||||
entries = semantic_pages(category)
|
res2 = _fetch_pageinfo_batch([v])
|
||||||
entry = entries.get(title)
|
if res2.get(v):
|
||||||
if not entry:
|
d = res2[v]
|
||||||
raise HTTPException(status_code=404, detail="Übung nicht gefunden im SMW-Ask-Ergebnis.")
|
return PageInfoResponse(pageid=d["pageid"], title=v, fullurl=d.get("fullurl", ""))
|
||||||
# Pageinfo via Core-API
|
|
||||||
info = page_info(title=title)
|
# 3. sauber 404
|
||||||
# Wikitext via parse
|
raise HTTPException(status_code=404, detail=f"Page not found: {title}")
|
||||||
parsed = parse_page(pageid=info.pageid, title=title)
|
|
||||||
return {
|
|
||||||
"title": title,
|
|
||||||
"pageid": info.pageid,
|
|
||||||
"fullurl": info.fullurl,
|
|
||||||
"printouts": entry.get("printouts", {}),
|
|
||||||
"wikitext": parsed.wikitext
|
|
||||||
}
|
|
||||||
+356
-124
@@ -1,183 +1,436 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Module: wiki_importer.py
|
wiki_importer.py – v2.3.8
|
||||||
Beschreibung:
|
|
||||||
- Importiert Übungen aus dem MediaWiki via FastAPI wiki_router
|
|
||||||
- Führt vor dem Import einen Login gegen /import/wiki/login durch (falls nicht via --skip-login deaktiviert)
|
|
||||||
- Holt Liste aller Übungs-Titel (SMW-Ask) via `/semantic/pages`
|
|
||||||
- Für jede Übung:
|
|
||||||
* Fetch pageinfo (pageid, fullurl) via `/info`
|
|
||||||
* Parse Wikitext (Templates: ÜbungInfoBox, Übungsbeschreibung, SkillDevelopment) via `/parsepage`
|
|
||||||
* Baut Payload entsprechend Exercise-Datenmodell
|
|
||||||
* POST an `/exercise` Endpoint (exercise_router)
|
|
||||||
- Unterstützt Single-Import via `--title` (oder ENV `WIKI_EXERCISE_TITLE`) und Full-Import via `--all`
|
|
||||||
- Optional: Credentials via CLI (--username/--password) oder `.env` (WIKI_BOT_USER / WIKI_BOT_PASSWORD)
|
|
||||||
|
|
||||||
Version: 2.1.0
|
Ziel dieses Patches: Die Felder `discipline`, `execution`, `keywords`, `equipment`, `duration_minutes` usw.
|
||||||
|
kommen bei dir teilweise leer an. Ursache sind zu aggressive Normalisierungen/Matcher.
|
||||||
|
|
||||||
|
Fix (konservativ & robust):
|
||||||
|
- Parser liest jetzt **gezielt** die bekannten Templates **ohne** Over-Normalisierung:
|
||||||
|
• `{{ÜbungInfoBox}}` / `{{UebungInfoBox}}`
|
||||||
|
• `{{Übungsbeschreibung}}` / `{{Uebungsbeschreibung}}`
|
||||||
|
• `{{Hilfsmittel}}`
|
||||||
|
• `{{SkillDevelopment}}`
|
||||||
|
- Feld-Extraktion nutzt **zuerst die exakten Wiki-Parameternamen** (deutsch/mit Umlauten),
|
||||||
|
erst danach schmale Synonym-Fallbacks. Das stellt sicher, dass z. B. `Schlüsselworte=`
|
||||||
|
wirklich in `keywords` landet.
|
||||||
|
- `imported_at` wird bei **Create und Update** gesetzt.
|
||||||
|
- Optionales Debugging: `--debug-raw` druckt die gefundenen Raw-Keys (einfach, nachvollziehbar).
|
||||||
|
|
||||||
|
Bestehende API-Endpunkte bleiben unverändert.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import argparse
|
import argparse
|
||||||
from typing import Dict, Any
|
from typing import Dict, Any, Tuple, Optional, List
|
||||||
import requests
|
import requests
|
||||||
import mwparserfromhell
|
import mwparserfromhell
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
|
||||||
# ----- Konfiguration / Defaults -----
|
# ----- Konfiguration / Defaults -----
|
||||||
load_dotenv() # .env laden, falls vorhanden
|
load_dotenv()
|
||||||
|
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki")
|
||||||
|
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise")
|
||||||
|
DEFAULT_CAT = os.getenv("WIKI_CATEGORY", "Übungen")
|
||||||
|
DEFAULT_TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
|
||||||
|
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "60"))
|
||||||
|
|
||||||
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki") # FastAPI-Wiki-Proxy
|
# ----- Helpers für Wiki-Router -----
|
||||||
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise") # Exercise-Endpoint
|
|
||||||
DEFAULT_CAT = os.getenv("WIKI_CATEGORY", "Übungen")
|
|
||||||
DEFAULT_TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
|
|
||||||
|
|
||||||
# ---- Hilfsfunktionen für Wiki-Router ----
|
|
||||||
def wiki_health() -> None:
|
def wiki_health() -> None:
|
||||||
r = requests.get(f"{API_BASE_URL}/health", timeout=15)
|
r = requests.get(f"{API_BASE_URL}/health", timeout=15)
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
print("[Sanity] Wiki health OK")
|
print("[Sanity] Wiki health OK")
|
||||||
|
|
||||||
|
|
||||||
def wiki_login(username: str, password: str) -> None:
|
def wiki_login(username: str, password: str) -> None:
|
||||||
"""
|
|
||||||
Führt einen Login gegen den wiki_router durch.
|
|
||||||
Erwartet: {"status":"success"} bei Erfolg.
|
|
||||||
"""
|
|
||||||
payload = {"username": username, "password": password}
|
payload = {"username": username, "password": password}
|
||||||
r = requests.post(f"{API_BASE_URL}/login", json=payload, timeout=30)
|
r = requests.post(f"{API_BASE_URL}/login", json=payload, timeout=30)
|
||||||
# kein raise_for_status(), wir wollen die JSON-Fehler sauber ausgeben
|
|
||||||
try:
|
try:
|
||||||
data = r.json()
|
data = r.json()
|
||||||
except Exception:
|
except Exception:
|
||||||
print(f"[Login] HTTP {r.status_code}: {r.text}")
|
print(f"[Login] HTTP {r.status_code}: {r.text}")
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
|
|
||||||
status = (data or {}).get("status")
|
status = (data or {}).get("status")
|
||||||
if status != "success":
|
if status != "success":
|
||||||
msg = (data or {}).get("message", "Login fehlgeschlagen")
|
msg = (data or {}).get("message", "Login fehlgeschlagen")
|
||||||
raise RuntimeError(f"[Login] {msg}")
|
raise RuntimeError(f"[Login] {msg}")
|
||||||
print("[Login] success")
|
print("[Login] success")
|
||||||
|
|
||||||
|
|
||||||
def fetch_all_pages(category: str) -> Dict[str, Any]:
|
def fetch_all_pages(category: str) -> Dict[str, Any]:
|
||||||
resp = requests.get(f"{API_BASE_URL}/semantic/pages", params={"category": category}, timeout=60)
|
resp = requests.get(f"{API_BASE_URL}/semantic/pages", params={"category": category}, timeout=REQUEST_TIMEOUT)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
return resp.json()
|
return resp.json()
|
||||||
|
|
||||||
|
|
||||||
def fetch_page_info(title: str) -> Dict[str, Any]:
|
def fetch_page_info(title: str) -> Dict[str, Any]:
|
||||||
r = requests.get(f"{API_BASE_URL}/info", params={"title": title}, timeout=30)
|
r = requests.get(f"{API_BASE_URL}/info", params={"title": title}, timeout=30)
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
info = r.json()
|
info = r.json()
|
||||||
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
|
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
|
||||||
|
|
||||||
|
# ----- Parser (konservativ) -----
|
||||||
|
|
||||||
|
T_INFOS = {"ÜbungInfoBox", "UebungInfoBox"}
|
||||||
|
T_BESCHR = {"Übungsbeschreibung", "Uebungsbeschreibung"}
|
||||||
|
T_HILFS = {"Hilfsmittel"}
|
||||||
|
T_SKILL = {"SkillDevelopment"}
|
||||||
|
|
||||||
|
|
||||||
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
|
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
|
||||||
print(f"[Parse] Lade '{title}' (ID={pageid})")
|
print(f"[Parse] Lade '{title}' (ID={pageid})")
|
||||||
resp = requests.get(
|
resp = requests.get(
|
||||||
f"{API_BASE_URL}/parsepage",
|
f"{API_BASE_URL}/parsepage",
|
||||||
params={"pageid": pageid, "title": title},
|
params={"pageid": pageid, "title": title},
|
||||||
timeout=60
|
timeout=REQUEST_TIMEOUT,
|
||||||
)
|
)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
wikitext = resp.json().get("wikitext", "")
|
wikitext = resp.json().get("wikitext", "")
|
||||||
wikicode = mwparserfromhell.parse(wikitext)
|
wikicode = mwparserfromhell.parse(wikitext)
|
||||||
|
|
||||||
raw: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid}
|
raw: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid, "wikitext": wikitext}
|
||||||
|
|
||||||
for tpl in wikicode.filter_templates():
|
for tpl in wikicode.filter_templates():
|
||||||
name = str(tpl.name).strip()
|
name = str(tpl.name).strip()
|
||||||
if name == "ÜbungInfoBox":
|
if name in T_INFOS or name in T_BESCHR or name in T_HILFS:
|
||||||
for p in tpl.params:
|
for p in tpl.params:
|
||||||
raw[str(p.name).strip()] = str(p.value).strip()
|
key = str(p.name).strip()
|
||||||
elif name == "Übungsbeschreibung":
|
val = str(p.value).strip()
|
||||||
for p in tpl.params:
|
raw[key] = val
|
||||||
raw[str(p.name).strip()] = str(p.value).strip()
|
elif name in T_SKILL:
|
||||||
elif name == "SkillDevelopment":
|
|
||||||
raw.setdefault("capabilities", [])
|
raw.setdefault("capabilities", [])
|
||||||
|
def _getp(t, k):
|
||||||
|
try:
|
||||||
|
return str(t.get(k).value).strip()
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
cap = _getp(tpl, "PrimaryCapability")
|
||||||
|
lvl = _getp(tpl, "CapabilityLevel")
|
||||||
try:
|
try:
|
||||||
cap = str(tpl.get("PrimaryCapability").value).strip()
|
lvl_i = int(lvl)
|
||||||
except Exception:
|
except Exception:
|
||||||
cap = ""
|
lvl_i = 0
|
||||||
try:
|
|
||||||
lvl = int(str(tpl.get("CapabilityLevel").value).strip())
|
|
||||||
except Exception:
|
|
||||||
lvl = 0
|
|
||||||
if cap:
|
if cap:
|
||||||
raw["capabilities"].append({"capability": cap, "level": lvl})
|
raw["capabilities"].append({"capability": cap, "level": lvl_i})
|
||||||
raw["wikitext"] = wikitext
|
|
||||||
return raw
|
return raw
|
||||||
|
|
||||||
def build_payload(raw: Dict[str, Any], fullurl: str, category: str) -> Dict[str, Any]:
|
# ----- Fingerprint (stabil, wie zuvor) -----
|
||||||
# Exercise.capabilities erwartet Dict[str,int]
|
|
||||||
caps_list = raw.get("capabilities", [])
|
def _normalize(v: Any) -> str:
|
||||||
capabilities = {}
|
if v is None:
|
||||||
for c in caps_list:
|
return ""
|
||||||
cap = c.get("capability")
|
if isinstance(v, (list, tuple)):
|
||||||
lvl = c.get("level")
|
return ",".join(_normalize(x) for x in v)
|
||||||
|
if isinstance(v, dict):
|
||||||
|
return json.dumps(v, sort_keys=True, ensure_ascii=False)
|
||||||
|
return str(v).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _norm_text(s: str) -> str:
|
||||||
|
if s is None:
|
||||||
|
return ""
|
||||||
|
s = str(s).replace("\u00a0", " ")
|
||||||
|
s = " ".join(s.split())
|
||||||
|
return s.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _canon_title(t: str) -> str:
|
||||||
|
t = (t or "").strip().replace("_", " ")
|
||||||
|
return t.replace("–", "-").replace("—", "-")
|
||||||
|
|
||||||
|
|
||||||
|
def compute_fingerprint(payload: Dict[str, Any]) -> str:
|
||||||
|
kws = payload.get("keywords") or []
|
||||||
|
kws = [k.replace("\u2013", "-").replace("\u2014", "-") for k in kws]
|
||||||
|
kws = sorted({(k or "").strip() for k in kws if (k or "").strip()}, key=str.casefold)
|
||||||
|
dur = payload.get("duration_minutes") or 0
|
||||||
|
try:
|
||||||
|
dur = int(round(float(dur)))
|
||||||
|
except Exception:
|
||||||
|
dur = 0
|
||||||
|
fields = [
|
||||||
|
_canon_title(payload.get("title", "")),
|
||||||
|
_norm_text(payload.get("summary", "")),
|
||||||
|
_norm_text(payload.get("execution", "")),
|
||||||
|
_norm_text(payload.get("notes", "")),
|
||||||
|
dur,
|
||||||
|
payload.get("capabilities", {}),
|
||||||
|
kws,
|
||||||
|
]
|
||||||
|
base = "|".join(_normalize(f) for f in fields)
|
||||||
|
return hashlib.sha256(base.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
# ----- Payload (exakte DE-Keys zuerst, dann schmale Fallbacks) -----
|
||||||
|
|
||||||
|
EXACT_KEYS = {
|
||||||
|
"summary": ["Summary", "Kurzbeschreibung"],
|
||||||
|
"execution": ["Durchführung", "Durchfuehrung", "Ablauf"],
|
||||||
|
"duration": ["Dauer", "Zeit"],
|
||||||
|
"keywords": ["Schlüsselworte", "Schlüsselwörter", "Schluesselworte", "Schluesselwoerter", "Keywords", "Tags"],
|
||||||
|
"equipment_prim": ["Hilfsmittel"],
|
||||||
|
"equipment_alt": ["Geräte", "Geraete", "Gerät", "Geraet", "Material"],
|
||||||
|
"discipline": ["Übungstyp", "Uebungstyp", "Disziplin"],
|
||||||
|
"group": ["Gruppengröße", "Gruppengroesse", "Group"],
|
||||||
|
"age_group": ["Altersgruppe"],
|
||||||
|
"target_group": ["Zielgruppe"],
|
||||||
|
"purpose": ["Ziel", "Zweck"],
|
||||||
|
"notes": ["Hinweise", "Notes"],
|
||||||
|
"preparation": ["Vorbereitung", "RefMethode"],
|
||||||
|
"method": ["Methode", "Method"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _first_any(raw: Dict[str, Any], keys: List[str]) -> Optional[str]:
|
||||||
|
for k in keys:
|
||||||
|
v = raw.get(k)
|
||||||
|
if isinstance(v, str) and v.strip():
|
||||||
|
return v.strip()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def build_payload(raw: Dict[str, Any], fullurl: str, category: str, *, mutate: bool = False) -> Dict[str, Any]:
|
||||||
|
# Capabilities -> Dict[str,int]
|
||||||
|
capabilities: Dict[str, int] = {}
|
||||||
|
for c in raw.get("capabilities", []) or []:
|
||||||
|
cap = c.get("capability"); lvl = c.get("level")
|
||||||
if isinstance(cap, str) and cap:
|
if isinstance(cap, str) and cap:
|
||||||
try:
|
try:
|
||||||
capabilities[cap] = int(lvl)
|
capabilities[cap] = int(lvl)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Defaults/Fallbacks
|
# Exakte Schlüssel zuerst
|
||||||
duration = 0.0
|
summary = _first_any(raw, EXACT_KEYS["summary"]) or ""
|
||||||
|
execution = _first_any(raw, EXACT_KEYS["execution"]) or ""
|
||||||
|
duration = _first_any(raw, EXACT_KEYS["duration"]) or "0"
|
||||||
|
|
||||||
|
kw_raw = _first_any(raw, EXACT_KEYS["keywords"]) or ""
|
||||||
|
if kw_raw:
|
||||||
|
parts = [p.strip() for p in kw_raw.replace("\n", ",").split(",")]
|
||||||
|
keywords = [p for p in parts if p]
|
||||||
|
else:
|
||||||
|
keywords = []
|
||||||
|
|
||||||
|
eq_raw = _first_any(raw, EXACT_KEYS["equipment_prim"]) or _first_any(raw, EXACT_KEYS["equipment_alt"]) or ""
|
||||||
|
if eq_raw:
|
||||||
|
equipment = [e.strip() for e in eq_raw.replace("\n", ",").split(",") if e.strip()]
|
||||||
|
else:
|
||||||
|
equipment = []
|
||||||
|
|
||||||
|
notes = _first_any(raw, EXACT_KEYS["notes"]) or ""
|
||||||
|
discipline = _first_any(raw, EXACT_KEYS["discipline"]) or ""
|
||||||
|
group = _first_any(raw, EXACT_KEYS["group"]) or None
|
||||||
|
age_group = _first_any(raw, EXACT_KEYS["age_group"]) or ""
|
||||||
|
target_group = _first_any(raw, EXACT_KEYS["target_group"]) or ""
|
||||||
|
purpose = _first_any(raw, EXACT_KEYS["purpose"]) or ""
|
||||||
|
preparation = _first_any(raw, EXACT_KEYS["preparation"]) or ""
|
||||||
|
method = _first_any(raw, EXACT_KEYS["method"]) or ""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
duration = float(raw.get("Dauer", 0) or 0)
|
duration_f = float(duration or 0)
|
||||||
except Exception:
|
except Exception:
|
||||||
duration = 0.0
|
duration_f = 0.0
|
||||||
|
|
||||||
keywords = []
|
|
||||||
kw_raw = raw.get("Schlüsselworte", "")
|
|
||||||
if isinstance(kw_raw, str):
|
|
||||||
keywords = [k.strip() for k in kw_raw.split(",") if k.strip()]
|
|
||||||
|
|
||||||
equipment = []
|
|
||||||
eq_raw = raw.get("equipment", [])
|
|
||||||
if isinstance(eq_raw, str):
|
|
||||||
equipment = [e.strip() for e in eq_raw.split(",") if e.strip()]
|
|
||||||
elif isinstance(eq_raw, list):
|
|
||||||
equipment = [str(e).strip() for e in eq_raw if str(e).strip()]
|
|
||||||
|
|
||||||
payload: Dict[str, Any] = {
|
payload: Dict[str, Any] = {
|
||||||
"title": raw.get("title") or "",
|
"title": raw.get("title") or "",
|
||||||
"summary": raw.get("Summary", "") or "",
|
"summary": summary,
|
||||||
"short_description": raw.get("Summary", "") or "",
|
"short_description": summary,
|
||||||
"keywords": keywords,
|
"keywords": keywords,
|
||||||
"link": fullurl or "",
|
"link": fullurl or "",
|
||||||
"discipline": raw.get("Übungstyp", "") or "",
|
"discipline": discipline,
|
||||||
"group": raw.get("Gruppengröße", "") or None,
|
"group": group,
|
||||||
"age_group": raw.get("Altersgruppe", "") or "",
|
"age_group": age_group,
|
||||||
"target_group": raw.get("Zielgruppe", "") or "",
|
"target_group": target_group,
|
||||||
"min_participants": 1,
|
"min_participants": 1,
|
||||||
"duration_minutes": int(round(duration)), # Exercise erwartet int
|
"duration_minutes": int(round(duration_f)),
|
||||||
"capabilities": capabilities,
|
"capabilities": capabilities,
|
||||||
"category": category or "",
|
"category": category or "",
|
||||||
"purpose": raw.get("Ziel", "") or "",
|
"purpose": purpose,
|
||||||
"execution": raw.get("Durchführung", "") or "",
|
"execution": execution,
|
||||||
"notes": raw.get("Hinweise", "") or "",
|
"notes": (notes + (" [auto-update]" if mutate else "")).strip(),
|
||||||
"preparation": raw.get("RefMethode", "") or "",
|
"preparation": preparation,
|
||||||
"method": raw.get("method", "") or "", # falls im Wikitext vorhanden
|
"method": method,
|
||||||
"equipment": equipment,
|
"equipment": equipment,
|
||||||
"fullurl": fullurl or "", # optionales Feld
|
"fullurl": fullurl or "",
|
||||||
# Idempotenz (optional nutzbar in exercise_router):
|
"external_id": f"mw:{raw.get('pageid')}",
|
||||||
"external_id": f"wiki:{raw.get('pageid')}",
|
"source": "MediaWiki",
|
||||||
"source": "MediaWiki"
|
|
||||||
}
|
}
|
||||||
|
payload["fingerprint"] = compute_fingerprint(payload)
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
def ingest_exercise(payload: Dict[str, Any]) -> None:
|
# ----- Lookup/Upsert -----
|
||||||
|
|
||||||
|
def lookup_by_external_id(external_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[int]]:
|
||||||
|
url = f"{EXERCISE_API}/by-external-id"
|
||||||
|
try:
|
||||||
|
r = requests.get(url, params={"external_id": external_id}, timeout=REQUEST_TIMEOUT)
|
||||||
|
if r.status_code == 404:
|
||||||
|
return None, 404
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json(), r.status_code
|
||||||
|
except requests.HTTPError as e:
|
||||||
|
return {"error": str(e), "status_code": getattr(e.response, "status_code", None)}, getattr(e.response, "status_code", None)
|
||||||
|
except Exception as e:
|
||||||
|
return {"error": str(e)}, None
|
||||||
|
|
||||||
|
|
||||||
|
def _payload_subset_for_fp(p: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"title": p.get("title"),
|
||||||
|
"summary": p.get("summary"),
|
||||||
|
"execution": p.get("execution"),
|
||||||
|
"notes": p.get("notes"),
|
||||||
|
"duration_minutes": p.get("duration_minutes"),
|
||||||
|
"capabilities": p.get("capabilities") or {},
|
||||||
|
"keywords": p.get("keywords") or [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _print_diff(before: Dict[str, Any], after: Dict[str, Any]) -> None:
|
||||||
|
keys = ["title","summary","execution","notes","duration_minutes","capabilities","keywords"]
|
||||||
|
b = {k: before.get(k) for k in keys}
|
||||||
|
a = {k: after.get(k) for k in keys}
|
||||||
|
def _kws(x):
|
||||||
|
return sorted({(k or "").strip() for k in (x or [])}, key=str.casefold)
|
||||||
|
b_norm = {
|
||||||
|
"title": _canon_title(b.get("title")),
|
||||||
|
"summary": _norm_text(b.get("summary")),
|
||||||
|
"execution": _norm_text(b.get("execution")),
|
||||||
|
"notes": _norm_text(b.get("notes")),
|
||||||
|
"duration_minutes": b.get("duration_minutes"),
|
||||||
|
"capabilities": b.get("capabilities"),
|
||||||
|
"keywords": _kws(b.get("keywords")),
|
||||||
|
}
|
||||||
|
a_norm = {
|
||||||
|
"title": _canon_title(a.get("title")),
|
||||||
|
"summary": _norm_text(a.get("summary")),
|
||||||
|
"execution": _norm_text(a.get("execution")),
|
||||||
|
"notes": _norm_text(a.get("notes")),
|
||||||
|
"duration_minutes": a.get("duration_minutes"),
|
||||||
|
"capabilities": a.get("capabilities"),
|
||||||
|
"keywords": _kws(a.get("keywords")),
|
||||||
|
}
|
||||||
|
diff = {k: (b_norm[k], a_norm[k]) for k in keys if b_norm.get(k) != a_norm.get(k)}
|
||||||
|
if diff:
|
||||||
|
print("[Diff] changes:", json.dumps(diff, ensure_ascii=False))
|
||||||
|
else:
|
||||||
|
print("[Diff] (none in hash fields)")
|
||||||
|
|
||||||
|
|
||||||
|
def _now_iso() -> str:
|
||||||
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_exercise(payload: Dict[str, Any], *, dry_run: bool = False) -> str:
|
||||||
title = payload.get("title", "<ohne Titel>")
|
title = payload.get("title", "<ohne Titel>")
|
||||||
resp = requests.post(EXERCISE_API, json=payload, timeout=60)
|
ext_id = payload.get("external_id")
|
||||||
if resp.status_code == 422:
|
fp_new = payload.get("fingerprint")
|
||||||
print(f"[Ingest] '{title}' -> FAILED 422:\n{resp.text}")
|
|
||||||
|
found, status = lookup_by_external_id(ext_id)
|
||||||
|
|
||||||
|
action = "create"; reason = "not found (lookup 404)"; found_payload = {}
|
||||||
|
|
||||||
|
if not (status == 404 or found is None):
|
||||||
|
if isinstance(found, dict):
|
||||||
|
found_payload = found.get("payload", found)
|
||||||
|
fp_old_stored = found.get("fingerprint") or found_payload.get("fingerprint")
|
||||||
|
fp_old_recalc = compute_fingerprint(_payload_subset_for_fp(found_payload))
|
||||||
|
if fp_new == fp_old_stored or fp_new == fp_old_recalc:
|
||||||
|
action, reason = "skip", "fingerprint unchanged"
|
||||||
|
else:
|
||||||
|
action, reason = "update", "fingerprint changed"
|
||||||
|
else:
|
||||||
|
action, reason = "create", "unexpected lookup type"
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
print(f"[DryRun] {action.upper():6} '{title}' ({ext_id}) – {reason}")
|
||||||
|
if action == "update":
|
||||||
|
_print_diff(found_payload, payload)
|
||||||
|
return action
|
||||||
|
|
||||||
|
if action == "create":
|
||||||
|
body = dict(payload); body["imported_at"] = _now_iso()
|
||||||
|
resp = requests.post(EXERCISE_API, json=body, timeout=REQUEST_TIMEOUT)
|
||||||
|
if resp.status_code == 422:
|
||||||
|
print(f"[Create] '{title}' -> FAILED 422:\n{resp.text}")
|
||||||
|
try: resp.raise_for_status()
|
||||||
|
except Exception: pass
|
||||||
|
else:
|
||||||
|
resp.raise_for_status(); print(f"[Create] '{title}' – {reason} -> OK")
|
||||||
|
elif action == "update":
|
||||||
|
body = dict(payload); body["imported_at"] = _now_iso()
|
||||||
|
resp = requests.post(EXERCISE_API, json=body, timeout=REQUEST_TIMEOUT)
|
||||||
|
if resp.status_code == 422:
|
||||||
|
print(f"[Update] '{title}' -> FAILED 422:\n{resp.text}")
|
||||||
|
try: resp.raise_for_status()
|
||||||
|
except Exception: pass
|
||||||
|
else:
|
||||||
|
resp.raise_for_status(); print(f"[Update] '{title}' – {reason} -> OK"); _print_diff(found_payload, payload)
|
||||||
|
else:
|
||||||
|
print(f"[Skip] '{title}' – {reason}")
|
||||||
|
return action
|
||||||
|
|
||||||
|
# ----- Orchestrierung -----
|
||||||
|
|
||||||
|
def process_one(title: str, category: str, *, mutate: bool = False, dry_run: bool = False, debug_raw: bool = False) -> str:
|
||||||
|
info = fetch_page_info(title)
|
||||||
|
pid = info.get("pageid"); fullurl = info.get("fullurl") or ""
|
||||||
|
if not pid:
|
||||||
|
print(f"[Error] pageid für '{title}' nicht gefunden.", file=sys.stderr); return "failed"
|
||||||
|
raw = parse_exercise(title, pid)
|
||||||
|
if debug_raw:
|
||||||
|
print("[Debug] Raw-Keys:", sorted([k for k in raw.keys() if k not in {"wikitext"}]))
|
||||||
|
payload = build_payload(raw, fullurl, category, mutate=mutate)
|
||||||
|
return upsert_exercise(payload, dry_run=dry_run)
|
||||||
|
|
||||||
|
|
||||||
|
def process_all(category: str, *, dry_run: bool = False, debug_raw: bool = False) -> Dict[str, int]:
|
||||||
|
stats = {"created": 0, "updated": 0, "skipped": 0, "failed": 0}
|
||||||
|
print(f"[Main] Lade Liste der Übungen aus Kategorie '{category}'…")
|
||||||
|
pages = fetch_all_pages(category)
|
||||||
|
print(f"[Main] {len(pages)} Seiten gefunden.")
|
||||||
|
|
||||||
|
for idx, (title, entry) in enumerate(pages.items(), 1):
|
||||||
try:
|
try:
|
||||||
resp.raise_for_status()
|
getter = getattr(entry, "get", None)
|
||||||
except Exception:
|
pid = getter("pageid") if callable(getter) else None
|
||||||
pass
|
fullurl = getter("fullurl") if callable(getter) else None
|
||||||
return
|
if not pid:
|
||||||
resp.raise_for_status()
|
info = fetch_page_info(title); pid = info.get("pageid"); fullurl = fullurl or info.get("fullurl")
|
||||||
print(f"[Ingest] '{title}' -> OK")
|
if not pid:
|
||||||
|
print(f"[Skip] '{title}' hat keine pageid"); stats["failed"] += 1; continue
|
||||||
|
raw = parse_exercise(title, pid)
|
||||||
|
if debug_raw and idx <= 5:
|
||||||
|
print(f"[Debug] #{idx} '{title}' Raw-Keys:", sorted([k for k in raw.keys() if k not in {"wikitext"}]))
|
||||||
|
payload = build_payload(raw, fullurl or "", category)
|
||||||
|
act = upsert_exercise(payload, dry_run=dry_run)
|
||||||
|
stats["created" if act=="create" else "updated" if act=="update" else "skipped"] += 1
|
||||||
|
except requests.HTTPError as e:
|
||||||
|
code = getattr(e, "response", None).status_code if getattr(e, "response", None) else None
|
||||||
|
if code == 404:
|
||||||
|
print(f"[Skip] '{title}': page not found (404)"); stats["failed"] += 1
|
||||||
|
else:
|
||||||
|
print(f"[Error] '{title}': {e}"); stats["failed"] += 1
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[Error] '{title}': {e}"); stats["failed"] += 1
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def run_smoke_test(title: str, category: str, *, debug_raw: bool = False) -> None:
|
||||||
|
print("\n[SmokeTest] Lauf 1/3: CREATE (Erstimport)"); act1 = process_one(title, category, mutate=False, debug_raw=debug_raw); print("[SmokeTest] Aktion:", act1)
|
||||||
|
print("\n[SmokeTest] Lauf 2/3: SKIP (Wiederholung, unverändert)"); act2 = process_one(title, category, mutate=False, debug_raw=debug_raw); print("[SmokeTest] Aktion:", act2)
|
||||||
|
print("\n[SmokeTest] Lauf 3/3: UPDATE (simulierte Wiki-Änderung an 'notes')"); act3 = process_one(title, category, mutate=True, debug_raw=debug_raw); print("[SmokeTest] Aktion:", act3)
|
||||||
|
print("\n[SmokeTest] Zusammenfassung:"); print(json.dumps({"run1": act1, "run2": act2, "run3": act3}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
# ----- Main -----
|
# ----- Main -----
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
parser = argparse.ArgumentParser(description="Import exercises from Wiki to Qdrant (via FastAPI wiki_router)")
|
parser = argparse.ArgumentParser(description="Import exercises from Wiki to Qdrant (via FastAPI wiki_router)")
|
||||||
parser.add_argument("--all", action="store_true", help="Alle Übungen importieren (SMW-Ask)")
|
parser.add_argument("--all", action="store_true", help="Alle Übungen importieren (SMW-Ask)")
|
||||||
@@ -186,13 +439,13 @@ def main() -> None:
|
|||||||
parser.add_argument("--username", type=str, default=os.getenv("WIKI_BOT_USER"), help="Wiki-Login Benutzer (überschreibt .env)")
|
parser.add_argument("--username", type=str, default=os.getenv("WIKI_BOT_USER"), help="Wiki-Login Benutzer (überschreibt .env)")
|
||||||
parser.add_argument("--password", type=str, default=os.getenv("WIKI_BOT_PASSWORD"), help="Wiki-Login Passwort (überschreibt .env)")
|
parser.add_argument("--password", type=str, default=os.getenv("WIKI_BOT_PASSWORD"), help="Wiki-Login Passwort (überschreibt .env)")
|
||||||
parser.add_argument("--skip-login", action="store_true", help="Login-Schritt überspringen (falls Session schon aktiv)")
|
parser.add_argument("--skip-login", action="store_true", help="Login-Schritt überspringen (falls Session schon aktiv)")
|
||||||
|
parser.add_argument("--dry-run", action="store_true", help="Kein Schreiben; nur Entscheidungen (create/update/skip) + Gründe loggen")
|
||||||
|
parser.add_argument("--smoke-test", action="store_true", help="3 Durchläufe (create→skip→update) für --title")
|
||||||
|
parser.add_argument("--debug-raw", action="store_true", help="Zeigt die aus dem Wiki gelesenen Roh-Keys je Seite")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
# Sanity
|
|
||||||
wiki_health()
|
wiki_health()
|
||||||
|
|
||||||
# Login (sofern nicht explizit übersprungen)
|
|
||||||
if not args.skip_login:
|
if not args.skip_login:
|
||||||
if not args.username or not args.password:
|
if not args.username or not args.password:
|
||||||
print("[Login] Fehler: fehlende Credentials. Setze .env (WIKI_BOT_USER/WIKI_BOT_PASSWORD) oder CLI --username/--password.", file=sys.stderr)
|
print("[Login] Fehler: fehlende Credentials. Setze .env (WIKI_BOT_USER/WIKI_BOT_PASSWORD) oder CLI --username/--password.", file=sys.stderr)
|
||||||
@@ -200,39 +453,18 @@ def main() -> None:
|
|||||||
try:
|
try:
|
||||||
wiki_login(args.username, args.password)
|
wiki_login(args.username, args.password)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(str(e), file=sys.stderr)
|
print(str(e), file=sys.stderr); sys.exit(1)
|
||||||
sys.exit(1)
|
|
||||||
|
if args.smoke_test:
|
||||||
|
run_smoke_test(args.title, args.category, debug_raw=args.debug_raw); return
|
||||||
|
|
||||||
# Einzel- oder Vollimport
|
|
||||||
if args.all:
|
if args.all:
|
||||||
print(f"[Main] Lade Liste der Übungen aus Kategorie '{args.category}'…")
|
stats = process_all(args.category, dry_run=args.dry_run, debug_raw=args.debug_raw)
|
||||||
pages = fetch_all_pages(args.category)
|
print("\n[Stats] created={created} updated={updated} skipped={skipped} failed={failed}".format(**stats))
|
||||||
print(f"[Main] {len(pages)} Seiten gefunden.")
|
|
||||||
for title, entry in pages.items():
|
|
||||||
pid = entry.get("pageid")
|
|
||||||
fullurl = entry.get("fullurl")
|
|
||||||
if not pid:
|
|
||||||
# Core-Info nachschlagen
|
|
||||||
info = fetch_page_info(title)
|
|
||||||
pid = info.get("pageid")
|
|
||||||
fullurl = fullurl or info.get("fullurl")
|
|
||||||
if not pid:
|
|
||||||
print(f"[Skip] '{title}' hat keine pageid")
|
|
||||||
continue
|
|
||||||
raw = parse_exercise(title, pid)
|
|
||||||
payload = build_payload(raw, fullurl or "", args.category)
|
|
||||||
ingest_exercise(payload)
|
|
||||||
else:
|
else:
|
||||||
print(f"[Main] Import single exercise: {args.title}")
|
print(f"[Main] Import single exercise: {args.title}")
|
||||||
info = fetch_page_info(args.title)
|
result = process_one(args.title, args.category, mutate=False, dry_run=args.dry_run, debug_raw=args.debug_raw)
|
||||||
pid = info.get("pageid")
|
print(f"[Result] {result}")
|
||||||
fullurl = info.get("fullurl") or ""
|
|
||||||
if not pid:
|
|
||||||
print(f"[Error] pageid für '{args.title}' nicht gefunden.", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
raw = parse_exercise(args.title, pid)
|
|
||||||
payload = build_payload(raw, fullurl, args.category)
|
|
||||||
ingest_exercise(payload)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
Reference in new issue
Block a user