init: add knowledge, llm-api, scripts + repo settings

This commit is contained in:
Lars committed 2025-08-10 08:06:10 +02:00
commit 75f13af1f1
64 files changed
+6924

No files matched your search

+34
View File
@@ -0,0 +1,34 @@
import re
def chunk_text_paragraphs(text: str, max_length: int = 500) -> list[str]:
# Absätze trennen
paragraphs = re.split(r'\n\s*\n', text.strip())
chunks: list[str] = []
current_chunk = ""
for para in paragraphs:
para = para.strip()
if not para:
continue
# Passt der Absatz noch zum aktuellen Chunk? +2 für die später hinzugefügten "\n\n"
if len(current_chunk) + len(para) + 2 <= max_length:
current_chunk = (current_chunk + "\n\n" + para) if current_chunk else para
else:
# Aktuellen Chunk abschließen
if current_chunk:
chunks.append(current_chunk)
# Ist der Absatz selbst zu groß? Dann hart splitten
if len(para) > max_length:
for i in range(0, len(para), max_length):
chunks.append(para[i:i + max_length])
current_chunk = ""
else:
current_chunk = para
# Letzten Chunk nicht vergessen
if current_chunk:
chunks.append(current_chunk)
return chunks
+93
View File
@@ -0,0 +1,93 @@
import os
import sys
import shutil
import requests
from tqdm import tqdm
# 📌 Konfiguration
API_URL = "http://localhost:8000/embed"
CHUNK_SIZE = 500
OVERLAP = 100
BATCH_SIZE = 20
# 📁 Kommandozeilenparameter auswerten
if len(sys.argv) != 2:
print("❌ Bitte gib eine Kategorie an, z. B.: python index_documents_advanced.py karatetrainer")
sys.exit(1)
CATEGORY = sys.argv[1]
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
ARCHIVE_DIR = os.path.join(SOURCE_DIR, "_imported")
COLLECTION = CATEGORY
if not os.path.exists(SOURCE_DIR):
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
sys.exit(1)
os.makedirs(ARCHIVE_DIR, exist_ok=True)
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
print(f"📂 Archivierte Dateien: {ARCHIVE_DIR}")
print(f"🎯 Ziel-Collection: {COLLECTION}")
# 🔧 Text in überlappende Chunks aufteilen
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
chunks = []
start = 0
while start < len(text):
end = min(start + size, len(text))
chunks.append(text[start:end])
start += size - overlap
return chunks
# 📚 Alle .txt-Dateien im Ordner lesen
def read_all_text_files(folder):
file_chunk_map = {} # Map: filename → chunks
for filename in os.listdir(folder):
if filename.endswith(".txt"):
path = os.path.join(folder, filename)
with open(path, "r", encoding="utf-8") as f:
text = f.read()
file_chunk_map[filename] = chunk_text(text)
return file_chunk_map
# 📤 An API senden
def embed_chunks_in_batches(chunks, collection):
results = []
for i in tqdm(range(0, len(chunks), BATCH_SIZE), desc="📡 Embedding"):
batch = chunks[i:i + BATCH_SIZE]
response = requests.post(API_URL, json={"texts": batch, "collection": collection})
response.raise_for_status()
results.append(response.json())
return results
# 🚀 Hauptlogik
if __name__ == "__main__":
file_chunk_map = read_all_text_files(SOURCE_DIR)
all_chunks = []
processed_files = []
for filename, chunks in file_chunk_map.items():
if chunks:
all_chunks.extend(chunks)
processed_files.append(filename)
if not all_chunks:
print("⚠️ Keine Textabschnitte gefunden.")
sys.exit(0)
print(f"📦 {len(all_chunks)} Textabschnitte aus {len(processed_files)} Dateien gefunden.")
try:
result = embed_chunks_in_batches(all_chunks, COLLECTION)
print(f"\n✅ Embedding abgeschlossen: {len(result)} API-Antwort(en) erhalten.")
# 🗃️ Verarbeitete Dateien archivieren
for filename in processed_files:
src = os.path.join(SOURCE_DIR, filename)
dst = os.path.join(ARCHIVE_DIR, filename)
shutil.move(src, dst)
print(f"📁 {len(processed_files)} Dateien verschoben nach _imported.")
except Exception as e:
print(f"❌ Fehler beim Senden: {e}")
+61
View File
@@ -0,0 +1,61 @@
import requests
import os
import sys
# 📌 Konfiguration
API_URL = "http://localhost:8000/embed"
CHUNK_SIZE = 500
OVERLAP = 100
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
"""
Teilt einen Text in überlappende Abschnitte auf.
"""
chunks = []
start = 0
while start < len(text):
end = min(start + size, len(text))
chunks.append(text[start:end])
start += size - overlap
return chunks
def read_text_file(path):
"""
Liest den Inhalt einer Textdatei ein.
"""
with open(path, "r", encoding="utf-8") as f:
return f.read()
def import_chunks(chunks, collection):
"""
Sendet die Textabschnitte an die API.
"""
response = requests.post(API_URL, json={
"texts": chunks,
"collection": collection
})
response.raise_for_status()
return response.json()
if __name__ == "__main__":
if len(sys.argv) != 3:
print("❌ Nutzung: python import_textfile.py <collection> <pfad_zur_txt_datei>")
sys.exit(1)
collection = sys.argv[1]
filepath = os.path.expanduser(sys.argv[2])
if not os.path.isfile(filepath):
print(f"❌ Datei nicht gefunden: {filepath}")
sys.exit(1)
print(f"📄 Lade Datei: {filepath}")
text = read_text_file(filepath)
chunks = chunk_text(text)
print(f"📦 {len(chunks)} Abschnitte vorbereitet – sende an Collection '{collection}'...")
try:
result = import_chunks(chunks, collection)
print(f"✅ Import erfolgreich: {result}")
except Exception as e:
print(f"❌ Fehler beim Import: {e}")
+31
View File
@@ -0,0 +1,31 @@
import requests
import sys
# 📌 Konfiguration
API_URL = "http://localhost:8000/embed"
def import_text(text, collection="default"):
"""
Sendet einen einzelnen Textabschnitt an die Embed-API zur Indexierung.
"""
response = requests.post(API_URL, json={
"texts": [text],
"collection": collection
})
response.raise_for_status()
return response.json()
if __name__ == "__main__":
if len(sys.argv) < 3:
print("❌ Nutzung: python import_texts.py <collection> \"<text>\"")
sys.exit(1)
collection = sys.argv[1]
text = sys.argv[2]
print(f"📤 Sende an Collection '{collection}': {text}")
try:
result = import_text(text, collection)
print(f"✅ Antwort: {result}")
except Exception as e:
print(f"❌ Fehler beim Importieren: {e}")
+66
View File
@@ -0,0 +1,66 @@
import os
import sys
import requests
# 📌 Konfiguration
API_URL = "http://localhost:8000/embed"
CHUNK_SIZE = 500
OVERLAP = 100
# 📁 Kommandozeilenparameter auswerten
if len(sys.argv) != 2:
print("❌ Bitte gib eine Kategorie an, z. B.: python index_documents.py karatetrainer")
sys.exit(1)
CATEGORY = sys.argv[1]
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
COLLECTION = CATEGORY
if not os.path.exists(SOURCE_DIR):
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
sys.exit(1)
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
print(f"🎯 Ziel-Collection: {COLLECTION}")
# 🔧 Text in überlappende Chunks aufteilen
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
chunks = []
start = 0
while start < len(text):
end = min(start + size, len(text))
chunks.append(text[start:end])
start += size - overlap
return chunks
# 📚 Alle .txt-Dateien im Ordner lesen
def read_all_text_files(folder):
chunks = []
for filename in os.listdir(folder):
if filename.endswith(".txt"):
path = os.path.join(folder, filename)
with open(path, "r", encoding="utf-8") as f:
text = f.read()
file_chunks = chunk_text(text)
chunks.extend(file_chunks)
return chunks
# 📤 An API senden
def embed_chunks(chunks, collection):
response = requests.post(API_URL, json={"texts": chunks, "collection": collection})
response.raise_for_status()
return response.json()
# 🚀 Hauptlogik
if __name__ == "__main__":
texts = read_all_text_files(SOURCE_DIR)
if not texts:
print("⚠️ Keine Textabschnitte gefunden.")
sys.exit(0)
print(f"📦 {len(texts)} Textabschnitte gefunden, sende an {API_URL}...")
try:
result = embed_chunks(texts, COLLECTION)
print(f"✅ Ergebnis: {result}")
except Exception as e:
print(f"❌ Fehler beim Senden: {e}")
+46
View File
@@ -0,0 +1,46 @@
import re
def chunk_text_paragraphs(text, max_length=500, overlap=1):
"""
Zerteilt den Text absatzweise in Chunks bis `max_length` Zeichen.
Optional wird `paragraph_overlap` Absatz(e) vom vorherigen Chunk übernommen.
"""
paragraphs = [p.strip() for p in re.split(r'\n{2,}', text) if p.strip()]
chunks = []
current_chunk = []
current_len = 0
i = 0
while i < len(paragraphs):
para = paragraphs[i]
para_len = len(para)
# Wenn dieser Absatz den max Chunk sprengt → neuen Chunk
if current_len + para_len + 2 > max_length: # +2 für Leerzeile
if current_chunk:
chunks.append("\n\n".join(current_chunk))
# Optional: letzte N Absätze behalten
if overlap > 0:
current_chunk = current_chunk[-overlap:]
current_len = sum(len(p) for p in current_chunk) + 2 * len(current_chunk)
else:
current_chunk = []
current_len = 0
else:
# Einzelner Absatz ist zu groß → hart splitten
chunks.append(para[:max_length])
para = para[max_length:]
paragraphs.insert(i + 1, para) # Rest zurück in Liste
i += 1
continue
else:
current_chunk.append(para)
current_len += para_len + 2 # +2 für Trennung
i += 1
if current_chunk:
chunks.append("\n\n".join(current_chunk))
return chunks
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# --------------------------------------------------
# chunking_utils.py
#
# Enthält robuste Text-Chunking-Logik:
# 1. Absatzbasiertes Chunking
# 2. Satzbasiertes Chunking per Regex (kein NLTK)
# 3. Satz-Overlap-Chunking
# --------------------------------------------------
import re
# --------------------------------------------------
# Hilfsfunktion: split_sentences
# Zweck:
# - Teilt Text in Sätze auf, basierend auf Punkt, Ausrufe- und Fragezeichen
# - Trennt bei ".!? " (Satzzeichen gefolgt von Leerraum)
# Parameter:
# text : Volltext als String
# Rückgabe:
# Liste von Satz-Strings
# --------------------------------------------------
def split_sentences(text: str) -> list[str]:
# Regex: lookbehind für . ! oder ?, dann ein oder mehrere Whitespace-Zeichen
return re.split(r'(?<=[\.!?])\s+', text.strip())
# --------------------------------------------------
# Funktion: chunk_text_paragraphs
# Zweck:
# - Trennt Text absatzweise in Chunks mit bis zu max_length Zeichen
# - Absätze werden an doppelten Zeilenumbrüchen getrennt
# - Zu große Absätze werden hart in max_length-Teile gesplittet
# Parameter:
# text : Volltext als String
# max_length : Maximale Länge eines Chunks (Standard 500)
# Rückgabe:
# Liste von Strings (Chunks)
# --------------------------------------------------
def chunk_text_paragraphs(text: str, max_length: int = 500) -> list[str]:
paragraphs = re.split(r'\n\s*\n', text.strip())
chunks: list[str] = []
current_chunk = ""
for para in paragraphs:
para = para.strip()
if not para:
continue
# Prüfen, ob Absatz noch in aktuellen Chunk passt (+2 für "\n\n")
if len(current_chunk) + len(para) + 2 <= max_length:
if current_chunk:
current_chunk += "\n\n" + para
else:
current_chunk = para
else:
# Bislang gesammelten Chunk speichern
if current_chunk:
chunks.append(current_chunk)
# Absatz hart splitten, wenn er allein zu groß ist
if len(para) > max_length:
for i in range(0, len(para), max_length):
part = para[i:i + max_length]
chunks.append(part)
current_chunk = ""
else:
# Neuer Chunk beginnt mit diesem Absatz
current_chunk = para
# Letzten Chunk nicht vergessen
if current_chunk:
chunks.append(current_chunk)
return chunks
# --------------------------------------------------
# Funktion: chunk_by_sentences
# Zweck:
# - Zerlegt Text in Sätze per Regex-Split
# - Baut daraus Chunks mit ganzen Sätzen bis max_length
# Parameter:
# text : Volltext als String
# max_length : Maximale Länge eines Chunks (Standard 500)
# Rückgabe:
# Liste von Strings (Chunks)
# --------------------------------------------------
def chunk_by_sentences(text: str, max_length: int = 500) -> list[str]:
sentences = split_sentences(text)
chunks: list[str] = []
current_chunk = ""
for sent in sentences:
sent = sent.strip()
if not sent:
continue
# Prüfen, ob Satz noch in aktuellen Chunk passt (+1 für Leerzeichen)
if len(current_chunk) + len(sent) + 1 <= max_length:
if current_chunk:
current_chunk += " " + sent
else:
current_chunk = sent
else:
# Bisher gesammelten Chunk speichern
if current_chunk:
chunks.append(current_chunk)
# Einzelnen Satz hart splitten, falls er zu lang ist
if len(sent) > max_length:
for i in range(0, len(sent), max_length):
chunks.append(sent[i:i + max_length])
current_chunk = ""
else:
current_chunk = sent
# Letzten Chunk nicht vergessen
if current_chunk:
chunks.append(current_chunk)
return chunks
# --------------------------------------------------
# Funktion: chunk_with_sentence_overlap
# Zweck:
# - Baut zunächst sentence-basierte Chunks
# - Fügt vom vorherigen Chunk overlap_sents Sätze vorne an
# Parameter:
# text : Volltext als String
# max_length : Maximale Länge eines Chunks (Standard 500)
# overlap_sents : Anzahl Sätze, die überlappend übernommen werden (Standard 1)
# Rückgabe:
# Liste von Strings (Chunks mit Kontext-Overlap)
# --------------------------------------------------
def chunk_with_sentence_overlap(
text: str,
max_length: int = 500,
overlap_sents: int = 1
) -> list[str]:
base_chunks = chunk_by_sentences(text, max_length)
overlapped: list[str] = []
for idx, chunk in enumerate(base_chunks):
if idx == 0 or overlap_sents <= 0:
overlapped.append(chunk)
else:
prev = base_chunks[idx - 1]
prev_sents = split_sentences(prev)
context = " ".join(prev_sents[-overlap_sents:])
overlapped.append((context + " " + chunk).strip())
return overlapped
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import sys
import subprocess
from tqdm import tqdm
def main():
import argparse
parser = argparse.ArgumentParser(
description="Importiert alle .txt-Dateien in einer Kategorie mittels import_single_file.py"
)
parser.add_argument(
"category",
help="Name der Kategorie (Ordner unter ~/knowledge/)"
)
parser.add_argument(
"--topic", "-t",
help="Optionaler Kontext/Topic für alle Importe",
default="default"
)
parser.add_argument(
"--script-path", "-s",
help="Pfad zum import_single_file.py",
default="import_single_file.py"
)
args = parser.parse_args()
category = args.category
topic = args.topic
script_path = args.script_path
base_dir = os.path.expanduser(f"~/knowledge/{category}")
if not os.path.isdir(base_dir):
print(f"❌ Kategorie-Ordner nicht gefunden: {base_dir}")
sys.exit(1)
files = [f for f in os.listdir(base_dir) if f.endswith(".txt")]
if not files:
print(f"⚠️ Keine .txt-Dateien in {base_dir} gefunden.")
sys.exit(0)
print(f"📁 Starte Ordner-Import: {len(files)} Dateien in Kategorie '{category}' (Topic: '{topic}')")
for filename in tqdm(files, desc="Importiere Dateien"):
cmd = [
sys.executable,
script_path,
category,
filename,
topic
]
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
print(f"❌ Fehler beim Import von {filename}: {e}")
print("✅ Ordner-Import abgeschlossen.")
if __name__ == "__main__":
main()
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
import_pdf_file.py
Importiert eine einzelne PDF-Datei:
- Text-Extraktion via pdfplumber
- Chunking mit Satz-Overlap
- POST an /embed
- Verschieben nach _imported
"""
import os
import sys
import shutil
import requests
from uuid import uuid4
from datetime import datetime, timezone
import pdfplumber
from chunking_utils import chunk_with_sentence_overlap
# Konfiguration
API_URL = "http://localhost:8000/embed"
MAX_LENGTH = 500
if len(sys.argv) < 3:
print("? Usage: python import_pdf_file.py <category> <filename> [topic]")
sys.exit(1)
category = sys.argv[1]
filename = sys.argv[2]
topic = sys.argv[3] if len(sys.argv) > 3 else "default"
source_dir = os.path.expanduser(f"~/knowledge/{category}")
file_path = os.path.join(source_dir, filename)
archive_dir= os.path.join(source_dir, "_imported")
if not os.path.isfile(file_path):
print(f"? Datei nicht gefunden: {file_path}")
sys.exit(1)
os.makedirs(archive_dir, exist_ok=True)
print(f"?? PDF-Import: {filename} in Kategorie {category} (Topic={topic})")
# 1) PDF-Text extrahieren
text = ""
with pdfplumber.open(file_path) as pdf:
for page in pdf.pages:
page_text = page.extract_text() or ""
text += page_text + "\n\n"
# 2) Chunking
chunks = chunk_with_sentence_overlap(
text,
max_length=MAX_LENGTH,
overlap_sents=2
)
print(f"?? {len(chunks)} Chunks erzeugt.")
# 3) Payload bauen
now = datetime.now(timezone.utc).isoformat()
payload = {"collection": category, "chunks": []}
for idx, chunk in enumerate(chunks):
payload["chunks"].append({
"text": chunk,
"source": filename,
"source_type": "pdf",
"title": os.path.splitext(filename)[0],
"version": "v1.0",
"related_to": category,
"tags": [category],
"owner": "karate-agent",
"context_tag": topic,
"imported_at": now,
"chunk_index": idx,
"category": category
})
# 4) Senden
try:
res = requests.post(API_URL, json=payload)
res.raise_for_status()
print(f"? {len(chunks)} Chunks erfolgreich eingebettet.")
except Exception as e:
print(f"? Fehler beim Senden: {e}")
sys.exit(1)
# 5) Archivieren
shutil.move(file_path, os.path.join(archive_dir, filename))
print("?? PDF verschoben nach _imported/")
+85
View File
@@ -0,0 +1,85 @@
import os
import sys
import shutil
import requests
from datetime import datetime, timezone
#from text_chunker import chunk_text_paragraphs
from uuid import uuid4
from chunking_utils import (
chunk_text_paragraphs,
chunk_by_sentences,
chunk_with_sentence_overlap
)
# ?? Konfiguration
API_URL = "http://localhost:8000/embed"
CHUNK_SIZE = 500
OVERLAP = 100
# ?? Eingabeparameter: Kategorie, Dateiname, optionale Metadaten
if len(sys.argv) < 3:
print("? Aufruf: python import_single_file.py <category> <filename> [topic]")
sys.exit(1)
CATEGORY = sys.argv[1]
FILENAME = sys.argv[2]
TOPIC = sys.argv[3] if len(sys.argv) > 3 else None
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
ARCHIVE_DIR = os.path.join(SOURCE_DIR, "_imported")
FILEPATH = os.path.join(SOURCE_DIR, FILENAME)
# ?? Validierung
if not os.path.exists(FILEPATH):
print(f"? Datei nicht gefunden: {FILEPATH}")
sys.exit(1)
os.makedirs(ARCHIVE_DIR, exist_ok=True)
print(f"?? Importiere Datei: {FILENAME} aus Kategorie: {CATEGORY}")
# ?? Inhalte lesen und in Chunks zerteilen
with open(FILEPATH, "r", encoding="utf-8") as f:
content = f.read()
chunks = chunk_with_sentence_overlap(content, max_length=CHUNK_SIZE, overlap_sents=2)
print(f"?? {len(chunks)} Textabschnitte erzeugt.")
# ?? Metadaten vorbereiten
now = datetime.now(timezone.utc).isoformat()
payload = {
"chunks": [],
"collection": CATEGORY
}
for i, chunk in enumerate(chunks):
payload["chunks"].append({
"text": chunk,
"source": FILENAME,
"source_type": "file",
"title": FILENAME.replace(".txt", ""),
"version": "v1.0",
"related_to": CATEGORY,
"tags": [CATEGORY],
"owner": "karate-agent",
"context_tag": TOPIC or "default",
"imported_at": now,
"chunk_index": i,
"category": CATEGORY
})
# ?? An API senden
try:
res = requests.post(API_URL, json=payload)
res.raise_for_status()
print(f"? {len(chunks)} Abschnitte erfolgreich eingebettet.")
except Exception as e:
print(f"? Fehler beim Senden: {e}")
sys.exit(1)
# ??? Datei archivieren
shutil.move(FILEPATH, os.path.join(ARCHIVE_DIR, FILENAME))
print(f"?? Datei nach _imported verschoben.")
+135
View File
@@ -0,0 +1,135 @@
import os
import sys
import shutil
import requests
import re
from tqdm import tqdm
from datetime import datetime
# 📌 Konfiguration
API_URL = "http://localhost:8000/embed"
CHUNK_SIZE = 500
OVERLAP = 100
BATCH_SIZE = 20
# 📁 Kommandozeilenparameter auswerten
if len(sys.argv) != 2:
print("❌ Bitte gib eine Kategorie an, z. B.: python import_txtdocuments.py karatetrainer")
sys.exit(1)
CATEGORY = sys.argv[1]
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
ARCHIVE_DIR = os.path.join(SOURCE_DIR, "_imported")
COLLECTION = CATEGORY
if not os.path.exists(SOURCE_DIR):
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
sys.exit(1)
os.makedirs(ARCHIVE_DIR, exist_ok=True)
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
print(f"📂 Archivierte Dateien: {ARCHIVE_DIR}")
print(f"🎯 Ziel-Collection: {COLLECTION}")
# 🔧 Text in überlappende Chunks aufteilen
#def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
# chunks = []
# start = 0
# while start < len(text):
# end = min(start + size, len(text))
# chunks.append(text[start:end])
# start += size - overlap
# return chunks
def chunk_text_paragraphs(text, max_length=500):
paragraphs = re.split(r'\n\s*\n', text.strip()) # Absatztrennung
chunks = []
current_chunk = ""
for para in paragraphs:
if len(current_chunk) + len(para) + 2 <= max_length:
current_chunk += para + "\n\n"
else:
if current_chunk:
chunks.append(current_chunk.strip())
# Falls einzelner Absatz zu groß ist, hart splitten
if len(para) > max_length:
for i in range(0, len(para), max_length):
chunks.append(para[i:i+max_length].strip())
current_chunk = ""
else:
current_chunk = para + "\n\n"
if current_chunk:
chunks.append(current_chunk.strip())
return chunks
# 📚 Alle .txt-Dateien im Ordner lesen
def read_all_text_files(folder):
file_chunk_map = {} # Map: filename → chunks
for filename in os.listdir(folder):
if filename.endswith(".txt"):
path = os.path.join(folder, filename)
with open(path, "r", encoding="utf-8") as f:
text = f.read()
file_chunk_map[filename] = chunk_text_paragraphs(text)
return file_chunk_map
# 🧱 Strukturierte Payloads vorbereiten
def prepare_payloads(file_chunk_map, collection):
payloads = []
imported_at = datetime.now().isoformat()
for filename, chunks in file_chunk_map.items():
for local_index, chunk in enumerate(chunks):
payload = {
"text": chunk,
"source": filename,
"type": "file",
"category": collection,
"imported_at": imported_at,
"chunk_index": local_index
}
payloads.append(payload)
for p in payloads:
print(f"{p['source']}: chunk_index={p['chunk_index']}")
return payloads
# 📤 An API senden
def embed_chunks_in_batches(payloads, collection):
results = []
for i in tqdm(range(0, len(payloads), BATCH_SIZE), desc="📡 Embedding"):
batch = payloads[i:i + BATCH_SIZE]
response = requests.post(API_URL, json={"chunks": batch, "collection": collection})
response.raise_for_status()
results.append(response.json())
return results
# 🚀 Hauptlogik
if __name__ == "__main__":
file_chunk_map = read_all_text_files(SOURCE_DIR)
processed_files = list(file_chunk_map.keys())
payloads = prepare_payloads(file_chunk_map, COLLECTION)
if not payloads:
print("⚠️ Keine Textabschnitte gefunden.")
sys.exit(0)
print(f"📦 {len(payloads)} Textabschnitte aus {len(processed_files)} Dateien gefunden. Sende an API...")
try:
result = embed_chunks_in_batches(payloads, COLLECTION)
print(f"\n✅ Embedding abgeschlossen: {len(result)} API-Antwort(en) erhalten.")
# 🗃️ Verarbeitete Dateien archivieren
for filename in processed_files:
src = os.path.join(SOURCE_DIR, filename)
dst = os.path.join(ARCHIVE_DIR, filename)
shutil.move(src, dst)
print(f"📁 {len(processed_files)} Dateien verschoben nach _imported.")
except Exception as e:
print(f"❌ Fehler beim Senden: {e}")
+32
View File
@@ -0,0 +1,32 @@
import sys
import requests
API_URL = "http://localhost:8000/prompt"
if len(sys.argv) < 3:
print("❌ Verwendung: python prompt_documents.py <collection> <frage>")
sys.exit(1)
collection = sys.argv[1]
query = " ".join(sys.argv[2:])
data = {
"query": query,
"collection": collection,
"context_limit": 3
}
print(f"🤖 Anfrage an LLM aus Collection '{collection}': {query}")
try:
response = requests.post(API_URL, json=data)
response.raise_for_status()
result = response.json()
except Exception as e:
print(f"❌ Fehler bei der Anfrage: {e}")
sys.exit(1)
print("\n📄 Kontext:")
print(result["context"])
print("\n💡 Antwort:")
print(result["answer"])
+45
View File
@@ -0,0 +1,45 @@
import os
import sys
import shutil
def print_usage():
print("❌ Bitte gib eine Kategorie an, z. B.:")
print(" python restore_imported_files.py karatetrainer")
sys.exit(1)
# --- Eingabe prüfen ---
if len(sys.argv) < 2:
print_usage()
CATEGORY = sys.argv[1]
FORCE = "--force" in sys.argv
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}/_imported")
TARGET_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
if not os.path.isdir(SOURCE_DIR):
print(f"❌ Quellordner '{SOURCE_DIR}' existiert nicht.")
sys.exit(1)
files = [f for f in os.listdir(SOURCE_DIR) if os.path.isfile(os.path.join(SOURCE_DIR, f))]
if not files:
print("⚠️ Keine Dateien zum Wiederherstellen gefunden.")
sys.exit(0)
print(f"♻️ Wiederherstellung von {len(files)} Dateien nach '{TARGET_DIR}'")
restored = 0
for file in files:
source_path = os.path.join(SOURCE_DIR, file)
target_path = os.path.join(TARGET_DIR, file)
if not FORCE:
confirm = input(f"🔁 Datei '{file}' zurückkopieren? [j/N] ").strip().lower()
if confirm != "j":
continue
shutil.move(source_path, target_path)
print(f"✅ '{file}' wurde zurückkopiert.")
restored += 1
print(f"\n🎉 {restored} Datei(en) wurden erfolgreich wiederhergestellt.")
+36
View File
@@ -0,0 +1,36 @@
import os
import sys
import shutil
def print_usage():
print("❌ Bitte gib Kategorie und Dateinamen an, z. B.:")
print(" python restore_single_file.py karatetrainer mae_geri.txt")
sys.exit(1)
# Eingabe prüfen
if len(sys.argv) != 3:
print_usage()
CATEGORY = sys.argv[1]
FILENAME = sys.argv[2]
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}/_imported")
TARGET_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
SOURCE_FILE = os.path.join(SOURCE_DIR, FILENAME)
TARGET_FILE = os.path.join(TARGET_DIR, FILENAME)
if not os.path.isfile(SOURCE_FILE):
print(f"❌ Die Datei '{FILENAME}' wurde im Archivordner '{SOURCE_DIR}' nicht gefunden.")
sys.exit(1)
if os.path.exists(TARGET_FILE):
confirm = input(f"⚠️ Datei '{FILENAME}' existiert im Zielordner. Überschreiben? [j/N] ").strip().lower()
if confirm != "j":
print("⏹️ Abgebrochen.")
sys.exit(0)
try:
shutil.move(SOURCE_FILE, TARGET_FILE)
print(f"✅ '{FILENAME}' wurde zurück nach '{TARGET_DIR}' verschoben.")
except Exception as e:
print(f"❌ Fehler beim Verschieben: {e}")
+31
View File
@@ -0,0 +1,31 @@
import sys
import requests
API_URL = "http://localhost:8000/search"
if len(sys.argv) < 3:
print("❌ Verwendung: python search_documents.py <collection> <suchfrage>")
sys.exit(1)
collection = sys.argv[1]
query = " ".join(sys.argv[2:])
params = {
"query": query,
"collection": collection,
"limit": 5
}
print(f"🔍 Suche in Collection '{collection}': {query}")
try:
response = requests.get(API_URL, params=params)
response.raise_for_status()
results = response.json()
except Exception as e:
print(f"❌ Fehler bei der Anfrage: {e}")
sys.exit(1)
print("📚 Ergebnisse:")
for r in results:
print(f"\n✅ Score: {r['score']:.4f}")
print(f"{r['text']}")
+238
View File
@@ -0,0 +1,238 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Module: wiki_importer.py
Beschreibung:
- Importiert Übungen aus dem MediaWiki via FastAPI wiki_router
- Führt vor dem Import einen Login gegen /import/wiki/login durch (falls nicht via --skip-login deaktiviert)
- Holt Liste aller Übungs-Titel (SMW-Ask) via `/semantic/pages`
- Für jede Übung:
* Fetch pageinfo (pageid, fullurl) via `/info`
* Parse Wikitext (Templates: ÜbungInfoBox, Übungsbeschreibung, SkillDevelopment) via `/parsepage`
* Baut Payload entsprechend Exercise-Datenmodell
* POST an `/exercise` Endpoint (exercise_router)
- Unterstützt Single-Import via `--title` (oder ENV `WIKI_EXERCISE_TITLE`) und Full-Import via `--all`
- Optional: Credentials via CLI (--username/--password) oder `.env` (WIKI_BOT_USER / WIKI_BOT_PASSWORD)
Version: 2.1.0
"""
import os
import sys
import argparse
from typing import Dict, Any
import requests
import mwparserfromhell
from dotenv import load_dotenv
# ----- Konfiguration / Defaults -----
load_dotenv() # .env laden, falls vorhanden
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki") # FastAPI-Wiki-Proxy
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise") # Exercise-Endpoint
DEFAULT_CAT = os.getenv("WIKI_CATEGORY", "Übungen")
DEFAULT_TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
# ---- Hilfsfunktionen für Wiki-Router ----
def wiki_health() -> None:
r = requests.get(f"{API_BASE_URL}/health", timeout=15)
r.raise_for_status()
print("[Sanity] Wiki health OK")
def wiki_login(username: str, password: str) -> None:
"""
Führt einen Login gegen den wiki_router durch.
Erwartet: {"status":"success"} bei Erfolg.
"""
payload = {"username": username, "password": password}
r = requests.post(f"{API_BASE_URL}/login", json=payload, timeout=30)
# kein raise_for_status(), wir wollen die JSON-Fehler sauber ausgeben
try:
data = r.json()
except Exception:
print(f"[Login] HTTP {r.status_code}: {r.text}")
r.raise_for_status()
status = (data or {}).get("status")
if status != "success":
msg = (data or {}).get("message", "Login fehlgeschlagen")
raise RuntimeError(f"[Login] {msg}")
print("[Login] success")
def fetch_all_pages(category: str) -> Dict[str, Any]:
resp = requests.get(f"{API_BASE_URL}/semantic/pages", params={"category": category}, timeout=60)
resp.raise_for_status()
return resp.json()
def fetch_page_info(title: str) -> Dict[str, Any]:
r = requests.get(f"{API_BASE_URL}/info", params={"title": title}, timeout=30)
r.raise_for_status()
info = r.json()
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
print(f"[Parse] Lade '{title}' (ID={pageid})")
resp = requests.get(
f"{API_BASE_URL}/parsepage",
params={"pageid": pageid, "title": title},
timeout=60
)
resp.raise_for_status()
wikitext = resp.json().get("wikitext", "")
wikicode = mwparserfromhell.parse(wikitext)
raw: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid}
for tpl in wikicode.filter_templates():
name = str(tpl.name).strip()
if name == "ÜbungInfoBox":
for p in tpl.params:
raw[str(p.name).strip()] = str(p.value).strip()
elif name == "Übungsbeschreibung":
for p in tpl.params:
raw[str(p.name).strip()] = str(p.value).strip()
elif name == "SkillDevelopment":
raw.setdefault("capabilities", [])
try:
cap = str(tpl.get("PrimaryCapability").value).strip()
except Exception:
cap = ""
try:
lvl = int(str(tpl.get("CapabilityLevel").value).strip())
except Exception:
lvl = 0
if cap:
raw["capabilities"].append({"capability": cap, "level": lvl})
raw["wikitext"] = wikitext
return raw
def build_payload(raw: Dict[str, Any], fullurl: str, category: str) -> Dict[str, Any]:
# Exercise.capabilities erwartet Dict[str,int]
caps_list = raw.get("capabilities", [])
capabilities = {}
for c in caps_list:
cap = c.get("capability")
lvl = c.get("level")
if isinstance(cap, str) and cap:
try:
capabilities[cap] = int(lvl)
except Exception:
pass
# Defaults/Fallbacks
duration = 0.0
try:
duration = float(raw.get("Dauer", 0) or 0)
except Exception:
duration = 0.0
keywords = []
kw_raw = raw.get("Schlüsselworte", "")
if isinstance(kw_raw, str):
keywords = [k.strip() for k in kw_raw.split(",") if k.strip()]
equipment = []
eq_raw = raw.get("equipment", [])
if isinstance(eq_raw, str):
equipment = [e.strip() for e in eq_raw.split(",") if e.strip()]
elif isinstance(eq_raw, list):
equipment = [str(e).strip() for e in eq_raw if str(e).strip()]
payload: Dict[str, Any] = {
"title": raw.get("title") or "",
"summary": raw.get("Summary", "") or "",
"short_description": raw.get("Summary", "") or "",
"keywords": keywords,
"link": fullurl or "",
"discipline": raw.get("Übungstyp", "") or "",
"group": raw.get("Gruppengröße", "") or None,
"age_group": raw.get("Altersgruppe", "") or "",
"target_group": raw.get("Zielgruppe", "") or "",
"min_participants": 1,
"duration_minutes": int(round(duration)), # Exercise erwartet int
"capabilities": capabilities,
"category": category or "",
"purpose": raw.get("Ziel", "") or "",
"execution": raw.get("Durchführung", "") or "",
"notes": raw.get("Hinweise", "") or "",
"preparation": raw.get("RefMethode", "") or "",
"method": raw.get("method", "") or "", # falls im Wikitext vorhanden
"equipment": equipment,
"fullurl": fullurl or "", # optionales Feld
# Idempotenz (optional nutzbar in exercise_router):
"external_id": f"wiki:{raw.get('pageid')}",
"source": "MediaWiki"
}
return payload
def ingest_exercise(payload: Dict[str, Any]) -> None:
title = payload.get("title", "<ohne Titel>")
resp = requests.post(EXERCISE_API, json=payload, timeout=60)
if resp.status_code == 422:
print(f"[Ingest] '{title}' -> FAILED 422:\n{resp.text}")
try:
resp.raise_for_status()
except Exception:
pass
return
resp.raise_for_status()
print(f"[Ingest] '{title}' -> OK")
# ----- Main -----
def main() -> None:
parser = argparse.ArgumentParser(description="Import exercises from Wiki to Qdrant (via FastAPI wiki_router)")
parser.add_argument("--all", action="store_true", help="Alle Übungen importieren (SMW-Ask)")
parser.add_argument("--title", type=str, default=DEFAULT_TITLE, help="Einzelimport eines Übungstitels")
parser.add_argument("--category", type=str, default=DEFAULT_CAT, help="Wiki-Kategorie (z.B. 'Übungen')")
parser.add_argument("--username", type=str, default=os.getenv("WIKI_BOT_USER"), help="Wiki-Login Benutzer (überschreibt .env)")
parser.add_argument("--password", type=str, default=os.getenv("WIKI_BOT_PASSWORD"), help="Wiki-Login Passwort (überschreibt .env)")
parser.add_argument("--skip-login", action="store_true", help="Login-Schritt überspringen (falls Session schon aktiv)")
args = parser.parse_args()
# Sanity
wiki_health()
# Login (sofern nicht explizit übersprungen)
if not args.skip_login:
if not args.username or not args.password:
print("[Login] Fehler: fehlende Credentials. Setze .env (WIKI_BOT_USER/WIKI_BOT_PASSWORD) oder CLI --username/--password.", file=sys.stderr)
sys.exit(1)
try:
wiki_login(args.username, args.password)
except Exception as e:
print(str(e), file=sys.stderr)
sys.exit(1)
# Einzel- oder Vollimport
if args.all:
print(f"[Main] Lade Liste der Übungen aus Kategorie '{args.category}'…")
pages = fetch_all_pages(args.category)
print(f"[Main] {len(pages)} Seiten gefunden.")
for title, entry in pages.items():
pid = entry.get("pageid")
fullurl = entry.get("fullurl")
if not pid:
# Core-Info nachschlagen
info = fetch_page_info(title)
pid = info.get("pageid")
fullurl = fullurl or info.get("fullurl")
if not pid:
print(f"[Skip] '{title}' hat keine pageid")
continue
raw = parse_exercise(title, pid)
payload = build_payload(raw, fullurl or "", args.category)
ingest_exercise(payload)
else:
print(f"[Main] Import single exercise: {args.title}")
info = fetch_page_info(args.title)
pid = info.get("pageid")
fullurl = info.get("fullurl") or ""
if not pid:
print(f"[Error] pageid für '{args.title}' nicht gefunden.", file=sys.stderr)
sys.exit(1)
raw = parse_exercise(args.title, pid)
payload = build_payload(raw, fullurl, args.category)
ingest_exercise(payload)
if __name__ == "__main__":
main()
+117
View File
@@ -0,0 +1,117 @@
"""
Module: wiki_importer.py
Beschreibung:
- Importiert zunächst nur eine Übung aus dem Wiki
- Liest Wikitext einer Übung aus
- Parsen mit mwparserfromhell
- Extrahiert Felder aus Templates:
* ÜbungInfoBox
* Übungsbeschreibung
* SkillDevelopment (mehrfach)
- Baut ein Exercise-Objekt zusammen
- Speichert per POST /exercise Endpoint in Qdrant
- Detailliertes Error-Logging für 422 und allgemeine Fehler
Version: 1.1.2
"""
import requests
import mwparserfromhell
import os
import sys
from typing import Dict, Any
# Konfiguration über Umgebungsvariablen
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki")
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise")
# Übungstitel, der importiert werden soll
TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
# Helper: Holt pageid und fullurl per Core-API
def fetch_page_info(title: str) -> Dict[str, Any]:
r = requests.get(f"{API_BASE_URL}/info", params={"title": title})
r.raise_for_status()
info = r.json()
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
# Parser: Lädt und parst eine Übung
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
print(f"[Parse] Loading '{title}' (ID={pageid})")
resp = requests.get(f"{API_BASE_URL}/parsepage", params={"pageid": pageid, "title": title})
resp.raise_for_status()
wikitext = resp.json().get("wikitext", "")
wikicode = mwparserfromhell.parse(wikitext)
data: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid}
for tpl in wikicode.filter_templates():
name = tpl.name.strip()
if name == "ÜbungInfoBox":
for param in tpl.params:
data[param.name.strip()] = str(param.value).strip()
elif name == "Übungsbeschreibung":
for param in tpl.params:
data[param.name.strip()] = str(param.value).strip()
elif name == "SkillDevelopment":
data.setdefault("capabilities", [])
primary = str(tpl.get("PrimaryCapability").value).strip()
level = int(str(tpl.get("CapabilityLevel").value).strip())
data["capabilities"].append({"capability": primary, "level": level})
data["wikitext"] = wikitext
return data
# Ingestion: Sendet einen Datensatz an Qdrant mit detailliertem Error-Logging
def ingest_exercise(ex_data: Dict[str, Any]) -> None:
title = ex_data.get("title")
try:
resp = requests.post(EXERCISE_API, json=ex_data)
if resp.status_code == 422:
print(f"[Ingest] '{title}' -> FAILED 422:")
print(resp.text)
resp.raise_for_status()
print(f"[Ingest] '{title}' -> OK")
except requests.HTTPError as e:
msg = resp.text if 'resp' in locals() else str(e)
print(f"[Ingest] '{title}' -> HTTPError: {e} - {msg}")
except Exception as e:
print(f"[Ingest] '{title}' -> FAILED: {e}")
# Main: Einmaliger Import für TITLE
if __name__ == "__main__":
print(f"[Main] Import single exercise: {TITLE}")
try:
info = fetch_page_info(TITLE)
pageid = info.get("pageid")
fullurl = info.get("fullurl")
if not pageid:
print(f"Error: pageid für '{TITLE}' nicht gefunden.")
sys.exit(1)
raw = parse_exercise(TITLE, pageid)
# capabilities als Dict wandeln
caps_list = raw.get("capabilities", [])
capabilities = {c["capability"]: c["level"] for c in caps_list}
# Payload entsprechend Datenmodell
exercise_payload = {
"title": raw.get("title"),
"summary": raw.get("Summary", ""),
"short_description": raw.get("Summary", ""),
"keywords": raw.get("Schlüsselworte", "").split(', '),
"link": fullurl,
"discipline": raw.get("Übungstyp", ""),
"group": raw.get("Gruppengröße", ""),
"age_group": raw.get("Altersgruppe", ""),
"target_group": raw.get("Zielgruppe", ""),
"min_participants": 1,
"duration_minutes": float(raw.get("Dauer", 0)),
"capabilities": capabilities,
"category": raw.get("category", "Übungen"),
"purpose": raw.get("Ziel", ""),
"execution": raw.get("Durchführung", ""),
"notes": raw.get("Hinweise", ""),
"preparation": raw.get("RefMethode", ""),
"method": raw.get("method", ""),
"equipment": raw.get("equipment", []),
"fullurl": fullurl
}
ingest_exercise(exercise_payload)
except Exception as e:
print(f"Fatal error: {e}")
sys.exit(1)
+136
View File
@@ -0,0 +1,136 @@
"""
Module: wiki_importer.py
Beschreibung:
- Importiert alle Übungen aus dem Wiki
- Holt Liste aller Übungs-Titel und pageids via `/semantic/pages`
- Für jede Übung:
* Fetch pageinfo (pageid, fullurl)
* Parse Wikitext (Templates: ÜbungInfoBox, Übungsbeschreibung, SkillDevelopment)
* Baut Payload entsprechend Datenmodell
* POST an `/exercise` Endpoint
- Unterstützt Single-Import via Umgebungsvariable `WIKI_EXERCISE_TITLE` und Full-Import via `--all`
Version: 2.0.0
"""
import requests
import mwparserfromhell
import os
import sys
from typing import Dict, Any, List
import argparse
# Konfiguration
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki")
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise")
DEFAULT_CATEGORY = os.getenv("WIKI_CATEGORY", "Übungen")
DEFAULT_TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
# Helper: Holt Liste aller Übungen (Titel-> entry)
def fetch_all_pages(category: str) -> Dict[str, Any]:
resp = requests.get(f"{API_BASE_URL}/semantic/pages", params={"category": category})
resp.raise_for_status()
return resp.json()
# Helper: Holt pageid und fullurl per Core-API
def fetch_page_info(title: str) -> Dict[str, Any]:
r = requests.get(f"{API_BASE_URL}/info", params={"title": title})
r.raise_for_status()
info = r.json()
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
# Parser: Lädt und parst eine Übung
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
print(f"[Parse] {title} (ID={pageid})")
resp = requests.get(f"{API_BASE_URL}/parsepage", params={"pageid": pageid, "title": title})
resp.raise_for_status()
wikitext = resp.json().get("wikitext", "")
wikicode = mwparserfromhell.parse(wikitext)
raw: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid}
for tpl in wikicode.filter_templates():
name = tpl.name.strip()
if name == "ÜbungInfoBox":
for p in tpl.params:
raw[p.name.strip()] = str(p.value).strip()
elif name == "Übungsbeschreibung":
for p in tpl.params:
raw[p.name.strip()] = str(p.value).strip()
elif name == "SkillDevelopment":
raw.setdefault("capabilities", [])
cap = str(tpl.get("PrimaryCapability").value).strip()
lvl = int(str(tpl.get("CapabilityLevel").value).strip())
raw["capabilities"].append({"capability": cap, "level": lvl})
raw["wikitext"] = wikitext
return raw
# Ingestion
def ingest_exercise(payload: Dict[str, Any]) -> None:
title = payload.get("title")
resp = requests.post(EXERCISE_API, json=payload)
if resp.status_code == 422:
print(f"[Error] {title} -> 422: {resp.text}")
return
resp.raise_for_status()
print(f"[Ingest] {title} -> OK")
# Build payload
def build_payload(raw: Dict[str, Any], fullurl: str, category: str) -> Dict[str, Any]:
caps_list = raw.get("capabilities", [])
capabilities = {c["capability"]: c["level"] for c in caps_list}
return {
"title": raw.get("title"),
"summary": raw.get("Summary", ""),
"short_description": raw.get("Summary", ""),
"keywords": raw.get("Schlüsselworte", "").split(', '),
"link": fullurl,
"discipline": raw.get("Übungstyp", ""),
"group": raw.get("Gruppengröße", ""),
"age_group": raw.get("Altersgruppe", ""),
"target_group": raw.get("Zielgruppe", ""),
"min_participants": 1,
"duration_minutes": float(raw.get("Dauer", 0)),
"capabilities": capabilities,
"category": category,
"purpose": raw.get("Ziel", ""),
"execution": raw.get("Durchführung", ""),
"notes": raw.get("Hinweise", ""),
"preparation": raw.get("RefMethode", ""),
"method": raw.get("method", ""),
"equipment": raw.get("equipment", []),
"fullurl": fullurl
}
# Main
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Import exercises from Wiki to Qdrant")
parser.add_argument("--all", action="store_true", help="Import all exercises")
parser.add_argument("--title", type=str, default=DEFAULT_TITLE, help="Single exercise title")
parser.add_argument("--category", type=str, default=DEFAULT_CATEGORY, help="Wiki category")
args = parser.parse_args()
if args.all:
pages = fetch_all_pages(args.category)
print(f"Found {len(pages)} exercises in category '{args.category}'")
for title, entry in pages.items():
pid = entry.get("pageid")
if not pid:
info = fetch_page_info(title)
pid = info.get("pageid")
fullurl = info.get("fullurl")
else:
fullurl = entry.get("fullurl") or fetch_page_info(title)["fullurl"]
if not pid:
print(f"Skip {title}, no pageid")
continue
raw = parse_exercise(title, pid)
payload = build_payload(raw, fullurl, args.category)
ingest_exercise(payload)
else:
info = fetch_page_info(args.title)
pid = info.get("pageid")
fullurl = info.get("fullurl")
if not pid:
print(f"Error: pageid for '{args.title}' not found")
sys.exit(1)
raw = parse_exercise(args.title, pid)
payload = build_payload(raw, fullurl, args.category)
ingest_exercise(payload)