init: add knowledge, llm-api, scripts + repo settings
This commit is contained in:
commit
75f13af1f1
64 files changed
+6924
No files matched your search
@@ -0,0 +1,34 @@
|
||||
import re
|
||||
|
||||
def chunk_text_paragraphs(text: str, max_length: int = 500) -> list[str]:
|
||||
# Absätze trennen
|
||||
paragraphs = re.split(r'\n\s*\n', text.strip())
|
||||
chunks: list[str] = []
|
||||
current_chunk = ""
|
||||
|
||||
for para in paragraphs:
|
||||
para = para.strip()
|
||||
if not para:
|
||||
continue
|
||||
|
||||
# Passt der Absatz noch zum aktuellen Chunk? +2 für die später hinzugefügten "\n\n"
|
||||
if len(current_chunk) + len(para) + 2 <= max_length:
|
||||
current_chunk = (current_chunk + "\n\n" + para) if current_chunk else para
|
||||
else:
|
||||
# Aktuellen Chunk abschließen
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
# Ist der Absatz selbst zu groß? Dann hart splitten
|
||||
if len(para) > max_length:
|
||||
for i in range(0, len(para), max_length):
|
||||
chunks.append(para[i:i + max_length])
|
||||
current_chunk = ""
|
||||
else:
|
||||
current_chunk = para
|
||||
|
||||
# Letzten Chunk nicht vergessen
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
return chunks
|
||||
@@ -0,0 +1,93 @@
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
# 📌 Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
CHUNK_SIZE = 500
|
||||
OVERLAP = 100
|
||||
BATCH_SIZE = 20
|
||||
|
||||
# 📁 Kommandozeilenparameter auswerten
|
||||
if len(sys.argv) != 2:
|
||||
print("❌ Bitte gib eine Kategorie an, z. B.: python index_documents_advanced.py karatetrainer")
|
||||
sys.exit(1)
|
||||
|
||||
CATEGORY = sys.argv[1]
|
||||
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
|
||||
ARCHIVE_DIR = os.path.join(SOURCE_DIR, "_imported")
|
||||
COLLECTION = CATEGORY
|
||||
|
||||
if not os.path.exists(SOURCE_DIR):
|
||||
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(ARCHIVE_DIR, exist_ok=True)
|
||||
|
||||
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
|
||||
print(f"📂 Archivierte Dateien: {ARCHIVE_DIR}")
|
||||
print(f"🎯 Ziel-Collection: {COLLECTION}")
|
||||
|
||||
# 🔧 Text in überlappende Chunks aufteilen
|
||||
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
|
||||
chunks = []
|
||||
start = 0
|
||||
while start < len(text):
|
||||
end = min(start + size, len(text))
|
||||
chunks.append(text[start:end])
|
||||
start += size - overlap
|
||||
return chunks
|
||||
|
||||
# 📚 Alle .txt-Dateien im Ordner lesen
|
||||
def read_all_text_files(folder):
|
||||
file_chunk_map = {} # Map: filename → chunks
|
||||
for filename in os.listdir(folder):
|
||||
if filename.endswith(".txt"):
|
||||
path = os.path.join(folder, filename)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
file_chunk_map[filename] = chunk_text(text)
|
||||
return file_chunk_map
|
||||
|
||||
# 📤 An API senden
|
||||
def embed_chunks_in_batches(chunks, collection):
|
||||
results = []
|
||||
for i in tqdm(range(0, len(chunks), BATCH_SIZE), desc="📡 Embedding"):
|
||||
batch = chunks[i:i + BATCH_SIZE]
|
||||
response = requests.post(API_URL, json={"texts": batch, "collection": collection})
|
||||
response.raise_for_status()
|
||||
results.append(response.json())
|
||||
return results
|
||||
|
||||
# 🚀 Hauptlogik
|
||||
if __name__ == "__main__":
|
||||
file_chunk_map = read_all_text_files(SOURCE_DIR)
|
||||
all_chunks = []
|
||||
processed_files = []
|
||||
|
||||
for filename, chunks in file_chunk_map.items():
|
||||
if chunks:
|
||||
all_chunks.extend(chunks)
|
||||
processed_files.append(filename)
|
||||
|
||||
if not all_chunks:
|
||||
print("⚠️ Keine Textabschnitte gefunden.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"📦 {len(all_chunks)} Textabschnitte aus {len(processed_files)} Dateien gefunden.")
|
||||
|
||||
try:
|
||||
result = embed_chunks_in_batches(all_chunks, COLLECTION)
|
||||
print(f"\n✅ Embedding abgeschlossen: {len(result)} API-Antwort(en) erhalten.")
|
||||
|
||||
# 🗃️ Verarbeitete Dateien archivieren
|
||||
for filename in processed_files:
|
||||
src = os.path.join(SOURCE_DIR, filename)
|
||||
dst = os.path.join(ARCHIVE_DIR, filename)
|
||||
shutil.move(src, dst)
|
||||
print(f"📁 {len(processed_files)} Dateien verschoben nach _imported.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Senden: {e}")
|
||||
@@ -0,0 +1,61 @@
|
||||
import requests
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 📌 Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
CHUNK_SIZE = 500
|
||||
OVERLAP = 100
|
||||
|
||||
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
|
||||
"""
|
||||
Teilt einen Text in überlappende Abschnitte auf.
|
||||
"""
|
||||
chunks = []
|
||||
start = 0
|
||||
while start < len(text):
|
||||
end = min(start + size, len(text))
|
||||
chunks.append(text[start:end])
|
||||
start += size - overlap
|
||||
return chunks
|
||||
|
||||
def read_text_file(path):
|
||||
"""
|
||||
Liest den Inhalt einer Textdatei ein.
|
||||
"""
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
def import_chunks(chunks, collection):
|
||||
"""
|
||||
Sendet die Textabschnitte an die API.
|
||||
"""
|
||||
response = requests.post(API_URL, json={
|
||||
"texts": chunks,
|
||||
"collection": collection
|
||||
})
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("❌ Nutzung: python import_textfile.py <collection> <pfad_zur_txt_datei>")
|
||||
sys.exit(1)
|
||||
|
||||
collection = sys.argv[1]
|
||||
filepath = os.path.expanduser(sys.argv[2])
|
||||
|
||||
if not os.path.isfile(filepath):
|
||||
print(f"❌ Datei nicht gefunden: {filepath}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"📄 Lade Datei: {filepath}")
|
||||
text = read_text_file(filepath)
|
||||
chunks = chunk_text(text)
|
||||
|
||||
print(f"📦 {len(chunks)} Abschnitte vorbereitet – sende an Collection '{collection}'...")
|
||||
try:
|
||||
result = import_chunks(chunks, collection)
|
||||
print(f"✅ Import erfolgreich: {result}")
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Import: {e}")
|
||||
@@ -0,0 +1,31 @@
|
||||
import requests
|
||||
import sys
|
||||
|
||||
# 📌 Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
|
||||
def import_text(text, collection="default"):
|
||||
"""
|
||||
Sendet einen einzelnen Textabschnitt an die Embed-API zur Indexierung.
|
||||
"""
|
||||
response = requests.post(API_URL, json={
|
||||
"texts": [text],
|
||||
"collection": collection
|
||||
})
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print("❌ Nutzung: python import_texts.py <collection> \"<text>\"")
|
||||
sys.exit(1)
|
||||
|
||||
collection = sys.argv[1]
|
||||
text = sys.argv[2]
|
||||
|
||||
print(f"📤 Sende an Collection '{collection}': {text}")
|
||||
try:
|
||||
result = import_text(text, collection)
|
||||
print(f"✅ Antwort: {result}")
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Importieren: {e}")
|
||||
@@ -0,0 +1,66 @@
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
|
||||
# 📌 Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
CHUNK_SIZE = 500
|
||||
OVERLAP = 100
|
||||
|
||||
# 📁 Kommandozeilenparameter auswerten
|
||||
if len(sys.argv) != 2:
|
||||
print("❌ Bitte gib eine Kategorie an, z. B.: python index_documents.py karatetrainer")
|
||||
sys.exit(1)
|
||||
|
||||
CATEGORY = sys.argv[1]
|
||||
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
|
||||
COLLECTION = CATEGORY
|
||||
|
||||
if not os.path.exists(SOURCE_DIR):
|
||||
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
|
||||
print(f"🎯 Ziel-Collection: {COLLECTION}")
|
||||
|
||||
# 🔧 Text in überlappende Chunks aufteilen
|
||||
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
|
||||
chunks = []
|
||||
start = 0
|
||||
while start < len(text):
|
||||
end = min(start + size, len(text))
|
||||
chunks.append(text[start:end])
|
||||
start += size - overlap
|
||||
return chunks
|
||||
|
||||
# 📚 Alle .txt-Dateien im Ordner lesen
|
||||
def read_all_text_files(folder):
|
||||
chunks = []
|
||||
for filename in os.listdir(folder):
|
||||
if filename.endswith(".txt"):
|
||||
path = os.path.join(folder, filename)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
file_chunks = chunk_text(text)
|
||||
chunks.extend(file_chunks)
|
||||
return chunks
|
||||
|
||||
# 📤 An API senden
|
||||
def embed_chunks(chunks, collection):
|
||||
response = requests.post(API_URL, json={"texts": chunks, "collection": collection})
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
# 🚀 Hauptlogik
|
||||
if __name__ == "__main__":
|
||||
texts = read_all_text_files(SOURCE_DIR)
|
||||
if not texts:
|
||||
print("⚠️ Keine Textabschnitte gefunden.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"📦 {len(texts)} Textabschnitte gefunden, sende an {API_URL}...")
|
||||
try:
|
||||
result = embed_chunks(texts, COLLECTION)
|
||||
print(f"✅ Ergebnis: {result}")
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Senden: {e}")
|
||||
@@ -0,0 +1,46 @@
|
||||
import re
|
||||
|
||||
def chunk_text_paragraphs(text, max_length=500, overlap=1):
|
||||
"""
|
||||
Zerteilt den Text absatzweise in Chunks bis `max_length` Zeichen.
|
||||
Optional wird `paragraph_overlap` Absatz(e) vom vorherigen Chunk übernommen.
|
||||
"""
|
||||
|
||||
paragraphs = [p.strip() for p in re.split(r'\n{2,}', text) if p.strip()]
|
||||
chunks = []
|
||||
current_chunk = []
|
||||
current_len = 0
|
||||
|
||||
i = 0
|
||||
while i < len(paragraphs):
|
||||
para = paragraphs[i]
|
||||
para_len = len(para)
|
||||
|
||||
# Wenn dieser Absatz den max Chunk sprengt → neuen Chunk
|
||||
if current_len + para_len + 2 > max_length: # +2 für Leerzeile
|
||||
if current_chunk:
|
||||
chunks.append("\n\n".join(current_chunk))
|
||||
# Optional: letzte N Absätze behalten
|
||||
if overlap > 0:
|
||||
current_chunk = current_chunk[-overlap:]
|
||||
current_len = sum(len(p) for p in current_chunk) + 2 * len(current_chunk)
|
||||
else:
|
||||
current_chunk = []
|
||||
current_len = 0
|
||||
else:
|
||||
# Einzelner Absatz ist zu groß → hart splitten
|
||||
chunks.append(para[:max_length])
|
||||
para = para[max_length:]
|
||||
paragraphs.insert(i + 1, para) # Rest zurück in Liste
|
||||
i += 1
|
||||
continue
|
||||
else:
|
||||
current_chunk.append(para)
|
||||
current_len += para_len + 2 # +2 für Trennung
|
||||
|
||||
i += 1
|
||||
|
||||
if current_chunk:
|
||||
chunks.append("\n\n".join(current_chunk))
|
||||
|
||||
return chunks
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# --------------------------------------------------
|
||||
# chunking_utils.py
|
||||
#
|
||||
# Enthält robuste Text-Chunking-Logik:
|
||||
# 1. Absatzbasiertes Chunking
|
||||
# 2. Satzbasiertes Chunking per Regex (kein NLTK)
|
||||
# 3. Satz-Overlap-Chunking
|
||||
# --------------------------------------------------
|
||||
|
||||
import re
|
||||
|
||||
# --------------------------------------------------
|
||||
# Hilfsfunktion: split_sentences
|
||||
# Zweck:
|
||||
# - Teilt Text in Sätze auf, basierend auf Punkt, Ausrufe- und Fragezeichen
|
||||
# - Trennt bei ".!? " (Satzzeichen gefolgt von Leerraum)
|
||||
# Parameter:
|
||||
# text : Volltext als String
|
||||
# Rückgabe:
|
||||
# Liste von Satz-Strings
|
||||
# --------------------------------------------------
|
||||
def split_sentences(text: str) -> list[str]:
|
||||
# Regex: lookbehind für . ! oder ?, dann ein oder mehrere Whitespace-Zeichen
|
||||
return re.split(r'(?<=[\.!?])\s+', text.strip())
|
||||
|
||||
|
||||
# --------------------------------------------------
|
||||
# Funktion: chunk_text_paragraphs
|
||||
# Zweck:
|
||||
# - Trennt Text absatzweise in Chunks mit bis zu max_length Zeichen
|
||||
# - Absätze werden an doppelten Zeilenumbrüchen getrennt
|
||||
# - Zu große Absätze werden hart in max_length-Teile gesplittet
|
||||
# Parameter:
|
||||
# text : Volltext als String
|
||||
# max_length : Maximale Länge eines Chunks (Standard 500)
|
||||
# Rückgabe:
|
||||
# Liste von Strings (Chunks)
|
||||
# --------------------------------------------------
|
||||
def chunk_text_paragraphs(text: str, max_length: int = 500) -> list[str]:
|
||||
paragraphs = re.split(r'\n\s*\n', text.strip())
|
||||
chunks: list[str] = []
|
||||
current_chunk = ""
|
||||
|
||||
for para in paragraphs:
|
||||
para = para.strip()
|
||||
if not para:
|
||||
continue
|
||||
|
||||
# Prüfen, ob Absatz noch in aktuellen Chunk passt (+2 für "\n\n")
|
||||
if len(current_chunk) + len(para) + 2 <= max_length:
|
||||
if current_chunk:
|
||||
current_chunk += "\n\n" + para
|
||||
else:
|
||||
current_chunk = para
|
||||
else:
|
||||
# Bislang gesammelten Chunk speichern
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
# Absatz hart splitten, wenn er allein zu groß ist
|
||||
if len(para) > max_length:
|
||||
for i in range(0, len(para), max_length):
|
||||
part = para[i:i + max_length]
|
||||
chunks.append(part)
|
||||
current_chunk = ""
|
||||
else:
|
||||
# Neuer Chunk beginnt mit diesem Absatz
|
||||
current_chunk = para
|
||||
|
||||
# Letzten Chunk nicht vergessen
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
# --------------------------------------------------
|
||||
# Funktion: chunk_by_sentences
|
||||
# Zweck:
|
||||
# - Zerlegt Text in Sätze per Regex-Split
|
||||
# - Baut daraus Chunks mit ganzen Sätzen bis max_length
|
||||
# Parameter:
|
||||
# text : Volltext als String
|
||||
# max_length : Maximale Länge eines Chunks (Standard 500)
|
||||
# Rückgabe:
|
||||
# Liste von Strings (Chunks)
|
||||
# --------------------------------------------------
|
||||
def chunk_by_sentences(text: str, max_length: int = 500) -> list[str]:
|
||||
sentences = split_sentences(text)
|
||||
chunks: list[str] = []
|
||||
current_chunk = ""
|
||||
|
||||
for sent in sentences:
|
||||
sent = sent.strip()
|
||||
if not sent:
|
||||
continue
|
||||
|
||||
# Prüfen, ob Satz noch in aktuellen Chunk passt (+1 für Leerzeichen)
|
||||
if len(current_chunk) + len(sent) + 1 <= max_length:
|
||||
if current_chunk:
|
||||
current_chunk += " " + sent
|
||||
else:
|
||||
current_chunk = sent
|
||||
else:
|
||||
# Bisher gesammelten Chunk speichern
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
# Einzelnen Satz hart splitten, falls er zu lang ist
|
||||
if len(sent) > max_length:
|
||||
for i in range(0, len(sent), max_length):
|
||||
chunks.append(sent[i:i + max_length])
|
||||
current_chunk = ""
|
||||
else:
|
||||
current_chunk = sent
|
||||
|
||||
# Letzten Chunk nicht vergessen
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
# --------------------------------------------------
|
||||
# Funktion: chunk_with_sentence_overlap
|
||||
# Zweck:
|
||||
# - Baut zunächst sentence-basierte Chunks
|
||||
# - Fügt vom vorherigen Chunk overlap_sents Sätze vorne an
|
||||
# Parameter:
|
||||
# text : Volltext als String
|
||||
# max_length : Maximale Länge eines Chunks (Standard 500)
|
||||
# overlap_sents : Anzahl Sätze, die überlappend übernommen werden (Standard 1)
|
||||
# Rückgabe:
|
||||
# Liste von Strings (Chunks mit Kontext-Overlap)
|
||||
# --------------------------------------------------
|
||||
def chunk_with_sentence_overlap(
|
||||
text: str,
|
||||
max_length: int = 500,
|
||||
overlap_sents: int = 1
|
||||
) -> list[str]:
|
||||
base_chunks = chunk_by_sentences(text, max_length)
|
||||
overlapped: list[str] = []
|
||||
|
||||
for idx, chunk in enumerate(base_chunks):
|
||||
if idx == 0 or overlap_sents <= 0:
|
||||
overlapped.append(chunk)
|
||||
else:
|
||||
prev = base_chunks[idx - 1]
|
||||
prev_sents = split_sentences(prev)
|
||||
context = " ".join(prev_sents[-overlap_sents:])
|
||||
overlapped.append((context + " " + chunk).strip())
|
||||
|
||||
return overlapped
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from tqdm import tqdm
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Importiert alle .txt-Dateien in einer Kategorie mittels import_single_file.py"
|
||||
)
|
||||
parser.add_argument(
|
||||
"category",
|
||||
help="Name der Kategorie (Ordner unter ~/knowledge/)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--topic", "-t",
|
||||
help="Optionaler Kontext/Topic für alle Importe",
|
||||
default="default"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--script-path", "-s",
|
||||
help="Pfad zum import_single_file.py",
|
||||
default="import_single_file.py"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
category = args.category
|
||||
topic = args.topic
|
||||
script_path = args.script_path
|
||||
|
||||
base_dir = os.path.expanduser(f"~/knowledge/{category}")
|
||||
if not os.path.isdir(base_dir):
|
||||
print(f"❌ Kategorie-Ordner nicht gefunden: {base_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
files = [f for f in os.listdir(base_dir) if f.endswith(".txt")]
|
||||
if not files:
|
||||
print(f"⚠️ Keine .txt-Dateien in {base_dir} gefunden.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"📁 Starte Ordner-Import: {len(files)} Dateien in Kategorie '{category}' (Topic: '{topic}')")
|
||||
for filename in tqdm(files, desc="Importiere Dateien"):
|
||||
cmd = [
|
||||
sys.executable,
|
||||
script_path,
|
||||
category,
|
||||
filename,
|
||||
topic
|
||||
]
|
||||
try:
|
||||
subprocess.run(cmd, check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"❌ Fehler beim Import von {filename}: {e}")
|
||||
|
||||
print("✅ Ordner-Import abgeschlossen.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
import_pdf_file.py
|
||||
|
||||
Importiert eine einzelne PDF-Datei:
|
||||
- Text-Extraktion via pdfplumber
|
||||
- Chunking mit Satz-Overlap
|
||||
- POST an /embed
|
||||
- Verschieben nach _imported
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import requests
|
||||
from uuid import uuid4
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pdfplumber
|
||||
from chunking_utils import chunk_with_sentence_overlap
|
||||
|
||||
# Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
MAX_LENGTH = 500
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("? Usage: python import_pdf_file.py <category> <filename> [topic]")
|
||||
sys.exit(1)
|
||||
|
||||
category = sys.argv[1]
|
||||
filename = sys.argv[2]
|
||||
topic = sys.argv[3] if len(sys.argv) > 3 else "default"
|
||||
|
||||
source_dir = os.path.expanduser(f"~/knowledge/{category}")
|
||||
file_path = os.path.join(source_dir, filename)
|
||||
archive_dir= os.path.join(source_dir, "_imported")
|
||||
|
||||
if not os.path.isfile(file_path):
|
||||
print(f"? Datei nicht gefunden: {file_path}")
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(archive_dir, exist_ok=True)
|
||||
print(f"?? PDF-Import: {filename} in Kategorie {category} (Topic={topic})")
|
||||
|
||||
# 1) PDF-Text extrahieren
|
||||
text = ""
|
||||
with pdfplumber.open(file_path) as pdf:
|
||||
for page in pdf.pages:
|
||||
page_text = page.extract_text() or ""
|
||||
text += page_text + "\n\n"
|
||||
|
||||
# 2) Chunking
|
||||
chunks = chunk_with_sentence_overlap(
|
||||
text,
|
||||
max_length=MAX_LENGTH,
|
||||
overlap_sents=2
|
||||
)
|
||||
print(f"?? {len(chunks)} Chunks erzeugt.")
|
||||
|
||||
# 3) Payload bauen
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
payload = {"collection": category, "chunks": []}
|
||||
|
||||
for idx, chunk in enumerate(chunks):
|
||||
payload["chunks"].append({
|
||||
"text": chunk,
|
||||
"source": filename,
|
||||
"source_type": "pdf",
|
||||
"title": os.path.splitext(filename)[0],
|
||||
"version": "v1.0",
|
||||
"related_to": category,
|
||||
"tags": [category],
|
||||
"owner": "karate-agent",
|
||||
"context_tag": topic,
|
||||
"imported_at": now,
|
||||
"chunk_index": idx,
|
||||
"category": category
|
||||
})
|
||||
|
||||
# 4) Senden
|
||||
try:
|
||||
res = requests.post(API_URL, json=payload)
|
||||
res.raise_for_status()
|
||||
print(f"? {len(chunks)} Chunks erfolgreich eingebettet.")
|
||||
except Exception as e:
|
||||
print(f"? Fehler beim Senden: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# 5) Archivieren
|
||||
shutil.move(file_path, os.path.join(archive_dir, filename))
|
||||
print("?? PDF verschoben nach _imported/")
|
||||
@@ -0,0 +1,85 @@
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import requests
|
||||
from datetime import datetime, timezone
|
||||
#from text_chunker import chunk_text_paragraphs
|
||||
from uuid import uuid4
|
||||
from chunking_utils import (
|
||||
chunk_text_paragraphs,
|
||||
chunk_by_sentences,
|
||||
chunk_with_sentence_overlap
|
||||
)
|
||||
|
||||
|
||||
# ?? Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
CHUNK_SIZE = 500
|
||||
OVERLAP = 100
|
||||
|
||||
# ?? Eingabeparameter: Kategorie, Dateiname, optionale Metadaten
|
||||
if len(sys.argv) < 3:
|
||||
print("? Aufruf: python import_single_file.py <category> <filename> [topic]")
|
||||
sys.exit(1)
|
||||
|
||||
CATEGORY = sys.argv[1]
|
||||
FILENAME = sys.argv[2]
|
||||
TOPIC = sys.argv[3] if len(sys.argv) > 3 else None
|
||||
|
||||
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
|
||||
ARCHIVE_DIR = os.path.join(SOURCE_DIR, "_imported")
|
||||
FILEPATH = os.path.join(SOURCE_DIR, FILENAME)
|
||||
|
||||
# ?? Validierung
|
||||
if not os.path.exists(FILEPATH):
|
||||
print(f"? Datei nicht gefunden: {FILEPATH}")
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(ARCHIVE_DIR, exist_ok=True)
|
||||
|
||||
print(f"?? Importiere Datei: {FILENAME} aus Kategorie: {CATEGORY}")
|
||||
|
||||
# ?? Inhalte lesen und in Chunks zerteilen
|
||||
with open(FILEPATH, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
chunks = chunk_with_sentence_overlap(content, max_length=CHUNK_SIZE, overlap_sents=2)
|
||||
print(f"?? {len(chunks)} Textabschnitte erzeugt.")
|
||||
|
||||
# ?? Metadaten vorbereiten
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
payload = {
|
||||
"chunks": [],
|
||||
"collection": CATEGORY
|
||||
}
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
payload["chunks"].append({
|
||||
"text": chunk,
|
||||
"source": FILENAME,
|
||||
"source_type": "file",
|
||||
"title": FILENAME.replace(".txt", ""),
|
||||
"version": "v1.0",
|
||||
"related_to": CATEGORY,
|
||||
"tags": [CATEGORY],
|
||||
"owner": "karate-agent",
|
||||
"context_tag": TOPIC or "default",
|
||||
"imported_at": now,
|
||||
"chunk_index": i,
|
||||
"category": CATEGORY
|
||||
})
|
||||
|
||||
# ?? An API senden
|
||||
try:
|
||||
res = requests.post(API_URL, json=payload)
|
||||
res.raise_for_status()
|
||||
print(f"? {len(chunks)} Abschnitte erfolgreich eingebettet.")
|
||||
except Exception as e:
|
||||
print(f"? Fehler beim Senden: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# ??? Datei archivieren
|
||||
shutil.move(FILEPATH, os.path.join(ARCHIVE_DIR, FILENAME))
|
||||
print(f"?? Datei nach _imported verschoben.")
|
||||
@@ -0,0 +1,135 @@
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import requests
|
||||
import re
|
||||
from tqdm import tqdm
|
||||
from datetime import datetime
|
||||
|
||||
# 📌 Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
CHUNK_SIZE = 500
|
||||
OVERLAP = 100
|
||||
BATCH_SIZE = 20
|
||||
|
||||
# 📁 Kommandozeilenparameter auswerten
|
||||
if len(sys.argv) != 2:
|
||||
print("❌ Bitte gib eine Kategorie an, z. B.: python import_txtdocuments.py karatetrainer")
|
||||
sys.exit(1)
|
||||
|
||||
CATEGORY = sys.argv[1]
|
||||
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
|
||||
ARCHIVE_DIR = os.path.join(SOURCE_DIR, "_imported")
|
||||
COLLECTION = CATEGORY
|
||||
|
||||
if not os.path.exists(SOURCE_DIR):
|
||||
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(ARCHIVE_DIR, exist_ok=True)
|
||||
|
||||
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
|
||||
print(f"📂 Archivierte Dateien: {ARCHIVE_DIR}")
|
||||
print(f"🎯 Ziel-Collection: {COLLECTION}")
|
||||
|
||||
# 🔧 Text in überlappende Chunks aufteilen
|
||||
#def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
|
||||
# chunks = []
|
||||
# start = 0
|
||||
# while start < len(text):
|
||||
# end = min(start + size, len(text))
|
||||
# chunks.append(text[start:end])
|
||||
# start += size - overlap
|
||||
# return chunks
|
||||
|
||||
def chunk_text_paragraphs(text, max_length=500):
|
||||
paragraphs = re.split(r'\n\s*\n', text.strip()) # Absatztrennung
|
||||
chunks = []
|
||||
current_chunk = ""
|
||||
|
||||
for para in paragraphs:
|
||||
if len(current_chunk) + len(para) + 2 <= max_length:
|
||||
current_chunk += para + "\n\n"
|
||||
else:
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk.strip())
|
||||
# Falls einzelner Absatz zu groß ist, hart splitten
|
||||
if len(para) > max_length:
|
||||
for i in range(0, len(para), max_length):
|
||||
chunks.append(para[i:i+max_length].strip())
|
||||
current_chunk = ""
|
||||
else:
|
||||
current_chunk = para + "\n\n"
|
||||
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk.strip())
|
||||
return chunks
|
||||
|
||||
|
||||
# 📚 Alle .txt-Dateien im Ordner lesen
|
||||
def read_all_text_files(folder):
|
||||
file_chunk_map = {} # Map: filename → chunks
|
||||
for filename in os.listdir(folder):
|
||||
if filename.endswith(".txt"):
|
||||
path = os.path.join(folder, filename)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
file_chunk_map[filename] = chunk_text_paragraphs(text)
|
||||
return file_chunk_map
|
||||
|
||||
# 🧱 Strukturierte Payloads vorbereiten
|
||||
def prepare_payloads(file_chunk_map, collection):
|
||||
payloads = []
|
||||
imported_at = datetime.now().isoformat()
|
||||
|
||||
for filename, chunks in file_chunk_map.items():
|
||||
for local_index, chunk in enumerate(chunks):
|
||||
payload = {
|
||||
"text": chunk,
|
||||
"source": filename,
|
||||
"type": "file",
|
||||
"category": collection,
|
||||
"imported_at": imported_at,
|
||||
"chunk_index": local_index
|
||||
}
|
||||
payloads.append(payload)
|
||||
for p in payloads:
|
||||
print(f"{p['source']}: chunk_index={p['chunk_index']}")
|
||||
|
||||
return payloads
|
||||
|
||||
# 📤 An API senden
|
||||
def embed_chunks_in_batches(payloads, collection):
|
||||
results = []
|
||||
for i in tqdm(range(0, len(payloads), BATCH_SIZE), desc="📡 Embedding"):
|
||||
batch = payloads[i:i + BATCH_SIZE]
|
||||
response = requests.post(API_URL, json={"chunks": batch, "collection": collection})
|
||||
response.raise_for_status()
|
||||
results.append(response.json())
|
||||
return results
|
||||
|
||||
# 🚀 Hauptlogik
|
||||
if __name__ == "__main__":
|
||||
file_chunk_map = read_all_text_files(SOURCE_DIR)
|
||||
processed_files = list(file_chunk_map.keys())
|
||||
payloads = prepare_payloads(file_chunk_map, COLLECTION)
|
||||
|
||||
if not payloads:
|
||||
print("⚠️ Keine Textabschnitte gefunden.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"📦 {len(payloads)} Textabschnitte aus {len(processed_files)} Dateien gefunden. Sende an API...")
|
||||
|
||||
try:
|
||||
result = embed_chunks_in_batches(payloads, COLLECTION)
|
||||
print(f"\n✅ Embedding abgeschlossen: {len(result)} API-Antwort(en) erhalten.")
|
||||
|
||||
# 🗃️ Verarbeitete Dateien archivieren
|
||||
for filename in processed_files:
|
||||
src = os.path.join(SOURCE_DIR, filename)
|
||||
dst = os.path.join(ARCHIVE_DIR, filename)
|
||||
shutil.move(src, dst)
|
||||
print(f"📁 {len(processed_files)} Dateien verschoben nach _imported.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Senden: {e}")
|
||||
@@ -0,0 +1,32 @@
|
||||
import sys
|
||||
import requests
|
||||
|
||||
API_URL = "http://localhost:8000/prompt"
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("❌ Verwendung: python prompt_documents.py <collection> <frage>")
|
||||
sys.exit(1)
|
||||
|
||||
collection = sys.argv[1]
|
||||
query = " ".join(sys.argv[2:])
|
||||
|
||||
data = {
|
||||
"query": query,
|
||||
"collection": collection,
|
||||
"context_limit": 3
|
||||
}
|
||||
|
||||
print(f"🤖 Anfrage an LLM aus Collection '{collection}': {query}")
|
||||
try:
|
||||
response = requests.post(API_URL, json=data)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler bei der Anfrage: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
print("\n📄 Kontext:")
|
||||
print(result["context"])
|
||||
|
||||
print("\n💡 Antwort:")
|
||||
print(result["answer"])
|
||||
@@ -0,0 +1,45 @@
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
|
||||
def print_usage():
|
||||
print("❌ Bitte gib eine Kategorie an, z. B.:")
|
||||
print(" python restore_imported_files.py karatetrainer")
|
||||
sys.exit(1)
|
||||
|
||||
# --- Eingabe prüfen ---
|
||||
if len(sys.argv) < 2:
|
||||
print_usage()
|
||||
|
||||
CATEGORY = sys.argv[1]
|
||||
FORCE = "--force" in sys.argv
|
||||
|
||||
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}/_imported")
|
||||
TARGET_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
|
||||
|
||||
if not os.path.isdir(SOURCE_DIR):
|
||||
print(f"❌ Quellordner '{SOURCE_DIR}' existiert nicht.")
|
||||
sys.exit(1)
|
||||
|
||||
files = [f for f in os.listdir(SOURCE_DIR) if os.path.isfile(os.path.join(SOURCE_DIR, f))]
|
||||
if not files:
|
||||
print("⚠️ Keine Dateien zum Wiederherstellen gefunden.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"♻️ Wiederherstellung von {len(files)} Dateien nach '{TARGET_DIR}'")
|
||||
|
||||
restored = 0
|
||||
for file in files:
|
||||
source_path = os.path.join(SOURCE_DIR, file)
|
||||
target_path = os.path.join(TARGET_DIR, file)
|
||||
|
||||
if not FORCE:
|
||||
confirm = input(f"🔁 Datei '{file}' zurückkopieren? [j/N] ").strip().lower()
|
||||
if confirm != "j":
|
||||
continue
|
||||
|
||||
shutil.move(source_path, target_path)
|
||||
print(f"✅ '{file}' wurde zurückkopiert.")
|
||||
restored += 1
|
||||
|
||||
print(f"\n🎉 {restored} Datei(en) wurden erfolgreich wiederhergestellt.")
|
||||
@@ -0,0 +1,36 @@
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
|
||||
def print_usage():
|
||||
print("❌ Bitte gib Kategorie und Dateinamen an, z. B.:")
|
||||
print(" python restore_single_file.py karatetrainer mae_geri.txt")
|
||||
sys.exit(1)
|
||||
|
||||
# Eingabe prüfen
|
||||
if len(sys.argv) != 3:
|
||||
print_usage()
|
||||
|
||||
CATEGORY = sys.argv[1]
|
||||
FILENAME = sys.argv[2]
|
||||
|
||||
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}/_imported")
|
||||
TARGET_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
|
||||
SOURCE_FILE = os.path.join(SOURCE_DIR, FILENAME)
|
||||
TARGET_FILE = os.path.join(TARGET_DIR, FILENAME)
|
||||
|
||||
if not os.path.isfile(SOURCE_FILE):
|
||||
print(f"❌ Die Datei '{FILENAME}' wurde im Archivordner '{SOURCE_DIR}' nicht gefunden.")
|
||||
sys.exit(1)
|
||||
|
||||
if os.path.exists(TARGET_FILE):
|
||||
confirm = input(f"⚠️ Datei '{FILENAME}' existiert im Zielordner. Überschreiben? [j/N] ").strip().lower()
|
||||
if confirm != "j":
|
||||
print("⏹️ Abgebrochen.")
|
||||
sys.exit(0)
|
||||
|
||||
try:
|
||||
shutil.move(SOURCE_FILE, TARGET_FILE)
|
||||
print(f"✅ '{FILENAME}' wurde zurück nach '{TARGET_DIR}' verschoben.")
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Verschieben: {e}")
|
||||
@@ -0,0 +1,31 @@
|
||||
import sys
|
||||
import requests
|
||||
|
||||
API_URL = "http://localhost:8000/search"
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("❌ Verwendung: python search_documents.py <collection> <suchfrage>")
|
||||
sys.exit(1)
|
||||
|
||||
collection = sys.argv[1]
|
||||
query = " ".join(sys.argv[2:])
|
||||
|
||||
params = {
|
||||
"query": query,
|
||||
"collection": collection,
|
||||
"limit": 5
|
||||
}
|
||||
|
||||
print(f"🔍 Suche in Collection '{collection}': {query}")
|
||||
try:
|
||||
response = requests.get(API_URL, params=params)
|
||||
response.raise_for_status()
|
||||
results = response.json()
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler bei der Anfrage: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
print("📚 Ergebnisse:")
|
||||
for r in results:
|
||||
print(f"\n✅ Score: {r['score']:.4f}")
|
||||
print(f"{r['text']}")
|
||||
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Module: wiki_importer.py
|
||||
Beschreibung:
|
||||
- Importiert Übungen aus dem MediaWiki via FastAPI wiki_router
|
||||
- Führt vor dem Import einen Login gegen /import/wiki/login durch (falls nicht via --skip-login deaktiviert)
|
||||
- Holt Liste aller Übungs-Titel (SMW-Ask) via `/semantic/pages`
|
||||
- Für jede Übung:
|
||||
* Fetch pageinfo (pageid, fullurl) via `/info`
|
||||
* Parse Wikitext (Templates: ÜbungInfoBox, Übungsbeschreibung, SkillDevelopment) via `/parsepage`
|
||||
* Baut Payload entsprechend Exercise-Datenmodell
|
||||
* POST an `/exercise` Endpoint (exercise_router)
|
||||
- Unterstützt Single-Import via `--title` (oder ENV `WIKI_EXERCISE_TITLE`) und Full-Import via `--all`
|
||||
- Optional: Credentials via CLI (--username/--password) oder `.env` (WIKI_BOT_USER / WIKI_BOT_PASSWORD)
|
||||
|
||||
Version: 2.1.0
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
from typing import Dict, Any
|
||||
import requests
|
||||
import mwparserfromhell
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# ----- Konfiguration / Defaults -----
|
||||
load_dotenv() # .env laden, falls vorhanden
|
||||
|
||||
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki") # FastAPI-Wiki-Proxy
|
||||
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise") # Exercise-Endpoint
|
||||
DEFAULT_CAT = os.getenv("WIKI_CATEGORY", "Übungen")
|
||||
DEFAULT_TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
|
||||
|
||||
# ---- Hilfsfunktionen für Wiki-Router ----
|
||||
def wiki_health() -> None:
|
||||
r = requests.get(f"{API_BASE_URL}/health", timeout=15)
|
||||
r.raise_for_status()
|
||||
print("[Sanity] Wiki health OK")
|
||||
|
||||
def wiki_login(username: str, password: str) -> None:
|
||||
"""
|
||||
Führt einen Login gegen den wiki_router durch.
|
||||
Erwartet: {"status":"success"} bei Erfolg.
|
||||
"""
|
||||
payload = {"username": username, "password": password}
|
||||
r = requests.post(f"{API_BASE_URL}/login", json=payload, timeout=30)
|
||||
# kein raise_for_status(), wir wollen die JSON-Fehler sauber ausgeben
|
||||
try:
|
||||
data = r.json()
|
||||
except Exception:
|
||||
print(f"[Login] HTTP {r.status_code}: {r.text}")
|
||||
r.raise_for_status()
|
||||
|
||||
status = (data or {}).get("status")
|
||||
if status != "success":
|
||||
msg = (data or {}).get("message", "Login fehlgeschlagen")
|
||||
raise RuntimeError(f"[Login] {msg}")
|
||||
print("[Login] success")
|
||||
|
||||
def fetch_all_pages(category: str) -> Dict[str, Any]:
|
||||
resp = requests.get(f"{API_BASE_URL}/semantic/pages", params={"category": category}, timeout=60)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def fetch_page_info(title: str) -> Dict[str, Any]:
|
||||
r = requests.get(f"{API_BASE_URL}/info", params={"title": title}, timeout=30)
|
||||
r.raise_for_status()
|
||||
info = r.json()
|
||||
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
|
||||
|
||||
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
|
||||
print(f"[Parse] Lade '{title}' (ID={pageid})")
|
||||
resp = requests.get(
|
||||
f"{API_BASE_URL}/parsepage",
|
||||
params={"pageid": pageid, "title": title},
|
||||
timeout=60
|
||||
)
|
||||
resp.raise_for_status()
|
||||
wikitext = resp.json().get("wikitext", "")
|
||||
wikicode = mwparserfromhell.parse(wikitext)
|
||||
|
||||
raw: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid}
|
||||
for tpl in wikicode.filter_templates():
|
||||
name = str(tpl.name).strip()
|
||||
if name == "ÜbungInfoBox":
|
||||
for p in tpl.params:
|
||||
raw[str(p.name).strip()] = str(p.value).strip()
|
||||
elif name == "Übungsbeschreibung":
|
||||
for p in tpl.params:
|
||||
raw[str(p.name).strip()] = str(p.value).strip()
|
||||
elif name == "SkillDevelopment":
|
||||
raw.setdefault("capabilities", [])
|
||||
try:
|
||||
cap = str(tpl.get("PrimaryCapability").value).strip()
|
||||
except Exception:
|
||||
cap = ""
|
||||
try:
|
||||
lvl = int(str(tpl.get("CapabilityLevel").value).strip())
|
||||
except Exception:
|
||||
lvl = 0
|
||||
if cap:
|
||||
raw["capabilities"].append({"capability": cap, "level": lvl})
|
||||
raw["wikitext"] = wikitext
|
||||
return raw
|
||||
|
||||
def build_payload(raw: Dict[str, Any], fullurl: str, category: str) -> Dict[str, Any]:
|
||||
# Exercise.capabilities erwartet Dict[str,int]
|
||||
caps_list = raw.get("capabilities", [])
|
||||
capabilities = {}
|
||||
for c in caps_list:
|
||||
cap = c.get("capability")
|
||||
lvl = c.get("level")
|
||||
if isinstance(cap, str) and cap:
|
||||
try:
|
||||
capabilities[cap] = int(lvl)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Defaults/Fallbacks
|
||||
duration = 0.0
|
||||
try:
|
||||
duration = float(raw.get("Dauer", 0) or 0)
|
||||
except Exception:
|
||||
duration = 0.0
|
||||
|
||||
keywords = []
|
||||
kw_raw = raw.get("Schlüsselworte", "")
|
||||
if isinstance(kw_raw, str):
|
||||
keywords = [k.strip() for k in kw_raw.split(",") if k.strip()]
|
||||
|
||||
equipment = []
|
||||
eq_raw = raw.get("equipment", [])
|
||||
if isinstance(eq_raw, str):
|
||||
equipment = [e.strip() for e in eq_raw.split(",") if e.strip()]
|
||||
elif isinstance(eq_raw, list):
|
||||
equipment = [str(e).strip() for e in eq_raw if str(e).strip()]
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"title": raw.get("title") or "",
|
||||
"summary": raw.get("Summary", "") or "",
|
||||
"short_description": raw.get("Summary", "") or "",
|
||||
"keywords": keywords,
|
||||
"link": fullurl or "",
|
||||
"discipline": raw.get("Übungstyp", "") or "",
|
||||
"group": raw.get("Gruppengröße", "") or None,
|
||||
"age_group": raw.get("Altersgruppe", "") or "",
|
||||
"target_group": raw.get("Zielgruppe", "") or "",
|
||||
"min_participants": 1,
|
||||
"duration_minutes": int(round(duration)), # Exercise erwartet int
|
||||
"capabilities": capabilities,
|
||||
"category": category or "",
|
||||
"purpose": raw.get("Ziel", "") or "",
|
||||
"execution": raw.get("Durchführung", "") or "",
|
||||
"notes": raw.get("Hinweise", "") or "",
|
||||
"preparation": raw.get("RefMethode", "") or "",
|
||||
"method": raw.get("method", "") or "", # falls im Wikitext vorhanden
|
||||
"equipment": equipment,
|
||||
"fullurl": fullurl or "", # optionales Feld
|
||||
# Idempotenz (optional nutzbar in exercise_router):
|
||||
"external_id": f"wiki:{raw.get('pageid')}",
|
||||
"source": "MediaWiki"
|
||||
}
|
||||
return payload
|
||||
|
||||
def ingest_exercise(payload: Dict[str, Any]) -> None:
|
||||
title = payload.get("title", "<ohne Titel>")
|
||||
resp = requests.post(EXERCISE_API, json=payload, timeout=60)
|
||||
if resp.status_code == 422:
|
||||
print(f"[Ingest] '{title}' -> FAILED 422:\n{resp.text}")
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
resp.raise_for_status()
|
||||
print(f"[Ingest] '{title}' -> OK")
|
||||
|
||||
# ----- Main -----
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Import exercises from Wiki to Qdrant (via FastAPI wiki_router)")
|
||||
parser.add_argument("--all", action="store_true", help="Alle Übungen importieren (SMW-Ask)")
|
||||
parser.add_argument("--title", type=str, default=DEFAULT_TITLE, help="Einzelimport eines Übungstitels")
|
||||
parser.add_argument("--category", type=str, default=DEFAULT_CAT, help="Wiki-Kategorie (z.B. 'Übungen')")
|
||||
parser.add_argument("--username", type=str, default=os.getenv("WIKI_BOT_USER"), help="Wiki-Login Benutzer (überschreibt .env)")
|
||||
parser.add_argument("--password", type=str, default=os.getenv("WIKI_BOT_PASSWORD"), help="Wiki-Login Passwort (überschreibt .env)")
|
||||
parser.add_argument("--skip-login", action="store_true", help="Login-Schritt überspringen (falls Session schon aktiv)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Sanity
|
||||
wiki_health()
|
||||
|
||||
# Login (sofern nicht explizit übersprungen)
|
||||
if not args.skip_login:
|
||||
if not args.username or not args.password:
|
||||
print("[Login] Fehler: fehlende Credentials. Setze .env (WIKI_BOT_USER/WIKI_BOT_PASSWORD) oder CLI --username/--password.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
wiki_login(args.username, args.password)
|
||||
except Exception as e:
|
||||
print(str(e), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Einzel- oder Vollimport
|
||||
if args.all:
|
||||
print(f"[Main] Lade Liste der Übungen aus Kategorie '{args.category}'…")
|
||||
pages = fetch_all_pages(args.category)
|
||||
print(f"[Main] {len(pages)} Seiten gefunden.")
|
||||
for title, entry in pages.items():
|
||||
pid = entry.get("pageid")
|
||||
fullurl = entry.get("fullurl")
|
||||
if not pid:
|
||||
# Core-Info nachschlagen
|
||||
info = fetch_page_info(title)
|
||||
pid = info.get("pageid")
|
||||
fullurl = fullurl or info.get("fullurl")
|
||||
if not pid:
|
||||
print(f"[Skip] '{title}' hat keine pageid")
|
||||
continue
|
||||
raw = parse_exercise(title, pid)
|
||||
payload = build_payload(raw, fullurl or "", args.category)
|
||||
ingest_exercise(payload)
|
||||
else:
|
||||
print(f"[Main] Import single exercise: {args.title}")
|
||||
info = fetch_page_info(args.title)
|
||||
pid = info.get("pageid")
|
||||
fullurl = info.get("fullurl") or ""
|
||||
if not pid:
|
||||
print(f"[Error] pageid für '{args.title}' nicht gefunden.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
raw = parse_exercise(args.title, pid)
|
||||
payload = build_payload(raw, fullurl, args.category)
|
||||
ingest_exercise(payload)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
Module: wiki_importer.py
|
||||
Beschreibung:
|
||||
- Importiert zunächst nur eine Übung aus dem Wiki
|
||||
- Liest Wikitext einer Übung aus
|
||||
- Parsen mit mwparserfromhell
|
||||
- Extrahiert Felder aus Templates:
|
||||
* ÜbungInfoBox
|
||||
* Übungsbeschreibung
|
||||
* SkillDevelopment (mehrfach)
|
||||
- Baut ein Exercise-Objekt zusammen
|
||||
- Speichert per POST /exercise Endpoint in Qdrant
|
||||
- Detailliertes Error-Logging für 422 und allgemeine Fehler
|
||||
Version: 1.1.2
|
||||
"""
|
||||
import requests
|
||||
import mwparserfromhell
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, Any
|
||||
|
||||
# Konfiguration über Umgebungsvariablen
|
||||
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki")
|
||||
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise")
|
||||
# Übungstitel, der importiert werden soll
|
||||
TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
|
||||
|
||||
# Helper: Holt pageid und fullurl per Core-API
|
||||
def fetch_page_info(title: str) -> Dict[str, Any]:
|
||||
r = requests.get(f"{API_BASE_URL}/info", params={"title": title})
|
||||
r.raise_for_status()
|
||||
info = r.json()
|
||||
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
|
||||
|
||||
# Parser: Lädt und parst eine Übung
|
||||
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
|
||||
print(f"[Parse] Loading '{title}' (ID={pageid})")
|
||||
resp = requests.get(f"{API_BASE_URL}/parsepage", params={"pageid": pageid, "title": title})
|
||||
resp.raise_for_status()
|
||||
wikitext = resp.json().get("wikitext", "")
|
||||
|
||||
wikicode = mwparserfromhell.parse(wikitext)
|
||||
data: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid}
|
||||
|
||||
for tpl in wikicode.filter_templates():
|
||||
name = tpl.name.strip()
|
||||
if name == "ÜbungInfoBox":
|
||||
for param in tpl.params:
|
||||
data[param.name.strip()] = str(param.value).strip()
|
||||
elif name == "Übungsbeschreibung":
|
||||
for param in tpl.params:
|
||||
data[param.name.strip()] = str(param.value).strip()
|
||||
elif name == "SkillDevelopment":
|
||||
data.setdefault("capabilities", [])
|
||||
primary = str(tpl.get("PrimaryCapability").value).strip()
|
||||
level = int(str(tpl.get("CapabilityLevel").value).strip())
|
||||
data["capabilities"].append({"capability": primary, "level": level})
|
||||
data["wikitext"] = wikitext
|
||||
return data
|
||||
|
||||
# Ingestion: Sendet einen Datensatz an Qdrant mit detailliertem Error-Logging
|
||||
def ingest_exercise(ex_data: Dict[str, Any]) -> None:
|
||||
title = ex_data.get("title")
|
||||
try:
|
||||
resp = requests.post(EXERCISE_API, json=ex_data)
|
||||
if resp.status_code == 422:
|
||||
print(f"[Ingest] '{title}' -> FAILED 422:")
|
||||
print(resp.text)
|
||||
resp.raise_for_status()
|
||||
print(f"[Ingest] '{title}' -> OK")
|
||||
except requests.HTTPError as e:
|
||||
msg = resp.text if 'resp' in locals() else str(e)
|
||||
print(f"[Ingest] '{title}' -> HTTPError: {e} - {msg}")
|
||||
except Exception as e:
|
||||
print(f"[Ingest] '{title}' -> FAILED: {e}")
|
||||
|
||||
# Main: Einmaliger Import für TITLE
|
||||
if __name__ == "__main__":
|
||||
print(f"[Main] Import single exercise: {TITLE}")
|
||||
try:
|
||||
info = fetch_page_info(TITLE)
|
||||
pageid = info.get("pageid")
|
||||
fullurl = info.get("fullurl")
|
||||
if not pageid:
|
||||
print(f"Error: pageid für '{TITLE}' nicht gefunden.")
|
||||
sys.exit(1)
|
||||
raw = parse_exercise(TITLE, pageid)
|
||||
# capabilities als Dict wandeln
|
||||
caps_list = raw.get("capabilities", [])
|
||||
capabilities = {c["capability"]: c["level"] for c in caps_list}
|
||||
# Payload entsprechend Datenmodell
|
||||
exercise_payload = {
|
||||
"title": raw.get("title"),
|
||||
"summary": raw.get("Summary", ""),
|
||||
"short_description": raw.get("Summary", ""),
|
||||
"keywords": raw.get("Schlüsselworte", "").split(', '),
|
||||
"link": fullurl,
|
||||
"discipline": raw.get("Übungstyp", ""),
|
||||
"group": raw.get("Gruppengröße", ""),
|
||||
"age_group": raw.get("Altersgruppe", ""),
|
||||
"target_group": raw.get("Zielgruppe", ""),
|
||||
"min_participants": 1,
|
||||
"duration_minutes": float(raw.get("Dauer", 0)),
|
||||
"capabilities": capabilities,
|
||||
"category": raw.get("category", "Übungen"),
|
||||
"purpose": raw.get("Ziel", ""),
|
||||
"execution": raw.get("Durchführung", ""),
|
||||
"notes": raw.get("Hinweise", ""),
|
||||
"preparation": raw.get("RefMethode", ""),
|
||||
"method": raw.get("method", ""),
|
||||
"equipment": raw.get("equipment", []),
|
||||
"fullurl": fullurl
|
||||
}
|
||||
ingest_exercise(exercise_payload)
|
||||
except Exception as e:
|
||||
print(f"Fatal error: {e}")
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,136 @@
|
||||
"""
|
||||
Module: wiki_importer.py
|
||||
Beschreibung:
|
||||
- Importiert alle Übungen aus dem Wiki
|
||||
- Holt Liste aller Übungs-Titel und pageids via `/semantic/pages`
|
||||
- Für jede Übung:
|
||||
* Fetch pageinfo (pageid, fullurl)
|
||||
* Parse Wikitext (Templates: ÜbungInfoBox, Übungsbeschreibung, SkillDevelopment)
|
||||
* Baut Payload entsprechend Datenmodell
|
||||
* POST an `/exercise` Endpoint
|
||||
- Unterstützt Single-Import via Umgebungsvariable `WIKI_EXERCISE_TITLE` und Full-Import via `--all`
|
||||
Version: 2.0.0
|
||||
"""
|
||||
import requests
|
||||
import mwparserfromhell
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, Any, List
|
||||
import argparse
|
||||
|
||||
# Konfiguration
|
||||
API_BASE_URL = os.getenv("API_BASE_URL", "http://localhost:8000/import/wiki")
|
||||
EXERCISE_API = os.getenv("EXERCISE_API_URL", "http://localhost:8000/exercise")
|
||||
DEFAULT_CATEGORY = os.getenv("WIKI_CATEGORY", "Übungen")
|
||||
DEFAULT_TITLE = os.getenv("WIKI_EXERCISE_TITLE", "Affenklatschen")
|
||||
|
||||
# Helper: Holt Liste aller Übungen (Titel-> entry)
|
||||
def fetch_all_pages(category: str) -> Dict[str, Any]:
|
||||
resp = requests.get(f"{API_BASE_URL}/semantic/pages", params={"category": category})
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
# Helper: Holt pageid und fullurl per Core-API
|
||||
def fetch_page_info(title: str) -> Dict[str, Any]:
|
||||
r = requests.get(f"{API_BASE_URL}/info", params={"title": title})
|
||||
r.raise_for_status()
|
||||
info = r.json()
|
||||
return {"pageid": info.get("pageid"), "fullurl": info.get("fullurl")}
|
||||
|
||||
# Parser: Lädt und parst eine Übung
|
||||
def parse_exercise(title: str, pageid: int) -> Dict[str, Any]:
|
||||
print(f"[Parse] {title} (ID={pageid})")
|
||||
resp = requests.get(f"{API_BASE_URL}/parsepage", params={"pageid": pageid, "title": title})
|
||||
resp.raise_for_status()
|
||||
wikitext = resp.json().get("wikitext", "")
|
||||
wikicode = mwparserfromhell.parse(wikitext)
|
||||
|
||||
raw: Dict[str, Any] = {"title": title, "source": "MediaWiki", "pageid": pageid}
|
||||
for tpl in wikicode.filter_templates():
|
||||
name = tpl.name.strip()
|
||||
if name == "ÜbungInfoBox":
|
||||
for p in tpl.params:
|
||||
raw[p.name.strip()] = str(p.value).strip()
|
||||
elif name == "Übungsbeschreibung":
|
||||
for p in tpl.params:
|
||||
raw[p.name.strip()] = str(p.value).strip()
|
||||
elif name == "SkillDevelopment":
|
||||
raw.setdefault("capabilities", [])
|
||||
cap = str(tpl.get("PrimaryCapability").value).strip()
|
||||
lvl = int(str(tpl.get("CapabilityLevel").value).strip())
|
||||
raw["capabilities"].append({"capability": cap, "level": lvl})
|
||||
raw["wikitext"] = wikitext
|
||||
return raw
|
||||
|
||||
# Ingestion
|
||||
def ingest_exercise(payload: Dict[str, Any]) -> None:
|
||||
title = payload.get("title")
|
||||
resp = requests.post(EXERCISE_API, json=payload)
|
||||
if resp.status_code == 422:
|
||||
print(f"[Error] {title} -> 422: {resp.text}")
|
||||
return
|
||||
resp.raise_for_status()
|
||||
print(f"[Ingest] {title} -> OK")
|
||||
|
||||
# Build payload
|
||||
def build_payload(raw: Dict[str, Any], fullurl: str, category: str) -> Dict[str, Any]:
|
||||
caps_list = raw.get("capabilities", [])
|
||||
capabilities = {c["capability"]: c["level"] for c in caps_list}
|
||||
return {
|
||||
"title": raw.get("title"),
|
||||
"summary": raw.get("Summary", ""),
|
||||
"short_description": raw.get("Summary", ""),
|
||||
"keywords": raw.get("Schlüsselworte", "").split(', '),
|
||||
"link": fullurl,
|
||||
"discipline": raw.get("Übungstyp", ""),
|
||||
"group": raw.get("Gruppengröße", ""),
|
||||
"age_group": raw.get("Altersgruppe", ""),
|
||||
"target_group": raw.get("Zielgruppe", ""),
|
||||
"min_participants": 1,
|
||||
"duration_minutes": float(raw.get("Dauer", 0)),
|
||||
"capabilities": capabilities,
|
||||
"category": category,
|
||||
"purpose": raw.get("Ziel", ""),
|
||||
"execution": raw.get("Durchführung", ""),
|
||||
"notes": raw.get("Hinweise", ""),
|
||||
"preparation": raw.get("RefMethode", ""),
|
||||
"method": raw.get("method", ""),
|
||||
"equipment": raw.get("equipment", []),
|
||||
"fullurl": fullurl
|
||||
}
|
||||
|
||||
# Main
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Import exercises from Wiki to Qdrant")
|
||||
parser.add_argument("--all", action="store_true", help="Import all exercises")
|
||||
parser.add_argument("--title", type=str, default=DEFAULT_TITLE, help="Single exercise title")
|
||||
parser.add_argument("--category", type=str, default=DEFAULT_CATEGORY, help="Wiki category")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.all:
|
||||
pages = fetch_all_pages(args.category)
|
||||
print(f"Found {len(pages)} exercises in category '{args.category}'")
|
||||
for title, entry in pages.items():
|
||||
pid = entry.get("pageid")
|
||||
if not pid:
|
||||
info = fetch_page_info(title)
|
||||
pid = info.get("pageid")
|
||||
fullurl = info.get("fullurl")
|
||||
else:
|
||||
fullurl = entry.get("fullurl") or fetch_page_info(title)["fullurl"]
|
||||
if not pid:
|
||||
print(f"Skip {title}, no pageid")
|
||||
continue
|
||||
raw = parse_exercise(title, pid)
|
||||
payload = build_payload(raw, fullurl, args.category)
|
||||
ingest_exercise(payload)
|
||||
else:
|
||||
info = fetch_page_info(args.title)
|
||||
pid = info.get("pageid")
|
||||
fullurl = info.get("fullurl")
|
||||
if not pid:
|
||||
print(f"Error: pageid for '{args.title}' not found")
|
||||
sys.exit(1)
|
||||
raw = parse_exercise(args.title, pid)
|
||||
payload = build_payload(raw, fullurl, args.category)
|
||||
ingest_exercise(payload)
|
||||
Reference in new issue
Block a user