init: add knowledge, llm-api, scripts + repo settings

This commit is contained in:
Lars committed 2025-08-10 08:06:10 +02:00
commit 75f13af1f1
64 files changed
+6924

No files matched your search

+34
View File
@@ -0,0 +1,34 @@
import re
def chunk_text_paragraphs(text: str, max_length: int = 500) -> list[str]:
# Absätze trennen
paragraphs = re.split(r'\n\s*\n', text.strip())
chunks: list[str] = []
current_chunk = ""
for para in paragraphs:
para = para.strip()
if not para:
continue
# Passt der Absatz noch zum aktuellen Chunk? +2 für die später hinzugefügten "\n\n"
if len(current_chunk) + len(para) + 2 <= max_length:
current_chunk = (current_chunk + "\n\n" + para) if current_chunk else para
else:
# Aktuellen Chunk abschließen
if current_chunk:
chunks.append(current_chunk)
# Ist der Absatz selbst zu groß? Dann hart splitten
if len(para) > max_length:
for i in range(0, len(para), max_length):
chunks.append(para[i:i + max_length])
current_chunk = ""
else:
current_chunk = para
# Letzten Chunk nicht vergessen
if current_chunk:
chunks.append(current_chunk)
return chunks
+93
View File
@@ -0,0 +1,93 @@
import os
import sys
import shutil
import requests
from tqdm import tqdm
# 📌 Konfiguration
API_URL = "http://localhost:8000/embed"
CHUNK_SIZE = 500
OVERLAP = 100
BATCH_SIZE = 20
# 📁 Kommandozeilenparameter auswerten
if len(sys.argv) != 2:
print("❌ Bitte gib eine Kategorie an, z. B.: python index_documents_advanced.py karatetrainer")
sys.exit(1)
CATEGORY = sys.argv[1]
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
ARCHIVE_DIR = os.path.join(SOURCE_DIR, "_imported")
COLLECTION = CATEGORY
if not os.path.exists(SOURCE_DIR):
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
sys.exit(1)
os.makedirs(ARCHIVE_DIR, exist_ok=True)
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
print(f"📂 Archivierte Dateien: {ARCHIVE_DIR}")
print(f"🎯 Ziel-Collection: {COLLECTION}")
# 🔧 Text in überlappende Chunks aufteilen
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
chunks = []
start = 0
while start < len(text):
end = min(start + size, len(text))
chunks.append(text[start:end])
start += size - overlap
return chunks
# 📚 Alle .txt-Dateien im Ordner lesen
def read_all_text_files(folder):
file_chunk_map = {} # Map: filename → chunks
for filename in os.listdir(folder):
if filename.endswith(".txt"):
path = os.path.join(folder, filename)
with open(path, "r", encoding="utf-8") as f:
text = f.read()
file_chunk_map[filename] = chunk_text(text)
return file_chunk_map
# 📤 An API senden
def embed_chunks_in_batches(chunks, collection):
results = []
for i in tqdm(range(0, len(chunks), BATCH_SIZE), desc="📡 Embedding"):
batch = chunks[i:i + BATCH_SIZE]
response = requests.post(API_URL, json={"texts": batch, "collection": collection})
response.raise_for_status()
results.append(response.json())
return results
# 🚀 Hauptlogik
if __name__ == "__main__":
file_chunk_map = read_all_text_files(SOURCE_DIR)
all_chunks = []
processed_files = []
for filename, chunks in file_chunk_map.items():
if chunks:
all_chunks.extend(chunks)
processed_files.append(filename)
if not all_chunks:
print("⚠️ Keine Textabschnitte gefunden.")
sys.exit(0)
print(f"📦 {len(all_chunks)} Textabschnitte aus {len(processed_files)} Dateien gefunden.")
try:
result = embed_chunks_in_batches(all_chunks, COLLECTION)
print(f"\n✅ Embedding abgeschlossen: {len(result)} API-Antwort(en) erhalten.")
# 🗃️ Verarbeitete Dateien archivieren
for filename in processed_files:
src = os.path.join(SOURCE_DIR, filename)
dst = os.path.join(ARCHIVE_DIR, filename)
shutil.move(src, dst)
print(f"📁 {len(processed_files)} Dateien verschoben nach _imported.")
except Exception as e:
print(f"❌ Fehler beim Senden: {e}")
+61
View File
@@ -0,0 +1,61 @@
import requests
import os
import sys
# 📌 Konfiguration
API_URL = "http://localhost:8000/embed"
CHUNK_SIZE = 500
OVERLAP = 100
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
"""
Teilt einen Text in überlappende Abschnitte auf.
"""
chunks = []
start = 0
while start < len(text):
end = min(start + size, len(text))
chunks.append(text[start:end])
start += size - overlap
return chunks
def read_text_file(path):
"""
Liest den Inhalt einer Textdatei ein.
"""
with open(path, "r", encoding="utf-8") as f:
return f.read()
def import_chunks(chunks, collection):
"""
Sendet die Textabschnitte an die API.
"""
response = requests.post(API_URL, json={
"texts": chunks,
"collection": collection
})
response.raise_for_status()
return response.json()
if __name__ == "__main__":
if len(sys.argv) != 3:
print("❌ Nutzung: python import_textfile.py <collection> <pfad_zur_txt_datei>")
sys.exit(1)
collection = sys.argv[1]
filepath = os.path.expanduser(sys.argv[2])
if not os.path.isfile(filepath):
print(f"❌ Datei nicht gefunden: {filepath}")
sys.exit(1)
print(f"📄 Lade Datei: {filepath}")
text = read_text_file(filepath)
chunks = chunk_text(text)
print(f"📦 {len(chunks)} Abschnitte vorbereitet – sende an Collection '{collection}'...")
try:
result = import_chunks(chunks, collection)
print(f"✅ Import erfolgreich: {result}")
except Exception as e:
print(f"❌ Fehler beim Import: {e}")
+31
View File
@@ -0,0 +1,31 @@
import requests
import sys
# 📌 Konfiguration
API_URL = "http://localhost:8000/embed"
def import_text(text, collection="default"):
"""
Sendet einen einzelnen Textabschnitt an die Embed-API zur Indexierung.
"""
response = requests.post(API_URL, json={
"texts": [text],
"collection": collection
})
response.raise_for_status()
return response.json()
if __name__ == "__main__":
if len(sys.argv) < 3:
print("❌ Nutzung: python import_texts.py <collection> \"<text>\"")
sys.exit(1)
collection = sys.argv[1]
text = sys.argv[2]
print(f"📤 Sende an Collection '{collection}': {text}")
try:
result = import_text(text, collection)
print(f"✅ Antwort: {result}")
except Exception as e:
print(f"❌ Fehler beim Importieren: {e}")
+66
View File
@@ -0,0 +1,66 @@
import os
import sys
import requests
# 📌 Konfiguration
API_URL = "http://localhost:8000/embed"
CHUNK_SIZE = 500
OVERLAP = 100
# 📁 Kommandozeilenparameter auswerten
if len(sys.argv) != 2:
print("❌ Bitte gib eine Kategorie an, z. B.: python index_documents.py karatetrainer")
sys.exit(1)
CATEGORY = sys.argv[1]
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
COLLECTION = CATEGORY
if not os.path.exists(SOURCE_DIR):
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
sys.exit(1)
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
print(f"🎯 Ziel-Collection: {COLLECTION}")
# 🔧 Text in überlappende Chunks aufteilen
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
chunks = []
start = 0
while start < len(text):
end = min(start + size, len(text))
chunks.append(text[start:end])
start += size - overlap
return chunks
# 📚 Alle .txt-Dateien im Ordner lesen
def read_all_text_files(folder):
chunks = []
for filename in os.listdir(folder):
if filename.endswith(".txt"):
path = os.path.join(folder, filename)
with open(path, "r", encoding="utf-8") as f:
text = f.read()
file_chunks = chunk_text(text)
chunks.extend(file_chunks)
return chunks
# 📤 An API senden
def embed_chunks(chunks, collection):
response = requests.post(API_URL, json={"texts": chunks, "collection": collection})
response.raise_for_status()
return response.json()
# 🚀 Hauptlogik
if __name__ == "__main__":
texts = read_all_text_files(SOURCE_DIR)
if not texts:
print("⚠️ Keine Textabschnitte gefunden.")
sys.exit(0)
print(f"📦 {len(texts)} Textabschnitte gefunden, sende an {API_URL}...")
try:
result = embed_chunks(texts, COLLECTION)
print(f"✅ Ergebnis: {result}")
except Exception as e:
print(f"❌ Fehler beim Senden: {e}")
+46
View File
@@ -0,0 +1,46 @@
import re
def chunk_text_paragraphs(text, max_length=500, overlap=1):
"""
Zerteilt den Text absatzweise in Chunks bis `max_length` Zeichen.
Optional wird `paragraph_overlap` Absatz(e) vom vorherigen Chunk übernommen.
"""
paragraphs = [p.strip() for p in re.split(r'\n{2,}', text) if p.strip()]
chunks = []
current_chunk = []
current_len = 0
i = 0
while i < len(paragraphs):
para = paragraphs[i]
para_len = len(para)
# Wenn dieser Absatz den max Chunk sprengt → neuen Chunk
if current_len + para_len + 2 > max_length: # +2 für Leerzeile
if current_chunk:
chunks.append("\n\n".join(current_chunk))
# Optional: letzte N Absätze behalten
if overlap > 0:
current_chunk = current_chunk[-overlap:]
current_len = sum(len(p) for p in current_chunk) + 2 * len(current_chunk)
else:
current_chunk = []
current_len = 0
else:
# Einzelner Absatz ist zu groß → hart splitten
chunks.append(para[:max_length])
para = para[max_length:]
paragraphs.insert(i + 1, para) # Rest zurück in Liste
i += 1
continue
else:
current_chunk.append(para)
current_len += para_len + 2 # +2 für Trennung
i += 1
if current_chunk:
chunks.append("\n\n".join(current_chunk))
return chunks