init: add knowledge, llm-api, scripts + repo settings
This commit is contained in:
commit
75f13af1f1
64 files changed
+6924
No files matched your search
@@ -0,0 +1,34 @@
|
||||
import re
|
||||
|
||||
def chunk_text_paragraphs(text: str, max_length: int = 500) -> list[str]:
|
||||
# Absätze trennen
|
||||
paragraphs = re.split(r'\n\s*\n', text.strip())
|
||||
chunks: list[str] = []
|
||||
current_chunk = ""
|
||||
|
||||
for para in paragraphs:
|
||||
para = para.strip()
|
||||
if not para:
|
||||
continue
|
||||
|
||||
# Passt der Absatz noch zum aktuellen Chunk? +2 für die später hinzugefügten "\n\n"
|
||||
if len(current_chunk) + len(para) + 2 <= max_length:
|
||||
current_chunk = (current_chunk + "\n\n" + para) if current_chunk else para
|
||||
else:
|
||||
# Aktuellen Chunk abschließen
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
# Ist der Absatz selbst zu groß? Dann hart splitten
|
||||
if len(para) > max_length:
|
||||
for i in range(0, len(para), max_length):
|
||||
chunks.append(para[i:i + max_length])
|
||||
current_chunk = ""
|
||||
else:
|
||||
current_chunk = para
|
||||
|
||||
# Letzten Chunk nicht vergessen
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
return chunks
|
||||
@@ -0,0 +1,93 @@
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
# 📌 Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
CHUNK_SIZE = 500
|
||||
OVERLAP = 100
|
||||
BATCH_SIZE = 20
|
||||
|
||||
# 📁 Kommandozeilenparameter auswerten
|
||||
if len(sys.argv) != 2:
|
||||
print("❌ Bitte gib eine Kategorie an, z. B.: python index_documents_advanced.py karatetrainer")
|
||||
sys.exit(1)
|
||||
|
||||
CATEGORY = sys.argv[1]
|
||||
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
|
||||
ARCHIVE_DIR = os.path.join(SOURCE_DIR, "_imported")
|
||||
COLLECTION = CATEGORY
|
||||
|
||||
if not os.path.exists(SOURCE_DIR):
|
||||
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(ARCHIVE_DIR, exist_ok=True)
|
||||
|
||||
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
|
||||
print(f"📂 Archivierte Dateien: {ARCHIVE_DIR}")
|
||||
print(f"🎯 Ziel-Collection: {COLLECTION}")
|
||||
|
||||
# 🔧 Text in überlappende Chunks aufteilen
|
||||
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
|
||||
chunks = []
|
||||
start = 0
|
||||
while start < len(text):
|
||||
end = min(start + size, len(text))
|
||||
chunks.append(text[start:end])
|
||||
start += size - overlap
|
||||
return chunks
|
||||
|
||||
# 📚 Alle .txt-Dateien im Ordner lesen
|
||||
def read_all_text_files(folder):
|
||||
file_chunk_map = {} # Map: filename → chunks
|
||||
for filename in os.listdir(folder):
|
||||
if filename.endswith(".txt"):
|
||||
path = os.path.join(folder, filename)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
file_chunk_map[filename] = chunk_text(text)
|
||||
return file_chunk_map
|
||||
|
||||
# 📤 An API senden
|
||||
def embed_chunks_in_batches(chunks, collection):
|
||||
results = []
|
||||
for i in tqdm(range(0, len(chunks), BATCH_SIZE), desc="📡 Embedding"):
|
||||
batch = chunks[i:i + BATCH_SIZE]
|
||||
response = requests.post(API_URL, json={"texts": batch, "collection": collection})
|
||||
response.raise_for_status()
|
||||
results.append(response.json())
|
||||
return results
|
||||
|
||||
# 🚀 Hauptlogik
|
||||
if __name__ == "__main__":
|
||||
file_chunk_map = read_all_text_files(SOURCE_DIR)
|
||||
all_chunks = []
|
||||
processed_files = []
|
||||
|
||||
for filename, chunks in file_chunk_map.items():
|
||||
if chunks:
|
||||
all_chunks.extend(chunks)
|
||||
processed_files.append(filename)
|
||||
|
||||
if not all_chunks:
|
||||
print("⚠️ Keine Textabschnitte gefunden.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"📦 {len(all_chunks)} Textabschnitte aus {len(processed_files)} Dateien gefunden.")
|
||||
|
||||
try:
|
||||
result = embed_chunks_in_batches(all_chunks, COLLECTION)
|
||||
print(f"\n✅ Embedding abgeschlossen: {len(result)} API-Antwort(en) erhalten.")
|
||||
|
||||
# 🗃️ Verarbeitete Dateien archivieren
|
||||
for filename in processed_files:
|
||||
src = os.path.join(SOURCE_DIR, filename)
|
||||
dst = os.path.join(ARCHIVE_DIR, filename)
|
||||
shutil.move(src, dst)
|
||||
print(f"📁 {len(processed_files)} Dateien verschoben nach _imported.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Senden: {e}")
|
||||
@@ -0,0 +1,61 @@
|
||||
import requests
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 📌 Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
CHUNK_SIZE = 500
|
||||
OVERLAP = 100
|
||||
|
||||
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
|
||||
"""
|
||||
Teilt einen Text in überlappende Abschnitte auf.
|
||||
"""
|
||||
chunks = []
|
||||
start = 0
|
||||
while start < len(text):
|
||||
end = min(start + size, len(text))
|
||||
chunks.append(text[start:end])
|
||||
start += size - overlap
|
||||
return chunks
|
||||
|
||||
def read_text_file(path):
|
||||
"""
|
||||
Liest den Inhalt einer Textdatei ein.
|
||||
"""
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
def import_chunks(chunks, collection):
|
||||
"""
|
||||
Sendet die Textabschnitte an die API.
|
||||
"""
|
||||
response = requests.post(API_URL, json={
|
||||
"texts": chunks,
|
||||
"collection": collection
|
||||
})
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("❌ Nutzung: python import_textfile.py <collection> <pfad_zur_txt_datei>")
|
||||
sys.exit(1)
|
||||
|
||||
collection = sys.argv[1]
|
||||
filepath = os.path.expanduser(sys.argv[2])
|
||||
|
||||
if not os.path.isfile(filepath):
|
||||
print(f"❌ Datei nicht gefunden: {filepath}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"📄 Lade Datei: {filepath}")
|
||||
text = read_text_file(filepath)
|
||||
chunks = chunk_text(text)
|
||||
|
||||
print(f"📦 {len(chunks)} Abschnitte vorbereitet – sende an Collection '{collection}'...")
|
||||
try:
|
||||
result = import_chunks(chunks, collection)
|
||||
print(f"✅ Import erfolgreich: {result}")
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Import: {e}")
|
||||
@@ -0,0 +1,31 @@
|
||||
import requests
|
||||
import sys
|
||||
|
||||
# 📌 Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
|
||||
def import_text(text, collection="default"):
|
||||
"""
|
||||
Sendet einen einzelnen Textabschnitt an die Embed-API zur Indexierung.
|
||||
"""
|
||||
response = requests.post(API_URL, json={
|
||||
"texts": [text],
|
||||
"collection": collection
|
||||
})
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print("❌ Nutzung: python import_texts.py <collection> \"<text>\"")
|
||||
sys.exit(1)
|
||||
|
||||
collection = sys.argv[1]
|
||||
text = sys.argv[2]
|
||||
|
||||
print(f"📤 Sende an Collection '{collection}': {text}")
|
||||
try:
|
||||
result = import_text(text, collection)
|
||||
print(f"✅ Antwort: {result}")
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Importieren: {e}")
|
||||
@@ -0,0 +1,66 @@
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
|
||||
# 📌 Konfiguration
|
||||
API_URL = "http://localhost:8000/embed"
|
||||
CHUNK_SIZE = 500
|
||||
OVERLAP = 100
|
||||
|
||||
# 📁 Kommandozeilenparameter auswerten
|
||||
if len(sys.argv) != 2:
|
||||
print("❌ Bitte gib eine Kategorie an, z. B.: python index_documents.py karatetrainer")
|
||||
sys.exit(1)
|
||||
|
||||
CATEGORY = sys.argv[1]
|
||||
SOURCE_DIR = os.path.expanduser(f"~/knowledge/{CATEGORY}")
|
||||
COLLECTION = CATEGORY
|
||||
|
||||
if not os.path.exists(SOURCE_DIR):
|
||||
print(f"❌ Der Ordner '{SOURCE_DIR}' existiert nicht.")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"📁 Lese Dokumente aus: {SOURCE_DIR}")
|
||||
print(f"🎯 Ziel-Collection: {COLLECTION}")
|
||||
|
||||
# 🔧 Text in überlappende Chunks aufteilen
|
||||
def chunk_text(text, size=CHUNK_SIZE, overlap=OVERLAP):
|
||||
chunks = []
|
||||
start = 0
|
||||
while start < len(text):
|
||||
end = min(start + size, len(text))
|
||||
chunks.append(text[start:end])
|
||||
start += size - overlap
|
||||
return chunks
|
||||
|
||||
# 📚 Alle .txt-Dateien im Ordner lesen
|
||||
def read_all_text_files(folder):
|
||||
chunks = []
|
||||
for filename in os.listdir(folder):
|
||||
if filename.endswith(".txt"):
|
||||
path = os.path.join(folder, filename)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
file_chunks = chunk_text(text)
|
||||
chunks.extend(file_chunks)
|
||||
return chunks
|
||||
|
||||
# 📤 An API senden
|
||||
def embed_chunks(chunks, collection):
|
||||
response = requests.post(API_URL, json={"texts": chunks, "collection": collection})
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
# 🚀 Hauptlogik
|
||||
if __name__ == "__main__":
|
||||
texts = read_all_text_files(SOURCE_DIR)
|
||||
if not texts:
|
||||
print("⚠️ Keine Textabschnitte gefunden.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"📦 {len(texts)} Textabschnitte gefunden, sende an {API_URL}...")
|
||||
try:
|
||||
result = embed_chunks(texts, COLLECTION)
|
||||
print(f"✅ Ergebnis: {result}")
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Senden: {e}")
|
||||
@@ -0,0 +1,46 @@
|
||||
import re
|
||||
|
||||
def chunk_text_paragraphs(text, max_length=500, overlap=1):
|
||||
"""
|
||||
Zerteilt den Text absatzweise in Chunks bis `max_length` Zeichen.
|
||||
Optional wird `paragraph_overlap` Absatz(e) vom vorherigen Chunk übernommen.
|
||||
"""
|
||||
|
||||
paragraphs = [p.strip() for p in re.split(r'\n{2,}', text) if p.strip()]
|
||||
chunks = []
|
||||
current_chunk = []
|
||||
current_len = 0
|
||||
|
||||
i = 0
|
||||
while i < len(paragraphs):
|
||||
para = paragraphs[i]
|
||||
para_len = len(para)
|
||||
|
||||
# Wenn dieser Absatz den max Chunk sprengt → neuen Chunk
|
||||
if current_len + para_len + 2 > max_length: # +2 für Leerzeile
|
||||
if current_chunk:
|
||||
chunks.append("\n\n".join(current_chunk))
|
||||
# Optional: letzte N Absätze behalten
|
||||
if overlap > 0:
|
||||
current_chunk = current_chunk[-overlap:]
|
||||
current_len = sum(len(p) for p in current_chunk) + 2 * len(current_chunk)
|
||||
else:
|
||||
current_chunk = []
|
||||
current_len = 0
|
||||
else:
|
||||
# Einzelner Absatz ist zu groß → hart splitten
|
||||
chunks.append(para[:max_length])
|
||||
para = para[max_length:]
|
||||
paragraphs.insert(i + 1, para) # Rest zurück in Liste
|
||||
i += 1
|
||||
continue
|
||||
else:
|
||||
current_chunk.append(para)
|
||||
current_len += para_len + 2 # +2 für Trennung
|
||||
|
||||
i += 1
|
||||
|
||||
if current_chunk:
|
||||
chunks.append("\n\n".join(current_chunk))
|
||||
|
||||
return chunks
|
||||
Reference in new issue
Block a user