chunker mit LLM initial
This commit is contained in:
1 parent
714763f92f
commit
13ab2b7d68
3 files changed
+197
-34
No files matched your search
@@ -0,0 +1,98 @@
|
||||
"""
|
||||
app/services/semantic_analyzer.py
|
||||
Kapselt die LLM-Strategie für Chunking und Kanten-Extraktion.
|
||||
Nutzt die Matrix-Logik aus DiscoveryService für konsistente Kanten-Typen.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import List, Dict, Any, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.services.llm_service import LLMService
|
||||
from app.services.discovery import DiscoveryService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class SemanticChunkResult:
|
||||
content: str
|
||||
suggested_edges: List[str] # Format: "kind:Target"
|
||||
|
||||
class SemanticAnalyzer:
|
||||
def __init__(self):
|
||||
self.llm = LLMService()
|
||||
self.discovery = DiscoveryService() # Wiederverwendung der Matrix-Logik
|
||||
|
||||
async def analyze_and_chunk(self, text: str, source_type: str) -> List[SemanticChunkResult]:
|
||||
"""
|
||||
Zerlegt Text mittels LLM in semantische Abschnitte und extrahiert Kanten.
|
||||
"""
|
||||
# 1. Prompt bauen
|
||||
system_prompt = (
|
||||
"Du bist ein Knowledge Graph Experte. Deine Aufgabe ist es, Rohtext in "
|
||||
"thematisch geschlossene Abschnitte (Chunks) zu zerlegen.\n"
|
||||
"Analysiere jeden Abschnitt auf Beziehungen zu anderen Konzepten.\n"
|
||||
"Antworte AUSSCHLIESSLICH mit validem JSON in diesem Format:\n"
|
||||
"[\n"
|
||||
" {\n"
|
||||
" \"content\": \"Der Text des Abschnitts...\",\n"
|
||||
" \"relations\": [{\"target\": \"Qdrant\", \"type\": \"depends_on\"}]\n"
|
||||
" }\n"
|
||||
"]\n"
|
||||
"Halte die Chunks mittellang (ca. 100-300 Wörter). Verändere den Inhalt nicht, nur die Struktur."
|
||||
)
|
||||
|
||||
user_prompt = f"Dokument-Typ: {source_type}\n\nTEXT:\n{text}"
|
||||
|
||||
try:
|
||||
# 2. LLM Call
|
||||
response_json = await self.llm.generate_raw_response(user_prompt, system=system_prompt)
|
||||
|
||||
# 3. JSON Parsing & Validierung
|
||||
# Markdown Code-Block entfernen falls vorhanden
|
||||
clean_json = response_json.replace("```json", "").replace("```", "").strip()
|
||||
data = json.loads(clean_json)
|
||||
|
||||
results = []
|
||||
for item in data:
|
||||
content = item.get("content", "").strip()
|
||||
if not content: continue
|
||||
|
||||
raw_rels = item.get("relations", [])
|
||||
refined_edges = []
|
||||
|
||||
for rel in raw_rels:
|
||||
target = rel.get("target")
|
||||
raw_type = rel.get("type", "related_to")
|
||||
|
||||
if target:
|
||||
# 4. Matrix-Logik anwenden (Active Intelligence)
|
||||
# Wir versuchen, den Typ des Ziels zu erraten oder nutzen Matrix blind
|
||||
# Hier vereinfacht: Wir nutzen Discovery Logic um den Edge-Typ zu validieren
|
||||
# (Wir nehmen an, Target Type ist unbekannt -> 'concept')
|
||||
final_kind = self.discovery._resolve_edge_type(source_type, "concept")
|
||||
|
||||
# Wenn LLM spezifischer war (z.B. 'blocks'), nehmen wir das LLM,
|
||||
# sonst den Matrix-Vorschlag
|
||||
if raw_type in ["related_to", "link"] and final_kind != "related_to":
|
||||
edge_str = f"{final_kind}:{target}"
|
||||
else:
|
||||
edge_str = f"{raw_type}:{target}"
|
||||
|
||||
refined_edges.append(edge_str)
|
||||
|
||||
results.append(SemanticChunkResult(content=content, suggested_edges=refined_edges))
|
||||
|
||||
return results
|
||||
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("SemanticAnalyzer: LLM lieferte kein valides JSON. Fallback auf Raw Text.")
|
||||
return [SemanticChunkResult(content=text, suggested_edges=[])]
|
||||
except Exception as e:
|
||||
logger.error(f"SemanticAnalyzer Error: {e}")
|
||||
return [SemanticChunkResult(content=text, suggested_edges=[])]
|
||||
|
||||
async def close(self):
|
||||
await self.llm.close()
|
||||
Reference in new issue
Block a user