fix und umbau fast and slow lane für UI und batch
This commit is contained in:
1 parent
1fe9582cbe
commit
fc4c868eaa
2 files changed
+40
-35
No files matched your search
+28
-22
@@ -1,6 +1,6 @@
|
||||
"""
|
||||
app/services/llm_service.py — LLM Client (Ollama)
|
||||
Version: 0.5.2 (Fix: Removed strict limits, increased Context)
|
||||
app/services/llm_service.py — LLM Client
|
||||
Version: 2.7.0 (Clean Architecture: Explicit Priority Queues)
|
||||
"""
|
||||
|
||||
import httpx
|
||||
@@ -9,13 +9,12 @@ import logging
|
||||
import os
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
from typing import Optional, Dict, Any, Literal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class Settings:
|
||||
OLLAMA_URL = os.getenv("MINDNET_OLLAMA_URL", "http://127.0.0.1:11434")
|
||||
# Timeout für die Generierung (lang)
|
||||
LLM_TIMEOUT = float(os.getenv("MINDNET_LLM_TIMEOUT", 300.0))
|
||||
LLM_MODEL = os.getenv("MINDNET_LLM_MODEL", "phi3:mini")
|
||||
PROMPTS_PATH = os.getenv("MINDNET_PROMPTS_PATH", "./config/prompts.yaml")
|
||||
@@ -24,16 +23,13 @@ def get_settings():
|
||||
return Settings()
|
||||
|
||||
class LLMService:
|
||||
# GLOBALER SEMAPHOR (Drosselung für Hintergrund-Prozesse)
|
||||
_background_semaphore = asyncio.Semaphore(2)
|
||||
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
self.prompts = self._load_prompts()
|
||||
|
||||
# FIX 1: Keine künstlichen Limits mehr. httpx defaults (100) sind besser.
|
||||
# Wir wollen nicht, dass der Chat wartet, nur weil im Hintergrund Embeddings laufen.
|
||||
|
||||
# Timeout-Konfiguration:
|
||||
# connect=10.0: Wenn Ollama nicht da ist, failen wir schnell.
|
||||
# read=LLM_TIMEOUT: Wenn Ollama denkt, geben wir ihm Zeit.
|
||||
self.timeout = httpx.Timeout(self.settings.LLM_TIMEOUT, connect=10.0)
|
||||
|
||||
self.client = httpx.AsyncClient(
|
||||
@@ -43,11 +39,9 @@ class LLMService:
|
||||
|
||||
def _load_prompts(self) -> dict:
|
||||
path = Path(self.settings.PROMPTS_PATH)
|
||||
if not path.exists():
|
||||
return {}
|
||||
if not path.exists(): return {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return yaml.safe_load(f)
|
||||
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load prompts: {e}")
|
||||
return {}
|
||||
@@ -58,19 +52,31 @@ class LLMService:
|
||||
system: str = None,
|
||||
force_json: bool = False,
|
||||
max_retries: int = 0,
|
||||
base_delay: float = 2.0
|
||||
base_delay: float = 2.0,
|
||||
priority: Literal["realtime", "background"] = "realtime" # <--- NEU & EXPLIZIT
|
||||
) -> str:
|
||||
"""
|
||||
Führt einen LLM Call aus.
|
||||
Führt einen LLM Call aus.
|
||||
priority="realtime": Chat (Sofort, keine Bremse).
|
||||
priority="background": Import/Analyse (Gedrosselt durch Semaphore).
|
||||
"""
|
||||
|
||||
# Entscheidung basierend auf explizitem Parameter, nicht Format!
|
||||
use_semaphore = (priority == "background")
|
||||
|
||||
if use_semaphore:
|
||||
async with LLMService._background_semaphore:
|
||||
return await self._execute_request(prompt, system, force_json, max_retries, base_delay)
|
||||
else:
|
||||
return await self._execute_request(prompt, system, force_json, max_retries, base_delay)
|
||||
|
||||
async def _execute_request(self, prompt, system, force_json, max_retries, base_delay):
|
||||
payload: Dict[str, Any] = {
|
||||
"model": self.settings.LLM_MODEL,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.1 if force_json else 0.7,
|
||||
# FIX 2: Kontext auf 8192 erhöht.
|
||||
# Wichtig für komplexe Schemas und JSON-Stabilität.
|
||||
"num_ctx": 8192
|
||||
}
|
||||
}
|
||||
@@ -97,7 +103,6 @@ class LLMService:
|
||||
attempt += 1
|
||||
if attempt > max_retries:
|
||||
logger.error(f"LLM Final Error (Versuch {attempt}): {e}")
|
||||
# Wir werfen den Fehler weiter, damit der Router nicht "Interner Fehler" als Typ interpretiert
|
||||
raise e
|
||||
|
||||
wait_time = base_delay * (2 ** (attempt - 1))
|
||||
@@ -106,8 +111,7 @@ class LLMService:
|
||||
|
||||
async def generate_rag_response(self, query: str, context_str: str) -> str:
|
||||
"""
|
||||
WICHTIG FÜR CHAT:
|
||||
Kein JSON, keine Retries (User-Latency).
|
||||
Chat-Wrapper: Immer Realtime.
|
||||
"""
|
||||
system_prompt = self.prompts.get("system_prompt", "")
|
||||
rag_template = self.prompts.get("rag_template", "{context_str}\n\n{query}")
|
||||
@@ -117,7 +121,9 @@ class LLMService:
|
||||
return await self.generate_raw_response(
|
||||
final_prompt,
|
||||
system=system_prompt,
|
||||
max_retries=0
|
||||
max_retries=0,
|
||||
force_json=False,
|
||||
priority="realtime" # <--- Standard
|
||||
)
|
||||
|
||||
async def close(self):
|
||||
|
||||
Reference in new issue
Block a user