komplette openrouter integration
This commit is contained in:
1 parent
c60aba63a4
commit
36fb27edf0
4 files changed
+96
-53
No files matched your search
+18
-14
@@ -5,7 +5,7 @@ DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen (Notes
|
||||
WP-22: Integration von Content Lifecycle (Status Gate) und Edge Registry Validation.
|
||||
WP-22: Kontextsensitive Kanten-Validierung mit Fundort-Reporting (Zeilennummern).
|
||||
WP-22: Multi-Hash Refresh für konsistente Change Detection.
|
||||
VERSION: 2.11.0 (WP-20 Full Integration: Hybrid Smart Edges)
|
||||
VERSION: 2.11.1 (WP-20 Quota Protection: OpenRouter Priority)
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.services.llm_service, app.services.edge_registry
|
||||
EXTERNAL_CONFIG: config/types.yaml, config/prompts.yaml
|
||||
@@ -111,7 +111,7 @@ class IngestionService:
|
||||
self.dim = self.cfg.dim if hasattr(self.cfg, 'dim') else self.settings.VECTOR_SIZE
|
||||
self.registry = load_type_registry()
|
||||
self.embedder = EmbeddingsClient()
|
||||
self.llm = LLMService() # WP-20 Integration
|
||||
self.llm = LLMService()
|
||||
|
||||
# Change Detection Modus (full oder body)
|
||||
self.active_hash_mode = os.getenv("MINDNET_CHANGE_DETECTION_MODE", "full")
|
||||
@@ -135,32 +135,36 @@ class IngestionService:
|
||||
async def _perform_smart_edge_allocation(self, text: str, note_id: str) -> List[Dict]:
|
||||
"""
|
||||
WP-20: Nutzt den Hybrid LLM Service für die semantische Kanten-Extraktion.
|
||||
Verwendet provider-spezifische Prompts aus der config.
|
||||
QUOTEN-SCHUTZ: Priorisiert OpenRouter (Gemma), um Gemini-Tageslimits zu schonen.
|
||||
"""
|
||||
# Wir priorisieren Gemma für Ingestion, falls verfügbar (OpenRouter/Cloud)
|
||||
model = getattr(self.settings, "GEMMA_MODEL", None)
|
||||
provider = self.settings.MINDNET_LLM_PROVIDER
|
||||
# Bestimme den Provider für die Ingestion (OpenRouter bevorzugt, falls Key vorhanden)
|
||||
provider = "openrouter" if getattr(self.settings, "OPENROUTER_API_KEY", None) else self.settings.MINDNET_LLM_PROVIDER
|
||||
|
||||
template = self.llm.get_prompt("edge_extraction")
|
||||
# Nutze Gemma-Modell für hohe Ingestion-Quoten (14.4K RPD) via OpenRouter oder Google
|
||||
model = getattr(self.settings, "GEMMA_MODEL", None)
|
||||
|
||||
# Hole Prompt aus der YAML (Kaskade: Provider -> gemini -> ollama)
|
||||
template = self.llm.get_prompt("edge_extraction", provider)
|
||||
prompt = template.format(text=text[:6000], note_id=note_id)
|
||||
|
||||
try:
|
||||
# Hintergrund-Task mit Semaphore
|
||||
# Hintergrund-Task mit Semaphore via LLMService
|
||||
response_json = await self.llm.generate_raw_response(
|
||||
prompt=prompt,
|
||||
priority="background",
|
||||
force_json=True,
|
||||
provider=provider,
|
||||
model_override=model
|
||||
)
|
||||
data = json.loads(response_json)
|
||||
|
||||
# Provenance für die EdgeRegistry
|
||||
# Provenance für die EdgeRegistry Dokumentation
|
||||
for item in data:
|
||||
item["provenance"] = "semantic_ai"
|
||||
item["line"] = f"ai-{provider}"
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning(f"Smart Edge Allocation skipped for {note_id}: {e}")
|
||||
logger.warning(f"Smart Edge Allocation failed for {note_id} on {provider}: {e}")
|
||||
return []
|
||||
|
||||
async def process_file(
|
||||
@@ -214,7 +218,7 @@ class IngestionService:
|
||||
logger.error(f"Payload build failed: {e}")
|
||||
return {**result, "error": f"Payload build failed: {str(e)}"}
|
||||
|
||||
# 4. Change Detection
|
||||
# 4. Change Detection (Multi-Hash)
|
||||
old_payload = None
|
||||
if not force_replace:
|
||||
old_payload = self._fetch_note_payload(note_id)
|
||||
@@ -255,7 +259,7 @@ class IngestionService:
|
||||
edges = []
|
||||
context = {"file": file_path, "note_id": note_id}
|
||||
|
||||
# A. Explizite User-Kanten
|
||||
# A. Explizite User-Kanten (Wiki-Links)
|
||||
explicit_edges = extract_edges_with_context(parsed)
|
||||
for e in explicit_edges:
|
||||
e["kind"] = edge_registry.resolve(edge_type=e["kind"], provenance="explicit", context={**context, "line": e.get("line")})
|
||||
@@ -267,7 +271,7 @@ class IngestionService:
|
||||
e["kind"] = edge_registry.resolve(edge_type=e.get("kind"), provenance="semantic_ai", context={**context, "line": e.get("line")})
|
||||
edges.append(e)
|
||||
|
||||
# C. System-Kanten
|
||||
# C. System-Kanten (Graph-Struktur)
|
||||
try:
|
||||
raw_system_edges = build_edges_for_note(note_id, chunk_pls, note_level_references=note_pl.get("references", []), include_note_scope_refs=note_scope_refs)
|
||||
except TypeError:
|
||||
@@ -282,7 +286,7 @@ class IngestionService:
|
||||
logger.error(f"Processing failed: {e}", exc_info=True)
|
||||
return {**result, "error": f"Processing failed: {str(e)}"}
|
||||
|
||||
# 6. Upsert
|
||||
# 6. Upsert in Qdrant
|
||||
try:
|
||||
if purge_before and has_old: self._purge_artifacts(note_id)
|
||||
|
||||
|
||||
Reference in new issue
Block a user