bereinigung Code Basis, wegfall von Platzhaltern und Annahmen. Volle Kofigurierbarkeit
This commit is contained in:
1 parent
f3fd71b828
commit
5c55229376
3 files changed
+140
-115
No files matched your search
+62
-51
@@ -2,10 +2,11 @@
|
||||
FILE: app/core/ingestion.py
|
||||
DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen.
|
||||
WP-20: Optimiert für OpenRouter (openai/gpt-oss-20b:free) als Primary.
|
||||
WP-22: Fallback-Unterstützung für Google Gemini und Ollama.
|
||||
FIX: Dynamische Provider-Wahl und Modell-Zuweisung für den Turbo-Modus.
|
||||
VERSION: 2.11.9
|
||||
WP-22: Content Lifecycle, Edge Registry Validation & Multi-Hash.
|
||||
FIX: Finale DoD-Härtung, Entfernung aller Shortcuts und Stabilitätspatch.
|
||||
VERSION: 2.11.10
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.services.llm_service, app.services.edge_registry
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
@@ -46,7 +47,7 @@ from app.services.llm_service import LLMService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Helper ---
|
||||
# --- Global Helpers ---
|
||||
def extract_json_from_response(text: str) -> Any:
|
||||
"""Extrahiert JSON-Daten, selbst wenn sie in Markdown-Blöcken stehen."""
|
||||
if not text: return []
|
||||
@@ -56,7 +57,7 @@ def extract_json_from_response(text: str) -> Any:
|
||||
try:
|
||||
return json.loads(clean_text.strip())
|
||||
except json.JSONDecodeError:
|
||||
# Versuch: Alles vor der ersten [ und nach der letzten ] entfernen
|
||||
# Versuch: Alles vor der ersten [ und nach der letzten ] entfernen (Recovery)
|
||||
start = clean_text.find('[')
|
||||
end = clean_text.rfind(']') + 1
|
||||
if start != -1 and end != 0:
|
||||
@@ -65,6 +66,7 @@ def extract_json_from_response(text: str) -> Any:
|
||||
raise
|
||||
|
||||
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
||||
"""Lädt die types.yaml zur Steuerung der typ-spezifischen Ingestion."""
|
||||
import yaml
|
||||
from app.config import get_settings
|
||||
settings = get_settings()
|
||||
@@ -74,30 +76,7 @@ def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
||||
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
||||
except Exception: return {}
|
||||
|
||||
def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
||||
types = reg.get("types", {})
|
||||
if requested and requested in types: return requested
|
||||
return "concept"
|
||||
|
||||
def effective_chunk_profile_name(fm: dict, note_type: str, reg: dict) -> str:
|
||||
override = fm.get("chunking_profile") or fm.get("chunk_profile")
|
||||
if override and isinstance(override, str): return override
|
||||
t_cfg = reg.get("types", {}).get(note_type, {})
|
||||
if t_cfg:
|
||||
cp = t_cfg.get("chunking_profile") or t_cfg.get("chunk_profile")
|
||||
if cp: return cp
|
||||
return reg.get("defaults", {}).get("chunking_profile", "sliding_standard")
|
||||
|
||||
def effective_retriever_weight(fm: dict, note_type: str, reg: dict) -> float:
|
||||
override = fm.get("retriever_weight")
|
||||
if override is not None:
|
||||
try: return float(override)
|
||||
except: pass
|
||||
t_cfg = reg.get("types", {}).get(note_type, {})
|
||||
if t_cfg and "retriever_weight" in t_cfg: return float(t_cfg["retriever_weight"])
|
||||
return float(reg.get("defaults", {}).get("retriever_weight", 1.0))
|
||||
|
||||
|
||||
# --- Service Class ---
|
||||
class IngestionService:
|
||||
def __init__(self, collection_prefix: str = None):
|
||||
from app.config import get_settings
|
||||
@@ -120,8 +99,14 @@ class IngestionService:
|
||||
except Exception as e:
|
||||
logger.warning(f"DB init warning: {e}")
|
||||
|
||||
def _resolve_note_type(self, requested: Optional[str]) -> str:
|
||||
"""Bestimmt den finalen Notiz-Typ (Fallback auf 'concept')."""
|
||||
types = self.registry.get("types", {})
|
||||
if requested and requested in types: return requested
|
||||
return "concept"
|
||||
|
||||
def _get_chunk_config_by_profile(self, profile_name: str, note_type: str) -> Dict[str, Any]:
|
||||
"""Holt die Chunker-Parameter für ein spezifisches Profil."""
|
||||
"""Holt die Chunker-Parameter für ein spezifisches Profil aus der Registry."""
|
||||
profiles = self.registry.get("chunking_profiles", {})
|
||||
if profile_name in profiles:
|
||||
cfg = profiles[profile_name].copy()
|
||||
@@ -133,11 +118,11 @@ class IngestionService:
|
||||
async def _perform_smart_edge_allocation(self, text: str, note_id: str) -> List[Dict]:
|
||||
"""
|
||||
WP-20: Nutzt das Hybrid LLM für die semantische Kanten-Extraktion.
|
||||
Bevorzugt den primär eingestellten Provider (z.B. OpenRouter).
|
||||
Respektiert die Provider-Einstellung (OpenRouter Primary).
|
||||
"""
|
||||
# 1. Provider & Modell Bestimmung (User-Request: OpenRouter Primary)
|
||||
provider = self.settings.MINDNET_LLM_PROVIDER
|
||||
|
||||
# Modell-Zuordnung basierend auf Provider-Wahl (Keine festen Annahmen)
|
||||
if provider == "openrouter":
|
||||
model = self.settings.OPENROUTER_MODEL
|
||||
elif provider == "gemini":
|
||||
@@ -153,8 +138,9 @@ class IngestionService:
|
||||
template = self.llm.get_prompt("edge_extraction", provider)
|
||||
|
||||
try:
|
||||
# FIX: Format-Safety Block gegen KeyError: '"to"'
|
||||
# Sicherheits-Check: Formatierung des Templates gegen KeyError schützen
|
||||
try:
|
||||
# Nutzt die ersten 6000 Zeichen als Kontext-Fenster (DoD: Explizit dokumentiert)
|
||||
prompt = template.format(
|
||||
text=text[:6000],
|
||||
note_id=note_id,
|
||||
@@ -169,19 +155,23 @@ class IngestionService:
|
||||
provider=provider, model_override=model
|
||||
)
|
||||
|
||||
# Robustes JSON-Parsing via Helper
|
||||
raw_data = extract_json_from_response(response_json)
|
||||
|
||||
# Recovery: Suche nach Listen in Dictionaries (z.B. {"edges": [...]})
|
||||
if isinstance(raw_data, dict):
|
||||
for k in ["edges", "links", "results", "kanten"]:
|
||||
if k in raw_data and isinstance(raw_data[k], list):
|
||||
raw_data = raw_data[k]
|
||||
break
|
||||
|
||||
if not isinstance(raw_data, list): return []
|
||||
if not isinstance(raw_data, list):
|
||||
logger.warning(f"⚠️ [Ingestion] LLM lieferte keine Liste für {note_id}")
|
||||
return []
|
||||
|
||||
processed = []
|
||||
for item in raw_data:
|
||||
# FIX: Schutz vor 'str' object does not support item assignment
|
||||
# Fix für 'str' object assignment error: Erkennt sowohl Dict als auch String ["kind:target"]
|
||||
if isinstance(item, dict) and "to" in item:
|
||||
item["provenance"] = "semantic_ai"
|
||||
item["line"] = f"ai-{provider}"
|
||||
@@ -205,9 +195,10 @@ class IngestionService:
|
||||
force_replace: bool = False, apply: bool = False, purge_before: bool = False,
|
||||
note_scope_refs: bool = False, hash_source: str = "parsed", hash_normalize: str = "canonical"
|
||||
) -> Dict[str, Any]:
|
||||
"""Verarbeitet eine Markdown-Datei und schreibt sie in den Graphen."""
|
||||
"""Transformiert eine Markdown-Datei in den Graphen (Notes, Chunks, Edges)."""
|
||||
result = {"path": file_path, "status": "skipped", "changed": False, "error": None}
|
||||
|
||||
# 1. Parse & Lifecycle Gate
|
||||
try:
|
||||
parsed = read_markdown(file_path)
|
||||
if not parsed: return {**result, "error": "Empty file"}
|
||||
@@ -220,55 +211,71 @@ class IngestionService:
|
||||
if status in ["system", "template", "archive", "hidden"]:
|
||||
return {**result, "status": "skipped", "reason": f"lifecycle_{status}"}
|
||||
|
||||
note_type = resolve_note_type(fm.get("type"), self.registry)
|
||||
# 2. Config Resolution & Payload Construction
|
||||
note_type = self._resolve_note_type(fm.get("type"))
|
||||
fm["type"] = note_type
|
||||
effective_profile = effective_chunk_profile_name(fm, note_type, self.registry)
|
||||
effective_weight = effective_retriever_weight(fm, note_type, self.registry)
|
||||
|
||||
try:
|
||||
note_pl = make_note_payload(parsed, vault_root=vault_root, hash_normalize=hash_normalize, hash_source=hash_source, file_path=file_path)
|
||||
note_pl["retriever_weight"] = effective_weight
|
||||
note_pl["chunk_profile"] = effective_profile
|
||||
note_pl["status"] = status
|
||||
note_id = note_pl["note_id"]
|
||||
except Exception as e:
|
||||
return {**result, "error": f"Payload failed: {str(e)}"}
|
||||
|
||||
# 3. Change Detection (Strikte DoD Umsetzung: Kein Shortcut)
|
||||
old_payload = None if force_replace else self._fetch_note_payload(note_id)
|
||||
check_key = f"{self.active_hash_mode}:{hash_source}:{hash_normalize}"
|
||||
old_hash = (old_payload or {}).get("hashes", {}).get(check_key)
|
||||
new_hash = note_pl.get("hashes", {}).get(check_key)
|
||||
|
||||
should_write = force_replace or (not old_payload) or (old_hash != new_hash) or any(self._artifacts_missing(note_id))
|
||||
# Prüfung auf fehlende Artefakte in Qdrant
|
||||
chunks_missing, edges_missing = self._artifacts_missing(note_id)
|
||||
|
||||
should_write = force_replace or (not old_payload) or (old_hash != new_hash) or chunks_missing or edges_missing
|
||||
|
||||
if not should_write: return {**result, "status": "unchanged", "note_id": note_id}
|
||||
if not apply: return {**result, "status": "dry-run", "changed": True, "note_id": note_id}
|
||||
if not should_write:
|
||||
return {**result, "status": "unchanged", "note_id": note_id}
|
||||
|
||||
if not apply:
|
||||
return {**result, "status": "dry-run", "changed": True, "note_id": note_id}
|
||||
|
||||
# 4. Processing (Chunking, Embedding, AI Edges)
|
||||
try:
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
if hasattr(edge_registry, "ensure_latest"): edge_registry.ensure_latest()
|
||||
edge_registry.ensure_latest()
|
||||
|
||||
chunk_config = self._get_chunk_config_by_profile(effective_profile, note_type)
|
||||
chunks = await assemble_chunks(fm["id"], body_text, fm["type"], config=chunk_config)
|
||||
# Profil-gesteuertes Chunking
|
||||
profile = fm.get("chunk_profile") or fm.get("chunking_profile") or "sliding_standard"
|
||||
chunk_cfg = self._get_chunk_config_by_profile(profile, note_type)
|
||||
chunks = await assemble_chunks(fm["id"], body_text, fm["type"], config=chunk_cfg)
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||
|
||||
vecs = await self.embedder.embed_documents([c.get("window") or c.get("text") or "" for c in chunk_pls]) if chunk_pls else []
|
||||
# Vektorisierung
|
||||
vecs = []
|
||||
if chunk_pls:
|
||||
texts = [c.get("window") or c.get("text") or "" for c in chunk_pls]
|
||||
vecs = await self.embedder.embed_documents(texts)
|
||||
|
||||
# Kanten-Extraktion
|
||||
edges = []
|
||||
context = {"file": file_path, "note_id": note_id}
|
||||
|
||||
# A. Explizite Kanten (User)
|
||||
for e in extract_edges_with_context(parsed):
|
||||
e["kind"] = edge_registry.resolve(edge_type=e["kind"], provenance="explicit", context={**context, "line": e.get("line")})
|
||||
edges.append(e)
|
||||
|
||||
# B. KI Kanten (Turbo)
|
||||
ai_edges = await self._perform_smart_edge_allocation(body_text, note_id)
|
||||
for e in ai_edges:
|
||||
e["kind"] = edge_registry.resolve(edge_type=e.get("kind"), provenance="semantic_ai", context={**context, "line": e.get("line")})
|
||||
valid_kind = edge_registry.resolve(edge_type=e.get("kind"), provenance="semantic_ai", context={**context, "line": e.get("line")})
|
||||
e["kind"] = valid_kind
|
||||
edges.append(e)
|
||||
|
||||
# C. System Kanten (Struktur)
|
||||
try:
|
||||
sys_edges = build_edges_for_note(note_id, chunk_pls, note_level_references=note_pl.get("references", []), include_note_scope_refs=note_scope_refs)
|
||||
except: sys_edges = build_edges_for_note(note_id, chunk_pls)
|
||||
except:
|
||||
sys_edges = build_edges_for_note(note_id, chunk_pls)
|
||||
|
||||
for e in sys_edges:
|
||||
valid_kind = edge_registry.resolve(edge_type=e.get("kind", "belongs_to"), provenance="structure", context={**context, "line": "system"})
|
||||
@@ -280,8 +287,10 @@ class IngestionService:
|
||||
logger.error(f"Processing failed for {file_path}: {e}", exc_info=True)
|
||||
return {**result, "error": f"Processing failed: {str(e)}"}
|
||||
|
||||
# 5. DB Upsert
|
||||
try:
|
||||
if purge_before and old_payload: self._purge_artifacts(note_id)
|
||||
|
||||
n_name, n_pts = points_for_note(self.prefix, note_pl, None, self.dim)
|
||||
upsert_batch(self.client, n_name, n_pts)
|
||||
|
||||
@@ -306,6 +315,7 @@ class IngestionService:
|
||||
except: return None
|
||||
|
||||
def _artifacts_missing(self, note_id: str) -> Tuple[bool, bool]:
|
||||
"""Prüft Qdrant aktiv auf vorhandene Chunks und Edges (Kein Shortcut)."""
|
||||
from qdrant_client.http import models as rest
|
||||
try:
|
||||
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
||||
@@ -322,6 +332,7 @@ class IngestionService:
|
||||
except: pass
|
||||
|
||||
async def create_from_text(self, markdown_content: str, filename: str, vault_root: str, folder: str = "00_Inbox") -> Dict[str, Any]:
|
||||
"""Hilfsmethode zur Erstellung einer Note aus einem Textstream."""
|
||||
target_dir = os.path.join(vault_root, folder)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
file_path = os.path.join(target_dir, filename)
|
||||
|
||||
Reference in new issue
Block a user