305 lines
11 KiB
Python
305 lines
11 KiB
Python
"""
|
|
app/core/ingestion.py
|
|
|
|
Zentraler Service für die Transformation von Markdown-Dateien in Qdrant-Objekte.
|
|
Version: 2.5.2 (Full Feature: Change Detection + Robust IO + Clean Config)
|
|
"""
|
|
import os
|
|
import logging
|
|
import asyncio
|
|
import time
|
|
from typing import Dict, List, Optional, Tuple, Any
|
|
|
|
# Core Module Imports
|
|
from app.core.parser import (
|
|
read_markdown,
|
|
normalize_frontmatter,
|
|
validate_required_frontmatter,
|
|
)
|
|
from app.core.note_payload import make_note_payload
|
|
from app.core.chunker import assemble_chunks, get_chunk_config
|
|
from app.core.chunk_payload import make_chunk_payloads
|
|
|
|
# Fallback für Edges
|
|
try:
|
|
from app.core.derive_edges import build_edges_for_note
|
|
except ImportError:
|
|
def build_edges_for_note(*args, **kwargs): return []
|
|
|
|
from app.core.qdrant import QdrantConfig, get_client, ensure_collections, ensure_payload_indexes
|
|
from app.core.qdrant_points import (
|
|
points_for_chunks,
|
|
points_for_note,
|
|
points_for_edges,
|
|
upsert_batch,
|
|
)
|
|
|
|
from app.services.embeddings_client import EmbeddingsClient
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# --- Helper ---
|
|
def load_type_registry(custom_path: Optional[str] = None) -> dict:
|
|
import yaml
|
|
path = custom_path or os.getenv("MINDNET_TYPES_FILE", "config/types.yaml")
|
|
if not os.path.exists(path): return {}
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) or {}
|
|
except Exception: return {}
|
|
|
|
def resolve_note_type(requested: Optional[str], reg: dict) -> str:
|
|
types = reg.get("types", {})
|
|
if requested and requested in types: return requested
|
|
return "concept"
|
|
|
|
def effective_chunk_profile(note_type: str, reg: dict) -> str:
|
|
t_cfg = reg.get("types", {}).get(note_type, {})
|
|
if t_cfg and t_cfg.get("chunk_profile"):
|
|
return t_cfg.get("chunk_profile")
|
|
return reg.get("defaults", {}).get("chunk_profile", "default")
|
|
|
|
def effective_retriever_weight(note_type: str, reg: dict) -> float:
|
|
t_cfg = reg.get("types", {}).get(note_type, {})
|
|
if t_cfg and "retriever_weight" in t_cfg:
|
|
return float(t_cfg["retriever_weight"])
|
|
return float(reg.get("defaults", {}).get("retriever_weight", 1.0))
|
|
|
|
|
|
class IngestionService:
|
|
def __init__(self, collection_prefix: str = None):
|
|
env_prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
|
|
self.prefix = collection_prefix or env_prefix
|
|
|
|
self.cfg = QdrantConfig.from_env()
|
|
self.cfg.prefix = self.prefix
|
|
self.client = get_client(self.cfg)
|
|
self.dim = self.cfg.dim
|
|
self.registry = load_type_registry()
|
|
self.embedder = EmbeddingsClient()
|
|
|
|
try:
|
|
ensure_collections(self.client, self.prefix, self.dim)
|
|
ensure_payload_indexes(self.client, self.prefix)
|
|
except Exception as e:
|
|
logger.warning(f"DB init warning: {e}")
|
|
|
|
async def process_file(
|
|
self,
|
|
file_path: str,
|
|
vault_root: str,
|
|
force_replace: bool = False,
|
|
apply: bool = False,
|
|
purge_before: bool = False,
|
|
note_scope_refs: bool = False,
|
|
hash_mode: str = "body",
|
|
hash_source: str = "parsed",
|
|
hash_normalize: str = "canonical"
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Verarbeitet eine einzelne Datei (ASYNC).
|
|
Inklusive Change Detection (Hash-Check) gegen Qdrant.
|
|
"""
|
|
result = {
|
|
"path": file_path,
|
|
"status": "skipped",
|
|
"changed": False,
|
|
"error": None
|
|
}
|
|
|
|
# 1. Parse & Frontmatter Validation
|
|
try:
|
|
parsed = read_markdown(file_path)
|
|
if not parsed:
|
|
return {**result, "error": "Empty or unreadable file"}
|
|
|
|
fm = normalize_frontmatter(parsed.frontmatter)
|
|
validate_required_frontmatter(fm)
|
|
except Exception as e:
|
|
logger.error(f"Validation failed for {file_path}: {e}")
|
|
return {**result, "error": f"Validation failed: {str(e)}"}
|
|
|
|
# 2. Type & Config Resolution
|
|
note_type = resolve_note_type(fm.get("type"), self.registry)
|
|
fm["type"] = note_type
|
|
fm["chunk_profile"] = effective_chunk_profile(note_type, self.registry)
|
|
|
|
weight = fm.get("retriever_weight")
|
|
if weight is None:
|
|
weight = effective_retriever_weight(note_type, self.registry)
|
|
fm["retriever_weight"] = float(weight)
|
|
|
|
# 3. Build Note Payload
|
|
try:
|
|
note_pl = make_note_payload(
|
|
parsed,
|
|
vault_root=vault_root,
|
|
hash_mode=hash_mode,
|
|
hash_normalize=hash_normalize,
|
|
hash_source=hash_source,
|
|
file_path=file_path
|
|
)
|
|
if not note_pl.get("fulltext"):
|
|
note_pl["fulltext"] = getattr(parsed, "body", "") or ""
|
|
note_pl["retriever_weight"] = fm["retriever_weight"]
|
|
|
|
note_id = note_pl["note_id"]
|
|
except Exception as e:
|
|
logger.error(f"Payload build failed: {e}")
|
|
return {**result, "error": f"Payload build failed: {str(e)}"}
|
|
|
|
# 4. Change Detection (Das fehlende Stück!)
|
|
old_payload = None
|
|
if not force_replace:
|
|
old_payload = self._fetch_note_payload(note_id)
|
|
|
|
has_old = old_payload is not None
|
|
key_current = f"{hash_mode}:{hash_source}:{hash_normalize}"
|
|
old_hash = (old_payload or {}).get("hashes", {}).get(key_current)
|
|
new_hash = note_pl.get("hashes", {}).get(key_current)
|
|
|
|
hash_changed = (old_hash != new_hash)
|
|
chunks_missing, edges_missing = self._artifacts_missing(note_id)
|
|
|
|
should_write = force_replace or (not has_old) or hash_changed or chunks_missing or edges_missing
|
|
|
|
if not should_write:
|
|
return {**result, "status": "unchanged", "note_id": note_id}
|
|
|
|
if not apply:
|
|
return {**result, "status": "dry-run", "changed": True, "note_id": note_id}
|
|
|
|
# 5. Processing (Chunking, Embedding, Edges)
|
|
try:
|
|
body_text = getattr(parsed, "body", "") or ""
|
|
|
|
# --- Config Loading (Clean) ---
|
|
chunk_config = get_chunk_config(note_type)
|
|
# Hier greift die Logik aus types.yaml (smart=True/False)
|
|
|
|
chunks = await assemble_chunks(fm["id"], body_text, fm["type"], config=chunk_config)
|
|
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
|
|
|
# Embedding
|
|
vecs = []
|
|
if chunk_pls:
|
|
texts = [c.get("window") or c.get("text") or "" for c in chunk_pls]
|
|
try:
|
|
if hasattr(self.embedder, 'embed_documents'):
|
|
vecs = await self.embedder.embed_documents(texts)
|
|
else:
|
|
for t in texts:
|
|
v = await self.embedder.embed_query(t)
|
|
vecs.append(v)
|
|
except Exception as e:
|
|
logger.error(f"Embedding failed: {e}")
|
|
raise RuntimeError(f"Embedding failed: {e}")
|
|
|
|
# Edges
|
|
try:
|
|
edges = build_edges_for_note(
|
|
note_id,
|
|
chunk_pls,
|
|
note_level_references=note_pl.get("references", []),
|
|
include_note_scope_refs=note_scope_refs
|
|
)
|
|
except TypeError:
|
|
edges = build_edges_for_note(note_id, chunk_pls)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Processing failed: {e}", exc_info=True)
|
|
return {**result, "error": f"Processing failed: {str(e)}"}
|
|
|
|
# 6. Upsert Action
|
|
try:
|
|
if purge_before and has_old:
|
|
self._purge_artifacts(note_id)
|
|
|
|
n_name, n_pts = points_for_note(self.prefix, note_pl, None, self.dim)
|
|
upsert_batch(self.client, n_name, n_pts)
|
|
|
|
if chunk_pls and vecs:
|
|
c_name, c_pts = points_for_chunks(self.prefix, chunk_pls, vecs)
|
|
upsert_batch(self.client, c_name, c_pts)
|
|
|
|
if edges:
|
|
e_name, e_pts = points_for_edges(self.prefix, edges)
|
|
upsert_batch(self.client, e_name, e_pts)
|
|
|
|
return {
|
|
"path": file_path,
|
|
"status": "success",
|
|
"changed": True,
|
|
"note_id": note_id,
|
|
"chunks_count": len(chunk_pls),
|
|
"edges_count": len(edges)
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Upsert failed: {e}", exc_info=True)
|
|
return {**result, "error": f"DB Upsert failed: {e}"}
|
|
|
|
# --- Qdrant Helper (Restored) ---
|
|
|
|
def _fetch_note_payload(self, note_id: str) -> Optional[dict]:
|
|
from qdrant_client.http import models as rest
|
|
col = f"{self.prefix}_notes"
|
|
try:
|
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
|
pts, _ = self.client.scroll(collection_name=col, scroll_filter=f, limit=1, with_payload=True)
|
|
return pts[0].payload if pts else None
|
|
except: return None
|
|
|
|
def _artifacts_missing(self, note_id: str) -> Tuple[bool, bool]:
|
|
from qdrant_client.http import models as rest
|
|
c_col = f"{self.prefix}_chunks"
|
|
e_col = f"{self.prefix}_edges"
|
|
try:
|
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
|
c_pts, _ = self.client.scroll(collection_name=c_col, scroll_filter=f, limit=1)
|
|
e_pts, _ = self.client.scroll(collection_name=e_col, scroll_filter=f, limit=1)
|
|
return (not bool(c_pts)), (not bool(e_pts))
|
|
except: return True, True
|
|
|
|
def _purge_artifacts(self, note_id: str):
|
|
from qdrant_client.http import models as rest
|
|
f = rest.Filter(must=[rest.FieldCondition(key="note_id", match=rest.MatchValue(value=note_id))])
|
|
selector = rest.FilterSelector(filter=f)
|
|
for suffix in ["chunks", "edges"]:
|
|
try:
|
|
self.client.delete(collection_name=f"{self.prefix}_{suffix}", points_selector=selector)
|
|
except Exception: pass
|
|
|
|
async def create_from_text(
|
|
self,
|
|
markdown_content: str,
|
|
filename: str,
|
|
vault_root: str,
|
|
folder: str = "00_Inbox"
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
WP-11 Persistence API Entrypoint.
|
|
"""
|
|
target_dir = os.path.join(vault_root, folder)
|
|
os.makedirs(target_dir, exist_ok=True)
|
|
|
|
file_path = os.path.join(target_dir, filename)
|
|
|
|
try:
|
|
# Robust Write: Ensure Flush & Sync
|
|
with open(file_path, "w", encoding="utf-8") as f:
|
|
f.write(markdown_content)
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
|
|
await asyncio.sleep(0.1)
|
|
|
|
logger.info(f"Written file to {file_path}")
|
|
except Exception as e:
|
|
return {"status": "error", "error": f"Disk write failed: {str(e)}"}
|
|
|
|
return await self.process_file(
|
|
file_path=file_path,
|
|
vault_root=vault_root,
|
|
apply=True,
|
|
force_replace=True,
|
|
purge_before=True
|
|
) |