Update chunking system to version 3.9.9, synchronizing parameters with the orchestrator and enhancing edge detection. Implement robust parsing to prevent duplicate edges in section propagation. Adjust comments for clarity and consistency across the codebase.

This commit is contained in:
Lars committed 2025-12-30 08:40:19 +01:00
1 parent 33dff04d47
commit 6aa6b32a6c
5 files changed
+68 -29

No files matched your search

+28 -6
View File
@@ -4,8 +4,8 @@ DESCRIPTION: Der zentrale IngestionService (Orchestrator).
WP-14: Modularisierung der Datenbank-Ebene (app.core.database).
WP-15b: Two-Pass Workflow mit globalem Kontext-Cache.
WP-20/22: Cloud-Resilienz und Content-Lifecycle integriert.
AUDIT v2.13.10: Umstellung auf app.core.database Infrastruktur.
VERSION: 2.13.10
AUDIT v2.13.11: Synchronisierung mit Atomic-Chunking v3.9.9.
VERSION: 2.13.11
STATUS: Active
"""
import logging
@@ -60,6 +60,7 @@ class IngestionService:
self.embedder = EmbeddingsClient()
self.llm = LLMService()
# Festlegen, welcher Hash für die Change-Detection maßgeblich ist
self.active_hash_mode = self.settings.CHANGE_DETECTION_MODE
self.batch_cache: Dict[str, NoteContext] = {} # WP-15b LocalBatchCache
@@ -130,12 +131,18 @@ class IngestionService:
)
note_id = note_pl["note_id"]
# Abgleich mit der Datenbank (Qdrant)
old_payload = None if force_replace else fetch_note_payload(self.client, self.prefix, note_id)
# Prüfung gegen den konfigurierten Hash-Modus (body vs. full)
check_key = f"{self.active_hash_mode}:{hash_source}:{hash_normalize}"
old_hash = (old_payload or {}).get("hashes", {}).get(check_key)
new_hash = note_pl.get("hashes", {}).get(check_key)
# Check ob Chunks oder Kanten in der DB fehlen (Reparatur-Modus)
c_miss, e_miss = artifacts_missing(self.client, self.prefix, note_id)
# Wenn Hash identisch und Artefakte vorhanden -> Skip
if not (force_replace or not old_payload or old_hash != new_hash or c_miss or e_miss):
return {**result, "status": "unchanged", "note_id": note_id}
@@ -146,36 +153,46 @@ class IngestionService:
try:
body_text = getattr(parsed, "body", "") or ""
edge_registry.ensure_latest()
# Profil-Auflösung via Registry
profile = fm.get("chunk_profile") or fm.get("chunking_profile") or "sliding_standard"
chunk_cfg = get_chunk_config_by_profile(self.registry, profile, note_type)
enable_smart = chunk_cfg.get("enable_smart_edge_allocation", False)
# WP-15b: Chunker-Aufruf bereitet Candidate-Pool vor
# WP-15b: Chunker-Aufruf bereitet den Candidate-Pool pro Chunk vor.
# assemble_chunks (v3.3.4) führt intern auch die Propagierung durch.
chunks = await assemble_chunks(note_id, body_text, note_type, config=chunk_cfg)
# Semantische Kanten-Validierung (Smart Edge Allocation)
for ch in chunks:
filtered = []
for cand in getattr(ch, "candidate_pool", []):
# WP-15b: Nur global_pool Kandidaten erfordern binäre Validierung
# Nur global_pool Kandidaten (aus dem Pool am Ende) erfordern KI-Validierung
if cand.get("provenance") == "global_pool" and enable_smart:
if await validate_edge_candidate(ch.text, cand, self.batch_cache, self.llm, self.settings.MINDNET_LLM_PROVIDER):
filtered.append(cand)
else:
# Explizite Kanten (Wikilinks/Callouts) werden ungeprüft übernommen
filtered.append(cand)
ch.candidate_pool = filtered
# Payload-Erstellung via interne Module
# Payload-Erstellung für die Chunks
chunk_pls = make_chunk_payloads(
fm, note_pl["path"], chunks, file_path=file_path,
types_cfg=self.registry
)
# Vektorisierung der Fenster-Texte
vecs = await self.embedder.embed_documents([c.get("window") or "" for c in chunk_pls]) if chunk_pls else []
# Kanten-Aggregation
# Aggregation aller finalen Kanten (Edges)
edges = build_edges_for_note(
note_id, chunk_pls,
note_level_references=note_pl.get("references", []),
include_note_scope_refs=note_scope_refs
)
# Kanten-Typen via Registry validieren/auflösen
for e in edges:
e["kind"] = edge_registry.resolve(
e.get("kind", "related_to"),
@@ -184,16 +201,20 @@ class IngestionService:
)
# 4. DB Upsert via modularisierter Points-Logik
# WICHTIG: Wenn sich der Inhalt geändert hat, löschen wir erst alle alten Fragmente.
if purge_before and old_payload:
purge_artifacts(self.client, self.prefix, note_id)
# Speichern der Haupt-Note
n_name, n_pts = points_for_note(self.prefix, note_pl, None, self.dim)
upsert_batch(self.client, n_name, n_pts)
# Speichern der Chunks
if chunk_pls and vecs:
c_pts = points_for_chunks(self.prefix, chunk_pls, vecs)[1]
upsert_batch(self.client, f"{self.prefix}_chunks", c_pts)
# Speichern der Kanten
if edges:
e_pts = points_for_edges(self.prefix, edges)[1]
upsert_batch(self.client, f"{self.prefix}_edges", e_pts)
@@ -217,4 +238,5 @@ class IngestionService:
with open(target_path, "w", encoding="utf-8") as f:
f.write(markdown_content)
await asyncio.sleep(0.1)
# Triggert sofortigen Import mit force_replace/purge_before
return await self.process_file(file_path=target_path, vault_root=vault_root, apply=True, force_replace=True, purge_before=True)