Update chunking system to version 3.9.9, synchronizing parameters with the orchestrator and enhancing edge detection. Implement robust parsing to prevent duplicate edges in section propagation. Adjust comments for clarity and consistency across the codebase.
This commit is contained in:
1 parent
33dff04d47
commit
6aa6b32a6c
5 files changed
+68
-29
No files matched your search
@@ -4,8 +4,8 @@ DESCRIPTION: Der zentrale IngestionService (Orchestrator).
|
||||
WP-14: Modularisierung der Datenbank-Ebene (app.core.database).
|
||||
WP-15b: Two-Pass Workflow mit globalem Kontext-Cache.
|
||||
WP-20/22: Cloud-Resilienz und Content-Lifecycle integriert.
|
||||
AUDIT v2.13.10: Umstellung auf app.core.database Infrastruktur.
|
||||
VERSION: 2.13.10
|
||||
AUDIT v2.13.11: Synchronisierung mit Atomic-Chunking v3.9.9.
|
||||
VERSION: 2.13.11
|
||||
STATUS: Active
|
||||
"""
|
||||
import logging
|
||||
@@ -60,6 +60,7 @@ class IngestionService:
|
||||
self.embedder = EmbeddingsClient()
|
||||
self.llm = LLMService()
|
||||
|
||||
# Festlegen, welcher Hash für die Change-Detection maßgeblich ist
|
||||
self.active_hash_mode = self.settings.CHANGE_DETECTION_MODE
|
||||
self.batch_cache: Dict[str, NoteContext] = {} # WP-15b LocalBatchCache
|
||||
|
||||
@@ -130,12 +131,18 @@ class IngestionService:
|
||||
)
|
||||
note_id = note_pl["note_id"]
|
||||
|
||||
# Abgleich mit der Datenbank (Qdrant)
|
||||
old_payload = None if force_replace else fetch_note_payload(self.client, self.prefix, note_id)
|
||||
|
||||
# Prüfung gegen den konfigurierten Hash-Modus (body vs. full)
|
||||
check_key = f"{self.active_hash_mode}:{hash_source}:{hash_normalize}"
|
||||
old_hash = (old_payload or {}).get("hashes", {}).get(check_key)
|
||||
new_hash = note_pl.get("hashes", {}).get(check_key)
|
||||
|
||||
# Check ob Chunks oder Kanten in der DB fehlen (Reparatur-Modus)
|
||||
c_miss, e_miss = artifacts_missing(self.client, self.prefix, note_id)
|
||||
|
||||
# Wenn Hash identisch und Artefakte vorhanden -> Skip
|
||||
if not (force_replace or not old_payload or old_hash != new_hash or c_miss or e_miss):
|
||||
return {**result, "status": "unchanged", "note_id": note_id}
|
||||
|
||||
@@ -146,36 +153,46 @@ class IngestionService:
|
||||
try:
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
edge_registry.ensure_latest()
|
||||
|
||||
# Profil-Auflösung via Registry
|
||||
profile = fm.get("chunk_profile") or fm.get("chunking_profile") or "sliding_standard"
|
||||
chunk_cfg = get_chunk_config_by_profile(self.registry, profile, note_type)
|
||||
enable_smart = chunk_cfg.get("enable_smart_edge_allocation", False)
|
||||
|
||||
# WP-15b: Chunker-Aufruf bereitet Candidate-Pool vor
|
||||
# WP-15b: Chunker-Aufruf bereitet den Candidate-Pool pro Chunk vor.
|
||||
# assemble_chunks (v3.3.4) führt intern auch die Propagierung durch.
|
||||
chunks = await assemble_chunks(note_id, body_text, note_type, config=chunk_cfg)
|
||||
|
||||
# Semantische Kanten-Validierung (Smart Edge Allocation)
|
||||
for ch in chunks:
|
||||
filtered = []
|
||||
for cand in getattr(ch, "candidate_pool", []):
|
||||
# WP-15b: Nur global_pool Kandidaten erfordern binäre Validierung
|
||||
# Nur global_pool Kandidaten (aus dem Pool am Ende) erfordern KI-Validierung
|
||||
if cand.get("provenance") == "global_pool" and enable_smart:
|
||||
if await validate_edge_candidate(ch.text, cand, self.batch_cache, self.llm, self.settings.MINDNET_LLM_PROVIDER):
|
||||
filtered.append(cand)
|
||||
else:
|
||||
# Explizite Kanten (Wikilinks/Callouts) werden ungeprüft übernommen
|
||||
filtered.append(cand)
|
||||
ch.candidate_pool = filtered
|
||||
|
||||
# Payload-Erstellung via interne Module
|
||||
# Payload-Erstellung für die Chunks
|
||||
chunk_pls = make_chunk_payloads(
|
||||
fm, note_pl["path"], chunks, file_path=file_path,
|
||||
types_cfg=self.registry
|
||||
)
|
||||
|
||||
# Vektorisierung der Fenster-Texte
|
||||
vecs = await self.embedder.embed_documents([c.get("window") or "" for c in chunk_pls]) if chunk_pls else []
|
||||
|
||||
# Kanten-Aggregation
|
||||
# Aggregation aller finalen Kanten (Edges)
|
||||
edges = build_edges_for_note(
|
||||
note_id, chunk_pls,
|
||||
note_level_references=note_pl.get("references", []),
|
||||
include_note_scope_refs=note_scope_refs
|
||||
)
|
||||
|
||||
# Kanten-Typen via Registry validieren/auflösen
|
||||
for e in edges:
|
||||
e["kind"] = edge_registry.resolve(
|
||||
e.get("kind", "related_to"),
|
||||
@@ -184,16 +201,20 @@ class IngestionService:
|
||||
)
|
||||
|
||||
# 4. DB Upsert via modularisierter Points-Logik
|
||||
# WICHTIG: Wenn sich der Inhalt geändert hat, löschen wir erst alle alten Fragmente.
|
||||
if purge_before and old_payload:
|
||||
purge_artifacts(self.client, self.prefix, note_id)
|
||||
|
||||
# Speichern der Haupt-Note
|
||||
n_name, n_pts = points_for_note(self.prefix, note_pl, None, self.dim)
|
||||
upsert_batch(self.client, n_name, n_pts)
|
||||
|
||||
# Speichern der Chunks
|
||||
if chunk_pls and vecs:
|
||||
c_pts = points_for_chunks(self.prefix, chunk_pls, vecs)[1]
|
||||
upsert_batch(self.client, f"{self.prefix}_chunks", c_pts)
|
||||
|
||||
# Speichern der Kanten
|
||||
if edges:
|
||||
e_pts = points_for_edges(self.prefix, edges)[1]
|
||||
upsert_batch(self.client, f"{self.prefix}_edges", e_pts)
|
||||
@@ -217,4 +238,5 @@ class IngestionService:
|
||||
with open(target_path, "w", encoding="utf-8") as f:
|
||||
f.write(markdown_content)
|
||||
await asyncio.sleep(0.1)
|
||||
# Triggert sofortigen Import mit force_replace/purge_before
|
||||
return await self.process_file(file_path=target_path, vault_root=vault_root, apply=True, force_replace=True, purge_before=True)
|
||||
Reference in new issue
Block a user