Update graph_derive_edges.py and ingestion_chunk_payload.py for version 4.3.0: Introduce debug logging for data transfer audits and candidate pool handling to address potential data loss. Ensure candidate_pool is explicitly retained for accurate chunk attribution, enhancing traceability and reliability in edge processing.
This commit is contained in:
parent
727de50290
commit
3a17b646e1
|
|
@ -17,7 +17,10 @@ DESCRIPTION: Hauptlogik zur Kanten-Aggregation und De-Duplizierung.
|
|||
- Gruppierung nach (kind, source, target, section) unabhängig vom Scope
|
||||
- Scope-Entscheidung: explicit:note_zone > chunk-Scope
|
||||
- ID-Berechnung erst nach Scope-Entscheidung
|
||||
VERSION: 4.2.2 (WP-24c: Semantische De-Duplizierung)
|
||||
WP-24c v4.3.0: Lokalisierung des Datenverlusts
|
||||
- Debug-Logik für Audit des Datentransfers
|
||||
- Verifizierung der candidate_pool Übertragung
|
||||
VERSION: 4.3.0 (WP-24c: Datenverlust-Lokalisierung)
|
||||
STATUS: Active
|
||||
"""
|
||||
import re
|
||||
|
|
@ -302,12 +305,23 @@ def build_edges_for_note(
|
|||
# PHASE 1 (Sicherung der Chunk-Autorität): Sammle alle Callout-Keys aus candidate_pool
|
||||
# BEVOR der globale Markdown-Scan oder der Loop über die Chunks beginnt
|
||||
# Dies stellt sicher, dass bereits geerntete Callouts nicht dupliziert werden
|
||||
# WP-24c v4.3.0: Debug-Logik für Audit des Datentransfers
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
for ch in chunks:
|
||||
cid = _get(ch, "chunk_id", "id")
|
||||
if not cid: continue
|
||||
|
||||
# Iteriere durch candidate_pool und sammle explicit:callout Kanten
|
||||
pool = ch.get("candidate_pool") or ch.get("candidate_edges") or []
|
||||
|
||||
# WP-24c v4.3.0: Debug-Logik - Ausgabe der Pool-Größe
|
||||
pool_size = len(pool)
|
||||
explicit_callout_count = sum(1 for cand in pool if cand.get("provenance") == "explicit:callout")
|
||||
if pool_size > 0:
|
||||
logger.debug(f"Note [{note_id}]: Chunk [{ch.get('index', '?')}] hat {pool_size} Kanten im Candidate-Pool ({explicit_callout_count} explicit:callout)")
|
||||
|
||||
for cand in pool:
|
||||
raw_t = cand.get("to")
|
||||
k = cand.get("kind", "related_to")
|
||||
|
|
@ -320,6 +334,13 @@ def build_edges_for_note(
|
|||
# Key-Format: (kind, target, section) für Multigraph-Präzision
|
||||
# Dies verhindert, dass der globale Scan diese Kante als Note-Scope neu anlegt
|
||||
all_chunk_callout_keys.add((k, t, sec))
|
||||
logger.debug(f"Note [{note_id}]: Callout-Key gesammelt: ({k}, {t}, {sec})")
|
||||
|
||||
# WP-24c v4.3.0: Debug-Logik - Ausgabe der gesammelten Keys
|
||||
if all_chunk_callout_keys:
|
||||
logger.debug(f"Note [{note_id}]: Gesammelt {len(all_chunk_callout_keys)} Callout-Keys aus candidate_pools")
|
||||
else:
|
||||
logger.warning(f"Note [{note_id}]: KEINE Callout-Keys in candidate_pools gefunden - möglicher Datenverlust!")
|
||||
|
||||
# WP-24c v4.2.9: PHASE 2: Verarbeite Chunks und erstelle Kanten
|
||||
for ch in chunks:
|
||||
|
|
@ -440,11 +461,21 @@ def build_edges_for_note(
|
|||
# dürfen nicht erneut als Note-Scope Kanten angelegt werden.
|
||||
callout_edges_from_markdown: List[dict] = []
|
||||
if markdown_body:
|
||||
# WP-24c v4.3.0: Debug-Logik - Ausgabe vor globalem Scan
|
||||
logger.debug(f"Note [{note_id}]: Starte globalen Markdown-Scan mit {len(all_chunk_callout_keys)} ausgeschlossenen Callout-Keys")
|
||||
|
||||
callout_edges_from_markdown = extract_callouts_from_markdown(
|
||||
markdown_body,
|
||||
note_id,
|
||||
existing_chunk_callouts=all_chunk_callout_keys # WP-24c v4.2.9 Fix B: Strikte Respektierung
|
||||
)
|
||||
|
||||
# WP-24c v4.3.0: Debug-Logik - Ausgabe nach globalem Scan
|
||||
if callout_edges_from_markdown:
|
||||
logger.debug(f"Note [{note_id}]: Globaler Scan erzeugte {len(callout_edges_from_markdown)} Note-Scope Callout-Kanten")
|
||||
else:
|
||||
logger.debug(f"Note [{note_id}]: Globaler Scan erzeugte KEINE Note-Scope Callout-Kanten (alle bereits in Chunks)")
|
||||
|
||||
edges.extend(callout_edges_from_markdown)
|
||||
|
||||
# 6) WP-24c v4.2.2: Semantische De-Duplizierung mit Scope-Entscheidung
|
||||
|
|
@ -500,6 +531,11 @@ def build_edges_for_note(
|
|||
# WP-24c v4.2.2: Berechne edge_id mit finalem Scope
|
||||
final_edge_id = _mk_edge_id(kind, final_source, target_id, final_scope, target_section=target_section)
|
||||
winner["edge_id"] = final_edge_id
|
||||
|
||||
# WP-24c v4.3.0: Debug-Logik - Ausgabe für Callout-Kanten
|
||||
if winner.get("provenance") == "explicit:callout":
|
||||
logger.debug(f"Note [{note_id}]: Finale Callout-Kante (single): scope={final_scope}, source={final_source}, target={target_id}, section={target_section}")
|
||||
|
||||
final_edges.append(winner)
|
||||
else:
|
||||
# Mehrere Kanten mit gleichem semantischen Schlüssel: Scope-Entscheidung
|
||||
|
|
@ -531,6 +567,11 @@ def build_edges_for_note(
|
|||
|
||||
final_edge_id = _mk_edge_id(kind, final_source, target_id, final_scope, target_section=target_section)
|
||||
winner["edge_id"] = final_edge_id
|
||||
|
||||
# WP-24c v4.3.0: Debug-Logik - Ausgabe für Callout-Kanten bei Deduplizierung
|
||||
if winner.get("provenance") == "explicit:callout":
|
||||
logger.debug(f"Note [{note_id}]: Finale Callout-Kante (deduped, {len(group)} Kandidaten): scope={final_scope}, source={final_source}, target={target_id}, section={target_section}")
|
||||
|
||||
final_edges.append(winner)
|
||||
|
||||
return final_edges
|
||||
|
|
@ -2,7 +2,8 @@
|
|||
FILE: app/core/ingestion/ingestion_chunk_payload.py
|
||||
DESCRIPTION: Baut das JSON-Objekt für 'mindnet_chunks'.
|
||||
Fix v2.4.3: Integration der zentralen Registry (WP-14) für konsistente Defaults.
|
||||
VERSION: 2.4.3
|
||||
WP-24c v4.3.0: candidate_pool wird explizit übernommen für Chunk-Attribution.
|
||||
VERSION: 2.4.4 (WP-24c v4.3.0)
|
||||
STATUS: Active
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
|
@ -85,6 +86,8 @@ def make_chunk_payloads(note: Dict[str, Any], note_path: str, chunks_from_chunke
|
|||
prev_id = getattr(ch, "neighbors_prev", None) if not is_dict else ch.get("neighbors_prev")
|
||||
next_id = getattr(ch, "neighbors_next", None) if not is_dict else ch.get("neighbors_next")
|
||||
section = getattr(ch, "section_title", "") if not is_dict else ch.get("section", "")
|
||||
# WP-24c v4.3.0: candidate_pool muss erhalten bleiben für Chunk-Attribution
|
||||
candidate_pool = getattr(ch, "candidate_pool", []) if not is_dict else ch.get("candidate_pool", [])
|
||||
|
||||
pl: Dict[str, Any] = {
|
||||
"note_id": nid or fm.get("id"),
|
||||
|
|
@ -102,7 +105,8 @@ def make_chunk_payloads(note: Dict[str, Any], note_path: str, chunks_from_chunke
|
|||
"path": note_path,
|
||||
"source_path": kwargs.get("file_path") or note_path,
|
||||
"retriever_weight": rw,
|
||||
"chunk_profile": cp
|
||||
"chunk_profile": cp,
|
||||
"candidate_pool": candidate_pool # WP-24c v4.3.0: Kritisch für Chunk-Attribution
|
||||
}
|
||||
|
||||
# Audit: Cleanup Pop (Vermeidung von redundanten Alias-Feldern)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user