Update graph_derive_edges.py to version 4.2.1: Implement Clean-Context enhancements, including consolidated callout extraction and smart scope prioritization. Refactor callout handling to avoid duplicates and improve processing efficiency. Update documentation to reflect changes in edge extraction logic and prioritization strategy.
This commit is contained in:
1 parent
003a270548
commit
dfff46e45c
2 files changed
+387
-20
No files matched your search
@@ -9,11 +9,15 @@ DESCRIPTION: Hauptlogik zur Kanten-Aggregation und De-Duplizierung.
|
||||
- Header-basierte Identifikation von Note-Scope Zonen
|
||||
- Automatische Scope-Umschaltung (chunk -> note)
|
||||
- Priorisierung: Note-Scope Links haben Vorrang bei Duplikaten
|
||||
VERSION: 4.2.0 (WP-24c: Note-Scope Zones)
|
||||
WP-24c v4.2.1: Clean-Context Bereinigung
|
||||
- Konsolidierte Callout-Extraktion (keine Duplikate)
|
||||
- Smart Scope-Priorisierung (chunk bevorzugt, außer bei höherer Provenance)
|
||||
- Effiziente Verarbeitung ohne redundante Scans
|
||||
VERSION: 4.2.1 (WP-24c: Clean-Context Bereinigung)
|
||||
STATUS: Active
|
||||
"""
|
||||
import re
|
||||
from typing import List, Optional, Dict, Tuple
|
||||
from typing import List, Optional, Dict, Tuple, Set
|
||||
from .graph_utils import (
|
||||
_get, _edge, _mk_edge_id, _dedupe_seq, parse_link_target,
|
||||
PROVENANCE_PRIORITY, load_types_registry, get_edge_defaults_for
|
||||
@@ -104,9 +108,7 @@ def extract_note_scope_zones(markdown_body: str) -> List[Tuple[str, str]]:
|
||||
wikilinks = extract_wikilinks(zone_text)
|
||||
for wl in wikilinks:
|
||||
edges.append(("related_to", wl))
|
||||
# Extrahiere Callouts
|
||||
callouts, _ = extract_callout_relations(zone_text)
|
||||
edges.extend(callouts)
|
||||
# WP-24c v4.2.1: Callouts werden NICHT hier extrahiert, da sie global abgedeckt werden
|
||||
in_zone = False
|
||||
zone_content = []
|
||||
|
||||
@@ -122,8 +124,71 @@ def extract_note_scope_zones(markdown_body: str) -> List[Tuple[str, str]]:
|
||||
wikilinks = extract_wikilinks(zone_text)
|
||||
for wl in wikilinks:
|
||||
edges.append(("related_to", wl))
|
||||
callouts, _ = extract_callout_relations(zone_text)
|
||||
edges.extend(callouts)
|
||||
# WP-24c v4.2.1: Callouts werden NICHT hier extrahiert, da sie global abgedeckt werden
|
||||
|
||||
return edges
|
||||
|
||||
def extract_callouts_from_markdown(
|
||||
markdown_body: str,
|
||||
note_id: str,
|
||||
existing_chunk_callouts: Optional[Set[Tuple[str, str, Optional[str]]]] = None
|
||||
) -> List[dict]:
|
||||
"""
|
||||
WP-24c v4.2.1: Extrahiert Callouts aus dem Original-Markdown.
|
||||
|
||||
Smart Logic: Nur Callouts, die NICHT in Chunks vorkommen (z.B. in Edge-Zonen),
|
||||
werden mit scope: "note" angelegt. Callouts, die bereits in Chunks erfasst wurden,
|
||||
werden übersprungen, um Duplikate zu vermeiden.
|
||||
|
||||
Args:
|
||||
markdown_body: Original-Markdown-Text (vor Chunking-Filterung)
|
||||
note_id: ID der Note
|
||||
existing_chunk_callouts: Set von (kind, target, section) Tupeln aus Chunks
|
||||
|
||||
Returns:
|
||||
List[dict]: Liste von Edge-Payloads mit scope: "note"
|
||||
"""
|
||||
if not markdown_body:
|
||||
return []
|
||||
|
||||
if existing_chunk_callouts is None:
|
||||
existing_chunk_callouts = set()
|
||||
|
||||
edges: List[dict] = []
|
||||
|
||||
# Extrahiere alle Callouts aus dem gesamten Markdown
|
||||
call_pairs, _ = extract_callout_relations(markdown_body)
|
||||
|
||||
for k, raw_t in call_pairs:
|
||||
t, sec = parse_link_target(raw_t, note_id)
|
||||
if not t:
|
||||
continue
|
||||
|
||||
# WP-24c v4.2.1: Prüfe, ob dieser Callout bereits in einem Chunk vorkommt
|
||||
callout_key = (k, t, sec)
|
||||
if callout_key in existing_chunk_callouts:
|
||||
# Callout ist bereits in Chunk erfasst -> überspringe (wird mit chunk-Scope angelegt)
|
||||
continue
|
||||
|
||||
# WP-24c v4.2.1: Callout ist NICHT in Chunks -> lege mit scope: "note" an
|
||||
# (typischerweise in Edge-Zonen, die nicht gechunkt werden)
|
||||
payload = {
|
||||
"edge_id": _mk_edge_id(k, note_id, t, "note", target_section=sec),
|
||||
"provenance": "explicit:callout",
|
||||
"rule_id": "callout:edge",
|
||||
"confidence": PROVENANCE_PRIORITY.get("callout:edge", 1.0)
|
||||
}
|
||||
if sec:
|
||||
payload["target_section"] = sec
|
||||
|
||||
edges.append(_edge(
|
||||
kind=k,
|
||||
scope="note",
|
||||
source_id=note_id,
|
||||
target_id=t,
|
||||
note_id=note_id,
|
||||
extra=payload
|
||||
))
|
||||
|
||||
return edges
|
||||
|
||||
@@ -151,7 +216,10 @@ def build_edges_for_note(
|
||||
|
||||
# WP-24c v4.2.0: Note-Scope Zonen Extraktion (VOR Chunk-Verarbeitung)
|
||||
note_scope_edges: List[dict] = []
|
||||
|
||||
if markdown_body:
|
||||
# 1. Note-Scope Zonen (Wikilinks und Typed Relations)
|
||||
# WP-24c v4.2.1: Callouts werden NICHT hier extrahiert, da sie separat behandelt werden
|
||||
zone_links = extract_note_scope_zones(markdown_body)
|
||||
for kind, raw_target in zone_links:
|
||||
target, sec = parse_link_target(raw_target, note_id)
|
||||
@@ -213,6 +281,9 @@ def build_edges_for_note(
|
||||
reg = load_types_registry()
|
||||
defaults = get_edge_defaults_for(note_type, reg)
|
||||
refs_all: List[str] = []
|
||||
|
||||
# WP-24c v4.2.1: Sammle alle Callout-Keys aus Chunks für Smart Logic
|
||||
all_chunk_callout_keys: Set[Tuple[str, str, Optional[str]]] = set()
|
||||
|
||||
for ch in chunks:
|
||||
cid = _get(ch, "chunk_id", "id")
|
||||
@@ -249,12 +320,15 @@ def build_edges_for_note(
|
||||
if sec: payload["target_section"] = sec
|
||||
edges.append(_edge(k, "chunk", cid, t, note_id, payload))
|
||||
|
||||
# C. Callouts (> [!edge])
|
||||
# C. Callouts (> [!edge]) - WP-24c v4.2.1: Sammle für Smart Logic
|
||||
call_pairs, rem2 = extract_callout_relations(rem)
|
||||
for k, raw_t in call_pairs:
|
||||
t, sec = parse_link_target(raw_t, note_id)
|
||||
if not t: continue
|
||||
|
||||
# WP-24c v4.2.1: Tracke Callout für spätere Deduplizierung (global sammeln)
|
||||
all_chunk_callout_keys.add((k, t, sec))
|
||||
|
||||
# WP-24c v4.1.0: target_section fließt nun fest in die ID-Generierung ein
|
||||
payload = {
|
||||
"chunk_id": cid,
|
||||
@@ -312,11 +386,23 @@ def build_edges_for_note(
|
||||
}))
|
||||
|
||||
# 4) WP-24c v4.2.0: Note-Scope Edges hinzufügen (VOR De-Duplizierung)
|
||||
# Diese werden mit höherer Priorität behandelt, da sie explizite Note-Level Verbindungen sind
|
||||
edges.extend(note_scope_edges)
|
||||
|
||||
# 5) WP-24c v4.2.1: Callout-Extraktion aus Markdown (NACH Chunk-Verarbeitung)
|
||||
# Smart Logic: Nur Callouts, die NICHT in Chunks vorkommen, werden mit scope: "note" angelegt
|
||||
callout_edges_from_markdown: List[dict] = []
|
||||
if markdown_body:
|
||||
callout_edges_from_markdown = extract_callouts_from_markdown(
|
||||
markdown_body,
|
||||
note_id,
|
||||
existing_chunk_callouts=all_chunk_callout_keys
|
||||
)
|
||||
edges.extend(callout_edges_from_markdown)
|
||||
|
||||
# 5) De-Duplizierung (In-Place) mit Priorisierung
|
||||
# WP-24c v4.2.0: Note-Scope Links haben Vorrang bei Duplikaten
|
||||
# 6) De-Duplizierung (In-Place) mit Priorisierung
|
||||
# WP-24c v4.2.1: Smart Scope-Priorisierung
|
||||
# - chunk-Scope wird bevorzugt (präzisere Information für RAG)
|
||||
# - note-Scope gewinnt nur bei höherer Provenance-Priorität (z.B. explicit:note_zone)
|
||||
# WP-24c v4.1.0: Da die EDGE-ID nun auf 5 Parametern basiert (inkl. target_section),
|
||||
# bleiben Links auf unterschiedliche Abschnitte derselben Note als eigenständige
|
||||
# Kanten erhalten. Nur identische Sektions-Links werden nach Confidence und Provenance konsolidiert.
|
||||
@@ -324,18 +410,17 @@ def build_edges_for_note(
|
||||
for e in edges:
|
||||
eid = e["edge_id"]
|
||||
|
||||
# WP-24c v4.2.0: Priorisierung bei Duplikaten
|
||||
# 1. Note-Scope Links (explicit:note_zone) haben höchste Priorität
|
||||
# 2. Dann Confidence
|
||||
# 3. Dann Provenance-Priority
|
||||
if eid not in unique_map:
|
||||
unique_map[eid] = e
|
||||
else:
|
||||
existing = unique_map[eid]
|
||||
existing_scope = existing.get("scope", "chunk")
|
||||
new_scope = e.get("scope", "chunk")
|
||||
existing_prov = existing.get("provenance", "")
|
||||
new_prov = e.get("provenance", "")
|
||||
|
||||
# Note-Scope Zone Links haben Vorrang
|
||||
# WP-24c v4.2.1: Scope-Priorisierung
|
||||
# 1. explicit:note_zone hat höchste Priorität (unabhängig von Scope)
|
||||
is_existing_note_zone = existing_prov == "explicit:note_zone"
|
||||
is_new_note_zone = new_prov == "explicit:note_zone"
|
||||
|
||||
@@ -346,10 +431,27 @@ def build_edges_for_note(
|
||||
# Bestehender Link ist Note-Scope Zone -> behalte
|
||||
pass
|
||||
else:
|
||||
# Beide sind Note-Scope oder beide nicht -> vergleiche Confidence
|
||||
existing_conf = existing.get("confidence", 0)
|
||||
new_conf = e.get("confidence", 0)
|
||||
if new_conf > existing_conf:
|
||||
# 2. chunk-Scope bevorzugen (präzisere Information)
|
||||
if existing_scope == "chunk" and new_scope == "note":
|
||||
# Bestehender chunk-Scope -> behalte
|
||||
pass
|
||||
elif existing_scope == "note" and new_scope == "chunk":
|
||||
# Neuer chunk-Scope -> ersetze (präziser)
|
||||
unique_map[eid] = e
|
||||
else:
|
||||
# Gleicher Scope -> vergleiche Confidence und Provenance-Priority
|
||||
existing_conf = existing.get("confidence", 0)
|
||||
new_conf = e.get("confidence", 0)
|
||||
|
||||
# Provenance-Priority berücksichtigen
|
||||
existing_priority = PROVENANCE_PRIORITY.get(existing_prov, 0.7)
|
||||
new_priority = PROVENANCE_PRIORITY.get(new_prov, 0.7)
|
||||
|
||||
# Kombinierter Score: Confidence * Priority
|
||||
existing_score = existing_conf * existing_priority
|
||||
new_score = new_conf * new_priority
|
||||
|
||||
if new_score > existing_score:
|
||||
unique_map[eid] = e
|
||||
|
||||
return list(unique_map.values())
|
||||
Reference in new issue
Block a user