Enhance chunking strategies and graph utilities for section-type transitions
- Implemented WP-26 v1.1: Section-Type-Wechsel erzwingt immer einen neuen Chunk, um konsistente Chunking-Verhalten bei unterschiedlichen section_types zu gewährleisten. - Introduced automatic Intra-Note-Edges zwischen Sektionen mit unterschiedlichen Typen, um semantische Beziehungen zu erfassen. - Updated graph utilities to support automatic edge type derivation based on section transitions. - Added unit tests for section-type changes and automatic edge generation to ensure functionality and reliability.
This commit is contained in:
1 parent
cc258008dc
commit
af3cc0a254
4 files changed
+505
-50
No files matched your search
@@ -7,6 +7,8 @@ DESCRIPTION: Strategien für atomares Sektions-Chunking v3.9.9.
|
||||
- Fix: Synchronisierung der Parameter mit dem Orchestrator (context_prefix).
|
||||
WP-24c v4.2.5: Strict-Mode ohne Carry-Over - Bei strict_heading_split wird nach jeder Sektion geflasht.
|
||||
WP-26 v1.0: section_type und block_id werden an Chunks weitergegeben.
|
||||
WP-26 v1.1: Section-Type-Wechsel erzwingt IMMER einen neuen Chunk (unabhängig vom Profil).
|
||||
Gleiche section_types folgen dem normalen Chunking-Verhalten.
|
||||
"""
|
||||
from typing import List, Dict, Any, Optional
|
||||
from .chunking_models import RawBlock, Chunk
|
||||
@@ -53,33 +55,18 @@ def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id:
|
||||
|
||||
# --- SCHRITT 1: Gruppierung in atomare Sektions-Einheiten ---
|
||||
# WP-26 v1.0: Erweitert um section_type und block_id Tracking
|
||||
# WP-26 v1.1: Section-Type-Wechsel erzwingt IMMER eine neue Sektion (unabhängig vom Profil)
|
||||
sections: List[Dict[str, Any]] = []
|
||||
curr_blocks = []
|
||||
for b in blocks:
|
||||
if b.kind == "heading" and b.level <= split_level:
|
||||
if curr_blocks:
|
||||
# WP-26 v1.0: Finde den effektiven section_type und block_id für diese Sektion
|
||||
# Priorisiere den ersten Block mit section_type, sonst den Heading-Block
|
||||
effective_section_type = None
|
||||
effective_block_id = None
|
||||
for cb in curr_blocks:
|
||||
if cb.section_type and effective_section_type is None:
|
||||
effective_section_type = cb.section_type
|
||||
if cb.block_id and effective_block_id is None:
|
||||
effective_block_id = cb.block_id
|
||||
|
||||
sections.append({
|
||||
"text": "\n\n".join([x.text for x in curr_blocks]),
|
||||
"meta": curr_blocks[0],
|
||||
"is_empty": len(curr_blocks) == 1 and curr_blocks[0].kind == "heading",
|
||||
"section_type": effective_section_type,
|
||||
"block_id": effective_block_id
|
||||
})
|
||||
curr_blocks = [b]
|
||||
else:
|
||||
curr_blocks.append(b)
|
||||
if curr_blocks:
|
||||
# WP-26 v1.0: Gleiche Logik für den letzten Block
|
||||
current_section_type = None # WP-26 v1.1: Tracking des aktuellen section_type
|
||||
|
||||
def _flush_section():
|
||||
"""Hilfsfunktion zum Abschließen einer Sektion."""
|
||||
nonlocal curr_blocks, current_section_type
|
||||
if not curr_blocks:
|
||||
return
|
||||
# WP-26 v1.0: Finde den effektiven section_type und block_id für diese Sektion
|
||||
# Priorisiere den ersten Block mit section_type, sonst den Heading-Block
|
||||
effective_section_type = None
|
||||
effective_block_id = None
|
||||
for cb in curr_blocks:
|
||||
@@ -95,6 +82,41 @@ def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id:
|
||||
"section_type": effective_section_type,
|
||||
"block_id": effective_block_id
|
||||
})
|
||||
curr_blocks = []
|
||||
current_section_type = effective_section_type
|
||||
|
||||
for b in blocks:
|
||||
# WP-26 v1.1: Prüfe auf Section-Type-Wechsel BEVOR wir den Block hinzufügen
|
||||
# Wenn sich der section_type ändert, muss die aktuelle Sektion abgeschlossen werden
|
||||
block_section_type = b.section_type
|
||||
|
||||
# Section-Type-Wechsel erkannt?
|
||||
# (Wechsel ist: alter Typ != neuer Typ UND mindestens einer ist nicht None)
|
||||
is_section_type_change = (
|
||||
curr_blocks and # Es gibt bereits Blöcke
|
||||
block_section_type is not None and # Neuer Block hat expliziten section_type
|
||||
current_section_type != block_section_type # Typ hat sich geändert
|
||||
)
|
||||
|
||||
if b.kind == "heading" and b.level <= split_level:
|
||||
# Heading-basierter Split (Standard-Verhalten)
|
||||
_flush_section()
|
||||
curr_blocks = [b]
|
||||
current_section_type = block_section_type # Update tracking
|
||||
elif is_section_type_change:
|
||||
# WP-26 v1.1: Section-Type-Wechsel erzwingt neuen Chunk
|
||||
_flush_section()
|
||||
curr_blocks = [b]
|
||||
current_section_type = block_section_type
|
||||
else:
|
||||
# Normales Hinzufügen zum aktuellen Block
|
||||
curr_blocks.append(b)
|
||||
# Update section_type wenn Block einen hat und wir noch keinen haben
|
||||
if block_section_type and not current_section_type:
|
||||
current_section_type = block_section_type
|
||||
|
||||
# Letzte Sektion abschließen
|
||||
_flush_section()
|
||||
|
||||
# --- SCHRITT 2: Verarbeitung der Queue ---
|
||||
queue = list(sections)
|
||||
@@ -217,30 +239,17 @@ def strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note
|
||||
"""
|
||||
Standard-Sliding-Window für flache Texte ohne Sektionsfokus.
|
||||
WP-26 v1.0: Erweitert um section_type und block_id Weitergabe.
|
||||
WP-26 v1.1: Section-Type-Wechsel erzwingt IMMER einen neuen Chunk.
|
||||
"""
|
||||
target = config.get("target", 400); max_tokens = config.get("max", 600)
|
||||
chunks: List[Chunk] = []; buf: List[RawBlock] = []
|
||||
current_section_type = None # WP-26 v1.1: Tracking des aktuellen section_type
|
||||
|
||||
for b in blocks:
|
||||
b_tokens = estimate_tokens(b.text)
|
||||
curr_tokens = sum(estimate_tokens(x.text) for x in buf) if buf else 0
|
||||
if curr_tokens + b_tokens > max_tokens and buf:
|
||||
txt = "\n\n".join([x.text for x in buf]); idx = len(chunks)
|
||||
win = _create_win(context_prefix, buf[0].section_title, txt)
|
||||
# WP-26 v1.0: Finde effektiven section_type und block_id
|
||||
effective_section_type = next((b.section_type for b in buf if b.section_type), None)
|
||||
effective_block_id = next((b.block_id for b in buf if b.block_id), None)
|
||||
chunks.append(Chunk(
|
||||
id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx,
|
||||
text=txt, window=win, token_count=curr_tokens,
|
||||
section_title=buf[0].section_title, section_path=buf[0].section_path,
|
||||
neighbors_prev=None, neighbors_next=None,
|
||||
section_type=effective_section_type, block_id=effective_block_id
|
||||
))
|
||||
buf = []
|
||||
buf.append(b)
|
||||
|
||||
if buf:
|
||||
def _flush_buffer():
|
||||
"""Hilfsfunktion zum Flushen des Buffers."""
|
||||
nonlocal buf, current_section_type
|
||||
if not buf:
|
||||
return
|
||||
txt = "\n\n".join([x.text for x in buf]); idx = len(chunks)
|
||||
win = _create_win(context_prefix, buf[0].section_title, txt)
|
||||
# WP-26 v1.0: Finde effektiven section_type und block_id
|
||||
@@ -248,10 +257,36 @@ def strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note
|
||||
effective_block_id = next((b.block_id for b in buf if b.block_id), None)
|
||||
chunks.append(Chunk(
|
||||
id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx,
|
||||
text=txt, window=win, token_count=estimate_tokens(txt),
|
||||
text=txt, window=win, token_count=sum(estimate_tokens(x.text) for x in buf),
|
||||
section_title=buf[0].section_title, section_path=buf[0].section_path,
|
||||
neighbors_prev=None, neighbors_next=None,
|
||||
section_type=effective_section_type, block_id=effective_block_id
|
||||
))
|
||||
buf = []
|
||||
current_section_type = effective_section_type
|
||||
|
||||
for b in blocks:
|
||||
b_tokens = estimate_tokens(b.text)
|
||||
curr_tokens = sum(estimate_tokens(x.text) for x in buf) if buf else 0
|
||||
block_section_type = b.section_type
|
||||
|
||||
# WP-26 v1.1: Prüfe auf Section-Type-Wechsel
|
||||
is_section_type_change = (
|
||||
buf and # Es gibt bereits Blöcke im Buffer
|
||||
block_section_type is not None and # Neuer Block hat expliziten section_type
|
||||
current_section_type != block_section_type # Typ hat sich geändert
|
||||
)
|
||||
|
||||
# Flush wenn: Token-Limit überschritten ODER Section-Type-Wechsel
|
||||
if (curr_tokens + b_tokens > max_tokens and buf) or is_section_type_change:
|
||||
_flush_buffer()
|
||||
|
||||
buf.append(b)
|
||||
# Update section_type wenn Block einen hat und wir noch keinen haben
|
||||
if block_section_type and not current_section_type:
|
||||
current_section_type = block_section_type
|
||||
|
||||
# Letzten Buffer flushen
|
||||
_flush_buffer()
|
||||
|
||||
return chunks
|
||||
Reference in new issue
Block a user