Enhance chunking strategies and graph utilities for section-type transitions and block ID extraction
- Implemented WP-26 v1.1: Section-Type-Wechsel erzwingt Split auch in SMART MODE (Schritt 2) zur Verbesserung der Chunking-Logik. - Updated `parse_link_target` to extract block IDs from section strings, ensuring accurate handling of links with block references. - Added unit tests to validate section-type change behavior and block ID extraction functionality, enhancing overall reliability.
This commit is contained in:
1 parent
af3cc0a254
commit
52fdc425f7
3 files changed
+121
No files matched your search
@@ -191,6 +191,36 @@ def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id:
|
||||
continue
|
||||
|
||||
# FALL B: SMART MODE (Regel 1-3)
|
||||
# WP-26 v1.1: Prüfe auf Section-Type-Wechsel AUCH in Schritt 2
|
||||
# Wenn sich der section_type zwischen current_meta und item ändert, muss gesplittet werden
|
||||
item_section_type = item.get("section_type")
|
||||
current_section_type_meta = current_meta.get("section_type")
|
||||
|
||||
# Section-Type-Wechsel: Von None zu einem Typ ODER von einem Typ zu einem anderen
|
||||
is_section_type_change_step2 = (
|
||||
current_chunk_text and # Es gibt bereits Content
|
||||
(
|
||||
# Wechsel von None zu einem Typ
|
||||
(current_section_type_meta is None and item_section_type is not None) or
|
||||
# Wechsel von einem Typ zu None
|
||||
(current_section_type_meta is not None and item_section_type is None) or
|
||||
# Wechsel zwischen verschiedenen Typen
|
||||
(current_section_type_meta is not None and item_section_type is not None
|
||||
and current_section_type_meta != item_section_type)
|
||||
)
|
||||
)
|
||||
|
||||
if is_section_type_change_step2:
|
||||
# WP-26 v1.1: Section-Type-Wechsel erzwingt Split
|
||||
_emit(current_chunk_text, current_meta["title"], current_meta["path"],
|
||||
current_meta["section_type"], current_meta["block_id"])
|
||||
current_chunk_text = ""
|
||||
# Reset Meta für nächsten Chunk
|
||||
current_meta["title"] = item["meta"].section_title
|
||||
current_meta["path"] = item["meta"].section_path
|
||||
current_meta["section_type"] = item_section_type
|
||||
current_meta["block_id"] = item.get("block_id")
|
||||
|
||||
combined_text = (current_chunk_text + "\n\n" + item_text).strip() if current_chunk_text else item_text
|
||||
combined_est = estimate_tokens(combined_text)
|
||||
|
||||
|
||||
Reference in new issue
Block a user