Enhance chunking system with WP-24c v4.2.6 and v4.2.7 updates: Introduce is_meta_content flag for callouts in RawBlock, ensuring they are chunked but later removed for clean context. Update parse_blocks and propagate_section_edges to handle callout edges with explicit provenance for chunk attribution. Implement clean-context logic to remove callout syntax post-processing, maintaining chunk integrity. Adjust get_chunk_config to prioritize frontmatter overrides for chunking profiles. Update documentation to reflect these changes.
This commit is contained in:
1 parent
4d43cc526e
commit
55b64c331a
8 files changed
+231
-43
No files matched your search
@@ -4,10 +4,11 @@ DESCRIPTION: Zerlegt Markdown in logische Einheiten (RawBlocks).
|
||||
Hält alle Überschriftenebenen (H1-H6) im Stream.
|
||||
Stellt die Funktion parse_edges_robust zur Verfügung.
|
||||
WP-24c v4.2.0: Identifiziert Edge-Zonen und markiert sie für Chunking-Ausschluss.
|
||||
WP-24c v4.2.5: Callout-Exclusion - Callouts werden als separate RawBlocks identifiziert und ausgeschlossen.
|
||||
"""
|
||||
import re
|
||||
import os
|
||||
from typing import List, Tuple, Set
|
||||
from typing import List, Tuple, Set, Dict, Any
|
||||
from .chunking_models import RawBlock
|
||||
from .chunking_utils import extract_frontmatter_from_text
|
||||
|
||||
@@ -25,6 +26,7 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
"""
|
||||
Zerlegt Text in logische Einheiten (RawBlocks), inklusive H1-H6.
|
||||
WP-24c v4.2.0: Identifiziert Edge-Zonen (LLM-Validierung & Note-Scope) und markiert sie für Chunking-Ausschluss.
|
||||
WP-24c v4.2.6: Callouts werden mit is_meta_content=True markiert (werden gechunkt, aber später entfernt).
|
||||
"""
|
||||
blocks = []
|
||||
h1_title = "Dokument"
|
||||
@@ -67,9 +69,61 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
lines = text_without_fm.split('\n')
|
||||
buffer = []
|
||||
|
||||
for line in lines:
|
||||
# WP-24c v4.2.5: Callout-Erkennung (auch verschachtelt: >>)
|
||||
# Regex für Callouts: >\s*[!edge] oder >\s*[!abstract] (auch mit mehreren >)
|
||||
callout_pattern = re.compile(r'^\s*>{1,}\s*\[!(edge|abstract)\]', re.IGNORECASE)
|
||||
|
||||
# WP-24c v4.2.5: Markiere verarbeitete Zeilen, um sie zu überspringen
|
||||
processed_indices = set()
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if i in processed_indices:
|
||||
continue
|
||||
|
||||
stripped = line.strip()
|
||||
|
||||
# WP-24c v4.2.5: Callout-Erkennung (VOR Heading-Erkennung)
|
||||
# Prüfe, ob diese Zeile ein Callout startet
|
||||
callout_match = callout_pattern.match(line)
|
||||
if callout_match:
|
||||
# Vorherigen Text-Block abschließen
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content:
|
||||
blocks.append(RawBlock(
|
||||
"paragraph", content, None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
buffer = []
|
||||
|
||||
# Sammle alle Zeilen des Callout-Blocks
|
||||
callout_lines = [line]
|
||||
leading_gt_count = len(line) - len(line.lstrip('>'))
|
||||
processed_indices.add(i)
|
||||
|
||||
# Sammle alle Zeilen, die zum Callout gehören (gleiche oder höhere Einrückung)
|
||||
j = i + 1
|
||||
while j < len(lines):
|
||||
next_line = lines[j]
|
||||
if not next_line.strip().startswith('>'):
|
||||
break
|
||||
next_leading_gt = len(next_line) - len(next_line.lstrip('>'))
|
||||
if next_leading_gt < leading_gt_count:
|
||||
break
|
||||
callout_lines.append(next_line)
|
||||
processed_indices.add(j)
|
||||
j += 1
|
||||
|
||||
# WP-24c v4.2.6: Erstelle Callout-Block mit is_meta_content = True
|
||||
# Callouts werden gechunkt (für Chunk-Attribution), aber später entfernt (Clean-Context)
|
||||
callout_content = "\n".join(callout_lines)
|
||||
blocks.append(RawBlock(
|
||||
"callout", callout_content, None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone, # Nur Edge-Zonen werden ausgeschlossen
|
||||
is_meta_content=True # WP-24c v4.2.6: Markierung für spätere Entfernung
|
||||
))
|
||||
continue
|
||||
|
||||
# Heading-Erkennung (H1 bis H6)
|
||||
heading_match = re.match(r'^(#{1,6})\s+(.*)', stripped)
|
||||
if heading_match:
|
||||
@@ -148,15 +202,22 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
|
||||
return blocks, h1_title
|
||||
|
||||
def parse_edges_robust(text: str) -> Set[str]:
|
||||
"""Extrahiert Kanten-Kandidaten aus Wikilinks und Callouts."""
|
||||
found_edges = set()
|
||||
def parse_edges_robust(text: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Extrahiert Kanten-Kandidaten aus Wikilinks und Callouts.
|
||||
WP-24c v4.2.7: Gibt Liste von Dicts zurück mit is_callout Flag für Chunk-Attribution.
|
||||
|
||||
Returns:
|
||||
List[Dict] mit keys: "edge" (str: "kind:target"), "is_callout" (bool)
|
||||
"""
|
||||
found_edges: List[Dict[str, any]] = []
|
||||
# 1. Wikilinks [[rel:kind|target]]
|
||||
inlines = re.findall(r'\[\[rel:([^\|\]]+)\|?([^\]]*)\]\]', text)
|
||||
for kind, target in inlines:
|
||||
k = kind.strip().lower()
|
||||
t = target.strip()
|
||||
if k and t: found_edges.add(f"{k}:{t}")
|
||||
if k and t:
|
||||
found_edges.append({"edge": f"{k}:{t}", "is_callout": False})
|
||||
|
||||
# 2. Callout Edges > [!edge] kind
|
||||
lines = text.split('\n')
|
||||
@@ -169,13 +230,15 @@ def parse_edges_robust(text: str) -> Set[str]:
|
||||
# Links in der gleichen Zeile des Callouts
|
||||
links = re.findall(r'\[\[([^\]]+)\]\]', stripped)
|
||||
for l in links:
|
||||
if "rel:" not in l: found_edges.add(f"{current_edge_type}:{l}")
|
||||
if "rel:" not in l:
|
||||
found_edges.append({"edge": f"{current_edge_type}:{l}", "is_callout": True})
|
||||
continue
|
||||
# Links in Folgezeilen des Callouts
|
||||
if current_edge_type and stripped.startswith('>'):
|
||||
links = re.findall(r'\[\[([^\]]+)\]\]', stripped)
|
||||
for l in links:
|
||||
if "rel:" not in l: found_edges.add(f"{current_edge_type}:{l}")
|
||||
if "rel:" not in l:
|
||||
found_edges.append({"edge": f"{current_edge_type}:{l}", "is_callout": True})
|
||||
elif not stripped.startswith('>'):
|
||||
current_edge_type = None
|
||||
return found_edges
|
||||
Reference in new issue
Block a user