Aktualisierung des Chunking-Parsers zur Verbesserung der Blockverarbeitung und Beschreibung. Anpassungen an der atomaren Sektions-Chunking-Strategie mit optimierter Token-Schätzung und neuen Hilfsfunktionen zur besseren Handhabung von großen Sektionen. Einführung einer präziseren Schätzung für deutsche Texte und Anpassungen an der Logik zur Handhabung von Sektionen.
This commit is contained in:
1 parent
96b4f65cd1
commit
680c36ab59
2 files changed
+47
-75
No files matched your search
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
FILE: app/core/chunking/chunking_parser.py
|
||||
DESCRIPTION: Zerlegt Markdown in logische Einheiten (RawBlocks).
|
||||
Hält H1-Überschriften im Inhalts-Stream.
|
||||
DESCRIPTION: Zerlegt Markdown in logische Blöcke (RawBlocks).
|
||||
Gewährleistet, dass H1 und Trenner im Stream verbleiben.
|
||||
"""
|
||||
import re
|
||||
from typing import List, Tuple, Set
|
||||
@@ -20,28 +20,21 @@ def split_sentences(text: str) -> list[str]:
|
||||
def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
"""Zerlegt Text in logische Einheiten (RawBlocks), inklusive H1-H6."""
|
||||
blocks = []
|
||||
h1_title = "Dokument"
|
||||
section_path = "/"
|
||||
current_section_title = None
|
||||
|
||||
# Frontmatter entfernen
|
||||
h1_title = "Dokument"; section_path = "/"; current_section_title = None
|
||||
fm, text_without_fm = extract_frontmatter_from_text(md_text)
|
||||
|
||||
# H1 für Note-Titel extrahieren
|
||||
# H1 für Metadaten extrahieren
|
||||
h1_match = re.search(r'^#\s+(.*)', text_without_fm, re.MULTILINE)
|
||||
if h1_match:
|
||||
h1_title = h1_match.group(1).strip()
|
||||
if h1_match: h1_title = h1_match.group(1).strip()
|
||||
|
||||
lines = text_without_fm.split('\n')
|
||||
buffer = []
|
||||
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
|
||||
# Heading-Erkennung (H1 bis H6)
|
||||
heading_match = re.match(r'^(#{1,6})\s+(.*)', stripped)
|
||||
|
||||
if heading_match:
|
||||
# Vorherigen Text-Block abschließen
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content:
|
||||
@@ -51,19 +44,14 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
level = len(heading_match.group(1))
|
||||
title = heading_match.group(2).strip()
|
||||
|
||||
# Pfad- und Titel-Update für die Metadaten der folgenden Blöcke
|
||||
if level == 1:
|
||||
current_section_title = title
|
||||
section_path = "/"
|
||||
current_section_title = title; section_path = "/"
|
||||
elif level == 2:
|
||||
current_section_title = title
|
||||
section_path = f"/{current_section_title}"
|
||||
current_section_title = title; section_path = f"/{current_section_title}"
|
||||
|
||||
# Die Überschrift selbst als Block hinzufügen
|
||||
blocks.append(RawBlock("heading", stripped, level, section_path, current_section_title))
|
||||
continue
|
||||
|
||||
# Trenner oder Leerzeilen beenden Blöcke, außer innerhalb von Callouts
|
||||
if (not stripped or stripped == "---") and not line.startswith('>'):
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
@@ -77,22 +65,19 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
if content: blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
|
||||
return blocks, h1_title
|
||||
|
||||
def parse_edges_robust(text: str) -> Set[str]:
|
||||
"""Extrahiert Kanten-Kandidaten aus Wikilinks und Callouts."""
|
||||
"""Extrahiert Kanten aus Wikilinks und Callouts."""
|
||||
found_edges = set()
|
||||
inlines = re.findall(r'\[\[rel:([^\|\]]+)\|?([^\]]*)\]\]', text)
|
||||
for kind, target in inlines:
|
||||
k = kind.strip().lower()
|
||||
t = target.strip()
|
||||
k = kind.strip().lower(); t = target.strip()
|
||||
if k and t: found_edges.add(f"{k}:{t}")
|
||||
|
||||
lines = text.split('\n')
|
||||
current_edge_type = None
|
||||
lines = text.split('\n'); current_edge_type = None
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
callout_match = re.match(r'>\s*\[!edge\]\s*([^:\s]+)', stripped)
|
||||
@@ -106,6 +91,5 @@ def parse_edges_robust(text: str) -> Set[str]:
|
||||
links = re.findall(r'\[\[([^\]]+)\]\]', stripped)
|
||||
for l in links:
|
||||
if "rel:" not in l: found_edges.add(f"{current_edge_type}:{l}")
|
||||
elif not stripped.startswith('>'):
|
||||
current_edge_type = None
|
||||
elif not stripped.startswith('>'): current_edge_type = None
|
||||
return found_edges
|
||||
Reference in new issue
Block a user