Implement WP-24c v4.2.0: Introduce configurable header names and levels for LLM validation and Note-Scope zones in the chunking system. Update chunking models, parser, and processor to support exclusion of edge zones during chunking. Enhance documentation and configuration files to reflect new environment variables for improved flexibility in Markdown processing.
This commit is contained in:
1 parent
39fd15b565
commit
003a270548
7 files changed
+401
-28
No files matched your search
@@ -3,8 +3,10 @@ FILE: app/core/chunking/chunking_parser.py
|
||||
DESCRIPTION: Zerlegt Markdown in logische Einheiten (RawBlocks).
|
||||
Hält alle Überschriftenebenen (H1-H6) im Stream.
|
||||
Stellt die Funktion parse_edges_robust zur Verfügung.
|
||||
WP-24c v4.2.0: Identifiziert Edge-Zonen und markiert sie für Chunking-Ausschluss.
|
||||
"""
|
||||
import re
|
||||
import os
|
||||
from typing import List, Tuple, Set
|
||||
from .chunking_models import RawBlock
|
||||
from .chunking_utils import extract_frontmatter_from_text
|
||||
@@ -20,7 +22,10 @@ def split_sentences(text: str) -> list[str]:
|
||||
return [p.strip() for p in _SENT_SPLIT.split(text) if p.strip()]
|
||||
|
||||
def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
"""Zerlegt Text in logische Einheiten (RawBlocks), inklusive H1-H6."""
|
||||
"""
|
||||
Zerlegt Text in logische Einheiten (RawBlocks), inklusive H1-H6.
|
||||
WP-24c v4.2.0: Identifiziert Edge-Zonen (LLM-Validierung & Note-Scope) und markiert sie für Chunking-Ausschluss.
|
||||
"""
|
||||
blocks = []
|
||||
h1_title = "Dokument"
|
||||
section_path = "/"
|
||||
@@ -29,6 +34,31 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
# Frontmatter entfernen
|
||||
fm, text_without_fm = extract_frontmatter_from_text(md_text)
|
||||
|
||||
# WP-24c v4.2.0: Konfigurierbare Header-Namen und -Ebenen
|
||||
llm_validation_headers = os.getenv(
|
||||
"MINDNET_LLM_VALIDATION_HEADERS",
|
||||
"Unzugeordnete Kanten,Edge Pool,Candidates"
|
||||
)
|
||||
llm_validation_header_list = [h.strip() for h in llm_validation_headers.split(",") if h.strip()]
|
||||
if not llm_validation_header_list:
|
||||
llm_validation_header_list = ["Unzugeordnete Kanten", "Edge Pool", "Candidates"]
|
||||
|
||||
note_scope_headers = os.getenv(
|
||||
"MINDNET_NOTE_SCOPE_ZONE_HEADERS",
|
||||
"Smart Edges,Relationen,Global Links,Note-Level Relations,Globale Verbindungen"
|
||||
)
|
||||
note_scope_header_list = [h.strip() for h in note_scope_headers.split(",") if h.strip()]
|
||||
if not note_scope_header_list:
|
||||
note_scope_header_list = ["Smart Edges", "Relationen", "Global Links", "Note-Level Relations", "Globale Verbindungen"]
|
||||
|
||||
# Header-Ebenen konfigurierbar (Default: LLM=3, Note-Scope=2)
|
||||
llm_validation_level = int(os.getenv("MINDNET_LLM_VALIDATION_HEADER_LEVEL", "3"))
|
||||
note_scope_level = int(os.getenv("MINDNET_NOTE_SCOPE_HEADER_LEVEL", "2"))
|
||||
|
||||
# Status-Tracking für Edge-Zonen
|
||||
in_exclusion_zone = False
|
||||
exclusion_zone_type = None # "llm_validation" oder "note_scope"
|
||||
|
||||
# H1 für Note-Titel extrahieren (Metadaten-Zweck)
|
||||
h1_match = re.search(r'^#\s+(.*)', text_without_fm, re.MULTILINE)
|
||||
if h1_match:
|
||||
@@ -47,20 +77,47 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
blocks.append(RawBlock(
|
||||
"paragraph", content, None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
buffer = []
|
||||
|
||||
level = len(heading_match.group(1))
|
||||
title = heading_match.group(2).strip()
|
||||
|
||||
# WP-24c v4.2.0: Prüfe, ob dieser Header eine Edge-Zone startet
|
||||
is_llm_validation_zone = (
|
||||
level == llm_validation_level and
|
||||
any(title.lower() == h.lower() for h in llm_validation_header_list)
|
||||
)
|
||||
is_note_scope_zone = (
|
||||
level == note_scope_level and
|
||||
any(title.lower() == h.lower() for h in note_scope_header_list)
|
||||
)
|
||||
|
||||
if is_llm_validation_zone:
|
||||
in_exclusion_zone = True
|
||||
exclusion_zone_type = "llm_validation"
|
||||
elif is_note_scope_zone:
|
||||
in_exclusion_zone = True
|
||||
exclusion_zone_type = "note_scope"
|
||||
elif in_exclusion_zone:
|
||||
# Neuer Header gefunden, der keine Edge-Zone ist -> Zone beendet
|
||||
in_exclusion_zone = False
|
||||
exclusion_zone_type = None
|
||||
|
||||
# Pfad- und Titel-Update für die Metadaten der folgenden Blöcke
|
||||
if level == 1:
|
||||
current_section_title = title; section_path = "/"
|
||||
elif level == 2:
|
||||
current_section_title = title; section_path = f"/{current_section_title}"
|
||||
|
||||
# Die Überschrift selbst als regulären Block hinzufügen
|
||||
blocks.append(RawBlock("heading", stripped, level, section_path, current_section_title))
|
||||
# Die Überschrift selbst als regulären Block hinzufügen (auch markiert, wenn in Zone)
|
||||
blocks.append(RawBlock(
|
||||
"heading", stripped, level, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
continue
|
||||
|
||||
# Trenner (---) oder Leerzeilen beenden Blöcke, außer innerhalb von Callouts
|
||||
@@ -68,17 +125,26 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
blocks.append(RawBlock(
|
||||
"paragraph", content, None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
buffer = []
|
||||
if stripped == "---":
|
||||
blocks.append(RawBlock("separator", "---", None, section_path, current_section_title))
|
||||
blocks.append(RawBlock(
|
||||
"separator", "---", None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
else:
|
||||
buffer.append(line)
|
||||
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
blocks.append(RawBlock(
|
||||
"paragraph", content, None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
|
||||
return blocks, h1_title
|
||||
|
||||
|
||||
Reference in new issue
Block a user