Implement WP-24c v4.2.0: Introduce configurable header names and levels for LLM validation and Note-Scope zones in the chunking system. Update chunking models, parser, and processor to support exclusion of edge zones during chunking. Enhance documentation and configuration files to reflect new environment variables for improved flexibility in Markdown processing.

This commit is contained in:
Lars committed 2026-01-10 21:46:51 +01:00
1 parent 39fd15b565
commit 003a270548
7 files changed
+401 -28

No files matched your search

+1
View File
@@ -13,6 +13,7 @@ class RawBlock:
level: Optional[int]
section_path: str
section_title: Optional[str]
exclude_from_chunking: bool = False # WP-24c v4.2.0: Flag für Edge-Zonen, die nicht gechunkt werden sollen
@dataclass
class Chunk:
+73 -7
View File
@@ -3,8 +3,10 @@ FILE: app/core/chunking/chunking_parser.py
DESCRIPTION: Zerlegt Markdown in logische Einheiten (RawBlocks).
Hält alle Überschriftenebenen (H1-H6) im Stream.
Stellt die Funktion parse_edges_robust zur Verfügung.
WP-24c v4.2.0: Identifiziert Edge-Zonen und markiert sie für Chunking-Ausschluss.
"""
import re
import os
from typing import List, Tuple, Set
from .chunking_models import RawBlock
from .chunking_utils import extract_frontmatter_from_text
@@ -20,7 +22,10 @@ def split_sentences(text: str) -> list[str]:
return [p.strip() for p in _SENT_SPLIT.split(text) if p.strip()]
def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
"""Zerlegt Text in logische Einheiten (RawBlocks), inklusive H1-H6."""
"""
Zerlegt Text in logische Einheiten (RawBlocks), inklusive H1-H6.
WP-24c v4.2.0: Identifiziert Edge-Zonen (LLM-Validierung & Note-Scope) und markiert sie für Chunking-Ausschluss.
"""
blocks = []
h1_title = "Dokument"
section_path = "/"
@@ -29,6 +34,31 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
# Frontmatter entfernen
fm, text_without_fm = extract_frontmatter_from_text(md_text)
# WP-24c v4.2.0: Konfigurierbare Header-Namen und -Ebenen
llm_validation_headers = os.getenv(
"MINDNET_LLM_VALIDATION_HEADERS",
"Unzugeordnete Kanten,Edge Pool,Candidates"
)
llm_validation_header_list = [h.strip() for h in llm_validation_headers.split(",") if h.strip()]
if not llm_validation_header_list:
llm_validation_header_list = ["Unzugeordnete Kanten", "Edge Pool", "Candidates"]
note_scope_headers = os.getenv(
"MINDNET_NOTE_SCOPE_ZONE_HEADERS",
"Smart Edges,Relationen,Global Links,Note-Level Relations,Globale Verbindungen"
)
note_scope_header_list = [h.strip() for h in note_scope_headers.split(",") if h.strip()]
if not note_scope_header_list:
note_scope_header_list = ["Smart Edges", "Relationen", "Global Links", "Note-Level Relations", "Globale Verbindungen"]
# Header-Ebenen konfigurierbar (Default: LLM=3, Note-Scope=2)
llm_validation_level = int(os.getenv("MINDNET_LLM_VALIDATION_HEADER_LEVEL", "3"))
note_scope_level = int(os.getenv("MINDNET_NOTE_SCOPE_HEADER_LEVEL", "2"))
# Status-Tracking für Edge-Zonen
in_exclusion_zone = False
exclusion_zone_type = None # "llm_validation" oder "note_scope"
# H1 für Note-Titel extrahieren (Metadaten-Zweck)
h1_match = re.search(r'^#\s+(.*)', text_without_fm, re.MULTILINE)
if h1_match:
@@ -47,20 +77,47 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
if buffer:
content = "\n".join(buffer).strip()
if content:
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
blocks.append(RawBlock(
"paragraph", content, None, section_path, current_section_title,
exclude_from_chunking=in_exclusion_zone
))
buffer = []
level = len(heading_match.group(1))
title = heading_match.group(2).strip()
# WP-24c v4.2.0: Prüfe, ob dieser Header eine Edge-Zone startet
is_llm_validation_zone = (
level == llm_validation_level and
any(title.lower() == h.lower() for h in llm_validation_header_list)
)
is_note_scope_zone = (
level == note_scope_level and
any(title.lower() == h.lower() for h in note_scope_header_list)
)
if is_llm_validation_zone:
in_exclusion_zone = True
exclusion_zone_type = "llm_validation"
elif is_note_scope_zone:
in_exclusion_zone = True
exclusion_zone_type = "note_scope"
elif in_exclusion_zone:
# Neuer Header gefunden, der keine Edge-Zone ist -> Zone beendet
in_exclusion_zone = False
exclusion_zone_type = None
# Pfad- und Titel-Update für die Metadaten der folgenden Blöcke
if level == 1:
current_section_title = title; section_path = "/"
elif level == 2:
current_section_title = title; section_path = f"/{current_section_title}"
# Die Überschrift selbst als regulären Block hinzufügen
blocks.append(RawBlock("heading", stripped, level, section_path, current_section_title))
# Die Überschrift selbst als regulären Block hinzufügen (auch markiert, wenn in Zone)
blocks.append(RawBlock(
"heading", stripped, level, section_path, current_section_title,
exclude_from_chunking=in_exclusion_zone
))
continue
# Trenner (---) oder Leerzeilen beenden Blöcke, außer innerhalb von Callouts
@@ -68,17 +125,26 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
if buffer:
content = "\n".join(buffer).strip()
if content:
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
blocks.append(RawBlock(
"paragraph", content, None, section_path, current_section_title,
exclude_from_chunking=in_exclusion_zone
))
buffer = []
if stripped == "---":
blocks.append(RawBlock("separator", "---", None, section_path, current_section_title))
blocks.append(RawBlock(
"separator", "---", None, section_path, current_section_title,
exclude_from_chunking=in_exclusion_zone
))
else:
buffer.append(line)
if buffer:
content = "\n".join(buffer).strip()
if content:
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
blocks.append(RawBlock(
"paragraph", content, None, section_path, current_section_title,
exclude_from_chunking=in_exclusion_zone
))
return blocks, h1_title
+30 -9
View File
@@ -6,9 +6,11 @@ DESCRIPTION: Der zentrale Orchestrator für das Chunking-System.
- Integriert physikalische Kanten-Injektion (Propagierung).
- Stellt H1-Kontext-Fenster sicher.
- Baut den Candidate-Pool für die WP-15b Ingestion auf.
WP-24c v4.2.0: Konfigurierbare Header-Namen für LLM-Validierung.
"""
import asyncio
import re
import os
import logging
from typing import List, Dict, Optional
from .chunking_models import Chunk
@@ -31,6 +33,10 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
fm, body_text = extract_frontmatter_from_text(md_text)
blocks, doc_title = parse_blocks(md_text)
# WP-24c v4.2.0: Filtere Blöcke aus Edge-Zonen (LLM-Validierung & Note-Scope)
# Diese Bereiche sollen nicht als Chunks angelegt werden, sondern nur die Kanten extrahiert werden
blocks_for_chunking = [b for b in blocks if not getattr(b, 'exclude_from_chunking', False)]
# Vorbereitung des H1-Präfix für die Embedding-Fenster (Breadcrumbs)
h1_prefix = f"# {doc_title}" if doc_title else ""
@@ -38,11 +44,11 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
# Alle Strategien nutzen nun einheitlich context_prefix für die Window-Bildung.
if config.get("strategy") == "by_heading":
chunks = await asyncio.to_thread(
strategy_by_heading, blocks, config, note_id, context_prefix=h1_prefix
strategy_by_heading, blocks_for_chunking, config, note_id, context_prefix=h1_prefix
)
else:
chunks = await asyncio.to_thread(
strategy_sliding_window, blocks, config, note_id, context_prefix=h1_prefix
strategy_sliding_window, blocks_for_chunking, config, note_id, context_prefix=h1_prefix
)
if not chunks:
@@ -63,14 +69,29 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
k, t = parts
ch.candidate_pool.append({"kind": k, "to": t, "provenance": "explicit"})
# 5. Global Pool (Unzugeordnete Kanten aus dem Dokument-Ende)
# Sucht nach dem Edge-Pool Block im Original-Markdown.
pool_match = re.search(
r'###?\s*(?:Unzugeordnete Kanten|Edge Pool|Candidates)\s*\n(.*?)(?:\n#|$)',
body_text,
re.DOTALL | re.IGNORECASE
# 5. Global Pool (Unzugeordnete Kanten - kann mitten im Dokument oder am Ende stehen)
# WP-24c v4.2.0: Konfigurierbare Header-Namen und -Ebene via .env
# Sucht nach ALLEN Edge-Pool Blöcken im Original-Markdown (nicht nur am Ende).
llm_validation_headers = os.getenv(
"MINDNET_LLM_VALIDATION_HEADERS",
"Unzugeordnete Kanten,Edge Pool,Candidates"
)
if pool_match:
header_list = [h.strip() for h in llm_validation_headers.split(",") if h.strip()]
# Fallback auf Defaults, falls leer
if not header_list:
header_list = ["Unzugeordnete Kanten", "Edge Pool", "Candidates"]
# Header-Ebene konfigurierbar (Default: 3 für ###)
llm_validation_level = int(os.getenv("MINDNET_LLM_VALIDATION_HEADER_LEVEL", "3"))
header_level_pattern = "#" * llm_validation_level
# Regex-Pattern mit konfigurierbaren Headern und Ebene
# WP-24c v4.2.0: finditer statt search, um ALLE Zonen zu finden (auch mitten im Dokument)
# Zone endet bei einem neuen Header (jeder Ebene) oder am Dokument-Ende
header_pattern = "|".join(re.escape(h) for h in header_list)
zone_pattern = rf'^{re.escape(header_level_pattern)}\s*(?:{header_pattern})\s*\n(.*?)(?=\n#|$)'
for pool_match in re.finditer(zone_pattern, body_text, re.DOTALL | re.IGNORECASE | re.MULTILINE):
global_edges = parse_edges_robust(pool_match.group(1))
for e_str in global_edges:
parts = e_str.split(':', 1)