Implement WP-24c v4.2.0: Introduce configurable header names and levels for LLM validation and Note-Scope zones in the chunking system. Update chunking models, parser, and processor to support exclusion of edge zones during chunking. Enhance documentation and configuration files to reflect new environment variables for improved flexibility in Markdown processing.
This commit is contained in:
1 parent
39fd15b565
commit
003a270548
7 files changed
+401
-28
No files matched your search
@@ -13,6 +13,7 @@ class RawBlock:
|
||||
level: Optional[int]
|
||||
section_path: str
|
||||
section_title: Optional[str]
|
||||
exclude_from_chunking: bool = False # WP-24c v4.2.0: Flag für Edge-Zonen, die nicht gechunkt werden sollen
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
|
||||
@@ -3,8 +3,10 @@ FILE: app/core/chunking/chunking_parser.py
|
||||
DESCRIPTION: Zerlegt Markdown in logische Einheiten (RawBlocks).
|
||||
Hält alle Überschriftenebenen (H1-H6) im Stream.
|
||||
Stellt die Funktion parse_edges_robust zur Verfügung.
|
||||
WP-24c v4.2.0: Identifiziert Edge-Zonen und markiert sie für Chunking-Ausschluss.
|
||||
"""
|
||||
import re
|
||||
import os
|
||||
from typing import List, Tuple, Set
|
||||
from .chunking_models import RawBlock
|
||||
from .chunking_utils import extract_frontmatter_from_text
|
||||
@@ -20,7 +22,10 @@ def split_sentences(text: str) -> list[str]:
|
||||
return [p.strip() for p in _SENT_SPLIT.split(text) if p.strip()]
|
||||
|
||||
def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
"""Zerlegt Text in logische Einheiten (RawBlocks), inklusive H1-H6."""
|
||||
"""
|
||||
Zerlegt Text in logische Einheiten (RawBlocks), inklusive H1-H6.
|
||||
WP-24c v4.2.0: Identifiziert Edge-Zonen (LLM-Validierung & Note-Scope) und markiert sie für Chunking-Ausschluss.
|
||||
"""
|
||||
blocks = []
|
||||
h1_title = "Dokument"
|
||||
section_path = "/"
|
||||
@@ -29,6 +34,31 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
# Frontmatter entfernen
|
||||
fm, text_without_fm = extract_frontmatter_from_text(md_text)
|
||||
|
||||
# WP-24c v4.2.0: Konfigurierbare Header-Namen und -Ebenen
|
||||
llm_validation_headers = os.getenv(
|
||||
"MINDNET_LLM_VALIDATION_HEADERS",
|
||||
"Unzugeordnete Kanten,Edge Pool,Candidates"
|
||||
)
|
||||
llm_validation_header_list = [h.strip() for h in llm_validation_headers.split(",") if h.strip()]
|
||||
if not llm_validation_header_list:
|
||||
llm_validation_header_list = ["Unzugeordnete Kanten", "Edge Pool", "Candidates"]
|
||||
|
||||
note_scope_headers = os.getenv(
|
||||
"MINDNET_NOTE_SCOPE_ZONE_HEADERS",
|
||||
"Smart Edges,Relationen,Global Links,Note-Level Relations,Globale Verbindungen"
|
||||
)
|
||||
note_scope_header_list = [h.strip() for h in note_scope_headers.split(",") if h.strip()]
|
||||
if not note_scope_header_list:
|
||||
note_scope_header_list = ["Smart Edges", "Relationen", "Global Links", "Note-Level Relations", "Globale Verbindungen"]
|
||||
|
||||
# Header-Ebenen konfigurierbar (Default: LLM=3, Note-Scope=2)
|
||||
llm_validation_level = int(os.getenv("MINDNET_LLM_VALIDATION_HEADER_LEVEL", "3"))
|
||||
note_scope_level = int(os.getenv("MINDNET_NOTE_SCOPE_HEADER_LEVEL", "2"))
|
||||
|
||||
# Status-Tracking für Edge-Zonen
|
||||
in_exclusion_zone = False
|
||||
exclusion_zone_type = None # "llm_validation" oder "note_scope"
|
||||
|
||||
# H1 für Note-Titel extrahieren (Metadaten-Zweck)
|
||||
h1_match = re.search(r'^#\s+(.*)', text_without_fm, re.MULTILINE)
|
||||
if h1_match:
|
||||
@@ -47,20 +77,47 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
blocks.append(RawBlock(
|
||||
"paragraph", content, None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
buffer = []
|
||||
|
||||
level = len(heading_match.group(1))
|
||||
title = heading_match.group(2).strip()
|
||||
|
||||
# WP-24c v4.2.0: Prüfe, ob dieser Header eine Edge-Zone startet
|
||||
is_llm_validation_zone = (
|
||||
level == llm_validation_level and
|
||||
any(title.lower() == h.lower() for h in llm_validation_header_list)
|
||||
)
|
||||
is_note_scope_zone = (
|
||||
level == note_scope_level and
|
||||
any(title.lower() == h.lower() for h in note_scope_header_list)
|
||||
)
|
||||
|
||||
if is_llm_validation_zone:
|
||||
in_exclusion_zone = True
|
||||
exclusion_zone_type = "llm_validation"
|
||||
elif is_note_scope_zone:
|
||||
in_exclusion_zone = True
|
||||
exclusion_zone_type = "note_scope"
|
||||
elif in_exclusion_zone:
|
||||
# Neuer Header gefunden, der keine Edge-Zone ist -> Zone beendet
|
||||
in_exclusion_zone = False
|
||||
exclusion_zone_type = None
|
||||
|
||||
# Pfad- und Titel-Update für die Metadaten der folgenden Blöcke
|
||||
if level == 1:
|
||||
current_section_title = title; section_path = "/"
|
||||
elif level == 2:
|
||||
current_section_title = title; section_path = f"/{current_section_title}"
|
||||
|
||||
# Die Überschrift selbst als regulären Block hinzufügen
|
||||
blocks.append(RawBlock("heading", stripped, level, section_path, current_section_title))
|
||||
# Die Überschrift selbst als regulären Block hinzufügen (auch markiert, wenn in Zone)
|
||||
blocks.append(RawBlock(
|
||||
"heading", stripped, level, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
continue
|
||||
|
||||
# Trenner (---) oder Leerzeilen beenden Blöcke, außer innerhalb von Callouts
|
||||
@@ -68,17 +125,26 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
blocks.append(RawBlock(
|
||||
"paragraph", content, None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
buffer = []
|
||||
if stripped == "---":
|
||||
blocks.append(RawBlock("separator", "---", None, section_path, current_section_title))
|
||||
blocks.append(RawBlock(
|
||||
"separator", "---", None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
else:
|
||||
buffer.append(line)
|
||||
|
||||
if buffer:
|
||||
content = "\n".join(buffer).strip()
|
||||
if content:
|
||||
blocks.append(RawBlock("paragraph", content, None, section_path, current_section_title))
|
||||
blocks.append(RawBlock(
|
||||
"paragraph", content, None, section_path, current_section_title,
|
||||
exclude_from_chunking=in_exclusion_zone
|
||||
))
|
||||
|
||||
return blocks, h1_title
|
||||
|
||||
|
||||
@@ -6,9 +6,11 @@ DESCRIPTION: Der zentrale Orchestrator für das Chunking-System.
|
||||
- Integriert physikalische Kanten-Injektion (Propagierung).
|
||||
- Stellt H1-Kontext-Fenster sicher.
|
||||
- Baut den Candidate-Pool für die WP-15b Ingestion auf.
|
||||
WP-24c v4.2.0: Konfigurierbare Header-Namen für LLM-Validierung.
|
||||
"""
|
||||
import asyncio
|
||||
import re
|
||||
import os
|
||||
import logging
|
||||
from typing import List, Dict, Optional
|
||||
from .chunking_models import Chunk
|
||||
@@ -31,6 +33,10 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
fm, body_text = extract_frontmatter_from_text(md_text)
|
||||
blocks, doc_title = parse_blocks(md_text)
|
||||
|
||||
# WP-24c v4.2.0: Filtere Blöcke aus Edge-Zonen (LLM-Validierung & Note-Scope)
|
||||
# Diese Bereiche sollen nicht als Chunks angelegt werden, sondern nur die Kanten extrahiert werden
|
||||
blocks_for_chunking = [b for b in blocks if not getattr(b, 'exclude_from_chunking', False)]
|
||||
|
||||
# Vorbereitung des H1-Präfix für die Embedding-Fenster (Breadcrumbs)
|
||||
h1_prefix = f"# {doc_title}" if doc_title else ""
|
||||
|
||||
@@ -38,11 +44,11 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
# Alle Strategien nutzen nun einheitlich context_prefix für die Window-Bildung.
|
||||
if config.get("strategy") == "by_heading":
|
||||
chunks = await asyncio.to_thread(
|
||||
strategy_by_heading, blocks, config, note_id, context_prefix=h1_prefix
|
||||
strategy_by_heading, blocks_for_chunking, config, note_id, context_prefix=h1_prefix
|
||||
)
|
||||
else:
|
||||
chunks = await asyncio.to_thread(
|
||||
strategy_sliding_window, blocks, config, note_id, context_prefix=h1_prefix
|
||||
strategy_sliding_window, blocks_for_chunking, config, note_id, context_prefix=h1_prefix
|
||||
)
|
||||
|
||||
if not chunks:
|
||||
@@ -63,14 +69,29 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
k, t = parts
|
||||
ch.candidate_pool.append({"kind": k, "to": t, "provenance": "explicit"})
|
||||
|
||||
# 5. Global Pool (Unzugeordnete Kanten aus dem Dokument-Ende)
|
||||
# Sucht nach dem Edge-Pool Block im Original-Markdown.
|
||||
pool_match = re.search(
|
||||
r'###?\s*(?:Unzugeordnete Kanten|Edge Pool|Candidates)\s*\n(.*?)(?:\n#|$)',
|
||||
body_text,
|
||||
re.DOTALL | re.IGNORECASE
|
||||
# 5. Global Pool (Unzugeordnete Kanten - kann mitten im Dokument oder am Ende stehen)
|
||||
# WP-24c v4.2.0: Konfigurierbare Header-Namen und -Ebene via .env
|
||||
# Sucht nach ALLEN Edge-Pool Blöcken im Original-Markdown (nicht nur am Ende).
|
||||
llm_validation_headers = os.getenv(
|
||||
"MINDNET_LLM_VALIDATION_HEADERS",
|
||||
"Unzugeordnete Kanten,Edge Pool,Candidates"
|
||||
)
|
||||
if pool_match:
|
||||
header_list = [h.strip() for h in llm_validation_headers.split(",") if h.strip()]
|
||||
# Fallback auf Defaults, falls leer
|
||||
if not header_list:
|
||||
header_list = ["Unzugeordnete Kanten", "Edge Pool", "Candidates"]
|
||||
|
||||
# Header-Ebene konfigurierbar (Default: 3 für ###)
|
||||
llm_validation_level = int(os.getenv("MINDNET_LLM_VALIDATION_HEADER_LEVEL", "3"))
|
||||
header_level_pattern = "#" * llm_validation_level
|
||||
|
||||
# Regex-Pattern mit konfigurierbaren Headern und Ebene
|
||||
# WP-24c v4.2.0: finditer statt search, um ALLE Zonen zu finden (auch mitten im Dokument)
|
||||
# Zone endet bei einem neuen Header (jeder Ebene) oder am Dokument-Ende
|
||||
header_pattern = "|".join(re.escape(h) for h in header_list)
|
||||
zone_pattern = rf'^{re.escape(header_level_pattern)}\s*(?:{header_pattern})\s*\n(.*?)(?=\n#|$)'
|
||||
|
||||
for pool_match in re.finditer(zone_pattern, body_text, re.DOTALL | re.IGNORECASE | re.MULTILINE):
|
||||
global_edges = parse_edges_robust(pool_match.group(1))
|
||||
for e_str in global_edges:
|
||||
parts = e_str.split(':', 1)
|
||||
|
||||
Reference in new issue
Block a user