Implement WP-24c v4.2.0: Introduce configurable header names and levels for LLM validation and Note-Scope zones in the chunking system. Update chunking models, parser, and processor to support exclusion of edge zones during chunking. Enhance documentation and configuration files to reflect new environment variables for improved flexibility in Markdown processing.
This commit is contained in:
1 parent
39fd15b565
commit
003a270548
7 files changed
+401
-28
No files matched your search
@@ -6,9 +6,11 @@ DESCRIPTION: Der zentrale Orchestrator für das Chunking-System.
|
||||
- Integriert physikalische Kanten-Injektion (Propagierung).
|
||||
- Stellt H1-Kontext-Fenster sicher.
|
||||
- Baut den Candidate-Pool für die WP-15b Ingestion auf.
|
||||
WP-24c v4.2.0: Konfigurierbare Header-Namen für LLM-Validierung.
|
||||
"""
|
||||
import asyncio
|
||||
import re
|
||||
import os
|
||||
import logging
|
||||
from typing import List, Dict, Optional
|
||||
from .chunking_models import Chunk
|
||||
@@ -31,6 +33,10 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
fm, body_text = extract_frontmatter_from_text(md_text)
|
||||
blocks, doc_title = parse_blocks(md_text)
|
||||
|
||||
# WP-24c v4.2.0: Filtere Blöcke aus Edge-Zonen (LLM-Validierung & Note-Scope)
|
||||
# Diese Bereiche sollen nicht als Chunks angelegt werden, sondern nur die Kanten extrahiert werden
|
||||
blocks_for_chunking = [b for b in blocks if not getattr(b, 'exclude_from_chunking', False)]
|
||||
|
||||
# Vorbereitung des H1-Präfix für die Embedding-Fenster (Breadcrumbs)
|
||||
h1_prefix = f"# {doc_title}" if doc_title else ""
|
||||
|
||||
@@ -38,11 +44,11 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
# Alle Strategien nutzen nun einheitlich context_prefix für die Window-Bildung.
|
||||
if config.get("strategy") == "by_heading":
|
||||
chunks = await asyncio.to_thread(
|
||||
strategy_by_heading, blocks, config, note_id, context_prefix=h1_prefix
|
||||
strategy_by_heading, blocks_for_chunking, config, note_id, context_prefix=h1_prefix
|
||||
)
|
||||
else:
|
||||
chunks = await asyncio.to_thread(
|
||||
strategy_sliding_window, blocks, config, note_id, context_prefix=h1_prefix
|
||||
strategy_sliding_window, blocks_for_chunking, config, note_id, context_prefix=h1_prefix
|
||||
)
|
||||
|
||||
if not chunks:
|
||||
@@ -63,14 +69,29 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
k, t = parts
|
||||
ch.candidate_pool.append({"kind": k, "to": t, "provenance": "explicit"})
|
||||
|
||||
# 5. Global Pool (Unzugeordnete Kanten aus dem Dokument-Ende)
|
||||
# Sucht nach dem Edge-Pool Block im Original-Markdown.
|
||||
pool_match = re.search(
|
||||
r'###?\s*(?:Unzugeordnete Kanten|Edge Pool|Candidates)\s*\n(.*?)(?:\n#|$)',
|
||||
body_text,
|
||||
re.DOTALL | re.IGNORECASE
|
||||
# 5. Global Pool (Unzugeordnete Kanten - kann mitten im Dokument oder am Ende stehen)
|
||||
# WP-24c v4.2.0: Konfigurierbare Header-Namen und -Ebene via .env
|
||||
# Sucht nach ALLEN Edge-Pool Blöcken im Original-Markdown (nicht nur am Ende).
|
||||
llm_validation_headers = os.getenv(
|
||||
"MINDNET_LLM_VALIDATION_HEADERS",
|
||||
"Unzugeordnete Kanten,Edge Pool,Candidates"
|
||||
)
|
||||
if pool_match:
|
||||
header_list = [h.strip() for h in llm_validation_headers.split(",") if h.strip()]
|
||||
# Fallback auf Defaults, falls leer
|
||||
if not header_list:
|
||||
header_list = ["Unzugeordnete Kanten", "Edge Pool", "Candidates"]
|
||||
|
||||
# Header-Ebene konfigurierbar (Default: 3 für ###)
|
||||
llm_validation_level = int(os.getenv("MINDNET_LLM_VALIDATION_HEADER_LEVEL", "3"))
|
||||
header_level_pattern = "#" * llm_validation_level
|
||||
|
||||
# Regex-Pattern mit konfigurierbaren Headern und Ebene
|
||||
# WP-24c v4.2.0: finditer statt search, um ALLE Zonen zu finden (auch mitten im Dokument)
|
||||
# Zone endet bei einem neuen Header (jeder Ebene) oder am Dokument-Ende
|
||||
header_pattern = "|".join(re.escape(h) for h in header_list)
|
||||
zone_pattern = rf'^{re.escape(header_level_pattern)}\s*(?:{header_pattern})\s*\n(.*?)(?=\n#|$)'
|
||||
|
||||
for pool_match in re.finditer(zone_pattern, body_text, re.DOTALL | re.IGNORECASE | re.MULTILINE):
|
||||
global_edges = parse_edges_robust(pool_match.group(1))
|
||||
for e_str in global_edges:
|
||||
parts = e_str.split(':', 1)
|
||||
|
||||
Reference in new issue
Block a user