Update chunking system to version 3.9.9, synchronizing parameters with the orchestrator and enhancing edge detection. Implement robust parsing to prevent duplicate edges in section propagation. Adjust comments for clarity and consistency across the codebase.

This commit is contained in:
Lars committed 2025-12-30 08:40:19 +01:00
1 parent 33dff04d47
commit 6aa6b32a6c
5 files changed
+68 -29

No files matched your search

+13 -9
View File
@@ -1,25 +1,29 @@
"""
FILE: app/core/chunking/chunking_strategies.py
DESCRIPTION: Strategien für atomares Sektions-Chunking v3.9.8.
DESCRIPTION: Strategien für atomares Sektions-Chunking v3.9.9.
Implementiert das 'Pack-and-Carry-Over' Verfahren nach Regel 1-3.
- Keine redundante Kanten-Injektion.
- Strikte Einhaltung von Sektionsgrenzen via Look-Ahead.
- Fix: Synchronisierung der Parameter mit dem Orchestrator (context_prefix).
"""
from typing import List, Dict, Any, Optional
from .chunking_models import RawBlock, Chunk
from .chunking_utils import estimate_tokens
from .chunking_parser import split_sentences
def _create_win(doc_title: str, sec_title: Optional[str], text: str) -> str:
def _create_win(context_prefix: str, sec_title: Optional[str], text: str) -> str:
"""Baut den Breadcrumb-Kontext für das Embedding-Fenster."""
parts = [doc_title] if doc_title else []
if sec_title and sec_title != doc_title: parts.append(sec_title)
parts = [context_prefix] if context_prefix else []
# Verhindert Dopplung, falls der Context-Prefix (H1) bereits den Sektionsnamen enthält
if sec_title and f"# {sec_title}" != context_prefix and sec_title not in (context_prefix or ""):
parts.append(sec_title)
prefix = " > ".join(parts)
return f"{prefix}\n{text}".strip() if prefix else text
def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, doc_title: str = "") -> List[Chunk]:
def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, context_prefix: str = "") -> List[Chunk]:
"""
Universelle Heading-Strategie mit Carry-Over Logik.
Synchronisiert auf context_prefix für Kompatibilität mit dem Orchestrator.
"""
smart_edge = config.get("enable_smart_edge_allocation", True)
strict = config.get("strict_heading_split", False)
@@ -34,7 +38,7 @@ def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id:
def _emit(txt, title, path):
"""Schreibt den finalen Chunk ohne Text-Modifikationen."""
idx = len(chunks)
win = _create_win(doc_title, title, txt)
win = _create_win(context_prefix, title, txt)
chunks.append(Chunk(
id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx,
text=txt, window=win, token_count=estimate_tokens(txt),
@@ -139,7 +143,7 @@ def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id:
return chunks
def strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, doc_title: str = "") -> List[Chunk]:
def strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, context_prefix: str = "") -> List[Chunk]:
"""Standard-Sliding-Window für flache Texte ohne Sektionsfokus."""
target = config.get("target", 400); max_tokens = config.get("max", 600)
chunks: List[Chunk] = []; buf: List[RawBlock] = []
@@ -149,14 +153,14 @@ def strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note
curr_tokens = sum(estimate_tokens(x.text) for x in buf) if buf else 0
if curr_tokens + b_tokens > max_tokens and buf:
txt = "\n\n".join([x.text for x in buf]); idx = len(chunks)
win = _create_win(doc_title, buf[0].section_title, txt)
win = _create_win(context_prefix, buf[0].section_title, txt)
chunks.append(Chunk(id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx, text=txt, window=win, token_count=curr_tokens, section_title=buf[0].section_title, section_path=buf[0].section_path, neighbors_prev=None, neighbors_next=None))
buf = []
buf.append(b)
if buf:
txt = "\n\n".join([x.text for x in buf]); idx = len(chunks)
win = _create_win(doc_title, buf[0].section_title, txt)
win = _create_win(context_prefix, buf[0].section_title, txt)
chunks.append(Chunk(id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx, text=txt, window=win, token_count=estimate_tokens(txt), section_title=buf[0].section_title, section_path=buf[0].section_path, neighbors_prev=None, neighbors_next=None))
return chunks