Update chunking system to version 3.9.9, synchronizing parameters with the orchestrator and enhancing edge detection. Implement robust parsing to prevent duplicate edges in section propagation. Adjust comments for clarity and consistency across the codebase.
This commit is contained in:
1 parent
33dff04d47
commit
6aa6b32a6c
5 files changed
+68
-29
No files matched your search
@@ -1,7 +1,8 @@
|
||||
"""
|
||||
FILE: app/core/chunking/chunking_processor.py
|
||||
DESCRIPTION: Der zentrale Orchestrator für das Chunking-System.
|
||||
AUDIT v3.3.3: Wiederherstellung der "Gold-Standard" Qualität.
|
||||
AUDIT v3.3.4: Wiederherstellung der "Gold-Standard" Qualität.
|
||||
- Fix: Synchronisierung der Parameter (context_prefix) für alle Strategien.
|
||||
- Integriert physikalische Kanten-Injektion (Propagierung).
|
||||
- Stellt H1-Kontext-Fenster sicher.
|
||||
- Baut den Candidate-Pool für die WP-15b Ingestion auf.
|
||||
@@ -30,16 +31,19 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
fm, body_text = extract_frontmatter_from_text(md_text)
|
||||
blocks, doc_title = parse_blocks(md_text)
|
||||
|
||||
# Vorbereitung des H1-Präfix für die Embedding-Fenster
|
||||
# Vorbereitung des H1-Präfix für die Embedding-Fenster (Breadcrumbs)
|
||||
h1_prefix = f"# {doc_title}" if doc_title else ""
|
||||
|
||||
# 2. Anwendung der Splitting-Strategie
|
||||
# Wir übergeben den Dokument-Titel/Präfix für die Window-Bildung.
|
||||
# Alle Strategien nutzen nun einheitlich context_prefix für die Window-Bildung.
|
||||
if config.get("strategy") == "by_heading":
|
||||
chunks = await asyncio.to_thread(strategy_by_heading, blocks, config, note_id, doc_title)
|
||||
chunks = await asyncio.to_thread(
|
||||
strategy_by_heading, blocks, config, note_id, context_prefix=h1_prefix
|
||||
)
|
||||
else:
|
||||
# sliding_window nutzt nun den context_prefix für das Window-Feld.
|
||||
chunks = await asyncio.to_thread(strategy_sliding_window, blocks, config, note_id, context_prefix=h1_prefix)
|
||||
chunks = await asyncio.to_thread(
|
||||
strategy_sliding_window, blocks, config, note_id, context_prefix=h1_prefix
|
||||
)
|
||||
|
||||
if not chunks:
|
||||
return []
|
||||
@@ -52,6 +56,7 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
# Zuerst die explizit im Text vorhandenen Kanten sammeln.
|
||||
for ch in chunks:
|
||||
# Wir extrahieren aus dem bereits (durch Propagation) angereicherten Text.
|
||||
# ch.candidate_pool wird im Modell-Konstruktor als leere Liste initialisiert.
|
||||
for e_str in parse_edges_robust(ch.text):
|
||||
parts = e_str.split(':', 1)
|
||||
if len(parts) == 2:
|
||||
@@ -71,7 +76,7 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
parts = e_str.split(':', 1)
|
||||
if len(parts) == 2:
|
||||
k, t = parts
|
||||
# Diese Kanten werden als "Global Pool" markiert für die spätere KI-Prüfung.
|
||||
# Diese Kanten werden als "global_pool" markiert für die spätere KI-Prüfung.
|
||||
for ch in chunks:
|
||||
ch.candidate_pool.append({"kind": k, "to": t, "provenance": "global_pool"})
|
||||
|
||||
@@ -80,6 +85,7 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
seen = set()
|
||||
unique = []
|
||||
for c in ch.candidate_pool:
|
||||
# Eindeutigkeit über Typ, Ziel und Herkunft (Provenance)
|
||||
key = (c["kind"], c["to"], c["provenance"])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""
|
||||
FILE: app/core/chunking/chunking_propagation.py
|
||||
DESCRIPTION: Injiziert Sektions-Kanten physisch in den Text (Embedding-Enrichment).
|
||||
Fix v3.3.5: Erkennt Wikilink-Targets, um Dopplungen zu verhindern.
|
||||
Fix v3.3.6: Nutzt robustes Parsing zur Erkennung vorhandener Kanten,
|
||||
um Dopplungen direkt hinter [!edge] Callouts format-agnostisch zu verhindern.
|
||||
"""
|
||||
from typing import List, Dict, Set
|
||||
from .chunking_models import Chunk
|
||||
@@ -34,15 +35,19 @@ def propagate_section_edges(chunks: List[Chunk]) -> List[Chunk]:
|
||||
if not edges_to_add:
|
||||
continue
|
||||
|
||||
# Vorhandene Kanten (Typ:Ziel) in DIESEM Chunk ermitteln,
|
||||
# um Dopplungen (z.B. durch Callouts) zu vermeiden.
|
||||
existing_edges = parse_edges_robust(ch.text)
|
||||
|
||||
injections = []
|
||||
for e_str in edges_to_add:
|
||||
kind, target = e_str.split(':', 1)
|
||||
|
||||
# DER FIX: Wir prüfen, ob das Ziel (target) bereits im Text vorkommt.
|
||||
# Wir suchen nach [[target]] (Callout-Stil) oder |target]] (Rel-Stil).
|
||||
if f"[[{target}]]" in ch.text or f"|{target}]]" in ch.text:
|
||||
# Sortierung für deterministische Ergebnisse
|
||||
for e_str in sorted(list(edges_to_add)):
|
||||
# Wenn die Kante (Typ + Ziel) bereits vorhanden ist (egal welches Format),
|
||||
# überspringen wir die Injektion für diesen Chunk.
|
||||
if e_str in existing_edges:
|
||||
continue
|
||||
|
||||
kind, target = e_str.split(':', 1)
|
||||
injections.append(f"[[rel:{kind}|{target}]]")
|
||||
|
||||
if injections:
|
||||
|
||||
@@ -1,25 +1,29 @@
|
||||
"""
|
||||
FILE: app/core/chunking/chunking_strategies.py
|
||||
DESCRIPTION: Strategien für atomares Sektions-Chunking v3.9.8.
|
||||
DESCRIPTION: Strategien für atomares Sektions-Chunking v3.9.9.
|
||||
Implementiert das 'Pack-and-Carry-Over' Verfahren nach Regel 1-3.
|
||||
- Keine redundante Kanten-Injektion.
|
||||
- Strikte Einhaltung von Sektionsgrenzen via Look-Ahead.
|
||||
- Fix: Synchronisierung der Parameter mit dem Orchestrator (context_prefix).
|
||||
"""
|
||||
from typing import List, Dict, Any, Optional
|
||||
from .chunking_models import RawBlock, Chunk
|
||||
from .chunking_utils import estimate_tokens
|
||||
from .chunking_parser import split_sentences
|
||||
|
||||
def _create_win(doc_title: str, sec_title: Optional[str], text: str) -> str:
|
||||
def _create_win(context_prefix: str, sec_title: Optional[str], text: str) -> str:
|
||||
"""Baut den Breadcrumb-Kontext für das Embedding-Fenster."""
|
||||
parts = [doc_title] if doc_title else []
|
||||
if sec_title and sec_title != doc_title: parts.append(sec_title)
|
||||
parts = [context_prefix] if context_prefix else []
|
||||
# Verhindert Dopplung, falls der Context-Prefix (H1) bereits den Sektionsnamen enthält
|
||||
if sec_title and f"# {sec_title}" != context_prefix and sec_title not in (context_prefix or ""):
|
||||
parts.append(sec_title)
|
||||
prefix = " > ".join(parts)
|
||||
return f"{prefix}\n{text}".strip() if prefix else text
|
||||
|
||||
def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, doc_title: str = "") -> List[Chunk]:
|
||||
def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, context_prefix: str = "") -> List[Chunk]:
|
||||
"""
|
||||
Universelle Heading-Strategie mit Carry-Over Logik.
|
||||
Synchronisiert auf context_prefix für Kompatibilität mit dem Orchestrator.
|
||||
"""
|
||||
smart_edge = config.get("enable_smart_edge_allocation", True)
|
||||
strict = config.get("strict_heading_split", False)
|
||||
@@ -34,7 +38,7 @@ def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id:
|
||||
def _emit(txt, title, path):
|
||||
"""Schreibt den finalen Chunk ohne Text-Modifikationen."""
|
||||
idx = len(chunks)
|
||||
win = _create_win(doc_title, title, txt)
|
||||
win = _create_win(context_prefix, title, txt)
|
||||
chunks.append(Chunk(
|
||||
id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx,
|
||||
text=txt, window=win, token_count=estimate_tokens(txt),
|
||||
@@ -139,7 +143,7 @@ def strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id:
|
||||
|
||||
return chunks
|
||||
|
||||
def strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, doc_title: str = "") -> List[Chunk]:
|
||||
def strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, context_prefix: str = "") -> List[Chunk]:
|
||||
"""Standard-Sliding-Window für flache Texte ohne Sektionsfokus."""
|
||||
target = config.get("target", 400); max_tokens = config.get("max", 600)
|
||||
chunks: List[Chunk] = []; buf: List[RawBlock] = []
|
||||
@@ -149,14 +153,14 @@ def strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note
|
||||
curr_tokens = sum(estimate_tokens(x.text) for x in buf) if buf else 0
|
||||
if curr_tokens + b_tokens > max_tokens and buf:
|
||||
txt = "\n\n".join([x.text for x in buf]); idx = len(chunks)
|
||||
win = _create_win(doc_title, buf[0].section_title, txt)
|
||||
win = _create_win(context_prefix, buf[0].section_title, txt)
|
||||
chunks.append(Chunk(id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx, text=txt, window=win, token_count=curr_tokens, section_title=buf[0].section_title, section_path=buf[0].section_path, neighbors_prev=None, neighbors_next=None))
|
||||
buf = []
|
||||
buf.append(b)
|
||||
|
||||
if buf:
|
||||
txt = "\n\n".join([x.text for x in buf]); idx = len(chunks)
|
||||
win = _create_win(doc_title, buf[0].section_title, txt)
|
||||
win = _create_win(context_prefix, buf[0].section_title, txt)
|
||||
chunks.append(Chunk(id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx, text=txt, window=win, token_count=estimate_tokens(txt), section_title=buf[0].section_title, section_path=buf[0].section_path, neighbors_prev=None, neighbors_next=None))
|
||||
|
||||
return chunks
|
||||
Reference in new issue
Block a user