neue chunker strategie mit smart chunking

This commit is contained in:
Lars committed 2025-12-12 10:10:05 +01:00
1 parent 49cdc9a13f
commit cd28a9d35c
4 files changed
+329 -487

No files matched your search

+34 -34
View File
@@ -1,41 +1,42 @@
version: 1.2 # Update für Smart Chunking Config
version: 1.3 # Update für Smart Edge Allocation
# --- CHUNKING DEFINITIONEN ---
# Hier definieren wir die technischen Strategien zentral.
# Hier definieren wir die technischen Strategien und den Smart Edge Filter.
chunking_profiles:
# Standard für Fließtexte (Sliding Window)
# 1. Standard Profile (Sliding Window, KEIN LLM-Filter)
sliding_short:
strategy: sliding_window
enable_smart_edge_allocation: false # Sekundärverfeinerung deaktiviert
target: 200
max: 350
overlap: [30, 50]
sliding_standard:
strategy: sliding_window
# 2. Smart Edge Allocation Profile (Sliding Window + LLM-Filter)
sliding_smart_edges:
strategy: sliding_window # Primärzerlegung: Sliding Window
enable_smart_edge_allocation: true # SEKUNDÄRVERFEINERUNG: LLM-Filter aktiv
target: 400
max: 600
overlap: [50, 80]
sliding_large:
strategy: sliding_window
target: 500
max: 800
overlap: [60, 100]
# Smart Chunking für Strukturen (Harte Splits)
# 3. Strukturierte Profile (By Heading, KEIN LLM-Filter)
structured_strict:
strategy: by_heading
enable_smart_edge_allocation: false
split_level: 2
max: 600 # Fallback Limit
target: 400 # Fallback Target bei Sub-Chunking
overlap: [50, 80] # Overlap bei Sub-Chunking
# NEU: LLM-basierte semantische Zerlegung (Chunker.py ruft semantic_analyzer.py)
semantic_llm:
strategy: semantic_llm
# Da das LLM die Längensteuerung übernimmt, dienen diese als Fallback/Empfehlung
target: 400
max: 800
max: 600
target: 400
overlap: [50, 80]
# 4. Strukturierte Profile (By Heading + LLM-Filter)
structured_smart_edges:
strategy: by_heading # Primärzerlegung: Harte Trennung
enable_smart_edge_allocation: true # SEKUNDÄRVERFEINERUNG: LLM-Filter aktiv
split_level: 2
max: 600
target: 400
overlap: [50, 80]
defaults:
retriever_weight: 1.0
@@ -45,12 +46,12 @@ defaults:
types:
# --- WISSENSBAUSTEINE ---
concept:
chunking_profile: sliding_standard
chunking_profile: sliding_smart_edges # Nutzt Kantenfilterung
retriever_weight: 0.60
edge_defaults: ["references", "related_to"]
source:
chunking_profile: sliding_standard
chunking_profile: sliding_short # Kein LLM-Filter
retriever_weight: 0.50
edge_defaults: []
@@ -61,17 +62,17 @@ types:
# --- IDENTITÄT & PERSÖNLICHKEIT ---
profile:
chunking_profile: structured_strict
chunking_profile: structured_smart_edges # Strukturiert + Kantenfilterung
retriever_weight: 0.70
edge_defaults: ["references", "related_to"]
value:
chunking_profile: structured_strict
chunking_profile: structured_smart_edges
retriever_weight: 1.00
edge_defaults: ["related_to"]
principle:
chunking_profile: structured_strict
chunking_profile: structured_smart_edges
retriever_weight: 0.95
edge_defaults: ["derived_from", "references"]
@@ -81,18 +82,18 @@ types:
edge_defaults: ["related_to"]
experience:
chunking_profile: sliding_standard
chunking_profile: sliding_smart_edges
retriever_weight: 0.90
edge_defaults: ["derived_from", "references"]
# --- STRATEGIE & ENTSCHEIDUNG ---
goal:
chunking_profile: sliding_standard
chunking_profile: sliding_smart_edges
retriever_weight: 0.95
edge_defaults: ["depends_on", "related_to"]
decision:
chunking_profile: structured_strict
chunking_profile: structured_smart_edges
retriever_weight: 1.00
edge_defaults: ["caused_by", "references"]
@@ -108,7 +109,7 @@ types:
# --- OPERATIV ---
project:
chunking_profile: sliding_large
chunking_profile: sliding_smart_edges
retriever_weight: 0.97
edge_defaults: ["references", "depends_on"]
@@ -118,7 +119,6 @@ types:
edge_defaults: ["depends_on", "part_of"]
journal:
# NEUE ZUWEISUNG: Journale profitieren am meisten von der semantischen Analyse
chunking_profile: semantic_llm
chunking_profile: sliding_smart_edges # Fließtext + Kantenfilterung
retriever_weight: 0.80
edge_defaults: ["references", "related_to"]