neue chunker strategie mit smart chunking
This commit is contained in:
1 parent
49cdc9a13f
commit
cd28a9d35c
4 files changed
+329
-487
No files matched your search
+34
-34
@@ -1,41 +1,42 @@
|
||||
version: 1.2 # Update für Smart Chunking Config
|
||||
version: 1.3 # Update für Smart Edge Allocation
|
||||
|
||||
# --- CHUNKING DEFINITIONEN ---
|
||||
# Hier definieren wir die technischen Strategien zentral.
|
||||
# Hier definieren wir die technischen Strategien und den Smart Edge Filter.
|
||||
chunking_profiles:
|
||||
# Standard für Fließtexte (Sliding Window)
|
||||
|
||||
# 1. Standard Profile (Sliding Window, KEIN LLM-Filter)
|
||||
sliding_short:
|
||||
strategy: sliding_window
|
||||
enable_smart_edge_allocation: false # Sekundärverfeinerung deaktiviert
|
||||
target: 200
|
||||
max: 350
|
||||
overlap: [30, 50]
|
||||
|
||||
sliding_standard:
|
||||
strategy: sliding_window
|
||||
|
||||
# 2. Smart Edge Allocation Profile (Sliding Window + LLM-Filter)
|
||||
sliding_smart_edges:
|
||||
strategy: sliding_window # Primärzerlegung: Sliding Window
|
||||
enable_smart_edge_allocation: true # SEKUNDÄRVERFEINERUNG: LLM-Filter aktiv
|
||||
target: 400
|
||||
max: 600
|
||||
overlap: [50, 80]
|
||||
|
||||
sliding_large:
|
||||
strategy: sliding_window
|
||||
target: 500
|
||||
max: 800
|
||||
overlap: [60, 100]
|
||||
|
||||
# Smart Chunking für Strukturen (Harte Splits)
|
||||
# 3. Strukturierte Profile (By Heading, KEIN LLM-Filter)
|
||||
structured_strict:
|
||||
strategy: by_heading
|
||||
enable_smart_edge_allocation: false
|
||||
split_level: 2
|
||||
max: 600 # Fallback Limit
|
||||
target: 400 # Fallback Target bei Sub-Chunking
|
||||
overlap: [50, 80] # Overlap bei Sub-Chunking
|
||||
|
||||
# NEU: LLM-basierte semantische Zerlegung (Chunker.py ruft semantic_analyzer.py)
|
||||
semantic_llm:
|
||||
strategy: semantic_llm
|
||||
# Da das LLM die Längensteuerung übernimmt, dienen diese als Fallback/Empfehlung
|
||||
target: 400
|
||||
max: 800
|
||||
max: 600
|
||||
target: 400
|
||||
overlap: [50, 80]
|
||||
|
||||
# 4. Strukturierte Profile (By Heading + LLM-Filter)
|
||||
structured_smart_edges:
|
||||
strategy: by_heading # Primärzerlegung: Harte Trennung
|
||||
enable_smart_edge_allocation: true # SEKUNDÄRVERFEINERUNG: LLM-Filter aktiv
|
||||
split_level: 2
|
||||
max: 600
|
||||
target: 400
|
||||
overlap: [50, 80]
|
||||
|
||||
defaults:
|
||||
retriever_weight: 1.0
|
||||
@@ -45,12 +46,12 @@ defaults:
|
||||
types:
|
||||
# --- WISSENSBAUSTEINE ---
|
||||
concept:
|
||||
chunking_profile: sliding_standard
|
||||
chunking_profile: sliding_smart_edges # Nutzt Kantenfilterung
|
||||
retriever_weight: 0.60
|
||||
edge_defaults: ["references", "related_to"]
|
||||
|
||||
source:
|
||||
chunking_profile: sliding_standard
|
||||
chunking_profile: sliding_short # Kein LLM-Filter
|
||||
retriever_weight: 0.50
|
||||
edge_defaults: []
|
||||
|
||||
@@ -61,17 +62,17 @@ types:
|
||||
|
||||
# --- IDENTITÄT & PERSÖNLICHKEIT ---
|
||||
profile:
|
||||
chunking_profile: structured_strict
|
||||
chunking_profile: structured_smart_edges # Strukturiert + Kantenfilterung
|
||||
retriever_weight: 0.70
|
||||
edge_defaults: ["references", "related_to"]
|
||||
|
||||
value:
|
||||
chunking_profile: structured_strict
|
||||
chunking_profile: structured_smart_edges
|
||||
retriever_weight: 1.00
|
||||
edge_defaults: ["related_to"]
|
||||
|
||||
principle:
|
||||
chunking_profile: structured_strict
|
||||
chunking_profile: structured_smart_edges
|
||||
retriever_weight: 0.95
|
||||
edge_defaults: ["derived_from", "references"]
|
||||
|
||||
@@ -81,18 +82,18 @@ types:
|
||||
edge_defaults: ["related_to"]
|
||||
|
||||
experience:
|
||||
chunking_profile: sliding_standard
|
||||
chunking_profile: sliding_smart_edges
|
||||
retriever_weight: 0.90
|
||||
edge_defaults: ["derived_from", "references"]
|
||||
|
||||
# --- STRATEGIE & ENTSCHEIDUNG ---
|
||||
goal:
|
||||
chunking_profile: sliding_standard
|
||||
chunking_profile: sliding_smart_edges
|
||||
retriever_weight: 0.95
|
||||
edge_defaults: ["depends_on", "related_to"]
|
||||
|
||||
decision:
|
||||
chunking_profile: structured_strict
|
||||
chunking_profile: structured_smart_edges
|
||||
retriever_weight: 1.00
|
||||
edge_defaults: ["caused_by", "references"]
|
||||
|
||||
@@ -108,7 +109,7 @@ types:
|
||||
|
||||
# --- OPERATIV ---
|
||||
project:
|
||||
chunking_profile: sliding_large
|
||||
chunking_profile: sliding_smart_edges
|
||||
retriever_weight: 0.97
|
||||
edge_defaults: ["references", "depends_on"]
|
||||
|
||||
@@ -118,7 +119,6 @@ types:
|
||||
edge_defaults: ["depends_on", "part_of"]
|
||||
|
||||
journal:
|
||||
# NEUE ZUWEISUNG: Journale profitieren am meisten von der semantischen Analyse
|
||||
chunking_profile: semantic_llm
|
||||
chunking_profile: sliding_smart_edges # Fließtext + Kantenfilterung
|
||||
retriever_weight: 0.80
|
||||
edge_defaults: ["references", "related_to"]
|
||||
Reference in new issue
Block a user