bug fixing

This commit is contained in:
Lars committed 2025-12-12 11:45:43 +01:00
1 parent df971f9c56
commit 3c19e192bc
2 files changed
+165 -142

No files matched your search

+126 -58
View File
@@ -13,12 +13,12 @@ import logging
# Services
from app.services.semantic_analyzer import get_semantic_analyzer
# Core Imports (mit Fehlerbehandlung für Tests)
# Core Imports
try:
from app.core.derive_edges import build_edges_for_note
except ImportError:
# Mock für Standalone-Tests ohne vollständige App-Struktur
def build_edges_for_note(*args, **kwargs): return []
# Mock für Tests
def build_edges_for_note(md_text, note_id, note_type, chunks=[], references=[]): return []
logger = logging.getLogger(__name__)
@@ -70,7 +70,8 @@ def extract_frontmatter_from_text(md_text: str) -> Tuple[Dict[str, Any], str]:
# 2. DATA CLASSES
# ==========================================
_SENT_SPLIT = re.compile(r'(?<=[.!?])\s+(?=[A-ZÄÖÜ0-9„(])'); _WS = re.compile(r'\s+')
_SENT_SPLIT = re.compile(r'(?<=[.!?])\s+(?=[A-ZÄÖÜ0-9„(])')
_WS = re.compile(r'\s+')
def estimate_tokens(text: str) -> int:
return max(1, math.ceil(len(text.strip()) / 4))
@@ -90,7 +91,6 @@ class Chunk:
id: str; note_id: str; index: int; text: str; window: str; token_count: int
section_title: Optional[str]; section_path: str
neighbors_prev: Optional[str]; neighbors_next: Optional[str]
# NEU: Speichert Kanten, die der Algorithmus diesem Chunk zugewiesen hat
suggested_edges: Optional[List[str]] = None
# ==========================================
@@ -98,30 +98,73 @@ class Chunk:
# ==========================================
def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
md = MarkdownIt("commonmark").enable("table")
tokens = md.parse(md_text)
blocks = []; h1_title = "Dokument"; h2 = None; section_path = "/"
"""
Zerlegt Text in logische Blöcke (Absätze, Header).
Verbesserte Version: Splittet auch reine Absätze.
"""
blocks = []
h1_title = "Dokument"
section_path = "/"
current_h2 = None
fm, text_without_fm = extract_frontmatter_from_text(md_text)
# Fallback Body Block
if text_without_fm.strip():
blocks.append(RawBlock("paragraph", text_without_fm.strip(), None, section_path, h2))
# Versuche echten Titel zu finden
# H1 suchen
h1_match = re.search(r'^#\s+(.*)', text_without_fm, re.MULTILINE)
if h1_match: h1_title = h1_match.group(1).strip()
if h1_match:
h1_title = h1_match.group(1).strip()
# Rudimentäres Parsing (Markdown-It ist komplex einzubinden ohne vollen Visitor)
# Wir splitten hier einfach an Doppel-Newlines für Paragraphen, wenn keine Header da sind.
# Zuerst Header-Struktur bewahren
lines = text_without_fm.split('\n')
buffer = []
for line in lines:
stripped = line.strip()
if stripped.startswith('# '): # H1 ignorieren wir im Body meist
continue
elif stripped.startswith('## '):
# Flush buffer
if buffer:
content = "\n".join(buffer).strip()
if content:
blocks.append(RawBlock("paragraph", content, None, section_path, current_h2))
buffer = []
current_h2 = stripped[3:].strip()
section_path = f"/{current_h2}"
blocks.append(RawBlock("heading", stripped, 2, section_path, current_h2))
elif not stripped:
# Leere Zeile -> Absatzende
if buffer:
content = "\n".join(buffer).strip()
if content:
blocks.append(RawBlock("paragraph", content, None, section_path, current_h2))
buffer = []
else:
buffer.append(line)
if buffer:
content = "\n".join(buffer).strip()
if content:
blocks.append(RawBlock("paragraph", content, None, section_path, current_h2))
return blocks, h1_title
def _strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, doc_title: str = "", context_prefix: str = "") -> List[Chunk]:
target = config.get("target", 400); max_tokens = config.get("max", 600)
target = config.get("target", 400)
max_tokens = config.get("max", 600)
overlap_val = config.get("overlap", (50, 80))
overlap = sum(overlap_val) // 2 if isinstance(overlap_val, tuple) else overlap_val
chunks = []; buf = []
chunks = []
buf = [] # Buffer für Blöcke
def _add_chunk(txt, win, sec, path):
def _create_chunk(txt, win, sec, path):
idx = len(chunks)
chunks.append(Chunk(
id=f"{note_id}#c{len(chunks):02d}", note_id=note_id, index=len(chunks),
id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx,
text=txt, window=win, token_count=estimate_tokens(txt),
section_title=sec, section_path=path, neighbors_prev=None, neighbors_next=None,
suggested_edges=[]
@@ -130,27 +173,74 @@ def _strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], not
def flush_buffer():
nonlocal buf
if not buf: return
text_body = "\n\n".join([b.text for b in buf])
win_body = f"{context_prefix}\n{text_body}".strip() if context_prefix else text_body
# Simple Logic for brevity: Just add chunk if small enough, else split sentences
# Kombiniere Blöcke im Buffer
text_body = "\n\n".join([b.text for b in buf])
sec_title = buf[-1].section_title if buf else None
sec_path = buf[-1].section_path if buf else "/"
# Check Größe
if estimate_tokens(text_body) <= max_tokens:
_add_chunk(text_body, win_body, buf[-1].section_title, buf[-1].section_path)
win_body = f"{context_prefix}\n{text_body}".strip() if context_prefix else text_body
_create_chunk(text_body, win_body, sec_title, sec_path)
else:
# Fallback naive split
_add_chunk(text_body[:max_tokens*4], win_body[:max_tokens*4], buf[-1].section_title, buf[-1].section_path)
# Text ist zu groß -> Splitte nach Sätzen
sentences = split_sentences(text_body)
current_chunk_sents = []
current_len = 0
for sent in sentences:
sent_len = estimate_tokens(sent)
if current_len + sent_len > target and current_chunk_sents:
# Chunk abschließen
c_txt = " ".join(current_chunk_sents)
c_win = f"{context_prefix}\n{c_txt}".strip() if context_prefix else c_txt
_create_chunk(c_txt, c_win, sec_title, sec_path)
# Overlap für nächsten Chunk
# Wir nehmen die letzten Sätze, die in den Overlap passen
overlap_sents = []
ov_len = 0
for s in reversed(current_chunk_sents):
if ov_len + estimate_tokens(s) < overlap:
overlap_sents.insert(0, s)
ov_len += estimate_tokens(s)
else:
break
current_chunk_sents = list(overlap_sents)
current_chunk_sents.append(sent)
current_len = ov_len + sent_len
else:
current_chunk_sents.append(sent)
current_len += sent_len
# Rest verarbeiten
if current_chunk_sents:
c_txt = " ".join(current_chunk_sents)
c_win = f"{context_prefix}\n{c_txt}".strip() if context_prefix else c_txt
_create_chunk(c_txt, c_win, sec_title, sec_path)
buf = []
for b in blocks:
if estimate_tokens("\n\n".join([x.text for x in buf] + [b.text])) >= target:
if b.kind == "heading": continue # Header nicht direkt in Text mischen, dienen nur Struktur
# Wenn Buffer + neuer Block zu groß -> Flush
current_buf_text = "\n\n".join([x.text for x in buf])
if estimate_tokens(current_buf_text) + estimate_tokens(b.text) >= target:
flush_buffer()
buf.append(b)
# Wenn der Block selbst riesig ist (größer als Target), sofort flushen und splitten
if estimate_tokens(b.text) >= target:
flush_buffer()
flush_buffer()
return chunks
def _strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, doc_title: str = "") -> List[Chunk]:
# Wrapper für Struktur-basiertes Chunking
# Im echten System ist hier die komplexe Logik. Wir nutzen hier sliding_window als Fallback.
return _strategy_sliding_window(blocks, config, note_id, doc_title, context_prefix=f"# {doc_title}")
# ==========================================
@@ -158,11 +248,6 @@ def _strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id
# ==========================================
async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Optional[Dict] = None) -> List[Chunk]:
"""
Hauptfunktion. Orchestriert das Chunking.
Unterstützt Dependency Injection für Config (Tests).
"""
# 1. Config & Status
if config is None:
config = get_chunk_config(note_type)
@@ -172,15 +257,12 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
primary_strategy = config.get("strategy", "sliding_window")
enable_smart_edges = config.get("enable_smart_edge_allocation", False)
# 2. Safety Override: Keine AI-Allocation bei Drafts (spart Ressourcen/Zeit)
if enable_smart_edges and note_status in ["draft", "initial_gen"]:
logger.info(f"Chunker: Skipping Smart Edges for draft '{note_id}'.")
enable_smart_edges = False
# 3. Step 1: Parsing & Primär-Zerlegung (Deterministisch)
blocks, doc_title = parse_blocks(md_text)
# Wähle Strategie
if primary_strategy == "by_heading":
chunks = await asyncio.to_thread(_strategy_by_heading, blocks, config, note_id, doc_title)
else:
@@ -189,11 +271,9 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
if not chunks:
return []
# 4. Step 2: Smart Edge Allocation (Optional)
if enable_smart_edges:
chunks = await _run_smart_edge_allocation(chunks, md_text, note_id, note_type)
# 5. Post-Processing (Neighbors)
for i, ch in enumerate(chunks):
ch.neighbors_prev = chunks[i-1].id if i > 0 else None
ch.neighbors_next = chunks[i+1].id if i < len(chunks)-1 else None
@@ -201,59 +281,47 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
return chunks
async def _run_smart_edge_allocation(chunks: List[Chunk], full_text: str, note_id: str, note_type: str) -> List[Chunk]:
"""
Führt die LLM-basierte Kantenzuordnung durch.
"""
analyzer = get_semantic_analyzer()
# A. Alle potenziellen Kanten der Notiz sammeln
# Wir rufen derive_edges auf dem GESAMTEN Text auf.
# WICHTIG: chunks=[] übergeben, damit er nur Note-Level References findet.
# FIX: Positional Argument für text übergeben, um TypeError zu vermeiden
raw_edges = build_edges_for_note(
text=full_text,
full_text,
note_id=note_id,
note_type=note_type,
chunks=[],
references=[]
references=[] # Falls die Signatur references erwartet
)
# Formatieren als "kind:Target" Liste
all_candidates = set()
for e in raw_edges:
# Nur Kanten mit Ziel und Typ, keine internen Strukturkanten
if e.get("target_id") and e.get("kind") not in ["next", "prev", "belongs_to"]:
all_candidates.add(f"{e['kind']}:{e['target_id']}")
# Robustheit: raw_edges könnte None sein, falls der Mock schlecht ist
if raw_edges:
for e in raw_edges:
if e.get("target_id") and e.get("kind") not in ["next", "prev", "belongs_to"]:
all_candidates.add(f"{e['kind']}:{e['target_id']}")
candidate_list = list(all_candidates)
if not candidate_list:
return chunks # Keine Kanten zu verteilen
return chunks
# B. LLM Filterung pro Chunk (Parallel)
tasks = []
for chunk in chunks:
tasks.append(analyzer.assign_edges_to_chunk(chunk.text, candidate_list, note_type))
# Alle Ergebnisse sammeln
results_per_chunk = await asyncio.gather(*tasks)
# C. Injection & Fallback
assigned_edges_global = set()
for i, confirmed_edges in enumerate(results_per_chunk):
chunk = chunks[i]
# Speichere bestätigte Kanten
chunk.suggested_edges = confirmed_edges
assigned_edges_global.update(confirmed_edges)
# Injiziere in den Text (für Indexierung)
if confirmed_edges:
injection_str = "\n" + " ".join([f"[[rel:{e.split(':')[0]}|{e.split(':')[1]}]]" for e in confirmed_edges if ':' in e])
chunk.text += injection_str
chunk.window += injection_str
# D. Fallback: Kanten, die NIRGENDS zugeordnet wurden, landen in allen Chunks (Sicherheit)
unassigned = set(candidate_list) - assigned_edges_global
if unassigned:
fallback_str = "\n" + " ".join([f"[[rel:{e.split(':')[0]}|{e.split(':')[1]}]]" for e in unassigned if ':' in e])