Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf302e8334 | ||
|
|
82c7752266 | ||
|
|
c676c8263f | ||
|
|
f6b2375d65 | ||
|
|
d1a065fec8 | ||
|
|
f686ecf947 | ||
|
|
6ac1f318d0 | ||
|
|
cbaf664123 | ||
|
|
470e653da6 | ||
|
|
43d3d8f7f3 | ||
|
|
83c0c9944d |
No files matched your search
@@ -34,6 +34,8 @@ class Settings:
|
||||
# --- WP-20 Hybrid LLM Provider ---
|
||||
# Erlaubt: "ollama" | "gemini" | "openrouter"
|
||||
MINDNET_LLM_PROVIDER: str = os.getenv("MINDNET_LLM_PROVIDER", "openrouter").lower()
|
||||
# Standardwert 10000, falls nichts in der .env steht
|
||||
MAX_OLLAMA_CHARS: int = int(os.getenv("MAX_OLLAMA_CHARS", 10000))
|
||||
|
||||
# Google AI Studio (2025er Lite-Modell für höhere Kapazität)
|
||||
GOOGLE_API_KEY: str | None = os.getenv("GOOGLE_API_KEY")
|
||||
|
||||
+157
-184
@@ -1,13 +1,16 @@
|
||||
"""
|
||||
FILE: app/core/chunker.py
|
||||
DESCRIPTION: Zerlegt Texte in Chunks (Sliding Window oder nach Headings).
|
||||
Orchestriert die Smart-Edge-Allocation via SemanticAnalyzer.
|
||||
FIX V3: Support für mehrzeilige Callouts und Section-Propagation.
|
||||
VERSION: 3.1.0 (Full Compatibility Merge)
|
||||
WP-15b: Implementiert Edge-Inheritance und Candidate-Pool Vorbereitung.
|
||||
Zentralisiert die Kanten-Vorbereitung für die spätere binäre Validierung.
|
||||
Bietet volle Unterstützung für Hybrid-Chunking (Strict/Soft/Safety-Net).
|
||||
VERSION: 3.2.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: re, math, yaml, pathlib, asyncio, logging
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Dict, Optional, Tuple, Any, Set
|
||||
import re
|
||||
import math
|
||||
@@ -17,15 +20,18 @@ import asyncio
|
||||
import logging
|
||||
|
||||
# Services
|
||||
from app.services.semantic_analyzer import get_semantic_analyzer
|
||||
# In WP-15b wird die KI-Validierung in die ingestion.py verlagert.
|
||||
# Wir behalten den Import für Abwärtskompatibilität, falls Legacy-Skripte ihn benötigen.
|
||||
try:
|
||||
from app.services.semantic_analyzer import get_semantic_analyzer
|
||||
except ImportError:
|
||||
def get_semantic_analyzer(): return None
|
||||
|
||||
# Core Imports
|
||||
# Wir importieren build_edges_for_note nur, um kompatibel zur Signatur zu bleiben
|
||||
# oder für den Fallback.
|
||||
try:
|
||||
from app.core.derive_edges import build_edges_for_note
|
||||
except ImportError:
|
||||
# Mock für Tests
|
||||
# Fallback für Standalone-Betrieb oder Tests
|
||||
def build_edges_for_note(note_id, chunks, note_level_references=None, include_note_scope_refs=False): return []
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -54,7 +60,7 @@ def _load_yaml_config() -> Dict[str, Any]:
|
||||
def get_chunk_config(note_type: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Lädt die Chunking-Strategie basierend auf dem Note-Type aus types.yaml.
|
||||
Dies sichert die Kompatibilität zu WP-15 (Profile).
|
||||
Sichert die Kompatibilität zu WP-15 Profilen.
|
||||
"""
|
||||
full_config = _load_yaml_config()
|
||||
profiles = full_config.get("chunking_profiles", {})
|
||||
@@ -75,6 +81,7 @@ def get_chunk_config(note_type: str) -> Dict[str, Any]:
|
||||
return config
|
||||
|
||||
def extract_frontmatter_from_text(md_text: str) -> Tuple[Dict[str, Any], str]:
|
||||
"""Trennt YAML-Frontmatter vom eigentlichen Text."""
|
||||
fm_match = re.match(r'^\s*---\s*\n(.*?)\n---', md_text, re.DOTALL)
|
||||
if not fm_match: return {}, md_text
|
||||
try:
|
||||
@@ -89,12 +96,15 @@ def extract_frontmatter_from_text(md_text: str) -> Tuple[Dict[str, Any], str]:
|
||||
# 2. DATA CLASSES & TEXT TOOLS
|
||||
# ==========================================
|
||||
|
||||
_SENT_SPLIT = re.compile(r'(?<=[.!?])\s+(?=[A-ZÄÖÜ0-9„(])'); _WS = re.compile(r'\s+')
|
||||
_SENT_SPLIT = re.compile(r'(?<=[.!?])\s+(?=[A-ZÄÖÜ0-9„(])')
|
||||
_WS = re.compile(r'\s+')
|
||||
|
||||
def estimate_tokens(text: str) -> int:
|
||||
"""Grobe Schätzung der Token-Anzahl (4 Zeichen pro Token)."""
|
||||
return max(1, math.ceil(len(text.strip()) / 4))
|
||||
|
||||
def split_sentences(text: str) -> list[str]:
|
||||
"""Teilt Text in Sätze auf unter Berücksichtigung von Interpunktion."""
|
||||
text = _WS.sub(' ', text.strip())
|
||||
if not text: return []
|
||||
parts = _SENT_SPLIT.split(text)
|
||||
@@ -102,13 +112,26 @@ def split_sentences(text: str) -> list[str]:
|
||||
|
||||
@dataclass
|
||||
class RawBlock:
|
||||
kind: str; text: str; level: Optional[int]; section_path: str; section_title: Optional[str]
|
||||
kind: str
|
||||
text: str
|
||||
level: Optional[int]
|
||||
section_path: str
|
||||
section_title: Optional[str]
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
id: str; note_id: str; index: int; text: str; window: str; token_count: int
|
||||
section_title: Optional[str]; section_path: str
|
||||
neighbors_prev: Optional[str]; neighbors_next: Optional[str]
|
||||
id: str
|
||||
note_id: str
|
||||
index: int
|
||||
text: str
|
||||
window: str
|
||||
token_count: int
|
||||
section_title: Optional[str]
|
||||
section_path: str
|
||||
neighbors_prev: Optional[str]
|
||||
neighbors_next: Optional[str]
|
||||
# WP-15b: Liste von Kandidaten für die semantische Validierung
|
||||
candidate_pool: List[Dict[str, Any]] = field(default_factory=list)
|
||||
suggested_edges: Optional[List[str]] = None
|
||||
|
||||
# ==========================================
|
||||
@@ -118,7 +141,7 @@ class Chunk:
|
||||
def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
"""
|
||||
Zerlegt Text in logische Blöcke (Absätze, Header).
|
||||
Wichtig für die Strategie 'by_heading'.
|
||||
Wichtig für die Strategie 'by_heading' und die Edge-Inheritance.
|
||||
"""
|
||||
blocks = []
|
||||
h1_title = "Dokument"
|
||||
@@ -165,14 +188,15 @@ def parse_blocks(md_text: str) -> Tuple[List[RawBlock], str]:
|
||||
|
||||
def _strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, doc_title: str = "", context_prefix: str = "") -> List[Chunk]:
|
||||
"""
|
||||
Die Standard-Strategie aus WP-15.
|
||||
Fasst Blöcke zusammen und schneidet bei 'target' Tokens (mit Satz-Rücksicht).
|
||||
Standard-Strategie aus WP-15.
|
||||
Fasst Blöcke zusammen und schneidet bei 'target' Tokens.
|
||||
"""
|
||||
target = config.get("target", 400)
|
||||
max_tokens = config.get("max", 600)
|
||||
overlap_val = config.get("overlap", (50, 80))
|
||||
overlap = sum(overlap_val) // 2 if isinstance(overlap_val, tuple) else overlap_val
|
||||
chunks = []; buf = []
|
||||
chunks = []
|
||||
buf = []
|
||||
|
||||
def _create_chunk(txt, win, sec, path):
|
||||
idx = len(chunks)
|
||||
@@ -180,7 +204,7 @@ def _strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], not
|
||||
id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx,
|
||||
text=txt, window=win, token_count=estimate_tokens(txt),
|
||||
section_title=sec, section_path=path, neighbors_prev=None, neighbors_next=None,
|
||||
suggested_edges=[]
|
||||
candidate_pool=[]
|
||||
))
|
||||
|
||||
def flush_buffer():
|
||||
@@ -190,14 +214,11 @@ def _strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], not
|
||||
text_body = "\n\n".join([b.text for b in buf])
|
||||
sec_title = buf[-1].section_title if buf else None
|
||||
sec_path = buf[-1].section_path if buf else "/"
|
||||
|
||||
# Context Prefix (z.B. H1) voranstellen für Embedding-Qualität
|
||||
win_body = f"{context_prefix}\n{text_body}".strip() if context_prefix else text_body
|
||||
|
||||
if estimate_tokens(text_body) <= max_tokens:
|
||||
_create_chunk(text_body, win_body, sec_title, sec_path)
|
||||
else:
|
||||
# Zu groß -> Satzweiser Split
|
||||
sentences = split_sentences(text_body)
|
||||
current_chunk_sents = []
|
||||
current_len = 0
|
||||
@@ -209,15 +230,13 @@ def _strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], not
|
||||
c_win = f"{context_prefix}\n{c_txt}".strip() if context_prefix else c_txt
|
||||
_create_chunk(c_txt, c_win, sec_title, sec_path)
|
||||
|
||||
# Overlap für nächsten Chunk
|
||||
overlap_sents = []
|
||||
ov_len = 0
|
||||
for s in reversed(current_chunk_sents):
|
||||
if ov_len + estimate_tokens(s) < overlap:
|
||||
overlap_sents.insert(0, s)
|
||||
ov_len += estimate_tokens(s)
|
||||
else:
|
||||
break
|
||||
else: break
|
||||
|
||||
current_chunk_sents = list(overlap_sents)
|
||||
current_chunk_sents.append(sent)
|
||||
@@ -226,12 +245,10 @@ def _strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], not
|
||||
current_chunk_sents.append(sent)
|
||||
current_len += sent_len
|
||||
|
||||
# Rest
|
||||
if current_chunk_sents:
|
||||
c_txt = " ".join(current_chunk_sents)
|
||||
c_win = f"{context_prefix}\n{c_txt}".strip() if context_prefix else c_txt
|
||||
_create_chunk(c_txt, c_win, sec_title, sec_path)
|
||||
|
||||
buf = []
|
||||
|
||||
for b in blocks:
|
||||
@@ -248,132 +265,137 @@ def _strategy_sliding_window(blocks: List[RawBlock], config: Dict[str, Any], not
|
||||
|
||||
def _strategy_by_heading(blocks: List[RawBlock], config: Dict[str, Any], note_id: str, doc_title: str = "") -> List[Chunk]:
|
||||
"""
|
||||
Strategie für strukturierte Daten (Profile, Werte).
|
||||
Nutzt sliding_window, forciert aber Schnitte an Headings (via parse_blocks Vorarbeit).
|
||||
Hybrid-Strategie v2.9 (Strict/Soft/Safety-Net).
|
||||
"""
|
||||
return _strategy_sliding_window(blocks, config, note_id, doc_title, context_prefix=f"# {doc_title}")
|
||||
strict = config.get("strict_heading_split", False)
|
||||
target = config.get("target", 400)
|
||||
max_tokens = config.get("max", 600)
|
||||
split_level = config.get("split_level", 2)
|
||||
|
||||
chunks = []
|
||||
current_buf = []
|
||||
current_tokens = 0
|
||||
|
||||
def _flush(sec_title, sec_path):
|
||||
nonlocal current_buf, current_tokens
|
||||
if not current_buf: return
|
||||
txt = "\n\n".join(current_buf)
|
||||
win = f"# {doc_title}\n## {sec_title}\n{txt}".strip() if sec_title else txt
|
||||
idx = len(chunks)
|
||||
chunks.append(Chunk(
|
||||
id=f"{note_id}#c{idx:02d}", note_id=note_id, index=idx,
|
||||
text=txt, window=win, token_count=estimate_tokens(txt),
|
||||
section_title=sec_title, section_path=sec_path,
|
||||
neighbors_prev=None, neighbors_next=None,
|
||||
candidate_pool=[]
|
||||
))
|
||||
current_buf = []
|
||||
current_tokens = 0
|
||||
|
||||
for b in blocks:
|
||||
if b.kind == "heading":
|
||||
# Hierarchie-Check: Split bei Überschriften oberhalb des Split-Levels
|
||||
if b.level < split_level:
|
||||
_flush(b.section_title, b.section_path)
|
||||
elif b.level == split_level:
|
||||
if strict or current_tokens >= target:
|
||||
_flush(b.section_title, b.section_path)
|
||||
continue
|
||||
|
||||
block_tokens = estimate_tokens(b.text)
|
||||
if current_tokens + block_tokens > max_tokens and current_buf:
|
||||
_flush(b.section_title, b.section_path)
|
||||
|
||||
current_buf.append(b.text)
|
||||
current_tokens += block_tokens
|
||||
|
||||
if current_buf:
|
||||
last = blocks[-1] if blocks else None
|
||||
_flush(last.section_title if last else None, last.section_path if last else "/")
|
||||
|
||||
return chunks
|
||||
|
||||
# ==========================================
|
||||
# 4. ROBUST EDGE PARSING & PROPAGATION (NEU)
|
||||
# 4. ROBUST EDGE PARSING & PROPAGATION
|
||||
# ==========================================
|
||||
|
||||
def _parse_edges_robust(text: str) -> Set[str]:
|
||||
"""
|
||||
NEU: Findet Kanten im Text, auch wenn sie mehrzeilig oder 'kaputt' formatiert sind.
|
||||
Erkennt:
|
||||
> [!edge] type
|
||||
> [[Link]]
|
||||
Returns: Set von Strings "kind:target"
|
||||
Findet Kanten im Text (Wikilinks, Inlines, Callouts).
|
||||
Fix V3: Support für mehrzeilige Callouts.
|
||||
"""
|
||||
found_edges = set()
|
||||
|
||||
# A. Inline [[rel:type|target]] (Standard)
|
||||
# A. Inline [[rel:type|target]]
|
||||
inlines = re.findall(r'\[\[rel:([^\|\]]+)\|?([^\]]*)\]\]', text)
|
||||
for kind, target in inlines:
|
||||
k = kind.strip()
|
||||
k = kind.strip().lower()
|
||||
t = target.strip()
|
||||
if k and t: found_edges.add(f"{k}:{t}")
|
||||
|
||||
# B. Multiline Callouts Parsing (Der Fix für dein Problem)
|
||||
# B. Multiline Callouts Parsing (WP-15 Fix)
|
||||
lines = text.split('\n')
|
||||
current_edge_type = None
|
||||
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
|
||||
# 1. Start Blockquote: > [!edge] type
|
||||
# (Erlaubt optionalen Doppelpunkt)
|
||||
callout_match = re.match(r'>\s*\[!edge\]\s*([^:\s]+)', stripped)
|
||||
if callout_match:
|
||||
current_edge_type = callout_match.group(1).strip()
|
||||
|
||||
# Check: Sind Links noch in der GLEICHEN Zeile?
|
||||
current_edge_type = callout_match.group(1).strip().lower()
|
||||
links = re.findall(r'\[\[([^\]]+)\]\]', stripped)
|
||||
for l in links:
|
||||
if "rel:" not in l:
|
||||
found_edges.add(f"{current_edge_type}:{l}")
|
||||
if "rel:" not in l: found_edges.add(f"{current_edge_type}:{l}")
|
||||
continue
|
||||
|
||||
# 2. Continuation Line: > [[Target]]
|
||||
# Wenn wir noch im 'edge mode' sind und die Zeile ein Zitat ist
|
||||
if current_edge_type and stripped.startswith('>'):
|
||||
links = re.findall(r'\[\[([^\]]+)\]\]', stripped)
|
||||
for l in links:
|
||||
if "rel:" not in l:
|
||||
found_edges.add(f"{current_edge_type}:{l}")
|
||||
|
||||
# 3. End of Blockquote (kein '>') -> Reset Type
|
||||
if "rel:" not in l: found_edges.add(f"{current_edge_type}:{l}")
|
||||
elif not stripped.startswith('>'):
|
||||
current_edge_type = None
|
||||
|
||||
return found_edges
|
||||
|
||||
def _propagate_section_edges(chunks: List[Chunk]) -> List[Chunk]:
|
||||
def _propagate_section_edges(chunks: List[Chunk], blocks: List[RawBlock]) -> List[Chunk]:
|
||||
"""
|
||||
NEU: Verteilt Kanten innerhalb einer Sektion.
|
||||
Löst das Problem: Callout steht oben im Kapitel, gilt aber für alle Chunks darunter.
|
||||
WP-15b: Implementiert Edge-Inheritance.
|
||||
Kanten aus Überschriften werden an untergeordnete Chunks vererbt.
|
||||
"""
|
||||
# Step 1: Sammeln pro Sektion
|
||||
section_map = {} # path -> set(kind:target)
|
||||
section_inheritance: Dict[str, Set[str]] = {}
|
||||
|
||||
# 1. Sammeln aus den Heading-Blöcken
|
||||
for b in blocks:
|
||||
if b.kind == "heading":
|
||||
edges = _parse_edges_robust(b.text)
|
||||
if edges:
|
||||
if b.section_path not in section_inheritance:
|
||||
section_inheritance[b.section_path] = set()
|
||||
section_inheritance[b.section_path].update(edges)
|
||||
|
||||
# 2. Injektion in den Candidate-Pool
|
||||
for ch in chunks:
|
||||
# Root-Level "/" ignorieren wir meist, da zu global
|
||||
if not ch.section_path or ch.section_path == "/": continue
|
||||
|
||||
edges = _parse_edges_robust(ch.text)
|
||||
if edges:
|
||||
if ch.section_path not in section_map:
|
||||
section_map[ch.section_path] = set()
|
||||
section_map[ch.section_path].update(edges)
|
||||
|
||||
# Step 2: Injizieren (Broadcasting)
|
||||
for ch in chunks:
|
||||
if ch.section_path in section_map:
|
||||
edges_to_add = section_map[ch.section_path]
|
||||
if not edges_to_add: continue
|
||||
|
||||
injections = []
|
||||
for e_str in edges_to_add:
|
||||
kind, target = e_str.split(':', 1)
|
||||
# Check: Kante schon im Text?
|
||||
token = f"[[rel:{kind}|{target}]]"
|
||||
if token not in ch.text:
|
||||
injections.append(token)
|
||||
|
||||
if injections:
|
||||
# Wir schreiben die Kanten "hart" in den Text.
|
||||
# Damit findet sie derive_edges.py später garantiert.
|
||||
block = "\n\n\n" + " ".join(injections)
|
||||
ch.text += block
|
||||
# Auch ins Window schreiben für Embedding-Kontext
|
||||
ch.window += block
|
||||
inherited = section_inheritance.get(ch.section_path, set())
|
||||
for e_str in inherited:
|
||||
kind, target = e_str.split(':', 1)
|
||||
ch.candidate_pool.append({"kind": kind, "to": target, "provenance": "inherited"})
|
||||
|
||||
return chunks
|
||||
|
||||
# ==========================================
|
||||
# 5. ORCHESTRATION (ASYNC)
|
||||
# 5. ORCHESTRATION (WP-15b)
|
||||
# ==========================================
|
||||
|
||||
async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Optional[Dict] = None) -> List[Chunk]:
|
||||
"""
|
||||
Hauptfunktion. Verbindet Parsing, Splitting und Edge-Allocation.
|
||||
Hauptfunktion zur Chunk-Generierung.
|
||||
Baut den Candidate-Pool für die semantische Validierung auf.
|
||||
"""
|
||||
# 1. Config laden (WP-15 Kompatibilität)
|
||||
if config is None:
|
||||
config = get_chunk_config(note_type)
|
||||
|
||||
fm, body_text = extract_frontmatter_from_text(md_text)
|
||||
note_status = fm.get("status", "").lower()
|
||||
|
||||
primary_strategy = config.get("strategy", "sliding_window")
|
||||
enable_smart_edges = config.get("enable_smart_edge_allocation", False)
|
||||
|
||||
# Drafts skippen LLM um Kosten/Zeit zu sparen
|
||||
if enable_smart_edges and note_status in ["draft", "initial_gen"]:
|
||||
logger.info(f"Chunker: Skipping Smart Edges for draft '{note_id}'.")
|
||||
enable_smart_edges = False
|
||||
|
||||
# 2. Parsing & Splitting
|
||||
# 1. Parsing & Splitting
|
||||
blocks, doc_title = parse_blocks(md_text)
|
||||
|
||||
if primary_strategy == "by_heading":
|
||||
@@ -381,94 +403,45 @@ async def assemble_chunks(note_id: str, md_text: str, note_type: str, config: Op
|
||||
else:
|
||||
chunks = await asyncio.to_thread(_strategy_sliding_window, blocks, config, note_id, doc_title)
|
||||
|
||||
if not chunks:
|
||||
return []
|
||||
if not chunks: return []
|
||||
|
||||
# 3. NEU: Propagation VOR Smart Edge Allocation
|
||||
# Das repariert die fehlenden Kanten aus deinen Callouts.
|
||||
chunks = _propagate_section_edges(chunks)
|
||||
# 2. WP-15b: Candidate Pool Vorbereitung
|
||||
|
||||
# A. Edge Inheritance (Sektions-Propagation)
|
||||
chunks = _propagate_section_edges(chunks, blocks)
|
||||
|
||||
# B. Explicit Edges (Direkt im Chunk-Text enthalten)
|
||||
for ch in chunks:
|
||||
explicit = _parse_edges_robust(ch.text)
|
||||
for e_str in explicit:
|
||||
kind, target = e_str.split(':', 1)
|
||||
ch.candidate_pool.append({"kind": kind, "to": target, "provenance": "explicit"})
|
||||
|
||||
# 4. Smart Edges (LLM)
|
||||
if enable_smart_edges:
|
||||
chunks = await _run_smart_edge_allocation(chunks, md_text, note_id, note_type)
|
||||
# C. Global "Unassigned Pool" Detection (Safety Net)
|
||||
# Sucht nach einer Sektion "Unzugeordnete Kanten" im Body
|
||||
unassigned_pool = set()
|
||||
pool_match = re.search(r'###?\s*(?:Unzugeordnete Kanten|Edge Pool|Candidates)\s*\n(.*?)(?:\n#|$)', body_text, re.DOTALL | re.IGNORECASE)
|
||||
if pool_match:
|
||||
unassigned_pool = _parse_edges_robust(pool_match.group(1))
|
||||
for ch in chunks:
|
||||
for e_str in unassigned_pool:
|
||||
kind, target = e_str.split(':', 1)
|
||||
ch.candidate_pool.append({"kind": kind, "to": target, "provenance": "global_pool"})
|
||||
|
||||
# 5. Linking
|
||||
# D. De-Duplikation des Pools
|
||||
for ch in chunks:
|
||||
seen = set()
|
||||
unique_pool = []
|
||||
for cand in ch.candidate_pool:
|
||||
key = (cand["kind"], cand["to"])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique_pool.append(cand)
|
||||
ch.candidate_pool = unique_pool
|
||||
|
||||
# 3. Nachbarschafts-Verkettung (Struktur-Kanten)
|
||||
for i, ch in enumerate(chunks):
|
||||
ch.neighbors_prev = chunks[i-1].id if i > 0 else None
|
||||
ch.neighbors_next = chunks[i+1].id if i < len(chunks)-1 else None
|
||||
|
||||
return chunks
|
||||
|
||||
def _extract_all_edges_from_md(md_text: str, note_id: str, note_type: str) -> List[str]:
|
||||
"""
|
||||
Hilfsfunktion: Sammelt ALLE Kanten für den LLM-Kandidaten-Pool.
|
||||
"""
|
||||
# A. Via derive_edges (Standard)
|
||||
dummy_chunk = {
|
||||
"chunk_id": f"{note_id}#full",
|
||||
"text": md_text,
|
||||
"content": md_text,
|
||||
"window": md_text,
|
||||
"type": note_type
|
||||
}
|
||||
# Signatur-Anpassung beachten (WP-15 Fix)
|
||||
raw_edges = build_edges_for_note(
|
||||
note_id,
|
||||
[dummy_chunk],
|
||||
note_level_references=None,
|
||||
include_note_scope_refs=False
|
||||
)
|
||||
all_candidates = set()
|
||||
for e in raw_edges:
|
||||
kind = e.get("kind")
|
||||
target = e.get("target_id")
|
||||
if target and kind not in ["belongs_to", "next", "prev", "backlink"]:
|
||||
all_candidates.add(f"{kind}:{target}")
|
||||
|
||||
# B. Via Robust Parser (NEU) - fängt die multiline Callouts
|
||||
robust_edges = _parse_edges_robust(md_text)
|
||||
all_candidates.update(robust_edges)
|
||||
|
||||
return list(all_candidates)
|
||||
|
||||
async def _run_smart_edge_allocation(chunks: List[Chunk], full_text: str, note_id: str, note_type: str) -> List[Chunk]:
|
||||
"""
|
||||
Der LLM-Schritt (WP-15). Filtert irrelevante Kanten.
|
||||
"""
|
||||
analyzer = get_semantic_analyzer()
|
||||
candidate_list = _extract_all_edges_from_md(full_text, note_id, note_type)
|
||||
|
||||
if not candidate_list:
|
||||
return chunks
|
||||
|
||||
tasks = []
|
||||
for chunk in chunks:
|
||||
tasks.append(analyzer.assign_edges_to_chunk(chunk.text, candidate_list, note_type))
|
||||
|
||||
results_per_chunk = await asyncio.gather(*tasks)
|
||||
|
||||
assigned_edges_global = set()
|
||||
|
||||
for i, confirmed_edges in enumerate(results_per_chunk):
|
||||
chunk = chunks[i]
|
||||
chunk.suggested_edges = confirmed_edges
|
||||
assigned_edges_global.update(confirmed_edges)
|
||||
|
||||
if confirmed_edges:
|
||||
# Wir schreiben auch Smart Edges hart in den Text
|
||||
injection_str = "\n" + " ".join([f"[[rel:{e.split(':')[0]}|{e.split(':')[1]}]]" for e in confirmed_edges if ':' in e])
|
||||
chunk.text += injection_str
|
||||
chunk.window += injection_str
|
||||
|
||||
# Fallback für Kanten, die das LLM nirgendwo zugeordnet hat
|
||||
# (Damit nichts verloren geht -> Safety Fallback)
|
||||
unassigned = set(candidate_list) - assigned_edges_global
|
||||
if unassigned:
|
||||
fallback_str = "\n" + " ".join([f"[[rel:{e.split(':')[0]}|{e.split(':')[1]}]]" for e in unassigned if ':' in e])
|
||||
for chunk in chunks:
|
||||
chunk.text += fallback_str
|
||||
chunk.window += fallback_str
|
||||
if chunk.suggested_edges is None: chunk.suggested_edges = []
|
||||
chunk.suggested_edges.extend(list(unassigned))
|
||||
|
||||
return chunks
|
||||
+100
-126
@@ -1,17 +1,20 @@
|
||||
"""
|
||||
FILE: app/core/derive_edges.py
|
||||
DESCRIPTION: Extrahiert Graph-Kanten aus Text. Unterstützt Wikilinks, Inline-Relations ([[rel:type|target]]) und Obsidian Callouts.
|
||||
VERSION: 2.0.0
|
||||
WP-15b: Integration des Candidate-Pools und Provenance-Priorisierung.
|
||||
Sichert die Graph-Integrität durch confidence-basiertes De-Duplicating.
|
||||
VERSION: 2.1.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: re, os, yaml, typing
|
||||
DEPENDENCIES: re, os, yaml, typing, hashlib
|
||||
EXTERNAL_CONFIG: config/types.yaml
|
||||
LAST_ANALYSIS: 2025-12-15
|
||||
LAST_ANALYSIS: 2025-12-26
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import hashlib
|
||||
from typing import Iterable, List, Optional, Tuple, Set, Dict
|
||||
|
||||
try:
|
||||
@@ -20,17 +23,18 @@ except Exception: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Utilities
|
||||
# 1. Utilities & ID Generation
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _get(d: dict, *keys, default=None):
|
||||
"""Sicherer Zugriff auf verschachtelte Dictionary-Keys."""
|
||||
for k in keys:
|
||||
if isinstance(d, dict) and k in d and d[k] is not None:
|
||||
return d[k]
|
||||
return default
|
||||
|
||||
def _chunk_text_for_refs(chunk: dict) -> str:
|
||||
# bevorzugt 'window' → dann 'text' → 'content' → 'raw'
|
||||
"""Extrahiert den relevanten Text für die Referenzsuche (bevorzugt Window)."""
|
||||
return (
|
||||
_get(chunk, "window")
|
||||
or _get(chunk, "text")
|
||||
@@ -40,6 +44,7 @@ def _chunk_text_for_refs(chunk: dict) -> str:
|
||||
)
|
||||
|
||||
def _dedupe_seq(seq: Iterable[str]) -> List[str]:
|
||||
"""Dedupliziert eine Sequenz von Strings unter Beibehaltung der Reihenfolge."""
|
||||
seen: Set[str] = set()
|
||||
out: List[str] = []
|
||||
for s in seq:
|
||||
@@ -49,9 +54,10 @@ def _dedupe_seq(seq: Iterable[str]) -> List[str]:
|
||||
return out
|
||||
|
||||
def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, extra: Optional[dict] = None) -> dict:
|
||||
"""Konstruiert ein valides Kanten-Payload-Objekt für Qdrant."""
|
||||
pl = {
|
||||
"kind": kind,
|
||||
"relation": kind, # Alias (v2)
|
||||
"relation": kind, # Alias für Abwärtskompatibilität (v2)
|
||||
"scope": scope, # "chunk" | "note"
|
||||
"source_id": source_id,
|
||||
"target_id": target_id,
|
||||
@@ -62,25 +68,38 @@ def _edge(kind: str, scope: str, source_id: str, target_id: str, note_id: str, e
|
||||
return pl
|
||||
|
||||
def _mk_edge_id(kind: str, s: str, t: str, scope: str, rule_id: Optional[str] = None) -> str:
|
||||
"""Erzeugt eine deterministische 12-Byte ID mittels BLAKE2s."""
|
||||
base = f"{kind}:{s}->{t}#{scope}"
|
||||
if rule_id:
|
||||
base += f"|{rule_id}"
|
||||
try:
|
||||
import hashlib
|
||||
return hashlib.blake2s(base.encode("utf-8"), digest_size=12).hexdigest()
|
||||
except Exception: # pragma: no cover
|
||||
return base
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Typen-Registry (types.yaml)
|
||||
# 2. Konfiguration & Provenance-Skala
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
# WP-15b: Prioritäten-Ranking für die De-Duplizierung
|
||||
PROVENANCE_PRIORITY = {
|
||||
"explicit:wikilink": 1.00,
|
||||
"inline:rel": 0.95,
|
||||
"callout:edge": 0.90,
|
||||
"semantic_ai": 0.90, # Validierte KI-Kanten
|
||||
"structure:belongs_to": 1.00,
|
||||
"structure:order": 0.95, # next/prev
|
||||
"explicit:note_scope": 1.00,
|
||||
"derived:backlink": 0.90,
|
||||
"edge_defaults": 0.70 # Heuristik (types.yaml)
|
||||
}
|
||||
|
||||
def _env(n: str, default: Optional[str] = None) -> str:
|
||||
v = os.getenv(n)
|
||||
return v if v is not None else (default or "")
|
||||
|
||||
def _load_types_registry() -> dict:
|
||||
"""Lädt die YAML-Registry aus MINDNET_TYPES_FILE oder ./config/types.yaml"""
|
||||
"""Lädt die YAML-Registry zur Ermittlung von Standard-Kanten."""
|
||||
p = _env("MINDNET_TYPES_FILE", "./config/types.yaml")
|
||||
if not os.path.isfile(p) or yaml is None:
|
||||
return {}
|
||||
@@ -97,13 +116,7 @@ def _get_types_map(reg: dict) -> dict:
|
||||
return reg if isinstance(reg, dict) else {}
|
||||
|
||||
def _edge_defaults_for(note_type: Optional[str], reg: dict) -> List[str]:
|
||||
"""
|
||||
Liefert die edge_defaults-Liste für den gegebenen Notiztyp.
|
||||
Fallback-Reihenfolge:
|
||||
1) reg['types'][note_type]['edge_defaults']
|
||||
2) reg['defaults']['edge_defaults'] (oder 'default'/'global')
|
||||
3) []
|
||||
"""
|
||||
"""Liefert die edge_defaults-Liste für den gegebenen Notiztyp."""
|
||||
types_map = _get_types_map(reg)
|
||||
if note_type and isinstance(types_map, dict):
|
||||
t = types_map.get(note_type)
|
||||
@@ -116,29 +129,19 @@ def _edge_defaults_for(note_type: Optional[str], reg: dict) -> List[str]:
|
||||
return []
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Parser für Links / Relationen
|
||||
# 3. Parser für Links / Relationen (Core Logik v2.0.0)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
# Normale Wikilinks (Fallback)
|
||||
_WIKILINK_RE = re.compile(r"\[\[(?:[^\|\]]+\|)?([a-zA-Z0-9_\-#:. ]+)\]\]")
|
||||
|
||||
# Getypte Inline-Relationen:
|
||||
# [[rel:KIND | Target]]
|
||||
# [[rel:KIND Target]]
|
||||
# Getypte Inline-Relationen
|
||||
_REL_PIPE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s*\|\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||
_REL_SPACE = re.compile(r"\[\[\s*rel:(?P<kind>[a-z_]+)\s+(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||
# rel: KIND [[Target]] (reines Textmuster)
|
||||
_REL_TEXT = re.compile(r"rel\s*:\s*(?P<kind>[a-z_]+)\s*\[\[\s*(?P<target>[^\]]+?)\s*\]\]", re.IGNORECASE)
|
||||
|
||||
def _extract_typed_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||
"""
|
||||
Gibt Liste (kind, target) zurück und den Text mit entfernten getypten Relation-Links,
|
||||
damit die generische Wikilink-Erkennung sie nicht doppelt zählt.
|
||||
Unterstützt drei Varianten:
|
||||
- [[rel:KIND | Target]]
|
||||
- [[rel:KIND Target]]
|
||||
- rel: KIND [[Target]]
|
||||
"""
|
||||
"""Extrahiert [[rel:KIND|Target]] und entfernt sie zur Vermeidung von Dubletten."""
|
||||
pairs: List[Tuple[str,str]] = []
|
||||
def _collect(m):
|
||||
k = (m.group("kind") or "").strip().lower()
|
||||
@@ -152,17 +155,13 @@ def _extract_typed_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||
text = _REL_TEXT.sub(_collect, text)
|
||||
return pairs, text
|
||||
|
||||
# Obsidian Callout Parser
|
||||
# Obsidian Callout Parser für mehrzeilige Blöcke
|
||||
_CALLOUT_START = re.compile(r"^\s*>\s*\[!edge\]\s*(.*)$", re.IGNORECASE)
|
||||
_REL_LINE = re.compile(r"^(?P<kind>[a-z_]+)\s*:\s*(?P<targets>.+?)\s*$", re.IGNORECASE)
|
||||
_WIKILINKS_IN_LINE = re.compile(r"\[\[([^\]]+)\]\]")
|
||||
|
||||
def _extract_callout_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||
"""
|
||||
Findet [!edge]-Callouts und extrahiert (kind, target). Entfernt den gesamten
|
||||
Callout-Block aus dem Text (damit Wikilinks daraus nicht zusätzlich als
|
||||
"references" gezählt werden).
|
||||
"""
|
||||
"""Verarbeitet [!edge]-Callouts und entfernt diese aus dem Textfluss."""
|
||||
if not text:
|
||||
return [], text
|
||||
|
||||
@@ -205,21 +204,20 @@ def _extract_callout_relations(text: str) -> Tuple[List[Tuple[str,str]], str]:
|
||||
t = raw.strip()
|
||||
if t:
|
||||
out_pairs.append((kind, t))
|
||||
|
||||
# Callout wird NICHT in keep_lines übernommen
|
||||
continue
|
||||
|
||||
remainder = "\n".join(keep_lines)
|
||||
return out_pairs, remainder
|
||||
|
||||
def _extract_wikilinks(text: str) -> List[str]:
|
||||
"""Extrahiert Standard-Wikilinks aus dem verbleibenden Text."""
|
||||
ids: List[str] = []
|
||||
for m in _WIKILINK_RE.finditer(text or ""):
|
||||
ids.append(m.group(1).strip())
|
||||
return ids
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Hauptfunktion
|
||||
# 4. Hauptfunktion (build_edges_for_note)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def build_edges_for_note(
|
||||
@@ -229,24 +227,13 @@ def build_edges_for_note(
|
||||
include_note_scope_refs: bool = False,
|
||||
) -> List[dict]:
|
||||
"""
|
||||
Erzeugt Kanten für eine Note.
|
||||
|
||||
- belongs_to: für jeden Chunk (chunk -> note)
|
||||
- next / prev: zwischen aufeinanderfolgenden Chunks
|
||||
- references: pro Chunk aus window/text (via Wikilinks)
|
||||
- typed inline relations: [[rel:KIND | Target]] / [[rel:KIND Target]] / rel: KIND [[Target]]
|
||||
- Obsidian Callouts: > [!edge] KIND: [[Target]] [[Target2]]
|
||||
- optional note-scope references/backlinks: dedupliziert über alle Chunk-Funde + note_level_references
|
||||
- typenbasierte Default-Kanten (edge_defaults) je gefundener Referenz
|
||||
Erzeugt und aggregiert alle Kanten für eine Note inklusive WP-15b Candidate-Processing.
|
||||
Setzt Provenance-Ranking zur Graph-Stabilisierung ein.
|
||||
"""
|
||||
edges: List[dict] = []
|
||||
note_type = _get(chunks[0], "type") if chunks else "concept"
|
||||
|
||||
# Note-Typ (aus erstem Chunk erwartet)
|
||||
note_type = None
|
||||
if chunks:
|
||||
note_type = _get(chunks[0], "type")
|
||||
|
||||
# 1) belongs_to
|
||||
# 1) Struktur-Kanten: belongs_to (Chunk -> Note)
|
||||
for ch in chunks:
|
||||
cid = _get(ch, "chunk_id", "id")
|
||||
if not cid:
|
||||
@@ -254,12 +241,12 @@ def build_edges_for_note(
|
||||
edges.append(_edge("belongs_to", "chunk", cid, note_id, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id("belongs_to", cid, note_id, "chunk", "structure:belongs_to"),
|
||||
"provenance": "rule",
|
||||
"provenance": "structure",
|
||||
"rule_id": "structure:belongs_to",
|
||||
"confidence": 1.0,
|
||||
"confidence": PROVENANCE_PRIORITY["structure:belongs_to"],
|
||||
}))
|
||||
|
||||
# 2) next / prev
|
||||
# 2) Struktur-Kanten: next / prev (Sequenz)
|
||||
for i in range(len(chunks) - 1):
|
||||
a, b = chunks[i], chunks[i + 1]
|
||||
a_id = _get(a, "chunk_id", "id")
|
||||
@@ -269,19 +256,19 @@ def build_edges_for_note(
|
||||
edges.append(_edge("next", "chunk", a_id, b_id, note_id, {
|
||||
"chunk_id": a_id,
|
||||
"edge_id": _mk_edge_id("next", a_id, b_id, "chunk", "structure:order"),
|
||||
"provenance": "rule",
|
||||
"provenance": "structure",
|
||||
"rule_id": "structure:order",
|
||||
"confidence": 0.95,
|
||||
"confidence": PROVENANCE_PRIORITY["structure:order"],
|
||||
}))
|
||||
edges.append(_edge("prev", "chunk", b_id, a_id, note_id, {
|
||||
"chunk_id": b_id,
|
||||
"edge_id": _mk_edge_id("prev", b_id, a_id, "chunk", "structure:order"),
|
||||
"provenance": "rule",
|
||||
"provenance": "structure",
|
||||
"rule_id": "structure:order",
|
||||
"confidence": 0.95,
|
||||
"confidence": PROVENANCE_PRIORITY["structure:order"],
|
||||
}))
|
||||
|
||||
# 3) references + typed inline + callouts + defaults (chunk-scope)
|
||||
# 3) Inhaltliche Kanten (Refs, Inlines, Callouts, Candidates)
|
||||
reg = _load_types_registry()
|
||||
defaults = _edge_defaults_for(note_type, reg)
|
||||
refs_all: List[str] = []
|
||||
@@ -292,51 +279,49 @@ def build_edges_for_note(
|
||||
continue
|
||||
raw = _chunk_text_for_refs(ch)
|
||||
|
||||
# 3a) typed inline relations
|
||||
# 3a) Typed Inline Relations
|
||||
typed, remainder = _extract_typed_relations(raw)
|
||||
for kind, target in typed:
|
||||
kind = kind.strip().lower()
|
||||
if not kind or not target:
|
||||
continue
|
||||
edges.append(_edge(kind, "chunk", cid, target, note_id, {
|
||||
k = kind.strip().lower()
|
||||
if not k or not target: continue
|
||||
edges.append(_edge(k, "chunk", cid, target, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(kind, cid, target, "chunk", "inline:rel"),
|
||||
"edge_id": _mk_edge_id(k, cid, target, "chunk", "inline:rel"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "inline:rel",
|
||||
"confidence": 0.95,
|
||||
"confidence": PROVENANCE_PRIORITY["inline:rel"],
|
||||
}))
|
||||
if kind in {"related_to", "similar_to"}:
|
||||
edges.append(_edge(kind, "chunk", target, cid, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(kind, target, cid, "chunk", "inline:rel"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "inline:rel",
|
||||
"confidence": 0.95,
|
||||
}))
|
||||
|
||||
# 3b) callouts
|
||||
# 3b) WP-15b Candidate Pool Integration (KI-validierte Kanten)
|
||||
# Verarbeitet Kanten, die bereits in der Ingestion semantisch geprüft wurden.
|
||||
pool = ch.get("candidate_pool") or ch.get("candidate_edges") or []
|
||||
for cand in pool:
|
||||
target = cand.get("to")
|
||||
kind = cand.get("kind", "related_to")
|
||||
prov = cand.get("provenance", "semantic_ai")
|
||||
if not target: continue
|
||||
edges.append(_edge(kind, "chunk", cid, target, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(kind, cid, target, "chunk", f"candidate:{prov}"),
|
||||
"provenance": prov,
|
||||
"rule_id": f"candidate:{prov}",
|
||||
"confidence": PROVENANCE_PRIORITY.get(prov, 0.90),
|
||||
}))
|
||||
|
||||
# 3c) Obsidian Callouts
|
||||
call_pairs, remainder2 = _extract_callout_relations(remainder)
|
||||
for kind, target in call_pairs:
|
||||
k = (kind or "").strip().lower()
|
||||
if not k or not target:
|
||||
continue
|
||||
if not k or not target: continue
|
||||
edges.append(_edge(k, "chunk", cid, target, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(k, cid, target, "chunk", "callout:edge"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "callout:edge",
|
||||
"confidence": 0.95,
|
||||
"confidence": PROVENANCE_PRIORITY["callout:edge"],
|
||||
}))
|
||||
if k in {"related_to", "similar_to"}:
|
||||
edges.append(_edge(k, "chunk", target, cid, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(k, target, cid, "chunk", "callout:edge"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "callout:edge",
|
||||
"confidence": 0.95,
|
||||
}))
|
||||
|
||||
# 3c) generische Wikilinks → references (+ defaults je Ref)
|
||||
# 3d) Standard-Wikilinks -> references (+ defaults)
|
||||
refs = _extract_wikilinks(remainder2)
|
||||
for r in refs:
|
||||
edges.append(_edge("references", "chunk", cid, r, note_id, {
|
||||
@@ -345,76 +330,65 @@ def build_edges_for_note(
|
||||
"edge_id": _mk_edge_id("references", cid, r, "chunk", "explicit:wikilink"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "explicit:wikilink",
|
||||
"confidence": 1.0,
|
||||
"confidence": PROVENANCE_PRIORITY["explicit:wikilink"],
|
||||
}))
|
||||
# Regelbasierte Kanten aus types.yaml anhängen
|
||||
for rel in defaults:
|
||||
if rel == "references":
|
||||
continue
|
||||
if rel == "references": continue
|
||||
edges.append(_edge(rel, "chunk", cid, r, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(rel, cid, r, "chunk", f"edge_defaults:{note_type}:{rel}"),
|
||||
"provenance": "rule",
|
||||
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||
"confidence": 0.7,
|
||||
"confidence": PROVENANCE_PRIORITY["edge_defaults"],
|
||||
}))
|
||||
if rel in {"related_to", "similar_to"}:
|
||||
edges.append(_edge(rel, "chunk", r, cid, note_id, {
|
||||
"chunk_id": cid,
|
||||
"edge_id": _mk_edge_id(rel, r, cid, "chunk", f"edge_defaults:{note_type}:{rel}"),
|
||||
"provenance": "rule",
|
||||
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||
"confidence": 0.7,
|
||||
}))
|
||||
|
||||
refs_all.extend(refs)
|
||||
|
||||
# 4) optional note-scope refs/backlinks (+ defaults)
|
||||
# 4) Optionale Note-Scope Referenzen & Backlinks
|
||||
if include_note_scope_refs:
|
||||
refs_note = list(refs_all or [])
|
||||
if note_level_references:
|
||||
refs_note.extend([r for r in note_level_references if isinstance(r, str) and r])
|
||||
refs_note = _dedupe_seq(refs_note)
|
||||
|
||||
for r in refs_note:
|
||||
edges.append(_edge("references", "note", note_id, r, note_id, {
|
||||
"edge_id": _mk_edge_id("references", note_id, r, "note", "explicit:note_scope"),
|
||||
"provenance": "explicit",
|
||||
"rule_id": "explicit:note_scope",
|
||||
"confidence": 1.0,
|
||||
"confidence": PROVENANCE_PRIORITY["explicit:note_scope"],
|
||||
}))
|
||||
# Backlink-Erzeugung zur Graphen-Stärkung
|
||||
edges.append(_edge("backlink", "note", r, note_id, note_id, {
|
||||
"edge_id": _mk_edge_id("backlink", r, note_id, "note", "derived:backlink"),
|
||||
"provenance": "rule",
|
||||
"rule_id": "derived:backlink",
|
||||
"confidence": 0.9,
|
||||
"confidence": PROVENANCE_PRIORITY["derived:backlink"],
|
||||
}))
|
||||
for rel in defaults:
|
||||
if rel == "references":
|
||||
continue
|
||||
if rel == "references": continue
|
||||
edges.append(_edge(rel, "note", note_id, r, note_id, {
|
||||
"edge_id": _mk_edge_id(rel, note_id, r, "note", f"edge_defaults:{note_type}:{rel}"),
|
||||
"provenance": "rule",
|
||||
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||
"confidence": 0.7,
|
||||
"confidence": PROVENANCE_PRIORITY["edge_defaults"],
|
||||
}))
|
||||
if rel in {"related_to", "similar_to"}:
|
||||
edges.append(_edge(rel, "note", r, note_id, note_id, {
|
||||
"edge_id": _mk_edge_id(rel, r, note_id, "note", f"edge_defaults:{note_type}:{rel}"),
|
||||
"provenance": "rule",
|
||||
"rule_id": f"edge_defaults:{note_type}:{rel}",
|
||||
"confidence": 0.7,
|
||||
}))
|
||||
|
||||
# 5) De-Dupe (source_id, target_id, relation, rule_id)
|
||||
seen: Set[Tuple[str,str,str,str]] = set()
|
||||
out: List[dict] = []
|
||||
# 5) WP-15b: Confidence-basierte De-Duplizierung
|
||||
# Wenn dieselbe Relation mehrfach existiert, gewinnt die mit der höchsten Confidence.
|
||||
unique_map: Dict[Tuple[str, str, str], dict] = {}
|
||||
|
||||
for e in edges:
|
||||
s = str(e.get("source_id") or "")
|
||||
t = str(e.get("target_id") or "")
|
||||
s, t = str(e.get("source_id")), str(e.get("target_id"))
|
||||
rel = str(e.get("relation") or e.get("kind") or "edge")
|
||||
rule = str(e.get("rule_id") or "")
|
||||
key = (s, t, rel, rule)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(e)
|
||||
return out
|
||||
key = (s, t, rel)
|
||||
|
||||
if key not in unique_map:
|
||||
unique_map[key] = e
|
||||
else:
|
||||
# Vergleich der Vertrauenswürdigkeit (Provenance Ranking)
|
||||
if e.get("confidence", 0) > unique_map[key].get("confidence", 0):
|
||||
unique_map[key] = e
|
||||
|
||||
return list(unique_map.values())
|
||||
+125
-142
@@ -3,12 +3,14 @@ FILE: app/core/ingestion.py
|
||||
DESCRIPTION: Haupt-Ingestion-Logik. Transformiert Markdown in den Graphen.
|
||||
WP-20: Optimiert für OpenRouter (mistralai/mistral-7b-instruct:free).
|
||||
WP-22: Content Lifecycle, Edge Registry Validation & Multi-Hash.
|
||||
FIX: Deep Fallback Logic (v2.11.14). Erkennt Policy Violations auch in validen
|
||||
JSON-Objekten und erzwingt den lokalen Ollama-Sprung, um Kantenverlust
|
||||
bei umfangreichen Protokollen zu verhindern.
|
||||
VERSION: 2.11.14
|
||||
WP-15b: Two-Pass Ingestion mit LocalBatchCache & Candidate-Validation.
|
||||
Sichert, dass explizite Kanten direkt übernommen und nur Pool-Kanten validiert werden.
|
||||
FIX: Deep Fallback Logic (v2.11.14) für JSON-Recovery.
|
||||
Robust Lookup Fix: Adressiert Notizen im Cache via ID, Titel und Dateiname.
|
||||
VERSION: 2.12.2
|
||||
STATUS: Active
|
||||
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker, app.services.llm_service, app.services.edge_registry
|
||||
DEPENDENCIES: app.core.parser, app.core.note_payload, app.core.chunker,
|
||||
app.services.llm_service, app.services.edge_registry
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
@@ -21,9 +23,11 @@ from typing import Dict, List, Optional, Tuple, Any
|
||||
# Core Module Imports
|
||||
from app.core.parser import (
|
||||
read_markdown,
|
||||
pre_scan_markdown,
|
||||
normalize_frontmatter,
|
||||
validate_required_frontmatter,
|
||||
extract_edges_with_context,
|
||||
NoteContext
|
||||
)
|
||||
from app.core.note_payload import make_note_payload
|
||||
from app.core.chunker import assemble_chunks, get_chunk_config
|
||||
@@ -49,7 +53,7 @@ from app.services.llm_service import LLMService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Global Helpers ---
|
||||
# --- Global Helpers (Full Compatibility v2.11.14) ---
|
||||
def extract_json_from_response(text: str) -> Any:
|
||||
"""
|
||||
Extrahiert JSON-Daten und bereinigt LLM-Steuerzeichen (Mistral/Llama).
|
||||
@@ -115,6 +119,7 @@ class IngestionService:
|
||||
self.llm = LLMService()
|
||||
|
||||
self.active_hash_mode = self.settings.CHANGE_DETECTION_MODE
|
||||
self.batch_cache: Dict[str, NoteContext] = {} # WP-15b LocalBatchCache
|
||||
|
||||
try:
|
||||
ensure_collections(self.client, self.prefix, self.dim)
|
||||
@@ -122,6 +127,74 @@ class IngestionService:
|
||||
except Exception as e:
|
||||
logger.warning(f"DB init warning: {e}")
|
||||
|
||||
async def run_batch(self, file_paths: List[str], vault_root: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
WP-15b: Implementiert den Two-Pass Ingestion Workflow.
|
||||
Pass 1: Pre-Scan baut flüchtigen Kontext-Cache auf.
|
||||
Pass 2: Processing führt die eigentliche semantische Validierung durch.
|
||||
"""
|
||||
logger.info(f"🔍 [Pass 1] Pre-Scanning {len(file_paths)} files for Context Cache...")
|
||||
for path in file_paths:
|
||||
ctx = pre_scan_markdown(path)
|
||||
if ctx:
|
||||
# Mehrfache Indizierung für robusten Look-up (WP-15b Fix)
|
||||
self.batch_cache[ctx.note_id] = ctx
|
||||
self.batch_cache[ctx.title] = ctx
|
||||
# Dateiname ohne Endung als dritter Schlüssel
|
||||
fname = os.path.splitext(os.path.basename(path))[0]
|
||||
self.batch_cache[fname] = ctx
|
||||
|
||||
logger.info(f"🚀 [Pass 2] Semantic Processing of {len(file_paths)} files...")
|
||||
results = []
|
||||
for path in file_paths:
|
||||
res = await self.process_file(path, vault_root, apply=True)
|
||||
results.append(res)
|
||||
return results
|
||||
|
||||
async def _validate_candidate(self, chunk_text: str, edge: Dict) -> bool:
|
||||
"""
|
||||
WP-15b: Validiert einen Kanten-Kandidaten semantisch gegen das Ziel.
|
||||
Nutzt den Cache aus Pass 1, um dem LLM Kontext der Ziel-Note zu geben.
|
||||
"""
|
||||
target_id = edge.get("to")
|
||||
target_ctx = self.batch_cache.get(target_id)
|
||||
|
||||
# Fallback Look-up für Links mit Ankern (Anchor entfernen)
|
||||
if not target_ctx and "#" in target_id:
|
||||
base_id = target_id.split("#")[0]
|
||||
target_ctx = self.batch_cache.get(base_id)
|
||||
|
||||
# Sicherheits-Fallback: Wenn Zielnotiz nicht im aktuellen Batch ist,
|
||||
# lassen wir die Kante als 'explicit' durch (Hard-Link Integrity).
|
||||
if not target_ctx:
|
||||
logger.info(f"ℹ️ [VALIDATION SKIP] No context for '{target_id}' - allowing link.")
|
||||
return True
|
||||
|
||||
provider = self.settings.MINDNET_LLM_PROVIDER
|
||||
template = self.llm.get_prompt("edge_validation", provider)
|
||||
|
||||
try:
|
||||
logger.info(f"⚖️ [VALIDATING] Relation '{edge.get('kind')}' -> '{target_id}'...")
|
||||
prompt = template.format(
|
||||
chunk_text=chunk_text[:1500],
|
||||
target_title=target_ctx.title,
|
||||
target_summary=target_ctx.summary,
|
||||
edge_kind=edge.get("kind", "related_to")
|
||||
)
|
||||
|
||||
response = await self.llm.generate_raw_response(prompt, priority="background")
|
||||
is_valid = "YES" in response.upper()
|
||||
|
||||
if is_valid:
|
||||
logger.info(f"✅ [VALIDATED] Relation to '{target_id}' confirmed.")
|
||||
else:
|
||||
logger.info(f"🚫 [REJECTED] Relation to '{target_id}' irrelevant for this chunk.")
|
||||
|
||||
return is_valid
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Semantic validation error for {target_id}: {e}")
|
||||
return True # Fallback: Im Zweifel Link behalten
|
||||
|
||||
def _resolve_note_type(self, requested: Optional[str]) -> str:
|
||||
"""Bestimmt den finalen Notiz-Typ (Fallback auf 'concept')."""
|
||||
types = self.registry.get("types", {})
|
||||
@@ -138,109 +211,12 @@ class IngestionService:
|
||||
return cfg
|
||||
return get_chunk_config(note_type)
|
||||
|
||||
async def _perform_smart_edge_allocation(self, text: str, note_id: str) -> List[Dict]:
|
||||
"""
|
||||
KI-Extraktion mit Deep-Fallback Logik.
|
||||
Erzwingt den lokalen Ollama-Sprung, wenn die Cloud-Antwort keine verwertbaren
|
||||
Kanten liefert (häufig bei Policy Violations auf OpenRouter).
|
||||
"""
|
||||
provider = self.settings.MINDNET_LLM_PROVIDER
|
||||
model = self.settings.OPENROUTER_MODEL if provider == "openrouter" else self.settings.GEMINI_MODEL
|
||||
|
||||
logger.info(f"🚀 [Ingestion] Turbo-Mode: Extracting edges for '{note_id}' using {model} on {provider}")
|
||||
|
||||
edge_registry.ensure_latest()
|
||||
valid_types_str = ", ".join(sorted(list(edge_registry.valid_types)))
|
||||
|
||||
template = self.llm.get_prompt("edge_extraction", provider)
|
||||
|
||||
try:
|
||||
try:
|
||||
# Wir begrenzen den Kontext auf 6000 Zeichen (ca. 1500 Token)
|
||||
prompt = template.format(
|
||||
text=text[:6000],
|
||||
note_id=note_id,
|
||||
valid_types=valid_types_str
|
||||
)
|
||||
except KeyError as ke:
|
||||
logger.error(f"❌ [Ingestion] Prompt-Template Fehler (Variable {ke} fehlt).")
|
||||
return []
|
||||
|
||||
# 1. Versuch: Anfrage an den primären Cloud-Provider
|
||||
response_json = await self.llm.generate_raw_response(
|
||||
prompt=prompt, priority="background", force_json=True,
|
||||
provider=provider, model_override=model
|
||||
)
|
||||
|
||||
# Initiales Parsing
|
||||
raw_data = extract_json_from_response(response_json)
|
||||
|
||||
# 2. Dictionary Recovery (Versuche Liste aus Dict zu extrahieren)
|
||||
candidates = []
|
||||
if isinstance(raw_data, list):
|
||||
candidates = raw_data
|
||||
elif isinstance(raw_data, dict):
|
||||
logger.info(f"ℹ️ [Ingestion] LLM returned dict, checking for embedded lists in {note_id}")
|
||||
for k in ["edges", "links", "results", "kanten", "matches", "edge_list"]:
|
||||
if k in raw_data and isinstance(raw_data[k], list):
|
||||
candidates = raw_data[k]
|
||||
break
|
||||
# Wenn immer noch keine Liste gefunden, versuche Key-Value Paare (Dict Recovery)
|
||||
if not candidates:
|
||||
for k, v in raw_data.items():
|
||||
if isinstance(v, str): candidates.append(f"{k}:{v}")
|
||||
elif isinstance(v, list): [candidates.append(f"{k}:{i}") for i in v if isinstance(i, str)]
|
||||
|
||||
# 3. DEEP FALLBACK: Wenn nach allen Recovery-Versuchen die Liste leer ist UND wir in der Cloud waren
|
||||
# Triggert den Fallback bei "Data Policy Violations" (leere oder Fehler-JSONs).
|
||||
if not candidates and provider != "ollama" and self.settings.LLM_FALLBACK_ENABLED:
|
||||
logger.warning(
|
||||
f"🛑 [Ingestion] Cloud-Antwort für {note_id} lieferte keine verwertbaren Kanten. "
|
||||
f"Mögliche Policy Violation oder Refusal. Erzwinge LOKALEN FALLBACK via Ollama..."
|
||||
)
|
||||
response_json_local = await self.llm.generate_raw_response(
|
||||
prompt=prompt, priority="background", force_json=True, provider="ollama"
|
||||
)
|
||||
raw_data_local = extract_json_from_response(response_json_local)
|
||||
|
||||
# Wiederhole Recovery für lokale Antwort
|
||||
if isinstance(raw_data_local, list):
|
||||
candidates = raw_data_local
|
||||
elif isinstance(raw_data_local, dict):
|
||||
for k in ["edges", "links", "results"]:
|
||||
if k in raw_data_local and isinstance(raw_data_local[k], list):
|
||||
candidates = raw_data_local[k]; break
|
||||
|
||||
if not candidates:
|
||||
logger.warning(f"⚠️ [Ingestion] Auch nach Fallback keine extrahierbaren Kanten für {note_id}")
|
||||
return []
|
||||
|
||||
processed = []
|
||||
for item in candidates:
|
||||
if isinstance(item, dict) and "to" in item:
|
||||
item["provenance"] = "semantic_ai"
|
||||
item["line"] = f"ai-{provider}"
|
||||
processed.append(item)
|
||||
elif isinstance(item, str) and ":" in item:
|
||||
parts = item.split(":", 1)
|
||||
processed.append({
|
||||
"to": parts[1].strip(),
|
||||
"kind": parts[0].strip(),
|
||||
"provenance": "semantic_ai",
|
||||
"line": f"ai-{provider}"
|
||||
})
|
||||
return processed
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ [Ingestion] Smart Edge Allocation failed for {note_id}: {e}")
|
||||
return []
|
||||
|
||||
async def process_file(
|
||||
self, file_path: str, vault_root: str,
|
||||
force_replace: bool = False, apply: bool = False, purge_before: bool = False,
|
||||
note_scope_refs: bool = False, hash_source: str = "parsed", hash_normalize: str = "canonical"
|
||||
) -> Dict[str, Any]:
|
||||
"""Transformiert eine Markdown-Datei in den Graphen (Notes, Chunks, Edges)."""
|
||||
"""Transformiert eine Markdown-Datei in den Graphen."""
|
||||
result = {"path": file_path, "status": "skipped", "changed": False, "error": None}
|
||||
|
||||
# 1. Parse & Lifecycle Gate
|
||||
@@ -252,12 +228,12 @@ class IngestionService:
|
||||
except Exception as e:
|
||||
return {**result, "error": f"Validation failed: {str(e)}"}
|
||||
|
||||
# WP-22: Filter für Systemdateien und Entwürfe
|
||||
# Lifecycle Filter (WP-22)
|
||||
status = fm.get("status", "draft").lower().strip()
|
||||
if status in ["system", "template", "archive", "hidden"]:
|
||||
return {**result, "status": "skipped", "reason": f"lifecycle_{status}"}
|
||||
|
||||
# 2. Config Resolution & Payload Construction
|
||||
# 2. Config Resolution & Payload
|
||||
note_type = self._resolve_note_type(fm.get("type"))
|
||||
fm["type"] = note_type
|
||||
|
||||
@@ -267,15 +243,13 @@ class IngestionService:
|
||||
except Exception as e:
|
||||
return {**result, "error": f"Payload failed: {str(e)}"}
|
||||
|
||||
# 3. Change Detection (Strikte DoD Umsetzung)
|
||||
# 3. Change Detection (v2.11.14 Logic)
|
||||
old_payload = None if force_replace else self._fetch_note_payload(note_id)
|
||||
check_key = f"{self.active_hash_mode}:{hash_source}:{hash_normalize}"
|
||||
old_hash = (old_payload or {}).get("hashes", {}).get(check_key)
|
||||
new_hash = note_pl.get("hashes", {}).get(check_key)
|
||||
|
||||
# Prüfung auf fehlende Artefakte in Qdrant
|
||||
chunks_missing, edges_missing = self._artifacts_missing(note_id)
|
||||
|
||||
should_write = force_replace or (not old_payload) or (old_hash != new_hash) or chunks_missing or edges_missing
|
||||
|
||||
if not should_write:
|
||||
@@ -284,50 +258,57 @@ class IngestionService:
|
||||
if not apply:
|
||||
return {**result, "status": "dry-run", "changed": True, "note_id": note_id}
|
||||
|
||||
# 4. Processing (Chunking, Embedding, AI Edges)
|
||||
# 4. Processing (Chunking, Embedding, Validated Edges)
|
||||
try:
|
||||
body_text = getattr(parsed, "body", "") or ""
|
||||
edge_registry.ensure_latest()
|
||||
|
||||
# Profil-gesteuertes Chunking
|
||||
# Chunker Resolution
|
||||
profile = fm.get("chunk_profile") or fm.get("chunking_profile") or "sliding_standard"
|
||||
chunk_cfg = self._get_chunk_config_by_profile(profile, note_type)
|
||||
enable_smart_edges = chunk_cfg.get("enable_smart_edge_allocation", False)
|
||||
|
||||
# WP-15b: Chunker bereitet nun den Candidate-Pool vor (inkl. Inheritance).
|
||||
chunks = await assemble_chunks(fm["id"], body_text, fm["type"], config=chunk_cfg)
|
||||
|
||||
# WP-15b: Validierung NUR für Kandidaten aus dem global_pool (Unzugeordnete Kanten)
|
||||
for ch_obj in chunks:
|
||||
filtered_pool = []
|
||||
for cand in getattr(ch_obj, "candidate_pool", []):
|
||||
# Nur 'global_pool' erfordert LLM-Validierung.
|
||||
# 'explicit' und 'inherited' werden direkt akzeptiert.
|
||||
if cand.get("provenance") == "global_pool" and enable_smart_edges:
|
||||
if await self._validate_candidate(ch_obj.text, cand):
|
||||
filtered_pool.append(cand)
|
||||
else:
|
||||
filtered_pool.append(cand)
|
||||
ch_obj.candidate_pool = filtered_pool
|
||||
|
||||
chunk_pls = make_chunk_payloads(fm, note_pl["path"], chunks, note_text=body_text)
|
||||
|
||||
# Vektorisierung
|
||||
# Embeddings generieren
|
||||
vecs = []
|
||||
if chunk_pls:
|
||||
texts = [c.get("window") or c.get("text") or "" for c in chunk_pls]
|
||||
vecs = await self.embedder.embed_documents(texts)
|
||||
|
||||
# Kanten-Extraktion
|
||||
edges = []
|
||||
context = {"file": file_path, "note_id": note_id}
|
||||
|
||||
# A. Explizite Kanten (User / Wikilinks)
|
||||
for e in extract_edges_with_context(parsed):
|
||||
e["kind"] = edge_registry.resolve(edge_type=e["kind"], provenance="explicit", context={**context, "line": e.get("line")})
|
||||
edges.append(e)
|
||||
|
||||
# B. KI Kanten (Turbo Mode mit v2.11.14 Fallback)
|
||||
ai_edges = await self._perform_smart_edge_allocation(body_text, note_id)
|
||||
for e in ai_edges:
|
||||
valid_kind = edge_registry.resolve(edge_type=e.get("kind"), provenance="semantic_ai", context={**context, "line": e.get("line")})
|
||||
e["kind"] = valid_kind
|
||||
edges.append(e)
|
||||
|
||||
# C. System Kanten (Struktur)
|
||||
try:
|
||||
sys_edges = build_edges_for_note(note_id, chunk_pls, note_level_references=note_pl.get("references", []), include_note_scope_refs=note_scope_refs)
|
||||
except:
|
||||
sys_edges = build_edges_for_note(note_id, chunk_pls)
|
||||
# Kanten finalisieren via derive_edges Aggregator (WP-15b kompatibel)
|
||||
# Nutzt das Provenance-Ranking (v2.1.0).
|
||||
edges = build_edges_for_note(
|
||||
note_id,
|
||||
chunk_pls,
|
||||
note_level_references=note_pl.get("references", []),
|
||||
include_note_scope_refs=note_scope_refs
|
||||
)
|
||||
|
||||
for e in sys_edges:
|
||||
valid_kind = edge_registry.resolve(edge_type=e.get("kind", "belongs_to"), provenance="structure", context={**context, "line": "system"})
|
||||
if valid_kind:
|
||||
e["kind"] = valid_kind
|
||||
edges.append(e)
|
||||
# Alias-Auflösung & Registry Enforcement
|
||||
context = {"file": file_path, "note_id": note_id}
|
||||
for e in edges:
|
||||
e["kind"] = edge_registry.resolve(
|
||||
edge_type=e.get("kind", "related_to"),
|
||||
provenance=e.get("provenance", "explicit"),
|
||||
context={**context, "line": e.get("line", "system")}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Processing failed for {file_path}: {e}", exc_info=True)
|
||||
@@ -341,12 +322,14 @@ class IngestionService:
|
||||
upsert_batch(self.client, n_name, n_pts)
|
||||
|
||||
if chunk_pls and vecs:
|
||||
c_name, c_pts = points_for_chunks(self.prefix, chunk_pls, vecs)
|
||||
upsert_batch(self.client, c_name, c_pts)
|
||||
# v2.11.14 Points-Extraction Logic
|
||||
c_pts = points_for_chunks(self.prefix, chunk_pls, vecs)[1]
|
||||
upsert_batch(self.client, f"{self.prefix}_chunks", c_pts)
|
||||
|
||||
if edges:
|
||||
e_name, e_pts = points_for_edges(self.prefix, edges)
|
||||
upsert_batch(self.client, e_name, e_pts)
|
||||
# v2.11.14 Points-Extraction Logic
|
||||
e_pts = points_for_edges(self.prefix, edges)[1]
|
||||
upsert_batch(self.client, f"{self.prefix}_edges", e_pts)
|
||||
|
||||
return {"path": file_path, "status": "success", "changed": True, "note_id": note_id, "chunks_count": len(chunk_pls), "edges_count": len(edges)}
|
||||
except Exception as e:
|
||||
|
||||
+38
-2
@@ -2,10 +2,11 @@
|
||||
FILE: app/core/parser.py
|
||||
DESCRIPTION: Liest Markdown-Dateien fehlertolerant (Encoding-Fallback). Trennt Frontmatter (YAML) vom Body.
|
||||
WP-22 Erweiterung: Kanten-Extraktion mit Zeilennummern für die EdgeRegistry.
|
||||
VERSION: 1.8.0
|
||||
WP-15b: Implementierung NoteContext und pre_scan_markdown für Pass 1 Ingestion.
|
||||
VERSION: 1.9.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: yaml, re, dataclasses, json, io, os
|
||||
LAST_ANALYSIS: 2025-12-23
|
||||
LAST_ANALYSIS: 2025-12-26
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -32,6 +33,15 @@ class ParsedNote:
|
||||
body: str
|
||||
path: str
|
||||
|
||||
@dataclass
|
||||
class NoteContext:
|
||||
"""Metadaten-Container für den flüchtigen LocalBatchCache (Pass 1)."""
|
||||
note_id: str
|
||||
title: str
|
||||
type: str
|
||||
summary: str
|
||||
tags: List[str]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Frontmatter-Erkennung
|
||||
@@ -152,6 +162,32 @@ def read_markdown(path: str) -> Optional[ParsedNote]:
|
||||
return ParsedNote(frontmatter=fm or {}, body=body or "", path=path)
|
||||
|
||||
|
||||
def pre_scan_markdown(path: str) -> Optional[NoteContext]:
|
||||
"""
|
||||
WP-15b: Schneller Scan für den LocalBatchCache (Pass 1).
|
||||
Extrahiert nur Identität und Kurz-Kontext zur semantischen Validierung.
|
||||
"""
|
||||
parsed = read_markdown(path)
|
||||
if not parsed:
|
||||
return None
|
||||
|
||||
fm = parsed.frontmatter
|
||||
# ID-Findung: Frontmatter ID oder Dateiname als Fallback
|
||||
note_id = str(fm.get("id") or os.path.splitext(os.path.basename(path))[0])
|
||||
|
||||
# Erstelle Kurz-Zusammenfassung (erste 500 Zeichen des Body, bereinigt)
|
||||
clean_body = re.sub(r'[#*`>]', '', parsed.body[:600]).strip()
|
||||
summary = clean_body[:500] + "..." if len(clean_body) > 500 else clean_body
|
||||
|
||||
return NoteContext(
|
||||
note_id=note_id,
|
||||
title=str(fm.get("title", note_id)),
|
||||
type=str(fm.get("type", "concept")),
|
||||
summary=summary,
|
||||
tags=fm.get("tags", []) if isinstance(fm.get("tags"), list) else []
|
||||
)
|
||||
|
||||
|
||||
def validate_required_frontmatter(fm: Dict[str, Any],
|
||||
required: Tuple[str, ...] = ("id", "title")) -> None:
|
||||
"""
|
||||
|
||||
+69
-36
@@ -1,11 +1,12 @@
|
||||
"""
|
||||
FILE: app/routers/chat.py
|
||||
DESCRIPTION: Haupt-Chat-Interface (RAG & Interview). Enthält Intent-Router (Keywords/LLM) und Prompt-Construction.
|
||||
VERSION: 2.7.1 (WP-22 Semantic Graph Routing)
|
||||
VERSION: 2.7.8 (Full Unabridged Stability Edition)
|
||||
STATUS: Active
|
||||
FIX: Umstellung auf llm.get_prompt() zur Behebung des 500 Server Errors (Dictionary replace crash).
|
||||
DEPENDENCIES: app.config, app.models.dto, app.services.llm_service, app.core.retriever, app.services.feedback_service
|
||||
EXTERNAL_CONFIG: config/decision_engine.yaml, config/types.yaml
|
||||
FIX:
|
||||
1. Implementiert Context-Throttling für Ollama (MAX_OLLAMA_CHARS).
|
||||
2. Deaktiviert LLM-Retries für den Chat (max_retries=0).
|
||||
3. Behebt Double-Fallback-Schleifen und Silent Refusals.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
@@ -36,7 +37,7 @@ def _load_decision_config() -> Dict[str, Any]:
|
||||
path = Path(settings.DECISION_CONFIG_PATH)
|
||||
default_config = {
|
||||
"strategies": {
|
||||
"FACT": {"trigger_keywords": []}
|
||||
"FACT": {"trigger_keywords": [], "preferred_provider": "openrouter"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,7 +160,7 @@ def _is_question(query: str) -> bool:
|
||||
q = query.strip().lower()
|
||||
if "?" in q: return True
|
||||
|
||||
# W-Fragen Indikatoren (falls User das ? vergisst)
|
||||
# W-Fragen Indikatoren
|
||||
starters = ["wer", "wie", "was", "wo", "wann", "warum", "weshalb", "wozu", "welche", "bist du", "entspricht"]
|
||||
if any(q.startswith(s + " ") for s in starters):
|
||||
return True
|
||||
@@ -170,7 +171,7 @@ async def _classify_intent(query: str, llm: LLMService) -> tuple[str, str]:
|
||||
"""
|
||||
Hybrid Router v5:
|
||||
1. Decision Keywords (Strategie) -> Prio 1
|
||||
2. Type Keywords (Interview Trigger) -> Prio 2, ABER NUR WENN KEINE FRAGE!
|
||||
2. Type Keywords (Interview Trigger) -> Prio 2
|
||||
3. LLM (Fallback) -> Prio 3
|
||||
"""
|
||||
config = get_full_config()
|
||||
@@ -179,7 +180,7 @@ async def _classify_intent(query: str, llm: LLMService) -> tuple[str, str]:
|
||||
|
||||
query_lower = query.lower()
|
||||
|
||||
# 1. FAST PATH A: Strategie Keywords (z.B. "Soll ich...")
|
||||
# 1. FAST PATH A: Strategie Keywords
|
||||
for intent_name, strategy in strategies.items():
|
||||
if intent_name == "FACT": continue
|
||||
keywords = strategy.get("trigger_keywords", [])
|
||||
@@ -187,7 +188,7 @@ async def _classify_intent(query: str, llm: LLMService) -> tuple[str, str]:
|
||||
if k.lower() in query_lower:
|
||||
return intent_name, "Keyword (Strategy)"
|
||||
|
||||
# 2. FAST PATH B: Type Keywords (z.B. "Projekt", "Werte") -> INTERVIEW
|
||||
# 2. FAST PATH B: Type Keywords -> INTERVIEW
|
||||
if not _is_question(query_lower):
|
||||
types_cfg = get_types_config()
|
||||
types_def = types_cfg.get("types", {})
|
||||
@@ -200,15 +201,15 @@ async def _classify_intent(query: str, llm: LLMService) -> tuple[str, str]:
|
||||
|
||||
# 3. SLOW PATH: LLM Router
|
||||
if settings.get("llm_fallback_enabled", False):
|
||||
# FIX: Nutze get_prompt statt direktem Zugriff auf dict
|
||||
router_prompt_template = llm.get_prompt("router_prompt")
|
||||
router_prompt_template = llm.get_prompt("llm_router_prompt")
|
||||
|
||||
if router_prompt_template:
|
||||
prompt = router_prompt_template.replace("{query}", query)
|
||||
logger.info("Keywords failed (or Question detected). Asking LLM for Intent...")
|
||||
|
||||
try:
|
||||
raw_response = await llm.generate_raw_response(prompt, priority="realtime")
|
||||
# FIX: Auch beim Routing keine Retries im Chat-Fluss
|
||||
raw_response = await llm.generate_raw_response(prompt, priority="realtime", max_retries=0)
|
||||
llm_output_upper = raw_response.upper()
|
||||
|
||||
if "INTERVIEW" in llm_output_upper or "CREATE" in llm_output_upper:
|
||||
@@ -241,9 +242,11 @@ async def chat_endpoint(
|
||||
# Strategy Load
|
||||
strategy = get_decision_strategy(intent)
|
||||
prompt_key = strategy.get("prompt_template", "rag_template")
|
||||
preferred_provider = strategy.get("preferred_provider")
|
||||
|
||||
sources_hits = []
|
||||
final_prompt = ""
|
||||
context_str = ""
|
||||
|
||||
if intent == "INTERVIEW":
|
||||
# --- INTERVIEW MODE ---
|
||||
@@ -264,7 +267,6 @@ async def chat_endpoint(
|
||||
logger.info(f"[{query_id}] Interview Type: {target_type}. Fields: {len(fields_list)}")
|
||||
fields_str = "\n- " + "\n- ".join(fields_list)
|
||||
|
||||
# FIX: Nutze get_prompt() zur Auflösung der provider-spezifischen Templates
|
||||
template = llm.get_prompt(prompt_key)
|
||||
final_prompt = template.replace("{context_str}", "Dialogverlauf...") \
|
||||
.replace("{query}", request.message) \
|
||||
@@ -274,14 +276,10 @@ async def chat_endpoint(
|
||||
sources_hits = []
|
||||
|
||||
else:
|
||||
# --- RAG MODE ---
|
||||
# --- RAG MODE (FACT, DECISION, EMPATHY, CODING) ---
|
||||
inject_types = strategy.get("inject_types", [])
|
||||
prepend_instr = strategy.get("prepend_instruction", "")
|
||||
|
||||
# --- WP-22: Semantic Graph Routing (Teil C) ---
|
||||
edge_boosts = strategy.get("edge_boosts", {})
|
||||
if edge_boosts:
|
||||
logger.info(f"[{query_id}] Applying Edge Boosts: {edge_boosts}")
|
||||
|
||||
query_req = QueryRequest(
|
||||
query=request.message,
|
||||
@@ -308,32 +306,66 @@ async def chat_endpoint(
|
||||
if strat_hit.node_id not in existing_ids:
|
||||
hits.append(strat_hit)
|
||||
|
||||
if not hits:
|
||||
context_str = "Keine relevanten Notizen gefunden."
|
||||
else:
|
||||
context_str = _build_enriched_context(hits)
|
||||
|
||||
# FIX: Nutze get_prompt() zur Auflösung der provider-spezifischen Templates
|
||||
template = llm.get_prompt(prompt_key)
|
||||
context_str = _build_enriched_context(hits) if hits else "Keine relevanten Notizen gefunden."
|
||||
|
||||
if not template:
|
||||
template = "{context_str}\n\n{query}"
|
||||
# --- STABILITY FIX: OLLAMA CONTEXT THROTTLE ---
|
||||
# Begrenzt den Text, um den "decode: cannot decode batches" Fehler zu vermeiden.
|
||||
# MAX_OLLAMA_CHARS = 10000
|
||||
|
||||
settings = get_settings() # Falls noch nicht im Scope vorhanden
|
||||
max_chars = getattr(settings, "MAX_OLLAMA_CHARS", 10000)
|
||||
if preferred_provider == "ollama" and len(context_str) > max_chars:
|
||||
logger.warning(f"⚠️ [{query_id}] Context zu groß für Ollama ({len(context_str)} chars). Kürze auf {max_chars}.")
|
||||
context_str = context_str[:max_chars] + "\n[...gekürzt zur Stabilität...]"
|
||||
|
||||
template = llm.get_prompt(prompt_key) or "{context_str}\n\n{query}"
|
||||
|
||||
if prepend_instr:
|
||||
context_str = f"{prepend_instr}\n\n{context_str}"
|
||||
|
||||
final_prompt = template.replace("{context_str}", context_str).replace("{query}", request.message)
|
||||
sources_hits = hits
|
||||
|
||||
# --- GENERATION ---
|
||||
# --- DEBUG SPOT 1: PROMPT CONSTRUCTION ---
|
||||
logger.info(f"[{query_id}] PROMPT CONSTRUCTION COMPLETE. Length: {len(final_prompt)} chars.")
|
||||
if not final_prompt.strip():
|
||||
logger.error(f"[{query_id}] CRITICAL: Final prompt is empty before sending to LLM!")
|
||||
|
||||
# --- GENERATION WITH NO-RETRY & DEEP FALLBACK ---
|
||||
system_prompt = llm.get_prompt("system_prompt")
|
||||
|
||||
# Chat nutzt IMMER realtime priority
|
||||
answer_text = await llm.generate_raw_response(
|
||||
prompt=final_prompt,
|
||||
system=system_prompt,
|
||||
priority="realtime"
|
||||
)
|
||||
# --- DEBUG SPOT 2: PRIMARY CALL ---
|
||||
logger.info(f"[{query_id}] PRIMARY CALL: Sending request to provider '{preferred_provider}' (No Retries)...")
|
||||
|
||||
answer_text = ""
|
||||
try:
|
||||
# FIX: max_retries=0 verhindert Hänger durch Retry-Kaskaden im Chat
|
||||
answer_text = await llm.generate_raw_response(
|
||||
prompt=final_prompt,
|
||||
system=system_prompt,
|
||||
priority="realtime",
|
||||
provider=preferred_provider,
|
||||
max_retries=0
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"🛑 [{query_id}] Primary Provider '{preferred_provider}' failed: {e}")
|
||||
|
||||
# DEEP FALLBACK: Wenn die Antwort leer ist (Silent Refusal) oder der Primary abgestürzt ist
|
||||
if not answer_text.strip() and preferred_provider != "ollama":
|
||||
# --- DEBUG SPOT 3: FALLBACK TRIGGER ---
|
||||
logger.warning(f"🛑 [{query_id}] PRIMARY '{preferred_provider}' returned EMPTY or FAILED. Triggering Deep Fallback to Ollama...")
|
||||
|
||||
try:
|
||||
answer_text = await llm.generate_raw_response(
|
||||
prompt=final_prompt,
|
||||
system=system_prompt,
|
||||
priority="realtime",
|
||||
provider="ollama",
|
||||
max_retries=0
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"🛑 [{query_id}] Deep Fallback to Ollama also failed: {e}")
|
||||
answer_text = "Entschuldigung, das System ist aktuell überlastet. Bitte versuche es in einem Moment erneut."
|
||||
|
||||
duration_ms = int((time.time() - start_time) * 1000)
|
||||
|
||||
@@ -344,7 +376,7 @@ async def chat_endpoint(
|
||||
query_text=request.message,
|
||||
results=sources_hits,
|
||||
mode="interview" if intent == "INTERVIEW" else "chat_rag",
|
||||
metadata={"intent": intent, "source": intent_source}
|
||||
metadata={"intent": intent, "source": intent_source, "provider": preferred_provider}
|
||||
)
|
||||
except: pass
|
||||
|
||||
@@ -359,4 +391,5 @@ async def chat_endpoint(
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in chat endpoint: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
# Wir geben eine benutzerfreundliche Meldung zurück, statt nur den Error-Stack
|
||||
raise HTTPException(status_code=500, detail="Das System konnte die Anfrage nicht verarbeiten.")
|
||||
@@ -1,11 +1,14 @@
|
||||
"""
|
||||
FILE: app/services/edge_registry.py
|
||||
DESCRIPTION: Single Source of Truth für Kanten-Typen mit dynamischem Reload.
|
||||
WP-15b: Erweiterte Provenance-Prüfung für die Candidate-Validation.
|
||||
Sichert die Graph-Integrität durch strikte Trennung von System- und Inhaltskanten.
|
||||
WP-22: Fix für absolute Pfade außerhalb des Vaults (Prod-Dictionary).
|
||||
WP-20: Synchronisation mit zentralen Settings (v0.6.2).
|
||||
VERSION: 0.7.5
|
||||
VERSION: 0.8.0
|
||||
STATUS: Active
|
||||
DEPENDENCIES: re, os, json, logging, time, app.config
|
||||
LAST_ANALYSIS: 2025-12-26
|
||||
"""
|
||||
import re
|
||||
import os
|
||||
@@ -19,7 +22,12 @@ from app.config import get_settings
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class EdgeRegistry:
|
||||
"""
|
||||
Zentraler Verwalter für das Kanten-Vokabular.
|
||||
Implementiert das Singleton-Pattern für konsistente Validierung über alle Services.
|
||||
"""
|
||||
_instance = None
|
||||
# System-Kanten, die nicht durch User oder KI gesetzt werden dürfen
|
||||
FORBIDDEN_SYSTEM_EDGES = {"next", "prev", "belongs_to"}
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
@@ -51,7 +59,7 @@ class EdgeRegistry:
|
||||
def ensure_latest(self):
|
||||
"""
|
||||
Prüft den Zeitstempel der Vokabular-Datei und lädt bei Bedarf neu.
|
||||
Verhindert den AttributeError in der Ingestion-Pipeline.
|
||||
Verhindert Inkonsistenzen bei Laufzeit-Updates des Dictionaries.
|
||||
"""
|
||||
if not os.path.exists(self.full_vocab_path):
|
||||
logger.error(f"!!! [EDGE-REGISTRY ERROR] File not found: {self.full_vocab_path} !!!")
|
||||
@@ -66,7 +74,10 @@ class EdgeRegistry:
|
||||
logger.error(f"!!! [EDGE-REGISTRY] Error checking file time: {e}")
|
||||
|
||||
def _load_vocabulary(self):
|
||||
"""Parst das Markdown-Wörterbuch und baut die Canonical-Map auf."""
|
||||
"""
|
||||
Parst das Markdown-Wörterbuch und baut die Canonical-Map auf.
|
||||
Erkennt Tabellen-Strukturen und extrahiert fettgedruckte System-Typen.
|
||||
"""
|
||||
self.canonical_map.clear()
|
||||
self.valid_types.clear()
|
||||
|
||||
@@ -101,8 +112,8 @@ class EdgeRegistry:
|
||||
|
||||
def resolve(self, edge_type: str, provenance: str = "explicit", context: dict = None) -> str:
|
||||
"""
|
||||
Validiert einen Kanten-Typ gegen das Vokabular.
|
||||
Loggt unbekannte Typen für die spätere manuelle Pflege.
|
||||
WP-15b: Validiert einen Kanten-Typ gegen das Vokabular und prüft Berechtigungen.
|
||||
Sichert, dass nur strukturelle Prozesse System-Kanten setzen dürfen.
|
||||
"""
|
||||
self.ensure_latest()
|
||||
if not edge_type:
|
||||
@@ -112,20 +123,23 @@ class EdgeRegistry:
|
||||
clean_type = edge_type.lower().strip().replace(" ", "_").replace("-", "_")
|
||||
ctx = context or {}
|
||||
|
||||
# System-Kanten dürfen nicht manuell vergeben werden
|
||||
if provenance == "explicit" and clean_type in self.FORBIDDEN_SYSTEM_EDGES:
|
||||
self._log_issue(clean_type, "forbidden_system_usage", ctx)
|
||||
# WP-15b: System-Kanten dürfen weder manuell noch durch KI/Vererbung gesetzt werden.
|
||||
# Nur Provenienz 'structure' (interne Prozesse) ist autorisiert.
|
||||
# Wir blockieren hier alle Provenienzen außer 'structure'.
|
||||
restricted_provenance = ["explicit", "semantic_ai", "inherited", "global_pool", "rule"]
|
||||
if provenance in restricted_provenance and clean_type in self.FORBIDDEN_SYSTEM_EDGES:
|
||||
self._log_issue(clean_type, f"forbidden_usage_by_{provenance}", ctx)
|
||||
return "related_to"
|
||||
|
||||
# System-Kanten sind nur bei struktureller Provenienz erlaubt
|
||||
# System-Kanten sind NUR bei struktureller Provenienz erlaubt
|
||||
if provenance == "structure" and clean_type in self.FORBIDDEN_SYSTEM_EDGES:
|
||||
return clean_type
|
||||
|
||||
# Mapping auf kanonischen Namen
|
||||
# Mapping auf kanonischen Namen (Alias-Auflösung)
|
||||
if clean_type in self.canonical_map:
|
||||
return self.canonical_map[clean_type]
|
||||
|
||||
# Fallback und Logging
|
||||
# Fallback und Logging unbekannter Typen für Admin-Review
|
||||
self._log_issue(clean_type, "unknown_type", ctx)
|
||||
return clean_type
|
||||
|
||||
@@ -139,12 +153,13 @@ class EdgeRegistry:
|
||||
"error": error_kind,
|
||||
"file": ctx.get("file", "unknown"),
|
||||
"line": ctx.get("line", "unknown"),
|
||||
"note_id": ctx.get("note_id", "unknown")
|
||||
"note_id": ctx.get("note_id", "unknown"),
|
||||
"provenance": ctx.get("provenance", "unknown")
|
||||
}
|
||||
with open(self.unknown_log_path, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(entry) + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Singleton Export
|
||||
# Singleton Export für systemweiten Zugriff
|
||||
registry = EdgeRegistry()
|
||||
+20
-23
@@ -6,9 +6,12 @@ DESCRIPTION: Hybrid-Client für Ollama, Google GenAI (Gemini) und OpenRouter.
|
||||
WP-20 Fix: Bulletproof Prompt-Auflösung für format() Aufrufe.
|
||||
WP-22/JSON: Optionales JSON-Schema + strict (für OpenRouter structured outputs).
|
||||
FIX: Intelligente Rate-Limit Erkennung (429 Handling), v1-API Sync & Timeouts.
|
||||
VERSION: 3.3.6
|
||||
VERSION: 3.3.7
|
||||
STATUS: Active
|
||||
DEPENDENCIES: httpx, yaml, logging, asyncio, json, google-genai, openai, app.config
|
||||
FIX:
|
||||
- Implementiert striktes max_retries Handling für alle Provider (v.a. für Chat-Stabilität).
|
||||
- Synchronisiert Rate-Limit Retries mit dem max_retries Parameter.
|
||||
- Optimiert Logging für sofortige Fehlererkennung.
|
||||
"""
|
||||
import httpx
|
||||
import yaml
|
||||
@@ -84,8 +87,6 @@ class LLMService:
|
||||
Hole provider-spezifisches Template mit intelligenter Text-Kaskade.
|
||||
HINWEIS: Dies ist nur ein Text-Lookup und verbraucht kein API-Kontingent.
|
||||
Kaskade: Gewählter Provider -> Gemini (Cloud-Stil) -> Ollama (Basis-Stil).
|
||||
|
||||
WP-20 Fix: Garantiert die Rückgabe eines Strings, um AttributeError zu vermeiden.
|
||||
"""
|
||||
active_provider = provider or self.settings.MINDNET_LLM_PROVIDER
|
||||
data = self.prompts.get(key, "")
|
||||
@@ -118,11 +119,6 @@ class LLMService:
|
||||
) -> str:
|
||||
"""
|
||||
Haupteinstiegspunkt für LLM-Anfragen mit Priorisierung.
|
||||
|
||||
force_json:
|
||||
- Ollama: nutzt payload["format"]="json"
|
||||
- Gemini: nutzt response_mime_type="application/json"
|
||||
- OpenRouter: nutzt response_format=json_object (Fallback) oder json_schema
|
||||
"""
|
||||
target_provider = provider or self.settings.MINDNET_LLM_PROVIDER
|
||||
|
||||
@@ -154,11 +150,12 @@ class LLMService:
|
||||
strict_json_schema: bool
|
||||
) -> str:
|
||||
"""
|
||||
Routet die Anfrage mit intelligenter Rate-Limit Erkennung (WP-20 + WP-76).
|
||||
Schleife läuft über MINDNET_LLM_RATE_LIMIT_RETRIES.
|
||||
Routet die Anfrage mit intelligenter Rate-Limit Erkennung.
|
||||
Nutzt max_retries um die Rate-Limit Schleife zu begrenzen.
|
||||
"""
|
||||
rate_limit_attempts = 0
|
||||
max_rate_retries = getattr(self.settings, "LLM_RATE_LIMIT_RETRIES", 3)
|
||||
# FIX: Wir nutzen max_retries als Limit für Rate-Limit Versuche, wenn explizit klein gewählt (z.B. Chat)
|
||||
max_rate_retries = min(max_retries, getattr(self.settings, "LLM_RATE_LIMIT_RETRIES", 3))
|
||||
wait_time = getattr(self.settings, "LLM_RATE_LIMIT_WAIT", 60.0)
|
||||
|
||||
while rate_limit_attempts <= max_rate_retries:
|
||||
@@ -182,18 +179,17 @@ class LLMService:
|
||||
|
||||
except Exception as e:
|
||||
err_str = str(e)
|
||||
# Intelligente 429 Erkennung für alle Cloud-Provider
|
||||
# Intelligente 429 Erkennung
|
||||
is_rate_limit = any(x in err_str for x in ["429", "RESOURCE_EXHAUSTED", "rate_limited", "Too Many Requests"])
|
||||
|
||||
if is_rate_limit and rate_limit_attempts < max_rate_retries:
|
||||
rate_limit_attempts += 1
|
||||
logger.warning(
|
||||
f"⏳ [LLMService] Rate Limit (429) detected from {provider}. "
|
||||
f"Attempt {rate_limit_attempts}/{max_rate_retries}. "
|
||||
f"Waiting {wait_time}s before cloud retry..."
|
||||
f"⏳ [LLMService] Rate Limit detected from {provider}. "
|
||||
f"Attempt {rate_limit_attempts}/{max_rate_retries}. Waiting {wait_time}s..."
|
||||
)
|
||||
await asyncio.sleep(wait_time)
|
||||
continue # Nächster Versuch in der Cloud-Schleife
|
||||
continue
|
||||
|
||||
# Wenn kein Rate-Limit oder Retries erschöpft -> Fallback zu Ollama (falls aktiviert)
|
||||
if self.settings.LLM_FALLBACK_ENABLED and provider != "ollama":
|
||||
@@ -206,14 +202,12 @@ class LLMService:
|
||||
async def _execute_google(self, prompt, system, force_json, model_override):
|
||||
"""Native Google SDK Integration (Gemini) mit v1 Fix."""
|
||||
model = model_override or self.settings.GEMINI_MODEL
|
||||
# Fix: Bereinige Modellnamen (Entfernung von 'models/' Präfix)
|
||||
clean_model = model.replace("models/", "")
|
||||
|
||||
config = types.GenerateContentConfig(
|
||||
system_instruction=system,
|
||||
response_mime_type="application/json" if force_json else "text/plain"
|
||||
)
|
||||
# Thread-Offloading mit striktem Timeout gegen "Hangs"
|
||||
response = await asyncio.wait_for(
|
||||
asyncio.to_thread(
|
||||
self.google_client.models.generate_content,
|
||||
@@ -233,7 +227,7 @@ class LLMService:
|
||||
json_schema_name: str = "mindnet_json",
|
||||
strict_json_schema: bool = True
|
||||
) -> str:
|
||||
"""OpenRouter API Integration (OpenAI-kompatibel) mit Schema-Support."""
|
||||
"""OpenRouter API Integration (OpenAI-kompatibel)."""
|
||||
model = model_override or self.settings.OPENROUTER_MODEL
|
||||
messages = []
|
||||
if system:
|
||||
@@ -262,14 +256,14 @@ class LLMService:
|
||||
return response.choices[0].message.content.strip()
|
||||
|
||||
async def _execute_ollama(self, prompt, system, force_json, max_retries, base_delay):
|
||||
"""Lokaler Ollama Call mit exponentiellem Backoff."""
|
||||
"""Lokaler Ollama Call mit striktem Retry-Limit."""
|
||||
payload = {
|
||||
"model": self.settings.LLM_MODEL,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.1 if force_json else 0.7,
|
||||
"num_ctx": 8192
|
||||
"num_ctx": 8192 # Begrenzung für Stabilität (WP-20)
|
||||
}
|
||||
}
|
||||
if force_json:
|
||||
@@ -285,9 +279,11 @@ class LLMService:
|
||||
return res.json().get("response", "").strip()
|
||||
except Exception as e:
|
||||
attempt += 1
|
||||
# WICHTIG: Wenn max_retries=0 (Chat), bricht dies nach dem 1. Versuch (attempt=1) sofort ab.
|
||||
if attempt > max_retries:
|
||||
logger.error(f"❌ Ollama Error after {attempt} retries: {e}")
|
||||
logger.error(f"❌ Ollama request failed after {attempt} attempt(s): {e}")
|
||||
raise e
|
||||
|
||||
wait_time = base_delay * (2 ** (attempt - 1))
|
||||
logger.warning(f"⚠️ Ollama attempt {attempt} failed. Retrying in {wait_time}s...")
|
||||
await asyncio.sleep(wait_time)
|
||||
@@ -300,6 +296,7 @@ class LLMService:
|
||||
|
||||
final_prompt = rag_template.format(context_str=context_str, query=query)
|
||||
|
||||
# RAG Aufrufe im Chat nutzen nun standardmäßig max_retries=2 (überschreibbar)
|
||||
return await self.generate_raw_response(
|
||||
final_prompt,
|
||||
system=system_prompt,
|
||||
|
||||
@@ -82,7 +82,7 @@ strategies:
|
||||
# 3. Empathie / "Ich"-Modus (Lokal & Privat via Ollama)
|
||||
EMPATHY:
|
||||
description: "Reaktion auf emotionale Zustände."
|
||||
preferred_provider: "ollama"
|
||||
preferred_provider: "openrouter"
|
||||
trigger_keywords:
|
||||
- "ich fühle"
|
||||
- "traurig"
|
||||
@@ -127,7 +127,7 @@ strategies:
|
||||
# 5. Interview / Datenerfassung (Lokal)
|
||||
INTERVIEW:
|
||||
description: "Der User möchte Wissen erfassen."
|
||||
preferred_provider: "ollama"
|
||||
preferred_provider: "openrouter"
|
||||
trigger_keywords:
|
||||
- "neue notiz"
|
||||
- "etwas notieren"
|
||||
|
||||
+39
-5
@@ -1,6 +1,7 @@
|
||||
# config/prompts.yaml — Final V2.5.5 (OpenRouter Hardening)
|
||||
# config/prompts.yaml — Final V2.6.0 (WP-15b Candidate-Validation)
|
||||
# WP-20: Optimierte Cloud-Templates zur Unterdrückung von Modell-Geschwätz.
|
||||
# FIX: Explizite Verbote für Einleitungstexte zur Vermeidung von JSON-Parsing-Fehlern.
|
||||
# WP-15b: Integration der binären edge_validation für den Two-Pass Workflow.
|
||||
# OLLAMA: UNVERÄNDERT laut Benutzeranweisung.
|
||||
|
||||
system_prompt: |
|
||||
@@ -215,7 +216,7 @@ edge_extraction:
|
||||
4. Antworte AUSSCHLIESSLICH in validem JSON als Liste von Objekten.
|
||||
|
||||
BEISPIEL:
|
||||
[[ {{"to": "Ziel-Konzept", "kind": "beziehungs_typ"}} ]]
|
||||
[[ {{"to": "Ziel-Konzept", \"kind\": \"beziehungs_typ\"}} ]]
|
||||
|
||||
TEXT:
|
||||
"""
|
||||
@@ -227,13 +228,46 @@ edge_extraction:
|
||||
Analysiere '{note_id}'. Extrahiere semantische Beziehungen.
|
||||
ERLAUBTE TYPEN: {valid_types}
|
||||
TEXT: {text}
|
||||
OUTPUT: STRIKT JSON-Array von Objekten: [[{{"to":"Ziel","kind":"typ"}}]]. Kein Text davor/danach. Wenn nichts: [].
|
||||
OUTPUT: STRIKT JSON-Array von Objekten: [[{{"to\":\"Ziel\",\"kind\":\"typ\"}}]]. Kein Text davor/danach. Wenn nichts: [].
|
||||
openrouter: |
|
||||
TASK: Extrahiere semantische Relationen für '{note_id}'.
|
||||
ERLAUBTE TYPEN: {valid_types}
|
||||
TEXT: {text}
|
||||
ANWEISUNG: Antworte AUSSCHLIESSLICH mit einem JSON-Array von Objekten.
|
||||
FORMAT: [[{{"to":"Ziel-Begriff","kind":"typ"}}]]
|
||||
FORMAT: [[{{"to\":\"Ziel-Begriff\",\"kind\":\"typ\"}}]]
|
||||
STRIKTES VERBOT: Schreibe keine Einleitung, keine Analyse und keine Erklärungen.
|
||||
Wenn keine Relationen existieren, antworte NUR mit: []
|
||||
OUTPUT:
|
||||
OUTPUT:
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# 8. WP-15b: EDGE VALIDATION (Intent: VALIDATE)
|
||||
# ---------------------------------------------------------
|
||||
edge_validation:
|
||||
gemini: |
|
||||
Bewerte die semantische Validität dieser Verbindung im Wissensgraph.
|
||||
|
||||
KONTEXT DER QUELLE (Chunk):
|
||||
"{chunk_text}"
|
||||
|
||||
ZIEL-NOTIZ: "{target_title}"
|
||||
ZIEL-BESCHREIBUNG (Zusammenfassung):
|
||||
"{target_summary}"
|
||||
|
||||
GEPLANTE RELATION: "{edge_kind}"
|
||||
|
||||
FRAGE: Bestätigt der Kontext der Quelle die Beziehung '{edge_kind}' zum Ziel?
|
||||
REGEL: Antworte NUR mit 'YES' oder 'NO'. Keine Erklärungen oder Smalltalk.
|
||||
openrouter: |
|
||||
Verify semantic relation for graph construction.
|
||||
Source Context: {chunk_text}
|
||||
Target Note: {target_title}
|
||||
Target Summary: {target_summary}
|
||||
Proposed Relation: {edge_kind}
|
||||
Instruction: Does the source context support this relation to the target?
|
||||
Result: Respond ONLY with 'YES' or 'NO'.
|
||||
ollama: |
|
||||
Bewerte die semantische Korrektheit dieser Verbindung.
|
||||
QUELLE: {chunk_text}
|
||||
ZIEL: {target_title} ({target_summary})
|
||||
BEZIEHUNG: {edge_kind}
|
||||
Ist diese Verbindung valide? Antworte NUR mit YES oder NO.
|
||||
@@ -53,6 +53,20 @@ Wenn der Intent `FACT` oder `DECISION` ist, wird folgender Flow ausgeführt:
|
||||
|
||||
Der `LLMService` (`app/services/llm_service.py`) fungiert als zentraler Hybrid-Client für OpenRouter, Google Gemini und Ollama. Er schützt das System vor Überlastung und verwaltet Quoten.
|
||||
|
||||
Mit Version 2.8.1 wurde die Architektur der Antwort-Generierung grundlegend gehärtet:
|
||||
|
||||
A. Fail-Fast Prinzip (No-Retry Chat)
|
||||
Im Gegensatz zur Ingestion-Pipeline nutzt das Chat-Backend für Echtzeit-Anfragen keine internen Retries (max_retries=0).
|
||||
|
||||
Scheitert ein Provider (Timeout/Fehler), wird sofort die Fallback-Kaskade eingeleitet oder der Deep Fallback zu Ollama getriggert.
|
||||
|
||||
Dies verhindert die kumulative Wartezeit von mehreren Minuten bei Provider-Störungen.
|
||||
|
||||
B. Context-Throttling & Memory Guard
|
||||
Drosselung: Vor der Übergabe an Ollama prüft die chat.py, ob der Kontext (RAG-Hits) die Grenze von MAX_OLLAMA_CHARS überschreitet und kürzt diesen ggf..
|
||||
|
||||
Modell-Lock: Der LLMService erzwingt im Ollama-Payload den Parameter num_ctx: 8192. Dies stabilisiert den VRAM-Verbrauch und verhindert, dass das Modell versucht, speicherintensive 128k-Kontexte zu reservieren.
|
||||
|
||||
### 2.1 Prioritäts-Semaphor
|
||||
|
||||
Jeder LLM-Request steuert über ein `priority`-Flag den Zugriff auf Hardware- und API-Ressourcen:
|
||||
|
||||
@@ -29,8 +29,8 @@ Diese Variablen steuern die Infrastruktur, Pfade und globale Timeouts.
|
||||
| `MINDNET_PROMPTS_PATH` | `config/prompts.yaml` | Pfad zu LLM Prompts. |
|
||||
| `MINDNET_LLM_PROVIDER` | `openrouter` | **Neu (WP-20):** Aktiver Provider (`openrouter`, `gemini`, `ollama`). |
|
||||
| `MINDNET_LLM_FALLBACK` | `true` | **Neu (WP-20):** Aktiviert automatischen Ollama-Fallback bei Cloud-Fehlern. |
|
||||
| `MINDNET_LLM_RATE_LIMIT_WAIT`| `60.0` | **Neu (WP-76):** Wartezeit in Sekunden bei HTTP 429 (Rate Limit). |
|
||||
| `MINDNET_LLM_RATE_LIMIT_RETRIES`| `3` | **Neu (WP-76):** Anzahl Cloud-Retries vor lokalem Fallback. |
|
||||
| `MINDNET_LLM_RATE_LIMIT_WAIT`| `60.0` | **Neu (WP-20):** Wartezeit in Sekunden bei HTTP 429 (Rate Limit). |
|
||||
| `MINDNET_LLM_RATE_LIMIT_RETRIES`| `3` | **Neu (WP-20):** Anzahl Cloud-Retries vor lokalem Fallback. |
|
||||
| `GOOGLE_API_KEY` | *(Key)* | API Key für Google AI Studio. |
|
||||
| `MINDNET_GEMINI_MODEL` | `gemini-2.5-flash-lite` | **Update 2025:** Optimiertes Lite-Modell für hohe Quoten. |
|
||||
| `OPENROUTER_API_KEY` | *(Key)* | API Key für OpenRouter Integration. |
|
||||
@@ -38,6 +38,7 @@ Diese Variablen steuern die Infrastruktur, Pfade und globale Timeouts.
|
||||
| `MINDNET_LLM_MODEL` | `phi3:mini` | Name des lokalen Chat-Modells (Ollama). |
|
||||
| `MINDNET_EMBEDDING_MODEL` | `nomic-embed-text` | Name des Embedding-Modells (Ollama). |
|
||||
| `MINDNET_OLLAMA_URL` | `http://127.0.0.1:11434`| URL zum lokalen LLM-Server. |
|
||||
| `MAX_OLLAMA_CHARS` | `10000`| Maximale Länge des Kontext-Strings, der an das lokale Modell gesendet wird. Verhindert Batch-Decoding-Fehler bei sehr großen Notiz-Historien. |
|
||||
| `MINDNET_LLM_TIMEOUT` | `300.0` | Timeout in Sekunden für LLM-Anfragen. |
|
||||
| `MINDNET_API_TIMEOUT` | `300.0` | Globales API-Timeout für das Frontend. |
|
||||
| `MINDNET_LL_BACKGROUND_LIMIT`| `2` | **Traffic Control:** Max. parallele Hintergrund-Tasks (Semaphore). |
|
||||
|
||||
@@ -19,14 +19,31 @@ context: "Installationsanleitung, Systemd-Units und Wartungsprozesse für Mindne
|
||||
### 1.2 Qdrant (Docker)
|
||||
Startet die Vektor-DB mit persistentem Storage auf Port 6333.
|
||||
|
||||
dockercompose.yaml
|
||||
```bash
|
||||
docker run -d \
|
||||
--name mindnet_qdrant \
|
||||
--restart always \
|
||||
-p 6333:6333 \
|
||||
-v $(pwd)/qdrant_storage:/qdrant/storage \
|
||||
qdrant/qdrant
|
||||
services:
|
||||
qdrant:
|
||||
image: qdrant/qdrant
|
||||
container_name: qdrant
|
||||
ports:
|
||||
- "6333:6333"
|
||||
volumes:
|
||||
- ./qdrant_data:/qdrant/storage
|
||||
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 65535
|
||||
hard: 65535
|
||||
restart: unless-stopped
|
||||
|
||||
```
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
|
||||
Um Abstürze der Vektordatenbank bei einer hohen Anzahl an Collections (z. B. durch viele Notiz-Typen oder Dev-Umgebungen) zu vermeiden, müssen die System-Limits für den Container angepasst werden.
|
||||
Hintergrund: Qdrant öffnet für jedes Segment einer Collection mehrere Dateien. Ohne diese Erhöhung führt das Standard-Linux-Limit (1024) zum Absturz mit dem Fehler os error 24 (Too many open files).
|
||||
|
||||
### 1.3 Ollama (Modelle)
|
||||
**Wichtig:** Seit v2.4 ist `nomic-embed-text` Pflicht für Embeddings.
|
||||
|
||||
@@ -185,44 +185,42 @@ Der bisherige WP-15 Ansatz litt unter Halluzinationen (erfundene Kantentypen), h
|
||||
2. **Single Source of Truth (SSOT):** Die Registry nutzt `01_edge_vocabulary.md` als führende Konfiguration.
|
||||
3. **Self-Learning Loop:** Protokollierung unbekannter Kanten in `unknown_edges.jsonl`.
|
||||
|
||||
## 23: Agentic Multi-Stream Reasoning (Mindnet 2025)
|
||||
### WP-23: Agentic Multi-Stream Reasoning (Mindnet 2025)
|
||||
|
||||
### 1. Zielsetzung & Problemstellung
|
||||
#### 1. Zielsetzung & Problemstellung
|
||||
Das bisherige System basiert auf einem globalen Scoring-Modell, bei dem Notizen unterschiedlicher Typen (z. B. `insight` vs. `belief`) in einem einzigen Retrieval-Topf konkurrieren. Dies führt dazu, dass leiser gewichtete, aber fundamentale Identitätsmerkmale oft durch hochgewichtete aktuelle Erkenntnisse verdrängt werden. Ziel dieses Pakets ist die Einführung einer parallelen **Stream-Architektur**, um die Vielschichtigkeit menschlicher Entscheidungsprozesse (Werte + Erfahrung + Absicht) im LLM-Kontext zu garantieren.
|
||||
|
||||
---
|
||||
|
||||
### 2. Funktionsbeschreibung: Die Streams
|
||||
#### 2. Funktionsbeschreibung: Die Streams
|
||||
Die Daten aus der `types.yaml` werden in drei logische Verarbeitungseinheiten unterteilt:
|
||||
|
||||
#### A. Identity Stream (Die Wahrheitsebene)
|
||||
##### A. Identity Stream (Die Wahrheitsebene)
|
||||
* **Inhalt:** `value`, `belief`, `trait`, `principle`, `need`, `boundary`, `bias`.
|
||||
* **Zweck:** Definition des moralischen Kompasses, der psychologischen Grundbedürfnisse und kognitiven Muster.
|
||||
* **Wirkung:** Liefert das "Warum" hinter jeder Handlung.
|
||||
|
||||
#### B. History Stream (Die Evidenzebene)
|
||||
##### B. History Stream (Die Evidenzebene)
|
||||
* **Inhalt:** `experience`, `event`, `source`, `journal`, `person`.
|
||||
* **Zweck:** Bereitstellung empirischer Belege aus der Vergangenheit und sozialer Kontexte.
|
||||
* **Wirkung:** Verankert die Antwort in real erlebten Mustern und Fakten.
|
||||
|
||||
#### C. Action Stream (Die Dynamikebene)
|
||||
##### C. Action Stream (Die Dynamikebene)
|
||||
* **Inhalt:** `project`, `decision`, `goal`, `task`, `risk`, `motivation`, `habit`, `state`.
|
||||
* **Zweck:** Analyse der aktuellen Richtung, geplanter Vorhaben und des gegenwärtigen Zustands.
|
||||
* **Wirkung:** Liefert den Kontext für die Umsetzung und zukünftige Ziele.
|
||||
|
||||
|
||||
### 3. Technische Wirkungsweise (Solution Sketch)
|
||||
#### 3. Technische Wirkungsweise (Solution Sketch)
|
||||
|
||||
#### Schritt 1: Query-Decomposition
|
||||
##### Schritt 1: Query-Decomposition
|
||||
Ein initialer Klassifizierungs-Agent analysiert die Nutzeranfrage und bestimmt, welcher Stream primär angesprochen werden muss (z. B. "Wie soll ich mich entscheiden?" boostet den Identity Stream).
|
||||
|
||||
#### Schritt 2: Parallel Stream Retrieval
|
||||
##### Schritt 2: Parallel Stream Retrieval
|
||||
Anstelle einer Suche werden drei unabhängige Vektor-Suchen mit Typ-Filtern durchgeführt:
|
||||
* **Search_A (Identity):** Top-5 Ergebnisse aus Identitäts-Notizen.
|
||||
* **Search_B (History):** Top-5 Ergebnisse aus biografischen/externen Notizen.
|
||||
* **Search_C (Action):** Top-5 Ergebnisse aus operativen/strategischen Notizen.
|
||||
|
||||
#### Schritt 3: Agentic Synthesis (The Reasoning)
|
||||
##### Schritt 3: Agentic Synthesis (The Reasoning)
|
||||
Ein Synthese-Agent (LLM) erhält die aggregierten Ergebnisse in getrennten Sektionen. Die Anweisung lautet:
|
||||
1. **Prüfung:** Steht das aktuelle Vorhaben (Action) im Einklang mit den Werten (Identity)?
|
||||
2. **Abgleich:** Welche vergangenen Erfahrungen (History) stützen oder widersprechen diesem Weg?
|
||||
@@ -230,12 +228,39 @@ Ein Synthese-Agent (LLM) erhält die aggregierten Ergebnisse in getrennten Sekti
|
||||
|
||||
|
||||
|
||||
### 4. Erwartete Ergebnisse
|
||||
#### 4. Erwartete Ergebnisse
|
||||
* **Höhere Resonanz:** Antworten wirken authentischer, da sie explizit auf das Wertesystem des Nutzers Bezug nehmen.
|
||||
* **Widerspruchs-Erkennung:** Das System kann den Nutzer aktiv warnen, wenn ein Projekt gegen seine `principles` oder `needs` verstößt.
|
||||
* **Robustes Retrieval:** Wichtige Identitäts-Informationen gehen nicht mehr im "Rauschen" von hunderten Journal-Einträgen verloren.
|
||||
---
|
||||
|
||||
### WP-24 – Proactive Discovery & Agentic Knowledge Mining
|
||||
**Status:** 🚀 In Planung (Nächster Architektur-Sprung)
|
||||
**Ziel:** Transformation von Mindnet von einem reaktiven Archiv zu einem aktiven Denkpartner. Das System soll aktiv Wissenslücken schließen und verborgene Querverbindungen in großen Vaults sowie in Chat-Dialogen aufspüren.
|
||||
|
||||
**Herausforderung:**
|
||||
1. **Silo-Effekt:** Bei wachsenden Vaults vergisst der Nutzer existierende Notizen und erstellt redundante Inhalte ohne Verknüpfung.
|
||||
2. **Insight-Verlust:** Im Chat entstehen wertvolle Synthesen, die momentan im flüchtigen Chat-Log vergraben bleiben.
|
||||
|
||||
**Lösungsskizze & Strategie:**
|
||||
|
||||
#### A. Proactive Discovery (Vault-Scanning)
|
||||
Das System nutzt die existierende `candidate_pool` Logik aus WP-15b, befüllt diese jedoch automatisiert:
|
||||
* **Vector Similarity Search**: Beim Import einer Note (oder als periodischer Hintergrundprozess) sucht der neue `RecommenderService` in Qdrant nach den Top-X semantisch ähnlichsten Chunks im gesamten Vault.
|
||||
* **Auto-Injection**: Diese Funde werden automatisch als `related_to` Kandidaten in den `candidate_pool` der neuen Note injiziert.
|
||||
* **WP-15b Filter**: Das LLM validiert diese Vorschläge im zweiten Pass der Ingestion gegen den Kontext. Nur was semantisch wirklich passt, wird als Kante im Graphen persistiert.
|
||||
|
||||
#### B. Agentic Knowledge Mining (Chat-to-Vault)
|
||||
Integration von Informationen aus dem Dialog direkt in den Graphen:
|
||||
* **Intent Detection**: Das Chat-Backend erkennt „notierwürdige“ Informationen (z.B. neue Prinzipien, Strategie-Entwürfe oder Werte-Anpassungen).
|
||||
* **Auto-Drafting**: Das LLM nutzt das `interview_template`, um aus dem Chat-Fragment eine valide Markdown-Datei mit Frontmatter (Status: `draft`) zu generieren.
|
||||
* **Real-Time Linking**: Die neue Datei wird sofort dem „Discovery-Lauf“ (Teil A) unterzogen, um sie mit dem bestehenden Wissensschatz zu vernetzen.
|
||||
* **User Review**: Die generierte Notiz erscheint im `00_Inbox` Ordner. Der Nutzer muss lediglich den Status auf `stable` setzen, um die Entdeckungen final zu integrieren.
|
||||
|
||||
**Erwartete Ergebnisse:**
|
||||
* Eliminierung von Wissens-Silos durch automatische Vernetzung.
|
||||
* Nahtloser Übergang von der Exploration (Chat) zur Konsolidierung (Vault).
|
||||
* Vermeidung von Dubletten durch Ähnlichkeits-Warnungen beim Import.
|
||||
## 4. Abhängigkeiten & Release-Plan
|
||||
|
||||
```mermaid
|
||||
@@ -244,6 +269,8 @@ graph TD
|
||||
WP19a --> WP17(Memory)
|
||||
WP15(Smart Edges) --> WP16(Auto-Discovery)
|
||||
WP15 --> WP14(Refactoring)
|
||||
WP15(Smart Edges) --> WP15b(Candidate Validation)
|
||||
WP15b --> WP24(Proactive Discovery)
|
||||
WP03(Import) --> WP18(Health Check)
|
||||
WP03 --> WP13(MCP)
|
||||
WP04 --> WP13(MCP)
|
||||
@@ -253,4 +280,5 @@ graph TD
|
||||
WP22 --> WP14
|
||||
WP15(Smart Edges) --> WP21
|
||||
WP20(Cloud Hybrid) --> WP15b
|
||||
WP24 --> WP23(Multi-Stream Reasoning)
|
||||
```
|
||||
@@ -315,4 +315,45 @@ Die Gewichtung findet **Pre-Retrieval** (im Scoring-Algorithmus) statt, **nicht*
|
||||
2. Zeige die Integration in `ingestion.py` (Status-Filter & Edge-Validierung).
|
||||
3. Zeige die Erweiterung in `scoring.py` (Status-Gewicht & Dynamic Edge Boosting).
|
||||
|
||||
Bitte bestätige die Übernahme dieses Architektur-Pakets.
|
||||
Bitte bestätige die Übernahme dieses Architektur-Pakets.
|
||||
|
||||
---
|
||||
|
||||
# Übergabe Arbeitspaket: WP-24 – Proactive Discovery & Agentic Knowledge Mining
|
||||
|
||||
## 1. Projekt-Kontext
|
||||
Wir arbeiten an **Mindnet**, einem System für einen "digitalen Zwilling". Das System nutzt einen Wissensgraph (Qdrant), asynchrone Ingestion und eine hybride LLM-Infrastruktur (Cloud/Lokal).
|
||||
|
||||
## 2. Status Quo (Abgeschlossen: WP-15b)
|
||||
Das Arbeitspaket **WP-15b (Candidate-Based Validation)** wurde gerade erfolgreich implementiert.
|
||||
* **Two-Pass Workflow:** In Pass 1 wird ein globaler `LocalBatchCache` aufgebaut (ID, Titel, Dateiname). In Pass 2 findet eine semantische binäre Validierung (YES/NO) statt.
|
||||
* **Edge Inheritance:** Kanten werden aus Sektionen und Frontmatter an Chunks vererbt.
|
||||
* **Candidate Pool:** Nur Kanten in der Sektion `## Unzugeordnete Kanten` (Provenienz: `global_pool`) werden vom LLM geprüft. Explizite Kanten (`[!edge]` im Text) werden direkt übernommen.
|
||||
|
||||
## 3. Auftrag: WP-24 – Proactive Discovery & Agentic Knowledge Mining
|
||||
Das Ziel ist die Transformation von Mindnet zu einem aktiven Denkpartner.
|
||||
|
||||
### Teil A: Proactive Discovery (Vault-Scanning)
|
||||
* **Mechanismus:** Automatisches Befüllen des `candidate_pool` via Vektor-Ähnlichkeit.
|
||||
* **Logik:** Beim Import einer Note sucht ein neuer Service in Qdrant nach den semantisch ähnlichsten Chunks im Vault und fügt diese als `related_to` Kandidaten hinzu.
|
||||
* **Filter:** Die WP-15b Validierungs-Logik filtert diese Vorschläge anschließend.
|
||||
|
||||
### Teil B: Agentic Knowledge Mining (Chat-to-Vault)
|
||||
* **Mechanismus:** Extraktion notierwürdiger Informationen aus dem Chat.
|
||||
* **Logik:** Erstellung von Markdown-Drafts im `00_Inbox` Ordner basierend auf dem Chat-Kontext unter Nutzung des `interview_template`.
|
||||
|
||||
## 4. Erforderliche Code-Basis (Dateien)
|
||||
Stelle sicher, dass dir folgende Dateien vorliegen, um die Logik zu verstehen und zu erweitern:
|
||||
|
||||
1. **`app/core/ingestion.py` (v2.12.2):** Zentraler Two-Pass Workflow und Validierungsgate.
|
||||
2. **`app/core/chunker.py` (v3.2.0):** Vorbereitung des Candidate-Pools und Vererbungslogik.
|
||||
3. **`scripts/import_markdown.py` (v2.4.1):** Entry-Point und Pre-Scan Harvester für den Cache.
|
||||
4. **`app/core/derive_edges.py` (v2.1.0):** Aggregator für Kanten mit Provenance-Priorisierung.
|
||||
5. **`app/services/edge_registry.py` (v0.8.0):** Validierung gegen das Kanten-Vokabular.
|
||||
6. **`config/prompts.yaml` (v2.6.0):** Enthält die `edge_validation` und `interview_template` Prompts.
|
||||
7. **`06_active_roadmap.md` (v2.9.0):** Enthält die detaillierte Planung für WP-24.
|
||||
|
||||
## 5. Nächste technische Schritte
|
||||
1. Entwurf eines `RecommenderService` für die Vektor-Suche in Qdrant.
|
||||
2. Integration des Services in die `ingestion.py` zur automatischen Befüllung des `candidate_pool`.
|
||||
3. Erweiterung des Chat-Backends um die "Capture-to-Vault" Funktionalität.
|
||||
+43
-17
@@ -2,7 +2,10 @@
|
||||
"""
|
||||
scripts/import_markdown.py
|
||||
CLI-Tool zum Importieren von Markdown-Dateien in Qdrant.
|
||||
Updated for Mindnet v2.3.6 (Async Ingestion Support).
|
||||
WP-15b: Implementiert den Two-Pass Workflow (Pre-Scan + Processing).
|
||||
Sorgt dafür, dass der LocalBatchCache vor der Verarbeitung robust gefüllt wird.
|
||||
Indiziert Notizen nach ID, Titel und Dateiname für maximale Link-Kompatibilität.
|
||||
VERSION: 2.4.1
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
@@ -11,21 +14,16 @@ import logging
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
import logging
|
||||
# Setzt das Level global auf INFO, damit Sie den Fortschritt sehen
|
||||
# Setzt das Level global auf INFO, damit der Fortschritt im Log sichtbar ist
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
|
||||
# Wenn Sie TIEFE Einblicke wollen, setzen Sie den SemanticAnalyzer spezifisch auf DEBUG:
|
||||
logging.getLogger("app.services.semantic_analyzer").setLevel(logging.DEBUG)
|
||||
|
||||
# Importiere den neuen Async Service
|
||||
# Stellen wir sicher, dass der Pfad stimmt (Pythonpath)
|
||||
# Importiere den neuen Async Service und stelle Python-Pfad sicher
|
||||
import sys
|
||||
sys.path.append(os.getcwd())
|
||||
|
||||
from app.core.ingestion import IngestionService
|
||||
from app.core.parser import pre_scan_markdown
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("importer")
|
||||
|
||||
async def main_async(args):
|
||||
@@ -34,7 +32,7 @@ async def main_async(args):
|
||||
logger.error(f"Vault path does not exist: {vault_path}")
|
||||
return
|
||||
|
||||
# Service initialisieren (startet Async Clients)
|
||||
# 1. Service initialisieren
|
||||
logger.info(f"Initializing IngestionService (Prefix: {args.prefix})")
|
||||
service = IngestionService(collection_prefix=args.prefix)
|
||||
|
||||
@@ -46,14 +44,42 @@ async def main_async(args):
|
||||
|
||||
logger.info(f"Found {len(files)} markdown files.")
|
||||
|
||||
stats = {"processed": 0, "skipped": 0, "errors": 0}
|
||||
# =========================================================================
|
||||
# PASS 1: Global Pre-Scan (WP-15b Harvester)
|
||||
# Füllt den LocalBatchCache für die semantische Kanten-Validierung.
|
||||
# Nutzt ID, Titel und Filename für robusten Look-up.
|
||||
# =========================================================================
|
||||
logger.info(f"🔍 [Pass 1] Pre-scanning {len(files)} files for global context cache...")
|
||||
for f_path in files:
|
||||
try:
|
||||
ctx = pre_scan_markdown(str(f_path))
|
||||
if ctx:
|
||||
# 1. Look-up via Note ID (UUID oder Frontmatter ID)
|
||||
service.batch_cache[ctx.note_id] = ctx
|
||||
|
||||
# 2. Look-up via Titel (Wichtig für Wikilinks [[Titel]])
|
||||
service.batch_cache[ctx.title] = ctx
|
||||
|
||||
# 3. Look-up via Dateiname (Wichtig für Wikilinks [[Filename]])
|
||||
fname = os.path.splitext(f_path.name)[0]
|
||||
service.batch_cache[fname] = ctx
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Could not pre-scan {f_path.name}: {e}")
|
||||
|
||||
# Wir nutzen eine Semaphore, um nicht zu viele Files gleichzeitig zu öffnen/embedden
|
||||
sem = asyncio.Semaphore(5) # Max 5 concurrent files to avoid OOM or Rate Limit
|
||||
logger.info(f"✅ Context Cache populated for {len(files)} notes.")
|
||||
|
||||
# =========================================================================
|
||||
# PASS 2: Processing (Semantic Batch-Verarbeitung)
|
||||
# Nutzt den gefüllten Cache zur binären Validierung semantischer Kanten.
|
||||
# =========================================================================
|
||||
stats = {"processed": 0, "skipped": 0, "errors": 0}
|
||||
sem = asyncio.Semaphore(5) # Max 5 parallele Dateien für Cloud-Stabilität
|
||||
|
||||
async def process_with_limit(f_path):
|
||||
async with sem:
|
||||
try:
|
||||
# Nutzt den nun gefüllten Batch-Cache in der process_file Logik
|
||||
res = await service.process_file(
|
||||
file_path=str(f_path),
|
||||
vault_root=str(vault_path),
|
||||
@@ -65,8 +91,8 @@ async def main_async(args):
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": str(e), "path": str(f_path)}
|
||||
|
||||
# Batch Processing
|
||||
# Wir verarbeiten in Chunks, um den Progress zu sehen
|
||||
logger.info(f"🚀 [Pass 2] Starting semantic processing in batches...")
|
||||
|
||||
batch_size = 20
|
||||
for i in range(0, len(files), batch_size):
|
||||
batch = files[i:i+batch_size]
|
||||
@@ -92,7 +118,7 @@ def main():
|
||||
load_dotenv()
|
||||
default_prefix = os.getenv("COLLECTION_PREFIX", "mindnet")
|
||||
|
||||
parser = argparse.ArgumentParser(description="Import Vault to Qdrant (Async)")
|
||||
parser = argparse.ArgumentParser(description="Two-Pass Markdown Ingestion for Mindnet")
|
||||
parser.add_argument("--vault", default="./vault", help="Path to vault root")
|
||||
parser.add_argument("--prefix", default=default_prefix, help="Collection prefix")
|
||||
parser.add_argument("--force", action="store_true", help="Force re-index all files")
|
||||
@@ -100,7 +126,7 @@ def main():
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Starte den Async Loop
|
||||
# Starte den asynchronen Haupt-Loop
|
||||
asyncio.run(main_async(args))
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user